Adjust structure of e2e test codes

Put every test moduels into seperate packages and all velero install parameters integrated into one struct

Signed-off-by: Ming <mqiu@vmware.com>

changelogs/unreleased/4386-mqiu

@@ -0,0 +1 @@
+Adjust structure of e2e test codes

test/e2e/backup/backup.go

@@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-package e2e
+package backup
 
 import (
 	"context"
@@ -23,98 +23,93 @@ import (
 	"github.com/google/uuid"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
+	. "github.com/vmware-tanzu/velero/test/e2e"
+	k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
+	kibishiiutils "github.com/vmware-tanzu/velero/test/e2e/util/kibishii"
+	veleroutils "github.com/vmware-tanzu/velero/test/e2e/util/velero"
 )
 
-var (
+func BackupRestoreWithSnapshots() {
-	uuidgen uuid.UUID
+	BackupRestoreTest(true)
-)
-
-// Test backup and restore of Kibishi using restic
-var _ = Describe("[Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", backup_restore_with_restic)
-
-var _ = Describe("[Snapshot] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", backup_restore_with_snapshots)
-
-func backup_restore_with_snapshots() {
-	backup_restore_test(true)
 }
 
-func backup_restore_with_restic() {
+func BackupRestoreWithRestic() {
-	backup_restore_test(false)
+	BackupRestoreTest(false)
 }
 
-func backup_restore_test(useVolumeSnapshots bool) {
+func BackupRestoreTest(useVolumeSnapshots bool) {
 	var (
 		backupName, restoreName string
 	)
 
-	client, err := newTestClient()
+	client, err := k8sutils.NewTestClient()
 	Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
 
 	BeforeEach(func() {
-		if useVolumeSnapshots && cloudProvider == "kind" {
+		if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" {
 			Skip("Volume snapshots not supported on kind")
 		}
 		var err error
 		flag.Parse()
-		uuidgen, err = uuid.NewRandom()
+		UUIDgen, err = uuid.NewRandom()
 		Expect(err).To(Succeed())
-		if installVelero {
+		if VeleroCfg.InstallVelero {
-			Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, plugins, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots,
+			Expect(veleroutils.VeleroInstall(context.Background(), &VeleroCfg, "", useVolumeSnapshots)).To(Succeed())
-				cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed())
 		}
 	})
 
 	AfterEach(func() {
-		if installVelero {
+		if VeleroCfg.InstallVelero {
-			err = veleroUninstall(context.Background(), veleroCLI, veleroNamespace)
+			err = veleroutils.VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)
 			Expect(err).To(Succeed())
 		}
 	})
 
 	When("kibishii is the sample workload", func() {
 		It("should be successfully backed up and restored to the default BackupStorageLocation", func() {
-			backupName = "backup-" + uuidgen.String()
+			backupName = "backup-" + UUIDgen.String()
-			restoreName = "restore-" + uuidgen.String()
+			restoreName = "restore-" + UUIDgen.String()
 			// Even though we are using Velero's CloudProvider plugin for object storage, the kubernetes cluster is running on
 			// KinD. So use the kind installation for Kibishii.
-			Expect(runKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, "", useVolumeSnapshots, registryCredentialFile)).To(Succeed(),
+			Expect(kibishiiutils.RunKibishiiTests(client, VeleroCfg.CloudProvider, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, restoreName, "", useVolumeSnapshots, VeleroCfg.RegistryCredentialFile)).To(Succeed(),
 				"Failed to successfully backup and restore Kibishii namespace")
 		})
 
 		It("should successfully back up and restore to an additional BackupStorageLocation with unique credentials", func() {
-			if additionalBSLProvider == "" {
+			if VeleroCfg.AdditionalBSLProvider == "" {
 				Skip("no additional BSL provider given, not running multiple BackupStorageLocation with unique credentials tests")
 			}
 
-			if additionalBSLBucket == "" {
+			if VeleroCfg.AdditionalBSLBucket == "" {
 				Skip("no additional BSL bucket given, not running multiple BackupStorageLocation with unique credentials tests")
 			}
 
-			if additionalBSLCredentials == "" {
+			if VeleroCfg.AdditionalBSLCredentials == "" {
 				Skip("no additional BSL credentials given, not running multiple BackupStorageLocation with unique credentials tests")
 			}
 
-			Expect(veleroAddPluginsForProvider(context.TODO(), veleroCLI, veleroNamespace, additionalBSLProvider, addBSLPlugins)).To(Succeed())
+			Expect(veleroutils.VeleroAddPluginsForProvider(context.TODO(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, VeleroCfg.AdditionalBSLProvider, VeleroCfg.AddBSLPlugins)).To(Succeed())
 
 			// Create Secret for additional BSL
-			secretName := fmt.Sprintf("bsl-credentials-%s", uuidgen)
+			secretName := fmt.Sprintf("bsl-credentials-%s", UUIDgen)
-			secretKey := fmt.Sprintf("creds-%s", additionalBSLProvider)
+			secretKey := fmt.Sprintf("creds-%s", VeleroCfg.AdditionalBSLProvider)
 			files := map[string]string{
-				secretKey: additionalBSLCredentials,
+				secretKey: VeleroCfg.AdditionalBSLCredentials,
 			}
 
-			Expect(createSecretFromFiles(context.TODO(), client, veleroNamespace, secretName, files)).To(Succeed())
+			Expect(k8sutils.CreateSecretFromFiles(context.TODO(), client, VeleroCfg.VeleroNamespace, secretName, files)).To(Succeed())
 
 			// Create additional BSL using credential
-			additionalBsl := fmt.Sprintf("bsl-%s", uuidgen)
+			additionalBsl := fmt.Sprintf("bsl-%s", UUIDgen)
-			Expect(veleroCreateBackupLocation(context.TODO(),
+			Expect(veleroutils.VeleroCreateBackupLocation(context.TODO(),
-				veleroCLI,
+				VeleroCfg.VeleroCLI,
-				veleroNamespace,
+				VeleroCfg.VeleroNamespace,
 				additionalBsl,
-				additionalBSLProvider,
+				VeleroCfg.AdditionalBSLProvider,
-				additionalBSLBucket,
+				VeleroCfg.AdditionalBSLBucket,
-				additionalBSLPrefix,
+				VeleroCfg.AdditionalBSLPrefix,
-				additionalBSLConfig,
+				VeleroCfg.AdditionalBSLConfig,
 				secretName,
 				secretKey,
 			)).To(Succeed())
@@ -127,11 +122,10 @@ func backup_restore_test(useVolumeSnapshots bool) {
 				// We limit the length of backup name here to avoid the issue of vsphere plugin https://github.com/vmware-tanzu/velero-plugin-for-vsphere/issues/370
 				// We can remove the logic once the issue is fixed
 				if bsl == "default" {
-					backupName = fmt.Sprintf("%s-%s", backupName, uuidgen)
+					backupName = fmt.Sprintf("%s-%s", backupName, UUIDgen)
-					restoreName = fmt.Sprintf("%s-%s", restoreName, uuidgen)
+					restoreName = fmt.Sprintf("%s-%s", restoreName, UUIDgen)
 				}
-
+				Expect(kibishiiutils.RunKibishiiTests(client, VeleroCfg.CloudProvider, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, restoreName, bsl, useVolumeSnapshots, VeleroCfg.RegistryCredentialFile)).To(Succeed(),
-				Expect(runKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, bsl, useVolumeSnapshots, registryCredentialFile)).To(Succeed(),
 					"Failed to successfully backup and restore Kibishii namespace using BSL %s", bsl)
 			}
 		})

test/e2e/basic/enable_api_group_versions.go

@@ -14,11 +14,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e
+package basic
 
 import (
 	"context"
 	"encoding/json"
+	"flag"
 	"fmt"
 	"os/exec"
 	"strconv"
@@ -34,46 +35,31 @@ import (
 
 	"github.com/vmware-tanzu/velero/pkg/builder"
 	veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
+	. "github.com/vmware-tanzu/velero/test/e2e"
+	k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
+	veleroutils "github.com/vmware-tanzu/velero/test/e2e/util/velero"
 )
 
-var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", func() {
+func APIGropuVersionsTest() {
 	var (
 		resource, group string
 		err             error
 		ctx             = context.Background()
 	)
 
-	client, err := newTestClient()
+	client, err := k8sutils.NewTestClient()
 	Expect(err).To(Succeed(), "Failed to instantiate cluster client for group version tests")
 
 	BeforeEach(func() {
 		resource = "rockbands"
 		group = "music.example.io"
-
+		UUIDgen, err = uuid.NewRandom()
-		uuidgen, err = uuid.NewRandom()
 		Expect(err).NotTo(HaveOccurred())
-
+		flag.Parse()
 		// TODO: install Velero once for the test suite once feature flag is
 		// removed and velero installation becomes the same as other e2e tests.
-		if installVelero {
+		if VeleroCfg.InstallVelero {
-			err = veleroInstall(
+			err = veleroutils.VeleroInstall(context.Background(), &VeleroCfg, "EnableAPIGroupVersions", false)
-				context.Background(),
-				veleroCLI,
-				veleroImage,
-				resticHelperImage,
-				plugins,
-				veleroNamespace,
-				cloudProvider,
-				objectStoreProvider,
-				false,
-				cloudCredentialsFile,
-				bslBucket,
-				bslPrefix,
-				bslConfig,
-				vslConfig,
-				crdsVersion,
-				"EnableAPIGroupVersions", // TODO: remove when feature flag is removed
-				registryCredentialFile)
 			Expect(err).NotTo(HaveOccurred())
 		}
 	})
@@ -88,8 +74,8 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions",
 		}
 		Expect(err).NotTo(HaveOccurred())
 
-		if installVelero {
+		if VeleroCfg.InstallVelero {
-			err = veleroUninstall(ctx, veleroCLI, veleroNamespace)
+			err = veleroutils.VeleroUninstall(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)
 			Expect(err).NotTo(HaveOccurred())
 		}
 
@@ -105,9 +91,9 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions",
 			)).To(Succeed(), "Failed to successfully backup and restore multiple API Groups")
 		})
 	})
-})
+}
 
-func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, resource, group string) error {
+func runEnableAPIGroupVersionsTests(ctx context.Context, client k8sutils.TestClient, resource, group string) error {
 	tests := []struct {
 		name       string
 		namespaces []string
@@ -180,7 +166,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso
 			},
 			tgtCrdYaml: "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml",
 			tgtVer:     "v2beta1",
-			cm: builder.ForConfigMap(veleroNamespace, "enableapigroupversions").Data(
+			cm: builder.ForConfigMap(VeleroCfg.VeleroNamespace, "enableapigroupversions").Data(
 				"restoreResourcesVersionPriority",
 				`rockbands.music.example.io=v2beta1,v2beta2,v2`,
 			).Result(),
@@ -224,11 +210,11 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso
 		for version, cr := range tc.srcCRs {
 			ns := resource + "-src-" + version
 
-			if err := createNamespace(ctx, client, ns); err != nil {
+			if err := k8sutils.CreateNamespace(ctx, client, ns); err != nil {
 				return errors.Wrapf(err, "create %s namespace", ns)
 			}
 			defer func(namespace string) {
-				if err = deleteNamespace(ctx, client, namespace, true); err != nil {
+				if err = k8sutils.DeleteNamespace(ctx, client, namespace, true); err != nil {
 					fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", ns))
 				}
 			}(ns)
@@ -243,16 +229,16 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso
 		// Restart Velero pods in order to recognize music-system CRD right away
 		// instead of waiting for discovery helper to refresh. See
 		// https://github.com/vmware-tanzu/velero/issues/3471.
-		if err := restartPods(ctx, veleroNamespace); err != nil {
+		if err := restartPods(ctx, VeleroCfg.VeleroNamespace); err != nil {
 			return errors.Wrapf(err, "restart Velero pods")
 		}
 
-		backup := "backup-rockbands-" + uuidgen.String() + "-" + strconv.Itoa(i)
+		backup := "backup-rockbands-" + UUIDgen.String() + "-" + strconv.Itoa(i)
 		namespacesStr := strings.Join(tc.namespaces, ",")
 
-		err = veleroBackupNamespace(ctx, veleroCLI, veleroNamespace, backup, namespacesStr, "", false)
+		err = veleroutils.VeleroBackupNamespace(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup, namespacesStr, "", false)
 		if err != nil {
-			runDebug(context.Background(), veleroCLI, veleroNamespace, backup, "")
+			veleroutils.RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup, "")
 			return errors.Wrapf(err, "back up %s namespaces on source cluster", namespacesStr)
 		}
 
@@ -261,7 +247,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso
 		}
 
 		for _, ns := range tc.namespaces {
-			if err := deleteNamespace(ctx, client, ns, true); err != nil {
+			if err := k8sutils.DeleteNamespace(ctx, client, ns, true); err != nil {
 				return errors.Wrapf(err, "delete %s namespace from source cluster", ns)
 			}
 		}
@@ -275,23 +261,23 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso
 
 		// Apply config map if there is one.
 		if tc.cm != nil {
-			_, err := client.clientGo.CoreV1().ConfigMaps(veleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{})
+			_, err := client.ClientGo.CoreV1().ConfigMaps(VeleroCfg.VeleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{})
 			if err != nil {
 				return errors.Wrap(err, "create config map with user version priorities")
 			}
 		}
 
 		// Reset Velero to recognize music-system CRD.
-		if err := restartPods(ctx, veleroNamespace); err != nil {
+		if err := restartPods(ctx, VeleroCfg.VeleroNamespace); err != nil {
 			return errors.Wrapf(err, "restart Velero pods")
 		}
 
 		// Restore rockbands namespaces.
-		restore := "restore-rockbands-" + uuidgen.String() + "-" + strconv.Itoa(i)
+		restore := "restore-rockbands-" + UUIDgen.String() + "-" + strconv.Itoa(i)
 
 		if tc.want != nil {
-			if err := veleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup); err != nil {
+			if err := veleroutils.VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restore, backup); err != nil {
-				runDebug(context.Background(), veleroCLI, veleroNamespace, "", restore)
+				veleroutils.RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", restore)
 				return errors.Wrapf(err, "restore %s namespaces on target cluster", namespacesStr)
 			}
 
@@ -329,7 +315,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso
 		} else {
 			// No custom resource should have been restored. Expect "no resource found"
 			// error during restore.
-			err := veleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup)
+			err := veleroutils.VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restore, backup)
 
 			if err.Error() != "Unexpected restore phase got PartiallyFailed, expecting Completed" {
 				return errors.New("expected error but not none")

test/e2e/e2e_suite_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e
+package e2e_test
 
 import (
 	"flag"
@@ -23,44 +23,59 @@ import (
 	. "github.com/onsi/ginkgo"
 	"github.com/onsi/ginkgo/reporters"
 	. "github.com/onsi/gomega"
-)
 
-var (
+	. "github.com/vmware-tanzu/velero/test/e2e"
-	veleroCLI, veleroImage, veleroVersion, cloudCredentialsFile, bslConfig, bslBucket, bslPrefix, vslConfig, cloudProvider, objectStoreProvider, veleroNamespace, crdsVersion string
+	. "github.com/vmware-tanzu/velero/test/e2e/backup"
-	additionalBSLProvider, additionalBSLBucket, additionalBSLPrefix, additionalBSLConfig, additionalBSLCredentials, registryCredentialFile, resticHelperImage                 string
+	. "github.com/vmware-tanzu/velero/test/e2e/basic"
-	upgradeFromVeleroVersion, upgradeFromVeleroCLI, plugins, addBSLPlugins                                                                                                    string
+	. "github.com/vmware-tanzu/velero/test/e2e/scale"
-	installVelero                                                                                                                                                             bool
+	. "github.com/vmware-tanzu/velero/test/e2e/upgrade"
 )
 
 func init() {
-	flag.StringVar(&cloudProvider, "cloud-provider", "", "cloud that Velero will be installed into.  Required.")
+	flag.StringVar(&VeleroCfg.CloudProvider, "cloud-provider", "", "cloud that Velero will be installed into.  Required.")
-	flag.StringVar(&objectStoreProvider, "object-store-provider", "", "provider of object store plugin. Required if cloud-provider is kind, otherwise ignored.")
+	flag.StringVar(&VeleroCfg.ObjectStoreProvider, "object-store-provider", "", "provider of object store plugin. Required if cloud-provider is kind, otherwise ignored.")
-	flag.StringVar(&bslBucket, "bucket", "", "name of the object storage bucket where backups from e2e tests should be stored. Required.")
+	flag.StringVar(&VeleroCfg.BSLBucket, "bucket", "", "name of the object storage bucket where backups from e2e tests should be stored. Required.")
-	flag.StringVar(&cloudCredentialsFile, "credentials-file", "", "file containing credentials for backup and volume provider. Required.")
+	flag.StringVar(&VeleroCfg.CloudCredentialsFile, "credentials-file", "", "file containing credentials for backup and volume provider. Required.")
-	flag.StringVar(&veleroCLI, "velerocli", "velero", "path to the velero application to use.")
+	flag.StringVar(&VeleroCfg.VeleroCLI, "velerocli", "velero", "path to the velero application to use.")
-	flag.StringVar(&veleroImage, "velero-image", "velero/velero:main", "image for the velero server to be tested.")
+	flag.StringVar(&VeleroCfg.VeleroImage, "velero-image", "velero/velero:main", "image for the velero server to be tested.")
-	flag.StringVar(&plugins, "plugins", "", "provider plugins to be tested.")
+	flag.StringVar(&VeleroCfg.Plugins, "plugins", "", "provider plugins to be tested.")
-	flag.StringVar(&addBSLPlugins, "additional-bsl-plugins", "", "additional plugins to be tested.")
+	flag.StringVar(&VeleroCfg.AddBSLPlugins, "additional-bsl-plugins", "", "additional plugins to be tested.")
-	flag.StringVar(&veleroVersion, "velero-version", "main", "image version for the velero server to be tested with.")
+	flag.StringVar(&VeleroCfg.VeleroVersion, "velero-version", "main", "image version for the velero server to be tested with.")
-	flag.StringVar(&resticHelperImage, "restic-helper-image", "", "image for the velero restic restore helper to be tested.")
+	flag.StringVar(&VeleroCfg.ResticHelperImage, "restic-helper-image", "", "image for the velero restic restore helper to be tested.")
-	flag.StringVar(&upgradeFromVeleroCLI, "upgrade-from-velero-cli", "", "path to the pre-upgrade velero application to use.")
+	flag.StringVar(&VeleroCfg.UpgradeFromVeleroCLI, "upgrade-from-velero-cli", "", "path to the pre-upgrade velero application to use.")
-	flag.StringVar(&upgradeFromVeleroVersion, "upgrade-from-velero-version", "v1.6.3", "image for the pre-upgrade velero server to be tested.")
+	flag.StringVar(&VeleroCfg.UpgradeFromVeleroVersion, "upgrade-from-velero-version", "v1.6.3", "image for the pre-upgrade velero server to be tested.")
-	flag.StringVar(&bslConfig, "bsl-config", "", "configuration to use for the backup storage location. Format is key1=value1,key2=value2")
+	flag.StringVar(&VeleroCfg.BSLConfig, "bsl-config", "", "configuration to use for the backup storage location. Format is key1=value1,key2=value2")
-	flag.StringVar(&bslPrefix, "prefix", "", "prefix under which all Velero data should be stored within the bucket. Optional.")
+	flag.StringVar(&VeleroCfg.BSLPrefix, "prefix", "", "prefix under which all Velero data should be stored within the bucket. Optional.")
-	flag.StringVar(&vslConfig, "vsl-config", "", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2")
+	flag.StringVar(&VeleroCfg.VSLConfig, "vsl-config", "", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2")
-	flag.StringVar(&veleroNamespace, "velero-namespace", "velero", "namespace to install Velero into")
+	flag.StringVar(&VeleroCfg.VeleroNamespace, "velero-namespace", "velero", "namespace to install Velero into")
-	flag.BoolVar(&installVelero, "install-velero", true, "install/uninstall velero during the test.  Optional.")
+	flag.BoolVar(&VeleroCfg.InstallVelero, "install-velero", true, "install/uninstall velero during the test.  Optional.")
-	flag.StringVar(&registryCredentialFile, "registry-credential-file", "", "file containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. Optional.")
+	flag.StringVar(&VeleroCfg.RegistryCredentialFile, "registry-credential-file", "", "file containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. Optional.")
 
 	// Flags to create an additional BSL for multiple credentials test
-	flag.StringVar(&additionalBSLProvider, "additional-bsl-object-store-provider", "", "Provider of object store plugin for additional backup storage location. Required if testing multiple credentials support.")
+	flag.StringVar(&VeleroCfg.AdditionalBSLProvider, "additional-bsl-object-store-provider", "", "Provider of object store plugin for additional backup storage location. Required if testing multiple credentials support.")
-	flag.StringVar(&additionalBSLBucket, "additional-bsl-bucket", "", "name of the object storage bucket for additional backup storage location. Required if testing multiple credentials support.")
+	flag.StringVar(&VeleroCfg.AdditionalBSLBucket, "additional-bsl-bucket", "", "name of the object storage bucket for additional backup storage location. Required if testing multiple credentials support.")
-	flag.StringVar(&additionalBSLPrefix, "additional-bsl-prefix", "", "prefix under which all Velero data should be stored within the bucket for additional backup storage location. Optional.")
+	flag.StringVar(&VeleroCfg.AdditionalBSLPrefix, "additional-bsl-prefix", "", "prefix under which all Velero data should be stored within the bucket for additional backup storage location. Optional.")
-	flag.StringVar(&additionalBSLConfig, "additional-bsl-config", "", "configuration to use for the additional backup storage location. Format is key1=value1,key2=value2")
+	flag.StringVar(&VeleroCfg.AdditionalBSLConfig, "additional-bsl-config", "", "configuration to use for the additional backup storage location. Format is key1=value1,key2=value2")
-	flag.StringVar(&additionalBSLCredentials, "additional-bsl-credentials-file", "", "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.")
+	flag.StringVar(&VeleroCfg.AdditionalBSLCredentials, "additional-bsl-credentials-file", "", "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.")
-	flag.StringVar(&crdsVersion, "crds-version", "v1", "CRD apiVersion for velero CRD creation.")
+	flag.StringVar(&VeleroCfg.CRDsVersion, "crds-version", "v1", "CRD apiVersion for velero CRD creation.")
 }
 
+var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", APIGropuVersionsTest)
+
+// Test backup and restore of Kibishi using restic
+var _ = Describe("[Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupRestoreWithRestic)
+
+var _ = Describe("[Snapshot] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreWithSnapshots)
+
+var _ = Describe("[Basic] Backup/restore of 2 namespaces", BasicBackupRestore)
+
+var _ = Describe("[Scale] Backup/restore of 2500 namespaces", MultiNSBackupRestore)
+
+// Upgrade test by Kibishi using restic
+var _ = Describe("[Upgrade][Restic] Velero upgrade tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupUpgradeRestoreWithRestic)
+
+var _ = Describe("[Upgrade][Snapshot] Velero upgrade tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupUpgradeRestoreWithSnapshots)
+
 func TestE2e(t *testing.T) {
 	// Skip running E2E tests when running only "short" tests because:
 	// 1. E2E tests are long running tests involving installation of Velero and performing backup and restore operations.

test/e2e/scale/multiple_namespaces.go

@@ -1,40 +1,58 @@
-package e2e
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package scale
 
 import (
 	"context"
 	"flag"
 	"fmt"
-	"strings"
 	"time"
 
 	"github.com/google/uuid"
 
+	k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
+	veleroutils "github.com/vmware-tanzu/velero/test/e2e/util/velero"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-
 	"github.com/pkg/errors"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	. "github.com/vmware-tanzu/velero/test/e2e"
 )
 
-var _ = Describe("[Basic] Backup/restore of 2 namespaces", func() {
+func BasicBackupRestore() {
 
-	client, err := newTestClient()
+	client, err := k8sutils.NewTestClient()
 	Expect(err).To(Succeed(), "Failed to instantiate cluster client for multiple namespace tests")
 
 	BeforeEach(func() {
 		var err error
 		flag.Parse()
-		uuidgen, err = uuid.NewRandom()
+		UUIDgen, err = uuid.NewRandom()
 		Expect(err).To(Succeed())
-		if installVelero {
+		if VeleroCfg.InstallVelero {
-			Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, plugins, veleroNamespace, cloudProvider, objectStoreProvider, false,
+			Expect(veleroutils.VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed())
-				cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed())
 		}
 	})
 
 	AfterEach(func() {
-		if installVelero {
+		if VeleroCfg.InstallVelero {
-			err := veleroUninstall(context.Background(), veleroCLI, veleroNamespace)
+			err := veleroutils.VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)
 			Expect(err).To(Succeed())
 		}
 
@@ -42,34 +60,33 @@ var _ = Describe("[Basic] Backup/restore of 2 namespaces", func() {
 
 	Context("When I create 2 namespaces", func() {
 		It("should be successfully backed up and restored", func() {
-			backupName := "backup-" + uuidgen.String()
+			backupName := "backup-" + UUIDgen.String()
-			restoreName := "restore-" + uuidgen.String()
+			restoreName := "restore-" + UUIDgen.String()
 			fiveMinTimeout, _ := context.WithTimeout(context.Background(), 5*time.Minute)
-			Expect(RunMultipleNamespaceTest(fiveMinTimeout, client, "nstest-"+uuidgen.String(), 2,
+			Expect(RunMultipleNamespaceTest(fiveMinTimeout, client, "nstest-"+UUIDgen.String(), 2,
 				backupName, restoreName)).To(Succeed(), "Failed to successfully backup and restore multiple namespaces")
 		})
 	})
-})
+}
 
-var _ = Describe("[Scale] Backup/restore of 2500 namespaces", func() {
+func MultiNSBackupRestore() {
 
-	client, err := newTestClient()
+	client, err := k8sutils.NewTestClient()
 	Expect(err).To(Succeed(), "Failed to instantiate cluster client for multiple namespace tests")
 
 	BeforeEach(func() {
 		var err error
 		flag.Parse()
-		uuidgen, err = uuid.NewRandom()
+		UUIDgen, err = uuid.NewRandom()
 		Expect(err).To(Succeed())
-		if installVelero {
+		if VeleroCfg.InstallVelero {
-			Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, plugins, veleroNamespace, cloudProvider, objectStoreProvider, false,
+			Expect(veleroutils.VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed())
-				cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed())
 		}
 	})
 
 	AfterEach(func() {
-		if installVelero {
+		if VeleroCfg.InstallVelero {
-			err := veleroUninstall(context.Background(), veleroCLI, veleroNamespace)
+			err := veleroutils.VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)
 			Expect(err).To(Succeed())
 		}
 
@@ -77,22 +94,22 @@ var _ = Describe("[Scale] Backup/restore of 2500 namespaces", func() {
 
 	Context("When I create 2500 namespaces", func() {
 		It("should be successfully backed up and restored", func() {
-			backupName := "backup-" + uuidgen.String()
+			backupName := "backup-" + UUIDgen.String()
-			restoreName := "restore-" + uuidgen.String()
+			restoreName := "restore-" + UUIDgen.String()
 			twoHourTimeout, _ := context.WithTimeout(context.Background(), 2*time.Hour)
-			Expect(RunMultipleNamespaceTest(twoHourTimeout, client, "nstest-"+uuidgen.String(), 2500,
+			Expect(RunMultipleNamespaceTest(twoHourTimeout, client, "nstest-"+UUIDgen.String(), 2500,
 				backupName, restoreName)).To(Succeed(), "Failed to successfully backup and restore multiple namespaces")
 		})
 	})
-})
+}
 
-func RunMultipleNamespaceTest(ctx context.Context, client testClient, nsBaseName string, numberOfNamespaces int, backupName string, restoreName string) error {
+func RunMultipleNamespaceTest(ctx context.Context, client k8sutils.TestClient, nsBaseName string, numberOfNamespaces int, backupName string, restoreName string) error {
-	defer cleanupNamespaces(context.Background(), client, nsBaseName) // Run at exit for final cleanup
+	defer k8sutils.CleanupNamespaces(context.Background(), client, nsBaseName) // Run at exit for final cleanup
 	var excludeNamespaces []string
 
 	// Currently it's hard to build a large list of namespaces to include and wildcards do not work so instead
 	// we will exclude all of the namespaces that existed prior to the test from the backup
-	namespaces, err := client.clientGo.CoreV1().Namespaces().List(ctx, v1.ListOptions{})
+	namespaces, err := client.ClientGo.CoreV1().Namespaces().List(ctx, v1.ListOptions{})
 	if err != nil {
 		return errors.Wrap(err, "Could not retrieve namespaces")
 	}
@@ -104,30 +121,30 @@ func RunMultipleNamespaceTest(ctx context.Context, client testClient, nsBaseName
 	fmt.Printf("Creating namespaces ...\n")
 	for nsNum := 0; nsNum < numberOfNamespaces; nsNum++ {
 		createNSName := fmt.Sprintf("%s-%00000d", nsBaseName, nsNum)
-		if err := createNamespace(ctx, client, createNSName); err != nil {
+		if err := k8sutils.CreateNamespace(ctx, client, createNSName); err != nil {
 			return errors.Wrapf(err, "Failed to create namespace %s", createNSName)
 		}
 	}
-	if err := veleroBackupExcludeNamespaces(ctx, veleroCLI, veleroNamespace, backupName, excludeNamespaces); err != nil {
+	if err := veleroutils.VeleroBackupExcludeNamespaces(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, excludeNamespaces); err != nil {
-		runDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "")
+		veleroutils.RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, "")
 		return errors.Wrapf(err, "Failed to backup backup namespaces %s-*", nsBaseName)
 	}
 
-	err = cleanupNamespaces(ctx, client, nsBaseName)
+	err = k8sutils.CleanupNamespaces(ctx, client, nsBaseName)
 	if err != nil {
 		return errors.Wrap(err, "Could cleanup retrieve namespaces")
 	}
 
-	err = veleroRestore(ctx, veleroCLI, veleroNamespace, restoreName, backupName)
+	err = veleroutils.VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restoreName, backupName)
 	if err != nil {
-		runDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName)
+		veleroutils.RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", restoreName)
 		return errors.Wrap(err, "Restore failed")
 	}
 
 	// Verify that we got back all of the namespaces we created
 	for nsNum := 0; nsNum < numberOfNamespaces; nsNum++ {
 		checkNSName := fmt.Sprintf("%s-%00000d", nsBaseName, nsNum)
-		checkNS, err := getNamespace(ctx, client, checkNSName)
+		checkNS, err := k8sutils.GetNamespace(ctx, client, checkNSName)
 		if err != nil {
 			return errors.Wrapf(err, "Could not retrieve test namespace %s", checkNSName)
 		}
@@ -138,21 +155,3 @@ func RunMultipleNamespaceTest(ctx context.Context, client testClient, nsBaseName
 	// Cleanup is automatic on the way out
 	return nil
 }
-
-func cleanupNamespaces(ctx context.Context, client testClient, nsBaseName string) error {
-	namespaces, err := client.clientGo.CoreV1().Namespaces().List(ctx, v1.ListOptions{})
-	if err != nil {
-		return errors.Wrap(err, "Could not retrieve namespaces")
-	}
-
-	fmt.Printf("Cleaning up namespaces ...\n")
-	for _, checkNamespace := range namespaces.Items {
-		if strings.HasPrefix(checkNamespace.Name, nsBaseName) {
-			err = client.clientGo.CoreV1().Namespaces().Delete(ctx, checkNamespace.Name, v1.DeleteOptions{})
-			if err != nil {
-				return errors.Wrapf(err, "Could not delete namespace %s", checkNamespace.Name)
-			}
-		}
-	}
-	return nil
-}

test/e2e/types.go

@@ -0,0 +1,52 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"github.com/google/uuid"
+)
+
+var UUIDgen uuid.UUID
+
+var VeleroCfg VerleroConfig
+
+type VerleroConfig struct {
+	VeleroCLI                string
+	VeleroImage              string
+	VeleroVersion            string
+	CloudCredentialsFile     string
+	BSLConfig                string
+	BSLBucket                string
+	BSLPrefix                string
+	VSLConfig                string
+	CloudProvider            string
+	ObjectStoreProvider      string
+	VeleroNamespace          string
+	CRDsVersion              string
+	AdditionalBSLProvider    string
+	AdditionalBSLBucket      string
+	AdditionalBSLPrefix      string
+	AdditionalBSLConfig      string
+	AdditionalBSLCredentials string
+	RegistryCredentialFile   string
+	ResticHelperImage        string
+	UpgradeFromVeleroVersion string
+	UpgradeFromVeleroCLI     string
+	Plugins                  string
+	AddBSLPlugins            string
+	InstallVelero            bool
+}

test/e2e/upgrade/upgrade.go

@@ -0,0 +1,170 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package upgrade
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"time"
+
+	"github.com/google/uuid"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/pkg/errors"
+
+	. "github.com/vmware-tanzu/velero/test/e2e"
+	k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
+	kibishiiutils "github.com/vmware-tanzu/velero/test/e2e/util/kibishii"
+	veleroutils "github.com/vmware-tanzu/velero/test/e2e/util/velero"
+)
+
+const (
+	upgradeNamespace = "upgrade-workload"
+)
+
+func BackupUpgradeRestoreWithSnapshots() {
+	BackupUpgradeRestoreTest(true)
+}
+
+func BackupUpgradeRestoreWithRestic() {
+	BackupUpgradeRestoreTest(false)
+}
+
+func BackupUpgradeRestoreTest(useVolumeSnapshots bool) {
+	var (
+		backupName, restoreName, upgradeFromVeleroCLI string
+	)
+
+	client, err := k8sutils.NewTestClient()
+	Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
+
+	BeforeEach(func() {
+		if (len(VeleroCfg.UpgradeFromVeleroVersion)) == 0 {
+			Skip("An original velero version is required to run upgrade test, please run test with upgrade-from-velero-version=<version>")
+		}
+		if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" {
+			Skip("Volume snapshots not supported on kind")
+		}
+		//Assume tag of velero server image is identical to velero CLI version
+		//Download velero CLI if it's empty according to velero CLI version
+		if (len(VeleroCfg.UpgradeFromVeleroCLI)) == 0 {
+			upgradeFromVeleroCLI, err = veleroutils.InstallVeleroCLI(VeleroCfg.UpgradeFromVeleroVersion)
+			Expect(err).To(Succeed())
+		}
+
+		var err error
+		flag.Parse()
+		UUIDgen, err = uuid.NewRandom()
+		Expect(err).To(Succeed())
+		if VeleroCfg.InstallVelero {
+			//Set VeleroImage and ResticHelperImage to blank
+			//VeleroImage and ResticHelperImage should be the default value in originalCli
+			tmpCfg := VeleroCfg
+			tmpCfg.VeleroCLI = upgradeFromVeleroCLI
+			tmpCfg.VeleroImage = ""
+			tmpCfg.ResticHelperImage = ""
+			tmpCfg.Plugins = ""
+			tmpCfg.CRDsVersion = ""
+			Expect(veleroutils.VeleroInstall(context.Background(), &tmpCfg, "", useVolumeSnapshots)).To(Succeed())
+			Expect(veleroutils.CheckVeleroVersion(context.Background(), upgradeFromVeleroCLI, tmpCfg.UpgradeFromVeleroVersion)).To(Succeed())
+		} else {
+			Skip("Upgrade test is skipped since user don't want to install any other velero")
+		}
+	})
+
+	AfterEach(func() {
+		if VeleroCfg.InstallVelero {
+			err = veleroutils.VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)
+			Expect(err).To(Succeed())
+		}
+	})
+
+	When("kibishii is the sample workload", func() {
+		It("should be successfully backed up and restored to the default BackupStorageLocation", func() {
+			backupName = "backup-" + UUIDgen.String()
+			restoreName = "restore-" + UUIDgen.String()
+			Expect(runUpgradeTests(client, &VeleroCfg, upgradeFromVeleroCLI, backupName, restoreName, "", useVolumeSnapshots)).To(Succeed(),
+				"Failed to successfully backup and restore Kibishii namespace")
+		})
+	})
+}
+
+// runUpgradeTests runs upgrade test on the provider by kibishii.
+func runUpgradeTests(client k8sutils.TestClient, veleroCfg *VerleroConfig, upgradeFromVeleroCLI, backupName, restoreName, backupLocation string,
+	useVolumeSnapshots bool) error {
+	if veleroCfg.VeleroCLI == "" {
+		return errors.New("empty")
+	}
+	oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
+	if err := k8sutils.CreateNamespace(oneHourTimeout, client, upgradeNamespace); err != nil {
+		return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", upgradeNamespace)
+	}
+	defer func() {
+		if err := k8sutils.DeleteNamespace(context.Background(), client, upgradeNamespace, true); err != nil {
+			fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", upgradeNamespace))
+		}
+	}()
+	if err := kibishiiutils.KibishiiPrepareBeforeBackup(oneHourTimeout, client, veleroCfg.CloudProvider, upgradeNamespace, veleroCfg.RegistryCredentialFile); err != nil {
+		return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", upgradeNamespace)
+	}
+
+	if err := veleroutils.VeleroBackupNamespace(oneHourTimeout, upgradeFromVeleroCLI, veleroCfg.VeleroNamespace, backupName, upgradeNamespace, backupLocation, useVolumeSnapshots); err != nil {
+		// TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command
+		// TODO move to "veleroutils.RunDebug" after we bump up to 1.7 in the upgrade case
+		veleroutils.VeleroBackupLogs(context.Background(), upgradeFromVeleroCLI, veleroCfg.VeleroNamespace, backupName)
+		return errors.Wrapf(err, "Failed to backup kibishii namespace %s", upgradeNamespace)
+	}
+
+	if veleroCfg.CloudProvider == "vsphere" && useVolumeSnapshots {
+		// Wait for uploads started by the Velero Plug-in for vSphere to complete
+		// TODO - remove after upload progress monitoring is implemented
+		fmt.Println("Waiting for vSphere uploads to complete")
+		if err := veleroutils.WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, upgradeNamespace); err != nil {
+			return errors.Wrapf(err, "Error waiting for uploads to complete")
+		}
+	}
+	fmt.Printf("Simulating a disaster by removing namespace %s\n", upgradeNamespace)
+	if err := k8sutils.DeleteNamespace(oneHourTimeout, client, upgradeNamespace, true); err != nil {
+		return errors.Wrapf(err, "failed to delete namespace %s", upgradeNamespace)
+	}
+
+	// the snapshots of AWS may be still in pending status when do the restore, wait for a while
+	// to avoid this https://github.com/vmware-tanzu/velero/issues/1799
+	// TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed
+	if veleroCfg.CloudProvider == "aws" && useVolumeSnapshots {
+		fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...")
+		time.Sleep(5 * time.Minute)
+	}
+
+	if err := veleroutils.VeleroInstall(context.Background(), veleroCfg, "", useVolumeSnapshots); err != nil {
+		return errors.Wrapf(err, "Failed to install velero from image %s", veleroCfg.VeleroImage)
+	}
+	if err := veleroutils.CheckVeleroVersion(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroVersion); err != nil {
+		return errors.Wrapf(err, "Velero install version mismatch.")
+	}
+	if err := veleroutils.VeleroRestore(oneHourTimeout, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, restoreName, backupName); err != nil {
+		veleroutils.RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, "", restoreName)
+		return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName)
+	}
+
+	if err := kibishiiutils.KibishiiVerifyAfterRestore(client, upgradeNamespace, oneHourTimeout); err != nil {
+		return errors.Wrapf(err, "Error verifying kibishii after restore")
+	}
+
+	fmt.Printf("Upgrade test completed successfully\n")
+	return nil
+}

test/e2e/upgrade_test.go

@@ -1,164 +0,0 @@
-/*
-Copyright the Velero contributors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-package e2e
-
-import (
-	"context"
-	"flag"
-	"fmt"
-	"time"
-
-	"github.com/google/uuid"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-	"github.com/pkg/errors"
-)
-
-const (
-	upgradeNamespace = "upgrade-workload"
-)
-
-// Upgrade test by Kibishi using restic
-var _ = Describe("[Upgrade][Restic] Velero upgrade tests on cluster using the plugin provider for object storage and Restic for volume backups", backup_upgrade_restore_with_restic)
-
-var _ = Describe("[Upgrade][Snapshot] Velero upgrade tests on cluster using the plugin provider for object storage and snapshots for volume backups", backup_upgrade_restore_with_snapshots)
-
-func backup_upgrade_restore_with_snapshots() {
-	backup_upgrade_restore_test(true)
-}
-
-func backup_upgrade_restore_with_restic() {
-	backup_upgrade_restore_test(false)
-}
-
-func backup_upgrade_restore_test(useVolumeSnapshots bool) {
-	var (
-		backupName, restoreName string
-	)
-	upgradeFromVeleroCLI := upgradeFromVeleroCLI
-
-	client, err := newTestClient()
-	Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
-
-	BeforeEach(func() {
-		if (len(upgradeFromVeleroVersion)) == 0 {
-			Skip("An original velero version is required to run upgrade test, please run test with upgrade-from-velero-version=<version>")
-		}
-		if useVolumeSnapshots && cloudProvider == "kind" {
-			Skip("Volume snapshots not supported on kind")
-		}
-		//Assume tag of velero server image is identical to velero CLI version
-		//Download velero CLI if it's empty according to velero CLI version
-		if (len(upgradeFromVeleroCLI)) == 0 {
-			upgradeFromVeleroCLI, err = installVeleroCLI(upgradeFromVeleroVersion)
-			Expect(err).To(Succeed())
-		}
-
-		var err error
-		flag.Parse()
-		uuidgen, err = uuid.NewRandom()
-		Expect(err).To(Succeed())
-		if installVelero {
-			//Set veleroImage and resticHelperImage to blank
-			//veleroImage and resticHelperImage should be the default value in originalCli
-			Expect(veleroInstall(context.Background(), upgradeFromVeleroCLI, "", "", "", veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots,
-				cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, "", "", registryCredentialFile)).To(Succeed())
-			Expect(checkVeleroVersion(context.Background(), upgradeFromVeleroCLI, upgradeFromVeleroVersion)).To(Succeed())
-		} else {
-			Skip("Upgrade test is skipped since user don't want to install any other velero")
-		}
-	})
-
-	AfterEach(func() {
-		if installVelero {
-			err = veleroUninstall(context.Background(), veleroCLI, veleroNamespace)
-			Expect(err).To(Succeed())
-		}
-	})
-
-	When("kibishii is the sample workload", func() {
-		It("should be successfully backed up and restored to the default BackupStorageLocation", func() {
-			backupName = "backup-" + uuidgen.String()
-			restoreName = "restore-" + uuidgen.String()
-			Expect(runUpgradeTests(client, veleroImage, veleroVersion, cloudProvider, upgradeFromVeleroCLI, veleroNamespace, backupName, restoreName, "", useVolumeSnapshots, registryCredentialFile)).To(Succeed(),
-				"Failed to successfully backup and restore Kibishii namespace")
-		})
-	})
-}
-
-// runUpgradeTests runs upgrade test on the provider by kibishii.
-func runUpgradeTests(client testClient, upgradeToVeleroImage, upgradeToVeleroVersion, providerName, upgradeFromVeleroCLI, veleroNamespace, backupName, restoreName, backupLocation string,
-	useVolumeSnapshots bool, registryCredentialFile string) error {
-	oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
-	if err := createNamespace(oneHourTimeout, client, upgradeNamespace); err != nil {
-		return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", upgradeNamespace)
-	}
-	defer func() {
-		if err := deleteNamespace(context.Background(), client, upgradeNamespace, true); err != nil {
-			fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", upgradeNamespace))
-		}
-	}()
-	if err := kibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, upgradeNamespace, registryCredentialFile); err != nil {
-		return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", upgradeNamespace)
-	}
-
-	if err := veleroBackupNamespace(oneHourTimeout, upgradeFromVeleroCLI, veleroNamespace, backupName, upgradeNamespace, backupLocation, useVolumeSnapshots); err != nil {
-		// TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command
-		// TODO move to "runDebug" after we bump up to 1.7 in the upgrade case
-		veleroBackupLogs(context.Background(), upgradeFromVeleroCLI, veleroNamespace, backupName)
-		return errors.Wrapf(err, "Failed to backup kibishii namespace %s", upgradeNamespace)
-	}
-
-	if providerName == "vsphere" && useVolumeSnapshots {
-		// Wait for uploads started by the Velero Plug-in for vSphere to complete
-		// TODO - remove after upload progress monitoring is implemented
-		fmt.Println("Waiting for vSphere uploads to complete")
-		if err := waitForVSphereUploadCompletion(oneHourTimeout, time.Hour, upgradeNamespace); err != nil {
-			return errors.Wrapf(err, "Error waiting for uploads to complete")
-		}
-	}
-	fmt.Printf("Simulating a disaster by removing namespace %s\n", upgradeNamespace)
-	if err := deleteNamespace(oneHourTimeout, client, upgradeNamespace, true); err != nil {
-		return errors.Wrapf(err, "failed to delete namespace %s", upgradeNamespace)
-	}
-
-	// the snapshots of AWS may be still in pending status when do the restore, wait for a while
-	// to avoid this https://github.com/vmware-tanzu/velero/issues/1799
-	// TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed
-	if providerName == "aws" && useVolumeSnapshots {
-		fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...")
-		time.Sleep(5 * time.Minute)
-	}
-
-	if err := veleroInstall(context.Background(), veleroCLI, upgradeToVeleroImage, resticHelperImage, plugins, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots,
-		cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile); err != nil {
-		return errors.Wrapf(err, "Failed to install velero from image %s", upgradeToVeleroImage)
-	}
-	if err := checkVeleroVersion(context.Background(), veleroCLI, upgradeToVeleroVersion); err != nil {
-		return errors.Wrapf(err, "Velero install version mismatch.")
-	}
-	if err := veleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil {
-		runDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName)
-		return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName)
-	}
-
-	if err := kibishiiVerifyAfterRestore(client, upgradeNamespace, oneHourTimeout); err != nil {
-		return errors.Wrapf(err, "Error verifying kibishii after restore")
-	}
-
-	fmt.Printf("Upgrade test completed successfully\n")
-	return nil
-}

test/e2e/util/k8s/client.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e
+package k8s
 
 import (
 	"k8s.io/client-go/kubernetes"
@@ -23,10 +23,10 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/client"
 )
 
-// testClient contains different API clients that are in use throughout
+// TestClient contains different API clients that are in use throughout
 // the e2e tests.
 
-type testClient struct {
+type TestClient struct {
 	kubebuilder kbclient.Client
 
 	// clientGo returns a client-go API client.
@@ -34,7 +34,7 @@ type testClient struct {
 	// Deprecated, TODO(2.0): presuming all controllers and resources are converted to the
 	// controller runtime framework by v2.0, it is the intent to remove all
 	// client-go API clients. Please use the controller runtime to make API calls for tests.
-	clientGo kubernetes.Interface
+	ClientGo kubernetes.Interface
 
 	// dynamicFactory returns a client-go API client for retrieving dynamic clients
 	// for GroupVersionResources and GroupVersionKinds.
@@ -45,35 +45,35 @@ type testClient struct {
 	dynamicFactory client.DynamicFactory
 }
 
-// newTestClient returns a set of ready-to-use API clients.
+// k8sutils.NewTestClient returns a set of ready-to-use API clients.
-func newTestClient() (testClient, error) {
+func NewTestClient() (TestClient, error) {
 	config, err := client.LoadConfig()
 	if err != nil {
-		return testClient{}, err
+		return TestClient{}, err
 	}
 
 	f := client.NewFactory("e2e", config)
 
 	clientGo, err := f.KubeClient()
 	if err != nil {
-		return testClient{}, err
+		return TestClient{}, err
 	}
 
 	kb, err := f.KubebuilderClient()
 	if err != nil {
-		return testClient{}, err
+		return TestClient{}, err
 	}
 
 	dynamicClient, err := f.DynamicClient()
 	if err != nil {
-		return testClient{}, err
+		return TestClient{}, err
 	}
 
 	factory := client.NewDynamicFactory(dynamicClient)
 
-	return testClient{
+	return TestClient{
 		kubebuilder:    kb,
-		clientGo:       clientGo,
+		ClientGo:       clientGo,
 		dynamicFactory: factory,
 	}, nil
 }

test/e2e/util/k8s/common.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e
+package k8s
 
 import (
 	"fmt"
@@ -32,11 +32,11 @@ import (
 )
 
 // ensureClusterExists returns whether or not a kubernetes cluster exists for tests to be run on.
-func ensureClusterExists(ctx context.Context) error {
+func EnsureClusterExists(ctx context.Context) error {
 	return exec.CommandContext(ctx, "kubectl", "cluster-info").Run()
 }
 
-func createSecretFromFiles(ctx context.Context, client testClient, namespace string, name string, files map[string]string) error {
+func CreateSecretFromFiles(ctx context.Context, client TestClient, namespace string, name string, files map[string]string) error {
 	data := make(map[string][]byte)
 
 	for key, filePath := range files {
@@ -49,17 +49,17 @@ func createSecretFromFiles(ctx context.Context, client testClient, namespace str
 	}
 
 	secret := builder.ForSecret(namespace, name).Data(data).Result()
-	_, err := client.clientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
+	_, err := client.ClientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
 	return err
 }
 
-// waitForPods waits until all of the pods have gone to PodRunning state
+// WaitForPods waits until all of the pods have gone to PodRunning state
-func waitForPods(ctx context.Context, client testClient, namespace string, pods []string) error {
+func WaitForPods(ctx context.Context, client TestClient, namespace string, pods []string) error {
 	timeout := 10 * time.Minute
 	interval := 5 * time.Second
 	err := wait.PollImmediate(interval, timeout, func() (bool, error) {
 		for _, podName := range pods {
-			checkPod, err := client.clientGo.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{})
+			checkPod, err := client.ClientGo.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{})
 			if err != nil {
 				return false, errors.WithMessage(err, fmt.Sprintf("Failed to verify pod %s/%s is %s", namespace, podName, corev1api.PodRunning))
 			}

test/e2e/util/k8s/namespace.go

@@ -14,11 +14,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e
+package k8s
 
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -27,26 +28,27 @@ import (
 	corev1api "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	waitutil "k8s.io/apimachinery/pkg/util/wait"
 
 	"github.com/vmware-tanzu/velero/pkg/builder"
 )
 
-func createNamespace(ctx context.Context, client testClient, namespace string) error {
+func CreateNamespace(ctx context.Context, client TestClient, namespace string) error {
 	ns := builder.ForNamespace(namespace).Result()
-	_, err := client.clientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{})
+	_, err := client.ClientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{})
 	if apierrors.IsAlreadyExists(err) {
 		return nil
 	}
 	return err
 }
 
-func getNamespace(ctx context.Context, client testClient, namespace string) (*corev1api.Namespace, error) {
+func GetNamespace(ctx context.Context, client TestClient, namespace string) (*corev1api.Namespace, error) {
-	return client.clientGo.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{})
+	return client.ClientGo.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{})
 }
 
-func deleteNamespace(ctx context.Context, client testClient, namespace string, wait bool) error {
+func DeleteNamespace(ctx context.Context, client TestClient, namespace string, wait bool) error {
-	if err := client.clientGo.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}); err != nil {
+	if err := client.ClientGo.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}); err != nil {
 		return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace))
 	}
 	if !wait {
@@ -55,7 +57,7 @@ func deleteNamespace(ctx context.Context, client testClient, namespace string, w
 
 	return waitutil.PollImmediateInfinite(5*time.Second,
 		func() (bool, error) {
-			if _, err := client.clientGo.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}); err != nil {
+			if _, err := client.ClientGo.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}); err != nil {
 				if apierrors.IsNotFound(err) {
 					return true, nil
 				}
@@ -65,3 +67,19 @@ func deleteNamespace(ctx context.Context, client testClient, namespace string, w
 			return false, nil
 		})
 }
+
+func CleanupNamespaces(ctx context.Context, client TestClient, nsBaseName string) error {
+	namespaces, err := client.ClientGo.CoreV1().Namespaces().List(ctx, v1.ListOptions{})
+	if err != nil {
+		return errors.Wrap(err, "Could not retrieve namespaces")
+	}
+	for _, checkNamespace := range namespaces.Items {
+		if strings.HasPrefix(checkNamespace.Name, nsBaseName) {
+			err = client.ClientGo.CoreV1().Namespaces().Delete(ctx, checkNamespace.Name, v1.DeleteOptions{})
+			if err != nil {
+				return errors.Wrapf(err, "Could not delete namespace %s", checkNamespace.Name)
+			}
+		}
+	}
+	return nil
+}

test/e2e/util/k8s/serviceaccount.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e
+package k8s
 
 import (
 	"context"
@@ -33,10 +33,10 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-func waitUntilServiceAccountCreated(ctx context.Context, client testClient, namespace, serviceAccount string, timeout time.Duration) error {
+func WaitUntilServiceAccountCreated(ctx context.Context, client TestClient, namespace, serviceAccount string, timeout time.Duration) error {
 	return wait.PollImmediate(5*time.Second, timeout,
 		func() (bool, error) {
-			if _, err := client.clientGo.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccount, metav1.GetOptions{}); err != nil {
+			if _, err := client.ClientGo.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccount, metav1.GetOptions{}); err != nil {
 				if !apierrors.IsNotFound(err) {
 					return false, err
 				}
@@ -46,7 +46,7 @@ func waitUntilServiceAccountCreated(ctx context.Context, client testClient, name
 		})
 }
 
-func patchServiceAccountWithImagePullSecret(ctx context.Context, client testClient, namespace, serviceAccount, dockerCredentialFile string) error {
+func PatchServiceAccountWithImagePullSecret(ctx context.Context, client TestClient, namespace, serviceAccount, dockerCredentialFile string) error {
 	credential, err := ioutil.ReadFile(dockerCredentialFile)
 	if err != nil {
 		return errors.Wrapf(err, "failed to read the docker credential file %q", dockerCredentialFile)
@@ -54,11 +54,11 @@ func patchServiceAccountWithImagePullSecret(ctx context.Context, client testClie
 	secretName := "image-pull-secret"
 	secret := builder.ForSecret(namespace, secretName).Data(map[string][]byte{".dockerconfigjson": credential}).Result()
 	secret.Type = corev1.SecretTypeDockerConfigJson
-	if _, err = client.clientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil {
+	if _, err = client.ClientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil {
 		return errors.Wrapf(err, "failed to create secret %q under namespace %q", secretName, namespace)
 	}
 
-	if _, err = client.clientGo.CoreV1().ServiceAccounts(namespace).Patch(ctx, serviceAccount, types.StrategicMergePatchType,
+	if _, err = client.ClientGo.CoreV1().ServiceAccounts(namespace).Patch(ctx, serviceAccount, types.StrategicMergePatchType,
 		[]byte(fmt.Sprintf(`{"imagePullSecrets": [{"name": "%s"}]}`, secretName)), metav1.PatchOptions{}); err != nil {
 		return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccount, namespace)
 	}

test/e2e/util/kibishii/kibishii_utils.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e
+package kibishii
 
 import (
 	"fmt"
@@ -26,6 +26,8 @@ import (
 	"golang.org/x/net/context"
 
 	veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
+	k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
+	veleroutils "github.com/vmware-tanzu/velero/test/e2e/util/velero"
 )
 
 const (
@@ -33,25 +35,24 @@ const (
 	jumpPadPod        = "jump-pad"
 )
 
-// runKibishiiTests runs kibishii tests on the provider.
+// RunKibishiiTests runs kibishii tests on the provider.
-func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string,
+func RunKibishiiTests(client k8sutils.TestClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string,
 	useVolumeSnapshots bool, registryCredentialFile string) error {
 	oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
-
+	if err := k8sutils.CreateNamespace(oneHourTimeout, client, kibishiiNamespace); err != nil {
-	if err := createNamespace(oneHourTimeout, client, kibishiiNamespace); err != nil {
 		return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace)
 	}
 	defer func() {
-		if err := deleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil {
+		if err := k8sutils.DeleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil {
 			fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", kibishiiNamespace))
 		}
 	}()
-	if err := kibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, kibishiiNamespace, registryCredentialFile); err != nil {
+	if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, kibishiiNamespace, registryCredentialFile); err != nil {
 		return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", kibishiiNamespace)
 	}
 
-	if err := veleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots); err != nil {
+	if err := veleroutils.VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots); err != nil {
-		runDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "")
+		veleroutils.RunDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "")
 		return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace)
 	}
 
@@ -59,12 +60,12 @@ func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespac
 		// Wait for uploads started by the Velero Plug-in for vSphere to complete
 		// TODO - remove after upload progress monitoring is implemented
 		fmt.Println("Waiting for vSphere uploads to complete")
-		if err := waitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil {
+		if err := veleroutils.WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil {
 			return errors.Wrapf(err, "Error waiting for uploads to complete")
 		}
 	}
 	fmt.Printf("Simulating a disaster by removing namespace %s\n", kibishiiNamespace)
-	if err := deleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil {
+	if err := k8sutils.DeleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil {
 		return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace)
 	}
 
@@ -76,12 +77,12 @@ func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespac
 		time.Sleep(5 * time.Minute)
 	}
 
-	if err := veleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil {
+	if err := veleroutils.VeleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil {
-		runDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName)
+		veleroutils.RunDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName)
 		return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName)
 	}
 
-	if err := kibishiiVerifyAfterRestore(client, kibishiiNamespace, oneHourTimeout); err != nil {
+	if err := KibishiiVerifyAfterRestore(client, kibishiiNamespace, oneHourTimeout); err != nil {
 		return errors.Wrapf(err, "Error verifying kibishii after restore")
 	}
 
@@ -144,19 +145,19 @@ func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel
 	return nil
 }
 
-func waitForKibishiiPods(ctx context.Context, client testClient, kibishiiNamespace string) error {
+func waitForKibishiiPods(ctx context.Context, client k8sutils.TestClient, kibishiiNamespace string) error {
-	return waitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"})
+	return k8sutils.WaitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"})
 }
 
-func kibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client testClient, providerName, kibishiiNamespace, registryCredentialFile string) error {
+func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client k8sutils.TestClient, providerName, kibishiiNamespace, registryCredentialFile string) error {
 	serviceAccountName := "default"
 
 	// wait until the service account is created before patch the image pull secret
-	if err := waitUntilServiceAccountCreated(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, 10*time.Minute); err != nil {
+	if err := k8sutils.WaitUntilServiceAccountCreated(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, 10*time.Minute); err != nil {
 		return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, kibishiiNamespace)
 	}
 	// add the image pull secret to avoid the image pull limit issue of Docker Hub
-	if err := patchServiceAccountWithImagePullSecret(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, registryCredentialFile); err != nil {
+	if err := k8sutils.PatchServiceAccountWithImagePullSecret(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, registryCredentialFile); err != nil {
 		return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, kibishiiNamespace)
 	}
 
@@ -177,7 +178,7 @@ func kibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client testClie
 	return nil
 }
 
-func kibishiiVerifyAfterRestore(client testClient, kibishiiNamespace string, oneHourTimeout context.Context) error {
+func KibishiiVerifyAfterRestore(client k8sutils.TestClient, kibishiiNamespace string, oneHourTimeout context.Context) error {
 	// wait for kibishii pod startup
 	// TODO - Fix kibishii so we can check that it is ready to go
 	fmt.Printf("Waiting for kibishii pods to be ready\n")

test/e2e/util/velero/install.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e
+package velero
 
 import (
 	"bytes"
@@ -35,6 +35,8 @@ import (
 
 	"github.com/vmware-tanzu/velero/pkg/cmd/cli/install"
 	velerexec "github.com/vmware-tanzu/velero/pkg/util/exec"
+	. "github.com/vmware-tanzu/velero/test/e2e"
+	k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
 )
 
 // we provide more install options other than the standard install.InstallOptions in E2E test
@@ -44,53 +46,49 @@ type installOptions struct {
 	ResticHelperImage      string
 }
 
-// TODO too many parameters for this function, better to make it a structure, we can introduces a structure `config` for the E2E to hold all configuration items
+func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, features string, useVolumeSnapshots bool) error {
-func veleroInstall(ctx context.Context, cli, veleroImage, resticHelperImage, providerPlugins, veleroNamespace, cloudProvider, objectStoreProvider string, useVolumeSnapshots bool,
+	if veleroCfg.CloudProvider != "kind" {
-	cloudCredentialsFile string, bslBucket string, bslPrefix string, bslConfig string, vslConfig string,
+		if veleroCfg.ObjectStoreProvider != "" {
-	crdsVersion string, features string, registryCredentialFile string) error {
-
-	if cloudProvider != "kind" {
-		if objectStoreProvider != "" {
 			return errors.New("For cloud platforms, object store plugin cannot be overridden") // Can't set an object store provider that is different than your cloud
 		}
-		objectStoreProvider = cloudProvider
+		veleroCfg.ObjectStoreProvider = veleroCfg.CloudProvider
 	} else {
-		if objectStoreProvider == "" {
+		if veleroCfg.ObjectStoreProvider == "" {
 			return errors.New("No object store provider specified - must be specified when using kind as the cloud provider") // Gotta have an object store provider
 		}
 	}
 
-	providerPluginsTmp, err := getProviderPlugins(ctx, cli, objectStoreProvider, providerPlugins)
+	providerPluginsTmp, err := getProviderPlugins(ctx, veleroCfg.VeleroCLI, veleroCfg.ObjectStoreProvider, veleroCfg.Plugins)
 	if err != nil {
 		return errors.WithMessage(err, "Failed to get provider plugins")
 	}
 	// TODO - handle this better
-	if cloudProvider == "vsphere" {
+	if veleroCfg.CloudProvider == "vsphere" {
-		// We overrider the objectStoreProvider here for vSphere because we want to use the aws plugin for the
+		// We overrider the ObjectStoreProvider here for vSphere because we want to use the aws plugin for the
 		// backup, but needed to pick up the provider plugins earlier.  vSphere plugin no longer needs a Volume
 		// Snapshot location specified
-		objectStoreProvider = "aws"
+		veleroCfg.ObjectStoreProvider = "aws"
 	}
-	err = ensureClusterExists(ctx)
+	err = k8sutils.EnsureClusterExists(ctx)
 	if err != nil {
 		return errors.WithMessage(err, "Failed to ensure Kubernetes cluster exists")
 	}
 
-	veleroInstallOptions, err := getProviderVeleroInstallOptions(objectStoreProvider, cloudCredentialsFile, bslBucket,
+	veleroInstallOptions, err := getProviderVeleroInstallOptions(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
-		bslPrefix, bslConfig, vslConfig, providerPluginsTmp, features)
+		veleroCfg.BSLPrefix, veleroCfg.BSLConfig, veleroCfg.VSLConfig, providerPluginsTmp, features)
 	if err != nil {
-		return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", objectStoreProvider)
+		return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", veleroCfg.ObjectStoreProvider)
 	}
 	veleroInstallOptions.UseVolumeSnapshots = useVolumeSnapshots
 	veleroInstallOptions.UseRestic = !useVolumeSnapshots
-	veleroInstallOptions.Image = veleroImage
+	veleroInstallOptions.Image = veleroCfg.VeleroImage
-	veleroInstallOptions.CRDsVersion = crdsVersion
+	veleroInstallOptions.CRDsVersion = veleroCfg.CRDsVersion
-	veleroInstallOptions.Namespace = veleroNamespace
+	veleroInstallOptions.Namespace = veleroCfg.VeleroNamespace
 
-	err = installVeleroServer(ctx, cli, &installOptions{
+	err = installVeleroServer(ctx, veleroCfg.VeleroCLI, &installOptions{
 		InstallOptions:         veleroInstallOptions,
-		RegistryCredentialFile: registryCredentialFile,
+		RegistryCredentialFile: veleroCfg.RegistryCredentialFile,
-		ResticHelperImage:      resticHelperImage,
+		ResticHelperImage:      veleroCfg.ResticHelperImage,
 	})
 	if err != nil {
 		return errors.WithMessagef(err, "Failed to install Velero in the cluster")
@@ -195,7 +193,7 @@ func createVelereResources(ctx context.Context, cli, namespace string, args []st
 		return errors.Wrapf(err, "failed to unmarshal the resources: %s", string(stdout))
 	}
 
-	if err = patchResources(ctx, resources, namespace, registryCredentialFile, resticHelperImage); err != nil {
+	if err = patchResources(ctx, resources, namespace, registryCredentialFile, VeleroCfg.ResticHelperImage); err != nil {
 		return errors.Wrapf(err, "failed to patch resources")
 	}
 
@@ -261,7 +259,7 @@ func patchResources(ctx context.Context, resources *unstructured.UnstructuredLis
 	}
 
 	// customize the restic restore helper image
-	if len(resticHelperImage) > 0 {
+	if len(VeleroCfg.ResticHelperImage) > 0 {
 		restoreActionConfig := corev1.ConfigMap{
 			TypeMeta: metav1.TypeMeta{
 				Kind:       "ConfigMap",
@@ -276,7 +274,7 @@ func patchResources(ctx context.Context, resources *unstructured.UnstructuredLis
 				},
 			},
 			Data: map[string]string{
-				"image": resticHelperImage,
+				"image": VeleroCfg.ResticHelperImage,
 			},
 		}
 
@@ -337,7 +335,7 @@ func waitVeleroReady(ctx context.Context, namespace string, useRestic bool) erro
 	return nil
 }
 
-func veleroUninstall(ctx context.Context, cli, namespace string) error {
+func VeleroUninstall(ctx context.Context, cli, namespace string) error {
 	stdout, stderr, err := velerexec.RunCommand(exec.CommandContext(ctx, cli, "uninstall", "--force", "-n", namespace))
 	if err != nil {
 		return errors.Wrapf(err, "failed to uninstall velero, stdout=%s, stderr=%s", stdout, stderr)

test/e2e/util/velero/velero_utils.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package e2e
+package velero
 
 import (
 	"bytes"
@@ -98,7 +98,7 @@ func getProviderVeleroInstallOptions(
 	pluginProvider,
 	credentialsFile,
 	objectStoreBucket,
-	objectStorePrefix string,
+	objectStorePrefix,
 	bslConfig,
 	vslConfig string,
 	plugins []string,
@@ -134,7 +134,7 @@ func getProviderVeleroInstallOptions(
 	return io, nil
 }
 
-// checkBackupPhase uses veleroCLI to inspect the phase of a Velero backup.
+// checkBackupPhase uses VeleroCLI to inspect the phase of a Velero backup.
 func checkBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string,
 	expectedPhase velerov1api.BackupPhase) error {
 	checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "get", "-o", "json",
@@ -178,7 +178,7 @@ func checkBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace str
 	return nil
 }
 
-// checkRestorePhase uses veleroCLI to inspect the phase of a Velero restore.
+// checkRestorePhase uses VeleroCLI to inspect the phase of a Velero restore.
 func checkRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string,
 	expectedPhase velerov1api.RestorePhase) error {
 	checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "restore", "get", "-o", "json",
@@ -222,8 +222,8 @@ func checkRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace st
 	return nil
 }
 
-// veleroBackupNamespace uses the veleroCLI to backup a namespace.
+// VeleroBackupNamespace uses the veleroCLI to backup a namespace.
-func veleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string, backupLocation string,
+func VeleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string, backupLocation string,
 	useVolumeSnapshots bool) error {
 	args := []string{
 		"--namespace", veleroNamespace,
@@ -246,71 +246,80 @@ func veleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespac
 		args = append(args, "--storage-location", backupLocation)
 	}
 
-	backupCmd := exec.CommandContext(ctx, veleroCLI, args...)
+	return VeleroBackupExec(ctx, veleroCLI, veleroNamespace, backupName, args)
-	backupCmd.Stdout = os.Stdout
-	backupCmd.Stderr = os.Stderr
-	fmt.Printf("backup cmd =%v\n", backupCmd)
-	err := backupCmd.Run()
-	if err != nil {
-		return err
-	}
-	err = checkBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted)
-
-	return err
 }
 
-// veleroBackupExcludeNamespaces uses the veleroCLI to backup a namespace.
+// VeleroBackupExcludeNamespaces uses the veleroCLI to backup a namespace.
-func veleroBackupExcludeNamespaces(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, excludeNamespaces []string) error {
+func VeleroBackupExcludeNamespaces(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, excludeNamespaces []string) error {
 	namespaces := strings.Join(excludeNamespaces, ",")
-	backupCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "create", "backup", backupName,
+	args := []string{
+		"--namespace", veleroNamespace, "create", "backup", backupName,
 		"--exclude-namespaces", namespaces,
-		"--default-volumes-to-restic", "--wait")
+		"--default-volumes-to-restic", "--wait",
-	backupCmd.Stdout = os.Stdout
-	backupCmd.Stderr = os.Stderr
-	fmt.Printf("backup cmd =%v\n", backupCmd)
-	err := backupCmd.Run()
-	if err != nil {
-		return err
 	}
-	err = checkBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted)
+	return VeleroBackupExec(ctx, veleroCLI, veleroNamespace, backupName, args)
-
-	return err
 }
 
-// veleroRestore uses the veleroCLI to restore from a Velero backup.
+// VeleroBackupIncludeNamespaces uses the veleroCLI to backup a namespace.
-func veleroRestore(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, backupName string) error {
+func VeleroBackupIncludeNamespaces(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, includeNamespaces []string) error {
-	restoreCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "create", "restore", restoreName,
+	namespaces := strings.Join(includeNamespaces, ",")
-		"--from-backup", backupName, "--wait")
+	args := []string{
+		"--namespace", veleroNamespace, "create", "backup", backupName,
+		"--include-namespaces", namespaces,
+		"--default-volumes-to-restic", "--wait",
+	}
+	return VeleroBackupExec(ctx, veleroCLI, veleroNamespace, backupName, args)
+}
 
-	restoreCmd.Stdout = os.Stdout
+// VeleroRestore uses the VeleroCLI to restore from a Velero backup.
-	restoreCmd.Stderr = os.Stderr
+func VeleroRestore(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, backupName string) error {
-	fmt.Printf("restore cmd =%v\n", restoreCmd)
+	args := []string{
-	err := restoreCmd.Run()
+		"--namespace", veleroNamespace, "create", "restore", restoreName,
-	if err != nil {
+		"--from-backup", backupName, "--wait",
+	}
+	return VeleroRestoreExec(ctx, veleroCLI, veleroNamespace, restoreName, args)
+}
+
+func VeleroRestoreExec(ctx context.Context, veleroCLI, veleroNamespace, restoreName string, args []string) error {
+	if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil {
 		return err
 	}
 	return checkRestorePhase(ctx, veleroCLI, veleroNamespace, restoreName, velerov1api.RestorePhaseCompleted)
 }
 
-func veleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error {
+func VeleroBackupExec(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, args []string) error {
-	describeCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "describe", backupName)
+	if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil {
-	describeCmd.Stdout = os.Stdout
-	describeCmd.Stderr = os.Stderr
-	err := describeCmd.Run()
-	if err != nil {
 		return err
 	}
-	logCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "logs", backupName)
+	return checkBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted)
-	logCmd.Stdout = os.Stdout
-	logCmd.Stderr = os.Stderr
-	err = logCmd.Run()
-	if err != nil {
-		return err
-	}
-	return nil
 }
 
-func runDebug(ctx context.Context, veleroCLI, veleroNamespace, backup, restore string) {
+func VeleroCmdExec(ctx context.Context, veleroCLI string, args []string) error {
+	cmd := exec.CommandContext(ctx, veleroCLI, args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	fmt.Printf("velero cmd =%v\n", cmd)
+	err := cmd.Run()
+	if err != nil {
+		return err
+	}
+	return err
+}
+
+func VeleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error {
+	args := []string{
+		"--namespace", veleroNamespace, "backup", "describe", backupName,
+	}
+	if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil {
+		return err
+	}
+	args = []string{
+		"--namespace", veleroNamespace, "backup", "logs", backupName,
+	}
+	return VeleroCmdExec(ctx, veleroCLI, args)
+}
+
+func RunDebug(ctx context.Context, veleroCLI, veleroNamespace, backup, restore string) {
 	output := fmt.Sprintf("debug-bundle-%d.tar.gz", time.Now().UnixNano())
 	args := []string{"debug", "--namespace", veleroNamespace, "--output", output, "--verbose"}
 	if len(backup) > 0 {
@@ -319,26 +328,21 @@ func runDebug(ctx context.Context, veleroCLI, veleroNamespace, backup, restore s
 	if len(restore) > 0 {
 		args = append(args, "--restore", restore)
 	}
-
-	cmd := exec.CommandContext(ctx, veleroCLI, args...)
-	cmd.Stdout = os.Stdout
-	cmd.Stdin = os.Stdin
-	fmt.Printf("debug cmd=%s\n", cmd.String())
 	fmt.Printf("Generating the debug tarball at %s\n", output)
-	if err := cmd.Run(); err != nil {
+	if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil {
 		fmt.Println(errors.Wrapf(err, "failed to run the debug command"))
 	}
 }
 
-func veleroCreateBackupLocation(ctx context.Context,
+func VeleroCreateBackupLocation(ctx context.Context,
-	veleroCLI string,
+	veleroCLI,
-	veleroNamespace string,
+	veleroNamespace,
-	name string,
+	name,
-	objectStoreProvider string,
+	objectStoreProvider,
-	bucket string,
+	bucket,
-	prefix string,
+	prefix,
-	config string,
+	config,
-	secretName string,
+	secretName,
 	secretKey string,
 ) error {
 	args := []string{
@@ -359,12 +363,7 @@ func veleroCreateBackupLocation(ctx context.Context,
 	if secretName != "" && secretKey != "" {
 		args = append(args, "--credential", fmt.Sprintf("%s=%s", secretName, secretKey))
 	}
-
+	return VeleroCmdExec(ctx, veleroCLI, args)
-	bslCreateCmd := exec.CommandContext(ctx, veleroCLI, args...)
-	bslCreateCmd.Stdout = os.Stdout
-	bslCreateCmd.Stderr = os.Stderr
-
-	return bslCreateCmd.Run()
 }
 
 func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, providerPlugins string) ([]string, error) {
@@ -385,9 +384,9 @@ func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, pro
 	return plugins, nil
 }
 
-// veleroAddPluginsForProvider determines which plugins need to be installed for a provider and
+// VeleroAddPluginsForProvider determines which plugins need to be installed for a provider and
 // installs them in the current Velero installation, skipping over those that are already installed.
-func veleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, addPlugins string) error {
+func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, addPlugins string) error {
 	plugins, err := getProviderPlugins(ctx, veleroCLI, provider, addPlugins)
 	if err != nil {
 		return errors.WithMessage(err, "Failed to get plugins")
@@ -417,9 +416,9 @@ func veleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNa
 	return nil
 }
 
-// waitForVSphereUploadCompletion waits for uploads started by the Velero Plug-in for vSphere to complete
+// WaitForVSphereUploadCompletion waits for uploads started by the Velero Plug-in for vSphere to complete
 // TODO - remove after upload progress monitoring is implemented
-func waitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, namespace string) error {
+func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, namespace string) error {
 	err := wait.PollImmediate(time.Minute, timeout, func() (bool, error) {
 		checkSnapshotCmd := exec.CommandContext(ctx, "kubectl",
 			"get", "-n", namespace, "snapshots.backupdriver.cnsdp.vmware.com", "-o=jsonpath='{range .items[*]}{.spec.resourceHandle.name}{\"=\"}{.status.phase}{\"\\n\"}{end}'")
@@ -447,6 +446,7 @@ func waitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration,
 			//             status will move to Canceling.  The snapshot ID will be removed from the status status if has been filled in
 			//             and the snapshot ID will not longer be valid for a Clone operation
 			// Canceled - the operation was canceled, the snapshot ID is not valid
+			// Canceled - the operation was canceled, the snapshot ID is not valid
 			if len(comps) == 2 {
 				phase := comps[1]
 				switch phase {
@@ -497,7 +497,7 @@ func getVeleroVersion(ctx context.Context, veleroCLI string, clientOnly bool) (s
 	return versionMatches[1], nil
 }
 
-func checkVeleroVersion(ctx context.Context, veleroCLI string, expectedVer string) error {
+func CheckVeleroVersion(ctx context.Context, veleroCLI string, expectedVer string) error {
 	tag := expectedVer
 	tagInstalled, err := getVeleroVersion(ctx, veleroCLI, false)
 	if err != nil {
@@ -510,7 +510,7 @@ func checkVeleroVersion(ctx context.Context, veleroCLI string, expectedVer strin
 	return nil
 }
 
-func installVeleroCLI(version string) (string, error) {
+func InstallVeleroCLI(version string) (string, error) {
 	name := "velero-" + version + "-" + runtime.GOOS + "-" + runtime.GOARCH
 	postfix := ".tar.gz"
 	tarball := name + postfix