Adjust structure of e2e test codes

Put every test moduels into seperate packages and all velero install parameters integrated into one struct

Signed-off-by: Ming <mqiu@vmware.com>

test/e2e/util/k8s/client.go

@@ -0,0 +1,79 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package k8s
+
+import (
+	"k8s.io/client-go/kubernetes"
+	kbclient "sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/vmware-tanzu/velero/pkg/client"
+)
+
+// TestClient contains different API clients that are in use throughout
+// the e2e tests.
+
+type TestClient struct {
+	kubebuilder kbclient.Client
+
+	// clientGo returns a client-go API client.
+	//
+	// Deprecated, TODO(2.0): presuming all controllers and resources are converted to the
+	// controller runtime framework by v2.0, it is the intent to remove all
+	// client-go API clients. Please use the controller runtime to make API calls for tests.
+	ClientGo kubernetes.Interface
+
+	// dynamicFactory returns a client-go API client for retrieving dynamic clients
+	// for GroupVersionResources and GroupVersionKinds.
+	//
+	// Deprecated, TODO(2.0): presuming all controllers and resources are converted to the
+	// controller runtime framework by v2.0, it is the intent to remove all
+	// client-go API clients. Please use the controller runtime to make API calls for tests.
+	dynamicFactory client.DynamicFactory
+}
+
+// k8sutils.NewTestClient returns a set of ready-to-use API clients.
+func NewTestClient() (TestClient, error) {
+	config, err := client.LoadConfig()
+	if err != nil {
+		return TestClient{}, err
+	}
+
+	f := client.NewFactory("e2e", config)
+
+	clientGo, err := f.KubeClient()
+	if err != nil {
+		return TestClient{}, err
+	}
+
+	kb, err := f.KubebuilderClient()
+	if err != nil {
+		return TestClient{}, err
+	}
+
+	dynamicClient, err := f.DynamicClient()
+	if err != nil {
+		return TestClient{}, err
+	}
+
+	factory := client.NewDynamicFactory(dynamicClient)
+
+	return TestClient{
+		kubebuilder:    kb,
+		ClientGo:       clientGo,
+		dynamicFactory: factory,
+	}, nil
+}

test/e2e/util/k8s/common.go

@@ -0,0 +1,79 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package k8s
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os/exec"
+	"time"
+
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	corev1api "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	"github.com/vmware-tanzu/velero/pkg/builder"
+)
+
+// ensureClusterExists returns whether or not a kubernetes cluster exists for tests to be run on.
+func EnsureClusterExists(ctx context.Context) error {
+	return exec.CommandContext(ctx, "kubectl", "cluster-info").Run()
+}
+
+func CreateSecretFromFiles(ctx context.Context, client TestClient, namespace string, name string, files map[string]string) error {
+	data := make(map[string][]byte)
+
+	for key, filePath := range files {
+		contents, err := ioutil.ReadFile(filePath)
+		if err != nil {
+			return errors.WithMessagef(err, "Failed to read secret file %q", filePath)
+		}
+
+		data[key] = contents
+	}
+
+	secret := builder.ForSecret(namespace, name).Data(data).Result()
+	_, err := client.ClientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
+	return err
+}
+
+// WaitForPods waits until all of the pods have gone to PodRunning state
+func WaitForPods(ctx context.Context, client TestClient, namespace string, pods []string) error {
+	timeout := 10 * time.Minute
+	interval := 5 * time.Second
+	err := wait.PollImmediate(interval, timeout, func() (bool, error) {
+		for _, podName := range pods {
+			checkPod, err := client.ClientGo.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{})
+			if err != nil {
+				return false, errors.WithMessage(err, fmt.Sprintf("Failed to verify pod %s/%s is %s", namespace, podName, corev1api.PodRunning))
+			}
+			// If any pod is still waiting we don't need to check any more so return and wait for next poll interval
+			if checkPod.Status.Phase != corev1api.PodRunning {
+				fmt.Printf("Pod %s is in state %s waiting for it to be %s\n", podName, checkPod.Status.Phase, corev1api.PodRunning)
+				return false, nil
+			}
+		}
+		// All pods were in PodRunning state, we're successful
+		return true, nil
+	})
+	if err != nil {
+		return errors.Wrapf(err, fmt.Sprintf("Failed to wait for pods in namespace %s to start running", namespace))
+	}
+	return nil
+}

test/e2e/util/k8s/namespace.go

@@ -0,0 +1,85 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package k8s
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+
+	corev1api "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	waitutil "k8s.io/apimachinery/pkg/util/wait"
+
+	"github.com/vmware-tanzu/velero/pkg/builder"
+)
+
+func CreateNamespace(ctx context.Context, client TestClient, namespace string) error {
+	ns := builder.ForNamespace(namespace).Result()
+	_, err := client.ClientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{})
+	if apierrors.IsAlreadyExists(err) {
+		return nil
+	}
+	return err
+}
+
+func GetNamespace(ctx context.Context, client TestClient, namespace string) (*corev1api.Namespace, error) {
+	return client.ClientGo.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{})
+}
+
+func DeleteNamespace(ctx context.Context, client TestClient, namespace string, wait bool) error {
+	if err := client.ClientGo.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace))
+	}
+	if !wait {
+		return nil
+	}
+
+	return waitutil.PollImmediateInfinite(5*time.Second,
+		func() (bool, error) {
+			if _, err := client.ClientGo.CoreV1().Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}); err != nil {
+				if apierrors.IsNotFound(err) {
+					return true, nil
+				}
+				return false, err
+			}
+			logrus.Debugf("namespace %q is still being deleted...", namespace)
+			return false, nil
+		})
+}
+
+func CleanupNamespaces(ctx context.Context, client TestClient, nsBaseName string) error {
+	namespaces, err := client.ClientGo.CoreV1().Namespaces().List(ctx, v1.ListOptions{})
+	if err != nil {
+		return errors.Wrap(err, "Could not retrieve namespaces")
+	}
+	for _, checkNamespace := range namespaces.Items {
+		if strings.HasPrefix(checkNamespace.Name, nsBaseName) {
+			err = client.ClientGo.CoreV1().Namespaces().Delete(ctx, checkNamespace.Name, v1.DeleteOptions{})
+			if err != nil {
+				return errors.Wrapf(err, "Could not delete namespace %s", checkNamespace.Name)
+			}
+		}
+	}
+	return nil
+}

test/e2e/util/k8s/serviceaccount.go

@@ -0,0 +1,66 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package k8s
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"time"
+
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	"github.com/vmware-tanzu/velero/pkg/builder"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func WaitUntilServiceAccountCreated(ctx context.Context, client TestClient, namespace, serviceAccount string, timeout time.Duration) error {
+	return wait.PollImmediate(5*time.Second, timeout,
+		func() (bool, error) {
+			if _, err := client.ClientGo.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccount, metav1.GetOptions{}); err != nil {
+				if !apierrors.IsNotFound(err) {
+					return false, err
+				}
+				return false, nil
+			}
+			return true, nil
+		})
+}
+
+func PatchServiceAccountWithImagePullSecret(ctx context.Context, client TestClient, namespace, serviceAccount, dockerCredentialFile string) error {
+	credential, err := ioutil.ReadFile(dockerCredentialFile)
+	if err != nil {
+		return errors.Wrapf(err, "failed to read the docker credential file %q", dockerCredentialFile)
+	}
+	secretName := "image-pull-secret"
+	secret := builder.ForSecret(namespace, secretName).Data(map[string][]byte{".dockerconfigjson": credential}).Result()
+	secret.Type = corev1.SecretTypeDockerConfigJson
+	if _, err = client.ClientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil {
+		return errors.Wrapf(err, "failed to create secret %q under namespace %q", secretName, namespace)
+	}
+
+	if _, err = client.ClientGo.CoreV1().ServiceAccounts(namespace).Patch(ctx, serviceAccount, types.StrategicMergePatchType,
+		[]byte(fmt.Sprintf(`{"imagePullSecrets": [{"name": "%s"}]}`, secretName)), metav1.PatchOptions{}); err != nil {
+		return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccount, namespace)
+	}
+	return nil
+}

test/e2e/util/kibishii/kibishii_utils.go

@@ -0,0 +1,195 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kibishii
+
+import (
+	"fmt"
+	"os/exec"
+	"strconv"
+	"time"
+
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+
+	veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
+	k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
+	veleroutils "github.com/vmware-tanzu/velero/test/e2e/util/velero"
+)
+
+const (
+	kibishiiNamespace = "kibishii-workload"
+	jumpPadPod        = "jump-pad"
+)
+
+// RunKibishiiTests runs kibishii tests on the provider.
+func RunKibishiiTests(client k8sutils.TestClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string,
+	useVolumeSnapshots bool, registryCredentialFile string) error {
+	oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
+	if err := k8sutils.CreateNamespace(oneHourTimeout, client, kibishiiNamespace); err != nil {
+		return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace)
+	}
+	defer func() {
+		if err := k8sutils.DeleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil {
+			fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", kibishiiNamespace))
+		}
+	}()
+	if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, kibishiiNamespace, registryCredentialFile); err != nil {
+		return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", kibishiiNamespace)
+	}
+
+	if err := veleroutils.VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots); err != nil {
+		veleroutils.RunDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "")
+		return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace)
+	}
+
+	if providerName == "vsphere" && useVolumeSnapshots {
+		// Wait for uploads started by the Velero Plug-in for vSphere to complete
+		// TODO - remove after upload progress monitoring is implemented
+		fmt.Println("Waiting for vSphere uploads to complete")
+		if err := veleroutils.WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil {
+			return errors.Wrapf(err, "Error waiting for uploads to complete")
+		}
+	}
+	fmt.Printf("Simulating a disaster by removing namespace %s\n", kibishiiNamespace)
+	if err := k8sutils.DeleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil {
+		return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace)
+	}
+
+	// the snapshots of AWS may be still in pending status when do the restore, wait for a while
+	// to avoid this https://github.com/vmware-tanzu/velero/issues/1799
+	// TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed
+	if providerName == "aws" && useVolumeSnapshots {
+		fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...")
+		time.Sleep(5 * time.Minute)
+	}
+
+	if err := veleroutils.VeleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil {
+		veleroutils.RunDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName)
+		return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName)
+	}
+
+	if err := KibishiiVerifyAfterRestore(client, kibishiiNamespace, oneHourTimeout); err != nil {
+		return errors.Wrapf(err, "Error verifying kibishii after restore")
+	}
+
+	fmt.Printf("kibishii test completed successfully\n")
+	return nil
+}
+
+func installKibishii(ctx context.Context, namespace string, cloudPlatform string) error {
+	// We use kustomize to generate YAML for Kibishii from the checked-in yaml directories
+	kibishiiInstallCmd := exec.CommandContext(ctx, "kubectl", "apply", "-n", namespace, "-k",
+		"github.com/vmware-tanzu-experiments/distributed-data-generator/kubernetes/yaml/"+cloudPlatform)
+	_, stderr, err := veleroexec.RunCommand(kibishiiInstallCmd)
+	if err != nil {
+		return errors.Wrapf(err, "failed to install kibishii, stderr=%s", stderr)
+	}
+
+	kibishiiSetWaitCmd := exec.CommandContext(ctx, "kubectl", "rollout", "status", "statefulset.apps/kibishii-deployment",
+		"-n", namespace, "-w", "--timeout=30m")
+	_, stderr, err = veleroexec.RunCommand(kibishiiSetWaitCmd)
+	if err != nil {
+		return errors.Wrapf(err, "failed to rollout, stderr=%s", stderr)
+	}
+
+	fmt.Printf("Waiting for kibishii jump-pad pod to be ready\n")
+	jumpPadWaitCmd := exec.CommandContext(ctx, "kubectl", "wait", "--for=condition=ready", "-n", namespace, "pod/jump-pad")
+	_, stderr, err = veleroexec.RunCommand(jumpPadWaitCmd)
+	if err != nil {
+		return errors.Wrapf(err, "Failed to wait for ready status of pod %s/%s, stderr=%s", namespace, jumpPadPod, stderr)
+	}
+
+	return err
+}
+
+func generateData(ctx context.Context, namespace string, levels int, filesPerLevel int, dirsPerLevel int, fileSize int,
+	blockSize int, passNum int, expectedNodes int) error {
+	kibishiiGenerateCmd := exec.CommandContext(ctx, "kubectl", "exec", "-n", namespace, "jump-pad", "--",
+		"/usr/local/bin/generate.sh", strconv.Itoa(levels), strconv.Itoa(filesPerLevel), strconv.Itoa(dirsPerLevel), strconv.Itoa(fileSize),
+		strconv.Itoa(blockSize), strconv.Itoa(passNum), strconv.Itoa(expectedNodes))
+	fmt.Printf("kibishiiGenerateCmd cmd =%v\n", kibishiiGenerateCmd)
+
+	_, stderr, err := veleroexec.RunCommand(kibishiiGenerateCmd)
+	if err != nil {
+		return errors.Wrapf(err, "failed to generate, stderr=%s", stderr)
+	}
+
+	return nil
+}
+
+func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel int, dirsPerLevel int, fileSize int,
+	blockSize int, passNum int, expectedNodes int) error {
+	kibishiiVerifyCmd := exec.CommandContext(ctx, "kubectl", "exec", "-n", namespace, "jump-pad", "--",
+		"/usr/local/bin/verify.sh", strconv.Itoa(levels), strconv.Itoa(filesPerLevel), strconv.Itoa(dirsPerLevel), strconv.Itoa(fileSize),
+		strconv.Itoa(blockSize), strconv.Itoa(passNum), strconv.Itoa(expectedNodes))
+	fmt.Printf("kibishiiVerifyCmd cmd =%v\n", kibishiiVerifyCmd)
+
+	stdout, stderr, err := veleroexec.RunCommand(kibishiiVerifyCmd)
+	if err != nil {
+		return errors.Wrapf(err, "failed to verify, stderr=%s, stdout=%s", stderr, stdout)
+	}
+	return nil
+}
+
+func waitForKibishiiPods(ctx context.Context, client k8sutils.TestClient, kibishiiNamespace string) error {
+	return k8sutils.WaitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"})
+}
+
+func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client k8sutils.TestClient, providerName, kibishiiNamespace, registryCredentialFile string) error {
+	serviceAccountName := "default"
+
+	// wait until the service account is created before patch the image pull secret
+	if err := k8sutils.WaitUntilServiceAccountCreated(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, 10*time.Minute); err != nil {
+		return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, kibishiiNamespace)
+	}
+	// add the image pull secret to avoid the image pull limit issue of Docker Hub
+	if err := k8sutils.PatchServiceAccountWithImagePullSecret(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, registryCredentialFile); err != nil {
+		return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, kibishiiNamespace)
+	}
+
+	if err := installKibishii(oneHourTimeout, kibishiiNamespace, providerName); err != nil {
+		return errors.Wrap(err, "Failed to install Kibishii workload")
+	}
+
+	// wait for kibishii pod startup
+	// TODO - Fix kibishii so we can check that it is ready to go
+	fmt.Printf("Waiting for kibishii pods to be ready\n")
+	if err := waitForKibishiiPods(oneHourTimeout, client, kibishiiNamespace); err != nil {
+		return errors.Wrapf(err, "Failed to wait for ready status of kibishii pods in %s", kibishiiNamespace)
+	}
+
+	if err := generateData(oneHourTimeout, kibishiiNamespace, 2, 10, 10, 1024, 1024, 0, 2); err != nil {
+		return errors.Wrap(err, "Failed to generate data")
+	}
+	return nil
+}
+
+func KibishiiVerifyAfterRestore(client k8sutils.TestClient, kibishiiNamespace string, oneHourTimeout context.Context) error {
+	// wait for kibishii pod startup
+	// TODO - Fix kibishii so we can check that it is ready to go
+	fmt.Printf("Waiting for kibishii pods to be ready\n")
+	if err := waitForKibishiiPods(oneHourTimeout, client, kibishiiNamespace); err != nil {
+		return errors.Wrapf(err, "Failed to wait for ready status of kibishii pods in %s", kibishiiNamespace)
+	}
+
+	// TODO - check that namespace exists
+	fmt.Printf("running kibishii verify\n")
+	if err := verifyData(oneHourTimeout, kibishiiNamespace, 2, 10, 10, 1024, 1024, 0, 2); err != nil {
+		return errors.Wrap(err, "Failed to verify data generated by kibishii")
+	}
+	return nil
+}

test/e2e/util/velero/install.go

@@ -0,0 +1,345 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package velero
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"time"
+
+	"github.com/pkg/errors"
+	apps "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	"github.com/vmware-tanzu/velero/pkg/cmd/cli/install"
+	velerexec "github.com/vmware-tanzu/velero/pkg/util/exec"
+	. "github.com/vmware-tanzu/velero/test/e2e"
+	k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
+)
+
+// we provide more install options other than the standard install.InstallOptions in E2E test
+type installOptions struct {
+	*install.InstallOptions
+	RegistryCredentialFile string
+	ResticHelperImage      string
+}
+
+func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, features string, useVolumeSnapshots bool) error {
+	if veleroCfg.CloudProvider != "kind" {
+		if veleroCfg.ObjectStoreProvider != "" {
+			return errors.New("For cloud platforms, object store plugin cannot be overridden") // Can't set an object store provider that is different than your cloud
+		}
+		veleroCfg.ObjectStoreProvider = veleroCfg.CloudProvider
+	} else {
+		if veleroCfg.ObjectStoreProvider == "" {
+			return errors.New("No object store provider specified - must be specified when using kind as the cloud provider") // Gotta have an object store provider
+		}
+	}
+
+	providerPluginsTmp, err := getProviderPlugins(ctx, veleroCfg.VeleroCLI, veleroCfg.ObjectStoreProvider, veleroCfg.Plugins)
+	if err != nil {
+		return errors.WithMessage(err, "Failed to get provider plugins")
+	}
+	// TODO - handle this better
+	if veleroCfg.CloudProvider == "vsphere" {
+		// We overrider the ObjectStoreProvider here for vSphere because we want to use the aws plugin for the
+		// backup, but needed to pick up the provider plugins earlier.  vSphere plugin no longer needs a Volume
+		// Snapshot location specified
+		veleroCfg.ObjectStoreProvider = "aws"
+	}
+	err = k8sutils.EnsureClusterExists(ctx)
+	if err != nil {
+		return errors.WithMessage(err, "Failed to ensure Kubernetes cluster exists")
+	}
+
+	veleroInstallOptions, err := getProviderVeleroInstallOptions(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
+		veleroCfg.BSLPrefix, veleroCfg.BSLConfig, veleroCfg.VSLConfig, providerPluginsTmp, features)
+	if err != nil {
+		return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", veleroCfg.ObjectStoreProvider)
+	}
+	veleroInstallOptions.UseVolumeSnapshots = useVolumeSnapshots
+	veleroInstallOptions.UseRestic = !useVolumeSnapshots
+	veleroInstallOptions.Image = veleroCfg.VeleroImage
+	veleroInstallOptions.CRDsVersion = veleroCfg.CRDsVersion
+	veleroInstallOptions.Namespace = veleroCfg.VeleroNamespace
+
+	err = installVeleroServer(ctx, veleroCfg.VeleroCLI, &installOptions{
+		InstallOptions:         veleroInstallOptions,
+		RegistryCredentialFile: veleroCfg.RegistryCredentialFile,
+		ResticHelperImage:      veleroCfg.ResticHelperImage,
+	})
+	if err != nil {
+		return errors.WithMessagef(err, "Failed to install Velero in the cluster")
+	}
+
+	return nil
+}
+
+func installVeleroServer(ctx context.Context, cli string, options *installOptions) error {
+	args := []string{"install"}
+	namespace := "velero"
+	if len(options.Namespace) > 0 {
+		args = append(args, "--namespace", options.Namespace)
+		namespace = options.Namespace
+	}
+	if len(options.CRDsVersion) > 0 {
+		args = append(args, "--crds-version", options.CRDsVersion)
+	}
+	if len(options.Image) > 0 {
+		args = append(args, "--image", options.Image)
+	}
+	if options.UseRestic {
+		args = append(args, "--use-restic")
+	}
+	if options.UseVolumeSnapshots {
+		args = append(args, "--use-volume-snapshots")
+	}
+	if len(options.ProviderName) > 0 {
+		args = append(args, "--provider", options.ProviderName)
+	}
+	if len(options.BackupStorageConfig.Data()) > 0 {
+		args = append(args, "--backup-location-config", options.BackupStorageConfig.String())
+	}
+	if len(options.BucketName) > 0 {
+		args = append(args, "--bucket", options.BucketName)
+	}
+	if len(options.Prefix) > 0 {
+		args = append(args, "--prefix", options.Prefix)
+	}
+	if len(options.SecretFile) > 0 {
+		args = append(args, "--secret-file", options.SecretFile)
+	}
+	if len(options.VolumeSnapshotConfig.Data()) > 0 {
+		args = append(args, "--snapshot-location-config", options.VolumeSnapshotConfig.String())
+	}
+	if len(options.Plugins) > 0 {
+		args = append(args, "--plugins", options.Plugins.String())
+	}
+	if len(options.Features) > 0 {
+		args = append(args, "--features", options.Features)
+	}
+
+	if err := createVelereResources(ctx, cli, namespace, args, options.RegistryCredentialFile, options.ResticHelperImage); err != nil {
+		return err
+	}
+
+	return waitVeleroReady(ctx, namespace, options.UseRestic)
+}
+
+func createVelereResources(ctx context.Context, cli, namespace string, args []string, registryCredentialFile, resticHelperImage string) error {
+	args = append(args, "--dry-run", "--output", "json", "--crds-only")
+
+	// get the CRD definitions
+	cmd := exec.CommandContext(ctx, cli, args...)
+	fmt.Printf("Running cmd %q \n", cmd.String())
+	stdout, stderr, err := velerexec.RunCommand(cmd)
+	if err != nil {
+		return errors.Wrapf(err, "failed to run velero install dry run command with \"--crds-only\" option, stdout=%s, stderr=%s", stdout, stderr)
+	}
+
+	// apply the CRDs first
+	cmd = exec.CommandContext(ctx, "kubectl", "apply", "-f", "-")
+	cmd.Stdin = bytes.NewBufferString(stdout)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	fmt.Printf("Applying velero CRDs...\n")
+	if err = cmd.Run(); err != nil {
+		return errors.Wrapf(err, "failed to apply the CRDs")
+	}
+
+	// wait until the CRDs are ready
+	cmd = exec.CommandContext(ctx, "kubectl", "wait", "--for", "condition=established", "-f", "-")
+	cmd.Stdin = bytes.NewBufferString(stdout)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	fmt.Printf("Waiting velero CRDs ready...\n")
+	if err = cmd.Run(); err != nil {
+		return errors.Wrapf(err, "failed to wait the CRDs be ready")
+	}
+
+	// remove the "--crds-only" option from the args
+	args = args[:len(args)-1]
+	cmd = exec.CommandContext(ctx, cli, args...)
+	fmt.Printf("Running cmd %q \n", cmd.String())
+	stdout, stderr, err = velerexec.RunCommand(cmd)
+	if err != nil {
+		return errors.Wrapf(err, "failed to run velero install dry run command, stdout=%s, stderr=%s", stdout, stderr)
+	}
+
+	resources := &unstructured.UnstructuredList{}
+	if err := json.Unmarshal([]byte(stdout), resources); err != nil {
+		return errors.Wrapf(err, "failed to unmarshal the resources: %s", string(stdout))
+	}
+
+	if err = patchResources(ctx, resources, namespace, registryCredentialFile, VeleroCfg.ResticHelperImage); err != nil {
+		return errors.Wrapf(err, "failed to patch resources")
+	}
+
+	data, err := json.Marshal(resources)
+	if err != nil {
+		return errors.Wrapf(err, "failed to marshal resources")
+	}
+
+	cmd = exec.CommandContext(ctx, "kubectl", "apply", "-f", "-")
+	cmd.Stdin = bytes.NewReader(data)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	fmt.Printf("Running cmd %q \n", cmd.String())
+	if err = cmd.Run(); err != nil {
+		return errors.Wrapf(err, "failed to apply velere resources")
+	}
+
+	return nil
+}
+
+// patch the velero resources
+func patchResources(ctx context.Context, resources *unstructured.UnstructuredList, namespace, registryCredentialFile, resticHelperImage string) error {
+	// apply the image pull secret to avoid the image pull limit of Docker Hub
+	if len(registryCredentialFile) > 0 {
+		credential, err := ioutil.ReadFile(registryCredentialFile)
+		if err != nil {
+			return errors.Wrapf(err, "failed to read the registry credential file %s", registryCredentialFile)
+		}
+
+		imagePullSecret := corev1.Secret{
+			TypeMeta: metav1.TypeMeta{
+				Kind:       "Secret",
+				APIVersion: corev1.SchemeGroupVersion.String(),
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "image-pull-secret",
+				Namespace: namespace,
+			},
+			Type: corev1.SecretTypeDockerConfigJson,
+			Data: map[string][]byte{
+				".dockerconfigjson": credential,
+			},
+		}
+
+		for resourceIndex, resource := range resources.Items {
+			if resource.GetKind() == "ServiceAccount" && resource.GetName() == "velero" {
+				resource.Object["imagePullSecrets"] = []map[string]interface{}{
+					{
+						"name": "image-pull-secret",
+					},
+				}
+				resources.Items[resourceIndex] = resource
+				fmt.Printf("image pull secret %q set for velero serviceaccount \n", "image-pull-secret")
+				break
+			}
+		}
+
+		un, err := toUnstructured(imagePullSecret)
+		if err != nil {
+			return errors.Wrapf(err, "failed to convert pull secret to unstructure")
+		}
+		resources.Items = append(resources.Items, un)
+	}
+
+	// customize the restic restore helper image
+	if len(VeleroCfg.ResticHelperImage) > 0 {
+		restoreActionConfig := corev1.ConfigMap{
+			TypeMeta: metav1.TypeMeta{
+				Kind:       "ConfigMap",
+				APIVersion: corev1.SchemeGroupVersion.String(),
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "restic-restore-action-config",
+				Namespace: namespace,
+				Labels: map[string]string{
+					"velero.io/plugin-config": "",
+					"velero.io/restic":        "RestoreItemAction",
+				},
+			},
+			Data: map[string]string{
+				"image": VeleroCfg.ResticHelperImage,
+			},
+		}
+
+		un, err := toUnstructured(restoreActionConfig)
+		if err != nil {
+			return errors.Wrapf(err, "failed to convert restore action config to unstructure")
+		}
+		resources.Items = append(resources.Items, un)
+		fmt.Printf("the restic restore helper image is set by the configmap %q \n", "restic-restore-action-config")
+	}
+
+	return nil
+}
+
+func toUnstructured(res interface{}) (unstructured.Unstructured, error) {
+	un := unstructured.Unstructured{}
+	data, err := json.Marshal(res)
+	if err != nil {
+		return un, err
+	}
+	err = json.Unmarshal(data, &un)
+	return un, err
+}
+
+func waitVeleroReady(ctx context.Context, namespace string, useRestic bool) error {
+	fmt.Println("Waiting for Velero deployment to be ready.")
+	// when doing upgrade by the "kubectl apply" the command "kubectl wait --for=condition=available deployment/velero -n velero --timeout=600s" returns directly
+	// use "rollout status" instead to avoid this. For more detail information, refer to https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#complete-deployment
+	stdout, stderr, err := velerexec.RunCommand(exec.CommandContext(ctx, "kubectl", "rollout", "status",
+		"deployment/velero", "-n", namespace))
+	if err != nil {
+		return errors.Wrapf(err, "fail to wait for the velero deployment ready, stdout=%s, stderr=%s", stdout, stderr)
+	}
+
+	if useRestic {
+		fmt.Println("Waiting for Velero restic daemonset to be ready.")
+		err := wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
+			stdout, stderr, err := velerexec.RunCommand(exec.CommandContext(ctx, "kubectl", "get", "daemonset/restic",
+				"-o", "json", "-n", namespace))
+			if err != nil {
+				return false, errors.Wrapf(err, "failed to get the restic daemonset, stdout=%s, stderr=%s", stdout, stderr)
+			}
+			restic := &apps.DaemonSet{}
+			if err = json.Unmarshal([]byte(stdout), restic); err != nil {
+				return false, errors.Wrapf(err, "failed to unmarshal the restic daemonset")
+			}
+			if restic.Status.DesiredNumberScheduled == restic.Status.NumberAvailable {
+				return true, nil
+			}
+			return false, nil
+		})
+		if err != nil {
+			return errors.Wrap(err, "fail to wait for the velero restic ready")
+		}
+	}
+
+	fmt.Printf("Velero is installed and ready to be tested in the %s namespace! ⛵ \n", namespace)
+	return nil
+}
+
+func VeleroUninstall(ctx context.Context, cli, namespace string) error {
+	stdout, stderr, err := velerexec.RunCommand(exec.CommandContext(ctx, cli, "uninstall", "--force", "-n", namespace))
+	if err != nil {
+		return errors.Wrapf(err, "failed to uninstall velero, stdout=%s, stderr=%s", stdout, stderr)
+	}
+	fmt.Println("Velero uninstalled ⛵")
+	return nil
+}

test/e2e/util/velero/velero_utils.go

@@ -0,0 +1,559 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package velero
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+	cliinstall "github.com/vmware-tanzu/velero/pkg/cmd/cli/install"
+	"github.com/vmware-tanzu/velero/pkg/cmd/util/flag"
+	veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
+)
+
+var pluginsMatrix = map[string]map[string][]string{
+	"v1.4": {
+		"aws":     {"velero/velero-plugin-for-aws:v1.1.0"},
+		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.1.2"},
+		"vsphere": {"velero/velero-plugin-for-aws:v1.1.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.0.2"},
+		"gcp":     {"velero/velero-plugin-for-gcp:v1.1.0"},
+	},
+	"v1.5": {
+		"aws":     {"velero/velero-plugin-for-aws:v1.1.0"},
+		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.1.2"},
+		"vsphere": {"velero/velero-plugin-for-aws:v1.1.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"},
+		"gcp":     {"velero/velero-plugin-for-gcp:v1.1.0"},
+	},
+	"v1.6": {
+		"aws":     {"velero/velero-plugin-for-aws:v1.2.1"},
+		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.2.1"},
+		"vsphere": {"velero/velero-plugin-for-aws:v1.2.1", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"},
+		"gcp":     {"velero/velero-plugin-for-gcp:v1.2.1"},
+	},
+	"v1.7": {
+		"aws":     {"velero/velero-plugin-for-aws:v1.3.0"},
+		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.3.0"},
+		"vsphere": {"velero/velero-plugin-for-aws:v1.3.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"},
+		"gcp":     {"velero/velero-plugin-for-gcp:v1.3.0"},
+	},
+	"main": {
+		"aws":     {"velero/velero-plugin-for-aws:main"},
+		"azure":   {"velero/velero-plugin-for-microsoft-azure:main"},
+		"vsphere": {"velero/velero-plugin-for-aws:main", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"},
+		"gcp":     {"velero/velero-plugin-for-gcp:main"},
+	},
+}
+
+func getProviderPluginsByVersion(version, providerName string) ([]string, error) {
+	var cloudMap map[string][]string
+	arr := strings.Split(version, ".")
+	if len(arr) >= 3 {
+		cloudMap = pluginsMatrix[arr[0]+"."+arr[1]]
+	}
+	if len(cloudMap) == 0 {
+		cloudMap = pluginsMatrix["main"]
+		if len(cloudMap) == 0 {
+			return nil, errors.Errorf("fail to get plugins by version: main")
+		}
+	}
+	plugins, ok := cloudMap[providerName]
+	if !ok {
+		return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, providerName)
+	}
+	return plugins, nil
+}
+
+// getProviderVeleroInstallOptions returns Velero InstallOptions for the provider.
+func getProviderVeleroInstallOptions(
+	pluginProvider,
+	credentialsFile,
+	objectStoreBucket,
+	objectStorePrefix,
+	bslConfig,
+	vslConfig string,
+	plugins []string,
+	features string,
+) (*cliinstall.InstallOptions, error) {
+
+	if credentialsFile == "" {
+		return nil, errors.Errorf("No credentials were supplied to use for E2E tests")
+	}
+
+	realPath, err := filepath.Abs(credentialsFile)
+	if err != nil {
+		return nil, err
+	}
+
+	io := cliinstall.NewInstallOptions()
+	// always wait for velero and restic pods to be running.
+	io.Wait = true
+	io.ProviderName = pluginProvider
+	io.SecretFile = credentialsFile
+
+	io.BucketName = objectStoreBucket
+	io.Prefix = objectStorePrefix
+	io.BackupStorageConfig = flag.NewMap()
+	io.BackupStorageConfig.Set(bslConfig)
+
+	io.VolumeSnapshotConfig = flag.NewMap()
+	io.VolumeSnapshotConfig.Set(vslConfig)
+
+	io.SecretFile = realPath
+	io.Plugins = flag.NewStringArray(plugins...)
+	io.Features = features
+	return io, nil
+}
+
+// checkBackupPhase uses VeleroCLI to inspect the phase of a Velero backup.
+func checkBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string,
+	expectedPhase velerov1api.BackupPhase) error {
+	checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "get", "-o", "json",
+		backupName)
+
+	fmt.Printf("get backup cmd =%v\n", checkCMD)
+	stdoutPipe, err := checkCMD.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	jsonBuf := make([]byte, 16*1024) // If the YAML is bigger than 16K, there's probably something bad happening
+
+	err = checkCMD.Start()
+	if err != nil {
+		return err
+	}
+
+	bytesRead, err := io.ReadFull(stdoutPipe, jsonBuf)
+
+	if err != nil && err != io.ErrUnexpectedEOF {
+		return err
+	}
+	if bytesRead == len(jsonBuf) {
+		return errors.New("yaml returned bigger than max allowed")
+	}
+
+	jsonBuf = jsonBuf[0:bytesRead]
+	err = checkCMD.Wait()
+	if err != nil {
+		return err
+	}
+	backup := velerov1api.Backup{}
+	err = json.Unmarshal(jsonBuf, &backup)
+	if err != nil {
+		return err
+	}
+	if backup.Status.Phase != expectedPhase {
+		return errors.Errorf("Unexpected backup phase got %s, expecting %s", backup.Status.Phase, expectedPhase)
+	}
+	return nil
+}
+
+// checkRestorePhase uses VeleroCLI to inspect the phase of a Velero restore.
+func checkRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string,
+	expectedPhase velerov1api.RestorePhase) error {
+	checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "restore", "get", "-o", "json",
+		restoreName)
+
+	fmt.Printf("get restore cmd =%v\n", checkCMD)
+	stdoutPipe, err := checkCMD.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	jsonBuf := make([]byte, 16*1024) // If the YAML is bigger than 16K, there's probably something bad happening
+
+	err = checkCMD.Start()
+	if err != nil {
+		return err
+	}
+
+	bytesRead, err := io.ReadFull(stdoutPipe, jsonBuf)
+
+	if err != nil && err != io.ErrUnexpectedEOF {
+		return err
+	}
+	if bytesRead == len(jsonBuf) {
+		return errors.New("yaml returned bigger than max allowed")
+	}
+
+	jsonBuf = jsonBuf[0:bytesRead]
+	err = checkCMD.Wait()
+	if err != nil {
+		return err
+	}
+	restore := velerov1api.Restore{}
+	err = json.Unmarshal(jsonBuf, &restore)
+	if err != nil {
+		return err
+	}
+	if restore.Status.Phase != expectedPhase {
+		return errors.Errorf("Unexpected restore phase got %s, expecting %s", restore.Status.Phase, expectedPhase)
+	}
+	return nil
+}
+
+// VeleroBackupNamespace uses the veleroCLI to backup a namespace.
+func VeleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string, backupLocation string,
+	useVolumeSnapshots bool) error {
+	args := []string{
+		"--namespace", veleroNamespace,
+		"create", "backup", backupName,
+		"--include-namespaces", namespace,
+		"--wait",
+	}
+
+	if useVolumeSnapshots {
+		args = append(args, "--snapshot-volumes")
+	} else {
+		args = append(args, "--default-volumes-to-restic")
+		// To workaround https://github.com/vmware-tanzu/velero-plugin-for-vsphere/issues/347 for vsphere plugin v1.1.1
+		// if the "--snapshot-volumes=false" isn't specified explicitly, the vSphere plugin will always take snapshots
+		// for the volumes even though the "--default-volumes-to-restic" is specified
+		// TODO This can be removed if the logic of vSphere plugin bump up to 1.3
+		args = append(args, "--snapshot-volumes=false")
+	}
+	if backupLocation != "" {
+		args = append(args, "--storage-location", backupLocation)
+	}
+
+	return VeleroBackupExec(ctx, veleroCLI, veleroNamespace, backupName, args)
+}
+
+// VeleroBackupExcludeNamespaces uses the veleroCLI to backup a namespace.
+func VeleroBackupExcludeNamespaces(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, excludeNamespaces []string) error {
+	namespaces := strings.Join(excludeNamespaces, ",")
+	args := []string{
+		"--namespace", veleroNamespace, "create", "backup", backupName,
+		"--exclude-namespaces", namespaces,
+		"--default-volumes-to-restic", "--wait",
+	}
+	return VeleroBackupExec(ctx, veleroCLI, veleroNamespace, backupName, args)
+}
+
+// VeleroBackupIncludeNamespaces uses the veleroCLI to backup a namespace.
+func VeleroBackupIncludeNamespaces(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, includeNamespaces []string) error {
+	namespaces := strings.Join(includeNamespaces, ",")
+	args := []string{
+		"--namespace", veleroNamespace, "create", "backup", backupName,
+		"--include-namespaces", namespaces,
+		"--default-volumes-to-restic", "--wait",
+	}
+	return VeleroBackupExec(ctx, veleroCLI, veleroNamespace, backupName, args)
+}
+
+// VeleroRestore uses the VeleroCLI to restore from a Velero backup.
+func VeleroRestore(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, backupName string) error {
+	args := []string{
+		"--namespace", veleroNamespace, "create", "restore", restoreName,
+		"--from-backup", backupName, "--wait",
+	}
+	return VeleroRestoreExec(ctx, veleroCLI, veleroNamespace, restoreName, args)
+}
+
+func VeleroRestoreExec(ctx context.Context, veleroCLI, veleroNamespace, restoreName string, args []string) error {
+	if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil {
+		return err
+	}
+	return checkRestorePhase(ctx, veleroCLI, veleroNamespace, restoreName, velerov1api.RestorePhaseCompleted)
+}
+
+func VeleroBackupExec(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, args []string) error {
+	if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil {
+		return err
+	}
+	return checkBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted)
+}
+
+func VeleroCmdExec(ctx context.Context, veleroCLI string, args []string) error {
+	cmd := exec.CommandContext(ctx, veleroCLI, args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	fmt.Printf("velero cmd =%v\n", cmd)
+	err := cmd.Run()
+	if err != nil {
+		return err
+	}
+	return err
+}
+
+func VeleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error {
+	args := []string{
+		"--namespace", veleroNamespace, "backup", "describe", backupName,
+	}
+	if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil {
+		return err
+	}
+	args = []string{
+		"--namespace", veleroNamespace, "backup", "logs", backupName,
+	}
+	return VeleroCmdExec(ctx, veleroCLI, args)
+}
+
+func RunDebug(ctx context.Context, veleroCLI, veleroNamespace, backup, restore string) {
+	output := fmt.Sprintf("debug-bundle-%d.tar.gz", time.Now().UnixNano())
+	args := []string{"debug", "--namespace", veleroNamespace, "--output", output, "--verbose"}
+	if len(backup) > 0 {
+		args = append(args, "--backup", backup)
+	}
+	if len(restore) > 0 {
+		args = append(args, "--restore", restore)
+	}
+	fmt.Printf("Generating the debug tarball at %s\n", output)
+	if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil {
+		fmt.Println(errors.Wrapf(err, "failed to run the debug command"))
+	}
+}
+
+func VeleroCreateBackupLocation(ctx context.Context,
+	veleroCLI,
+	veleroNamespace,
+	name,
+	objectStoreProvider,
+	bucket,
+	prefix,
+	config,
+	secretName,
+	secretKey string,
+) error {
+	args := []string{
+		"--namespace", veleroNamespace,
+		"create", "backup-location", name,
+		"--provider", objectStoreProvider,
+		"--bucket", bucket,
+	}
+
+	if prefix != "" {
+		args = append(args, "--prefix", prefix)
+	}
+
+	if config != "" {
+		args = append(args, "--config", config)
+	}
+
+	if secretName != "" && secretKey != "" {
+		args = append(args, "--credential", fmt.Sprintf("%s=%s", secretName, secretKey))
+	}
+	return VeleroCmdExec(ctx, veleroCLI, args)
+}
+
+func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, providerPlugins string) ([]string, error) {
+	// Fetch the plugins for the provider before checking for the object store provider below.
+	var plugins []string
+	if len(providerPlugins) > 0 {
+		plugins = strings.Split(providerPlugins, ",")
+	} else {
+		version, err := getVeleroVersion(ctx, veleroCLI, true)
+		if err != nil {
+			return nil, errors.WithMessage(err, "failed to get velero version")
+		}
+		plugins, err = getProviderPluginsByVersion(version, objectStoreProvider)
+		if err != nil {
+			return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", objectStoreProvider, version)
+		}
+	}
+	return plugins, nil
+}
+
+// VeleroAddPluginsForProvider determines which plugins need to be installed for a provider and
+// installs them in the current Velero installation, skipping over those that are already installed.
+func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, addPlugins string) error {
+	plugins, err := getProviderPlugins(ctx, veleroCLI, provider, addPlugins)
+	if err != nil {
+		return errors.WithMessage(err, "Failed to get plugins")
+	}
+	for _, plugin := range plugins {
+		stdoutBuf := new(bytes.Buffer)
+		stderrBuf := new(bytes.Buffer)
+
+		installPluginCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "plugin", "add", plugin)
+		installPluginCmd.Stdout = stdoutBuf
+		installPluginCmd.Stderr = stderrBuf
+
+		err := installPluginCmd.Run()
+
+		fmt.Fprint(os.Stdout, stdoutBuf)
+		fmt.Fprint(os.Stderr, stderrBuf)
+
+		if err != nil {
+			// If the plugin failed to install as it was already installed, ignore the error and continue
+			// TODO: Check which plugins are already installed by inspecting `velero plugin get`
+			if !strings.Contains(stderrBuf.String(), "Duplicate value") {
+				return errors.WithMessagef(err, "error installing plugin %s", plugin)
+			}
+		}
+	}
+
+	return nil
+}
+
+// WaitForVSphereUploadCompletion waits for uploads started by the Velero Plug-in for vSphere to complete
+// TODO - remove after upload progress monitoring is implemented
+func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, namespace string) error {
+	err := wait.PollImmediate(time.Minute, timeout, func() (bool, error) {
+		checkSnapshotCmd := exec.CommandContext(ctx, "kubectl",
+			"get", "-n", namespace, "snapshots.backupdriver.cnsdp.vmware.com", "-o=jsonpath='{range .items[*]}{.spec.resourceHandle.name}{\"=\"}{.status.phase}{\"\\n\"}{end}'")
+		fmt.Printf("checkSnapshotCmd cmd =%v\n", checkSnapshotCmd)
+		stdout, stderr, err := veleroexec.RunCommand(checkSnapshotCmd)
+		if err != nil {
+			fmt.Print(stdout)
+			fmt.Print(stderr)
+			return false, errors.Wrap(err, "failed to verify")
+		}
+		lines := strings.Split(stdout, "\n")
+		complete := true
+		for _, curLine := range lines {
+			fmt.Println(curLine)
+			comps := strings.Split(curLine, "=")
+			// SnapshotPhase represents the lifecycle phase of a Snapshot.
+			// New - No work yet, next phase is InProgress
+			// InProgress - snapshot being taken
+			// Snapshotted - local snapshot complete, next phase is Protecting or SnapshotFailed
+			// SnapshotFailed - end state, snapshot was not able to be taken
+			// Uploading - snapshot is being moved to durable storage
+			// Uploaded - end state, snapshot has been protected
+			// UploadFailed - end state, unable to move to durable storage
+			// Canceling - when the SanpshotCancel flag is set, if the Snapshot has not already moved into a terminal state, the
+			//             status will move to Canceling.  The snapshot ID will be removed from the status status if has been filled in
+			//             and the snapshot ID will not longer be valid for a Clone operation
+			// Canceled - the operation was canceled, the snapshot ID is not valid
+			// Canceled - the operation was canceled, the snapshot ID is not valid
+			if len(comps) == 2 {
+				phase := comps[1]
+				switch phase {
+				case "Uploaded":
+				case "New", "InProgress", "Snapshotted", "Uploading":
+					complete = false
+				default:
+					return false, fmt.Errorf("unexpected snapshot phase: %s", phase)
+				}
+			}
+		}
+		return complete, nil
+	})
+
+	return err
+}
+
+func getVeleroVersion(ctx context.Context, veleroCLI string, clientOnly bool) (string, error) {
+	args := []string{"version", "--timeout", "60s"}
+	if clientOnly {
+		args = append(args, "--client-only")
+	}
+	cmd := exec.CommandContext(ctx, veleroCLI, args...)
+	fmt.Println("Get Version Command:" + cmd.String())
+	stdout, stderr, err := veleroexec.RunCommand(cmd)
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to get velero version, stdout=%s, stderr=%s", stdout, stderr)
+	}
+
+	output := strings.Replace(stdout, "\n", " ", -1)
+	fmt.Println("Version:" + output)
+	resultCount := 3
+	regexpRule := `(?i)client\s*:\s*version\s*:\s*(\S+).+server\s*:\s*version\s*:\s*(\S+)`
+	if clientOnly {
+		resultCount = 2
+		regexpRule = `(?i)client\s*:\s*version\s*:\s*(\S+)`
+	}
+	regCompiler := regexp.MustCompile(regexpRule)
+	versionMatches := regCompiler.FindStringSubmatch(output)
+	if len(versionMatches) != resultCount {
+		return "", errors.New("failed to parse velero version from output")
+	}
+	if !clientOnly {
+		if versionMatches[1] != versionMatches[2] {
+			return "", errors.New("velero server and client version are not matched")
+		}
+	}
+	return versionMatches[1], nil
+}
+
+func CheckVeleroVersion(ctx context.Context, veleroCLI string, expectedVer string) error {
+	tag := expectedVer
+	tagInstalled, err := getVeleroVersion(ctx, veleroCLI, false)
+	if err != nil {
+		return errors.WithMessagef(err, "failed to get Velero version")
+	}
+	if strings.Trim(tag, " ") != strings.Trim(tagInstalled, " ") {
+		return errors.New(fmt.Sprintf("velero version %s is not as expected %s", tagInstalled, tag))
+	}
+	fmt.Printf("Velero version %s is as expected %s\n", tagInstalled, tag)
+	return nil
+}
+
+func InstallVeleroCLI(version string) (string, error) {
+	name := "velero-" + version + "-" + runtime.GOOS + "-" + runtime.GOARCH
+	postfix := ".tar.gz"
+	tarball := name + postfix
+	tempFile, err := getVeleroCliTarball("https://github.com/vmware-tanzu/velero/releases/download/" + version + "/" + tarball)
+	if err != nil {
+		return "", errors.WithMessagef(err, "failed to get Velero CLI tarball")
+	}
+	tempVeleroCliDir, err := ioutil.TempDir("", "velero-test")
+	if err != nil {
+		return "", errors.WithMessagef(err, "failed to create temp dir for tarball extraction")
+	}
+
+	cmd := exec.Command("tar", "-xvf", tempFile.Name(), "-C", tempVeleroCliDir)
+	defer os.Remove(tempFile.Name())
+
+	if _, err := cmd.Output(); err != nil {
+		return "", errors.WithMessagef(err, "failed to extract file from velero CLI tarball")
+	}
+	return tempVeleroCliDir + "/" + name + "/velero", nil
+}
+
+func getVeleroCliTarball(cliTarballUrl string) (*os.File, error) {
+	lastInd := strings.LastIndex(cliTarballUrl, "/")
+	tarball := cliTarballUrl[lastInd+1:]
+
+	resp, err := http.Get(cliTarballUrl)
+	if err != nil {
+		return nil, errors.WithMessagef(err, "failed to access Velero CLI tarball")
+	}
+	defer resp.Body.Close()
+
+	tarballBuf, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.WithMessagef(err, "failed to read buffer for tarball %s.", tarball)
+	}
+	tmpfile, err := ioutil.TempFile("", tarball)
+	if err != nil {
+		return nil, errors.WithMessagef(err, "failed to create temp file for tarball %s locally.", tarball)
+	}
+
+	if _, err := tmpfile.Write(tarballBuf); err != nil {
+		return nil, errors.WithMessagef(err, "failed to write tarball file %s locally.", tarball)
+	}
+
+	return tmpfile, nil
+}