From 45d53178ae4bc4e6809e862350e1477964ef70ae Mon Sep 17 00:00:00 2001 From: "David L. Smith-Uchida" Date: Thu, 18 Feb 2021 16:16:59 -0800 Subject: [PATCH] E2E tests now run in multiple clouds in addition to KIND (#3286) Split plug-in provider into cloud provider/object provider Moved velero install/uninstall for tests into velero_utils Added remove of CRDs to test v elero uninstall Added remove of cluster role binding to test velero uninstall Added dump of velero describe and logs on error Added velero namespace argument to velero_utils functions Modified api group versions e2e tests to use VeleroInstall Added velero logs dumps for api group versions e2e testing Added DeleteNamespace to test/e2e/common.go Fixed VeleroInstall to use the image specified Changed enable_api_group_versions_test to use veleroNamespace instead of hardcoded "velero" Signed-off-by: Dave Smith-Uchida --- test/e2e/Makefile | 12 +- test/e2e/README.md | 25 +++- test/e2e/backup_test.go | 57 ++++---- test/e2e/common.go | 34 ++++- test/e2e/e2e_suite_test.go | 9 +- test/e2e/enable_api_group_versions_test.go | 74 +++++----- test/e2e/kibishii_tests.go | 18 ++- test/e2e/kind_test.go | 24 ---- test/e2e/velero_utils.go | 152 +++++++++++++++++++-- 9 files changed, 282 insertions(+), 123 deletions(-) delete mode 100644 test/e2e/kind_test.go diff --git a/test/e2e/Makefile b/test/e2e/Makefile index 3cc3869b4..931c55ce6 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -48,12 +48,14 @@ OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin GINKGO_FOCUS ?= VELERO_CLI ?=$$(pwd)/../../_output/bin/$(GOOS)/$(GOARCH)/velero VELERO_IMAGE ?= velero/velero:main +VELERO_NAMESPACE ?= CREDS_FILE ?= BSL_BUCKET ?= BSL_PREFIX ?= BSL_CONFIG ?= VSL_CONFIG ?= -PLUGIN_PROVIDER ?= +CLOUD_PROVIDER ?= +OBJECT_STORE_PROVIDER ?= .PHONY:ginkgo ginkgo: # Make sure ginkgo is in $GOPATH/bin @@ -65,16 +67,18 @@ run: ginkgo ( echo "A credentials file is required to run E2E tests, please re-run the make target with CREDS_FILE="; exit 1 ) @[ "${BSL_BUCKET}" ] && echo "Using bucket ${BSL_BUCKET} to store backups from E2E tests" || \ (echo "Bucket to store the backups from E2E tests is required, please re-run with BSL_BUCKET="; exit 1 ) - @[ "${PLUGIN_PROVIDER}" ] && echo "Using plugin provider ${PLUGIN_PROVIDER} for object storage and volume snaphotting" || \ - (echo "Plugin provider to store the backups from E2E tests is required, please re-run with PLUGIN_PROVIDER="; exit 1 ) + @[ "${CLOUD_PROVIDER}" ] && echo "Using cloud provider ${CLOUD_PROVIDER}" || \ + (echo "Cloud provider for target cloud/plug-in provider is required, please rerun with CLOUD_PROVIDER="; exit 1) @$(GINKGO) -v -focus="$(GINKGO_FOCUS)" . -- -velerocli=$(VELERO_CLI) \ -velero-image=$(VELERO_IMAGE) \ + -velero-namespace=$(VELERO_NAMESPACE) \ -credentials-file=$(CREDS_FILE) \ -bucket=$(BSL_BUCKET) \ -prefix=$(BSL_PREFIX) \ -bsl-config=$(BSL_CONFIG) \ -vsl-config=$(VSL_CONFIG) \ - -plugin-provider=$(PLUGIN_PROVIDER) + -cloud-provider=$(CLOUD_PROVIDER) \ + -object-store-provider="$(OBJECT_STORE_PROVIDER)" build: ginkgo mkdir -p $(OUTPUT_DIR) diff --git a/test/e2e/README.md b/test/e2e/README.md index 25ccadde8..b6a4e9b3b 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -31,15 +31,21 @@ These configuration parameters are expected as values to the following command l 1. `-credentials-file`: File containing credentials for backup and volume provider. Required. 1. `-bucket`: Name of the object storage bucket where backups from e2e tests should be stored. Required. -1. `-plugin-provider`: Provider of object store and volume snapshotter plugins. Required. +1. `-cloud-provider`: The cloud the tests will be run in. Appropriate plug-ins will be installed except for kind which requires +the object-store-provider to be specified. +1. `-object-store-provider`: Object store provider to use. Required when kind is the cloud provider. 1. `-velerocli`: Path to the velero application to use. Optional, by default uses `velero` in the `$PATH` 1. `-velero-image`: Image for the velero server to be tested. Optional, by default uses `velero/velero:main` 1. `-bsl-config`: Configuration to use for the backup storage location. Format is key1=value1,key2=value2. Optional. 1. `-prefix`: Prefix in the `bucket`, under which all Velero data should be stored within the bucket. Optional. 1. `-vsl-config`: Configuration to use for the volume snapshot location. Format is key1=value1,key2=value2. Optional. +1. `-velero-namespace`: Namespace to install velero in. Optional, defaults to "velero". +1. `-install-velero`: Specifies whether to install/uninstall velero for the tests. Optional, defaults to "true". These configurations or parameters are used to generate install options for Velero for each test suite. +Tests can be run with the Kubernetes cluster hosted in various cloud providers or in a _kind_ cluster with storage in +a specified object store type. Currently supported cloud provider types are _aws_, _azure_, _vsphere_ and _kind_. ## Running tests locally ### Running using `make` @@ -49,22 +55,27 @@ E2E tests can be run from the Velero repository root by running `make test-e2e`. Below is a mapping between `make` variables to E2E configuration flags. 1. `CREDS_FILE`: `-credentials-file`. Required. 1. `BSL_BUCKET`: `-bucket`. Required. -1. `PLUGIN_PROVIDER`: `-plugin-provider`. Required. +1. `CLOUD_PROVIDER`: `-cloud-provider`. Required +1. `OBJECT_STORE_PROVIDER`: `-object-store-provider`. Required when kind is the cloud provider. 1. `VELERO_CLI`: the `-velerocli`. Optional. 1. `VELERO_IMAGE`: the `-velero-image`. Optional. 1. `BSL_PREFIX`: `-prefix`. Optional. 1. `BSL_CONFIG`: `-bsl-config`. Optional. 1. `VSL_CONFIG`: `-vsl-config`. Optional. -For example, E2E tests can be run from Velero repository roots using the below command: +For example, E2E tests can be run from Velero repository roots using the commands below: -1. Run Velero tests using AWS as the storage provider: +1. Run Velero tests in a kind cluster with AWS (or Minio) as the storage provider: ```bash - BSL_PREFIX= BSL_BUCKET= CREDS_FILE=/path/to/aws-creds PLUGIN_PROVIDER=aws make test-e2e + BSL_PREFIX= BSL_BUCKET= CREDS_FILE=/path/to/aws-creds CLOUD_PROVIDER=kind OBJECT_STORE_PROVIDER=aws make test-e2e ``` -1. Run Velero tests using Microsoft Azure as the storage provider: +1. Run Velero tests in an AWS cluster: ```bash - BSL_CONFIG="resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID" BSL_BUCKET=velero CREDS_FILE=~/bin/velero-dev/aks-creds PLUGIN_PROVIDER=azure make test-e2e + BSL_PREFIX= BSL_BUCKET= CREDS_FILE=/path/to/aws-creds CLOUD_PROVIDER=aws make test-e2e + ``` +1. Run Velero tests in a Microsoft Azure cluster: + ```bash + BSL_CONFIG="resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID" BSL_BUCKET= CREDS_FILE=/path/to/azure-creds CLOUD_PROVIDER=azure make test-e2e ``` Please refer to `velero-plugin-for-microsoft-azure` documentation for instruction to [set up permissions for Velero](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#set-permissions-for-velero) and to [set up azure storage account and blob container](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#setup-azure-storage-account-and-blob-container) 1. Run Ginko-focused Restore Multi-API Groups tests using an image built for PR #3133 and Minio as the backup storage location: diff --git a/test/e2e/backup_test.go b/test/e2e/backup_test.go index c74e70533..22a676faa 100644 --- a/test/e2e/backup_test.go +++ b/test/e2e/backup_test.go @@ -1,63 +1,60 @@ package e2e import ( + "context" "flag" - "fmt" "time" "github.com/google/uuid" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "golang.org/x/net/context" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/client-go/kubernetes" - - "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" ) var ( - uuidgen uuid.UUID - veleroInstallOptions *install.InstallOptions + uuidgen uuid.UUID ) -func veleroInstall(pluginProvider string, useRestic bool) { - var err error - flag.Parse() - Expect(EnsureClusterExists(context.TODO())).To(Succeed(), "Failed to ensure kubernetes cluster exists") - uuidgen, err = uuid.NewRandom() - Expect(err).To(Succeed()) - veleroInstallOptions, err = GetProviderVeleroInstallOptions(pluginProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, getProviderPlugins(pluginProvider)) - Expect(err).To(Succeed(), fmt.Sprintf("Failed to get Velero InstallOptions for plugin provider %s", pluginProvider)) - veleroInstallOptions.UseRestic = useRestic - Expect(InstallVeleroServer(veleroInstallOptions)).To(Succeed(), "Failed to install Velero on KinD cluster") -} - // Test backup and restore of Kibishi using restic -var _ = Describe("[Restic] [KinD] Velero tests on KinD cluster using the plugin provider for object storage and Restic for volume backups", func() { +var _ = Describe("[Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", func() { var ( - client *kubernetes.Clientset - backupName string - restoreName string + client *kubernetes.Clientset + extensionsClient *apiextensionsclientset.Clientset + backupName string + restoreName string ) + BeforeEach(func() { var err error - veleroInstall(pluginProvider, true) - client, err = GetClusterClient() + flag.Parse() + uuidgen, err = uuid.NewRandom() + Expect(err).To(Succeed()) + if installVelero { + VeleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, + cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, "") + } + client, extensionsClient, err = GetClusterClient() Expect(err).To(Succeed(), "Failed to instantiate cluster client") }) + AfterEach(func() { - timeoutCTX, _ := context.WithTimeout(context.Background(), time.Minute) - err := client.CoreV1().Namespaces().Delete(timeoutCTX, veleroInstallOptions.Namespace, metav1.DeleteOptions{}) - Expect(err).To(Succeed()) + if installVelero { + timeoutCTX, _ := context.WithTimeout(context.Background(), time.Minute) + err := VeleroUninstall(timeoutCTX, client, extensionsClient, veleroNamespace) + Expect(err).To(Succeed()) + } + }) + Context("When kibishii is the sample workload", func() { It("should be successfully backed up and restored", func() { backupName = "backup-" + uuidgen.String() restoreName = "restore-" + uuidgen.String() // Even though we are using Velero's CloudProvider plugin for object storage, the kubernetes cluster is running on // KinD. So use the kind installation for Kibishii. - Expect(RunKibishiiTests(client, "kind", veleroCLI, backupName, restoreName)).To(Succeed(), "Failed to successfully backup and restore Kibishii namespace") + Expect(RunKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName)).To(Succeed(), + "Failed to successfully backup and restore Kibishii namespace") }) }) }) diff --git a/test/e2e/common.go b/test/e2e/common.go index 1d5b07dbe..e3cbf8fec 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -1,10 +1,15 @@ package e2e import ( + "fmt" "os/exec" + "time" + + "k8s.io/apimachinery/pkg/util/wait" "github.com/pkg/errors" "golang.org/x/net/context" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -19,20 +24,26 @@ func EnsureClusterExists(ctx context.Context) error { } // GetClusterClient instantiates and returns a client for the cluster. -func GetClusterClient() (*kubernetes.Clientset, error) { +func GetClusterClient() (*kubernetes.Clientset, *apiextensionsclientset.Clientset, error) { loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() configOverrides := &clientcmd.ConfigOverrides{} kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) clientConfig, err := kubeConfig.ClientConfig() if err != nil { - return nil, errors.WithStack(err) + return nil, nil, errors.WithStack(err) } client, err := kubernetes.NewForConfig(clientConfig) if err != nil { - return nil, errors.WithStack(err) + return nil, nil, errors.WithStack(err) } - return client, nil + + extensionClientSet, err := apiextensionsclientset.NewForConfig(clientConfig) + if err != nil { + return nil, nil, errors.WithStack(err) + } + + return client, extensionClientSet, nil } // CreateNamespace creates a kubernetes namespace @@ -44,3 +55,18 @@ func CreateNamespace(ctx context.Context, client *kubernetes.Clientset, namespac } return err } + +func DeleteNamespace(ctx context.Context, client *kubernetes.Clientset, namespace string) error { + err := client.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}) + if err != nil { + return errors.WithMessagef(err, "Delete namespace failed removing namespace %s", namespace) + } + return wait.Poll(1*time.Second, 3*time.Minute, func() (bool, error) { + _, err := client.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + if err != nil { + fmt.Printf("Namespaces.Get after delete return err %v\n", err) + return true, nil // Assume any error means the delete was successful + } + return false, nil + }) +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 59c74e1aa..f6ec08630 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -9,11 +9,13 @@ import ( ) var ( - veleroCLI, veleroImage, cloudCredentialsFile, bslConfig, bslBucket, bslPrefix, vslConfig, pluginProvider string + veleroCLI, veleroImage, cloudCredentialsFile, bslConfig, bslBucket, bslPrefix, vslConfig, cloudProvider, objectStoreProvider, veleroNamespace string + installVelero, useVolumeSnapshots bool ) func init() { - flag.StringVar(&pluginProvider, "plugin-provider", "", "Provider of object store and volume snapshotter plugins. Required.") + flag.StringVar(&cloudProvider, "cloud-provider", "", "Cloud that Velero will be installed into. Required.") + flag.StringVar(&objectStoreProvider, "object-store-provider", "", "Provider of object store plugin. Required if cloud-provider is kind, otherwise ignored.") flag.StringVar(&bslBucket, "bucket", "", "name of the object storage bucket where backups from e2e tests should be stored. Required.") flag.StringVar(&cloudCredentialsFile, "credentials-file", "", "file containing credentials for backup and volume provider. Required.") flag.StringVar(&veleroCLI, "velerocli", "velero", "path to the velero application to use.") @@ -21,6 +23,9 @@ func init() { flag.StringVar(&bslConfig, "bsl-config", "", "configuration to use for the backup storage location. Format is key1=value1,key2=value2") flag.StringVar(&bslPrefix, "prefix", "", "prefix under which all Velero data should be stored within the bucket. Optional.") flag.StringVar(&vslConfig, "vsl-config", "", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2") + flag.StringVar(&veleroNamespace, "velero-namespace", "velero", "Namespace to install Velero into") + flag.BoolVar(&installVelero, "install-velero", true, "Install/uninstall velero during the test. Optional.") + flag.BoolVar(&useVolumeSnapshots, "use-volume-snapshots", false, "Use volume-snapshotter plugin for volume backup. Optional") } func TestE2e(t *testing.T) { diff --git a/test/e2e/enable_api_group_versions_test.go b/test/e2e/enable_api_group_versions_test.go index 028f8ce8e..b75a508fe 100644 --- a/test/e2e/enable_api_group_versions_test.go +++ b/test/e2e/enable_api_group_versions_test.go @@ -10,6 +10,8 @@ import ( "strings" "time" + apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + "github.com/google/uuid" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -23,13 +25,14 @@ import ( veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" ) -var _ = Describe("[KinD] Velero tests on KinD clusters with various CRD API group versions", func() { +var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", func() { var ( - resource, group string - certMgrCRD map[string]string - client *kubernetes.Clientset - err error - ctx = context.Background() + resource, group string + certMgrCRD map[string]string + client *kubernetes.Clientset + extensionsClient *apiextensionsclient.Clientset + err error + ctx = context.Background() ) BeforeEach(func() { @@ -40,7 +43,7 @@ var _ = Describe("[KinD] Velero tests on KinD clusters with various CRD API grou "namespace": "cert-manager", } - client, err = GetClusterClient() + client, extensionsClient, err = GetClusterClient() // Currently we ignore the API extensions client Expect(err).NotTo(HaveOccurred()) err = InstallCRD(ctx, certMgrCRD["url"], certMgrCRD["namespace"]) @@ -58,15 +61,6 @@ var _ = Describe("[KinD] Velero tests on KinD clusters with various CRD API grou _, _, _ = veleroexec.RunCommand(cmd) _ = DeleteCRD(ctx, certMgrCRD["url"], certMgrCRD["namespace"]) - - // Uninstall Velero. - if client != nil { - _ = client.CoreV1().Namespaces().Delete( - context.Background(), - "velero", - metav1.DeleteOptions{}, - ) - } }) Context("When EnableAPIGroupVersions flag is set", func() { @@ -76,12 +70,14 @@ var _ = Describe("[KinD] Velero tests on KinD clusters with various CRD API grou resource, group, client, + extensionsClient, )).To(Succeed(), "Failed to successfully backup and restore multiple API Groups") }) }) }) -func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, client *kubernetes.Clientset) error { +func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, client *kubernetes.Clientset, + extensionsClient *apiextensionsclient.Clientset) error { tests := []struct { name string namespaces []string @@ -181,7 +177,7 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, "namespace": "music-system", }, tgtVer: "v2beta1", - cm: builder.ForConfigMap("velero", "enableapigroupversions").Data( + cm: builder.ForConfigMap(veleroNamespace, "enableapigroupversions").Data( "restoreResourcesVersionPriority", `rockbands.music.example.io=v2beta1,v2beta2,v2`, ).Result(), @@ -220,17 +216,22 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, tc.namespaces = append(tc.namespaces, ns) } - if err := installVeleroForAPIGroups(ctx); err != nil { - return errors.Wrap(err, "install velero") + // TODO - Velero needs to be installed AFTER CRDs are installed because of https://github.com/vmware-tanzu/velero/issues/3471 + // Once that issue is fixed, we should install Velero once for the test suite + if installVelero { + VeleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, + cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, + "EnableAPIGroupVersions" /* TODO - remove this when the feature flag is removed */) + fmt.Println("Sleep 20s to wait for Velero to stabilize after install.") + time.Sleep(time.Second * 20) } - fmt.Println("Sleep 20s to wait for Velero to stabilize after install.") - time.Sleep(time.Second * 20) backup := "backup-rockbands-" + uuidgen.String() + "-" + strconv.Itoa(i) namespacesStr := strings.Join(tc.namespaces, ",") - err = VeleroBackupNamespace(ctx, veleroCLI, backup, namespacesStr) + err = VeleroBackupNamespace(ctx, veleroCLI, veleroNamespace, backup, namespacesStr) if err != nil { + VeleroBackupLogs(ctx, veleroCLI, veleroNamespace, backup) return errors.Wrapf(err, "backing up %s namespaces on source cluster", namespacesStr) } @@ -256,14 +257,14 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, // Apply config map if there is one. if tc.cm != nil { - _, err := client.CoreV1().ConfigMaps("velero").Create(ctx, tc.cm, metav1.CreateOptions{}) + _, err := client.CoreV1().ConfigMaps(veleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{}) if err != nil { return errors.Wrap(err, "creating config map with user version priorities") } } // Reset Velero to recognize music-system CRD. - if err := RestartPods(ctx, "velero"); err != nil { + if err := RestartPods(ctx, veleroNamespace); err != nil { return errors.Wrapf(err, "restarting Velero pods") } fmt.Println("Sleep 20s to wait for Velero to stabilize after restart.") @@ -273,7 +274,8 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, restore := "restore-rockbands-" + uuidgen.String() + "-" + strconv.Itoa(i) if tc.want != nil { - if err := VeleroRestore(ctx, veleroCLI, restore, backup); err != nil { + if err := VeleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup); err != nil { + VeleroRestoreLogs(ctx, veleroCLI, veleroNamespace, restore) return errors.Wrapf(err, "restoring %s namespaces on target cluster", namespacesStr) } @@ -310,7 +312,7 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, } else { // No custom resource should have been restored. Expect "no resource found" // error during restore. - err := VeleroRestore(ctx, veleroCLI, restore, backup) + err := VeleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup) if err.Error() != "Unexpected restore phase got PartiallyFailed, expecting Completed" { return errors.New("expected error but not none") @@ -338,9 +340,13 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, tc.srcCRD["namespace"], ) - // Delete Velero namespace - _ = client.CoreV1().Namespaces().Delete(ctx, "velero", metav1.DeleteOptions{}) - _ = WaitNamespaceDelete(ctx, "velero") + // Uninstall Velero + if installVelero { + err = VeleroUninstall(ctx, client, extensionsClient, veleroNamespace) + if err != nil { + return err + } + } } return nil @@ -353,13 +359,14 @@ func installVeleroForAPIGroups(ctx context.Context) error { // Pass global variables to option parameters. options, err := GetProviderVeleroInstallOptions( - pluginProvider, + cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, - getProviderPlugins(pluginProvider), + getProviderPlugins(cloudProvider), + "EnableAPIGroupVersions", ) if err != nil { return errors.Wrap(err, "get velero install options") @@ -428,7 +435,7 @@ func WaitForPodContainers(ctx context.Context, ns string) error { func DeleteCRD(ctx context.Context, crdFile, ns string) error { fmt.Println("Delete CRD", crdFile) - cmd := exec.CommandContext(ctx, "kubectl", "delete", "-f", crdFile) + cmd := exec.CommandContext(ctx, "kubectl", "delete", "-f", crdFile, "--wait") _, stderr, err := veleroexec.RunCommand(cmd) if strings.Contains(stderr, "not found") { @@ -454,7 +461,6 @@ func DeleteCRD(ctx context.Context, crdFile, ns string) error { re := regexp.MustCompile(ns) return re.MatchString(stdout), nil }) - return err } diff --git a/test/e2e/kibishii_tests.go b/test/e2e/kibishii_tests.go index 7329a3e40..0f025f1e1 100644 --- a/test/e2e/kibishii_tests.go +++ b/test/e2e/kibishii_tests.go @@ -37,9 +37,15 @@ func installKibishii(ctx context.Context, namespace string, cloudPlatform string return err } + fmt.Printf("Waiting for kibishii jump-pad ready\n") jumpPadWaitCmd := exec.CommandContext(ctx, "kubectl", "wait", "--for=condition=ready", "-n", namespace, "pod/jump-pad") _, _, err = veleroexec.RunCommand(jumpPadWaitCmd) + // TODO - Fix kibishii so we can check that it is ready to go + if err == nil { + fmt.Printf("Sleeping for Kibishii startup\n") + time.Sleep(time.Minute * 1) + } return err } @@ -73,23 +79,24 @@ func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel } // RunKibishiiTests runs kibishii tests on the provider. -func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, backupName, restoreName string) error { +func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, veleroNamespace, backupName, restoreName string) error { fiveMinTimeout, _ := context.WithTimeout(context.Background(), 5*time.Minute) oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) if err := CreateNamespace(fiveMinTimeout, client, kibishiiNamespace); err != nil { - errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace) + return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace) } if err := installKibishii(fiveMinTimeout, kibishiiNamespace, providerName); err != nil { - errors.Wrap(err, "Failed to install Kibishii workload") + return errors.Wrap(err, "Failed to install Kibishii workload") } if err := generateData(oneHourTimeout, kibishiiNamespace, 2, 10, 10, 1024, 1024, 0, 2); err != nil { return errors.Wrap(err, "Failed to generate data") } - if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, backupName, kibishiiNamespace); err != nil { + if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace); err != nil { + VeleroBackupLogs(fiveMinTimeout, veleroCLI, veleroNamespace, backupName) return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace) } @@ -98,7 +105,8 @@ func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, bac return errors.Wrap(err, "Failed to simulate a disaster") } - if err := VeleroRestore(oneHourTimeout, veleroCLI, restoreName, backupName); err != nil { + if err := VeleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil { + VeleroRestoreLogs(fiveMinTimeout, veleroCLI, veleroNamespace, restoreName) return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName) } diff --git a/test/e2e/kind_test.go b/test/e2e/kind_test.go deleted file mode 100644 index e4a1012c2..000000000 --- a/test/e2e/kind_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package e2e - -import ( - "context" - "flag" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -var _ = Describe("Testing Velero on a kind cluster", func() { - BeforeEach(func() { - flag.Parse() - ctx := context.TODO() - err := EnsureClusterExists(ctx) - Expect(err).To(Succeed()) - }) - Describe("Dummy test", func() { - Context("Dummy test", func() { - It("is a dummy test", func() { - }) - }) - }) -}) diff --git a/test/e2e/velero_utils.go b/test/e2e/velero_utils.go index f50cf0b25..47bc1897e 100644 --- a/test/e2e/velero_utils.go +++ b/test/e2e/velero_utils.go @@ -9,7 +9,13 @@ import ( "os/exec" "path/filepath" + "k8s.io/apimachinery/pkg/labels" + "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" @@ -34,13 +40,14 @@ func getProviderPlugins(providerName string) []string { // GetProviderVeleroInstallOptions returns Velero InstallOptions for the provider. func GetProviderVeleroInstallOptions( - providerName, + pluginProvider, credentialsFile, objectStoreBucket, objectStorePrefix string, bslConfig, vslConfig string, plugins []string, + features string, ) (*cliinstall.InstallOptions, error) { if credentialsFile == "" { @@ -55,7 +62,7 @@ func GetProviderVeleroInstallOptions( io := cliinstall.NewInstallOptions() // always wait for velero and restic pods to be running. io.Wait = true - io.ProviderName = providerName + io.ProviderName = pluginProvider io.SecretFile = credentialsFile io.BucketName = objectStoreBucket @@ -68,6 +75,7 @@ func GetProviderVeleroInstallOptions( io.SecretFile = realPath io.Plugins = flag.NewStringArray(plugins...) + io.Features = features return io, nil } @@ -101,7 +109,7 @@ func InstallVeleroServer(io *cliinstall.InstallOptions) error { } fmt.Println("Waiting for Velero deployment to be ready.") - if _, err = install.DeploymentIsReady(factory, "velero"); err != nil { + if _, err = install.DeploymentIsReady(factory, io.Namespace); err != nil { return errors.Wrap(err, errorMsg) } @@ -116,8 +124,11 @@ func InstallVeleroServer(io *cliinstall.InstallOptions) error { } // CheckBackupPhase uses veleroCLI to inspect the phase of a Velero backup. -func CheckBackupPhase(ctx context.Context, veleroCLI string, backupName string, expectedPhase velerov1api.BackupPhase) error { - checkCMD := exec.CommandContext(ctx, veleroCLI, "backup", "get", "-o", "json", backupName) +func CheckBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, + expectedPhase velerov1api.BackupPhase) error { + checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "get", "-o", "json", + backupName) + fmt.Printf("get backup cmd =%v\n", checkCMD) stdoutPipe, err := checkCMD.StdoutPipe() if err != nil { @@ -157,8 +168,11 @@ func CheckBackupPhase(ctx context.Context, veleroCLI string, backupName string, } // CheckRestorePhase uses veleroCLI to inspect the phase of a Velero restore. -func CheckRestorePhase(ctx context.Context, veleroCLI string, restoreName string, expectedPhase velerov1api.RestorePhase) error { - checkCMD := exec.CommandContext(ctx, veleroCLI, "restore", "get", "-o", "json", restoreName) +func CheckRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, + expectedPhase velerov1api.RestorePhase) error { + checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "restore", "get", "-o", "json", + restoreName) + fmt.Printf("get restore cmd =%v\n", checkCMD) stdoutPipe, err := checkCMD.StdoutPipe() if err != nil { @@ -198,23 +212,135 @@ func CheckRestorePhase(ctx context.Context, veleroCLI string, restoreName string } // VeleroBackupNamespace uses the veleroCLI to backup a namespace. -func VeleroBackupNamespace(ctx context.Context, veleroCLI string, backupName string, namespace string) error { - backupCmd := exec.CommandContext(ctx, veleroCLI, "create", "backup", backupName, "--include-namespaces", namespace, +func VeleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string) error { + backupCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "create", "backup", backupName, + "--include-namespaces", namespace, "--default-volumes-to-restic", "--wait") + + backupCmd.Stdout = os.Stdout + backupCmd.Stderr = os.Stderr + fmt.Printf("backup cmd =%v\n", backupCmd) err := backupCmd.Run() if err != nil { return err } - return CheckBackupPhase(ctx, veleroCLI, backupName, velerov1api.BackupPhaseCompleted) + err = CheckBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted) + + return err } // VeleroRestore uses the veleroCLI to restore from a Velero backup. -func VeleroRestore(ctx context.Context, veleroCLI string, restoreName string, backupName string) error { - restoreCmd := exec.CommandContext(ctx, veleroCLI, "create", "restore", restoreName, "--from-backup", backupName, "--wait") +func VeleroRestore(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, backupName string) error { + restoreCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "create", "restore", restoreName, + "--from-backup", backupName, "--wait") + + restoreCmd.Stdout = os.Stdout + restoreCmd.Stderr = os.Stderr fmt.Printf("restore cmd =%v\n", restoreCmd) err := restoreCmd.Run() if err != nil { return err } - return CheckRestorePhase(ctx, veleroCLI, restoreName, velerov1api.RestorePhaseCompleted) + return CheckRestorePhase(ctx, veleroCLI, veleroNamespace, restoreName, velerov1api.RestorePhaseCompleted) +} + +func VeleroInstall(ctx context.Context, veleroImage string, veleroNamespace string, cloudProvider string, objectStoreProvider string, useVolumeSnapshots bool, + cloudCredentialsFile string, bslBucket string, bslPrefix string, bslConfig string, vslConfig string, + features string) error { + + if cloudProvider != "kind" { + if objectStoreProvider != "" { + return errors.New("For cloud platforms, object store plugin cannot be overridden") // Can't set an object store provider that is different than your cloud + } + objectStoreProvider = cloudProvider + } else { + if objectStoreProvider == "" { + return errors.New("No object store provider specified - must be specified when using kind as the cloud provider") // Gotta have an object store provider + } + } + err := EnsureClusterExists(ctx) + if err != nil { + return errors.WithMessage(err, "Failed to ensure kubernetes cluster exists") + } + veleroInstallOptions, err := GetProviderVeleroInstallOptions(objectStoreProvider, cloudCredentialsFile, bslBucket, + bslPrefix, bslConfig, vslConfig, getProviderPlugins(objectStoreProvider), features) + if err != nil { + return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", objectStoreProvider) + } + veleroInstallOptions.UseRestic = !useVolumeSnapshots + veleroInstallOptions.Image = veleroImage + veleroInstallOptions.Namespace = veleroNamespace + err = InstallVeleroServer(veleroInstallOptions) + if err != nil { + return errors.WithMessagef(err, "Failed to install Velero in cluster") + } + return nil +} + +func VeleroUninstall(ctx context.Context, client *kubernetes.Clientset, extensionsClient *apiextensionsclient.Clientset, + veleroNamespace string) error { + // TODO - replace with invocation of "velero uninstall" when that becomes available + err := DeleteNamespace(ctx, client, veleroNamespace) + if err != nil { + return errors.WithMessagef(err, "Uninstall failed removing Velero namespace %s", veleroNamespace) + } + + return err + rolebinding := install.ClusterRoleBinding(veleroNamespace) + + err = client.RbacV1().ClusterRoleBindings().Delete(ctx, rolebinding.Name, metav1.DeleteOptions{}) + if err != nil { + return errors.WithMessagef(err, "Uninstall failed removing Velero cluster role binding %s", rolebinding) + } + veleroLabels := labels.FormatLabels(install.Labels()) + + crds, err := extensionsClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{ + LabelSelector: veleroLabels, + }) + if err != nil { + return errors.WithMessagef(err, "Uninstall failed listing Velero crds") + } + for _, removeCRD := range crds.Items { + err = extensionsClient.ApiextensionsV1().CustomResourceDefinitions().Delete(ctx, removeCRD.ObjectMeta.Name, metav1.DeleteOptions{}) + if err != nil { + return errors.WithMessagef(err, "Uninstall failed removing CRD %s", removeCRD.ObjectMeta.Name) + } + } + return nil +} + +func VeleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error { + describeCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "describe", backupName) + describeCmd.Stdout = os.Stdout + describeCmd.Stderr = os.Stderr + err := describeCmd.Run() + if err != nil { + return err + } + logCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "logs", backupName) + logCmd.Stdout = os.Stdout + logCmd.Stderr = os.Stderr + err = logCmd.Run() + if err != nil { + return err + } + return nil +} + +func VeleroRestoreLogs(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string) error { + describeCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "restore", "describe", restoreName) + describeCmd.Stdout = os.Stdout + describeCmd.Stderr = os.Stderr + err := describeCmd.Run() + if err != nil { + return err + } + logCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "restore", "logs", restoreName) + logCmd.Stdout = os.Stdout + logCmd.Stderr = os.Stderr + err = logCmd.Run() + if err != nil { + return err + } + return nil }