From ccac3d41301c8883b62e2d3d3429711a5f8088d6 Mon Sep 17 00:00:00 2001 From: Carlisia Thompson Date: Wed, 28 Apr 2021 09:54:17 -0700 Subject: [PATCH] A small refactor of the e2e tests (#3726) * A small refactor of the e2e tests Signed-off-by: Carlisia * Add copyright header Signed-off-by: Carlisia * Fix CI Signed-off-by: Carlisia * Revert unneeded changes Signed-off-by: Carlisia * Remove file that doesnt belong here Signed-off-by: Carlisia --- go.mod | 2 +- go.sum | 6 +- pkg/cmd/cli/install/install.go | 5 +- pkg/install/resources.go | 4 +- test/e2e/backup_test.go | 27 ++++++-- test/e2e/common.go | 33 ++++++--- test/e2e/e2e_suite_test.go | 13 ++++ test/e2e/enable_api_group_versions_test.go | 79 +++++++++++++--------- test/e2e/kibishii_tests.go | 33 ++++++--- test/e2e/velero_utils.go | 69 +++++++++++-------- 10 files changed, 172 insertions(+), 99 deletions(-) diff --git a/go.mod b/go.mod index 8e27497ef..ca5af08fe 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/hashicorp/go-plugin v0.0.0-20190610192547-a1bc61569a26 github.com/joho/godotenv v1.3.0 github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0 - github.com/onsi/ginkgo v1.15.2 + github.com/onsi/ginkgo v1.16.1 github.com/onsi/gomega v1.10.2 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.7.1 diff --git a/go.sum b/go.sum index 55a53c12d..d10afce7e 100644 --- a/go.sum +++ b/go.sum @@ -214,6 +214,8 @@ github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+ github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobuffalo/flect v0.2.2 h1:PAVD7sp0KOdfswjAw9BpLCU9hXo7wFSzgpQ+zNeks/A= github.com/gobuffalo/flect v0.2.2/go.mod h1:vmkQwuZYhN5Pc4ljYQZzP+1sq+NEkK+lh20jmEmX3jc= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= @@ -431,8 +433,8 @@ github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.15.2 h1:l77YT15o814C2qVL47NOyjV/6RbaP7kKdrvZnxQ3Org= -github.com/onsi/ginkgo v1.15.2/go.mod h1:Dd6YFfwBW84ETqqtL0CPyPXillHgY6XhQH3uuCCTr/o= +github.com/onsi/ginkgo v1.16.1 h1:foqVmeWDD6yYpK+Yz3fHyNIxFYNxswxqNFjSKe+vI54= +github.com/onsi/ginkgo v1.16.1/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index eb82cbbfd..d2ed61c6a 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -255,10 +255,7 @@ func (o *InstallOptions) Run(c *cobra.Command, f client.Factory) error { return err } - resources, err = install.AllResources(vo) - if err != nil { - return err - } + resources = install.AllResources(vo) } if _, err := output.PrintWithFormat(c, resources); err != nil { diff --git a/pkg/install/resources.go b/pkg/install/resources.go index 885232045..9ca639388 100644 --- a/pkg/install/resources.go +++ b/pkg/install/resources.go @@ -245,7 +245,7 @@ func AllCRDs() *unstructured.UnstructuredList { // AllResources returns a list of all resources necessary to install Velero, in the appropriate order, into a Kubernetes cluster. // Items are unstructured, since there are different data types returned. -func AllResources(o *VeleroOptions) (*unstructured.UnstructuredList, error) { +func AllResources(o *VeleroOptions) *unstructured.UnstructuredList { resources := AllCRDs() ns := Namespace(o.Namespace) @@ -317,5 +317,5 @@ func AllResources(o *VeleroOptions) (*unstructured.UnstructuredList, error) { appendUnstructured(resources, ds) } - return resources, nil + return resources } diff --git a/test/e2e/backup_test.go b/test/e2e/backup_test.go index 20dc96ecb..5de97a7c1 100644 --- a/test/e2e/backup_test.go +++ b/test/e2e/backup_test.go @@ -1,3 +1,16 @@ +/* +Copyright the Velero contributors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package e2e import ( @@ -49,7 +62,7 @@ func backup_restore_test(useVolumeSnapshots bool) { uuidgen, err = uuid.NewRandom() Expect(err).To(Succeed()) if installVelero { - Expect(VeleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, + Expect(veleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, "")).To(Succeed()) } @@ -60,7 +73,7 @@ func backup_restore_test(useVolumeSnapshots bool) { AfterEach(func() { if installVelero { timeoutCTX, _ := context.WithTimeout(context.Background(), time.Minute) - err := VeleroUninstall(timeoutCTX, client, extensionsClient, veleroNamespace) + err := veleroUninstall(timeoutCTX, client, extensionsClient, veleroNamespace) Expect(err).To(Succeed()) } @@ -72,7 +85,7 @@ func backup_restore_test(useVolumeSnapshots bool) { restoreName = "restore-" + uuidgen.String() // Even though we are using Velero's CloudProvider plugin for object storage, the kubernetes cluster is running on // KinD. So use the kind installation for Kibishii. - Expect(RunKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, "", useVolumeSnapshots)).To(Succeed(), + Expect(runKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, "", useVolumeSnapshots)).To(Succeed(), "Failed to successfully backup and restore Kibishii namespace") }) @@ -89,7 +102,7 @@ func backup_restore_test(useVolumeSnapshots bool) { Skip("no additional BSL credentials given, not running multiple BackupStorageLocation with unique credentials tests") } - Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCLI, veleroNamespace, additionalBSLProvider)).To(Succeed()) + Expect(veleroAddPluginsForProvider(context.TODO(), veleroCLI, veleroNamespace, additionalBSLProvider)).To(Succeed()) // Create Secret for additional BSL secretName := fmt.Sprintf("bsl-credentials-%s", uuidgen) @@ -98,11 +111,11 @@ func backup_restore_test(useVolumeSnapshots bool) { secretKey: additionalBSLCredentials, } - Expect(CreateSecretFromFiles(context.TODO(), client, veleroNamespace, secretName, files)).To(Succeed()) + Expect(createSecretFromFiles(context.TODO(), client, veleroNamespace, secretName, files)).To(Succeed()) // Create additional BSL using credential additionalBsl := fmt.Sprintf("bsl-%s", uuidgen) - Expect(VeleroCreateBackupLocation(context.TODO(), + Expect(veleroCreateBackupLocation(context.TODO(), veleroCLI, veleroNamespace, additionalBsl, @@ -120,7 +133,7 @@ func backup_restore_test(useVolumeSnapshots bool) { backupName = fmt.Sprintf("backup-%s-%s", bsl, uuidgen) restoreName = fmt.Sprintf("restore-%s-%s", bsl, uuidgen) - Expect(RunKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, bsl, useVolumeSnapshots)).To(Succeed(), + Expect(runKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, bsl, useVolumeSnapshots)).To(Succeed(), "Failed to successfully backup and restore Kibishii namespace using BSL %s", bsl) } }) diff --git a/test/e2e/common.go b/test/e2e/common.go index 6dbfc0291..d080e3746 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -1,3 +1,16 @@ +/* +Copyright the Velero contributors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package e2e import ( @@ -18,13 +31,13 @@ import ( "github.com/vmware-tanzu/velero/pkg/builder" ) -// EnsureClusterExists returns whether or not a kubernetes cluster exists for tests to be run on. -func EnsureClusterExists(ctx context.Context) error { +// ensureClusterExists returns whether or not a kubernetes cluster exists for tests to be run on. +func ensureClusterExists(ctx context.Context) error { return exec.CommandContext(ctx, "kubectl", "cluster-info").Run() } -// CreateNamespace creates a kubernetes namespace -func CreateNamespace(ctx context.Context, client *kubernetes.Clientset, namespace string) error { +// createNamespace creates a kubernetes namespace +func createNamespace(ctx context.Context, client *kubernetes.Clientset, namespace string) error { ns := builder.ForNamespace(namespace).Result() _, err := client.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { @@ -33,8 +46,8 @@ func CreateNamespace(ctx context.Context, client *kubernetes.Clientset, namespac return err } -// WaitForNamespaceDeletion Waits for namespace to be deleted. -func WaitForNamespaceDeletion(interval, timeout time.Duration, client *kubernetes.Clientset, ns string) error { +// waitForNamespaceDeletion waits for namespace to be deleted. +func waitForNamespaceDeletion(interval, timeout time.Duration, client *kubernetes.Clientset, ns string) error { err := wait.PollImmediate(interval, timeout, func() (bool, error) { _, err := client.CoreV1().Namespaces().Get(context.TODO(), ns, metav1.GetOptions{}) if err != nil { @@ -49,7 +62,7 @@ func WaitForNamespaceDeletion(interval, timeout time.Duration, client *kubernete return err } -func CreateSecretFromFiles(ctx context.Context, client *kubernetes.Clientset, namespace string, name string, files map[string]string) error { +func createSecretFromFiles(ctx context.Context, client *kubernetes.Clientset, namespace string, name string, files map[string]string) error { data := make(map[string][]byte) for key, filePath := range files { @@ -66,10 +79,8 @@ func CreateSecretFromFiles(ctx context.Context, client *kubernetes.Clientset, na return err } -/* - Waits until all of the pods have gone to PodRunning state -*/ -func WaitForPods(ctx context.Context, client *kubernetes.Clientset, namespace string, pods []string) error { +// waitForPods waits until all of the pods have gone to PodRunning state +func waitForPods(ctx context.Context, client *kubernetes.Clientset, namespace string, pods []string) error { timeout := 10 * time.Minute interval := 5 * time.Second err := wait.PollImmediate(interval, timeout, func() (bool, error) { diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index ecac96bd4..e00969283 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -1,3 +1,16 @@ +/* +Copyright the Velero contributors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package e2e import ( diff --git a/test/e2e/enable_api_group_versions_test.go b/test/e2e/enable_api_group_versions_test.go index 20be75bf6..7198f0e4a 100644 --- a/test/e2e/enable_api_group_versions_test.go +++ b/test/e2e/enable_api_group_versions_test.go @@ -1,3 +1,16 @@ +/* +Copyright the Velero contributors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package e2e import ( @@ -49,7 +62,7 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", client, extensionsClient, err = kube.GetClusterClient() // Currently we ignore the API extensions client Expect(err).NotTo(HaveOccurred()) - err = InstallCRD(ctx, certMgrCRD["url"], certMgrCRD["namespace"]) + err = installCRD(ctx, certMgrCRD["url"], certMgrCRD["namespace"]) Expect(err).NotTo(HaveOccurred()) uuidgen, err = uuid.NewRandom() @@ -63,12 +76,12 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", cmd = exec.CommandContext(ctx, "kubectl", "delete", "crd", "rockbands.music.example.io") _, _, _ = veleroexec.RunCommand(cmd) - _ = DeleteCRD(ctx, certMgrCRD["url"], certMgrCRD["namespace"]) + _ = deleteCRD(ctx, certMgrCRD["url"], certMgrCRD["namespace"]) }) Context("When EnableAPIGroupVersions flag is set", func() { It("Should back up API group version and restore by version priority", func() { - Expect(RunEnableAPIGroupVersionsTests( + Expect(runEnableAPIGroupVersionsTests( ctx, resource, group, @@ -79,7 +92,7 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", }) }) -func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, client *kubernetes.Clientset, +func runEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, client *kubernetes.Clientset, extensionsClient *apiextensionsclient.Clientset) error { tests := []struct { name string @@ -200,7 +213,7 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, for i, tc := range tests { fmt.Printf("\n====== Test Case %d ======\n", i) - err := InstallCRD(ctx, tc.srcCRD["url"], tc.srcCRD["namespace"]) + err := installCRD(ctx, tc.srcCRD["url"], tc.srcCRD["namespace"]) if err != nil { return errors.Wrap(err, "installing music-system CRD for source cluster") } @@ -208,11 +221,11 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, for version, cr := range tc.srcCRs { ns := resource + "-src-" + version - if err := CreateNamespace(ctx, client, ns); err != nil { + if err := createNamespace(ctx, client, ns); err != nil { return errors.Wrapf(err, "creating %s namespace", ns) } - if err := InstallCR(ctx, cr, ns); err != nil { + if err := installCR(ctx, cr, ns); err != nil { return errors.Wrapf(err, "installing %s custom resource on source cluster namespace %s", cr, ns) } @@ -222,7 +235,7 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, // TODO - Velero needs to be installed AFTER CRDs are installed because of https://github.com/vmware-tanzu/velero/issues/3471 // Once that issue is fixed, we should install Velero once for the test suite if installVelero { - VeleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, false, + veleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, false, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, "EnableAPIGroupVersions" /* TODO - remove this when the feature flag is removed */) fmt.Println("Sleep 20s to wait for Velero to stabilize after install.") @@ -232,14 +245,14 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, backup := "backup-rockbands-" + uuidgen.String() + "-" + strconv.Itoa(i) namespacesStr := strings.Join(tc.namespaces, ",") - err = VeleroBackupNamespace(ctx, veleroCLI, veleroNamespace, backup, namespacesStr, "", false) + err = veleroBackupNamespace(ctx, veleroCLI, veleroNamespace, backup, namespacesStr, "", false) if err != nil { - VeleroBackupLogs(ctx, veleroCLI, veleroNamespace, backup) + veleroBackupLogs(ctx, veleroCLI, veleroNamespace, backup) return errors.Wrapf(err, "backing up %s namespaces on source cluster", namespacesStr) } // Delete music-system CRD and controllers installed on source cluster. - if err := DeleteCRD(ctx, tc.srcCRD["url"], tc.srcCRD["namespace"]); err != nil { + if err := deleteCRD(ctx, tc.srcCRD["url"], tc.srcCRD["namespace"]); err != nil { return errors.Wrapf(err, "deleting music-system CRD from source cluster") } @@ -248,13 +261,13 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, return errors.Wrapf(err, "deleting %s namespace from source cluster", ns) } - if err := WaitNamespaceDelete(ctx, ns); err != nil { + if err := waitNamespaceDelete(ctx, ns); err != nil { return errors.Wrapf(err, "deleting %s namespace from source cluster", ns) } } // Install music-system CRD for target cluster. - if err := InstallCRD(ctx, tc.tgtCRD["url"], tc.tgtCRD["namespace"]); err != nil { + if err := installCRD(ctx, tc.tgtCRD["url"], tc.tgtCRD["namespace"]); err != nil { return errors.Wrapf(err, "installing music-system CRD for target cluster") } @@ -267,7 +280,7 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, } // Reset Velero to recognize music-system CRD. - if err := RestartPods(ctx, veleroNamespace); err != nil { + if err := restartPods(ctx, veleroNamespace); err != nil { return errors.Wrapf(err, "restarting Velero pods") } fmt.Println("Sleep 20s to wait for Velero to stabilize after restart.") @@ -277,8 +290,8 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, restore := "restore-rockbands-" + uuidgen.String() + "-" + strconv.Itoa(i) if tc.want != nil { - if err := VeleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup); err != nil { - VeleroRestoreLogs(ctx, veleroCLI, veleroNamespace, restore) + if err := veleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup); err != nil { + veleroRestoreLogs(ctx, veleroCLI, veleroNamespace, restore) return errors.Wrapf(err, "restoring %s namespaces on target cluster", namespacesStr) } @@ -315,7 +328,7 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, } else { // No custom resource should have been restored. Expect "no resource found" // error during restore. - err := VeleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup) + err := veleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup) if err.Error() != "Unexpected restore phase got PartiallyFailed, expecting Completed" { return errors.New("expected error but not none") @@ -326,18 +339,18 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, for _, ns := range tc.namespaces { fmt.Println("Delete namespace", ns) _ = client.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}) - _ = WaitNamespaceDelete(ctx, ns) + _ = waitNamespaceDelete(ctx, ns) } // Delete source cluster music-system CRD - _ = DeleteCRD( + _ = deleteCRD( ctx, tc.srcCRD["url"], tc.srcCRD["namespace"], ) // Delete target cluster music-system CRD - _ = DeleteCRD( + _ = deleteCRD( ctx, tc.tgtCRD["url"], tc.srcCRD["namespace"], @@ -345,7 +358,7 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, // Uninstall Velero if installVelero { - err = VeleroUninstall(ctx, client, extensionsClient, veleroNamespace) + err = veleroUninstall(ctx, client, extensionsClient, veleroNamespace) if err != nil { return err } @@ -356,12 +369,12 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, } func installVeleroForAPIGroups(ctx context.Context) error { - if err := EnsureClusterExists(ctx); err != nil { + if err := ensureClusterExists(ctx); err != nil { return errors.Wrap(err, "check cluster exists") } // Pass global variables to option parameters. - options, err := GetProviderVeleroInstallOptions( + options, err := getProviderVeleroInstallOptions( cloudProvider, cloudCredentialsFile, bslBucket, @@ -379,14 +392,14 @@ func installVeleroForAPIGroups(ctx context.Context) error { options.Features = "EnableAPIGroupVersions" options.Image = veleroImage - if err := InstallVeleroServer(options); err != nil { + if err := installVeleroServer(options); err != nil { return errors.Wrap(err, "install velero server") } return nil } -func InstallCRD(ctx context.Context, crdFile, ns string) error { +func installCRD(ctx context.Context, crdFile, ns string) error { fmt.Printf("Install CRD %s.\n", crdFile) cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", crdFile) @@ -396,17 +409,17 @@ func InstallCRD(ctx context.Context, crdFile, ns string) error { } fmt.Println("Wait for CRD to be ready.") - if err := WaitForPodContainers(ctx, ns); err != nil { + if err := waitForPodContainers(ctx, ns); err != nil { return err } return err } -// WaitForPodContainers will get the pods and container status in a namespace. +// waitForPodContainers will get the pods and container status in a namespace. // If the ratio of the number of containers running to total in a pod is not 1, // it is not ready. Otherwise, if all container ratios are 1, the pod is running. -func WaitForPodContainers(ctx context.Context, ns string) error { +func waitForPodContainers(ctx context.Context, ns string) error { err := wait.Poll(3*time.Second, 4*time.Minute, func() (bool, error) { cmd := exec.CommandContext(ctx, "kubectl", "get", "pods", "-n", ns) stdout, stderr, err := veleroexec.RunCommand(cmd) @@ -436,7 +449,7 @@ func WaitForPodContainers(ctx context.Context, ns string) error { return err } -func DeleteCRD(ctx context.Context, crdFile, ns string) error { +func deleteCRD(ctx context.Context, crdFile, ns string) error { fmt.Println("Delete CRD", crdFile) cmd := exec.CommandContext(ctx, "kubectl", "delete", "-f", crdFile, "--wait") @@ -467,7 +480,7 @@ func DeleteCRD(ctx context.Context, crdFile, ns string) error { return err } -func RestartPods(ctx context.Context, ns string) error { +func restartPods(ctx context.Context, ns string) error { fmt.Printf("Restart pods in %s namespace.\n", ns) cmd := exec.CommandContext(ctx, "kubectl", "delete", "pod", "--all", "-n", ns) @@ -475,7 +488,7 @@ func RestartPods(ctx context.Context, ns string) error { if err == nil { fmt.Println("Wait for pods to be ready.") - if err := WaitForPodContainers(ctx, ns); err != nil { + if err := waitForPodContainers(ctx, ns); err != nil { return err } } @@ -483,7 +496,7 @@ func RestartPods(ctx context.Context, ns string) error { return err } -func InstallCR(ctx context.Context, crFile, ns string) error { +func installCR(ctx context.Context, crFile, ns string) error { retries := 5 var stderr string var err error @@ -503,7 +516,7 @@ func InstallCR(ctx context.Context, crFile, ns string) error { return errors.Wrap(err, stderr) } -func WaitNamespaceDelete(ctx context.Context, ns string) error { +func waitNamespaceDelete(ctx context.Context, ns string) error { err := wait.Poll(1*time.Second, 3*time.Minute, func() (bool, error) { cmd := exec.CommandContext(ctx, "kubectl", "get", "namespace", ns) diff --git a/test/e2e/kibishii_tests.go b/test/e2e/kibishii_tests.go index 09fffeedf..edb0ab5af 100644 --- a/test/e2e/kibishii_tests.go +++ b/test/e2e/kibishii_tests.go @@ -1,3 +1,16 @@ +/* +Copyright the Velero contributors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package e2e import ( @@ -77,15 +90,15 @@ func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel return nil } -// RunKibishiiTests runs kibishii tests on the provider. -func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string, +// runKibishiiTests runs kibishii tests on the provider. +func runKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string, useVolumeSnapshots bool) error { fiveMinTimeout, _ := context.WithTimeout(context.Background(), 5*time.Minute) oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) timeout := 10 * time.Minute interval := 5 * time.Second - if err := CreateNamespace(fiveMinTimeout, client, kibishiiNamespace); err != nil { + if err := createNamespace(fiveMinTimeout, client, kibishiiNamespace); err != nil { return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace) } @@ -104,8 +117,8 @@ func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, vel return errors.Wrap(err, "Failed to generate data") } - if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots); err != nil { - VeleroBackupLogs(fiveMinTimeout, veleroCLI, veleroNamespace, backupName) + if err := veleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots); err != nil { + veleroBackupLogs(fiveMinTimeout, veleroCLI, veleroNamespace, backupName) return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace) } @@ -122,13 +135,13 @@ func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, vel return errors.Wrap(err, "Failed to simulate a disaster") } // wait for ns delete - err := WaitForNamespaceDeletion(interval, timeout, client, kibishiiNamespace) + err := waitForNamespaceDeletion(interval, timeout, client, kibishiiNamespace) if err != nil { return errors.Wrapf(err, fmt.Sprintf("Failed to wait for deletion of namespace %s", kibishiiNamespace)) } - if err := VeleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil { - VeleroRestoreLogs(fiveMinTimeout, veleroCLI, veleroNamespace, restoreName) + if err := veleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil { + veleroRestoreLogs(fiveMinTimeout, veleroCLI, veleroNamespace, restoreName) return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName) } @@ -149,7 +162,7 @@ func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, vel return errors.Wrapf(err, "Failed to cleanup %s wrokload namespace", kibishiiNamespace) } // wait for ns delete - if err = WaitForNamespaceDeletion(interval, timeout, client, kibishiiNamespace); err != nil { + if err = waitForNamespaceDeletion(interval, timeout, client, kibishiiNamespace); err != nil { return errors.Wrapf(err, fmt.Sprintf("Failed to wait for deletion of namespace %s", kibishiiNamespace)) } fmt.Printf("kibishii test completed successfully\n") @@ -157,5 +170,5 @@ func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, vel } func waitForKibishiiPods(ctx context.Context, client *kubernetes.Clientset, kibishiiNamespace string) error { - return WaitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"}) + return waitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"}) } diff --git a/test/e2e/velero_utils.go b/test/e2e/velero_utils.go index 9162e6687..9b6fba708 100644 --- a/test/e2e/velero_utils.go +++ b/test/e2e/velero_utils.go @@ -1,3 +1,16 @@ +/* +Copyright the Velero contributors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package e2e import ( @@ -43,8 +56,8 @@ func getProviderPlugins(providerName string) []string { } } -// GetProviderVeleroInstallOptions returns Velero InstallOptions for the provider. -func GetProviderVeleroInstallOptions( +// getProviderVeleroInstallOptions returns Velero InstallOptions for the provider. +func getProviderVeleroInstallOptions( pluginProvider, credentialsFile, objectStoreBucket, @@ -84,8 +97,8 @@ func GetProviderVeleroInstallOptions( return io, nil } -// InstallVeleroServer installs velero in the cluster. -func InstallVeleroServer(io *cliinstall.InstallOptions) error { +// installVeleroServer installs velero in the cluster. +func installVeleroServer(io *cliinstall.InstallOptions) error { config, err := client.LoadConfig() if err != nil { return err @@ -97,7 +110,7 @@ func InstallVeleroServer(io *cliinstall.InstallOptions) error { } f := client.NewFactory("e2e", config) - resources, err := install.AllResources(vo) + resources := install.AllResources(vo) if err != nil { return errors.Wrap(err, "Failed to install Velero in the cluster") } @@ -128,8 +141,8 @@ func InstallVeleroServer(io *cliinstall.InstallOptions) error { return nil } -// CheckBackupPhase uses veleroCLI to inspect the phase of a Velero backup. -func CheckBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, +// checkBackupPhase uses veleroCLI to inspect the phase of a Velero backup. +func checkBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, expectedPhase velerov1api.BackupPhase) error { checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "get", "-o", "json", backupName) @@ -172,8 +185,8 @@ func CheckBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace str return nil } -// CheckRestorePhase uses veleroCLI to inspect the phase of a Velero restore. -func CheckRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, +// checkRestorePhase uses veleroCLI to inspect the phase of a Velero restore. +func checkRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, expectedPhase velerov1api.RestorePhase) error { checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "restore", "get", "-o", "json", restoreName) @@ -216,8 +229,8 @@ func CheckRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace st return nil } -// VeleroBackupNamespace uses the veleroCLI to backup a namespace. -func VeleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string, backupLocation string, +// veleroBackupNamespace uses the veleroCLI to backup a namespace. +func veleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string, backupLocation string, useVolumeSnapshots bool) error { args := []string{ "--namespace", veleroNamespace, @@ -243,13 +256,13 @@ func VeleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespac if err != nil { return err } - err = CheckBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted) + err = checkBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted) return err } -// VeleroRestore uses the veleroCLI to restore from a Velero backup. -func VeleroRestore(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, backupName string) error { +// veleroRestore uses the veleroCLI to restore from a Velero backup. +func veleroRestore(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, backupName string) error { restoreCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "create", "restore", restoreName, "--from-backup", backupName, "--wait") @@ -260,10 +273,10 @@ func VeleroRestore(ctx context.Context, veleroCLI string, veleroNamespace string if err != nil { return err } - return CheckRestorePhase(ctx, veleroCLI, veleroNamespace, restoreName, velerov1api.RestorePhaseCompleted) + return checkRestorePhase(ctx, veleroCLI, veleroNamespace, restoreName, velerov1api.RestorePhaseCompleted) } -func VeleroInstall(ctx context.Context, veleroImage string, veleroNamespace string, cloudProvider string, objectStoreProvider string, useVolumeSnapshots bool, +func veleroInstall(ctx context.Context, veleroImage string, veleroNamespace string, cloudProvider string, objectStoreProvider string, useVolumeSnapshots bool, cloudCredentialsFile string, bslBucket string, bslPrefix string, bslConfig string, vslConfig string, features string) error { @@ -287,11 +300,11 @@ func VeleroInstall(ctx context.Context, veleroImage string, veleroNamespace stri // Snapshot location specified objectStoreProvider = "aws" } - err := EnsureClusterExists(ctx) + err := ensureClusterExists(ctx) if err != nil { return errors.WithMessage(err, "Failed to ensure kubernetes cluster exists") } - veleroInstallOptions, err := GetProviderVeleroInstallOptions(objectStoreProvider, cloudCredentialsFile, bslBucket, + veleroInstallOptions, err := getProviderVeleroInstallOptions(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, providerPlugins, features) if useVolumeSnapshots { if cloudProvider != "vsphere" { @@ -309,18 +322,18 @@ func VeleroInstall(ctx context.Context, veleroImage string, veleroNamespace stri veleroInstallOptions.Image = veleroImage veleroInstallOptions.Namespace = veleroNamespace - err = InstallVeleroServer(veleroInstallOptions) + err = installVeleroServer(veleroInstallOptions) if err != nil { return errors.WithMessagef(err, "Failed to install Velero in cluster") } return nil } -func VeleroUninstall(ctx context.Context, client *kubernetes.Clientset, extensionsClient *apiextensionsclient.Clientset, veleroNamespace string) error { +func veleroUninstall(ctx context.Context, client *kubernetes.Clientset, extensionsClient *apiextensionsclient.Clientset, veleroNamespace string) error { return uninstall.Run(ctx, client, extensionsClient, veleroNamespace, true) } -func VeleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error { +func veleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error { describeCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "describe", backupName) describeCmd.Stdout = os.Stdout describeCmd.Stderr = os.Stderr @@ -338,7 +351,7 @@ func VeleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace str return nil } -func VeleroRestoreLogs(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string) error { +func veleroRestoreLogs(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string) error { describeCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "restore", "describe", restoreName) describeCmd.Stdout = os.Stdout describeCmd.Stderr = os.Stderr @@ -356,7 +369,7 @@ func VeleroRestoreLogs(ctx context.Context, veleroCLI string, veleroNamespace st return nil } -func VeleroCreateBackupLocation(ctx context.Context, +func veleroCreateBackupLocation(ctx context.Context, veleroCLI string, veleroNamespace string, name string, @@ -393,9 +406,9 @@ func VeleroCreateBackupLocation(ctx context.Context, return bslCreateCmd.Run() } -// VeleroAddPluginsForProvider determines which plugins need to be installed for a provider and +// veleroAddPluginsForProvider determines which plugins need to be installed for a provider and // installs them in the current Velero installation, skipping over those that are already installed. -func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string) error { +func veleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string) error { for _, plugin := range getProviderPlugins(provider) { stdoutBuf := new(bytes.Buffer) stderrBuf := new(bytes.Buffer) @@ -421,10 +434,8 @@ func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNa return nil } -/* - Waits for uploads started by the Velero Plug-in for vSphere to complete - TODO - remove after upload progress monitoring is implemented -*/ +// waitForVSphereUploadCompletion waits for uploads started by the Velero Plug-in for vSphere to complete +// TODO - remove after upload progress monitoring is implemented func waitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, namespace string) error { err := wait.PollImmediate(time.Minute, timeout, func() (bool, error) { checkSnapshotCmd := exec.CommandContext(ctx, "kubectl",