diff --git a/test/e2e/Makefile b/test/e2e/Makefile index f676fabd6..53d52364e 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -89,6 +89,9 @@ ADDITIONAL_BSL_CONFIG ?= FEATURES ?= DEBUG_E2E_TEST ?= false +# Parameters to run migration tests along with all other E2E tests, and both of them should +# be provided or left them all empty to skip migration tests with no influence to other +# E2E tests. DEFAULT_CLUSTER ?= STANDBY_CLUSTER ?= diff --git a/test/e2e/backup/backup.go b/test/e2e/backup/backup.go index e4e62db59..550773718 100644 --- a/test/e2e/backup/backup.go +++ b/test/e2e/backup/backup.go @@ -59,8 +59,11 @@ func BackupRestoreTest(useVolumeSnapshots bool) { }) AfterEach(func() { - if VeleroCfg.InstallVelero { - if !VeleroCfg.Debug { + if !VeleroCfg.Debug { + By("Clean backups after test", func() { + DeleteBackups(context.Background(), *VeleroCfg.ClientToInstallVelero) + }) + if VeleroCfg.InstallVelero { err = VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) Expect(err).To(Succeed()) } diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index 0663e554a..59944d6aa 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -64,12 +64,16 @@ func backup_deletion_test(useVolumeSnapshots bool) { }) AfterEach(func() { - if VeleroCfg.InstallVelero { - if !VeleroCfg.Debug { + if !VeleroCfg.Debug { + By("Clean backups after test", func() { + DeleteBackups(context.Background(), *VeleroCfg.ClientToInstallVelero) + }) + if VeleroCfg.InstallVelero { err = VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) Expect(err).To(Succeed()) } } + }) When("kibishii is the sample workload", func() { diff --git a/test/e2e/backups/sync_backups.go b/test/e2e/backups/sync_backups.go index db062e67d..fc2c27337 100644 --- a/test/e2e/backups/sync_backups.go +++ b/test/e2e/backups/sync_backups.go @@ -64,11 +64,15 @@ func BackupsSyncTest() { }) AfterEach(func() { - if VeleroCfg.InstallVelero { - if !VeleroCfg.Debug { + if !VeleroCfg.Debug { + By("Clean backups after test", func() { + DeleteBackups(context.Background(), *VeleroCfg.ClientToInstallVelero) + }) + if VeleroCfg.InstallVelero { Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed()) } } + }) It("Backups in object storage should be synced to a new Velero successfully", func() { @@ -76,10 +80,11 @@ func BackupsSyncTest() { By(fmt.Sprintf("Prepare workload as target to backup by creating namespace %s namespace", test.testNS)) Expect(CreateNamespace(test.ctx, *VeleroCfg.ClientToInstallVelero, test.testNS)).To(Succeed(), fmt.Sprintf("Failed to create %s namespace", test.testNS)) - - defer func() { - Expect(DeleteNamespace(test.ctx, *VeleroCfg.ClientToInstallVelero, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS)) - }() + if !VeleroCfg.Debug { + defer func() { + Expect(DeleteNamespace(test.ctx, *VeleroCfg.ClientToInstallVelero, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS)) + }() + } var BackupCfg BackupConfig BackupCfg.BackupName = test.backupName @@ -120,7 +125,6 @@ func BackupsSyncTest() { fmt.Sprintf("Failed to delete the namespace %s", test.testNS)) }() } - var BackupCfg BackupConfig BackupCfg.BackupName = test.backupName BackupCfg.Namespace = test.testNS diff --git a/test/e2e/backups/ttl.go b/test/e2e/backups/ttl.go index 78f1915a1..b2e05a17b 100644 --- a/test/e2e/backups/ttl.go +++ b/test/e2e/backups/ttl.go @@ -76,12 +76,15 @@ func TTLTest() { }) AfterEach(func() { - if VeleroCfg.InstallVelero { - VeleroCfg.GCFrequency = "" - if !VeleroCfg.Debug { + VeleroCfg.GCFrequency = "" + if !VeleroCfg.Debug { + By("Clean backups after test", func() { + DeleteBackups(context.Background(), *VeleroCfg.ClientToInstallVelero) + }) + if VeleroCfg.InstallVelero { Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed()) - Expect(DeleteNamespace(test.ctx, client, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS)) } + Expect(DeleteNamespace(test.ctx, client, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS)) } }) diff --git a/test/e2e/basic/enable_api_group_versions.go b/test/e2e/basic/enable_api_group_versions.go index 86afaf892..220f6a029 100644 --- a/test/e2e/basic/enable_api_group_versions.go +++ b/test/e2e/basic/enable_api_group_versions.go @@ -63,22 +63,25 @@ func APIGropuVersionsTest() { }) AfterEach(func() { - fmt.Printf("Clean up resource: kubectl delete crd %s.%s\n", resource, group) - cmd := exec.CommandContext(ctx, "kubectl", "delete", "crd", resource+"."+group) - _, stderr, err := veleroexec.RunCommand(cmd) - if strings.Contains(stderr, "NotFound") { - fmt.Printf("Ignore error: %v\n", stderr) - err = nil - } - Expect(err).NotTo(HaveOccurred()) + if !VeleroCfg.Debug { + fmt.Printf("Clean up resource: kubectl delete crd %s.%s\n", resource, group) + cmd := exec.CommandContext(ctx, "kubectl", "delete", "crd", resource+"."+group) + _, stderr, err := veleroexec.RunCommand(cmd) + if strings.Contains(stderr, "NotFound") { + fmt.Printf("Ignore error: %v\n", stderr) + err = nil + } + Expect(err).NotTo(HaveOccurred()) + By("Clean backups after test", func() { + DeleteBackups(context.Background(), *VeleroCfg.ClientToInstallVelero) + }) + if VeleroCfg.InstallVelero { - if VeleroCfg.InstallVelero { - if !VeleroCfg.Debug { - err = VeleroUninstall(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) - Expect(err).NotTo(HaveOccurred()) + By("Uninstall Velero", func() { + Expect(VeleroUninstall(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).NotTo(HaveOccurred()) + }) } } - }) Context("When EnableAPIGroupVersions flag is set", func() { diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index 9800237d4..d7504342a 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -71,15 +71,23 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) AfterEach(func() { - if VeleroCfg.InstallVelero { - if !VeleroCfg.Debug { + if !VeleroCfg.Debug { + By("Clean backups after test", func() { + DeleteBackups(context.Background(), *VeleroCfg.DefaultClient) + }) + By(fmt.Sprintf("Delete sample workload namespace %s", bslDeletionTestNs), func() { Expect(DeleteNamespace(context.Background(), *VeleroCfg.ClientToInstallVelero, bslDeletionTestNs, true)).To(Succeed(), fmt.Sprintf("failed to delete the namespace %q", bslDeletionTestNs)) - Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace)).To(Succeed()) + }) + if VeleroCfg.InstallVelero { + By("Uninstall Velero", func() { + Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, + VeleroCfg.VeleroNamespace)).To(Succeed()) + }) } } + }) When("kibishii is the sample workload", func() { diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index f0f376f61..5c31ca9b7 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -68,8 +68,11 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) } }) AfterEach(func() { - if VeleroCfg.InstallVelero { - if !VeleroCfg.Debug { + if !VeleroCfg.Debug { + By("Clean backups after test", func() { + DeleteBackups(context.Background(), *VeleroCfg.DefaultClient) + }) + if VeleroCfg.InstallVelero { By(fmt.Sprintf("Uninstall Velero and delete sample workload namespace %s", migrationNamespace), func() { Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed()) Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, @@ -81,12 +84,13 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) VeleroCfg.VeleroNamespace)).To(Succeed()) DeleteNamespace(context.Background(), *VeleroCfg.StandbyClient, migrationNamespace, true) }) - By(fmt.Sprintf("Switch to default kubeconfig context %s", VeleroCfg.DefaultClient), func() { - Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed()) - VeleroCfg.ClientToInstallVelero = VeleroCfg.DefaultClient - }) } + By(fmt.Sprintf("Switch to default kubeconfig context %s", VeleroCfg.DefaultClient), func() { + Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed()) + VeleroCfg.ClientToInstallVelero = VeleroCfg.DefaultClient + }) } + }) When("kibishii is the sample workload", func() { It("should be successfully backed up and restored to the default BackupStorageLocation", func() { diff --git a/test/e2e/resource-filtering/exclude_label.go b/test/e2e/resource-filtering/exclude_label.go index 72307fe3f..2147221c9 100644 --- a/test/e2e/resource-filtering/exclude_label.go +++ b/test/e2e/resource-filtering/exclude_label.go @@ -19,12 +19,14 @@ package filtering import ( "context" "fmt" - "strings" "time" - "github.com/pkg/errors" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" apierrors "k8s.io/apimachinery/pkg/api/errors" + "github.com/pkg/errors" + . "github.com/vmware-tanzu/velero/test/e2e" . "github.com/vmware-tanzu/velero/test/e2e/test" . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" @@ -62,7 +64,7 @@ func (e *ExcludeFromBackup) Init() error { e.BackupArgs = []string{ "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", e.BackupName, - "--include-namespaces", strings.Join(*e.NSIncluded, ","), + "--include-namespaces", e.NSBaseName, "--default-volumes-to-restic", "--wait", } @@ -75,63 +77,87 @@ func (e *ExcludeFromBackup) Init() error { func (e *ExcludeFromBackup) CreateResources() error { e.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute) - for nsNum := 0; nsNum < e.NamespacesTotal; nsNum++ { - namespace := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) - fmt.Printf("Creating resources in namespace ...%s\n", namespace) - labels := e.labels - if nsNum%2 == 0 { - labels = map[string]string{ - "velero.io/exclude-from-backup": "false", - } - } - if err := CreateNamespaceWithLabel(e.Ctx, e.Client, namespace, labels); err != nil { - return errors.Wrapf(err, "Failed to create namespace %s", namespace) - } - serviceAccountName := "default" - // wait until the service account is created before patch the image pull secret - if err := WaitUntilServiceAccountCreated(e.Ctx, e.Client, namespace, serviceAccountName, 10*time.Minute); err != nil { - return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, namespace) - } - // add the image pull secret to avoid the image pull limit issue of Docker Hub - if err := PatchServiceAccountWithImagePullSecret(e.Ctx, e.Client, namespace, serviceAccountName, VeleroCfg.RegistryCredentialFile); err != nil { - return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, namespace) - } - //Create deployment - fmt.Printf("Creating deployment in namespaces ...%s\n", namespace) - - deployment := NewDeployment(e.NSBaseName, namespace, e.replica, labels) - deployment, err := CreateDeployment(e.Client.ClientGo, namespace, deployment) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) - } - err = WaitForReadyDeployment(e.Client.ClientGo, namespace, deployment.Name) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("failed to ensure deployment completion in namespace: %q", namespace)) - } + namespace := e.NSBaseName + // These 2 labels for resources to be included + label1 := map[string]string{ + "meaningless-label-resource-to-include": "true", + } + label2 := map[string]string{ + "velero.io/exclude-from-backup": "false", + } + fmt.Printf("Creating resources in namespace ...%s\n", namespace) + if err := CreateNamespace(e.Ctx, e.Client, namespace); err != nil { + return errors.Wrapf(err, "Failed to create namespace %s", namespace) + } + serviceAccountName := "default" + // wait until the service account is created before patch the image pull secret + if err := WaitUntilServiceAccountCreated(e.Ctx, e.Client, namespace, serviceAccountName, 10*time.Minute); err != nil { + return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, namespace) + } + // add the image pull secret to avoid the image pull limit issue of Docker Hub + if err := PatchServiceAccountWithImagePullSecret(e.Ctx, e.Client, namespace, serviceAccountName, VeleroCfg.RegistryCredentialFile); err != nil { + return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, namespace) + } + //Create deployment: to be included + fmt.Printf("Creating deployment in namespaces ...%s\n", namespace) + deployment := NewDeployment(e.NSBaseName, namespace, e.replica, label2) + deployment, err := CreateDeployment(e.Client.ClientGo, namespace, deployment) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) + } + err = WaitForReadyDeployment(e.Client.ClientGo, namespace, deployment.Name) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to ensure job completion in namespace: %q", namespace)) + } + //Create Secret + secretName := e.NSBaseName + fmt.Printf("Creating secret %s in namespaces ...%s\n", secretName, namespace) + _, err = CreateSecret(e.Client.ClientGo, namespace, secretName, e.labels) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to create secret in the namespace %q", namespace)) + } + err = WaitForSecretsComplete(e.Client.ClientGo, namespace, secretName) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to ensure secret completion in namespace: %q", namespace)) + } + By(fmt.Sprintf("Checking secret %s should exists in namespaces ...%s\n", secretName, namespace), func() { + _, err = GetSecret(e.Client.ClientGo, namespace, e.NSBaseName) + Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) + }) + //Create Configmap: to be included + configmaptName := e.NSBaseName + fmt.Printf("Creating configmap %s in namespaces ...%s\n", configmaptName, namespace) + _, err = CreateConfigMap(e.Client.ClientGo, namespace, configmaptName, label1) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to create configmap in the namespace %q", namespace)) + } + err = WaitForConfigMapComplete(e.Client.ClientGo, namespace, configmaptName) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to ensure secret completion in namespace: %q", namespace)) } return nil } func (e *ExcludeFromBackup) Verify() error { - for nsNum := 0; nsNum < e.NamespacesTotal; nsNum++ { - namespace := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) - fmt.Printf("Checking resources in namespaces ...%s\n", namespace) - //Check deployment - _, err := GetDeployment(e.Client.ClientGo, namespace, e.NSBaseName) - if nsNum%2 == 0 { //include - if err != nil { - return errors.Wrap(err, fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) - } - } else { //exclude - if err == nil { - return fmt.Errorf("failed to exclude deployment in namespaces %q", namespace) - } else { - if apierrors.IsNotFound(err) { //resource should be excluded - return nil - } - return errors.Wrap(err, fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) - } - } - } + namespace := e.NSBaseName + By(fmt.Sprintf("Checking resources in namespaces ...%s\n", namespace), func() { + //Check namespace + checkNS, err := GetNamespace(e.Ctx, e.Client, namespace) + Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("Could not retrieve test namespace %s", namespace)) + Expect(checkNS.Name == namespace).To(Equal(true), fmt.Sprintf("Retrieved namespace for %s has name %s instead", namespace, checkNS.Name)) + + //Check deployment: should be included + _, err = GetDeployment(e.Client.ClientGo, namespace, e.NSBaseName) + Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) + + //Check secrets: secrets should not be included + _, err = GetSecret(e.Client.ClientGo, namespace, e.NSBaseName) + Expect(err).Should(HaveOccurred(), fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) + Expect(apierrors.IsNotFound(err)).To(Equal(true)) + + //Check configmap: should be included + _, err = GetConfigmap(e.Client.ClientGo, namespace, e.NSBaseName) + Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("failed to list configmap in namespace: %q", namespace)) + }) return nil } diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go index fe0ad7e9d..dc8597512 100644 --- a/test/e2e/test/test.go +++ b/test/e2e/test/test.go @@ -83,8 +83,8 @@ func TestFunc(test VeleroBackupRestoreTest) func() { } }) AfterEach(func() { - if VeleroCfg.InstallVelero { - if !VeleroCfg.Debug { + if !VeleroCfg.Debug { + if VeleroCfg.InstallVelero { Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To((Succeed())) } } @@ -117,9 +117,11 @@ func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() { }) AfterEach(func() { - if VeleroCfg.InstallVelero { - if countIt == len(tests) && !VeleroCfg.Debug { - Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To((Succeed())) + if !VeleroCfg.Debug { + if VeleroCfg.InstallVelero { + if countIt == len(tests) && !VeleroCfg.Debug { + Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To((Succeed())) + } } } }) @@ -174,7 +176,15 @@ func (t *TestCase) Verify() error { } func (t *TestCase) Clean() error { - return CleanupNamespacesWithPoll(t.Ctx, t.Client, t.NSBaseName) + if !VeleroCfg.Debug { + By(fmt.Sprintf("Clean namespace with prefix %s after test", t.NSBaseName), func() { + CleanupNamespaces(t.Ctx, t.Client, t.NSBaseName) + }) + By("Clean backups after test", func() { + DeleteBackups(t.Ctx, t.Client) + }) + } + return nil } func (t *TestCase) GetTestMsg() *TestMSG { diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index 488e25961..c86273038 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -70,11 +70,14 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC } }) AfterEach(func() { - if VeleroCfg.InstallVelero { - if !VeleroCfg.Debug { - By(fmt.Sprintf("Delete sample workload namespace %s", upgradeNamespace), func() { - DeleteNamespace(context.Background(), *VeleroCfg.ClientToInstallVelero, upgradeNamespace, true) - }) + if !VeleroCfg.Debug { + By("Clean backups after test", func() { + DeleteBackups(context.Background(), *VeleroCfg.ClientToInstallVelero) + }) + By(fmt.Sprintf("Delete sample workload namespace %s", upgradeNamespace), func() { + DeleteNamespace(context.Background(), *VeleroCfg.ClientToInstallVelero, upgradeNamespace, true) + }) + if VeleroCfg.InstallVelero { By("Uninstall Velero", func() { Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed()) diff --git a/test/e2e/util/velero/velero_utils.go b/test/e2e/util/velero/velero_utils.go index aa54a0cff..e7bc0404e 100644 --- a/test/e2e/util/velero/velero_utils.go +++ b/test/e2e/util/velero/velero_utils.go @@ -37,6 +37,8 @@ import ( "github.com/pkg/errors" "k8s.io/apimachinery/pkg/util/wait" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" cliinstall "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" @@ -433,6 +435,9 @@ func VeleroCmdExec(ctx context.Context, veleroCLI string, args []string) error { cmd.Stderr = os.Stderr fmt.Printf("velero cmd =%v\n", cmd) err := cmd.Run() + if strings.Contains(fmt.Sprint(cmd.Stdout), "Failed") { + return errors.New(fmt.Sprintf("velero cmd =%v return with failure\n", cmd)) + } if err != nil { return err } @@ -1008,3 +1013,16 @@ func GetVersionList(veleroCli, veleroVersion string) []VeleroCLI2Version { } return veleroCLI2VersionList } +func DeleteBackups(ctx context.Context, client TestClient) error { + backupList := new(velerov1api.BackupList) + if err := client.Kubebuilder.List(ctx, backupList, &kbclient.ListOptions{Namespace: VeleroCfg.VeleroNamespace}); err != nil { + return fmt.Errorf("failed to list backup object in %s namespace with err %v", VeleroCfg.VeleroNamespace, err) + } + for _, backup := range backupList.Items { + fmt.Printf("Backup %s is going to be deleted...", backup.Name) + if err := VeleroBackupDelete(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup.Name); err != nil { + return err + } + } + return nil +}