diff --git a/changelogs/unreleased/4404-mqiu b/changelogs/unreleased/4404-mqiu new file mode 100644 index 000000000..99fa88dca --- /dev/null +++ b/changelogs/unreleased/4404-mqiu @@ -0,0 +1 @@ +Add resource filtering test cases diff --git a/test/e2e/backup/backup.go b/test/e2e/backup/backup.go index e5ff9bf26..21ac1c509 100644 --- a/test/e2e/backup/backup.go +++ b/test/e2e/backup/backup.go @@ -25,9 +25,9 @@ import ( . "github.com/onsi/gomega" . "github.com/vmware-tanzu/velero/test/e2e" - k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s" - kibishiiutils "github.com/vmware-tanzu/velero/test/e2e/util/kibishii" - veleroutils "github.com/vmware-tanzu/velero/test/e2e/util/velero" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + . "github.com/vmware-tanzu/velero/test/e2e/util/kibishii" + . "github.com/vmware-tanzu/velero/test/e2e/util/velero" ) func BackupRestoreWithSnapshots() { @@ -43,7 +43,7 @@ func BackupRestoreTest(useVolumeSnapshots bool) { backupName, restoreName string ) - client, err := k8sutils.NewTestClient() + client, err := NewTestClient() Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") BeforeEach(func() { @@ -55,13 +55,13 @@ func BackupRestoreTest(useVolumeSnapshots bool) { UUIDgen, err = uuid.NewRandom() Expect(err).To(Succeed()) if VeleroCfg.InstallVelero { - Expect(veleroutils.VeleroInstall(context.Background(), &VeleroCfg, "", useVolumeSnapshots)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &VeleroCfg, "", useVolumeSnapshots)).To(Succeed()) } }) AfterEach(func() { if VeleroCfg.InstallVelero { - err = veleroutils.VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) + err = VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) Expect(err).To(Succeed()) } }) @@ -72,7 +72,7 @@ func BackupRestoreTest(useVolumeSnapshots bool) { restoreName = "restore-" + UUIDgen.String() // Even though we are using Velero's CloudProvider plugin for object storage, the kubernetes cluster is running on // KinD. So use the kind installation for Kibishii. - Expect(kibishiiutils.RunKibishiiTests(client, VeleroCfg.CloudProvider, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, restoreName, "", useVolumeSnapshots, VeleroCfg.RegistryCredentialFile)).To(Succeed(), + Expect(RunKibishiiTests(client, VeleroCfg.CloudProvider, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, restoreName, "", useVolumeSnapshots, VeleroCfg.RegistryCredentialFile)).To(Succeed(), "Failed to successfully backup and restore Kibishii namespace") }) @@ -89,7 +89,7 @@ func BackupRestoreTest(useVolumeSnapshots bool) { Skip("no additional BSL credentials given, not running multiple BackupStorageLocation with unique credentials tests") } - Expect(veleroutils.VeleroAddPluginsForProvider(context.TODO(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, VeleroCfg.AdditionalBSLProvider, VeleroCfg.AddBSLPlugins)).To(Succeed()) + Expect(VeleroAddPluginsForProvider(context.TODO(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, VeleroCfg.AdditionalBSLProvider, VeleroCfg.AddBSLPlugins)).To(Succeed()) // Create Secret for additional BSL secretName := fmt.Sprintf("bsl-credentials-%s", UUIDgen) @@ -98,11 +98,11 @@ func BackupRestoreTest(useVolumeSnapshots bool) { secretKey: VeleroCfg.AdditionalBSLCredentials, } - Expect(k8sutils.CreateSecretFromFiles(context.TODO(), client, VeleroCfg.VeleroNamespace, secretName, files)).To(Succeed()) + Expect(CreateSecretFromFiles(context.TODO(), client, VeleroCfg.VeleroNamespace, secretName, files)).To(Succeed()) // Create additional BSL using credential additionalBsl := fmt.Sprintf("bsl-%s", UUIDgen) - Expect(veleroutils.VeleroCreateBackupLocation(context.TODO(), + Expect(VeleroCreateBackupLocation(context.TODO(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, additionalBsl, @@ -125,7 +125,7 @@ func BackupRestoreTest(useVolumeSnapshots bool) { backupName = fmt.Sprintf("%s-%s", backupName, UUIDgen) restoreName = fmt.Sprintf("%s-%s", restoreName, UUIDgen) } - Expect(kibishiiutils.RunKibishiiTests(client, VeleroCfg.CloudProvider, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, restoreName, bsl, useVolumeSnapshots, VeleroCfg.RegistryCredentialFile)).To(Succeed(), + Expect(RunKibishiiTests(client, VeleroCfg.CloudProvider, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, restoreName, bsl, useVolumeSnapshots, VeleroCfg.RegistryCredentialFile)).To(Succeed(), "Failed to successfully backup and restore Kibishii namespace using BSL %s", bsl) } }) diff --git a/test/e2e/basic/enable_api_group_versions.go b/test/e2e/basic/enable_api_group_versions.go index d9c13e280..215fdf773 100644 --- a/test/e2e/basic/enable_api_group_versions.go +++ b/test/e2e/basic/enable_api_group_versions.go @@ -36,8 +36,8 @@ import ( "github.com/vmware-tanzu/velero/pkg/builder" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" . "github.com/vmware-tanzu/velero/test/e2e" - k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s" - veleroutils "github.com/vmware-tanzu/velero/test/e2e/util/velero" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + . "github.com/vmware-tanzu/velero/test/e2e/util/velero" ) func APIGropuVersionsTest() { @@ -47,7 +47,7 @@ func APIGropuVersionsTest() { ctx = context.Background() ) - client, err := k8sutils.NewTestClient() + client, err := NewTestClient() Expect(err).To(Succeed(), "Failed to instantiate cluster client for group version tests") BeforeEach(func() { @@ -59,7 +59,7 @@ func APIGropuVersionsTest() { // TODO: install Velero once for the test suite once feature flag is // removed and velero installation becomes the same as other e2e tests. if VeleroCfg.InstallVelero { - err = veleroutils.VeleroInstall(context.Background(), &VeleroCfg, "EnableAPIGroupVersions", false) + err = VeleroInstall(context.Background(), &VeleroCfg, "EnableAPIGroupVersions", false) Expect(err).NotTo(HaveOccurred()) } }) @@ -75,7 +75,7 @@ func APIGropuVersionsTest() { Expect(err).NotTo(HaveOccurred()) if VeleroCfg.InstallVelero { - err = veleroutils.VeleroUninstall(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) + err = VeleroUninstall(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) Expect(err).NotTo(HaveOccurred()) } @@ -93,7 +93,7 @@ func APIGropuVersionsTest() { }) } -func runEnableAPIGroupVersionsTests(ctx context.Context, client k8sutils.TestClient, resource, group string) error { +func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, resource, group string) error { tests := []struct { name string namespaces []string @@ -210,11 +210,11 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client k8sutils.TestCli for version, cr := range tc.srcCRs { ns := resource + "-src-" + version - if err := k8sutils.CreateNamespace(ctx, client, ns); err != nil { + if err := CreateNamespace(ctx, client, ns); err != nil { return errors.Wrapf(err, "create %s namespace", ns) } defer func(namespace string) { - if err = k8sutils.DeleteNamespace(ctx, client, namespace, true); err != nil { + if err = DeleteNamespace(ctx, client, namespace, true); err != nil { fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", ns)) } }(ns) @@ -236,9 +236,9 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client k8sutils.TestCli backup := "backup-rockbands-" + UUIDgen.String() + "-" + strconv.Itoa(i) namespacesStr := strings.Join(tc.namespaces, ",") - err = veleroutils.VeleroBackupNamespace(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup, namespacesStr, "", false) + err = VeleroBackupNamespace(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup, namespacesStr, "", false) if err != nil { - veleroutils.RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup, "") + RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup, "") return errors.Wrapf(err, "back up %s namespaces on source cluster", namespacesStr) } @@ -247,7 +247,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client k8sutils.TestCli } for _, ns := range tc.namespaces { - if err := k8sutils.DeleteNamespace(ctx, client, ns, true); err != nil { + if err := DeleteNamespace(ctx, client, ns, true); err != nil { return errors.Wrapf(err, "delete %s namespace from source cluster", ns) } } @@ -276,8 +276,8 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client k8sutils.TestCli restore := "restore-rockbands-" + UUIDgen.String() + "-" + strconv.Itoa(i) if tc.want != nil { - if err := veleroutils.VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restore, backup); err != nil { - veleroutils.RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", restore) + if err := VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restore, backup); err != nil { + RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", restore) return errors.Wrapf(err, "restore %s namespaces on target cluster", namespacesStr) } @@ -315,7 +315,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client k8sutils.TestCli } else { // No custom resource should have been restored. Expect "no resource found" // error during restore. - err := veleroutils.VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restore, backup) + err := VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restore, backup) if err.Error() != "Unexpected restore phase got PartiallyFailed, expecting Completed" { return errors.New("expected error but not none") diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 825ce2938..7753c60e8 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -27,6 +27,7 @@ import ( . "github.com/vmware-tanzu/velero/test/e2e" . "github.com/vmware-tanzu/velero/test/e2e/backup" . "github.com/vmware-tanzu/velero/test/e2e/basic" + . "github.com/vmware-tanzu/velero/test/e2e/resource-filtering" . "github.com/vmware-tanzu/velero/test/e2e/scale" . "github.com/vmware-tanzu/velero/test/e2e/upgrade" ) @@ -73,9 +74,20 @@ var _ = Describe("[Scale] Backup/restore of 2500 namespaces", MultiNSBackupResto // Upgrade test by Kibishi using restic var _ = Describe("[Upgrade][Restic] Velero upgrade tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupUpgradeRestoreWithRestic) - var _ = Describe("[Upgrade][Snapshot] Velero upgrade tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupUpgradeRestoreWithSnapshots) +// test filter objects by namespace, type, or labels when backup or restore. +var _ = Describe("[ResourceFiltering][ExcludeFromBackup] Resources with the label velero.io/exclude-from-backup=true are not included in backup", ExcludeFromBackupTest) +var _ = Describe("[ResourceFiltering][ExcludeNamespaces][Backup] Velero test on exclude namespace from the cluster backup", BackupWithExcludeNamespaces) +var _ = Describe("[ResourceFiltering][ExcludeNamespaces][Restore] Velero test on exclude namespace from the cluster restore", RestoreWithExcludeNamespaces) +var _ = Describe("[ResourceFiltering][ExcludeResources][Backup] Velero test on exclude resources from the cluster backup", BackupWithExcludeResources) +var _ = Describe("[ResourceFiltering][ExcludeResources][Restore] Velero test on exclude resources from the cluster restore", RestoreWithExcludeResources) +var _ = Describe("[ResourceFiltering][IncludeNamespaces][Backup] Velero test on include namespace from the cluster backup", BackupWithIncludeNamespaces) +var _ = Describe("[ResourceFiltering][IncludeNamespaces][Restore] Velero test on include namespace from the cluster restore", RestoreWithIncludeNamespaces) +var _ = Describe("[ResourceFiltering][IncludeResources][Backup] Velero test on include resources from the cluster backup", BackupWithIncludeResources) +var _ = Describe("[ResourceFiltering][IncludeResources][Restore] Velero test on include resources from the cluster restore", RestoreWithIncludeResources) +var _ = Describe("[ResourceFiltering][LabelSelector] Velero test on backup include resources matching the label selector", BackupWithLabelSelector) + func TestE2e(t *testing.T) { // Skip running E2E tests when running only "short" tests because: // 1. E2E tests are long running tests involving installation of Velero and performing backup and restore operations. diff --git a/test/e2e/resource-filtering/base.go b/test/e2e/resource-filtering/base.go new file mode 100644 index 000000000..77d97da19 --- /dev/null +++ b/test/e2e/resource-filtering/base.go @@ -0,0 +1,146 @@ +/* +Copyright 2021 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filtering + +import ( + "context" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" +) + +type FilteringCase struct { + TestCase + IsTestInBackup bool + replica int32 + labels map[string]string + labelSelector string +} + +var testInBackup = FilteringCase{IsTestInBackup: true} +var testInRestore = FilteringCase{IsTestInBackup: false} + +func (f *FilteringCase) Init() { + UUIDgen, _ = uuid.NewRandom() + f.replica = int32(2) + f.labels = map[string]string{"resourcefiltering": "true"} + f.labelSelector = "resourcefiltering" + f.Client = TestClientInstance + + f.NamespacesTotal = 5 + f.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", f.BackupName, + "--default-volumes-to-restic", "--wait", + } + + f.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", f.RestoreName, + "--from-backup", f.BackupName, "--wait", + } +} + +func (f *FilteringCase) CreateResources() error { + f.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute) + for nsNum := 0; nsNum < f.NamespacesTotal; nsNum++ { + namespace := fmt.Sprintf("%s-%00000d", f.NSBaseName, nsNum) + fmt.Printf("Creating resources in namespace ...%s\n", namespace) + if err := CreateNamespace(f.Ctx, f.Client, namespace); err != nil { + return errors.Wrapf(err, "Failed to create namespace %s", namespace) + } + + //Create deployment + fmt.Printf("Creating deployment in namespaces ...%s\n", namespace) + deployment := NewDeployment(f.NSBaseName, namespace, f.replica, f.labels) + deployment, err := CreateDeployment(f.Client.ClientGo, namespace, deployment) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) + } + err = WaitForReadyDeployment(f.Client.ClientGo, namespace, deployment.Name) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to ensure job completion in namespace: %q", namespace)) + } + //Create Secret + secretName := f.NSBaseName + fmt.Printf("Creating secret %s in namespaces ...%s\n", secretName, namespace) + _, err = CreateSecret(f.Client.ClientGo, namespace, secretName, f.labels) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to create secret in the namespace %q", namespace)) + } + err = WaitForSecretsComplete(f.Client.ClientGo, namespace, secretName) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to ensure secret completion in namespace: %q", namespace)) + } + //Create Configmap + configmaptName := f.NSBaseName + fmt.Printf("Creating configmap %s in namespaces ...%s\n", configmaptName, namespace) + _, err = CreateConfigMap(f.Client.ClientGo, namespace, configmaptName, f.labels) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to create configmap in the namespace %q", namespace)) + } + err = WaitForConfigMapComplete(f.Client.ClientGo, namespace, configmaptName) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to ensure secret completion in namespace: %q", namespace)) + } + } + return nil +} + +func (f *FilteringCase) Verify() error { + for nsNum := 0; nsNum < f.NamespacesTotal; nsNum++ { + namespace := fmt.Sprintf("%s-%00000d", f.NSBaseName, nsNum) + fmt.Printf("Checking resources in namespaces ...%s\n", namespace) + //Check namespace + checkNS, err := GetNamespace(f.Ctx, f.Client, namespace) + if err != nil { + return errors.Wrapf(err, "Could not retrieve test namespace %s", namespace) + } + if checkNS.Name != namespace { + return errors.Errorf("Retrieved namespace for %s has name %s instead", namespace, checkNS.Name) + } + //Check deployment + _, err = GetDeployment(f.Client.ClientGo, namespace, f.NSBaseName) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) + } + + //Check secrets + secretsList, err := f.Client.ClientGo.CoreV1().Secrets(namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: f.labelSelector}) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to list secrets in namespace: %q", namespace)) + } else if len(secretsList.Items) == 0 { + return errors.Wrap(err, fmt.Sprintf("no secrets found in namespace: %q", namespace)) + } + + //Check configmap + configmapList, err := f.Client.ClientGo.CoreV1().ConfigMaps(namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: f.labelSelector}) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to list configmap in namespace: %q", namespace)) + } else if len(configmapList.Items) == 0 { + return errors.Wrap(err, fmt.Sprintf("no configmap found in namespace: %q", namespace)) + } + } + return nil +} diff --git a/test/e2e/resource-filtering/exclude_label.go b/test/e2e/resource-filtering/exclude_label.go new file mode 100644 index 000000000..919be15f5 --- /dev/null +++ b/test/e2e/resource-filtering/exclude_label.go @@ -0,0 +1,125 @@ +/* +Copyright 2021 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filtering + +import ( + "context" + "fmt" + "math/rand" + "time" + + "github.com/google/uuid" + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" +) + +/* +Resources with the label velero.io/exclude-from-backup=true are not included +in backup, even if it contains a matching selector label. +*/ + +type ExcludeFromBackup struct { + FilteringCase +} + +var ExcludeFromBackupTest func() = TestFunc(&ExcludeFromBackup{testInBackup}) + +func (e *ExcludeFromBackup) Init() { + rand.Seed(time.Now().UnixNano()) + UUIDgen, _ = uuid.NewRandom() + e.FilteringCase.Init() + e.BackupName = "backup-exclude-from-backup-" + UUIDgen.String() + e.RestoreName = "restore-exclude-from-backup-" + UUIDgen.String() + e.NSBaseName = "exclude-from-backup-" + UUIDgen.String() + e.TestMsg = &TestMSG{ + Desc: "Backup with the label velero.io/exclude-from-backup=true are not included test", + Text: "Should not backup resources with the label velero.io/exclude-from-backup=true", + FailedMSG: "Failed to backup resources with the label velero.io/exclude-from-backup=true", + } + e.labels = map[string]string{ + "velero.io/exclude-from-backup": "true", + } + e.labelSelector = "velero.io/exclude-from-backup" + e.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", e.BackupName, + "--default-volumes-to-restic", "--wait", + } + + e.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", e.RestoreName, + "--from-backup", e.BackupName, "--wait", + } +} + +func (e *ExcludeFromBackup) CreateResources() error { + e.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute) + for nsNum := 0; nsNum < e.NamespacesTotal; nsNum++ { + namespace := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) + fmt.Printf("Creating resources in namespace ...%s\n", namespace) + labels := e.labels + if nsNum%2 == 0 { + labels = map[string]string{ + "velero.io/exclude-from-backup": "false", + } + } + if err := CreateNamespaceWithLabel(e.Ctx, e.Client, namespace, labels); err != nil { + return errors.Wrapf(err, "Failed to create namespace %s", namespace) + } + + //Create deployment + fmt.Printf("Creating deployment in namespaces ...%s\n", namespace) + + deployment := NewDeployment(e.NSBaseName, namespace, e.replica, labels) + deployment, err := CreateDeployment(e.Client.ClientGo, namespace, deployment) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) + } + err = WaitForReadyDeployment(e.Client.ClientGo, namespace, deployment.Name) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to ensure deployment completion in namespace: %q", namespace)) + } + } + return nil +} + +func (e *ExcludeFromBackup) Verify() error { + for nsNum := 0; nsNum < e.NamespacesTotal; nsNum++ { + namespace := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) + fmt.Printf("Checking resources in namespaces ...%s\n", namespace) + //Check deployment + _, err := GetDeployment(e.Client.ClientGo, namespace, e.NSBaseName) + if nsNum%2 == 0 { //include + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) + } + } else { //exclude + if err == nil { + return fmt.Errorf("failed to exclude deployment in namespaces %q", namespace) + } else { + if apierrors.IsNotFound(err) { //resource should be excluded + return nil + } + return errors.Wrap(err, fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) + } + } + } + return nil +} diff --git a/test/e2e/resource-filtering/exclude_namespaces.go b/test/e2e/resource-filtering/exclude_namespaces.go new file mode 100644 index 000000000..88ccb446e --- /dev/null +++ b/test/e2e/resource-filtering/exclude_namespaces.go @@ -0,0 +1,144 @@ +/* +Copyright 2021 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filtering + +import ( + "context" + "fmt" + "math/rand" + "strings" + "time" + + "github.com/google/uuid" + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" +) + +/* +exclude-namespaces +Exclude namespace1 from the cluster backup. +velero backup create --exclude-namespaces + +Exclude two namespaces during a restore. +velero restore create --exclude-namespaces , +*/ + +type ExcludeNamespaces struct { + FilteringCase + nsExcluded *[]string + namespacesExcluded int +} + +var BackupWithExcludeNamespaces func() = TestFunc(&ExcludeNamespaces{FilteringCase: testInBackup}) +var RestoreWithExcludeNamespaces func() = TestFunc(&ExcludeNamespaces{FilteringCase: testInRestore}) + +func (e *ExcludeNamespaces) Init() { + rand.Seed(time.Now().UnixNano()) + UUIDgen, _ = uuid.NewRandom() + e.FilteringCase.Init() + e.BackupName = "backup-exclude-namespaces-" + UUIDgen.String() + e.RestoreName = "restore-exclude-namespaces-" + UUIDgen.String() + e.namespacesExcluded = e.NamespacesTotal / 2 + e.NSBaseName = "exclude-namespaces-" + UUIDgen.String() + if e.IsTestInBackup { + e.TestMsg = &TestMSG{ + Desc: "Backup resources with exclude namespace test", + FailedMSG: "Failed to backup and restore with namespace include", + Text: fmt.Sprintf("should not backup %d namespaces of %d", e.namespacesExcluded, e.NamespacesTotal), + } + } else { + e.TestMsg = &TestMSG{ + Desc: "Restore resources with exclude namespace test", + FailedMSG: "Failed to restore with namespace exclude", + Text: fmt.Sprintf("should not backup %d namespaces of %d", e.namespacesExcluded, e.NamespacesTotal), + } + } + e.nsExcluded = &[]string{} + for nsNum := 0; nsNum < e.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) + if nsNum < e.namespacesExcluded { + *e.nsExcluded = append(*e.nsExcluded, createNSName) + } + } + if e.IsTestInBackup { + e.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", e.BackupName, + "--exclude-namespaces", strings.Join(*e.nsExcluded, ","), + "--default-volumes-to-restic", "--wait", + } + + e.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", e.RestoreName, + "--from-backup", e.BackupName, "--wait", + } + + } else { + e.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", e.BackupName, + "--default-volumes-to-restic", "--wait", + } + + e.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", e.RestoreName, + "--exclude-namespaces", strings.Join(*e.nsExcluded, ","), + "--from-backup", e.BackupName, "--wait", + } + } +} + +func (e *ExcludeNamespaces) CreateResources() error { + e.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute) + for nsNum := 0; nsNum < e.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) + fmt.Printf("Creating namespaces ...%s\n", createNSName) + if err := CreateNamespace(e.Ctx, e.Client, createNSName); err != nil { + return errors.Wrapf(err, "Failed to create namespace %s", createNSName) + } + } + return nil +} + +func (e *ExcludeNamespaces) Verify() error { + // Verify that we got back all of the namespaces we created + for nsNum := 0; nsNum < e.namespacesExcluded; nsNum++ { + excludeNSName := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) + _, err := GetNamespace(e.Ctx, e.Client, excludeNSName) + if err == nil { + return errors.Wrapf(err, "Resource filtering with exclude namespace but exclude namespace %s exist", excludeNSName) + } + + if !apierrors.IsNotFound(err) { + return errors.Wrapf(err, "Resource filtering with exclude namespace failed with checking namespace %s", excludeNSName) + } + } + + for nsNum := e.namespacesExcluded; nsNum < e.NamespacesTotal; nsNum++ { + checkNSName := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) + checkNS, err := GetNamespace(e.Ctx, e.Client, checkNSName) + if err != nil { + return errors.Wrapf(err, "Could not retrieve test namespace %s", checkNSName) + } + if checkNS.Name != checkNSName { + return errors.Errorf("Retrieved namespace for %s has name %s instead", checkNSName, checkNS.Name) + } + } + return nil +} diff --git a/test/e2e/resource-filtering/exclude_resources.go b/test/e2e/resource-filtering/exclude_resources.go new file mode 100644 index 000000000..81fd9a133 --- /dev/null +++ b/test/e2e/resource-filtering/exclude_resources.go @@ -0,0 +1,124 @@ +/* +Copyright 2021 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filtering + +import ( + "context" + "fmt" + "math/rand" + "time" + + "github.com/google/uuid" + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" +) + +/* +exclude-resources +Exclude secrets from the backup. + +velero backup create --exclude-resources secrets +Exclude secrets and rolebindings. + +velero backup create --exclude-resources secrets +*/ + +type ExcludeResources struct { + FilteringCase +} + +var BackupWithExcludeResources func() = TestFunc(&ExcludeResources{testInBackup}) +var RestoreWithExcludeResources func() = TestFunc(&ExcludeResources{testInRestore}) + +func (e *ExcludeResources) Init() { + rand.Seed(time.Now().UnixNano()) + UUIDgen, _ = uuid.NewRandom() + e.FilteringCase.Init() + e.NSBaseName = "exclude-resources-" + UUIDgen.String() + if e.IsTestInBackup { // testing case backup with exclude-resources option + e.TestMsg = &TestMSG{ + Desc: "Backup resources with resources included test", + Text: "Should not backup resources which is excluded others should be backup", + FailedMSG: "Failed to backup with resource exclude", + } + e.BackupName = "backup-exclude-resources-" + UUIDgen.String() + e.RestoreName = "restore-" + UUIDgen.String() + e.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", e.BackupName, + "--exclude-resources", "secrets", + "--default-volumes-to-restic", "--wait", + } + + e.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", e.RestoreName, + "--from-backup", e.BackupName, "--wait", + } + } else { // testing case restore with exclude-resources option + e.TestMsg = &TestMSG{ + Desc: "Restore resources with resources included test", + Text: "Should not restore resources which is excluded others should be backup", + FailedMSG: "Failed to restore with resource exclude", + } + e.BackupName = "backup-" + UUIDgen.String() + e.RestoreName = "restore-exclude-resources-" + UUIDgen.String() + e.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", e.BackupName, + "--default-volumes-to-restic", "--wait", + } + e.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", e.RestoreName, + "--exclude-resources", "secrets", + "--from-backup", e.BackupName, "--wait", + } + } +} + +func (e *ExcludeResources) Verify() error { + for nsNum := 0; nsNum < e.NamespacesTotal; nsNum++ { + namespace := fmt.Sprintf("%s-%00000d", e.NSBaseName, nsNum) + fmt.Printf("Checking resources in namespaces ...%s\n", namespace) + //Check deployment + _, err := GetDeployment(e.Client.ClientGo, namespace, e.NSBaseName) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) + } + //Check secrets + secretsList, err := e.Client.ClientGo.CoreV1().Secrets(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: e.labelSelector}) + if err != nil { + if apierrors.IsNotFound(err) { //resource should be excluded + return nil + } + return errors.Wrap(err, fmt.Sprintf("failed to list secrets in namespace: %q", namespace)) + } else if len(secretsList.Items) != 0 { + return errors.Errorf(fmt.Sprintf("Should no secrets found %s in namespace: %q", secretsList.Items[0].Name, namespace)) + } + + //Check configmap + configmapList, err := e.Client.ClientGo.CoreV1().ConfigMaps(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: e.labelSelector}) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to list configmap in namespace: %q", namespace)) + } else if len(configmapList.Items) == 0 { + return errors.Errorf(fmt.Sprintf("Should have configmap found in namespace: %q", namespace)) + } + } + return nil +} diff --git a/test/e2e/resource-filtering/include_namespaces.go b/test/e2e/resource-filtering/include_namespaces.go new file mode 100644 index 000000000..7476beb0c --- /dev/null +++ b/test/e2e/resource-filtering/include_namespaces.go @@ -0,0 +1,148 @@ +/* +Copyright 2021 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filtering + +import ( + "context" + "fmt" + "math/rand" + "strings" + "time" + + "github.com/google/uuid" + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" +) + +/* +include-namespaces +Backup a namespace and it's objects. + +velero backup create --include-namespaces +Restore two namespaces and their objects. + +velero restore create --include-namespaces , +*/ + +type IncludeNamespaces struct { + nsIncluded *[]string + namespacesIncluded int + FilteringCase +} + +var BackupWithIncludeNamespaces func() = TestFunc(&IncludeNamespaces{FilteringCase: testInBackup}) +var RestoreWithIncludeNamespaces func() = TestFunc(&IncludeNamespaces{FilteringCase: testInRestore}) + +func (i *IncludeNamespaces) Init() { + rand.Seed(time.Now().UnixNano()) + UUIDgen, _ = uuid.NewRandom() + i.FilteringCase.Init() + i.namespacesIncluded = i.NamespacesTotal / 2 + i.nsIncluded = &[]string{} + i.NSBaseName = "include-namespaces-" + UUIDgen.String() + for nsNum := 0; nsNum < i.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", i.NSBaseName, nsNum) + if nsNum < i.namespacesIncluded { + *i.nsIncluded = append(*i.nsIncluded, createNSName) + } + } + + if i.IsTestInBackup { + i.BackupName = "backup-include-namespaces-" + UUIDgen.String() + i.RestoreName = "restore-" + UUIDgen.String() + i.TestMsg = &TestMSG{ + Desc: "Backup resources with include namespace test", + FailedMSG: "Failed to backup with namespace include", + Text: fmt.Sprintf("should backup %d namespaces of %d", i.namespacesIncluded, i.NamespacesTotal), + } + i.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", i.BackupName, + "--include-namespaces", strings.Join(*i.nsIncluded, ","), + "--default-volumes-to-restic", "--wait", + } + + i.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", i.RestoreName, + "--from-backup", i.BackupName, "--wait", + } + + } else { + i.BackupName = "backup-" + UUIDgen.String() + i.RestoreName = "restore-include-namespaces-" + UUIDgen.String() + i.TestMsg = &TestMSG{ + Desc: "Restore resources with include namespace test", + FailedMSG: "Failed to restore with namespace include", + Text: fmt.Sprintf("should restore %d namespaces of %d", i.namespacesIncluded, i.NamespacesTotal), + } + i.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", i.BackupName, + "--default-volumes-to-restic", "--wait", + } + + i.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", i.RestoreName, + "--include-namespaces", strings.Join(*i.nsIncluded, ","), + "--from-backup", i.BackupName, "--wait", + } + } +} + +func (i *IncludeNamespaces) CreateResources() error { + i.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute) + for nsNum := 0; nsNum < i.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", i.NSBaseName, nsNum) + fmt.Printf("Creating namespaces ...%s\n", createNSName) + if err := CreateNamespace(i.Ctx, i.Client, createNSName); err != nil { + return errors.Wrapf(err, "Failed to create namespace %s", createNSName) + } + if nsNum <= i.namespacesIncluded { + *i.nsIncluded = append(*i.nsIncluded, createNSName) + } + } + return nil +} + +func (i *IncludeNamespaces) Verify() error { + // Verify that we got back all of the namespaces we created + for nsNum := 0; nsNum < i.namespacesIncluded; nsNum++ { + checkNSName := fmt.Sprintf("%s-%00000d", i.NSBaseName, nsNum) + checkNS, err := GetNamespace(i.Ctx, i.Client, checkNSName) + if err != nil { + return errors.Wrapf(err, "Could not retrieve test namespace %s", checkNSName) + } + if checkNS.Name != checkNSName { + return errors.Errorf("Retrieved namespace for %s has name %s instead", checkNSName, checkNS.Name) + } + } + + for nsNum := i.namespacesIncluded; nsNum < i.NamespacesTotal; nsNum++ { + excludeNSName := fmt.Sprintf("%s-%00000d", i.NSBaseName, nsNum) + _, err := GetNamespace(i.Ctx, i.Client, excludeNSName) + if err == nil { + return errors.Wrapf(err, "Resource filtering with include namespace but exclude namespace %s exist", excludeNSName) + } + + if !apierrors.IsNotFound(err) { + return errors.Wrapf(err, "Resource filtering with include namespace failed with checking namespace %s", excludeNSName) + } + } + return nil +} diff --git a/test/e2e/resource-filtering/include_resources.go b/test/e2e/resource-filtering/include_resources.go new file mode 100644 index 000000000..a56e94091 --- /dev/null +++ b/test/e2e/resource-filtering/include_resources.go @@ -0,0 +1,122 @@ +/* +Copyright 2021 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filtering + +import ( + "context" + "fmt" + "math/rand" + "time" + + "github.com/google/uuid" + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" +) + +/* +Backup all deployments in the cluster. + velero backup create --include-resources deployments, configmaps + +Restore all deployments and configmaps in the cluster. + velero restore create --include-resources deployments,configmaps +*/ + +type IncludeResources struct { + FilteringCase +} + +var BackupWithIncludeResources func() = TestFunc(&IncludeResources{testInBackup}) +var RestoreWithIncludeResources func() = TestFunc(&IncludeResources{testInRestore}) + +func (i *IncludeResources) Init() { + rand.Seed(time.Now().UnixNano()) + UUIDgen, _ = uuid.NewRandom() + i.FilteringCase.Init() + i.NSBaseName = "include-resources-" + UUIDgen.String() + if i.IsTestInBackup { // testing case backup with include-resources option + i.TestMsg = &TestMSG{ + Desc: "Backup resources with resources included test", + Text: "Should backup resources which is included others should not be backup", + FailedMSG: "Failed to backup with resource include", + } + i.BackupName = "backup-include-resources-" + UUIDgen.String() + i.RestoreName = "restore-" + UUIDgen.String() + i.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", i.BackupName, + "--include-resources", "deployments,configmaps", + "--default-volumes-to-restic", "--wait", + } + + i.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", i.RestoreName, + "--from-backup", i.BackupName, "--wait", + } + } else { // testing case restore with include-resources option + i.TestMsg = &TestMSG{ + Desc: "Restore resources with resources included test", + Text: "Should restore resources which is included others should not be backup", + FailedMSG: "Failed to restore with resource include", + } + i.BackupName = "backup-" + UUIDgen.String() + i.RestoreName = "restore-include-resources-" + UUIDgen.String() + i.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", i.BackupName, + "--default-volumes-to-restic", "--wait", + } + i.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", i.RestoreName, + "--include-resources", "deployments,configmaps", + "--from-backup", i.BackupName, "--wait", + } + } +} + +func (i *IncludeResources) Verify() error { + for nsNum := 0; nsNum < i.NamespacesTotal; nsNum++ { + namespace := fmt.Sprintf("%s-%00000d", i.NSBaseName, nsNum) + fmt.Printf("Checking resources in namespaces ...%s\n", namespace) + //Check deployment + _, err := GetDeployment(i.Client.ClientGo, namespace, i.NSBaseName) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) + } + //Check secrets + secretsList, err := i.Client.ClientGo.CoreV1().Secrets(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: i.labelSelector}) + if err != nil { + if apierrors.IsNotFound(err) { //resource should be excluded + return nil + } + return errors.Wrap(err, fmt.Sprintf("failed to list secrets in namespace: %q", namespace)) + } else if len(secretsList.Items) != 0 { + return errors.Errorf(fmt.Sprintf("Should no secrets found %s in namespace: %q", secretsList.Items[0].Name, namespace)) + } + + //Check configmap + configmapList, err := i.Client.ClientGo.CoreV1().ConfigMaps(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: i.labelSelector}) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to list configmap in namespace: %q", namespace)) + } else if len(configmapList.Items) == 0 { + return errors.Errorf(fmt.Sprintf("Should have configmap found in namespace: %q", namespace)) + } + } + return nil +} diff --git a/test/e2e/resource-filtering/label_selector.go b/test/e2e/resource-filtering/label_selector.go new file mode 100644 index 000000000..57d663964 --- /dev/null +++ b/test/e2e/resource-filtering/label_selector.go @@ -0,0 +1,160 @@ +/* +Copyright 2021 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package filtering + +import ( + "context" + "fmt" + "math/rand" + "time" + + "github.com/google/uuid" + "github.com/pkg/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" +) + +/* +Include resources matching the label selector. + velero backup create --selector = +*/ + +type LabelSelector struct { + FilteringCase +} + +var BackupWithLabelSelector func() = TestFunc(&LabelSelector{testInBackup}) + +func (l *LabelSelector) Init() { + rand.Seed(time.Now().UnixNano()) + UUIDgen, _ = uuid.NewRandom() + l.FilteringCase.Init() + l.BackupName = "backup-label-selector-" + UUIDgen.String() + l.RestoreName = "restore-label-selector-" + UUIDgen.String() + l.NSBaseName = "backup-label-selector-" + UUIDgen.String() + l.TestMsg = &TestMSG{ + Desc: "Backup with the label selector test", + Text: "Should backup resources with selected label resource", + FailedMSG: "Failed to backup resources with selected label", + } + l.labels = map[string]string{ + "resourcefiltering": "true", + } + l.labelSelector = "resourcefiltering" + l.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", l.BackupName, + "--selector", "resourcefiltering=true", + "--default-volumes-to-restic", "--wait", + } + + l.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", l.RestoreName, + "--from-backup", l.BackupName, "--wait", + } +} + +func (l *LabelSelector) CreateResources() error { + l.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute) + for nsNum := 0; nsNum < l.NamespacesTotal; nsNum++ { + namespace := fmt.Sprintf("%s-%00000d", l.NSBaseName, nsNum) + fmt.Printf("Creating resources in namespace ...%s\n", namespace) + labels := l.labels + if nsNum%2 == 0 { + labels = map[string]string{ + "resourcefiltering": "false", + } + } + if err := CreateNamespaceWithLabel(l.Ctx, l.Client, namespace, labels); err != nil { + return errors.Wrapf(err, "Failed to create namespace %s", namespace) + } + + //Create deployment + fmt.Printf("Creating deployment in namespaces ...%s\n", namespace) + + deployment := NewDeployment(l.NSBaseName, namespace, l.replica, labels) + deployment, err := CreateDeployment(l.Client.ClientGo, namespace, deployment) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) + } + err = WaitForReadyDeployment(l.Client.ClientGo, namespace, deployment.Name) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to ensure job completion in namespace: %q", namespace)) + } + //Create Secret + secretName := l.NSBaseName + fmt.Printf("Creating secret %s in namespaces ...%s\n", secretName, namespace) + _, err = CreateSecret(l.Client.ClientGo, namespace, secretName, l.labels) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to create secret in the namespace %q", namespace)) + } + err = WaitForSecretsComplete(l.Client.ClientGo, namespace, secretName) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to ensure secret completion in namespace: %q", namespace)) + } + } + return nil +} + +func (l *LabelSelector) Verify() error { + for nsNum := 0; nsNum < l.NamespacesTotal; nsNum++ { + namespace := fmt.Sprintf("%s-%00000d", l.NSBaseName, nsNum) + fmt.Printf("Checking resources in namespaces ...%s\n", namespace) + //Check deployment + _, err := GetDeployment(l.Client.ClientGo, namespace, l.NSBaseName) + if nsNum%2 == 1 { //include + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) + } + } else { //exclude + if err == nil { + return fmt.Errorf("failed to exclude deployment in namespaces %q", namespace) + } else { + if apierrors.IsNotFound(err) { //resource should be excluded + return nil + } + return errors.Wrap(err, fmt.Sprintf("failed to list deployment in namespace: %q", namespace)) + } + } + + //Check secrets + secretsList, err := l.Client.ClientGo.CoreV1().Secrets(namespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: l.labelSelector, + }) + + if nsNum%2 == 0 { //include + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to list secrets in namespace: %q", namespace)) + } else if len(secretsList.Items) == 0 { + return errors.Errorf(fmt.Sprintf("no secrets found in namespace: %q", namespace)) + } + } else { //exclude + if err == nil { + return fmt.Errorf("failed to exclude secrets in namespaces %q", namespace) + } else { + if apierrors.IsNotFound(err) { //resource should be excluded + return nil + } + return errors.Wrap(err, fmt.Sprintf("failed to list secrets in namespace: %q", namespace)) + } + } + } + return nil +} diff --git a/test/e2e/scale/multiple_namespaces.go b/test/e2e/scale/multiple_namespaces.go index dd7832d3b..91e8b5636 100644 --- a/test/e2e/scale/multiple_namespaces.go +++ b/test/e2e/scale/multiple_namespaces.go @@ -24,8 +24,8 @@ import ( "github.com/google/uuid" - k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s" - veleroutils "github.com/vmware-tanzu/velero/test/e2e/util/velero" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + . "github.com/vmware-tanzu/velero/test/e2e/util/velero" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -37,7 +37,7 @@ import ( func BasicBackupRestore() { - client, err := k8sutils.NewTestClient() + client, err := NewTestClient() Expect(err).To(Succeed(), "Failed to instantiate cluster client for multiple namespace tests") BeforeEach(func() { @@ -46,13 +46,13 @@ func BasicBackupRestore() { UUIDgen, err = uuid.NewRandom() Expect(err).To(Succeed()) if VeleroCfg.InstallVelero { - Expect(veleroutils.VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed()) } }) AfterEach(func() { if VeleroCfg.InstallVelero { - err := veleroutils.VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) + err := VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) Expect(err).To(Succeed()) } @@ -71,7 +71,7 @@ func BasicBackupRestore() { func MultiNSBackupRestore() { - client, err := k8sutils.NewTestClient() + client, err := NewTestClient() Expect(err).To(Succeed(), "Failed to instantiate cluster client for multiple namespace tests") BeforeEach(func() { @@ -80,13 +80,13 @@ func MultiNSBackupRestore() { UUIDgen, err = uuid.NewRandom() Expect(err).To(Succeed()) if VeleroCfg.InstallVelero { - Expect(veleroutils.VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed()) } }) AfterEach(func() { if VeleroCfg.InstallVelero { - err := veleroutils.VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) + err := VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) Expect(err).To(Succeed()) } @@ -103,8 +103,8 @@ func MultiNSBackupRestore() { }) } -func RunMultipleNamespaceTest(ctx context.Context, client k8sutils.TestClient, nsBaseName string, numberOfNamespaces int, backupName string, restoreName string) error { - defer k8sutils.CleanupNamespaces(context.Background(), client, nsBaseName) // Run at exit for final cleanup +func RunMultipleNamespaceTest(ctx context.Context, client TestClient, nsBaseName string, numberOfNamespaces int, backupName string, restoreName string) error { + defer CleanupNamespaces(context.Background(), client, nsBaseName) // Run at exit for final cleanup var excludeNamespaces []string // Currently it's hard to build a large list of namespaces to include and wildcards do not work so instead @@ -121,30 +121,30 @@ func RunMultipleNamespaceTest(ctx context.Context, client k8sutils.TestClient, n fmt.Printf("Creating namespaces ...\n") for nsNum := 0; nsNum < numberOfNamespaces; nsNum++ { createNSName := fmt.Sprintf("%s-%00000d", nsBaseName, nsNum) - if err := k8sutils.CreateNamespace(ctx, client, createNSName); err != nil { + if err := CreateNamespace(ctx, client, createNSName); err != nil { return errors.Wrapf(err, "Failed to create namespace %s", createNSName) } } - if err := veleroutils.VeleroBackupExcludeNamespaces(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, excludeNamespaces); err != nil { - veleroutils.RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, "") + if err := VeleroBackupExcludeNamespaces(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, excludeNamespaces); err != nil { + RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, "") return errors.Wrapf(err, "Failed to backup backup namespaces %s-*", nsBaseName) } - err = k8sutils.CleanupNamespaces(ctx, client, nsBaseName) + err = CleanupNamespaces(ctx, client, nsBaseName) if err != nil { return errors.Wrap(err, "Could cleanup retrieve namespaces") } - err = veleroutils.VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restoreName, backupName) + err = VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restoreName, backupName) if err != nil { - veleroutils.RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", restoreName) + RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", restoreName) return errors.Wrap(err, "Restore failed") } // Verify that we got back all of the namespaces we created for nsNum := 0; nsNum < numberOfNamespaces; nsNum++ { checkNSName := fmt.Sprintf("%s-%00000d", nsBaseName, nsNum) - checkNS, err := k8sutils.GetNamespace(ctx, client, checkNSName) + checkNS, err := GetNamespace(ctx, client, checkNSName) if err != nil { return errors.Wrapf(err, "Could not retrieve test namespace %s", checkNSName) } diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go new file mode 100644 index 000000000..4b61f6cd3 --- /dev/null +++ b/test/e2e/test/test.go @@ -0,0 +1,164 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "context" + "flag" + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/pkg/errors" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + . "github.com/vmware-tanzu/velero/test/e2e/util/velero" +) + +type VeleroTest interface { + Init() + CreateResources() error + Backup() error + Destroy() error + Restore() error + Verify() error + Clean() error + GetTestMsg() *TestMSG +} + +type TestMSG struct { + Desc string + Text string + FailedMSG string +} + +type TestCase struct { + BackupName string + RestoreName string + NSBaseName string + BackupArgs []string + RestoreArgs []string + NamespacesTotal int + TestMsg *TestMSG + Client TestClient + Ctx context.Context +} + +var TestClientInstance TestClient + +func TestFunc(test VeleroTest) func() { + return func() { + var err error + TestClientInstance, err = NewTestClient() + Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") + test.Init() + BeforeEach(func() { + flag.Parse() + if VeleroCfg.InstallVelero { + Expect(VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed()) + } + + }) + + AfterEach(func() { + if VeleroCfg.InstallVelero { + err := VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) + Expect(err).To(Succeed()) + } + }) + + It(test.GetTestMsg().Text, func() { + Expect(RunTestCase(test)).To(Succeed(), test.GetTestMsg().FailedMSG) + }) + } +} + +func (t *TestCase) Init() { +} + +func (t *TestCase) CreateResources() error { + return nil +} + +func (t *TestCase) Backup() error { + if err := VeleroCmdExec(t.Ctx, VeleroCfg.VeleroCLI, t.BackupArgs); err != nil { + RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.BackupName, "") + return errors.Wrapf(err, "Failed to backup resources") + } + return nil +} + +func (t *TestCase) Destroy() error { + err := CleanupNamespacesWithPoll(t.Ctx, t.Client, t.NSBaseName) + if err != nil { + return errors.Wrap(err, "Could cleanup retrieve namespaces") + } + return nil +} + +func (t *TestCase) Restore() error { + if err := VeleroCmdExec(t.Ctx, VeleroCfg.VeleroCLI, t.RestoreArgs); err != nil { + RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.BackupName, "") + return errors.Wrapf(err, "Failed to restore resources") + } + return nil +} + +func (t *TestCase) Verify() error { + return nil +} + +func (t *TestCase) Clean() error { + return CleanupNamespaces(t.Ctx, t.Client, t.NSBaseName) +} + +func (t *TestCase) GetTestMsg() *TestMSG { + return t.TestMsg +} + +func RunTestCase(test VeleroTest) error { + fmt.Printf("Running test case %s\n", test.GetTestMsg().Desc) + if test == nil { + return errors.New("No case should be tested") + } + + defer test.Clean() + + err := test.CreateResources() + if err != nil { + return err + } + err = test.Backup() + if err != nil { + return err + } + err = test.Destroy() + if err != nil { + return err + } + err = test.Restore() + if err != nil { + return err + } + err = test.Verify() + if err != nil { + return err + } + return nil +} diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index e5825b79d..a5306f278 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -27,9 +27,9 @@ import ( "github.com/pkg/errors" . "github.com/vmware-tanzu/velero/test/e2e" - k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s" - kibishiiutils "github.com/vmware-tanzu/velero/test/e2e/util/kibishii" - veleroutils "github.com/vmware-tanzu/velero/test/e2e/util/velero" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + . "github.com/vmware-tanzu/velero/test/e2e/util/kibishii" + . "github.com/vmware-tanzu/velero/test/e2e/util/velero" ) const ( @@ -49,7 +49,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool) { backupName, restoreName, upgradeFromVeleroCLI string ) - client, err := k8sutils.NewTestClient() + client, err := NewTestClient() Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") BeforeEach(func() { @@ -62,7 +62,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool) { //Assume tag of velero server image is identical to velero CLI version //Download velero CLI if it's empty according to velero CLI version if (len(VeleroCfg.UpgradeFromVeleroCLI)) == 0 { - upgradeFromVeleroCLI, err = veleroutils.InstallVeleroCLI(VeleroCfg.UpgradeFromVeleroVersion) + upgradeFromVeleroCLI, err = InstallVeleroCLI(VeleroCfg.UpgradeFromVeleroVersion) Expect(err).To(Succeed()) } @@ -79,8 +79,8 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool) { tmpCfg.ResticHelperImage = "" tmpCfg.Plugins = "" tmpCfg.CRDsVersion = "" - Expect(veleroutils.VeleroInstall(context.Background(), &tmpCfg, "", useVolumeSnapshots)).To(Succeed()) - Expect(veleroutils.CheckVeleroVersion(context.Background(), upgradeFromVeleroCLI, tmpCfg.UpgradeFromVeleroVersion)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &tmpCfg, "", useVolumeSnapshots)).To(Succeed()) + Expect(CheckVeleroVersion(context.Background(), upgradeFromVeleroCLI, tmpCfg.UpgradeFromVeleroVersion)).To(Succeed()) } else { Skip("Upgrade test is skipped since user don't want to install any other velero") } @@ -88,7 +88,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool) { AfterEach(func() { if VeleroCfg.InstallVelero { - err = veleroutils.VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) + err = VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) Expect(err).To(Succeed()) } }) @@ -104,28 +104,28 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool) { } // runUpgradeTests runs upgrade test on the provider by kibishii. -func runUpgradeTests(client k8sutils.TestClient, veleroCfg *VerleroConfig, upgradeFromVeleroCLI, backupName, restoreName, backupLocation string, +func runUpgradeTests(client TestClient, veleroCfg *VerleroConfig, upgradeFromVeleroCLI, backupName, restoreName, backupLocation string, useVolumeSnapshots bool) error { if veleroCfg.VeleroCLI == "" { return errors.New("empty") } oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) - if err := k8sutils.CreateNamespace(oneHourTimeout, client, upgradeNamespace); err != nil { + if err := CreateNamespace(oneHourTimeout, client, upgradeNamespace); err != nil { return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", upgradeNamespace) } defer func() { - if err := k8sutils.DeleteNamespace(context.Background(), client, upgradeNamespace, true); err != nil { + if err := DeleteNamespace(context.Background(), client, upgradeNamespace, true); err != nil { fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", upgradeNamespace)) } }() - if err := kibishiiutils.KibishiiPrepareBeforeBackup(oneHourTimeout, client, veleroCfg.CloudProvider, upgradeNamespace, veleroCfg.RegistryCredentialFile); err != nil { + if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, veleroCfg.CloudProvider, upgradeNamespace, veleroCfg.RegistryCredentialFile); err != nil { return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", upgradeNamespace) } - if err := veleroutils.VeleroBackupNamespace(oneHourTimeout, upgradeFromVeleroCLI, veleroCfg.VeleroNamespace, backupName, upgradeNamespace, backupLocation, useVolumeSnapshots); err != nil { + if err := VeleroBackupNamespace(oneHourTimeout, upgradeFromVeleroCLI, veleroCfg.VeleroNamespace, backupName, upgradeNamespace, backupLocation, useVolumeSnapshots); err != nil { // TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command - // TODO move to "veleroutils.RunDebug" after we bump up to 1.7 in the upgrade case - veleroutils.VeleroBackupLogs(context.Background(), upgradeFromVeleroCLI, veleroCfg.VeleroNamespace, backupName) + // TODO move to "RunDebug" after we bump up to 1.7 in the upgrade case + VeleroBackupLogs(context.Background(), upgradeFromVeleroCLI, veleroCfg.VeleroNamespace, backupName) return errors.Wrapf(err, "Failed to backup kibishii namespace %s", upgradeNamespace) } @@ -133,12 +133,12 @@ func runUpgradeTests(client k8sutils.TestClient, veleroCfg *VerleroConfig, upgra // Wait for uploads started by the Velero Plug-in for vSphere to complete // TODO - remove after upload progress monitoring is implemented fmt.Println("Waiting for vSphere uploads to complete") - if err := veleroutils.WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, upgradeNamespace); err != nil { + if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, upgradeNamespace); err != nil { return errors.Wrapf(err, "Error waiting for uploads to complete") } } fmt.Printf("Simulating a disaster by removing namespace %s\n", upgradeNamespace) - if err := k8sutils.DeleteNamespace(oneHourTimeout, client, upgradeNamespace, true); err != nil { + if err := DeleteNamespace(oneHourTimeout, client, upgradeNamespace, true); err != nil { return errors.Wrapf(err, "failed to delete namespace %s", upgradeNamespace) } @@ -150,18 +150,18 @@ func runUpgradeTests(client k8sutils.TestClient, veleroCfg *VerleroConfig, upgra time.Sleep(5 * time.Minute) } - if err := veleroutils.VeleroInstall(context.Background(), veleroCfg, "", useVolumeSnapshots); err != nil { + if err := VeleroInstall(context.Background(), veleroCfg, "", useVolumeSnapshots); err != nil { return errors.Wrapf(err, "Failed to install velero from image %s", veleroCfg.VeleroImage) } - if err := veleroutils.CheckVeleroVersion(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroVersion); err != nil { + if err := CheckVeleroVersion(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroVersion); err != nil { return errors.Wrapf(err, "Velero install version mismatch.") } - if err := veleroutils.VeleroRestore(oneHourTimeout, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, restoreName, backupName); err != nil { - veleroutils.RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, "", restoreName) + if err := VeleroRestore(oneHourTimeout, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, restoreName, backupName); err != nil { + RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, "", restoreName) return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName) } - if err := kibishiiutils.KibishiiVerifyAfterRestore(client, upgradeNamespace, oneHourTimeout); err != nil { + if err := KibishiiVerifyAfterRestore(client, upgradeNamespace, oneHourTimeout); err != nil { return errors.Wrapf(err, "Error verifying kibishii after restore") } diff --git a/test/e2e/util/k8s/client.go b/test/e2e/util/k8s/client.go index 839be2b83..ae7286988 100644 --- a/test/e2e/util/k8s/client.go +++ b/test/e2e/util/k8s/client.go @@ -17,6 +17,8 @@ limitations under the License. package k8s import ( + "sync" + "k8s.io/client-go/kubernetes" kbclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -45,8 +47,21 @@ type TestClient struct { dynamicFactory client.DynamicFactory } -// k8sutils.NewTestClient returns a set of ready-to-use API clients. +var ( + once sync.Once + testClient TestClient + err error +) + func NewTestClient() (TestClient, error) { + once.Do(func() { // <-- atomic, does not allow repeating + testClient, err = InitTestClient() // <-- thread safe + }) + return testClient, err +} + +// NewTestClient returns a set of ready-to-use API clients. +func InitTestClient() (TestClient, error) { config, err := client.LoadConfig() if err != nil { return TestClient{}, err diff --git a/test/e2e/util/k8s/configmap.go b/test/e2e/util/k8s/configmap.go new file mode 100644 index 000000000..73bb1911c --- /dev/null +++ b/test/e2e/util/k8s/configmap.go @@ -0,0 +1,47 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8s + +import ( + "golang.org/x/net/context" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" +) + +func CreateConfigMap(c clientset.Interface, ns, name string, data map[string]string) (*v1.ConfigMap, error) { + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: data, + }, + Data: data, + } + return c.CoreV1().ConfigMaps(ns).Create(context.TODO(), cm, metav1.CreateOptions{}) +} + +// WaitForConfigMapComplete uses c to wait for completions to complete for the Job jobName in namespace ns. +func WaitForConfigMapComplete(c clientset.Interface, ns, configmapName string) error { + return wait.Poll(PollInterval, PollTimeout, func() (bool, error) { + _, err := c.CoreV1().ConfigMaps(ns).Get(context.TODO(), configmapName, metav1.GetOptions{}) + if err != nil { + return false, err + } + return true, nil + }) +} diff --git a/test/e2e/util/k8s/deployment.go b/test/e2e/util/k8s/deployment.go new file mode 100644 index 000000000..6763c3e9b --- /dev/null +++ b/test/e2e/util/k8s/deployment.go @@ -0,0 +1,95 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8s + +import ( + "fmt" + "time" + + "golang.org/x/net/context" + apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" +) + +const ( + JobSelectorKey = "job" + // Poll is how often to Poll pods, nodes and claims. + PollInterval = 2 * time.Second + PollTimeout = 15 * time.Minute +) + +// newDeployment returns a RollingUpdate Deployment with a fake container image +func NewDeployment(name, ns string, replicas int32, labels map[string]string) *apps.Deployment { + return &apps.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: name, + Labels: labels, + }, + Spec: apps.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{MatchLabels: labels}, + Strategy: apps.DeploymentStrategy{ + Type: apps.RollingUpdateDeploymentStrategyType, + RollingUpdate: new(apps.RollingUpdateDeployment), + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: name, + Image: "busybox:latest", + Command: []string{"sleep", "1000000"}, + }, + }, + }, + }, + }, + } +} + +func CreateDeployment(c clientset.Interface, ns string, deployment *apps.Deployment) (*apps.Deployment, error) { + return c.AppsV1().Deployments(ns).Create(context.TODO(), deployment, metav1.CreateOptions{}) +} + +func GetDeployment(c clientset.Interface, ns, name string) (*apps.Deployment, error) { + return c.AppsV1().Deployments(ns).Get(context.TODO(), name, metav1.GetOptions{}) +} + +// WaitForReadyDeployment waits for number of ready replicas to equal number of replicas. +func WaitForReadyDeployment(c clientset.Interface, ns, name string) error { + if err := wait.PollImmediate(PollInterval, PollTimeout, func() (bool, error) { + deployment, err := c.AppsV1().Deployments(ns).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return false, fmt.Errorf("failed to get deployment %q: %v", name, err) + } + return deployment.Status.ReadyReplicas == *deployment.Spec.Replicas, nil + }); err != nil { + return fmt.Errorf("failed to wait for .readyReplicas to equal .replicas: %v", err) + } + return nil +} diff --git a/test/e2e/util/k8s/namespace.go b/test/e2e/util/k8s/namespace.go index 2c33e32d5..b6a83a62b 100644 --- a/test/e2e/util/k8s/namespace.go +++ b/test/e2e/util/k8s/namespace.go @@ -43,6 +43,16 @@ func CreateNamespace(ctx context.Context, client TestClient, namespace string) e return err } +func CreateNamespaceWithLabel(ctx context.Context, client TestClient, namespace string, label map[string]string) error { + ns := builder.ForNamespace(namespace).Result() + ns.Labels = label + _, err := client.ClientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + return nil + } + return err +} + func GetNamespace(ctx context.Context, client TestClient, namespace string) (*corev1api.Namespace, error) { return client.ClientGo.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) } @@ -68,6 +78,22 @@ func DeleteNamespace(ctx context.Context, client TestClient, namespace string, w }) } +func CleanupNamespacesWithPoll(ctx context.Context, client TestClient, nsBaseName string) error { + namespaces, err := client.ClientGo.CoreV1().Namespaces().List(ctx, v1.ListOptions{}) + if err != nil { + return errors.Wrap(err, "Could not retrieve namespaces") + } + for _, checkNamespace := range namespaces.Items { + if strings.HasPrefix(checkNamespace.Name, nsBaseName) { + err := DeleteNamespace(ctx, client, checkNamespace.Name, true) + if err != nil { + return errors.Wrapf(err, "Could not delete namespace %s", checkNamespace.Name) + } + } + } + return nil +} + func CleanupNamespaces(ctx context.Context, client TestClient, nsBaseName string) error { namespaces, err := client.ClientGo.CoreV1().Namespaces().List(ctx, v1.ListOptions{}) if err != nil { diff --git a/test/e2e/util/k8s/secret.go b/test/e2e/util/k8s/secret.go new file mode 100644 index 000000000..9701559eb --- /dev/null +++ b/test/e2e/util/k8s/secret.go @@ -0,0 +1,46 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8s + +import ( + "golang.org/x/net/context" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" +) + +func CreateSecret(c clientset.Interface, ns, name string, labels map[string]string) (*v1.Secret, error) { + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: labels, + }, + } + return c.CoreV1().Secrets(ns).Create(context.TODO(), secret, metav1.CreateOptions{}) +} + +// WaitForSecretsComplete uses c to wait for completions to complete for the Job jobName in namespace ns. +func WaitForSecretsComplete(c clientset.Interface, ns, secretName string) error { + return wait.Poll(PollInterval, PollTimeout, func() (bool, error) { + _, err := c.CoreV1().Secrets(ns).Get(context.TODO(), secretName, metav1.GetOptions{}) + if err != nil { + return false, err + } + return true, nil + }) +} diff --git a/test/e2e/util/kibishii/kibishii_utils.go b/test/e2e/util/kibishii/kibishii_utils.go index 3221f6da8..945006c54 100644 --- a/test/e2e/util/kibishii/kibishii_utils.go +++ b/test/e2e/util/kibishii/kibishii_utils.go @@ -26,8 +26,8 @@ import ( "golang.org/x/net/context" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" - k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s" - veleroutils "github.com/vmware-tanzu/velero/test/e2e/util/velero" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + . "github.com/vmware-tanzu/velero/test/e2e/util/velero" ) const ( @@ -36,14 +36,14 @@ const ( ) // RunKibishiiTests runs kibishii tests on the provider. -func RunKibishiiTests(client k8sutils.TestClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string, +func RunKibishiiTests(client TestClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string, useVolumeSnapshots bool, registryCredentialFile string) error { oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) - if err := k8sutils.CreateNamespace(oneHourTimeout, client, kibishiiNamespace); err != nil { + if err := CreateNamespace(oneHourTimeout, client, kibishiiNamespace); err != nil { return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace) } defer func() { - if err := k8sutils.DeleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil { + if err := DeleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil { fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", kibishiiNamespace)) } }() @@ -51,8 +51,8 @@ func RunKibishiiTests(client k8sutils.TestClient, providerName, veleroCLI, veler return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", kibishiiNamespace) } - if err := veleroutils.VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots); err != nil { - veleroutils.RunDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "") + if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots); err != nil { + RunDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "") return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace) } @@ -60,12 +60,12 @@ func RunKibishiiTests(client k8sutils.TestClient, providerName, veleroCLI, veler // Wait for uploads started by the Velero Plug-in for vSphere to complete // TODO - remove after upload progress monitoring is implemented fmt.Println("Waiting for vSphere uploads to complete") - if err := veleroutils.WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil { + if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil { return errors.Wrapf(err, "Error waiting for uploads to complete") } } fmt.Printf("Simulating a disaster by removing namespace %s\n", kibishiiNamespace) - if err := k8sutils.DeleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil { + if err := DeleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil { return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace) } @@ -77,8 +77,8 @@ func RunKibishiiTests(client k8sutils.TestClient, providerName, veleroCLI, veler time.Sleep(5 * time.Minute) } - if err := veleroutils.VeleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil { - veleroutils.RunDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName) + if err := VeleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil { + RunDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName) return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName) } @@ -145,19 +145,19 @@ func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel return nil } -func waitForKibishiiPods(ctx context.Context, client k8sutils.TestClient, kibishiiNamespace string) error { - return k8sutils.WaitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"}) +func waitForKibishiiPods(ctx context.Context, client TestClient, kibishiiNamespace string) error { + return WaitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"}) } -func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client k8sutils.TestClient, providerName, kibishiiNamespace, registryCredentialFile string) error { +func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClient, providerName, kibishiiNamespace, registryCredentialFile string) error { serviceAccountName := "default" // wait until the service account is created before patch the image pull secret - if err := k8sutils.WaitUntilServiceAccountCreated(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, 10*time.Minute); err != nil { + if err := WaitUntilServiceAccountCreated(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, 10*time.Minute); err != nil { return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, kibishiiNamespace) } // add the image pull secret to avoid the image pull limit issue of Docker Hub - if err := k8sutils.PatchServiceAccountWithImagePullSecret(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, registryCredentialFile); err != nil { + if err := PatchServiceAccountWithImagePullSecret(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, registryCredentialFile); err != nil { return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, kibishiiNamespace) } @@ -178,7 +178,7 @@ func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client k8sutils return nil } -func KibishiiVerifyAfterRestore(client k8sutils.TestClient, kibishiiNamespace string, oneHourTimeout context.Context) error { +func KibishiiVerifyAfterRestore(client TestClient, kibishiiNamespace string, oneHourTimeout context.Context) error { // wait for kibishii pod startup // TODO - Fix kibishii so we can check that it is ready to go fmt.Printf("Waiting for kibishii pods to be ready\n") diff --git a/test/e2e/util/velero/install.go b/test/e2e/util/velero/install.go index 38c58f18a..b843fb0be 100644 --- a/test/e2e/util/velero/install.go +++ b/test/e2e/util/velero/install.go @@ -36,7 +36,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" velerexec "github.com/vmware-tanzu/velero/pkg/util/exec" . "github.com/vmware-tanzu/velero/test/e2e" - k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" ) // we provide more install options other than the standard install.InstallOptions in E2E test @@ -69,7 +69,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, features strin // Snapshot location specified veleroCfg.ObjectStoreProvider = "aws" } - err = k8sutils.EnsureClusterExists(ctx) + err = EnsureClusterExists(ctx) if err != nil { return errors.WithMessage(err, "Failed to ensure Kubernetes cluster exists") }