diff --git a/test/e2e/README.md b/test/e2e/README.md index 223a7d0c3..151f0ebb8 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -111,5 +111,30 @@ The `-focus` flag is passed to ginkgo using the `GINKGO_FOCUS` make variable. Th ### API clients When adding a test, aim to instantiate an API client only once at the beginning of the test. There is a constructor `newTestClient` that facilitates the configuration and instantiation of clients. Also, please use the `kubebuilder` runtime controller client for any new test, as we will phase out usage of `client-go` API clients. +## TestCase frame related +TestCase frame provide a serials of interface to concatenate one complete e2e test. it's makes the testing be concise and explicit. +### VeleroBackupRestoreTest interface +VeleroBackupRestoreTest interface provided a standard workflow of backup and restore, which makes the whole testing process clearer and code reusability. + +For programming conventions, Take ResourcePoliciesCase case for example: +##### Init +- It's need first call the base `TestCase` Init function to generate random number as UUIDgen and set one default timeout duration +- Assigning CaseBaseName variable with a case related prefix, all others variables will follow the prefix +- Assigning NamespacesTotal variable, and generating namespaces +- Assigning values to the inner variable for specific case +- For BackupArgs, as Velero installation is using global Velero configuration, it's NEED to specify the value of the variable snapshot-volumes or default-volumes-to-fs-backup explicitly. +##### CreateResources +- It's better to set a global timeout in CreateResources function which is the real beginning of one e2e test +##### Destroy +- It's only clean up resources in currently test namespaces, if wantting to clean up all resources including resources created which is not in currently test namespaces, it's better to override base Destroy function +##### Clean +- Clean function only clean resources in namespaces which has the prefix CaseBaseName. So the the names of test namespaces should start with prefix of CaseBaseName. +- If wantting to clean up all resources including resources created which is not in currently test namespaces, it's better to override base Clean function +#### Velero Installation +- Velero is installed with global velero config before the E2E test start, so if the case (such as upgrade/migration, etc.) does not want to use the global velero config, it is NEED TO UNINSTALL velero, or the global velero config may affect the current test. +### TestFunc +The TestFunc function concatenate all the flows in a test. +It will reduce the frequency of velero installation by installing and checking Velero with the global Velero. It's Need to explicit reinstall Velero if the case has specicial configuration, such as API Group test case we need to enable feature EnableCSI. + ### Tips Look for the ⛵ emoji printed at the end of each install and uninstall log. There should not be two install/unintall in a row, and there should be tests between an install and an uninstall. \ No newline at end of file diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index df90b7014..6e59afcc9 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -47,7 +47,6 @@ func BackupDeletionWithRestic() { func backup_deletion_test(useVolumeSnapshots bool) { var ( backupName string - err error veleroCfg VeleroConfig ) veleroCfg = VeleroCfg @@ -60,11 +59,11 @@ func backup_deletion_test(useVolumeSnapshots bool) { } var err error flag.Parse() + if veleroCfg.InstallVelero { + Expect(PrepareVelero(context.Background(), "backup deletion")).To(Succeed()) + } UUIDgen, err = uuid.NewRandom() Expect(err).To(Succeed()) - if veleroCfg.InstallVelero { - Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) - } }) AfterEach(func() { @@ -72,10 +71,6 @@ func backup_deletion_test(useVolumeSnapshots bool) { By("Clean backups after test", func() { DeleteBackups(context.Background(), *veleroCfg.ClientToInstallVelero) }) - if veleroCfg.InstallVelero { - err = VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace) - Expect(err).To(Succeed()) - } } }) diff --git a/test/e2e/basic/storage-class-changing.go b/test/e2e/basic/storage-class-changing.go index a653ec549..05eb7e855 100644 --- a/test/e2e/basic/storage-class-changing.go +++ b/test/e2e/basic/storage-class-changing.go @@ -48,8 +48,6 @@ func (s *StorageClasssChanging) Init() error { Text: "Change the storage class of persistent volumes and persistent" + " volume claims during restores", } - s.BackupName = "backup-sc-" + UUIDgen.String() - s.RestoreName = "restore-" + UUIDgen.String() s.srcStorageClass = "default" s.desStorageClass = StorageClassName s.labels = map[string]string{"velero.io/change-storage-class": "RestoreItemAction", @@ -58,9 +56,6 @@ func (s *StorageClasssChanging) Init() error { s.configmaptName = "change-storage-class-config" s.volume = "volume-1" s.podName = "pod-1" - - s.BackupName = s.BackupName + "backup-" + UUIDgen.String() - s.RestoreName = s.RestoreName + "restore-" + UUIDgen.String() s.BackupArgs = []string{ "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", s.BackupName, "--include-namespaces", s.namespace, diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index f68342efb..e15307369 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -70,7 +70,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { UUIDgen, err = uuid.NewRandom() Expect(err).To(Succeed()) if veleroCfg.InstallVelero { - Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) + Expect(PrepareVelero(context.Background(), "BSL Deletion")).To(Succeed()) } }) @@ -84,14 +84,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { true)).To(Succeed(), fmt.Sprintf("failed to delete the namespace %q", bslDeletionTestNs)) }) - if veleroCfg.InstallVelero { - By("Uninstall Velero", func() { - Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, - veleroCfg.VeleroNamespace)).To(Succeed()) - }) - } } - }) When("kibishii is the sample workload", func() { diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index f4e1a4ddf..651affb0e 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -43,6 +43,7 @@ import ( . "github.com/vmware-tanzu/velero/test/e2e/schedule" . "github.com/vmware-tanzu/velero/test/e2e/upgrade" . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + . "github.com/vmware-tanzu/velero/test/e2e/util/velero" ) func init() { @@ -65,6 +66,8 @@ func init() { flag.StringVar(&VeleroCfg.VSLConfig, "vsl-config", "", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2") flag.StringVar(&VeleroCfg.VeleroNamespace, "velero-namespace", "velero", "namespace to install Velero into") flag.BoolVar(&VeleroCfg.InstallVelero, "install-velero", true, "install/uninstall velero during the test. Optional.") + flag.BoolVar(&VeleroCfg.UseNodeAgent, "use-node-agent", true, "whether deploy node agent daemonset velero during the test. Optional.") + flag.BoolVar(&VeleroCfg.UseVolumeSnapshots, "use-volume-snapshots", true, "Whether or not to create snapshot location automatically. Set to false if you do not plan to create volume snapshots via a storage provider.") flag.StringVar(&VeleroCfg.RegistryCredentialFile, "registry-credential-file", "", "file containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. Optional.") flag.StringVar(&VeleroCfg.KibishiiDirectory, "kibishii-directory", "github.com/vmware-tanzu-experiments/distributed-data-generator/kubernetes/yaml/", "The file directory or URL path to install Kibishii. Optional.") //vmware-tanzu-experiments @@ -189,3 +192,17 @@ func TestE2e(t *testing.T) { junitReporter := reporters.NewJUnitReporter("report.xml") RunSpecsWithDefaultAndCustomReporters(t, "E2e Suite", []Reporter{junitReporter}) } + +var _ = BeforeSuite(func() { + if VeleroCfg.InstallVelero { + By("Install test resources before testing") + Expect(PrepareVelero(context.Background(), "install resource before testing")).To(Succeed()) + } +}) + +var _ = AfterSuite(func() { + if VeleroCfg.InstallVelero && !VeleroCfg.Debug { + By("release test resources after testing") + Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed()) + } +}) diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index c9b7571a6..0a133f23e 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -69,6 +69,13 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) if veleroCfg.DefaultCluster == "" && veleroCfg.StandbyCluster == "" { Skip("Migration test needs 2 clusters") } + // need to uninstall Velero first in case of the affection of the existing global velero installation + if veleroCfg.InstallVelero { + By("Uninstall Velero", func() { + Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace)).To(Succeed()) + }) + } }) AfterEach(func() { if !veleroCfg.Debug { diff --git a/test/e2e/privilegesmgmt/ssr.go b/test/e2e/privilegesmgmt/ssr.go index 704d4d191..1f840db68 100644 --- a/test/e2e/privilegesmgmt/ssr.go +++ b/test/e2e/privilegesmgmt/ssr.go @@ -44,15 +44,7 @@ func SSRTest() { flag.Parse() veleroCfg.UseVolumeSnapshots = false if veleroCfg.InstallVelero { - Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) - } - }) - - AfterEach(func() { - if veleroCfg.InstallVelero { - if !veleroCfg.Debug { - Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed()) - } + Expect(PrepareVelero(context.Background(), "SSR test")).To(Succeed()) } }) @@ -102,7 +94,7 @@ func SSRTest() { Expect(veleroCfg.ClientToInstallVelero.Kubebuilder.List(ctx, ssrListResp, &kbclient.ListOptions{Namespace: testNS})).To(Succeed(), fmt.Sprintf("Failed to list ssr object in %s namespace", testNS)) Expect(len(ssrListResp.Items)).To(BeNumerically("==", 1), - fmt.Sprintf("Count of ssr object in %s namespace is not 1", testNS)) + fmt.Sprintf("Count of ssr object in %s namespace is not 1 but %d", testNS, len(ssrListResp.Items))) Expect(ssrListResp.Items[0].Status.Phase).To(BeEmpty(), fmt.Sprintf("Status of ssr object in %s namespace should be empty", testNS)) Expect(ssrListResp.Items[0].Status.ServerVersion).To(BeEmpty(), diff --git a/test/e2e/pv-backup/pv-backup-filter.go b/test/e2e/pv-backup/pv-backup-filter.go index 2ed6f2cf5..503afd1c4 100644 --- a/test/e2e/pv-backup/pv-backup-filter.go +++ b/test/e2e/pv-backup/pv-backup-filter.go @@ -66,7 +66,7 @@ func (p *PVBackupFiltering) Init() error { } func (p *PVBackupFiltering) CreateResources() error { - p.Ctx, p.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) + p.Ctx, p.CtxCancel = context.WithTimeout(context.Background(), 30*time.Minute) err := InstallStorageClass(p.Ctx, fmt.Sprintf("testdata/storage-class/%s.yaml", VeleroCfg.CloudProvider)) if err != nil { return errors.Wrapf(err, "failed to install storage class for pv backup filtering test") diff --git a/test/e2e/resourcepolicies/resource_policies.go b/test/e2e/resourcepolicies/resource_policies.go index c8cf5db34..9f60dab72 100644 --- a/test/e2e/resourcepolicies/resource_policies.go +++ b/test/e2e/resourcepolicies/resource_policies.go @@ -59,25 +59,31 @@ type ResourcePoliciesCase struct { var ResourcePoliciesTest func() = TestFunc(&ResourcePoliciesCase{}) func (r *ResourcePoliciesCase) Init() error { + // generate random number as UUIDgen and set one default timeout duration r.TestCase.Init() - r.CaseBaseName = "resource-policies-" + r.UUIDgen - r.cmName = "cm-" + r.CaseBaseName - r.yamlConfig = yamlData - r.VeleroCfg = VeleroCfg - r.Client = *r.VeleroCfg.ClientToInstallVelero - r.VeleroCfg.UseVolumeSnapshots = false - r.VeleroCfg.UseNodeAgent = true - r.NamespacesTotal = 3 + // generate variable names based on CaseBaseName + UUIDgen + r.CaseBaseName = "resource-policies-" + r.UUIDgen + r.BackupName = "backup-" + r.CaseBaseName + r.RestoreName = "restore-" + r.CaseBaseName + r.cmName = "cm-" + r.CaseBaseName + + // generate namespaces by NamespacesTotal + r.NamespacesTotal = 3 r.NSIncluded = &[]string{} for nsNum := 0; nsNum < r.NamespacesTotal; nsNum++ { createNSName := fmt.Sprintf("%s-%00000d", r.CaseBaseName, nsNum) *r.NSIncluded = append(*r.NSIncluded, createNSName) } - r.BackupName = "backup-resource-policies-" + UUIDgen.String() - r.RestoreName = "restore-resource-policies-" + UUIDgen.String() + // assign values to the inner variable for specific case + r.yamlConfig = yamlData + r.VeleroCfg = VeleroCfg + r.Client = *r.VeleroCfg.ClientToInstallVelero + r.VeleroCfg.UseVolumeSnapshots = false + r.VeleroCfg.UseNodeAgent = true + // NEED explicitly specify the value of the variables for snapshot-volumes or default-volumes-to-fs-backup r.BackupArgs = []string{ "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", r.BackupName, "--resource-policies-configmap", r.cmName, @@ -91,6 +97,7 @@ func (r *ResourcePoliciesCase) Init() error { "--from-backup", r.BackupName, "--wait", } + // Message output by ginkgo r.TestMsg = &TestMSG{ Desc: "Skip backup of volume by resource policies", FailedMSG: "Failed to skip backup of volume by resource policies", @@ -100,7 +107,9 @@ func (r *ResourcePoliciesCase) Init() error { } func (r *ResourcePoliciesCase) CreateResources() error { + // It's better to set a global timeout in CreateResources function which is the real beginning of one e2e test r.Ctx, r.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) + By(("Installing storage class..."), func() { Expect(r.installTestStorageClasses(fmt.Sprintf("testdata/storage-class/%s.yaml", VeleroCfg.CloudProvider))).To(Succeed(), "Failed to install storage class") }) @@ -178,6 +187,7 @@ func (r *ResourcePoliciesCase) Verify() error { } func (r *ResourcePoliciesCase) Clean() error { + // If created some resources which is not in current test namespace, we NEED to override the base Clean function if !r.VeleroCfg.Debug { if err := r.deleteTestStorageClassList([]string{"e2e-storage-class", "e2e-storage-class-2"}); err != nil { return err @@ -187,7 +197,7 @@ func (r *ResourcePoliciesCase) Clean() error { return err } - return r.GetTestCase().Clean() + return r.GetTestCase().Clean() // only clean up resources in test namespace } return nil } diff --git a/test/e2e/schedule/ordered_resources.go b/test/e2e/schedule/ordered_resources.go index 3bc4c7bed..ce9eb97c0 100644 --- a/test/e2e/schedule/ordered_resources.go +++ b/test/e2e/schedule/ordered_resources.go @@ -128,6 +128,10 @@ func (o *OrderedResources) Backup() error { return nil } +func (o *OrderedResources) Destroy() error { + return nil +} + func (o *OrderedResources) Verify() error { By(fmt.Sprintf("Checking resource order in %s schedule cr", o.ScheduleName), func() { err := CheckScheduleWithResourceOrder(o.Ctx, o.VeleroCfg.VeleroCLI, o.VeleroCfg.VeleroNamespace, o.ScheduleName, o.OrderMap) diff --git a/test/e2e/schedule/schedule-backup-creation.go b/test/e2e/schedule/schedule-backup-creation.go index 02f5b1540..57fbab136 100644 --- a/test/e2e/schedule/schedule-backup-creation.go +++ b/test/e2e/schedule/schedule-backup-creation.go @@ -36,7 +36,7 @@ var ScheduleBackupCreationTest func() = TestFunc(&ScheduleBackupCreation{}) func (n *ScheduleBackupCreation) Init() error { n.TestCase.Init() n.CaseBaseName = "schedule-backup-creation-test" + n.UUIDgen - n.ScheduleName = n.ScheduleName + "schedule-" + UUIDgen.String() + n.ScheduleName = "schedule-" + n.CaseBaseName n.namespace = n.CaseBaseName n.VeleroCfg = VeleroCfg n.Client = *n.VeleroCfg.ClientToInstallVelero diff --git a/test/e2e/schedule/schedule.go b/test/e2e/schedule/schedule.go index eb1b0d9e4..7b7f2682f 100644 --- a/test/e2e/schedule/schedule.go +++ b/test/e2e/schedule/schedule.go @@ -32,7 +32,7 @@ func (n *ScheduleBackup) Init() error { n.CaseBaseName = "schedule-backup-" + n.UUIDgen n.NSIncluded = &[]string{n.CaseBaseName} n.ScheduleName = "schedule-" + n.CaseBaseName - + n.RestoreName = "restore-" + n.CaseBaseName n.VeleroCfg = VeleroCfg n.Client = *n.VeleroCfg.ClientToInstallVelero n.Period = 3 // Unit is minute @@ -46,6 +46,7 @@ func (n *ScheduleBackup) Init() error { "--include-namespaces", strings.Join(*n.NSIncluded, ","), "--schedule=*/" + fmt.Sprintf("%v", n.Period) + " * * * *", } + Expect(n.Period < 30).To(Equal(true)) return nil } @@ -61,7 +62,7 @@ func (n *ScheduleBackup) CreateResources() error { _, err := CreateConfigMap(n.Client.ClientGo, ns, configmaptName, nil, nil) Expect(err).To(Succeed(), fmt.Sprintf("failed to create configmap in the namespace %q", ns)) Expect(WaitForConfigMapComplete(n.Client.ClientGo, ns, configmaptName)).To(Succeed(), - fmt.Sprintf("ailed to ensure secret completion in namespace: %q", ns)) + fmt.Sprintf("failed to ensure secret completion in namespace: %q", ns)) } return nil } @@ -139,9 +140,11 @@ func (n *ScheduleBackup) Backup() error { n.BackupName = strings.Replace(n.randBackupName, " ", "", -1) - By("Delete all namespaces", func() { - Expect(CleanupNamespacesWithPoll(n.Ctx, n.Client, n.CaseBaseName)).To(Succeed(), "Could cleanup retrieve namespaces") - }) + n.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", n.RestoreName, + "--from-backup", n.BackupName, + "--wait", + } backupsInfo, err := GetScheduledBackupsCreationTime(n.Ctx, VeleroCfg.VeleroCLI, "default", n.ScheduleName) Expect(err).To(Succeed(), fmt.Sprintf("Fail to get backups from schedule %s", n.ScheduleName)) @@ -205,10 +208,6 @@ func (n *ScheduleBackup) Verify() error { return nil } -func (n *ScheduleBackup) Destroy() error { - return nil -} - func (n *ScheduleBackup) Clean() error { if !n.VeleroCfg.Debug { Expect(VeleroScheduleDelete(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, n.ScheduleName)).To(Succeed()) diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go index fd51ad420..9148030eb 100644 --- a/test/e2e/test/test.go +++ b/test/e2e/test/test.go @@ -30,6 +30,7 @@ import ( "github.com/pkg/errors" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/test/e2e" . "github.com/vmware-tanzu/velero/test/e2e" . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" . "github.com/vmware-tanzu/velero/test/e2e/util/velero" @@ -80,24 +81,16 @@ type TestCase struct { func TestFunc(test VeleroBackupRestoreTest) func() { return func() { Expect(test.Init()).To(Succeed(), "Failed to instantiate test cases") - veleroCfg := test.GetTestCase().VeleroCfg BeforeEach(func() { flag.Parse() - veleroCfg := test.GetTestCase().VeleroCfg + // Using the global velero config which covered the installation for most common cases + veleroCfg := e2e.VeleroCfg // TODO: Skip nodeport test until issue https://github.com/kubernetes/kubernetes/issues/114384 fixed if veleroCfg.CloudProvider == "azure" && strings.Contains(test.GetTestCase().CaseBaseName, "nodeport") { Skip("Skip due to issue https://github.com/kubernetes/kubernetes/issues/114384 on AKS") } if veleroCfg.InstallVelero { - veleroCfg.UseVolumeSnapshots = test.GetTestCase().UseVolumeSnapshots - Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) - } - }) - AfterEach(func() { - if !veleroCfg.Debug { - if veleroCfg.InstallVelero { - Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To((Succeed())) - } + Expect(PrepareVelero(context.Background(), test.GetTestCase().CaseBaseName)).To(Succeed()) } }) It(test.GetTestMsg().Text, func() { @@ -108,35 +101,16 @@ func TestFunc(test VeleroBackupRestoreTest) func() { func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() { return func() { - var countIt int - var useVolumeSnapshots bool - var veleroCfg VeleroConfig + veleroCfg := e2e.VeleroCfg for k := range tests { Expect(tests[k].Init()).To(Succeed(), fmt.Sprintf("Failed to instantiate test %s case", tests[k].GetTestMsg().Desc)) - veleroCfg = tests[k].GetTestCase().VeleroCfg - useVolumeSnapshots = tests[k].GetTestCase().UseVolumeSnapshots defer tests[k].GetTestCase().CtxCancel() } BeforeEach(func() { flag.Parse() if veleroCfg.InstallVelero { - if countIt == 0 { - veleroCfg.UseVolumeSnapshots = useVolumeSnapshots - veleroCfg.UseNodeAgent = !useVolumeSnapshots - Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) - } - countIt++ - } - }) - - AfterEach(func() { - if !veleroCfg.Debug { - if veleroCfg.InstallVelero { - if countIt == len(tests) && !veleroCfg.Debug { - Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To((Succeed())) - } - } + Expect(PrepareVelero(context.Background(), tests[0].GetTestCase().CaseBaseName)).To(Succeed()) } }) diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index d2fe8830d..1e661d823 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -74,6 +74,13 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC if veleroCfg.VeleroCLI == "" { Skip("VeleroCLI should be provide") } + // need to uninstall Velero first in case of the affection of the existing global velero installation + if veleroCfg.InstallVelero { + By("Uninstall Velero", func() { + Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace)).To(Succeed()) + }) + } }) AfterEach(func() { if !veleroCfg.Debug { diff --git a/test/e2e/util/k8s/common.go b/test/e2e/util/k8s/common.go index 694051e72..cf18e7f0a 100644 --- a/test/e2e/util/k8s/common.go +++ b/test/e2e/util/k8s/common.go @@ -238,7 +238,7 @@ func GetPVByPodName(client TestClient, namespace, podName string) (string, error return "", err } if len(pvcList) != 1 { - return "", errors.New(fmt.Sprintf("Only 1 PVC of pod %s should be found under namespace %s", podName, namespace)) + return "", errors.New(fmt.Sprintf("Only 1 PVC of pod %s should be found under namespace %s but got %v", podName, namespace, pvcList)) } pvList, err := GetPvByPvc(context.Background(), namespace, pvcList[0]) if err != nil { diff --git a/test/e2e/util/kibishii/kibishii_utils.go b/test/e2e/util/kibishii/kibishii_utils.go index 66ad2116d..120c984db 100644 --- a/test/e2e/util/kibishii/kibishii_utils.go +++ b/test/e2e/util/kibishii/kibishii_utils.go @@ -291,6 +291,18 @@ func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClie providerName, kibishiiNamespace, registryCredentialFile, veleroFeatures, kibishiiDirectory string, useVolumeSnapshots bool, kibishiiData *KibishiiData) error { fmt.Printf("installKibishii %s\n", time.Now().Format("2006-01-02 15:04:05")) + serviceAccountName := "default" + + // wait until the service account is created before patch the image pull secret + if err := WaitUntilServiceAccountCreated(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, 10*time.Minute); err != nil { + return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, kibishiiNamespace) + } + + // add the image pull secret to avoid the image pull limit issue of Docker Hub + if err := PatchServiceAccountWithImagePullSecret(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, registryCredentialFile); err != nil { + return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, kibishiiNamespace) + } + if err := installKibishii(oneHourTimeout, kibishiiNamespace, providerName, veleroFeatures, kibishiiDirectory, useVolumeSnapshots); err != nil { return errors.Wrap(err, "Failed to install Kibishii workload") diff --git a/test/e2e/util/velero/install.go b/test/e2e/util/velero/install.go index 112e041d5..5a37a1ba1 100644 --- a/test/e2e/util/velero/install.go +++ b/test/e2e/util/velero/install.go @@ -35,6 +35,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" velerexec "github.com/vmware-tanzu/velero/pkg/util/exec" . "github.com/vmware-tanzu/velero/test/e2e" @@ -461,6 +462,70 @@ func waitVeleroReady(ctx context.Context, namespace string, useNodeAgent bool) e return nil } +func IsVeleroReady(ctx context.Context, namespace string, useNodeAgent bool) (bool, error) { + if useNodeAgent { + stdout, stderr, err := velerexec.RunCommand(exec.CommandContext(ctx, "kubectl", "get", "daemonset/node-agent", + "-o", "json", "-n", namespace)) + if err != nil { + return false, errors.Wrapf(err, "failed to get the node-agent daemonset, stdout=%s, stderr=%s", stdout, stderr) + } else { + daemonset := &apps.DaemonSet{} + if err = json.Unmarshal([]byte(stdout), daemonset); err != nil { + return false, errors.Wrapf(err, "failed to unmarshal the node-agent daemonset") + } + if daemonset.Status.DesiredNumberScheduled != daemonset.Status.NumberAvailable { + return false, fmt.Errorf("the available number pod %d in node-agent daemonset not equal to scheduled number %d", daemonset.Status.NumberAvailable, daemonset.Status.DesiredNumberScheduled) + } + } + } + + stdout, stderr, err := velerexec.RunCommand(exec.CommandContext(ctx, "kubectl", "get", "deployment/velero", + "-o", "json", "-n", namespace)) + if err != nil { + return false, errors.Wrapf(err, "failed to get the velero deployment stdout=%s, stderr=%s", stdout, stderr) + } else { + deployment := &apps.Deployment{} + if err = json.Unmarshal([]byte(stdout), deployment); err != nil { + return false, errors.Wrapf(err, "failed to unmarshal the velero deployment") + } + if deployment.Status.AvailableReplicas != deployment.Status.Replicas { + return false, fmt.Errorf("the available replicas %d in velero deployment not equal to replicas %d", deployment.Status.AvailableReplicas, deployment.Status.Replicas) + } + } + + // Check BSL + stdout, stderr, err = velerexec.RunCommand(exec.CommandContext(ctx, "kubectl", "get", "bsl", "default", + "-o", "json", "-n", namespace)) + if err != nil { + return false, errors.Wrapf(err, "failed to get bsl %s stdout=%s, stderr=%s", VeleroCfg.BSLBucket, stdout, stderr) + } else { + bsl := &velerov1api.BackupStorageLocation{} + if err = json.Unmarshal([]byte(stdout), bsl); err != nil { + return false, errors.Wrapf(err, "failed to unmarshal the velero bsl") + } + if bsl.Status.Phase != velerov1api.BackupStorageLocationPhaseAvailable { + return false, fmt.Errorf("current bsl %s is not available", VeleroCfg.BSLBucket) + } + } + + return true, nil +} + +func PrepareVelero(ctx context.Context, caseName string) error { + ready, err := IsVeleroReady(context.Background(), VeleroCfg.VeleroNamespace, VeleroCfg.UseNodeAgent) + if err != nil { + fmt.Printf("error in checking velero status with %v", err) + VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) + ready = false + } + if ready { + fmt.Printf("velero is ready for case %s \n", caseName) + return nil + } + fmt.Printf("need to install velero for case %s \n", caseName) + return VeleroInstall(context.Background(), &VeleroCfg) +} + func VeleroUninstall(ctx context.Context, cli, namespace string) error { stdout, stderr, err := velerexec.RunCommand(exec.CommandContext(ctx, cli, "uninstall", "--force", "-n", namespace)) if err != nil {