From 996d2a025fa33b31cba4eec57ad48fa7193a6d13 Mon Sep 17 00:00:00 2001 From: Xun Jiang Date: Thu, 28 Aug 2025 14:05:56 +0800 Subject: [PATCH] Add E2E test cases for repository maintenance job configuration. Signed-off-by: Xun Jiang --- pkg/builder/priority_class_builder.go | 52 ++++ test/e2e/e2e_suite_test.go | 24 ++ test/e2e/migration/migration.go | 6 + .../repo_maintenance_config.go | 247 ++++++++++++++++++ test/e2e/test/test.go | 8 +- test/types.go | 5 + test/util/velero/install.go | 25 +- test/util/velero/velero_utils.go | 47 ++++ 8 files changed, 410 insertions(+), 4 deletions(-) create mode 100644 pkg/builder/priority_class_builder.go create mode 100644 test/e2e/repomaintenance/repo_maintenance_config.go diff --git a/pkg/builder/priority_class_builder.go b/pkg/builder/priority_class_builder.go new file mode 100644 index 000000000..b197e6d74 --- /dev/null +++ b/pkg/builder/priority_class_builder.go @@ -0,0 +1,52 @@ +/* +Copyright 2019 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + corev1api "k8s.io/api/core/v1" + schedulingv1api "k8s.io/api/scheduling/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type PriorityClassBuilder struct { + object *schedulingv1api.PriorityClass +} + +func ForPriorityClass(name string) *PriorityClassBuilder { + return &PriorityClassBuilder{ + object: &schedulingv1api.PriorityClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + }, + } +} + +func (p *PriorityClassBuilder) Value(value int) *PriorityClassBuilder { + p.object.Value = int32(value) + return p +} + +func (p *PriorityClassBuilder) PreemptionPolicy(policy string) *PriorityClassBuilder { + preemptionPolicy := corev1api.PreemptionPolicy(policy) + p.object.PreemptionPolicy = &preemptionPolicy + return p +} + +func (p *PriorityClassBuilder) Result() *schedulingv1api.PriorityClass { + return p.object +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 0159af8af..0e632dcc4 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -43,6 +43,7 @@ import ( . "github.com/vmware-tanzu/velero/test/e2e/parallelfilesupload" . "github.com/vmware-tanzu/velero/test/e2e/privilegesmgmt" . "github.com/vmware-tanzu/velero/test/e2e/pv-backup" + . "github.com/vmware-tanzu/velero/test/e2e/repomaintenance" . "github.com/vmware-tanzu/velero/test/e2e/resource-filtering" . "github.com/vmware-tanzu/velero/test/e2e/resourcemodifiers" . "github.com/vmware-tanzu/velero/test/e2e/resourcepolicies" @@ -660,6 +661,18 @@ var _ = Describe( ParallelFilesDownloadTest, ) +var _ = Describe( + "Test Repository Maintenance Job Configuration's global part", + Label("RepoMaintenance", "LongTime"), + GlobalRepoMaintenanceTest, +) + +var _ = Describe( + "Test Repository Maintenance Job Configuration's specific part", + Label("RepoMaintenance", "LongTime"), + SpecificRepoMaintenanceTest, +) + func GetKubeConfigContext() error { var err error var tcDefault, tcStandby k8s.TestClient @@ -740,6 +753,12 @@ var _ = BeforeSuite(func() { ).To(Succeed()) } + // Create the needed PriorityClasses + Expect(veleroutil.CreatePriorityClasses( + context.Background(), + test.VeleroCfg.ClientToInstallVelero.Kubebuilder, + )).To(Succeed()) + if test.InstallVelero { By("Install test resources before testing") Expect( @@ -783,6 +802,11 @@ var _ = AfterSuite(func() { ).To(Succeed()) } + Expect(veleroutil.DeletePriorityClasses( + ctx, + test.VeleroCfg.ClientToInstallVelero.Kubebuilder, + )).To(Succeed()) + // If the Velero is installed during test, and the FailFast is not enabled, // uninstall Velero. If not, either Velero is not installed, or kept it for debug on failure. if test.InstallVelero && (testSuitePassed || !test.VeleroCfg.FailFast) { diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 1c896aa9f..b26f87f38 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -342,6 +342,12 @@ func (m *migrationE2E) Restore() error { Expect(veleroutil.InstallStorageClasses( m.VeleroCfg.StandbyClusterCloudProvider)).To(Succeed()) + // Create the needed PriorityClasses + Expect(veleroutil.CreatePriorityClasses( + context.Background(), + test.VeleroCfg.StandbyClient.Kubebuilder, + )).To(Succeed()) + if strings.EqualFold(m.VeleroCfg.Features, test.FeatureCSI) && m.VeleroCfg.UseVolumeSnapshots { By("Install VolumeSnapshotClass for E2E.") diff --git a/test/e2e/repomaintenance/repo_maintenance_config.go b/test/e2e/repomaintenance/repo_maintenance_config.go new file mode 100644 index 000000000..711d9bb89 --- /dev/null +++ b/test/e2e/repomaintenance/repo_maintenance_config.go @@ -0,0 +1,247 @@ +/* +Copyright 2021 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package repomaintenance + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + . "github.com/onsi/gomega" + "github.com/pkg/errors" + batchv1api "k8s.io/api/batch/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/builder" + velerotypes "github.com/vmware-tanzu/velero/pkg/types" + "github.com/vmware-tanzu/velero/pkg/util/kube" + velerokubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" + "github.com/vmware-tanzu/velero/test" + . "github.com/vmware-tanzu/velero/test/e2e/test" + k8sutil "github.com/vmware-tanzu/velero/test/util/k8s" + veleroutil "github.com/vmware-tanzu/velero/test/util/velero" +) + +type RepoMaintenanceTestCase struct { + TestCase + repoMaintenanceConfigMapName string + repoMaintenanceConfigKey string + jobConfigs velerotypes.JobConfigs +} + +var keepJobNum = 1 + +var GlobalRepoMaintenanceTest func() = TestFunc(&RepoMaintenanceTestCase{ + repoMaintenanceConfigKey: "global", + repoMaintenanceConfigMapName: "global", + jobConfigs: velerotypes.JobConfigs{ + KeepLatestMaintenanceJobs: &keepJobNum, + PodResources: &velerokubeutil.PodResources{ + CPURequest: "100m", + MemoryRequest: "100Mi", + CPULimit: "200m", + MemoryLimit: "200Mi", + }, + PriorityClassName: test.PriorityClassNameForRepoMaintenance, + }, +}) + +var SpecificRepoMaintenanceTest func() = TestFunc(&RepoMaintenanceTestCase{ + repoMaintenanceConfigKey: "", + repoMaintenanceConfigMapName: "specific", + jobConfigs: velerotypes.JobConfigs{ + KeepLatestMaintenanceJobs: &keepJobNum, + PodResources: &velerokubeutil.PodResources{ + CPURequest: "100m", + MemoryRequest: "100Mi", + CPULimit: "200m", + MemoryLimit: "200Mi", + }, + PriorityClassName: test.PriorityClassNameForRepoMaintenance, + }, +}) + +func (r *RepoMaintenanceTestCase) Init() error { + // generate random number as UUIDgen and set one default timeout duration + r.TestCase.Init() + + // generate variable names based on CaseBaseName + UUIDgen + r.CaseBaseName = "repo-maintenance-" + r.UUIDgen + r.BackupName = "backup-" + r.CaseBaseName + r.RestoreName = "restore-" + r.CaseBaseName + + // generate namespaces by NamespacesTotal + r.NamespacesTotal = 1 + r.NSIncluded = &[]string{} + for nsNum := 0; nsNum < r.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", r.CaseBaseName, nsNum) + *r.NSIncluded = append(*r.NSIncluded, createNSName) + } + + // If repoMaintenanceConfigKey is not set, it means testing the specific repo case. + // Need to assemble the BackupRepository name. The format is "volumeNamespace-bslName-uploaderName" + if r.repoMaintenanceConfigKey == "" { + r.repoMaintenanceConfigKey = (*r.NSIncluded)[0] + "-" + "default" + "-" + test.UploaderTypeKopia + } + + // assign values to the inner variable for specific case + r.VeleroCfg.UseNodeAgent = true + r.VeleroCfg.UseNodeAgentWindows = true + + r.BackupArgs = []string{ + "create", "--namespace", r.VeleroCfg.VeleroNamespace, "backup", r.BackupName, + "--include-namespaces", strings.Join(*r.NSIncluded, ","), + "--snapshot-volumes=true", "--snapshot-move-data", "--wait", + } + + // Message output by ginkgo + r.TestMsg = &TestMSG{ + Desc: "Validate Repository Maintenance Job configuration", + FailedMSG: "Failed to apply and / or validate configuration in repository maintenance jobs.", + Text: "Should be able to apply and validate configuration in repository maintenance jobs.", + } + return nil +} + +func (r *RepoMaintenanceTestCase) InstallVelero() error { + // Because this test needs to use customized repository maintenance ConfigMap, + // need to uninstall and reinstall Velero. + + fmt.Println("Start to uninstall Velero") + if err := veleroutil.VeleroUninstall(r.Ctx, r.VeleroCfg); err != nil { + fmt.Printf("Fail to uninstall Velero: %s\n", err.Error()) + return err + } + + result, err := json.Marshal(r.jobConfigs) + if err != nil { + return err + } + + repoMaintenanceConfig := builder.ForConfigMap(r.VeleroCfg.VeleroNamespace, r.repoMaintenanceConfigMapName). + Data(r.repoMaintenanceConfigKey, string(result)).Result() + + r.VeleroCfg.RepoMaintenanceJobConfigMap = r.repoMaintenanceConfigMapName + + return veleroutil.PrepareVelero( + r.Ctx, + r.CaseBaseName, + r.VeleroCfg, + repoMaintenanceConfig, + ) +} + +func (r *RepoMaintenanceTestCase) CreateResources() error { + for _, ns := range *r.NSIncluded { + if err := k8sutil.CreateNamespace(r.Ctx, r.Client, ns); err != nil { + fmt.Printf("Fail to create ns %s: %s\n", ns, err.Error()) + return err + } + + pvc, err := k8sutil.CreatePVC(r.Client, ns, "volume-1", test.StorageClassName, nil) + if err != nil { + fmt.Printf("Fail to create PVC %s: %s\n", "volume-1", err.Error()) + return err + } + + vols := k8sutil.CreateVolumes(pvc.Name, []string{"volume-1"}) + + deployment := k8sutil.NewDeployment( + r.CaseBaseName, + (*r.NSIncluded)[0], + 1, + map[string]string{"app": "test"}, + r.VeleroCfg.ImageRegistryProxy, + r.VeleroCfg.WorkerOS, + ).WithVolume(vols).Result() + + deployment, err = k8sutil.CreateDeployment(r.Client.ClientGo, ns, deployment) + if err != nil { + fmt.Printf("Fail to create deployment %s: %s \n", deployment.Name, err.Error()) + return errors.Wrap(err, fmt.Sprintf("failed to create deployment: %s", err.Error())) + } + + if err := k8sutil.WaitForReadyDeployment(r.Client.ClientGo, deployment.Namespace, deployment.Name); err != nil { + fmt.Printf("Fail to create deployment %s: %s\n", r.CaseBaseName, err.Error()) + return err + } + } + + return nil +} + +func (r *RepoMaintenanceTestCase) Verify() error { + // Reduce the MaintenanceFrequency to 1 minute. + backupRepositoryList := new(velerov1api.BackupRepositoryList) + if err := r.VeleroCfg.ClientToInstallVelero.Kubebuilder.List( + r.Ctx, + backupRepositoryList, + &client.ListOptions{ + Namespace: r.VeleroCfg.Namespace, + LabelSelector: labels.SelectorFromSet(map[string]string{velerov1api.VolumeNamespaceLabel: (*r.NSIncluded)[0]}), + }, + ); err != nil { + return err + } + + if len(backupRepositoryList.Items) <= 0 { + return fmt.Errorf("fail list BackupRepository. no item is returned") + } + + backupRepository := backupRepositoryList.Items[0] + + updated := backupRepository.DeepCopy() + updated.Spec.MaintenanceFrequency = metav1.Duration{Duration: time.Minute} + if err := r.VeleroCfg.ClientToInstallVelero.Kubebuilder.Patch(r.Ctx, updated, client.MergeFrom(&backupRepository)); err != nil { + fmt.Printf("failed to patch BackupRepository %q: %s", backupRepository.GetName(), err.Error()) + return err + } + + // The minimal time unit of Repository Maintenance is 5 minutes. + // Wait for more than one cycles to make sure the result is valid. + time.Sleep(6 * time.Minute) + + jobList := new(batchv1api.JobList) + if err := r.VeleroCfg.ClientToInstallVelero.Kubebuilder.List(r.Ctx, jobList, &client.ListOptions{ + Namespace: r.VeleroCfg.Namespace, + LabelSelector: labels.SelectorFromSet(map[string]string{"velero.io/repo-name": backupRepository.Name}), + }); err != nil { + return nil + } + + resources, err := kube.ParseResourceRequirements( + r.jobConfigs.PodResources.CPURequest, + r.jobConfigs.PodResources.MemoryRequest, + r.jobConfigs.PodResources.CPULimit, + r.jobConfigs.PodResources.MemoryLimit, + ) + if err != nil { + return errors.Wrap(err, "failed to parse resource requirements for maintenance job") + } + + Expect(jobList.Items[0].Spec.Template.Spec.Containers[0].Resources).To(Equal(resources)) + + Expect(jobList.Items).To(HaveLen(*r.jobConfigs.KeepLatestMaintenanceJobs)) + + Expect(jobList.Items[0].Spec.Template.Spec.PriorityClassName).To(Equal(r.jobConfigs.PriorityClassName)) + + return nil +} diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go index 78704a4bb..1a91c115f 100644 --- a/test/e2e/test/test.go +++ b/test/e2e/test/test.go @@ -41,6 +41,7 @@ depends on your test patterns. */ type VeleroBackupRestoreTest interface { Init() error + InstallVelero() error CreateResources() error Backup() error Destroy() error @@ -109,6 +110,10 @@ func (t *TestCase) GenerateUUID() string { return fmt.Sprintf("%08d", rand.IntN(100000000)) } +func (t *TestCase) InstallVelero() error { + return PrepareVelero(context.Background(), t.GetTestCase().CaseBaseName, t.GetTestCase().VeleroCfg) +} + func (t *TestCase) CreateResources() error { return nil } @@ -221,7 +226,8 @@ func RunTestCase(test VeleroBackupRestoreTest) error { fmt.Printf("Running test case %s %s\n", test.GetTestMsg().Desc, time.Now().Format("2006-01-02 15:04:05")) if InstallVelero { - Expect(PrepareVelero(context.Background(), test.GetTestCase().CaseBaseName, test.GetTestCase().VeleroCfg)).To(Succeed()) + fmt.Printf("Install Velero for test case %s: %s", test.GetTestCase().CaseBaseName, time.Now().Format("2006-01-02 15:04:05")) + Expect(test.InstallVelero()).To(Succeed()) } defer test.Clean() diff --git a/test/types.go b/test/types.go index 01ea91c35..ef9582ce3 100644 --- a/test/types.go +++ b/test/types.go @@ -57,6 +57,11 @@ const ( BackupRepositoryConfigName = "backup-repository-config" ) +const ( + PriorityClassNameForDataMover = "data-mover" + PriorityClassNameForRepoMaintenance = "repo-maintenance" +) + var PublicCloudProviders = []string{AWS, Azure, GCP, Vsphere} var LocalCloudProviders = []string{Kind, VanillaZFS} var CloudProviders = append(PublicCloudProviders, LocalCloudProviders...) diff --git a/test/util/velero/install.go b/test/util/velero/install.go index 2f4eb43de..ca6e207d1 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -36,6 +36,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" @@ -56,7 +57,17 @@ type installOptions struct { WorkerOS string } -func VeleroInstall(ctx context.Context, veleroCfg *test.VeleroConfig, isStandbyCluster bool) error { +/* +VeleroInstall is used to install Velero for E2E test + +params: + + ctx: The context + veleroCfg: Velero E2E case configuration + isStandbyCluster: Whether Velero is installed on standby cluster + objects: The objects are installed in Velero installed namespace, e.g. the ConfigMaps. +*/ +func VeleroInstall(ctx context.Context, veleroCfg *test.VeleroConfig, isStandbyCluster bool, objects ...client.Object) error { fmt.Printf("Velero install %s\n", time.Now().Format("2006-01-02 15:04:05")) // veleroCfg struct including a set of BSL params and a set of additional BSL params, @@ -153,6 +164,14 @@ func VeleroInstall(ctx context.Context, veleroCfg *test.VeleroConfig, isStandbyC ) } + // Install the passed-in objects in Velero installed namespace + for _, obj := range objects { + if err := veleroCfg.ClientToInstallVelero.Kubebuilder.Create(ctx, obj); err != nil { + fmt.Printf("fail to create object %s in namespace %s: %s\n", obj.GetName(), obj.GetNamespace(), err.Error()) + return fmt.Errorf("fail to create object %s in namespace %s: %w", obj.GetName(), obj.GetNamespace(), err) + } + } + // For AWS IRSA credential test, AWS IAM service account is required, so if ServiceAccountName and EKSPolicyARN // are both provided, we assume IRSA test is running, otherwise skip this IAM service account creation part. if veleroCfg.CloudProvider == test.AWS && veleroInstallOptions.ServiceAccountName != "" { @@ -790,7 +809,7 @@ func CheckBSL(ctx context.Context, ns string, bslName string) error { return err } -func PrepareVelero(ctx context.Context, caseName string, veleroCfg test.VeleroConfig) error { +func PrepareVelero(ctx context.Context, caseName string, veleroCfg test.VeleroConfig, objects ...client.Object) error { ready, err := IsVeleroReady(context.Background(), &veleroCfg) if err != nil { fmt.Printf("error in checking velero status with %v", err) @@ -804,7 +823,7 @@ func PrepareVelero(ctx context.Context, caseName string, veleroCfg test.VeleroCo return nil } fmt.Printf("need to install velero for case %s \n", caseName) - return VeleroInstall(context.Background(), &veleroCfg, false) + return VeleroInstall(context.Background(), &veleroCfg, false, objects...) } func VeleroUninstall(ctx context.Context, veleroCfg test.VeleroConfig) error { diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index 5985dd509..65f56bf03 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -38,6 +38,8 @@ import ( "github.com/pkg/errors" "golang.org/x/mod/semver" + schedulingv1api "k8s.io/api/scheduling/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" ver "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/util/wait" @@ -45,9 +47,11 @@ import ( "github.com/vmware-tanzu/velero/internal/volume" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/builder" cliinstall "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" + "github.com/vmware-tanzu/velero/test" . "github.com/vmware-tanzu/velero/test" common "github.com/vmware-tanzu/velero/test/util/common" . "github.com/vmware-tanzu/velero/test/util/k8s" @@ -274,6 +278,9 @@ func getProviderVeleroInstallOptions(veleroCfg *VeleroConfig, io.ItemBlockWorkerCount = veleroCfg.ItemBlockWorkerCount io.ServerPriorityClassName = veleroCfg.ServerPriorityClassName io.NodeAgentPriorityClassName = veleroCfg.NodeAgentPriorityClassName + io.RepoMaintenanceJobConfigMap = veleroCfg.RepoMaintenanceJobConfigMap + io.BackupRepoConfigMap = veleroCfg.BackupRepoConfigMap + io.NodeAgentConfigMap = veleroCfg.NodeAgentConfigMap return io, nil } @@ -1812,3 +1819,43 @@ func KubectlGetAllDeleteBackupRequest(ctx context.Context, backupName, veleroNam return common.GetListByCmdPipes(ctx, cmds) } + +func CreatePriorityClasses(ctx context.Context, client kbclient.Client) error { + dataMoverPriorityClass := builder.ForPriorityClass(test.PriorityClassNameForDataMover). + Value(90000).PreemptionPolicy("Never").Result() + if err := client.Create(ctx, dataMoverPriorityClass); err != nil { + fmt.Printf("Fail to create PriorityClass %s: %s\n", test.PriorityClassNameForDataMover, err.Error()) + return fmt.Errorf("fail to create PriorityClass %s: %w", test.PriorityClassNameForDataMover, err) + } + + repoMaintenancePriorityClass := builder.ForPriorityClass(test.PriorityClassNameForRepoMaintenance). + Value(80000).PreemptionPolicy("Never").Result() + if err := client.Create(ctx, repoMaintenancePriorityClass); err != nil { + fmt.Printf("Fail to create PriorityClass %s: %s\n", test.PriorityClassNameForRepoMaintenance, err.Error()) + return fmt.Errorf("fail to create PriorityClass %s: %w", test.PriorityClassNameForRepoMaintenance, err) + } + + return nil +} + +func DeletePriorityClasses(ctx context.Context, client kbclient.Client) error { + priorityClassDataMover := &schedulingv1api.PriorityClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: test.PriorityClassNameForDataMover, + }, + } + if err := client.Delete(ctx, priorityClassDataMover); err != nil { + return err + } + + priorityClassRepoMaintenance := &schedulingv1api.PriorityClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: test.PriorityClassNameForRepoMaintenance, + }, + } + if err := client.Delete(ctx, priorityClassRepoMaintenance); err != nil { + return err + } + + return nil +}