Add E2E test cases for repository maintenance job configuration.

Signed-off-by: Xun Jiang <xun.jiang@broadcom.com>

pkg/builder/priority_class_builder.go

@@ -0,0 +1,52 @@
+/*
+Copyright 2019 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package builder
+
+import (
+	corev1api "k8s.io/api/core/v1"
+	schedulingv1api "k8s.io/api/scheduling/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type PriorityClassBuilder struct {
+	object *schedulingv1api.PriorityClass
+}
+
+func ForPriorityClass(name string) *PriorityClassBuilder {
+	return &PriorityClassBuilder{
+		object: &schedulingv1api.PriorityClass{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: name,
+			},
+		},
+	}
+}
+
+func (p *PriorityClassBuilder) Value(value int) *PriorityClassBuilder {
+	p.object.Value = int32(value)
+	return p
+}
+
+func (p *PriorityClassBuilder) PreemptionPolicy(policy string) *PriorityClassBuilder {
+	preemptionPolicy := corev1api.PreemptionPolicy(policy)
+	p.object.PreemptionPolicy = &preemptionPolicy
+	return p
+}
+
+func (p *PriorityClassBuilder) Result() *schedulingv1api.PriorityClass {
+	return p.object
+}

test/e2e/e2e_suite_test.go

@@ -43,6 +43,7 @@ import (
 	. "github.com/vmware-tanzu/velero/test/e2e/parallelfilesupload"
 	. "github.com/vmware-tanzu/velero/test/e2e/privilegesmgmt"
 	. "github.com/vmware-tanzu/velero/test/e2e/pv-backup"
+	. "github.com/vmware-tanzu/velero/test/e2e/repomaintenance"
 	. "github.com/vmware-tanzu/velero/test/e2e/resource-filtering"
 	. "github.com/vmware-tanzu/velero/test/e2e/resourcemodifiers"
 	. "github.com/vmware-tanzu/velero/test/e2e/resourcepolicies"
@@ -660,6 +661,18 @@ var _ = Describe(
 	ParallelFilesDownloadTest,
 )
 
+var _ = Describe(
+	"Test Repository Maintenance Job Configuration's global part",
+	Label("RepoMaintenance", "LongTime"),
+	GlobalRepoMaintenanceTest,
+)
+
+var _ = Describe(
+	"Test Repository Maintenance Job Configuration's specific part",
+	Label("RepoMaintenance", "LongTime"),
+	SpecificRepoMaintenanceTest,
+)
+
 func GetKubeConfigContext() error {
 	var err error
 	var tcDefault, tcStandby k8s.TestClient
@@ -740,6 +753,12 @@ var _ = BeforeSuite(func() {
 		).To(Succeed())
 	}
 
+	// Create the needed PriorityClasses
+	Expect(veleroutil.CreatePriorityClasses(
+		context.Background(),
+		test.VeleroCfg.ClientToInstallVelero.Kubebuilder,
+	)).To(Succeed())
+
 	if test.InstallVelero {
 		By("Install test resources before testing")
 		Expect(
@@ -783,6 +802,11 @@ var _ = AfterSuite(func() {
 		).To(Succeed())
 	}
 
+	Expect(veleroutil.DeletePriorityClasses(
+		ctx,
+		test.VeleroCfg.ClientToInstallVelero.Kubebuilder,
+	)).To(Succeed())
+
 	// If the Velero is installed during test, and the FailFast is not enabled,
 	// uninstall Velero. If not, either Velero is not installed, or kept it for debug on failure.
 	if test.InstallVelero && (testSuitePassed || !test.VeleroCfg.FailFast) {

test/e2e/migration/migration.go

@@ -342,6 +342,12 @@ func (m *migrationE2E) Restore() error {
 		Expect(veleroutil.InstallStorageClasses(
 			m.VeleroCfg.StandbyClusterCloudProvider)).To(Succeed())
 
+		// Create the needed PriorityClasses
+		Expect(veleroutil.CreatePriorityClasses(
+			context.Background(),
+			test.VeleroCfg.StandbyClient.Kubebuilder,
+		)).To(Succeed())
+
 		if strings.EqualFold(m.VeleroCfg.Features, test.FeatureCSI) &&
 			m.VeleroCfg.UseVolumeSnapshots {
 			By("Install VolumeSnapshotClass for E2E.")

test/e2e/repomaintenance/repo_maintenance_config.go

@@ -0,0 +1,247 @@
+/*
+Copyright 2021 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package repomaintenance
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	. "github.com/onsi/gomega"
+	"github.com/pkg/errors"
+	batchv1api "k8s.io/api/batch/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+	"github.com/vmware-tanzu/velero/pkg/builder"
+	velerotypes "github.com/vmware-tanzu/velero/pkg/types"
+	"github.com/vmware-tanzu/velero/pkg/util/kube"
+	velerokubeutil "github.com/vmware-tanzu/velero/pkg/util/kube"
+	"github.com/vmware-tanzu/velero/test"
+	. "github.com/vmware-tanzu/velero/test/e2e/test"
+	k8sutil "github.com/vmware-tanzu/velero/test/util/k8s"
+	veleroutil "github.com/vmware-tanzu/velero/test/util/velero"
+)
+
+type RepoMaintenanceTestCase struct {
+	TestCase
+	repoMaintenanceConfigMapName string
+	repoMaintenanceConfigKey     string
+	jobConfigs                   velerotypes.JobConfigs
+}
+
+var keepJobNum = 1
+
+var GlobalRepoMaintenanceTest func() = TestFunc(&RepoMaintenanceTestCase{
+	repoMaintenanceConfigKey:     "global",
+	repoMaintenanceConfigMapName: "global",
+	jobConfigs: velerotypes.JobConfigs{
+		KeepLatestMaintenanceJobs: &keepJobNum,
+		PodResources: &velerokubeutil.PodResources{
+			CPURequest:    "100m",
+			MemoryRequest: "100Mi",
+			CPULimit:      "200m",
+			MemoryLimit:   "200Mi",
+		},
+		PriorityClassName: test.PriorityClassNameForRepoMaintenance,
+	},
+})
+
+var SpecificRepoMaintenanceTest func() = TestFunc(&RepoMaintenanceTestCase{
+	repoMaintenanceConfigKey:     "",
+	repoMaintenanceConfigMapName: "specific",
+	jobConfigs: velerotypes.JobConfigs{
+		KeepLatestMaintenanceJobs: &keepJobNum,
+		PodResources: &velerokubeutil.PodResources{
+			CPURequest:    "100m",
+			MemoryRequest: "100Mi",
+			CPULimit:      "200m",
+			MemoryLimit:   "200Mi",
+		},
+		PriorityClassName: test.PriorityClassNameForRepoMaintenance,
+	},
+})
+
+func (r *RepoMaintenanceTestCase) Init() error {
+	// generate random number as UUIDgen and set one default timeout duration
+	r.TestCase.Init()
+
+	// generate variable names based on CaseBaseName + UUIDgen
+	r.CaseBaseName = "repo-maintenance-" + r.UUIDgen
+	r.BackupName = "backup-" + r.CaseBaseName
+	r.RestoreName = "restore-" + r.CaseBaseName
+
+	// generate namespaces by NamespacesTotal
+	r.NamespacesTotal = 1
+	r.NSIncluded = &[]string{}
+	for nsNum := 0; nsNum < r.NamespacesTotal; nsNum++ {
+		createNSName := fmt.Sprintf("%s-%00000d", r.CaseBaseName, nsNum)
+		*r.NSIncluded = append(*r.NSIncluded, createNSName)
+	}
+
+	// If repoMaintenanceConfigKey is not set, it means testing the specific repo case.
+	// Need to assemble the BackupRepository name. The format is "volumeNamespace-bslName-uploaderName"
+	if r.repoMaintenanceConfigKey == "" {
+		r.repoMaintenanceConfigKey = (*r.NSIncluded)[0] + "-" + "default" + "-" + test.UploaderTypeKopia
+	}
+
+	// assign values to the inner variable for specific case
+	r.VeleroCfg.UseNodeAgent = true
+	r.VeleroCfg.UseNodeAgentWindows = true
+
+	r.BackupArgs = []string{
+		"create", "--namespace", r.VeleroCfg.VeleroNamespace, "backup", r.BackupName,
+		"--include-namespaces", strings.Join(*r.NSIncluded, ","),
+		"--snapshot-volumes=true", "--snapshot-move-data", "--wait",
+	}
+
+	// Message output by ginkgo
+	r.TestMsg = &TestMSG{
+		Desc:      "Validate Repository Maintenance Job configuration",
+		FailedMSG: "Failed to apply and / or validate configuration in repository maintenance jobs.",
+		Text:      "Should be able to apply and validate configuration in repository maintenance jobs.",
+	}
+	return nil
+}
+
+func (r *RepoMaintenanceTestCase) InstallVelero() error {
+	// Because this test needs to use customized repository maintenance ConfigMap,
+	// need to uninstall and reinstall Velero.
+
+	fmt.Println("Start to uninstall Velero")
+	if err := veleroutil.VeleroUninstall(r.Ctx, r.VeleroCfg); err != nil {
+		fmt.Printf("Fail to uninstall Velero: %s\n", err.Error())
+		return err
+	}
+
+	result, err := json.Marshal(r.jobConfigs)
+	if err != nil {
+		return err
+	}
+
+	repoMaintenanceConfig := builder.ForConfigMap(r.VeleroCfg.VeleroNamespace, r.repoMaintenanceConfigMapName).
+		Data(r.repoMaintenanceConfigKey, string(result)).Result()
+
+	r.VeleroCfg.RepoMaintenanceJobConfigMap = r.repoMaintenanceConfigMapName
+
+	return veleroutil.PrepareVelero(
+		r.Ctx,
+		r.CaseBaseName,
+		r.VeleroCfg,
+		repoMaintenanceConfig,
+	)
+}
+
+func (r *RepoMaintenanceTestCase) CreateResources() error {
+	for _, ns := range *r.NSIncluded {
+		if err := k8sutil.CreateNamespace(r.Ctx, r.Client, ns); err != nil {
+			fmt.Printf("Fail to create ns %s: %s\n", ns, err.Error())
+			return err
+		}
+
+		pvc, err := k8sutil.CreatePVC(r.Client, ns, "volume-1", test.StorageClassName, nil)
+		if err != nil {
+			fmt.Printf("Fail to create PVC %s: %s\n", "volume-1", err.Error())
+			return err
+		}
+
+		vols := k8sutil.CreateVolumes(pvc.Name, []string{"volume-1"})
+
+		deployment := k8sutil.NewDeployment(
+			r.CaseBaseName,
+			(*r.NSIncluded)[0],
+			1,
+			map[string]string{"app": "test"},
+			r.VeleroCfg.ImageRegistryProxy,
+			r.VeleroCfg.WorkerOS,
+		).WithVolume(vols).Result()
+
+		deployment, err = k8sutil.CreateDeployment(r.Client.ClientGo, ns, deployment)
+		if err != nil {
+			fmt.Printf("Fail to create deployment %s: %s \n", deployment.Name, err.Error())
+			return errors.Wrap(err, fmt.Sprintf("failed to create deployment: %s", err.Error()))
+		}
+
+		if err := k8sutil.WaitForReadyDeployment(r.Client.ClientGo, deployment.Namespace, deployment.Name); err != nil {
+			fmt.Printf("Fail to create deployment %s: %s\n", r.CaseBaseName, err.Error())
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (r *RepoMaintenanceTestCase) Verify() error {
+	// Reduce the MaintenanceFrequency to 1 minute.
+	backupRepositoryList := new(velerov1api.BackupRepositoryList)
+	if err := r.VeleroCfg.ClientToInstallVelero.Kubebuilder.List(
+		r.Ctx,
+		backupRepositoryList,
+		&client.ListOptions{
+			Namespace:     r.VeleroCfg.Namespace,
+			LabelSelector: labels.SelectorFromSet(map[string]string{velerov1api.VolumeNamespaceLabel: (*r.NSIncluded)[0]}),
+		},
+	); err != nil {
+		return err
+	}
+
+	if len(backupRepositoryList.Items) <= 0 {
+		return fmt.Errorf("fail list BackupRepository. no item is returned")
+	}
+
+	backupRepository := backupRepositoryList.Items[0]
+
+	updated := backupRepository.DeepCopy()
+	updated.Spec.MaintenanceFrequency = metav1.Duration{Duration: time.Minute}
+	if err := r.VeleroCfg.ClientToInstallVelero.Kubebuilder.Patch(r.Ctx, updated, client.MergeFrom(&backupRepository)); err != nil {
+		fmt.Printf("failed to patch BackupRepository %q: %s", backupRepository.GetName(), err.Error())
+		return err
+	}
+
+	// The minimal time unit of Repository Maintenance is 5 minutes.
+	// Wait for more than one cycles to make sure the result is valid.
+	time.Sleep(6 * time.Minute)
+
+	jobList := new(batchv1api.JobList)
+	if err := r.VeleroCfg.ClientToInstallVelero.Kubebuilder.List(r.Ctx, jobList, &client.ListOptions{
+		Namespace:     r.VeleroCfg.Namespace,
+		LabelSelector: labels.SelectorFromSet(map[string]string{"velero.io/repo-name": backupRepository.Name}),
+	}); err != nil {
+		return nil
+	}
+
+	resources, err := kube.ParseResourceRequirements(
+		r.jobConfigs.PodResources.CPURequest,
+		r.jobConfigs.PodResources.MemoryRequest,
+		r.jobConfigs.PodResources.CPULimit,
+		r.jobConfigs.PodResources.MemoryLimit,
+	)
+	if err != nil {
+		return errors.Wrap(err, "failed to parse resource requirements for maintenance job")
+	}
+
+	Expect(jobList.Items[0].Spec.Template.Spec.Containers[0].Resources).To(Equal(resources))
+
+	Expect(jobList.Items).To(HaveLen(*r.jobConfigs.KeepLatestMaintenanceJobs))
+
+	Expect(jobList.Items[0].Spec.Template.Spec.PriorityClassName).To(Equal(r.jobConfigs.PriorityClassName))
+
+	return nil
+}

test/e2e/test/test.go

@@ -41,6 +41,7 @@ depends on your test patterns.
 */
 type VeleroBackupRestoreTest interface {
 	Init() error
+	InstallVelero() error
 	CreateResources() error
 	Backup() error
 	Destroy() error
@@ -109,6 +110,10 @@ func (t *TestCase) GenerateUUID() string {
 	return fmt.Sprintf("%08d", rand.IntN(100000000))
 }
 
+func (t *TestCase) InstallVelero() error {
+	return PrepareVelero(context.Background(), t.GetTestCase().CaseBaseName, t.GetTestCase().VeleroCfg)
+}
+
 func (t *TestCase) CreateResources() error {
 	return nil
 }
@@ -221,7 +226,8 @@ func RunTestCase(test VeleroBackupRestoreTest) error {
 	fmt.Printf("Running test case %s %s\n", test.GetTestMsg().Desc, time.Now().Format("2006-01-02 15:04:05"))
 
 	if InstallVelero {
-		Expect(PrepareVelero(context.Background(), test.GetTestCase().CaseBaseName, test.GetTestCase().VeleroCfg)).To(Succeed())
+		fmt.Printf("Install Velero for test case %s: %s", test.GetTestCase().CaseBaseName, time.Now().Format("2006-01-02 15:04:05"))
+		Expect(test.InstallVelero()).To(Succeed())
 	}
 
 	defer test.Clean()

test/types.go

@@ -57,6 +57,11 @@ const (
 	BackupRepositoryConfigName    = "backup-repository-config"
 )
 
+const (
+	PriorityClassNameForDataMover       = "data-mover"
+	PriorityClassNameForRepoMaintenance = "repo-maintenance"
+)
+
 var PublicCloudProviders = []string{AWS, Azure, GCP, Vsphere}
 var LocalCloudProviders = []string{Kind, VanillaZFS}
 var CloudProviders = append(PublicCloudProviders, LocalCloudProviders...)

test/util/velero/install.go

@@ -36,6 +36,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/cmd/cli/install"
@@ -56,7 +57,17 @@ type installOptions struct {
 	WorkerOS                         string
 }
 
-func VeleroInstall(ctx context.Context, veleroCfg *test.VeleroConfig, isStandbyCluster bool) error {
+/*
+VeleroInstall is used to install Velero for E2E test
+
+params:
+
+	ctx:              The context
+	veleroCfg:        Velero E2E case configuration
+	isStandbyCluster: Whether Velero is installed on standby cluster
+	objects:          The objects are installed in Velero installed namespace, e.g. the ConfigMaps.
+*/
+func VeleroInstall(ctx context.Context, veleroCfg *test.VeleroConfig, isStandbyCluster bool, objects ...client.Object) error {
 	fmt.Printf("Velero install %s\n", time.Now().Format("2006-01-02 15:04:05"))
 
 	// veleroCfg struct including a set of BSL params and a set of additional BSL params,
@@ -153,6 +164,14 @@ func VeleroInstall(ctx context.Context, veleroCfg *test.VeleroConfig, isStandbyC
 		)
 	}
 
+	// Install the passed-in objects in Velero installed namespace
+	for _, obj := range objects {
+		if err := veleroCfg.ClientToInstallVelero.Kubebuilder.Create(ctx, obj); err != nil {
+			fmt.Printf("fail to create object %s in namespace %s: %s\n", obj.GetName(), obj.GetNamespace(), err.Error())
+			return fmt.Errorf("fail to create object %s in namespace %s: %w", obj.GetName(), obj.GetNamespace(), err)
+		}
+	}
+
 	// For AWS IRSA credential test, AWS IAM service account is required, so if ServiceAccountName and EKSPolicyARN
 	// are both provided, we assume IRSA test is running, otherwise skip this IAM service account creation part.
 	if veleroCfg.CloudProvider == test.AWS && veleroInstallOptions.ServiceAccountName != "" {
@@ -790,7 +809,7 @@ func CheckBSL(ctx context.Context, ns string, bslName string) error {
 	return err
 }
 
-func PrepareVelero(ctx context.Context, caseName string, veleroCfg test.VeleroConfig) error {
+func PrepareVelero(ctx context.Context, caseName string, veleroCfg test.VeleroConfig, objects ...client.Object) error {
 	ready, err := IsVeleroReady(context.Background(), &veleroCfg)
 	if err != nil {
 		fmt.Printf("error in checking velero status with %v", err)
@@ -804,7 +823,7 @@ func PrepareVelero(ctx context.Context, caseName string, veleroCfg test.VeleroCo
 		return nil
 	}
 	fmt.Printf("need to install velero for case %s \n", caseName)
-	return VeleroInstall(context.Background(), &veleroCfg, false)
+	return VeleroInstall(context.Background(), &veleroCfg, false, objects...)
 }
 
 func VeleroUninstall(ctx context.Context, veleroCfg test.VeleroConfig) error {

test/util/velero/velero_utils.go

@@ -38,6 +38,8 @@ import (
 
 	"github.com/pkg/errors"
 	"golang.org/x/mod/semver"
+	schedulingv1api "k8s.io/api/scheduling/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	ver "k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -45,9 +47,11 @@ import (
 
 	"github.com/vmware-tanzu/velero/internal/volume"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+	"github.com/vmware-tanzu/velero/pkg/builder"
 	cliinstall "github.com/vmware-tanzu/velero/pkg/cmd/cli/install"
 	"github.com/vmware-tanzu/velero/pkg/cmd/util/flag"
 	veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
+	"github.com/vmware-tanzu/velero/test"
 	. "github.com/vmware-tanzu/velero/test"
 	common "github.com/vmware-tanzu/velero/test/util/common"
 	. "github.com/vmware-tanzu/velero/test/util/k8s"
@@ -274,6 +278,9 @@ func getProviderVeleroInstallOptions(veleroCfg *VeleroConfig,
 	io.ItemBlockWorkerCount = veleroCfg.ItemBlockWorkerCount
 	io.ServerPriorityClassName = veleroCfg.ServerPriorityClassName
 	io.NodeAgentPriorityClassName = veleroCfg.NodeAgentPriorityClassName
+	io.RepoMaintenanceJobConfigMap = veleroCfg.RepoMaintenanceJobConfigMap
+	io.BackupRepoConfigMap = veleroCfg.BackupRepoConfigMap
+	io.NodeAgentConfigMap = veleroCfg.NodeAgentConfigMap
 
 	return io, nil
 }
@@ -1812,3 +1819,43 @@ func KubectlGetAllDeleteBackupRequest(ctx context.Context, backupName, veleroNam
 
 	return common.GetListByCmdPipes(ctx, cmds)
 }
+
+func CreatePriorityClasses(ctx context.Context, client kbclient.Client) error {
+	dataMoverPriorityClass := builder.ForPriorityClass(test.PriorityClassNameForDataMover).
+		Value(90000).PreemptionPolicy("Never").Result()
+	if err := client.Create(ctx, dataMoverPriorityClass); err != nil {
+		fmt.Printf("Fail to create PriorityClass %s: %s\n", test.PriorityClassNameForDataMover, err.Error())
+		return fmt.Errorf("fail to create PriorityClass %s: %w", test.PriorityClassNameForDataMover, err)
+	}
+
+	repoMaintenancePriorityClass := builder.ForPriorityClass(test.PriorityClassNameForRepoMaintenance).
+		Value(80000).PreemptionPolicy("Never").Result()
+	if err := client.Create(ctx, repoMaintenancePriorityClass); err != nil {
+		fmt.Printf("Fail to create PriorityClass %s: %s\n", test.PriorityClassNameForRepoMaintenance, err.Error())
+		return fmt.Errorf("fail to create PriorityClass %s: %w", test.PriorityClassNameForRepoMaintenance, err)
+	}
+
+	return nil
+}
+
+func DeletePriorityClasses(ctx context.Context, client kbclient.Client) error {
+	priorityClassDataMover := &schedulingv1api.PriorityClass{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: test.PriorityClassNameForDataMover,
+		},
+	}
+	if err := client.Delete(ctx, priorityClassDataMover); err != nil {
+		return err
+	}
+
+	priorityClassRepoMaintenance := &schedulingv1api.PriorityClass{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: test.PriorityClassNameForRepoMaintenance,
+		},
+	}
+	if err := client.Delete(ctx, priorityClassRepoMaintenance); err != nil {
+		return err
+	}
+
+	return nil
+}