/* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package maintenance import ( "context" "fmt" "math" "strings" "testing" "time" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" batchv1api "k8s.io/api/batch/v1" corev1api "k8s.io/api/core/v1" schedulingv1 "k8s.io/api/scheduling/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" velerolabel "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/repository/provider" velerotest "github.com/vmware-tanzu/velero/pkg/test" velerotypes "github.com/vmware-tanzu/velero/pkg/types" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/logging" ) func TestGenerateJobName(t *testing.T) { testCases := []struct { repo string expectedStart string }{ { repo: "example", expectedStart: "example-maintain-job-", }, { repo: strings.Repeat("a", 60), expectedStart: "repo-maintain-job-", }, } for _, tc := range testCases { t.Run(tc.repo, func(t *testing.T) { // Call the function to test jobName := GenerateJobName(tc.repo) // Check if the generated job name starts with the expected prefix if !strings.HasPrefix(jobName, tc.expectedStart) { t.Errorf("generated job name does not start with expected prefix") } // Check if the length of the generated job name exceeds the Kubernetes limit if len(jobName) > 63 { t.Errorf("generated job name exceeds Kubernetes limit") } }) } } func TestDeleteOldJobs(t *testing.T) { // Set up test repo and keep value repo := &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Name: "label with more than 63 characters should be modified", Namespace: velerov1api.DefaultNamespace, }, } keep := 1 jobArray := []client.Object{ &batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "job-0", Namespace: velerov1api.DefaultNamespace, Labels: map[string]string{RepositoryNameLabel: velerolabel.ReturnNameOrHash(repo.Name)}, }, Spec: batchv1api.JobSpec{}, }, &batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "job-1", Namespace: velerov1api.DefaultNamespace, Labels: map[string]string{RepositoryNameLabel: velerolabel.ReturnNameOrHash(repo.Name)}, }, Spec: batchv1api.JobSpec{}, }, } newJob := &batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "job-new", Namespace: velerov1api.DefaultNamespace, Labels: map[string]string{RepositoryNameLabel: velerolabel.ReturnNameOrHash(repo.Name)}, }, Spec: batchv1api.JobSpec{}, } // Create a fake Kubernetes client with 2 jobs. scheme := runtime.NewScheme() _ = batchv1api.AddToScheme(scheme) cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(jobArray...).Build() // Create a new job require.NoError(t, cli.Create(t.Context(), newJob)) // Call the function require.NoError(t, DeleteOldJobs(cli, *repo, keep, velerotest.NewLogger())) // Get the remaining jobs jobList := &batchv1api.JobList{} require.NoError(t, cli.List(t.Context(), jobList, client.MatchingLabels(map[string]string{RepositoryNameLabel: repo.Name}))) // We expect the number of jobs to be equal to 'keep' assert.Len(t, jobList.Items, keep) // Only the new created job should be left. assert.Equal(t, jobList.Items[0].Name, newJob.Name) } func TestWaitForJobComplete(t *testing.T) { // Set up test job job := &batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "test-job", Namespace: "default", }, Status: batchv1api.JobStatus{}, } schemeFail := runtime.NewScheme() scheme := runtime.NewScheme() batchv1api.AddToScheme(scheme) waitCompletionBackOff1 := wait.Backoff{ Duration: time.Second, Steps: math.MaxInt, Factor: 2, Cap: time.Second * 12, } waitCompletionBackOff2 := wait.Backoff{ Duration: time.Second, Steps: math.MaxInt, Factor: 2, Cap: time.Second * 2, } // Define test cases tests := []struct { description string // Test case description kubeClientObj []runtime.Object runtimeScheme *runtime.Scheme jobStatus batchv1api.JobStatus // Job status to set for the test logBackOff wait.Backoff updateAfter time.Duration expectedLogs int expectError bool // Whether an error is expected }{ { description: "wait error", runtimeScheme: schemeFail, expectError: true, }, { description: "Job Succeeded", runtimeScheme: scheme, kubeClientObj: []runtime.Object{ job, }, jobStatus: batchv1api.JobStatus{ Succeeded: 1, }, expectError: false, }, { description: "Job Failed", runtimeScheme: scheme, kubeClientObj: []runtime.Object{ job, }, jobStatus: batchv1api.JobStatus{ Failed: 1, }, expectError: false, }, { description: "Log backoff not to cap", runtimeScheme: scheme, kubeClientObj: []runtime.Object{ job, }, logBackOff: waitCompletionBackOff1, updateAfter: time.Second * 8, expectedLogs: 3, }, { description: "Log backoff to cap", runtimeScheme: scheme, kubeClientObj: []runtime.Object{ job, }, logBackOff: waitCompletionBackOff2, updateAfter: time.Second * 6, expectedLogs: 3, }, } // Run tests for _, tc := range tests { t.Run(tc.description, func(t *testing.T) { // Set the job status job.Status = tc.jobStatus // Create a fake Kubernetes client fakeClientBuilder := fake.NewClientBuilder() fakeClientBuilder = fakeClientBuilder.WithScheme(tc.runtimeScheme) fakeClient := fakeClientBuilder.WithRuntimeObjects(tc.kubeClientObj...).Build() buffer := []string{} logger := velerotest.NewMultipleLogger(&buffer) waitCompletionBackOff = tc.logBackOff if tc.updateAfter != 0 { go func() { time.Sleep(tc.updateAfter) original := job.DeepCopy() job.Status.Succeeded = 1 err := fakeClient.Status().Patch(t.Context(), job, client.MergeFrom(original)) require.NoError(t, err) }() } // Call the function _, err := waitForJobComplete(t.Context(), fakeClient, job.Namespace, job.Name, logger) // Check if the error matches the expectation if tc.expectError { require.Error(t, err) } else { require.NoError(t, err) } assert.LessOrEqual(t, len(buffer), tc.expectedLogs) }) } } func TestGetResultFromJob(t *testing.T) { // Set up test job job := &batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "test-job", Namespace: "default", }, } // Set up test pod with no status pod := &corev1api.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pod", Namespace: "default", Labels: map[string]string{"job-name": job.Name}, }, } // Create a fake Kubernetes client cli := fake.NewClientBuilder().Build() // test an error should be returned result, err := getResultFromJob(cli, job) require.EqualError(t, err, "no pod found for job test-job") assert.Empty(t, result) cli = fake.NewClientBuilder().WithObjects(job, pod).Build() // test an error should be returned result, err = getResultFromJob(cli, job) require.EqualError(t, err, "no container statuses found for job test-job") assert.Empty(t, result) // Set a non-terminated container status to the pod pod.Status = corev1api.PodStatus{ ContainerStatuses: []corev1api.ContainerStatus{ { State: corev1api.ContainerState{}, }, }, } // Test an error should be returned cli = fake.NewClientBuilder().WithObjects(job, pod).Build() result, err = getResultFromJob(cli, job) require.EqualError(t, err, "container for job test-job is not terminated") assert.Empty(t, result) // Set a terminated container status to the pod pod.Status = corev1api.PodStatus{ ContainerStatuses: []corev1api.ContainerStatus{ { State: corev1api.ContainerState{ Terminated: &corev1api.ContainerStateTerminated{}, }, }, }, } // This call should return the termination message with no error cli = fake.NewClientBuilder().WithObjects(job, pod).Build() result, err = getResultFromJob(cli, job) require.NoError(t, err) assert.Empty(t, result) // Set a terminated container status with invalidate message to the pod pod.Status = corev1api.PodStatus{ ContainerStatuses: []corev1api.ContainerStatus{ { State: corev1api.ContainerState{ Terminated: &corev1api.ContainerStateTerminated{ Message: "fake-message", }, }, }, }, } cli = fake.NewClientBuilder().WithObjects(job, pod).Build() result, err = getResultFromJob(cli, job) require.EqualError(t, err, "error to locate repo maintenance error indicator from termination message") assert.Empty(t, result) // Set a terminated container status with empty maintenance error to the pod pod.Status = corev1api.PodStatus{ ContainerStatuses: []corev1api.ContainerStatus{ { State: corev1api.ContainerState{ Terminated: &corev1api.ContainerStateTerminated{ Message: "Repo maintenance error: ", }, }, }, }, } cli = fake.NewClientBuilder().WithObjects(job, pod).Build() result, err = getResultFromJob(cli, job) require.EqualError(t, err, "nothing after repo maintenance error indicator in termination message") assert.Empty(t, result) // Set a terminated container status with maintenance error to the pod pod.Status = corev1api.PodStatus{ ContainerStatuses: []corev1api.ContainerStatus{ { State: corev1api.ContainerState{ Terminated: &corev1api.ContainerStateTerminated{ Message: "Repo maintenance error: fake-error", }, }, }, }, } cli = fake.NewClientBuilder().WithObjects(job, pod).Build() result, err = getResultFromJob(cli, job) require.NoError(t, err) assert.Equal(t, "fake-error", result) } func TestGetJobConfig(t *testing.T) { keepLatestMaintenanceJobs := 1 ctx := t.Context() logger := logrus.New() veleroNamespace := "velero" repoMaintenanceJobConfig := "repo-maintenance-job-config" repo := &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: veleroNamespace, Name: repoMaintenanceJobConfig, }, Spec: velerov1api.BackupRepositorySpec{ BackupStorageLocation: "default", RepositoryType: "kopia", VolumeNamespace: "test", }, } testCases := []struct { name string repoJobConfig *corev1api.ConfigMap expectedConfig *velerotypes.JobConfigs expectedError error }{ { name: "Config not exist", expectedConfig: nil, expectedError: nil, }, { name: "Invalid JSON", repoJobConfig: &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: veleroNamespace, Name: repoMaintenanceJobConfig, }, Data: map[string]string{ "test-default-kopia": "{\"cpuRequest:\"100m\"}", }, }, expectedConfig: nil, expectedError: fmt.Errorf("fail to unmarshal configs from %s", repoMaintenanceJobConfig), }, { name: "Find config specific for BackupRepository", repoJobConfig: &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: veleroNamespace, Name: repoMaintenanceJobConfig, }, Data: map[string]string{ "test-default-kopia": "{\"podResources\":{\"cpuRequest\":\"100m\",\"cpuLimit\":\"200m\",\"memoryRequest\":\"100Mi\",\"memoryLimit\":\"200Mi\"},\"loadAffinity\":[{\"nodeSelector\":{\"matchExpressions\":[{\"key\":\"cloud.google.com/machine-family\",\"operator\":\"In\",\"values\":[\"e2\"]}]}}]}", }, }, expectedConfig: &velerotypes.JobConfigs{ PodResources: &kube.PodResources{ CPURequest: "100m", CPULimit: "200m", MemoryRequest: "100Mi", MemoryLimit: "200Mi", }, LoadAffinities: []*kube.LoadAffinity{ { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "cloud.google.com/machine-family", Operator: metav1.LabelSelectorOpIn, Values: []string{"e2"}, }, }, }, }, }, }, expectedError: nil, }, { name: "Find config specific for global", repoJobConfig: &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: veleroNamespace, Name: repoMaintenanceJobConfig, }, Data: map[string]string{ GlobalKeyForRepoMaintenanceJobCM: "{\"podResources\":{\"cpuRequest\":\"50m\",\"cpuLimit\":\"100m\",\"memoryRequest\":\"50Mi\",\"memoryLimit\":\"100Mi\"},\"loadAffinity\":[{\"nodeSelector\":{\"matchExpressions\":[{\"key\":\"cloud.google.com/machine-family\",\"operator\":\"In\",\"values\":[\"n2\"]}]}}]}", }, }, expectedConfig: &velerotypes.JobConfigs{ PodResources: &kube.PodResources{ CPURequest: "50m", CPULimit: "100m", MemoryRequest: "50Mi", MemoryLimit: "100Mi", }, LoadAffinities: []*kube.LoadAffinity{ { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "cloud.google.com/machine-family", Operator: metav1.LabelSelectorOpIn, Values: []string{"n2"}, }, }, }, }, }, }, expectedError: nil, }, { name: "Specific config supersede global config", repoJobConfig: &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: veleroNamespace, Name: repoMaintenanceJobConfig, }, Data: map[string]string{ GlobalKeyForRepoMaintenanceJobCM: "{\"keepLatestMaintenanceJobs\":1,\"podResources\":{\"cpuRequest\":\"50m\",\"cpuLimit\":\"100m\",\"memoryRequest\":\"50Mi\",\"memoryLimit\":\"100Mi\"},\"loadAffinity\":[{\"nodeSelector\":{\"matchExpressions\":[{\"key\":\"cloud.google.com/machine-family\",\"operator\":\"In\",\"values\":[\"n2\"]}]}}]}", "test-default-kopia": "{\"podResources\":{\"cpuRequest\":\"100m\",\"cpuLimit\":\"200m\",\"memoryRequest\":\"100Mi\",\"memoryLimit\":\"200Mi\"},\"loadAffinity\":[{\"nodeSelector\":{\"matchExpressions\":[{\"key\":\"cloud.google.com/machine-family\",\"operator\":\"In\",\"values\":[\"e2\"]}]}}]}", }, }, expectedConfig: &velerotypes.JobConfigs{ KeepLatestMaintenanceJobs: &keepLatestMaintenanceJobs, PodResources: &kube.PodResources{ CPURequest: "100m", CPULimit: "200m", MemoryRequest: "100Mi", MemoryLimit: "200Mi", }, LoadAffinities: []*kube.LoadAffinity{ { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "cloud.google.com/machine-family", Operator: metav1.LabelSelectorOpIn, Values: []string{"e2"}, }, }, }, }, }, }, expectedError: nil, }, { name: "Configs only exist in global section should supersede specific config", repoJobConfig: &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: veleroNamespace, Name: repoMaintenanceJobConfig, }, Data: map[string]string{ GlobalKeyForRepoMaintenanceJobCM: "{\"keepLatestMaintenanceJobs\":1,\"podResources\":{\"cpuRequest\":\"50m\",\"cpuLimit\":\"100m\",\"memoryRequest\":\"50Mi\",\"memoryLimit\":\"100Mi\"},\"loadAffinity\":[{\"nodeSelector\":{\"matchExpressions\":[{\"key\":\"cloud.google.com/machine-family\",\"operator\":\"In\",\"values\":[\"n2\"]}]}}],\"priorityClassName\":\"global-priority\",\"podAnnotations\":{\"global-key\":\"global-value\"},\"podLabels\":{\"global-key\":\"global-value\"},\"tolerations\":[{\"key\":\"global-taint\",\"operator\":\"Exists\",\"effect\":\"NoSchedule\"}]}", "test-default-kopia": "{\"podResources\":{\"cpuRequest\":\"100m\",\"cpuLimit\":\"200m\",\"memoryRequest\":\"100Mi\",\"memoryLimit\":\"200Mi\"},\"loadAffinity\":[{\"nodeSelector\":{\"matchExpressions\":[{\"key\":\"cloud.google.com/machine-family\",\"operator\":\"In\",\"values\":[\"e2\"]}]}}],\"priorityClassName\":\"specific-priority\",\"podAnnotations\":{\"specific-key\":\"specific-value\"},\"podLabels\":{\"specific-key\":\"specific-value\"},\"tolerations\":[{\"key\":\"specific-taint\",\"operator\":\"Equal\",\"value\":\"dedicated\",\"effect\":\"NoSchedule\"}]}", }, }, expectedConfig: &velerotypes.JobConfigs{ KeepLatestMaintenanceJobs: &keepLatestMaintenanceJobs, PodResources: &kube.PodResources{ CPURequest: "100m", CPULimit: "200m", MemoryRequest: "100Mi", MemoryLimit: "200Mi", }, LoadAffinities: []*kube.LoadAffinity{ { NodeSelector: metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{ { Key: "cloud.google.com/machine-family", Operator: metav1.LabelSelectorOpIn, Values: []string{"e2"}, }, }, }, }, }, PriorityClassName: "global-priority", PodAnnotations: map[string]string{"global-key": "global-value"}, PodLabels: map[string]string{"global-key": "global-value"}, Tolerations: []corev1api.Toleration{ { Key: "global-taint", Operator: corev1api.TolerationOpExists, Effect: corev1api.TaintEffectNoSchedule, }, }, }, expectedError: nil, }, { name: "Find config with tolerations in global section", repoJobConfig: &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: veleroNamespace, Name: repoMaintenanceJobConfig, }, Data: map[string]string{ GlobalKeyForRepoMaintenanceJobCM: "{\"tolerations\":[{\"key\":\"global-taint\",\"operator\":\"Exists\",\"effect\":\"NoSchedule\"}]}", }, }, expectedConfig: &velerotypes.JobConfigs{ Tolerations: []corev1api.Toleration{ { Key: "global-taint", Operator: corev1api.TolerationOpExists, Effect: corev1api.TaintEffectNoSchedule, }, }, }, expectedError: nil, }, { name: "Tolerations in specific config should be ignored", repoJobConfig: &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: veleroNamespace, Name: repoMaintenanceJobConfig, }, Data: map[string]string{ "test-default-kopia": "{\"podResources\":{\"cpuRequest\":\"100m\"},\"tolerations\":[{\"key\":\"specific-taint\",\"operator\":\"Equal\",\"value\":\"dedicated\",\"effect\":\"NoSchedule\"}]}", }, }, expectedConfig: &velerotypes.JobConfigs{ PodResources: &kube.PodResources{ CPURequest: "100m", }, }, expectedError: nil, }, { name: "Global tolerations apply when specific config does not specify global configs", repoJobConfig: &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: veleroNamespace, Name: repoMaintenanceJobConfig, }, Data: map[string]string{ GlobalKeyForRepoMaintenanceJobCM: "{\"tolerations\":[{\"key\":\"global-taint\",\"operator\":\"Exists\",\"effect\":\"NoSchedule\"}]}", "test-default-kopia": "{\"podResources\":{\"cpuRequest\":\"100m\"}}", }, }, expectedConfig: &velerotypes.JobConfigs{ PodResources: &kube.PodResources{ CPURequest: "100m", }, Tolerations: []corev1api.Toleration{ { Key: "global-taint", Operator: corev1api.TolerationOpExists, Effect: corev1api.TaintEffectNoSchedule, }, }, }, expectedError: nil, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var fakeClient client.Client if tc.repoJobConfig != nil { fakeClient = velerotest.NewFakeControllerRuntimeClient(t, tc.repoJobConfig) } else { fakeClient = velerotest.NewFakeControllerRuntimeClient(t) } jobConfig, err := getJobConfig( ctx, fakeClient, logger, veleroNamespace, repoMaintenanceJobConfig, repo, ) if tc.expectedError != nil { require.ErrorContains(t, err, tc.expectedError.Error()) } else { require.NoError(t, err) } require.Equal(t, tc.expectedConfig, jobConfig) }) } } func TestWaitAllJobsComplete(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), time.Second*2) veleroNamespace := "velero" repo := &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: veleroNamespace, Name: "label with more than 63 characters should be modified", }, Spec: velerov1api.BackupRepositorySpec{ BackupStorageLocation: "default", RepositoryType: "kopia", VolumeNamespace: "test", }, } now := time.Now().Round(time.Second) jobOtherLabel := &batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "job1", Namespace: veleroNamespace, Labels: map[string]string{RepositoryNameLabel: "other-repo"}, CreationTimestamp: metav1.Time{Time: now}, }, } jobIncomplete := &batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "job1", Namespace: veleroNamespace, Labels: map[string]string{RepositoryNameLabel: velerolabel.ReturnNameOrHash(repo.Name)}, CreationTimestamp: metav1.Time{Time: now}, }, } jobSucceeded1 := &batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "job1", Namespace: veleroNamespace, Labels: map[string]string{RepositoryNameLabel: velerolabel.ReturnNameOrHash(repo.Name)}, CreationTimestamp: metav1.Time{Time: now}, }, Status: batchv1api.JobStatus{ StartTime: &metav1.Time{Time: now}, CompletionTime: &metav1.Time{Time: now.Add(time.Hour)}, Succeeded: 1, }, } jobPodSucceeded1 := builder.ForPod(veleroNamespace, "job1").Labels(map[string]string{"job-name": "job1"}).ContainerStatuses(&corev1api.ContainerStatus{ State: corev1api.ContainerState{ Terminated: &corev1api.ContainerStateTerminated{}, }, }).Result() jobFailed1 := &batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "job2", Namespace: veleroNamespace, Labels: map[string]string{RepositoryNameLabel: velerolabel.ReturnNameOrHash(repo.Name)}, CreationTimestamp: metav1.Time{Time: now.Add(time.Hour)}, }, Status: batchv1api.JobStatus{ StartTime: &metav1.Time{Time: now.Add(time.Hour)}, Failed: 1, }, } jobPodFailed1 := builder.ForPod(veleroNamespace, "job2").Labels(map[string]string{"job-name": "job2"}).ContainerStatuses(&corev1api.ContainerStatus{ State: corev1api.ContainerState{ Terminated: &corev1api.ContainerStateTerminated{ Message: "Repo maintenance error: fake-message-2", }, }, }).Result() jobSucceeded2 := &batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "job3", Namespace: veleroNamespace, Labels: map[string]string{RepositoryNameLabel: velerolabel.ReturnNameOrHash(repo.Name)}, CreationTimestamp: metav1.Time{Time: now.Add(time.Hour * 2)}, }, Status: batchv1api.JobStatus{ StartTime: &metav1.Time{Time: now.Add(time.Hour * 2)}, CompletionTime: &metav1.Time{Time: now.Add(time.Hour * 3)}, Succeeded: 1, }, } jobPodSucceeded2 := builder.ForPod(veleroNamespace, "job3").Labels(map[string]string{"job-name": "job3"}).ContainerStatuses(&corev1api.ContainerStatus{ State: corev1api.ContainerState{ Terminated: &corev1api.ContainerStateTerminated{}, }, }).Result() jobSucceeded3 := &batchv1api.Job{ ObjectMeta: metav1.ObjectMeta{ Name: "job4", Namespace: veleroNamespace, Labels: map[string]string{RepositoryNameLabel: velerolabel.ReturnNameOrHash(repo.Name)}, CreationTimestamp: metav1.Time{Time: now.Add(time.Hour * 3)}, }, Status: batchv1api.JobStatus{ StartTime: &metav1.Time{Time: now.Add(time.Hour * 3)}, CompletionTime: &metav1.Time{Time: now.Add(time.Hour * 4)}, Succeeded: 1, }, } jobPodSucceeded3 := builder.ForPod(veleroNamespace, "job4").Labels(map[string]string{"job-name": "job4"}).ContainerStatuses(&corev1api.ContainerStatus{ State: corev1api.ContainerState{ Terminated: &corev1api.ContainerStateTerminated{}, }, }).Result() schemeFail := runtime.NewScheme() scheme := runtime.NewScheme() batchv1api.AddToScheme(scheme) corev1api.AddToScheme(scheme) testCases := []struct { name string ctx context.Context kubeClientObj []runtime.Object runtimeScheme *runtime.Scheme expectedStatus []velerov1api.BackupRepositoryMaintenanceStatus expectedError string }{ { name: "list job error", runtimeScheme: schemeFail, expectedError: "error listing maintenance job for repo label with more than 63 characters should be modified: no kind is registered for the type v1.JobList in scheme", }, { name: "job not exist", runtimeScheme: scheme, }, { name: "no matching job", runtimeScheme: scheme, kubeClientObj: []runtime.Object{ jobOtherLabel, }, }, { name: "wait complete error", ctx: ctx, runtimeScheme: scheme, kubeClientObj: []runtime.Object{ jobIncomplete, }, expectedError: "error waiting maintenance job[job1] complete: context deadline exceeded", }, { name: "get result error on succeeded job", ctx: t.Context(), runtimeScheme: scheme, kubeClientObj: []runtime.Object{ jobSucceeded1, }, expectedStatus: []velerov1api.BackupRepositoryMaintenanceStatus{ { Result: velerov1api.BackupRepositoryMaintenanceSucceeded, StartTimestamp: &metav1.Time{Time: now}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, }, }, }, { name: "get result error on failed job", ctx: t.Context(), runtimeScheme: scheme, kubeClientObj: []runtime.Object{ jobFailed1, }, expectedStatus: []velerov1api.BackupRepositoryMaintenanceStatus{ { Result: velerov1api.BackupRepositoryMaintenanceFailed, StartTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, Message: "Repo maintenance failed but result is not retrievable, err: no pod found for job job2", }, }, }, { name: "less than limit", ctx: t.Context(), runtimeScheme: scheme, kubeClientObj: []runtime.Object{ jobFailed1, jobSucceeded1, jobPodSucceeded1, jobPodFailed1, }, expectedStatus: []velerov1api.BackupRepositoryMaintenanceStatus{ { Result: velerov1api.BackupRepositoryMaintenanceSucceeded, StartTimestamp: &metav1.Time{Time: now}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, }, { Result: velerov1api.BackupRepositoryMaintenanceFailed, StartTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, Message: "fake-message-2", }, }, }, { name: "equal to limit", ctx: t.Context(), runtimeScheme: scheme, kubeClientObj: []runtime.Object{ jobSucceeded2, jobFailed1, jobSucceeded1, jobPodSucceeded1, jobPodFailed1, jobPodSucceeded2, }, expectedStatus: []velerov1api.BackupRepositoryMaintenanceStatus{ { Result: velerov1api.BackupRepositoryMaintenanceSucceeded, StartTimestamp: &metav1.Time{Time: now}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, }, { Result: velerov1api.BackupRepositoryMaintenanceFailed, StartTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, Message: "fake-message-2", }, { Result: velerov1api.BackupRepositoryMaintenanceSucceeded, StartTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 3)}, }, }, }, { name: "more than limit", ctx: t.Context(), runtimeScheme: scheme, kubeClientObj: []runtime.Object{ jobSucceeded3, jobSucceeded2, jobFailed1, jobSucceeded1, jobPodSucceeded1, jobPodFailed1, jobPodSucceeded2, jobPodSucceeded3, }, expectedStatus: []velerov1api.BackupRepositoryMaintenanceStatus{ { Result: velerov1api.BackupRepositoryMaintenanceFailed, StartTimestamp: &metav1.Time{Time: now.Add(time.Hour)}, Message: "fake-message-2", }, { Result: velerov1api.BackupRepositoryMaintenanceSucceeded, StartTimestamp: &metav1.Time{Time: now.Add(time.Hour * 2)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 3)}, }, { Result: velerov1api.BackupRepositoryMaintenanceSucceeded, StartTimestamp: &metav1.Time{Time: now.Add(time.Hour * 3)}, CompleteTimestamp: &metav1.Time{Time: now.Add(time.Hour * 4)}, }, }, }, } for _, test := range testCases { t.Run(test.name, func(t *testing.T) { fakeClientBuilder := fake.NewClientBuilder() fakeClientBuilder = fakeClientBuilder.WithScheme(test.runtimeScheme) fakeClient := fakeClientBuilder.WithRuntimeObjects(test.kubeClientObj...).Build() history, err := WaitAllJobsComplete(test.ctx, fakeClient, repo, 3, velerotest.NewLogger()) if test.expectedError != "" { require.ErrorContains(t, err, test.expectedError) } else { require.NoError(t, err) } assert.Len(t, history, len(test.expectedStatus)) for i := 0; i < len(test.expectedStatus); i++ { assert.Equal(t, test.expectedStatus[i].Result, history[i].Result) assert.Equal(t, test.expectedStatus[i].Message, history[i].Message) assert.Equal(t, test.expectedStatus[i].StartTimestamp.Time, history[i].StartTimestamp.Time) if test.expectedStatus[i].CompleteTimestamp == nil { assert.Nil(t, history[i].CompleteTimestamp) } else { assert.Equal(t, test.expectedStatus[i].CompleteTimestamp.Time, history[i].CompleteTimestamp.Time) } } }) } cancel() } func TestBuildJob(t *testing.T) { deploy := appsv1api.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "velero", Namespace: "velero", }, Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ SecurityContext: &corev1api.PodSecurityContext{ RunAsNonRoot: boolptr.True(), }, Containers: []corev1api.Container{ { Name: "velero-repo-maintenance-container", Image: "velero-image", SecurityContext: &corev1api.SecurityContext{ RunAsNonRoot: boolptr.True(), }, Env: []corev1api.EnvVar{ { Name: "test-name", Value: "test-value", }, }, EnvFrom: []corev1api.EnvFromSource{ { ConfigMapRef: &corev1api.ConfigMapEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-configmap", }, }, }, { SecretRef: &corev1api.SecretEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-secret", }, }, }, }, }, }, ImagePullSecrets: []corev1api.LocalObjectReference{ { Name: "imagePullSecret1", }, }, }, }, }, } deploy2 := deploy.DeepCopy() deploy2.Spec.Template.Labels = map[string]string{"azure.workload.identity/use": "fake-label-value"} deploy2.Spec.Template.Spec.SecurityContext = nil deploy2.Spec.Template.Spec.Containers[0].SecurityContext = nil testCases := []struct { name string m *velerotypes.JobConfigs deploy *appsv1api.Deployment logLevel logrus.Level logFormat *logging.FormatFlag expectedJobName string expectedError bool expectedEnv []corev1api.EnvVar expectedEnvFrom []corev1api.EnvFromSource expectedPodLabel map[string]string expectedPodAnnotation map[string]string expectedSecurityContext *corev1api.SecurityContext expectedPodSecurityContext *corev1api.PodSecurityContext expectedImagePullSecrets []corev1api.LocalObjectReference backupRepository *velerov1api.BackupRepository }{ { name: "Valid maintenance job without third party labels", m: &velerotypes.JobConfigs{ PodResources: &kube.PodResources{ CPURequest: "100m", MemoryRequest: "128Mi", CPULimit: "200m", MemoryLimit: "256Mi", }, }, deploy: &deploy, logLevel: logrus.InfoLevel, logFormat: logging.NewFormatFlag(), expectedJobName: "test-123-maintain-job", expectedError: false, expectedEnv: []corev1api.EnvVar{ { Name: "test-name", Value: "test-value", }, }, expectedEnvFrom: []corev1api.EnvFromSource{ { ConfigMapRef: &corev1api.ConfigMapEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-configmap", }, }, }, { SecretRef: &corev1api.SecretEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-secret", }, }, }, }, expectedPodLabel: map[string]string{ RepositoryNameLabel: "test-123", }, expectedSecurityContext: &corev1api.SecurityContext{ RunAsNonRoot: boolptr.True(), }, expectedPodSecurityContext: &corev1api.PodSecurityContext{ RunAsNonRoot: boolptr.True(), }, expectedImagePullSecrets: []corev1api.LocalObjectReference{ { Name: "imagePullSecret1", }, }, }, { name: "Valid maintenance job with third party labels", m: &velerotypes.JobConfigs{ PodResources: &kube.PodResources{ CPURequest: "100m", MemoryRequest: "128Mi", CPULimit: "200m", MemoryLimit: "256Mi", }, }, deploy: deploy2, logLevel: logrus.InfoLevel, logFormat: logging.NewFormatFlag(), expectedJobName: "test-123-maintain-job", expectedError: false, expectedEnv: []corev1api.EnvVar{ { Name: "test-name", Value: "test-value", }, }, expectedEnvFrom: []corev1api.EnvFromSource{ { ConfigMapRef: &corev1api.ConfigMapEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-configmap", }, }, }, { SecretRef: &corev1api.SecretEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-secret", }, }, }, }, expectedPodLabel: map[string]string{ RepositoryNameLabel: "test-123", "azure.workload.identity/use": "fake-label-value", }, expectedSecurityContext: nil, expectedPodSecurityContext: nil, expectedImagePullSecrets: []corev1api.LocalObjectReference{ { Name: "imagePullSecret1", }, }, }, { name: "Error getting Velero server deployment", m: &velerotypes.JobConfigs{ PodResources: &kube.PodResources{ CPURequest: "100m", MemoryRequest: "128Mi", CPULimit: "200m", MemoryLimit: "256Mi", }, }, logLevel: logrus.InfoLevel, logFormat: logging.NewFormatFlag(), expectedJobName: "", expectedError: true, }, { name: "Valid maintenance job customized labels and annotations", m: &velerotypes.JobConfigs{ PodResources: &kube.PodResources{ CPURequest: "100m", MemoryRequest: "128Mi", CPULimit: "200m", MemoryLimit: "256Mi", }, PodLabels: map[string]string{ "global-label-1": "global-label-value-1", "global-label-2": "global-label-value-2", }, PodAnnotations: map[string]string{ "global-annotation-1": "global-annotation-value-1", "global-annotation-2": "global-annotation-value-2", }, }, deploy: deploy2, logLevel: logrus.InfoLevel, logFormat: logging.NewFormatFlag(), expectedError: false, expectedJobName: "test-123-maintain-job", expectedEnv: []corev1api.EnvVar{ { Name: "test-name", Value: "test-value", }, }, expectedEnvFrom: []corev1api.EnvFromSource{ { ConfigMapRef: &corev1api.ConfigMapEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-configmap", }, }, }, { SecretRef: &corev1api.SecretEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-secret", }, }, }, }, expectedPodLabel: map[string]string{ "global-label-1": "global-label-value-1", "global-label-2": "global-label-value-2", RepositoryNameLabel: "test-123", }, expectedPodAnnotation: map[string]string{ "global-annotation-1": "global-annotation-value-1", "global-annotation-2": "global-annotation-value-2", }, expectedSecurityContext: nil, expectedPodSecurityContext: nil, expectedImagePullSecrets: []corev1api.LocalObjectReference{ { Name: "imagePullSecret1", }, }, }, { name: "Valid maintenance job with third party labels and BackupRepository name longer than 63", m: &velerotypes.JobConfigs{ PodResources: &kube.PodResources{ CPURequest: "100m", MemoryRequest: "128Mi", CPULimit: "200m", MemoryLimit: "256Mi", }, }, deploy: deploy2, logLevel: logrus.InfoLevel, logFormat: logging.NewFormatFlag(), expectedError: false, expectedEnv: []corev1api.EnvVar{ { Name: "test-name", Value: "test-value", }, }, expectedEnvFrom: []corev1api.EnvFromSource{ { ConfigMapRef: &corev1api.ConfigMapEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-configmap", }, }, }, { SecretRef: &corev1api.SecretEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-secret", }, }, }, }, expectedPodLabel: map[string]string{ RepositoryNameLabel: velerolabel.ReturnNameOrHash("label with more than 63 characters should be modified"), "azure.workload.identity/use": "fake-label-value", }, expectedSecurityContext: nil, expectedPodSecurityContext: nil, expectedImagePullSecrets: []corev1api.LocalObjectReference{ { Name: "imagePullSecret1", }, }, backupRepository: &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "label with more than 63 characters should be modified", }, Spec: velerov1api.BackupRepositorySpec{ VolumeNamespace: "test-123", RepositoryType: "kopia", }, }, }, { name: "Invalid label key is skipped", m: &velerotypes.JobConfigs{ PodResources: &kube.PodResources{ CPURequest: "100m", MemoryRequest: "128Mi", CPULimit: "200m", MemoryLimit: "256Mi", }, PodLabels: map[string]string{ "valid-label": "valid-value", "INVALID KEY!!": "some-value", }, }, deploy: deploy2, logLevel: logrus.InfoLevel, logFormat: logging.NewFormatFlag(), expectedJobName: "test-123-maintain-job", expectedError: false, expectedEnv: []corev1api.EnvVar{ { Name: "test-name", Value: "test-value", }, }, expectedEnvFrom: []corev1api.EnvFromSource{ { ConfigMapRef: &corev1api.ConfigMapEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-configmap", }, }, }, { SecretRef: &corev1api.SecretEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-secret", }, }, }, }, expectedPodLabel: map[string]string{ RepositoryNameLabel: "test-123", "valid-label": "valid-value", }, expectedSecurityContext: nil, expectedPodSecurityContext: nil, expectedImagePullSecrets: []corev1api.LocalObjectReference{ { Name: "imagePullSecret1", }, }, }, { name: "Invalid label value is skipped", m: &velerotypes.JobConfigs{ PodResources: &kube.PodResources{ CPURequest: "100m", MemoryRequest: "128Mi", CPULimit: "200m", MemoryLimit: "256Mi", }, PodLabels: map[string]string{ "valid-label": "valid-value", "another-label": "this value has spaces and is invalid", }, }, deploy: deploy2, logLevel: logrus.InfoLevel, logFormat: logging.NewFormatFlag(), expectedJobName: "test-123-maintain-job", expectedError: false, expectedEnv: []corev1api.EnvVar{ { Name: "test-name", Value: "test-value", }, }, expectedEnvFrom: []corev1api.EnvFromSource{ { ConfigMapRef: &corev1api.ConfigMapEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-configmap", }, }, }, { SecretRef: &corev1api.SecretEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-secret", }, }, }, }, expectedPodLabel: map[string]string{ RepositoryNameLabel: "test-123", "valid-label": "valid-value", }, expectedSecurityContext: nil, expectedPodSecurityContext: nil, expectedImagePullSecrets: []corev1api.LocalObjectReference{ { Name: "imagePullSecret1", }, }, }, { name: "Label value exceeding 63 characters is skipped", m: &velerotypes.JobConfigs{ PodResources: &kube.PodResources{ CPURequest: "100m", MemoryRequest: "128Mi", CPULimit: "200m", MemoryLimit: "256Mi", }, PodLabels: map[string]string{ "valid-label": "valid-value", "long-value-label": "this-value-is-way-too-long-for-a-kubernetes-label-value-and-exceeds-sixty-three-characters", }, }, deploy: deploy2, logLevel: logrus.InfoLevel, logFormat: logging.NewFormatFlag(), expectedJobName: "test-123-maintain-job", expectedError: false, expectedEnv: []corev1api.EnvVar{ { Name: "test-name", Value: "test-value", }, }, expectedEnvFrom: []corev1api.EnvFromSource{ { ConfigMapRef: &corev1api.ConfigMapEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-configmap", }, }, }, { SecretRef: &corev1api.SecretEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-secret", }, }, }, }, expectedPodLabel: map[string]string{ RepositoryNameLabel: "test-123", "valid-label": "valid-value", }, expectedSecurityContext: nil, expectedPodSecurityContext: nil, expectedImagePullSecrets: []corev1api.LocalObjectReference{ { Name: "imagePullSecret1", }, }, }, { name: "User-provided label cannot overwrite RepositoryNameLabel", m: &velerotypes.JobConfigs{ PodResources: &kube.PodResources{ CPURequest: "100m", MemoryRequest: "128Mi", CPULimit: "200m", MemoryLimit: "256Mi", }, PodLabels: map[string]string{ RepositoryNameLabel: "user-override-attempt", "valid-label": "valid-value", }, }, deploy: deploy2, logLevel: logrus.InfoLevel, logFormat: logging.NewFormatFlag(), expectedJobName: "test-123-maintain-job", expectedError: false, expectedEnv: []corev1api.EnvVar{ { Name: "test-name", Value: "test-value", }, }, expectedEnvFrom: []corev1api.EnvFromSource{ { ConfigMapRef: &corev1api.ConfigMapEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-configmap", }, }, }, { SecretRef: &corev1api.SecretEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-secret", }, }, }, }, expectedPodLabel: map[string]string{ RepositoryNameLabel: "test-123", "valid-label": "valid-value", }, expectedSecurityContext: nil, expectedPodSecurityContext: nil, expectedImagePullSecrets: []corev1api.LocalObjectReference{ { Name: "imagePullSecret1", }, }, }, { name: "Invalid annotation key is skipped", m: &velerotypes.JobConfigs{ PodResources: &kube.PodResources{ CPURequest: "100m", MemoryRequest: "128Mi", CPULimit: "200m", MemoryLimit: "256Mi", }, PodAnnotations: map[string]string{ "valid-annotation": "any value is fine for annotations, even with spaces!", "INVALID KEY ANNO!": "some-value", }, }, deploy: deploy2, logLevel: logrus.InfoLevel, logFormat: logging.NewFormatFlag(), expectedJobName: "test-123-maintain-job", expectedError: false, expectedEnv: []corev1api.EnvVar{ { Name: "test-name", Value: "test-value", }, }, expectedEnvFrom: []corev1api.EnvFromSource{ { ConfigMapRef: &corev1api.ConfigMapEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-configmap", }, }, }, { SecretRef: &corev1api.SecretEnvSource{ LocalObjectReference: corev1api.LocalObjectReference{ Name: "test-secret", }, }, }, }, expectedPodLabel: map[string]string{ RepositoryNameLabel: "test-123", "azure.workload.identity/use": "fake-label-value", }, expectedPodAnnotation: map[string]string{ "valid-annotation": "any value is fine for annotations, even with spaces!", }, expectedSecurityContext: nil, expectedPodSecurityContext: nil, expectedImagePullSecrets: []corev1api.LocalObjectReference{ { Name: "imagePullSecret1", }, }, }, } param := provider.RepoParam{ BackupRepo: &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "test-123", }, Spec: velerov1api.BackupRepositorySpec{ VolumeNamespace: "test-123", RepositoryType: "kopia", }, }, BackupLocation: &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "test-location", }, }, } defaultBackupRepo := param.BackupRepo for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { if tc.backupRepository != nil { param.BackupRepo = tc.backupRepository } else { param.BackupRepo = defaultBackupRepo } // Create a fake clientset with resources objs := []runtime.Object{param.BackupLocation, param.BackupRepo} if tc.deploy != nil { objs = append(objs, tc.deploy) } scheme := runtime.NewScheme() _ = appsv1api.AddToScheme(scheme) _ = velerov1api.AddToScheme(scheme) cli := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(objs...).Build() // Call the function to test job, err := buildJob( cli, t.Context(), param.BackupRepo, param.BackupLocation.Name, tc.m, tc.logLevel, tc.logFormat, logrus.New(), ) // Check the error if tc.expectedError { require.Error(t, err) assert.Nil(t, job) } else { require.NoError(t, err) assert.NotNil(t, job) assert.Contains(t, job.Name, tc.expectedJobName) assert.Equal(t, param.BackupRepo.Namespace, job.Namespace) assert.Equal(t, param.BackupRepo.Name, job.Labels[RepositoryNameLabel]) assert.Equal(t, param.BackupRepo.Name, job.Spec.Template.ObjectMeta.Labels[RepositoryNameLabel]) // Check container assert.Len(t, job.Spec.Template.Spec.Containers, 1) container := job.Spec.Template.Spec.Containers[0] assert.Equal(t, "velero-repo-maintenance-container", container.Name) assert.Equal(t, "velero-image", container.Image) assert.Equal(t, corev1api.PullIfNotPresent, container.ImagePullPolicy) // Check container env assert.Equal(t, tc.expectedEnv, container.Env) assert.Equal(t, tc.expectedEnvFrom, container.EnvFrom) // Check security context assert.Equal(t, tc.expectedPodSecurityContext, job.Spec.Template.Spec.SecurityContext) assert.Equal(t, tc.expectedSecurityContext, container.SecurityContext) // Check resources expectedResources := corev1api.ResourceRequirements{ Requests: corev1api.ResourceList{ corev1api.ResourceCPU: resource.MustParse(tc.m.PodResources.CPURequest), corev1api.ResourceMemory: resource.MustParse(tc.m.PodResources.MemoryRequest), }, Limits: corev1api.ResourceList{ corev1api.ResourceCPU: resource.MustParse(tc.m.PodResources.CPULimit), corev1api.ResourceMemory: resource.MustParse(tc.m.PodResources.MemoryLimit), }, } assert.Equal(t, expectedResources, container.Resources) // Check args expectedArgs := []string{ "repo-maintenance", fmt.Sprintf("--repo-name=%s", param.BackupRepo.Spec.VolumeNamespace), fmt.Sprintf("--repo-type=%s", param.BackupRepo.Spec.RepositoryType), fmt.Sprintf("--backup-storage-location=%s", param.BackupLocation.Name), fmt.Sprintf("--log-level=%s", tc.logLevel.String()), fmt.Sprintf("--log-format=%s", tc.logFormat.String()), } assert.Equal(t, expectedArgs, container.Args) assert.Equal(t, tc.expectedPodLabel, job.Spec.Template.Labels) if tc.expectedPodAnnotation != nil { assert.Equal(t, tc.expectedPodAnnotation, job.Spec.Template.Annotations) } assert.Equal(t, tc.expectedImagePullSecrets, job.Spec.Template.Spec.ImagePullSecrets) } }) } } func TestGetKeepLatestMaintenanceJobs(t *testing.T) { tests := []struct { name string repoMaintenanceJobConfig string configMap *corev1api.ConfigMap repo *velerov1api.BackupRepository expectedValue int expectError bool }{ { name: "no config map name provided", repoMaintenanceJobConfig: "", configMap: nil, repo: mockBackupRepo(), expectedValue: 3, expectError: false, }, { name: "config map not found", repoMaintenanceJobConfig: "non-existent-config", configMap: nil, repo: mockBackupRepo(), expectedValue: 3, expectError: false, }, { name: "config map with global keepLatestMaintenanceJobs", repoMaintenanceJobConfig: "repo-job-config", configMap: &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "repo-job-config", }, Data: map[string]string{ "global": `{"keepLatestMaintenanceJobs": 5}`, }, }, repo: mockBackupRepo(), expectedValue: 5, expectError: false, }, { name: "config map with specific repo keepLatestMaintenanceJobs overriding global", repoMaintenanceJobConfig: "repo-job-config", configMap: &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "repo-job-config", }, Data: map[string]string{ "global": `{"keepLatestMaintenanceJobs": 5}`, "test-ns-default-kopia": `{"keepLatestMaintenanceJobs": 10}`, }, }, repo: mockBackupRepo(), expectedValue: 10, expectError: false, }, { name: "config map with no keepLatestMaintenanceJobs specified", repoMaintenanceJobConfig: "repo-job-config", configMap: &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "repo-job-config", }, Data: map[string]string{ "global": `{"podResources": {"cpuRequest": "100m"}}`, }, }, repo: mockBackupRepo(), expectedValue: 3, expectError: false, }, { name: "config map with invalid JSON", repoMaintenanceJobConfig: "repo-job-config", configMap: &corev1api.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "repo-job-config", }, Data: map[string]string{ "global": `{"keepLatestMaintenanceJobs": invalid}`, }, }, repo: mockBackupRepo(), expectedValue: 3, expectError: true, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { scheme := runtime.NewScheme() corev1api.AddToScheme(scheme) var objects []runtime.Object if test.configMap != nil { objects = append(objects, test.configMap) } client := fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(objects...).Build() logger := velerotest.NewLogger() result, err := GetKeepLatestMaintenanceJobs( t.Context(), client, logger, "velero", test.repoMaintenanceJobConfig, test.repo, ) if test.expectError { require.Error(t, err) } else { require.NoError(t, err) assert.Equal(t, test.expectedValue, result) } }) } } func mockBackupRepo() *velerov1api.BackupRepository { return &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", Name: "test-repo", }, Spec: velerov1api.BackupRepositorySpec{ VolumeNamespace: "test-ns", BackupStorageLocation: "default", RepositoryType: "kopia", }, } } func TestGetPriorityClassName(t *testing.T) { testCases := []struct { name string config *velerotypes.JobConfigs priorityClassExists bool expectedValue string expectedLogContains string expectedLogLevel string }{ { name: "empty priority class name should return empty string", config: &velerotypes.JobConfigs{PriorityClassName: ""}, expectedValue: "", expectedLogContains: "", }, { name: "nil config should return empty string", config: nil, expectedValue: "", expectedLogContains: "", }, { name: "existing priority class should log info and return name", config: &velerotypes.JobConfigs{PriorityClassName: "high-priority"}, priorityClassExists: true, expectedValue: "high-priority", expectedLogContains: "Validated priority class \\\"high-priority\\\" exists in cluster", expectedLogLevel: "info", }, { name: "non-existing priority class should log warning and still return name", config: &velerotypes.JobConfigs{PriorityClassName: "missing-priority"}, priorityClassExists: false, expectedValue: "missing-priority", expectedLogContains: "Priority class \\\"missing-priority\\\" not found in cluster", expectedLogLevel: "warning", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // Create a new scheme and add necessary API types localScheme := runtime.NewScheme() err := schedulingv1.AddToScheme(localScheme) require.NoError(t, err) // Create fake client builder clientBuilder := fake.NewClientBuilder().WithScheme(localScheme) // Add priority class if it should exist if tc.priorityClassExists { priorityClass := &schedulingv1.PriorityClass{ ObjectMeta: metav1.ObjectMeta{ Name: tc.config.PriorityClassName, }, Value: 1000, } clientBuilder = clientBuilder.WithObjects(priorityClass) } client := clientBuilder.Build() // Capture logs var logBuffer strings.Builder logger := logrus.New() logger.SetOutput(&logBuffer) logger.SetLevel(logrus.InfoLevel) // Call the function result := getPriorityClassName(t.Context(), client, tc.config, logger) // Verify the result assert.Equal(t, tc.expectedValue, result) // Verify log output logOutput := logBuffer.String() if tc.expectedLogContains != "" { assert.Contains(t, logOutput, tc.expectedLogContains) } // Verify log level if tc.expectedLogLevel == "warning" { assert.Contains(t, logOutput, "level=warning") } else if tc.expectedLogLevel == "info" { assert.Contains(t, logOutput, "level=info") } }) } } func TestBuildJobWithPriorityClassName(t *testing.T) { testCases := []struct { name string priorityClassName string expectedValue string }{ { name: "with priority class name", priorityClassName: "high-priority", expectedValue: "high-priority", }, { name: "without priority class name", priorityClassName: "", expectedValue: "", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // Create a new scheme and add necessary API types localScheme := runtime.NewScheme() err := velerov1api.AddToScheme(localScheme) require.NoError(t, err) err = appsv1api.AddToScheme(localScheme) require.NoError(t, err) err = batchv1api.AddToScheme(localScheme) require.NoError(t, err) err = schedulingv1.AddToScheme(localScheme) require.NoError(t, err) // Create a fake client client := fake.NewClientBuilder().WithScheme(localScheme).Build() // Create a deployment with the specified priority class name deployment := &appsv1api.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "velero", Namespace: "velero", }, Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Name: "velero", Image: "velero/velero:latest", }, }, PriorityClassName: tc.priorityClassName, }, }, }, } // Create a backup repository repo := &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Name: "test-repo", Namespace: "velero", }, Spec: velerov1api.BackupRepositorySpec{ VolumeNamespace: "velero", BackupStorageLocation: "default", }, } // Create the deployment in the fake client err = client.Create(t.Context(), deployment) require.NoError(t, err) // Create minimal job configs and resources jobConfig := &velerotypes.JobConfigs{ PriorityClassName: tc.priorityClassName, } logLevel := logrus.InfoLevel logFormat := logging.NewFormatFlag() logFormat.Set("text") // Call buildJob job, err := buildJob(client, t.Context(), repo, "default", jobConfig, logLevel, logFormat, logrus.New()) require.NoError(t, err) // Verify the priority class name is set correctly assert.Equal(t, tc.expectedValue, job.Spec.Template.Spec.PriorityClassName) }) } } func TestBuildTolerationsForMaintenanceJob(t *testing.T) { windowsToleration := corev1api.Toleration{ Key: "os", Operator: "Equal", Effect: "NoSchedule", Value: "windows", } testCases := []struct { name string deploymentTolerations []corev1api.Toleration configuredTolerations []corev1api.Toleration expectedTolerations []corev1api.Toleration }{ { name: "no tolerations should only include Windows toleration", deploymentTolerations: nil, configuredTolerations: nil, expectedTolerations: []corev1api.Toleration{ windowsToleration, }, }, { name: "empty tolerations should only include Windows toleration", deploymentTolerations: []corev1api.Toleration{}, configuredTolerations: []corev1api.Toleration{}, expectedTolerations: []corev1api.Toleration{ windowsToleration, }, }, { name: "non-allowed deployment toleration should not be inherited", deploymentTolerations: []corev1api.Toleration{ { Key: "vng-ondemand", Operator: "Equal", Effect: "NoSchedule", Value: "amd64", }, }, configuredTolerations: nil, expectedTolerations: []corev1api.Toleration{ windowsToleration, }, }, { name: "allowed deployment tolerations should be inherited", deploymentTolerations: []corev1api.Toleration{ { Key: "kubernetes.azure.com/scalesetpriority", Operator: "Equal", Effect: "NoSchedule", Value: "spot", }, { Key: "CriticalAddonsOnly", Operator: "Exists", Effect: "NoSchedule", }, }, configuredTolerations: nil, expectedTolerations: []corev1api.Toleration{ windowsToleration, { Key: "kubernetes.azure.com/scalesetpriority", Operator: "Equal", Effect: "NoSchedule", Value: "spot", }, { Key: "CriticalAddonsOnly", Operator: "Exists", Effect: "NoSchedule", }, }, }, { name: "mixed allowed and non-allowed deployment tolerations should only inherit allowed", deploymentTolerations: []corev1api.Toleration{ { Key: "vng-ondemand", Operator: "Equal", Effect: "NoSchedule", Value: "amd64", }, { Key: "CriticalAddonsOnly", Operator: "Exists", Effect: "NoSchedule", }, { Key: "custom-key", Operator: "Equal", Effect: "NoExecute", Value: "custom-value", }, }, configuredTolerations: nil, expectedTolerations: []corev1api.Toleration{ windowsToleration, { Key: "CriticalAddonsOnly", Operator: "Exists", Effect: "NoSchedule", }, }, }, { name: "configured tolerations should be included along with Windows toleration", deploymentTolerations: nil, configuredTolerations: []corev1api.Toleration{ { Key: "dedicated", Operator: "Equal", Effect: "NoSchedule", Value: "backup", }, }, expectedTolerations: []corev1api.Toleration{ { Key: "dedicated", Operator: "Equal", Effect: "NoSchedule", Value: "backup", }, windowsToleration, }, }, { name: "configured tolerations merged with allowed deployment tolerations and Windows toleration", deploymentTolerations: []corev1api.Toleration{ { Key: "CriticalAddonsOnly", Operator: "Exists", Effect: "NoSchedule", }, { Key: "unallowed-key", Operator: "Exists", Effect: "NoSchedule", }, }, configuredTolerations: []corev1api.Toleration{ { Key: "karpenter.sh/nodepool", Operator: "Equal", Effect: "NoSchedule", Value: "maintenance", }, }, expectedTolerations: []corev1api.Toleration{ { Key: "karpenter.sh/nodepool", Operator: "Equal", Effect: "NoSchedule", Value: "maintenance", }, windowsToleration, { Key: "CriticalAddonsOnly", Operator: "Exists", Effect: "NoSchedule", }, }, }, { name: "duplicate tolerations in configured list should be deduplicated", deploymentTolerations: nil, configuredTolerations: []corev1api.Toleration{ { Key: "dedicated", Operator: "Equal", Effect: "NoSchedule", Value: "backup", }, { Key: "dedicated", Operator: "Equal", Effect: "NoSchedule", Value: "backup", }, }, expectedTolerations: []corev1api.Toleration{ { Key: "dedicated", Operator: "Equal", Effect: "NoSchedule", Value: "backup", }, windowsToleration, }, }, { name: "configured toleration duplicate of allowed deployment toleration keeps configured", deploymentTolerations: []corev1api.Toleration{ { Key: "CriticalAddonsOnly", Operator: "Exists", Effect: "NoSchedule", }, }, configuredTolerations: []corev1api.Toleration{ { Key: "CriticalAddonsOnly", Operator: "Exists", Effect: "NoSchedule", }, }, expectedTolerations: []corev1api.Toleration{ { Key: "CriticalAddonsOnly", Operator: "Exists", Effect: "NoSchedule", }, windowsToleration, }, }, { name: "configured toleration duplicate of Windows toleration keeps configured", deploymentTolerations: nil, configuredTolerations: []corev1api.Toleration{ windowsToleration, }, expectedTolerations: []corev1api.Toleration{ windowsToleration, }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // Create a deployment with the specified tolerations var deployment *appsv1api.Deployment if tc.deploymentTolerations != nil { deployment = &appsv1api.Deployment{ Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Tolerations: tc.deploymentTolerations, }, }, }, } } result := buildTolerationsForMaintenanceJob(deployment, tc.configuredTolerations) assert.Equal(t, tc.expectedTolerations, result) }) } } func TestBuildJobWithTolerationsInheritance(t *testing.T) { // Windows toleration that should always be present windowsToleration := corev1api.Toleration{ Key: "os", Operator: "Equal", Effect: "NoSchedule", Value: "windows", } testCases := []struct { name string deploymentTolerations []corev1api.Toleration jobConfig *velerotypes.JobConfigs expectedTolerations []corev1api.Toleration }{ { name: "no tolerations on deployment should only have Windows toleration", deploymentTolerations: nil, expectedTolerations: []corev1api.Toleration{ windowsToleration, }, }, { name: "allowed tolerations should be inherited along with Windows toleration", deploymentTolerations: []corev1api.Toleration{ { Key: "kubernetes.azure.com/scalesetpriority", Operator: "Equal", Effect: "NoSchedule", Value: "spot", }, { Key: "custom-taint", Operator: "Equal", Effect: "NoExecute", Value: "dedicated", }, }, expectedTolerations: []corev1api.Toleration{ windowsToleration, { Key: "kubernetes.azure.com/scalesetpriority", Operator: "Equal", Effect: "NoSchedule", Value: "spot", }, }, }, { name: "configured tolerations should be applied to job", deploymentTolerations: []corev1api.Toleration{ { Key: "CriticalAddonsOnly", Operator: "Exists", Effect: "NoSchedule", }, }, jobConfig: &velerotypes.JobConfigs{ Tolerations: []corev1api.Toleration{ { Key: "karpenter.sh/nodepool", Operator: "Equal", Effect: "NoSchedule", Value: "maintenance", }, }, }, expectedTolerations: []corev1api.Toleration{ { Key: "karpenter.sh/nodepool", Operator: "Equal", Effect: "NoSchedule", Value: "maintenance", }, windowsToleration, { Key: "CriticalAddonsOnly", Operator: "Exists", Effect: "NoSchedule", }, }, }, { name: "duplicate tolerations between configured and deployment should be deduplicated", deploymentTolerations: []corev1api.Toleration{ { Key: "CriticalAddonsOnly", Operator: "Exists", Effect: "NoSchedule", }, }, jobConfig: &velerotypes.JobConfigs{ Tolerations: []corev1api.Toleration{ { Key: "CriticalAddonsOnly", Operator: "Exists", Effect: "NoSchedule", }, }, }, expectedTolerations: []corev1api.Toleration{ { Key: "CriticalAddonsOnly", Operator: "Exists", Effect: "NoSchedule", }, windowsToleration, }, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { // Create a new scheme and add necessary API types localScheme := runtime.NewScheme() err := velerov1api.AddToScheme(localScheme) require.NoError(t, err) err = appsv1api.AddToScheme(localScheme) require.NoError(t, err) err = batchv1api.AddToScheme(localScheme) require.NoError(t, err) // Create a deployment with the specified tolerations deployment := &appsv1api.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "velero", Namespace: "velero", }, Spec: appsv1api.DeploymentSpec{ Template: corev1api.PodTemplateSpec{ Spec: corev1api.PodSpec{ Containers: []corev1api.Container{ { Name: "velero", Image: "velero/velero:latest", }, }, Tolerations: tc.deploymentTolerations, }, }, }, } // Create a backup repository repo := &velerov1api.BackupRepository{ ObjectMeta: metav1.ObjectMeta{ Name: "test-repo", Namespace: "velero", }, Spec: velerov1api.BackupRepositorySpec{ VolumeNamespace: "velero", BackupStorageLocation: "default", }, } // Create fake client and add the deployment client := fake.NewClientBuilder().WithScheme(localScheme).WithObjects(deployment).Build() // Create minimal job configs and resources jobConfig := tc.jobConfig if jobConfig == nil { jobConfig = &velerotypes.JobConfigs{} } logLevel := logrus.InfoLevel logFormat := logging.NewFormatFlag() logFormat.Set("text") // Call buildJob job, err := buildJob(client, t.Context(), repo, "default", jobConfig, logLevel, logFormat, logrus.New()) require.NoError(t, err) // Verify the tolerations are set correctly assert.Equal(t, tc.expectedTolerations, job.Spec.Template.Spec.Tolerations) }) } }