Customize the tolerations of maintenance job. (#10553)

Signed-off-by: Xun Jiang <xun.jiang@broadcom.com>

pkg/repository/maintenance/maintenance.go

@@ -257,6 +257,9 @@ func getJobConfig(
 				repoMaintenanceJobConfig,
 				repoJobConfigKey)
 		}
+
+		// Tolerations are only read from global config, not per-repository
+		result.Tolerations = nil
 	}
 
 	if _, ok := cm.Data[GlobalKeyForRepoMaintenanceJobCM]; ok {
@@ -302,6 +305,11 @@ func getJobConfig(
 		if len(globalResult.PodAnnotations) > 0 {
 			result.PodAnnotations = globalResult.PodAnnotations
 		}
+
+		// Tolerations are only read from global config, not per-repository
+		if len(globalResult.Tolerations) > 0 {
+			result.Tolerations = globalResult.Tolerations
+		}
 	}
 
 	logger.Debugf("Configuration content for repository %s is %+v", repo.Name, result)
@@ -481,24 +489,45 @@ func StartNewJob(
 	return maintenanceJob.Name, nil
 }
 
-// buildTolerationsForMaintenanceJob builds the tolerations for maintenance jobs.
-// It includes the required Windows toleration for backward compatibility and
-// inherits all tolerations from the Velero deployment.
-func buildTolerationsForMaintenanceJob(deployment *appsv1api.Deployment) []corev1api.Toleration {
-	// Start with the Windows toleration for backward compatibility
+// buildTolerationsForMaintenanceJob builds the tolerations for maintenance jobs:
+// the explicitly configured tolerations (sourced from the maintenance job ConfigMap),
+// plus the required Windows toleration for backward compatibility, plus any toleration
+// on the Velero deployment whose key is in util.ThirdPartyTolerations. The combined
+// list is deduplicated by kube.DeduplicateTolerations.
+//
+// configuredTolerations is appended first so it wins: DeduplicateTolerations
+// keeps only the first occurrence of each exact (Key, Operator, Value, Effect) combination,
+// so an allowlisted deployment toleration or default Windows toleration identical to one
+// already set in the ConfigMap is dropped as a duplicate rather than overriding it.
+func buildTolerationsForMaintenanceJob(deployment *appsv1api.Deployment, configuredTolerations []corev1api.Toleration) []corev1api.Toleration {
 	windowsToleration := corev1api.Toleration{
 		Key:      "os",
 		Operator: "Equal",
 		Effect:   "NoSchedule",
 		Value:    "windows",
 	}
-	result := []corev1api.Toleration{windowsToleration}
 
-	// Inherit all tolerations from the Velero deployment
-	deploymentTolerations := veleroutil.GetTolerationsFromVeleroServer(deployment)
-	result = append(result, deploymentTolerations...)
+	var deploymentTolerations []corev1api.Toleration
+	if deployment != nil {
+		deploymentTolerations = veleroutil.GetTolerationsFromVeleroServer(deployment)
+	}
 
-	return result
+	merged := make([]corev1api.Toleration, 0, len(configuredTolerations)+1+len(deploymentTolerations))
+	merged = append(merged, configuredTolerations...)
+	merged = append(merged, windowsToleration)
+
+	allowedTolerations := make(map[string]struct{}, len(util.ThirdPartyTolerations))
+	for _, allowed := range util.ThirdPartyTolerations {
+		allowedTolerations[allowed] = struct{}{}
+	}
+
+	for _, t := range deploymentTolerations {
+		if _, ok := allowedTolerations[t.Key]; ok {
+			merged = append(merged, t)
+		}
+	}
+
+	return kube.DeduplicateTolerations(merged)
 }
 
 func getPriorityClassName(ctx context.Context, cli client.Client, config *velerotypes.JobConfigs, logger logrus.FieldLogger) string {
@@ -649,6 +678,11 @@ func buildJob(
 	args = append(args, fmt.Sprintf("--log-level=%s", logLevel.String()))
 	args = append(args, fmt.Sprintf("--log-format=%s", logFormat.String()))
 
+	var configuredTolerations []corev1api.Toleration
+	if config != nil && len(config.Tolerations) > 0 {
+		configuredTolerations = config.Tolerations
+	}
+
 	// build the maintenance job
 	job := &batchv1api.Job{
 		ObjectMeta: metav1.ObjectMeta{
@@ -689,7 +723,7 @@ func buildJob(
 					SecurityContext:    podSecurityContext,
 					Volumes:            volumes,
 					ServiceAccountName: serviceAccount,
-					Tolerations:        buildTolerationsForMaintenanceJob(deployment),
+					Tolerations:        buildTolerationsForMaintenanceJob(deployment, configuredTolerations),
 					ImagePullSecrets:   imagePullSecrets,
 				},
 			},

pkg/repository/maintenance/maintenance_test.go

@@ -546,8 +546,8 @@ func TestGetJobConfig(t *testing.T) {
 					Name:      repoMaintenanceJobConfig,
 				},
 				Data: map[string]string{
-					GlobalKeyForRepoMaintenanceJobCM: "{\"keepLatestMaintenanceJobs\":1,\"podResources\":{\"cpuRequest\":\"50m\",\"cpuLimit\":\"100m\",\"memoryRequest\":\"50Mi\",\"memoryLimit\":\"100Mi\"},\"loadAffinity\":[{\"nodeSelector\":{\"matchExpressions\":[{\"key\":\"cloud.google.com/machine-family\",\"operator\":\"In\",\"values\":[\"n2\"]}]}}],\"priorityClassName\":\"global-priority\",\"podAnnotations\":{\"global-key\":\"global-value\"},\"podLabels\":{\"global-key\":\"global-value\"}}",
-					"test-default-kopia":             "{\"podResources\":{\"cpuRequest\":\"100m\",\"cpuLimit\":\"200m\",\"memoryRequest\":\"100Mi\",\"memoryLimit\":\"200Mi\"},\"loadAffinity\":[{\"nodeSelector\":{\"matchExpressions\":[{\"key\":\"cloud.google.com/machine-family\",\"operator\":\"In\",\"values\":[\"e2\"]}]}}],\"priorityClassName\":\"specific-priority\",\"podAnnotations\":{\"specific-key\":\"specific-value\"},\"podLabels\":{\"specific-key\":\"specific-value\"}}",
+					GlobalKeyForRepoMaintenanceJobCM: "{\"keepLatestMaintenanceJobs\":1,\"podResources\":{\"cpuRequest\":\"50m\",\"cpuLimit\":\"100m\",\"memoryRequest\":\"50Mi\",\"memoryLimit\":\"100Mi\"},\"loadAffinity\":[{\"nodeSelector\":{\"matchExpressions\":[{\"key\":\"cloud.google.com/machine-family\",\"operator\":\"In\",\"values\":[\"n2\"]}]}}],\"priorityClassName\":\"global-priority\",\"podAnnotations\":{\"global-key\":\"global-value\"},\"podLabels\":{\"global-key\":\"global-value\"},\"tolerations\":[{\"key\":\"global-taint\",\"operator\":\"Exists\",\"effect\":\"NoSchedule\"}]}",
+					"test-default-kopia":             "{\"podResources\":{\"cpuRequest\":\"100m\",\"cpuLimit\":\"200m\",\"memoryRequest\":\"100Mi\",\"memoryLimit\":\"200Mi\"},\"loadAffinity\":[{\"nodeSelector\":{\"matchExpressions\":[{\"key\":\"cloud.google.com/machine-family\",\"operator\":\"In\",\"values\":[\"e2\"]}]}}],\"priorityClassName\":\"specific-priority\",\"podAnnotations\":{\"specific-key\":\"specific-value\"},\"podLabels\":{\"specific-key\":\"specific-value\"},\"tolerations\":[{\"key\":\"specific-taint\",\"operator\":\"Equal\",\"value\":\"dedicated\",\"effect\":\"NoSchedule\"}]}",
 				},
 			},
 			expectedConfig: &velerotypes.JobConfigs{
@@ -574,6 +574,79 @@ func TestGetJobConfig(t *testing.T) {
 				PriorityClassName: "global-priority",
 				PodAnnotations:    map[string]string{"global-key": "global-value"},
 				PodLabels:         map[string]string{"global-key": "global-value"},
+				Tolerations: []corev1api.Toleration{
+					{
+						Key:      "global-taint",
+						Operator: corev1api.TolerationOpExists,
+						Effect:   corev1api.TaintEffectNoSchedule,
+					},
+				},
+			},
+			expectedError: nil,
+		},
+		{
+			name: "Find config with tolerations in global section",
+			repoJobConfig: &corev1api.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: veleroNamespace,
+					Name:      repoMaintenanceJobConfig,
+				},
+				Data: map[string]string{
+					GlobalKeyForRepoMaintenanceJobCM: "{\"tolerations\":[{\"key\":\"global-taint\",\"operator\":\"Exists\",\"effect\":\"NoSchedule\"}]}",
+				},
+			},
+			expectedConfig: &velerotypes.JobConfigs{
+				Tolerations: []corev1api.Toleration{
+					{
+						Key:      "global-taint",
+						Operator: corev1api.TolerationOpExists,
+						Effect:   corev1api.TaintEffectNoSchedule,
+					},
+				},
+			},
+			expectedError: nil,
+		},
+		{
+			name: "Tolerations in specific config should be ignored",
+			repoJobConfig: &corev1api.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: veleroNamespace,
+					Name:      repoMaintenanceJobConfig,
+				},
+				Data: map[string]string{
+					"test-default-kopia": "{\"podResources\":{\"cpuRequest\":\"100m\"},\"tolerations\":[{\"key\":\"specific-taint\",\"operator\":\"Equal\",\"value\":\"dedicated\",\"effect\":\"NoSchedule\"}]}",
+				},
+			},
+			expectedConfig: &velerotypes.JobConfigs{
+				PodResources: &kube.PodResources{
+					CPURequest: "100m",
+				},
+			},
+			expectedError: nil,
+		},
+		{
+			name: "Global tolerations apply when specific config does not specify global configs",
+			repoJobConfig: &corev1api.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: veleroNamespace,
+					Name:      repoMaintenanceJobConfig,
+				},
+				Data: map[string]string{
+					GlobalKeyForRepoMaintenanceJobCM: "{\"tolerations\":[{\"key\":\"global-taint\",\"operator\":\"Exists\",\"effect\":\"NoSchedule\"}]}",
+					"test-default-kopia":             "{\"podResources\":{\"cpuRequest\":\"100m\"}}",
+				},
+			},
+			expectedConfig: &velerotypes.JobConfigs{
+				PodResources: &kube.PodResources{
+					CPURequest: "100m",
+				},
+				Tolerations: []corev1api.Toleration{
+					{
+						Key:      "global-taint",
+						Operator: corev1api.TolerationOpExists,
+						Effect:   corev1api.TaintEffectNoSchedule,
+					},
+				},
 			},
 			expectedError: nil,
 		},
@@ -1937,11 +2010,13 @@ func TestBuildTolerationsForMaintenanceJob(t *testing.T) {
 	testCases := []struct {
 		name                  string
 		deploymentTolerations []corev1api.Toleration
+		configuredTolerations []corev1api.Toleration
 		expectedTolerations   []corev1api.Toleration
 	}{
 		{
 			name:                  "no tolerations should only include Windows toleration",
 			deploymentTolerations: nil,
+			configuredTolerations: nil,
 			expectedTolerations: []corev1api.Toleration{
 				windowsToleration,
 			},
@@ -1949,12 +2024,59 @@ func TestBuildTolerationsForMaintenanceJob(t *testing.T) {
 		{
 			name:                  "empty tolerations should only include Windows toleration",
 			deploymentTolerations: []corev1api.Toleration{},
+			configuredTolerations: []corev1api.Toleration{},
 			expectedTolerations: []corev1api.Toleration{
 				windowsToleration,
 			},
 		},
 		{
-			name: "all tolerations should be inherited",
+			name: "non-allowed deployment toleration should not be inherited",
+			deploymentTolerations: []corev1api.Toleration{
+				{
+					Key:      "vng-ondemand",
+					Operator: "Equal",
+					Effect:   "NoSchedule",
+					Value:    "amd64",
+				},
+			},
+			configuredTolerations: nil,
+			expectedTolerations: []corev1api.Toleration{
+				windowsToleration,
+			},
+		},
+		{
+			name: "allowed deployment tolerations should be inherited",
+			deploymentTolerations: []corev1api.Toleration{
+				{
+					Key:      "kubernetes.azure.com/scalesetpriority",
+					Operator: "Equal",
+					Effect:   "NoSchedule",
+					Value:    "spot",
+				},
+				{
+					Key:      "CriticalAddonsOnly",
+					Operator: "Exists",
+					Effect:   "NoSchedule",
+				},
+			},
+			configuredTolerations: nil,
+			expectedTolerations: []corev1api.Toleration{
+				windowsToleration,
+				{
+					Key:      "kubernetes.azure.com/scalesetpriority",
+					Operator: "Equal",
+					Effect:   "NoSchedule",
+					Value:    "spot",
+				},
+				{
+					Key:      "CriticalAddonsOnly",
+					Operator: "Exists",
+					Effect:   "NoSchedule",
+				},
+			},
+		},
+		{
+			name: "mixed allowed and non-allowed deployment tolerations should only inherit allowed",
 			deploymentTolerations: []corev1api.Toleration{
 				{
 					Key:      "vng-ondemand",
@@ -1974,43 +2096,155 @@ func TestBuildTolerationsForMaintenanceJob(t *testing.T) {
 					Value:    "custom-value",
 				},
 			},
+			configuredTolerations: nil,
 			expectedTolerations: []corev1api.Toleration{
 				windowsToleration,
 				{
-					Key:      "vng-ondemand",
+					Key:      "CriticalAddonsOnly",
+					Operator: "Exists",
+					Effect:   "NoSchedule",
+				},
+			},
+		},
+		{
+			name:                  "configured tolerations should be included along with Windows toleration",
+			deploymentTolerations: nil,
+			configuredTolerations: []corev1api.Toleration{
+				{
+					Key:      "dedicated",
 					Operator: "Equal",
 					Effect:   "NoSchedule",
-					Value:    "amd64",
+					Value:    "backup",
 				},
+			},
+			expectedTolerations: []corev1api.Toleration{
+				{
+					Key:      "dedicated",
+					Operator: "Equal",
+					Effect:   "NoSchedule",
+					Value:    "backup",
+				},
+				windowsToleration,
+			},
+		},
+		{
+			name: "configured tolerations merged with allowed deployment tolerations and Windows toleration",
+			deploymentTolerations: []corev1api.Toleration{
 				{
 					Key:      "CriticalAddonsOnly",
 					Operator: "Exists",
 					Effect:   "NoSchedule",
 				},
 				{
-					Key:      "custom-key",
-					Operator: "Equal",
-					Effect:   "NoExecute",
-					Value:    "custom-value",
+					Key:      "unallowed-key",
+					Operator: "Exists",
+					Effect:   "NoSchedule",
 				},
 			},
+			configuredTolerations: []corev1api.Toleration{
+				{
+					Key:      "karpenter.sh/nodepool",
+					Operator: "Equal",
+					Effect:   "NoSchedule",
+					Value:    "maintenance",
+				},
+			},
+			expectedTolerations: []corev1api.Toleration{
+				{
+					Key:      "karpenter.sh/nodepool",
+					Operator: "Equal",
+					Effect:   "NoSchedule",
+					Value:    "maintenance",
+				},
+				windowsToleration,
+				{
+					Key:      "CriticalAddonsOnly",
+					Operator: "Exists",
+					Effect:   "NoSchedule",
+				},
+			},
+		},
+		{
+			name:                  "duplicate tolerations in configured list should be deduplicated",
+			deploymentTolerations: nil,
+			configuredTolerations: []corev1api.Toleration{
+				{
+					Key:      "dedicated",
+					Operator: "Equal",
+					Effect:   "NoSchedule",
+					Value:    "backup",
+				},
+				{
+					Key:      "dedicated",
+					Operator: "Equal",
+					Effect:   "NoSchedule",
+					Value:    "backup",
+				},
+			},
+			expectedTolerations: []corev1api.Toleration{
+				{
+					Key:      "dedicated",
+					Operator: "Equal",
+					Effect:   "NoSchedule",
+					Value:    "backup",
+				},
+				windowsToleration,
+			},
+		},
+		{
+			name: "configured toleration duplicate of allowed deployment toleration keeps configured",
+			deploymentTolerations: []corev1api.Toleration{
+				{
+					Key:      "CriticalAddonsOnly",
+					Operator: "Exists",
+					Effect:   "NoSchedule",
+				},
+			},
+			configuredTolerations: []corev1api.Toleration{
+				{
+					Key:      "CriticalAddonsOnly",
+					Operator: "Exists",
+					Effect:   "NoSchedule",
+				},
+			},
+			expectedTolerations: []corev1api.Toleration{
+				{
+					Key:      "CriticalAddonsOnly",
+					Operator: "Exists",
+					Effect:   "NoSchedule",
+				},
+				windowsToleration,
+			},
+		},
+		{
+			name:                  "configured toleration duplicate of Windows toleration keeps configured",
+			deploymentTolerations: nil,
+			configuredTolerations: []corev1api.Toleration{
+				windowsToleration,
+			},
+			expectedTolerations: []corev1api.Toleration{
+				windowsToleration,
+			},
 		},
 	}
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			// Create a deployment with the specified tolerations
-			deployment := &appsv1api.Deployment{
-				Spec: appsv1api.DeploymentSpec{
-					Template: corev1api.PodTemplateSpec{
-						Spec: corev1api.PodSpec{
-							Tolerations: tc.deploymentTolerations,
+			var deployment *appsv1api.Deployment
+			if tc.deploymentTolerations != nil {
+				deployment = &appsv1api.Deployment{
+					Spec: appsv1api.DeploymentSpec{
+						Template: corev1api.PodTemplateSpec{
+							Spec: corev1api.PodSpec{
+								Tolerations: tc.deploymentTolerations,
+							},
 						},
 					},
-				},
+				}
 			}
 
-			result := buildTolerationsForMaintenanceJob(deployment)
+			result := buildTolerationsForMaintenanceJob(deployment, tc.configuredTolerations)
 			assert.Equal(t, tc.expectedTolerations, result)
 		})
 	}
@@ -2028,6 +2262,7 @@ func TestBuildJobWithTolerationsInheritance(t *testing.T) {
 	testCases := []struct {
 		name                  string
 		deploymentTolerations []corev1api.Toleration
+		jobConfig             *velerotypes.JobConfigs
 		expectedTolerations   []corev1api.Toleration
 	}{
 		{
@@ -2038,7 +2273,7 @@ func TestBuildJobWithTolerationsInheritance(t *testing.T) {
 			},
 		},
 		{
-			name: "all tolerations should be inherited along with Windows toleration",
+			name: "allowed tolerations should be inherited along with Windows toleration",
 			deploymentTolerations: []corev1api.Toleration{
 				{
 					Key:      "kubernetes.azure.com/scalesetpriority",
@@ -2061,13 +2296,68 @@ func TestBuildJobWithTolerationsInheritance(t *testing.T) {
 					Effect:   "NoSchedule",
 					Value:    "spot",
 				},
+			},
+		},
+		{
+			name: "configured tolerations should be applied to job",
+			deploymentTolerations: []corev1api.Toleration{
 				{
-					Key:      "custom-taint",
-					Operator: "Equal",
-					Effect:   "NoExecute",
-					Value:    "dedicated",
+					Key:      "CriticalAddonsOnly",
+					Operator: "Exists",
+					Effect:   "NoSchedule",
 				},
 			},
+			jobConfig: &velerotypes.JobConfigs{
+				Tolerations: []corev1api.Toleration{
+					{
+						Key:      "karpenter.sh/nodepool",
+						Operator: "Equal",
+						Effect:   "NoSchedule",
+						Value:    "maintenance",
+					},
+				},
+			},
+			expectedTolerations: []corev1api.Toleration{
+				{
+					Key:      "karpenter.sh/nodepool",
+					Operator: "Equal",
+					Effect:   "NoSchedule",
+					Value:    "maintenance",
+				},
+				windowsToleration,
+				{
+					Key:      "CriticalAddonsOnly",
+					Operator: "Exists",
+					Effect:   "NoSchedule",
+				},
+			},
+		},
+		{
+			name: "duplicate tolerations between configured and deployment should be deduplicated",
+			deploymentTolerations: []corev1api.Toleration{
+				{
+					Key:      "CriticalAddonsOnly",
+					Operator: "Exists",
+					Effect:   "NoSchedule",
+				},
+			},
+			jobConfig: &velerotypes.JobConfigs{
+				Tolerations: []corev1api.Toleration{
+					{
+						Key:      "CriticalAddonsOnly",
+						Operator: "Exists",
+						Effect:   "NoSchedule",
+					},
+				},
+			},
+			expectedTolerations: []corev1api.Toleration{
+				{
+					Key:      "CriticalAddonsOnly",
+					Operator: "Exists",
+					Effect:   "NoSchedule",
+				},
+				windowsToleration,
+			},
 		},
 	}
 
@@ -2119,7 +2409,10 @@ func TestBuildJobWithTolerationsInheritance(t *testing.T) {
 			client := fake.NewClientBuilder().WithScheme(localScheme).WithObjects(deployment).Build()
 
 			// Create minimal job configs and resources
-			jobConfig := &velerotypes.JobConfigs{}
+			jobConfig := tc.jobConfig
+			if jobConfig == nil {
+				jobConfig = &velerotypes.JobConfigs{}
+			}
 			logLevel := logrus.InfoLevel
 			logFormat := logging.NewFormatFlag()
 			logFormat.Set("text")