issue 9343: include PV topology to data mover pod affinities

Signed-off-by: Lyndon-Li <lyonghui@vmware.com>

pkg/exposer/csi_snapshot.go

@@ -124,6 +124,15 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O
 		"owner": ownerObject.Name,
 	})
 
+	volumeTopology, err := kube.GetVolumeTopology(ctx, e.kubeClient.CoreV1(), e.kubeClient.StorageV1(), csiExposeParam.SourcePVName, csiExposeParam.StorageClass)
+	if err != nil {
+		return errors.Wrapf(err, "error getting volume topology for PV %s, storage class %s", csiExposeParam.SourcePVName, csiExposeParam.StorageClass)
+	}
+
+	if volumeTopology != nil {
+		curLog.Infof("Using volume topology %v", volumeTopology)
+	}
+
 	curLog.Info("Exposing CSI snapshot")
 
 	volumeSnapshot, err := csi.WaitVolumeSnapshotReady(ctx, e.csiSnapshotClient, csiExposeParam.SnapshotName, csiExposeParam.SourceNamespace, csiExposeParam.ExposeTimeout, curLog)
@@ -254,6 +263,7 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O
 		csiExposeParam.NodeOS,
 		csiExposeParam.PriorityClassName,
 		intoleratableNodes,
+		volumeTopology,
 	)
 	if err != nil {
 		return errors.Wrap(err, "error to create backup pod")
@@ -588,6 +598,7 @@ func (e *csiSnapshotExposer) createBackupPod(
 	nodeOS string,
 	priorityClassName string,
 	intoleratableNodes []string,
+	volumeTopology *corev1api.NodeSelector,
 ) (*corev1api.Pod, error) {
 	podName := ownerObject.Name
 
@@ -701,7 +712,7 @@ func (e *csiSnapshotExposer) createBackupPod(
 	}
 
 	if affinity != nil {
-		podAffinity = kube.ToSystemAffinity([]*kube.LoadAffinity{affinity})
+		podAffinity = kube.ToSystemAffinity(affinity, volumeTopology)
 	}
 
 	pod := &corev1api.Pod{

pkg/exposer/csi_snapshot_priority_test.go

@@ -154,6 +154,7 @@ func TestCreateBackupPodWithPriorityClass(t *testing.T) {
 				kube.NodeOSLinux,
 				tc.expectedPriorityClass,
 				nil,
+				nil,
 			)
 
 			require.NoError(t, err, tc.description)
@@ -239,6 +240,7 @@ func TestCreateBackupPodWithMissingConfigMap(t *testing.T) {
 		kube.NodeOSLinux,
 		"", // empty priority class since config map is missing
 		nil,
+		nil,
 	)
 
 	// Should succeed even when config map is missing

pkg/exposer/generic_restore.go

@@ -498,7 +498,7 @@ func (e *genericRestoreExposer) createRestorePod(
 		e.log.Infof("No selected node for restore pod. Try to get affinity from the node-agent config.")
 
 		if affinity != nil {
-			podAffinity = kube.ToSystemAffinity([]*kube.LoadAffinity{affinity})
+			podAffinity = kube.ToSystemAffinity(affinity, nil)
 		}
 	}
 

pkg/repository/maintenance/maintenance.go

@@ -671,8 +671,7 @@ func buildJob(
 	}
 
 	if config != nil && len(config.LoadAffinities) > 0 {
-		// Maintenance job only takes the first loadAffinity.
-		affinity := kube.ToSystemAffinity([]*kube.LoadAffinity{config.LoadAffinities[0]})
+		affinity := kube.ToSystemAffinity(config.LoadAffinities[0], nil)
 		job.Spec.Template.Spec.Affinity = affinity
 	}
 

pkg/util/kube/pod.go

@@ -230,14 +230,9 @@ func CollectPodLogs(ctx context.Context, podGetter corev1client.CoreV1Interface,
 	return nil
 }
 
-func ToSystemAffinity(loadAffinities []*LoadAffinity) *corev1api.Affinity {
-	if len(loadAffinities) == 0 {
-		return nil
-	}
-	nodeSelectorTermList := make([]corev1api.NodeSelectorTerm, 0)
-
-	for _, loadAffinity := range loadAffinities {
-		requirements := []corev1api.NodeSelectorRequirement{}
+func ToSystemAffinity(loadAffinity *LoadAffinity, volumeTopolpogy *corev1api.NodeSelector) *corev1api.Affinity {
+	requirements := []corev1api.NodeSelectorRequirement{}
+	if loadAffinity != nil {
 		for k, v := range loadAffinity.NodeSelector.MatchLabels {
 			requirements = append(requirements, corev1api.NodeSelectorRequirement{
 				Key:      k,
@@ -253,25 +248,25 @@ func ToSystemAffinity(loadAffinities []*LoadAffinity) *corev1api.Affinity {
 				Operator: corev1api.NodeSelectorOperator(exp.Operator),
 			})
 		}
-
-		nodeSelectorTermList = append(
-			nodeSelectorTermList,
-			corev1api.NodeSelectorTerm{
-				MatchExpressions: requirements,
-			},
-		)
 	}
 
-	if len(nodeSelectorTermList) > 0 {
-		result := new(corev1api.Affinity)
-		result.NodeAffinity = new(corev1api.NodeAffinity)
-		result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = new(corev1api.NodeSelector)
-		result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = nodeSelectorTermList
+	result := new(corev1api.Affinity)
+	result.NodeAffinity = new(corev1api.NodeAffinity)
+	result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = new(corev1api.NodeSelector)
 
-		return result
+	if volumeTopolpogy != nil {
+		result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append(result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, volumeTopolpogy.NodeSelectorTerms...)
+	} else if len(requirements) > 0 {
+		result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = make([]corev1api.NodeSelectorTerm, 1)
+	} else {
+		return nil
 	}
 
-	return nil
+	for i := range result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms {
+		result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[i].MatchExpressions = append(result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[i].MatchExpressions, requirements...)
+	}
+
+	return result
 }
 
 func DiagnosePod(pod *corev1api.Pod, events *corev1api.EventList) string {

pkg/util/kube/pod_test.go

@@ -747,24 +747,23 @@ func TestCollectPodLogs(t *testing.T) {
 func TestToSystemAffinity(t *testing.T) {
 	tests := []struct {
 		name           string
-		loadAffinities []*LoadAffinity
+		loadAffinity   *LoadAffinity
+		volumeTopology *corev1api.NodeSelector
 		expected       *corev1api.Affinity
 	}{
 		{
 			name: "loadAffinity is nil",
 		},
 		{
-			name:           "loadAffinity is empty",
-			loadAffinities: []*LoadAffinity{},
+			name:         "loadAffinity is empty",
+			loadAffinity: &LoadAffinity{},
 		},
 		{
 			name: "with match label",
-			loadAffinities: []*LoadAffinity{
-				{
-					NodeSelector: metav1.LabelSelector{
-						MatchLabels: map[string]string{
-							"key-1": "value-1",
-						},
+			loadAffinity: &LoadAffinity{
+				NodeSelector: metav1.LabelSelector{
+					MatchLabels: map[string]string{
+						"key-1": "value-1",
 					},
 				},
 			},
@@ -788,23 +787,21 @@ func TestToSystemAffinity(t *testing.T) {
 		},
 		{
 			name: "with match expression",
-			loadAffinities: []*LoadAffinity{
-				{
-					NodeSelector: metav1.LabelSelector{
-						MatchLabels: map[string]string{
-							"key-2": "value-2",
+			loadAffinity: &LoadAffinity{
+				NodeSelector: metav1.LabelSelector{
+					MatchLabels: map[string]string{
+						"key-2": "value-2",
+					},
+					MatchExpressions: []metav1.LabelSelectorRequirement{
+						{
+							Key:      "key-3",
+							Values:   []string{"value-3-1", "value-3-2"},
+							Operator: metav1.LabelSelectorOpNotIn,
 						},
-						MatchExpressions: []metav1.LabelSelectorRequirement{
-							{
-								Key:      "key-3",
-								Values:   []string{"value-3-1", "value-3-2"},
-								Operator: metav1.LabelSelectorOpNotIn,
-							},
-							{
-								Key:      "key-4",
-								Values:   []string{"value-4-1", "value-4-2", "value-4-3"},
-								Operator: metav1.LabelSelectorOpDoesNotExist,
-							},
+						{
+							Key:      "key-4",
+							Values:   []string{"value-4-1", "value-4-2", "value-4-3"},
+							Operator: metav1.LabelSelectorOpDoesNotExist,
 						},
 					},
 				},
@@ -838,19 +835,49 @@ func TestToSystemAffinity(t *testing.T) {
 			},
 		},
 		{
-			name: "multiple load affinities",
-			loadAffinities: []*LoadAffinity{
-				{
-					NodeSelector: metav1.LabelSelector{
-						MatchLabels: map[string]string{
-							"key-1": "value-1",
+			name: "with olume topology",
+			volumeTopology: &corev1api.NodeSelector{
+				NodeSelectorTerms: []corev1api.NodeSelectorTerm{
+					{
+						MatchExpressions: []corev1api.NodeSelectorRequirement{
+							{
+								Key:      "key-5",
+								Values:   []string{"value-5-1", "value-5-2", "value-5-3"},
+								Operator: corev1api.NodeSelectorOpGt,
+							},
+							{
+								Key:      "key-6",
+								Values:   []string{"value-5-1", "value-5-2", "value-5-3"},
+								Operator: corev1api.NodeSelectorOpGt,
+							},
 						},
 					},
-				},
-				{
-					NodeSelector: metav1.LabelSelector{
-						MatchLabels: map[string]string{
-							"key-2": "value-2",
+					{
+						MatchExpressions: []corev1api.NodeSelectorRequirement{
+							{
+								Key:      "key-7",
+								Values:   []string{"value-7-1", "value-7-2", "value-7-3"},
+								Operator: corev1api.NodeSelectorOpGt,
+							},
+							{
+								Key:      "key-8",
+								Values:   []string{"value-8-1", "value-8-2", "value-8-3"},
+								Operator: corev1api.NodeSelectorOpGt,
+							},
+						},
+					},
+					{
+						MatchFields: []corev1api.NodeSelectorRequirement{
+							{
+								Key:      "key-9",
+								Values:   []string{"value-9-1", "value-9-2", "value-9-3"},
+								Operator: corev1api.NodeSelectorOpGt,
+							},
+							{
+								Key:      "key-a",
+								Values:   []string{"value-a-1", "value-a-2", "value-a-3"},
+								Operator: corev1api.NodeSelectorOpGt,
+							},
 						},
 					},
 				},
@@ -862,10 +889,177 @@ func TestToSystemAffinity(t *testing.T) {
 							{
 								MatchExpressions: []corev1api.NodeSelectorRequirement{
 									{
-										Key:      "key-1",
-										Values:   []string{"value-1"},
+										Key:      "key-5",
+										Values:   []string{"value-5-1", "value-5-2", "value-5-3"},
+										Operator: corev1api.NodeSelectorOpGt,
+									},
+									{
+										Key:      "key-6",
+										Values:   []string{"value-5-1", "value-5-2", "value-5-3"},
+										Operator: corev1api.NodeSelectorOpGt,
+									},
+								},
+							},
+							{
+								MatchExpressions: []corev1api.NodeSelectorRequirement{
+									{
+										Key:      "key-7",
+										Values:   []string{"value-7-1", "value-7-2", "value-7-3"},
+										Operator: corev1api.NodeSelectorOpGt,
+									},
+									{
+										Key:      "key-8",
+										Values:   []string{"value-8-1", "value-8-2", "value-8-3"},
+										Operator: corev1api.NodeSelectorOpGt,
+									},
+								},
+							},
+							{
+								MatchFields: []corev1api.NodeSelectorRequirement{
+									{
+										Key:      "key-9",
+										Values:   []string{"value-9-1", "value-9-2", "value-9-3"},
+										Operator: corev1api.NodeSelectorOpGt,
+									},
+									{
+										Key:      "key-a",
+										Values:   []string{"value-a-1", "value-a-2", "value-a-3"},
+										Operator: corev1api.NodeSelectorOpGt,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "with match expression and volume topology",
+			loadAffinity: &LoadAffinity{
+				NodeSelector: metav1.LabelSelector{
+					MatchLabels: map[string]string{
+						"key-2": "value-2",
+					},
+					MatchExpressions: []metav1.LabelSelectorRequirement{
+						{
+							Key:      "key-3",
+							Values:   []string{"value-3-1", "value-3-2"},
+							Operator: metav1.LabelSelectorOpNotIn,
+						},
+						{
+							Key:      "key-4",
+							Values:   []string{"value-4-1", "value-4-2", "value-4-3"},
+							Operator: metav1.LabelSelectorOpDoesNotExist,
+						},
+					},
+				},
+			},
+			volumeTopology: &corev1api.NodeSelector{
+				NodeSelectorTerms: []corev1api.NodeSelectorTerm{
+					{
+						MatchExpressions: []corev1api.NodeSelectorRequirement{
+							{
+								Key:      "key-5",
+								Values:   []string{"value-5-1", "value-5-2", "value-5-3"},
+								Operator: corev1api.NodeSelectorOpGt,
+							},
+							{
+								Key:      "key-6",
+								Values:   []string{"value-5-1", "value-5-2", "value-5-3"},
+								Operator: corev1api.NodeSelectorOpGt,
+							},
+						},
+					},
+					{
+						MatchExpressions: []corev1api.NodeSelectorRequirement{
+							{
+								Key:      "key-7",
+								Values:   []string{"value-7-1", "value-7-2", "value-7-3"},
+								Operator: corev1api.NodeSelectorOpGt,
+							},
+							{
+								Key:      "key-8",
+								Values:   []string{"value-8-1", "value-8-2", "value-8-3"},
+								Operator: corev1api.NodeSelectorOpGt,
+							},
+						},
+					},
+					{
+						MatchFields: []corev1api.NodeSelectorRequirement{
+							{
+								Key:      "key-9",
+								Values:   []string{"value-9-1", "value-9-2", "value-9-3"},
+								Operator: corev1api.NodeSelectorOpGt,
+							},
+							{
+								Key:      "key-a",
+								Values:   []string{"value-a-1", "value-a-2", "value-a-3"},
+								Operator: corev1api.NodeSelectorOpGt,
+							},
+						},
+					},
+				},
+			},
+			expected: &corev1api.Affinity{
+				NodeAffinity: &corev1api.NodeAffinity{
+					RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{
+						NodeSelectorTerms: []corev1api.NodeSelectorTerm{
+							{
+								MatchExpressions: []corev1api.NodeSelectorRequirement{
+									{
+										Key:      "key-5",
+										Values:   []string{"value-5-1", "value-5-2", "value-5-3"},
+										Operator: corev1api.NodeSelectorOpGt,
+									},
+									{
+										Key:      "key-6",
+										Values:   []string{"value-5-1", "value-5-2", "value-5-3"},
+										Operator: corev1api.NodeSelectorOpGt,
+									},
+									{
+										Key:      "key-2",
+										Values:   []string{"value-2"},
 										Operator: corev1api.NodeSelectorOpIn,
 									},
+									{
+										Key:      "key-3",
+										Values:   []string{"value-3-1", "value-3-2"},
+										Operator: corev1api.NodeSelectorOpNotIn,
+									},
+									{
+										Key:      "key-4",
+										Values:   []string{"value-4-1", "value-4-2", "value-4-3"},
+										Operator: corev1api.NodeSelectorOpDoesNotExist,
+									},
+								},
+							},
+							{
+								MatchExpressions: []corev1api.NodeSelectorRequirement{
+									{
+										Key:      "key-7",
+										Values:   []string{"value-7-1", "value-7-2", "value-7-3"},
+										Operator: corev1api.NodeSelectorOpGt,
+									},
+									{
+										Key:      "key-8",
+										Values:   []string{"value-8-1", "value-8-2", "value-8-3"},
+										Operator: corev1api.NodeSelectorOpGt,
+									},
+									{
+										Key:      "key-2",
+										Values:   []string{"value-2"},
+										Operator: corev1api.NodeSelectorOpIn,
+									},
+									{
+										Key:      "key-3",
+										Values:   []string{"value-3-1", "value-3-2"},
+										Operator: corev1api.NodeSelectorOpNotIn,
+									},
+									{
+										Key:      "key-4",
+										Values:   []string{"value-4-1", "value-4-2", "value-4-3"},
+										Operator: corev1api.NodeSelectorOpDoesNotExist,
+									},
 								},
 							},
 							{
@@ -875,6 +1069,28 @@ func TestToSystemAffinity(t *testing.T) {
 										Values:   []string{"value-2"},
 										Operator: corev1api.NodeSelectorOpIn,
 									},
+									{
+										Key:      "key-3",
+										Values:   []string{"value-3-1", "value-3-2"},
+										Operator: corev1api.NodeSelectorOpNotIn,
+									},
+									{
+										Key:      "key-4",
+										Values:   []string{"value-4-1", "value-4-2", "value-4-3"},
+										Operator: corev1api.NodeSelectorOpDoesNotExist,
+									},
+								},
+								MatchFields: []corev1api.NodeSelectorRequirement{
+									{
+										Key:      "key-9",
+										Values:   []string{"value-9-1", "value-9-2", "value-9-3"},
+										Operator: corev1api.NodeSelectorOpGt,
+									},
+									{
+										Key:      "key-a",
+										Values:   []string{"value-a-1", "value-a-2", "value-a-3"},
+										Operator: corev1api.NodeSelectorOpGt,
+									},
 								},
 							},
 						},
@@ -886,7 +1102,7 @@ func TestToSystemAffinity(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			affinity := ToSystemAffinity(test.loadAffinities)
+			affinity := ToSystemAffinity(test.loadAffinity, test.volumeTopology)
 			assert.True(t, reflect.DeepEqual(affinity, test.expected))
 		})
 	}

pkg/util/kube/pvc_pv.go

@@ -580,3 +580,25 @@ func GetPVAttachedNodes(ctx context.Context, pv string, storageClient storagev1.
 
 	return nodes, nil
 }
+
+func GetVolumeTopology(ctx context.Context, volumeClient corev1client.CoreV1Interface, storageClient storagev1.StorageV1Interface, pvName string, scName string) (*corev1api.NodeSelector, error) {
+	sc, err := storageClient.StorageClasses().Get(ctx, scName, metav1.GetOptions{})
+	if err != nil {
+		return nil, errors.Wrapf(err, "error getting storage class %s", scName)
+	}
+
+	if sc.VolumeBindingMode == nil || *sc.VolumeBindingMode != storagev1api.VolumeBindingWaitForFirstConsumer {
+		return nil, nil
+	}
+
+	pv, err := volumeClient.PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{})
+	if err != nil {
+		return nil, errors.Wrapf(err, "error getting PV %s", pvName)
+	}
+
+	if pv.Spec.NodeAffinity == nil {
+		return nil, nil
+	}
+
+	return pv.Spec.NodeAffinity.Required, nil
+}

test/e2e/nodeagentconfig/node-agent-config.go

@@ -240,7 +240,7 @@ func (n *NodeAgentConfigTestCase) Backup() error {
 	Expect(backupPodList.Items[0].Spec.PriorityClassName).To(Equal(n.nodeAgentConfigs.PriorityClassName))
 
 	// In backup, only the second element of LoadAffinity array should be used.
-	expectedAffinity := velerokubeutil.ToSystemAffinity(n.nodeAgentConfigs.LoadAffinity[1:])
+	expectedAffinity := velerokubeutil.ToSystemAffinity(n.nodeAgentConfigs.LoadAffinity[1], nil)
 
 	Expect(backupPodList.Items[0].Spec.Affinity).To(Equal(expectedAffinity))
 
@@ -317,7 +317,7 @@ func (n *NodeAgentConfigTestCase) Restore() error {
 	Expect(restorePodList.Items[0].Spec.PriorityClassName).To(Equal(n.nodeAgentConfigs.PriorityClassName))
 
 	// In restore, only the first element of LoadAffinity array should be used.
-	expectedAffinity := velerokubeutil.ToSystemAffinity(n.nodeAgentConfigs.LoadAffinity[:1])
+	expectedAffinity := velerokubeutil.ToSystemAffinity(n.nodeAgentConfigs.LoadAffinity[0], nil)
 
 	Expect(restorePodList.Items[0].Spec.Affinity).To(Equal(expectedAffinity))