diff --git a/pkg/exposer/csi_snapshot.go b/pkg/exposer/csi_snapshot.go index 5acb229d2..637cd7c07 100644 --- a/pkg/exposer/csi_snapshot.go +++ b/pkg/exposer/csi_snapshot.go @@ -124,6 +124,15 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O "owner": ownerObject.Name, }) + volumeTopology, err := kube.GetVolumeTopology(ctx, e.kubeClient.CoreV1(), e.kubeClient.StorageV1(), csiExposeParam.SourcePVName, csiExposeParam.StorageClass) + if err != nil { + return errors.Wrapf(err, "error getting volume topology for PV %s, storage class %s", csiExposeParam.SourcePVName, csiExposeParam.StorageClass) + } + + if volumeTopology != nil { + curLog.Infof("Using volume topology %v", volumeTopology) + } + curLog.Info("Exposing CSI snapshot") volumeSnapshot, err := csi.WaitVolumeSnapshotReady(ctx, e.csiSnapshotClient, csiExposeParam.SnapshotName, csiExposeParam.SourceNamespace, csiExposeParam.ExposeTimeout, curLog) @@ -254,6 +263,7 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1api.O csiExposeParam.NodeOS, csiExposeParam.PriorityClassName, intoleratableNodes, + volumeTopology, ) if err != nil { return errors.Wrap(err, "error to create backup pod") @@ -588,6 +598,7 @@ func (e *csiSnapshotExposer) createBackupPod( nodeOS string, priorityClassName string, intoleratableNodes []string, + volumeTopology *corev1api.NodeSelector, ) (*corev1api.Pod, error) { podName := ownerObject.Name @@ -701,7 +712,7 @@ func (e *csiSnapshotExposer) createBackupPod( } if affinity != nil { - podAffinity = kube.ToSystemAffinity([]*kube.LoadAffinity{affinity}) + podAffinity = kube.ToSystemAffinity(affinity, volumeTopology) } pod := &corev1api.Pod{ diff --git a/pkg/exposer/csi_snapshot_priority_test.go b/pkg/exposer/csi_snapshot_priority_test.go index 345d5b327..d1ffa4700 100644 --- a/pkg/exposer/csi_snapshot_priority_test.go +++ b/pkg/exposer/csi_snapshot_priority_test.go @@ -154,6 +154,7 @@ func TestCreateBackupPodWithPriorityClass(t *testing.T) { kube.NodeOSLinux, tc.expectedPriorityClass, nil, + nil, ) require.NoError(t, err, tc.description) @@ -239,6 +240,7 @@ func TestCreateBackupPodWithMissingConfigMap(t *testing.T) { kube.NodeOSLinux, "", // empty priority class since config map is missing nil, + nil, ) // Should succeed even when config map is missing diff --git a/pkg/exposer/generic_restore.go b/pkg/exposer/generic_restore.go index c10370072..f7cf24d8d 100644 --- a/pkg/exposer/generic_restore.go +++ b/pkg/exposer/generic_restore.go @@ -498,7 +498,7 @@ func (e *genericRestoreExposer) createRestorePod( e.log.Infof("No selected node for restore pod. Try to get affinity from the node-agent config.") if affinity != nil { - podAffinity = kube.ToSystemAffinity([]*kube.LoadAffinity{affinity}) + podAffinity = kube.ToSystemAffinity(affinity, nil) } } diff --git a/pkg/repository/maintenance/maintenance.go b/pkg/repository/maintenance/maintenance.go index 496d07703..426cb44d4 100644 --- a/pkg/repository/maintenance/maintenance.go +++ b/pkg/repository/maintenance/maintenance.go @@ -671,8 +671,7 @@ func buildJob( } if config != nil && len(config.LoadAffinities) > 0 { - // Maintenance job only takes the first loadAffinity. - affinity := kube.ToSystemAffinity([]*kube.LoadAffinity{config.LoadAffinities[0]}) + affinity := kube.ToSystemAffinity(config.LoadAffinities[0], nil) job.Spec.Template.Spec.Affinity = affinity } diff --git a/pkg/util/kube/pod.go b/pkg/util/kube/pod.go index 2aeb45a1c..b5cd9a62c 100644 --- a/pkg/util/kube/pod.go +++ b/pkg/util/kube/pod.go @@ -230,14 +230,9 @@ func CollectPodLogs(ctx context.Context, podGetter corev1client.CoreV1Interface, return nil } -func ToSystemAffinity(loadAffinities []*LoadAffinity) *corev1api.Affinity { - if len(loadAffinities) == 0 { - return nil - } - nodeSelectorTermList := make([]corev1api.NodeSelectorTerm, 0) - - for _, loadAffinity := range loadAffinities { - requirements := []corev1api.NodeSelectorRequirement{} +func ToSystemAffinity(loadAffinity *LoadAffinity, volumeTopolpogy *corev1api.NodeSelector) *corev1api.Affinity { + requirements := []corev1api.NodeSelectorRequirement{} + if loadAffinity != nil { for k, v := range loadAffinity.NodeSelector.MatchLabels { requirements = append(requirements, corev1api.NodeSelectorRequirement{ Key: k, @@ -253,25 +248,25 @@ func ToSystemAffinity(loadAffinities []*LoadAffinity) *corev1api.Affinity { Operator: corev1api.NodeSelectorOperator(exp.Operator), }) } - - nodeSelectorTermList = append( - nodeSelectorTermList, - corev1api.NodeSelectorTerm{ - MatchExpressions: requirements, - }, - ) } - if len(nodeSelectorTermList) > 0 { - result := new(corev1api.Affinity) - result.NodeAffinity = new(corev1api.NodeAffinity) - result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = new(corev1api.NodeSelector) - result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = nodeSelectorTermList + result := new(corev1api.Affinity) + result.NodeAffinity = new(corev1api.NodeAffinity) + result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = new(corev1api.NodeSelector) - return result + if volumeTopolpogy != nil { + result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append(result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, volumeTopolpogy.NodeSelectorTerms...) + } else if len(requirements) > 0 { + result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = make([]corev1api.NodeSelectorTerm, 1) + } else { + return nil } - return nil + for i := range result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { + result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[i].MatchExpressions = append(result.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[i].MatchExpressions, requirements...) + } + + return result } func DiagnosePod(pod *corev1api.Pod, events *corev1api.EventList) string { diff --git a/pkg/util/kube/pod_test.go b/pkg/util/kube/pod_test.go index 6751e8b6e..aa8d4db99 100644 --- a/pkg/util/kube/pod_test.go +++ b/pkg/util/kube/pod_test.go @@ -747,24 +747,23 @@ func TestCollectPodLogs(t *testing.T) { func TestToSystemAffinity(t *testing.T) { tests := []struct { name string - loadAffinities []*LoadAffinity + loadAffinity *LoadAffinity + volumeTopology *corev1api.NodeSelector expected *corev1api.Affinity }{ { name: "loadAffinity is nil", }, { - name: "loadAffinity is empty", - loadAffinities: []*LoadAffinity{}, + name: "loadAffinity is empty", + loadAffinity: &LoadAffinity{}, }, { name: "with match label", - loadAffinities: []*LoadAffinity{ - { - NodeSelector: metav1.LabelSelector{ - MatchLabels: map[string]string{ - "key-1": "value-1", - }, + loadAffinity: &LoadAffinity{ + NodeSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "key-1": "value-1", }, }, }, @@ -788,23 +787,21 @@ func TestToSystemAffinity(t *testing.T) { }, { name: "with match expression", - loadAffinities: []*LoadAffinity{ - { - NodeSelector: metav1.LabelSelector{ - MatchLabels: map[string]string{ - "key-2": "value-2", + loadAffinity: &LoadAffinity{ + NodeSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "key-2": "value-2", + }, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "key-3", + Values: []string{"value-3-1", "value-3-2"}, + Operator: metav1.LabelSelectorOpNotIn, }, - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "key-3", - Values: []string{"value-3-1", "value-3-2"}, - Operator: metav1.LabelSelectorOpNotIn, - }, - { - Key: "key-4", - Values: []string{"value-4-1", "value-4-2", "value-4-3"}, - Operator: metav1.LabelSelectorOpDoesNotExist, - }, + { + Key: "key-4", + Values: []string{"value-4-1", "value-4-2", "value-4-3"}, + Operator: metav1.LabelSelectorOpDoesNotExist, }, }, }, @@ -838,19 +835,49 @@ func TestToSystemAffinity(t *testing.T) { }, }, { - name: "multiple load affinities", - loadAffinities: []*LoadAffinity{ - { - NodeSelector: metav1.LabelSelector{ - MatchLabels: map[string]string{ - "key-1": "value-1", + name: "with olume topology", + volumeTopology: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "key-5", + Values: []string{"value-5-1", "value-5-2", "value-5-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-6", + Values: []string{"value-5-1", "value-5-2", "value-5-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, }, }, - }, - { - NodeSelector: metav1.LabelSelector{ - MatchLabels: map[string]string{ - "key-2": "value-2", + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "key-7", + Values: []string{"value-7-1", "value-7-2", "value-7-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-8", + Values: []string{"value-8-1", "value-8-2", "value-8-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + }, + }, + { + MatchFields: []corev1api.NodeSelectorRequirement{ + { + Key: "key-9", + Values: []string{"value-9-1", "value-9-2", "value-9-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-a", + Values: []string{"value-a-1", "value-a-2", "value-a-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, }, }, }, @@ -862,10 +889,177 @@ func TestToSystemAffinity(t *testing.T) { { MatchExpressions: []corev1api.NodeSelectorRequirement{ { - Key: "key-1", - Values: []string{"value-1"}, + Key: "key-5", + Values: []string{"value-5-1", "value-5-2", "value-5-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-6", + Values: []string{"value-5-1", "value-5-2", "value-5-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + }, + }, + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "key-7", + Values: []string{"value-7-1", "value-7-2", "value-7-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-8", + Values: []string{"value-8-1", "value-8-2", "value-8-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + }, + }, + { + MatchFields: []corev1api.NodeSelectorRequirement{ + { + Key: "key-9", + Values: []string{"value-9-1", "value-9-2", "value-9-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-a", + Values: []string{"value-a-1", "value-a-2", "value-a-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "with match expression and volume topology", + loadAffinity: &LoadAffinity{ + NodeSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "key-2": "value-2", + }, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "key-3", + Values: []string{"value-3-1", "value-3-2"}, + Operator: metav1.LabelSelectorOpNotIn, + }, + { + Key: "key-4", + Values: []string{"value-4-1", "value-4-2", "value-4-3"}, + Operator: metav1.LabelSelectorOpDoesNotExist, + }, + }, + }, + }, + volumeTopology: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "key-5", + Values: []string{"value-5-1", "value-5-2", "value-5-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-6", + Values: []string{"value-5-1", "value-5-2", "value-5-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + }, + }, + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "key-7", + Values: []string{"value-7-1", "value-7-2", "value-7-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-8", + Values: []string{"value-8-1", "value-8-2", "value-8-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + }, + }, + { + MatchFields: []corev1api.NodeSelectorRequirement{ + { + Key: "key-9", + Values: []string{"value-9-1", "value-9-2", "value-9-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-a", + Values: []string{"value-a-1", "value-a-2", "value-a-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + }, + }, + }, + }, + expected: &corev1api.Affinity{ + NodeAffinity: &corev1api.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1api.NodeSelector{ + NodeSelectorTerms: []corev1api.NodeSelectorTerm{ + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "key-5", + Values: []string{"value-5-1", "value-5-2", "value-5-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-6", + Values: []string{"value-5-1", "value-5-2", "value-5-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-2", + Values: []string{"value-2"}, Operator: corev1api.NodeSelectorOpIn, }, + { + Key: "key-3", + Values: []string{"value-3-1", "value-3-2"}, + Operator: corev1api.NodeSelectorOpNotIn, + }, + { + Key: "key-4", + Values: []string{"value-4-1", "value-4-2", "value-4-3"}, + Operator: corev1api.NodeSelectorOpDoesNotExist, + }, + }, + }, + { + MatchExpressions: []corev1api.NodeSelectorRequirement{ + { + Key: "key-7", + Values: []string{"value-7-1", "value-7-2", "value-7-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-8", + Values: []string{"value-8-1", "value-8-2", "value-8-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-2", + Values: []string{"value-2"}, + Operator: corev1api.NodeSelectorOpIn, + }, + { + Key: "key-3", + Values: []string{"value-3-1", "value-3-2"}, + Operator: corev1api.NodeSelectorOpNotIn, + }, + { + Key: "key-4", + Values: []string{"value-4-1", "value-4-2", "value-4-3"}, + Operator: corev1api.NodeSelectorOpDoesNotExist, + }, }, }, { @@ -875,6 +1069,28 @@ func TestToSystemAffinity(t *testing.T) { Values: []string{"value-2"}, Operator: corev1api.NodeSelectorOpIn, }, + { + Key: "key-3", + Values: []string{"value-3-1", "value-3-2"}, + Operator: corev1api.NodeSelectorOpNotIn, + }, + { + Key: "key-4", + Values: []string{"value-4-1", "value-4-2", "value-4-3"}, + Operator: corev1api.NodeSelectorOpDoesNotExist, + }, + }, + MatchFields: []corev1api.NodeSelectorRequirement{ + { + Key: "key-9", + Values: []string{"value-9-1", "value-9-2", "value-9-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, + { + Key: "key-a", + Values: []string{"value-a-1", "value-a-2", "value-a-3"}, + Operator: corev1api.NodeSelectorOpGt, + }, }, }, }, @@ -886,7 +1102,7 @@ func TestToSystemAffinity(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - affinity := ToSystemAffinity(test.loadAffinities) + affinity := ToSystemAffinity(test.loadAffinity, test.volumeTopology) assert.True(t, reflect.DeepEqual(affinity, test.expected)) }) } diff --git a/pkg/util/kube/pvc_pv.go b/pkg/util/kube/pvc_pv.go index 5209cf6ea..c82040ed4 100644 --- a/pkg/util/kube/pvc_pv.go +++ b/pkg/util/kube/pvc_pv.go @@ -580,3 +580,25 @@ func GetPVAttachedNodes(ctx context.Context, pv string, storageClient storagev1. return nodes, nil } + +func GetVolumeTopology(ctx context.Context, volumeClient corev1client.CoreV1Interface, storageClient storagev1.StorageV1Interface, pvName string, scName string) (*corev1api.NodeSelector, error) { + sc, err := storageClient.StorageClasses().Get(ctx, scName, metav1.GetOptions{}) + if err != nil { + return nil, errors.Wrapf(err, "error getting storage class %s", scName) + } + + if sc.VolumeBindingMode == nil || *sc.VolumeBindingMode != storagev1api.VolumeBindingWaitForFirstConsumer { + return nil, nil + } + + pv, err := volumeClient.PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) + if err != nil { + return nil, errors.Wrapf(err, "error getting PV %s", pvName) + } + + if pv.Spec.NodeAffinity == nil { + return nil, nil + } + + return pv.Spec.NodeAffinity.Required, nil +} diff --git a/test/e2e/nodeagentconfig/node-agent-config.go b/test/e2e/nodeagentconfig/node-agent-config.go index 1b46eed65..01cc6e38c 100644 --- a/test/e2e/nodeagentconfig/node-agent-config.go +++ b/test/e2e/nodeagentconfig/node-agent-config.go @@ -240,7 +240,7 @@ func (n *NodeAgentConfigTestCase) Backup() error { Expect(backupPodList.Items[0].Spec.PriorityClassName).To(Equal(n.nodeAgentConfigs.PriorityClassName)) // In backup, only the second element of LoadAffinity array should be used. - expectedAffinity := velerokubeutil.ToSystemAffinity(n.nodeAgentConfigs.LoadAffinity[1:]) + expectedAffinity := velerokubeutil.ToSystemAffinity(n.nodeAgentConfigs.LoadAffinity[1], nil) Expect(backupPodList.Items[0].Spec.Affinity).To(Equal(expectedAffinity)) @@ -317,7 +317,7 @@ func (n *NodeAgentConfigTestCase) Restore() error { Expect(restorePodList.Items[0].Spec.PriorityClassName).To(Equal(n.nodeAgentConfigs.PriorityClassName)) // In restore, only the first element of LoadAffinity array should be used. - expectedAffinity := velerokubeutil.ToSystemAffinity(n.nodeAgentConfigs.LoadAffinity[:1]) + expectedAffinity := velerokubeutil.ToSystemAffinity(n.nodeAgentConfigs.LoadAffinity[0], nil) Expect(restorePodList.Items[0].Spec.Affinity).To(Equal(expectedAffinity))