From b170892e648d694471f781f6a338aec56041ffbd Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Thu, 6 Mar 2025 13:04:49 +0800 Subject: [PATCH] issue 8754: add third party annotation support Signed-off-by: Lyndon-Li --- changelogs/unreleased/8770-Lyndon-Li | 1 + pkg/controller/data_download_controller.go | 28 +++-- pkg/controller/data_upload_controller.go | 36 ++++-- pkg/exposer/csi_snapshot.go | 8 +- pkg/exposer/generic_restore.go | 10 +- pkg/nodeagent/node_agent.go | 28 ++++- pkg/nodeagent/node_agent_test.go | 129 +++++++++++++++++++++ pkg/util/third_party.go | 4 + 8 files changed, 218 insertions(+), 26 deletions(-) create mode 100644 changelogs/unreleased/8770-Lyndon-Li diff --git a/changelogs/unreleased/8770-Lyndon-Li b/changelogs/unreleased/8770-Lyndon-Li new file mode 100644 index 000000000..e40cb5583 --- /dev/null +++ b/changelogs/unreleased/8770-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #8754, add third party annotation support for data mover \ No newline at end of file diff --git a/pkg/controller/data_download_controller.go b/pkg/controller/data_download_controller.go index 3d5b2965a..b05b3f47a 100644 --- a/pkg/controller/data_download_controller.go +++ b/pkg/controller/data_download_controller.go @@ -748,15 +748,27 @@ func (r *DataDownloadReconciler) setupExposeParam(dd *velerov2alpha1api.DataDown } } + hostingPodAnnotation := map[string]string{} + for _, k := range util.ThirdPartyAnnotations { + if v, err := nodeagent.GetAnnotationValue(context.Background(), r.kubeClient, dd.Namespace, k, nodeOS); err != nil { + if err != nodeagent.ErrNodeAgentAnnotationNotFound { + log.WithError(err).Warnf("Failed to check node-agent annotation, skip adding host pod annotation %s", k) + } + } else { + hostingPodAnnotation[k] = v + } + } + return exposer.GenericRestoreExposeParam{ - TargetPVCName: dd.Spec.TargetVolume.PVC, - TargetNamespace: dd.Spec.TargetVolume.Namespace, - HostingPodLabels: hostingPodLabels, - Resources: r.podResources, - OperationTimeout: dd.Spec.OperationTimeout.Duration, - ExposeTimeout: r.preparingTimeout, - NodeOS: nodeOS, - RestorePVCConfig: r.restorePVCConfig, + TargetPVCName: dd.Spec.TargetVolume.PVC, + TargetNamespace: dd.Spec.TargetVolume.Namespace, + HostingPodLabels: hostingPodLabels, + HostingPodAnnotations: hostingPodAnnotation, + Resources: r.podResources, + OperationTimeout: dd.Spec.OperationTimeout.Duration, + ExposeTimeout: r.preparingTimeout, + NodeOS: nodeOS, + RestorePVCConfig: r.restorePVCConfig, }, nil } diff --git a/pkg/controller/data_upload_controller.go b/pkg/controller/data_upload_controller.go index 182d9da8c..96ac91e1b 100644 --- a/pkg/controller/data_upload_controller.go +++ b/pkg/controller/data_upload_controller.go @@ -835,19 +835,31 @@ func (r *DataUploadReconciler) setupExposeParam(du *velerov2alpha1api.DataUpload } } + hostingPodAnnotation := map[string]string{} + for _, k := range util.ThirdPartyAnnotations { + if v, err := nodeagent.GetAnnotationValue(context.Background(), r.kubeClient, du.Namespace, k, nodeOS); err != nil { + if err != nodeagent.ErrNodeAgentAnnotationNotFound { + log.WithError(err).Warnf("Failed to check node-agent annotation, skip adding host pod annotation %s", k) + } + } else { + hostingPodAnnotation[k] = v + } + } + return &exposer.CSISnapshotExposeParam{ - SnapshotName: du.Spec.CSISnapshot.VolumeSnapshot, - SourceNamespace: du.Spec.SourceNamespace, - StorageClass: du.Spec.CSISnapshot.StorageClass, - HostingPodLabels: hostingPodLabels, - AccessMode: accessMode, - OperationTimeout: du.Spec.OperationTimeout.Duration, - ExposeTimeout: r.preparingTimeout, - VolumeSize: pvc.Spec.Resources.Requests[corev1.ResourceStorage], - Affinity: r.loadAffinity, - BackupPVCConfig: r.backupPVCConfig, - Resources: r.podResources, - NodeOS: nodeOS, + SnapshotName: du.Spec.CSISnapshot.VolumeSnapshot, + SourceNamespace: du.Spec.SourceNamespace, + StorageClass: du.Spec.CSISnapshot.StorageClass, + HostingPodLabels: hostingPodLabels, + HostingPodAnnotations: hostingPodAnnotation, + AccessMode: accessMode, + OperationTimeout: du.Spec.OperationTimeout.Duration, + ExposeTimeout: r.preparingTimeout, + VolumeSize: pvc.Spec.Resources.Requests[corev1.ResourceStorage], + Affinity: r.loadAffinity, + BackupPVCConfig: r.backupPVCConfig, + Resources: r.podResources, + NodeOS: nodeOS, }, nil } diff --git a/pkg/exposer/csi_snapshot.go b/pkg/exposer/csi_snapshot.go index e8676c976..df5936c3b 100644 --- a/pkg/exposer/csi_snapshot.go +++ b/pkg/exposer/csi_snapshot.go @@ -56,6 +56,9 @@ type CSISnapshotExposeParam struct { // HostingPodLabels is the labels that are going to apply to the hosting pod HostingPodLabels map[string]string + // HostingPodAnnotations is the annotations that are going to apply to the hosting pod + HostingPodAnnotations map[string]string + // OperationTimeout specifies the time wait for resources operations in Expose OperationTimeout time.Duration @@ -211,6 +214,7 @@ func (e *csiSnapshotExposer) Expose(ctx context.Context, ownerObject corev1.Obje backupPVC, csiExposeParam.OperationTimeout, csiExposeParam.HostingPodLabels, + csiExposeParam.HostingPodAnnotations, csiExposeParam.Affinity, csiExposeParam.Resources, backupPVCReadOnly, @@ -523,6 +527,7 @@ func (e *csiSnapshotExposer) createBackupPod( backupPVC *corev1.PersistentVolumeClaim, operationTimeout time.Duration, label map[string]string, + annotation map[string]string, affinity *kube.LoadAffinity, resources corev1.ResourceRequirements, backupPVCReadOnly bool, @@ -633,7 +638,8 @@ func (e *csiSnapshotExposer) createBackupPod( Controller: boolptr.True(), }, }, - Labels: label, + Labels: label, + Annotations: annotation, }, Spec: corev1.PodSpec{ TopologySpreadConstraints: []corev1.TopologySpreadConstraint{ diff --git a/pkg/exposer/generic_restore.go b/pkg/exposer/generic_restore.go index 06c680d8c..2445ecc85 100644 --- a/pkg/exposer/generic_restore.go +++ b/pkg/exposer/generic_restore.go @@ -46,6 +46,9 @@ type GenericRestoreExposeParam struct { // HostingPodLabels is the labels that are going to apply to the hosting pod HostingPodLabels map[string]string + // HostingPodAnnotations is the annotations that are going to apply to the hosting pod + HostingPodAnnotations map[string]string + // Resources defines the resource requirements of the hosting pod Resources corev1.ResourceRequirements @@ -119,7 +122,7 @@ func (e *genericRestoreExposer) Expose(ctx context.Context, ownerObject corev1.O return errors.Errorf("Target PVC %s/%s has already been bound, abort", param.TargetNamespace, param.TargetPVCName) } - restorePod, err := e.createRestorePod(ctx, ownerObject, targetPVC, param.OperationTimeout, param.HostingPodLabels, selectedNode, param.Resources, param.NodeOS) + restorePod, err := e.createRestorePod(ctx, ownerObject, targetPVC, param.OperationTimeout, param.HostingPodLabels, param.HostingPodAnnotations, selectedNode, param.Resources, param.NodeOS) if err != nil { return errors.Wrapf(err, "error to create restore pod") } @@ -374,7 +377,7 @@ func (e *genericRestoreExposer) RebindVolume(ctx context.Context, ownerObject co } func (e *genericRestoreExposer) createRestorePod(ctx context.Context, ownerObject corev1.ObjectReference, targetPVC *corev1.PersistentVolumeClaim, - operationTimeout time.Duration, label map[string]string, selectedNode string, resources corev1.ResourceRequirements, nodeType string) (*corev1.Pod, error) { + operationTimeout time.Duration, label map[string]string, annotation map[string]string, selectedNode string, resources corev1.ResourceRequirements, nodeType string) (*corev1.Pod, error) { restorePodName := ownerObject.Name restorePVCName := ownerObject.Name @@ -464,7 +467,8 @@ func (e *genericRestoreExposer) createRestorePod(ctx context.Context, ownerObjec Controller: boolptr.True(), }, }, - Labels: label, + Labels: label, + Annotations: annotation, }, Spec: corev1.PodSpec{ TopologySpreadConstraints: []corev1.TopologySpreadConstraint{ diff --git a/pkg/nodeagent/node_agent.go b/pkg/nodeagent/node_agent.go index 44d0cebd2..1eadc435b 100644 --- a/pkg/nodeagent/node_agent.go +++ b/pkg/nodeagent/node_agent.go @@ -44,8 +44,9 @@ const ( ) var ( - ErrDaemonSetNotFound = errors.New("daemonset not found") - ErrNodeAgentLabelNotFound = errors.New("node-agent label not found") + ErrDaemonSetNotFound = errors.New("daemonset not found") + ErrNodeAgentLabelNotFound = errors.New("node-agent label not found") + ErrNodeAgentAnnotationNotFound = errors.New("node-agent annotation not found") ) type LoadConcurrency struct { @@ -225,3 +226,26 @@ func GetLabelValue(ctx context.Context, kubeClient kubernetes.Interface, namespa return val, nil } + +func GetAnnotationValue(ctx context.Context, kubeClient kubernetes.Interface, namespace string, key string, osType string) (string, error) { + dsName := daemonSet + if osType == kube.NodeOSWindows { + dsName = daemonsetWindows + } + + ds, err := kubeClient.AppsV1().DaemonSets(namespace).Get(ctx, dsName, metav1.GetOptions{}) + if err != nil { + return "", errors.Wrapf(err, "error getting %s daemonset", dsName) + } + + if ds.Spec.Template.Annotations == nil { + return "", ErrNodeAgentAnnotationNotFound + } + + val, found := ds.Spec.Template.Annotations[key] + if !found { + return "", ErrNodeAgentAnnotationNotFound + } + + return val, nil +} diff --git a/pkg/nodeagent/node_agent_test.go b/pkg/nodeagent/node_agent_test.go index a153e1e8c..9ea905789 100644 --- a/pkg/nodeagent/node_agent_test.go +++ b/pkg/nodeagent/node_agent_test.go @@ -461,3 +461,132 @@ func TestGetLabelValue(t *testing.T) { }) } } + +func TestGetAnnotationValue(t *testing.T) { + daemonSet := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "fake-ns", + Name: "node-agent", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "DaemonSet", + }, + } + + daemonSetWithOtherAnnotation := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "fake-ns", + Name: "node-agent", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "DaemonSet", + }, + Spec: appsv1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "fake-other-annotation": "fake-value-1", + }, + }, + }, + }, + } + + daemonSetWithAnnotation := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "fake-ns", + Name: "node-agent", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "DaemonSet", + }, + Spec: appsv1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "fake-annotation": "fake-value-2", + }, + }, + }, + }, + } + + daemonSetWithEmptyAnnotation := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "fake-ns", + Name: "node-agent", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "DaemonSet", + }, + Spec: appsv1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "fake-annotation": "", + }, + }, + }, + }, + } + + tests := []struct { + name string + kubeClientObj []runtime.Object + namespace string + expectedValue string + expectErr string + }{ + { + name: "ds get error", + namespace: "fake-ns", + expectErr: "error getting node-agent daemonset: daemonsets.apps \"node-agent\" not found", + }, + { + name: "no annotation", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + daemonSet, + }, + expectErr: ErrNodeAgentAnnotationNotFound.Error(), + }, + { + name: "no expecting annotation", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + daemonSetWithOtherAnnotation, + }, + expectErr: ErrNodeAgentAnnotationNotFound.Error(), + }, + { + name: "expecting annotation", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + daemonSetWithAnnotation, + }, + expectedValue: "fake-value-2", + }, + { + name: "expecting empty annotation", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + daemonSetWithEmptyAnnotation, + }, + expectedValue: "", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...) + + value, err := GetAnnotationValue(context.TODO(), fakeKubeClient, test.namespace, "fake-annotation", kube.NodeOSLinux) + if test.expectErr == "" { + assert.NoError(t, err) + assert.Equal(t, test.expectedValue, value) + } else { + assert.EqualError(t, err, test.expectErr) + } + }) + } +} diff --git a/pkg/util/third_party.go b/pkg/util/third_party.go index b0b73d68e..04f30ccb0 100644 --- a/pkg/util/third_party.go +++ b/pkg/util/third_party.go @@ -19,3 +19,7 @@ package util var ThirdPartyLabels = []string{ "azure.workload.identity/use", } + +var ThirdPartyAnnotations = []string{ + "iam.amazonaws.com/role", +}