From 41687279a8c7ed2a87c95db6e9f5dcc3be7d4a99 Mon Sep 17 00:00:00 2001 From: lyndon-li <98304688+Lyndon-Li@users.noreply.github.com> Date: Wed, 26 Aug 2026 02:14:26 +0800 Subject: [PATCH] fix node agent rediness check issue (#10397) Signed-off-by: Lyndon-Li --- pkg/backup/actions/csi/pvc_action.go | 2 +- pkg/nodeagent/node_agent.go | 35 ++++----- pkg/nodeagent/node_agent_test.go | 108 ++++++++++++++------------- 3 files changed, 77 insertions(+), 68 deletions(-) diff --git a/pkg/backup/actions/csi/pvc_action.go b/pkg/backup/actions/csi/pvc_action.go index 817844bfa..01f4e3d1a 100644 --- a/pkg/backup/actions/csi/pvc_action.go +++ b/pkg/backup/actions/csi/pvc_action.go @@ -351,7 +351,7 @@ func (p *pvcBackupItemAction) Execute( // created but never processed (the DataUpload controller runs inside node-agent), // causing the backup to hang until itemOperationTimeout expires. if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) && datamover.IsBuiltInDataMover(backup.Spec.DataMover) { - if err := nodeagent.IsReady(ctx, backup.Namespace, p.crClient, p.log); err != nil { + if err := nodeagent.IsReady(ctx, backup.Namespace, p.crClient); err != nil { p.log.WithError(err).Error("cannot perform snapshot data movement without running node-agent pods") return nil, nil, "", nil, errors.Wrap(err, "CSI PVC BIA cannot proceed: node-agent is not ready for snapshot data movement") } diff --git a/pkg/nodeagent/node_agent.go b/pkg/nodeagent/node_agent.go index 61dff9299..b449a91f4 100644 --- a/pkg/nodeagent/node_agent.go +++ b/pkg/nodeagent/node_agent.go @@ -22,7 +22,6 @@ import ( "fmt" "github.com/cockroachdb/errors" - "github.com/sirupsen/logrus" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -83,28 +82,30 @@ func KbClientIsRunningInNode(ctx context.Context, namespace string, nodeName str } // IsReady checks whether the node-agent daemonset has at least one ready pod -// by inspecting the DaemonSet status. It only checks the daemonset for node -// OS types that are present in the cluster, following the same pattern as -// server.checkNodeAgent. -func IsReady(ctx context.Context, namespace string, crClient ctrlclient.Client, log logrus.FieldLogger) error { - if kube.WithLinuxNode(ctx, crClient, log) { - ds := new(appsv1api.DaemonSet) - if err := crClient.Get(ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: daemonSet}, ds); err != nil { +// by inspecting the DaemonSet status. +func IsReady(ctx context.Context, namespace string, crClient ctrlclient.Client) error { + dsLinux := new(appsv1api.DaemonSet) + if err := crClient.Get(ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: daemonSet}, dsLinux); err != nil { + dsLinux = nil + if !apierrors.IsNotFound(err) { return errors.Wrap(err, "failed to get linux node-agent daemonset") } - if ds.Status.NumberReady > 0 { - return nil - } } - if kube.WithWindowsNode(ctx, crClient, log) { - ds := new(appsv1api.DaemonSet) - if err := crClient.Get(ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: daemonsetWindows}, ds); err != nil { + dsWindows := new(appsv1api.DaemonSet) + if err := crClient.Get(ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: daemonsetWindows}, dsWindows); err != nil { + dsWindows = nil + if !apierrors.IsNotFound(err) { return errors.Wrap(err, "failed to get windows node-agent daemonset") } - if ds.Status.NumberReady > 0 { - return nil - } + } + + if dsLinux != nil && dsLinux.Status.NumberReady > 0 { + return nil + } + + if dsWindows != nil && dsWindows.Status.NumberReady > 0 { + return nil } return errors.New("node-agent is not ready: no ready pods found") diff --git a/pkg/nodeagent/node_agent_test.go b/pkg/nodeagent/node_agent_test.go index a523bf15a..9bba67ec4 100644 --- a/pkg/nodeagent/node_agent_test.go +++ b/pkg/nodeagent/node_agent_test.go @@ -17,10 +17,10 @@ limitations under the License. package nodeagent import ( + "context" "testing" "github.com/cockroachdb/errors" - "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" @@ -29,7 +29,9 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/fake" clientTesting "k8s.io/client-go/testing" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "github.com/vmware-tanzu/velero/pkg/builder" velerotypes "github.com/vmware-tanzu/velero/pkg/types" @@ -217,22 +219,6 @@ func TestIsRunningInNode(t *testing.T) { func TestIsReady(t *testing.T) { scheme := runtime.NewScheme() appsv1api.AddToScheme(scheme) - corev1api.AddToScheme(scheme) - - log := logrus.New() - - linuxNode := &corev1api.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "linux-node", - Labels: map[string]string{kube.NodeOSLabel: kube.NodeOSLinux}, - }, - } - windowsNode := &corev1api.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "windows-node", - Labels: map[string]string{kube.NodeOSLabel: kube.NodeOSWindows}, - }, - } dsLinuxNotReady := &appsv1api.DaemonSet{ ObjectMeta: metav1.ObjectMeta{Namespace: "fake-ns", Name: "node-agent"}, @@ -255,96 +241,118 @@ func TestIsReady(t *testing.T) { name string kubeClientObj []runtime.Object namespace string + interceptor *interceptor.Funcs expectErr string }{ { - name: "no nodes in cluster", + name: "both daemonsets not found", namespace: "fake-ns", expectErr: "node-agent is not ready: no ready pods found", }, { - name: "linux node exists but daemonset not found", + name: "linux daemonset get error", namespace: "fake-ns", - kubeClientObj: []runtime.Object{ - linuxNode, + interceptor: &interceptor.Funcs{ + Get: func(ctx context.Context, c ctrlclient.WithWatch, key ctrlclient.ObjectKey, obj ctrlclient.Object, opts ...ctrlclient.GetOption) error { + if key.Name == "node-agent" { + return errors.New("fake-get-error") + } + return c.Get(ctx, key, obj, opts...) + }, }, - expectErr: "failed to get linux node-agent daemonset", + expectErr: "failed to get linux node-agent daemonset: fake-get-error", }, { - name: "linux node and daemonset exist but no ready pods", + name: "windows daemonset get error", + namespace: "fake-ns", + interceptor: &interceptor.Funcs{ + Get: func(ctx context.Context, c ctrlclient.WithWatch, key ctrlclient.ObjectKey, obj ctrlclient.Object, opts ...ctrlclient.GetOption) error { + if key.Name == "node-agent-windows" { + return errors.New("fake-get-error") + } + return c.Get(ctx, key, obj, opts...) + }, + }, + expectErr: "failed to get windows node-agent daemonset: fake-get-error", + }, + { + name: "linux ds exist but no ready pods", namespace: "fake-ns", kubeClientObj: []runtime.Object{ - linuxNode, dsLinuxNotReady, }, expectErr: "node-agent is not ready: no ready pods found", }, { - name: "linux node and daemonset with ready pods", + name: "linux ds with ready pods", namespace: "fake-ns", kubeClientObj: []runtime.Object{ - linuxNode, dsLinuxReady, }, }, { - name: "windows node and daemonset with ready pods", + name: "windows ds exist but no ready pods", namespace: "fake-ns", kubeClientObj: []runtime.Object{ - windowsNode, - dsWindowsReady, - }, - }, - { - name: "windows node and daemonset with no ready pods", - namespace: "fake-ns", - kubeClientObj: []runtime.Object{ - windowsNode, dsWindowsNotReady, }, expectErr: "node-agent is not ready: no ready pods found", }, { - name: "both node types with both daemonsets ready", + name: "windows ds with ready pods", namespace: "fake-ns", kubeClientObj: []runtime.Object{ - linuxNode, - windowsNode, - dsLinuxReady, dsWindowsReady, }, }, { - name: "both node types but neither daemonset has ready pods", + name: "both daemonsets exist but no ready pods", namespace: "fake-ns", kubeClientObj: []runtime.Object{ - linuxNode, - windowsNode, dsLinuxNotReady, dsWindowsNotReady, }, expectErr: "node-agent is not ready: no ready pods found", }, { - name: "linux not ready but windows ready", + name: "both daemonsets exist, linux ready", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + dsLinuxReady, + dsWindowsNotReady, + }, + }, + { + name: "both daemonsets exist, windows ready", namespace: "fake-ns", kubeClientObj: []runtime.Object{ - linuxNode, - windowsNode, dsLinuxNotReady, dsWindowsReady, }, }, + { + name: "both daemonsets exist, both ready", + namespace: "fake-ns", + kubeClientObj: []runtime.Object{ + dsLinuxReady, + dsWindowsReady, + }, + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - fakeClient := clientFake.NewClientBuilder(). + builder := clientFake.NewClientBuilder(). WithScheme(scheme). - WithRuntimeObjects(test.kubeClientObj...). - Build() + WithRuntimeObjects(test.kubeClientObj...) - err := IsReady(t.Context(), test.namespace, fakeClient, log) + if test.interceptor != nil { + builder = builder.WithInterceptorFuncs(*test.interceptor) + } + + fakeClient := builder.Build() + + err := IsReady(t.Context(), test.namespace, fakeClient) if test.expectErr == "" { assert.NoError(t, err) } else {