mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-08-28 03:46:14 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix node agent rediness check issue (#10397)

Browse Source 
Signed-off-by: Lyndon-Li <lyonghui@vmware.com>
This commit is contained in:
lyndon-li
2026-08-25 14:14:26 -04:00
committed by GitHub
parent 2e692dfd3a
commit 41687279a8
3 changed files with 77 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/backup/actions/csi/pvc_action.go
+1 -1
View File
@@ -351,7 +351,7 @@ func (p *pvcBackupItemAction) Execute(
// created but never processed (the DataUpload controller runs inside node-agent),
// causing the backup to hang until itemOperationTimeout expires.
if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) && datamover.IsBuiltInDataMover(backup.Spec.DataMover) {
if err := nodeagent.IsReady(ctx, backup.Namespace, p.crClient, p.log); err != nil {
if err := nodeagent.IsReady(ctx, backup.Namespace, p.crClient); err != nil {
p.log.WithError(err).Error("cannot perform snapshot data movement without running node-agent pods")
return nil, nil, "", nil, errors.Wrap(err, "CSI PVC BIA cannot proceed: node-agent is not ready for snapshot data movement")
}
pkg/nodeagent/node_agent.go
+18 -17
View File
@@ -22,7 +22,6 @@ import (
"fmt"
"github.com/cockroachdb/errors"
"github.com/sirupsen/logrus"
appsv1api "k8s.io/api/apps/v1"
corev1api "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -83,28 +82,30 @@ func KbClientIsRunningInNode(ctx context.Context, namespace string, nodeName str
}
// IsReady checks whether the node-agent daemonset has at least one ready pod
// by inspecting the DaemonSet status. It only checks the daemonset for node
// OS types that are present in the cluster, following the same pattern as
// server.checkNodeAgent.
func IsReady(ctx context.Context, namespace string, crClient ctrlclient.Client, log logrus.FieldLogger) error {
if kube.WithLinuxNode(ctx, crClient, log) {
ds := new(appsv1api.DaemonSet)
if err := crClient.Get(ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: daemonSet}, ds); err != nil {
// by inspecting the DaemonSet status.
func IsReady(ctx context.Context, namespace string, crClient ctrlclient.Client) error {
dsLinux := new(appsv1api.DaemonSet)
if err := crClient.Get(ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: daemonSet}, dsLinux); err != nil {
dsLinux = nil
if !apierrors.IsNotFound(err) {
return errors.Wrap(err, "failed to get linux node-agent daemonset")
}
if ds.Status.NumberReady > 0 {
return nil
}
}
if kube.WithWindowsNode(ctx, crClient, log) {
ds := new(appsv1api.DaemonSet)
if err := crClient.Get(ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: daemonsetWindows}, ds); err != nil {
dsWindows := new(appsv1api.DaemonSet)
if err := crClient.Get(ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: daemonsetWindows}, dsWindows); err != nil {
dsWindows = nil
if !apierrors.IsNotFound(err) {
return errors.Wrap(err, "failed to get windows node-agent daemonset")
}
if ds.Status.NumberReady > 0 {
return nil
}
}
if dsLinux != nil && dsLinux.Status.NumberReady > 0 {
return nil
}
if dsWindows != nil && dsWindows.Status.NumberReady > 0 {
return nil
}
return errors.New("node-agent is not ready: no ready pods found")
pkg/nodeagent/node_agent_test.go
+58 -50
View File
@@ -17,10 +17,10 @@ limitations under the License.
package nodeagent
import (
"context"
"testing"
"github.com/cockroachdb/errors"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
appsv1api "k8s.io/api/apps/v1"
@@ -29,7 +29,9 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/fake"
clientTesting "k8s.io/client-go/testing"
ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
clientFake "sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
"github.com/vmware-tanzu/velero/pkg/builder"
velerotypes "github.com/vmware-tanzu/velero/pkg/types"
@@ -217,22 +219,6 @@ func TestIsRunningInNode(t *testing.T) {
func TestIsReady(t *testing.T) {
scheme := runtime.NewScheme()
appsv1api.AddToScheme(scheme)
corev1api.AddToScheme(scheme)
log := logrus.New()
linuxNode := &corev1api.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "linux-node",
Labels: map[string]string{kube.NodeOSLabel: kube.NodeOSLinux},
},
}
windowsNode := &corev1api.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "windows-node",
Labels: map[string]string{kube.NodeOSLabel: kube.NodeOSWindows},
},
}
dsLinuxNotReady := &appsv1api.DaemonSet{
ObjectMeta: metav1.ObjectMeta{Namespace: "fake-ns", Name: "node-agent"},
@@ -255,96 +241,118 @@ func TestIsReady(t *testing.T) {
name string
kubeClientObj []runtime.Object
namespace string
interceptor *interceptor.Funcs
expectErr string
}{
{
name: "no nodes in cluster",
name: "both daemonsets not found",
namespace: "fake-ns",
expectErr: "node-agent is not ready: no ready pods found",
},
{
name: "linux node exists but daemonset not found",
name: "linux daemonset get error",
namespace: "fake-ns",
kubeClientObj: []runtime.Object{
linuxNode,
interceptor: &interceptor.Funcs{
Get: func(ctx context.Context, c ctrlclient.WithWatch, key ctrlclient.ObjectKey, obj ctrlclient.Object, opts ...ctrlclient.GetOption) error {
if key.Name == "node-agent" {
return errors.New("fake-get-error")
}
return c.Get(ctx, key, obj, opts...)
},
},
expectErr: "failed to get linux node-agent daemonset",
expectErr: "failed to get linux node-agent daemonset: fake-get-error",
},
{
name: "linux node and daemonset exist but no ready pods",
name: "windows daemonset get error",
namespace: "fake-ns",
interceptor: &interceptor.Funcs{
Get: func(ctx context.Context, c ctrlclient.WithWatch, key ctrlclient.ObjectKey, obj ctrlclient.Object, opts ...ctrlclient.GetOption) error {
if key.Name == "node-agent-windows" {
return errors.New("fake-get-error")
}
return c.Get(ctx, key, obj, opts...)
},
},
expectErr: "failed to get windows node-agent daemonset: fake-get-error",
},
{
name: "linux ds exist but no ready pods",
namespace: "fake-ns",
kubeClientObj: []runtime.Object{
linuxNode,
dsLinuxNotReady,
},
expectErr: "node-agent is not ready: no ready pods found",
},
{
name: "linux node and daemonset with ready pods",
name: "linux ds with ready pods",
namespace: "fake-ns",
kubeClientObj: []runtime.Object{
linuxNode,
dsLinuxReady,
},
},
{
name: "windows node and daemonset with ready pods",
name: "windows ds exist but no ready pods",
namespace: "fake-ns",
kubeClientObj: []runtime.Object{
windowsNode,
dsWindowsReady,
},
},
{
name: "windows node and daemonset with no ready pods",
namespace: "fake-ns",
kubeClientObj: []runtime.Object{
windowsNode,
dsWindowsNotReady,
},
expectErr: "node-agent is not ready: no ready pods found",
},
{
name: "both node types with both daemonsets ready",
name: "windows ds with ready pods",
namespace: "fake-ns",
kubeClientObj: []runtime.Object{
linuxNode,
windowsNode,
dsLinuxReady,
dsWindowsReady,
},
},
{
name: "both node types but neither daemonset has ready pods",
name: "both daemonsets exist but no ready pods",
namespace: "fake-ns",
kubeClientObj: []runtime.Object{
linuxNode,
windowsNode,
dsLinuxNotReady,
dsWindowsNotReady,
},
expectErr: "node-agent is not ready: no ready pods found",
},
{
name: "linux not ready but windows ready",
name: "both daemonsets exist, linux ready",
namespace: "fake-ns",
kubeClientObj: []runtime.Object{
dsLinuxReady,
dsWindowsNotReady,
},
},
{
name: "both daemonsets exist, windows ready",
namespace: "fake-ns",
kubeClientObj: []runtime.Object{
linuxNode,
windowsNode,
dsLinuxNotReady,
dsWindowsReady,
},
},
{
name: "both daemonsets exist, both ready",
namespace: "fake-ns",
kubeClientObj: []runtime.Object{
dsLinuxReady,
dsWindowsReady,
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
fakeClient := clientFake.NewClientBuilder().
builder := clientFake.NewClientBuilder().
WithScheme(scheme).
WithRuntimeObjects(test.kubeClientObj...).
Build()
WithRuntimeObjects(test.kubeClientObj...)
err := IsReady(t.Context(), test.namespace, fakeClient, log)
if test.interceptor != nil {
builder = builder.WithInterceptorFuncs(*test.interceptor)
}
fakeClient := builder.Build()
err := IsReady(t.Context(), test.namespace, fakeClient)
if test.expectErr == "" {
assert.NoError(t, err)
} else {