diff --git a/changelogs/unreleased/6523-Lyndon-Li b/changelogs/unreleased/6523-Lyndon-Li
new file mode 100644
index 000000000..c1fd3f8d9
--- /dev/null
+++ b/changelogs/unreleased/6523-Lyndon-Li
@@ -0,0 +1 @@
+Fix issue #6519. Restrict the client manager of node-agent server to include only Velero resources from the server's namespace, otherwise, the controllers will try to reconcile CRs from all the installed Velero namespaces.
\ No newline at end of file
diff --git a/pkg/cmd/cli/nodeagent/server.go b/pkg/cmd/cli/nodeagent/server.go
index ffbb5ca74..a7b3e053d 100644
--- a/pkg/cmd/cli/nodeagent/server.go
+++ b/pkg/cmd/cli/nodeagent/server.go
@@ -169,6 +169,18 @@ func newNodeAgentServer(logger logrus.FieldLogger, factory client.Factory, confi
 			&v1.Pod{}: {
 				Field: fields.Set{"spec.nodeName": nodeName}.AsSelector(),
 			},
+			&velerov1api.PodVolumeBackup{}: {
+				Field: fields.Set{"metadata.namespace": factory.Namespace()}.AsSelector(),
+			},
+			&velerov1api.PodVolumeRestore{}: {
+				Field: fields.Set{"metadata.namespace": factory.Namespace()}.AsSelector(),
+			},
+			&velerov2alpha1api.DataUpload{}: {
+				Field: fields.Set{"metadata.namespace": factory.Namespace()}.AsSelector(),
+			},
+			&velerov2alpha1api.DataDownload{}: {
+				Field: fields.Set{"metadata.namespace": factory.Namespace()}.AsSelector(),
+			},
 		},
 	}
 	mgr, err := ctrl.NewManager(clientConfig, ctrl.Options{