Merge branch 'main' into issue-fix-6693

pkg/cmd/cli/nodeagent/server.go

@@ -19,6 +19,7 @@ package nodeagent
 import (
 	"context"
 	"fmt"
+	"math"
 	"net/http"
 	"os"
 	"strings"
@@ -32,6 +33,7 @@ import (
 	storagev1api "k8s.io/api/storage/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -53,7 +55,9 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/cmd"
 	"github.com/vmware-tanzu/velero/pkg/cmd/util/signals"
 	"github.com/vmware-tanzu/velero/pkg/controller"
+	"github.com/vmware-tanzu/velero/pkg/datapath"
 	"github.com/vmware-tanzu/velero/pkg/metrics"
+	"github.com/vmware-tanzu/velero/pkg/nodeagent"
 	"github.com/vmware-tanzu/velero/pkg/repository"
 	"github.com/vmware-tanzu/velero/pkg/util/filesystem"
 	"github.com/vmware-tanzu/velero/pkg/util/logging"
@@ -73,6 +77,7 @@ const (
 
 	defaultResourceTimeout         = 10 * time.Minute
 	defaultDataMoverPrepareTimeout = 30 * time.Minute
+	defaultDataPathConcurrentNum   = 1
 )
 
 type nodeAgentServerConfig struct {
@@ -132,6 +137,7 @@ type nodeAgentServer struct {
 	config            nodeAgentServerConfig
 	kubeClient        kubernetes.Interface
 	csiSnapshotClient *snapshotv1client.Clientset
+	dataPathMgr       *datapath.Manager
 }
 
 func newNodeAgentServer(logger logrus.FieldLogger, factory client.Factory, config nodeAgentServerConfig) (*nodeAgentServer, error) {
@@ -219,6 +225,10 @@ func newNodeAgentServer(logger logrus.FieldLogger, factory client.Factory, confi
 	if err != nil {
 		return nil, err
 	}
+
+	dataPathConcurrentNum := s.getDataPathConcurrentNum(defaultDataPathConcurrentNum)
+	s.dataPathMgr = datapath.NewManager(dataPathConcurrentNum)
+
 	return s, nil
 }
 
@@ -263,24 +273,24 @@ func (s *nodeAgentServer) run() {
 
 	credentialGetter := &credentials.CredentialGetter{FromFile: credentialFileStore, FromSecret: credSecretStore}
 	repoEnsurer := repository.NewEnsurer(s.mgr.GetClient(), s.logger, s.config.resourceTimeout)
-	pvbReconciler := controller.NewPodVolumeBackupReconciler(s.mgr.GetClient(), repoEnsurer,
+	pvbReconciler := controller.NewPodVolumeBackupReconciler(s.mgr.GetClient(), s.dataPathMgr, repoEnsurer,
 		credentialGetter, s.nodeName, s.mgr.GetScheme(), s.metrics, s.logger)
 
 	if err := pvbReconciler.SetupWithManager(s.mgr); err != nil {
 		s.logger.Fatal(err, "unable to create controller", "controller", controller.PodVolumeBackup)
 	}
 
-	if err = controller.NewPodVolumeRestoreReconciler(s.mgr.GetClient(), repoEnsurer, credentialGetter, s.logger).SetupWithManager(s.mgr); err != nil {
+	if err = controller.NewPodVolumeRestoreReconciler(s.mgr.GetClient(), s.dataPathMgr, repoEnsurer, credentialGetter, s.logger).SetupWithManager(s.mgr); err != nil {
 		s.logger.WithError(err).Fatal("Unable to create the pod volume restore controller")
 	}
 
-	dataUploadReconciler := controller.NewDataUploadReconciler(s.mgr.GetClient(), s.kubeClient, s.csiSnapshotClient.SnapshotV1(), repoEnsurer, clock.RealClock{}, credentialGetter, s.nodeName, s.fileSystem, s.config.dataMoverPrepareTimeout, s.logger, s.metrics)
+	dataUploadReconciler := controller.NewDataUploadReconciler(s.mgr.GetClient(), s.kubeClient, s.csiSnapshotClient.SnapshotV1(), s.dataPathMgr, repoEnsurer, clock.RealClock{}, credentialGetter, s.nodeName, s.fileSystem, s.config.dataMoverPrepareTimeout, s.logger, s.metrics)
 	s.markDataUploadsCancel(dataUploadReconciler)
 	if err = dataUploadReconciler.SetupWithManager(s.mgr); err != nil {
 		s.logger.WithError(err).Fatal("Unable to create the data upload controller")
 	}
 
-	dataDownloadReconciler := controller.NewDataDownloadReconciler(s.mgr.GetClient(), s.kubeClient, repoEnsurer, credentialGetter, s.nodeName, s.config.dataMoverPrepareTimeout, s.logger, s.metrics)
+	dataDownloadReconciler := controller.NewDataDownloadReconciler(s.mgr.GetClient(), s.kubeClient, s.dataPathMgr, repoEnsurer, credentialGetter, s.nodeName, s.config.dataMoverPrepareTimeout, s.logger, s.metrics)
 	s.markDataDownloadsCancel(dataDownloadReconciler)
 	if err = dataDownloadReconciler.SetupWithManager(s.mgr); err != nil {
 		s.logger.WithError(err).Fatal("Unable to create the data download controller")
@@ -478,3 +488,65 @@ func (s *nodeAgentServer) markInProgressPVRsFailed(client ctrlclient.Client) {
 		s.logger.WithField("podvolumerestore", pvr.GetName()).Warn(pvr.Status.Message)
 	}
 }
+
+var getConfigsFunc = nodeagent.GetConfigs
+
+func (s *nodeAgentServer) getDataPathConcurrentNum(defaultNum int) int {
+	configs, err := getConfigsFunc(s.ctx, s.namespace, s.kubeClient)
+	if err != nil {
+		s.logger.WithError(err).Warn("Failed to get node agent configs")
+		return defaultNum
+	}
+
+	if configs == nil || configs.DataPathConcurrency == nil {
+		s.logger.Infof("Concurrency configs are not found, use the default number %v", defaultNum)
+		return defaultNum
+	}
+
+	globalNum := configs.DataPathConcurrency.GlobalConfig
+
+	if globalNum <= 0 {
+		s.logger.Warnf("Global number %v is invalid, use the default value %v", globalNum, defaultNum)
+		globalNum = defaultNum
+	}
+
+	if len(configs.DataPathConcurrency.PerNodeConfig) == 0 {
+		return globalNum
+	}
+
+	curNode, err := s.kubeClient.CoreV1().Nodes().Get(s.ctx, s.nodeName, metav1.GetOptions{})
+	if err != nil {
+		s.logger.WithError(err).Warnf("Failed to get node info for %s, use the global number %v", s.nodeName, globalNum)
+		return globalNum
+	}
+
+	concurrentNum := math.MaxInt32
+
+	for _, rule := range configs.DataPathConcurrency.PerNodeConfig {
+		selector, err := metav1.LabelSelectorAsSelector(&rule.NodeSelector)
+		if err != nil {
+			s.logger.WithError(err).Warnf("Failed to parse rule with label selector %s, skip it", rule.NodeSelector.String())
+			continue
+		}
+
+		if rule.Number <= 0 {
+			s.logger.Warnf("Rule with label selector %s is with an invalid number %v, skip it", rule.NodeSelector.String(), rule.Number)
+			continue
+		}
+
+		if selector.Matches(labels.Set(curNode.GetLabels())) {
+			if concurrentNum > rule.Number {
+				concurrentNum = rule.Number
+			}
+		}
+	}
+
+	if concurrentNum == math.MaxInt32 {
+		s.logger.Infof("Per node number for node %s is not found, use the global number %v", s.nodeName, globalNum)
+		concurrentNum = globalNum
+	} else {
+		s.logger.Infof("Use the per node number %v over global number %v for node %s", concurrentNum, globalNum, s.nodeName)
+	}
+
+	return concurrentNum
+}

pkg/cmd/cli/nodeagent/server_test.go

@@ -17,16 +17,22 @@ package nodeagent
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 
+	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/fake"
 
 	"github.com/vmware-tanzu/velero/pkg/builder"
+	"github.com/vmware-tanzu/velero/pkg/nodeagent"
 	testutil "github.com/vmware-tanzu/velero/pkg/test"
 )
 
@@ -107,3 +113,259 @@ func Test_validatePodVolumesHostPath(t *testing.T) {
 		})
 	}
 }
+
+func Test_getDataPathConcurrentNum(t *testing.T) {
+	defaultNum := 100001
+	globalNum := 6
+	nodeName := "node-agent-node"
+	node1 := builder.ForNode("node-agent-node").Result()
+	node2 := builder.ForNode("node-agent-node").Labels(map[string]string{
+		"host-name": "node-1",
+		"xxxx":      "yyyyy",
+	}).Result()
+
+	invalidLabelSelector := metav1.LabelSelector{
+		MatchLabels: map[string]string{
+			"inva/lid": "inva/lid",
+		},
+	}
+	validLabelSelector1 := metav1.LabelSelector{
+		MatchLabels: map[string]string{
+			"host-name": "node-1",
+		},
+	}
+	validLabelSelector2 := metav1.LabelSelector{
+		MatchLabels: map[string]string{
+			"xxxx": "yyyyy",
+		},
+	}
+
+	tests := []struct {
+		name          string
+		getFunc       func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error)
+		setKubeClient bool
+		kubeClientObj []runtime.Object
+		expectNum     int
+		expectLog     string
+	}{
+		{
+			name: "failed to get configs",
+			getFunc: func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error) {
+				return nil, errors.New("fake-get-error")
+			},
+			expectLog: "Failed to get node agent configs",
+			expectNum: defaultNum,
+		},
+		{
+			name: "configs cm not found",
+			getFunc: func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error) {
+				return nil, nil
+			},
+			expectLog: fmt.Sprintf("Concurrency configs are not found, use the default number %v", defaultNum),
+			expectNum: defaultNum,
+		},
+		{
+			name: "configs cm's data path concurrency is nil",
+			getFunc: func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error) {
+				return &nodeagent.Configs{}, nil
+			},
+			expectLog: fmt.Sprintf("Concurrency configs are not found, use the default number %v", defaultNum),
+			expectNum: defaultNum,
+		},
+		{
+			name: "global number is invalid",
+			getFunc: func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error) {
+				return &nodeagent.Configs{
+					DataPathConcurrency: &nodeagent.DataPathConcurrency{
+						GlobalConfig: -1,
+					},
+				}, nil
+			},
+			expectLog: fmt.Sprintf("Global number %v is invalid, use the default value %v", -1, defaultNum),
+			expectNum: defaultNum,
+		},
+		{
+			name: "global number is valid",
+			getFunc: func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error) {
+				return &nodeagent.Configs{
+					DataPathConcurrency: &nodeagent.DataPathConcurrency{
+						GlobalConfig: globalNum,
+					},
+				}, nil
+			},
+			expectNum: globalNum,
+		},
+		{
+			name: "node is not found",
+			getFunc: func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error) {
+				return &nodeagent.Configs{
+					DataPathConcurrency: &nodeagent.DataPathConcurrency{
+						GlobalConfig: globalNum,
+						PerNodeConfig: []nodeagent.RuledConfigs{
+							{
+								Number: 100,
+							},
+						},
+					},
+				}, nil
+			},
+			setKubeClient: true,
+			expectLog:     fmt.Sprintf("Failed to get node info for %s, use the global number %v", nodeName, globalNum),
+			expectNum:     globalNum,
+		},
+		{
+			name: "failed to get selector",
+			getFunc: func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error) {
+				return &nodeagent.Configs{
+					DataPathConcurrency: &nodeagent.DataPathConcurrency{
+						GlobalConfig: globalNum,
+						PerNodeConfig: []nodeagent.RuledConfigs{
+							{
+								NodeSelector: invalidLabelSelector,
+								Number:       100,
+							},
+						},
+					},
+				}, nil
+			},
+			setKubeClient: true,
+			kubeClientObj: []runtime.Object{node1},
+			expectLog:     fmt.Sprintf("Failed to parse rule with label selector %s, skip it", invalidLabelSelector.String()),
+			expectNum:     globalNum,
+		},
+		{
+			name: "rule number is invalid",
+			getFunc: func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error) {
+				return &nodeagent.Configs{
+					DataPathConcurrency: &nodeagent.DataPathConcurrency{
+						GlobalConfig: globalNum,
+						PerNodeConfig: []nodeagent.RuledConfigs{
+							{
+								NodeSelector: validLabelSelector1,
+								Number:       -1,
+							},
+						},
+					},
+				}, nil
+			},
+			setKubeClient: true,
+			kubeClientObj: []runtime.Object{node1},
+			expectLog:     fmt.Sprintf("Rule with label selector %s is with an invalid number %v, skip it", validLabelSelector1.String(), -1),
+			expectNum:     globalNum,
+		},
+		{
+			name: "label doesn't match",
+			getFunc: func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error) {
+				return &nodeagent.Configs{
+					DataPathConcurrency: &nodeagent.DataPathConcurrency{
+						GlobalConfig: globalNum,
+						PerNodeConfig: []nodeagent.RuledConfigs{
+							{
+								NodeSelector: validLabelSelector1,
+								Number:       -1,
+							},
+						},
+					},
+				}, nil
+			},
+			setKubeClient: true,
+			kubeClientObj: []runtime.Object{node1},
+			expectLog:     fmt.Sprintf("Per node number for node %s is not found, use the global number %v", nodeName, globalNum),
+			expectNum:     globalNum,
+		},
+		{
+			name: "match one rule",
+			getFunc: func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error) {
+				return &nodeagent.Configs{
+					DataPathConcurrency: &nodeagent.DataPathConcurrency{
+						GlobalConfig: globalNum,
+						PerNodeConfig: []nodeagent.RuledConfigs{
+							{
+								NodeSelector: validLabelSelector1,
+								Number:       66,
+							},
+						},
+					},
+				}, nil
+			},
+			setKubeClient: true,
+			kubeClientObj: []runtime.Object{node2},
+			expectLog:     fmt.Sprintf("Use the per node number %v over global number %v for node %s", 66, globalNum, nodeName),
+			expectNum:     66,
+		},
+		{
+			name: "match multiple rules",
+			getFunc: func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error) {
+				return &nodeagent.Configs{
+					DataPathConcurrency: &nodeagent.DataPathConcurrency{
+						GlobalConfig: globalNum,
+						PerNodeConfig: []nodeagent.RuledConfigs{
+							{
+								NodeSelector: validLabelSelector1,
+								Number:       66,
+							},
+							{
+								NodeSelector: validLabelSelector2,
+								Number:       36,
+							},
+						},
+					},
+				}, nil
+			},
+			setKubeClient: true,
+			kubeClientObj: []runtime.Object{node2},
+			expectLog:     fmt.Sprintf("Use the per node number %v over global number %v for node %s", 36, globalNum, nodeName),
+			expectNum:     36,
+		},
+		{
+			name: "match multiple rules 2",
+			getFunc: func(context.Context, string, kubernetes.Interface) (*nodeagent.Configs, error) {
+				return &nodeagent.Configs{
+					DataPathConcurrency: &nodeagent.DataPathConcurrency{
+						GlobalConfig: globalNum,
+						PerNodeConfig: []nodeagent.RuledConfigs{
+							{
+								NodeSelector: validLabelSelector1,
+								Number:       36,
+							},
+							{
+								NodeSelector: validLabelSelector2,
+								Number:       66,
+							},
+						},
+					},
+				}, nil
+			},
+			setKubeClient: true,
+			kubeClientObj: []runtime.Object{node2},
+			expectLog:     fmt.Sprintf("Use the per node number %v over global number %v for node %s", 36, globalNum, nodeName),
+			expectNum:     36,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			fakeKubeClient := fake.NewSimpleClientset(test.kubeClientObj...)
+
+			logBuffer := ""
+
+			s := &nodeAgentServer{
+				nodeName: nodeName,
+				logger:   testutil.NewSingleLogger(&logBuffer),
+			}
+
+			if test.setKubeClient {
+				s.kubeClient = fakeKubeClient
+			}
+
+			getConfigsFunc = test.getFunc
+
+			num := s.getDataPathConcurrentNum(defaultNum)
+			assert.Equal(t, test.expectNum, num)
+			if test.expectLog == "" {
+				assert.Equal(t, "", logBuffer)
+			} else {
+				assert.True(t, strings.Contains(logBuffer, test.expectLog))
+			}
+		})
+	}
+}