issue 6663: changes for configurable data path concurrency

Signed-off-by: Lyndon-Li <lyonghui@vmware.com>

pkg/cmd/cli/nodeagent/server.go

@@ -19,6 +19,7 @@ package nodeagent
 import (
 	"context"
 	"fmt"
+	"math"
 	"net/http"
 	"os"
 	"strings"
@@ -32,6 +33,7 @@ import (
 	storagev1api "k8s.io/api/storage/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -53,7 +55,9 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/cmd"
 	"github.com/vmware-tanzu/velero/pkg/cmd/util/signals"
 	"github.com/vmware-tanzu/velero/pkg/controller"
+	"github.com/vmware-tanzu/velero/pkg/datapath"
 	"github.com/vmware-tanzu/velero/pkg/metrics"
+	"github.com/vmware-tanzu/velero/pkg/nodeagent"
 	"github.com/vmware-tanzu/velero/pkg/repository"
 	"github.com/vmware-tanzu/velero/pkg/util/filesystem"
 	"github.com/vmware-tanzu/velero/pkg/util/logging"
@@ -73,12 +77,14 @@ const (
 
 	defaultResourceTimeout         = 10 * time.Minute
 	defaultDataMoverPrepareTimeout = 30 * time.Minute
+	defaultDataPathConcurrentNum   = 1
 )
 
 type nodeAgentServerConfig struct {
 	metricsAddress          string
 	resourceTimeout         time.Duration
 	dataMoverPrepareTimeout time.Duration
+	dataPathConcurrentNum   int
 }
 
 func NewServerCommand(f client.Factory) *cobra.Command {
@@ -88,6 +94,7 @@ func NewServerCommand(f client.Factory) *cobra.Command {
 		metricsAddress:          defaultMetricsAddress,
 		resourceTimeout:         defaultResourceTimeout,
 		dataMoverPrepareTimeout: defaultDataMoverPrepareTimeout,
+		dataPathConcurrentNum:   defaultDataPathConcurrentNum,
 	}
 
 	command := &cobra.Command{
@@ -114,6 +121,7 @@ func NewServerCommand(f client.Factory) *cobra.Command {
 	command.Flags().Var(formatFlag, "log-format", fmt.Sprintf("The format for log output. Valid values are %s.", strings.Join(formatFlag.AllowedValues(), ", ")))
 	command.Flags().DurationVar(&config.resourceTimeout, "resource-timeout", config.resourceTimeout, "How long to wait for resource processes which are not covered by other specific timeout parameters. Default is 10 minutes.")
 	command.Flags().DurationVar(&config.dataMoverPrepareTimeout, "data-mover-prepare-timeout", config.dataMoverPrepareTimeout, "How long to wait for preparing a DataUpload/DataDownload. Default is 30 minutes.")
+	command.Flags().IntVar(&config.dataPathConcurrentNum, "data-path-concurrent-num", config.dataPathConcurrentNum, "The concurrent number of data path in the current node. Default is 1.")
 
 	return command
 }
@@ -131,6 +139,7 @@ type nodeAgentServer struct {
 	config            nodeAgentServerConfig
 	kubeClient        kubernetes.Interface
 	csiSnapshotClient *snapshotv1client.Clientset
+	dataPathMgr       *datapath.Manager
 }
 
 func newNodeAgentServer(logger logrus.FieldLogger, factory client.Factory, config nodeAgentServerConfig) (*nodeAgentServer, error) {
@@ -217,6 +226,16 @@ func newNodeAgentServer(logger logrus.FieldLogger, factory client.Factory, confi
 	if err != nil {
 		return nil, err
 	}
+
+	dataPathConcurrentNum := config.dataPathConcurrentNum
+	if dataPathConcurrentNum <= 0 {
+		dataPathConcurrentNum = defaultDataPathConcurrentNum
+		s.logger.Warnf("Data path concurrency number %v is invalid, use the default value %v", config.dataPathConcurrentNum, defaultDataPathConcurrentNum)
+	}
+
+	dataPathConcurrentNum = s.getDataPathConcurrentNum(dataPathConcurrentNum, s.logger)
+	s.dataPathMgr = datapath.NewManager(dataPathConcurrentNum)
+
 	return s, nil
 }
 
@@ -261,24 +280,24 @@ func (s *nodeAgentServer) run() {
 
 	credentialGetter := &credentials.CredentialGetter{FromFile: credentialFileStore, FromSecret: credSecretStore}
 	repoEnsurer := repository.NewEnsurer(s.mgr.GetClient(), s.logger, s.config.resourceTimeout)
-	pvbReconciler := controller.NewPodVolumeBackupReconciler(s.mgr.GetClient(), repoEnsurer,
+	pvbReconciler := controller.NewPodVolumeBackupReconciler(s.mgr.GetClient(), s.dataPathMgr, repoEnsurer,
 		credentialGetter, s.nodeName, s.mgr.GetScheme(), s.metrics, s.logger)
 
 	if err := pvbReconciler.SetupWithManager(s.mgr); err != nil {
 		s.logger.Fatal(err, "unable to create controller", "controller", controller.PodVolumeBackup)
 	}
 
-	if err = controller.NewPodVolumeRestoreReconciler(s.mgr.GetClient(), repoEnsurer, credentialGetter, s.logger).SetupWithManager(s.mgr); err != nil {
+	if err = controller.NewPodVolumeRestoreReconciler(s.mgr.GetClient(), s.dataPathMgr, repoEnsurer, credentialGetter, s.logger).SetupWithManager(s.mgr); err != nil {
 		s.logger.WithError(err).Fatal("Unable to create the pod volume restore controller")
 	}
 
-	dataUploadReconciler := controller.NewDataUploadReconciler(s.mgr.GetClient(), s.kubeClient, s.csiSnapshotClient.SnapshotV1(), repoEnsurer, clock.RealClock{}, credentialGetter, s.nodeName, s.fileSystem, s.config.dataMoverPrepareTimeout, s.logger, s.metrics)
+	dataUploadReconciler := controller.NewDataUploadReconciler(s.mgr.GetClient(), s.kubeClient, s.csiSnapshotClient.SnapshotV1(), s.dataPathMgr, repoEnsurer, clock.RealClock{}, credentialGetter, s.nodeName, s.fileSystem, s.config.dataMoverPrepareTimeout, s.logger, s.metrics)
 	s.markDataUploadsCancel(dataUploadReconciler)
 	if err = dataUploadReconciler.SetupWithManager(s.mgr); err != nil {
 		s.logger.WithError(err).Fatal("Unable to create the data upload controller")
 	}
 
-	dataDownloadReconciler := controller.NewDataDownloadReconciler(s.mgr.GetClient(), s.kubeClient, repoEnsurer, credentialGetter, s.nodeName, s.config.dataMoverPrepareTimeout, s.logger, s.metrics)
+	dataDownloadReconciler := controller.NewDataDownloadReconciler(s.mgr.GetClient(), s.kubeClient, s.dataPathMgr, repoEnsurer, credentialGetter, s.nodeName, s.config.dataMoverPrepareTimeout, s.logger, s.metrics)
 	s.markDataDownloadsCancel(dataDownloadReconciler)
 	if err = dataDownloadReconciler.SetupWithManager(s.mgr); err != nil {
 		s.logger.WithError(err).Fatal("Unable to create the data download controller")
@@ -476,3 +495,55 @@ func (s *nodeAgentServer) markInProgressPVRsFailed(client ctrlclient.Client) {
 		s.logger.WithField("podvolumerestore", pvr.GetName()).Warn(pvr.Status.Message)
 	}
 }
+
+func (s *nodeAgentServer) getDataPathConcurrentNum(globalNum int, logger logrus.FieldLogger) int {
+	configs, err := nodeagent.GetConfigs(s.ctx, s.namespace, s.kubeClient.CoreV1())
+	if err != nil {
+		logger.WithError(err).Warn("Failed to get node agent configs")
+	}
+
+	if configs == nil || configs.DataPathConcurrency == nil {
+		logger.Infof("Node agent configs are not found, use the global number %v", globalNum)
+		return globalNum
+	}
+
+	curNode, err := s.kubeClient.CoreV1().Nodes().Get(s.ctx, s.nodeName, metav1.GetOptions{})
+	if err != nil {
+		logger.WithError(err).Warnf("Failed to get node info for %s, use the global number %v", s.nodeName, globalNum)
+		return globalNum
+	}
+
+	selectors := map[labels.Selector]int{}
+	for rule, number := range configs.DataPathConcurrency.ConfigRules {
+		selector, err := labels.Parse(rule)
+		if err != nil {
+			logger.WithError(err).Warnf("Failed to parse rule with label selector %s, skip it", rule)
+			continue
+		}
+
+		if number <= 0 {
+			logger.Warnf("Rule with label selector %s is with an invalid number %v, skip it", rule, number)
+			continue
+		}
+
+		selectors[selector] = number
+	}
+
+	concurrentNum := math.MaxInt32
+	for selector, number := range selectors {
+		if selector.Matches(labels.Set(curNode.GetLabels())) {
+			if concurrentNum > number {
+				concurrentNum = number
+			}
+		}
+	}
+
+	if concurrentNum == math.MaxInt32 {
+		logger.Infof("Per node number for node %s is not found, use the global number %v", s.nodeName, globalNum)
+		concurrentNum = globalNum
+	} else {
+		logger.Infof("Use the per node number %v over global number %v for node %s", concurrentNum, globalNum, s.nodeName)
+	}
+
+	return concurrentNum
+}