Enable parameterized kubelet mount path during node-agent installation (#9074)

Enable parameterized kubelet mount path during node-agent installation

Signed-off-by: longyuxiang <longyuxiang@kylinos.cn>

changelogs/unreleased/9074-longxiucai

@@ -0,0 +1 @@
+Enable parameterized kubelet mount path during node-agent installation

pkg/cmd/cli/install/install.go

@@ -90,6 +90,7 @@ type Options struct {
 	NodeAgentConfigMap              string
 	ItemBlockWorkerCount            int
 	NodeAgentDisableHostPath        bool
+	kubeletRootDir                  string
 }
 
 // BindFlags adds command line values to the options struct.
@@ -114,6 +115,8 @@ func (o *Options) BindFlags(flags *pflag.FlagSet) {
 	flags.StringVar(&o.NodeAgentPodMemRequest, "node-agent-pod-mem-request", o.NodeAgentPodMemRequest, `Memory request for node-agent pod. A value of "0" is treated as unbounded. Optional.`)
 	flags.StringVar(&o.NodeAgentPodCPULimit, "node-agent-pod-cpu-limit", o.NodeAgentPodCPULimit, `CPU limit for node-agent pod. A value of "0" is treated as unbounded. Optional.`)
 	flags.StringVar(&o.NodeAgentPodMemLimit, "node-agent-pod-mem-limit", o.NodeAgentPodMemLimit, `Memory limit for node-agent pod. A value of "0" is treated as unbounded. Optional.`)
+	flags.StringVar(&o.kubeletRootDir, "kubelet-root-dir", o.kubeletRootDir, `Kubelet root directory for the node agent. Optional.`)
+
 	flags.Var(&o.BackupStorageConfig, "backup-location-config", "Configuration to use for the backup storage location. Format is key1=value1,key2=value2")
 	flags.Var(&o.VolumeSnapshotConfig, "snapshot-location-config", "Configuration to use for the volume snapshot location. Format is key1=value1,key2=value2")
 	flags.BoolVar(&o.UseVolumeSnapshots, "use-volume-snapshots", o.UseVolumeSnapshots, "Whether or not to create snapshot location automatically. Set to false if you do not plan to create volume snapshots via a storage provider.")
@@ -220,6 +223,7 @@ func NewInstallOptions() *Options {
 		DefaultSnapshotMoveData:  false,
 		DisableInformerCache:     false,
 		ScheduleSkipImmediately:  false,
+		kubeletRootDir:           install.DefaultKubeletRootDir,
 		NodeAgentDisableHostPath: false,
 	}
 }
@@ -295,6 +299,7 @@ func (o *Options) AsVeleroOptions() (*install.VeleroOptions, error) {
 		RepoMaintenanceJobConfigMap:     o.RepoMaintenanceJobConfigMap,
 		NodeAgentConfigMap:              o.NodeAgentConfigMap,
 		ItemBlockWorkerCount:            o.ItemBlockWorkerCount,
+		KubeletRootDir:                  o.kubeletRootDir,
 		NodeAgentDisableHostPath:        o.NodeAgentDisableHostPath,
 	}, nil
 }

pkg/install/daemonset.go

@@ -18,6 +18,7 @@ package install
 
 import (
 	"fmt"
+	"path/filepath"
 	"strings"
 
 	appsv1api "k8s.io/api/apps/v1"
@@ -63,7 +64,8 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1api.DaemonSet
 	if c.forWindows {
 		dsName = "node-agent-windows"
 	}
-
+	hostPodsVolumePath := filepath.Join(c.kubeletRootDir, "pods")
+	hostPluginsVolumePath := filepath.Join(c.kubeletRootDir, "plugins")
 	volumes := []corev1api.Volume{}
 	volumeMounts := []corev1api.VolumeMount{}
 	if !c.nodeAgentDisableHostPath {
@@ -72,7 +74,7 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1api.DaemonSet
 				Name: "host-pods",
 				VolumeSource: corev1api.VolumeSource{
 					HostPath: &corev1api.HostPathVolumeSource{
-						Path: "/var/lib/kubelet/pods",
+						Path: hostPodsVolumePath,
 					},
 				},
 			},
@@ -80,7 +82,7 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1api.DaemonSet
 				Name: "host-plugins",
 				VolumeSource: corev1api.VolumeSource{
 					HostPath: &corev1api.HostPathVolumeSource{
-						Path: "/var/lib/kubelet/plugins",
+						Path: hostPluginsVolumePath,
 					},
 				},
 			},

pkg/install/daemonset_test.go

@@ -63,6 +63,10 @@ func TestDaemonSet(t *testing.T) {
 	ds = DaemonSet("velero", WithServiceAccountName("test-sa"))
 	assert.Equal(t, "test-sa", ds.Spec.Template.Spec.ServiceAccountName)
 
+	ds = DaemonSet("velero", WithKubeletRootDir("/data/test/kubelet"))
+	assert.Equal(t, "/data/test/kubelet/pods", ds.Spec.Template.Spec.Volumes[0].HostPath.Path)
+	assert.Equal(t, "/data/test/kubelet/plugins", ds.Spec.Template.Spec.Volumes[1].HostPath.Path)
+
 	ds = DaemonSet("velero", WithNodeAgentDisableHostPath(true))
 	assert.Len(t, ds.Spec.Template.Spec.Volumes, 1)
 	assert.Len(t, ds.Spec.Template.Spec.Containers[0].VolumeMounts, 1)

pkg/install/deployment.go

@@ -60,6 +60,7 @@ type podTemplateConfig struct {
 	nodeAgentConfigMap              string
 	itemBlockWorkerCount            int
 	forWindows                      bool
+	kubeletRootDir                  string
 	nodeAgentDisableHostPath        bool
 }
 
@@ -228,6 +229,12 @@ func WithForWindows() podTemplateOption {
 	}
 }
 
+func WithKubeletRootDir(kubeletRootDir string) podTemplateOption {
+	return func(c *podTemplateConfig) {
+		c.kubeletRootDir = kubeletRootDir
+	}
+}
+
 func WithNodeAgentDisableHostPath(disable bool) podTemplateOption {
 	return func(c *podTemplateConfig) {
 		c.nodeAgentDisableHostPath = disable

pkg/install/resources.go

@@ -54,6 +54,8 @@ var (
 	DefaultNodeAgentPodMemLimit   = "0"
 
 	DefaultVeleroNamespace = "velero"
+
+	DefaultKubeletRootDir = "/var/lib/kubelet"
 )
 
 func Labels() map[string]string {
@@ -269,6 +271,7 @@ type VeleroOptions struct {
 	RepoMaintenanceJobConfigMap     string
 	NodeAgentConfigMap              string
 	ItemBlockWorkerCount            int
+	KubeletRootDir                  string
 	NodeAgentDisableHostPath        bool
 }
 
@@ -417,6 +420,10 @@ func AllResources(o *VeleroOptions) *unstructured.UnstructuredList {
 			dsOpts = append(dsOpts, WithNodeAgentConfigMap(o.NodeAgentConfigMap))
 		}
 
+		if len(o.KubeletRootDir) > 0 {
+			dsOpts = append(dsOpts, WithKubeletRootDir(o.KubeletRootDir))
+		}
+
 		if o.UseNodeAgent {
 			ds := DaemonSet(o.Namespace, dsOpts...)
 			if err := appendUnstructured(resources, ds); err != nil {

site/content/docs/main/customize-installation.md

@@ -31,6 +31,9 @@ For some use cases, Velero node-agent requires to run under privileged mode. For
 
 If you've already run `velero install` without the `--use-node-agent` or `--privileged-node-agent` flag, you can run the same command again, including the `--use-node-agent` or `--privileged-node-agent` flag, to add CSI snapshot data movement to your existing install.
 
+## Customize the kubelet root path of the node-agent
+When installing with the `--use-node-agent` flag, the node-agent will mount the default kubelet paths `/var/lib/kubelet/pods` and `/var/lib/kubelet/plugins` (hostPath). To customize these kubelet mount paths, use the `--kubelet-root-dir` flag.
+
 ## Default Pod Volume backup to file system backup
 
 By default, `velero install` does not enable the use of File System Backup (FSB) to take backups of all pod volumes. You must apply an [annotation](file-system-backup.md/#using-opt-in-pod-volume-backup) to every pod which contains volumes for Velero to use FSB for the backup.

site/content/docs/main/velero-install.md

@@ -21,6 +21,7 @@ velero install \
     --velero-pod-mem-request <MEMORY_REQUEST> \
     --velero-pod-cpu-limit <CPU_LIMIT> \
     --velero-pod-mem-limit <MEMORY_LIMIT> \
+    --kubelet-root-dir <PATH_TO_KUBELET_ROOT_DIR> \
     [--use-node-agent] \
     [--default-volumes-to-fs-backup] \
     [--node-agent-pod-cpu-request <CPU_REQUEST>] \