Merge pull request #9574 from blackpiglet/xj014661/main/ephemeral_storage_config
Run the E2E test on kind / get-go-version (push) Failing after 1m24s
Run the E2E test on kind / build (push) Has been skipped
Run the E2E test on kind / setup-test-matrix (push) Successful in 4s
Run the E2E test on kind / run-e2e-test (push) Has been skipped
Main CI / get-go-version (push) Successful in 17s
Main CI / Build (push) Failing after 40s
Close stale issues and PRs / stale (push) Successful in 16s
Trivy Nightly Scan / Trivy nightly scan (velero, main) (push) Failing after 1m28s
Trivy Nightly Scan / Trivy nightly scan (velero-plugin-for-aws, main) (push) Failing after 41s
Trivy Nightly Scan / Trivy nightly scan (velero-plugin-for-gcp, main) (push) Failing after 35s
Trivy Nightly Scan / Trivy nightly scan (velero-plugin-for-microsoft-azure, main) (push) Failing after 25s

Add ephemeral storage limit and request support for data mover and maintenance job
This commit is contained in:
Xun Jiang/Bruce Jiang
2026-03-05 22:43:16 +08:00
committed by GitHub
14 changed files with 191 additions and 52 deletions
+1
View File
@@ -0,0 +1 @@
Add ephemeral storage limit and request support for data mover and maintenance job
+12 -2
View File
@@ -275,11 +275,21 @@ func (o *Options) AsVeleroOptions() (*install.VeleroOptions, error) {
return nil, err
}
}
veleroPodResources, err := kubeutil.ParseResourceRequirements(o.VeleroPodCPURequest, o.VeleroPodMemRequest, o.VeleroPodCPULimit, o.VeleroPodMemLimit)
veleroPodResources, err := kubeutil.ParseCPUAndMemoryResources(
o.VeleroPodCPURequest,
o.VeleroPodMemRequest,
o.VeleroPodCPULimit,
o.VeleroPodMemLimit,
)
if err != nil {
return nil, err
}
nodeAgentPodResources, err := kubeutil.ParseResourceRequirements(o.NodeAgentPodCPURequest, o.NodeAgentPodMemRequest, o.NodeAgentPodCPULimit, o.NodeAgentPodMemLimit)
nodeAgentPodResources, err := kubeutil.ParseCPUAndMemoryResources(
o.NodeAgentPodCPURequest,
o.NodeAgentPodMemRequest,
o.NodeAgentPodCPULimit,
o.NodeAgentPodMemLimit,
)
if err != nil {
return nil, err
}
+19 -1
View File
@@ -323,7 +323,25 @@ func (s *nodeAgentServer) run() {
podResources := corev1api.ResourceRequirements{}
if s.dataPathConfigs != nil && s.dataPathConfigs.PodResources != nil {
if res, err := kube.ParseResourceRequirements(s.dataPathConfigs.PodResources.CPURequest, s.dataPathConfigs.PodResources.MemoryRequest, s.dataPathConfigs.PodResources.CPULimit, s.dataPathConfigs.PodResources.MemoryLimit); err != nil {
// To make the PodResources ConfigMap without ephemeral storage request/limit backward compatible,
// need to avoid set value as empty, because empty string will cause parsing error.
ephemeralStorageRequest := constant.DefaultEphemeralStorageRequest
if s.dataPathConfigs.PodResources.EphemeralStorageRequest != "" {
ephemeralStorageRequest = s.dataPathConfigs.PodResources.EphemeralStorageRequest
}
ephemeralStorageLimit := constant.DefaultEphemeralStorageLimit
if s.dataPathConfigs.PodResources.EphemeralStorageLimit != "" {
ephemeralStorageLimit = s.dataPathConfigs.PodResources.EphemeralStorageLimit
}
if res, err := kube.ParseResourceRequirements(
s.dataPathConfigs.PodResources.CPURequest,
s.dataPathConfigs.PodResources.MemoryRequest,
ephemeralStorageRequest,
s.dataPathConfigs.PodResources.CPULimit,
s.dataPathConfigs.PodResources.MemoryLimit,
ephemeralStorageLimit,
); err != nil {
s.logger.WithError(err).Warn("Pod resource requirements are invalid, ignore")
} else {
podResources = res
+3
View File
@@ -23,4 +23,7 @@ const (
PluginCSIPVCRestoreRIA = "velero.io/csi-pvc-restorer"
PluginCsiVolumeSnapshotRestoreRIA = "velero.io/csi-volumesnapshot-restorer"
DefaultEphemeralStorageRequest = "0"
DefaultEphemeralStorageLimit = "0"
)
+19 -1
View File
@@ -38,6 +38,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
"github.com/vmware-tanzu/velero/pkg/constant"
velerolabel "github.com/vmware-tanzu/velero/pkg/label"
velerotypes "github.com/vmware-tanzu/velero/pkg/types"
"github.com/vmware-tanzu/velero/pkg/util"
@@ -574,15 +575,32 @@ func buildJob(
// Set resource limits and requests
cpuRequest := DefaultMaintenanceJobCPURequest
memRequest := DefaultMaintenanceJobMemRequest
ephemeralStorageRequest := constant.DefaultEphemeralStorageRequest
cpuLimit := DefaultMaintenanceJobCPULimit
memLimit := DefaultMaintenanceJobMemLimit
ephemeralStorageLimit := constant.DefaultEphemeralStorageLimit
if config != nil && config.PodResources != nil {
cpuRequest = config.PodResources.CPURequest
memRequest = config.PodResources.MemoryRequest
cpuLimit = config.PodResources.CPULimit
memLimit = config.PodResources.MemoryLimit
// To make the PodResources ConfigMap without ephemeral storage request/limit backward compatible,
// need to avoid set value as empty, because empty string will cause parsing error.
if config.PodResources.EphemeralStorageRequest != "" {
ephemeralStorageRequest = config.PodResources.EphemeralStorageRequest
}
if config.PodResources.EphemeralStorageLimit != "" {
ephemeralStorageLimit = config.PodResources.EphemeralStorageLimit
}
}
resources, err := kube.ParseResourceRequirements(cpuRequest, memRequest, cpuLimit, memLimit)
resources, err := kube.ParseResourceRequirements(
cpuRequest,
memRequest,
ephemeralStorageRequest,
cpuLimit,
memLimit,
ephemeralStorageLimit,
)
if err != nil {
return nil, errors.Wrap(err, "failed to parse resource requirements for maintenance job")
}
@@ -163,12 +163,19 @@ func (a *PodVolumeRestoreAction) Execute(input *velero.RestoreItemActionExecuteI
memLimit = defaultMemRequestLimit
}
resourceReqs, err := kube.ParseResourceRequirements(cpuRequest, memRequest, cpuLimit, memLimit)
resourceReqs, err := kube.ParseCPUAndMemoryResources(
cpuRequest,
memRequest,
cpuLimit,
memLimit,
)
if err != nil {
log.Errorf("couldn't parse resource requirements: %s.", err)
resourceReqs, _ = kube.ParseResourceRequirements(
defaultCPURequestLimit, defaultMemRequestLimit, // requests
defaultCPURequestLimit, defaultMemRequestLimit, // limits
resourceReqs, _ = kube.ParseCPUAndMemoryResources(
defaultCPURequestLimit,
defaultMemRequestLimit,
defaultCPURequestLimit,
defaultMemRequestLimit,
)
}
@@ -117,9 +117,11 @@ func TestGetImage(t *testing.T) {
// TestPodVolumeRestoreActionExecute tests the pod volume restore item action plugin's Execute method.
func TestPodVolumeRestoreActionExecute(t *testing.T) {
resourceReqs, _ := kube.ParseResourceRequirements(
defaultCPURequestLimit, defaultMemRequestLimit, // requests
defaultCPURequestLimit, defaultMemRequestLimit, // limits
resourceReqs, _ := kube.ParseCPUAndMemoryResources(
defaultCPURequestLimit,
defaultMemRequestLimit,
defaultCPURequestLimit,
defaultMemRequestLimit,
)
id := int64(1000)
securityContext := corev1api.SecurityContext{
+6 -4
View File
@@ -40,10 +40,12 @@ type LoadAffinity struct {
}
type PodResources struct {
CPURequest string `json:"cpuRequest,omitempty"`
MemoryRequest string `json:"memoryRequest,omitempty"`
CPULimit string `json:"cpuLimit,omitempty"`
MemoryLimit string `json:"memoryLimit,omitempty"`
CPURequest string `json:"cpuRequest,omitempty"`
CPULimit string `json:"cpuLimit,omitempty"`
MemoryRequest string `json:"memoryRequest,omitempty"`
MemoryLimit string `json:"memoryLimit,omitempty"`
EphemeralStorageRequest string `json:"ephemeralStorageRequest,omitempty"`
EphemeralStorageLimit string `json:"ephemeralStorageLimit,omitempty"`
}
// IsPodRunning does a well-rounded check to make sure the specified pod is running stably.
+44 -2
View File
@@ -20,12 +20,34 @@ import (
"github.com/pkg/errors"
corev1api "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"github.com/vmware-tanzu/velero/pkg/constant"
)
// ParseResourceRequirements takes a set of CPU and memory requests and limit string
// ParseCPUAndMemoryResources is a helper function that parses CPU and memory requests and limits,
// using default values for ephemeral storage.
func ParseCPUAndMemoryResources(cpuRequest, memRequest, cpuLimit, memLimit string) (corev1api.ResourceRequirements, error) {
return ParseResourceRequirements(
cpuRequest,
memRequest,
constant.DefaultEphemeralStorageRequest,
cpuLimit,
memLimit,
constant.DefaultEphemeralStorageLimit,
)
}
// ParseResourceRequirements takes a set of CPU, memory, ephemeral storage requests and limit string
// values and returns a ResourceRequirements struct to be used in a Container.
// An error is returned if we cannot parse the request/limit.
func ParseResourceRequirements(cpuRequest, memRequest, cpuLimit, memLimit string) (corev1api.ResourceRequirements, error) {
func ParseResourceRequirements(
cpuRequest,
memRequest,
ephemeralStorageRequest,
cpuLimit,
memLimit,
ephemeralStorageLimit string,
) (corev1api.ResourceRequirements, error) {
resources := corev1api.ResourceRequirements{
Requests: corev1api.ResourceList{},
Limits: corev1api.ResourceList{},
@@ -41,6 +63,11 @@ func ParseResourceRequirements(cpuRequest, memRequest, cpuLimit, memLimit string
return resources, errors.Wrapf(err, `couldn't parse memory request "%s"`, memRequest)
}
parsedEphemeralStorageRequest, err := resource.ParseQuantity(ephemeralStorageRequest)
if err != nil {
return resources, errors.Wrapf(err, `couldn't parse ephemeral storage request "%s"`, ephemeralStorageRequest)
}
parsedCPULimit, err := resource.ParseQuantity(cpuLimit)
if err != nil {
return resources, errors.Wrapf(err, `couldn't parse CPU limit "%s"`, cpuLimit)
@@ -51,6 +78,11 @@ func ParseResourceRequirements(cpuRequest, memRequest, cpuLimit, memLimit string
return resources, errors.Wrapf(err, `couldn't parse memory limit "%s"`, memLimit)
}
parsedEphemeralStorageLimit, err := resource.ParseQuantity(ephemeralStorageLimit)
if err != nil {
return resources, errors.Wrapf(err, `couldn't parse ephemeral storage limit "%s"`, ephemeralStorageLimit)
}
// A quantity of 0 is treated as unbounded
unbounded := resource.MustParse("0")
@@ -62,6 +94,10 @@ func ParseResourceRequirements(cpuRequest, memRequest, cpuLimit, memLimit string
return resources, errors.WithStack(errors.Errorf(`Memory request "%s" must be less than or equal to Memory limit "%s"`, memRequest, memLimit))
}
if parsedEphemeralStorageLimit != unbounded && parsedEphemeralStorageRequest.Cmp(parsedEphemeralStorageLimit) > 0 {
return resources, errors.WithStack(errors.Errorf(`Ephemeral storage request "%s" must be less than or equal to Ephemeral storage limit "%s"`, ephemeralStorageRequest, ephemeralStorageLimit))
}
// Only set resources if they are not unbounded
if parsedCPURequest != unbounded {
resources.Requests[corev1api.ResourceCPU] = parsedCPURequest
@@ -69,12 +105,18 @@ func ParseResourceRequirements(cpuRequest, memRequest, cpuLimit, memLimit string
if parsedMemRequest != unbounded {
resources.Requests[corev1api.ResourceMemory] = parsedMemRequest
}
if parsedEphemeralStorageRequest != unbounded {
resources.Requests[corev1api.ResourceEphemeralStorage] = parsedEphemeralStorageRequest
}
if parsedCPULimit != unbounded {
resources.Limits[corev1api.ResourceCPU] = parsedCPULimit
}
if parsedMemLimit != unbounded {
resources.Limits[corev1api.ResourceMemory] = parsedMemLimit
}
if parsedEphemeralStorageLimit != unbounded {
resources.Limits[corev1api.ResourceEphemeralStorage] = parsedEphemeralStorageLimit
}
return resources, nil
}
+51 -29
View File
@@ -27,10 +27,12 @@ import (
func TestParseResourceRequirements(t *testing.T) {
type args struct {
cpuRequest string
memRequest string
cpuLimit string
memLimit string
cpuRequest string
memRequest string
ephemeralStorageRequest string
cpuLimit string
memLimit string
ephemeralStorageLimit string
}
tests := []struct {
name string
@@ -38,43 +40,61 @@ func TestParseResourceRequirements(t *testing.T) {
wantErr bool
expected *corev1api.ResourceRequirements
}{
{"unbounded quantities", args{"0", "0", "0", "0"}, false, &corev1api.ResourceRequirements{
{"unbounded quantities", args{"0", "0", "0", "0", "0", "0"}, false, &corev1api.ResourceRequirements{
Requests: corev1api.ResourceList{},
Limits: corev1api.ResourceList{},
}},
{"valid quantities", args{"100m", "128Mi", "200m", "256Mi"}, false, nil},
{"CPU request with unbounded limit", args{"100m", "128Mi", "0", "256Mi"}, false, &corev1api.ResourceRequirements{
{"valid quantities", args{"100m", "128Mi", "5Gi", "200m", "256Mi", "10Gi"}, false, nil},
{"CPU request with unbounded limit", args{"100m", "128Mi", "5Gi", "0", "256Mi", "10Gi"}, false, &corev1api.ResourceRequirements{
Requests: corev1api.ResourceList{
corev1api.ResourceCPU: resource.MustParse("100m"),
corev1api.ResourceMemory: resource.MustParse("128Mi"),
corev1api.ResourceCPU: resource.MustParse("100m"),
corev1api.ResourceMemory: resource.MustParse("128Mi"),
corev1api.ResourceEphemeralStorage: resource.MustParse("5Gi"),
},
Limits: corev1api.ResourceList{
corev1api.ResourceMemory: resource.MustParse("256Mi"),
corev1api.ResourceEphemeralStorage: resource.MustParse("10Gi"),
},
}},
{"Mem request with unbounded limit", args{"100m", "128Mi", "5Gi", "200m", "0", "10Gi"}, false, &corev1api.ResourceRequirements{
Requests: corev1api.ResourceList{
corev1api.ResourceCPU: resource.MustParse("100m"),
corev1api.ResourceMemory: resource.MustParse("128Mi"),
corev1api.ResourceEphemeralStorage: resource.MustParse("5Gi"),
},
Limits: corev1api.ResourceList{
corev1api.ResourceCPU: resource.MustParse("200m"),
corev1api.ResourceEphemeralStorage: resource.MustParse("10Gi"),
},
}},
{"Ephemeral storage request with unbounded limit", args{"100m", "128Mi", "5Gi", "200m", "256Mi", "0"}, false, &corev1api.ResourceRequirements{
Requests: corev1api.ResourceList{
corev1api.ResourceCPU: resource.MustParse("100m"),
corev1api.ResourceMemory: resource.MustParse("128Mi"),
corev1api.ResourceEphemeralStorage: resource.MustParse("5Gi"),
},
Limits: corev1api.ResourceList{
corev1api.ResourceCPU: resource.MustParse("200m"),
corev1api.ResourceMemory: resource.MustParse("256Mi"),
},
}},
{"Mem request with unbounded limit", args{"100m", "128Mi", "200m", "0"}, false, &corev1api.ResourceRequirements{
{"CPU/Mem/EphemeralStorage requests with unbounded limits", args{"100m", "128Mi", "5Gi", "0", "0", "0"}, false, &corev1api.ResourceRequirements{
Requests: corev1api.ResourceList{
corev1api.ResourceCPU: resource.MustParse("100m"),
corev1api.ResourceMemory: resource.MustParse("128Mi"),
},
Limits: corev1api.ResourceList{
corev1api.ResourceCPU: resource.MustParse("200m"),
},
}},
{"CPU/Mem requests with unbounded limits", args{"100m", "128Mi", "0", "0"}, false, &corev1api.ResourceRequirements{
Requests: corev1api.ResourceList{
corev1api.ResourceCPU: resource.MustParse("100m"),
corev1api.ResourceMemory: resource.MustParse("128Mi"),
corev1api.ResourceCPU: resource.MustParse("100m"),
corev1api.ResourceMemory: resource.MustParse("128Mi"),
corev1api.ResourceEphemeralStorage: resource.MustParse("5Gi"),
},
Limits: corev1api.ResourceList{},
}},
{"invalid quantity", args{"100m", "invalid", "200m", "256Mi"}, true, nil},
{"CPU request greater than limit", args{"300m", "128Mi", "200m", "256Mi"}, true, nil},
{"memory request greater than limit", args{"100m", "512Mi", "200m", "256Mi"}, true, nil},
{"invalid quantity", args{"100m", "invalid", "1Gi", "200m", "256Mi", "valid"}, true, nil},
{"CPU request greater than limit", args{"300m", "128Mi", "5Gi", "200m", "256Mi", "10Gi"}, true, nil},
{"memory request greater than limit", args{"100m", "512Mi", "5Gi", "200m", "256Mi", "10Gi"}, true, nil},
{"ephemeral storage request greater than limit", args{"100m", "128Mi", "10Gi", "200m", "256Mi", "5Gi"}, true, nil},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := ParseResourceRequirements(tt.args.cpuRequest, tt.args.memRequest, tt.args.cpuLimit, tt.args.memLimit)
got, err := ParseResourceRequirements(tt.args.cpuRequest, tt.args.memRequest, tt.args.ephemeralStorageRequest, tt.args.cpuLimit, tt.args.memLimit, tt.args.ephemeralStorageLimit)
if tt.wantErr {
assert.Error(t, err)
return
@@ -85,12 +105,14 @@ func TestParseResourceRequirements(t *testing.T) {
if tt.expected == nil {
expected = corev1api.ResourceRequirements{
Requests: corev1api.ResourceList{
corev1api.ResourceCPU: resource.MustParse(tt.args.cpuRequest),
corev1api.ResourceMemory: resource.MustParse(tt.args.memRequest),
corev1api.ResourceCPU: resource.MustParse(tt.args.cpuRequest),
corev1api.ResourceMemory: resource.MustParse(tt.args.memRequest),
corev1api.ResourceEphemeralStorage: resource.MustParse(tt.args.ephemeralStorageRequest),
},
Limits: corev1api.ResourceList{
corev1api.ResourceCPU: resource.MustParse(tt.args.cpuLimit),
corev1api.ResourceMemory: resource.MustParse(tt.args.memLimit),
corev1api.ResourceCPU: resource.MustParse(tt.args.cpuLimit),
corev1api.ResourceMemory: resource.MustParse(tt.args.memLimit),
corev1api.ResourceEphemeralStorage: resource.MustParse(tt.args.ephemeralStorageLimit),
},
}
} else {
@@ -7,7 +7,7 @@ During [CSI Snapshot Data Movement][1], Velero built-in data mover launches data
During [fs-backup][2], Velero also launches data mover pods to run the data transfer.
The data transfer is a time and resource consuming activity.
Velero by default uses the [BestEffort QoS][2] for the data mover pods, which guarantees the best performance of the data movement activities. On the other hand, it may take lots of cluster resource, i.e., CPU, memory, and how many resources are taken is decided by the concurrency and the scale of data to be moved.
Velero by default uses the [BestEffort QoS][2] for the data mover pods, which guarantees the best performance of the data movement activities. On the other hand, it may take lots of cluster resource, i.e., CPU, memory, ephemeral storage, and how many resources are taken is decided by the concurrency and the scale of data to be moved.
If the cluster nodes don't have sufficient resource, Velero also allows you to customize the resources for the data mover pods.
Note: If less resources are assigned to data mover pods, the data movement activities may take longer time; or the data mover pods may be OOM killed if the assigned memory resource doesn't meet the requirements. Consequently, the dataUpload/dataDownload may run longer or fail.
@@ -25,6 +25,8 @@ Here is a sample of the configMap with ```podResources```:
"podResources": {
"cpuRequest": "1000m",
"cpuLimit": "1000m",
"ephemeralStorageRequest": "5Gi",
"ephemeralStorageLimit": "10Gi",
"memoryRequest": "512Mi",
"memoryLimit": "1Gi"
}
@@ -72,6 +72,8 @@ data:
"podResources": {
"cpuRequest": "100m",
"cpuLimit": "200m",
"ephemeralStorageRequest": "5Gi",
"ephemeralStorageLimit": "10Gi",
"memoryRequest": "100Mi",
"memoryLimit": "200Mi"
},
@@ -99,6 +101,8 @@ data:
"podResources": {
"cpuRequest": "200m",
"cpuLimit": "400m",
"ephemeralStorageRequest": "5Gi",
"ephemeralStorageLimit": "10Gi",
"memoryRequest": "200Mi",
"memoryLimit": "400Mi"
},
@@ -224,7 +224,7 @@ Configure different node selection rules for specific storage classes:
```
### Pod Resources (`podResources`)
Configure CPU and memory resources for Data Mover Pods to optimize performance and prevent resource conflict.
Configure CPU, memory and ephemeral storage resources for Data Mover Pods to optimize performance and prevent resource conflict.
The configurations work for PodVolumeBackup, PodVolumeRestore, DataUpload, and DataDownload pods.
@@ -233,6 +233,8 @@ The configurations work for PodVolumeBackup, PodVolumeRestore, DataUpload, and D
"podResources": {
"cpuRequest": "1000m",
"cpuLimit": "2000m",
"ephemeralStorageRequest": "5Gi",
"ephemeralStorageLimit": "10Gi",
"memoryRequest": "1Gi",
"memoryLimit": "4Gi"
}
@@ -535,6 +537,8 @@ Here's a comprehensive example showing how all configuration sections work toget
"podResources": {
"cpuRequest": "500m",
"cpuLimit": "1000m",
"ephemeralStorageRequest": "5Gi",
"ephemeralStorageLimit": "10Gi",
"memoryRequest": "1Gi",
"memoryLimit": "2Gi"
},
@@ -70,10 +70,12 @@ var SpecificRepoMaintenanceTest func() = TestFunc(&RepoMaintenanceTestCase{
jobConfigs: velerotypes.JobConfigs{
KeepLatestMaintenanceJobs: &keepJobNum,
PodResources: &velerokubeutil.PodResources{
CPURequest: "100m",
MemoryRequest: "100Mi",
CPULimit: "200m",
MemoryLimit: "200Mi",
CPURequest: "100m",
MemoryRequest: "100Mi",
EphemeralStorageRequest: "5Gi",
CPULimit: "200m",
MemoryLimit: "200Mi",
EphemeralStorageLimit: "10Gi",
},
PriorityClassName: test.PriorityClassNameForRepoMaintenance,
},
@@ -230,8 +232,10 @@ func (r *RepoMaintenanceTestCase) Verify() error {
resources, err := kube.ParseResourceRequirements(
r.jobConfigs.PodResources.CPURequest,
r.jobConfigs.PodResources.MemoryRequest,
r.jobConfigs.PodResources.EphemeralStorageRequest,
r.jobConfigs.PodResources.CPULimit,
r.jobConfigs.PodResources.MemoryLimit,
r.jobConfigs.PodResources.EphemeralStorageLimit,
)
if err != nil {
return errors.Wrap(err, "failed to parse resource requirements for maintenance job")