From 248ee89123377c38d2840d456b30a11e473cf132 Mon Sep 17 00:00:00 2001 From: Adnan Abdulhussein Date: Mon, 29 Jul 2019 12:13:06 -0700 Subject: [PATCH] add configurable CPU/memory requests/limits for velero pod on install (#1678) * add configurable CPU/memory requests/limits for velero pod on install Signed-off-by: Adnan Abdulhussein --- changelogs/unreleased/1678-prydonius | 1 + pkg/cmd/cli/install/install.go | 68 ++++++++++++++++++++++++++++ pkg/cmd/cli/install/install_test.go | 66 +++++++++++++++++++++++++++ pkg/install/deployment.go | 8 ++++ pkg/install/resources.go | 12 ++++- site/docs/master/aws-config.md | 3 ++ site/docs/master/azure-config.md | 3 ++ site/docs/master/gcp-config.md | 3 ++ site/docs/master/ibm-config.md | 3 ++ site/docs/master/install-overview.md | 19 ++++++++ 10 files changed, 184 insertions(+), 2 deletions(-) create mode 100644 changelogs/unreleased/1678-prydonius create mode 100644 pkg/cmd/cli/install/install_test.go diff --git a/changelogs/unreleased/1678-prydonius b/changelogs/unreleased/1678-prydonius new file mode 100644 index 000000000..89c929119 --- /dev/null +++ b/changelogs/unreleased/1678-prydonius @@ -0,0 +1 @@ +Adds configurable CPU/memory requests and limits to the Velero Deployment generated by velero install. diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index 932170b3e..d3794839a 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -26,6 +26,8 @@ import ( "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" api "github.com/heptio/velero/pkg/apis/velero/v1" "github.com/heptio/velero/pkg/client" @@ -43,6 +45,10 @@ type InstallOptions struct { Prefix string ProviderName string PodAnnotations flag.Map + VeleroPodCPURequest string + VeleroPodMemRequest string + VeleroPodCPULimit string + VeleroPodMemLimit string RestoreOnly bool SecretFile string DryRun bool @@ -62,6 +68,10 @@ func (o *InstallOptions) BindFlags(flags *pflag.FlagSet) { flags.StringVar(&o.Prefix, "prefix", o.Prefix, "prefix under which all Velero data should be stored within the bucket. Optional.") flags.Var(&o.PodAnnotations, "pod-annotations", "annotations to add to the Velero and Restic pods. Optional. Format is key1=value1,key2=value2") flags.StringVar(&o.Namespace, "namespace", o.Namespace, "namespace to install Velero and associated data into. Optional.") + flags.StringVar(&o.VeleroPodCPURequest, "velero-pod-cpu-request", o.VeleroPodCPURequest, "CPU request for Velero pod. Optional.") + flags.StringVar(&o.VeleroPodMemRequest, "velero-pod-mem-request", o.VeleroPodMemRequest, "memory request for Velero pod. Optional.") + flags.StringVar(&o.VeleroPodCPULimit, "velero-pod-cpu-limit", o.VeleroPodCPULimit, "CPU limit for Velero pod. Optional.") + flags.StringVar(&o.VeleroPodMemLimit, "velero-pod-mem-limit", o.VeleroPodMemLimit, "memory limit for Velero pod. Optional.") flags.Var(&o.BackupStorageConfig, "backup-location-config", "configuration to use for the backup storage location. Format is key1=value1,key2=value2") flags.Var(&o.VolumeSnapshotConfig, "snapshot-location-config", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2") flags.BoolVar(&o.UseVolumeSnapshots, "use-volume-snapshots", o.UseVolumeSnapshots, "whether or not to create snapshot location automatically. Set to false if you do not plan to create volume snapshots via a storage provider.") @@ -79,6 +89,10 @@ func NewInstallOptions() *InstallOptions { BackupStorageConfig: flag.NewMap(), VolumeSnapshotConfig: flag.NewMap(), PodAnnotations: flag.NewMap(), + VeleroPodCPURequest: install.DefaultVeleroPodCPURequest, + VeleroPodMemRequest: install.DefaultVeleroPodMemRequest, + VeleroPodCPULimit: install.DefaultVeleroPodCPULimit, + VeleroPodMemLimit: install.DefaultVeleroPodMemLimit, // Default to creating a VSL unless we're told otherwise UseVolumeSnapshots: true, } @@ -94,6 +108,11 @@ func (o *InstallOptions) AsVeleroOptions() (*install.VeleroOptions, error) { if err != nil { return nil, err } + veleroPodResources, err := parseResourceRequests(o.VeleroPodCPURequest, o.VeleroPodMemRequest, o.VeleroPodCPULimit, o.VeleroPodMemLimit) + if err != nil { + return nil, err + } + return &install.VeleroOptions{ Namespace: o.Namespace, Image: o.Image, @@ -101,6 +120,7 @@ func (o *InstallOptions) AsVeleroOptions() (*install.VeleroOptions, error) { Bucket: o.BucketName, Prefix: o.Prefix, PodAnnotations: o.PodAnnotations.Data(), + VeleroPodResources: veleroPodResources, SecretData: secretData, RestoreOnly: o.RestoreOnly, UseRestic: o.UseRestic, @@ -144,6 +164,8 @@ This is useful as a starting point for more customized installations. # velero install --bucket backups --provider aws --backup-location-config region=us-west-2 --secret-file ./an-empty-file --snapshot-location-config region=us-west-2 --pod-annotations iam.amazonaws.com/role=arn:aws:iam:::role/ + # velero install --bucket gcp-backups --provider gcp --secret-file ./gcp-creds.json --velero-pod-cpu-request=1000m --velero-pod-cpu-limit=5000m --velero-pod-mem-request=512Mi --velero-pod-mem-limit=1024Mi + `, Run: func(c *cobra.Command, args []string) { cmd.CheckError(o.Validate(c, args, f)) @@ -233,3 +255,49 @@ func (o *InstallOptions) Validate(c *cobra.Command, args []string, f client.Fact return nil } + +// parseResourceRequests takes a set of CPU and memory requests and limit string +// values and returns a ResourceRequirements struct to be used in a Container. +// An error is returned if we cannot parse the request/limit. +func parseResourceRequests(cpuRequest, memRequest, cpuLimit, memLimit string) (corev1.ResourceRequirements, error) { + var resources corev1.ResourceRequirements + + parsedCPURequest, err := resource.ParseQuantity(cpuRequest) + if err != nil { + return resources, errors.Wrapf(err, `couldn't parse CPU request "%s"`, cpuRequest) + } + + parsedMemRequest, err := resource.ParseQuantity(memRequest) + if err != nil { + return resources, errors.Wrapf(err, `couldn't parse memory request "%s"`, memRequest) + } + + parsedCPULimit, err := resource.ParseQuantity(cpuLimit) + if err != nil { + return resources, errors.Wrapf(err, `couldn't parse CPU limit "%s"`, cpuLimit) + } + + parsedMemLimit, err := resource.ParseQuantity(memLimit) + if err != nil { + return resources, errors.Wrapf(err, `couldn't parse memory limit "%s"`, memLimit) + } + + if parsedCPURequest.Cmp(parsedCPULimit) > 0 { + return resources, errors.WithStack(errors.Errorf(`CPU request "%s" must be less than or equal to CPU limit "%s"`, cpuRequest, cpuLimit)) + } + + if parsedMemRequest.Cmp(parsedMemLimit) > 0 { + return resources, errors.WithStack(errors.Errorf(`Memory request "%s" must be less than or equal to Memory limit "%s"`, memRequest, memLimit)) + } + + resources.Requests = corev1.ResourceList{ + corev1.ResourceCPU: parsedCPURequest, + corev1.ResourceMemory: parsedMemRequest, + } + resources.Limits = corev1.ResourceList{ + corev1.ResourceCPU: parsedCPULimit, + corev1.ResourceMemory: parsedMemLimit, + } + + return resources, nil +} diff --git a/pkg/cmd/cli/install/install_test.go b/pkg/cmd/cli/install/install_test.go new file mode 100644 index 000000000..1aaef7aa3 --- /dev/null +++ b/pkg/cmd/cli/install/install_test.go @@ -0,0 +1,66 @@ +/* +Copyright 2019 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package install + +import ( + "testing" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +func Test_parseResourceRequests(t *testing.T) { + type args struct { + cpuRequest string + memRequest string + cpuLimit string + memLimit string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {"unbounded quantities", args{"0", "0", "0", "0"}, false}, + {"valid quantities", args{"100m", "128Mi", "200m", "256Mi"}, false}, + {"invalid quantity", args{"100m", "invalid", "200m", "256Mi"}, true}, + {"CPU request greater than limit", args{"300m", "128Mi", "200m", "256Mi"}, true}, + {"memory request greater than limit", args{"100m", "512Mi", "200m", "256Mi"}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseResourceRequests(tt.args.cpuRequest, tt.args.memRequest, tt.args.cpuLimit, tt.args.memLimit) + if tt.wantErr { + assert.Error(t, err) + return + } + + expected := corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(tt.args.cpuRequest), + corev1.ResourceMemory: resource.MustParse(tt.args.memRequest), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(tt.args.cpuLimit), + corev1.ResourceMemory: resource.MustParse(tt.args.memLimit), + }, + } + assert.Equal(t, expected, got) + }) + } +} diff --git a/pkg/install/deployment.go b/pkg/install/deployment.go index d02bb5c7f..4c8b48941 100644 --- a/pkg/install/deployment.go +++ b/pkg/install/deployment.go @@ -32,6 +32,7 @@ type podTemplateConfig struct { envVars []corev1.EnvVar restoreOnly bool annotations map[string]string + resources corev1.ResourceRequirements } func WithImage(image string) podTemplateOption { @@ -74,6 +75,12 @@ func WithRestoreOnly() podTemplateOption { } } +func WithResources(resources corev1.ResourceRequirements) podTemplateOption { + return func(c *podTemplateConfig) { + c.resources = resources + } +} + func Deployment(namespace string, opts ...podTemplateOption) *appsv1.Deployment { // TODO: Add support for server args c := &podTemplateConfig{ @@ -150,6 +157,7 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1.Deployment Value: "/credentials/cloud", }, }, + Resources: c.resources, }, }, Volumes: []corev1.Volume{ diff --git a/pkg/install/resources.go b/pkg/install/resources.go index 6eb7024c6..9ec67a1ff 100644 --- a/pkg/install/resources.go +++ b/pkg/install/resources.go @@ -24,7 +24,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "github.com/heptio/velero/pkg/apis/velero/v1" + v1 "github.com/heptio/velero/pkg/apis/velero/v1" "github.com/heptio/velero/pkg/buildinfo" ) @@ -37,7 +37,13 @@ func imageVersion() string { } // DefaultImage is the default image to use for the Velero deployment and restic daemonset containers. -var DefaultImage = "gcr.io/heptio-images/velero:" + imageVersion() +var ( + DefaultImage = "gcr.io/heptio-images/velero:" + imageVersion() + DefaultVeleroPodCPURequest = "500m" + DefaultVeleroPodMemRequest = "128Mi" + DefaultVeleroPodCPULimit = "1000m" + DefaultVeleroPodMemLimit = "256Mi" +) func labels() map[string]string { return map[string]string{ @@ -189,6 +195,7 @@ type VeleroOptions struct { Bucket string Prefix string PodAnnotations map[string]string + VeleroPodResources corev1.ResourceRequirements SecretData []byte RestoreOnly bool UseRestic bool @@ -232,6 +239,7 @@ func AllResources(o *VeleroOptions) (*unstructured.UnstructuredList, error) { deploy := Deployment(o.Namespace, WithAnnotations(o.PodAnnotations), WithImage(o.Image), + WithResources(o.VeleroPodResources), ) if o.RestoreOnly { deploy = Deployment(o.Namespace, diff --git a/site/docs/master/aws-config.md b/site/docs/master/aws-config.md index ec18f9524..82522be08 100644 --- a/site/docs/master/aws-config.md +++ b/site/docs/master/aws-config.md @@ -161,6 +161,8 @@ Additionally, you can specify `--use-restic` to enable restic support, and `--wa (Optional) Specify [additional configurable parameters][6] for the `--snapshot-location-config` flag. +(Optional) Specify [CPU and memory resource requests and limits][22] for the Velero pod. + For more complex installation needs, use either the Helm chart, or add `--dry-run -o yaml` options for generating the YAML representation for the installation. ## Setting AWS_CLUSTER_NAME (Optional) @@ -303,3 +305,4 @@ Note that the `--secret-file` argument is required, but it can be an empty file. [14]: http://docs.aws.amazon.com/IAM/latest/UserGuide/introduction.html [20]: faq.md [21]: api-types/backupstoragelocation.md#aws +[22]: install-overview.md#velero-resource-requirements diff --git a/site/docs/master/azure-config.md b/site/docs/master/azure-config.md index 461f25e83..76bd981ba 100644 --- a/site/docs/master/azure-config.md +++ b/site/docs/master/azure-config.md @@ -157,6 +157,8 @@ Additionally, you can specify `--use-restic` to enable restic support, and `--wa (Optional) Specify [additional configurable parameters][8] for the `--snapshot-location-config` flag. +(Optional) Specify [CPU and memory resource requests and limits][23] for the Velero pod. + For more complex installation needs, use either the Helm chart, or add `--dry-run -o yaml` options for generating the YAML representation for the installation. [0]: namespace.md @@ -167,3 +169,4 @@ For more complex installation needs, use either the Helm chart, or add `--dry-ru [20]: faq.md [21]: api-types/backupstoragelocation.md#azure [22]: https://azure.microsoft.com/en-us/services/kubernetes-service/ +[23]: install-overview.md#velero-resource-requirements diff --git a/site/docs/master/gcp-config.md b/site/docs/master/gcp-config.md index 85bc1ed09..9586c1a75 100644 --- a/site/docs/master/gcp-config.md +++ b/site/docs/master/gcp-config.md @@ -132,6 +132,8 @@ Additionally, you can specify `--use-restic` to enable restic support, and `--wa (Optional) Specify [additional configurable parameters][8] for the `--snapshot-location-config` flag. +(Optional) Specify [CPU and memory resource requests and limits][23] for the Velero pod. + For more complex installation needs, use either the Helm chart, or add `--dry-run -o yaml` options for generating the YAML representation for the installation. [0]: namespace.md @@ -141,3 +143,4 @@ For more complex installation needs, use either the Helm chart, or add `--dry-ru [16]: https://cloud.google.com/sdk/docs/ [20]: faq.md [22]: https://cloud.google.com/kubernetes-engine/docs/how-to/role-based-access-control#iam-rolebinding-bootstrap +[23]: install-overview.md#velero-resource-requirements diff --git a/site/docs/master/ibm-config.md b/site/docs/master/ibm-config.md index f885e67e2..047f3d7c5 100644 --- a/site/docs/master/ibm-config.md +++ b/site/docs/master/ibm-config.md @@ -70,6 +70,8 @@ Velero does not currently have a volume snapshot plugin for IBM Cloud, so creati Additionally, you can specify `--use-restic` to enable restic support, and `--wait` to wait for the deployment to be ready. +(Optional) Specify [CPU and memory resource requests and limits][15] for the Velero pod. + Once the installation is complete, remove the default `VolumeSnapshotLocation` that was created by `velero install`, since it's specific to AWS and won't work for IBM Cloud: ```bash @@ -92,3 +94,4 @@ Uncomment `storageClassName: ` and replace with your `S [5]: https://console.bluemix.net/docs/containers/container_index.html#container_index [6]: api-types/backupstoragelocation.md#aws [14]: http://docs.aws.amazon.com/IAM/latest/UserGuide/introduction.html +[15]: install-overview.md#velero-resource-requirements diff --git a/site/docs/master/install-overview.md b/site/docs/master/install-overview.md index cb0409ee5..c21b30f3a 100644 --- a/site/docs/master/install-overview.md +++ b/site/docs/master/install-overview.md @@ -55,6 +55,24 @@ Whether you run Velero on a cloud provider or on-premises, if you have more than For details, see the documentation topics for individual cloud providers. +## Velero resource requirements + +By default, the Velero deployment requests 500m CPU, 128Mi memory and sets a limit of 1000m CPU, 256Mi. +If you need to customize these resource requests and limits, you can set the following flags in your `velero install` command: + +``` +velero install \ + --provider \ + --bucket \ + --secret-file \ + --velero-pod-cpu-request \ + --velero-pod-mem-request \ + --velero-pod-cpu-limit \ + --velero-pod-mem-limit \ +``` + +Values for these flags follow the same format as [Kubernetes resource requirements][103]. + ## Removing Velero If you would like to completely uninstall Velero from your cluster, the following commands will remove all resources created by `velero install`: @@ -149,3 +167,4 @@ After you set up the Velero server, try these examples: [100]: support-matrix.md#volume-snapshot-providers [101]: https://www.minio.io [102]: https://portworx.com +[103]: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/#meaning-of-cpu