diff --git a/changelogs/unreleased/1677-nrb b/changelogs/unreleased/1677-nrb new file mode 100644 index 000000000..ae4c945be --- /dev/null +++ b/changelogs/unreleased/1677-nrb @@ -0,0 +1 @@ +Add low cpu/memory limits to the restic init container. This allows for restoration into namespaces with quotas defined. diff --git a/pkg/builder/container_builder.go b/pkg/builder/container_builder.go new file mode 100644 index 000000000..848347763 --- /dev/null +++ b/pkg/builder/container_builder.go @@ -0,0 +1,68 @@ +/* +Copyright 2019 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + corev1api "k8s.io/api/core/v1" +) + +// ContainerBuilder builds Container objects +type ContainerBuilder struct { + object *corev1api.Container +} + +// ForContainer is the constructor for ContainerBuilder. +func ForContainer(name, image string) *ContainerBuilder { + return &ContainerBuilder{ + object: &corev1api.Container{ + Name: name, + Image: image, + }, + } +} + +// Result returns the built Container. +func (b *ContainerBuilder) Result() *corev1api.Container { + return b.object +} + +// Args sets the container's Args. +func (b *ContainerBuilder) Args(args ...string) *ContainerBuilder { + b.object.Args = append(b.object.Args, args...) + return b +} + +// VolumeMounts sets the container's VolumeMounts. +func (b *ContainerBuilder) VolumeMounts(volumeMounts ...*corev1api.VolumeMount) *ContainerBuilder { + for _, v := range volumeMounts { + b.object.VolumeMounts = append(b.object.VolumeMounts, *v) + } + return b +} + +// Resources sets the container's Resources. +func (b *ContainerBuilder) Resources(resources *corev1api.ResourceRequirements) *ContainerBuilder { + b.object.Resources = *resources + return b +} + +func (b *ContainerBuilder) Env(vars ...*corev1api.EnvVar) *ContainerBuilder { + for _, v := range vars { + b.object.Env = append(b.object.Env, *v) + } + return b +} diff --git a/pkg/builder/pod_builder.go b/pkg/builder/pod_builder.go index 1f12d3477..77c6be043 100644 --- a/pkg/builder/pod_builder.go +++ b/pkg/builder/pod_builder.go @@ -69,3 +69,10 @@ func (b *PodBuilder) NodeName(val string) *PodBuilder { b.object.Spec.NodeName = val return b } + +func (b *PodBuilder) InitContainers(containers ...*corev1api.Container) *PodBuilder { + for _, c := range containers { + b.object.Spec.InitContainers = append(b.object.Spec.InitContainers, *c) + } + return b +} diff --git a/pkg/builder/volume_mount_builder.go b/pkg/builder/volume_mount_builder.go new file mode 100644 index 000000000..9c794f5a1 --- /dev/null +++ b/pkg/builder/volume_mount_builder.go @@ -0,0 +1,41 @@ +/* +Copyright 2019 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package builder + +import ( + corev1api "k8s.io/api/core/v1" +) + +// VolumeMountBuilder builds VolumeMount objects. +type VolumeMountBuilder struct { + object *corev1api.VolumeMount +} + +// ForVolumeMount is the constructor for a VolumeMountBuilder. +func ForVolumeMount(name, mountPath string) *VolumeMountBuilder { + return &VolumeMountBuilder{ + object: &corev1api.VolumeMount{ + Name: name, + MountPath: mountPath, + }, + } +} + +// Result returns the built VolumeMount. +func (b *VolumeMountBuilder) Result() *corev1api.VolumeMount { + return b.object +} diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index 617993366..bbba51e61 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -26,8 +26,6 @@ import ( "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/pflag" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" api "github.com/heptio/velero/pkg/apis/velero/v1" "github.com/heptio/velero/pkg/client" @@ -35,6 +33,7 @@ import ( "github.com/heptio/velero/pkg/cmd/util/flag" "github.com/heptio/velero/pkg/cmd/util/output" "github.com/heptio/velero/pkg/install" + kubeutil "github.com/heptio/velero/pkg/util/kube" ) // InstallOptions collects all the options for installing Velero into a Kubernetes cluster. @@ -125,11 +124,11 @@ func (o *InstallOptions) AsVeleroOptions() (*install.VeleroOptions, error) { return nil, err } } - veleroPodResources, err := parseResourceRequests(o.VeleroPodCPURequest, o.VeleroPodMemRequest, o.VeleroPodCPULimit, o.VeleroPodMemLimit) + veleroPodResources, err := kubeutil.ParseResourceRequirements(o.VeleroPodCPURequest, o.VeleroPodMemRequest, o.VeleroPodCPULimit, o.VeleroPodMemLimit) if err != nil { return nil, err } - resticPodResources, err := parseResourceRequests(o.ResticPodCPURequest, o.ResticPodMemRequest, o.ResticPodCPULimit, o.ResticPodMemLimit) + resticPodResources, err := kubeutil.ParseResourceRequirements(o.ResticPodCPURequest, o.ResticPodMemRequest, o.ResticPodCPULimit, o.ResticPodMemLimit) if err != nil { return nil, err } @@ -285,49 +284,3 @@ func (o *InstallOptions) Validate(c *cobra.Command, args []string, f client.Fact return nil } - -// parseResourceRequests takes a set of CPU and memory requests and limit string -// values and returns a ResourceRequirements struct to be used in a Container. -// An error is returned if we cannot parse the request/limit. -func parseResourceRequests(cpuRequest, memRequest, cpuLimit, memLimit string) (corev1.ResourceRequirements, error) { - var resources corev1.ResourceRequirements - - parsedCPURequest, err := resource.ParseQuantity(cpuRequest) - if err != nil { - return resources, errors.Wrapf(err, `couldn't parse CPU request "%s"`, cpuRequest) - } - - parsedMemRequest, err := resource.ParseQuantity(memRequest) - if err != nil { - return resources, errors.Wrapf(err, `couldn't parse memory request "%s"`, memRequest) - } - - parsedCPULimit, err := resource.ParseQuantity(cpuLimit) - if err != nil { - return resources, errors.Wrapf(err, `couldn't parse CPU limit "%s"`, cpuLimit) - } - - parsedMemLimit, err := resource.ParseQuantity(memLimit) - if err != nil { - return resources, errors.Wrapf(err, `couldn't parse memory limit "%s"`, memLimit) - } - - if parsedCPURequest.Cmp(parsedCPULimit) > 0 { - return resources, errors.WithStack(errors.Errorf(`CPU request "%s" must be less than or equal to CPU limit "%s"`, cpuRequest, cpuLimit)) - } - - if parsedMemRequest.Cmp(parsedMemLimit) > 0 { - return resources, errors.WithStack(errors.Errorf(`Memory request "%s" must be less than or equal to Memory limit "%s"`, memRequest, memLimit)) - } - - resources.Requests = corev1.ResourceList{ - corev1.ResourceCPU: parsedCPURequest, - corev1.ResourceMemory: parsedMemRequest, - } - resources.Limits = corev1.ResourceList{ - corev1.ResourceCPU: parsedCPULimit, - corev1.ResourceMemory: parsedMemLimit, - } - - return resources, nil -} diff --git a/pkg/restore/restic_restore_action.go b/pkg/restore/restic_restore_action.go index 051b2c0c0..6b884a087 100644 --- a/pkg/restore/restic_restore_action.go +++ b/pkg/restore/restic_restore_action.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + "github.com/heptio/velero/pkg/builder" "github.com/heptio/velero/pkg/buildinfo" "github.com/heptio/velero/pkg/plugin/framework" "github.com/heptio/velero/pkg/plugin/velero" @@ -35,7 +36,11 @@ import ( "github.com/heptio/velero/pkg/util/kube" ) -const defaultImageBase = "gcr.io/heptio-images/velero-restic-restore-helper" +const ( + defaultImageBase = "gcr.io/heptio-images/velero-restic-restore-helper" + defaultCPURequestLimit = "100m" + defaultMemRequestLimit = "128Mi" +) type ResticRestoreAction struct { logger logrus.FieldLogger @@ -85,38 +90,30 @@ func (a *ResticRestoreAction) Execute(input *velero.RestoreItemActionExecuteInpu image := getImage(log, config) log.Infof("Using image %q", image) - initContainer := corev1.Container{ - Name: restic.InitContainer, - Image: image, - Args: []string{string(input.Restore.UID)}, - Env: []corev1.EnvVar{ - { - Name: "POD_NAMESPACE", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.namespace", - }, - }, - }, - { - Name: "POD_NAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.name", - }, - }, - }, - }, + cpuRequest, memRequest := getResourceRequests(log, config) + cpuLimit, memLimit := getResourceLimits(log, config) + + resourceReqs, err := kube.ParseResourceRequirements(cpuRequest, memRequest, cpuLimit, memLimit) + if err != nil { + log.Errorf("Using default resource values, couldn't parse resource requirements: %s.", err) + resourceReqs, _ = kube.ParseResourceRequirements( + defaultCPURequestLimit, defaultMemRequestLimit, // requests + defaultCPURequestLimit, defaultMemRequestLimit, // limits + ) } + initContainerBuilder := newResticInitContainerBuilder(image, string(input.Restore.UID)) + initContainerBuilder.Resources(&resourceReqs) + for volumeName := range volumeSnapshots { - mount := corev1.VolumeMount{ + mount := &corev1.VolumeMount{ Name: volumeName, MountPath: "/restores/" + volumeName, } - initContainer.VolumeMounts = append(initContainer.VolumeMounts, mount) + initContainerBuilder.VolumeMounts(mount) } + initContainer := *initContainerBuilder.Result() if len(pod.Spec.InitContainers) == 0 || pod.Spec.InitContainers[0].Name != restic.InitContainer { pod.Spec.InitContainers = append([]corev1.Container{initContainer}, pod.Spec.InitContainers...) } else { @@ -162,6 +159,28 @@ func getImage(log logrus.FieldLogger, config *corev1.ConfigMap) string { } } +// getResourceRequests extracts the CPU and memory requests from a ConfigMap. +// The 0 values are valid if the keys are not present +func getResourceRequests(log logrus.FieldLogger, config *corev1.ConfigMap) (string, string) { + if config == nil { + log.Debug("No config found for plugin") + return "", "" + } + + return config.Data["cpuRequest"], config.Data["memRequest"] +} + +// getResourceLimits extracts the CPU and memory limits from a ConfigMap. +// The 0 values are valid if the keys are not present +func getResourceLimits(log logrus.FieldLogger, config *corev1.ConfigMap) (string, string) { + if config == nil { + log.Debug("No config found for plugin") + return "", "" + } + + return config.Data["cpuLimit"], config.Data["memLimit"] +} + // TODO eventually this can move to pkg/plugin/framework since it'll be used across multiple // plugins. func getPluginConfig(kind framework.PluginKind, name string, client corev1client.ConfigMapInterface) (*corev1.ConfigMap, error) { @@ -191,6 +210,29 @@ func getPluginConfig(kind framework.PluginKind, name string, client corev1client return &list.Items[0], nil } +func newResticInitContainerBuilder(image, restoreUID string) *builder.ContainerBuilder { + return builder.ForContainer(restic.InitContainer, image). + Args(restoreUID). + Env([]*corev1.EnvVar{ + { + Name: "POD_NAMESPACE", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, + }, + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + }...) +} + func initContainerImage(imageBase string) string { tag := buildinfo.Version if tag == "" { diff --git a/pkg/restore/restic_restore_action_test.go b/pkg/restore/restic_restore_action_test.go index 4318771dd..7a842094e 100644 --- a/pkg/restore/restic_restore_action_test.go +++ b/pkg/restore/restic_restore_action_test.go @@ -20,16 +20,25 @@ import ( "fmt" "testing" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" - corev1 "k8s.io/api/core/v1" + "github.com/stretchr/testify/require" + corev1api "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + api "github.com/heptio/velero/pkg/apis/velero/v1" + "github.com/heptio/velero/pkg/builder" "github.com/heptio/velero/pkg/buildinfo" + "github.com/heptio/velero/pkg/plugin/velero" + "github.com/heptio/velero/pkg/util/kube" velerotest "github.com/heptio/velero/pkg/util/test" ) func TestGetImage(t *testing.T) { - configMapWithData := func(key, val string) *corev1.ConfigMap { - return &corev1.ConfigMap{ + configMapWithData := func(key, val string) *corev1api.ConfigMap { + return &corev1api.ConfigMap{ Data: map[string]string{ key: val, }, @@ -44,7 +53,7 @@ func TestGetImage(t *testing.T) { tests := []struct { name string - configMap *corev1.ConfigMap + configMap *corev1api.ConfigMap want string }{ { @@ -80,3 +89,81 @@ func TestGetImage(t *testing.T) { }) } } + +// TestResticRestoreActionExecute tests the restic restore item action plugin's Execute method. +func TestResticRestoreActionExecute(t *testing.T) { + resourceReqs, _ := kube.ParseResourceRequirements( + defaultCPURequestLimit, defaultMemRequestLimit, // requests + defaultCPURequestLimit, defaultMemRequestLimit, // limits + ) + + tests := []struct { + name string + pod *corev1api.Pod + want *corev1api.Pod + }{ + { + name: "Restoring pod with no other initContainers adds the restic initContainer", + pod: builder.ForPod("ns-1", "pod").ObjectMeta( + builder.WithAnnotations("snapshot.velero.io/myvol", "")). + Result(), + want: builder.ForPod("ns-1", "pod"). + ObjectMeta( + builder.WithAnnotations("snapshot.velero.io/myvol", "")). + InitContainers( + newResticInitContainerBuilder(initContainerImage(defaultImageBase), ""). + Resources(&resourceReqs). + VolumeMounts(builder.ForVolumeMount("myvol", "/restores/myvol").Result()).Result()). + Result(), + }, + { + name: "Restoring pod with other initContainers adds the restic initContainer as the first one", + pod: builder.ForPod("ns-1", "pod").ObjectMeta( + builder.WithAnnotations("snapshot.velero.io/myvol", "")). + InitContainers(builder.ForContainer("first-container", "").Result()). + Result(), + want: builder.ForPod("ns-1", "pod"). + ObjectMeta( + builder.WithAnnotations("snapshot.velero.io/myvol", "")). + InitContainers( + newResticInitContainerBuilder(initContainerImage(defaultImageBase), ""). + Resources(&resourceReqs). + VolumeMounts(builder.ForVolumeMount("myvol", "/restores/myvol").Result()).Result(), + builder.ForContainer("first-container", "").Result()). + Result(), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + unstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pod) + require.NoError(t, err) + + input := &velero.RestoreItemActionExecuteInput{ + Item: &unstructured.Unstructured{ + Object: unstructuredMap, + }, + Restore: builder.ForRestore("velero", "my-restore"). + Phase(api.RestorePhaseInProgress). + Result(), + } + + clientset := fake.NewSimpleClientset() + a := NewResticRestoreAction( + logrus.StandardLogger(), + clientset.CoreV1().ConfigMaps("velero"), + ) + + // method under test + res, err := a.Execute(input) + + assert.NoError(t, err) + + wantUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.want) + require.NoError(t, err) + + assert.Equal(t, &unstructured.Unstructured{Object: wantUnstructured}, res.UpdatedItem) + }) + } + +} diff --git a/pkg/util/kube/resource_requirements.go b/pkg/util/kube/resource_requirements.go new file mode 100644 index 000000000..a1c1f23c0 --- /dev/null +++ b/pkg/util/kube/resource_requirements.go @@ -0,0 +1,69 @@ +/* +Copyright 2019 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kube + +import ( + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +// ParseResourceRequirements takes a set of CPU and memory requests and limit string +// values and returns a ResourceRequirements struct to be used in a Container. +// An error is returned if we cannot parse the request/limit. +func ParseResourceRequirements(cpuRequest, memRequest, cpuLimit, memLimit string) (corev1.ResourceRequirements, error) { + var resources corev1.ResourceRequirements + + parsedCPURequest, err := resource.ParseQuantity(cpuRequest) + if err != nil { + return resources, errors.Wrapf(err, `couldn't parse CPU request "%s"`, cpuRequest) + } + + parsedMemRequest, err := resource.ParseQuantity(memRequest) + if err != nil { + return resources, errors.Wrapf(err, `couldn't parse memory request "%s"`, memRequest) + } + + parsedCPULimit, err := resource.ParseQuantity(cpuLimit) + if err != nil { + return resources, errors.Wrapf(err, `couldn't parse CPU limit "%s"`, cpuLimit) + } + + parsedMemLimit, err := resource.ParseQuantity(memLimit) + if err != nil { + return resources, errors.Wrapf(err, `couldn't parse memory limit "%s"`, memLimit) + } + + if parsedCPURequest.Cmp(parsedCPULimit) > 0 { + return resources, errors.WithStack(errors.Errorf(`CPU request "%s" must be less than or equal to CPU limit "%s"`, cpuRequest, cpuLimit)) + } + + if parsedMemRequest.Cmp(parsedMemLimit) > 0 { + return resources, errors.WithStack(errors.Errorf(`Memory request "%s" must be less than or equal to Memory limit "%s"`, memRequest, memLimit)) + } + + resources.Requests = corev1.ResourceList{ + corev1.ResourceCPU: parsedCPURequest, + corev1.ResourceMemory: parsedMemRequest, + } + resources.Limits = corev1.ResourceList{ + corev1.ResourceCPU: parsedCPULimit, + corev1.ResourceMemory: parsedMemLimit, + } + + return resources, nil +} diff --git a/pkg/cmd/cli/install/install_test.go b/pkg/util/kube/resource_requirements_test.go similarity index 90% rename from pkg/cmd/cli/install/install_test.go rename to pkg/util/kube/resource_requirements_test.go index 1aaef7aa3..efd151ca9 100644 --- a/pkg/cmd/cli/install/install_test.go +++ b/pkg/util/kube/resource_requirements_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package install +package kube import ( "testing" @@ -24,7 +24,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" ) -func Test_parseResourceRequests(t *testing.T) { +func TestParseResourceRequirements(t *testing.T) { type args struct { cpuRequest string memRequest string @@ -44,7 +44,7 @@ func Test_parseResourceRequests(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parseResourceRequests(tt.args.cpuRequest, tt.args.memRequest, tt.args.cpuLimit, tt.args.memLimit) + got, err := ParseResourceRequirements(tt.args.cpuRequest, tt.args.memRequest, tt.args.cpuLimit, tt.args.memLimit) if tt.wantErr { assert.Error(t, err) return diff --git a/site/docs/master/restic.md b/site/docs/master/restic.md index c726fd9a5..bef6d89dc 100644 --- a/site/docs/master/restic.md +++ b/site/docs/master/restic.md @@ -199,11 +199,15 @@ the next restic backup taken will be treated as a completely new backup, not an - Restic scans each file in a single thread. This means that large files (such as ones storing a database) will take a long time to scan for data deduplication, even if the actual difference is small. -## Customize Restore Helper Image +## Customize Restore Helper Container Velero uses a helper init container when performing a restic restore. By default, the image for this container is `gcr.io/heptio-images/velero-restic-restore-helper:`, where `VERSION` matches the version/tag of the main Velero image. You can customize the image that is used for this helper by creating a ConfigMap in the Velero namespace with -the alternate image. The ConfigMap must look like the following: +the alternate image. + +In addition, you can customize the resource requirements for the init container, should you need. + +The ConfigMap must look like the following: ```yaml apiVersion: v1 @@ -225,10 +229,28 @@ metadata: # that this ConfigMap is for. velero.io/restic: RestoreItemAction data: - # "image" is the only configurable key. The value can either - # include a tag or not; if the tag is *not* included, the - # tag from the main Velero image will automatically be used. + # The value for "image" can either include a tag or not; + # if the tag is *not* included, the tag from the main Velero + # image will automatically be used. image: myregistry.io/my-custom-helper-image[:OPTIONAL_TAG] + + # "cpuRequest" sets the request.cpu value on the restic init containers during restore. + # If not set, it will default to "100m". + cpuRequest: 200m + + # "memRequest" sets the request.memory value on the restic init containers during restore. + # If not set, it will default to "128Mi". + memRequest: 128Mi + + # "cpuLimit" sets the request.cpu value on the restic init containers during restore. + # If not set, it will default to "100m". + cpuLimit: 200m + + # "memLimit" sets the request.memory value on the restic init containers during restore. + # If not set, it will default to "128Mi". + memLimit: 128Mi + + ``` ## Troubleshooting