Add resource limits to restic init container (#1677)

* Add resource limits to restic init container

Fixes #1201

Signed-off-by: Nolan Brubaker <brubakern@vmware.com>

* Start restic restore item action tests

Signed-off-by: Nolan Brubaker <brubakern@vmware.com>

* Get initial tests for restore action working

Signed-off-by: Nolan Brubaker <brubakern@vmware.com>

* Add new test case

Signed-off-by: Nolan Brubaker <brubakern@vmware.com>

* Move resource parsing into a shared function

Signed-off-by: Nolan Brubaker <brubakern@vmware.com>

* Fetch request/limits from plugin's ConfigMap

Signed-off-by: Nolan Brubaker <brubakern@vmware.com>

* Use builders

Signed-off-by: Nolan Brubaker <brubakern@vmware.com>

* Use moved ParseResourceRequirements function

Signed-off-by: Nolan Brubaker <brubakern@vmware.com>

* Move init container building inline

Signed-off-by: Nolan Brubaker <brubakern@vmware.com>

* Drop CPU limit down a bit and clarify error message

Signed-off-by: Nolan Brubaker <brubakern@vmware.com>

* Fix godoc

Signed-off-by: Nolan Brubaker <brubakern@vmware.com>

* Add resource requirements to doc

Signed-off-by: Nolan Brubaker <brubakern@vmware.com>
This commit is contained in:
Nolan Brubaker
2019-08-05 15:18:11 -04:00
committed by Steve Kriss
parent 2254635bcb
commit a4e70456a1
10 changed files with 377 additions and 87 deletions

View File

@@ -0,0 +1 @@
Add low cpu/memory limits to the restic init container. This allows for restoration into namespaces with quotas defined.

View File

@@ -0,0 +1,68 @@
/*
Copyright 2019 the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package builder
import (
corev1api "k8s.io/api/core/v1"
)
// ContainerBuilder builds Container objects
type ContainerBuilder struct {
object *corev1api.Container
}
// ForContainer is the constructor for ContainerBuilder.
func ForContainer(name, image string) *ContainerBuilder {
return &ContainerBuilder{
object: &corev1api.Container{
Name: name,
Image: image,
},
}
}
// Result returns the built Container.
func (b *ContainerBuilder) Result() *corev1api.Container {
return b.object
}
// Args sets the container's Args.
func (b *ContainerBuilder) Args(args ...string) *ContainerBuilder {
b.object.Args = append(b.object.Args, args...)
return b
}
// VolumeMounts sets the container's VolumeMounts.
func (b *ContainerBuilder) VolumeMounts(volumeMounts ...*corev1api.VolumeMount) *ContainerBuilder {
for _, v := range volumeMounts {
b.object.VolumeMounts = append(b.object.VolumeMounts, *v)
}
return b
}
// Resources sets the container's Resources.
func (b *ContainerBuilder) Resources(resources *corev1api.ResourceRequirements) *ContainerBuilder {
b.object.Resources = *resources
return b
}
func (b *ContainerBuilder) Env(vars ...*corev1api.EnvVar) *ContainerBuilder {
for _, v := range vars {
b.object.Env = append(b.object.Env, *v)
}
return b
}

View File

@@ -69,3 +69,10 @@ func (b *PodBuilder) NodeName(val string) *PodBuilder {
b.object.Spec.NodeName = val
return b
}
func (b *PodBuilder) InitContainers(containers ...*corev1api.Container) *PodBuilder {
for _, c := range containers {
b.object.Spec.InitContainers = append(b.object.Spec.InitContainers, *c)
}
return b
}

View File

@@ -0,0 +1,41 @@
/*
Copyright 2019 the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package builder
import (
corev1api "k8s.io/api/core/v1"
)
// VolumeMountBuilder builds VolumeMount objects.
type VolumeMountBuilder struct {
object *corev1api.VolumeMount
}
// ForVolumeMount is the constructor for a VolumeMountBuilder.
func ForVolumeMount(name, mountPath string) *VolumeMountBuilder {
return &VolumeMountBuilder{
object: &corev1api.VolumeMount{
Name: name,
MountPath: mountPath,
},
}
}
// Result returns the built VolumeMount.
func (b *VolumeMountBuilder) Result() *corev1api.VolumeMount {
return b.object
}

View File

@@ -26,8 +26,6 @@ import (
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
api "github.com/heptio/velero/pkg/apis/velero/v1"
"github.com/heptio/velero/pkg/client"
@@ -35,6 +33,7 @@ import (
"github.com/heptio/velero/pkg/cmd/util/flag"
"github.com/heptio/velero/pkg/cmd/util/output"
"github.com/heptio/velero/pkg/install"
kubeutil "github.com/heptio/velero/pkg/util/kube"
)
// InstallOptions collects all the options for installing Velero into a Kubernetes cluster.
@@ -125,11 +124,11 @@ func (o *InstallOptions) AsVeleroOptions() (*install.VeleroOptions, error) {
return nil, err
}
}
veleroPodResources, err := parseResourceRequests(o.VeleroPodCPURequest, o.VeleroPodMemRequest, o.VeleroPodCPULimit, o.VeleroPodMemLimit)
veleroPodResources, err := kubeutil.ParseResourceRequirements(o.VeleroPodCPURequest, o.VeleroPodMemRequest, o.VeleroPodCPULimit, o.VeleroPodMemLimit)
if err != nil {
return nil, err
}
resticPodResources, err := parseResourceRequests(o.ResticPodCPURequest, o.ResticPodMemRequest, o.ResticPodCPULimit, o.ResticPodMemLimit)
resticPodResources, err := kubeutil.ParseResourceRequirements(o.ResticPodCPURequest, o.ResticPodMemRequest, o.ResticPodCPULimit, o.ResticPodMemLimit)
if err != nil {
return nil, err
}
@@ -285,49 +284,3 @@ func (o *InstallOptions) Validate(c *cobra.Command, args []string, f client.Fact
return nil
}
// parseResourceRequests takes a set of CPU and memory requests and limit string
// values and returns a ResourceRequirements struct to be used in a Container.
// An error is returned if we cannot parse the request/limit.
func parseResourceRequests(cpuRequest, memRequest, cpuLimit, memLimit string) (corev1.ResourceRequirements, error) {
var resources corev1.ResourceRequirements
parsedCPURequest, err := resource.ParseQuantity(cpuRequest)
if err != nil {
return resources, errors.Wrapf(err, `couldn't parse CPU request "%s"`, cpuRequest)
}
parsedMemRequest, err := resource.ParseQuantity(memRequest)
if err != nil {
return resources, errors.Wrapf(err, `couldn't parse memory request "%s"`, memRequest)
}
parsedCPULimit, err := resource.ParseQuantity(cpuLimit)
if err != nil {
return resources, errors.Wrapf(err, `couldn't parse CPU limit "%s"`, cpuLimit)
}
parsedMemLimit, err := resource.ParseQuantity(memLimit)
if err != nil {
return resources, errors.Wrapf(err, `couldn't parse memory limit "%s"`, memLimit)
}
if parsedCPURequest.Cmp(parsedCPULimit) > 0 {
return resources, errors.WithStack(errors.Errorf(`CPU request "%s" must be less than or equal to CPU limit "%s"`, cpuRequest, cpuLimit))
}
if parsedMemRequest.Cmp(parsedMemLimit) > 0 {
return resources, errors.WithStack(errors.Errorf(`Memory request "%s" must be less than or equal to Memory limit "%s"`, memRequest, memLimit))
}
resources.Requests = corev1.ResourceList{
corev1.ResourceCPU: parsedCPURequest,
corev1.ResourceMemory: parsedMemRequest,
}
resources.Limits = corev1.ResourceList{
corev1.ResourceCPU: parsedCPULimit,
corev1.ResourceMemory: parsedMemLimit,
}
return resources, nil
}

View File

@@ -28,6 +28,7 @@ import (
"k8s.io/apimachinery/pkg/runtime"
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
"github.com/heptio/velero/pkg/builder"
"github.com/heptio/velero/pkg/buildinfo"
"github.com/heptio/velero/pkg/plugin/framework"
"github.com/heptio/velero/pkg/plugin/velero"
@@ -35,7 +36,11 @@ import (
"github.com/heptio/velero/pkg/util/kube"
)
const defaultImageBase = "gcr.io/heptio-images/velero-restic-restore-helper"
const (
defaultImageBase = "gcr.io/heptio-images/velero-restic-restore-helper"
defaultCPURequestLimit = "100m"
defaultMemRequestLimit = "128Mi"
)
type ResticRestoreAction struct {
logger logrus.FieldLogger
@@ -85,38 +90,30 @@ func (a *ResticRestoreAction) Execute(input *velero.RestoreItemActionExecuteInpu
image := getImage(log, config)
log.Infof("Using image %q", image)
initContainer := corev1.Container{
Name: restic.InitContainer,
Image: image,
Args: []string{string(input.Restore.UID)},
Env: []corev1.EnvVar{
{
Name: "POD_NAMESPACE",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "metadata.namespace",
},
},
},
{
Name: "POD_NAME",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "metadata.name",
},
},
},
},
cpuRequest, memRequest := getResourceRequests(log, config)
cpuLimit, memLimit := getResourceLimits(log, config)
resourceReqs, err := kube.ParseResourceRequirements(cpuRequest, memRequest, cpuLimit, memLimit)
if err != nil {
log.Errorf("Using default resource values, couldn't parse resource requirements: %s.", err)
resourceReqs, _ = kube.ParseResourceRequirements(
defaultCPURequestLimit, defaultMemRequestLimit, // requests
defaultCPURequestLimit, defaultMemRequestLimit, // limits
)
}
initContainerBuilder := newResticInitContainerBuilder(image, string(input.Restore.UID))
initContainerBuilder.Resources(&resourceReqs)
for volumeName := range volumeSnapshots {
mount := corev1.VolumeMount{
mount := &corev1.VolumeMount{
Name: volumeName,
MountPath: "/restores/" + volumeName,
}
initContainer.VolumeMounts = append(initContainer.VolumeMounts, mount)
initContainerBuilder.VolumeMounts(mount)
}
initContainer := *initContainerBuilder.Result()
if len(pod.Spec.InitContainers) == 0 || pod.Spec.InitContainers[0].Name != restic.InitContainer {
pod.Spec.InitContainers = append([]corev1.Container{initContainer}, pod.Spec.InitContainers...)
} else {
@@ -162,6 +159,28 @@ func getImage(log logrus.FieldLogger, config *corev1.ConfigMap) string {
}
}
// getResourceRequests extracts the CPU and memory requests from a ConfigMap.
// The 0 values are valid if the keys are not present
func getResourceRequests(log logrus.FieldLogger, config *corev1.ConfigMap) (string, string) {
if config == nil {
log.Debug("No config found for plugin")
return "", ""
}
return config.Data["cpuRequest"], config.Data["memRequest"]
}
// getResourceLimits extracts the CPU and memory limits from a ConfigMap.
// The 0 values are valid if the keys are not present
func getResourceLimits(log logrus.FieldLogger, config *corev1.ConfigMap) (string, string) {
if config == nil {
log.Debug("No config found for plugin")
return "", ""
}
return config.Data["cpuLimit"], config.Data["memLimit"]
}
// TODO eventually this can move to pkg/plugin/framework since it'll be used across multiple
// plugins.
func getPluginConfig(kind framework.PluginKind, name string, client corev1client.ConfigMapInterface) (*corev1.ConfigMap, error) {
@@ -191,6 +210,29 @@ func getPluginConfig(kind framework.PluginKind, name string, client corev1client
return &list.Items[0], nil
}
func newResticInitContainerBuilder(image, restoreUID string) *builder.ContainerBuilder {
return builder.ForContainer(restic.InitContainer, image).
Args(restoreUID).
Env([]*corev1.EnvVar{
{
Name: "POD_NAMESPACE",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "metadata.namespace",
},
},
},
{
Name: "POD_NAME",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "metadata.name",
},
},
},
}...)
}
func initContainerImage(imageBase string) string {
tag := buildinfo.Version
if tag == "" {

View File

@@ -20,16 +20,25 @@ import (
"fmt"
"testing"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
"github.com/stretchr/testify/require"
corev1api "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/fake"
api "github.com/heptio/velero/pkg/apis/velero/v1"
"github.com/heptio/velero/pkg/builder"
"github.com/heptio/velero/pkg/buildinfo"
"github.com/heptio/velero/pkg/plugin/velero"
"github.com/heptio/velero/pkg/util/kube"
velerotest "github.com/heptio/velero/pkg/util/test"
)
func TestGetImage(t *testing.T) {
configMapWithData := func(key, val string) *corev1.ConfigMap {
return &corev1.ConfigMap{
configMapWithData := func(key, val string) *corev1api.ConfigMap {
return &corev1api.ConfigMap{
Data: map[string]string{
key: val,
},
@@ -44,7 +53,7 @@ func TestGetImage(t *testing.T) {
tests := []struct {
name string
configMap *corev1.ConfigMap
configMap *corev1api.ConfigMap
want string
}{
{
@@ -80,3 +89,81 @@ func TestGetImage(t *testing.T) {
})
}
}
// TestResticRestoreActionExecute tests the restic restore item action plugin's Execute method.
func TestResticRestoreActionExecute(t *testing.T) {
resourceReqs, _ := kube.ParseResourceRequirements(
defaultCPURequestLimit, defaultMemRequestLimit, // requests
defaultCPURequestLimit, defaultMemRequestLimit, // limits
)
tests := []struct {
name string
pod *corev1api.Pod
want *corev1api.Pod
}{
{
name: "Restoring pod with no other initContainers adds the restic initContainer",
pod: builder.ForPod("ns-1", "pod").ObjectMeta(
builder.WithAnnotations("snapshot.velero.io/myvol", "")).
Result(),
want: builder.ForPod("ns-1", "pod").
ObjectMeta(
builder.WithAnnotations("snapshot.velero.io/myvol", "")).
InitContainers(
newResticInitContainerBuilder(initContainerImage(defaultImageBase), "").
Resources(&resourceReqs).
VolumeMounts(builder.ForVolumeMount("myvol", "/restores/myvol").Result()).Result()).
Result(),
},
{
name: "Restoring pod with other initContainers adds the restic initContainer as the first one",
pod: builder.ForPod("ns-1", "pod").ObjectMeta(
builder.WithAnnotations("snapshot.velero.io/myvol", "")).
InitContainers(builder.ForContainer("first-container", "").Result()).
Result(),
want: builder.ForPod("ns-1", "pod").
ObjectMeta(
builder.WithAnnotations("snapshot.velero.io/myvol", "")).
InitContainers(
newResticInitContainerBuilder(initContainerImage(defaultImageBase), "").
Resources(&resourceReqs).
VolumeMounts(builder.ForVolumeMount("myvol", "/restores/myvol").Result()).Result(),
builder.ForContainer("first-container", "").Result()).
Result(),
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
unstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pod)
require.NoError(t, err)
input := &velero.RestoreItemActionExecuteInput{
Item: &unstructured.Unstructured{
Object: unstructuredMap,
},
Restore: builder.ForRestore("velero", "my-restore").
Phase(api.RestorePhaseInProgress).
Result(),
}
clientset := fake.NewSimpleClientset()
a := NewResticRestoreAction(
logrus.StandardLogger(),
clientset.CoreV1().ConfigMaps("velero"),
)
// method under test
res, err := a.Execute(input)
assert.NoError(t, err)
wantUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.want)
require.NoError(t, err)
assert.Equal(t, &unstructured.Unstructured{Object: wantUnstructured}, res.UpdatedItem)
})
}
}

View File

@@ -0,0 +1,69 @@
/*
Copyright 2019 the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package kube
import (
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
)
// ParseResourceRequirements takes a set of CPU and memory requests and limit string
// values and returns a ResourceRequirements struct to be used in a Container.
// An error is returned if we cannot parse the request/limit.
func ParseResourceRequirements(cpuRequest, memRequest, cpuLimit, memLimit string) (corev1.ResourceRequirements, error) {
var resources corev1.ResourceRequirements
parsedCPURequest, err := resource.ParseQuantity(cpuRequest)
if err != nil {
return resources, errors.Wrapf(err, `couldn't parse CPU request "%s"`, cpuRequest)
}
parsedMemRequest, err := resource.ParseQuantity(memRequest)
if err != nil {
return resources, errors.Wrapf(err, `couldn't parse memory request "%s"`, memRequest)
}
parsedCPULimit, err := resource.ParseQuantity(cpuLimit)
if err != nil {
return resources, errors.Wrapf(err, `couldn't parse CPU limit "%s"`, cpuLimit)
}
parsedMemLimit, err := resource.ParseQuantity(memLimit)
if err != nil {
return resources, errors.Wrapf(err, `couldn't parse memory limit "%s"`, memLimit)
}
if parsedCPURequest.Cmp(parsedCPULimit) > 0 {
return resources, errors.WithStack(errors.Errorf(`CPU request "%s" must be less than or equal to CPU limit "%s"`, cpuRequest, cpuLimit))
}
if parsedMemRequest.Cmp(parsedMemLimit) > 0 {
return resources, errors.WithStack(errors.Errorf(`Memory request "%s" must be less than or equal to Memory limit "%s"`, memRequest, memLimit))
}
resources.Requests = corev1.ResourceList{
corev1.ResourceCPU: parsedCPURequest,
corev1.ResourceMemory: parsedMemRequest,
}
resources.Limits = corev1.ResourceList{
corev1.ResourceCPU: parsedCPULimit,
corev1.ResourceMemory: parsedMemLimit,
}
return resources, nil
}

View File

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package install
package kube
import (
"testing"
@@ -24,7 +24,7 @@ import (
"k8s.io/apimachinery/pkg/api/resource"
)
func Test_parseResourceRequests(t *testing.T) {
func TestParseResourceRequirements(t *testing.T) {
type args struct {
cpuRequest string
memRequest string
@@ -44,7 +44,7 @@ func Test_parseResourceRequests(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := parseResourceRequests(tt.args.cpuRequest, tt.args.memRequest, tt.args.cpuLimit, tt.args.memLimit)
got, err := ParseResourceRequirements(tt.args.cpuRequest, tt.args.memRequest, tt.args.cpuLimit, tt.args.memLimit)
if tt.wantErr {
assert.Error(t, err)
return

View File

@@ -199,11 +199,15 @@ the next restic backup taken will be treated as a completely new backup, not an
- Restic scans each file in a single thread. This means that large files (such as ones storing a database) will take a long time to scan for data deduplication, even if the actual
difference is small.
## Customize Restore Helper Image
## Customize Restore Helper Container
Velero uses a helper init container when performing a restic restore. By default, the image for this container is `gcr.io/heptio-images/velero-restic-restore-helper:<VERSION>`,
where `VERSION` matches the version/tag of the main Velero image. You can customize the image that is used for this helper by creating a ConfigMap in the Velero namespace with
the alternate image. The ConfigMap must look like the following:
the alternate image.
In addition, you can customize the resource requirements for the init container, should you need.
The ConfigMap must look like the following:
```yaml
apiVersion: v1
@@ -225,10 +229,28 @@ metadata:
# that this ConfigMap is for.
velero.io/restic: RestoreItemAction
data:
# "image" is the only configurable key. The value can either
# include a tag or not; if the tag is *not* included, the
# tag from the main Velero image will automatically be used.
# The value for "image" can either include a tag or not;
# if the tag is *not* included, the tag from the main Velero
# image will automatically be used.
image: myregistry.io/my-custom-helper-image[:OPTIONAL_TAG]
# "cpuRequest" sets the request.cpu value on the restic init containers during restore.
# If not set, it will default to "100m".
cpuRequest: 200m
# "memRequest" sets the request.memory value on the restic init containers during restore.
# If not set, it will default to "128Mi".
memRequest: 128Mi
# "cpuLimit" sets the request.cpu value on the restic init containers during restore.
# If not set, it will default to "100m".
cpuLimit: 200m
# "memLimit" sets the request.memory value on the restic init containers during restore.
# If not set, it will default to "128Mi".
memLimit: 128Mi
```
## Troubleshooting