add restic integration for doing pod volume backups/restores

Signed-off-by: Steve Kriss <steve@heptio.com>

pkg/controller/pod_volume_backup_controller.go

@@ -0,0 +1,299 @@
+/*
+Copyright 2018 the Heptio Ark contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	jsonpatch "github.com/evanphx/json-patch"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	corev1informers "k8s.io/client-go/informers/core/v1"
+	corev1listers "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/tools/cache"
+
+	arkv1api "github.com/heptio/ark/pkg/apis/ark/v1"
+	arkv1client "github.com/heptio/ark/pkg/generated/clientset/versioned/typed/ark/v1"
+	informers "github.com/heptio/ark/pkg/generated/informers/externalversions/ark/v1"
+	listers "github.com/heptio/ark/pkg/generated/listers/ark/v1"
+	"github.com/heptio/ark/pkg/restic"
+	"github.com/heptio/ark/pkg/util/kube"
+)
+
+type podVolumeBackupController struct {
+	*genericController
+
+	podVolumeBackupClient arkv1client.PodVolumeBackupsGetter
+	podVolumeBackupLister listers.PodVolumeBackupLister
+	secretLister          corev1listers.SecretLister
+	podLister             corev1listers.PodLister
+	pvcLister             corev1listers.PersistentVolumeClaimLister
+	nodeName              string
+
+	processBackupFunc func(*arkv1api.PodVolumeBackup) error
+}
+
+// NewPodVolumeBackupController creates a new pod volume backup controller.
+func NewPodVolumeBackupController(
+	logger logrus.FieldLogger,
+	podVolumeBackupInformer informers.PodVolumeBackupInformer,
+	podVolumeBackupClient arkv1client.PodVolumeBackupsGetter,
+	podInformer cache.SharedIndexInformer,
+	secretInformer corev1informers.SecretInformer,
+	pvcInformer corev1informers.PersistentVolumeClaimInformer,
+	nodeName string,
+) Interface {
+	c := &podVolumeBackupController{
+		genericController:     newGenericController("pod-volume-backup", logger),
+		podVolumeBackupClient: podVolumeBackupClient,
+		podVolumeBackupLister: podVolumeBackupInformer.Lister(),
+		podLister:             corev1listers.NewPodLister(podInformer.GetIndexer()),
+		secretLister:          secretInformer.Lister(),
+		pvcLister:             pvcInformer.Lister(),
+		nodeName:              nodeName,
+	}
+
+	c.syncHandler = c.processQueueItem
+	c.cacheSyncWaiters = append(
+		c.cacheSyncWaiters,
+		podVolumeBackupInformer.Informer().HasSynced,
+		secretInformer.Informer().HasSynced,
+		podInformer.HasSynced,
+		pvcInformer.Informer().HasSynced,
+	)
+	c.processBackupFunc = c.processBackup
+
+	podVolumeBackupInformer.Informer().AddEventHandler(
+		cache.ResourceEventHandlerFuncs{
+			AddFunc:    c.enqueue,
+			UpdateFunc: func(_, obj interface{}) { c.enqueue(obj) },
+		},
+	)
+
+	return c
+}
+
+func (c *podVolumeBackupController) processQueueItem(key string) error {
+	log := c.logger.WithField("key", key)
+	log.Debug("Running processItem")
+
+	ns, name, err := cache.SplitMetaNamespaceKey(key)
+	if err != nil {
+		log.WithError(err).Error("error splitting queue key")
+		return nil
+	}
+
+	req, err := c.podVolumeBackupLister.PodVolumeBackups(ns).Get(name)
+	if apierrors.IsNotFound(err) {
+		log.Debug("Unable to find PodVolumeBackup")
+		return nil
+	}
+	if err != nil {
+		return errors.Wrap(err, "error getting PodVolumeBackup")
+	}
+
+	// only process new items
+	switch req.Status.Phase {
+	case "", arkv1api.PodVolumeBackupPhaseNew:
+	default:
+		return nil
+	}
+
+	// only process items for this node
+	if req.Spec.Node != c.nodeName {
+		return nil
+	}
+
+	// Don't mutate the shared cache
+	reqCopy := req.DeepCopy()
+	return c.processBackupFunc(reqCopy)
+}
+
+func (c *podVolumeBackupController) processBackup(req *arkv1api.PodVolumeBackup) error {
+	log := c.logger.WithFields(logrus.Fields{
+		"namespace": req.Namespace,
+		"name":      req.Name,
+	})
+
+	var err error
+
+	// update status to InProgress
+	req, err = c.patchPodVolumeBackup(req, updatePhaseFunc(arkv1api.PodVolumeBackupPhaseInProgress))
+	if err != nil {
+		log.WithError(err).Error("Error setting phase to InProgress")
+		return errors.WithStack(err)
+	}
+
+	pod, err := c.podLister.Pods(req.Spec.Pod.Namespace).Get(req.Spec.Pod.Name)
+	if err != nil {
+		log.WithError(err).Errorf("Error getting pod %s/%s", req.Spec.Pod.Namespace, req.Spec.Pod.Name)
+		return c.fail(req, errors.Wrap(err, "error getting pod").Error(), log)
+	}
+
+	volumeDir, err := kube.GetVolumeDirectory(pod, req.Spec.Volume, c.pvcLister)
+	if err != nil {
+		log.WithError(err).Error("Error getting volume directory name")
+		return c.fail(req, errors.Wrap(err, "error getting volume directory name").Error(), log)
+	}
+
+	path, err := singlePathMatch(fmt.Sprintf("/host_pods/%s/volumes/*/%s", string(req.Spec.Pod.UID), volumeDir))
+	if err != nil {
+		log.WithError(err).Error("Error uniquely identifying volume path")
+		return c.fail(req, errors.Wrap(err, "error getting volume path on host").Error(), log)
+	}
+
+	// temp creds
+	file, err := restic.TempCredentialsFile(c.secretLister, req.Spec.Pod.Namespace)
+	if err != nil {
+		log.WithError(err).Error("Error creating temp restic credentials file")
+		return c.fail(req, errors.Wrap(err, "error creating temp restic credentials file").Error(), log)
+	}
+	// ignore error since there's nothing we can do and it's a temp file.
+	defer os.Remove(file)
+
+	resticCmd := restic.BackupCommand(
+		req.Spec.RepoPrefix,
+		req.Spec.Pod.Namespace,
+		file,
+		path,
+		req.Spec.Tags,
+	)
+
+	var stdout, stderr string
+
+	if stdout, stderr, err = runCommand(resticCmd.Cmd()); err != nil {
+		log.WithError(errors.WithStack(err)).Errorf("Error running command=%s, stdout=%s, stderr=%s", resticCmd.String(), stdout, stderr)
+		return c.fail(req, fmt.Sprintf("error running restic backup, stderr=%s: %s", stderr, err.Error()), log)
+	}
+	log.Debugf("Ran command=%s, stdout=%s, stderr=%s", resticCmd.String(), stdout, stderr)
+
+	snapshotID, err := restic.GetSnapshotID(req.Spec.RepoPrefix, req.Spec.Pod.Namespace, file, req.Spec.Tags)
+	if err != nil {
+		log.WithError(err).Error("Error getting SnapshotID")
+		return c.fail(req, errors.Wrap(err, "error getting snapshot id").Error(), log)
+	}
+
+	// update status to Completed with path & snapshot id
+	req, err = c.patchPodVolumeBackup(req, func(r *arkv1api.PodVolumeBackup) {
+		r.Status.Path = path
+		r.Status.SnapshotID = snapshotID
+		r.Status.Phase = arkv1api.PodVolumeBackupPhaseCompleted
+	})
+	if err != nil {
+		log.WithError(err).Error("Error setting phase to Completed")
+		return err
+	}
+
+	return nil
+}
+
+// runCommand runs a command and returns its stdout, stderr, and its returned
+// error (if any). If there are errors reading stdout or stderr, their return
+// value(s) will contain the error as a string.
+func runCommand(cmd *exec.Cmd) (string, string, error) {
+	stdoutBuf := new(bytes.Buffer)
+	stderrBuf := new(bytes.Buffer)
+
+	cmd.Stdout = stdoutBuf
+	cmd.Stderr = stderrBuf
+
+	runErr := cmd.Run()
+
+	var stdout, stderr string
+
+	if res, readErr := ioutil.ReadAll(stdoutBuf); readErr != nil {
+		stdout = errors.Wrap(readErr, "error reading command's stdout").Error()
+	} else {
+		stdout = string(res)
+	}
+
+	if res, readErr := ioutil.ReadAll(stderrBuf); readErr != nil {
+		stderr = errors.Wrap(readErr, "error reading command's stderr").Error()
+	} else {
+		stderr = string(res)
+	}
+
+	return stdout, stderr, runErr
+}
+
+func (c *podVolumeBackupController) patchPodVolumeBackup(req *arkv1api.PodVolumeBackup, mutate func(*arkv1api.PodVolumeBackup)) (*arkv1api.PodVolumeBackup, error) {
+	// Record original json
+	oldData, err := json.Marshal(req)
+	if err != nil {
+		return nil, errors.Wrap(err, "error marshalling original PodVolumeBackup")
+	}
+
+	// Mutate
+	mutate(req)
+
+	// Record new json
+	newData, err := json.Marshal(req)
+	if err != nil {
+		return nil, errors.Wrap(err, "error marshalling updated PodVolumeBackup")
+	}
+
+	patchBytes, err := jsonpatch.CreateMergePatch(oldData, newData)
+	if err != nil {
+		return nil, errors.Wrap(err, "error creating json merge patch for PodVolumeBackup")
+	}
+
+	req, err = c.podVolumeBackupClient.PodVolumeBackups(req.Namespace).Patch(req.Name, types.MergePatchType, patchBytes)
+	if err != nil {
+		return nil, errors.Wrap(err, "error patching PodVolumeBackup")
+	}
+
+	return req, nil
+}
+
+func (c *podVolumeBackupController) fail(req *arkv1api.PodVolumeBackup, msg string, log logrus.FieldLogger) error {
+	if _, err := c.patchPodVolumeBackup(req, func(r *arkv1api.PodVolumeBackup) {
+		r.Status.Phase = arkv1api.PodVolumeBackupPhaseFailed
+		r.Status.Message = msg
+	}); err != nil {
+		log.WithError(err).Error("Error setting phase to Failed")
+		return err
+	}
+	return nil
+}
+
+func updatePhaseFunc(phase arkv1api.PodVolumeBackupPhase) func(r *arkv1api.PodVolumeBackup) {
+	return func(r *arkv1api.PodVolumeBackup) {
+		r.Status.Phase = phase
+	}
+}
+
+func singlePathMatch(path string) (string, error) {
+	matches, err := filepath.Glob(path)
+	if err != nil {
+		return "", errors.WithStack(err)
+	}
+
+	if len(matches) != 1 {
+		return "", errors.Errorf("expected one matching path, got %d", len(matches))
+	}
+
+	return matches[0], nil
+}