mirror of
https://github.com/vmware-tanzu/velero.git
synced 2026-01-05 04:55:22 +00:00
300 lines
9.1 KiB
Go
300 lines
9.1 KiB
Go
/*
|
|
Copyright 2018 the Heptio Ark contributors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package controller
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
|
|
jsonpatch "github.com/evanphx/json-patch"
|
|
"github.com/pkg/errors"
|
|
"github.com/sirupsen/logrus"
|
|
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
corev1informers "k8s.io/client-go/informers/core/v1"
|
|
corev1listers "k8s.io/client-go/listers/core/v1"
|
|
"k8s.io/client-go/tools/cache"
|
|
|
|
arkv1api "github.com/heptio/ark/pkg/apis/ark/v1"
|
|
arkv1client "github.com/heptio/ark/pkg/generated/clientset/versioned/typed/ark/v1"
|
|
informers "github.com/heptio/ark/pkg/generated/informers/externalversions/ark/v1"
|
|
listers "github.com/heptio/ark/pkg/generated/listers/ark/v1"
|
|
"github.com/heptio/ark/pkg/restic"
|
|
"github.com/heptio/ark/pkg/util/kube"
|
|
)
|
|
|
|
type podVolumeBackupController struct {
|
|
*genericController
|
|
|
|
podVolumeBackupClient arkv1client.PodVolumeBackupsGetter
|
|
podVolumeBackupLister listers.PodVolumeBackupLister
|
|
secretLister corev1listers.SecretLister
|
|
podLister corev1listers.PodLister
|
|
pvcLister corev1listers.PersistentVolumeClaimLister
|
|
nodeName string
|
|
|
|
processBackupFunc func(*arkv1api.PodVolumeBackup) error
|
|
}
|
|
|
|
// NewPodVolumeBackupController creates a new pod volume backup controller.
|
|
func NewPodVolumeBackupController(
|
|
logger logrus.FieldLogger,
|
|
podVolumeBackupInformer informers.PodVolumeBackupInformer,
|
|
podVolumeBackupClient arkv1client.PodVolumeBackupsGetter,
|
|
podInformer cache.SharedIndexInformer,
|
|
secretInformer corev1informers.SecretInformer,
|
|
pvcInformer corev1informers.PersistentVolumeClaimInformer,
|
|
nodeName string,
|
|
) Interface {
|
|
c := &podVolumeBackupController{
|
|
genericController: newGenericController("pod-volume-backup", logger),
|
|
podVolumeBackupClient: podVolumeBackupClient,
|
|
podVolumeBackupLister: podVolumeBackupInformer.Lister(),
|
|
podLister: corev1listers.NewPodLister(podInformer.GetIndexer()),
|
|
secretLister: secretInformer.Lister(),
|
|
pvcLister: pvcInformer.Lister(),
|
|
nodeName: nodeName,
|
|
}
|
|
|
|
c.syncHandler = c.processQueueItem
|
|
c.cacheSyncWaiters = append(
|
|
c.cacheSyncWaiters,
|
|
podVolumeBackupInformer.Informer().HasSynced,
|
|
secretInformer.Informer().HasSynced,
|
|
podInformer.HasSynced,
|
|
pvcInformer.Informer().HasSynced,
|
|
)
|
|
c.processBackupFunc = c.processBackup
|
|
|
|
podVolumeBackupInformer.Informer().AddEventHandler(
|
|
cache.ResourceEventHandlerFuncs{
|
|
AddFunc: c.enqueue,
|
|
UpdateFunc: func(_, obj interface{}) { c.enqueue(obj) },
|
|
},
|
|
)
|
|
|
|
return c
|
|
}
|
|
|
|
func (c *podVolumeBackupController) processQueueItem(key string) error {
|
|
log := c.logger.WithField("key", key)
|
|
log.Debug("Running processItem")
|
|
|
|
ns, name, err := cache.SplitMetaNamespaceKey(key)
|
|
if err != nil {
|
|
log.WithError(err).Error("error splitting queue key")
|
|
return nil
|
|
}
|
|
|
|
req, err := c.podVolumeBackupLister.PodVolumeBackups(ns).Get(name)
|
|
if apierrors.IsNotFound(err) {
|
|
log.Debug("Unable to find PodVolumeBackup")
|
|
return nil
|
|
}
|
|
if err != nil {
|
|
return errors.Wrap(err, "error getting PodVolumeBackup")
|
|
}
|
|
|
|
// only process new items
|
|
switch req.Status.Phase {
|
|
case "", arkv1api.PodVolumeBackupPhaseNew:
|
|
default:
|
|
return nil
|
|
}
|
|
|
|
// only process items for this node
|
|
if req.Spec.Node != c.nodeName {
|
|
return nil
|
|
}
|
|
|
|
// Don't mutate the shared cache
|
|
reqCopy := req.DeepCopy()
|
|
return c.processBackupFunc(reqCopy)
|
|
}
|
|
|
|
func (c *podVolumeBackupController) processBackup(req *arkv1api.PodVolumeBackup) error {
|
|
log := c.logger.WithFields(logrus.Fields{
|
|
"namespace": req.Namespace,
|
|
"name": req.Name,
|
|
})
|
|
|
|
var err error
|
|
|
|
// update status to InProgress
|
|
req, err = c.patchPodVolumeBackup(req, updatePhaseFunc(arkv1api.PodVolumeBackupPhaseInProgress))
|
|
if err != nil {
|
|
log.WithError(err).Error("Error setting phase to InProgress")
|
|
return errors.WithStack(err)
|
|
}
|
|
|
|
pod, err := c.podLister.Pods(req.Spec.Pod.Namespace).Get(req.Spec.Pod.Name)
|
|
if err != nil {
|
|
log.WithError(err).Errorf("Error getting pod %s/%s", req.Spec.Pod.Namespace, req.Spec.Pod.Name)
|
|
return c.fail(req, errors.Wrap(err, "error getting pod").Error(), log)
|
|
}
|
|
|
|
volumeDir, err := kube.GetVolumeDirectory(pod, req.Spec.Volume, c.pvcLister)
|
|
if err != nil {
|
|
log.WithError(err).Error("Error getting volume directory name")
|
|
return c.fail(req, errors.Wrap(err, "error getting volume directory name").Error(), log)
|
|
}
|
|
|
|
path, err := singlePathMatch(fmt.Sprintf("/host_pods/%s/volumes/*/%s", string(req.Spec.Pod.UID), volumeDir))
|
|
if err != nil {
|
|
log.WithError(err).Error("Error uniquely identifying volume path")
|
|
return c.fail(req, errors.Wrap(err, "error getting volume path on host").Error(), log)
|
|
}
|
|
|
|
// temp creds
|
|
file, err := restic.TempCredentialsFile(c.secretLister, req.Spec.Pod.Namespace)
|
|
if err != nil {
|
|
log.WithError(err).Error("Error creating temp restic credentials file")
|
|
return c.fail(req, errors.Wrap(err, "error creating temp restic credentials file").Error(), log)
|
|
}
|
|
// ignore error since there's nothing we can do and it's a temp file.
|
|
defer os.Remove(file)
|
|
|
|
resticCmd := restic.BackupCommand(
|
|
req.Spec.RepoPrefix,
|
|
req.Spec.Pod.Namespace,
|
|
file,
|
|
path,
|
|
req.Spec.Tags,
|
|
)
|
|
|
|
var stdout, stderr string
|
|
|
|
if stdout, stderr, err = runCommand(resticCmd.Cmd()); err != nil {
|
|
log.WithError(errors.WithStack(err)).Errorf("Error running command=%s, stdout=%s, stderr=%s", resticCmd.String(), stdout, stderr)
|
|
return c.fail(req, fmt.Sprintf("error running restic backup, stderr=%s: %s", stderr, err.Error()), log)
|
|
}
|
|
log.Debugf("Ran command=%s, stdout=%s, stderr=%s", resticCmd.String(), stdout, stderr)
|
|
|
|
snapshotID, err := restic.GetSnapshotID(req.Spec.RepoPrefix, req.Spec.Pod.Namespace, file, req.Spec.Tags)
|
|
if err != nil {
|
|
log.WithError(err).Error("Error getting SnapshotID")
|
|
return c.fail(req, errors.Wrap(err, "error getting snapshot id").Error(), log)
|
|
}
|
|
|
|
// update status to Completed with path & snapshot id
|
|
req, err = c.patchPodVolumeBackup(req, func(r *arkv1api.PodVolumeBackup) {
|
|
r.Status.Path = path
|
|
r.Status.SnapshotID = snapshotID
|
|
r.Status.Phase = arkv1api.PodVolumeBackupPhaseCompleted
|
|
})
|
|
if err != nil {
|
|
log.WithError(err).Error("Error setting phase to Completed")
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// runCommand runs a command and returns its stdout, stderr, and its returned
|
|
// error (if any). If there are errors reading stdout or stderr, their return
|
|
// value(s) will contain the error as a string.
|
|
func runCommand(cmd *exec.Cmd) (string, string, error) {
|
|
stdoutBuf := new(bytes.Buffer)
|
|
stderrBuf := new(bytes.Buffer)
|
|
|
|
cmd.Stdout = stdoutBuf
|
|
cmd.Stderr = stderrBuf
|
|
|
|
runErr := cmd.Run()
|
|
|
|
var stdout, stderr string
|
|
|
|
if res, readErr := ioutil.ReadAll(stdoutBuf); readErr != nil {
|
|
stdout = errors.Wrap(readErr, "error reading command's stdout").Error()
|
|
} else {
|
|
stdout = string(res)
|
|
}
|
|
|
|
if res, readErr := ioutil.ReadAll(stderrBuf); readErr != nil {
|
|
stderr = errors.Wrap(readErr, "error reading command's stderr").Error()
|
|
} else {
|
|
stderr = string(res)
|
|
}
|
|
|
|
return stdout, stderr, runErr
|
|
}
|
|
|
|
func (c *podVolumeBackupController) patchPodVolumeBackup(req *arkv1api.PodVolumeBackup, mutate func(*arkv1api.PodVolumeBackup)) (*arkv1api.PodVolumeBackup, error) {
|
|
// Record original json
|
|
oldData, err := json.Marshal(req)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "error marshalling original PodVolumeBackup")
|
|
}
|
|
|
|
// Mutate
|
|
mutate(req)
|
|
|
|
// Record new json
|
|
newData, err := json.Marshal(req)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "error marshalling updated PodVolumeBackup")
|
|
}
|
|
|
|
patchBytes, err := jsonpatch.CreateMergePatch(oldData, newData)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "error creating json merge patch for PodVolumeBackup")
|
|
}
|
|
|
|
req, err = c.podVolumeBackupClient.PodVolumeBackups(req.Namespace).Patch(req.Name, types.MergePatchType, patchBytes)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "error patching PodVolumeBackup")
|
|
}
|
|
|
|
return req, nil
|
|
}
|
|
|
|
func (c *podVolumeBackupController) fail(req *arkv1api.PodVolumeBackup, msg string, log logrus.FieldLogger) error {
|
|
if _, err := c.patchPodVolumeBackup(req, func(r *arkv1api.PodVolumeBackup) {
|
|
r.Status.Phase = arkv1api.PodVolumeBackupPhaseFailed
|
|
r.Status.Message = msg
|
|
}); err != nil {
|
|
log.WithError(err).Error("Error setting phase to Failed")
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func updatePhaseFunc(phase arkv1api.PodVolumeBackupPhase) func(r *arkv1api.PodVolumeBackup) {
|
|
return func(r *arkv1api.PodVolumeBackup) {
|
|
r.Status.Phase = phase
|
|
}
|
|
}
|
|
|
|
func singlePathMatch(path string) (string, error) {
|
|
matches, err := filepath.Glob(path)
|
|
if err != nil {
|
|
return "", errors.WithStack(err)
|
|
}
|
|
|
|
if len(matches) != 1 {
|
|
return "", errors.Errorf("expected one matching path, got %d", len(matches))
|
|
}
|
|
|
|
return matches[0], nil
|
|
}
|