mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-09-21 07:24:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Switch backup finalizer to DeleteBackupRequest

Browse Source 
We ran into a lot of problems using a finalizer on the backup to allow
the Ark server to clean up all associated backup data when deleting a
backup.

Users also found it less than desirable that deleting the heptio-ark
namespace resulted in all the backup data being deleted.

This removes the finalizer and replaces it with an explicit
DeleteBackupRequest that is created as a means of requesting the
deletion of a backup and all its associated data. This is what `ark
backup delete` does.

If you use kubectl to delete a backup or to delete the heptio-ark
namespace, this no longer deletes associated backups. Additionally, as
long as the heptio-ark namespace still exists, the Ark server's
BackupSyncController will continually sync backups into the heptio-ark
namespace from object storage.

Signed-off-by: Andy Goldstein <andy.goldstein@gmail.com>
This commit is contained in:
Andy Goldstein
2018-04-05 11:16:15 -04:00
parent b0a2a0bf7b
commit 74f60b1ee1
28 changed files with 1959 additions and 610 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/controller/gc_controller.go
+66 -166
View File
@@ -17,65 +17,40 @@ limitations under the License.
package controller
import (
"context"
"encoding/json"
"time"
pkgbackup "github.com/heptio/ark/pkg/backup"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/clock"
kerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/tools/cache"
"k8s.io/kubernetes/pkg/util/version"
api "github.com/heptio/ark/pkg/apis/ark/v1"
"github.com/heptio/ark/pkg/cloudprovider"
arkv1client "github.com/heptio/ark/pkg/generated/clientset/versioned/typed/ark/v1"
informers "github.com/heptio/ark/pkg/generated/informers/externalversions/ark/v1"
listers "github.com/heptio/ark/pkg/generated/listers/ark/v1"
"github.com/heptio/ark/pkg/util/kube"
"github.com/heptio/ark/pkg/util/stringslice"
)
// MinVersionForDelete is the minimum Kubernetes server version that Ark
// requires in order to be able to properly delete backups (including
// the associated snapshots and object storage files). This is because
// Ark uses finalizers on the backup CRD to implement garbage-collection
// and deletion.
var MinVersionForDelete = version.MustParseSemantic("1.7.5")
// gcController removes expired backup content from object storage.
// gcController creates DeleteBackupRequests for expired backups.
type gcController struct {
backupService cloudprovider.BackupService
snapshotService cloudprovider.SnapshotService
bucket string
syncPeriod time.Duration
clock clock.Clock
backupLister listers.BackupLister
backupListerSynced cache.InformerSynced
backupClient arkv1client.BackupsGetter
restoreLister listers.RestoreLister
restoreListerSynced cache.InformerSynced
restoreClient arkv1client.RestoresGetter
logger logrus.FieldLogger
*genericController
logger logrus.FieldLogger
backupLister listers.BackupLister
deleteBackupRequestClient arkv1client.DeleteBackupRequestsGetter
syncPeriod time.Duration
clock clock.Clock
}
// NewGCController constructs a new gcController.
func NewGCController(
backupService cloudprovider.BackupService,
snapshotService cloudprovider.SnapshotService,
bucket string,
syncPeriod time.Duration,
backupInformer informers.BackupInformer,
backupClient arkv1client.BackupsGetter,
restoreInformer informers.RestoreInformer,
restoreClient arkv1client.RestoresGetter,
logger logrus.FieldLogger,
backupInformer informers.BackupInformer,
deleteBackupRequestClient arkv1client.DeleteBackupRequestsGetter,
syncPeriod time.Duration,
) Interface {
if syncPeriod < time.Minute {
logger.WithField("syncPeriod", syncPeriod).Info("Provided GC sync period is too short. Setting to 1 minute")
@@ -83,161 +58,86 @@ func NewGCController(
}
c := &gcController{
backupService: backupService,
snapshotService: snapshotService,
bucket: bucket,
syncPeriod: syncPeriod,
clock: clock.RealClock{},
backupLister: backupInformer.Lister(),
backupListerSynced: backupInformer.Informer().HasSynced,
backupClient: backupClient,
restoreLister: restoreInformer.Lister(),
restoreListerSynced: restoreInformer.Informer().HasSynced,
restoreClient: restoreClient,
logger: logger,
genericController: newGenericController("gc-controller", logger),
syncPeriod: syncPeriod,
clock: clock.RealClock{},
backupLister: backupInformer.Lister(),
deleteBackupRequestClient: deleteBackupRequestClient,
logger: logger,
}
c.syncHandler = c.processQueueItem
c.cacheSyncWaiters = append(c.cacheSyncWaiters, backupInformer.Informer().HasSynced)
c.resyncPeriod = syncPeriod
c.resyncFunc = c.enqueueAllBackups
backupInformer.Informer().AddEventHandler(
cache.ResourceEventHandlerFuncs{
AddFunc: c.handleFinalizer,
UpdateFunc: func(_, upd interface{}) {
c.handleFinalizer(upd)
},
AddFunc: c.enqueue,
UpdateFunc: func(_, obj interface{}) { c.enqueue(obj) },
},
)
return c
}
// handleFinalizer runs garbage-collection on a backup that has the Ark GC
// finalizer and a deletionTimestamp.
func (c *gcController) handleFinalizer(obj interface{}) {
var (
backup = obj.(*api.Backup)
log = c.logger.WithField("backup", kube.NamespaceAndName(backup))
)
// enqueueAllBackups lists all backups from cache and enqueues all of them so we can check each one
// for expiration.
func (c *gcController) enqueueAllBackups() {
c.logger.Debug("gcController.enqueueAllBackups")
// we're only interested in backups that have a deletionTimestamp and at
// least one finalizer.
if backup.DeletionTimestamp == nil || len(backup.Finalizers) == 0 {
return
}
log.Debugf("Backup has finalizers %s", backup.Finalizers)
if !stringslice.Has(backup.Finalizers, api.GCFinalizer) {
return
}
log.Infof("Garbage-collecting backup")
if err := c.garbageCollect(backup, log); err != nil {
// if there were errors deleting related cloud resources, don't
// delete the backup API object because we don't want to orphan
// the cloud resources.
log.WithError(err).Error("Error deleting backup's related objects")
return
}
patchMap := map[string]interface{}{
"metadata": map[string]interface{}{
"finalizers": stringslice.Except(backup.Finalizers, api.GCFinalizer),
"resourceVersion": backup.ResourceVersion,
},
}
patchBytes, err := json.Marshal(patchMap)
if err != nil {
log.WithError(err).Error("Error marshaling finalizers patch")
return
}
if _, err = c.backupClient.Backups(backup.Namespace).Patch(backup.Name, types.MergePatchType, patchBytes); err != nil {
log.WithError(errors.WithStack(err)).Error("Error patching backup")
}
}
// Run is a blocking function that runs a single worker to garbage-collect backups
// from object/block storage and the Ark API. It will return when it receives on the
// ctx.Done() channel.
func (c *gcController) Run(ctx context.Context, workers int) error {
c.logger.Info("Waiting for caches to sync")
if !cache.WaitForCacheSync(ctx.Done(), c.backupListerSynced, c.restoreListerSynced) {
return errors.New("timed out waiting for caches to sync")
}
c.logger.Info("Caches are synced")
wait.Until(c.run, c.syncPeriod, ctx.Done())
return nil
}
func (c *gcController) run() {
now := c.clock.Now()
c.logger.Info("Garbage-collecting expired backups")
// Go thru API objects and delete expired ones (finalizer will GC their
// corresponding files/snapshots/restores). Note that we're ignoring backups
// in object storage that haven't been synced to Kubernetes yet; they'll
// be processed for GC (if applicable) once they've been synced.
backups, err := c.backupLister.List(labels.Everything())
if err != nil {
c.logger.WithError(errors.WithStack(err)).Error("Error getting all backups")
c.logger.WithError(errors.WithStack(err)).Error("error listing backups")
return
}
for _, backup := range backups {
log := c.logger.WithField("backup", kube.NamespaceAndName(backup))
if backup.Status.Expiration.Time.After(now) {
log.Debug("Backup has not expired yet, skipping")
continue
}
// since backups have a finalizer, this will actually have the effect of setting a deletionTimestamp and calling
// an update. The update will be handled by this controller and will result in a deletion of the obj storage
// files and the API object.
if err := c.backupClient.Backups(backup.Namespace).Delete(backup.Name, &metav1.DeleteOptions{}); err != nil {
log.WithError(errors.WithStack(err)).Error("Error deleting backup")
}
c.enqueue(backup)
}
}
// garbageCollect prepares for deleting an expired backup by deleting any
// associated backup files, volume snapshots, or restore API objects.
func (c *gcController) garbageCollect(backup *api.Backup, log logrus.FieldLogger) error {
// if the backup includes snapshots but we don't currently have a PVProvider, we don't
// want to orphan the snapshots so skip garbage-collection entirely.
if c.snapshotService == nil && len(backup.Status.VolumeBackups) > 0 {
return errors.New("cannot garbage-collect backup because it includes snapshots and Ark is not configured with a PersistentVolumeProvider")
func (c *gcController) processQueueItem(key string) error {
log := c.logger.WithField("backup", key)
ns, name, err := cache.SplitMetaNamespaceKey(key)
if err != nil {
return errors.Wrap(err, "error splitting queue key")
}
var errs []error
for _, volumeBackup := range backup.Status.VolumeBackups {
log.WithField("snapshotID", volumeBackup.SnapshotID).Info("Removing snapshot associated with backup")
if err := c.snapshotService.DeleteSnapshot(volumeBackup.SnapshotID); err != nil {
errs = append(errs, errors.Wrapf(err, "error deleting snapshot %s", volumeBackup.SnapshotID))
}
backup, err := c.backupLister.Backups(ns).Get(name)
if apierrors.IsNotFound(err) {
log.Debug("Unable to find backup")
return nil
}
if err != nil {
return errors.Wrap(err, "error getting backup")
}
log.Info("Removing backup from object storage")
if err := c.backupService.DeleteBackupDir(c.bucket, backup.Name); err != nil {
errs = append(errs, errors.Wrap(err, "error deleting backup from object storage"))
log = c.logger.WithFields(
logrus.Fields{
"backup": key,
"expiration": backup.Status.Expiration.Time,
},
)
now := c.clock.Now()
expiration := backup.Status.Expiration.Time
if expiration.IsZero() || expiration.After(now) {
log.Debug("Backup has not expired yet, skipping")
return nil
}
if restores, err := c.restoreLister.Restores(backup.Namespace).List(labels.Everything()); err != nil {
log.WithError(errors.WithStack(err)).Error("Error listing restore API objects")
} else {
for _, restore := range restores {
if restore.Spec.BackupName != backup.Name {
continue
}
log.Info("Backup has expired. Creating a DeleteBackupRequest.")
restoreLog := log.WithField("restore", kube.NamespaceAndName(restore))
req := pkgbackup.NewDeleteBackupRequest(name)
restoreLog.Info("Deleting restore referencing backup")
if err := c.restoreClient.Restores(restore.Namespace).Delete(restore.Name, &metav1.DeleteOptions{}); err != nil {
restoreLog.WithError(errors.WithStack(err)).Error("Error deleting restore")
}
}
_, err = c.deleteBackupRequestClient.DeleteBackupRequests(ns).Create(req)
if err != nil {
return errors.Wrap(err, "error creating DeleteBackupRequest")
}
return kerrors.NewAggregate(errs)
return nil
}