mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-09-08 09:06:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Refoctor backup controller with controller-runtime. (#5969)

Browse Source 
Signed-off-by: Ming <mqiu@vmware.com>
Signed-off-by: Xun Jiang <blackpiglet@gmail.com>
Co-authored-by: Ming <mqiu@vmware.com>
This commit is contained in:
Xun Jiang/Bruce Jiang
2023-03-10 08:59:40 +08:00
committed by GitHub
co-authored by Ming
parent 6d8f086283
commit 7d7e3fff0d
22 changed files with 616 additions and 818 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/controller/backup_controller.go
+154 -196
View File
@@ -27,7 +27,6 @@ import (
"sync"
"time"
jsonpatch "github.com/evanphx/json-patch"
snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned"
snapshotv1listers "github.com/kubernetes-csi/external-snapshotter/client/v4/listers/volumesnapshot/v1"
@@ -39,12 +38,11 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
kerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/tools/cache"
clocks "k8s.io/utils/clock"
"k8s.io/utils/clock"
ctrl "sigs.k8s.io/controller-runtime"
kbclient "sigs.k8s.io/controller-runtime/pkg/client"
"github.com/vmware-tanzu/velero/internal/credentials"
@@ -53,9 +51,6 @@ import (
pkgbackup "github.com/vmware-tanzu/velero/pkg/backup"
"github.com/vmware-tanzu/velero/pkg/discovery"
"github.com/vmware-tanzu/velero/pkg/features"
velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1"
velerov1informers "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions/velero/v1"
velerov1listers "github.com/vmware-tanzu/velero/pkg/generated/listers/velero/v1"
"github.com/vmware-tanzu/velero/pkg/label"
"github.com/vmware-tanzu/velero/pkg/metrics"
"github.com/vmware-tanzu/velero/pkg/persistence"
@@ -71,14 +66,17 @@ import (
"github.com/vmware-tanzu/velero/pkg/volume"
)
type backupController struct {
*genericController
const (
backupResyncPeriod = time.Minute
)
type backupReconciler struct {
ctx context.Context
logger logrus.FieldLogger
discoveryHelper discovery.Helper
backupper pkgbackup.Backupper
lister velerov1listers.BackupLister
client velerov1client.BackupsGetter
kbClient kbclient.Client
clock clocks.WithTickerAndDelayedExecution
clock clock.WithTickerAndDelayedExecution
backupLogLevel logrus.Level
newPluginManager func(logrus.FieldLogger) clientmgmt.Manager
backupTracker BackupTracker
@@ -87,7 +85,6 @@ type backupController struct {
defaultBackupTTL time.Duration
defaultCSISnapshotTimeout time.Duration
defaultItemOperationTimeout time.Duration
snapshotLocationLister velerov1listers.VolumeSnapshotLocationLister
defaultSnapshotLocations map[string]string
metrics *metrics.ServerMetrics
backupStoreGetter persistence.ObjectBackupStoreGetter
@@ -97,9 +94,8 @@ type backupController struct {
credentialFileStore credentials.FileStore
}
func NewBackupController(
backupInformer velerov1informers.BackupInformer,
client velerov1client.BackupsGetter,
func NewBackupReconciler(
ctx context.Context,
discoveryHelper discovery.Helper,
backupper pkgbackup.Backupper,
logger logrus.FieldLogger,
@@ -112,7 +108,6 @@ func NewBackupController(
defaultBackupTTL time.Duration,
defaultCSISnapshotTimeout time.Duration,
defaultItemOperationTimeout time.Duration,
volumeSnapshotLocationLister velerov1listers.VolumeSnapshotLocationLister,
defaultSnapshotLocations map[string]string,
metrics *metrics.ServerMetrics,
backupStoreGetter persistence.ObjectBackupStoreGetter,
@@ -120,14 +115,14 @@ func NewBackupController(
volumeSnapshotLister snapshotv1listers.VolumeSnapshotLister,
volumeSnapshotClient snapshotterClientSet.Interface,
credentialStore credentials.FileStore,
) Interface {
c := &backupController{
genericController: newGenericController(Backup, logger),
) *backupReconciler {
b := &backupReconciler{
ctx: ctx,
discoveryHelper: discoveryHelper,
backupper: backupper,
lister: backupInformer.Lister(),
client: client,
clock: &clocks.RealClock{},
clock: &clock.RealClock{},
logger: logger,
backupLogLevel: backupLogLevel,
newPluginManager: newPluginManager,
backupTracker: backupTracker,
@@ -137,7 +132,6 @@ func NewBackupController(
defaultBackupTTL: defaultBackupTTL,
defaultCSISnapshotTimeout: defaultCSISnapshotTimeout,
defaultItemOperationTimeout: defaultItemOperationTimeout,
snapshotLocationLister: volumeSnapshotLocationLister,
defaultSnapshotLocations: defaultSnapshotLocations,
metrics: metrics,
backupStoreGetter: backupStoreGetter,
@@ -146,61 +140,49 @@ func NewBackupController(
volumeSnapshotClient: volumeSnapshotClient,
credentialFileStore: credentialStore,
}
c.syncHandler = c.processBackup
c.resyncFunc = c.resync
c.resyncPeriod = time.Minute
backupInformer.Informer().AddEventHandler(
cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
backup := obj.(*velerov1api.Backup)
switch backup.Status.Phase {
case "", velerov1api.BackupPhaseNew:
// only process new backups
default:
c.logger.WithFields(logrus.Fields{
"backup": kubeutil.NamespaceAndName(backup),
"phase": backup.Status.Phase,
}).Debug("Backup is not new, skipping")
return
}
key, err := cache.MetaNamespaceKeyFunc(backup)
if err != nil {
c.logger.WithError(err).WithField(Backup, backup).Error("Error creating queue key, item not added to queue")
return
}
c.queue.Add(key)
},
},
)
return c
b.updateTotalBackupMetric()
return b
}
func (c *backupController) resync() {
// recompute backup_total metric
backups, err := c.lister.List(labels.Everything())
if err != nil {
c.logger.Error(err, "Error computing backup_total metric")
} else {
c.metrics.SetBackupTotal(int64(len(backups)))
}
func (b *backupReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&velerov1api.Backup{}).
Complete(b)
}
// recompute backup_last_successful_timestamp metric for each
// schedule (including the empty schedule, i.e. ad-hoc backups)
for schedule, timestamp := range getLastSuccessBySchedule(backups) {
c.metrics.SetBackupLastSuccessfulTimestamp(schedule, timestamp)
}
func (b *backupReconciler) updateTotalBackupMetric() {
go func() {
// Wait for 5 seconds to let controller-runtime to setup k8s clients.
time.Sleep(5 * time.Second)
wait.Until(
func() {
// recompute backup_total metric
backups := &velerov1api.BackupList{}
err := b.kbClient.List(context.Background(), backups, &kbclient.ListOptions{LabelSelector: labels.Everything()})
if err != nil {
b.logger.Error(err, "Error computing backup_total metric")
} else {
b.metrics.SetBackupTotal(int64(len(backups.Items)))
}
// recompute backup_last_successful_timestamp metric for each
// schedule (including the empty schedule, i.e. ad-hoc backups)
for schedule, timestamp := range getLastSuccessBySchedule(backups.Items) {
b.metrics.SetBackupLastSuccessfulTimestamp(schedule, timestamp)
}
},
backupResyncPeriod,
b.ctx.Done(),
)
}()
}
// getLastSuccessBySchedule finds the most recent completed backup for each schedule
// and returns a map of schedule name -> completion time of the most recent completed
// backup. This map includes an entry for ad-hoc/non-scheduled backups, where the key
// is the empty string.
func getLastSuccessBySchedule(backups []*velerov1api.Backup) map[string]time.Time {
func getLastSuccessBySchedule(backups []velerov1api.Backup) map[string]time.Time {
lastSuccessBySchedule := map[string]time.Time{}
for _, backup := range backups {
if backup.Status.Phase != velerov1api.BackupPhaseCompleted {
@@ -221,24 +203,23 @@ func getLastSuccessBySchedule(backups []*velerov1api.Backup) map[string]time.Tim
return lastSuccessBySchedule
}
func (c *backupController) processBackup(key string) error {
log := c.logger.WithField("key", key)
log.Debug("Running processBackup")
ns, name, err := cache.SplitMetaNamespaceKey(key)
if err != nil {
log.WithError(err).Errorf("error splitting key")
return nil
}
func (b *backupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
log := b.logger.WithFields(logrus.Fields{
"controller": Backup,
"backuprequest": req.String(),
})
log.Debug("Getting backup")
original, err := c.lister.Backups(ns).Get(name)
if apierrors.IsNotFound(err) {
log.Debugf("backup %s not found", name)
return nil
}
original := &velerov1api.Backup{}
err := b.kbClient.Get(ctx, req.NamespacedName, original)
if err != nil {
return errors.Wrap(err, "error getting backup")
if apierrors.IsNotFound(err) {
log.Debug("backup not found")
return ctrl.Result{}, nil
}
log.WithError(err).Error("error getting backup")
return ctrl.Result{}, err
}
// Double-check we have the correct phase. In the unlikely event that multiple controller
@@ -253,42 +234,44 @@ func (c *backupController) processBackup(key string) error {
case "", velerov1api.BackupPhaseNew:
// only process new backups
default:
return nil
b.logger.WithFields(logrus.Fields{
"backup": kubeutil.NamespaceAndName(original),
"phase": original.Status.Phase,
}).Debug("Backup is not handled")
return ctrl.Result{}, nil
}
log.Debug("Preparing backup request")
request := c.prepareBackupRequest(original, log)
request := b.prepareBackupRequest(original, log)
if len(request.Status.ValidationErrors) > 0 {
request.Status.Phase = velerov1api.BackupPhaseFailedValidation
} else {
request.Status.Phase = velerov1api.BackupPhaseInProgress
request.Status.StartTimestamp = &metav1.Time{Time: c.clock.Now()}
request.Status.StartTimestamp = &metav1.Time{Time: b.clock.Now()}
}
// update status
updatedBackup, err := patchBackup(original, request.Backup, c.client)
if err != nil {
return errors.Wrapf(err, "error updating Backup status to %s", request.Status.Phase)
if err := kubeutil.PatchResource(original, request.Backup, b.kbClient); err != nil {
return ctrl.Result{}, errors.Wrapf(err, "error updating Backup status to %s", request.Status.Phase)
}
// store ref to just-updated item for creating patch
original = updatedBackup
request.Backup = updatedBackup.DeepCopy()
original = request.Backup.DeepCopy()
if request.Status.Phase == velerov1api.BackupPhaseFailedValidation {
return nil
log.Debug("failed to validate backup status")
return ctrl.Result{}, nil
}
c.backupTracker.Add(request.Namespace, request.Name)
defer c.backupTracker.Delete(request.Namespace, request.Name)
b.backupTracker.Add(request.Namespace, request.Name)
defer b.backupTracker.Delete(request.Namespace, request.Name)
log.Debug("Running backup")
backupScheduleName := request.GetLabels()[velerov1api.ScheduleNameLabel]
c.metrics.RegisterBackupAttempt(backupScheduleName)
b.metrics.RegisterBackupAttempt(backupScheduleName)
// execution & upload of backup
if err := c.runBackup(request); err != nil {
if err := b.runBackup(request); err != nil {
// even though runBackup sets the backup's phase prior
// to uploading artifacts to object storage, we have to
// check for an error again here and update the phase if
@@ -302,52 +285,26 @@ func (c *backupController) processBackup(key string) error {
switch request.Status.Phase {
case velerov1api.BackupPhaseCompleted:
c.metrics.RegisterBackupSuccess(backupScheduleName)
c.metrics.RegisterBackupLastStatus(backupScheduleName, metrics.BackupLastStatusSucc)
b.metrics.RegisterBackupSuccess(backupScheduleName)
b.metrics.RegisterBackupLastStatus(backupScheduleName, metrics.BackupLastStatusSucc)
case velerov1api.BackupPhasePartiallyFailed:
c.metrics.RegisterBackupPartialFailure(backupScheduleName)
c.metrics.RegisterBackupLastStatus(backupScheduleName, metrics.BackupLastStatusFailure)
b.metrics.RegisterBackupPartialFailure(backupScheduleName)
b.metrics.RegisterBackupLastStatus(backupScheduleName, metrics.BackupLastStatusFailure)
case velerov1api.BackupPhaseFailed:
c.metrics.RegisterBackupFailed(backupScheduleName)
c.metrics.RegisterBackupLastStatus(backupScheduleName, metrics.BackupLastStatusFailure)
b.metrics.RegisterBackupFailed(backupScheduleName)
b.metrics.RegisterBackupLastStatus(backupScheduleName, metrics.BackupLastStatusFailure)
case velerov1api.BackupPhaseFailedValidation:
c.metrics.RegisterBackupValidationFailure(backupScheduleName)
c.metrics.RegisterBackupLastStatus(backupScheduleName, metrics.BackupLastStatusFailure)
b.metrics.RegisterBackupValidationFailure(backupScheduleName)
b.metrics.RegisterBackupLastStatus(backupScheduleName, metrics.BackupLastStatusFailure)
}
log.Debug("Updating backup's final status")
if _, err := patchBackup(original, request.Backup, c.client); err != nil {
log.Info("Updating backup's final status")
if err := kubeutil.PatchResource(original, request.Backup, b.kbClient); err != nil {
log.WithError(err).Error("error updating backup's final status")
}
return nil
return ctrl.Result{}, nil
}
func patchBackup(original, updated *velerov1api.Backup, client velerov1client.BackupsGetter) (*velerov1api.Backup, error) {
origBytes, err := json.Marshal(original)
if err != nil {
return nil, errors.Wrap(err, "error marshalling original backup")
}
updatedBytes, err := json.Marshal(updated)
if err != nil {
return nil, errors.Wrap(err, "error marshalling updated backup")
}
patchBytes, err := jsonpatch.CreateMergePatch(origBytes, updatedBytes)
if err != nil {
return nil, errors.Wrap(err, "error creating json merge patch for backup")
}
res, err := client.Backups(original.Namespace).Patch(context.TODO(), original.Name, types.MergePatchType, patchBytes, metav1.PatchOptions{})
if err != nil {
return nil, errors.Wrap(err, "error patching backup")
}
return res, nil
}
func (c *backupController) prepareBackupRequest(backup *velerov1api.Backup, logger logrus.FieldLogger) *pkgbackup.Request {
func (b *backupReconciler) prepareBackupRequest(backup *velerov1api.Backup, logger logrus.FieldLogger) *pkgbackup.Request {
request := &pkgbackup.Request{
Backup: backup.DeepCopy(), // don't modify items in the cache
}
@@ -360,21 +317,21 @@ func (c *backupController) prepareBackupRequest(backup *velerov1api.Backup, logg
if request.Spec.TTL.Duration == 0 {
// set default backup TTL
request.Spec.TTL.Duration = c.defaultBackupTTL
request.Spec.TTL.Duration = b.defaultBackupTTL
}
if request.Spec.CSISnapshotTimeout.Duration == 0 {
// set default CSI VolumeSnapshot timeout
request.Spec.CSISnapshotTimeout.Duration = c.defaultCSISnapshotTimeout
request.Spec.CSISnapshotTimeout.Duration = b.defaultCSISnapshotTimeout
}
if request.Spec.ItemOperationTimeout.Duration == 0 {
// set default item operation timeout
request.Spec.ItemOperationTimeout.Duration = c.defaultItemOperationTimeout
request.Spec.ItemOperationTimeout.Duration = b.defaultItemOperationTimeout
}
// calculate expiration
request.Status.Expiration = &metav1.Time{Time: c.clock.Now().Add(request.Spec.TTL.Duration)}
request.Status.Expiration = &metav1.Time{Time: b.clock.Now().Add(request.Spec.TTL.Duration)}
// TODO: post v1.10. Remove this code block after DefaultVolumesToRestic is removed from CRD
// For now, for CRs created by old versions, we need to respect the DefaultVolumesToRestic value if it is set true
@@ -384,17 +341,17 @@ func (c *backupController) prepareBackupRequest(backup *velerov1api.Backup, logg
}
if request.Spec.DefaultVolumesToFsBackup == nil {
request.Spec.DefaultVolumesToFsBackup = &c.defaultVolumesToFsBackup
request.Spec.DefaultVolumesToFsBackup = &b.defaultVolumesToFsBackup
}
// find which storage location to use
var serverSpecified bool
if request.Spec.StorageLocation == "" {
// when the user doesn't specify a location, use the server default unless there is an existing BSL marked as default
// TODO(2.0) c.defaultBackupLocation will be deprecated
request.Spec.StorageLocation = c.defaultBackupLocation
// TODO(2.0) b.defaultBackupLocation will be deprecated
request.Spec.StorageLocation = b.defaultBackupLocation
locationList, err := storage.ListBackupStorageLocations(context.Background(), c.kbClient, request.Namespace)
locationList, err := storage.ListBackupStorageLocations(context.Background(), b.kbClient, request.Namespace)
if err == nil {
for _, location := range locationList.Items {
if location.Spec.Default {
@@ -408,7 +365,7 @@ func (c *backupController) prepareBackupRequest(backup *velerov1api.Backup, logg
// get the storage location, and store the BackupStorageLocation API obj on the request
storageLocation := &velerov1api.BackupStorageLocation{}
if err := c.kbClient.Get(context.Background(), kbclient.ObjectKey{
if err := b.kbClient.Get(context.Background(), kbclient.ObjectKey{
Namespace: request.Namespace,
Name: request.Spec.StorageLocation,
}, storageLocation); err != nil {
@@ -439,7 +396,7 @@ func (c *backupController) prepareBackupRequest(backup *velerov1api.Backup, logg
// validate and get the backup's VolumeSnapshotLocations, and store the
// VolumeSnapshotLocation API objs on the request
if locs, errs := c.validateAndGetSnapshotLocations(request.Backup); len(errs) > 0 {
if locs, errs := b.validateAndGetSnapshotLocations(request.Backup); len(errs) > 0 {
request.Status.ValidationErrors = append(request.Status.ValidationErrors, errs...)
} else {
request.Spec.VolumeSnapshotLocations = nil
@@ -453,14 +410,14 @@ func (c *backupController) prepareBackupRequest(backup *velerov1api.Backup, logg
if request.Annotations == nil {
request.Annotations = make(map[string]string)
}
request.Annotations[velerov1api.SourceClusterK8sGitVersionAnnotation] = c.discoveryHelper.ServerVersion().String()
request.Annotations[velerov1api.SourceClusterK8sMajorVersionAnnotation] = c.discoveryHelper.ServerVersion().Major
request.Annotations[velerov1api.SourceClusterK8sMinorVersionAnnotation] = c.discoveryHelper.ServerVersion().Minor
request.Annotations[velerov1api.SourceClusterK8sGitVersionAnnotation] = b.discoveryHelper.ServerVersion().String()
request.Annotations[velerov1api.SourceClusterK8sMajorVersionAnnotation] = b.discoveryHelper.ServerVersion().Major
request.Annotations[velerov1api.SourceClusterK8sMinorVersionAnnotation] = b.discoveryHelper.ServerVersion().Minor
// Add namespaces with label velero.io/exclude-from-backup=true into request.Spec.ExcludedNamespaces
// Essentially, adding the label velero.io/exclude-from-backup=true to a namespace would be equivalent to setting spec.ExcludedNamespaces
namespaces := corev1api.NamespaceList{}
if err := c.kbClient.List(context.Background(), &namespaces, kbclient.MatchingLabels{"velero.io/exclude-from-backup": "true"}); err == nil {
if err := b.kbClient.List(context.Background(), &namespaces, kbclient.MatchingLabels{"velero.io/exclude-from-backup": "true"}); err == nil {
for _, ns := range namespaces.Items {
request.Spec.ExcludedNamespaces = append(request.Spec.ExcludedNamespaces, ns.Name)
}
@@ -495,7 +452,7 @@ func (c *backupController) prepareBackupRequest(backup *velerov1api.Backup, logg
// it will automatically be used)
//
// if backup has snapshotVolume disabled then it returns empty VSL
func (c *backupController) validateAndGetSnapshotLocations(backup *velerov1api.Backup) (map[string]*velerov1api.VolumeSnapshotLocation, []string) {
func (b *backupReconciler) validateAndGetSnapshotLocations(backup *velerov1api.Backup) (map[string]*velerov1api.VolumeSnapshotLocation, []string) {
errors := []string{}
providerLocations := make(map[string]*velerov1api.VolumeSnapshotLocation)
@@ -506,8 +463,8 @@ func (c *backupController) validateAndGetSnapshotLocations(backup *velerov1api.B
for _, locationName := range backup.Spec.VolumeSnapshotLocations {
// validate each locationName exists as a VolumeSnapshotLocation
location, err := c.snapshotLocationLister.VolumeSnapshotLocations(backup.Namespace).Get(locationName)
if err != nil {
location := &velerov1api.VolumeSnapshotLocation{}
if err := b.kbClient.Get(context.Background(), kbclient.ObjectKey{Namespace: backup.Namespace, Name: locationName}, location); err != nil {
if apierrors.IsNotFound(err) {
errors = append(errors, fmt.Sprintf("a VolumeSnapshotLocation CRD for the location %s with the name specified in the backup spec needs to be created before this snapshot can be executed. Error: %v", locationName, err))
} else {
@@ -532,8 +489,8 @@ func (c *backupController) validateAndGetSnapshotLocations(backup *velerov1api.B
if len(errors) > 0 {
return nil, errors
}
allLocations, err := c.snapshotLocationLister.VolumeSnapshotLocations(backup.Namespace).List(labels.Everything())
allLocations := &velerov1api.VolumeSnapshotLocationList{}
err := b.kbClient.List(context.Background(), allLocations, &kbclient.ListOptions{Namespace: backup.Namespace, LabelSelector: labels.Everything()})
if err != nil {
errors = append(errors, fmt.Sprintf("error listing volume snapshot locations: %v", err))
return nil, errors
@@ -541,9 +498,9 @@ func (c *backupController) validateAndGetSnapshotLocations(backup *velerov1api.B
// build a map of provider->list of all locations for the provider
allProviderLocations := make(map[string][]*velerov1api.VolumeSnapshotLocation)
for i := range allLocations {
loc := allLocations[i]
allProviderLocations[loc.Spec.Provider] = append(allProviderLocations[loc.Spec.Provider], loc)
for i := range allLocations.Items {
loc := allLocations.Items[i]
allProviderLocations[loc.Spec.Provider] = append(allProviderLocations[loc.Spec.Provider], &loc)
}
// go through each provider and make sure we have/can get a VSL
@@ -557,12 +514,13 @@ func (c *backupController) validateAndGetSnapshotLocations(backup *velerov1api.B
if len(locations) > 1 {
// more than one possible location for the provider: check
// the defaults
defaultLocation := c.defaultSnapshotLocations[provider]
defaultLocation := b.defaultSnapshotLocations[provider]
if defaultLocation == "" {
errors = append(errors, fmt.Sprintf("provider %s has more than one possible volume snapshot location, and none were specified explicitly or as a default", provider))
continue
}
location, err := c.snapshotLocationLister.VolumeSnapshotLocations(backup.Namespace).Get(defaultLocation)
location := &velerov1api.VolumeSnapshotLocation{}
b.kbClient.Get(context.Background(), kbclient.ObjectKey{Namespace: backup.Namespace, Name: defaultLocation}, location)
if err != nil {
errors = append(errors, fmt.Sprintf("error getting volume snapshot location named %s: %v", defaultLocation, err))
continue
@@ -582,7 +540,7 @@ func (c *backupController) validateAndGetSnapshotLocations(backup *velerov1api.B
// add credential to config for each location
for _, location := range providerLocations {
err = volume.UpdateVolumeSnapshotLocationWithCredentialConfig(location, c.credentialFileStore, c.logger)
err = volume.UpdateVolumeSnapshotLocationWithCredentialConfig(location, b.credentialFileStore, b.logger)
if err != nil {
errors = append(errors, fmt.Sprintf("error adding credentials to volume snapshot location named %s: %v", location.Name, err))
continue
@@ -595,17 +553,17 @@ func (c *backupController) validateAndGetSnapshotLocations(backup *velerov1api.B
// runBackup runs and uploads a validated backup. Any error returned from this function
// causes the backup to be Failed; if no error is returned, the backup's status's Errors
// field is checked to see if the backup was a partial failure.
func (c *backupController) runBackup(backup *pkgbackup.Request) error {
c.logger.WithField(Backup, kubeutil.NamespaceAndName(backup)).Info("Setting up backup log")
func (b *backupReconciler) runBackup(backup *pkgbackup.Request) error {
b.logger.WithField(Backup, kubeutil.NamespaceAndName(backup)).Info("Setting up backup log")
// Log the backup to both a backup log file and to stdout. This will help see what happened if the upload of the
// backup log failed for whatever reason.
logCounter := logging.NewLogHook()
backupLog, err := logging.NewTempFileLogger(c.backupLogLevel, c.formatFlag, logCounter, logrus.Fields{Backup: kubeutil.NamespaceAndName(backup)})
backupLog, err := logging.NewTempFileLogger(b.backupLogLevel, b.formatFlag, logCounter, logrus.Fields{Backup: kubeutil.NamespaceAndName(backup)})
if err != nil {
return errors.Wrap(err, "error creating dual mode logger for backup")
}
defer backupLog.Dispose(c.logger.WithField(Backup, kubeutil.NamespaceAndName(backup)))
defer backupLog.Dispose(b.logger.WithField(Backup, kubeutil.NamespaceAndName(backup)))
backupLog.Info("Setting up backup temp file")
backupFile, err := ioutil.TempFile("", "")
@@ -615,7 +573,7 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error {
defer closeAndRemoveFile(backupFile, backupLog)
backupLog.Info("Setting up plugin manager")
pluginManager := c.newPluginManager(backupLog)
pluginManager := b.newPluginManager(backupLog)
defer pluginManager.CleanupClients()
backupLog.Info("Getting backup item actions")
@@ -629,7 +587,7 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error {
}
backupLog.Info("Setting up backup store to check for backup existence")
backupStore, err := c.backupStoreGetter.Get(backup.StorageLocation, pluginManager, backupLog)
backupStore, err := b.backupStoreGetter.Get(backup.StorageLocation, pluginManager, backupLog)
if err != nil {
return err
}
@@ -637,7 +595,7 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error {
exists, err := backupStore.BackupExists(backup.StorageLocation.Spec.StorageType.ObjectStorage.Bucket, backup.Name)
if exists || err != nil {
backup.Status.Phase = velerov1api.BackupPhaseFailed
backup.Status.CompletionTimestamp = &metav1.Time{Time: c.clock.Now()}
backup.Status.CompletionTimestamp = &metav1.Time{Time: b.clock.Now()}
if err != nil {
return errors.Wrapf(err, "error checking if backup already exists in object storage")
}
@@ -648,7 +606,7 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error {
itemSnapshottersResolver := framework.NewItemSnapshotterResolver(itemSnapshotters)
var fatalErrs []error
if err := c.backupper.BackupWithResolvers(backupLog, backup, backupFile, backupItemActionsResolver,
if err := b.backupper.BackupWithResolvers(backupLog, backup, backupFile, backupItemActionsResolver,
itemSnapshottersResolver, pluginManager); err != nil {
fatalErrs = append(fatalErrs, err)
}
@@ -662,14 +620,14 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error {
selector := label.NewSelectorForBackup(backup.Name)
vscList := &snapshotv1api.VolumeSnapshotContentList{}
volumeSnapshots, err = c.waitVolumeSnapshotReadyToUse(context.Background(), backup.Spec.CSISnapshotTimeout.Duration, backup.Name)
volumeSnapshots, err = b.waitVolumeSnapshotReadyToUse(context.Background(), backup.Spec.CSISnapshotTimeout.Duration, backup.Name)
if err != nil {
backupLog.Errorf("fail to wait VolumeSnapshot change to Ready: %s", err.Error())
}
backup.CSISnapshots = volumeSnapshots
err = c.kbClient.List(context.Background(), vscList, &kbclient.ListOptions{LabelSelector: selector})
err = b.kbClient.List(context.Background(), vscList, &kbclient.ListOptions{LabelSelector: selector})
if err != nil {
backupLog.Error(err)
}
@@ -682,7 +640,7 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error {
// persist the volumesnapshotclasses referenced by vsc
if volumeSnapshotContents[index].Spec.VolumeSnapshotClassName != nil && !vsClassSet.Has(*volumeSnapshotContents[index].Spec.VolumeSnapshotClassName) {
vsClass := &snapshotv1api.VolumeSnapshotClass{}
if err := c.kbClient.Get(context.TODO(), kbclient.ObjectKey{Name: *volumeSnapshotContents[index].Spec.VolumeSnapshotClassName}, vsClass); err != nil {
if err := b.kbClient.Get(context.TODO(), kbclient.ObjectKey{Name: *volumeSnapshotContents[index].Spec.VolumeSnapshotClassName}, vsClass); err != nil {
backupLog.Error(err)
} else {
vsClassSet.Insert(*volumeSnapshotContents[index].Spec.VolumeSnapshotClassName)
@@ -697,7 +655,7 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error {
// Delete the VolumeSnapshots created in the backup, when CSI feature is enabled.
if len(volumeSnapshots) > 0 && len(volumeSnapshotContents) > 0 {
c.deleteVolumeSnapshot(volumeSnapshots, volumeSnapshotContents, backupLog)
b.deleteVolumeSnapshot(volumeSnapshots, volumeSnapshotContents, backupLog)
}
}
@@ -740,7 +698,7 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error {
"errors": backupErrors,
}
backupLog.DoneForPersist(c.logger.WithField(Backup, kubeutil.NamespaceAndName(backup)))
backupLog.DoneForPersist(b.logger.WithField(Backup, kubeutil.NamespaceAndName(backup)))
// Assign finalize phase as close to end as possible so that any errors
// logged to backupLog are captured. This is done before uploading the
@@ -767,14 +725,14 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error {
if backup.Status.Phase == velerov1api.BackupPhaseFailed ||
backup.Status.Phase == velerov1api.BackupPhasePartiallyFailed ||
backup.Status.Phase == velerov1api.BackupPhaseCompleted {
backup.Status.CompletionTimestamp = &metav1.Time{Time: c.clock.Now()}
backup.Status.CompletionTimestamp = &metav1.Time{Time: b.clock.Now()}
}
recordBackupMetrics(backupLog, backup.Backup, backupFile, c.metrics, false)
recordBackupMetrics(backupLog, backup.Backup, backupFile, b.metrics, false)
// re-instantiate the backup store because credentials could have changed since the original
// instantiation, if this was a long-running backup
backupLog.Info("Setting up backup store to persist the backup")
backupStore, err = c.backupStoreGetter.Get(backup.StorageLocation, pluginManager, backupLog)
backupStore, err = b.backupStoreGetter.Get(backup.StorageLocation, pluginManager, backupLog)
if err != nil {
return err
}
@@ -787,7 +745,7 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error {
}
}
c.logger.WithField(Backup, kubeutil.NamespaceAndName(backup)).Info("Backup completed")
b.logger.WithField(Backup, kubeutil.NamespaceAndName(backup)).Info("Backup completed")
// if we return a non-nil error, the calling function will update
// the backup's phase to Failed.
@@ -965,20 +923,19 @@ func encodeToJSONGzip(data interface{}, desc string) (*bytes.Buffer, []error) {
// using goroutine here instead of waiting in CSI plugin, because it's not easy to make BackupItemAction
// parallel by now. After BackupItemAction parallel is implemented, this logic should be moved to CSI plugin
// as https://github.com/vmware-tanzu/velero-plugin-for-csi/pull/100
func (c *backupController) waitVolumeSnapshotReadyToUse(ctx context.Context,
func (b *backupReconciler) waitVolumeSnapshotReadyToUse(ctx context.Context,
csiSnapshotTimeout time.Duration, backupName string) ([]snapshotv1api.VolumeSnapshot, error) {
eg, _ := errgroup.WithContext(ctx)
timeout := csiSnapshotTimeout
interval := 5 * time.Second
volumeSnapshots := make([]snapshotv1api.VolumeSnapshot, 0)
if c.volumeSnapshotLister != nil {
tmpVSs, err := c.volumeSnapshotLister.List(label.NewSelectorForBackup(backupName))
if b.volumeSnapshotLister != nil {
tmpVSs, err := b.volumeSnapshotLister.List(label.NewSelectorForBackup(backupName))
if err != nil {
c.logger.Error(err)
b.logger.Error(err)
return volumeSnapshots, err
}
for _, vs := range tmpVSs {
volumeSnapshots = append(volumeSnapshots, *vs)
}
@@ -991,22 +948,22 @@ func (c *backupController) waitVolumeSnapshotReadyToUse(ctx context.Context,
volumeSnapshot := volumeSnapshots[index]
eg.Go(func() error {
err := wait.PollImmediate(interval, timeout, func() (bool, error) {
tmpVS, err := c.volumeSnapshotClient.SnapshotV1().VolumeSnapshots(volumeSnapshot.Namespace).Get(ctx, volumeSnapshot.Name, metav1.GetOptions{})
tmpVS, err := b.volumeSnapshotClient.SnapshotV1().VolumeSnapshots(volumeSnapshot.Namespace).Get(b.ctx, volumeSnapshot.Name, metav1.GetOptions{})
if err != nil {
return false, errors.Wrapf(err, fmt.Sprintf("failed to get volumesnapshot %s/%s", volumeSnapshot.Namespace, volumeSnapshot.Name))
}
if tmpVS.Status == nil || tmpVS.Status.BoundVolumeSnapshotContentName == nil || !boolptr.IsSetToTrue(tmpVS.Status.ReadyToUse) {
c.logger.Infof("Waiting for CSI driver to reconcile volumesnapshot %s/%s. Retrying in %ds", volumeSnapshot.Namespace, volumeSnapshot.Name, interval/time.Second)
b.logger.Infof("Waiting for CSI driver to reconcile volumesnapshot %s/%s. Retrying in %ds", volumeSnapshot.Namespace, volumeSnapshot.Name, interval/time.Second)
return false, nil
}
c.logger.Debugf("VolumeSnapshot %s/%s turned into ReadyToUse.", volumeSnapshot.Namespace, volumeSnapshot.Name)
b.logger.Debugf("VolumeSnapshot %s/%s turned into ReadyToUse.", volumeSnapshot.Namespace, volumeSnapshot.Name)
// Put the ReadyToUse VolumeSnapshot element in the result channel.
vsChannel <- *tmpVS
return true, nil
})
if err == wait.ErrWaitTimeout {
c.logger.Errorf("Timed out awaiting reconciliation of volumesnapshot %s/%s", volumeSnapshot.Namespace, volumeSnapshot.Name)
b.logger.Errorf("Timed out awaiting reconciliation of volumesnapshot %s/%s", volumeSnapshot.Namespace, volumeSnapshot.Name)
}
return err
})
@@ -1028,7 +985,7 @@ func (c *backupController) waitVolumeSnapshotReadyToUse(ctx context.Context,
// which will cause snapshot deletion on cloud provider, then backup cannot restore the PV.
// If DeletionPolicy is Retain, just delete it. If DeletionPolicy is Delete, need to
// change DeletionPolicy to Retain before deleting VS, then change DeletionPolicy back to Delete.
func (c *backupController) deleteVolumeSnapshot(volumeSnapshots []snapshotv1api.VolumeSnapshot,
func (b *backupReconciler) deleteVolumeSnapshot(volumeSnapshots []snapshotv1api.VolumeSnapshot,
volumeSnapshotContents []snapshotv1api.VolumeSnapshotContent,
logger logrus.FieldLogger) {
var wg sync.WaitGroup
@@ -1067,14 +1024,15 @@ func (c *backupController) deleteVolumeSnapshot(volumeSnapshots []snapshotv1api.
logger.Debugf("Patching VolumeSnapshotContent %s", vsc.Name)
original := vsc.DeepCopy()
vsc.Spec.DeletionPolicy = snapshotv1api.VolumeSnapshotContentRetain
if err := c.kbClient.Patch(context.Background(), &vsc, kbclient.MergeFrom(original)); err != nil {
err := kubeutil.PatchResource(original, &vsc, b.kbClient)
if err != nil {
logger.Errorf("fail to modify VolumeSnapshotContent %s DeletionPolicy to Retain: %s", vsc.Name, err.Error())
return
}
defer func() {
logger.Debugf("Start to recreate VolumeSnapshotContent %s", vsc.Name)
err := c.recreateVolumeSnapshotContent(vsc)
err := b.recreateVolumeSnapshotContent(vsc)
if err != nil {
logger.Errorf("fail to recreate VolumeSnapshotContent %s: %s", vsc.Name, err.Error())
}
@@ -1083,7 +1041,7 @@ func (c *backupController) deleteVolumeSnapshot(volumeSnapshots []snapshotv1api.
// Delete VolumeSnapshot from cluster
logger.Debugf("Deleting VolumeSnapshot %s/%s", vs.Namespace, vs.Name)
err := c.volumeSnapshotClient.SnapshotV1().VolumeSnapshots(vs.Namespace).Delete(context.TODO(), vs.Name, metav1.DeleteOptions{})
err := b.volumeSnapshotClient.SnapshotV1().VolumeSnapshots(vs.Namespace).Delete(b.ctx, vs.Name, metav1.DeleteOptions{})
if err != nil {
logger.Errorf("fail to delete VolumeSnapshot %s/%s: %s", vs.Namespace, vs.Name, err.Error())
}
@@ -1098,11 +1056,11 @@ func (c *backupController) deleteVolumeSnapshot(volumeSnapshots []snapshotv1api.
// and Source. Source is updated to let csi-controller thinks the VSC is statically provsisioned with VS.
// Set VolumeSnapshotRef's UID to nil will let the csi-controller finds out the related VS is gone, then
// VSC can be deleted.
func (c *backupController) recreateVolumeSnapshotContent(vsc snapshotv1api.VolumeSnapshotContent) error {
func (b *backupReconciler) recreateVolumeSnapshotContent(vsc snapshotv1api.VolumeSnapshotContent) error {
timeout := 1 * time.Minute
interval := 1 * time.Second
err := c.kbClient.Delete(context.TODO(), &vsc)
err := b.kbClient.Delete(context.TODO(), &vsc)
if err != nil {
return errors.Wrapf(err, "fail to delete VolumeSnapshotContent: %s", vsc.Name)
}
@@ -1110,7 +1068,7 @@ func (c *backupController) recreateVolumeSnapshotContent(vsc snapshotv1api.Volum
// Check VolumeSnapshotContents is already deleted, before re-creating it.
err = wait.PollImmediate(interval, timeout, func() (bool, error) {
tmpVSC := &snapshotv1api.VolumeSnapshotContent{}
err := c.kbClient.Get(context.TODO(), kbclient.ObjectKey{Name: vsc.Name}, tmpVSC)
err := b.kbClient.Get(context.TODO(), kbclient.ObjectKey{Name: vsc.Name}, tmpVSC)
if err != nil {
if apierrors.IsNotFound(err) {
return true, nil
@@ -1139,7 +1097,7 @@ func (c *backupController) recreateVolumeSnapshotContent(vsc snapshotv1api.Volum
}
// ResourceVersion shouldn't exist for new creation.
vsc.ResourceVersion = ""
err = c.kbClient.Create(context.TODO(), &vsc)
err = b.kbClient.Create(context.TODO(), &vsc)
if err != nil {
return errors.Wrapf(err, "fail to create VolumeSnapshotContent %s", vsc.Name)
}