mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-08-19 05:36:08 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

add PartiallyFailed phase for backups, log+continue on errors (#1386)

Browse Source 
Signed-off-by: Steve Kriss <krisss@vmware.com>
This commit is contained in:
Steve Kriss
2019-04-26 09:14:26 -07:00
committed by KubeKween
parent 42f351b000
commit 8392e6d83f
13 changed files with 355 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/controller/backup_controller.go
+66 -31
View File
@@ -1,5 +1,5 @@
/*
Copyright 2017 the Velero contributors.
Copyright 2017, 2019 the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -214,16 +214,29 @@ func (c *backupController) processBackup(key string) error {
defer c.backupTracker.Delete(request.Namespace, request.Name)
log.Debug("Running backup")
// execution & upload of backup
backupScheduleName := request.GetLabels()[velerov1api.ScheduleNameLabel]
c.metrics.RegisterBackupAttempt(backupScheduleName)
// execution & upload of backup
if err := c.runBackup(request); err != nil {
// even though runBackup sets the backup's phase prior
// to uploading artifacts to object storage, we have to
// check for an error again here and update the phase if
// one is found, because there could've been an error
// while uploading artifacts to object storage, which would
// result in the backup being Failed.
log.WithError(err).Error("backup failed")
request.Status.Phase = velerov1api.BackupPhaseFailed
c.metrics.RegisterBackupFailed(backupScheduleName)
} else {
}
switch request.Status.Phase {
case velerov1api.BackupPhaseCompleted:
c.metrics.RegisterBackupSuccess(backupScheduleName)
case velerov1api.BackupPhasePartiallyFailed:
c.metrics.RegisterBackupPartialFailure(backupScheduleName)
case velerov1api.BackupPhaseFailed:
c.metrics.RegisterBackupFailed(backupScheduleName)
}
log.Debug("Updating backup's final status")
@@ -411,9 +424,11 @@ func (c *backupController) validateAndGetSnapshotLocations(backup *velerov1api.B
return providerLocations, nil
}
// runBackup runs and uploads a validated backup. Any error returned from this function
// causes the backup to be Failed; if no error is returned, the backup's status's Errors
// field is checked to see if the backup was a partial failure.
func (c *backupController) runBackup(backup *pkgbackup.Request) error {
log := c.logger.WithField("backup", kubeutil.NamespaceAndName(backup))
log.Info("Starting backup")
c.logger.WithField("backup", kubeutil.NamespaceAndName(backup)).Info("Setting up backup log")
logFile, err := ioutil.TempFile("", "")
if err != nil {
@@ -429,25 +444,31 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error {
// backup log failed for whatever reason.
logger := logging.DefaultLogger(c.backupLogLevel)
logger.Out = io.MultiWriter(os.Stdout, gzippedLogFile)
log = logger.WithField("backup", kubeutil.NamespaceAndName(backup))
log.Info("Starting backup")
logCounter := logging.NewLogCounterHook()
logger.Hooks.Add(logCounter)
backupLog := logger.WithField("backup", kubeutil.NamespaceAndName(backup))
backupLog.Info("Setting up backup temp file")
backupFile, err := ioutil.TempFile("", "")
if err != nil {
return errors.Wrap(err, "error creating temp file for backup")
}
defer closeAndRemoveFile(backupFile, log)
defer closeAndRemoveFile(backupFile, backupLog)
pluginManager := c.newPluginManager(log)
backupLog.Info("Setting up plugin manager")
pluginManager := c.newPluginManager(backupLog)
defer pluginManager.CleanupClients()
backupLog.Info("Getting backup item actions")
actions, err := pluginManager.GetBackupItemActions()
if err != nil {
return err
}
backupStore, err := c.newBackupStore(backup.StorageLocation, pluginManager, log)
backupLog.Info("Setting up backup store")
backupStore, err := c.newBackupStore(backup.StorageLocation, pluginManager, backupLog)
if err != nil {
return err
}
@@ -462,17 +483,9 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error {
return errors.Errorf("backup already exists in object storage")
}
// Do the actual backup
var errs []error
if err := c.backupper.Backup(log, backup, backupFile, actions, pluginManager); err != nil {
errs = append(errs, err)
backup.Status.Phase = velerov1api.BackupPhaseFailed
} else {
backup.Status.Phase = velerov1api.BackupPhaseCompleted
}
if err := gzippedLogFile.Close(); err != nil {
c.logger.WithError(err).Error("error closing gzippedLogFile")
var fatalErrs []error
if err := c.backupper.Backup(backupLog, backup, backupFile, actions, pluginManager); err != nil {
fatalErrs = append(fatalErrs, err)
}
// Mark completion timestamp before serializing and uploading.
@@ -486,21 +499,45 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error {
}
}
errs = append(errs, persistBackup(backup, backupFile, logFile, backupStore, c.logger)...)
errs = append(errs, recordBackupMetrics(backup.Backup, backupFile, c.metrics))
recordBackupMetrics(backupLog, backup.Backup, backupFile, c.metrics)
log.Info("Backup completed")
if err := gzippedLogFile.Close(); err != nil {
c.logger.WithError(err).Error("error closing gzippedLogFile")
}
return kerrors.NewAggregate(errs)
backup.Status.Warnings = logCounter.GetCount(logrus.WarnLevel)
backup.Status.Errors = logCounter.GetCount(logrus.ErrorLevel)
// Assign finalize phase as close to end as possible so that any errors
// logged to backupLog are captured. This is done before uploading the
// artifacts to object storage so that the JSON representation of the
// backup in object storage has the terminal phase set.
switch {
case len(fatalErrs) > 0:
backup.Status.Phase = velerov1api.BackupPhaseFailed
case logCounter.GetCount(logrus.ErrorLevel) > 0:
backup.Status.Phase = velerov1api.BackupPhasePartiallyFailed
default:
backup.Status.Phase = velerov1api.BackupPhaseCompleted
}
if errs := persistBackup(backup, backupFile, logFile, backupStore, c.logger); len(errs) > 0 {
fatalErrs = append(fatalErrs, errs...)
}
c.logger.Info("Backup completed")
// if we return a non-nil error, the calling function will update
// the backup's phase to Failed.
return kerrors.NewAggregate(fatalErrs)
}
func recordBackupMetrics(backup *velerov1api.Backup, backupFile *os.File, serverMetrics *metrics.ServerMetrics) error {
func recordBackupMetrics(log logrus.FieldLogger, backup *velerov1api.Backup, backupFile *os.File, serverMetrics *metrics.ServerMetrics) {
backupScheduleName := backup.GetLabels()[velerov1api.ScheduleNameLabel]
var backupSizeBytes int64
var err error
if backupFileStat, err := backupFile.Stat(); err != nil {
err = errors.Wrap(err, "error getting file info")
log.WithError(errors.WithStack(err)).Error("Error getting backup file info")
} else {
backupSizeBytes = backupFileStat.Size()
}
@@ -512,8 +549,6 @@ func recordBackupMetrics(backup *velerov1api.Backup, backupFile *os.File, server
serverMetrics.RegisterVolumeSnapshotAttempts(backupScheduleName, backup.Status.VolumeSnapshotsAttempted)
serverMetrics.RegisterVolumeSnapshotSuccesses(backupScheduleName, backup.Status.VolumeSnapshotsCompleted)
serverMetrics.RegisterVolumeSnapshotFailures(backupScheduleName, backup.Status.VolumeSnapshotsAttempted-backup.Status.VolumeSnapshotsCompleted)
return err
}
func persistBackup(backup *pkgbackup.Request, backupContents, backupLog *os.File, backupStore persistence.BackupStore, log logrus.FieldLogger) []error {