velero/pkg/controller/backup_finalizer_controller.go

/*
Copyright the Velero contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"bytes"
	"context"
	"os"
	"time"

	"github.com/cockroachdb/errors"
	"github.com/sirupsen/logrus"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/util/retry"
	clocks "k8s.io/utils/clock"
	ctrl "sigs.k8s.io/controller-runtime"
	kbclient "sigs.k8s.io/controller-runtime/pkg/client"

	velerov1api "github.com/velero-io/velero/pkg/apis/velero/v1"
	pkgbackup "github.com/velero-io/velero/pkg/backup"
	"github.com/velero-io/velero/pkg/client"
	"github.com/velero-io/velero/pkg/constant"
	"github.com/velero-io/velero/pkg/itemoperation"
	"github.com/velero-io/velero/pkg/kuberesource"
	"github.com/velero-io/velero/pkg/metrics"
	"github.com/velero-io/velero/pkg/persistence"
	"github.com/velero-io/velero/pkg/plugin/clientmgmt"
	"github.com/velero-io/velero/pkg/plugin/framework"
	"github.com/velero-io/velero/pkg/util/csi"
	"github.com/velero-io/velero/pkg/util/encode"
)

// objectStorageBackoff retries object storage writes (PutBackupMetadata,
// PutBackupContents). Bounded (Steps) so a persistent failure surfaces as an
// error within a bounded time rather than retrying forever; see the retry
// call sites below for rationale. Var (not const) so tests can shrink it for
// speed, matching pkg/repository/maintenance's waitCompletionBackOff pattern.
var objectStorageBackoff = wait.Backoff{
	Duration: time.Second,
	Factor:   2.0,
	Steps:    5,
	Jitter:   0.1,
}

// backupFinalizerReconciler reconciles a Backup object
type backupFinalizerReconciler struct {
	client            kbclient.Client
	globalCRClient    kbclient.Client
	clock             clocks.WithTickerAndDelayedExecution
	backupper         pkgbackup.Backupper
	newPluginManager  func(logrus.FieldLogger) clientmgmt.Manager
	backupTracker     BackupTracker
	metrics           *metrics.ServerMetrics
	backupStoreGetter persistence.ObjectBackupStoreGetter
	log               logrus.FieldLogger
	resourceTimeout   time.Duration
}

// NewBackupFinalizerReconciler initializes and returns backupFinalizerReconciler struct.
func NewBackupFinalizerReconciler(
	client kbclient.Client,
	globalCRClient kbclient.Client,
	clock clocks.WithTickerAndDelayedExecution,
	backupper pkgbackup.Backupper,
	newPluginManager func(logrus.FieldLogger) clientmgmt.Manager,
	backupTracker BackupTracker,
	backupStoreGetter persistence.ObjectBackupStoreGetter,
	log logrus.FieldLogger,
	metrics *metrics.ServerMetrics,
	resourceTimeout time.Duration,
) *backupFinalizerReconciler {
	return &backupFinalizerReconciler{
		client:            client,
		globalCRClient:    globalCRClient,
		clock:             clock,
		backupper:         backupper,
		newPluginManager:  newPluginManager,
		backupTracker:     backupTracker,
		backupStoreGetter: backupStoreGetter,
		log:               log,
		metrics:           metrics,
	}
}

// +kubebuilder:rbac:groups=velero.io,resources=backups,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=velero.io,resources=backups/status,verbs=get;update;patch

func (r *backupFinalizerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
	log := r.log.WithFields(logrus.Fields{
		"controller": "backup-finalizer",
		"backup":     req.NamespacedName,
	})

	// Fetch the Backup instance.
	log.Debug("Getting Backup")
	backup := &velerov1api.Backup{}
	if err := r.client.Get(ctx, req.NamespacedName, backup); err != nil {
		if apierrors.IsNotFound(err) {
			log.Debug("Unable to find Backup")
			return ctrl.Result{}, nil
		}

		log.WithError(err).Error("Error getting Backup")
		return ctrl.Result{}, errors.WithStack(err)
	}
	switch backup.Status.Phase {
	case velerov1api.BackupPhaseCompleted, velerov1api.BackupPhasePartiallyFailed, velerov1api.BackupPhaseFailed:
		// Terminal status is persisted only after the finalized archive has been
		// uploaded. Run group cleanup on this subsequent reconciliation so a
		// cleanup failure/restart retries deletion, not archive generation using
		// snapshots that may already have been deleted.
		if backup.Annotations[velerov1api.VolumeGroupSnapshotBackupAnnotation] == "true" &&
			backup.Annotations[velerov1api.VolumeGroupSnapshotCleanupCompletedAnnotation] != "true" {
			if err := csi.CleanupBackupVolumeGroupSnapshots(ctx, backup, r.globalCRClient, log); err != nil {
				return ctrl.Result{}, err
			}
			base := backup.DeepCopy()
			if backup.Annotations == nil {
				backup.Annotations = map[string]string{}
			}
			backup.Annotations[velerov1api.VolumeGroupSnapshotCleanupCompletedAnnotation] = "true"
			return ctrl.Result{}, r.client.Patch(ctx, backup, kbclient.MergeFrom(base))
		}
		return ctrl.Result{}, nil
	case velerov1api.BackupPhaseFinalizing, velerov1api.BackupPhaseFinalizingPartiallyFailed:
		// only process backups finalizing after  plugin operations are complete
	default:
		log.Debug("Backup is not awaiting finalizing, skipping")
		return ctrl.Result{}, nil
	}

	original := backup.DeepCopy()
	defer func() {
		switch backup.Status.Phase {
		case
			velerov1api.BackupPhaseCompleted,
			velerov1api.BackupPhasePartiallyFailed,
			velerov1api.BackupPhaseFailed,
			velerov1api.BackupPhaseFailedValidation:
			r.backupTracker.Delete(backup.Namespace, backup.Name)
		}
		// Always attempt to Patch the backup object and status after each reconciliation.
		//
		// if this patch fails, there may not be another opportunity to update the backup object without external update event.
		// so we retry
		// This retries updating Finalzing/FinalizingPartiallyFailed to Completed/PartiallyFailed
		if err := client.RetryOnErrorMaxBackOff(r.resourceTimeout, func() error { return r.client.Patch(ctx, backup, kbclient.MergeFrom(original)) }); err != nil {
			log.WithError(err).Error("Error updating backup")
			return
		}
	}()

	location := &velerov1api.BackupStorageLocation{}
	if err := r.client.Get(ctx, kbclient.ObjectKey{
		Namespace: backup.Namespace,
		Name:      backup.Spec.StorageLocation,
	}, location); err != nil {
		return ctrl.Result{}, errors.WithStack(err)
	}
	pluginManager := r.newPluginManager(log)
	defer pluginManager.CleanupClients()

	backupStore, err := r.backupStoreGetter.Get(location, pluginManager, log)
	if err != nil {
		log.WithError(err).Error("Error getting a backup store")
		return ctrl.Result{}, errors.WithStack(err)
	}

	// Download item operations list and backup contents
	operations, err := backupStore.GetBackupItemOperations(backup.Name)
	if err != nil {
		log.WithError(err).Error("Error getting backup item operations")
		return ctrl.Result{}, errors.WithStack(err)
	}

	backupRequest := &pkgbackup.Request{
		Backup:                        backup,
		StorageLocation:               location,
		SkippedVolumeTracker:          pkgbackup.NewSkipVolumeTracker(),
		BackedUpItems:                 pkgbackup.NewBackedUpItemsMap(),
		MustIncludeAdditionalItemPVCs: pkgbackup.NewBackedUpItemsMap(),
	}
	var outBackupFile *os.File
	if len(operations) > 0 {
		log.Info("Setting up finalized backup temp file")
		inBackupFile, err := downloadToTempFile(backup.Name, backupStore, log)
		if err != nil {
			return ctrl.Result{}, errors.Wrap(err, "error downloading backup")
		}
		defer closeAndRemoveFile(inBackupFile, log)
		outBackupFile, err = os.CreateTemp("", "")
		if err != nil {
			log.WithError(err).Error("error creating temp file for backup")
			return ctrl.Result{}, errors.WithStack(err)
		}
		defer closeAndRemoveFile(outBackupFile, log)

		log.Info("Getting backup item actions")
		actions, err := pluginManager.GetBackupItemActionsV2()
		if err != nil {
			log.WithError(err).Error("error getting Backup Item Actions")
			return ctrl.Result{}, errors.WithStack(err)
		}
		backupItemActionsResolver := framework.NewBackupItemActionResolverV2(actions)

		// Call itemBackupper.BackupItem for the list of items updated by async operations
		err = r.backupper.FinalizeBackup(
			log,
			backupRequest,
			inBackupFile,
			outBackupFile,
			backupItemActionsResolver,
			operations,
			backupStore,
		)
		if err != nil {
			log.WithError(err).Error("error finalizing Backup")
			return ctrl.Result{}, errors.WithStack(err)
		}
	}
	backupScheduleName := backupRequest.GetLabels()[velerov1api.ScheduleNameLabel]

	// Determine the final phase and completion timestamp, but do NOT set them
	// on the in-memory backup object yet. We first need to upload metadata and
	// contents to object storage. If the upload fails, the deferred patch must
	// NOT write a terminal phase to the API server so the controller can retry.
	var finalPhase velerov1api.BackupPhase
	switch backup.Status.Phase {
	case velerov1api.BackupPhaseFinalizing:
		finalPhase = velerov1api.BackupPhaseCompleted
	case velerov1api.BackupPhaseFinalizingPartiallyFailed:
		finalPhase = velerov1api.BackupPhasePartiallyFailed
	default:
		return ctrl.Result{}, nil
	}

	completionTimestamp := &metav1.Time{Time: r.clock.Now()}
	csiVolumeSnapshotsCompleted := updateCSIVolumeSnapshotsCompleted(operations)

	// Encode backup JSON with the final phase for object storage, so that the
	// metadata in storage reflects the completed state.
	backupForUpload := backup.DeepCopy()
	backupForUpload.Status.Phase = finalPhase
	backupForUpload.Status.CompletionTimestamp = completionTimestamp
	backupForUpload.Status.CSIVolumeSnapshotsCompleted = csiVolumeSnapshotsCompleted

	backupJSON := new(bytes.Buffer)
	if err := encode.To(backupForUpload, "json", backupJSON); err != nil {
		return ctrl.Result{}, errors.Wrap(err, "error encoding backup json")
	}
	// retry.DefaultBackoff is tuned for API server optimistic-concurrency
	// conflicts (4 steps, ~1.25s total) and gives up far too quickly for
	// object storage calls, which can see longer transient outages or
	// throttling. objectStorageBackoff (above) grows more slowly and gives
	// up after a bounded time instead of retrying forever, so a persistent
	// failure (e.g. an object-lock/immutability policy) surfaces as an error
	// promptly; controller-runtime requeues the Reconcile on error, so
	// retries continue across reconciles rather than blocking a worker
	// goroutine indefinitely on a call that will never succeed.
	if err := retry.OnError(objectStorageBackoff, func(err error) bool { return err != nil },
		func() error { return backupStore.PutBackupMetadata(backup.Name, bytes.NewReader(backupJSON.Bytes())) },
	); err != nil {
		return ctrl.Result{}, errors.Wrap(err, "error uploading backup json")
	}
	if len(operations) > 0 {
		if err := retry.OnError(objectStorageBackoff, func(err error) bool { return err != nil },
			func() error { return backupStore.PutBackupContents(backup.Name, outBackupFile) },
		); err != nil {
			return ctrl.Result{}, errors.Wrap(err, "error uploading backup final contents")
		}
	}

	// Uploads succeeded — now safe to set the final phase on the in-memory
	// backup object so the deferred patch writes it to the API server.
	backup.Status.Phase = finalPhase
	backup.Status.CompletionTimestamp = completionTimestamp
	backup.Status.CSIVolumeSnapshotsCompleted = csiVolumeSnapshotsCompleted

	switch finalPhase {
	case velerov1api.BackupPhaseCompleted:
		r.metrics.RegisterBackupSuccess(backupScheduleName)
		r.metrics.RegisterBackupLastStatus(backupScheduleName, metrics.BackupLastStatusSucc)
	case velerov1api.BackupPhasePartiallyFailed:
		r.metrics.RegisterBackupPartialFailure(backupScheduleName)
		r.metrics.RegisterBackupLastStatus(backupScheduleName, metrics.BackupLastStatusFailure)
	}

	recordBackupMetrics(log, backup, outBackupFile, r.metrics, true)

	return ctrl.Result{}, nil
}

func (r *backupFinalizerReconciler) SetupWithManager(mgr ctrl.Manager) error {
	return ctrl.NewControllerManagedBy(mgr).
		For(&velerov1api.Backup{}).
		Named(constant.ControllerBackupFinalizer).
		Complete(r)
}

// updateCSIVolumeSnapshotsCompleted calculate the completed VS number according to
// the backup's async operation list.
func updateCSIVolumeSnapshotsCompleted(
	operations []*itemoperation.BackupOperation) int {
	completedNum := 0

	for index := range operations {
		if operations[index].Spec.ResourceIdentifier.String() == kuberesource.VolumeSnapshots.String() &&
			operations[index].Status.Phase == itemoperation.OperationPhaseCompleted {
			completedNum++
		}
	}

	return completedNum
}