mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-09-26 09:54:23 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Make in-progress PVB/PVR as failed when restic controller restarts to avoid hanging backup/restore

Browse Source 
Make in-progress PVB/PVR as failed when restic controller restarts to avoid hanging backup/restore

Fixes #4772

Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com>
This commit is contained in:
Wenkai Yin(尹文开)
2022-05-07 16:51:06 +08:00
parent 6a42d1c4e1
commit 034cda2571
7 changed files with 224 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/controller/pod_volume_restore_controller.go
+31 -27
View File
@@ -31,7 +31,6 @@ import (
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/clock"
"sigs.k8s.io/cluster-api/util/patch"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/handler"
@@ -101,24 +100,21 @@ func (c *PodVolumeRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Req
if they interfere with volumes being restored: %s index %d`, restic.InitContainer, restic.InitContainer, resticInitContainerIndex)
}
patchHelper, err := patch.NewHelper(pvr, c.Client)
if err != nil {
log.WithError(err).Error("Unable to new patch helper")
return ctrl.Result{}, err
}
log.Info("Restore starting")
original := pvr.DeepCopy()
pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseInProgress
pvr.Status.StartTimestamp = &metav1.Time{Time: c.clock.Now()}
if err = patchHelper.Patch(ctx, pvr); err != nil {
if err = kube.Patch(ctx, original, pvr, c.Client); err != nil {
log.WithError(err).Error("Unable to update status to in progress")
return ctrl.Result{}, err
}
if err = c.processRestore(ctx, pvr, pod, log); err != nil {
original = pvr.DeepCopy()
pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseFailed
pvr.Status.Message = err.Error()
pvr.Status.CompletionTimestamp = &metav1.Time{Time: c.clock.Now()}
if e := patchHelper.Patch(ctx, pvr); e != nil {
if e := kube.Patch(ctx, original, pvr, c.Client); e != nil {
log.WithError(err).Error("Unable to update status to failed")
}
@@ -126,9 +122,10 @@ func (c *PodVolumeRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Req
return ctrl.Result{}, err
}
original = pvr.DeepCopy()
pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseCompleted
pvr.Status.CompletionTimestamp = &metav1.Time{Time: c.clock.Now()}
if err = patchHelper.Patch(ctx, pvr); err != nil {
if err = kube.Patch(ctx, original, pvr, c.Client); err != nil {
log.WithError(err).Error("Unable to update status to completed")
return ctrl.Result{}, err
}
@@ -137,11 +134,6 @@ func (c *PodVolumeRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Req
}
func (c *PodVolumeRestoreReconciler) shouldProcess(ctx context.Context, log logrus.FieldLogger, pvr *velerov1api.PodVolumeRestore) (bool, *corev1api.Pod, error) {
if !isPVRNew(pvr) {
log.Debug("PodVolumeRestore is not new, skip")
return false, nil, nil
}
// we filter the pods during the initialization of cache, if we can get a pod here, the pod must be in the same node with the controller
// so we don't need to compare the node anymore
pod := &corev1api.Pod{}
@@ -154,6 +146,28 @@ func (c *PodVolumeRestoreReconciler) shouldProcess(ctx context.Context, log logr
return false, nil, err
}
// the status checking logic must be put after getting the PVR's pod because that the getting pod logic
// makes sure the PVR's pod is on the same node with the controller. The controller should only process
// the PVRs on the same node
switch pvr.Status.Phase {
case "", velerov1api.PodVolumeRestorePhaseNew:
case velerov1api.PodVolumeRestorePhaseInProgress:
original := pvr.DeepCopy()
pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseFailed
pvr.Status.Message = fmt.Sprintf("got a PodVolumeRestore with unexpected status %q, this may be due to a restart of the controller during the restoring, mark it as %q",
velerov1api.PodVolumeRestorePhaseInProgress, pvr.Status.Phase)
pvr.Status.CompletionTimestamp = &metav1.Time{Time: c.clock.Now()}
if err := kube.Patch(ctx, original, pvr, c.Client); err != nil {
log.WithError(err).Error("Unable to update status to failed")
return false, nil, err
}
log.Warn(pvr.Status.Message)
return false, nil, nil
default:
log.Debug("PodVolumeRestore is not new or in-progress, skip")
return false, nil, nil
}
if !isResticInitContainerRunning(pod) {
log.Debug("Pod is not running restic-wait init container, skip")
return false, nil, nil
@@ -163,8 +177,6 @@ func (c *PodVolumeRestoreReconciler) shouldProcess(ctx context.Context, log logr
}
func (c *PodVolumeRestoreReconciler) SetupWithManager(mgr ctrl.Manager) error {
mgr.GetConfig()
// The pod may not being scheduled at the point when its PVRs are initially reconciled.
// By watching the pods, we can trigger the PVR reconciliation again once the pod is finally scheduled on the node.
return ctrl.NewControllerManagedBy(mgr).
@@ -197,10 +209,6 @@ func (c *PodVolumeRestoreReconciler) findVolumeRestoresForPod(pod client.Object)
return requests
}
func isPVRNew(pvr *velerov1api.PodVolumeRestore) bool {
return pvr.Status.Phase == "" || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseNew
}
func isResticInitContainerRunning(pod *corev1api.Pod) bool {
// Restic wait container can be anywhere in the list of init containers, but must be running.
i := getResticInitContainerIndex(pod)
@@ -337,13 +345,9 @@ func (c *PodVolumeRestoreReconciler) processRestore(ctx context.Context, req *ve
// the PVR with the new progress
func (c *PodVolumeRestoreReconciler) updateRestoreProgressFunc(req *velerov1api.PodVolumeRestore, log logrus.FieldLogger) func(velerov1api.PodVolumeOperationProgress) {
return func(progress velerov1api.PodVolumeOperationProgress) {
helper, err := patch.NewHelper(req, c.Client)
if err != nil {
log.WithError(err).Error("Unable to new patch helper")
return
}
original := req.DeepCopy()
req.Status.Progress = progress
if err = helper.Patch(context.Background(), req); err != nil {
if err := kube.Patch(context.Background(), original, req, c.Client); err != nil {
log.WithError(err).Error("Unable to update PodVolumeRestore progress")
}
}