mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-09-21 07:24:19 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #7117 from allenxu404/issue6567

Browse Source 
Add hook status to backup/restore CR
This commit is contained in:
Daniel Jiang
2023-11-28 16:54:11 +08:00
committed by GitHub
parent 6ac7ff1230 5d1a632be4
commit 85482aefaf
19 changed files with 1173 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/apis/velero/v1/backup_types.go
+18
View File
@@ -441,6 +441,11 @@ type BackupStatus struct {
// BackupItemAction operations for this backup which ended with an error.
// +optional
BackupItemOperationsFailed int `json:"backupItemOperationsFailed,omitempty"`
// HookStatus contains information about the status of the hooks.
// +optional
// +nullable
HookStatus *HookStatus `json:"hookStatus,omitempty"`
}
// BackupProgress stores information about the progress of a Backup's execution.
@@ -458,6 +463,19 @@ type BackupProgress struct {
ItemsBackedUp int `json:"itemsBackedUp,omitempty"`
}
// HookStatus stores information about the status of the hooks.
type HookStatus struct {
// HooksAttempted is the total number of attempted hooks
// Specifically, HooksAttempted represents the number of hooks that failed to execute
// and the number of hooks that executed successfully.
// +optional
HooksAttempted int `json:"hooksAttempted,omitempty"`
// HooksFailed is the total number of hooks which ended with an error
// +optional
HooksFailed int `json:"hooksFailed,omitempty"`
}
// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +kubebuilder:object:root=true
pkg/apis/velero/v1/restore_types.go
+5
View File
@@ -345,6 +345,11 @@ type RestoreStatus struct {
// RestoreItemAction operations for this restore which ended with an error.
// +optional
RestoreItemOperationsFailed int `json:"restoreItemOperationsFailed,omitempty"`
// HookStatus contains information about the status of the hooks.
// +optional
// +nullable
HookStatus *HookStatus `json:"hookStatus,omitempty"`
}
// RestoreProgress stores information about the restore's execution progress
pkg/apis/velero/v1/zz_generated.deepcopy.go
+25
View File
@@ -419,6 +419,11 @@ func (in *BackupStatus) DeepCopyInto(out *BackupStatus) {
*out = new(BackupProgress)
**out = **in
}
if in.HookStatus != nil {
in, out := &in.HookStatus, &out.HookStatus
*out = new(HookStatus)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupStatus.
@@ -802,6 +807,21 @@ func (in *ExecRestoreHook) DeepCopy() *ExecRestoreHook {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *HookStatus) DeepCopyInto(out *HookStatus) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HookStatus.
func (in *HookStatus) DeepCopy() *HookStatus {
if in == nil {
return nil
}
out := new(HookStatus)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *InitRestoreHook) DeepCopyInto(out *InitRestoreHook) {
*out = *in
@@ -1362,6 +1382,11 @@ func (in *RestoreStatus) DeepCopyInto(out *RestoreStatus) {
*out = new(RestoreProgress)
**out = **in
}
if in.HookStatus != nil {
in, out := &in.HookStatus, &out.HookStatus
*out = new(HookStatus)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreStatus.
pkg/backup/backup.go
+10 -1
View File
@@ -302,6 +302,7 @@ func (kb *kubernetesBackupper) BackupWithResolvers(log logrus.FieldLogger,
itemHookHandler: &hook.DefaultItemHookHandler{
PodCommandExecutor: kb.podCommandExecutor,
},
hookTracker: hook.NewHookTracker(),
}
// helper struct to send current progress between the main
@@ -427,8 +428,15 @@ func (kb *kubernetesBackupper) BackupWithResolvers(log logrus.FieldLogger,
updated.Status.Progress.TotalItems = len(backupRequest.BackedUpItems)
updated.Status.Progress.ItemsBackedUp = len(backupRequest.BackedUpItems)
// update the hooks execution status
if updated.Status.HookStatus == nil {
updated.Status.HookStatus = &velerov1api.HookStatus{}
}
updated.Status.HookStatus.HooksAttempted, updated.Status.HookStatus.HooksFailed = itemBackupper.hookTracker.Stat()
log.Infof("hookTracker: %+v, hookAttempted: %d, hookFailed: %d", itemBackupper.hookTracker.GetTracker(), updated.Status.HookStatus.HooksAttempted, updated.Status.HookStatus.HooksFailed)
if err := kube.PatchResource(backupRequest.Backup, updated, kb.kbClient); err != nil {
log.WithError(errors.WithStack((err))).Warn("Got error trying to update backup's status.progress")
log.WithError(errors.WithStack((err))).Warn("Got error trying to update backup's status.progress and hook status")
}
skippedPVSummary, _ := json.Marshal(backupRequest.SkippedPVTracker.Summary())
log.Infof("Summary for skipped PVs: %s", skippedPVSummary)
@@ -598,6 +606,7 @@ func (kb *kubernetesBackupper) FinalizeBackup(log logrus.FieldLogger,
discoveryHelper: kb.discoveryHelper,
itemHookHandler: &hook.NoOpItemHookHandler{},
podVolumeSnapshotTracker: newPVCSnapshotTracker(),
hookTracker: hook.NewHookTracker(),
}
updateFiles := make(map[string]FileForArchive)
backedUpGroupResources := map[schema.GroupResource]bool{}
pkg/backup/item_backupper.go
+4 -3
View File
@@ -78,6 +78,7 @@ type itemBackupper struct {
itemHookHandler hook.ItemHookHandler
snapshotLocationVolumeSnapshotters map[string]vsv1.VolumeSnapshotter
hookTracker *hook.HookTracker
}
type FileForArchive struct {
@@ -184,7 +185,7 @@ func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runti
)
log.Debug("Executing pre hooks")
if err := ib.itemHookHandler.HandleHooks(log, groupResource, obj, ib.backupRequest.ResourceHooks, hook.PhasePre); err != nil {
if err := ib.itemHookHandler.HandleHooks(log, groupResource, obj, ib.backupRequest.ResourceHooks, hook.PhasePre, ib.hookTracker); err != nil {
return false, itemFiles, err
}
if optedOut, podName := ib.podVolumeSnapshotTracker.OptedoutByPod(namespace, name); optedOut {
@@ -234,7 +235,7 @@ func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runti
// if there was an error running actions, execute post hooks and return
log.Debug("Executing post hooks")
if err := ib.itemHookHandler.HandleHooks(log, groupResource, obj, ib.backupRequest.ResourceHooks, hook.PhasePost); err != nil {
if err := ib.itemHookHandler.HandleHooks(log, groupResource, obj, ib.backupRequest.ResourceHooks, hook.PhasePost, ib.hookTracker); err != nil {
backupErrs = append(backupErrs, err)
}
return false, itemFiles, kubeerrs.NewAggregate(backupErrs)
@@ -293,7 +294,7 @@ func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runti
}
log.Debug("Executing post hooks")
if err := ib.itemHookHandler.HandleHooks(log, groupResource, obj, ib.backupRequest.ResourceHooks, hook.PhasePost); err != nil {
if err := ib.itemHookHandler.HandleHooks(log, groupResource, obj, ib.backupRequest.ResourceHooks, hook.PhasePost, ib.hookTracker); err != nil {
backupErrs = append(backupErrs, err)
}
pkg/cmd/util/output/backup_describer.go
+6
View File
@@ -392,6 +392,12 @@ func DescribeBackupStatus(ctx context.Context, kbClient kbclient.Client, d *Desc
}
d.Printf("Velero-Native Snapshots: <none included>\n")
if status.HookStatus != nil {
d.Println()
d.Printf("HooksAttempted:\t%d\n", status.HookStatus.HooksAttempted)
d.Printf("HooksFailed:\t%d\n", status.HookStatus.HooksFailed)
}
}
func describeBackupItemOperations(ctx context.Context, kbClient kbclient.Client, d *Describer, backup *velerov1api.Backup, details bool, insecureSkipTLSVerify bool, caCertPath string) {
pkg/cmd/util/output/backup_structured_describer.go
+5
View File
@@ -303,6 +303,11 @@ func DescribeBackupStatusInSF(ctx context.Context, kbClient kbclient.Client, d *
backupStatusInfo["veleroNativeSnapshotsDetail"] = snapshotDetails
return
}
if status.HookStatus != nil {
backupStatusInfo["hooksAttempted"] = status.HookStatus.HooksAttempted
backupStatusInfo["hooksFailed"] = status.HookStatus.HooksFailed
}
}
func describeBackupResourceListInSF(ctx context.Context, kbClient kbclient.Client, backupStatusInfo map[string]interface{}, backup *velerov1api.Backup, insecureSkipTLSVerify bool, caCertPath string) {
pkg/cmd/util/output/restore_describer.go
+6
View File
@@ -180,6 +180,12 @@ func DescribeRestore(ctx context.Context, kbClient kbclient.Client, restore *vel
d.Println()
describeRestoreItemOperations(ctx, kbClient, d, restore, details, insecureSkipTLSVerify, caCertFile)
if restore.Status.HookStatus != nil {
d.Println()
d.Printf("HooksAttempted: \t%d\n", restore.Status.HookStatus.HooksAttempted)
d.Printf("HooksFailed: \t%d\n", restore.Status.HookStatus.HooksFailed)
}
if details {
describeRestoreResourceList(ctx, kbClient, d, restore, insecureSkipTLSVerify, caCertFile)
d.Println()
pkg/restore/restore.go
+17 -6
View File
@@ -325,6 +325,7 @@ func (kr *kubernetesRestorer) RestoreWithResolvers(
resourceModifiers: req.ResourceModifiers,
disableInformerCache: req.DisableInformerCache,
featureVerifier: kr.featureVerifier,
hookTracker: hook.NewHookTracker(),
}
return restoreCtx.execute()
@@ -377,6 +378,7 @@ type restoreContext struct {
resourceModifiers *resourcemodifiers.ResourceModifiers
disableInformerCache bool
featureVerifier features.Verifier
hookTracker *hook.HookTracker
}
type resourceClientKey struct {
@@ -646,11 +648,6 @@ func (ctx *restoreContext) execute() (results.Result, results.Result) {
updated.Status.Progress.TotalItems = len(ctx.restoredItems)
updated.Status.Progress.ItemsRestored = len(ctx.restoredItems)
err = kube.PatchResource(ctx.restore, updated, ctx.kbClient)
if err != nil {
ctx.log.WithError(errors.WithStack((err))).Warn("Updating restore status.progress")
}
// Wait for all of the pod volume restore goroutines to be done, which is
// only possible once all of their errors have been received by the loop
// below, then close the podVolumeErrs channel so the loop terminates.
@@ -685,6 +682,19 @@ func (ctx *restoreContext) execute() (results.Result, results.Result) {
}
ctx.log.Info("Done waiting for all post-restore exec hooks to complete")
// update hooks execution status
if updated.Status.HookStatus == nil {
updated.Status.HookStatus = &velerov1api.HookStatus{}
}
updated.Status.HookStatus.HooksAttempted, updated.Status.HookStatus.HooksFailed = ctx.hookTracker.Stat()
ctx.log.Infof("hookTracker: %+v, hookAttempted: %d, hookFailed: %d", ctx.hookTracker.GetTracker(), updated.Status.HookStatus.HooksAttempted, updated.Status.HookStatus.HooksFailed)
// patch the restore status
err = kube.PatchResource(ctx.restore, updated, ctx.kbClient)
if err != nil {
ctx.log.WithError(errors.WithStack((err))).Warn("Updating restore status")
}
return warnings, errs
}
@@ -1971,6 +1981,7 @@ func (ctx *restoreContext) waitExec(createdObj *unstructured.Unstructured) {
ctx.resourceRestoreHooks,
pod,
ctx.log,
ctx.hookTracker,
)
if err != nil {
ctx.log.WithError(err).Errorf("error getting exec hooks for pod %s/%s", pod.Namespace, pod.Name)
@@ -1978,7 +1989,7 @@ func (ctx *restoreContext) waitExec(createdObj *unstructured.Unstructured) {
return
}
if errs := ctx.waitExecHookHandler.HandleHooks(ctx.hooksContext, ctx.log, pod, execHooksByContainer); len(errs) > 0 {
if errs := ctx.waitExecHookHandler.HandleHooks(ctx.hooksContext, ctx.log, pod, execHooksByContainer, ctx.hookTracker); len(errs) > 0 {
ctx.log.WithError(kubeerrs.NewAggregate(errs)).Error("unable to successfully execute post-restore hooks")
ctx.hooksCancelFunc()