Restore hooks exec (#2804)

* Exec hooks in restored pods

Signed-off-by: Andrew Reed <andrew@replicated.com>

* WaitExecHookHandler implements ItemHookHandler

This required adding a context.Context argument to the ItemHookHandler
interface which is unused by the DefaultItemHookHandler implementation.
It also means passing nil for the []ResourceHook argument since that
holds BackupResourceHook.

Signed-off-by: Andrew Reed <andrew@replicated.com>

* WaitExecHookHandler unit tests

Signed-off-by: Andrew Reed <andrew@replicated.com>

* Changelog and go fmt

Signed-off-by: Andrew Reed <andrew@replicated.com>

* Fix double import

Signed-off-by: Andrew Reed <andrew@replicated.com>

* Default to first contaienr in pod

Signed-off-by: Andrew Reed <andrew@replicated.com>

* Use constants for hook error modes in tests

Signed-off-by: Andrew Reed <andrew@replicated.com>

* Revert to separate WaitExecHookHandler interface

Signed-off-by: Andrew Reed <andrew@replicated.com>

* Negative tests for invalid timeout annotations

Signed-off-by: Andrew Reed <andrew@replicated.com>

* Rename NamedExecRestoreHook PodExecRestoreHook

Also make field names more descriptive.

Signed-off-by: Andrew Reed <andrew@replicated.com>

* Cleanup test names

Signed-off-by: Andrew Reed <andrew@replicated.com>

* Separate maxHookWait and add unit tests

Signed-off-by: Andrew Reed <andrew@replicated.com>

* Comment on maxWait <= 0

Also info log container is not running for hooks to execute in.
Also add context error to hooks not executed errors.

Signed-off-by: Andrew Reed <andrew@replicated.com>

* Remove log about default for invalid timeout

There is no default wait or exec timeout.

Signed-off-by: Andrew Reed <andrew@replicated.com>

* Linting

Signed-off-by: Andrew Reed <andrew@replicated.com>

* Fix log message and rename controller to podWatcher

Signed-off-by: Andrew Reed <andrew@replicated.com>

* Comment on exactly-once semantics for handler

Signed-off-by: Andrew Reed <andrew@replicated.com>

* Fix logging and comments

Use filed logger for pod in handler.
Add comment about pod changes in unit tests.
Use kube util NamespaceAndName in messages.

Signed-off-by: Andrew Reed <andrew@replicated.com>

* Fix maxHookWait

Signed-off-by: Andrew Reed <andrew@replicated.com>

changelogs/unreleased/2804-areed

@@ -0,0 +1 @@
+Implement post-restore exec hooks in pod containers

internal/hook/item_hook_handler.go

@@ -407,3 +407,114 @@ func GetRestoreHooksFromSpec(hooksSpec *velerov1api.RestoreHooks) ([]ResourceRes
 
 	return restoreHooks, nil
 }
+
+// getPodExecRestoreHookFromAnnotations returns an ExecRestoreHook based on restore annotations, as
+// long as the 'command' annotation is present. If it is absent, this returns nil.
+func getPodExecRestoreHookFromAnnotations(annotations map[string]string, log logrus.FieldLogger) *velerov1api.ExecRestoreHook {
+	commandValue := annotations[podRestoreHookCommandAnnotationKey]
+	if commandValue == "" {
+		return nil
+	}
+
+	container := annotations[podRestoreHookContainerAnnotationKey]
+
+	onError := velerov1api.HookErrorMode(annotations[podRestoreHookOnErrorAnnotationKey])
+	if onError != velerov1api.HookErrorModeContinue && onError != velerov1api.HookErrorModeFail {
+		onError = ""
+	}
+
+	var execTimeout time.Duration
+	execTimeoutString := annotations[podRestoreHookTimeoutAnnotationKey]
+	if execTimeoutString != "" {
+		if temp, err := time.ParseDuration(execTimeoutString); err == nil {
+			execTimeout = temp
+		} else {
+			log.Warn(errors.Wrapf(err, "Unable to parse exec timeout %s, ignoring", execTimeoutString))
+		}
+	}
+
+	var waitTimeout time.Duration
+	waitTimeoutString := annotations[podRestoreHookWaitTimeoutAnnotationKey]
+	if waitTimeoutString != "" {
+		if temp, err := time.ParseDuration(waitTimeoutString); err == nil {
+			waitTimeout = temp
+		} else {
+			log.Warn(errors.Wrapf(err, "Unable to parse wait timeout %s, ignoring", waitTimeoutString))
+		}
+	}
+
+	return &velerov1api.ExecRestoreHook{
+		Container:   container,
+		Command:     parseStringToCommand(commandValue),
+		OnError:     onError,
+		ExecTimeout: metav1.Duration{Duration: execTimeout},
+		WaitTimeout: metav1.Duration{Duration: waitTimeout},
+	}
+}
+
+type PodExecRestoreHook struct {
+	HookName   string
+	HookSource string
+	Hook       velerov1api.ExecRestoreHook
+	executed   bool
+}
+
+// GroupRestoreExecHooks returns a list of hooks to be executed in a pod grouped by
+// container name. If an exec hook is defined in annotation that is used, else applicable exec
+// hooks from the restore resource are accumulated.
+func GroupRestoreExecHooks(
+	resourceRestoreHooks []ResourceRestoreHook,
+	pod *corev1api.Pod,
+	log logrus.FieldLogger,
+) (map[string][]PodExecRestoreHook, error) {
+	byContainer := map[string][]PodExecRestoreHook{}
+
+	if pod == nil || len(pod.Spec.Containers) == 0 {
+		return byContainer, nil
+	}
+	metadata, err := meta.Accessor(pod)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+	hookFromAnnotation := getPodExecRestoreHookFromAnnotations(metadata.GetAnnotations(), log)
+	if hookFromAnnotation != nil {
+		// default to first container in pod if unset
+		if hookFromAnnotation.Container == "" {
+			hookFromAnnotation.Container = pod.Spec.Containers[0].Name
+		}
+		byContainer[hookFromAnnotation.Container] = []PodExecRestoreHook{
+			{
+				HookName:   "<from-annotation>",
+				HookSource: "annotation",
+				Hook:       *hookFromAnnotation,
+			},
+		}
+		return byContainer, nil
+	}
+
+	// No hook found on pod's annotations so check for applicable hooks from the restore spec
+	labels := metadata.GetLabels()
+	namespace := metadata.GetNamespace()
+	for _, rrh := range resourceRestoreHooks {
+		if !rrh.Selector.applicableTo(kuberesource.Pods, namespace, labels) {
+			continue
+		}
+		for _, rh := range rrh.RestoreHooks {
+			if rh.Exec == nil {
+				continue
+			}
+			named := PodExecRestoreHook{
+				HookName:   rrh.Name,
+				Hook:       *rh.Exec,
+				HookSource: "backupSpec",
+			}
+			// default to first container in pod if unset, without mutating resource restore hook
+			if named.Hook.Container == "" {
+				named.Hook.Container = pod.Spec.Containers[0].Name
+			}
+			byContainer[named.Hook.Container] = append(byContainer[named.Hook.Container], named)
+		}
+	}
+
+	return byContainer, nil
+}

internal/hook/item_hook_handler_test.go

@@ -710,6 +710,487 @@ func parseLabelSelectorOrDie(s string) labels.Selector {
 	return ret
 }
 
+func TestGetPodExecRestoreHookFromAnnotations(t *testing.T) {
+	testCases := []struct {
+		name             string
+		inputAnnotations map[string]string
+		expected         *velerov1api.ExecRestoreHook
+	}{
+		{
+			name:             "should return nil when command is missing",
+			inputAnnotations: nil,
+			expected:         nil,
+		},
+		{
+			name: "should return nil when command is empty string",
+			inputAnnotations: map[string]string{
+				podRestoreHookCommandAnnotationKey: "",
+			},
+			expected: nil,
+		},
+		{
+			name: "should return a hook when 1 item command is a string",
+			inputAnnotations: map[string]string{
+				podRestoreHookCommandAnnotationKey: "/usr/bin/foo",
+			},
+			expected: &velerov1api.ExecRestoreHook{
+				Command: []string{"/usr/bin/foo"},
+			},
+		},
+		{
+			name: "should return a multi-item hook when command is a json array",
+			inputAnnotations: map[string]string{
+				podRestoreHookCommandAnnotationKey: `["a","b","c"]`,
+			},
+			expected: &velerov1api.ExecRestoreHook{
+				Command: []string{"a", "b", "c"},
+			},
+		},
+		{
+			name: "error mode continue should be in returned hook when set in annotation",
+			inputAnnotations: map[string]string{
+				podRestoreHookCommandAnnotationKey: "/usr/bin/foo",
+				podRestoreHookOnErrorAnnotationKey: string(velerov1api.HookErrorModeContinue),
+			},
+			expected: &velerov1api.ExecRestoreHook{
+				Command: []string{"/usr/bin/foo"},
+				OnError: velerov1api.HookErrorModeContinue,
+			},
+		},
+		{
+			name: "error mode fail should be in returned hook when set in annotation",
+			inputAnnotations: map[string]string{
+				podRestoreHookCommandAnnotationKey: "/usr/bin/foo",
+				podRestoreHookOnErrorAnnotationKey: string(velerov1api.HookErrorModeFail),
+			},
+			expected: &velerov1api.ExecRestoreHook{
+				Command: []string{"/usr/bin/foo"},
+				OnError: velerov1api.HookErrorModeFail,
+			},
+		},
+		{
+			name: "exec and wait timeouts should be in returned hook when set in annotations",
+			inputAnnotations: map[string]string{
+				podRestoreHookCommandAnnotationKey:     "/usr/bin/foo",
+				podRestoreHookTimeoutAnnotationKey:     "45s",
+				podRestoreHookWaitTimeoutAnnotationKey: "1h",
+			},
+			expected: &velerov1api.ExecRestoreHook{
+				Command:     []string{"/usr/bin/foo"},
+				ExecTimeout: metav1.Duration{Duration: 45 * time.Second},
+				WaitTimeout: metav1.Duration{Duration: time.Hour},
+			},
+		},
+		{
+			name: "container should be in returned hook when set in annotation",
+			inputAnnotations: map[string]string{
+				podRestoreHookCommandAnnotationKey:   "/usr/bin/foo",
+				podRestoreHookContainerAnnotationKey: "my-app",
+			},
+			expected: &velerov1api.ExecRestoreHook{
+				Command:   []string{"/usr/bin/foo"},
+				Container: "my-app",
+			},
+		},
+		{
+			name: "bad exec timeout should be discarded",
+			inputAnnotations: map[string]string{
+				podRestoreHookCommandAnnotationKey:   "/usr/bin/foo",
+				podRestoreHookContainerAnnotationKey: "my-app",
+				podRestoreHookTimeoutAnnotationKey:   "none",
+			},
+			expected: &velerov1api.ExecRestoreHook{
+				Command:     []string{"/usr/bin/foo"},
+				Container:   "my-app",
+				ExecTimeout: metav1.Duration{0},
+			},
+		},
+		{
+			name: "bad wait timeout should be discarded",
+			inputAnnotations: map[string]string{
+				podRestoreHookCommandAnnotationKey:     "/usr/bin/foo",
+				podRestoreHookContainerAnnotationKey:   "my-app",
+				podRestoreHookWaitTimeoutAnnotationKey: "none",
+			},
+			expected: &velerov1api.ExecRestoreHook{
+				Command:     []string{"/usr/bin/foo"},
+				Container:   "my-app",
+				ExecTimeout: metav1.Duration{0},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			l := velerotest.NewLogger()
+			actual := getPodExecRestoreHookFromAnnotations(tc.inputAnnotations, l)
+			assert.Equal(t, tc.expected, actual)
+		})
+	}
+}
+
+func TestGroupRestoreExecHooks(t *testing.T) {
+	testCases := []struct {
+		name                 string
+		resourceRestoreHooks []ResourceRestoreHook
+		pod                  *corev1api.Pod
+		expected             map[string][]PodExecRestoreHook
+	}{
+		{
+			name:                 "should return empty map when neither spec hooks nor annotations hooks are set",
+			resourceRestoreHooks: nil,
+			pod:                  builder.ForPod("default", "my-pod").Result(),
+			expected:             map[string][]PodExecRestoreHook{},
+		},
+		{
+			name:                 "should return hook from annotation when no spec hooks are set",
+			resourceRestoreHooks: nil,
+			pod: builder.ForPod("default", "my-pod").
+				ObjectMeta(builder.WithAnnotations(
+					podRestoreHookCommandAnnotationKey, "/usr/bin/foo",
+					podRestoreHookContainerAnnotationKey, "container1",
+					podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue),
+					podRestoreHookTimeoutAnnotationKey, "1s",
+					podRestoreHookWaitTimeoutAnnotationKey, "1m",
+				)).
+				Containers(&corev1api.Container{
+					Name: "container1",
+				}).
+				Result(),
+			expected: map[string][]PodExecRestoreHook{
+				"container1": {
+					{
+						HookName:   "<from-annotation>",
+						HookSource: "annotation",
+						Hook: velerov1api.ExecRestoreHook{
+							Container:   "container1",
+							Command:     []string{"/usr/bin/foo"},
+							OnError:     velerov1api.HookErrorModeContinue,
+							ExecTimeout: metav1.Duration{time.Second},
+							WaitTimeout: metav1.Duration{time.Minute},
+						},
+					},
+				},
+			},
+		},
+		{
+			name:                 "should default to first pod container when not set in annotation",
+			resourceRestoreHooks: nil,
+			pod: builder.ForPod("default", "my-pod").
+				ObjectMeta(builder.WithAnnotations(
+					podRestoreHookCommandAnnotationKey, "/usr/bin/foo",
+					podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue),
+					podRestoreHookTimeoutAnnotationKey, "1s",
+					podRestoreHookWaitTimeoutAnnotationKey, "1m",
+				)).
+				Containers(&corev1api.Container{
+					Name: "container1",
+				}).
+				Result(),
+			expected: map[string][]PodExecRestoreHook{
+				"container1": {
+					{
+						HookName:   "<from-annotation>",
+						HookSource: "annotation",
+						Hook: velerov1api.ExecRestoreHook{
+							Container:   "container1",
+							Command:     []string{"/usr/bin/foo"},
+							OnError:     velerov1api.HookErrorModeContinue,
+							ExecTimeout: metav1.Duration{time.Second},
+							WaitTimeout: metav1.Duration{time.Minute},
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "should return hook from spec for pod with no hook annotations",
+			resourceRestoreHooks: []ResourceRestoreHook{
+				{
+					Name:     "hook1",
+					Selector: ResourceHookSelector{},
+					RestoreHooks: []velerov1api.RestoreResourceHook{
+						{
+							Exec: &velerov1api.ExecRestoreHook{
+								Container:   "container1",
+								Command:     []string{"/usr/bin/foo"},
+								OnError:     velerov1api.HookErrorModeContinue,
+								ExecTimeout: metav1.Duration{time.Second},
+								WaitTimeout: metav1.Duration{time.Minute},
+							},
+						},
+					},
+				},
+			},
+			pod: builder.ForPod("default", "my-pod").
+				Containers(&corev1api.Container{
+					Name: "container1",
+				}).
+				Result(),
+			expected: map[string][]PodExecRestoreHook{
+				"container1": {
+					{
+						HookName:   "hook1",
+						HookSource: "backupSpec",
+						Hook: velerov1api.ExecRestoreHook{
+							Container:   "container1",
+							Command:     []string{"/usr/bin/foo"},
+							OnError:     velerov1api.HookErrorModeContinue,
+							ExecTimeout: metav1.Duration{time.Second},
+							WaitTimeout: metav1.Duration{time.Minute},
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "should default to first container pod when unset in spec hook",
+			resourceRestoreHooks: []ResourceRestoreHook{
+				{
+					Name:     "hook1",
+					Selector: ResourceHookSelector{},
+					RestoreHooks: []velerov1api.RestoreResourceHook{
+						{
+							Exec: &velerov1api.ExecRestoreHook{
+								Command:     []string{"/usr/bin/foo"},
+								OnError:     velerov1api.HookErrorModeContinue,
+								ExecTimeout: metav1.Duration{time.Second},
+								WaitTimeout: metav1.Duration{time.Minute},
+							},
+						},
+					},
+				},
+			},
+			pod: builder.ForPod("default", "my-pod").
+				Containers(&corev1api.Container{
+					Name: "container1",
+				}).
+				Result(),
+			expected: map[string][]PodExecRestoreHook{
+				"container1": {
+					{
+						HookName:   "hook1",
+						HookSource: "backupSpec",
+						Hook: velerov1api.ExecRestoreHook{
+							Container:   "container1",
+							Command:     []string{"/usr/bin/foo"},
+							OnError:     velerov1api.HookErrorModeContinue,
+							ExecTimeout: metav1.Duration{time.Second},
+							WaitTimeout: metav1.Duration{time.Minute},
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "should return hook from annotation ignoring hooks in spec",
+			resourceRestoreHooks: []ResourceRestoreHook{
+				{
+					Name:     "hook1",
+					Selector: ResourceHookSelector{},
+					RestoreHooks: []velerov1api.RestoreResourceHook{
+						{
+							Exec: &velerov1api.ExecRestoreHook{
+								Container:   "container2",
+								Command:     []string{"/usr/bin/bar"},
+								OnError:     velerov1api.HookErrorModeFail,
+								ExecTimeout: metav1.Duration{time.Hour},
+								WaitTimeout: metav1.Duration{time.Hour},
+							},
+						},
+					},
+				},
+			},
+			pod: builder.ForPod("default", "my-pod").
+				ObjectMeta(builder.WithAnnotations(
+					podRestoreHookCommandAnnotationKey, "/usr/bin/foo",
+					podRestoreHookContainerAnnotationKey, "container1",
+					podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue),
+					podRestoreHookTimeoutAnnotationKey, "1s",
+					podRestoreHookWaitTimeoutAnnotationKey, "1m",
+				)).
+				Containers(&corev1api.Container{
+					Name: "container1",
+				}).
+				Result(),
+			expected: map[string][]PodExecRestoreHook{
+				"container1": {
+					{
+						HookName:   "<from-annotation>",
+						HookSource: "annotation",
+						Hook: velerov1api.ExecRestoreHook{
+							Container:   "container1",
+							Command:     []string{"/usr/bin/foo"},
+							OnError:     velerov1api.HookErrorModeContinue,
+							ExecTimeout: metav1.Duration{time.Second},
+							WaitTimeout: metav1.Duration{time.Minute},
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "should return empty map when only has init hook and pod has no hook annotations",
+			resourceRestoreHooks: []ResourceRestoreHook{
+				{
+					Name:     "hook1",
+					Selector: ResourceHookSelector{},
+					RestoreHooks: []velerov1api.RestoreResourceHook{
+						{
+							Init: &velerov1api.InitRestoreHook{},
+						},
+					},
+				},
+			},
+			pod: builder.ForPod("default", "my-pod").
+				Containers(&corev1api.Container{
+					Name: "container1",
+				}).
+				Result(),
+			expected: map[string][]PodExecRestoreHook{},
+		},
+		{
+			name: "should return empty map when spec has exec hook for pod in different namespace and pod has no hook annotations",
+			resourceRestoreHooks: []ResourceRestoreHook{
+				{
+					Name: "hook1",
+					Selector: ResourceHookSelector{
+						Namespaces: collections.NewIncludesExcludes().Includes("other"),
+					},
+					RestoreHooks: []velerov1api.RestoreResourceHook{
+						{
+							Exec: &velerov1api.ExecRestoreHook{
+								Container:   "container1",
+								Command:     []string{"/usr/bin/foo"},
+								OnError:     velerov1api.HookErrorModeContinue,
+								ExecTimeout: metav1.Duration{time.Second},
+								WaitTimeout: metav1.Duration{time.Minute},
+							},
+						},
+					},
+				},
+			},
+			pod:      builder.ForPod("default", "my-pod").Result(),
+			expected: map[string][]PodExecRestoreHook{},
+		},
+		{
+			name: "should return map with multiple keys when spec hooks apply to multiple containers in pod and has no hook annotations",
+			resourceRestoreHooks: []ResourceRestoreHook{
+				{
+					Name:     "hook1",
+					Selector: ResourceHookSelector{},
+					RestoreHooks: []velerov1api.RestoreResourceHook{
+						{
+							Exec: &velerov1api.ExecRestoreHook{
+								Container:   "container1",
+								Command:     []string{"/usr/bin/foo"},
+								OnError:     velerov1api.HookErrorModeFail,
+								ExecTimeout: metav1.Duration{time.Second},
+								WaitTimeout: metav1.Duration{time.Minute},
+							},
+						},
+						{
+							Exec: &velerov1api.ExecRestoreHook{
+								Container:   "container2",
+								Command:     []string{"/usr/bin/baz"},
+								OnError:     velerov1api.HookErrorModeContinue,
+								ExecTimeout: metav1.Duration{time.Second * 3},
+								WaitTimeout: metav1.Duration{time.Second * 3},
+							},
+						},
+						{
+							Exec: &velerov1api.ExecRestoreHook{
+								Container:   "container1",
+								Command:     []string{"/usr/bin/bar"},
+								OnError:     velerov1api.HookErrorModeContinue,
+								ExecTimeout: metav1.Duration{time.Second * 2},
+								WaitTimeout: metav1.Duration{time.Minute * 2},
+							},
+						},
+					},
+				},
+				{
+					Name:     "hook2",
+					Selector: ResourceHookSelector{},
+					RestoreHooks: []velerov1api.RestoreResourceHook{
+						{
+							Exec: &velerov1api.ExecRestoreHook{
+								Container:   "container1",
+								Command:     []string{"/usr/bin/aaa"},
+								OnError:     velerov1api.HookErrorModeContinue,
+								ExecTimeout: metav1.Duration{time.Second * 4},
+								WaitTimeout: metav1.Duration{time.Minute * 4},
+							},
+						},
+					},
+				},
+			},
+			pod: builder.ForPod("default", "my-pod").
+				Containers(&corev1api.Container{
+					Name: "container1",
+				}).
+				Result(),
+			expected: map[string][]PodExecRestoreHook{
+				"container1": {
+					{
+						HookName:   "hook1",
+						HookSource: "backupSpec",
+						Hook: velerov1api.ExecRestoreHook{
+							Container:   "container1",
+							Command:     []string{"/usr/bin/foo"},
+							OnError:     velerov1api.HookErrorModeFail,
+							ExecTimeout: metav1.Duration{time.Second},
+							WaitTimeout: metav1.Duration{time.Minute},
+						},
+					},
+					{
+						HookName:   "hook1",
+						HookSource: "backupSpec",
+						Hook: velerov1api.ExecRestoreHook{
+							Container:   "container1",
+							Command:     []string{"/usr/bin/bar"},
+							OnError:     velerov1api.HookErrorModeContinue,
+							ExecTimeout: metav1.Duration{time.Second * 2},
+							WaitTimeout: metav1.Duration{time.Minute * 2},
+						},
+					},
+					{
+						HookName:   "hook2",
+						HookSource: "backupSpec",
+						Hook: velerov1api.ExecRestoreHook{
+							Container:   "container1",
+							Command:     []string{"/usr/bin/aaa"},
+							OnError:     velerov1api.HookErrorModeContinue,
+							ExecTimeout: metav1.Duration{time.Second * 4},
+							WaitTimeout: metav1.Duration{time.Minute * 4},
+						},
+					},
+				},
+				"container2": {
+					{
+						HookName:   "hook1",
+						HookSource: "backupSpec",
+						Hook: velerov1api.ExecRestoreHook{
+							Container:   "container2",
+							Command:     []string{"/usr/bin/baz"},
+							OnError:     velerov1api.HookErrorModeContinue,
+							ExecTimeout: metav1.Duration{time.Second * 3},
+							WaitTimeout: metav1.Duration{time.Second * 3},
+						},
+					},
+				},
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			actual, err := GroupRestoreExecHooks(tc.resourceRestoreHooks, tc.pod, velerotest.NewLogger())
+			assert.Nil(t, err)
+			assert.Equal(t, tc.expected, actual)
+		})
+	}
+}
+
 func TestGetInitRestoreHookFromAnnotations(t *testing.T) {
 	testCases := []struct {
 		name             string

internal/hook/wait_exec_hook_handler.go

@@ -0,0 +1,275 @@
+/*
+Copyright 2020 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hook
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/cache"
+
+	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+	"github.com/vmware-tanzu/velero/pkg/podexec"
+	"github.com/vmware-tanzu/velero/pkg/util/kube"
+)
+
+type WaitExecHookHandler interface {
+	HandleHooks(
+		ctx context.Context,
+		log logrus.FieldLogger,
+		pod *v1.Pod,
+		byContainer map[string][]PodExecRestoreHook,
+	) []error
+}
+
+type ListWatchFactory interface {
+	NewListWatch(namespace string, selector fields.Selector) cache.ListerWatcher
+}
+
+type DefaultListWatchFactory struct {
+	PodsGetter cache.Getter
+}
+
+func (d *DefaultListWatchFactory) NewListWatch(namespace string, selector fields.Selector) cache.ListerWatcher {
+	return cache.NewListWatchFromClient(d.PodsGetter, "pods", namespace, selector)
+}
+
+var _ ListWatchFactory = &DefaultListWatchFactory{}
+
+type DefaultWaitExecHookHandler struct {
+	ListWatchFactory   ListWatchFactory
+	PodCommandExecutor podexec.PodCommandExecutor
+}
+
+var _ WaitExecHookHandler = &DefaultWaitExecHookHandler{}
+
+func (e *DefaultWaitExecHookHandler) HandleHooks(
+	ctx context.Context,
+	log logrus.FieldLogger,
+	pod *v1.Pod,
+	byContainer map[string][]PodExecRestoreHook,
+) []error {
+	if pod == nil {
+		return nil
+	}
+
+	// If hooks are defined for a container that does not exist in the pod log a warning and discard
+	// those hooks to avoid waiting for a container that will never become ready. After that if
+	// there are no hooks left to be executed return immediately.
+	for containerName := range byContainer {
+		if !podHasContainer(pod, containerName) {
+			log.Warningf("Pod %s does not have container %s: discarding post-restore exec hooks", kube.NamespaceAndName(pod), containerName)
+			delete(byContainer, containerName)
+		}
+	}
+	if len(byContainer) == 0 {
+		return nil
+	}
+
+	// Every hook in every container can have its own wait timeout. Rather than setting up separate
+	// contexts for each, find the largest wait timeout for any hook that should be executed in
+	// the pod and watch the pod for up to that long. Before executing any hook in a container,
+	// check if that hook has a timeout and skip execution if expired.
+	ctx, cancel := context.WithCancel(ctx)
+	maxWait := maxHookWait(byContainer)
+	// If no hook has a wait timeout then this function will continue waiting for containers to
+	// become ready until the shared hook context is canceled.
+	if maxWait > 0 {
+		ctx, cancel = context.WithTimeout(ctx, maxWait)
+	}
+	waitStart := time.Now()
+
+	var errors []error
+
+	// The first time this handler is called after a container starts running it will execute all
+	// pending hooks for that container. Subsequent invocations of this handler will never execute
+	// hooks in that container. It uses the byContainer map to keep track of which containers have
+	// not yet been observed to be running. It relies on the Informer not to be called concurrently.
+	// When a container is observed running and its hooks are executed, the container is deleted
+	// from the byContainer map. When the map is empty the watch is ended.
+	handler := func(newObj interface{}) {
+		newPod, ok := newObj.(*v1.Pod)
+		if !ok {
+			return
+		}
+
+		podLog := log.WithFields(
+			logrus.Fields{
+				"pod": kube.NamespaceAndName(newPod),
+			},
+		)
+
+		if newPod.Status.Phase == v1.PodSucceeded || newPod.Status.Phase == v1.PodFailed {
+			err := fmt.Errorf("Pod entered phase %s before some post-restore exec hooks ran", newPod.Status.Phase)
+			podLog.Warning(err)
+			cancel()
+			return
+		}
+
+		for containerName, hooks := range byContainer {
+			if !isContainerRunning(newPod, containerName) {
+				podLog.Infof("Container %s is not running: post-restore hooks will not yet be executed", containerName)
+				continue
+			}
+			podMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(newPod)
+			if err != nil {
+				podLog.WithError(err).Error("error unstructuring pod")
+				cancel()
+				return
+			}
+
+			// Sequentially run all hooks for the ready container. The container's hooks are not
+			// removed from the byContainer map until all have completed so that if one fails
+			// remaining unexecuted hooks can be handled by the outer function.
+			for i, hook := range hooks {
+				// This indicates to the outer function not to handle this hook as unexecuted in
+				// case of terminating before deleting this container's slice of hooks from the
+				// byContainer map.
+				byContainer[containerName][i].executed = true
+
+				hookLog := podLog.WithFields(
+					logrus.Fields{
+						"hookSource": hook.HookSource,
+						"hookType":   "exec",
+						"hookPhase":  "post",
+					},
+				)
+				// Check the individual hook's wait timeout is not expired
+				if hook.Hook.WaitTimeout.Duration != 0 && time.Since(waitStart) > hook.Hook.WaitTimeout.Duration {
+					err := fmt.Errorf("Hook %s in container %s expired before executing", hook.HookName, hook.Hook.Container)
+					hookLog.Error(err)
+					if hook.Hook.OnError == velerov1api.HookErrorModeFail {
+						errors = append(errors, err)
+						cancel()
+						return
+					}
+				}
+				eh := &velerov1api.ExecHook{
+					Container: hook.Hook.Container,
+					Command:   hook.Hook.Command,
+					OnError:   hook.Hook.OnError,
+					Timeout:   hook.Hook.ExecTimeout,
+				}
+				if err := e.PodCommandExecutor.ExecutePodCommand(hookLog, podMap, pod.Namespace, pod.Name, hook.HookName, eh); err != nil {
+					hookLog.WithError(err).Error("Error executing hook")
+					if hook.Hook.OnError == velerov1api.HookErrorModeFail {
+						errors = append(errors, err)
+						cancel()
+						return
+					}
+				}
+			}
+			delete(byContainer, containerName)
+		}
+		if len(byContainer) == 0 {
+			cancel()
+		}
+	}
+
+	selector := fields.OneTermEqualSelector("metadata.name", pod.Name)
+	lw := e.ListWatchFactory.NewListWatch(pod.Namespace, selector)
+
+	_, podWatcher := cache.NewInformer(lw, pod, 0, cache.ResourceEventHandlerFuncs{
+		AddFunc: handler,
+		UpdateFunc: func(_, newObj interface{}) {
+			handler(newObj)
+		},
+		DeleteFunc: func(obj interface{}) {
+			err := fmt.Errorf("Pod %s deleted before all hooks were executed", kube.NamespaceAndName(pod))
+			log.Error(err)
+			cancel()
+		},
+	})
+
+	podWatcher.Run(ctx.Done())
+
+	// There are some cases where this function could return with unexecuted hooks: the pod may
+	// be deleted, a hook with OnError mode Fail could fail, or it may timeout waiting for
+	// containers to become ready.
+	// Each unexecuted hook is logged as an error but only hooks with OnError mode Fail return
+	// an error from this function.
+	for _, hooks := range byContainer {
+		for _, hook := range hooks {
+			if hook.executed {
+				continue
+			}
+			err := fmt.Errorf("Hook %s in container %s in pod %s not executed: %v", hook.HookName, hook.Hook.Container, kube.NamespaceAndName(pod), ctx.Err())
+			hookLog := log.WithFields(
+				logrus.Fields{
+					"hookSource": hook.HookSource,
+					"hookType":   "exec",
+					"hookPhase":  "post",
+				},
+			)
+			hookLog.Error(err)
+			if hook.Hook.OnError == velerov1api.HookErrorModeFail {
+				errors = append(errors, err)
+			}
+		}
+	}
+
+	return errors
+}
+
+func podHasContainer(pod *v1.Pod, containerName string) bool {
+	if pod == nil {
+		return false
+	}
+	for _, c := range pod.Spec.Containers {
+		if c.Name == containerName {
+			return true
+		}
+	}
+
+	return false
+}
+
+func isContainerRunning(pod *v1.Pod, containerName string) bool {
+	if pod == nil {
+		return false
+	}
+	for _, cs := range pod.Status.ContainerStatuses {
+		if cs.Name != containerName {
+			continue
+		}
+		return cs.State.Running != nil
+	}
+
+	return false
+}
+
+// maxHookWait returns 0 to mean wait indefinitely. Any hook without a wait timeout will cause this
+// function to return 0.
+func maxHookWait(byContainer map[string][]PodExecRestoreHook) time.Duration {
+	var maxWait time.Duration
+	for _, hooks := range byContainer {
+		for _, hook := range hooks {
+			if hook.Hook.WaitTimeout.Duration <= 0 {
+				return 0
+			}
+			if hook.Hook.WaitTimeout.Duration > maxWait {
+				maxWait = hook.Hook.WaitTimeout.Duration
+			}
+		}
+	}
+	return maxWait
+}

internal/hook/wait_exec_hook_handler_test.go

@@ -0,0 +1,949 @@
+/*
+Copyright 2020 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hook
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/tools/cache"
+	fcache "k8s.io/client-go/tools/cache/testing"
+
+	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+	"github.com/vmware-tanzu/velero/pkg/builder"
+	velerotest "github.com/vmware-tanzu/velero/pkg/test"
+)
+
+type fakeListWatchFactory struct {
+	lw cache.ListerWatcher
+}
+
+func (f *fakeListWatchFactory) NewListWatch(ns string, selector fields.Selector) cache.ListerWatcher {
+	return f.lw
+}
+
+var _ ListWatchFactory = &fakeListWatchFactory{}
+
+func TestWaitExecHandleHooks(t *testing.T) {
+	type change struct {
+		// delta to wait since last change applied or pod added
+		wait    time.Duration
+		updated *v1.Pod
+	}
+	type expectedExecution struct {
+		hook  *velerov1api.ExecHook
+		name  string
+		error error
+		pod   *v1.Pod
+	}
+	tests := []struct {
+		name string
+		// Used as argument to HandleHooks and first state added to ListerWatcher
+		initialPod         *v1.Pod
+		groupResource      string
+		byContainer        map[string][]PodExecRestoreHook
+		expectedExecutions []expectedExecution
+		expectedErrors     []error
+		// changes represents the states of the pod over time. It can be used to test a container
+		// becoming ready at some point after it is first observed by the controller.
+		changes                   []change
+		sharedHooksContextTimeout time.Duration
+	}{
+		{
+			name: "should return no error when hook from annotation executes successfully",
+			initialPod: builder.ForPod("default", "my-pod").
+				ObjectMeta(builder.WithAnnotations(
+					podRestoreHookCommandAnnotationKey, "/usr/bin/foo",
+					podRestoreHookContainerAnnotationKey, "container1",
+					podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue),
+					podRestoreHookTimeoutAnnotationKey, "1s",
+					podRestoreHookWaitTimeoutAnnotationKey, "1m",
+				)).
+				Containers(&v1.Container{
+					Name: "container1",
+				}).
+				ContainerStatuses(&v1.ContainerStatus{
+					Name: "container1",
+					State: v1.ContainerState{
+						Running: &v1.ContainerStateRunning{},
+					},
+				}).
+				Result(),
+			groupResource: "pods",
+			byContainer: map[string][]PodExecRestoreHook{
+				"container1": {
+					{
+						HookName:   "<from-annotation>",
+						HookSource: "annotation",
+						Hook: velerov1api.ExecRestoreHook{
+							Container:   "container1",
+							Command:     []string{"/usr/bin/foo"},
+							OnError:     velerov1api.HookErrorModeContinue,
+							ExecTimeout: metav1.Duration{time.Second},
+							WaitTimeout: metav1.Duration{time.Minute},
+						},
+					},
+				},
+			},
+			expectedExecutions: []expectedExecution{
+				{
+					name: "<from-annotation>",
+					hook: &velerov1api.ExecHook{
+						Container: "container1",
+						Command:   []string{"/usr/bin/foo"},
+						OnError:   velerov1api.HookErrorModeContinue,
+						Timeout:   metav1.Duration{time.Second},
+					},
+					error: nil,
+					pod: builder.ForPod("default", "my-pod").
+						ObjectMeta(builder.WithResourceVersion("1")).
+						ObjectMeta(builder.WithAnnotations(
+							podRestoreHookCommandAnnotationKey, "/usr/bin/foo",
+							podRestoreHookContainerAnnotationKey, "container1",
+							podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue),
+							podRestoreHookTimeoutAnnotationKey, "1s",
+							podRestoreHookWaitTimeoutAnnotationKey, "1m",
+						)).
+						Containers(&v1.Container{
+							Name: "container1",
+						}).
+						ContainerStatuses(&v1.ContainerStatus{
+							Name: "container1",
+							State: v1.ContainerState{
+								Running: &v1.ContainerStateRunning{},
+							},
+						}).
+						Result(),
+				},
+			},
+			expectedErrors: nil,
+		},
+		{
+			name: "should return an error when hook from annotation fails with on error mode fail",
+			initialPod: builder.ForPod("default", "my-pod").
+				ObjectMeta(builder.WithAnnotations(
+					podRestoreHookCommandAnnotationKey, "/usr/bin/foo",
+					podRestoreHookContainerAnnotationKey, "container1",
+					podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeFail),
+					podRestoreHookTimeoutAnnotationKey, "1s",
+					podRestoreHookWaitTimeoutAnnotationKey, "1m",
+				)).
+				Containers(&v1.Container{
+					Name: "container1",
+				}).
+				ContainerStatuses(&v1.ContainerStatus{
+					Name: "container1",
+					State: v1.ContainerState{
+						Running: &v1.ContainerStateRunning{},
+					},
+				}).
+				Result(),
+			groupResource: "pods",
+			byContainer: map[string][]PodExecRestoreHook{
+				"container1": {
+					{
+						HookName:   "<from-annotation>",
+						HookSource: "annotation",
+						Hook: velerov1api.ExecRestoreHook{
+							Container:   "container1",
+							Command:     []string{"/usr/bin/foo"},
+							OnError:     velerov1api.HookErrorModeFail,
+							ExecTimeout: metav1.Duration{time.Second},
+							WaitTimeout: metav1.Duration{time.Minute},
+						},
+					},
+				},
+			},
+			expectedExecutions: []expectedExecution{
+				{
+					name: "<from-annotation>",
+					hook: &velerov1api.ExecHook{
+						Container: "container1",
+						Command:   []string{"/usr/bin/foo"},
+						OnError:   velerov1api.HookErrorModeFail,
+						Timeout:   metav1.Duration{time.Second},
+					},
+					error: errors.New("pod hook error"),
+					pod: builder.ForPod("default", "my-pod").
+						ObjectMeta(builder.WithResourceVersion("1")).
+						ObjectMeta(builder.WithAnnotations(
+							podRestoreHookCommandAnnotationKey, "/usr/bin/foo",
+							podRestoreHookContainerAnnotationKey, "container1",
+							podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeFail),
+							podRestoreHookTimeoutAnnotationKey, "1s",
+							podRestoreHookWaitTimeoutAnnotationKey, "1m",
+						)).
+						Containers(&v1.Container{
+							Name: "container1",
+						}).
+						ContainerStatuses(&v1.ContainerStatus{
+							Name: "container1",
+							State: v1.ContainerState{
+								Running: &v1.ContainerStateRunning{},
+							},
+						}).
+						Result(),
+				},
+			},
+			expectedErrors: []error{errors.New("pod hook error")},
+		},
+		{
+			name: "should return no error when hook from annotation fails with on error mode continue",
+			initialPod: builder.ForPod("default", "my-pod").
+				ObjectMeta(builder.WithAnnotations(
+					podRestoreHookCommandAnnotationKey, "/usr/bin/foo",
+					podRestoreHookContainerAnnotationKey, "container1",
+					podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue),
+					podRestoreHookTimeoutAnnotationKey, "1s",
+					podRestoreHookWaitTimeoutAnnotationKey, "1m",
+				)).
+				Containers(&v1.Container{
+					Name: "container1",
+				}).
+				ContainerStatuses(&v1.ContainerStatus{
+					Name: "container1",
+					State: v1.ContainerState{
+						Running: &v1.ContainerStateRunning{},
+					},
+				}).
+				Result(),
+			groupResource: "pods",
+			byContainer: map[string][]PodExecRestoreHook{
+				"container1": {
+					{
+						HookName:   "<from-annotation>",
+						HookSource: "annotation",
+						Hook: velerov1api.ExecRestoreHook{
+							Container:   "container1",
+							Command:     []string{"/usr/bin/foo"},
+							OnError:     velerov1api.HookErrorModeContinue,
+							ExecTimeout: metav1.Duration{time.Second},
+							WaitTimeout: metav1.Duration{time.Minute},
+						},
+					},
+				},
+			},
+			expectedExecutions: []expectedExecution{
+				{
+					name: "<from-annotation>",
+					hook: &velerov1api.ExecHook{
+						Container: "container1",
+						Command:   []string{"/usr/bin/foo"},
+						OnError:   velerov1api.HookErrorModeContinue,
+						Timeout:   metav1.Duration{time.Second},
+					},
+					error: errors.New("pod hook error"),
+					pod: builder.ForPod("default", "my-pod").
+						ObjectMeta(builder.WithResourceVersion("1")).
+						ObjectMeta(builder.WithAnnotations(
+							podRestoreHookCommandAnnotationKey, "/usr/bin/foo",
+							podRestoreHookContainerAnnotationKey, "container1",
+							podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue),
+							podRestoreHookTimeoutAnnotationKey, "1s",
+							podRestoreHookWaitTimeoutAnnotationKey, "1m",
+						)).
+						Containers(&v1.Container{
+							Name: "container1",
+						}).
+						ContainerStatuses(&v1.ContainerStatus{
+							Name: "container1",
+							State: v1.ContainerState{
+								Running: &v1.ContainerStateRunning{},
+							},
+						}).
+						Result(),
+				},
+			},
+			expectedErrors: nil,
+		},
+		{
+			name: "should return no error when hook from annotation executes after 10ms wait for container to start",
+			initialPod: builder.ForPod("default", "my-pod").
+				ObjectMeta(builder.WithAnnotations(
+					podRestoreHookCommandAnnotationKey, "/usr/bin/foo",
+					podRestoreHookContainerAnnotationKey, "container1",
+					podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue),
+					podRestoreHookTimeoutAnnotationKey, "1s",
+					podRestoreHookWaitTimeoutAnnotationKey, "1m",
+				)).
+				Containers(&v1.Container{
+					Name: "container1",
+				}).
+				ContainerStatuses(&v1.ContainerStatus{
+					Name: "container1",
+					State: v1.ContainerState{
+						Waiting: &v1.ContainerStateWaiting{},
+					},
+				}).
+				Result(),
+			groupResource: "pods",
+			byContainer: map[string][]PodExecRestoreHook{
+				"container1": {
+					{
+						HookName:   "<from-annotation>",
+						HookSource: "annotation",
+						Hook: velerov1api.ExecRestoreHook{
+							Container:   "container1",
+							Command:     []string{"/usr/bin/foo"},
+							OnError:     velerov1api.HookErrorModeContinue,
+							ExecTimeout: metav1.Duration{time.Second},
+							WaitTimeout: metav1.Duration{time.Minute},
+						},
+					},
+				},
+			},
+			expectedExecutions: []expectedExecution{
+				{
+					name: "<from-annotation>",
+					hook: &velerov1api.ExecHook{
+						Container: "container1",
+						Command:   []string{"/usr/bin/foo"},
+						OnError:   velerov1api.HookErrorModeContinue,
+						Timeout:   metav1.Duration{time.Second},
+					},
+					error: nil,
+					pod: builder.ForPod("default", "my-pod").
+						ObjectMeta(builder.WithResourceVersion("2")).
+						ObjectMeta(builder.WithAnnotations(
+							podRestoreHookCommandAnnotationKey, "/usr/bin/foo",
+							podRestoreHookContainerAnnotationKey, "container1",
+							podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue),
+							podRestoreHookTimeoutAnnotationKey, "1s",
+							podRestoreHookWaitTimeoutAnnotationKey, "1m",
+						)).
+						Containers(&v1.Container{
+							Name: "container1",
+						}).
+						ContainerStatuses(&v1.ContainerStatus{
+							Name: "container1",
+							State: v1.ContainerState{
+								Running: &v1.ContainerStateRunning{},
+							},
+						}).
+						Result(),
+				},
+			},
+			expectedErrors: nil,
+			changes: []change{
+				{
+					wait: 10 * time.Millisecond,
+					updated: builder.ForPod("default", "my-pod").
+						ObjectMeta(builder.WithAnnotations(
+							podRestoreHookCommandAnnotationKey, "/usr/bin/foo",
+							podRestoreHookContainerAnnotationKey, "container1",
+							podRestoreHookOnErrorAnnotationKey, string(velerov1api.HookErrorModeContinue),
+							podRestoreHookTimeoutAnnotationKey, "1s",
+							podRestoreHookWaitTimeoutAnnotationKey, "1m",
+						)).
+						Containers(&v1.Container{
+							Name: "container1",
+						}).
+						ContainerStatuses(&v1.ContainerStatus{
+							Name: "container1",
+							State: v1.ContainerState{
+								Running: &v1.ContainerStateRunning{},
+							},
+						}).
+						Result(),
+				},
+			},
+		},
+		{
+			name:          "should return no error when hook from spec executes successfully",
+			groupResource: "pods",
+			initialPod: builder.ForPod("default", "my-pod").
+				Containers(&v1.Container{
+					Name: "container1",
+				}).
+				ContainerStatuses(&v1.ContainerStatus{
+					Name: "container1",
+					State: v1.ContainerState{
+						Running: &v1.ContainerStateRunning{},
+					},
+				}).
+				Result(),
+			expectedErrors: nil,
+			byContainer: map[string][]PodExecRestoreHook{
+				"container1": {
+					{
+						HookName:   "my-hook-1",
+						HookSource: "backupSpec",
+						Hook: velerov1api.ExecRestoreHook{
+							Container: "container1",
+							Command:   []string{"/usr/bin/foo"},
+						},
+					},
+				},
+			},
+			expectedExecutions: []expectedExecution{
+				{
+					name: "my-hook-1",
+					hook: &velerov1api.ExecHook{
+						Container: "container1",
+						Command:   []string{"/usr/bin/foo"},
+					},
+					pod: builder.ForPod("default", "my-pod").
+						ObjectMeta(builder.WithResourceVersion("1")).
+						Containers(&v1.Container{
+							Name: "container1",
+						}).
+						ContainerStatuses(&v1.ContainerStatus{
+							Name: "container1",
+							State: v1.ContainerState{
+								Running: &v1.ContainerStateRunning{},
+							},
+						}).
+						Result(),
+				},
+			},
+		},
+		{
+			name:          "should return no error when spec hook with wait timeout expires with OnError mode Continue",
+			groupResource: "pods",
+			initialPod: builder.ForPod("default", "my-pod").
+				Containers(&v1.Container{
+					Name: "container1",
+				}).
+				ContainerStatuses(&v1.ContainerStatus{
+					Name: "container1",
+					State: v1.ContainerState{
+						Waiting: &v1.ContainerStateWaiting{},
+					},
+				}).
+				Result(),
+			expectedErrors: nil,
+			byContainer: map[string][]PodExecRestoreHook{
+				"container1": {
+					{
+						HookName:   "my-hook-1",
+						HookSource: "backupSpec",
+						Hook: velerov1api.ExecRestoreHook{
+							Container:   "container1",
+							Command:     []string{"/usr/bin/foo"},
+							OnError:     velerov1api.HookErrorModeContinue,
+							WaitTimeout: metav1.Duration{time.Millisecond},
+						},
+					},
+				},
+			},
+			expectedExecutions: []expectedExecution{},
+		},
+		{
+			name:          "should return an error when spec hook with wait timeout expires with OnError mode Fail",
+			groupResource: "pods",
+			initialPod: builder.ForPod("default", "my-pod").
+				Containers(&v1.Container{
+					Name: "container1",
+				}).
+				ContainerStatuses(&v1.ContainerStatus{
+					Name: "container1",
+					State: v1.ContainerState{
+						Waiting: &v1.ContainerStateWaiting{},
+					},
+				}).
+				Result(),
+			expectedErrors: []error{errors.New("Hook my-hook-1 in container container1 in pod default/my-pod not executed: context deadline exceeded")},
+			byContainer: map[string][]PodExecRestoreHook{
+				"container1": {
+					{
+						HookName:   "my-hook-1",
+						HookSource: "backupSpec",
+						Hook: velerov1api.ExecRestoreHook{
+							Container:   "container1",
+							Command:     []string{"/usr/bin/foo"},
+							OnError:     velerov1api.HookErrorModeFail,
+							WaitTimeout: metav1.Duration{time.Millisecond},
+						},
+					},
+				},
+			},
+			expectedExecutions: []expectedExecution{},
+		},
+		{
+			name:          "should return an error when shared hooks context is canceled before spec hook with OnError mode Fail executes",
+			groupResource: "pods",
+			initialPod: builder.ForPod("default", "my-pod").
+				Containers(&v1.Container{
+					Name: "container1",
+				}).
+				ContainerStatuses(&v1.ContainerStatus{
+					Name: "container1",
+					State: v1.ContainerState{
+						Waiting: &v1.ContainerStateWaiting{},
+					},
+				}).
+				Result(),
+			expectedErrors: []error{errors.New("Hook my-hook-1 in container container1 in pod default/my-pod not executed: context deadline exceeded")},
+			byContainer: map[string][]PodExecRestoreHook{
+				"container1": {
+					{
+						HookName:   "my-hook-1",
+						HookSource: "backupSpec",
+						Hook: velerov1api.ExecRestoreHook{
+							Container: "container1",
+							Command:   []string{"/usr/bin/foo"},
+							OnError:   velerov1api.HookErrorModeFail,
+						},
+					},
+				},
+			},
+			expectedExecutions:        []expectedExecution{},
+			sharedHooksContextTimeout: time.Millisecond,
+		},
+		{
+			name:           "should return no error when shared hooks context is canceled before spec hook with OnError mode Continue executes",
+			expectedErrors: nil,
+			groupResource:  "pods",
+			initialPod: builder.ForPod("default", "my-pod").
+				Containers(&v1.Container{
+					Name: "container1",
+				}).
+				ContainerStatuses(&v1.ContainerStatus{
+					Name: "container1",
+					State: v1.ContainerState{
+						Waiting: &v1.ContainerStateWaiting{},
+					},
+				}).
+				Result(),
+			byContainer: map[string][]PodExecRestoreHook{
+				"container1": {
+					{
+						HookName:   "my-hook-1",
+						HookSource: "backupSpec",
+						Hook: velerov1api.ExecRestoreHook{
+							Container: "container1",
+							Command:   []string{"/usr/bin/foo"},
+							OnError:   velerov1api.HookErrorModeContinue,
+						},
+					},
+				},
+			},
+			expectedExecutions:        []expectedExecution{},
+			sharedHooksContextTimeout: time.Millisecond,
+		},
+		{
+			name:          "shoudl return no error with 2 spec hooks in 2 different containers, 1st container starts running after 10ms, 2nd container after 20ms, both succeed",
+			groupResource: "pods",
+			initialPod: builder.ForPod("default", "my-pod").
+				Containers(&v1.Container{
+					Name: "container1",
+				}).
+				Containers(&v1.Container{
+					Name: "container2",
+				}).
+				// initially both are waiting
+				ContainerStatuses(&v1.ContainerStatus{
+					Name: "container1",
+					State: v1.ContainerState{
+						Waiting: &v1.ContainerStateWaiting{},
+					},
+				}).
+				ContainerStatuses(&v1.ContainerStatus{
+					Name: "container2",
+					State: v1.ContainerState{
+						Waiting: &v1.ContainerStateWaiting{},
+					},
+				}).
+				Result(),
+			expectedErrors: nil,
+			byContainer: map[string][]PodExecRestoreHook{
+				"container1": {
+					{
+						HookName:   "my-hook-1",
+						HookSource: "backupSpec",
+						Hook: velerov1api.ExecRestoreHook{
+							Container: "container1",
+							Command:   []string{"/usr/bin/foo"},
+						},
+					},
+				},
+				"container2": {
+					{
+						HookName:   "my-hook-1",
+						HookSource: "backupSpec",
+						Hook: velerov1api.ExecRestoreHook{
+							Container: "container2",
+							Command:   []string{"/usr/bin/bar"},
+						},
+					},
+				},
+			},
+			expectedExecutions: []expectedExecution{
+				{
+					name: "my-hook-1",
+					hook: &velerov1api.ExecHook{
+						Container: "container1",
+						Command:   []string{"/usr/bin/foo"},
+					},
+					error: nil,
+					pod: builder.ForPod("default", "my-pod").
+						ObjectMeta(builder.WithResourceVersion("2")).
+						Containers(&v1.Container{
+							Name: "container1",
+						}).
+						Containers(&v1.Container{
+							Name: "container2",
+						}).
+						ContainerStatuses(&v1.ContainerStatus{
+							Name: "container1",
+							State: v1.ContainerState{
+								Running: &v1.ContainerStateRunning{},
+							},
+						}).
+						// container 2 is still waiting when the first hook executes in container1
+						ContainerStatuses(&v1.ContainerStatus{
+							Name: "container2",
+							State: v1.ContainerState{
+								Waiting: &v1.ContainerStateWaiting{},
+							},
+						}).
+						Result(),
+				},
+				{
+					name: "my-hook-1",
+					hook: &velerov1api.ExecHook{
+						Container: "container2",
+						Command:   []string{"/usr/bin/bar"},
+					},
+					error: nil,
+					pod: builder.ForPod("default", "my-pod").
+						ObjectMeta(builder.WithResourceVersion("3")).
+						Containers(&v1.Container{
+							Name: "container1",
+						}).
+						Containers(&v1.Container{
+							Name: "container2",
+						}).
+						ContainerStatuses(&v1.ContainerStatus{
+							Name: "container1",
+							State: v1.ContainerState{
+								Running: &v1.ContainerStateRunning{},
+							},
+						}).
+						ContainerStatuses(&v1.ContainerStatus{
+							Name: "container2",
+							State: v1.ContainerState{
+								Running: &v1.ContainerStateRunning{},
+							},
+						}).
+						Result(),
+				},
+			},
+			changes: []change{
+				// 1st modification: container1 starts running, resourceVersion 2, container2 still waiting
+				{
+					wait: 10 * time.Millisecond,
+					updated: builder.ForPod("default", "my-pod").
+						ObjectMeta(builder.WithResourceVersion("2")).
+						Containers(&v1.Container{
+							Name: "container1",
+						}).
+						Containers(&v1.Container{
+							Name: "container2",
+						}).
+						ContainerStatuses(&v1.ContainerStatus{
+							Name: "container1",
+							State: v1.ContainerState{
+								Running: &v1.ContainerStateRunning{},
+							},
+						}).
+						ContainerStatuses(&v1.ContainerStatus{
+							Name: "container2",
+							State: v1.ContainerState{
+								Waiting: &v1.ContainerStateWaiting{},
+							},
+						}).
+						Result(),
+				},
+				// 2nd modification: container2 starts running, resourceVersion 3
+				{
+					wait: 10 * time.Millisecond,
+					updated: builder.ForPod("default", "my-pod").
+						ObjectMeta(builder.WithResourceVersion("3")).
+						Containers(&v1.Container{
+							Name: "container1",
+						}).
+						Containers(&v1.Container{
+							Name: "container2",
+						}).
+						ContainerStatuses(&v1.ContainerStatus{
+							Name: "container1",
+							State: v1.ContainerState{
+								Running: &v1.ContainerStateRunning{},
+							},
+						}).
+						ContainerStatuses(&v1.ContainerStatus{
+							Name: "container2",
+							State: v1.ContainerState{
+								Running: &v1.ContainerStateRunning{},
+							},
+						}).
+						Result(),
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+
+			source := fcache.NewFakeControllerSource()
+			go func() {
+				// This is the state of the pod that will be seen by the AddFunc handler.
+				source.Add(test.initialPod)
+				// Changes holds the versions of the pod over time. Each of these states
+				// will be seen by the UpdateFunc handler.
+				for _, change := range test.changes {
+					time.Sleep(change.wait)
+					source.Modify(change.updated)
+				}
+			}()
+
+			podCommandExecutor := &velerotest.MockPodCommandExecutor{}
+			defer podCommandExecutor.AssertExpectations(t)
+
+			h := &DefaultWaitExecHookHandler{
+				PodCommandExecutor: podCommandExecutor,
+				ListWatchFactory:   &fakeListWatchFactory{source},
+			}
+
+			for _, e := range test.expectedExecutions {
+				obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(e.pod)
+				assert.Nil(t, err)
+				podCommandExecutor.On("ExecutePodCommand", mock.Anything, obj, e.pod.Namespace, e.pod.Name, e.name, e.hook).Return(e.error)
+			}
+
+			ctx := context.Background()
+			if test.sharedHooksContextTimeout > 0 {
+				ctx, _ = context.WithTimeout(ctx, test.sharedHooksContextTimeout)
+			}
+
+			errs := h.HandleHooks(ctx, velerotest.NewLogger(), test.initialPod, test.byContainer)
+
+			// for i, ee := range test.expectedErrors {
+			require.Len(t, errs, len(test.expectedErrors))
+			for i, ee := range test.expectedErrors {
+				assert.EqualError(t, errs[i], ee.Error())
+			}
+		})
+	}
+}
+
+func TestPodHasContainer(t *testing.T) {
+	tests := []struct {
+		name      string
+		pod       *v1.Pod
+		container string
+		expect    bool
+	}{
+		{
+			name:      "has container",
+			expect:    true,
+			container: "container1",
+			pod: builder.ForPod("default", "my-pod").
+				Containers(&v1.Container{
+					Name: "container1",
+				}).
+				Result(),
+		},
+		{
+			name:      "does not have container",
+			expect:    false,
+			container: "container1",
+			pod: builder.ForPod("default", "my-pod").
+				Containers(&v1.Container{
+					Name: "container2",
+				}).
+				Result(),
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			actual := podHasContainer(test.pod, test.container)
+			assert.Equal(t, actual, test.expect)
+		})
+	}
+}
+
+func TestIsContainerRunning(t *testing.T) {
+	tests := []struct {
+		name      string
+		pod       *v1.Pod
+		container string
+		expect    bool
+	}{
+		{
+			name:      "should return true when running",
+			container: "container1",
+			expect:    true,
+			pod: builder.ForPod("default", "my-pod").
+				ContainerStatuses(&v1.ContainerStatus{
+					Name: "container1",
+					State: v1.ContainerState{
+						Running: &v1.ContainerStateRunning{},
+					},
+				}).
+				Result(),
+		},
+		{
+			name:      "should return false when no state is set",
+			container: "container1",
+			expect:    false,
+			pod: builder.ForPod("default", "my-pod").
+				ContainerStatuses(&v1.ContainerStatus{
+					Name:  "container1",
+					State: v1.ContainerState{},
+				}).
+				Result(),
+		},
+		{
+			name:      "should return false when waiting",
+			container: "container1",
+			expect:    false,
+			pod: builder.ForPod("default", "my-pod").
+				ContainerStatuses(&v1.ContainerStatus{
+					Name: "container1",
+					State: v1.ContainerState{
+						Waiting: &v1.ContainerStateWaiting{},
+					},
+				}).
+				Result(),
+		},
+		{
+			name:      "should return true when running and first container is terminated",
+			container: "container1",
+			expect:    true,
+			pod: builder.ForPod("default", "my-pod").
+				ContainerStatuses(&v1.ContainerStatus{
+					Name: "container0",
+					State: v1.ContainerState{
+						Terminated: &v1.ContainerStateTerminated{},
+					},
+				},
+					&v1.ContainerStatus{
+						Name: "container1",
+						State: v1.ContainerState{
+							Running: &v1.ContainerStateRunning{},
+						},
+					}).
+				Result(),
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			actual := isContainerRunning(test.pod, test.container)
+			assert.Equal(t, actual, test.expect)
+		})
+	}
+}
+
+func TestMaxHookWait(t *testing.T) {
+	tests := []struct {
+		name        string
+		byContainer map[string][]PodExecRestoreHook
+		expect      time.Duration
+	}{
+		{
+			name:        "should return 0 for nil map",
+			byContainer: nil,
+			expect:      0,
+		},
+		{
+			name:   "should return 0 if all hooks are 0 or negative",
+			expect: 0,
+			byContainer: map[string][]PodExecRestoreHook{
+				"container1": {
+					{
+						Hook: velerov1api.ExecRestoreHook{
+							ExecTimeout: metav1.Duration{time.Second},
+							WaitTimeout: metav1.Duration{0},
+						},
+					},
+					{
+						Hook: velerov1api.ExecRestoreHook{
+							WaitTimeout: metav1.Duration{-1},
+						},
+					},
+				},
+			},
+		},
+		{
+			name:   "should return biggest wait timeout from multiple hooks in multiple containers",
+			expect: time.Hour,
+			byContainer: map[string][]PodExecRestoreHook{
+				"container1": {
+					{
+						Hook: velerov1api.ExecRestoreHook{
+							WaitTimeout: metav1.Duration{time.Second},
+						},
+					},
+					{
+						Hook: velerov1api.ExecRestoreHook{
+							WaitTimeout: metav1.Duration{time.Second},
+						},
+					},
+				},
+				"container2": {
+					{
+						Hook: velerov1api.ExecRestoreHook{
+							WaitTimeout: metav1.Duration{time.Hour},
+						},
+					},
+					{
+						Hook: velerov1api.ExecRestoreHook{
+							WaitTimeout: metav1.Duration{time.Minute},
+						},
+					},
+				},
+			},
+		},
+		{
+			name:   "should return 0 if any hook does not have a wait timeout",
+			expect: 0,
+			byContainer: map[string][]PodExecRestoreHook{
+				"container1": {
+					{
+						Hook: velerov1api.ExecRestoreHook{
+							ExecTimeout: metav1.Duration{time.Second},
+							WaitTimeout: metav1.Duration{time.Second},
+						},
+					},
+					{
+						Hook: velerov1api.ExecRestoreHook{
+							WaitTimeout: metav1.Duration{0},
+						},
+					},
+				},
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			actual := maxHookWait(test.byContainer)
+			assert.Equal(t, actual, test.expect)
+		})
+	}
+}

pkg/builder/object_meta.go

@@ -34,6 +34,14 @@ func WithName(val string) func(obj metav1.Object) {
 	}
 }
 
+// WithResourceVersion is a functional option that applies the specified
+// resourceVersion to an object
+func WithResourceVersion(val string) func(obj metav1.Object) {
+	return func(obj metav1.Object) {
+		obj.SetResourceVersion(val)
+	}
+}
+
 // WithLabels is a functional option that applies the specified
 // label keys/values to an object.
 func WithLabels(vals ...string) func(obj metav1.Object) {

pkg/builder/pod_builder.go

@@ -82,3 +82,17 @@ func (b *PodBuilder) InitContainers(containers ...*corev1api.Container) *PodBuil
 	}
 	return b
 }
+
+func (b *PodBuilder) Containers(containers ...*corev1api.Container) *PodBuilder {
+	for _, c := range containers {
+		b.object.Spec.Containers = append(b.object.Spec.Containers, *c)
+	}
+	return b
+}
+
+func (b *PodBuilder) ContainerStatuses(containerStatuses ...*corev1api.ContainerStatus) *PodBuilder {
+	for _, c := range containerStatuses {
+		b.object.Status.ContainerStatuses = append(b.object.Status.ContainerStatuses, *c)
+	}
+	return b
+}

pkg/cmd/server/server.go

@@ -720,6 +720,8 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string
 			s.config.podVolumeOperationTimeout,
 			s.config.resourceTerminatingTimeout,
 			s.logger,
+			podexec.NewPodCommandExecutor(s.kubeClientConfig, s.kubeClient.CoreV1().RESTClient()),
+			s.kubeClient.CoreV1().RESTClient(),
 		)
 		cmd.CheckError(err)
 

pkg/restore/restore.go

@@ -42,7 +42,9 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/tools/cache"
 
+	"github.com/vmware-tanzu/velero/internal/hook"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	"github.com/vmware-tanzu/velero/pkg/archive"
 	"github.com/vmware-tanzu/velero/pkg/client"
@@ -51,6 +53,7 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/kuberesource"
 	"github.com/vmware-tanzu/velero/pkg/label"
 	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
+	"github.com/vmware-tanzu/velero/pkg/podexec"
 	"github.com/vmware-tanzu/velero/pkg/restic"
 	"github.com/vmware-tanzu/velero/pkg/util/boolptr"
 	"github.com/vmware-tanzu/velero/pkg/util/collections"
@@ -95,6 +98,8 @@ type kubernetesRestorer struct {
 	fileSystem                 filesystem.Interface
 	pvRenamer                  func(string) (string, error)
 	logger                     logrus.FieldLogger
+	podCommandExecutor         podexec.PodCommandExecutor
+	podGetter                  cache.Getter
 }
 
 // NewKubernetesRestorer creates a new kubernetesRestorer.
@@ -107,6 +112,8 @@ func NewKubernetesRestorer(
 	resticTimeout time.Duration,
 	resourceTerminatingTimeout time.Duration,
 	logger logrus.FieldLogger,
+	podCommandExecutor podexec.PodCommandExecutor,
+	podGetter cache.Getter,
 ) (Restorer, error) {
 	return &kubernetesRestorer{
 		discoveryHelper:            discoveryHelper,
@@ -125,7 +132,9 @@ func NewKubernetesRestorer(
 			veleroCloneName := "velero-clone-" + veleroCloneUuid.String()
 			return veleroCloneName, nil
 		},
-		fileSystem: filesystem.NewFileSystem(),
+		fileSystem:         filesystem.NewFileSystem(),
+		podCommandExecutor: podCommandExecutor,
+		podGetter:          podGetter,
 	}, nil
 }
 
@@ -186,6 +195,18 @@ func (kr *kubernetesRestorer) Restore(
 		}
 	}
 
+	resourceRestoreHooks, err := hook.GetRestoreHooksFromSpec(&req.Restore.Spec.Hooks)
+	if err != nil {
+		return Result{}, Result{Velero: []string{err.Error()}}
+	}
+	hooksCtx, hooksCancelFunc := go_context.WithCancel(go_context.Background())
+	waitExecHookHandler := &hook.DefaultWaitExecHookHandler{
+		PodCommandExecutor: kr.podCommandExecutor,
+		ListWatchFactory: &hook.DefaultListWatchFactory{
+			PodsGetter: kr.podGetter,
+		},
+	}
+
 	pvRestorer := &pvRestorer{
 		logger:                  req.Log,
 		backup:                  req.Backup,
@@ -222,6 +243,11 @@ func (kr *kubernetesRestorer) Restore(
 		pvRenamer:                  kr.pvRenamer,
 		discoveryHelper:            kr.discoveryHelper,
 		resourcePriorities:         kr.resourcePriorities,
+		resourceRestoreHooks:       resourceRestoreHooks,
+		hooksErrs:                  make(chan error),
+		waitExecHookHandler:        waitExecHookHandler,
+		hooksContext:               hooksCtx,
+		hooksCancelFunc:            hooksCancelFunc,
 	}
 
 	return restoreCtx.execute()
@@ -297,6 +323,10 @@ type restoreContext struct {
 	resourcePriorities         []string
 	hooksWaitGroup             sync.WaitGroup
 	hooksErrs                  chan error
+	resourceRestoreHooks       []hook.ResourceRestoreHook
+	waitExecHookHandler        hook.WaitExecHookHandler
+	hooksContext               go_context.Context
+	hooksCancelFunc            go_context.CancelFunc
 }
 
 type resourceClientKey struct {
@@ -465,6 +495,18 @@ func (ctx *restoreContext) execute() (Result, Result) {
 	}
 	ctx.log.Info("Done waiting for all restic restores to complete")
 
+	// wait for all post-restore exec hooks with same logic as restic wait above
+	go func() {
+		ctx.log.Info("Waiting for all post-restore-exec hooks to complete")
+
+		ctx.hooksWaitGroup.Wait()
+		close(ctx.hooksErrs)
+	}()
+	for err := range ctx.hooksErrs {
+		errs.Velero = append(errs.Velero, err.Error())
+	}
+	ctx.log.Info("Done waiting for all post-restore exec hooks to complete")
+
 	return warnings, errs
 }
 
@@ -1116,6 +1158,10 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
 		restorePodVolumeBackups(ctx, createdObj, originalNamespace)
 	}
 
+	if groupResource == kuberesource.Pods {
+		ctx.waitExec(createdObj)
+	}
+
 	// Wait for a CRD to be available for instantiating resources
 	// before continuing.
 	if groupResource == kuberesource.CustomResourceDefinitions {
@@ -1205,6 +1251,43 @@ func restorePodVolumeBackups(ctx *restoreContext, createdObj *unstructured.Unstr
 	}
 }
 
+// waitExec executes hooks in a restored pod's containers when they become ready
+func (ctx *restoreContext) waitExec(createdObj *unstructured.Unstructured) {
+	ctx.hooksWaitGroup.Add(1)
+	go func() {
+		// Done() will only be called after all errors have been successfully sent
+		// on the ctx.resticErrs channel
+		defer ctx.hooksWaitGroup.Done()
+
+		pod := new(v1.Pod)
+		if err := runtime.DefaultUnstructuredConverter.FromUnstructured(createdObj.UnstructuredContent(), &pod); err != nil {
+			ctx.log.WithError(err).Error("error converting unstructured pod")
+			ctx.hooksErrs <- err
+			return
+		}
+		execHooksByContainer, err := hook.GroupRestoreExecHooks(
+			ctx.resourceRestoreHooks,
+			pod,
+			ctx.log,
+		)
+		if err != nil {
+			ctx.log.WithError(err).Errorf("error getting exec hooks for pod %s/%s", pod.Namespace, pod.Name)
+			ctx.hooksErrs <- err
+			return
+		}
+
+		if errs := ctx.waitExecHookHandler.HandleHooks(ctx.hooksContext, ctx.log, pod, execHooksByContainer); len(errs) > 0 {
+			ctx.log.WithError(kubeerrs.NewAggregate(errs)).Error("unable to successfully execute post-restore hooks")
+			ctx.hooksCancelFunc()
+
+			for _, err := range errs {
+				// Errors are already logged in the HandleHooks method
+				ctx.hooksErrs <- err
+			}
+		}
+	}()
+}
+
 func hasSnapshot(pvName string, snapshots []*volume.Snapshot) bool {
 	for _, snapshot := range snapshots {
 		if snapshot.Spec.PersistentVolumeName == pvName {