Merge branch 'main' into optimize-sub-object-description

pkg/apis/velero/shared/constants.go

@@ -0,0 +1,22 @@
+/*
+Copyright The Velero Contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package shared
+
+const (
+	DataUploadParentSnapshotNone = "none"
+	DataUploadParentSnapshotAuto = "auto"
+)

pkg/apis/velero/v1/labels_annotations.go

@@ -166,6 +166,14 @@ const (
 
 	// Velero checks this annotation to determine whether to skip resource excluding check.
 	MustIncludeAdditionalItemAnnotation = "backup.velero.io/must-include-additional-items"
+	// MustIncludeAdditionalItemRestoreAnnotation is set by RestoreItemActions on the UpdatedItem
+	// to tell Velero to bypass global resource/namespace exclusion checks (and IncludeClusterResources=false)
+	// for that action's AdditionalItems. Value must be "true" to enable the bypass. The annotation is
+	// always stripped before the item is applied to the cluster when present, including non-"true" values.
+	//
+	// Notice: SkipRestore on the Execute output takes precedence. If SkipRestore is true, the
+	// annotation is never inspected and AdditionalItems are not processed.
+	MustIncludeAdditionalItemRestoreAnnotation = "restore.velero.io/must-include-additional-items"
 	// SkippedNoCSIPVAnnotation - Velero checks this annotation on processed PVC to
 	// find out if the snapshot was skipped b/c the PV is not provisioned via CSI
 	SkippedNoCSIPVAnnotation = "backup.velero.io/skipped-no-csi-pv"

pkg/apis/velero/v2alpha1/data_upload_types.go

@@ -64,6 +64,12 @@ type DataUploadSpec struct {
 	// SourceFSType is the file system type of the source volume.
 	// +optional
 	SourceFSType string `json:"sourceFSType,omitempty"`
+
+	// ParentSnapshot specifies the parent snapshot that current backup is based on.
+	// If its value is "" or "auto", the data mover finds the recent backup of the same volume as parent.
+	// If its value is "none", the data mover will do a full backup
+	// If its value is a specific snapshotID, the data mover finds the specific snapshot as parent.
+	ParentSnapshot string `json:"parentSnapshot,omitempty"`
 }
 
 type SnapshotType string

pkg/backup/actions/csi/pvc_action.go

@@ -42,6 +42,7 @@ import (
 	crclient "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
+	veleroshared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1"
 	veleroclient "github.com/vmware-tanzu/velero/pkg/client"
@@ -535,6 +536,16 @@ func newDataUpload(
 	vsc *snapshotv1api.VolumeSnapshotContent,
 	fsType string,
 ) *velerov2alpha1.DataUpload {
+	var parentSnapshot string
+	switch backup.Spec.BackupType {
+	case velerov1api.BackupTypeFull:
+		parentSnapshot = veleroshared.DataUploadParentSnapshotNone
+	case velerov1api.BackupTypeIncremental:
+		parentSnapshot = veleroshared.DataUploadParentSnapshotAuto
+	default:
+		parentSnapshot = veleroshared.DataUploadParentSnapshotAuto
+	}
+
 	dataUpload := &velerov2alpha1.DataUpload{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: velerov2alpha1.SchemeGroupVersion.String(),
@@ -572,6 +583,7 @@ func newDataUpload(
 			SourceNamespace:       pvc.Namespace,
 			OperationTimeout:      backup.Spec.CSISnapshotTimeout,
 			SourceFSType:          fsType,
+			ParentSnapshot:        parentSnapshot,
 		},
 	}
 

pkg/backup/actions/csi/pvc_action_test.go

@@ -23,40 +23,39 @@ import (
 	"testing"
 	"time"
 
-	"github.com/vmware-tanzu/velero/pkg/kuberesource"
-
-	volumegroupsnapshotv1beta2 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta2"
-	"github.com/stretchr/testify/assert"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/utils/ptr"
-
-	"github.com/vmware-tanzu/velero/pkg/label"
-
+	"github.com/cockroachdb/errors"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	volumegroupsnapshotv1beta2 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta2"
 	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
-
-	"github.com/cockroachdb/errors"
 	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	corev1api "k8s.io/api/core/v1"
 	storagev1api "k8s.io/api/storage/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/utils/ptr"
 	crclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
+	veleroshared "github.com/vmware-tanzu/velero/pkg/apis/velero/shared"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1"
 	"github.com/vmware-tanzu/velero/pkg/builder"
 	factorymocks "github.com/vmware-tanzu/velero/pkg/client/mocks"
+	"github.com/vmware-tanzu/velero/pkg/kuberesource"
+	"github.com/vmware-tanzu/velero/pkg/label"
 	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
 	velerotest "github.com/vmware-tanzu/velero/pkg/test"
+	uploaderUtil "github.com/vmware-tanzu/velero/pkg/uploader/util"
+	"github.com/vmware-tanzu/velero/pkg/util/boolptr"
 )
 
 const testDriver = "csi.example.com"
@@ -163,6 +162,7 @@ func TestExecute(t *testing.T) {
 					SourcePVC:        "testPVC",
 					SourceNamespace:  "velero",
 					OperationTimeout: metav1.Duration{Duration: 1 * time.Minute},
+					ParentSnapshot:   veleroshared.DataUploadParentSnapshotAuto,
 				},
 			},
 		},
@@ -2176,3 +2176,131 @@ func TestGetOrCreateVolumeHelper(t *testing.T) {
 	// The pvcPodCache should be the same instance
 	require.Same(t, cache1, action.pvcPodCache, "Expected same pvcPodCache instance on repeated calls")
 }
+
+func TestNewDataUpload(t *testing.T) {
+	tests := []struct {
+		name                 string
+		backupType           velerov1api.BackupType
+		vsClassName          *string
+		uploaderConfig       *velerov1api.UploaderConfigForBackup
+		expectedParentSnap   string
+		expectedDataMoverCfg map[string]string
+	}{
+		{
+			name:                 "Full backup type, no uploader config, no vs class name",
+			backupType:           velerov1api.BackupTypeFull,
+			vsClassName:          nil,
+			uploaderConfig:       nil,
+			expectedParentSnap:   "none",
+			expectedDataMoverCfg: nil,
+		},
+		{
+			name:               "Incremental backup type, with uploader config, with vs class name",
+			backupType:         velerov1api.BackupTypeIncremental,
+			vsClassName:        ptr.To("test-vs-class"),
+			uploaderConfig:     &velerov1api.UploaderConfigForBackup{ParallelFilesUpload: 10},
+			expectedParentSnap: "auto",
+			expectedDataMoverCfg: map[string]string{
+				uploaderUtil.ParallelFilesUpload: "10",
+			},
+		},
+		{
+			name:                 "Default backup type, uploader config with 0 parallel files",
+			backupType:           "",
+			vsClassName:          ptr.To("test-vs-class"),
+			uploaderConfig:       &velerov1api.UploaderConfigForBackup{ParallelFilesUpload: 0},
+			expectedParentSnap:   "auto",
+			expectedDataMoverCfg: nil,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			backup := &velerov1api.Backup{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-backup",
+					Namespace: "velero",
+					UID:       types.UID("backup-uid"),
+				},
+				Spec: velerov1api.BackupSpec{
+					BackupType:         tc.backupType,
+					DataMover:          "velero",
+					StorageLocation:    "default",
+					CSISnapshotTimeout: metav1.Duration{Duration: 10 * time.Minute},
+					UploaderConfig:     tc.uploaderConfig,
+				},
+			}
+
+			vs := &snapshotv1api.VolumeSnapshot{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test-vs",
+				},
+				Spec: snapshotv1api.VolumeSnapshotSpec{
+					VolumeSnapshotClassName: tc.vsClassName,
+				},
+			}
+
+			pvc := &corev1api.PersistentVolumeClaim{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-pvc",
+					Namespace: "test-ns",
+					UID:       types.UID("pvc-uid"),
+				},
+				Spec: corev1api.PersistentVolumeClaimSpec{
+					StorageClassName: ptr.To("test-storage-class"),
+				},
+			}
+
+			vsc := &snapshotv1api.VolumeSnapshotContent{
+				Spec: snapshotv1api.VolumeSnapshotContentSpec{
+					Driver: "test-driver",
+				},
+			}
+
+			operationID := "test-op-id"
+			fsType := "ext4"
+
+			du := newDataUpload(backup, vs, pvc, operationID, vsc, fsType)
+
+			require.NotNil(t, du)
+			assert.Equal(t, velerov2alpha1.SchemeGroupVersion.String(), du.APIVersion)
+			assert.Equal(t, "DataUpload", du.Kind)
+			assert.Equal(t, backup.Namespace, du.Namespace)
+			assert.Equal(t, backup.Name+"-", du.GenerateName)
+
+			require.Len(t, du.OwnerReferences, 1)
+			assert.Equal(t, velerov1api.SchemeGroupVersion.String(), du.OwnerReferences[0].APIVersion)
+			assert.Equal(t, "Backup", du.OwnerReferences[0].Kind)
+			assert.Equal(t, backup.Name, du.OwnerReferences[0].Name)
+			assert.Equal(t, backup.UID, du.OwnerReferences[0].UID)
+			assert.Equal(t, boolptr.True(), du.OwnerReferences[0].Controller)
+
+			expectedLabels := map[string]string{
+				velerov1api.BackupNameLabel:       label.GetValidName(backup.Name),
+				velerov1api.BackupUIDLabel:        string(backup.UID),
+				velerov1api.PVCUIDLabel:           string(pvc.UID),
+				velerov1api.AsyncOperationIDLabel: operationID,
+			}
+			assert.Equal(t, expectedLabels, du.Labels)
+
+			assert.Equal(t, velerov2alpha1.SnapshotTypeCSI, du.Spec.SnapshotType)
+			assert.Equal(t, vs.Name, du.Spec.CSISnapshot.VolumeSnapshot)
+			assert.Equal(t, *pvc.Spec.StorageClassName, du.Spec.CSISnapshot.StorageClass)
+			assert.Equal(t, vsc.Spec.Driver, du.Spec.CSISnapshot.Driver)
+			if tc.vsClassName != nil {
+				assert.Equal(t, *tc.vsClassName, du.Spec.CSISnapshot.SnapshotClass)
+			} else {
+				assert.Empty(t, du.Spec.CSISnapshot.SnapshotClass)
+			}
+
+			assert.Equal(t, pvc.Name, du.Spec.SourcePVC)
+			assert.Equal(t, backup.Spec.DataMover, du.Spec.DataMover)
+			assert.Equal(t, backup.Spec.StorageLocation, du.Spec.BackupStorageLocation)
+			assert.Equal(t, pvc.Namespace, du.Spec.SourceNamespace)
+			assert.Equal(t, backup.Spec.CSISnapshotTimeout, du.Spec.OperationTimeout)
+			assert.Equal(t, fsType, du.Spec.SourceFSType)
+			assert.Equal(t, tc.expectedParentSnap, du.Spec.ParentSnapshot)
+			assert.Equal(t, tc.expectedDataMoverCfg, du.Spec.DataMoverConfig)
+		})
+	}
+}

pkg/backup/backup.go

@@ -1428,22 +1428,20 @@ func resolveClusterScopedFilterPolicy(
 }
 
 func resolveResourceFilter(rf resourcepolicies.ResourceFilter) (*ResolvedResourceFilter, error) {
-	var selector labels.Selector
-	if len(rf.LabelSelector) > 0 {
-		var err error
-		selector, err = labels.ValidatedSelectorFromSet(labels.Set(rf.LabelSelector))
-		if err != nil {
-			return nil, fmt.Errorf("invalid label selector in resource filter: %w", err)
-		}
+	selector, err := resourcepolicies.SelectorFromPolicyLabelSelector(rf.LabelSelector)
+	if err != nil {
+		return nil, fmt.Errorf("invalid label selector in resource filter: %w", err)
 	}
 
 	var orSelectors []labels.Selector
 	for _, ols := range rf.OrLabelSelectors {
-		s, err := labels.ValidatedSelectorFromSet(labels.Set(ols))
+		s, err := resourcepolicies.SelectorFromPolicyLabelSelector(ols)
 		if err != nil {
 			return nil, fmt.Errorf("invalid OR label selector in resource filter: %w", err)
 		}
-		orSelectors = append(orSelectors, s)
+		if s != nil {
+			orSelectors = append(orSelectors, s)
+		}
 	}
 
 	var nameIE *collections.IncludesExcludes

pkg/backup/backup_test.go

@@ -5741,7 +5741,7 @@ func TestResolveResourceFilter(t *testing.T) {
 		{
 			name: "valid label selector",
 			rf: resourcepolicies.ResourceFilter{
-				LabelSelector: map[string]string{"app": "foo"},
+				LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"app": "foo"}},
 			},
 			expectErr: false,
 			checkResult: func(t *testing.T, r *ResolvedResourceFilter) {
@@ -5754,16 +5754,16 @@ func TestResolveResourceFilter(t *testing.T) {
 		{
 			name: "invalid label selector",
 			rf: resourcepolicies.ResourceFilter{
-				LabelSelector: map[string]string{"invalid/label/key": "value"},
+				LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"invalid/label/key": "value"}},
 			},
 			expectErr: true,
 		},
 		{
 			name: "valid or label selectors",
 			rf: resourcepolicies.ResourceFilter{
-				OrLabelSelectors: []map[string]string{
-					{"app": "foo"},
-					{"app": "bar"},
+				OrLabelSelectors: []*resourcepolicies.PolicyLabelSelector{
+					{MatchLabels: map[string]string{"app": "foo"}},
+					{MatchLabels: map[string]string{"app": "bar"}},
 				},
 			},
 			expectErr: false,
@@ -5776,8 +5776,8 @@ func TestResolveResourceFilter(t *testing.T) {
 		{
 			name: "invalid or label selectors",
 			rf: resourcepolicies.ResourceFilter{
-				OrLabelSelectors: []map[string]string{
-					{"invalid/label/key": "value"},
+				OrLabelSelectors: []*resourcepolicies.PolicyLabelSelector{
+					{MatchLabels: map[string]string{"invalid/label/key": "value"}},
 				},
 			},
 			expectErr: true,
@@ -5797,6 +5797,68 @@ func TestResolveResourceFilter(t *testing.T) {
 				assert.False(t, r.NameIE.ShouldInclude("exc1"))
 			},
 		},
+		{
+			name: "empty labelSelector is no filter",
+			rf: resourcepolicies.ResourceFilter{
+				LabelSelector: &resourcepolicies.PolicyLabelSelector{},
+			},
+			expectErr: false,
+			checkResult: func(t *testing.T, r *ResolvedResourceFilter) {
+				t.Helper()
+				require.NotNil(t, r)
+				assert.Nil(t, r.LabelSelector)
+			},
+		},
+		{
+			name: "set-based In and DoesNotExist",
+			rf: resourcepolicies.ResourceFilter{
+				LabelSelector: &resourcepolicies.PolicyLabelSelector{
+					MatchExpressions: []resourcepolicies.PolicyLabelSelectorRequirement{
+						{Key: "environment", Operator: "In", Values: []string{"prod", "staging"}},
+						{Key: "do-not-backup", Operator: "DoesNotExist"},
+					},
+				},
+			},
+			expectErr: false,
+			checkResult: func(t *testing.T, r *ResolvedResourceFilter) {
+				t.Helper()
+				require.NotNil(t, r.LabelSelector)
+				assert.True(t, r.LabelSelector.Matches(labels.Set{"environment": "prod"}))
+				assert.True(t, r.LabelSelector.Matches(labels.Set{"environment": "staging"}))
+				assert.False(t, r.LabelSelector.Matches(labels.Set{"environment": "dev"}))
+				assert.False(t, r.LabelSelector.Matches(labels.Set{"environment": "prod", "do-not-backup": "true"}))
+			},
+		},
+		{
+			name: "set-based NotIn and Exists",
+			rf: resourcepolicies.ResourceFilter{
+				LabelSelector: &resourcepolicies.PolicyLabelSelector{
+					MatchExpressions: []resourcepolicies.PolicyLabelSelectorRequirement{
+						{Key: "tier", Operator: "NotIn", Values: []string{"debug"}},
+						{Key: "app", Operator: "Exists"},
+					},
+				},
+			},
+			expectErr: false,
+			checkResult: func(t *testing.T, r *ResolvedResourceFilter) {
+				t.Helper()
+				require.NotNil(t, r.LabelSelector)
+				assert.True(t, r.LabelSelector.Matches(labels.Set{"app": "web", "tier": "frontend"}))
+				assert.False(t, r.LabelSelector.Matches(labels.Set{"app": "web", "tier": "debug"}))
+				assert.False(t, r.LabelSelector.Matches(labels.Set{"tier": "frontend"}))
+			},
+		},
+		{
+			name: "invalid operator",
+			rf: resourcepolicies.ResourceFilter{
+				LabelSelector: &resourcepolicies.PolicyLabelSelector{
+					MatchExpressions: []resourcepolicies.PolicyLabelSelectorRequirement{
+						{Key: "env", Operator: "Equals", Values: []string{"prod"}},
+					},
+				},
+			},
+			expectErr: true,
+		},
 	}
 
 	for _, tc := range tests {
@@ -5834,11 +5896,11 @@ func TestResolveClusterScopedFilterPolicy(t *testing.T) {
 		ResourceFilters: []resourcepolicies.ResourceFilter{
 			{
 				Kinds:         []string{"pods", "secrets"},
-				LabelSelector: map[string]string{"app": "foo"},
+				LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"app": "foo"}},
 			},
 			{
 				Kinds:         []string{"invalid-kind"},
-				LabelSelector: map[string]string{"invalid/label/key": "value"},
+				LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"invalid/label/key": "value"}},
 			},
 		},
 	}
@@ -5852,7 +5914,7 @@ func TestResolveClusterScopedFilterPolicy(t *testing.T) {
 		ResourceFilters: []resourcepolicies.ResourceFilter{
 			{
 				Kinds:         []string{"pods", "secrets"},
-				LabelSelector: map[string]string{"app": "foo"},
+				LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"app": "foo"}},
 			},
 		},
 	}
@@ -5900,11 +5962,11 @@ func TestResolveNamespacedFilterPolicies(t *testing.T) {
 			ResourceFilters: []resourcepolicies.ResourceFilter{
 				{
 					Kinds:         []string{"pods"},
-					LabelSelector: map[string]string{"app": "foo"},
+					LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"app": "foo"}},
 				},
 				{
 					Kinds:         []string{"*"},
-					LabelSelector: map[string]string{"catch": "all"},
+					LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"catch": "all"}},
 				},
 			},
 		},
@@ -5932,7 +5994,7 @@ func TestResolveNamespacedFilterPolicies(t *testing.T) {
 			ResourceFilters: []resourcepolicies.ResourceFilter{
 				{
 					Kinds:         []string{"pods"},
-					LabelSelector: map[string]string{"invalid/label/key": "value"},
+					LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"invalid/label/key": "value"}},
 				},
 			},
 		},
@@ -6016,7 +6078,7 @@ func TestBackupWithResPoliciesLogs(t *testing.T) {
 		ResourceFilters: []resourcepolicies.ResourceFilter{
 			{
 				Kinds:         []string{"pods"},
-				LabelSelector: map[string]string{"invalid/label/key": "value"},
+				LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"invalid/label/key": "value"}},
 			},
 		},
 	}
@@ -6035,7 +6097,7 @@ func TestBackupWithResPoliciesLogs(t *testing.T) {
 			ResourceFilters: []resourcepolicies.ResourceFilter{
 				{
 					Kinds:         []string{"pods"},
-					LabelSelector: map[string]string{"invalid/label/key": "value"},
+					LabelSelector: &resourcepolicies.PolicyLabelSelector{MatchLabels: map[string]string{"invalid/label/key": "value"}},
 				},
 			},
 		},

pkg/cmd/cli/datamover/backup.go

@@ -15,6 +15,7 @@ package datamover
 
 import (
 	"context"
+	"crypto/fips140"
 	"fmt"
 	"os"
 	"strings"
@@ -87,7 +88,10 @@ func NewBackupCommand(f client.Factory) *cobra.Command {
 				kube.ExitPodWithMessage(logger, false, "Failed to create data mover backup, %v", err)
 			}
 
-			s.run()
+			// Disable FIPS-140 compliance check, because Kopia doesn't support FIPS-140 yet.
+			fips140.WithoutEnforcement(func() {
+				s.run()
+			})
 		},
 	}
 

pkg/cmd/cli/datamover/restore.go

@@ -15,6 +15,7 @@ package datamover
 
 import (
 	"context"
+	"crypto/fips140"
 	"fmt"
 	"os"
 	"strings"
@@ -81,7 +82,10 @@ func NewRestoreCommand(f client.Factory) *cobra.Command {
 				kube.ExitPodWithMessage(logger, false, "Failed to create data mover restore, %v", err)
 			}
 
-			s.run()
+			// Disable FIPS-140 compliance check, because Kopia doesn't support FIPS-140 yet.
+			fips140.WithoutEnforcement(func() {
+				s.run()
+			})
 		},
 	}
 

pkg/cmd/cli/podvolume/backup.go

@@ -15,6 +15,7 @@ package podvolume
 
 import (
 	"context"
+	"crypto/fips140"
 	"fmt"
 	"os"
 	"strings"
@@ -80,7 +81,10 @@ func NewBackupCommand(f client.Factory) *cobra.Command {
 				kube.ExitPodWithMessage(logger, false, "Failed to create pod volume backup, %v", err)
 			}
 
-			s.run()
+			// Disable FIPS-140 compliance check, because Kopia doesn't support FIPS-140 yet.
+			fips140.WithoutEnforcement(func() {
+				s.run()
+			})
 		},
 	}
 

pkg/cmd/cli/podvolume/restore.go

@@ -15,6 +15,7 @@ package podvolume
 
 import (
 	"context"
+	"crypto/fips140"
 	"fmt"
 	"os"
 	"strings"
@@ -79,7 +80,10 @@ func NewRestoreCommand(f client.Factory) *cobra.Command {
 				kube.ExitPodWithMessage(logger, false, "Failed to create pod volume restore, %v", err)
 			}
 
-			s.run()
+			// Disable FIPS-140 compliance check, because Kopia doesn't support FIPS-140 yet.
+			fips140.WithoutEnforcement(func() {
+				s.run()
+			})
 		},
 	}
 

pkg/cmd/cli/repomantenance/maintenance.go

@@ -2,6 +2,7 @@ package repomantenance
 
 import (
 	"context"
+	"crypto/fips140"
 	"fmt"
 	"os"
 	"strings"
@@ -57,7 +58,10 @@ func NewCommand(f velerocli.Factory) *cobra.Command {
 		Hidden: true,
 		Short:  "VELERO INTERNAL COMMAND ONLY - not intended to be run directly by users",
 		Run: func(c *cobra.Command, args []string) {
-			o.Run(f)
+			// Disable FIPS-140 compliance check, because Kopia doesn't support FIPS-140 yet.
+			fips140.WithoutEnforcement(func() {
+				o.Run(f)
+			})
 		},
 	}
 

pkg/cmd/util/output/backup_describer.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
 	"sort"
 	"strconv"
 	"strings"
@@ -31,7 +30,6 @@ import (
 
 	"github.com/cockroachdb/errors"
 	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
-	"github.com/sirupsen/logrus"
 
 	"github.com/fatih/color"
 	kbclient "sigs.k8s.io/controller-runtime/pkg/client"
@@ -94,9 +92,6 @@ func DescribeBackup(
 		if backup.Spec.ResourcePolicy != nil {
 			d.Println()
 			DescribeResourcePolicies(d, backup.Spec.ResourcePolicy)
-
-			// Display fine-grained filter policies if they exist
-			DescribeFineGrainedFilterPolicies(ctx, kbClient, d, backup)
 		}
 
 		DescribeGlobalVolumePolicy(d, backup)
@@ -151,119 +146,6 @@ func DescribeGlobalVolumePolicy(d *Describer, backup *velerov1api.Backup) {
 	d.Printf("\tName:\t%s\n", name)
 }
 
-// DescribeFineGrainedFilterPolicies describes cluster-scoped and namespace-scoped filter policies if present
-func DescribeFineGrainedFilterPolicies(ctx context.Context, kbClient kbclient.Client, d *Describer, backup *velerov1api.Backup) {
-	if backup.Spec.ResourcePolicy == nil {
-		return
-	}
-
-	// Create a discard logger for the resource policies function since this is CLI output context
-	discardLogger := logrus.New()
-	discardLogger.Out = io.Discard
-
-	resourcePolicies, err := resourcepolicies.GetResourcePoliciesFromBackup(*backup, kbClient, discardLogger)
-	if err != nil {
-		// Don't fail the describe if we can't read policies, just skip
-		return
-	}
-
-	if resourcePolicies == nil {
-		return
-	}
-
-	clusterScopedFilterPolicy := resourcePolicies.GetClusterScopedFilterPolicy()
-	if clusterScopedFilterPolicy != nil {
-		d.Printf("\nCluster Scoped Filter Policy:\n")
-		d.Printf("  Resource Filters:\n")
-		for _, rf := range clusterScopedFilterPolicy.ResourceFilters {
-			kindsStr := strings.Join(rf.Kinds, ", ")
-			d.Printf("    %s:\n", kindsStr)
-
-			// Label selector
-			if len(rf.LabelSelector) > 0 {
-				selectorStr := formatLabelMap(rf.LabelSelector)
-				d.Printf("      Label selector:     %s\n", selectorStr)
-			} else if len(rf.OrLabelSelectors) > 0 {
-				var orStrs []string
-				for _, ols := range rf.OrLabelSelectors {
-					orStrs = append(orStrs, formatLabelMap(ols))
-				}
-				d.Printf("      OR label selectors: [%s]\n", strings.Join(orStrs, ", "))
-			} else {
-				d.Printf("      Label selector:     <none>\n")
-			}
-
-			// Name patterns
-			if len(rf.Names) > 0 {
-				d.Printf("      Included names:     [%s]\n", strings.Join(rf.Names, ", "))
-			} else {
-				d.Printf("      Included names:     <none>\n")
-			}
-
-			if len(rf.ExcludedNames) > 0 {
-				d.Printf("      Excluded names:     [%s]\n", strings.Join(rf.ExcludedNames, ", "))
-			} else {
-				d.Printf("      Excluded names:     <none>\n")
-			}
-		}
-	}
-
-	nfPolicies := resourcePolicies.GetNamespacedFilterPolicies()
-	if len(nfPolicies) > 0 {
-		d.Printf("\nNamespace-Scoped Filter Policies:\n")
-		for _, policy := range nfPolicies {
-			for _, ns := range policy.Namespaces {
-				d.Printf("  %s:\n", ns)
-				d.Printf("    Resource Filters:\n")
-				for _, rf := range policy.ResourceFilters {
-					var kindsStr string
-					if rf.IsCatchAll() {
-						kindsStr = "<catch-all> (all other kinds)"
-					} else {
-						kindsStr = strings.Join(rf.Kinds, ", ")
-					}
-					d.Printf("      %s:\n", kindsStr)
-
-					// Label selector
-					if len(rf.LabelSelector) > 0 {
-						selectorStr := formatLabelMap(rf.LabelSelector)
-						d.Printf("        Label selector:     %s\n", selectorStr)
-					} else if len(rf.OrLabelSelectors) > 0 {
-						var orStrs []string
-						for _, ols := range rf.OrLabelSelectors {
-							orStrs = append(orStrs, formatLabelMap(ols))
-						}
-						d.Printf("        OR label selectors: [%s]\n", strings.Join(orStrs, ", "))
-					} else {
-						d.Printf("        Label selector:     <none>\n")
-					}
-
-					// Name patterns
-					if len(rf.Names) > 0 {
-						d.Printf("        Included names:     [%s]\n", strings.Join(rf.Names, ", "))
-					} else {
-						d.Printf("        Included names:     <none>\n")
-					}
-
-					if len(rf.ExcludedNames) > 0 {
-						d.Printf("        Excluded names:     [%s]\n", strings.Join(rf.ExcludedNames, ", "))
-					} else {
-						d.Printf("        Excluded names:     <none>\n")
-					}
-				}
-			}
-		}
-	}
-}
-
-func formatLabelMap(labelMap map[string]string) string {
-	var pairs []string
-	for k, v := range labelMap {
-		pairs = append(pairs, fmt.Sprintf("%s=%s", k, v))
-	}
-	return strings.Join(pairs, ",")
-}
-
 // DescribeUploaderConfigForBackup describes uploader config in human-readable format
 func DescribeUploaderConfigForBackup(d *Describer, spec velerov1api.BackupSpec) {
 	d.Printf("Uploader config:\n")

pkg/cmd/util/output/backup_describer_test.go

@@ -18,7 +18,6 @@ package output
 
 import (
 	"bytes"
-	"context"
 	"testing"
 	"text/tabwriter"
 	"time"
@@ -26,8 +25,6 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	corev1api "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	"github.com/vmware-tanzu/velero/internal/volume"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
@@ -897,85 +894,3 @@ func TestDescribeBackupItemOperation(t *testing.T) {
 	d.out.Flush()
 	assert.Equal(t, expected, d.buf.String())
 }
-
-func TestDescribeFineGrainedFilterPolicies(t *testing.T) {
-	yamlData := `
-version: v1
-clusterScopedFilterPolicy:
-  resourceFilters:
-  - kinds: ["StorageClass"]
-    labelSelector: {"app": "velero"}
-  - kinds: ["ClusterRole"]
-    orLabelSelectors:
-    - {"app": "velero"}
-    - {"app": "test"}
-    names: ["role1"]
-    excludedNames: ["role2"]
-namespacedFilterPolicies:
-- namespaces: ["ns1", "ns2"]
-  resourceFilters:
-  - kinds: ["Pod", "ConfigMap"]
-    labelSelector: {"app": "velero"}
-  - kinds: ["*"]
-`
-	cm := &corev1api.ConfigMap{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test-policy",
-			Namespace: "velero",
-		},
-		Data: map[string]string{
-			"policy.yaml": yamlData,
-		},
-	}
-
-	client := fake.NewClientBuilder().WithRuntimeObjects(cm).Build()
-
-	backup := builder.ForBackup("velero", "test-backup").
-		ResourcePolicies("test-policy").Result()
-
-	d := &Describer{
-		Prefix: "",
-		out:    &tabwriter.Writer{},
-		buf:    &bytes.Buffer{},
-	}
-	d.out.Init(d.buf, 0, 8, 2, ' ', 0)
-
-	DescribeFineGrainedFilterPolicies(context.Background(), client, d, backup)
-	d.out.Flush()
-
-	expected := `
-Cluster Scoped Filter Policy:
-  Resource Filters:
-    StorageClass:
-      Label selector:     app=velero
-      Included names:     <none>
-      Excluded names:     <none>
-    ClusterRole:
-      OR label selectors: [app=velero, app=test]
-      Included names:     [role1]
-      Excluded names:     [role2]
-
-Namespace-Scoped Filter Policies:
-  ns1:
-    Resource Filters:
-      Pod, ConfigMap:
-        Label selector:     app=velero
-        Included names:     <none>
-        Excluded names:     <none>
-      <catch-all> (all other kinds):
-        Label selector:     <none>
-        Included names:     <none>
-        Excluded names:     <none>
-  ns2:
-    Resource Filters:
-      Pod, ConfigMap:
-        Label selector:     app=velero
-        Included names:     <none>
-        Excluded names:     <none>
-      <catch-all> (all other kinds):
-        Label selector:     <none>
-        Included names:     <none>
-        Excluded names:     <none>
-`
-	assert.Equal(t, expected, d.buf.String())
-}

pkg/cmd/util/output/backup_structured_describer.go

@@ -21,10 +21,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
 	"strings"
 
-	"github.com/sirupsen/logrus"
 	corev1api "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
@@ -57,7 +55,6 @@ func DescribeBackupInSF(
 
 		if backup.Spec.ResourcePolicy != nil {
 			DescribeResourcePoliciesInSF(d, backup.Spec.ResourcePolicy)
-			DescribeFineGrainedFilterPoliciesInSF(ctx, kbClient, d, backup)
 		}
 
 		DescribeGlobalVolumePolicyInSF(d, backup)
@@ -228,88 +225,6 @@ func DescribeBackupSpecInSF(d *StructuredDescriber, spec velerov1api.BackupSpec)
 	d.Describe("spec", backupSpecInfo)
 }
 
-// DescribeFineGrainedFilterPoliciesInSF adds the clusterScopedFilterPolicy
-// and namespacedFilterPolicies sections to the structured describer output when present
-// in the ResourcePolicy ConfigMap referenced by the backup.
-func DescribeFineGrainedFilterPoliciesInSF(ctx context.Context, kbClient kbclient.Client, d *StructuredDescriber, backup *velerov1api.Backup) {
-	if backup.Spec.ResourcePolicy == nil {
-		return
-	}
-
-	discardLogger := logrus.New()
-	discardLogger.Out = io.Discard
-
-	resPolicies, err := resourcepolicies.GetResourcePoliciesFromBackup(*backup, kbClient, discardLogger)
-	if err != nil || resPolicies == nil {
-		return
-	}
-
-	clusterScopedFilterPolicy := resPolicies.GetClusterScopedFilterPolicy()
-	if clusterScopedFilterPolicy != nil {
-		var clusterScopedFilters []map[string]any
-		for _, rf := range clusterScopedFilterPolicy.ResourceFilters {
-			entry := map[string]any{
-				"kinds": rf.Kinds,
-			}
-			if len(rf.LabelSelector) > 0 {
-				entry["labelSelector"] = rf.LabelSelector
-			}
-			if len(rf.OrLabelSelectors) > 0 {
-				entry["orLabelSelectors"] = rf.OrLabelSelectors
-			}
-			if len(rf.Names) > 0 {
-				entry["names"] = rf.Names
-			}
-			if len(rf.ExcludedNames) > 0 {
-				entry["excludedNames"] = rf.ExcludedNames
-			}
-			clusterScopedFilters = append(clusterScopedFilters, entry)
-		}
-		d.Describe("clusterScopedFilterPolicy", map[string]any{
-			"resourceFilters": clusterScopedFilters,
-		})
-	}
-
-	nfPolicies := resPolicies.GetNamespacedFilterPolicies()
-	if len(nfPolicies) == 0 {
-		return
-	}
-
-	var structuredPolicies []map[string]any
-	for _, policy := range nfPolicies {
-		for _, ns := range policy.Namespaces {
-			var rfEntries []map[string]any
-			for _, rf := range policy.ResourceFilters {
-				entry := map[string]any{}
-				if rf.IsCatchAll() {
-					entry["kinds"] = []string{}
-					entry["isCatchAll"] = true
-				} else {
-					entry["kinds"] = rf.Kinds
-				}
-				if len(rf.LabelSelector) > 0 {
-					entry["labelSelector"] = rf.LabelSelector
-				}
-				if len(rf.OrLabelSelectors) > 0 {
-					entry["orLabelSelectors"] = rf.OrLabelSelectors
-				}
-				if len(rf.Names) > 0 {
-					entry["names"] = rf.Names
-				}
-				if len(rf.ExcludedNames) > 0 {
-					entry["excludedNames"] = rf.ExcludedNames
-				}
-				rfEntries = append(rfEntries, entry)
-			}
-			structuredPolicies = append(structuredPolicies, map[string]any{
-				"namespace":       ns,
-				"resourceFilters": rfEntries,
-			})
-		}
-	}
-	d.Describe("namespacedFilterPolicies", structuredPolicies)
-}
-
 // DescribeBackupStatusInSF describes a backup status in structured format.
 func DescribeBackupStatusInSF(ctx context.Context, kbClient kbclient.Client, d *StructuredDescriber, backup *velerov1api.Backup, details bool,
 	insecureSkipTLSVerify bool, caCertPath string, podVolumeBackups []velerov1api.PodVolumeBackup) {

pkg/cmd/util/output/backup_structured_describer_test.go

@@ -17,7 +17,6 @@ limitations under the License.
 package output
 
 import (
-	"context"
 	"reflect"
 	"testing"
 	"time"
@@ -25,8 +24,6 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	corev1api "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	"github.com/vmware-tanzu/velero/internal/volume"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
@@ -731,96 +728,3 @@ func TestDescribeDeleteBackupRequestsInSF(t *testing.T) {
 		})
 	}
 }
-
-func TestDescribeFineGrainedFilterPoliciesInSF(t *testing.T) {
-	yamlData := `
-version: v1
-clusterScopedFilterPolicy:
-  resourceFilters:
-  - kinds: ["StorageClass"]
-    labelSelector: {"app": "velero"}
-  - kinds: ["ClusterRole"]
-    orLabelSelectors:
-    - {"app": "velero"}
-    - {"app": "test"}
-    names: ["role1"]
-    excludedNames: ["role2"]
-namespacedFilterPolicies:
-- namespaces: ["ns1", "ns2"]
-  resourceFilters:
-  - kinds: ["Pod", "ConfigMap"]
-    labelSelector: {"app": "velero"}
-  - kinds: ["*"]
-`
-	cm := &corev1api.ConfigMap{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test-policy",
-			Namespace: "velero",
-		},
-		Data: map[string]string{
-			"policy.yaml": yamlData,
-		},
-	}
-
-	client := fake.NewClientBuilder().WithRuntimeObjects(cm).Build()
-
-	backup := builder.ForBackup("velero", "test-backup").
-		ResourcePolicies("test-policy").Result()
-
-	sd := &StructuredDescriber{
-		output: make(map[string]any),
-		format: "",
-	}
-
-	DescribeFineGrainedFilterPoliciesInSF(context.Background(), client, sd, backup)
-
-	expect := map[string]any{
-		"clusterScopedFilterPolicy": map[string]any{
-			"resourceFilters": []map[string]any{
-				{
-					"kinds":         []string{"StorageClass"},
-					"labelSelector": map[string]string{"app": "velero"},
-				},
-				{
-					"kinds": []string{"ClusterRole"},
-					"orLabelSelectors": []map[string]string{
-						{"app": "velero"},
-						{"app": "test"},
-					},
-					"names":         []string{"role1"},
-					"excludedNames": []string{"role2"},
-				},
-			},
-		},
-		"namespacedFilterPolicies": []map[string]any{
-			{
-				"namespace": "ns1",
-				"resourceFilters": []map[string]any{
-					{
-						"kinds":         []string{"Pod", "ConfigMap"},
-						"labelSelector": map[string]string{"app": "velero"},
-					},
-					{
-						"kinds":      []string{},
-						"isCatchAll": true,
-					},
-				},
-			},
-			{
-				"namespace": "ns2",
-				"resourceFilters": []map[string]any{
-					{
-						"kinds":         []string{"Pod", "ConfigMap"},
-						"labelSelector": map[string]string{"app": "velero"},
-					},
-					{
-						"kinds":      []string{},
-						"isCatchAll": true,
-					},
-				},
-			},
-		},
-	}
-
-	assert.True(t, reflect.DeepEqual(sd.output, expect))
-}

pkg/controller/backup_controller.go

@@ -84,33 +84,34 @@ var autoExcludeClusterScopedResources = []string{
 }
 
 type backupReconciler struct {
-	ctx                           context.Context
-	logger                        logrus.FieldLogger
-	discoveryHelper               discovery.Helper
-	backupper                     pkgbackup.Backupper
-	kbClient                      kbclient.Client
-	clock                         clock.WithTickerAndDelayedExecution
-	backupLogLevel                logrus.Level
-	newPluginManager              func(logrus.FieldLogger) clientmgmt.Manager
-	backupTracker                 BackupTracker
-	defaultBackupLocation         string
-	defaultVolumesToFsBackup      bool
-	defaultBackupTTL              time.Duration
-	defaultVGSLabelKey            string
-	defaultCSISnapshotTimeout     time.Duration
-	resourceTimeout               time.Duration
-	defaultItemOperationTimeout   time.Duration
-	defaultSnapshotLocations      map[string]string
-	metrics                       *metrics.ServerMetrics
-	backupStoreGetter             persistence.ObjectBackupStoreGetter
-	formatFlag                    logging.Format
-	credentialFileStore           credentials.FileStore
-	maxConcurrentK8SConnections   int
-	defaultSnapshotMoveData       bool
-	globalCRClient                kbclient.Client
-	itemBlockWorkerCount          int
-	concurrentBackups             int
-	globalVolumePoliciesConfigMap string
+	ctx                                context.Context
+	logger                             logrus.FieldLogger
+	discoveryHelper                    discovery.Helper
+	backupper                          pkgbackup.Backupper
+	kbClient                           kbclient.Client
+	clock                              clock.WithTickerAndDelayedExecution
+	backupLogLevel                     logrus.Level
+	newPluginManager                   func(logrus.FieldLogger) clientmgmt.Manager
+	backupTracker                      BackupTracker
+	defaultBackupLocation              string
+	defaultVolumesToFsBackup           bool
+	defaultBackupTTL                   time.Duration
+	defaultVGSLabelKey                 string
+	defaultCSISnapshotTimeout          time.Duration
+	resourceTimeout                    time.Duration
+	defaultItemOperationTimeout        time.Duration
+	defaultSnapshotLocations           map[string]string
+	metrics                            *metrics.ServerMetrics
+	backupStoreGetter                  persistence.ObjectBackupStoreGetter
+	formatFlag                         logging.Format
+	credentialFileStore                credentials.FileStore
+	maxConcurrentK8SConnections        int
+	defaultSnapshotMoveData            bool
+	globalCRClient                     kbclient.Client
+	itemBlockWorkerCount               int
+	concurrentBackups                  int
+	globalVolumePoliciesConfigMap      string
+	knownSchedulesWithSuccessfulBackup sets.Set[string]
 }
 
 func NewBackupReconciler(
@@ -204,28 +205,43 @@ func (b *backupReconciler) updateTotalBackupMetric() {
 		time.Sleep(5 * time.Second)
 
 		wait.Until(
-			func() {
-				// recompute backup_total metric
-				backups := &velerov1api.BackupList{}
-				err := b.kbClient.List(context.Background(), backups, &kbclient.ListOptions{LabelSelector: labels.Everything()})
-				if err != nil {
-					b.logger.Error(err, "Error computing backup_total metric")
-				} else {
-					b.metrics.SetBackupTotal(int64(len(backups.Items)))
-				}
-
-				// recompute backup_last_successful_timestamp metric for each
-				// schedule (including the empty schedule, i.e. ad-hoc backups)
-				for schedule, timestamp := range getLastSuccessBySchedule(backups.Items) {
-					b.metrics.SetBackupLastSuccessfulTimestamp(schedule, timestamp)
-				}
-			},
+			b.resyncBackupMetrics,
 			backupResyncPeriod,
 			b.ctx.Done(),
 		)
 	}()
 }
 
+func (b *backupReconciler) resyncBackupMetrics() {
+	backups := &velerov1api.BackupList{}
+	err := b.kbClient.List(context.Background(), backups, &kbclient.ListOptions{LabelSelector: labels.Everything()})
+	if err != nil {
+		b.logger.Error(err, "Error computing backup_total metric")
+		return
+	}
+
+	b.metrics.SetBackupTotal(int64(len(backups.Items)))
+
+	currentSchedules := getLastSuccessBySchedule(backups.Items)
+	for schedule, timestamp := range currentSchedules {
+		b.metrics.SetBackupLastSuccessfulTimestamp(schedule, timestamp)
+	}
+
+	// Remove metrics for schedules that no longer have successful backups
+	if b.knownSchedulesWithSuccessfulBackup != nil {
+		for schedule := range b.knownSchedulesWithSuccessfulBackup {
+			if _, exists := currentSchedules[schedule]; !exists {
+				b.metrics.DeleteBackupLastSuccessfulTimestamp(schedule)
+			}
+		}
+	}
+
+	b.knownSchedulesWithSuccessfulBackup = sets.New[string]()
+	for schedule := range currentSchedules {
+		b.knownSchedulesWithSuccessfulBackup.Insert(schedule)
+	}
+}
+
 // getLastSuccessBySchedule finds the most recent completed backup for each schedule
 // and returns a map of schedule name -> completion time of the most recent completed
 // backup. This map includes an entry for ad-hoc/non-scheduled backups, where the key

pkg/controller/backup_controller_test.go

@@ -31,6 +31,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
+	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
@@ -2041,6 +2042,48 @@ func Test_getLastSuccessBySchedule(t *testing.T) {
 	}
 }
 
+// Test_resyncBackupMetrics_prunesStaleTimestamps verifies that resyncBackupMetrics
+// removes backupLastSuccessfulTimestamp entries for schedules that no longer have
+// any completed backups (e.g. after the schedule and its backups are deleted).
+func Test_resyncBackupMetrics_prunesStaleTimestamps(t *testing.T) {
+	baseTime, err := time.Parse(time.RFC1123, time.RFC1123)
+	require.NoError(t, err)
+
+	m := metrics.NewServerMetrics()
+	gauge := m.Metrics()["backup_last_successful_timestamp"]
+
+	activeBackup := builder.ForBackup("velero", "b1").
+		ObjectMeta(builder.WithLabels(velerov1api.ScheduleNameLabel, "active-schedule")).
+		Phase(velerov1api.BackupPhaseCompleted).
+		CompletionTimestamp(baseTime).
+		Result()
+
+	deletedBackup := builder.ForBackup("velero", "b2").
+		ObjectMeta(builder.WithLabels(velerov1api.ScheduleNameLabel, "deleted-schedule")).
+		Phase(velerov1api.BackupPhaseCompleted).
+		CompletionTimestamp(baseTime).
+		Result()
+
+	fakeClient := velerotest.NewFakeControllerRuntimeClient(t, activeBackup, deletedBackup)
+
+	c := &backupReconciler{
+		kbClient: fakeClient,
+		logger:   logrus.StandardLogger(),
+		metrics:  m,
+	}
+
+	// First resync: sets metrics for both schedules
+	c.resyncBackupMetrics()
+	assert.Equal(t, 2, testutil.CollectAndCount(gauge))
+
+	// Simulate schedule deletion: remove the backup for "deleted-schedule"
+	require.NoError(t, fakeClient.Delete(t.Context(), deletedBackup))
+
+	// Second resync: prunes "deleted-schedule" metric, keeps "active-schedule"
+	c.resyncBackupMetrics()
+	assert.Equal(t, 1, testutil.CollectAndCount(gauge))
+}
+
 // Unit tests to make sure that the backup's status is updated correctly during reconcile.
 // To clear up confusion whether status can be updated with Patch alone without status writer and not kbClient.Status().Patch()
 func TestPatchResourceWorksWithStatus(t *testing.T) {

pkg/controller/data_download_controller.go

@@ -454,7 +454,7 @@ func (r *DataDownloadReconciler) startCancelableDataPath(asyncBR datapath.AsyncB
 
 	if err := asyncBR.StartRestore(dd.Spec.SnapshotID, datapath.AccessPoint{
 		ByPath: res.ByPod.VolumeName,
-	}, dd.Spec.DataMoverConfig); err != nil {
+	}, dd.Spec.DataMoverConfig, nil); err != nil {
 		return errors.Wrapf(err, "error starting async restore for pod %s, volume %s", res.ByPod.HostingPod.Name, res.ByPod.VolumeName)
 	}
 
@@ -1096,7 +1096,7 @@ func (r *DataDownloadReconciler) resumeCancellableDataPath(ctx context.Context,
 
 	if err := asyncBR.StartRestore(dd.Spec.SnapshotID, datapath.AccessPoint{
 		ByPath: res.ByPod.VolumeName,
-	}, nil); err != nil {
+	}, nil, nil); err != nil {
 		return errors.Wrapf(err, "error to resume asyncBR watcher for dd %s", dd.Name)
 	}
 

pkg/controller/data_download_controller_test.go

@@ -529,7 +529,7 @@ func TestDataDownloadReconcile(t *testing.T) {
 				}
 
 				if test.mockStart {
-					asyncBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(test.mockStartErr)
+					asyncBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.mockStartErr)
 				}
 
 				if test.mockCancel {
@@ -1288,7 +1288,7 @@ func TestResumeCancellableRestore(t *testing.T) {
 			}
 
 			if test.mockStart {
-				mockAsyncBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(test.startWatcherErr)
+				mockAsyncBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.startWatcherErr)
 			}
 
 			if test.mockClose {

pkg/controller/data_upload_controller_test.go

@@ -348,7 +348,7 @@ func (f *fakeFSBR) StartBackup(source datapath.AccessPoint, uploaderConfigs map[
 	return f.startErr
 }
 
-func (f *fakeFSBR) StartRestore(snapshotID string, target datapath.AccessPoint, uploaderConfigs map[string]string) error {
+func (f *fakeFSBR) StartRestore(snapshotID string, target datapath.AccessPoint, uploaderConfigs map[string]string, param any) error {
 	return nil
 }
 

pkg/controller/pod_volume_restore_controller.go

@@ -236,9 +236,9 @@ func (r *PodVolumeRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Req
 			return ctrl.Result{}, nil
 		}
 
-		shouldProcess, pod, err := shouldProcess(ctx, r.client, log, pvr)
+		shouldProcess, pod, err := shouldProcess(ctx, r.client, log, pvr, r.resourceTimeout)
 		if err != nil {
-			return ctrl.Result{}, err
+			return r.errorOut(ctx, pvr, err, "Pod for this PVR is not ready", log)
 		}
 		if !shouldProcess {
 			return ctrl.Result{}, nil
@@ -528,7 +528,7 @@ func (r *PodVolumeRestoreReconciler) startCancelableDataPath(asyncBR datapath.As
 
 	if err := asyncBR.StartRestore(pvr.Spec.SnapshotID, datapath.AccessPoint{
 		ByPath: res.ByPod.VolumeName,
-	}, pvr.Spec.UploaderSettings); err != nil {
+	}, pvr.Spec.UploaderSettings, nil); err != nil {
 		return errors.Wrapf(err, "error starting async restore for pod %s, volume %s", res.ByPod.HostingPod.Name, res.ByPod.VolumeName)
 	}
 
@@ -565,7 +565,7 @@ func UpdatePVRStatusToFailed(ctx context.Context, c client.Client, pvr *velerov1
 	return err
 }
 
-func shouldProcess(ctx context.Context, client client.Client, log logrus.FieldLogger, pvr *velerov1api.PodVolumeRestore) (bool, *corev1api.Pod, error) {
+func shouldProcess(ctx context.Context, client client.Client, log logrus.FieldLogger, pvr *velerov1api.PodVolumeRestore, timeout time.Duration) (bool, *corev1api.Pod, error) {
 	if !isPVRNew(pvr) {
 		log.Debug("PVR is not new, skip")
 		return false, nil, nil
@@ -573,22 +573,63 @@ func shouldProcess(ctx context.Context, client client.Client, log logrus.FieldLo
 
 	// we filter the pods during the initialization of cache, if we can get a pod here, the pod must be in the same node with the controller
 	// so we don't need to compare the node anymore
-	pod := &corev1api.Pod{}
-	if err := client.Get(ctx, types.NamespacedName{Namespace: pvr.Spec.Pod.Namespace, Name: pvr.Spec.Pod.Name}, pod); err != nil {
-		if apierrors.IsNotFound(err) {
-			log.WithError(err).Debug("Pod not found on this node, skip")
-			return false, nil, nil
+	var targetPod *corev1api.Pod
+	err := wait.PollUntilContextTimeout(ctx, time.Millisecond*100, timeout, true, func(ctx context.Context) (bool, error) {
+		updated := &corev1api.Pod{}
+		if err := client.Get(ctx, types.NamespacedName{Namespace: pvr.Spec.Pod.Namespace, Name: pvr.Spec.Pod.Name}, updated); err != nil {
+			if apierrors.IsNotFound(err) {
+				return false, nil
+			}
+
+			return false, err
+		}
+
+		targetPod = updated
+
+		return true, nil
+	})
+
+	if err != nil {
+		if errors.Is(err, context.DeadlineExceeded) {
+			return false, nil, errors.Errorf("timeout to wait for pod %s/%s", pvr.Spec.Pod.Namespace, pvr.Spec.Pod.Name)
+		} else {
+			return false, nil, errors.Wrapf(err, "error waiting for pod %s/%s", pvr.Spec.Pod.Namespace, pvr.Spec.Pod.Name)
 		}
-		log.WithError(err).Error("Unable to get pod")
-		return false, nil, err
 	}
 
-	if !isInitContainerRunning(pod) {
+	if targetPod.Status.Phase == corev1api.PodFailed || targetPod.Status.Phase == corev1api.PodUnknown {
+		return false, nil, errors.Errorf("unexpected state for pod %s/%s", targetPod.Namespace, targetPod.Name)
+	}
+
+	idx := getInitContainerIndex(targetPod)
+	if idx < 0 {
+		return false, nil, errors.Errorf("no restore-wait init container in pod %s/%s", targetPod.Namespace, targetPod.Name)
+	}
+
+	if len(targetPod.Status.InitContainerStatuses) <= idx {
+		log.Debug("Pod init container statuses are not fully populated yet, skip")
+		return false, nil, nil
+	}
+
+	containerStatus := targetPod.Status.InitContainerStatuses[idx]
+
+	if containerStatus.State.Terminated != nil {
+		return false, nil, errors.Errorf("restore-wait init container has already completed in pod %s/%s", targetPod.Namespace, targetPod.Name)
+	}
+
+	if containerStatus.State.Waiting != nil {
+		reason := containerStatus.State.Waiting.Reason
+		if reason == "ImagePullBackOff" || reason == "ErrImageNeverPull" || reason == "CreateContainerConfigError" || reason == "CreateContainerError" || reason == "InvalidImageName" || reason == "ErrImagePull" {
+			return false, nil, errors.Errorf("restore-wait init container in pod %s/%s is in unrecoverable waiting state with reason %s", targetPod.Namespace, targetPod.Name, reason)
+		}
+	}
+
+	if containerStatus.State.Running == nil {
 		log.Debug("Pod is not running restore-wait init container, skip")
 		return false, nil, nil
 	}
 
-	return true, pod, nil
+	return true, targetPod, nil
 }
 
 func (r *PodVolumeRestoreReconciler) closeDataPath(ctx context.Context, pvrName string) {
@@ -770,14 +811,6 @@ func isPVRNew(pvr *velerov1api.PodVolumeRestore) bool {
 	return pvr.Status.Phase == "" || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseNew
 }
 
-func isInitContainerRunning(pod *corev1api.Pod) bool {
-	// Pod volume wait container can be anywhere in the list of init containers, but must be running.
-	i := getInitContainerIndex(pod)
-	return i >= 0 &&
-		len(pod.Status.InitContainerStatuses)-1 >= i &&
-		pod.Status.InitContainerStatuses[i].State.Running != nil
-}
-
 func getInitContainerIndex(pod *corev1api.Pod) int {
 	// Pod volume wait container can be anywhere in the list of init containers so locate it.
 	for i, initContainer := range pod.Spec.InitContainers {
@@ -1113,7 +1146,7 @@ func (r *PodVolumeRestoreReconciler) resumeCancellableDataPath(ctx context.Conte
 
 	if err := asyncBR.StartRestore(pvr.Spec.SnapshotID, datapath.AccessPoint{
 		ByPath: res.ByPod.VolumeName,
-	}, pvr.Spec.UploaderSettings); err != nil {
+	}, pvr.Spec.UploaderSettings, nil); err != nil {
 		return errors.Wrapf(err, "error to resume asyncBR watcher for PVR %s", pvr.Name)
 	}
 

pkg/controller/pod_volume_restore_controller_test.go

@@ -65,6 +65,8 @@ func TestShouldProcess(t *testing.T) {
 		obj             *velerov1api.PodVolumeRestore
 		pod             *corev1api.Pod
 		shouldProcessed bool
+		expectError     bool
+		errString       string
 	}{
 		{
 			name: "InProgress phase pvr should not be processed",
@@ -115,6 +117,8 @@ func TestShouldProcess(t *testing.T) {
 				},
 			},
 			shouldProcessed: false,
+			expectError:     true,
+			errString:       "timeout to wait for pod ns-1/pod-1",
 		},
 		{
 			name: "Empty phase pvr with pod on node not running init container should not be processed",
@@ -200,6 +204,268 @@ func TestShouldProcess(t *testing.T) {
 			},
 			shouldProcessed: true,
 		},
+		{
+			name: "pod is in failed phase should return error",
+			obj: &velerov1api.PodVolumeRestore{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "velero",
+					Name:      "pvr-1",
+				},
+				Spec: velerov1api.PodVolumeRestoreSpec{
+					Pod: corev1api.ObjectReference{
+						Namespace: "ns-1",
+						Name:      "pod-1",
+					},
+				},
+				Status: velerov1api.PodVolumeRestoreStatus{
+					Phase: "",
+				},
+			},
+			pod: &corev1api.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "ns-1",
+					Name:      "pod-1",
+				},
+				Status: corev1api.PodStatus{
+					Phase: corev1api.PodFailed,
+				},
+			},
+			shouldProcessed: false,
+			expectError:     true,
+			errString:       "unexpected state for pod",
+		},
+		{
+			name: "pod is in unknown phase should return error",
+			obj: &velerov1api.PodVolumeRestore{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "velero",
+					Name:      "pvr-1",
+				},
+				Spec: velerov1api.PodVolumeRestoreSpec{
+					Pod: corev1api.ObjectReference{
+						Namespace: "ns-1",
+						Name:      "pod-1",
+					},
+				},
+				Status: velerov1api.PodVolumeRestoreStatus{
+					Phase: "",
+				},
+			},
+			pod: &corev1api.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "ns-1",
+					Name:      "pod-1",
+				},
+				Status: corev1api.PodStatus{
+					Phase: corev1api.PodUnknown,
+				},
+			},
+			shouldProcessed: false,
+			expectError:     true,
+			errString:       "unexpected state for pod",
+		},
+		{
+			name: "pod with no init containers should return error",
+			obj: &velerov1api.PodVolumeRestore{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "velero",
+					Name:      "pvr-1",
+				},
+				Spec: velerov1api.PodVolumeRestoreSpec{
+					Pod: corev1api.ObjectReference{
+						Namespace: "ns-1",
+						Name:      "pod-1",
+					},
+				},
+				Status: velerov1api.PodVolumeRestoreStatus{
+					Phase: "",
+				},
+			},
+			pod: &corev1api.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "ns-1",
+					Name:      "pod-1",
+				},
+				Spec: corev1api.PodSpec{
+					NodeName: controllerNode,
+				},
+			},
+			shouldProcessed: false,
+			expectError:     true,
+			errString:       "no restore-wait init container",
+		},
+		{
+			name: "pod init container statuses are not fully populated yet should skip",
+			obj: &velerov1api.PodVolumeRestore{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "velero",
+					Name:      "pvr-1",
+				},
+				Spec: velerov1api.PodVolumeRestoreSpec{
+					Pod: corev1api.ObjectReference{
+						Namespace: "ns-1",
+						Name:      "pod-1",
+					},
+				},
+				Status: velerov1api.PodVolumeRestoreStatus{
+					Phase: "",
+				},
+			},
+			pod: &corev1api.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "ns-1",
+					Name:      "pod-1",
+				},
+				Spec: corev1api.PodSpec{
+					NodeName: controllerNode,
+					InitContainers: []corev1api.Container{
+						{
+							Name: restorehelper.WaitInitContainer,
+						},
+					},
+				},
+				Status: corev1api.PodStatus{
+					InitContainerStatuses: []corev1api.ContainerStatus{},
+				},
+			},
+			shouldProcessed: false,
+		},
+		{
+			name: "restore-wait init container has already completed should return error",
+			obj: &velerov1api.PodVolumeRestore{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "velero",
+					Name:      "pvr-1",
+				},
+				Spec: velerov1api.PodVolumeRestoreSpec{
+					Pod: corev1api.ObjectReference{
+						Namespace: "ns-1",
+						Name:      "pod-1",
+					},
+				},
+				Status: velerov1api.PodVolumeRestoreStatus{
+					Phase: "",
+				},
+			},
+			pod: &corev1api.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "ns-1",
+					Name:      "pod-1",
+				},
+				Spec: corev1api.PodSpec{
+					NodeName: controllerNode,
+					InitContainers: []corev1api.Container{
+						{
+							Name: restorehelper.WaitInitContainer,
+						},
+					},
+				},
+				Status: corev1api.PodStatus{
+					InitContainerStatuses: []corev1api.ContainerStatus{
+						{
+							State: corev1api.ContainerState{
+								Terminated: &corev1api.ContainerStateTerminated{
+									ExitCode: 0,
+								},
+							},
+						},
+					},
+				},
+			},
+			shouldProcessed: false,
+			expectError:     true,
+			errString:       "restore-wait init container has already completed",
+		},
+		{
+			name: "restore-wait init container is in unrecoverable waiting state should return error",
+			obj: &velerov1api.PodVolumeRestore{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "velero",
+					Name:      "pvr-1",
+				},
+				Spec: velerov1api.PodVolumeRestoreSpec{
+					Pod: corev1api.ObjectReference{
+						Namespace: "ns-1",
+						Name:      "pod-1",
+					},
+				},
+				Status: velerov1api.PodVolumeRestoreStatus{
+					Phase: "",
+				},
+			},
+			pod: &corev1api.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "ns-1",
+					Name:      "pod-1",
+				},
+				Spec: corev1api.PodSpec{
+					NodeName: controllerNode,
+					InitContainers: []corev1api.Container{
+						{
+							Name: restorehelper.WaitInitContainer,
+						},
+					},
+				},
+				Status: corev1api.PodStatus{
+					InitContainerStatuses: []corev1api.ContainerStatus{
+						{
+							State: corev1api.ContainerState{
+								Waiting: &corev1api.ContainerStateWaiting{
+									Reason: "ImagePullBackOff",
+								},
+							},
+						},
+					},
+				},
+			},
+			shouldProcessed: false,
+			expectError:     true,
+			errString:       "is in unrecoverable waiting state with reason ImagePullBackOff",
+		},
+		{
+			name: "restore-wait init container is in normal waiting state should skip",
+			obj: &velerov1api.PodVolumeRestore{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "velero",
+					Name:      "pvr-1",
+				},
+				Spec: velerov1api.PodVolumeRestoreSpec{
+					Pod: corev1api.ObjectReference{
+						Namespace: "ns-1",
+						Name:      "pod-1",
+					},
+				},
+				Status: velerov1api.PodVolumeRestoreStatus{
+					Phase: "",
+				},
+			},
+			pod: &corev1api.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "ns-1",
+					Name:      "pod-1",
+				},
+				Spec: corev1api.PodSpec{
+					NodeName: controllerNode,
+					InitContainers: []corev1api.Container{
+						{
+							Name: restorehelper.WaitInitContainer,
+						},
+					},
+				},
+				Status: corev1api.PodStatus{
+					InitContainerStatuses: []corev1api.ContainerStatus{
+						{
+							State: corev1api.ContainerState{
+								Waiting: &corev1api.ContainerStateWaiting{
+									Reason: "ContainerCreating",
+								},
+							},
+						},
+					},
+				},
+			},
+			shouldProcessed: false,
+		},
 	}
 
 	for _, ts := range tests {
@@ -221,179 +487,16 @@ func TestShouldProcess(t *testing.T) {
 				clock:  &clocks.RealClock{},
 			}
 
-			shouldProcess, _, _ := shouldProcess(ctx, c.client, c.logger, ts.obj)
+			shouldProcess, _, err := shouldProcess(ctx, c.client, c.logger, ts.obj, time.Second)
 			require.Equal(t, ts.shouldProcessed, shouldProcess)
-		})
-	}
-}
-
-func TestIsInitContainerRunning(t *testing.T) {
-	tests := []struct {
-		name     string
-		pod      *corev1api.Pod
-		expected bool
-	}{
-		{
-			name: "pod with no init containers should return false",
-			pod: &corev1api.Pod{
-				ObjectMeta: metav1.ObjectMeta{
-					Namespace: "ns-1",
-					Name:      "pod-1",
-				},
-			},
-			expected: false,
-		},
-		{
-			name: "pod with running init container that's not restore init should return false",
-			pod: &corev1api.Pod{
-				ObjectMeta: metav1.ObjectMeta{
-					Namespace: "ns-1",
-					Name:      "pod-1",
-				},
-				Spec: corev1api.PodSpec{
-					InitContainers: []corev1api.Container{
-						{
-							Name: "non-restore-init",
-						},
-					},
-				},
-				Status: corev1api.PodStatus{
-					InitContainerStatuses: []corev1api.ContainerStatus{
-						{
-							State: corev1api.ContainerState{
-								Running: &corev1api.ContainerStateRunning{StartedAt: metav1.Time{Time: time.Now()}},
-							},
-						},
-					},
-				},
-			},
-			expected: false,
-		},
-		{
-			name: "pod with running init container that's not first should still work",
-			pod: &corev1api.Pod{
-				ObjectMeta: metav1.ObjectMeta{
-					Namespace: "ns-1",
-					Name:      "pod-1",
-				},
-				Spec: corev1api.PodSpec{
-					InitContainers: []corev1api.Container{
-						{
-							Name: "non-restore-init",
-						},
-						{
-							Name: restorehelper.WaitInitContainer,
-						},
-					},
-				},
-				Status: corev1api.PodStatus{
-					InitContainerStatuses: []corev1api.ContainerStatus{
-						{
-							State: corev1api.ContainerState{
-								Running: &corev1api.ContainerStateRunning{StartedAt: metav1.Time{Time: time.Now()}},
-							},
-						},
-						{
-							State: corev1api.ContainerState{
-								Running: &corev1api.ContainerStateRunning{StartedAt: metav1.Time{Time: time.Now()}},
-							},
-						},
-					},
-				},
-			},
-			expected: true,
-		},
-		{
-			name: "pod with init container as first initContainer that's not running should return false",
-			pod: &corev1api.Pod{
-				ObjectMeta: metav1.ObjectMeta{
-					Namespace: "ns-1",
-					Name:      "pod-1",
-				},
-				Spec: corev1api.PodSpec{
-					InitContainers: []corev1api.Container{
-						{
-							Name: restorehelper.WaitInitContainer,
-						},
-						{
-							Name: "non-restore-init",
-						},
-					},
-				},
-				Status: corev1api.PodStatus{
-					InitContainerStatuses: []corev1api.ContainerStatus{
-						{
-							State: corev1api.ContainerState{},
-						},
-						{
-							State: corev1api.ContainerState{
-								Running: &corev1api.ContainerStateRunning{StartedAt: metav1.Time{Time: time.Now()}},
-							},
-						},
-					},
-				},
-			},
-			expected: false,
-		},
-		{
-			name: "pod with running init container as first initContainer should return true",
-			pod: &corev1api.Pod{
-				ObjectMeta: metav1.ObjectMeta{
-					Namespace: "ns-1",
-					Name:      "pod-1",
-				},
-				Spec: corev1api.PodSpec{
-					InitContainers: []corev1api.Container{
-						{
-							Name: restorehelper.WaitInitContainer,
-						},
-						{
-							Name: "non-restore-init",
-						},
-					},
-				},
-				Status: corev1api.PodStatus{
-					InitContainerStatuses: []corev1api.ContainerStatus{
-						{
-							State: corev1api.ContainerState{
-								Running: &corev1api.ContainerStateRunning{StartedAt: metav1.Time{Time: time.Now()}},
-							},
-						},
-						{
-							State: corev1api.ContainerState{
-								Running: &corev1api.ContainerStateRunning{StartedAt: metav1.Time{Time: time.Now()}},
-							},
-						},
-					},
-				},
-			},
-			expected: true,
-		},
-		{
-			name: "pod with init container with empty InitContainerStatuses should return 0",
-			pod: &corev1api.Pod{
-				ObjectMeta: metav1.ObjectMeta{
-					Namespace: "ns-1",
-					Name:      "pod-1",
-				},
-				Spec: corev1api.PodSpec{
-					InitContainers: []corev1api.Container{
-						{
-							Name: restorehelper.WaitInitContainer,
-						},
-					},
-				},
-				Status: corev1api.PodStatus{
-					InitContainerStatuses: []corev1api.ContainerStatus{},
-				},
-			},
-			expected: false,
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			assert.Equal(t, test.expected, isInitContainerRunning(test.pod))
+			if ts.expectError {
+				require.Error(t, err)
+				if ts.errString != "" {
+					assert.Contains(t, err.Error(), ts.errString)
+				}
+			} else {
+				require.NoError(t, err)
+			}
 		})
 	}
 }
@@ -996,7 +1099,7 @@ func TestPodVolumeRestoreReconcile(t *testing.T) {
 				}
 
 				if test.mockStart {
-					asyncBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(test.mockStartErr)
+					asyncBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.mockStartErr)
 				}
 
 				if test.mockCancel {
@@ -1798,7 +1901,7 @@ func TestResumeCancellablePodVolumeRestore(t *testing.T) {
 			}
 
 			if test.mockStart {
-				mockAsyncBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(test.startWatcherErr)
+				mockAsyncBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.startWatcherErr)
 			}
 
 			if test.mockClose {

pkg/datamover/backup_micro_service.go

@@ -204,7 +204,7 @@ func (r *BackupMicroService) RunCancelableDataPath(ctx context.Context) (string,
 
 	if err := dp.StartBackup(r.sourceTargetPath, du.Spec.DataMoverConfig, &datapath.BackupStartParam{
 		RealSource:     GetRealSource(du.Spec.SourceNamespace, du.Spec.SourcePVC),
-		ParentSnapshot: "",
+		ParentSnapshot: du.Spec.ParentSnapshot,
 		ForceFull:      false,
 		Tags:           tags,
 		VolumeID:       r.volumeID,

pkg/datamover/restore_micro_service.go

@@ -180,7 +180,7 @@ func (r *RestoreMicroService) RunCancelableDataPath(ctx context.Context) (string
 	}
 	log.Info("fs init")
 
-	if err := dp.StartRestore(dd.Spec.SnapshotID, r.sourceTargetPath, dd.Spec.DataMoverConfig); err != nil {
+	if err := dp.StartRestore(dd.Spec.SnapshotID, r.sourceTargetPath, dd.Spec.DataMoverConfig, &datapath.RestoreStartParam{}); err != nil {
 		return "", errors.Wrap(err, "error starting data path restore")
 	}
 

pkg/datamover/restore_micro_service_test.go

@@ -355,12 +355,12 @@ func TestRunCancelableRestore(t *testing.T) {
 
 				if test.startErr != nil {
 					fsBR.On("Init", mock.Anything, mock.Anything).Return(nil)
-					fsBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(test.startErr)
+					fsBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.startErr)
 				}
 
 				if test.dataPathStarted {
 					fsBR.On("Init", mock.Anything, mock.Anything).Return(nil)
-					fsBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(nil)
+					fsBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil)
 				}
 
 				return fsBR

pkg/datamover/util.go

@@ -19,12 +19,15 @@ package datamover
 import (
 	"fmt"
 
+	"github.com/vmware-tanzu/velero/pkg/uploader"
 	datamoverutil "github.com/vmware-tanzu/velero/pkg/util/datamover"
 )
 
 func GetUploaderType(dataMover string) string {
-	if datamoverutil.IsBuiltInDataMover(dataMover) {
-		return "kopia"
+	if datamoverutil.IsVeleroFSDataMover(dataMover) {
+		return uploader.KopiaType
+	} else if datamoverutil.IsVeleroBlockDataMover(dataMover) {
+		return uploader.BlockType
 	} else {
 		return dataMover
 	}

pkg/datamover/util_test.go

@@ -22,6 +22,16 @@ func TestGetUploaderType(t *testing.T) {
 			input: "velero",
 			want:  "kopia",
 		},
+		{
+			name:  "velero-fs dataMover is kopia",
+			input: "velero-fs",
+			want:  "kopia",
+		},
+		{
+			name:  "velero-block dataMover is velero-block",
+			input: "velero-block",
+			want:  "velero-block",
+		},
 		{
 			name:  "kopia dataMover is kopia",
 			input: "kopia",

pkg/datapath/data_path.go

@@ -59,6 +59,10 @@ type BackupStartParam struct {
 	SnapshotID     string
 }
 
+// RestoreStartParam define the input param for restore start
+type RestoreStartParam struct {
+}
+
 type generalDataPath struct {
 	ctx            context.Context
 	cancel         context.CancelFunc
@@ -221,7 +225,7 @@ func (dp *generalDataPath) StartBackup(source AccessPoint, uploaderConfig map[st
 	return nil
 }
 
-func (dp *generalDataPath) StartRestore(snapshotID string, target AccessPoint, uploaderConfigs map[string]string) error {
+func (dp *generalDataPath) StartRestore(snapshotID string, target AccessPoint, uploaderConfigs map[string]string, param any) error {
 	if !dp.initialized {
 		return errors.New("data path is not initialized")
 	}

pkg/datapath/data_path_test.go

@@ -190,7 +190,7 @@ func TestAsyncRestore(t *testing.T) {
 			dp.initialized = true
 			dp.callbacks = test.callbacks
 
-			err := dp.StartRestore(test.snapshot, AccessPoint{ByPath: test.path}, map[string]string{})
+			err := dp.StartRestore(test.snapshot, AccessPoint{ByPath: test.path}, map[string]string{}, &RestoreStartParam{})
 			require.NoError(t, err)
 
 			<-finish

pkg/datapath/micro_service_watcher.go

@@ -221,7 +221,7 @@ func (ms *microServiceBRWatcher) StartBackup(source AccessPoint, uploaderConfig
 	return nil
 }
 
-func (ms *microServiceBRWatcher) StartRestore(snapshotID string, target AccessPoint, uploaderConfigs map[string]string) error {
+func (ms *microServiceBRWatcher) StartRestore(snapshotID string, target AccessPoint, uploaderConfigs map[string]string, param any) error {
 	ms.log.Infof("Start watching restore ms to target %s, from snapshot %s", target.ByPath, snapshotID)
 
 	ms.startWatch()

pkg/datapath/mocks/asyncBR.go

@@ -60,17 +60,17 @@ func (_m *AsyncBR) StartBackup(source datapath.AccessPoint, dataMoverConfig map[
 	return r0
 }
 
-// StartRestore provides a mock function with given fields: snapshotID, target, dataMoverConfig
-func (_m *AsyncBR) StartRestore(snapshotID string, target datapath.AccessPoint, dataMoverConfig map[string]string) error {
-	ret := _m.Called(snapshotID, target, dataMoverConfig)
+// StartRestore provides a mock function with given fields: snapshotID, target, dataMoverConfig, param
+func (_m *AsyncBR) StartRestore(snapshotID string, target datapath.AccessPoint, dataMoverConfig map[string]string, param interface{}) error {
+	ret := _m.Called(snapshotID, target, dataMoverConfig, param)
 
 	if len(ret) == 0 {
 		panic("no return value specified for StartRestore")
 	}
 
 	var r0 error
-	if rf, ok := ret.Get(0).(func(string, datapath.AccessPoint, map[string]string) error); ok {
-		r0 = rf(snapshotID, target, dataMoverConfig)
+	if rf, ok := ret.Get(0).(func(string, datapath.AccessPoint, map[string]string, interface{}) error); ok {
+		r0 = rf(snapshotID, target, dataMoverConfig, param)
 	} else {
 		r0 = ret.Error(0)
 	}

pkg/datapath/types.go

@@ -66,7 +66,7 @@ type AsyncBR interface {
 	StartBackup(source AccessPoint, dataMoverConfig map[string]string, param any) error
 
 	// StartRestore starts an asynchronous data path instance for restore
-	StartRestore(snapshotID string, target AccessPoint, dataMoverConfig map[string]string) error
+	StartRestore(snapshotID string, target AccessPoint, dataMoverConfig map[string]string, param any) error
 
 	// Cancel cancels an asynchronous data path instance
 	Cancel()

pkg/install/deployment.go

@@ -139,7 +139,10 @@ func WithPodVolumeOperationTimeout(val time.Duration) podTemplateOption {
 
 func WithPlugins(plugins []string) podTemplateOption {
 	return func(c *podTemplateConfig) {
-		c.plugins = plugins
+		c.plugins = make([]string, 0, len(plugins))
+		for _, plugin := range plugins {
+			c.plugins = append(c.plugins, strings.TrimSpace(plugin))
+		}
 	}
 }
 

pkg/install/deployment_test.go

@@ -60,6 +60,15 @@ func TestDeployment(t *testing.T) {
 	assert.Len(t, deploy.Spec.Template.Spec.Containers[0].Args, 2)
 	assert.Equal(t, "--features=EnableCSI,foo,bar,baz", deploy.Spec.Template.Spec.Containers[0].Args[1])
 
+	deploy = Deployment("velero", WithPlugins([]string{
+		"harbor-repo.vmware.com/harbor-ci/velero/velero-plugin-for-aws:v1.2.0",
+		" \n vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1 ",
+	}))
+	assert.Len(t, deploy.Spec.Template.Spec.InitContainers, 2)
+	assert.Equal(t, "harbor-repo.vmware.com/harbor-ci/velero/velero-plugin-for-aws:v1.2.0", deploy.Spec.Template.Spec.InitContainers[0].Image)
+	assert.Equal(t, "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1", deploy.Spec.Template.Spec.InitContainers[1].Image)
+	assert.Equal(t, "vsphereveleroplugin-velero-plugin-for-vsphere", deploy.Spec.Template.Spec.InitContainers[1].Name)
+
 	deploy = Deployment("velero", WithUploaderType("kopia"))
 	assert.Len(t, deploy.Spec.Template.Spec.Containers[0].Args, 2)
 	assert.Equal(t, "--uploader-type=kopia", deploy.Spec.Template.Spec.Containers[0].Args[1])

pkg/metrics/metrics.go

@@ -758,6 +758,14 @@ func (m *ServerMetrics) RegisterPodVolumeOpLatencyGauge(node, pvbName, opName, b
 	}
 }
 
+// DeleteBackupLastSuccessfulTimestamp removes the backupLastSuccessfulTimestamp
+// metric for a single schedule.
+func (m *ServerMetrics) DeleteBackupLastSuccessfulTimestamp(scheduleName string) {
+	if g, ok := m.metrics[backupLastSuccessfulTimestamp].(*prometheus.GaugeVec); ok {
+		g.DeleteLabelValues(scheduleName)
+	}
+}
+
 // SetBackupTarballSizeBytesGauge records the size, in bytes, of a backup tarball.
 func (m *ServerMetrics) SetBackupTarballSizeBytesGauge(backupSchedule string, size int64) {
 	if g, ok := m.metrics[backupTarballSizeBytesGauge].(*prometheus.GaugeVec); ok {

pkg/metrics/metrics_test.go

@@ -21,6 +21,7 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
 	dto "github.com/prometheus/client_model/go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -457,6 +458,32 @@ func getHistogramCount(t *testing.T, vec *prometheus.HistogramVec, scheduleLabel
 	return 0
 }
 
+// TestDeleteBackupLastSuccessfulTimestamp verifies that DeleteBackupLastSuccessfulTimestamp
+// removes only the specified schedule's metric.
+func TestDeleteBackupLastSuccessfulTimestamp(t *testing.T) {
+	m := NewServerMetrics()
+
+	now := time.Now()
+	m.SetBackupLastSuccessfulTimestamp("schedule-1", now)
+	m.SetBackupLastSuccessfulTimestamp("schedule-2", now.Add(-time.Hour))
+	m.SetBackupLastSuccessfulTimestamp("", now.Add(-2*time.Hour))
+
+	g := m.metrics[backupLastSuccessfulTimestamp].(*prometheus.GaugeVec)
+	assert.Equal(t, 3, testutil.CollectAndCount(g))
+
+	m.DeleteBackupLastSuccessfulTimestamp("schedule-1")
+	assert.Equal(t, 2, testutil.CollectAndCount(g))
+	assert.Equal(t, float64(now.Add(-time.Hour).Unix()), testutil.ToFloat64(g.WithLabelValues("schedule-2")))
+	assert.Equal(t, float64(now.Add(-2*time.Hour).Unix()), testutil.ToFloat64(g.WithLabelValues("")))
+
+	m.DeleteBackupLastSuccessfulTimestamp("schedule-2")
+	assert.Equal(t, 1, testutil.CollectAndCount(g))
+	assert.Equal(t, float64(now.Add(-2*time.Hour).Unix()), testutil.ToFloat64(g.WithLabelValues("")))
+
+	m.DeleteBackupLastSuccessfulTimestamp("")
+	assert.Equal(t, 0, testutil.CollectAndCount(g))
+}
+
 // TestRepoMaintenanceMetrics verifies that repo maintenance metrics are properly recorded.
 func TestRepoMaintenanceMetrics(t *testing.T) {
 	tests := []struct {

pkg/podvolume/backupper.go

@@ -20,12 +20,15 @@ import (
 	"context"
 	"fmt"
 	"sync"
+	"time"
 
 	"github.com/cockroachdb/errors"
 	"github.com/sirupsen/logrus"
 	corev1api "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/tools/cache"
 	ctrlcache "sigs.k8s.io/controller-runtime/pkg/cache"
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
@@ -181,7 +184,7 @@ func newBackupper(
 					// the PVB in the indexer is already in final status, no need to call WaitGroup.Done()
 					if ok && (existPVB.Status.Phase == velerov1api.PodVolumeBackupPhaseCompleted ||
 						existPVB.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed ||
-						pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCanceled) {
+						existPVB.Status.Phase == velerov1api.PodVolumeBackupPhaseCanceled) {
 						statusChangedToFinal = false
 					}
 				}
@@ -411,6 +414,24 @@ func (b *backupper) WaitAllPodVolumesProcessed(log logrus.FieldLogger) []*velero
 	select {
 	case <-b.ctx.Done():
 		log.Error("timed out waiting for all PodVolumeBackups to complete")
+
+		for _, obj := range b.pvbIndexer.List() {
+			pvb, ok := obj.(*velerov1api.PodVolumeBackup)
+			if !ok {
+				log.Errorf("expected PVB, but got %T", obj)
+				continue
+			}
+
+			if pvb.Status.Phase != velerov1api.PodVolumeBackupPhaseCompleted &&
+				pvb.Status.Phase != velerov1api.PodVolumeBackupPhaseFailed &&
+				pvb.Status.Phase != velerov1api.PodVolumeBackupPhaseCanceled {
+				log.Infof("Setting cancel flag for ongoing PVB %s/%s", pvb.Namespace, pvb.Name)
+				if err := updatePVBWithRetry(context.Background(), b.crClient, pvb.Namespace, pvb.Name); err != nil {
+					log.WithError(err).Errorf("Failed to set cancel flag for PVB %s/%s", pvb.Namespace, pvb.Name)
+				}
+			}
+		}
+		<-done
 	case <-done:
 	}
 
@@ -432,6 +453,32 @@ func (b *backupper) WaitAllPodVolumesProcessed(log logrus.FieldLogger) []*velero
 	return podVolumeBackups
 }
 
+func updatePVBWithRetry(ctx context.Context, client ctrlclient.Client, namespace, name string) error {
+	return wait.PollUntilContextCancel(ctx, 100*time.Millisecond, true, func(ctx context.Context) (bool, error) {
+		pvb := &velerov1api.PodVolumeBackup{}
+		if err := client.Get(ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: name}, pvb); err != nil {
+			return false, errors.Wrap(err, "getting PVB")
+		}
+
+		if pvb.Spec.Cancel {
+			return true, nil
+		}
+
+		pvb.Spec.Cancel = true
+		pvb.Status.Message = "Cancel PVB on pod volume timeout"
+
+		err := client.Update(ctx, pvb)
+		if err != nil {
+			if apierrors.IsConflict(err) {
+				return false, nil
+			}
+			return false, errors.Wrapf(err, "error updating PVB %s/%s", pvb.Namespace, pvb.Name)
+		}
+
+		return true, nil
+	})
+}
+
 func (b *backupper) GetPodVolumeBackupByPodAndVolume(podNamespace, podName, volume string) (*velerov1api.PodVolumeBackup, error) {
 	obj, exist, err := b.pvbIndexer.GetByKey(fmt.Sprintf(pvbKeyPattern, podNamespace, podName, volume))
 	if err != nil {

pkg/podvolume/backupper_test.go

@@ -733,14 +733,14 @@ func TestListPodVolumeBackupsByPodp(t *testing.T) {
 }
 
 type logHook struct {
-	entry *logrus.Entry
+	entries []*logrus.Entry
 }
 
 func (l *logHook) Levels() []logrus.Level {
 	return []logrus.Level{logrus.ErrorLevel}
 }
 func (l *logHook) Fire(entry *logrus.Entry) error {
-	l.entry = entry
+	l.entries = append(l.entries, entry)
 	return nil
 }
 
@@ -808,12 +808,35 @@ func TestWaitAllPodVolumesProcessed(t *testing.T) {
 		logHook := &logHook{}
 		logger.Hooks.Add(logHook)
 
-		backuper := newBackupper(c.ctx, log, nil, nil, informer, nil, "", &velerov1api.Backup{})
+		backuper := newBackupper(c.ctx, log, nil, nil, informer, client, "", &velerov1api.Backup{})
 		if c.pvb != nil {
 			require.NoError(t, backuper.pvbIndexer.Add(c.pvb))
 			backuper.wg.Add(1)
 		}
 
+		if c.ctx == timeoutCtx && c.pvb != nil {
+			// Start a goroutine to simulate the controller's cancellation behavior
+			go func() {
+				// Wait a short time for the cancel flag to be set
+				ticker := time.NewTicker(10 * time.Millisecond)
+				defer ticker.Stop()
+				for range ticker.C {
+					pvb := &velerov1api.PodVolumeBackup{}
+					err := client.Get(t.Context(), ctrlclient.ObjectKey{Namespace: c.pvb.Namespace, Name: c.pvb.Name}, pvb)
+					if err == nil && pvb.Spec.Cancel {
+						oldPVB := pvb.DeepCopy()
+						pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseCanceled
+						pvb.Status.Message = "canceled"
+						_ = client.Update(t.Context(), pvb)
+						if informer.handler != nil {
+							informer.handler.OnUpdate(oldPVB, pvb)
+						}
+						return
+					}
+				}
+			}()
+		}
+
 		if c.statusToBeUpdated != nil {
 			pvb := &velerov1api.PodVolumeBackup{}
 			err := client.Get(t.Context(), ctrlclient.ObjectKey{Namespace: c.pvb.Namespace, Name: c.pvb.Name}, pvb)
@@ -831,9 +854,18 @@ func TestWaitAllPodVolumesProcessed(t *testing.T) {
 		pvbs := backuper.WaitAllPodVolumesProcessed(logger)
 
 		if c.expectedErr != "" {
-			assert.Equal(t, c.expectedErr, logHook.entry.Message)
+			found := false
+			var loggedMsgs []string
+			for _, entry := range logHook.entries {
+				loggedMsgs = append(loggedMsgs, entry.Message)
+				if entry.Message == c.expectedErr {
+					found = true
+					break
+				}
+			}
+			assert.True(t, found, "Expected error %q to be logged, but got %v", c.expectedErr, loggedMsgs)
 		} else {
-			assert.Nil(t, logHook.entry)
+			assert.Empty(t, logHook.entries)
 		}
 
 		if c.expectedPVBCount > 0 {

pkg/podvolume/restore_micro_service.go

@@ -184,7 +184,7 @@ func (r *RestoreMicroService) RunCancelableDataPath(ctx context.Context) (string
 
 	log.Info("Async fs br init")
 
-	if err := fsRestore.StartRestore(pvr.Spec.SnapshotID, r.sourceTargetPath, pvr.Spec.UploaderSettings); err != nil {
+	if err := fsRestore.StartRestore(pvr.Spec.SnapshotID, r.sourceTargetPath, pvr.Spec.UploaderSettings, &datapath.RestoreStartParam{}); err != nil {
 		return "", errors.Wrap(err, "error starting data path restore")
 	}
 

pkg/podvolume/restore_micro_service_test.go

@@ -436,12 +436,12 @@ func TestRunCancelableDataPathRestore(t *testing.T) {
 
 				if test.startErr != nil {
 					fsBR.On("Init", mock.Anything, mock.Anything).Return(nil)
-					fsBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(test.startErr)
+					fsBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(test.startErr)
 				}
 
 				if test.dataPathStarted {
 					fsBR.On("Init", mock.Anything, mock.Anything).Return(nil)
-					fsBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything).Return(nil)
+					fsBR.On("StartRestore", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil)
 				}
 
 				return fsBR

pkg/repository/manager/manager.go

@@ -18,6 +18,7 @@ package repository
 
 import (
 	"context"
+	"crypto/fips140"
 	"fmt"
 	"time"
 
@@ -173,7 +174,13 @@ func (m *manager) PrepareRepo(repo *velerov1api.BackupRepository) error {
 	if err != nil {
 		return errors.WithStack(err)
 	}
-	return prd.PrepareRepo(context.Background(), param)
+
+	// Disable FIPS-140 compliance check, because Kopia doesn't support FIPS-140 yet.
+	var prepareErr error
+	fips140.WithoutEnforcement(func() {
+		prepareErr = prd.PrepareRepo(context.Background(), param)
+	})
+	return prepareErr
 }
 
 func (m *manager) PruneRepo(repo *velerov1api.BackupRepository) error {
@@ -244,11 +251,20 @@ func (m *manager) BatchForget(ctx context.Context, repo *velerov1api.BackupRepos
 		return []error{errors.WithStack(err)}
 	}
 
-	if err := prd.BoostRepoConnect(context.Background(), param); err != nil {
+	// Disable FIPS-140 compliance check, because Kopia doesn't support FIPS-140 yet.
+	var connectErr error
+	fips140.WithoutEnforcement(func() {
+		connectErr = prd.BoostRepoConnect(context.Background(), param)
+	})
+	if connectErr != nil {
 		return []error{errors.WithStack(err)}
 	}
 
-	return prd.BatchForget(context.Background(), snapshots, param)
+	forgetErr := make([]error, 0)
+	fips140.WithoutEnforcement(func() {
+		forgetErr = prd.BatchForget(context.Background(), snapshots, param)
+	})
+	return forgetErr
 }
 
 func (m *manager) DefaultMaintenanceFrequency(repo *velerov1api.BackupRepository) (time.Duration, error) {

pkg/repository/udmrepo/kopialib/lib_repo_ex_test.go

@@ -1208,6 +1208,10 @@ func TestKopiaObjectWriterEx_MixedWriteAndWriteAt(t *testing.T) {
 	assert.Equal(t, int64(3072), kow.entries[3].Start)
 }
 
+// TestKopiaObjectWriterEx_ConcurrentAsyncErrors verifies the async error contract
+// under real scheduling: once an async block write fails, the error either fails a
+// subsequent Write call fast or surfaces at Result — it is never lost. Which of the
+// two happens first depends on goroutine scheduling, and both are correct.
 func TestKopiaObjectWriterEx_ConcurrentAsyncErrors(t *testing.T) {
 	mockRepoWriter := repomocks.NewMockRepositoryWriter(t)
 	mockWriter := repomocks.NewWriter(t)
@@ -1231,14 +1235,65 @@ func TestKopiaObjectWriterEx_ConcurrentAsyncErrors(t *testing.T) {
 
 	data := make([]byte, 1024)
 
-	// Issue multiple writes so they all spawn async goroutines
-	// First few writes shouldn't fail immediately until getWriteError catches the asynchronous fault
+	// Issue multiple writes so they all spawn async goroutines. A later Write may
+	// observe the stored async error and fail fast — that is correct behavior.
+	for i := 0; i < 10; i++ {
+		l, err := kow.Write(data)
+		if err != nil {
+			assert.Contains(t, err.Error(), "simulated async error")
+			break
+		}
+		assert.Equal(t, 1024, l)
+	}
+
+	// Regardless of whether a Write observed the error first, Result must report it.
+	id, err := kow.Result()
+
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "simulated async error")
+	assert.Equal(t, udmrepo.ID(""), id)
+}
+
+// TestKopiaObjectWriterEx_AsyncErrorSurfacesAtResult pins the late-error schedule:
+// async writes are held until all writes have been queued, so no Write call observes
+// the failure and Result alone must report it.
+func TestKopiaObjectWriterEx_AsyncErrorSurfacesAtResult(t *testing.T) {
+	mockRepoWriter := repomocks.NewMockRepositoryWriter(t)
+	mockWriter := repomocks.NewWriter(t)
+
+	releaseWrites := make(chan struct{})
+	mockWriter.On("Write", mock.Anything).Run(func(mock.Arguments) {
+		<-releaseWrites
+	}).Return(0, errors.New("simulated async error"))
+	mockWriter.On("Close").Return(nil)
+
+	mockRepoWriter.On("NewObjectWriter", mock.Anything, mock.Anything).Return(mockWriter)
+
+	sem := make(chan struct{}, 10)
+	buf := freelist.New(10*1024, 1024)
+
+	kow := &kopiaObjectWriterEx{
+		ctx:            context.Background(),
+		rawRepoWriter:  mockRepoWriter,
+		blockSize:      1024,
+		asyncWritesSem: sem,
+		asyncBuffer:    buf,
+		logger:         velerotest.NewLogger(),
+	}
+
+	data := make([]byte, 1024)
+
+	// All async writes block on releaseWrites, so no error can be stored yet and
+	// every Write must succeed.
 	for i := 0; i < 10; i++ {
 		l, err := kow.Write(data)
 		require.NoError(t, err)
 		assert.Equal(t, 1024, l)
 	}
 
+	close(releaseWrites)
+
+	// Result waits for the async writers to finish and must report their error.
 	id, err := kow.Result()
 
 	require.Error(t, err)

pkg/restore/restore.go

@@ -478,7 +478,15 @@ func (ctx *restoreContext) getNamespaceFilter(namespace string) *resolvedNamespa
 		return filter
 	}
 
-	// 2. Walk patterns in definition order (first-match semantics)
+	// 2. Check for exact match first (O(1) map lookup)
+	// This ensures exact namespace matches take precedence over globs,
+	// regardless of where they are listed in the configuration.
+	if filter, ok := ctx.namespacedFilterMap[namespace]; ok {
+		ctx.namespaceFilterCache[namespace] = filter
+		return filter
+	}
+
+	// 3. Walk patterns in definition order using pre-compiled globs
 	// Note: namespaceFilterCache is mutated below without synchronization. This is safe
 	// today because resource collection runs sequentially. If the restore loop is
 	// parallelized in the future, these map writes will need a lock to prevent data races.
@@ -489,14 +497,10 @@ func (ctx *restoreContext) getNamespaceFilter(namespace string) *resolvedNamespa
 				ctx.namespaceFilterCache[namespace] = filter
 				return filter
 			}
-		} else if p.pattern == namespace {
-			filter := ctx.namespacedFilterMap[p.pattern]
-			ctx.namespaceFilterCache[namespace] = filter
-			return filter
 		}
 	}
 
-	// 3. Cache the miss so we don't re-evaluate failed matches
+	// 4. Cache the miss so we don't re-evaluate failed matches
 	ctx.namespaceFilterCache[namespace] = nil
 	return nil
 }
@@ -634,21 +638,19 @@ func resolveRestoreNamespacedFilterPolicies(
 func resolveResourceFilter(
 	rf resourcepolicies.ResourceFilter,
 ) (*resolvedResourceFilter, error) {
-	var selector labels.Selector
-	if len(rf.LabelSelector) > 0 {
-		var err error
-		selector, err = labels.ValidatedSelectorFromSet(labels.Set(rf.LabelSelector))
-		if err != nil {
-			return nil, fmt.Errorf("invalid label selector in resource filter: %w", err)
-		}
+	selector, err := resourcepolicies.SelectorFromPolicyLabelSelector(rf.LabelSelector)
+	if err != nil {
+		return nil, fmt.Errorf("invalid label selector in resource filter: %w", err)
 	}
 	var orSelectors []labels.Selector
 	for _, ols := range rf.OrLabelSelectors {
-		s, err := labels.ValidatedSelectorFromSet(labels.Set(ols))
+		s, err := resourcepolicies.SelectorFromPolicyLabelSelector(ols)
 		if err != nil {
 			return nil, fmt.Errorf("invalid OR label selector in resource filter: %w", err)
 		}
-		orSelectors = append(orSelectors, s)
+		if s != nil {
+			orSelectors = append(orSelectors, s)
+		}
 	}
 	var nameIE *collections.IncludesExcludes
 	if len(rf.Names) > 0 || len(rf.ExcludedNames) > 0 {
@@ -1058,7 +1060,7 @@ func (ctx *restoreContext) processSelectedResource(
 				continue
 			}
 
-			w, e, _ := ctx.restoreItem(obj, groupResource, targetNS)
+			w, e, _ := ctx.restoreItem(obj, groupResource, targetNS, false)
 			warnings.Merge(&w)
 			errs.Merge(&e)
 			processedItems++
@@ -1384,7 +1386,7 @@ func (ctx *restoreContext) getResource(groupResource schema.GroupResource, obj *
 	return u, nil
 }
 
-func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupResource schema.GroupResource, namespace string) (results.Result, results.Result, bool) {
+func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupResource schema.GroupResource, namespace string, mustInclude bool) (results.Result, results.Result, bool) {
 	warnings, errs := results.Result{}, results.Result{}
 	// itemExists bool is used to determine whether to include this item in the "wait for additional items" list
 	itemExists := false
@@ -1401,27 +1403,41 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
 	// Check if group/resource should be restored. We need to do this here since
 	// this method may be getting called for an additional item which is a group/resource
 	// that's excluded.
-	if !ctx.resourceIncludesExcludes.ShouldInclude(groupResource.String()) && !ctx.resourceMustHave.Has(groupResource.String()) {
-		restoreLogger.Info("Not restoring item because resource is excluded")
-		return warnings, errs, itemExists
-	}
-
-	// Check if namespace/cluster-scoped resource should be restored. We need
-	// to do this here since this method may be getting called for an additional
-	// item which is in a namespace that's excluded, or which is cluster-scoped
-	// and should be excluded. Note that we're checking the object's namespace (
-	// via obj.GetNamespace()) instead of the namespace parameter, because we want
-	// to check the *original* namespace, not the remapped one if it's been remapped.
 	//
 	// Note: Additional items intentionally bypass fine-grained resource filter policies
 	// (like per-namespace label/name selectors) to avoid breaking semantic dependencies,
-	// but they must still pass the global exclusions enforced below.
-	if namespace != "" {
-		if !ctx.namespaceIncludesExcludes.ShouldInclude(obj.GetNamespace()) && !ctx.resourceMustHave.Has(groupResource.String()) {
-			restoreLogger.Info("Not restoring item because namespace is excluded")
+	// but they must still pass the global exclusions enforced below unless mustInclude is set.
+	if mustInclude {
+		restoreLogger.Info("Skipping the resource/namespace exclusion checks because the item is marked as must-include")
+	} else {
+		if !ctx.resourceIncludesExcludes.ShouldInclude(groupResource.String()) && !ctx.resourceMustHave.Has(groupResource.String()) {
+			restoreLogger.Info("Not restoring item because resource is excluded")
 			return warnings, errs, itemExists
 		}
 
+		// Check if namespace/cluster-scoped resource should be restored. We need
+		// to do this here since this method may be getting called for an additional
+		// item which is in a namespace that's excluded, or which is cluster-scoped
+		// and should be excluded. Note that we're checking the object's namespace (
+		// via obj.GetNamespace()) instead of the namespace parameter, because we want
+		// to check the *original* namespace, not the remapped one if it's been remapped.
+		if namespace != "" {
+			if !ctx.namespaceIncludesExcludes.ShouldInclude(obj.GetNamespace()) && !ctx.resourceMustHave.Has(groupResource.String()) {
+				restoreLogger.Info("Not restoring item because namespace is excluded")
+				return warnings, errs, itemExists
+			}
+		} else {
+			if boolptr.IsSetToFalse(ctx.restore.Spec.IncludeClusterResources) {
+				restoreLogger.Info("Not restoring item because it's cluster-scoped")
+				return warnings, errs, itemExists
+			}
+		}
+	}
+
+	// Namespace creation runs unconditionally when namespace != "", regardless of
+	// mustInclude. This ensures target namespaces exist for additional items that
+	// bypass the namespace-exclusion check above.
+	if namespace != "" {
 		// If the namespace scoped resource should be restored, ensure that the
 		// namespace into which the resource is being restored into exists.
 		// This is the *remapped* namespace that we are ensuring exists.
@@ -1440,11 +1456,6 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
 			}
 			ctx.restoredItems[itemKey] = restoredItemStatus{action: ItemRestoreResultCreated, itemExists: true, createdName: nsToEnsure.Name}
 		}
-	} else {
-		if boolptr.IsSetToFalse(ctx.restore.Spec.IncludeClusterResources) {
-			restoreLogger.Info("Not restoring item because it's cluster-scoped")
-			return warnings, errs, itemExists
-		}
 	}
 
 	// Make a copy of object retrieved from backup to make it available unchanged
@@ -1666,6 +1677,21 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
 
 		obj = unstructuredObj
 
+		mustIncludeAdditionalItems := false
+		if annotations := obj.GetAnnotations(); annotations != nil {
+			if _, present := annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation]; present {
+				// Only the string value "true" enables the bypass.
+				if annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] == "true" {
+					mustIncludeAdditionalItems = true
+					restoreLogger.Info("RestoreItemAction marked additional items as must-include; bypassing resource/namespace exclusion checks for them")
+				}
+				// Always strip the annotation so it never lands on the cluster,
+				// regardless of whether the value enabled the bypass.
+				delete(annotations, velerov1api.MustIncludeAdditionalItemRestoreAnnotation)
+				obj.SetAnnotations(annotations)
+			}
+		}
+
 		var filteredAdditionalItems []velero.ResourceIdentifier
 		for _, additionalItem := range executeOutput.AdditionalItems {
 			itemPath := archive.GetItemFilePath(ctx.restoreDir, additionalItem.GroupResource.String(), additionalItem.Namespace, additionalItem.Name)
@@ -1685,6 +1711,7 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
 			additionalObj, err := archive.Unmarshal(ctx.fileSystem, itemPath)
 			if err != nil {
 				errs.Add(namespace, errors.Wrapf(err, "error restoring additional item %s", additionalResourceID))
+				continue
 			}
 
 			additionalItemNamespace := additionalItem.Namespace
@@ -1694,7 +1721,7 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
 				}
 			}
 
-			w, e, additionalItemExists := ctx.restoreItem(additionalObj, additionalItem.GroupResource, additionalItemNamespace)
+			w, e, additionalItemExists := ctx.restoreItem(additionalObj, additionalItem.GroupResource, additionalItemNamespace, mustIncludeAdditionalItems)
 			if additionalItemExists {
 				filteredAdditionalItems = append(filteredAdditionalItems, additionalItem)
 			}

pkg/restore/restore_policies_test.go

@@ -62,7 +62,7 @@ namespacedFilterPolicies:
 			},
 		},
 		{
-			name:    "namespaced filter policy with glob namespace match and first-match semantics",
+			name:    "namespaced filter policy with exact match priority over glob (glob listed first)",
 			restore: defaultRestore().Result(),
 			backup:  defaultBackup().Result(),
 			policyYAML: `version: v1
@@ -94,7 +94,43 @@ namespacedFilterPolicies:
 				test.Pods(),
 			},
 			want: map[*test.APIResource][]string{
-				test.Pods(): {"ns-1/pod-1", "ns-2/pod-1"},
+				test.Pods(): {"ns-1/pod-2", "ns-2/pod-1"},
+			},
+		},
+		{
+			name:    "namespaced filter policy with exact match priority over glob (exact listed first)",
+			restore: defaultRestore().Result(),
+			backup:  defaultBackup().Result(),
+			policyYAML: `version: v1
+namespacedFilterPolicies:
+  - namespaces:
+      - ns-1
+    resourceFilters:
+      - kinds:
+          - pods
+        names:
+          - pod-2
+  - namespaces:
+      - ns-*
+    resourceFilters:
+      - kinds:
+          - pods
+        names:
+          - pod-1
+`,
+			tarball: test.NewTarWriter(t).
+				AddItems("pods",
+					builder.ForPod("ns-1", "pod-1").Result(),
+					builder.ForPod("ns-1", "pod-2").Result(),
+					builder.ForPod("ns-2", "pod-1").Result(),
+					builder.ForPod("ns-2", "pod-2").Result(),
+				).
+				Done(),
+			apiResources: []*test.APIResource{
+				test.Pods(),
+			},
+			want: map[*test.APIResource][]string{
+				test.Pods(): {"ns-1/pod-2", "ns-2/pod-1"},
 			},
 		},
 		{
@@ -134,7 +170,8 @@ namespacedFilterPolicies:
       - kinds:
           - '*'
         labelSelector:
-          app: test
+          matchLabels:
+            app: test
 `,
 			tarball: test.NewTarWriter(t).
 				AddItems("pods",

pkg/restore/restore_test.go

@@ -2150,6 +2150,102 @@ func TestRestoreActionAdditionalItems(t *testing.T) {
 				test.PVs():  nil,
 			},
 		},
+		{
+			name:    "must-include annotation bypasses resource exclusion for additional items",
+			restore: defaultRestore().IncludedResources("pods").Result(),
+			backup:  defaultBackup().Result(),
+			tarball: test.NewTarWriter(t).
+				AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()).
+				AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result()).
+				Done(),
+			apiResources: []*test.APIResource{test.Pods(), test.PVs()},
+			actions: []riav2.RestoreItemAction{
+				&pluggableAction{
+					executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) {
+						item := input.Item.(*unstructured.Unstructured)
+						annotations := item.GetAnnotations()
+						if annotations == nil {
+							annotations = map[string]string{}
+						}
+						annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true"
+						item.SetAnnotations(annotations)
+						return &velero.RestoreItemActionExecuteOutput{
+							UpdatedItem: item,
+							AdditionalItems: []velero.ResourceIdentifier{
+								{GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"},
+							},
+						}, nil
+					},
+				},
+			},
+			want: map[*test.APIResource][]string{
+				test.Pods(): {"ns-1/pod-1"},
+				test.PVs():  {"/pv-1"},
+			},
+		},
+		{
+			name:         "must-include annotation bypasses namespace exclusion for additional items",
+			restore:      defaultRestore().IncludedNamespaces("ns-1").Result(),
+			backup:       defaultBackup().Result(),
+			tarball:      test.NewTarWriter(t).AddItems("pods", builder.ForPod("ns-1", "pod-1").Result(), builder.ForPod("ns-2", "pod-2").Result()).Done(),
+			apiResources: []*test.APIResource{test.Pods()},
+			actions: []riav2.RestoreItemAction{
+				&pluggableAction{
+					selector: velero.ResourceSelector{IncludedNamespaces: []string{"ns-1"}},
+					executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) {
+						item := input.Item.(*unstructured.Unstructured)
+						annotations := item.GetAnnotations()
+						if annotations == nil {
+							annotations = map[string]string{}
+						}
+						annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true"
+						item.SetAnnotations(annotations)
+						return &velero.RestoreItemActionExecuteOutput{
+							UpdatedItem: item,
+							AdditionalItems: []velero.ResourceIdentifier{
+								{GroupResource: kuberesource.Pods, Namespace: "ns-2", Name: "pod-2"},
+							},
+						}, nil
+					},
+				},
+			},
+			want: map[*test.APIResource][]string{
+				test.Pods(): {"ns-1/pod-1", "ns-2/pod-2"},
+			},
+		},
+		{
+			name:    "must-include annotation bypasses IncludeClusterResources=false for additional items",
+			restore: defaultRestore().IncludeClusterResources(false).Result(),
+			backup:  defaultBackup().Result(),
+			tarball: test.NewTarWriter(t).
+				AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()).
+				AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result()).
+				Done(),
+			apiResources: []*test.APIResource{test.Pods(), test.PVs()},
+			actions: []riav2.RestoreItemAction{
+				&pluggableAction{
+					executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) {
+						item := input.Item.(*unstructured.Unstructured)
+						annotations := item.GetAnnotations()
+						if annotations == nil {
+							annotations = map[string]string{}
+						}
+						annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true"
+						item.SetAnnotations(annotations)
+						return &velero.RestoreItemActionExecuteOutput{
+							UpdatedItem: item,
+							AdditionalItems: []velero.ResourceIdentifier{
+								{GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"},
+							},
+						}, nil
+					},
+				},
+			},
+			want: map[*test.APIResource][]string{
+				test.Pods(): {"ns-1/pod-1"},
+				test.PVs():  {"/pv-1"},
+			},
+		},
 	}
 
 	for _, tc := range tests {
@@ -2180,6 +2276,324 @@ func TestRestoreActionAdditionalItems(t *testing.T) {
 	}
 }
 
+// TestRestoreMustIncludeAdditionalItems covers restore must-include edge cases beyond the
+// basic filter-bypass cases in TestRestoreActionAdditionalItems.
+func TestRestoreMustIncludeAdditionalItems(t *testing.T) {
+	t.Run("must-include annotation is stripped from the restored item", func(t *testing.T) {
+		h := newHarness(t)
+		h.AddItems(t, test.Pods())
+
+		data := &Request{
+			Log:     h.log,
+			Restore: defaultRestore().Result(),
+			Backup:  defaultBackup().Result(),
+			BackupReader: test.NewTarWriter(t).
+				AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()).
+				Done(),
+		}
+		warnings, errs := h.restorer.Restore(
+			data,
+			[]riav2.RestoreItemAction{
+				&pluggableAction{
+					executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) {
+						item := input.Item.(*unstructured.Unstructured)
+						annotations := item.GetAnnotations()
+						if annotations == nil {
+							annotations = map[string]string{}
+						}
+						annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true"
+						annotations["keep-me"] = "yes"
+						item.SetAnnotations(annotations)
+						return &velero.RestoreItemActionExecuteOutput{UpdatedItem: item}, nil
+					},
+				},
+			},
+			nil,
+		)
+
+		assertEmptyResults(t, warnings, errs)
+
+		got, err := h.DynamicClient.Resource(test.Pods().GVR()).Namespace("ns-1").Get(t.Context(), "pod-1", metav1.GetOptions{})
+		require.NoError(t, err)
+		annotations := got.GetAnnotations()
+		assert.NotContains(t, annotations, velerov1api.MustIncludeAdditionalItemRestoreAnnotation)
+		assert.Equal(t, "yes", annotations["keep-me"])
+	})
+
+	t.Run("non-true must-include annotation is stripped without bypassing filters", func(t *testing.T) {
+		h := newHarness(t)
+		h.AddItems(t, test.Pods())
+		h.AddItems(t, test.PVs())
+
+		data := &Request{
+			Log:     h.log,
+			Restore: defaultRestore().IncludedResources("pods").Result(),
+			Backup:  defaultBackup().Result(),
+			BackupReader: test.NewTarWriter(t).
+				AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()).
+				AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result()).
+				Done(),
+		}
+		warnings, errs := h.restorer.Restore(
+			data,
+			[]riav2.RestoreItemAction{
+				&pluggableAction{
+					executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) {
+						item := input.Item.(*unstructured.Unstructured)
+						annotations := item.GetAnnotations()
+						if annotations == nil {
+							annotations = map[string]string{}
+						}
+						annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "True"
+						annotations["keep-me"] = "yes"
+						item.SetAnnotations(annotations)
+						return &velero.RestoreItemActionExecuteOutput{
+							UpdatedItem: item,
+							AdditionalItems: []velero.ResourceIdentifier{
+								{GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"},
+							},
+						}, nil
+					},
+				},
+			},
+			nil,
+		)
+
+		assertEmptyResults(t, warnings, errs)
+		assertAPIContents(t, h, map[*test.APIResource][]string{
+			test.Pods(): {"ns-1/pod-1"},
+			test.PVs():  nil,
+		})
+
+		got, err := h.DynamicClient.Resource(test.Pods().GVR()).Namespace("ns-1").Get(t.Context(), "pod-1", metav1.GetOptions{})
+		require.NoError(t, err)
+		annotations := got.GetAnnotations()
+		assert.NotContains(t, annotations, velerov1api.MustIncludeAdditionalItemRestoreAnnotation)
+		assert.Equal(t, "yes", annotations["keep-me"])
+	})
+
+	t.Run("SkipRestore supersedes must-include annotation and skips additional items", func(t *testing.T) {
+		h := newHarness(t)
+		h.AddItems(t, test.Pods())
+		h.AddItems(t, test.PVs())
+
+		data := &Request{
+			Log:     h.log,
+			Restore: defaultRestore().IncludedResources("pods").Result(),
+			Backup:  defaultBackup().Result(),
+			BackupReader: test.NewTarWriter(t).
+				AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()).
+				AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result()).
+				Done(),
+		}
+		warnings, errs := h.restorer.Restore(
+			data,
+			[]riav2.RestoreItemAction{
+				&pluggableAction{
+					executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) {
+						item := input.Item.(*unstructured.Unstructured)
+						annotations := item.GetAnnotations()
+						if annotations == nil {
+							annotations = map[string]string{}
+						}
+						annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true"
+						item.SetAnnotations(annotations)
+						return &velero.RestoreItemActionExecuteOutput{
+							UpdatedItem: item,
+							SkipRestore: true,
+							AdditionalItems: []velero.ResourceIdentifier{
+								{GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"},
+							},
+						}, nil
+					},
+				},
+			},
+			nil,
+		)
+
+		assertEmptyResults(t, warnings, errs)
+		assertAPIContents(t, h, map[*test.APIResource][]string{
+			test.Pods(): nil,
+			test.PVs():  nil,
+		})
+	})
+
+	t.Run("must-include does not restore additional items missing from the backup tarball", func(t *testing.T) {
+		h := newHarness(t)
+		h.AddItems(t, test.Pods())
+		h.AddItems(t, test.PVs())
+
+		data := &Request{
+			Log:     h.log,
+			Restore: defaultRestore().IncludedResources("pods").Result(),
+			Backup:  defaultBackup().Result(),
+			BackupReader: test.NewTarWriter(t).
+				AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()).
+				Done(),
+		}
+		warnings, errs := h.restorer.Restore(
+			data,
+			[]riav2.RestoreItemAction{
+				&pluggableAction{
+					executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) {
+						item := input.Item.(*unstructured.Unstructured)
+						annotations := item.GetAnnotations()
+						if annotations == nil {
+							annotations = map[string]string{}
+						}
+						annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true"
+						item.SetAnnotations(annotations)
+						return &velero.RestoreItemActionExecuteOutput{
+							UpdatedItem: item,
+							AdditionalItems: []velero.ResourceIdentifier{
+								{GroupResource: kuberesource.PersistentVolumes, Name: "pv-missing"},
+							},
+						}, nil
+					},
+				},
+			},
+			nil,
+		)
+
+		assertEmptyResults(t, errs)
+		assertNonEmptyResults(t, "warning", warnings)
+		assertAPIContents(t, h, map[*test.APIResource][]string{
+			test.Pods(): {"ns-1/pod-1"},
+			test.PVs():  nil,
+		})
+	})
+
+	t.Run("transitive must-include requires each RIA level to re-set the annotation", func(t *testing.T) {
+		h := newHarness(t)
+		h.AddItems(t, test.Pods())
+		h.AddItems(t, test.PVs())
+		h.AddItems(t, test.PVCs())
+
+		data := &Request{
+			Log:     h.log,
+			Restore: defaultRestore().IncludedResources("pods").Result(),
+			Backup:  defaultBackup().Result(),
+			BackupReader: test.NewTarWriter(t).
+				AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()).
+				AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result()).
+				AddItems("persistentvolumeclaims", builder.ForPersistentVolumeClaim("ns-2", "pvc-1").Result()).
+				Done(),
+		}
+		warnings, errs := h.restorer.Restore(
+			data,
+			[]riav2.RestoreItemAction{
+				// Parent pod RIA force-includes the excluded PV.
+				&pluggableAction{
+					selector: velero.ResourceSelector{IncludedResources: []string{"pods"}},
+					executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) {
+						item := input.Item.(*unstructured.Unstructured)
+						annotations := item.GetAnnotations()
+						if annotations == nil {
+							annotations = map[string]string{}
+						}
+						annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true"
+						item.SetAnnotations(annotations)
+						return &velero.RestoreItemActionExecuteOutput{
+							UpdatedItem: item,
+							AdditionalItems: []velero.ResourceIdentifier{
+								{GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"},
+							},
+						}, nil
+					},
+				},
+				// Child PV RIA also re-sets the annotation to force-include an excluded PVC.
+				&pluggableAction{
+					selector: velero.ResourceSelector{IncludedResources: []string{"persistentvolumes"}},
+					executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) {
+						item := input.Item.(*unstructured.Unstructured)
+						annotations := item.GetAnnotations()
+						if annotations == nil {
+							annotations = map[string]string{}
+						}
+						annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true"
+						item.SetAnnotations(annotations)
+						return &velero.RestoreItemActionExecuteOutput{
+							UpdatedItem: item,
+							AdditionalItems: []velero.ResourceIdentifier{
+								{GroupResource: kuberesource.PersistentVolumeClaims, Namespace: "ns-2", Name: "pvc-1"},
+							},
+						}, nil
+					},
+				},
+			},
+			nil,
+		)
+
+		assertEmptyResults(t, warnings, errs)
+		assertAPIContents(t, h, map[*test.APIResource][]string{
+			test.Pods(): {"ns-1/pod-1"},
+			test.PVs():  {"/pv-1"},
+			test.PVCs(): {"ns-2/pvc-1"},
+		})
+	})
+
+	t.Run("without re-annotating, transitive additional items still respect filters", func(t *testing.T) {
+		h := newHarness(t)
+		h.AddItems(t, test.Pods())
+		h.AddItems(t, test.PVs())
+		h.AddItems(t, test.PVCs())
+
+		data := &Request{
+			Log:     h.log,
+			Restore: defaultRestore().IncludedResources("pods").Result(),
+			Backup:  defaultBackup().Result(),
+			BackupReader: test.NewTarWriter(t).
+				AddItems("pods", builder.ForPod("ns-1", "pod-1").Result()).
+				AddItems("persistentvolumes", builder.ForPersistentVolume("pv-1").Result()).
+				AddItems("persistentvolumeclaims", builder.ForPersistentVolumeClaim("ns-2", "pvc-1").Result()).
+				Done(),
+		}
+		warnings, errs := h.restorer.Restore(
+			data,
+			[]riav2.RestoreItemAction{
+				&pluggableAction{
+					selector: velero.ResourceSelector{IncludedResources: []string{"pods"}},
+					executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) {
+						item := input.Item.(*unstructured.Unstructured)
+						annotations := item.GetAnnotations()
+						if annotations == nil {
+							annotations = map[string]string{}
+						}
+						annotations[velerov1api.MustIncludeAdditionalItemRestoreAnnotation] = "true"
+						item.SetAnnotations(annotations)
+						return &velero.RestoreItemActionExecuteOutput{
+							UpdatedItem: item,
+							AdditionalItems: []velero.ResourceIdentifier{
+								{GroupResource: kuberesource.PersistentVolumes, Name: "pv-1"},
+							},
+						}, nil
+					},
+				},
+				// Child PV RIA returns an additional PVC but does NOT set must-include.
+				&pluggableAction{
+					selector: velero.ResourceSelector{IncludedResources: []string{"persistentvolumes"}},
+					executeFunc: func(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) {
+						return &velero.RestoreItemActionExecuteOutput{
+							UpdatedItem: input.Item,
+							AdditionalItems: []velero.ResourceIdentifier{
+								{GroupResource: kuberesource.PersistentVolumeClaims, Namespace: "ns-2", Name: "pvc-1"},
+							},
+						}, nil
+					},
+				},
+			},
+			nil,
+		)
+
+		assertEmptyResults(t, warnings, errs)
+		assertAPIContents(t, h, map[*test.APIResource][]string{
+			test.Pods(): {"ns-1/pod-1"},
+			test.PVs():  {"/pv-1"},
+			test.PVCs(): nil,
+		})
+	})
+}
+
 // TestShouldRestore runs the ShouldRestore function for various permutations of
 // existing/nonexisting/being-deleted PVs, PVCs, and namespaces, and verifies the
 // result/error matches expectations.

pkg/uploader/block/snapshot.go

@@ -121,7 +121,10 @@ func snapshotSource(
 		return "", 0, errors.Wrapf(err, "Failed to run uploader backup for si %v", source)
 	}
 
-	snap.Tags = make(map[string]string)
+	if snap.Tags == nil {
+		snap.Tags = make(map[string]string)
+	}
+
 	snap.Tags[uploader.CBTChangeIDTag] = cbtSource.ChangeID
 	snap.Tags[uploader.CBTVolumeIDTag] = cbtSource.VolumeID
 	if snapshotTags != nil {
@@ -222,7 +225,17 @@ func Restore(ctx context.Context, blkUp Uploader, rep udmrepo.BackupRepo, snapsh
 
 	defer destDev.Close()
 
-	size, err := blkUp.Restore(snapshot, destInfo{dev: destDev, path: destPath}, bitmap.Iterator(), uploaderCfg)
+	destSize, err := destDev.Seek(0, io.SeekEnd)
+	if err != nil {
+		return 0, errors.Wrapf(err, "error getting length of block device %s", dest)
+	}
+
+	_, err = destDev.Seek(0, io.SeekStart)
+	if err != nil {
+		return 0, errors.Wrapf(err, "error reset pos of block device %s", dest)
+	}
+
+	size, err := blkUp.Restore(snapshot, destInfo{dev: destDev, path: destPath, size: destSize}, bitmap.Iterator(), uploaderCfg)
 	if err != nil {
 		return 0, errors.Wrapf(err, "error restoring to block dev %s", destPath)
 	}

pkg/uploader/block/uploader.go

@@ -17,11 +17,13 @@ limitations under the License.
 package block
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io"
 	"os"
 	"runtime"
+	"strconv"
 	"strings"
 
 	"github.com/cockroachdb/errors"
@@ -36,8 +38,9 @@ import (
 var ErrCanceled = errors.New("uploader is canceled")
 
 const (
-	blockSize  = (1 << 20)
-	bufferSize = 100 << 20
+	blockSize         = (1 << 20)
+	bufferSize        = 100 << 20
+	bdevSourceSizeTag = "bdev-source-size"
 )
 
 type sourceInfo struct {
@@ -49,6 +52,7 @@ type sourceInfo struct {
 type destInfo struct {
 	dev  *os.File
 	path string
+	size int64
 }
 
 type Uploader interface {
@@ -135,12 +139,52 @@ func (blkup *blockUploader) Backup(source sourceInfo, parentObject udmrepo.ID, b
 			Type:        udmrepo.ObjectDataTypeMetadata,
 			Permissions: 0o777,
 		},
+		Tags: map[string]string{
+			bdevSourceSizeTag: strconv.FormatInt(source.size, 10),
+		},
 	}, backupSize, nil
 }
 
-// TODO implement in following PRs
 func (blkup *blockUploader) Restore(snapshot udmrepo.Snapshot, dest destInfo, bitmap cbt.Iterator, configs map[string]string) (int64, error) {
-	return 0, errors.New("not implemented")
+	if bitmap == nil {
+		return 0, errors.New("bitmap is not available")
+	}
+
+	meta, err := blkup.repoWriter.ReadMetadata(blkup.ctx, snapshot.RootObject.ID)
+	if err != nil {
+		return 0, errors.Wrapf(err, "error reading snapshot metadata for %s", snapshot.Description)
+	}
+
+	if len(meta.SubObjects) != 1 {
+		return 0, errors.Errorf("unexpected number of bdev object (%d) for snapshot %s", len(meta.SubObjects), snapshot.Description)
+	}
+
+	sourceSize, err := getSourceSize(snapshot)
+	if err != nil {
+		sourceSize = meta.SubObjects[0].Size
+		blkup.log.Warnf("Failed to get source size from snapshot %s, use backup size %v", snapshot.Description, sourceSize)
+	}
+
+	if sourceSize > meta.SubObjects[0].Size {
+		return 0, errors.Wrapf(err, "unexpected size (%v vs. %v) for bdev object %s", meta.SubObjects[0].Size, sourceSize, meta.SubObjects[0].Name)
+	}
+
+	if sourceSize > dest.size {
+		return 0, errors.Wrapf(err, "dest dev(%s) size is too small (%v vs. %v)", dest.path, dest.size, sourceSize)
+	}
+
+	reader, err := blkup.repoWriter.OpenObject(blkup.ctx, meta.SubObjects[0].ID)
+	if err != nil {
+		return 0, errors.Wrapf(err, "error opening bdev object %v", meta.SubObjects[0].Name)
+	}
+	defer reader.Close()
+
+	size, err := blkup.restoreData(reader, dest.dev, bitmap, sourceSize, dest.path)
+	if err != nil {
+		return 0, errors.Wrapf(err, "error restoring bdev object %s to volume %s", meta.SubObjects[0].Name, dest.path)
+	}
+
+	return size, nil
 }
 
 func (blkup *blockUploader) backupObject(dev *os.File, dest udmrepo.ObjectWriter, bitmap cbt.Iterator, totalLength int64) (udmrepo.ID, int64, int64, error) {
@@ -319,6 +363,202 @@ func getObjectName(source string) string {
 	return strings.Trim(s, "-")
 }
 
+func (blkup *blockUploader) restoreData(reader io.ReadSeeker, dest *os.File, bitmap cbt.Iterator, totalLength int64, destPath string) (int64, error) {
+	list := freelist.New(bufferSize, blockSize)
+	resultChan := make(chan readResult, list.Capacity())
+	zeroBlock := make([]byte, blockSize)
+	totalCount := bitmap.Count()
+
+	quit := make(chan struct{})
+	defer close(quit)
+
+	go func() {
+		defer close(resultChan)
+
+		offset, valid := bitmap.Next()
+		var buffer []byte
+		var nextPos = uint64(0)
+		for valid {
+			select {
+			case <-blkup.ctx.Done():
+				return
+			case <-quit:
+				return
+			case buffer = <-list.Chunks():
+			}
+
+			var err error
+
+			if nextPos != offset {
+				_, err = reader.Seek(int64(offset), io.SeekStart)
+			}
+
+			if err == nil {
+				var length int
+				length, err = io.ReadFull(reader, buffer)
+				if err == nil && length <= 0 {
+					err = io.ErrUnexpectedEOF
+				}
+			}
+
+			r := readResult{
+				buffer: buffer,
+				offset: int64(offset),
+				err:    err,
+			}
+
+			if r.err != nil {
+				r.resetBuffer(list)
+			}
+
+			resultChan <- r
+
+			if r.err != nil {
+				return
+			}
+
+			nextPos = offset + uint64(blockSize)
+			offset, valid = bitmap.Next()
+		}
+	}()
+
+	var written int64
+	var result readResult
+	var writeErr error
+	var readerRunning bool
+	var zeroStart int64 = -1
+	var zeroLength int64
+	var curCount int64
+
+	for curCount < int64(totalCount) {
+		select {
+		case <-blkup.ctx.Done():
+			writeErr = ErrCanceled
+		case result, readerRunning = <-resultChan:
+			if !readerRunning {
+				if blkup.ctx.Err() != nil {
+					writeErr = ErrCanceled
+				} else {
+					writeErr = io.ErrUnexpectedEOF
+				}
+			}
+		}
+
+		if writeErr != nil {
+			break
+		}
+
+		if result.err != nil {
+			writeErr = result.err
+			break
+		}
+
+		length := min(int64(blockSize), totalLength-result.offset)
+		if bytes.Equal(result.buffer, zeroBlock) {
+			if zeroStart == -1 {
+				zeroStart = result.offset
+				zeroLength = length
+			} else if result.offset == zeroStart+zeroLength {
+				zeroLength += length
+			} else {
+				if err := blkup.flushZeroBlocks(dest, zeroStart, zeroLength, zeroBlock, destPath); err != nil {
+					writeErr = errors.Wrapf(err, "error flushing zero blocks from %v, length %v", zeroStart, zeroLength)
+					break
+				}
+				zeroStart = result.offset
+				zeroLength = length
+			}
+		} else {
+			if zeroStart != -1 {
+				if err := blkup.flushZeroBlocks(dest, zeroStart, zeroLength, zeroBlock, destPath); err != nil {
+					writeErr = errors.Wrapf(err, "error flushing zero blocks from %v, length %v", zeroStart, zeroLength)
+					break
+				}
+
+				zeroStart = -1
+				zeroLength = 0
+			}
+
+			n, err := dest.WriteAt(result.buffer[:length], result.offset)
+			if err != nil {
+				writeErr = err
+				break
+			}
+
+			if length != int64(n) {
+				writeErr = io.ErrShortWrite
+				break
+			}
+		}
+
+		written += length
+		curCount++
+
+		result.resetBuffer(list)
+
+		blkup.progress.UpdateProgress(&uploader.Progress{BytesDone: written, TotalBytes: totalLength})
+	}
+
+	result.resetBuffer(list)
+
+	if writeErr != nil {
+		return written, writeErr
+	}
+
+	if zeroStart != -1 {
+		if err := blkup.flushZeroBlocks(dest, zeroStart, zeroLength, zeroBlock, destPath); err != nil {
+			return written, errors.Wrapf(err, "error flushing zero blocks from %v, length %v", zeroStart, zeroLength)
+		}
+	}
+
+	return written, nil
+}
+
+func (blkup *blockUploader) flushZeroBlocks(dest *os.File, start int64, length int64, zeroBlock []byte, destPath string) error {
+	err := blkZeroOut(dest, start, length)
+	if err == nil {
+		return nil
+	}
+
+	blkup.log.WithError(err).Warnf("Failed to call zero out from dev %s, start %v, length %v. Fallback to conservative way", destPath, start, length)
+
+	var written int64
+	for written < length {
+		writeSize := min(len(zeroBlock), int(length-written))
+
+		n, err := dest.WriteAt(zeroBlock[:writeSize], start+written)
+		if err != nil {
+			return errors.Wrapf(err, "error writing zero buffer at %v, length %v", start+written, writeSize)
+		}
+
+		if writeSize != n {
+			return errors.Wrapf(err, "short write zero buffer at %v, length %v", start+written, writeSize)
+		}
+
+		written += int64(writeSize)
+	}
+
+	return nil
+}
+
+func getSourceSize(snapshot udmrepo.Snapshot) (int64, error) {
+	if snapshot.Tags == nil {
+		return 0, errors.New("source size tag is empty")
+	}
+
+	s, found := snapshot.Tags[bdevSourceSizeTag]
+	if !found {
+		return 0, errors.New("source size tag is missing")
+	}
+
+	size, err := strconv.ParseInt(s, 10, 64)
+	if err != nil {
+		return 0, errors.Wrapf(err, "error parsing size from %s", s)
+	}
+
+	return size, nil
+}
+
 func loadObjectFromSnapshot(ctx context.Context, rep udmrepo.BackupRepo, snapshot *udmrepo.Snapshot) (udmrepo.ID, error) {
 	if snapshot == nil {
 		return "", errors.New("snapshot is empty")

pkg/uploader/block/uploader_test.go

@@ -460,3 +460,231 @@ func TestLoadObjectFromSnapshot(t *testing.T) {
 		})
 	}
 }
+
+func TestGetSourceSize(t *testing.T) {
+	testCases := []struct {
+		name      string
+		snapshot  udmrepo.Snapshot
+		expectErr bool
+		expected  int64
+	}{
+		{
+			name:      "nil tags",
+			snapshot:  udmrepo.Snapshot{},
+			expectErr: true,
+		},
+		{
+			name: "missing tag",
+			snapshot: udmrepo.Snapshot{
+				Tags: map[string]string{},
+			},
+			expectErr: true,
+		},
+		{
+			name: "invalid tag value",
+			snapshot: udmrepo.Snapshot{
+				Tags: map[string]string{
+					bdevSourceSizeTag: "abc",
+				},
+			},
+			expectErr: true,
+		},
+		{
+			name: "valid tag value",
+			snapshot: udmrepo.Snapshot{
+				Tags: map[string]string{
+					bdevSourceSizeTag: "1048576",
+				},
+			},
+			expectErr: false,
+			expected:  1048576,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			size, err := getSourceSize(tc.snapshot)
+			if tc.expectErr {
+				assert.Error(t, err)
+			} else {
+				require.NoError(t, err)
+				assert.Equal(t, tc.expected, size)
+			}
+		})
+	}
+}
+
+func TestFlushZeroBlocks(t *testing.T) {
+	t.Run("success via write fallback", func(t *testing.T) {
+		f, err := os.CreateTemp(t.TempDir(), "zerotest-*")
+		require.NoError(t, err)
+		defer os.Remove(f.Name())
+		defer f.Close()
+
+		require.NoError(t, f.Truncate(2048))
+
+		blkup := &blockUploader{
+			log: logrus.New(),
+		}
+		blkup.log.(*logrus.Logger).Out = io.Discard
+
+		zeroBlock := make([]byte, 1024)
+		err = blkup.flushZeroBlocks(f, 0, 2048, zeroBlock, f.Name())
+
+		require.NoError(t, err)
+
+		data, err := os.ReadFile(f.Name())
+		require.NoError(t, err)
+		assert.Equal(t, make([]byte, 2048), data)
+	})
+}
+
+type errReader struct {
+	err error
+}
+
+func (r *errReader) Read(p []byte) (n int, err error) {
+	return 0, r.err
+}
+
+func (r *errReader) Seek(offset int64, whence int) (int64, error) {
+	return 0, nil
+}
+
+func TestRestoreData(t *testing.T) {
+	t.Run("success", func(t *testing.T) {
+		ctx := context.Background()
+		progress := &mockProgressUpdater{}
+		progress.On("UpdateProgress", mock.Anything).Return()
+		blkup := &blockUploader{
+			ctx:      ctx,
+			progress: progress,
+			log:      logrus.New(),
+		}
+
+		f, err := os.CreateTemp(t.TempDir(), "restoretest-*")
+		require.NoError(t, err)
+		defer os.Remove(f.Name())
+		defer f.Close()
+
+		data := make([]byte, 1048576)
+		for i := range data {
+			data[i] = 1
+		}
+		reader := bytes.NewReader(data)
+
+		iterMock := cbtmocks.NewIterator(t)
+		iterMock.On("Count").Return(uint64(1))
+		iterMock.On("Next").Return(uint64(0), true).Once()
+		iterMock.On("Next").Return(uint64(0), false)
+
+		written, err := blkup.restoreData(reader, f, iterMock, 1048576, f.Name())
+		require.NoError(t, err)
+		assert.Equal(t, int64(1048576), written)
+
+		f.Seek(0, 0)
+		writtenData, err := io.ReadAll(f)
+		require.NoError(t, err)
+		assert.Equal(t, data, writtenData)
+	})
+
+	t.Run("read err", func(t *testing.T) {
+		ctx := context.Background()
+		blkup := &blockUploader{
+			ctx: ctx,
+			log: logrus.New(),
+		}
+
+		f, err := os.CreateTemp(t.TempDir(), "restoretest-*")
+		require.NoError(t, err)
+		defer os.Remove(f.Name())
+		defer f.Close()
+
+		reader := &errReader{err: errors.New("read error")}
+
+		iterMock := cbtmocks.NewIterator(t)
+		iterMock.On("Count").Return(uint64(1))
+		iterMock.On("Next").Return(uint64(0), true).Once()
+		iterMock.On("Next").Return(uint64(0), false)
+
+		_, err = blkup.restoreData(reader, f, iterMock, 1048576, f.Name())
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "read error")
+	})
+}
+
+func TestBlockUploaderRestore(t *testing.T) {
+	t.Run("missing metadata", func(t *testing.T) {
+		ctx := context.Background()
+		repoWriter := udmrepomocks.NewBackupRepo(t)
+		blkup := NewUploader(ctx, repoWriter, nil, logrus.New())
+
+		repoWriter.On("ReadMetadata", mock.Anything, udmrepo.ID("root-id")).Return(nil, errors.New("meta not found"))
+
+		iterMock := cbtmocks.NewIterator(t)
+		_, err := blkup.Restore(udmrepo.Snapshot{RootObject: udmrepo.ObjectMetadata{ID: "root-id"}}, destInfo{}, iterMock, nil)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "meta not found")
+	})
+
+	t.Run("success", func(t *testing.T) {
+		ctx := context.Background()
+		repoWriter := udmrepomocks.NewBackupRepo(t)
+		progress := &mockProgressUpdater{}
+		progress.On("UpdateProgress", mock.Anything).Return()
+
+		blkup := NewUploader(ctx, repoWriter, progress, logrus.New())
+
+		f, err := os.CreateTemp(t.TempDir(), "restoretest-*")
+		require.NoError(t, err)
+		defer os.Remove(f.Name())
+		defer f.Close()
+
+		meta := &udmrepo.Metadata{
+			SubObjects: []udmrepo.ObjectMetadata{
+				{
+					ID:   "data-id",
+					Name: "bdev",
+					Size: 1048576,
+				},
+			},
+		}
+
+		repoWriter.On("ReadMetadata", mock.Anything, udmrepo.ID("root-id")).Return(meta, nil)
+
+		objReader := udmrepomocks.NewObjectReader(t)
+		objReader.On("Read", mock.Anything).Run(func(args mock.Arguments) {
+			p := args.Get(0).([]byte)
+			for i := range p {
+				p[i] = 1
+			}
+		}).Return(1048576, io.EOF).Once()
+		objReader.On("Read", mock.Anything).Return(0, io.EOF)
+		objReader.On("Close").Return(nil)
+
+		repoWriter.On("OpenObject", mock.Anything, udmrepo.ID("data-id")).Return(objReader, nil)
+
+		snap := udmrepo.Snapshot{
+			Description: "test snapshot",
+			RootObject:  udmrepo.ObjectMetadata{ID: "root-id"},
+			Tags: map[string]string{
+				bdevSourceSizeTag: "1048576",
+			},
+		}
+
+		dest := destInfo{
+			dev:  f,
+			size: 2048576,
+			path: f.Name(),
+		}
+
+		iterMock := cbtmocks.NewIterator(t)
+		iterMock.On("Count").Return(uint64(1))
+		iterMock.On("Next").Return(uint64(0), true).Once()
+		iterMock.On("Next").Return(uint64(0), false)
+
+		written, err := blkup.Restore(snap, dest, iterMock, nil)
+		require.NoError(t, err)
+		assert.Equal(t, int64(1048576), written)
+	})
+}

pkg/util/datamover/datamover.go

@@ -32,7 +32,18 @@ const (
 // IsBuiltInDataMover reports whether the given data mover value refers to a
 // Velero built-in data mover (an empty value or the default "velero" alias).
 func IsBuiltInDataMover(dataMover string) bool {
-	return dataMover == "" || dataMover == DataMoverTypeVelero
+	return IsVeleroBlockDataMover(dataMover) || IsVeleroFSDataMover(dataMover)
+}
+
+func IsVeleroFSDataMover(dataMover string) bool {
+	if dataMover == "" || dataMover == DataMoverTypeVelero {
+		dataMover = DataMoverTypeVeleroFs
+	}
+	return dataMover == DataMoverTypeVeleroFs
+}
+
+func IsVeleroBlockDataMover(dataMover string) bool {
+	return dataMover == DataMoverTypeVeleroBlock
 }
 
 // GetDefaultBuiltInDataMover returns the data mover used when the default

pkg/util/datamover/datamover_test.go

@@ -38,6 +38,16 @@ func TestIsBuiltInDataMover(t *testing.T) {
 			dataMover: "velero",
 			want:      true,
 		},
+		{
+			name:      "velero-fs dataMover is builtin",
+			dataMover: "velero-fs",
+			want:      true,
+		},
+		{
+			name:      "velero-block dataMover is builtin",
+			dataMover: "velero-block",
+			want:      true,
+		},
 		{
 			name:      "kopia dataMover is not builtin",
 			dataMover: "kopia",
@@ -54,3 +64,61 @@ func TestIsBuiltInDataMover(t *testing.T) {
 func TestGetDefaultBuiltInDataMover(t *testing.T) {
 	assert.Equal(t, DataMoverTypeVeleroFs, GetDefaultBuiltInDataMover())
 }
+
+func TestIsFSDataMover(t *testing.T) {
+	testcases := []struct {
+		name      string
+		dataMover string
+		want      bool
+	}{
+		{
+			name:      "empty dataMover is fs",
+			dataMover: "",
+			want:      true,
+		},
+		{
+			name:      "velero dataMover is fs",
+			dataMover: "velero",
+			want:      true,
+		},
+		{
+			name:      "velero-fs dataMover is fs",
+			dataMover: "velero-fs",
+			want:      true,
+		},
+		{
+			name:      "velero-block dataMover is not fs",
+			dataMover: "velero-block",
+			want:      false,
+		},
+	}
+	for _, tc := range testcases {
+		t.Run(tc.name, func(tt *testing.T) {
+			assert.Equal(tt, tc.want, IsVeleroFSDataMover(tc.dataMover))
+		})
+	}
+}
+
+func TestIsBlockDataMover(t *testing.T) {
+	testcases := []struct {
+		name      string
+		dataMover string
+		want      bool
+	}{
+		{
+			name:      "velero-block dataMover is block",
+			dataMover: "velero-block",
+			want:      true,
+		},
+		{
+			name:      "velero-fs dataMover is not block",
+			dataMover: "velero-fs",
+			want:      false,
+		},
+	}
+	for _, tc := range testcases {
+		t.Run(tc.name, func(tt *testing.T) {
+			assert.Equal(tt, tc.want, IsVeleroBlockDataMover(tc.dataMover))
+		})
+	}
+}