From da144457dc9d627af609f2c78afd5525673a4b7d Mon Sep 17 00:00:00 2001 From: Shubham Pampattiwar Date: Tue, 10 Mar 2026 10:40:09 -0700 Subject: [PATCH 1/2] Fix DBR stuck when CSI snapshot no longer exists in cloud provider (#9581) * Fix DBR stuck when CSI snapshot no longer exists in cloud provider During backup deletion, VolumeSnapshotContentDeleteItemAction creates a new VSC with the snapshot handle from the backup and polls for readiness. If the underlying snapshot no longer exists (e.g., deleted externally), the CSI driver reports Status.Error but checkVSCReadiness() only checks ReadyToUse, causing it to poll for the full 10-minute timeout instead of failing fast. Additionally, the newly created VSC is never cleaned up on failure, leaving orphaned resources in the cluster. This commit: - Adds Status.Error detection in checkVSCReadiness() to fail immediately on permanent CSI driver errors (e.g., InvalidSnapshot.NotFound) - Cleans up the dangling VSC when readiness polling fails Fixes #9579 Signed-off-by: Shubham Pampattiwar * Add changelog for PR #9581 Signed-off-by: Shubham Pampattiwar * Fix typo in pod_volume_test.go: colume -> volume Signed-off-by: Shubham Pampattiwar --------- Signed-off-by: Shubham Pampattiwar Signed-off-by: Priyansh Choudhary --- .../unreleased/9581-shubham-pampattiwar | 1 + .../csi/volumesnapshotcontent_action.go | 11 ++++++ .../csi/volumesnapshotcontent_action_test.go | 39 +++++++++++++++++++ 3 files changed, 51 insertions(+) create mode 100644 changelogs/unreleased/9581-shubham-pampattiwar diff --git a/changelogs/unreleased/9581-shubham-pampattiwar b/changelogs/unreleased/9581-shubham-pampattiwar new file mode 100644 index 000000000..f369a8af5 --- /dev/null +++ b/changelogs/unreleased/9581-shubham-pampattiwar @@ -0,0 +1 @@ +Fix DBR stuck when CSI snapshot no longer exists in cloud provider diff --git a/internal/delete/actions/csi/volumesnapshotcontent_action.go b/internal/delete/actions/csi/volumesnapshotcontent_action.go index 8ac7d4d2a..39d570a8e 100644 --- a/internal/delete/actions/csi/volumesnapshotcontent_action.go +++ b/internal/delete/actions/csi/volumesnapshotcontent_action.go @@ -129,6 +129,10 @@ func (p *volumeSnapshotContentDeleteItemAction) Execute( return checkVSCReadiness(ctx, &snapCont, p.crClient) }, ); err != nil { + // Clean up the VSC we created since it can't become ready + if deleteErr := p.crClient.Delete(context.TODO(), &snapCont); deleteErr != nil && !apierrors.IsNotFound(deleteErr) { + p.log.WithError(deleteErr).Errorf("Failed to clean up VolumeSnapshotContent %s", snapCont.Name) + } return errors.Wrapf(err, "fail to wait VolumeSnapshotContent %s becomes ready.", snapCont.Name) } @@ -159,6 +163,13 @@ var checkVSCReadiness = func( return true, nil } + // Fail fast on permanent CSI driver errors (e.g., InvalidSnapshot.NotFound) + if tmpVSC.Status != nil && tmpVSC.Status.Error != nil && tmpVSC.Status.Error.Message != nil { + return false, errors.Errorf( + "VolumeSnapshotContent %s has error: %s", vsc.Name, *tmpVSC.Status.Error.Message, + ) + } + return false, nil } diff --git a/internal/delete/actions/csi/volumesnapshotcontent_action_test.go b/internal/delete/actions/csi/volumesnapshotcontent_action_test.go index 223b84a83..3803fe29f 100644 --- a/internal/delete/actions/csi/volumesnapshotcontent_action_test.go +++ b/internal/delete/actions/csi/volumesnapshotcontent_action_test.go @@ -94,6 +94,19 @@ func TestVSCExecute(t *testing.T) { return false, errors.Errorf("test error case") }, }, + { + name: "Error case with CSI error, dangling VSC should be cleaned up", + vsc: builder.ForVolumeSnapshotContent("bar").ObjectMeta(builder.WithLabelsMap(map[string]string{velerov1api.BackupNameLabel: "backup"})).Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: &snapshotHandleStr}).Result(), + backup: builder.ForBackup("velero", "backup").ObjectMeta(builder.WithAnnotationsMap(map[string]string{velerov1api.ResourceTimeoutAnnotation: "5s"})).Result(), + expectErr: true, + function: func( + ctx context.Context, + vsc *snapshotv1api.VolumeSnapshotContent, + client crclient.Client, + ) (bool, error) { + return false, errors.Errorf("VolumeSnapshotContent %s has error: InvalidSnapshot.NotFound", vsc.Name) + }, + }, } for _, test := range tests { @@ -190,6 +203,24 @@ func TestCheckVSCReadiness(t *testing.T) { expectErr: false, ready: false, }, + { + name: "VSC has error from CSI driver", + vsc: &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-1", + Namespace: "velero", + }, + Status: &snapshotv1api.VolumeSnapshotContentStatus{ + ReadyToUse: boolPtr(false), + Error: &snapshotv1api.VolumeSnapshotError{ + Message: stringPtr("InvalidSnapshot.NotFound: The snapshot 'snap-0abc123' does not exist."), + }, + }, + }, + createVSC: true, + expectErr: true, + ready: false, + }, } for _, test := range tests { @@ -208,3 +239,11 @@ func TestCheckVSCReadiness(t *testing.T) { }) } } + +func boolPtr(b bool) *bool { + return &b +} + +func stringPtr(s string) *string { + return &s +} From 6658ceb68b2367a74161111f5d83a0ad83a2be41 Mon Sep 17 00:00:00 2001 From: Priyansh Choudhary Date: Mon, 23 Mar 2026 11:04:08 +0530 Subject: [PATCH 2/2] Update Dockerfile to use setup-envtest for downloading envtest binaries Signed-off-by: Priyansh Choudhary --- changelogs/unreleased/9581-shubham-pampattiwar | 1 - hack/build-image/Dockerfile | 9 +++++---- pkg/util/podvolume/pod_volume_test.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) delete mode 100644 changelogs/unreleased/9581-shubham-pampattiwar diff --git a/changelogs/unreleased/9581-shubham-pampattiwar b/changelogs/unreleased/9581-shubham-pampattiwar deleted file mode 100644 index f369a8af5..000000000 --- a/changelogs/unreleased/9581-shubham-pampattiwar +++ /dev/null @@ -1 +0,0 @@ -Fix DBR stuck when CSI snapshot no longer exists in cloud provider diff --git a/hack/build-image/Dockerfile b/hack/build-image/Dockerfile index 069398e5f..15c9f9ca8 100644 --- a/hack/build-image/Dockerfile +++ b/hack/build-image/Dockerfile @@ -21,10 +21,11 @@ ENV GO111MODULE=on ENV GOPROXY=${GOPROXY} # kubebuilder test bundle is separated from kubebuilder. Need to setup it for CI test. -RUN curl -sSLo envtest-bins.tar.gz https://go.kubebuilder.io/test-tools/1.22.1/linux/$(go env GOARCH) && \ - mkdir /usr/local/kubebuilder && \ - tar -C /usr/local/kubebuilder --strip-components=1 -zvxf envtest-bins.tar.gz - +# Using setup-envtest to download envtest binaries +RUN go install sigs.k8s.io/controller-runtime/tools/setup-envtest@release-0.19 && \ + mkdir -p /usr/local/kubebuilder/bin && \ + ENVTEST_ASSETS_DIR=$(setup-envtest use 1.24.2 --bin-dir /usr/local/kubebuilder/bin -p path) && \ + cp -r ${ENVTEST_ASSETS_DIR}/* /usr/local/kubebuilder/bin/ RUN wget --quiet https://github.com/kubernetes-sigs/kubebuilder/releases/download/v3.2.0/kubebuilder_linux_$(go env GOARCH) && \ mv kubebuilder_linux_$(go env GOARCH) /usr/local/kubebuilder/bin/kubebuilder && \ chmod +x /usr/local/kubebuilder/bin/kubebuilder diff --git a/pkg/util/podvolume/pod_volume_test.go b/pkg/util/podvolume/pod_volume_test.go index 9f6738a65..2bc9da86f 100644 --- a/pkg/util/podvolume/pod_volume_test.go +++ b/pkg/util/podvolume/pod_volume_test.go @@ -156,7 +156,7 @@ func TestGetVolumesByPod(t *testing.T) { Volumes: []corev1api.Volume{ // PVB Volumes {Name: "pvbPV1"}, {Name: "pvbPV2"}, {Name: "pvbPV3"}, - /// Excluded from PVB because colume mounting default service account token + /// Excluded from PVB because volume mounting default service account token {Name: "default-token-5xq45"}, }, },