From df2686c14669b63f07c5713f559507504e0c1c5b Mon Sep 17 00:00:00 2001 From: Priyansh Choudhary Date: Wed, 15 Apr 2026 03:56:49 +0530 Subject: [PATCH] Add delay to avoid race conditions during VolumeSnapshotContent deletion (#9700) * Add delay to avoid race conditions during VolumeSnapshotContent deletion Signed-off-by: Priyansh Choudhary * updated changelog Signed-off-by: Priyansh Choudhary * Updated Changelog Signed-off-by: Priyansh Choudhary --- changelogs/unreleased/9700-priyansh17 | 1 + .../csi/volumesnapshotcontent_action.go | 8 +++ .../csi/volumesnapshotcontent_action_test.go | 49 +++++++++++++++++++ 3 files changed, 58 insertions(+) create mode 100644 changelogs/unreleased/9700-priyansh17 diff --git a/changelogs/unreleased/9700-priyansh17 b/changelogs/unreleased/9700-priyansh17 new file mode 100644 index 000000000..b3cb5af68 --- /dev/null +++ b/changelogs/unreleased/9700-priyansh17 @@ -0,0 +1 @@ +Fix issue #9699, add a 2-second gap between temporary CSI VolumeSnapshotContent create and delete operations \ No newline at end of file diff --git a/internal/delete/actions/csi/volumesnapshotcontent_action.go b/internal/delete/actions/csi/volumesnapshotcontent_action.go index 98e0fc03b..9473686e0 100644 --- a/internal/delete/actions/csi/volumesnapshotcontent_action.go +++ b/internal/delete/actions/csi/volumesnapshotcontent_action.go @@ -18,6 +18,7 @@ package csi import ( "context" + "time" "github.com/google/uuid" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" @@ -40,6 +41,10 @@ type volumeSnapshotContentDeleteItemAction struct { crClient crclient.Client } +const tempVSCCreateDeleteGap = 2 * time.Second + +var sleepBetweenTempVSCCreateAndDelete = time.Sleep + // AppliesTo returns information indicating // VolumeSnapshotContentRestoreItemAction action should be invoked // while restoring VolumeSnapshotContent.snapshot.storage.k8s.io resources @@ -123,6 +128,9 @@ func (p *volumeSnapshotContentDeleteItemAction) Execute( } p.log.Infof("Created temp VolumeSnapshotContent %s with DeletionPolicy=Delete to trigger cloud snapshot cleanup", snapCont.Name) + // Add a small delay before delete to avoid create/delete race conditions in CSI controllers. + sleepBetweenTempVSCCreateAndDelete(tempVSCCreateDeleteGap) + // Delete the temp VSC immediately to trigger cloud snapshot removal. // The CSI driver will handle the actual cloud snapshot deletion. if err := p.crClient.Delete( diff --git a/internal/delete/actions/csi/volumesnapshotcontent_action_test.go b/internal/delete/actions/csi/volumesnapshotcontent_action_test.go index 114bd752f..e8a0b5865 100644 --- a/internal/delete/actions/csi/volumesnapshotcontent_action_test.go +++ b/internal/delete/actions/csi/volumesnapshotcontent_action_test.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "testing" + "time" snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" "github.com/sirupsen/logrus" @@ -46,6 +47,21 @@ type fakeClientWithErrors struct { deleteError error } +type fakeClientWithCallTracking struct { + crclient.Client + events *[]string +} + +func (c *fakeClientWithCallTracking) Create(ctx context.Context, obj crclient.Object, opts ...crclient.CreateOption) error { + *c.events = append(*c.events, "create") + return c.Client.Create(ctx, obj, opts...) +} + +func (c *fakeClientWithCallTracking) Delete(ctx context.Context, obj crclient.Object, opts ...crclient.DeleteOption) error { + *c.events = append(*c.events, "delete") + return c.Client.Delete(ctx, obj, opts...) +} + func (c *fakeClientWithErrors) Get(ctx context.Context, key crclient.ObjectKey, obj crclient.Object, opts ...crclient.GetOption) error { if c.getError != nil { return c.getError @@ -325,6 +341,39 @@ func TestTryDeleteOriginalVSC(t *testing.T) { }) } +func TestVSCExecute_CreateSleepDeleteOrder(t *testing.T) { + snapshotHandleStr := "test" + vsc := builder.ForVolumeSnapshotContent("bar"). + ObjectMeta(builder.WithLabelsMap(map[string]string{velerov1api.BackupNameLabel: "backup"})). + Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: &snapshotHandleStr}). + Result() + + vscMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(vsc) + require.NoError(t, err) + + events := make([]string, 0, 3) + realClient := velerotest.NewFakeControllerRuntimeClient(t) + trackingClient := &fakeClientWithCallTracking{Client: realClient, events: &events} + + originalSleep := sleepBetweenTempVSCCreateAndDelete + t.Cleanup(func() { + sleepBetweenTempVSCCreateAndDelete = originalSleep + }) + + sleepBetweenTempVSCCreateAndDelete = func(d time.Duration) { + require.Equal(t, tempVSCCreateDeleteGap, d) + events = append(events, "sleep") + } + + p := volumeSnapshotContentDeleteItemAction{log: logrus.StandardLogger(), crClient: trackingClient} + err = p.Execute(&velero.DeleteItemActionExecuteInput{ + Item: &unstructured.Unstructured{Object: vscMap}, + Backup: builder.ForBackup("velero", "backup").Result(), + }) + require.NoError(t, err) + require.Equal(t, []string{"create", "sleep", "delete"}, events) +} + func boolPtr(b bool) *bool { return &b }