Use thread safe map for cancel recorder (#10255)

* use thread safe map for cancel recorder

Signed-off-by: Lyndon-Li <lyonghui@vmware.com>

* use atomic load and store

Signed-off-by: Lyndon-Li <lyonghui@vmware.com>

---------

Signed-off-by: Lyndon-Li <lyonghui@vmware.com>

pkg/controller/pod_volume_restore_controller_test.go

@@ -19,9 +19,12 @@ package controller
 import (
 	"context"
 	"fmt"
+	"sync"
 	"testing"
 	"time"
 
+	clocktesting "k8s.io/utils/clock/testing"
+
 	"github.com/cockroachdb/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
@@ -1087,7 +1090,7 @@ func TestPodVolumeRestoreReconcile(t *testing.T) {
 			}
 
 			if test.sportTime != nil {
-				r.cancelledPVR[test.pvr.Name] = test.sportTime.Time
+				r.cancelledPVR.Store(test.pvr.Name, test.sportTime.Time)
 			}
 
 			if test.constrained {
@@ -1208,9 +1211,15 @@ func TestPodVolumeRestoreReconcile(t *testing.T) {
 			}
 
 			if test.expectCancelRecord {
-				assert.Contains(t, r.cancelledPVR, test.pvr.Name)
+				_, ok := r.cancelledPVR.Load(test.pvr.Name)
+				assert.True(t, ok)
 			} else {
-				assert.Empty(t, r.cancelledPVR)
+				empty := true
+				r.cancelledPVR.Range(func(key, value any) bool {
+					empty = false
+					return false
+				})
+				assert.True(t, empty)
 			}
 
 			if isPVRInFinalState(&pvr) || pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseInProgress {
@@ -1935,3 +1944,47 @@ func TestResumeCancellablePodVolumeRestore(t *testing.T) {
 		})
 	}
 }
+
+type pvrSequenceClock struct {
+	*clocktesting.FakeClock
+	mu sync.Mutex
+}
+
+func (c *pvrSequenceClock) Now() time.Time {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.FakeClock.Step(time.Second)
+	return c.FakeClock.Now()
+}
+
+func TestPodVolumeRestoreCancelConcurrency(t *testing.T) {
+	ctx := t.Context()
+	pvr := builder.ForPodVolumeRestore(velerov1api.DefaultNamespace, "pvr-1").Cancel(true).Phase(velerov1api.PodVolumeRestorePhaseInProgress).Result()
+
+	r, err := initPodVolumeRestoreReconciler(nil, []client.Object{pvr})
+	require.NoError(t, err)
+
+	firstTime := time.Now()
+	// manually store the initial time
+	r.cancelledPVR.Store(pvr.Name, firstTime)
+
+	// Custom clock that returns a different time each call
+	r.clock = &pvrSequenceClock{FakeClock: clocktesting.NewFakeClock(firstTime)}
+
+	var wg sync.WaitGroup
+	routines := 50
+	wg.Add(routines)
+
+	for i := 0; i < routines; i++ {
+		go func() {
+			defer wg.Done()
+			_, _ = r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: pvr.Name, Namespace: pvr.Namespace}})
+		}()
+	}
+
+	wg.Wait()
+
+	v, ok := r.cancelledPVR.Load(pvr.Name)
+	assert.True(t, ok)
+	assert.Equal(t, firstTime, v.(time.Time), "The initially recorded timestamp should be preserved")
+}