Use thread safe map for cancel recorder (#10255)

* use thread safe map for cancel recorder

Signed-off-by: Lyndon-Li <lyonghui@vmware.com>

* use atomic load and store

Signed-off-by: Lyndon-Li <lyonghui@vmware.com>

---------

Signed-off-by: Lyndon-Li <lyonghui@vmware.com>

pkg/controller/pod_volume_backup_controller_test.go

@@ -19,9 +19,12 @@ package controller
 import (
 	"context"
 	"fmt"
+	"sync"
 	"testing"
 	"time"
 
+	clocktesting "k8s.io/utils/clock/testing"
+
 	"github.com/cockroachdb/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
@@ -489,7 +492,7 @@ func TestPVBReconcile(t *testing.T) {
 			}
 
 			if test.sportTime != nil {
-				r.cancelledPVB[test.pvb.Name] = test.sportTime.Time
+				r.cancelledPVB.Store(test.pvb.Name, test.sportTime.Time)
 			}
 
 			if test.constrained {
@@ -567,9 +570,15 @@ func TestPVBReconcile(t *testing.T) {
 			}
 
 			if test.expectCancelRecord {
-				assert.Contains(t, r.cancelledPVB, test.pvb.Name)
+				_, ok := r.cancelledPVB.Load(test.pvb.Name)
+				assert.True(t, ok)
 			} else {
-				assert.Empty(t, r.cancelledPVB)
+				empty := true
+				r.cancelledPVB.Range(func(key, value any) bool {
+					empty = false
+					return false
+				})
+				assert.True(t, empty)
 			}
 
 			if isPVBInFinalState(&pvb) || pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseInProgress {
@@ -1308,3 +1317,50 @@ func TestPodVolumeBackupSetupExposeParam(t *testing.T) {
 		})
 	}
 }
+
+type pvbSequenceClock struct {
+	*clocktesting.FakeClock
+	mu sync.Mutex
+}
+
+func (c *pvbSequenceClock) Now() time.Time {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.FakeClock.Step(time.Second)
+	return c.FakeClock.Now()
+}
+
+func TestPodVolumeBackupCancelConcurrency(t *testing.T) {
+	ctx := t.Context()
+	pvb := builder.ForPodVolumeBackup(velerov1api.DefaultNamespace, "pvb-1").Cancel(true).Phase(velerov1api.PodVolumeBackupPhaseInProgress).Result()
+
+	r, err := initPVBReconciler()
+	require.NoError(t, err)
+
+	err = r.client.Create(ctx, pvb)
+	require.NoError(t, err)
+
+	firstTime := time.Now()
+	// manually store the initial time
+	r.cancelledPVB.Store(pvb.Name, firstTime)
+
+	// Custom clock that returns a different time each call
+	r.clock = &pvbSequenceClock{FakeClock: clocktesting.NewFakeClock(firstTime)}
+
+	var wg sync.WaitGroup
+	routines := 50
+	wg.Add(routines)
+
+	for i := 0; i < routines; i++ {
+		go func() {
+			defer wg.Done()
+			_, _ = r.Reconcile(ctx, ctrl.Request{NamespacedName: types.NamespacedName{Name: pvb.Name, Namespace: pvb.Namespace}})
+		}()
+	}
+
+	wg.Wait()
+
+	v, ok := r.cancelledPVB.Load(pvb.Name)
+	assert.True(t, ok)
+	assert.Equal(t, firstTime, v.(time.Time), "The initially recorded timestamp should be preserved")
+}