From fa20e46016da7574cc7e956f3e0e094d65f966cf Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Tue, 10 Mar 2026 15:41:17 -0400 Subject: [PATCH 1/2] refactor: Optimize VSC handle readiness polling for VSS backups Co-authored-by: aider (gemini/gemini-2.5-pro) Signed-off-by: Scott Seago --- changelogs/unreleased/9602-sseago | 1 + pkg/util/csi/volume_snapshot.go | 143 ++++++++++++++++++------------ 2 files changed, 85 insertions(+), 59 deletions(-) create mode 100644 changelogs/unreleased/9602-sseago diff --git a/changelogs/unreleased/9602-sseago b/changelogs/unreleased/9602-sseago new file mode 100644 index 000000000..6bed2f243 --- /dev/null +++ b/changelogs/unreleased/9602-sseago @@ -0,0 +1 @@ +Optimize VSC handle readiness polling for VSS backups diff --git a/pkg/util/csi/volume_snapshot.go b/pkg/util/csi/volume_snapshot.go index 8cc7c043a..69f4b1a67 100644 --- a/pkg/util/csi/volume_snapshot.go +++ b/pkg/util/csi/volume_snapshot.go @@ -598,72 +598,97 @@ func WaitUntilVSCHandleIsReady( log logrus.FieldLogger, csiSnapshotTimeout time.Duration, ) (*snapshotv1api.VolumeSnapshotContent, error) { - // We'll wait 10m for the VSC to be reconciled polling - // every 5s unless backup's csiSnapshotTimeout is set - interval := 5 * time.Second + // We'll wait for the VSC to be reconciled, trying a fast poll interval first + // before falling back to a slower poll interval for the full csiSnapshotTimeout. vsc := new(snapshotv1api.VolumeSnapshotContent) + var interval time.Duration + pollFunc := func(ctx context.Context) (bool, error) { + vs := new(snapshotv1api.VolumeSnapshot) + if err := crClient.Get( + ctx, + crclient.ObjectKeyFromObject(volSnap), + vs, + ); err != nil { + return false, + errors.Wrapf( + err, + "failed to get volumesnapshot %s/%s", + volSnap.Namespace, volSnap.Name, + ) + } + + if vs.Status == nil || vs.Status.BoundVolumeSnapshotContentName == nil { + log.Infof("Waiting for CSI driver to reconcile volumesnapshot %s/%s. Retrying in %ds", + volSnap.Namespace, volSnap.Name, interval/time.Second) + return false, nil + } + + if err := crClient.Get( + ctx, + crclient.ObjectKey{ + Name: *vs.Status.BoundVolumeSnapshotContentName, + }, + vsc, + ); err != nil { + return false, + errors.Wrapf( + err, + "failed to get VolumeSnapshotContent %s for VolumeSnapshot %s/%s", + *vs.Status.BoundVolumeSnapshotContentName, vs.Namespace, vs.Name, + ) + } + + // we need to wait for the VolumeSnapshotContent + // to have a snapshot handle because during restore, + // we'll use that snapshot handle as the source for + // the VolumeSnapshotContent so it's statically + // bound to the existing snapshot. + if vsc.Status == nil || + vsc.Status.SnapshotHandle == nil { + log.Infof( + "Waiting for VolumeSnapshotContents %s to have snapshot handle. Retrying in %ds", + vsc.Name, interval/time.Second) + if vsc.Status != nil && + vsc.Status.Error != nil { + log.Warnf("VolumeSnapshotContent %s has error: %v", + vsc.Name, *vsc.Status.Error.Message) + } + return false, nil + } + + return true, nil + } + + // The short interval for the first ten seconds is due to the fact that + // Microsoft VSS backups have a hard-coded unfreeze call after 10 seconds, + // so we need to minimize waiting time during the first 10 seconds. + // First poll with a short interval and timeout. + interval = 1 * time.Second + timeout := 10 * time.Second err := wait.PollUntilContextTimeout( + context.Background(), + interval, + timeout, + true, + pollFunc, + ) + + if err == nil { + return vsc, nil + } + if !wait.Interrupted(err) { + return nil, err + } + + // If the first poll timed out, poll with a longer interval and the full timeout. + interval = 5 * time.Second + err = wait.PollUntilContextTimeout( context.Background(), interval, csiSnapshotTimeout, true, - func(ctx context.Context) (bool, error) { - vs := new(snapshotv1api.VolumeSnapshot) - if err := crClient.Get( - ctx, - crclient.ObjectKeyFromObject(volSnap), - vs, - ); err != nil { - return false, - errors.Wrapf( - err, - "failed to get volumesnapshot %s/%s", - volSnap.Namespace, volSnap.Name, - ) - } - - if vs.Status == nil || vs.Status.BoundVolumeSnapshotContentName == nil { - log.Infof("Waiting for CSI driver to reconcile volumesnapshot %s/%s. Retrying in %ds", - volSnap.Namespace, volSnap.Name, interval/time.Second) - return false, nil - } - - if err := crClient.Get( - ctx, - crclient.ObjectKey{ - Name: *vs.Status.BoundVolumeSnapshotContentName, - }, - vsc, - ); err != nil { - return false, - errors.Wrapf( - err, - "failed to get VolumeSnapshotContent %s for VolumeSnapshot %s/%s", - *vs.Status.BoundVolumeSnapshotContentName, vs.Namespace, vs.Name, - ) - } - - // we need to wait for the VolumeSnapshotContent - // to have a snapshot handle because during restore, - // we'll use that snapshot handle as the source for - // the VolumeSnapshotContent so it's statically - // bound to the existing snapshot. - if vsc.Status == nil || - vsc.Status.SnapshotHandle == nil { - log.Infof( - "Waiting for VolumeSnapshotContents %s to have snapshot handle. Retrying in %ds", - vsc.Name, interval/time.Second) - if vsc.Status != nil && - vsc.Status.Error != nil { - log.Warnf("VolumeSnapshotContent %s has error: %v", - vsc.Name, *vsc.Status.Error.Message) - } - return false, nil - } - - return true, nil - }, + pollFunc, ) if err != nil { From c60a5bcc7c7d3d9791ab6a3214d5a03c228e1f9f Mon Sep 17 00:00:00 2001 From: Scott Seago Date: Wed, 18 Mar 2026 18:07:18 -0400 Subject: [PATCH 2/2] feat: Implement early frequent polling for CSI snapshots Co-authored-by: aider (gemini/gemini-2.5-pro) Signed-off-by: Scott Seago --- .../unreleased/{9602-sseago => 9955-sseago} | 0 pkg/cmd/cli/install/install.go | 24 ++++++----- pkg/install/deployment.go | 16 +++++++ pkg/install/resources.go | 5 +++ pkg/util/csi/volume_snapshot.go | 43 +++++++++++-------- 5 files changed, 60 insertions(+), 28 deletions(-) rename changelogs/unreleased/{9602-sseago => 9955-sseago} (100%) diff --git a/changelogs/unreleased/9602-sseago b/changelogs/unreleased/9955-sseago similarity index 100% rename from changelogs/unreleased/9602-sseago rename to changelogs/unreleased/9955-sseago diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index 26b4f9384..0df53eb32 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -81,6 +81,7 @@ type Options struct { DefaultVolumesToFsBackup bool UploaderType string DefaultSnapshotMoveData bool + CSISnapshotEarlyFrequentPolling bool DisableInformerCache bool ScheduleSkipImmediately bool PodResources kubeutil.PodResources @@ -141,6 +142,7 @@ func (o *Options) BindFlags(flags *pflag.FlagSet) { flags.BoolVar(&o.DefaultVolumesToFsBackup, "default-volumes-to-fs-backup", o.DefaultVolumesToFsBackup, "Bool flag to configure Velero server to use pod volume file system backup by default for all volumes on all backups. Optional.") flags.StringVar(&o.UploaderType, "uploader-type", o.UploaderType, fmt.Sprintf("The type of uploader to transfer the data of pod volumes, supported value: '%s'", uploader.KopiaType)) flags.BoolVar(&o.DefaultSnapshotMoveData, "default-snapshot-move-data", o.DefaultSnapshotMoveData, "Bool flag to configure Velero server to move data by default for all snapshots supporting data movement. Optional.") + flags.BoolVar(&o.CSISnapshotEarlyFrequentPolling, "csi-snapshot-early-frequent-polling", o.CSISnapshotEarlyFrequentPolling, "Bool flag to configure Velero server to use early frequent polling by default for all CSI snapshots. Optional.") flags.BoolVar(&o.DisableInformerCache, "disable-informer-cache", o.DisableInformerCache, "Disable informer cache for Get calls on restore. With this enabled, it will speed up restore in cases where there are backup resources which already exist in the cluster, but for very large clusters this will increase velero memory usage. Default is false (don't disable). Optional.") flags.BoolVar(&o.ScheduleSkipImmediately, "schedule-skip-immediately", o.ScheduleSkipImmediately, "Skip the first scheduled backup immediately after creating a schedule. Default is false (don't skip).") flags.BoolVar(&o.NodeAgentDisableHostPath, "node-agent-disable-host-path", o.NodeAgentDisableHostPath, "Don't mount the pod volume host path to node-agent. Optional. Pod volume host path mount is required by fs-backup but could be disabled for other backup methods.") @@ -238,16 +240,17 @@ func NewInstallOptions() *Options { NodeAgentPodCPULimit: install.DefaultNodeAgentPodCPULimit, NodeAgentPodMemLimit: install.DefaultNodeAgentPodMemLimit, // Default to creating a VSL unless we're told otherwise - UseVolumeSnapshots: true, - NoDefaultBackupLocation: false, - CRDsOnly: false, - DefaultVolumesToFsBackup: false, - UploaderType: uploader.KopiaType, - DefaultSnapshotMoveData: false, - DisableInformerCache: false, - ScheduleSkipImmediately: false, - kubeletRootDir: install.DefaultKubeletRootDir, - NodeAgentDisableHostPath: false, + UseVolumeSnapshots: true, + NoDefaultBackupLocation: false, + CRDsOnly: false, + DefaultVolumesToFsBackup: false, + UploaderType: uploader.KopiaType, + DefaultSnapshotMoveData: false, + CSISnapshotEarlyFrequentPolling: false, + DisableInformerCache: false, + ScheduleSkipImmediately: false, + kubeletRootDir: install.DefaultKubeletRootDir, + NodeAgentDisableHostPath: false, } } @@ -324,6 +327,7 @@ func (o *Options) AsVeleroOptions() (*install.VeleroOptions, error) { DefaultVolumesToFsBackup: o.DefaultVolumesToFsBackup, UploaderType: o.UploaderType, DefaultSnapshotMoveData: o.DefaultSnapshotMoveData, + CSISnapshotEarlyFrequentPolling: o.CSISnapshotEarlyFrequentPolling, DisableInformerCache: o.DisableInformerCache, ScheduleSkipImmediately: o.ScheduleSkipImmediately, PodResources: o.PodResources, diff --git a/pkg/install/deployment.go b/pkg/install/deployment.go index 7af17bc53..4ce4b5a4f 100644 --- a/pkg/install/deployment.go +++ b/pkg/install/deployment.go @@ -50,6 +50,7 @@ type podTemplateConfig struct { serviceAccountName string uploaderType string defaultSnapshotMoveData bool + csiSnapshotEarlyFrequentPolling bool privilegedNodeAgent bool disableInformerCache bool scheduleSkipImmediately bool @@ -166,6 +167,12 @@ func WithDefaultSnapshotMoveData(b bool) podTemplateOption { } } +func WithCSISnapshotEarlyFrequentPolling(b bool) podTemplateOption { + return func(c *podTemplateConfig) { + c.csiSnapshotEarlyFrequentPolling = b + } +} + func WithDisableInformerCache(b bool) podTemplateOption { return func(c *podTemplateConfig) { c.disableInformerCache = b @@ -489,6 +496,15 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1api.Deployme }...) } + if c.csiSnapshotEarlyFrequentPolling { + deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, []corev1api.EnvVar{ + { + Name: "CSI_SNAPSHOT_EARLY_FREQUENT_POLLING", + Value: "true", + }, + }...) + } + deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, c.envVars...) if len(c.plugins) > 0 { diff --git a/pkg/install/resources.go b/pkg/install/resources.go index 5c7534774..c4ec6f1bc 100644 --- a/pkg/install/resources.go +++ b/pkg/install/resources.go @@ -263,6 +263,7 @@ type VeleroOptions struct { DefaultVolumesToFsBackup bool UploaderType string DefaultSnapshotMoveData bool + CSISnapshotEarlyFrequentPolling bool DisableInformerCache bool ScheduleSkipImmediately bool PodResources kube.PodResources @@ -390,6 +391,10 @@ func AllResources(o *VeleroOptions) *unstructured.UnstructuredList { deployOpts = append(deployOpts, WithDefaultSnapshotMoveData(true)) } + if o.CSISnapshotEarlyFrequentPolling { + deployOpts = append(deployOpts, WithCSISnapshotEarlyFrequentPolling(true)) + } + if o.DisableInformerCache { deployOpts = append(deployOpts, WithDisableInformerCache(true)) } diff --git a/pkg/util/csi/volume_snapshot.go b/pkg/util/csi/volume_snapshot.go index 69f4b1a67..b78455bc8 100644 --- a/pkg/util/csi/volume_snapshot.go +++ b/pkg/util/csi/volume_snapshot.go @@ -20,6 +20,8 @@ import ( "context" "encoding/json" "fmt" + "os" + "strconv" "strings" "time" @@ -660,25 +662,30 @@ func WaitUntilVSCHandleIsReady( return true, nil } - // The short interval for the first ten seconds is due to the fact that - // Microsoft VSS backups have a hard-coded unfreeze call after 10 seconds, - // so we need to minimize waiting time during the first 10 seconds. - // First poll with a short interval and timeout. - interval = 1 * time.Second - timeout := 10 * time.Second - err := wait.PollUntilContextTimeout( - context.Background(), - interval, - timeout, - true, - pollFunc, - ) + var err error + frequentPolling, err := strconv.ParseBool(os.Getenv("CSI_SNAPSHOT_EARLY_FREQUENT_POLLING")) - if err == nil { - return vsc, nil - } - if !wait.Interrupted(err) { - return nil, err + if err == nil && frequentPolling { + // The short interval for the first ten seconds is due to the fact that + // Microsoft VSS backups have a hard-coded unfreeze call after 10 seconds, + // so we need to minimize waiting time during the first 10 seconds. + // First poll with a short interval and timeout. + interval = 1 * time.Second + timeout := 10 * time.Second + err = wait.PollUntilContextTimeout( + context.Background(), + interval, + timeout, + true, + pollFunc, + ) + + if err == nil { + return vsc, nil + } + if !wait.Interrupted(err) { + return nil, err + } } // If the first poll timed out, poll with a longer interval and the full timeout.