mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-07-19 14:32:36 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #9955 from sseago/vss-main

Browse Source 
Early frequent polling for CSI snapshots to handle Windows VSS timeout
This commit is contained in:
lyndon-li
2026-07-06 11:11:52 +08:00
committed by GitHub
parent 96ae6586d0 c60a5bcc7c
commit cfbd1b6ae4
5 changed files with 128 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
changelogs/unreleased/9955-sseago
+1
View File
@@ -0,0 +1 @@
Optimize VSC handle readiness polling for VSS backups
pkg/cmd/cli/install/install.go
+14 -10
View File
@@ -81,6 +81,7 @@ type Options struct {
DefaultVolumesToFsBackup bool
UploaderType string
DefaultSnapshotMoveData bool
CSISnapshotEarlyFrequentPolling bool
DisableInformerCache bool
ScheduleSkipImmediately bool
PodResources kubeutil.PodResources
@@ -141,6 +142,7 @@ func (o *Options) BindFlags(flags *pflag.FlagSet) {
flags.BoolVar(&o.DefaultVolumesToFsBackup, "default-volumes-to-fs-backup", o.DefaultVolumesToFsBackup, "Bool flag to configure Velero server to use pod volume file system backup by default for all volumes on all backups. Optional.")
flags.StringVar(&o.UploaderType, "uploader-type", o.UploaderType, fmt.Sprintf("The type of uploader to transfer the data of pod volumes, supported value: '%s'", uploader.KopiaType))
flags.BoolVar(&o.DefaultSnapshotMoveData, "default-snapshot-move-data", o.DefaultSnapshotMoveData, "Bool flag to configure Velero server to move data by default for all snapshots supporting data movement. Optional.")
flags.BoolVar(&o.CSISnapshotEarlyFrequentPolling, "csi-snapshot-early-frequent-polling", o.CSISnapshotEarlyFrequentPolling, "Bool flag to configure Velero server to use early frequent polling by default for all CSI snapshots. Optional.")
flags.BoolVar(&o.DisableInformerCache, "disable-informer-cache", o.DisableInformerCache, "Disable informer cache for Get calls on restore. With this enabled, it will speed up restore in cases where there are backup resources which already exist in the cluster, but for very large clusters this will increase velero memory usage. Default is false (don't disable). Optional.")
flags.BoolVar(&o.ScheduleSkipImmediately, "schedule-skip-immediately", o.ScheduleSkipImmediately, "Skip the first scheduled backup immediately after creating a schedule. Default is false (don't skip).")
flags.BoolVar(&o.NodeAgentDisableHostPath, "node-agent-disable-host-path", o.NodeAgentDisableHostPath, "Don't mount the pod volume host path to node-agent. Optional. Pod volume host path mount is required by fs-backup but could be disabled for other backup methods.")
@@ -238,16 +240,17 @@ func NewInstallOptions() *Options {
NodeAgentPodCPULimit: install.DefaultNodeAgentPodCPULimit,
NodeAgentPodMemLimit: install.DefaultNodeAgentPodMemLimit,
// Default to creating a VSL unless we're told otherwise
UseVolumeSnapshots: true,
NoDefaultBackupLocation: false,
CRDsOnly: false,
DefaultVolumesToFsBackup: false,
UploaderType: uploader.KopiaType,
DefaultSnapshotMoveData: false,
DisableInformerCache: false,
ScheduleSkipImmediately: false,
kubeletRootDir: install.DefaultKubeletRootDir,
NodeAgentDisableHostPath: false,
UseVolumeSnapshots: true,
NoDefaultBackupLocation: false,
CRDsOnly: false,
DefaultVolumesToFsBackup: false,
UploaderType: uploader.KopiaType,
DefaultSnapshotMoveData: false,
CSISnapshotEarlyFrequentPolling: false,
DisableInformerCache: false,
ScheduleSkipImmediately: false,
kubeletRootDir: install.DefaultKubeletRootDir,
NodeAgentDisableHostPath: false,
}
}
@@ -324,6 +327,7 @@ func (o *Options) AsVeleroOptions() (*install.VeleroOptions, error) {
DefaultVolumesToFsBackup: o.DefaultVolumesToFsBackup,
UploaderType: o.UploaderType,
DefaultSnapshotMoveData: o.DefaultSnapshotMoveData,
CSISnapshotEarlyFrequentPolling: o.CSISnapshotEarlyFrequentPolling,
DisableInformerCache: o.DisableInformerCache,
ScheduleSkipImmediately: o.ScheduleSkipImmediately,
PodResources: o.PodResources,
pkg/install/deployment.go
+16
View File
@@ -50,6 +50,7 @@ type podTemplateConfig struct {
serviceAccountName string
uploaderType string
defaultSnapshotMoveData bool
csiSnapshotEarlyFrequentPolling bool
privilegedNodeAgent bool
disableInformerCache bool
scheduleSkipImmediately bool
@@ -166,6 +167,12 @@ func WithDefaultSnapshotMoveData(b bool) podTemplateOption {
}
}
func WithCSISnapshotEarlyFrequentPolling(b bool) podTemplateOption {
return func(c *podTemplateConfig) {
c.csiSnapshotEarlyFrequentPolling = b
}
}
func WithDisableInformerCache(b bool) podTemplateOption {
return func(c *podTemplateConfig) {
c.disableInformerCache = b
@@ -489,6 +496,15 @@ func Deployment(namespace string, opts ...podTemplateOption) *appsv1api.Deployme
}...)
}
if c.csiSnapshotEarlyFrequentPolling {
deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, []corev1api.EnvVar{
{
Name: "CSI_SNAPSHOT_EARLY_FREQUENT_POLLING",
Value: "true",
},
}...)
}
deployment.Spec.Template.Spec.Containers[0].Env = append(deployment.Spec.Template.Spec.Containers[0].Env, c.envVars...)
if len(c.plugins) > 0 {
pkg/install/resources.go
+5
View File
@@ -263,6 +263,7 @@ type VeleroOptions struct {
DefaultVolumesToFsBackup bool
UploaderType string
DefaultSnapshotMoveData bool
CSISnapshotEarlyFrequentPolling bool
DisableInformerCache bool
ScheduleSkipImmediately bool
PodResources kube.PodResources
@@ -390,6 +391,10 @@ func AllResources(o *VeleroOptions) *unstructured.UnstructuredList {
deployOpts = append(deployOpts, WithDefaultSnapshotMoveData(true))
}
if o.CSISnapshotEarlyFrequentPolling {
deployOpts = append(deployOpts, WithCSISnapshotEarlyFrequentPolling(true))
}
if o.DisableInformerCache {
deployOpts = append(deployOpts, WithDisableInformerCache(true))
}
pkg/util/csi/volume_snapshot.go
+92 -60
View File
@@ -20,6 +20,8 @@ import (
"context"
"encoding/json"
"fmt"
"os"
"strconv"
"strings"
"time"
@@ -598,72 +600,102 @@ func WaitUntilVSCHandleIsReady(
log logrus.FieldLogger,
csiSnapshotTimeout time.Duration,
) (*snapshotv1api.VolumeSnapshotContent, error) {
// We'll wait 10m for the VSC to be reconciled polling
// every 5s unless backup's csiSnapshotTimeout is set
interval := 5 * time.Second
// We'll wait for the VSC to be reconciled, trying a fast poll interval first
// before falling back to a slower poll interval for the full csiSnapshotTimeout.
vsc := new(snapshotv1api.VolumeSnapshotContent)
var interval time.Duration
err := wait.PollUntilContextTimeout(
pollFunc := func(ctx context.Context) (bool, error) {
vs := new(snapshotv1api.VolumeSnapshot)
if err := crClient.Get(
ctx,
crclient.ObjectKeyFromObject(volSnap),
vs,
); err != nil {
return false,
errors.Wrapf(
err,
"failed to get volumesnapshot %s/%s",
volSnap.Namespace, volSnap.Name,
)
}
if vs.Status == nil || vs.Status.BoundVolumeSnapshotContentName == nil {
log.Infof("Waiting for CSI driver to reconcile volumesnapshot %s/%s. Retrying in %ds",
volSnap.Namespace, volSnap.Name, interval/time.Second)
return false, nil
}
if err := crClient.Get(
ctx,
crclient.ObjectKey{
Name: *vs.Status.BoundVolumeSnapshotContentName,
},
vsc,
); err != nil {
return false,
errors.Wrapf(
err,
"failed to get VolumeSnapshotContent %s for VolumeSnapshot %s/%s",
*vs.Status.BoundVolumeSnapshotContentName, vs.Namespace, vs.Name,
)
}
// we need to wait for the VolumeSnapshotContent
// to have a snapshot handle because during restore,
// we'll use that snapshot handle as the source for
// the VolumeSnapshotContent so it's statically
// bound to the existing snapshot.
if vsc.Status == nil ||
vsc.Status.SnapshotHandle == nil {
log.Infof(
"Waiting for VolumeSnapshotContents %s to have snapshot handle. Retrying in %ds",
vsc.Name, interval/time.Second)
if vsc.Status != nil &&
vsc.Status.Error != nil {
log.Warnf("VolumeSnapshotContent %s has error: %v",
vsc.Name, *vsc.Status.Error.Message)
}
return false, nil
}
return true, nil
}
var err error
frequentPolling, err := strconv.ParseBool(os.Getenv("CSI_SNAPSHOT_EARLY_FREQUENT_POLLING"))
if err == nil && frequentPolling {
// The short interval for the first ten seconds is due to the fact that
// Microsoft VSS backups have a hard-coded unfreeze call after 10 seconds,
// so we need to minimize waiting time during the first 10 seconds.
// First poll with a short interval and timeout.
interval = 1 * time.Second
timeout := 10 * time.Second
err = wait.PollUntilContextTimeout(
context.Background(),
interval,
timeout,
true,
pollFunc,
)
if err == nil {
return vsc, nil
}
if !wait.Interrupted(err) {
return nil, err
}
}
// If the first poll timed out, poll with a longer interval and the full timeout.
interval = 5 * time.Second
err = wait.PollUntilContextTimeout(
context.Background(),
interval,
csiSnapshotTimeout,
true,
func(ctx context.Context) (bool, error) {
vs := new(snapshotv1api.VolumeSnapshot)
if err := crClient.Get(
ctx,
crclient.ObjectKeyFromObject(volSnap),
vs,
); err != nil {
return false,
errors.Wrapf(
err,
"failed to get volumesnapshot %s/%s",
volSnap.Namespace, volSnap.Name,
)
}
if vs.Status == nil || vs.Status.BoundVolumeSnapshotContentName == nil {
log.Infof("Waiting for CSI driver to reconcile volumesnapshot %s/%s. Retrying in %ds",
volSnap.Namespace, volSnap.Name, interval/time.Second)
return false, nil
}
if err := crClient.Get(
ctx,
crclient.ObjectKey{
Name: *vs.Status.BoundVolumeSnapshotContentName,
},
vsc,
); err != nil {
return false,
errors.Wrapf(
err,
"failed to get VolumeSnapshotContent %s for VolumeSnapshot %s/%s",
*vs.Status.BoundVolumeSnapshotContentName, vs.Namespace, vs.Name,
)
}
// we need to wait for the VolumeSnapshotContent
// to have a snapshot handle because during restore,
// we'll use that snapshot handle as the source for
// the VolumeSnapshotContent so it's statically
// bound to the existing snapshot.
if vsc.Status == nil ||
vsc.Status.SnapshotHandle == nil {
log.Infof(
"Waiting for VolumeSnapshotContents %s to have snapshot handle. Retrying in %ds",
vsc.Name, interval/time.Second)
if vsc.Status != nil &&
vsc.Status.Error != nil {
log.Warnf("VolumeSnapshotContent %s has error: %v",
vsc.Name, *vsc.Status.Error.Message)
}
return false, nil
}
return true, nil
},
pollFunc,
)
if err != nil {