Merge branch 'vmware-tanzu:release-1.9' into release-1.9

changelogs/unreleased/5186-jxun

@@ -0,0 +1 @@
+Add annotation "pv.kubernetes.io/migrated-to" for CSI checking.

changelogs/unreleased/5189-jxun

@@ -0,0 +1 @@
+Make CSI snapshot creation timeout configurable.

config/crd/v1/bases/velero.io_backups.yaml

@@ -37,6 +37,11 @@ spec:
           spec:
             description: BackupSpec defines the specification for a Velero backup.
             properties:
+              csiSnapshotTimeout:
+                description: CSISnapshotTimeout specifies the time used to wait for
+                  CSI VolumeSnapshot status turns to ReadyToUse during creation, before
+                  returning error as timeout. The default value is 10 minute.
+                type: string
               defaultVolumesToRestic:
                 description: DefaultVolumesToRestic specifies whether restic should
                   be used to take a backup of all pod volumes by default.

config/crd/v1/bases/velero.io_schedules.yaml

@@ -61,6 +61,11 @@ spec:
                 description: Template is the definition of the Backup to be run on
                   the provided schedule
                 properties:
+                  csiSnapshotTimeout:
+                    description: CSISnapshotTimeout specifies the time used to wait
+                      for CSI VolumeSnapshot status turns to ReadyToUse during creation,
+                      before returning error as timeout. The default value is 10 minute.
+                    type: string
                   defaultVolumesToRestic:
                     description: DefaultVolumesToRestic specifies whether restic should
                       be used to take a backup of all pod volumes by default.

pkg/apis/velero/v1/backup.go

@@ -110,6 +110,12 @@ type BackupSpec struct {
 	// +optional
 	// +nullable
 	OrderedResources map[string]string `json:"orderedResources,omitempty"`
+
+	// CSISnapshotTimeout specifies the time used to wait for CSI VolumeSnapshot status turns to
+	// ReadyToUse during creation, before returning error as timeout.
+	// The default value is 10 minute.
+	// +optional
+	CSISnapshotTimeout metav1.Duration `json:"csiSnapshotTimeout,omitempty"`
 }
 
 // BackupHooks contains custom behaviors that should be executed at different phases of the backup.

pkg/apis/velero/v1/zz_generated.deepcopy.go

@@ -250,6 +250,7 @@ func (in *BackupSpec) DeepCopyInto(out *BackupSpec) {
 			(*out)[key] = val
 		}
 	}
+	out.CSISnapshotTimeout = in.CSISnapshotTimeout
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupSpec.

pkg/builder/backup_builder.go

@@ -233,3 +233,9 @@ func (b *BackupBuilder) OrderedResources(orders map[string]string) *BackupBuilde
 	b.object.Spec.OrderedResources = orders
 	return b
 }
+
+// CSISnapshotTimeout sets the Backup's CSISnapshotTimeout
+func (b *BackupBuilder) CSISnapshotTimeout(timeout time.Duration) *BackupBuilder {
+	b.object.Spec.CSISnapshotTimeout.Duration = timeout
+	return b
+}

pkg/cmd/cli/backup/create.go

@@ -98,6 +98,7 @@ type CreateOptions struct {
 	SnapshotLocations       []string
 	FromSchedule            string
 	OrderedResources        string
+	CSISnapshotTimeout      time.Duration
 
 	client veleroclient.Interface
 }
@@ -122,6 +123,7 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) {
 	flags.StringSliceVar(&o.SnapshotLocations, "volume-snapshot-locations", o.SnapshotLocations, "List of locations (at most one per provider) where volume snapshots should be stored.")
 	flags.VarP(&o.Selector, "selector", "l", "Only back up resources matching this label selector.")
 	flags.StringVar(&o.OrderedResources, "ordered-resources", "", "Mapping Kinds to an ordered list of specific resources of that Kind.  Resource names are separated by commas and their names are in format 'namespace/resourcename'. For cluster scope resource, simply use resource name. Key-value pairs in the mapping are separated by semi-colon.  Example: 'pods=ns1/pod1,ns1/pod2;persistentvolumeclaims=ns1/pvc4,ns1/pvc8'.  Optional.")
+	flags.DurationVar(&o.CSISnapshotTimeout, "csi-snapshot-timeout", o.CSISnapshotTimeout, "How long to wait for CSI snapshot creation before timeout.")
 	f := flags.VarPF(&o.SnapshotVolumes, "snapshot-volumes", "", "Take snapshots of PersistentVolumes as part of the backup.")
 	// this allows the user to just specify "--snapshot-volumes" as shorthand for "--snapshot-volumes=true"
 	// like a normal bool flag
@@ -332,7 +334,8 @@ func (o *CreateOptions) BuildBackup(namespace string) (*velerov1api.Backup, erro
 			LabelSelector(o.Selector.LabelSelector).
 			TTL(o.TTL).
 			StorageLocation(o.StorageLocation).
-			VolumeSnapshotLocations(o.SnapshotLocations...)
+			VolumeSnapshotLocations(o.SnapshotLocations...).
+			CSISnapshotTimeout(o.CSISnapshotTimeout)
 		if len(o.OrderedResources) > 0 {
 			orders, err := ParseOrderedResources(o.OrderedResources)
 			if err != nil {

pkg/cmd/cli/backup/create_test.go

@@ -19,6 +19,7 @@ package backup
 import (
 	"context"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -35,6 +36,7 @@ func TestCreateOptions_BuildBackup(t *testing.T) {
 	o.Labels.Set("velero.io/test=true")
 	o.OrderedResources = "pods=p1,p2;persistentvolumeclaims=pvc1,pvc2"
 	orders, err := ParseOrderedResources(o.OrderedResources)
+	o.CSISnapshotTimeout = 20 * time.Minute
 	assert.NoError(t, err)
 
 	backup, err := o.BuildBackup(testNamespace)
@@ -46,6 +48,7 @@ func TestCreateOptions_BuildBackup(t *testing.T) {
 		SnapshotVolumes:         o.SnapshotVolumes.Value,
 		IncludeClusterResources: o.IncludeClusterResources.Value,
 		OrderedResources:        orders,
+		CSISnapshotTimeout:      metav1.Duration{Duration: o.CSISnapshotTimeout},
 	}, backup.Spec)
 
 	assert.Equal(t, map[string]string{

pkg/cmd/cli/schedule/create.go

@@ -145,6 +145,7 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
 				VolumeSnapshotLocations: o.BackupOptions.SnapshotLocations,
 				DefaultVolumesToRestic:  o.BackupOptions.DefaultVolumesToRestic.Value,
 				OrderedResources:        orders,
+				CSISnapshotTimeout:      metav1.Duration{Duration: o.BackupOptions.CSISnapshotTimeout},
 			},
 			Schedule:                   o.Schedule,
 			UseOwnerReferencesInBackup: &o.UseOwnerReferencesInBackup,

pkg/cmd/server/server.go

@@ -102,6 +102,8 @@ const (
 	// the default TTL for a backup
 	defaultBackupTTL = 30 * 24 * time.Hour
 
+	defaultCSISnapshotTimeout = 10 * time.Minute
+
 	// defaultCredentialsDirectory is the path on disk where credential
 	// files will be written to
 	defaultCredentialsDirectory = "/tmp/credentials"
@@ -111,7 +113,7 @@ type serverConfig struct {
 	// TODO(2.0) Deprecate defaultBackupLocation
 	pluginDir, metricsAddress, defaultBackupLocation                        string
 	backupSyncPeriod, podVolumeOperationTimeout, resourceTerminatingTimeout time.Duration
-	defaultBackupTTL, storeValidationFrequency                              time.Duration
+	defaultBackupTTL, storeValidationFrequency, defaultCSISnapshotTimeout   time.Duration
 	restoreResourcePriorities                                               []string
 	defaultVolumeSnapshotLocations                                          map[string]string
 	restoreOnly                                                             bool
@@ -142,6 +144,7 @@ func NewCommand(f client.Factory) *cobra.Command {
 			defaultVolumeSnapshotLocations:    make(map[string]string),
 			backupSyncPeriod:                  defaultBackupSyncPeriod,
 			defaultBackupTTL:                  defaultBackupTTL,
+			defaultCSISnapshotTimeout:         defaultCSISnapshotTimeout,
 			storeValidationFrequency:          defaultStoreValidationFrequency,
 			podVolumeOperationTimeout:         defaultPodVolumeOperationTimeout,
 			restoreResourcePriorities:         defaultRestorePriorities,
@@ -650,6 +653,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string
 			s.config.defaultBackupLocation,
 			s.config.defaultVolumesToRestic,
 			s.config.defaultBackupTTL,
+			s.config.defaultCSISnapshotTimeout,
 			s.sharedInformerFactory.Velero().V1().VolumeSnapshotLocations().Lister(),
 			defaultVolumeSnapshotLocations,
 			s.metrics,

pkg/controller/backup_controller.go

@@ -87,6 +87,7 @@ type backupController struct {
 	defaultBackupLocation       string
 	defaultVolumesToRestic      bool
 	defaultBackupTTL            time.Duration
+	defaultCSISnapshotTimeout   time.Duration
 	snapshotLocationLister      velerov1listers.VolumeSnapshotLocationLister
 	defaultSnapshotLocations    map[string]string
 	metrics                     *metrics.ServerMetrics
@@ -111,6 +112,7 @@ func NewBackupController(
 	defaultBackupLocation string,
 	defaultVolumesToRestic bool,
 	defaultBackupTTL time.Duration,
+	defaultCSISnapshotTimeout time.Duration,
 	volumeSnapshotLocationLister velerov1listers.VolumeSnapshotLocationLister,
 	defaultSnapshotLocations map[string]string,
 	metrics *metrics.ServerMetrics,
@@ -135,6 +137,7 @@ func NewBackupController(
 		defaultBackupLocation:       defaultBackupLocation,
 		defaultVolumesToRestic:      defaultVolumesToRestic,
 		defaultBackupTTL:            defaultBackupTTL,
+		defaultCSISnapshotTimeout:   defaultCSISnapshotTimeout,
 		snapshotLocationLister:      volumeSnapshotLocationLister,
 		defaultSnapshotLocations:    defaultSnapshotLocations,
 		metrics:                     metrics,
@@ -359,6 +362,11 @@ func (c *backupController) prepareBackupRequest(backup *velerov1api.Backup) *pkg
 		request.Spec.TTL.Duration = c.defaultBackupTTL
 	}
 
+	if request.Spec.CSISnapshotTimeout.Duration == 0 {
+		// set default CSI VolumeSnapshot timeout
+		request.Spec.CSISnapshotTimeout.Duration = c.defaultCSISnapshotTimeout
+	}
+
 	// calculate expiration
 	request.Status.Expiration = &metav1.Time{Time: c.clock.Now().Add(request.Spec.TTL.Duration)}
 
@@ -638,7 +646,7 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error {
 				backupLog.Error(err)
 			}
 
-			err = c.checkVolumeSnapshotReadyToUse(context.Background(), volumeSnapshots)
+			err = c.checkVolumeSnapshotReadyToUse(context.Background(), volumeSnapshots, backup.Spec.CSISnapshotTimeout.Duration)
 			if err != nil {
 				backupLog.Errorf("fail to wait VolumeSnapshot change to Ready: %s", err.Error())
 			}
@@ -879,9 +887,10 @@ func encodeToJSONGzip(data interface{}, desc string) (*bytes.Buffer, []error) {
 // using goroutine here instead of waiting in CSI plugin, because it's not easy to make BackupItemAction
 // parallel by now. After BackupItemAction parallel is implemented, this logic should be moved to CSI plugin
 // as https://github.com/vmware-tanzu/velero-plugin-for-csi/pull/100
-func (c *backupController) checkVolumeSnapshotReadyToUse(ctx context.Context, volumesnapshots []*snapshotv1api.VolumeSnapshot) error {
+func (c *backupController) checkVolumeSnapshotReadyToUse(ctx context.Context, volumesnapshots []*snapshotv1api.VolumeSnapshot,
+	csiSnapshotTimeout time.Duration) error {
 	eg, _ := errgroup.WithContext(ctx)
-	timeout := 10 * time.Minute
+	timeout := csiSnapshotTimeout
 	interval := 5 * time.Second
 
 	for _, vs := range volumesnapshots {

pkg/controller/pod_volume_backup_controller.go

@@ -208,19 +208,6 @@ func (r *PodVolumeBackupReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Complete(r)
 }
 
-func (r *PodVolumeBackupReconciler) singlePathMatch(path string) (string, error) {
-	matches, err := r.FileSystem.Glob(path)
-	if err != nil {
-		return "", errors.WithStack(err)
-	}
-
-	if len(matches) != 1 {
-		return "", errors.Errorf("expected one matching path: %s, got %d", path, len(matches))
-	}
-
-	return matches[0], nil
-}
-
 // getParentSnapshot finds the most recent completed PodVolumeBackup for the
 // specified PVC and returns its Restic snapshot ID. Any errors encountered are
 // logged but not returned since they do not prevent a backup from proceeding.
@@ -317,7 +304,7 @@ func (r *PodVolumeBackupReconciler) buildResticCommand(ctx context.Context, log
 	pathGlob := fmt.Sprintf("/host_pods/%s/volumes/*/%s", string(pvb.Spec.Pod.UID), volDir)
 	log.WithField("pathGlob", pathGlob).Debug("Looking for path matching glob")
 
-	path, err := r.singlePathMatch(pathGlob)
+	path, err := kube.SinglePathMatch(pathGlob, r.FileSystem, log)
 	if err != nil {
 		return nil, errors.Wrap(err, "identifying unique volume path on host")
 	}

pkg/controller/pod_volume_restore_controller.go

@@ -215,19 +215,6 @@ func getResticInitContainerIndex(pod *corev1api.Pod) int {
 	return -1
 }
 
-func singlePathMatch(path string) (string, error) {
-	matches, err := filepath.Glob(path)
-	if err != nil {
-		return "", errors.WithStack(err)
-	}
-
-	if len(matches) != 1 {
-		return "", errors.Errorf("expected one matching path: %s, got %d", path, len(matches))
-	}
-
-	return matches[0], nil
-}
-
 func (c *PodVolumeRestoreReconciler) processRestore(ctx context.Context, req *velerov1api.PodVolumeRestore, pod *corev1api.Pod, log logrus.FieldLogger) error {
 	volumeDir, err := kube.GetVolumeDirectory(ctx, log, pod, req.Spec.Volume, c.Client)
 	if err != nil {
@@ -236,7 +223,9 @@ func (c *PodVolumeRestoreReconciler) processRestore(ctx context.Context, req *ve
 
 	// Get the full path of the new volume's directory as mounted in the daemonset pod, which
 	// will look like: /host_pods/<new-pod-uid>/volumes/<volume-plugin-name>/<volume-dir>
-	volumePath, err := singlePathMatch(fmt.Sprintf("/host_pods/%s/volumes/*/%s", string(req.Spec.Pod.UID), volumeDir))
+	volumePath, err := kube.SinglePathMatch(
+		fmt.Sprintf("/host_pods/%s/volumes/*/%s", string(req.Spec.Pod.UID), volumeDir),
+		c.fileSystem, log)
 	if err != nil {
 		return errors.Wrap(err, "error identifying path of volume")
 	}

pkg/util/kube/utils.go

@@ -34,15 +34,20 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/vmware-tanzu/velero/pkg/util/filesystem"
 )
 
 // These annotations are taken from the Kubernetes persistent volume/persistent volume claim controller.
 // They cannot be directly importing because they are part of the kubernetes/kubernetes package, and importing that package is unsupported.
 // Their values are well-known and slow changing. They're duplicated here as constants to provide compile-time checking.
 // Originals can be found in kubernetes/kubernetes/pkg/controller/volume/persistentvolume/util/util.go.
-const KubeAnnBindCompleted = "pv.kubernetes.io/bind-completed"
-const KubeAnnBoundByController = "pv.kubernetes.io/bound-by-controller"
-const KubeAnnDynamicallyProvisioned = "pv.kubernetes.io/provisioned-by"
+const (
+	KubeAnnBindCompleted          = "pv.kubernetes.io/bind-completed"
+	KubeAnnBoundByController      = "pv.kubernetes.io/bound-by-controller"
+	KubeAnnDynamicallyProvisioned = "pv.kubernetes.io/provisioned-by"
+	KubeAnnMigratedTo             = "pv.kubernetes.io/migrated-to"
+)
 
 // NamespaceAndName returns a string in the format <namespace>/<name>
 func NamespaceAndName(objMeta metav1.Object) string {
@@ -163,6 +168,9 @@ func GetVolumeDirectory(ctx context.Context, log logrus.FieldLogger, pod *corev1
 	return pvc.Spec.VolumeName, nil
 }
 
+// isProvisionedByCSI function checks whether this is a CSI PV by annotation.
+// Either "pv.kubernetes.io/provisioned-by" or "pv.kubernetes.io/migrated-to" indicates
+// PV is provisioned by CSI.
 func isProvisionedByCSI(log logrus.FieldLogger, pv *corev1api.PersistentVolume, kbClient client.Client) (bool, error) {
 	if pv.Spec.CSI != nil {
 		return true, nil
@@ -171,14 +179,15 @@ func isProvisionedByCSI(log logrus.FieldLogger, pv *corev1api.PersistentVolume,
 	// Refer to https://github.com/vmware-tanzu/velero/issues/4496 for more details
 	if pv.Annotations != nil {
 		driverName := pv.Annotations[KubeAnnDynamicallyProvisioned]
-		if len(driverName) > 0 {
+		migratedDriver := pv.Annotations[KubeAnnMigratedTo]
+		if len(driverName) > 0 || len(migratedDriver) > 0 {
 			list := &storagev1api.CSIDriverList{}
 			if err := kbClient.List(context.TODO(), list); err != nil {
 				return false, err
 			}
 			for _, driver := range list.Items {
-				if driverName == driver.Name {
-					log.Debugf("the annotation %s=%s indicates the volume is provisioned by a CSI driver", KubeAnnDynamicallyProvisioned, driverName)
+				if driverName == driver.Name || migratedDriver == driver.Name {
+					log.Debugf("the annotation %s or %s equals to %s indicates the volume is provisioned by a CSI driver", KubeAnnDynamicallyProvisioned, KubeAnnMigratedTo, driverName)
 					return true, nil
 				}
 			}
@@ -187,6 +196,21 @@ func isProvisionedByCSI(log logrus.FieldLogger, pv *corev1api.PersistentVolume,
 	return false, nil
 }
 
+// SinglePathMatch function will be called by PVB and PVR controller to check whether pass-in volume path is valid.
+// Check whether there is only one match by the path's pattern (/host_pods/%s/volumes/*/volume_name/[mount|]).
+func SinglePathMatch(path string, fs filesystem.Interface, log logrus.FieldLogger) (string, error) {
+	matches, err := fs.Glob(path)
+	if err != nil {
+		return "", errors.WithStack(err)
+	}
+	if len(matches) != 1 {
+		return "", errors.Errorf("expected one matching path: %s, got %d", path, len(matches))
+	}
+
+	log.Debugf("This is a valid volume path: %s.", matches[0])
+	return matches[0], nil
+}
+
 // IsV1CRDReady checks a v1 CRD to see if it's ready, with both the Established and NamesAccepted conditions.
 func IsV1CRDReady(crd *apiextv1.CustomResourceDefinition) bool {
 	var isEstablished, namesAccepted bool

pkg/util/kube/utils_test.go

@@ -197,6 +197,13 @@ func TestGetVolumeDirectorySuccess(t *testing.T) {
 			pv:   builder.ForPersistentVolume("a-pv").ObjectMeta(builder.WithAnnotations(KubeAnnDynamicallyProvisioned, "csi.test.com")).Result(),
 			want: "a-pv/mount",
 		},
+		{
+			name: "Volume with CSI annotation 'pv.kubernetes.io/migrated-to' appends '/mount' to the volume name",
+			pod:  builder.ForPod("ns-1", "my-pod").Volumes(builder.ForVolume("my-vol").PersistentVolumeClaimSource("my-pvc").Result()).Result(),
+			pvc:  builder.ForPersistentVolumeClaim("ns-1", "my-pvc").VolumeName("a-pv").Result(),
+			pv:   builder.ForPersistentVolume("a-pv").ObjectMeta(builder.WithAnnotations(KubeAnnMigratedTo, "csi.test.com")).Result(),
+			want: "a-pv/mount",
+		},
 	}
 
 	csiDriver := storagev1api.CSIDriver{
@@ -425,3 +432,13 @@ func TestIsCRDReady(t *testing.T) {
 	_, err = IsCRDReady(obj)
 	assert.NotNil(t, err)
 }
+
+func TestSinglePathMatch(t *testing.T) {
+	fakeFS := velerotest.NewFakeFileSystem()
+	fakeFS.MkdirAll("testDir1/subpath", 0755)
+	fakeFS.MkdirAll("testDir2/subpath", 0755)
+
+	_, err := SinglePathMatch("./*/subpath", fakeFS, logrus.StandardLogger())
+	assert.NotNil(t, err)
+	assert.Contains(t, err.Error(), "expected one matching path")
+}

site/content/docs/main/api-types/backup.md

@@ -29,6 +29,10 @@ metadata:
   namespace: velero
 # Parameters about the backup. Required.
 spec:
+  # CSISnapshotTimeout specifies the time used to wait for
+  # CSI VolumeSnapshot status turns to ReadyToUse during creation, before
+  # returning error as timeout. The default value is 10 minute.
+  csiSnapshotTimeout: 10m
   # Array of namespaces to include in the backup. If unspecified, all namespaces are included.
   # Optional.
   includedNamespaces:

site/content/docs/main/api-types/schedule.md

@@ -34,6 +34,10 @@ spec:
   schedule: 0 7 * * *
   # Template is the spec that should be used for each backup triggered by this schedule.
   template:
+    # CSISnapshotTimeout specifies the time used to wait for
+    # CSI VolumeSnapshot status turns to ReadyToUse during creation, before
+    # returning error as timeout. The default value is 10 minute.
+    csiSnapshotTimeout: 10m
     # Array of namespaces to include in the scheduled backup. If unspecified, all namespaces are included.
     # Optional.
     includedNamespaces:

site/content/docs/main/examples.md

@@ -47,10 +47,10 @@ cd velero
     kubectl apply -f examples/nginx-app/with-pv.yaml
     ```
 
-1. Create a backup with PV snapshotting:
+1. Create a backup with PV snapshotting. `--csi-snapshot-timeout` is used to setup time to wait before CSI snapshot creation timeout. The default value is 10 minutes:
 
     ```bash
-    velero backup create nginx-backup --include-namespaces nginx-example
+    velero backup create nginx-backup --include-namespaces nginx-example --csi-snapshot-timeout=20m
     ```
 
 1. Simulate a disaster:

site/content/docs/v1.9/api-types/backup.md

@@ -29,6 +29,11 @@ metadata:
   namespace: velero
 # Parameters about the backup. Required.
 spec:
+  # Available since v1.9.1.
+  # CSISnapshotTimeout specifies the time used to wait for
+  # CSI VolumeSnapshot status turns to ReadyToUse during creation, before
+  # returning error as timeout. The default value is 10 minute.
+  csiSnapshotTimeout: 10m
   # Array of namespaces to include in the backup. If unspecified, all namespaces are included.
   # Optional.
   includedNamespaces:

site/content/docs/v1.9/api-types/schedule.md

@@ -34,6 +34,11 @@ spec:
   schedule: 0 7 * * *
   # Template is the spec that should be used for each backup triggered by this schedule.
   template:
+    # Available since v1.9.1.
+    # CSISnapshotTimeout specifies the time used to wait for
+    # CSI VolumeSnapshot status turns to ReadyToUse during creation, before
+    # returning error as timeout. The default value is 10 minute.
+    csiSnapshotTimeout: 10m
     # Array of namespaces to include in the scheduled backup. If unspecified, all namespaces are included.
     # Optional.
     includedNamespaces: