From 31e140919a62855ba6f9479cfce924e4d2e61d76 Mon Sep 17 00:00:00 2001 From: Xun Jiang Date: Fri, 29 Mar 2024 18:18:31 +0800 Subject: [PATCH] Merge CSI plugin code. Signed-off-by: Xun Jiang --- changelogs/unreleased/7609-blackpiglet | 1 + .../actions/csi/volumesnapshot_action.go | 120 ++ .../csi/volumesnapshotcontent_action.go | 126 ++ pkg/apis/velero/v1/labels_annotations.go | 40 + pkg/backup/actions/csi/pvc_action.go | 578 ++++++++++ pkg/backup/actions/csi/pvc_action_test.go | 367 ++++++ .../actions/csi/volumesnapshot_action.go | 379 ++++++ .../actions/csi/volumesnapshotclass_action.go | 124 ++ .../csi/volumesnapshotcontent_action.go | 141 +++ pkg/backup/backup_test.go | 22 +- pkg/backup/item_backupper.go | 25 +- pkg/client/factory.go | 3 + pkg/cmd/server/plugin/plugin.go | 209 +++- pkg/controller/backup_controller.go | 2 +- pkg/exposer/csi_snapshot.go | 19 +- pkg/exposer/csi_snapshot_test.go | 2 +- pkg/restore/actions/add_pv_from_pvc_action.go | 2 +- .../actions/add_pv_from_pvc_action_test.go | 2 +- .../actions/add_pvc_from_pod_action.go | 2 +- .../actions/add_pvc_from_pod_action_test.go | 2 +- .../actions/admissionwebhook_config_action.go | 2 +- .../admissionwebhook_config_action_test.go | 2 +- pkg/restore/actions/apiservice_action.go | 2 +- pkg/restore/actions/apiservice_action_test.go | 2 +- .../actions/change_image_name_action.go | 2 +- .../actions/change_image_name_action_test.go | 2 +- .../actions/change_pvc_node_selector.go | 2 +- .../actions/change_pvc_node_selector_test.go | 2 +- .../actions/change_storageclass_action.go | 2 +- .../change_storageclass_action_test.go | 2 +- .../actions/clusterrolebinding_action.go | 2 +- .../actions/clusterrolebinding_action_test.go | 2 +- .../crd_v1_preserve_unknown_fields_action.go | 2 +- ..._v1_preserve_unknown_fields_action_test.go | 2 +- pkg/restore/actions/csi/pvc_action.go | 616 ++++++++++ pkg/restore/actions/csi/pvc_action_test.go | 713 ++++++++++++ .../actions/csi/volumesnapshot_action.go | 217 ++++ .../actions/csi/volumesnapshot_action_test.go | 89 ++ .../actions/csi/volumesnapshotclass_action.go | 111 ++ .../csi/volumesnapshotcontent_action.go | 113 ++ .../actions/dataupload_retrieve_action.go | 2 +- .../dataupload_retrieve_action_test.go | 5 +- .../actions/init_restorehook_pod_action.go | 2 +- .../init_restorehook_pod_action_test.go | 2 +- pkg/restore/actions/job_action.go | 2 +- pkg/restore/actions/job_action_test.go | 2 +- pkg/restore/actions/pod_action.go | 2 +- pkg/restore/actions/pod_action_test.go | 2 +- .../actions/pod_volume_restore_action.go | 2 +- .../actions/pod_volume_restore_action_test.go | 2 +- pkg/restore/actions/rolebinding_action.go | 2 +- .../actions/rolebinding_action_test.go | 2 +- pkg/restore/actions/secret_action.go | 2 +- pkg/restore/actions/secret_action_test.go | 2 +- pkg/restore/actions/service_account_action.go | 2 +- .../actions/service_account_action_test.go | 2 +- pkg/restore/actions/service_action.go | 2 +- pkg/restore/actions/service_action_test.go | 2 +- pkg/test/fake_controller_runtime_client.go | 42 +- pkg/util/csi/volume_snapshot.go | 626 +++++++++- pkg/util/csi/volume_snapshot_test.go | 1020 ++++++++++++++++- pkg/util/kube/pvc_pv.go | 32 +- pkg/util/kube/pvc_pv_test.go | 137 +++ pkg/util/kube/utils.go | 30 + pkg/util/kube/utils_test.go | 184 +++ pkg/util/podvolume/pod_volume.go | 63 +- pkg/util/podvolume/pod_volume_test.go | 414 +++++++ pkg/util/scheme.go | 16 + pkg/util/util.go | 26 + pkg/util/util_test.go | 64 ++ test/e2e/resource-filtering/exclude_label.go | 7 +- 71 files changed, 6565 insertions(+), 188 deletions(-) create mode 100644 changelogs/unreleased/7609-blackpiglet create mode 100644 internal/delete/actions/csi/volumesnapshot_action.go create mode 100644 internal/delete/actions/csi/volumesnapshotcontent_action.go create mode 100644 pkg/backup/actions/csi/pvc_action.go create mode 100644 pkg/backup/actions/csi/pvc_action_test.go create mode 100644 pkg/backup/actions/csi/volumesnapshot_action.go create mode 100644 pkg/backup/actions/csi/volumesnapshotclass_action.go create mode 100644 pkg/backup/actions/csi/volumesnapshotcontent_action.go create mode 100644 pkg/restore/actions/csi/pvc_action.go create mode 100644 pkg/restore/actions/csi/pvc_action_test.go create mode 100644 pkg/restore/actions/csi/volumesnapshot_action.go create mode 100644 pkg/restore/actions/csi/volumesnapshot_action_test.go create mode 100644 pkg/restore/actions/csi/volumesnapshotclass_action.go create mode 100644 pkg/restore/actions/csi/volumesnapshotcontent_action.go create mode 100644 pkg/util/util.go create mode 100644 pkg/util/util_test.go diff --git a/changelogs/unreleased/7609-blackpiglet b/changelogs/unreleased/7609-blackpiglet new file mode 100644 index 000000000..48644ae05 --- /dev/null +++ b/changelogs/unreleased/7609-blackpiglet @@ -0,0 +1 @@ +Merge CSI plugin code into Velero. \ No newline at end of file diff --git a/internal/delete/actions/csi/volumesnapshot_action.go b/internal/delete/actions/csi/volumesnapshot_action.go new file mode 100644 index 000000000..bc6d9a2be --- /dev/null +++ b/internal/delete/actions/csi/volumesnapshot_action.go @@ -0,0 +1,120 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + "fmt" + + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/velero/pkg/client" + plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + "github.com/vmware-tanzu/velero/pkg/util/csi" + kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" +) + +// volumeSnapshotDeleteItemAction is a backup item action plugin for Velero. +type volumeSnapshotDeleteItemAction struct { + log logrus.FieldLogger + crClient crclient.Client +} + +// AppliesTo returns information indicating that the +// VolumeSnapshotBackupItemAction should be invoked to backup +// VolumeSnapshots. +func (p *volumeSnapshotDeleteItemAction) AppliesTo() (velero.ResourceSelector, error) { + p.log.Debug("VolumeSnapshotBackupItemAction AppliesTo") + + return velero.ResourceSelector{ + IncludedResources: []string{"volumesnapshots.snapshot.storage.k8s.io"}, + }, nil +} + +func (p *volumeSnapshotDeleteItemAction) Execute( + input *velero.DeleteItemActionExecuteInput, +) error { + p.log.Info("Starting VolumeSnapshotDeleteItemAction for volumeSnapshot") + + var vs snapshotv1api.VolumeSnapshot + + if err := runtime.DefaultUnstructuredConverter.FromUnstructured( + input.Item.UnstructuredContent(), + &vs, + ); err != nil { + return errors.Wrapf(err, "failed to convert input.Item from unstructured") + } + + // We don't want this DeleteItemAction plugin to delete VolumeSnapshot + // taken outside of Velero. So skip deleting VolumeSnapshot objects + // that were not created in the process of creating the Velero + // backup being deleted. + if !kubeutil.HasBackupLabel(&vs.ObjectMeta, input.Backup.Name) { + p.log.Info( + "VolumeSnapshot %s/%s was not taken by backup %s, skipping deletion", + vs.Namespace, vs.Name, input.Backup.Name, + ) + return nil + } + + p.log.Infof("Deleting VolumeSnapshot %s/%s", vs.Namespace, vs.Name) + if vs.Status != nil && vs.Status.BoundVolumeSnapshotContentName != nil { + // we patch the DeletionPolicy of the VolumeSnapshotContent + // to set it to Delete. This ensures that the volume snapshot + // in the storage provider is also deleted. + err := csi.SetVolumeSnapshotContentDeletionPolicy( + *vs.Status.BoundVolumeSnapshotContentName, + p.crClient, + ) + if err != nil && !apierrors.IsNotFound(err) { + return errors.Wrapf( + err, + fmt.Sprintf("failed to patch DeletionPolicy of volume snapshot %s/%s", + vs.Namespace, vs.Name), + ) + } + + if apierrors.IsNotFound(err) { + return nil + } + } + err := p.crClient.Delete(context.TODO(), &vs) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil +} + +func NewVolumeSnapshotDeleteItemAction(f client.Factory) plugincommon.HandlerInitializer { + return func(logger logrus.FieldLogger) (interface{}, error) { + crClient, err := f.KubebuilderClient() + if err != nil { + return nil, errors.WithStack(err) + } + + return &volumeSnapshotDeleteItemAction{ + log: logger, + crClient: crClient, + }, nil + } +} diff --git a/internal/delete/actions/csi/volumesnapshotcontent_action.go b/internal/delete/actions/csi/volumesnapshotcontent_action.go new file mode 100644 index 000000000..f3cfe41f7 --- /dev/null +++ b/internal/delete/actions/csi/volumesnapshotcontent_action.go @@ -0,0 +1,126 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + "fmt" + + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/velero/pkg/client" + plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + "github.com/vmware-tanzu/velero/pkg/util/csi" + kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" +) + +// volumeSnapshotContentDeleteItemAction is a restore item action plugin for Velero +type volumeSnapshotContentDeleteItemAction struct { + log logrus.FieldLogger + crClient crclient.Client +} + +// AppliesTo returns information indicating +// VolumeSnapshotContentRestoreItemAction action should be invoked +// while restoring VolumeSnapshotContent.snapshot.storage.k8s.io resources +func (p *volumeSnapshotContentDeleteItemAction) AppliesTo() (velero.ResourceSelector, error) { + return velero.ResourceSelector{ + IncludedResources: []string{"volumesnapshotcontent.snapshot.storage.k8s.io"}, + }, nil +} + +func (p *volumeSnapshotContentDeleteItemAction) Execute( + input *velero.DeleteItemActionExecuteInput, +) error { + p.log.Info("Starting VolumeSnapshotContentDeleteItemAction") + + var snapCont snapshotv1api.VolumeSnapshotContent + if err := runtime.DefaultUnstructuredConverter.FromUnstructured( + input.Item.UnstructuredContent(), + &snapCont, + ); err != nil { + return errors.Wrapf(err, "failed to convert VolumeSnapshotContent from unstructured") + } + + // We don't want this DeleteItemAction plugin to delete + // VolumeSnapshotContent taken outside of Velero. + // So skip deleting VolumeSnapshotContent not have the backup name + // in its labels. + if !kubeutil.HasBackupLabel(&snapCont.ObjectMeta, input.Backup.Name) { + p.log.Info( + "VolumeSnapshotContent %s was not taken by backup %s, skipping deletion", + snapCont.Name, + input.Backup.Name, + ) + return nil + } + + p.log.Infof("Deleting VolumeSnapshotContent %s", snapCont.Name) + + if err := csi.SetVolumeSnapshotContentDeletionPolicy( + snapCont.Name, + p.crClient, + ); err != nil { + // #4764: Leave a warning when VolumeSnapshotContent cannot be found for deletion. + // Manual deleting VolumeSnapshotContent can cause this. + // It's tricky for Velero to handle this inconsistency. + // Even if Velero restores the VolumeSnapshotContent, CSI snapshot controller + // may not delete it correctly due to the snapshot represented by VolumeSnapshotContent + // already deleted on cloud provider. + if apierrors.IsNotFound(err) { + p.log.Warnf( + "VolumeSnapshotContent %s of backup %s cannot be found. May leave orphan snapshot %s on cloud provider.", + snapCont.Name, input.Backup.Name, *snapCont.Status.SnapshotHandle) + return nil + } + return errors.Wrapf(err, fmt.Sprintf( + "failed to set DeletionPolicy on volumesnapshotcontent %s. Skipping deletion", + snapCont.Name)) + } + + if err := p.crClient.Delete( + context.TODO(), + &snapCont, + ); err != nil && !apierrors.IsNotFound(err) { + p.log.Infof("VolumeSnapshotContent %s not found", snapCont.Name) + return err + } + + return nil +} + +func NewVolumeSnapshotContentDeleteItemAction( + f client.Factory, +) plugincommon.HandlerInitializer { + return func(logger logrus.FieldLogger) (interface{}, error) { + crClient, err := f.KubebuilderClient() + if err != nil { + return nil, err + } + + return &volumeSnapshotContentDeleteItemAction{ + log: logger, + crClient: crClient, + }, nil + } +} diff --git a/pkg/apis/velero/v1/labels_annotations.go b/pkg/apis/velero/v1/labels_annotations.go index 08b6b2440..6a2b59d49 100644 --- a/pkg/apis/velero/v1/labels_annotations.go +++ b/pkg/apis/velero/v1/labels_annotations.go @@ -97,6 +97,10 @@ const ( // VolumesToExcludeAnnotation is the annotation on a pod whose mounted volumes // should be excluded from pod volume backup. VolumesToExcludeAnnotation = "backup.velero.io/backup-volumes-excludes" + + // ExcludeFromBackupLabel is the label to exclude k8s resource from backup, + // even if the resource contains a matching selector label. + ExcludeFromBackupLabel = "velero.io/exclude-from-backup" ) type AsyncOperationIDPrefix string @@ -111,3 +115,39 @@ type VeleroResourceUsage string const ( VeleroResourceUsageDataUploadResult VeleroResourceUsage = "DataUpload" ) + +// CSI related plugin actions' constant variable +const ( + VolumeSnapshotLabel = "velero.io/volume-snapshot-name" + VolumeSnapshotHandleAnnotation = "velero.io/csi-volumesnapshot-handle" + VolumeSnapshotRestoreSize = "velero.io/csi-volumesnapshot-restore-size" + DriverNameAnnotation = "velero.io/csi-driver-name" + DeleteSecretNameAnnotation = "velero.io/csi-deletesnapshotsecret-name" // #nosec G101 + DeleteSecretNamespaceAnnotation = "velero.io/csi-deletesnapshotsecret-namespace" // #nosec G101 + VSCDeletionPolicyAnnotation = "velero.io/csi-vsc-deletion-policy" + VolumeSnapshotClassSelectorLabel = "velero.io/csi-volumesnapshot-class" + VolumeSnapshotClassDriverBackupAnnotationPrefix = "velero.io/csi-volumesnapshot-class" + VolumeSnapshotClassDriverPVCAnnotation = "velero.io/csi-volumesnapshot-class" + + // There is no release w/ these constants exported. Using the strings for now. + // CSI Annotation volumesnapshotclass + // https://github.com/kubernetes-csi/external-snapshotter/blob/master/pkg/utils/util.go#L59-L60 + PrefixedListSecretNameAnnotation = "csi.storage.k8s.io/snapshotter-list-secret-name" // #nosec G101 + PrefixedListSecretNamespaceAnnotation = "csi.storage.k8s.io/snapshotter-list-secret-namespace" // #nosec G101 + + // CSI Annotation volumesnapshotcontents + PrefixedSecretNameAnnotation = "csi.storage.k8s.io/snapshotter-secret-name" // #nosec G101 + PrefixedSecretNamespaceAnnotation = "csi.storage.k8s.io/snapshotter-secret-namespace" // #nosec G101 + + // Velero checks this annotation to determine whether to skip resource excluding check. + MustIncludeAdditionalItemAnnotation = "backup.velero.io/must-include-additional-items" + // SkippedNoCSIPVAnnotation - Velero checks this annotation on processed PVC to + // find out if the snapshot was skipped b/c the PV is not provisioned via CSI + SkippedNoCSIPVAnnotation = "backup.velero.io/skipped-no-csi-pv" + + // DynamicPVRestoreLabel is the label key for dynamic PV restore + DynamicPVRestoreLabel = "velero.io/dynamic-pv-restore" + + // DataUploadNameAnnotation is the label key for the DataUpload name + DataUploadNameAnnotation = "velero.io/data-upload-name" +) diff --git a/pkg/backup/actions/csi/pvc_action.go b/pkg/backup/actions/csi/pvc_action.go new file mode 100644 index 000000000..1417e9120 --- /dev/null +++ b/pkg/backup/actions/csi/pvc_action.go @@ -0,0 +1,578 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + "fmt" + + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + corev1api "k8s.io/api/core/v1" + storagev1api "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + "github.com/vmware-tanzu/velero/pkg/client" + "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/label" + plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + biav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" + uploaderUtil "github.com/vmware-tanzu/velero/pkg/uploader/util" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/csi" + kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" + "github.com/vmware-tanzu/velero/pkg/util/podvolume" +) + +// pvcBackupItemAction is a backup item action plugin for Velero. +type pvcBackupItemAction struct { + log logrus.FieldLogger + crClient crclient.Client +} + +// AppliesTo returns information indicating that the PVCBackupItemAction +// should be invoked to backup PVCs. +func (p *pvcBackupItemAction) AppliesTo() (velero.ResourceSelector, error) { + return velero.ResourceSelector{ + IncludedResources: []string{"persistentvolumeclaims"}, + }, nil +} + +func (p *pvcBackupItemAction) validateBackup(backup velerov1api.Backup) (valid bool) { + // Do nothing if volume snapshots have not been requested in this backup + if boolptr.IsSetToFalse(backup.Spec.SnapshotVolumes) { + p.log.Infof( + "Volume snapshotting not requested for backup %s/%s", + backup.Namespace, backup.Name) + return false + } + + if backup.Status.Phase == velerov1api.BackupPhaseFinalizing || + backup.Status.Phase == velerov1api.BackupPhaseFinalizingPartiallyFailed { + p.log.WithFields( + logrus.Fields{ + "Backup": fmt.Sprintf("%s/%s", backup.Namespace, backup.Name), + "Phase": backup.Status.Phase, + }, + ).Debug("Backup is in finalizing phase. Skip this PVC.") + return false + } + + return true +} + +func (p *pvcBackupItemAction) validatePVCandPV( + pvc corev1api.PersistentVolumeClaim, + defaultVolumesToFsBackup *bool, + item runtime.Unstructured, +) ( + valid bool, + updateItem runtime.Unstructured, + err error, +) { + updateItem = item + + // no storage class: we don't know how to map to a VolumeSnapshotClass + if pvc.Spec.StorageClassName == nil { + return false, + updateItem, + errors.Errorf( + "Cannot snapshot PVC %s/%s, PVC has no storage class.", + pvc.Namespace, pvc.Name) + } + + p.log.Debugf( + "Fetching underlying PV for PVC %s", + fmt.Sprintf("%s/%s", pvc.Namespace, pvc.Name), + ) + + // Do nothing if this is not a CSI provisioned volume + pv, err := kubeutil.GetPVForPVC(&pvc, p.crClient) + if err != nil { + return false, updateItem, errors.WithStack(err) + } + + if pv.Spec.PersistentVolumeSource.CSI == nil { + p.log.Infof( + "Skipping PVC %s/%s, associated PV %s is not a CSI volume", + pvc.Namespace, pvc.Name, pv.Name) + + kubeutil.AddAnnotations( + &pvc.ObjectMeta, + map[string]string{ + velerov1api.SkippedNoCSIPVAnnotation: "true", + }) + data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pvc) + updateItem = &unstructured.Unstructured{Object: data} + return false, updateItem, err + } + + // Do nothing if FS uploader is used to backup this PV + isFSUploaderUsed, err := podvolume.IsPVCDefaultToFSBackup( + pvc.Namespace, + pvc.Name, + p.crClient, + boolptr.IsSetToTrue(defaultVolumesToFsBackup), + ) + if err != nil { + return false, updateItem, errors.WithStack(err) + } + if isFSUploaderUsed { + p.log.Infof( + "Skipping PVC %s/%s, PV %s will be backed up using FS uploader", + pvc.Namespace, pvc.Name, pv.Name) + return false, updateItem, nil + } + + return true, updateItem, nil +} + +func (p *pvcBackupItemAction) createVolumeSnapshot( + pvc corev1api.PersistentVolumeClaim, + backup *velerov1api.Backup, +) ( + vs *snapshotv1api.VolumeSnapshot, + err error, +) { + p.log.Debugf("Fetching storage class for PV %s", *pvc.Spec.StorageClassName) + storageClass := new(storagev1api.StorageClass) + if err := p.crClient.Get( + context.TODO(), crclient.ObjectKey{Name: *pvc.Spec.StorageClassName}, + storageClass, + ); err != nil { + return nil, errors.Wrap(err, "error getting storage class") + } + + p.log.Debugf("Fetching VolumeSnapshotClass for %s", storageClass.Provisioner) + vsClass, err := csi.GetVolumeSnapshotClass( + storageClass.Provisioner, + backup, + &pvc, + p.log, + p.crClient, + ) + if err != nil { + return nil, errors.Wrapf( + err, "failed to get VolumeSnapshotClass for StorageClass %s", + storageClass.Name, + ) + } + p.log.Infof("VolumeSnapshotClass=%s", vsClass.Name) + + vsLabels := map[string]string{} + for k, v := range pvc.ObjectMeta.Labels { + vsLabels[k] = v + } + vsLabels[velerov1api.BackupNameLabel] = label.GetValidName(backup.Name) + + // Craft the vs object to be created + vs = &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "velero-" + pvc.Name + "-", + Namespace: pvc.Namespace, + Labels: vsLabels, + }, + Spec: snapshotv1api.VolumeSnapshotSpec{ + Source: snapshotv1api.VolumeSnapshotSource{ + PersistentVolumeClaimName: &pvc.Name, + }, + VolumeSnapshotClassName: &vsClass.Name, + }, + } + + if err := p.crClient.Create(context.TODO(), vs); err != nil { + return nil, errors.Wrapf( + err, "error creating volume snapshot", + ) + } + p.log.Infof( + "Created VolumeSnapshot %s", + fmt.Sprintf("%s/%s", vs.Namespace, vs.Name), + ) + + return vs, nil +} + +// Execute recognizes PVCs backed by volumes provisioned by CSI drivers +// with VolumeSnapshotting capability and creates snapshots of the +// underlying PVs by creating VolumeSnapshot CSI API objects that will +// trigger the CSI driver to perform the snapshot operation on the volume. +func (p *pvcBackupItemAction) Execute( + item runtime.Unstructured, + backup *velerov1api.Backup, +) ( + runtime.Unstructured, + []velero.ResourceIdentifier, + string, + []velero.ResourceIdentifier, + error, +) { + p.log.Info("Starting PVCBackupItemAction") + + if valid := p.validateBackup(*backup); !valid { + return item, nil, "", nil, nil + } + + var pvc corev1api.PersistentVolumeClaim + if err := runtime.DefaultUnstructuredConverter.FromUnstructured( + item.UnstructuredContent(), + &pvc, + ); err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + if valid, item, err := p.validatePVCandPV( + pvc, + backup.Spec.DefaultVolumesToFsBackup, + item, + ); !valid { + if err != nil { + return nil, nil, "", nil, err + } + return item, nil, "", nil, nil + } + + vs, err := p.createVolumeSnapshot(pvc, backup) + if err != nil { + return nil, nil, "", nil, err + } + + labels := map[string]string{ + velerov1api.VolumeSnapshotLabel: vs.Name, + velerov1api.BackupNameLabel: backup.Name, + } + + annotations := map[string]string{ + velerov1api.VolumeSnapshotLabel: vs.Name, + velerov1api.MustIncludeAdditionalItemAnnotation: "true", + } + + var additionalItems []velero.ResourceIdentifier + operationID := "" + var itemToUpdate []velero.ResourceIdentifier + + if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) { + operationID = label.GetValidName( + string( + velerov1api.AsyncOperationIDPrefixDataUpload, + ) + string(backup.UID) + "." + string(pvc.UID), + ) + dataUploadLog := p.log.WithFields(logrus.Fields{ + "Source PVC": fmt.Sprintf("%s/%s", pvc.Namespace, pvc.Name), + "VolumeSnapshot": fmt.Sprintf("%s/%s", vs.Namespace, vs.Name), + "Operation ID": operationID, + "Backup": backup.Name, + }) + + // Wait until VS associated VSC snapshot handle created before + // returning with the Async operation for data mover. + _, err := csi.WaitUntilVSCHandleIsReady( + vs, + p.crClient, + p.log, + true, + backup.Spec.CSISnapshotTimeout.Duration, + ) + if err != nil { + dataUploadLog.Errorf( + "Fail to wait VolumeSnapshot turned to ReadyToUse: %s", + err.Error(), + ) + csi.CleanupVolumeSnapshot(vs, p.crClient, p.log) + return nil, nil, "", nil, errors.WithStack(err) + } + + dataUploadLog.Info("Starting data upload of backup") + + dataUpload, err := createDataUpload( + context.Background(), + backup, + p.crClient, + vs, + &pvc, + operationID, + ) + if err != nil { + dataUploadLog.WithError(err).Error("failed to submit DataUpload") + + // TODO: need to use DeleteVolumeSnapshotIfAny, after data mover + // adopting the controller-runtime client. + if deleteErr := p.crClient.Delete(context.TODO(), vs); deleteErr != nil { + if !apierrors.IsNotFound(deleteErr) { + dataUploadLog.WithError(deleteErr).Error("fail to delete VolumeSnapshot") + } + } + + // Return without modification to not fail the backup, + // and the above error log makes the backup partially fail. + return item, nil, "", nil, nil + } else { + itemToUpdate = []velero.ResourceIdentifier{ + { + GroupResource: schema.GroupResource{ + Group: "velero.io", + Resource: "datauploads", + }, + Namespace: dataUpload.Namespace, + Name: dataUpload.Name, + }, + } + // Set the DataUploadNameLabel, which is used for restore to + // let CSI plugin check whether it should handle the volume. + // If volume is CSI migration, PVC doesn't have the annotation. + annotations[velerov1api.DataUploadNameAnnotation] = dataUpload.Namespace + "/" + dataUpload.Name + + dataUploadLog.Info("DataUpload is submitted successfully.") + } + } else { + additionalItems = []velero.ResourceIdentifier{ + { + GroupResource: kuberesource.VolumeSnapshots, + Namespace: vs.Namespace, + Name: vs.Name, + }, + } + } + + kubeutil.AddAnnotations(&pvc.ObjectMeta, annotations) + kubeutil.AddLabels(&pvc.ObjectMeta, labels) + + p.log.Infof("Returning from PVCBackupItemAction with %d additionalItems to backup", + len(additionalItems)) + for _, ai := range additionalItems { + p.log.Debugf("%s: %s", ai.GroupResource.String(), ai.Name) + } + + pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pvc) + if err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + return &unstructured.Unstructured{Object: pvcMap}, + additionalItems, operationID, itemToUpdate, nil +} + +func (p *pvcBackupItemAction) Name() string { + return "PVCBackupItemAction" +} + +func (p *pvcBackupItemAction) Progress( + operationID string, + backup *velerov1api.Backup, +) (velero.OperationProgress, error) { + progress := velero.OperationProgress{} + if operationID == "" { + return progress, biav2.InvalidOperationIDError(operationID) + } + + dataUpload, err := getDataUpload(context.Background(), p.crClient, operationID) + if err != nil { + p.log.Errorf( + "fail to get DataUpload for backup %s/%s by operation ID %s: %s", + backup.Namespace, backup.Name, operationID, err.Error(), + ) + return progress, err + } + if dataUpload.Status.Phase == velerov2alpha1.DataUploadPhaseNew || + dataUpload.Status.Phase == "" { + p.log.Debugf("DataUpload is still not processed yet. Skip progress update.") + return progress, nil + } + + progress.Description = string(dataUpload.Status.Phase) + progress.OperationUnits = "Bytes" + progress.NCompleted = dataUpload.Status.Progress.BytesDone + progress.NTotal = dataUpload.Status.Progress.TotalBytes + + if dataUpload.Status.StartTimestamp != nil { + progress.Started = dataUpload.Status.StartTimestamp.Time + } + + if dataUpload.Status.CompletionTimestamp != nil { + progress.Updated = dataUpload.Status.CompletionTimestamp.Time + } + + if dataUpload.Status.Phase == velerov2alpha1.DataUploadPhaseCompleted { + progress.Completed = true + } else if dataUpload.Status.Phase == velerov2alpha1.DataUploadPhaseFailed { + progress.Completed = true + progress.Err = dataUpload.Status.Message + } else if dataUpload.Status.Phase == velerov2alpha1.DataUploadPhaseCanceled { + progress.Completed = true + progress.Err = "DataUpload is canceled" + } + + return progress, nil +} + +func (p *pvcBackupItemAction) Cancel(operationID string, backup *velerov1api.Backup) error { + if operationID == "" { + return biav2.InvalidOperationIDError(operationID) + } + + dataUpload, err := getDataUpload(context.Background(), p.crClient, operationID) + if err != nil { + p.log.Errorf( + "fail to get DataUpload for backup %s/%s: %s", + backup.Namespace, backup.Name, err.Error(), + ) + return err + } + + return cancelDataUpload(context.Background(), p.crClient, dataUpload) +} + +func newDataUpload( + backup *velerov1api.Backup, + vs *snapshotv1api.VolumeSnapshot, + pvc *corev1api.PersistentVolumeClaim, + operationID string, +) *velerov2alpha1.DataUpload { + dataUpload := &velerov2alpha1.DataUpload{ + TypeMeta: metav1.TypeMeta{ + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + Kind: "DataUpload", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: backup.Namespace, + GenerateName: backup.Name + "-", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: velerov1api.SchemeGroupVersion.String(), + Kind: "Backup", + Name: backup.Name, + UID: backup.UID, + Controller: boolptr.True(), + }, + }, + Labels: map[string]string{ + velerov1api.BackupNameLabel: label.GetValidName(backup.Name), + velerov1api.BackupUIDLabel: string(backup.UID), + velerov1api.PVCUIDLabel: string(pvc.UID), + velerov1api.AsyncOperationIDLabel: operationID, + }, + }, + Spec: velerov2alpha1.DataUploadSpec{ + SnapshotType: velerov2alpha1.SnapshotTypeCSI, + CSISnapshot: &velerov2alpha1.CSISnapshotSpec{ + VolumeSnapshot: vs.Name, + StorageClass: *pvc.Spec.StorageClassName, + SnapshotClass: *vs.Spec.VolumeSnapshotClassName, + }, + SourcePVC: pvc.Name, + DataMover: backup.Spec.DataMover, + BackupStorageLocation: backup.Spec.StorageLocation, + SourceNamespace: pvc.Namespace, + OperationTimeout: backup.Spec.CSISnapshotTimeout, + }, + } + + if backup.Spec.UploaderConfig != nil && + backup.Spec.UploaderConfig.ParallelFilesUpload > 0 { + dataUpload.Spec.DataMoverConfig = make(map[string]string) + dataUpload.Spec.DataMoverConfig[uploaderUtil.ParallelFilesUpload] = fmt.Sprintf( + "%d", backup.Spec.UploaderConfig.ParallelFilesUpload, + ) + } + + return dataUpload +} + +func createDataUpload( + ctx context.Context, + backup *velerov1api.Backup, + crClient crclient.Client, + vs *snapshotv1api.VolumeSnapshot, + pvc *corev1api.PersistentVolumeClaim, + operationID string, +) (*velerov2alpha1.DataUpload, error) { + dataUpload := newDataUpload(backup, vs, pvc, operationID) + + err := crClient.Create(ctx, dataUpload) + if err != nil { + return nil, errors.Wrap(err, "fail to create DataUpload CR") + } + + return dataUpload, err +} + +func getDataUpload( + ctx context.Context, + crClient crclient.Client, + operationID string, +) (*velerov2alpha1.DataUpload, error) { + dataUploadList := new(velerov2alpha1.DataUploadList) + err := crClient.List(ctx, dataUploadList, &crclient.ListOptions{ + LabelSelector: labels.SelectorFromSet( + map[string]string{velerov1api.AsyncOperationIDLabel: operationID}, + ), + }) + if err != nil { + return nil, errors.Wrapf(err, "error to list DataUpload") + } + + if len(dataUploadList.Items) == 0 { + return nil, errors.Errorf("not found DataUpload for operationID %s", operationID) + } + + if len(dataUploadList.Items) > 1 { + return nil, errors.Errorf("more than one DataUpload found operationID %s", operationID) + } + + return &dataUploadList.Items[0], nil +} + +func cancelDataUpload( + ctx context.Context, + crClient crclient.Client, + dataUpload *velerov2alpha1.DataUpload, +) error { + updatedDataUpload := dataUpload.DeepCopy() + updatedDataUpload.Spec.Cancel = true + + err := crClient.Patch(ctx, updatedDataUpload, crclient.MergeFrom(dataUpload)) + if err != nil { + return errors.Wrap(err, "error patch DataUpload") + } + + return nil +} + +func NewPvcBackupItemAction(f client.Factory) plugincommon.HandlerInitializer { + return func(logger logrus.FieldLogger) (interface{}, error) { + crClient, err := f.KubebuilderClient() + if err != nil { + return nil, errors.WithStack(err) + } + + return &pvcBackupItemAction{ + log: logger, + crClient: crClient, + }, nil + } +} diff --git a/pkg/backup/actions/csi/pvc_action_test.go b/pkg/backup/actions/csi/pvc_action_test.go new file mode 100644 index 000000000..4d8a0e910 --- /dev/null +++ b/pkg/backup/actions/csi/pvc_action_test.go @@ -0,0 +1,367 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + v1 "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + "github.com/vmware-tanzu/velero/pkg/builder" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + velerotest "github.com/vmware-tanzu/velero/pkg/test" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" +) + +func TestExecute(t *testing.T) { + boolTrue := true + tests := []struct { + name string + backup *velerov1api.Backup + pvc *corev1.PersistentVolumeClaim + pv *corev1.PersistentVolume + sc *storagev1.StorageClass + vsClass *snapshotv1api.VolumeSnapshotClass + operationID string + expectedErr error + expectedBackup *velerov1api.Backup + expectedDataUpload *velerov2alpha1.DataUpload + expectedPVC *corev1.PersistentVolumeClaim + }{ + { + name: "Skip PVC handling if SnapshotVolume set to false", + backup: builder.ForBackup("velero", "test").SnapshotVolumes(false).Result(), + expectedErr: nil, + }, + { + name: "Skip PVC BIA when backup is in finalizing phase", + backup: builder.ForBackup("velero", "test").Phase(velerov1api.BackupPhaseFinalizing).Result(), + expectedErr: nil, + }, + { + name: "Test SnapshotMoveData", + backup: builder.ForBackup("velero", "test").SnapshotMoveData(true).CSISnapshotTimeout(1 * time.Minute).Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").StorageClass("testSC").Phase(corev1.ClaimBound).Result(), + pv: builder.ForPersistentVolume("testPV").CSI("hostpath", "testVolume").Result(), + sc: builder.ForStorageClass("testSC").Provisioner("hostpath").Result(), + vsClass: builder.ForVolumeSnapshotClass("testVSClass").Driver("hostpath").ObjectMeta(builder.WithLabels(velerov1api.VolumeSnapshotClassSelectorLabel, "")).Result(), + operationID: ".", + expectedErr: nil, + expectedDataUpload: &velerov2alpha1.DataUpload{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataUpload", + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-", + Namespace: "velero", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test", + velerov1api.BackupUIDLabel: "", + velerov1api.PVCUIDLabel: "", + velerov1api.AsyncOperationIDLabel: "du-.", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "velero.io/v1", + Kind: "Backup", + Name: "test", + UID: "", + Controller: &boolTrue, + }, + }, + }, + Spec: velerov2alpha1.DataUploadSpec{ + SnapshotType: velerov2alpha1.SnapshotTypeCSI, + CSISnapshot: &velerov2alpha1.CSISnapshotSpec{ + VolumeSnapshot: "", + StorageClass: "testSC", + SnapshotClass: "testVSClass", + }, + SourcePVC: "testPVC", + SourceNamespace: "velero", + OperationTimeout: metav1.Duration{Duration: 1 * time.Minute}, + }, + }, + }, + { + name: "Verify PVC is modified as expected", + backup: builder.ForBackup("velero", "test").SnapshotMoveData(true).CSISnapshotTimeout(1 * time.Minute).Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").StorageClass("testSC").Phase(corev1.ClaimBound).Result(), + pv: builder.ForPersistentVolume("testPV").CSI("hostpath", "testVolume").Result(), + sc: builder.ForStorageClass("testSC").Provisioner("hostpath").Result(), + vsClass: builder.ForVolumeSnapshotClass("tescVSClass").Driver("hostpath").ObjectMeta(builder.WithLabels(velerov1api.VolumeSnapshotClassSelectorLabel, "")).Result(), + operationID: ".", + expectedErr: nil, + expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC"). + ObjectMeta(builder.WithAnnotations(velerov1api.MustIncludeAdditionalItemAnnotation, "true", velerov1api.DataUploadNameAnnotation, "velero/", velerov1api.VolumeSnapshotLabel, ""), + builder.WithLabels(velerov1api.BackupNameLabel, "test", velerov1api.VolumeSnapshotLabel, "")). + VolumeName("testPV").StorageClass("testSC").Phase(corev1.ClaimBound).Result(), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(*testing.T) { + crClient := velerotest.NewFakeControllerRuntimeClient(t) + logger := logrus.New() + logger.Level = logrus.DebugLevel + + if tc.pvc != nil { + require.NoError(t, crClient.Create(context.Background(), tc.pvc)) + } + if tc.pv != nil { + require.NoError(t, crClient.Create(context.Background(), tc.pv)) + } + if tc.sc != nil { + require.NoError(t, crClient.Create(context.Background(), tc.sc)) + } + if tc.vsClass != nil { + require.NoError(t, crClient.Create(context.Background(), tc.vsClass)) + } + + pvcBIA := pvcBackupItemAction{ + log: logger, + crClient: crClient, + } + + pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&tc.pvc) + require.NoError(t, err) + + if boolptr.IsSetToTrue(tc.backup.Spec.SnapshotMoveData) == true { + go func() { + var vsList v1.VolumeSnapshotList + err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 10*time.Second, true, func(ctx context.Context) (bool, error) { + err = pvcBIA.crClient.List(ctx, &vsList, &crclient.ListOptions{Namespace: tc.pvc.Namespace}) + + require.NoError(t, err) + if err != nil || len(vsList.Items) == 0 { + //lint:ignore nilerr reason + return false, nil // ignore + } + return true, nil + }) + + require.NoError(t, err) + vscName := "testVSC" + readyToUse := true + vsList.Items[0].Status = &v1.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &vscName, + ReadyToUse: &readyToUse, + } + err = pvcBIA.crClient.Update(context.Background(), &vsList.Items[0]) + require.NoError(t, err) + + handleName := "testHandle" + vsc := builder.ForVolumeSnapshotContent("testVSC").Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: &handleName}).Result() + err = pvcBIA.crClient.Create(context.Background(), vsc) + require.NoError(t, err) + }() + } + + resultUnstructed, _, _, _, err := pvcBIA.Execute(&unstructured.Unstructured{Object: pvcMap}, tc.backup) + if tc.expectedErr != nil { + require.Equal(t, err, tc.expectedErr) + } + + if tc.expectedDataUpload != nil { + dataUploadList := new(velerov2alpha1.DataUploadList) + err := crClient.List(context.Background(), dataUploadList, &crclient.ListOptions{LabelSelector: labels.SelectorFromSet(map[string]string{velerov1api.BackupNameLabel: tc.backup.Name})}) + require.NoError(t, err) + require.Len(t, dataUploadList.Items, 1) + require.True(t, cmp.Equal(tc.expectedDataUpload, &dataUploadList.Items[0], cmpopts.IgnoreFields(velerov2alpha1.DataUpload{}, "ResourceVersion", "Name", "Spec.CSISnapshot.VolumeSnapshot"))) + } + + if tc.expectedPVC != nil { + resultPVC := new(corev1.PersistentVolumeClaim) + runtime.DefaultUnstructuredConverter.FromUnstructured(resultUnstructed.UnstructuredContent(), resultPVC) + + require.True(t, cmp.Equal(tc.expectedPVC, resultPVC, cmpopts.IgnoreFields(corev1.PersistentVolumeClaim{}, "ResourceVersion", "Annotations", "Labels"))) + } + }) + } +} + +func TestProgress(t *testing.T) { + currentTime := time.Now() + tests := []struct { + name string + backup *velerov1api.Backup + dataUpload *velerov2alpha1.DataUpload + operationID string + expectedErr string + expectedProgress velero.OperationProgress + }{ + { + name: "DataUpload cannot be found", + backup: builder.ForBackup("velero", "test").Result(), + operationID: "testing", + expectedErr: "not found DataUpload for operationID testing", + }, + { + name: "DataUpload is found", + backup: builder.ForBackup("velero", "test").Result(), + dataUpload: &velerov2alpha1.DataUpload{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataUpload", + APIVersion: "v2alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "velero", + Name: "testing", + Labels: map[string]string{ + velerov1api.AsyncOperationIDLabel: "testing", + }, + }, + Status: velerov2alpha1.DataUploadStatus{ + Phase: velerov2alpha1.DataUploadPhaseFailed, + Progress: shared.DataMoveOperationProgress{ + BytesDone: 1000, + TotalBytes: 1000, + }, + StartTimestamp: &metav1.Time{Time: currentTime}, + CompletionTimestamp: &metav1.Time{Time: currentTime}, + Message: "Testing error", + }, + }, + operationID: "testing", + expectedProgress: velero.OperationProgress{ + Completed: true, + Err: "Testing error", + NCompleted: 1000, + NTotal: 1000, + OperationUnits: "Bytes", + Description: "Failed", + Started: currentTime, + Updated: currentTime, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(*testing.T) { + crClient := velerotest.NewFakeControllerRuntimeClient(t) + logger := logrus.New() + + pvcBIA := pvcBackupItemAction{ + log: logger, + crClient: crClient, + } + + if tc.dataUpload != nil { + err := crClient.Create(context.Background(), tc.dataUpload) + require.NoError(t, err) + } + + progress, err := pvcBIA.Progress(tc.operationID, tc.backup) + if tc.expectedErr != "" { + require.Equal(t, tc.expectedErr, err.Error()) + } + require.True(t, cmp.Equal(tc.expectedProgress, progress, cmpopts.IgnoreFields(velero.OperationProgress{}, "Started", "Updated"))) + }) + } +} + +func TestCancel(t *testing.T) { + tests := []struct { + name string + backup *velerov1api.Backup + dataUpload velerov2alpha1.DataUpload + operationID string + expectedErr error + expectedDataUpload velerov2alpha1.DataUpload + }{ + { + name: "Cancel DataUpload", + backup: builder.ForBackup("velero", "test").Result(), + dataUpload: velerov2alpha1.DataUpload{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataUpload", + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "velero", + Name: "testing", + Labels: map[string]string{ + velerov1api.AsyncOperationIDLabel: "testing", + }, + }, + }, + operationID: "testing", + expectedErr: nil, + expectedDataUpload: velerov2alpha1.DataUpload{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataUpload", + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "velero", + Name: "testing", + Labels: map[string]string{ + velerov1api.AsyncOperationIDLabel: "testing", + }, + }, + Spec: velerov2alpha1.DataUploadSpec{ + Cancel: true, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(*testing.T) { + crClient := velerotest.NewFakeControllerRuntimeClient(t) + logger := logrus.New() + + pvcBIA := pvcBackupItemAction{ + log: logger, + crClient: crClient, + } + + err := crClient.Create(context.Background(), &tc.dataUpload) + require.NoError(t, err) + + err = pvcBIA.Cancel(tc.operationID, tc.backup) + if tc.expectedErr != nil { + require.Equal(t, err, tc.expectedErr) + } + + du := new(velerov2alpha1.DataUpload) + err = crClient.Get(context.Background(), crclient.ObjectKey{Namespace: tc.dataUpload.Namespace, Name: tc.dataUpload.Name}, du) + require.NoError(t, err) + + require.True(t, cmp.Equal(tc.expectedDataUpload, *du, cmpopts.IgnoreFields(velerov2alpha1.DataUpload{}, "ResourceVersion"))) + }) + } +} diff --git a/pkg/backup/actions/csi/volumesnapshot_action.go b/pkg/backup/actions/csi/volumesnapshot_action.go new file mode 100644 index 000000000..64e38c3c7 --- /dev/null +++ b/pkg/backup/actions/csi/volumesnapshot_action.go @@ -0,0 +1,379 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + "fmt" + "strings" + "time" + + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/client" + "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/label" + plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + biav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/csi" + kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" +) + +// volumeSnapshotBackupItemAction is a backup item action plugin to backup +// CSI VolumeSnapshot objects using Velero +type volumeSnapshotBackupItemAction struct { + log logrus.FieldLogger + crClient crclient.Client +} + +// AppliesTo returns information indicating that the +// VolumeSnapshotBackupItemAction should be invoked to +// backup VolumeSnapshots. +func (p *volumeSnapshotBackupItemAction) AppliesTo() ( + velero.ResourceSelector, + error, +) { + return velero.ResourceSelector{ + IncludedResources: []string{"volumesnapshots.snapshot.storage.k8s.io"}, + }, nil +} + +// Execute backs up a CSI VolumeSnapshot object and captures, as labels and annotations, +// information from its associated VolumeSnapshotContents such as CSI driver name, +// storage snapshot handle and namespace and name of the snapshot delete secret, if any. +// It returns the VolumeSnapshotClass and the VolumeSnapshotContents as additional items +// to be backed up. +func (p *volumeSnapshotBackupItemAction) Execute( + item runtime.Unstructured, + backup *velerov1api.Backup, +) ( + runtime.Unstructured, + []velero.ResourceIdentifier, + string, + []velero.ResourceIdentifier, + error, +) { + p.log.Infof("Executing VolumeSnapshotBackupItemAction") + + vs := new(snapshotv1api.VolumeSnapshot) + if err := runtime.DefaultUnstructuredConverter.FromUnstructured( + item.UnstructuredContent(), vs); err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + additionalItems := []velero.ResourceIdentifier{ + { + GroupResource: kuberesource.VolumeSnapshotClasses, + Name: *vs.Spec.VolumeSnapshotClassName, + }, + } + + // determine if we are backing up a VolumeSnapshot that was created by velero while + // performing backup of a CSI backed PVC. + // For VolumeSnapshots that were created during the backup of a CSI backed PVC, + // we will wait for the VolumeSnapshotContents to be available. + // For VolumeSnapshots created outside of velero, we expect the VolumeSnapshotContent + // to be available prior to backing up the VolumeSnapshot. In case of a failure, + // backup should be re-attempted after the CSI driver has reconciled the VolumeSnapshot. + // existence of the velerov1api.BackupNameLabel indicates that the VolumeSnapshot was + // created while backing up a CSI backed PVC. + + // We want to await reconciliation of only those VolumeSnapshots created during the + // ongoing backup. For this we will wait only if the backup label exists on the + // VolumeSnapshot object and the backup name is the same as that of the value of the + // backup label. + backupOngoing := vs.Labels[velerov1api.BackupNameLabel] == label.GetValidName(backup.Name) + + p.log.Infof("Getting VolumesnapshotContent for Volumesnapshot %s/%s", + vs.Namespace, vs.Name) + + vsc, err := csi.WaitUntilVSCHandleIsReady( + vs, + p.crClient, + p.log, + backupOngoing, + backup.Spec.CSISnapshotTimeout.Duration, + ) + if err != nil { + csi.CleanupVolumeSnapshot(vs, p.crClient, p.log) + return nil, nil, "", nil, errors.WithStack(err) + } + + if backup.Status.Phase == velerov1api.BackupPhaseFinalizing || + backup.Status.Phase == velerov1api.BackupPhaseFinalizingPartiallyFailed { + p.log. + WithField("Backup", fmt.Sprintf("%s/%s", backup.Namespace, backup.Name)). + WithField("BackupPhase", backup.Status.Phase).Debugf("Clean VolumeSnapshots.") + + csi.DeleteVolumeSnapshot(*vs, *vsc, backup, p.crClient, p.log) + return item, nil, "", nil, nil + } + + annotations := make(map[string]string) + + if vsc != nil { + // when we are backing up VolumeSnapshots created outside of velero, we + // will not await VolumeSnapshot reconciliation and in this case + // GetVolumeSnapshotContentForVolumeSnapshot may not find the associated + // VolumeSnapshotContents to add to the backup. This is not an error + // encountered in the backup process. So we add the VolumeSnapshotContent + // to the backup only if one is found. + additionalItems = append(additionalItems, velero.ResourceIdentifier{ + GroupResource: kuberesource.VolumeSnapshotContents, + Name: vsc.Name, + }) + annotations[velerov1api.VSCDeletionPolicyAnnotation] = string(vsc.Spec.DeletionPolicy) + + if vsc.Status != nil { + if vsc.Status.SnapshotHandle != nil { + // Capture storage provider snapshot handle and CSI driver name + // to be used on restore to create a static VolumeSnapshotContent + // that will be the source of the VolumeSnapshot. + annotations[velerov1api.VolumeSnapshotHandleAnnotation] = *vsc.Status.SnapshotHandle + annotations[velerov1api.DriverNameAnnotation] = vsc.Spec.Driver + } + if vsc.Status.RestoreSize != nil { + annotations[velerov1api.VolumeSnapshotRestoreSize] = resource.NewQuantity( + *vsc.Status.RestoreSize, resource.BinarySI).String() + } + } + + if backupOngoing { + p.log.Infof("Patching VolumeSnapshotContent %s with velero BackupNameLabel", + vsc.Name) + // If we created the VolumeSnapshotContent object during this ongoing backup, + // we would have created it with a DeletionPolicy of Retain. + // But, we want to retain these VolumeSnapshotContent ONLY for the lifetime + // of the backup. To that effect, during velero backup + // deletion, we will update the DeletionPolicy of the VolumeSnapshotContent + // and then delete the VolumeSnapshot object which will cascade delete the + // VolumeSnapshotContent and the associated snapshot in the storage + // provider (handled by the CSI driver and the CSI common controller). + // However, in the event that the VolumeSnapshot object is deleted outside + // of the backup deletion process, it is possible that the dynamically created + // VolumeSnapshotContent object will be left as an orphaned and non-discoverable + // resource in the cluster as well as in the storage provider. To avoid piling + // up of such orphaned resources, we will want to discover and delete the + // dynamically created VolumeSnapshotContent. We do that by adding + // the "velero.io/backup-name" label on the VolumeSnapshotContent. + // Further, we want to add this label only on VolumeSnapshotContents that + // were created during an ongoing velero backup. + originVSC := vsc.DeepCopy() + kubeutil.AddLabels( + &vsc.ObjectMeta, + map[string]string{ + velerov1api.BackupNameLabel: label.GetValidName(backup.Name), + }, + ) + + if vscPatchError := p.crClient.Patch( + context.TODO(), + vsc, + crclient.MergeFrom(originVSC), + ); vscPatchError != nil { + p.log.Warnf("Failed to patch VolumeSnapshotContent %s: %v", + vsc.Name, vscPatchError) + } + } + } + + // Before applying the BIA v2, the in-cluster VS state is not persisted into backup. + // After the change, because the final state of VS will be stored in backup as the + // result of async operation result, need to patch the annotations into VS to work, + // because restore will check the annotations information. + originVS := vs.DeepCopy() + kubeutil.AddAnnotations(&vs.ObjectMeta, annotations) + if err := p.crClient.Patch( + context.TODO(), + vs, + crclient.MergeFrom(originVS), + ); err != nil { + p.log.Errorf("Fail to patch VolumeSnapshot: %s.", err.Error()) + return nil, nil, "", nil, errors.WithStack(err) + } + + annotations[velerov1api.MustIncludeAdditionalItemAnnotation] = "true" + // save newly applied annotations into the backed-up VolumeSnapshot item + kubeutil.AddAnnotations(&vs.ObjectMeta, annotations) + + vsMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(vs) + if err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + p.log.Infof("Returning from VolumeSnapshotBackupItemAction", + "with %d additionalItems to backup", len(additionalItems)) + for _, ai := range additionalItems { + p.log.Debugf("%s: %s", ai.GroupResource.String(), ai.Name) + } + + operationID := "" + var itemToUpdate []velero.ResourceIdentifier + + // Only return Async operation for VSC created for this backup. + if backupOngoing { + // The operationID is of the form // + operationID = vs.Namespace + "/" + vs.Name + "/" + time.Now().Format(time.RFC3339) + itemToUpdate = []velero.ResourceIdentifier{ + { + GroupResource: kuberesource.VolumeSnapshots, + Namespace: vs.Namespace, + Name: vs.Name, + }, + { + GroupResource: kuberesource.VolumeSnapshotContents, + Name: vsc.Name, + }, + } + } + + return &unstructured.Unstructured{Object: vsMap}, + additionalItems, operationID, itemToUpdate, nil +} + +// Name returns the plugin's name. +func (p *volumeSnapshotBackupItemAction) Name() string { + return "VolumeSnapshotBackupItemAction" +} + +func (p *volumeSnapshotBackupItemAction) Progress( + operationID string, + backup *velerov1api.Backup, +) (velero.OperationProgress, error) { + progress := velero.OperationProgress{} + if operationID == "" { + return progress, biav2.InvalidOperationIDError(operationID) + } + // The operationID is of the form // + operationIDParts := strings.Split(operationID, "/") + if len(operationIDParts) != 3 { + p.log.Errorf("invalid operation ID %s", operationID) + return progress, biav2.InvalidOperationIDError(operationID) + } + var err error + if progress.Started, err = time.Parse(time.RFC3339, operationIDParts[2]); err != nil { + p.log.Errorf("error parsing operation ID's StartedTime", + "part into time %s: %s", operationID, err.Error()) + return progress, errors.WithStack(err) + } + + vs := new(snapshotv1api.VolumeSnapshot) + if err := p.crClient.Get( + context.Background(), + crclient.ObjectKey{ + Namespace: operationIDParts[0], + Name: operationIDParts[1], + }, + vs); err != nil { + p.log.Errorf("error getting volumesnapshot %s/%s: %s", + operationIDParts[0], operationIDParts[1], err.Error()) + return progress, errors.WithStack(err) + } + + if vs.Status == nil { + p.log.Debugf("VolumeSnapshot %s/%s has an empty status.", + "Skip progress update.", vs.Namespace, vs.Name) + return progress, nil + } + + if boolptr.IsSetToTrue(vs.Status.ReadyToUse) { + p.log.Debugf("VolumeSnapshot %s/%s is ReadyToUse.", + "Continue on querying corresponding VolumeSnapshotContent.", + vs.Namespace, vs.Name) + } else if vs.Status.Error != nil { + errorMessage := "" + if vs.Status.Error.Message != nil { + errorMessage = *vs.Status.Error.Message + } + p.log.Warnf("VolumeSnapshot has a temporary error %s.", + "Snapshot controller will retry later.", errorMessage) + + return progress, nil + } + + if vs.Status != nil && vs.Status.BoundVolumeSnapshotContentName != nil { + vsc := new(snapshotv1api.VolumeSnapshotContent) + err := p.crClient.Get( + context.Background(), + crclient.ObjectKey{Name: *vs.Status.BoundVolumeSnapshotContentName}, + vsc, + ) + if err != nil { + p.log.Errorf("error getting VolumeSnapshotContent %s: %s", + *vs.Status.BoundVolumeSnapshotContentName, err.Error()) + return progress, errors.WithStack(err) + } + + if vsc.Status == nil { + p.log.Debugf("VolumeSnapshotContent %s has an empty Status.", + "Skip progress update.", vsc.Name) + return progress, nil + } + + now := time.Now() + + if boolptr.IsSetToTrue(vsc.Status.ReadyToUse) { + progress.Completed = true + progress.Updated = now + } else if vsc.Status.Error != nil { + progress.Completed = true + progress.Updated = now + if vsc.Status.Error.Message != nil { + progress.Err = *vsc.Status.Error.Message + } + p.log.Warnf("VolumeSnapshotContent meets an error %s.", progress.Err) + } + } + + return progress, nil +} + +// Cancel is not implemented for VolumeSnapshotBackupItemAction +func (p *volumeSnapshotBackupItemAction) Cancel( + operationID string, + backup *velerov1api.Backup, +) error { + // CSI Specification doesn't support canceling a snapshot creation. + return nil +} + +// NewVolumeSnapshotBackupItemAction returns +// VolumeSnapshotBackupItemAction instance. +func NewVolumeSnapshotBackupItemAction( + f client.Factory, +) plugincommon.HandlerInitializer { + return func(logger logrus.FieldLogger) (interface{}, error) { + crClient, err := f.KubebuilderClient() + if err != nil { + return nil, errors.WithStack(err) + } + + return &volumeSnapshotBackupItemAction{ + log: logger, + crClient: crClient, + }, nil + } +} diff --git a/pkg/backup/actions/csi/volumesnapshotclass_action.go b/pkg/backup/actions/csi/volumesnapshotclass_action.go new file mode 100644 index 000000000..d6f9ca4e5 --- /dev/null +++ b/pkg/backup/actions/csi/volumesnapshotclass_action.go @@ -0,0 +1,124 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + csiutil "github.com/vmware-tanzu/velero/pkg/util/csi" + kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" +) + +// volumeSnapshotClassBackupItemAction is a backup item action plugin to +// backup CSI VolumeSnapshotClass objects using Velero +type volumeSnapshotClassBackupItemAction struct { + log logrus.FieldLogger +} + +// AppliesTo returns information indicating that the +// VolumeSnapshotClassBackupItemAction action should be invoked +// to backup VolumeSnapshotClass. +func (p *volumeSnapshotClassBackupItemAction) AppliesTo() ( + velero.ResourceSelector, + error, +) { + return velero.ResourceSelector{ + IncludedResources: []string{"volumesnapshotclass.snapshot.storage.k8s.io"}, + }, nil +} + +// Execute backs up a VolumeSnapshotClass object and returns as additional +// items any snapshot lister secret that may be referenced in its annotations. +func (p *volumeSnapshotClassBackupItemAction) Execute( + item runtime.Unstructured, + backup *velerov1api.Backup, +) ( + runtime.Unstructured, + []velero.ResourceIdentifier, + string, + []velero.ResourceIdentifier, + error, +) { + p.log.Infof("Executing VolumeSnapshotClassBackupItemAction") + + var snapClass snapshotv1api.VolumeSnapshotClass + if err := runtime.DefaultUnstructuredConverter.FromUnstructured( + item.UnstructuredContent(), + &snapClass, + ); err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + additionalItems := []velero.ResourceIdentifier{} + if csiutil.IsVolumeSnapshotClassHasListerSecret(&snapClass) { + additionalItems = append(additionalItems, velero.ResourceIdentifier{ + GroupResource: kuberesource.Secrets, + Name: snapClass.Annotations[velerov1api.PrefixedListSecretNameAnnotation], + Namespace: snapClass.Annotations[velerov1api.PrefixedListSecretNamespaceAnnotation], + }) + + kubeutil.AddAnnotations(&snapClass.ObjectMeta, map[string]string{ + velerov1api.MustIncludeAdditionalItemAnnotation: "true", + }) + } + + snapClassMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&snapClass) + if err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + p.log.Infof( + "Returning from VolumeSnapshotClassBackupItemAction with %d additionalItems to backup", + len(additionalItems), + ) + return &unstructured.Unstructured{Object: snapClassMap}, additionalItems, "", nil, nil +} + +// Name returns the plugin's name. +func (p *volumeSnapshotClassBackupItemAction) Name() string { + return "VolumeSnapshotClassBackupItemAction" +} + +// Progress is not implemented for VolumeSnapshotClassBackupItemAction +func (p *volumeSnapshotClassBackupItemAction) Progress( + operationID string, + backup *velerov1api.Backup, +) (velero.OperationProgress, error) { + return velero.OperationProgress{}, nil +} + +// Cancel is not implemented for VolumeSnapshotClassBackupItemAction +func (p *volumeSnapshotClassBackupItemAction) Cancel( + operationID string, + backup *velerov1api.Backup, +) error { + return nil +} + +// NewVolumeSnapshotClassBackupItemAction returns a +// VolumeSnapshotClassBackupItemAction instance. +func NewVolumeSnapshotClassBackupItemAction(logger logrus.FieldLogger) (interface{}, error) { + return &volumeSnapshotClassBackupItemAction{log: logger}, nil +} diff --git a/pkg/backup/actions/csi/volumesnapshotcontent_action.go b/pkg/backup/actions/csi/volumesnapshotcontent_action.go new file mode 100644 index 000000000..e8e17280d --- /dev/null +++ b/pkg/backup/actions/csi/volumesnapshotcontent_action.go @@ -0,0 +1,141 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "fmt" + + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + csiutil "github.com/vmware-tanzu/velero/pkg/util/csi" + kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" +) + +// volumeSnapshotContentBackupItemAction is a backup item action plugin to backup +// CSI VolumeSnapshotContent objects using Velero +type volumeSnapshotContentBackupItemAction struct { + log logrus.FieldLogger +} + +// AppliesTo returns information indicating that the +// VolumeSnapshotContentBackupItemAction action should be invoked to +// backup VolumeSnapshotContents. +func (p *volumeSnapshotContentBackupItemAction) AppliesTo() (velero.ResourceSelector, error) { + return velero.ResourceSelector{ + IncludedResources: []string{"volumesnapshotcontent.snapshot.storage.k8s.io"}, + }, nil +} + +// Execute returns the unmodified VolumeSnapshotContent object along +// with the snapshot deletion secret, if any, from its annotation +// as additional items to backup. +func (p *volumeSnapshotContentBackupItemAction) Execute( + item runtime.Unstructured, + backup *velerov1api.Backup, +) ( + runtime.Unstructured, + []velero.ResourceIdentifier, + string, + []velero.ResourceIdentifier, + error, +) { + p.log.Infof("Executing VolumeSnapshotContentBackupItemAction") + + if backup.Status.Phase == velerov1api.BackupPhaseFinalizing || + backup.Status.Phase == velerov1api.BackupPhaseFinalizingPartiallyFailed { + p.log.WithField("Backup", fmt.Sprintf("%s/%s", backup.Namespace, backup.Name)). + WithField("BackupPhase", backup.Status.Phase). + Debug("Skipping VolumeSnapshotContentBackupItemAction", + "as backup is in finalizing phase.") + return item, nil, "", nil, nil + } + + var snapCont snapshotv1api.VolumeSnapshotContent + if err := runtime.DefaultUnstructuredConverter.FromUnstructured( + item.UnstructuredContent(), + &snapCont, + ); err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + additionalItems := []velero.ResourceIdentifier{} + + // we should backup the snapshot deletion secrets that may be referenced + // in the VolumeSnapshotContent's annotation + if csiutil.IsVolumeSnapshotContentHasDeleteSecret(&snapCont) { + additionalItems = append( + additionalItems, + velero.ResourceIdentifier{ + GroupResource: kuberesource.Secrets, + Name: snapCont.Annotations[velerov1api.PrefixedSecretNameAnnotation], + Namespace: snapCont.Annotations[velerov1api.PrefixedSecretNamespaceAnnotation], + }) + + kubeutil.AddAnnotations(&snapCont.ObjectMeta, map[string]string{ + velerov1api.MustIncludeAdditionalItemAnnotation: "true", + }) + } + + snapContMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&snapCont) + if err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + p.log.Infof( + "Returning from VolumeSnapshotContentBackupItemAction", + "with %d additionalItems to backup", + len(additionalItems), + ) + return &unstructured.Unstructured{Object: snapContMap}, additionalItems, "", nil, nil +} + +// Name returns the plugin's name. +func (p *volumeSnapshotContentBackupItemAction) Name() string { + return "VolumeSnapshotContentBackupItemAction" +} + +// Progress is not implemented for VolumeSnapshotContentBackupItemAction. +func (p *volumeSnapshotContentBackupItemAction) Progress( + operationID string, + backup *velerov1api.Backup, +) (velero.OperationProgress, error) { + return velero.OperationProgress{}, nil +} + +// Cancel is not implemented for VolumeSnapshotContentBackupItemAction. +func (p *volumeSnapshotContentBackupItemAction) Cancel( + operationID string, + backup *velerov1api.Backup, +) error { + // CSI Specification doesn't support canceling a snapshot creation. + return nil +} + +// NewVolumeSnapshotContentBackupItemAction returns a +// VolumeSnapshotContentBackupItemAction instance. +func NewVolumeSnapshotContentBackupItemAction( + logger logrus.FieldLogger, +) (interface{}, error) { + return &volumeSnapshotContentBackupItemAction{log: logger}, nil +} diff --git a/pkg/backup/backup_test.go b/pkg/backup/backup_test.go index 0a8c7db78..142570648 100644 --- a/pkg/backup/backup_test.go +++ b/pkg/backup/backup_test.go @@ -383,16 +383,16 @@ func TestBackupOldResourceFiltering(t *testing.T) { Result(), apiResources: []*test.APIResource{ test.Pods( - builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "true")).Result(), + builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true")).Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), - builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "true")).Result(), + builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true")).Result(), ), test.PVs( builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a", "b")).Result(), - builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "true")).Result(), + builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true")).Result(), ), }, want: []string{ @@ -411,16 +411,16 @@ func TestBackupOldResourceFiltering(t *testing.T) { Result(), apiResources: []*test.APIResource{ test.Pods( - builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "true", "a", "b")).Result(), + builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true", "a", "b")).Result(), builder.ForPod("zoo", "raz").ObjectMeta(builder.WithLabels("a", "b")).Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), - builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "true", "a", "b")).Result(), + builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true", "a", "b")).Result(), ), test.PVs( builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a", "b")).Result(), - builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels("a", "b", "velero.io/exclude-from-backup", "true")).Result(), + builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels("a", "b", velerov1.ExcludeFromBackupLabel, "true")).Result(), ), }, want: []string{ @@ -436,16 +436,16 @@ func TestBackupOldResourceFiltering(t *testing.T) { Result(), apiResources: []*test.APIResource{ test.Pods( - builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "false")).Result(), + builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "false")).Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), - builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "1")).Result(), + builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "1")).Result(), ), test.PVs( builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a", "b")).Result(), - builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "")).Result(), + builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "")).Result(), ), }, want: []string{ @@ -1273,7 +1273,7 @@ func (a *recordResourcesAction) Execute(item runtime.Unstructured, backup *veler a.backups = append(a.backups, *backup) if a.skippedCSISnapshot { u := &unstructured.Unstructured{Object: item.UnstructuredContent()} - u.SetAnnotations(map[string]string{skippedNoCSIPVAnnotation: "true"}) + u.SetAnnotations(map[string]string{velerov1.SkippedNoCSIPVAnnotation: "true"}) item = u a.additionalItems = nil } @@ -2028,7 +2028,7 @@ func TestBackupActionAdditionalItems(t *testing.T) { builder.ForPod("ns-1", "pod-1").Result(), ), test.PVs( - builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "true")).Result(), + builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true")).Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, diff --git a/pkg/backup/item_backupper.go b/pkg/backup/item_backupper.go index 4b589fae3..953b3191e 100644 --- a/pkg/backup/item_backupper.go +++ b/pkg/backup/item_backupper.go @@ -24,19 +24,17 @@ import ( "strings" "time" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" kubeerrs "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" - kbClient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/hook" @@ -58,11 +56,8 @@ import ( ) const ( - mustIncludeAdditionalItemAnnotation = "backup.velero.io/must-include-additional-items" - skippedNoCSIPVAnnotation = "backup.velero.io/skipped-no-csi-pv" - excludeFromBackupLabel = "velero.io/exclude-from-backup" - csiBIAPluginName = "velero.io/csi-pvc-backupper" - vsphereBIAPluginName = "velero.io/vsphere-pvc-backupper" + csiBIAPluginName = "velero.io/csi-pvc-backupper" + vsphereBIAPluginName = "velero.io/vsphere-pvc-backupper" ) // itemBackupper can back up individual items to a tar writer. @@ -129,9 +124,9 @@ func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runti if mustInclude { log.Infof("Skipping the exclusion checks for this resource") } else { - if metadata.GetLabels()[excludeFromBackupLabel] == "true" { - log.Infof("Excluding item because it has label %s=true", excludeFromBackupLabel) - ib.trackSkippedPV(obj, groupResource, "", fmt.Sprintf("item has label %s=true", excludeFromBackupLabel), log) + if metadata.GetLabels()[velerov1api.ExcludeFromBackupLabel] == "true" { + log.Infof("Excluding item because it has label %s=true", velerov1api.ExcludeFromBackupLabel) + ib.trackSkippedPV(obj, groupResource, "", fmt.Sprintf("item has label %s=true", velerov1api.ExcludeFromBackupLabel), log) return false, itemFiles, nil } // NOTE: we have to re-check namespace & resource includes/excludes because it's possible that @@ -384,18 +379,18 @@ func (ib *itemBackupper) executeActions( return nil, itemFiles, errors.Wrapf(err, "error executing custom action (groupResource=%s, namespace=%s, name=%s)", groupResource.String(), namespace, name) } u := &unstructured.Unstructured{Object: updatedItem.UnstructuredContent()} - if actionName == csiBIAPluginName && additionalItemIdentifiers == nil && u.GetAnnotations()[skippedNoCSIPVAnnotation] == "true" { + if actionName == csiBIAPluginName && additionalItemIdentifiers == nil && u.GetAnnotations()[velerov1api.SkippedNoCSIPVAnnotation] == "true" { // snapshot was skipped by CSI plugin ib.trackSkippedPV(obj, groupResource, csiSnapshotApproach, "skipped b/c it's not a CSI volume", log) - delete(u.GetAnnotations(), skippedNoCSIPVAnnotation) + delete(u.GetAnnotations(), velerov1api.SkippedNoCSIPVAnnotation) } else if (actionName == csiBIAPluginName || actionName == vsphereBIAPluginName) && !boolptr.IsSetToFalse(ib.backupRequest.Backup.Spec.SnapshotVolumes) { // the snapshot has been taken by the BIA plugin ib.unTrackSkippedPV(obj, groupResource, log) } - mustInclude := u.GetAnnotations()[mustIncludeAdditionalItemAnnotation] == "true" || finalize + mustInclude := u.GetAnnotations()[velerov1api.MustIncludeAdditionalItemAnnotation] == "true" || finalize // remove the annotation as it's for communication between BIA and velero server, // we don't want the resource be restored with this annotation. - delete(u.GetAnnotations(), mustIncludeAdditionalItemAnnotation) + delete(u.GetAnnotations(), velerov1api.MustIncludeAdditionalItemAnnotation) obj = u // If async plugin started async operation, add it to the ItemOperations list diff --git a/pkg/client/factory.go b/pkg/client/factory.go index d359ee229..c74f43b88 100644 --- a/pkg/client/factory.go +++ b/pkg/client/factory.go @@ -195,6 +195,9 @@ func (f *factory) KubebuilderWatchClient() (kbclient.WithWatch, error) { if err := apiextv1.AddToScheme(scheme); err != nil { return nil, err } + if err := snapshotv1api.AddToScheme(scheme); err != nil { + return nil, err + } kubebuilderWatchClient, err := kbclient.NewWithWatch(clientConfig, kbclient.Options{ Scheme: scheme, }) diff --git a/pkg/cmd/server/plugin/plugin.go b/pkg/cmd/server/plugin/plugin.go index c375c5437..0f729f3ac 100644 --- a/pkg/cmd/server/plugin/plugin.go +++ b/pkg/cmd/server/plugin/plugin.go @@ -23,14 +23,17 @@ import ( "github.com/vmware-tanzu/velero/pkg/datamover" + dia "github.com/vmware-tanzu/velero/internal/delete/actions/csi" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" bia "github.com/vmware-tanzu/velero/pkg/backup/actions" + csibia "github.com/vmware-tanzu/velero/pkg/backup/actions/csi" "github.com/vmware-tanzu/velero/pkg/client" velerodiscovery "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/features" veleroplugin "github.com/vmware-tanzu/velero/pkg/plugin/framework" plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" ria "github.com/vmware-tanzu/velero/pkg/restore/actions" + csiria "github.com/vmware-tanzu/velero/pkg/restore/actions/csi" ) func NewCommand(f client.Factory) *cobra.Command { @@ -41,32 +44,142 @@ func NewCommand(f client.Factory) *cobra.Command { Short: "INTERNAL COMMAND ONLY - not intended to be run directly by users", Run: func(c *cobra.Command, args []string) { pluginServer = pluginServer. - RegisterBackupItemAction("velero.io/pv", newPVBackupItemAction). - RegisterBackupItemAction("velero.io/pod", newPodBackupItemAction). - RegisterBackupItemAction("velero.io/service-account", newServiceAccountBackupItemAction(f)). - RegisterRestoreItemAction("velero.io/job", newJobRestoreItemAction). - RegisterRestoreItemAction("velero.io/pod", newPodRestoreItemAction). - RegisterRestoreItemAction("velero.io/pod-volume-restore", newPodVolumeRestoreItemAction(f)). - RegisterRestoreItemAction("velero.io/init-restore-hook", newInitRestoreHookPodAction). - RegisterRestoreItemAction("velero.io/service", newServiceRestoreItemAction). - RegisterRestoreItemAction("velero.io/service-account", newServiceAccountRestoreItemAction). - RegisterRestoreItemAction("velero.io/add-pvc-from-pod", newAddPVCFromPodRestoreItemAction). - RegisterRestoreItemAction("velero.io/add-pv-from-pvc", newAddPVFromPVCRestoreItemAction). - RegisterRestoreItemAction("velero.io/change-storage-class", newChangeStorageClassRestoreItemAction(f)). - RegisterRestoreItemAction("velero.io/change-image-name", newChangeImageNameRestoreItemAction(f)). - RegisterRestoreItemAction("velero.io/role-bindings", newRoleBindingItemAction). - RegisterRestoreItemAction("velero.io/cluster-role-bindings", newClusterRoleBindingItemAction). - RegisterRestoreItemAction("velero.io/crd-preserve-fields", newCRDV1PreserveUnknownFieldsItemAction). - RegisterRestoreItemAction("velero.io/change-pvc-node-selector", newChangePVCNodeSelectorItemAction(f)). - RegisterRestoreItemAction("velero.io/apiservice", newAPIServiceRestoreItemAction). - RegisterRestoreItemAction("velero.io/admission-webhook-configuration", newAdmissionWebhookConfigurationAction). - RegisterRestoreItemAction("velero.io/secret", newSecretRestoreItemAction(f)). - RegisterRestoreItemAction("velero.io/dataupload", newDataUploadRetrieveAction(f)). - RegisterDeleteItemAction("velero.io/dataupload-delete", newDateUploadDeleteItemAction(f)) + RegisterBackupItemAction( + "velero.io/pv", + newPVBackupItemAction, + ). + RegisterBackupItemAction( + "velero.io/pod", + newPodBackupItemAction, + ). + RegisterBackupItemAction( + "velero.io/service-account", + newServiceAccountBackupItemAction(f), + ). + RegisterRestoreItemAction( + "velero.io/job", + newJobRestoreItemAction, + ). + RegisterRestoreItemAction( + "velero.io/pod", + newPodRestoreItemAction, + ). + RegisterRestoreItemAction( + "velero.io/pod-volume-restore", + newPodVolumeRestoreItemAction(f), + ). + RegisterRestoreItemAction( + "velero.io/init-restore-hook", + newInitRestoreHookPodAction, + ). + RegisterRestoreItemAction( + "velero.io/service", + newServiceRestoreItemAction, + ). + RegisterRestoreItemAction( + "velero.io/service-account", + newServiceAccountRestoreItemAction, + ). + RegisterRestoreItemAction( + "velero.io/add-pvc-from-pod", + newAddPVCFromPodRestoreItemAction, + ). + RegisterRestoreItemAction( + "velero.io/add-pv-from-pvc", + newAddPVFromPVCRestoreItemAction, + ). + RegisterRestoreItemAction( + "velero.io/change-storage-class", + newChangeStorageClassRestoreItemAction(f), + ). + RegisterRestoreItemAction( + "velero.io/change-image-name", + newChangeImageNameRestoreItemAction(f), + ). + RegisterRestoreItemAction( + "velero.io/role-bindings", + newRoleBindingItemAction, + ). + RegisterRestoreItemAction( + "velero.io/cluster-role-bindings", + newClusterRoleBindingItemAction, + ). + RegisterRestoreItemAction( + "velero.io/crd-preserve-fields", + newCRDV1PreserveUnknownFieldsItemAction, + ). + RegisterRestoreItemAction( + "velero.io/change-pvc-node-selector", + newChangePVCNodeSelectorItemAction(f), + ). + RegisterRestoreItemAction( + "velero.io/apiservice", + newAPIServiceRestoreItemAction, + ). + RegisterRestoreItemAction( + "velero.io/admission-webhook-configuration", + newAdmissionWebhookConfigurationAction, + ). + RegisterRestoreItemAction( + "velero.io/secret", + newSecretRestoreItemAction(f), + ). + RegisterRestoreItemAction( + "velero.io/dataupload", + newDataUploadRetrieveAction(f), + ). + RegisterDeleteItemAction( + "velero.io/dataupload-delete", + newDateUploadDeleteItemAction(f), + ). + RegisterDeleteItemAction( + "velero.io/csi-volumesnapshot-delete", + newVolumeSnapshotDeleteItemAction(f), + ). + RegisterDeleteItemAction( + "velero.io/csi-volumesnapshotcontent-delete", + newVolumeSnapshotContentDeleteItemAction(f), + ). + RegisterBackupItemActionV2( + "velero.io/csi-pvc-backupper", + newPvcBackupItemAction(f), + ). + RegisterBackupItemActionV2( + "velero.io/csi-volumesnapshot-backupper", + newVolumeSnapshotBackupItemAction(f), + ). + RegisterBackupItemActionV2( + "velero.io/csi-volumesnapshotcontent-backupper", + newVolumeSnapshotContentBackupItemAction, + ). + RegisterBackupItemActionV2( + "velero.io/csi-volumesnapshotclass-backupper", + newVolumeSnapshotClassBackupItemAction, + ). + RegisterRestoreItemActionV2( + "velero.io/csi-pvc-restorer", + newPvcRestoreItemAction(f), + ). + RegisterRestoreItemActionV2( + "velero.io/csi-volumesnapshot-restorer", + newVolumeSnapshotRestoreItemAction(f), + ). + RegisterRestoreItemActionV2( + "velero.io/csi-volumesnapshotcontent-restorer", + newVolumeSnapshotContentRestoreItemAction, + ). + RegisterRestoreItemActionV2( + "velero.io/csi-volumesnapshotclass-restorer", + newVolumeSnapshotClassRestoreItemAction, + ) if !features.IsEnabled(velerov1api.APIGroupVersionsFeatureFlag) { - // Do not register crd-remap-version BIA if the API Group feature flag is enabled, so that the v1 CRD can be backed up - pluginServer = pluginServer.RegisterBackupItemAction("velero.io/crd-remap-version", newRemapCRDVersionAction(f)) + // Do not register crd-remap-version BIA if the API Group feature + // flag is enabled, so that the v1 CRD can be backed up. + pluginServer = pluginServer.RegisterBackupItemAction( + "velero.io/crd-remap-version", + newRemapCRDVersionAction(f), + ) } pluginServer.Serve() }, @@ -270,3 +383,51 @@ func newDateUploadDeleteItemAction(f client.Factory) plugincommon.HandlerInitial return datamover.NewDataUploadDeleteAction(logger, client), nil } } + +// CSI plugin init functions. + +// BackupItemAction plugins + +func newPvcBackupItemAction(f client.Factory) plugincommon.HandlerInitializer { + return csibia.NewPvcBackupItemAction(f) +} + +func newVolumeSnapshotBackupItemAction(f client.Factory) plugincommon.HandlerInitializer { + return csibia.NewVolumeSnapshotBackupItemAction(f) +} + +func newVolumeSnapshotContentBackupItemAction(logger logrus.FieldLogger) (interface{}, error) { + return csibia.NewVolumeSnapshotContentBackupItemAction(logger) +} + +func newVolumeSnapshotClassBackupItemAction(logger logrus.FieldLogger) (interface{}, error) { + return csibia.NewVolumeSnapshotClassBackupItemAction(logger) +} + +// DeleteItemAction plugins + +func newVolumeSnapshotDeleteItemAction(f client.Factory) plugincommon.HandlerInitializer { + return dia.NewVolumeSnapshotDeleteItemAction(f) +} + +func newVolumeSnapshotContentDeleteItemAction(f client.Factory) plugincommon.HandlerInitializer { + return dia.NewVolumeSnapshotContentDeleteItemAction(f) +} + +// RestoreItemAction plugins + +func newPvcRestoreItemAction(f client.Factory) plugincommon.HandlerInitializer { + return csiria.NewPvcRestoreItemAction(f) +} + +func newVolumeSnapshotRestoreItemAction(f client.Factory) plugincommon.HandlerInitializer { + return csiria.NewVolumeSnapshotRestoreItemAction(f) +} + +func newVolumeSnapshotContentRestoreItemAction(logger logrus.FieldLogger) (interface{}, error) { + return csiria.NewVolumeSnapshotContentRestoreItemAction(logger) +} + +func newVolumeSnapshotClassRestoreItemAction(logger logrus.FieldLogger) (interface{}, error) { + return csiria.NewVolumeSnapshotClassRestoreItemAction(logger) +} diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index da915ed7e..162d388f9 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -431,7 +431,7 @@ func (b *backupReconciler) prepareBackupRequest(backup *velerov1api.Backup, logg // Add namespaces with label velero.io/exclude-from-backup=true into request.Spec.ExcludedNamespaces // Essentially, adding the label velero.io/exclude-from-backup=true to a namespace would be equivalent to setting spec.ExcludedNamespaces namespaces := corev1api.NamespaceList{} - if err := b.kbClient.List(context.Background(), &namespaces, kbclient.MatchingLabels{"velero.io/exclude-from-backup": "true"}); err == nil { + if err := b.kbClient.List(context.Background(), &namespaces, kbclient.MatchingLabels{velerov1api.ExcludeFromBackupLabel: "true"}); err == nil { for _, ns := range namespaces.Items { request.Spec.ExcludedNamespaces = append(request.Spec.ExcludedNamespaces, ns.Name) } diff --git a/pkg/exposer/csi_snapshot.go b/pkg/exposer/csi_snapshot.go index 637d0805d..68dce2437 100644 --- a/pkg/exposer/csi_snapshot.go +++ b/pkg/exposer/csi_snapshot.go @@ -20,27 +20,22 @@ import ( "context" "time" + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + snapshotter "github.com/kubernetes-csi/external-snapshotter/client/v7/clientset/versioned/typed/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" - - "github.com/vmware-tanzu/velero/pkg/nodeagent" - "github.com/vmware-tanzu/velero/pkg/util/boolptr" - corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" - - snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" - - "github.com/vmware-tanzu/velero/pkg/util/csi" - "github.com/vmware-tanzu/velero/pkg/util/kube" - - snapshotter "github.com/kubernetes-csi/external-snapshotter/client/v7/clientset/versioned/typed/volumesnapshot/v1" "sigs.k8s.io/controller-runtime/pkg/client" - apierrors "k8s.io/apimachinery/pkg/api/errors" + "github.com/vmware-tanzu/velero/pkg/nodeagent" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/csi" + "github.com/vmware-tanzu/velero/pkg/util/kube" ) // CSISnapshotExposeParam define the input param for Expose of CSI snapshots diff --git a/pkg/exposer/csi_snapshot_test.go b/pkg/exposer/csi_snapshot_test.go index 6fbee7450..4bed98cd1 100644 --- a/pkg/exposer/csi_snapshot_test.go +++ b/pkg/exposer/csi_snapshot_test.go @@ -160,7 +160,7 @@ func TestExpose(t *testing.T) { OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, }, - err: "error wait volume snapshot ready: error to get volumesnapshot /fake-vs: volumesnapshots.snapshot.storage.k8s.io \"fake-vs\" not found", + err: "error wait volume snapshot ready: error to get VolumeSnapshot /fake-vs: volumesnapshots.snapshot.storage.k8s.io \"fake-vs\" not found", }, { name: "get vsc fail", diff --git a/pkg/restore/actions/add_pv_from_pvc_action.go b/pkg/restore/actions/add_pv_from_pvc_action.go index e9e882d07..d597a33fd 100644 --- a/pkg/restore/actions/add_pv_from_pvc_action.go +++ b/pkg/restore/actions/add_pv_from_pvc_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "github.com/pkg/errors" diff --git a/pkg/restore/actions/add_pv_from_pvc_action_test.go b/pkg/restore/actions/add_pv_from_pvc_action_test.go index 4124fb2d8..625b35dbd 100644 --- a/pkg/restore/actions/add_pv_from_pvc_action_test.go +++ b/pkg/restore/actions/add_pv_from_pvc_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "testing" diff --git a/pkg/restore/actions/add_pvc_from_pod_action.go b/pkg/restore/actions/add_pvc_from_pod_action.go index 70f33d985..3e88f796a 100644 --- a/pkg/restore/actions/add_pvc_from_pod_action.go +++ b/pkg/restore/actions/add_pvc_from_pod_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "github.com/pkg/errors" diff --git a/pkg/restore/actions/add_pvc_from_pod_action_test.go b/pkg/restore/actions/add_pvc_from_pod_action_test.go index 5e99006ad..44b5fb4ad 100644 --- a/pkg/restore/actions/add_pvc_from_pod_action_test.go +++ b/pkg/restore/actions/add_pvc_from_pod_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "testing" diff --git a/pkg/restore/actions/admissionwebhook_config_action.go b/pkg/restore/actions/admissionwebhook_config_action.go index 74fd1b3ff..c6c967699 100644 --- a/pkg/restore/actions/admissionwebhook_config_action.go +++ b/pkg/restore/actions/admissionwebhook_config_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "github.com/pkg/errors" diff --git a/pkg/restore/actions/admissionwebhook_config_action_test.go b/pkg/restore/actions/admissionwebhook_config_action_test.go index cbc019ff0..b0f24f7f5 100644 --- a/pkg/restore/actions/admissionwebhook_config_action_test.go +++ b/pkg/restore/actions/admissionwebhook_config_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "encoding/json" diff --git a/pkg/restore/actions/apiservice_action.go b/pkg/restore/actions/apiservice_action.go index 7f817a59e..f2433352b 100644 --- a/pkg/restore/actions/apiservice_action.go +++ b/pkg/restore/actions/apiservice_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "github.com/sirupsen/logrus" diff --git a/pkg/restore/actions/apiservice_action_test.go b/pkg/restore/actions/apiservice_action_test.go index 81f4a6171..0e37e8936 100644 --- a/pkg/restore/actions/apiservice_action_test.go +++ b/pkg/restore/actions/apiservice_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "testing" diff --git a/pkg/restore/actions/change_image_name_action.go b/pkg/restore/actions/change_image_name_action.go index 0bd96b0b4..66fac1f18 100644 --- a/pkg/restore/actions/change_image_name_action.go +++ b/pkg/restore/actions/change_image_name_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "context" diff --git a/pkg/restore/actions/change_image_name_action_test.go b/pkg/restore/actions/change_image_name_action_test.go index d851e7c0d..5aa409bf4 100644 --- a/pkg/restore/actions/change_image_name_action_test.go +++ b/pkg/restore/actions/change_image_name_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "context" diff --git a/pkg/restore/actions/change_pvc_node_selector.go b/pkg/restore/actions/change_pvc_node_selector.go index 5934cd266..2888b6b2d 100644 --- a/pkg/restore/actions/change_pvc_node_selector.go +++ b/pkg/restore/actions/change_pvc_node_selector.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "context" diff --git a/pkg/restore/actions/change_pvc_node_selector_test.go b/pkg/restore/actions/change_pvc_node_selector_test.go index 7cfbd170c..615f8e10d 100644 --- a/pkg/restore/actions/change_pvc_node_selector_test.go +++ b/pkg/restore/actions/change_pvc_node_selector_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "bytes" diff --git a/pkg/restore/actions/change_storageclass_action.go b/pkg/restore/actions/change_storageclass_action.go index 91384c64e..b4052eb2d 100644 --- a/pkg/restore/actions/change_storageclass_action.go +++ b/pkg/restore/actions/change_storageclass_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "context" diff --git a/pkg/restore/actions/change_storageclass_action_test.go b/pkg/restore/actions/change_storageclass_action_test.go index 7f701a0d6..4d32a6bad 100644 --- a/pkg/restore/actions/change_storageclass_action_test.go +++ b/pkg/restore/actions/change_storageclass_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "context" diff --git a/pkg/restore/actions/clusterrolebinding_action.go b/pkg/restore/actions/clusterrolebinding_action.go index 851b13f09..072d2f4a1 100644 --- a/pkg/restore/actions/clusterrolebinding_action.go +++ b/pkg/restore/actions/clusterrolebinding_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "github.com/pkg/errors" diff --git a/pkg/restore/actions/clusterrolebinding_action_test.go b/pkg/restore/actions/clusterrolebinding_action_test.go index cea1c5787..bf9284e78 100644 --- a/pkg/restore/actions/clusterrolebinding_action_test.go +++ b/pkg/restore/actions/clusterrolebinding_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "sort" diff --git a/pkg/restore/actions/crd_v1_preserve_unknown_fields_action.go b/pkg/restore/actions/crd_v1_preserve_unknown_fields_action.go index 3f7b78241..673edd1b6 100644 --- a/pkg/restore/actions/crd_v1_preserve_unknown_fields_action.go +++ b/pkg/restore/actions/crd_v1_preserve_unknown_fields_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "encoding/json" diff --git a/pkg/restore/actions/crd_v1_preserve_unknown_fields_action_test.go b/pkg/restore/actions/crd_v1_preserve_unknown_fields_action_test.go index 63e2cea94..329cfb433 100644 --- a/pkg/restore/actions/crd_v1_preserve_unknown_fields_action_test.go +++ b/pkg/restore/actions/crd_v1_preserve_unknown_fields_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "encoding/json" diff --git a/pkg/restore/actions/csi/pvc_action.go b/pkg/restore/actions/csi/pvc_action.go new file mode 100644 index 000000000..1f75889f9 --- /dev/null +++ b/pkg/restore/actions/csi/pvc_action.go @@ -0,0 +1,616 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + "encoding/json" + "fmt" + + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + corev1api "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + utilrand "k8s.io/apimachinery/pkg/util/rand" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + "github.com/vmware-tanzu/velero/pkg/client" + "github.com/vmware-tanzu/velero/pkg/label" + plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + riav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/restoreitemaction/v2" + uploaderUtil "github.com/vmware-tanzu/velero/pkg/uploader/util" + "github.com/vmware-tanzu/velero/pkg/util" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" +) + +const ( + AnnBindCompleted = "pv.kubernetes.io/bind-completed" + AnnBoundByController = "pv.kubernetes.io/bound-by-controller" + AnnStorageProvisioner = "volume.kubernetes.io/storage-provisioner" + AnnBetaStorageProvisioner = "volume.beta.kubernetes.io/storage-provisioner" + AnnSelectedNode = "volume.kubernetes.io/selected-node" +) + +const ( + GenerateNameRandomLength = 5 +) + +// pvcRestoreItemAction is a restore item action plugin for Velero +type pvcRestoreItemAction struct { + log logrus.FieldLogger + crClient crclient.Client +} + +// AppliesTo returns information indicating that the +// PVCRestoreItemAction should be run while restoring PVCs. +func (p *pvcRestoreItemAction) AppliesTo() (velero.ResourceSelector, error) { + return velero.ResourceSelector{ + IncludedResources: []string{"persistentvolumeclaims"}, + //TODO: add label selector volumeSnapshotLabel + }, nil +} + +func removePVCAnnotations(pvc *corev1api.PersistentVolumeClaim, remove []string) { + if pvc.Annotations == nil { + pvc.Annotations = make(map[string]string) + return + } + for k := range pvc.Annotations { + if util.Contains(remove, k) { + delete(pvc.Annotations, k) + } + } +} + +func resetPVCSpec(pvc *corev1api.PersistentVolumeClaim, vsName string) { + // Restore operation for the PVC will use the VolumeSnapshot as the data source. + // So clear out the volume name, which is a ref to the PV + pvc.Spec.VolumeName = "" + dataSource := &corev1api.TypedLocalObjectReference{ + APIGroup: &snapshotv1api.SchemeGroupVersion.Group, + Kind: "VolumeSnapshot", + Name: vsName, + } + pvc.Spec.DataSource = dataSource + pvc.Spec.DataSourceRef = nil +} + +func setPVCStorageResourceRequest( + pvc *corev1api.PersistentVolumeClaim, + restoreSize resource.Quantity, + log logrus.FieldLogger, +) { + { + if pvc.Spec.Resources.Requests == nil { + pvc.Spec.Resources.Requests = corev1api.ResourceList{} + } + + storageReq, exists := pvc.Spec.Resources.Requests[corev1api.ResourceStorage] + if !exists || storageReq.Cmp(restoreSize) < 0 { + pvc.Spec.Resources.Requests[corev1api.ResourceStorage] = restoreSize + rs := pvc.Spec.Resources.Requests[corev1api.ResourceStorage] + log.Infof("Resetting storage requests for PVC %s/%s to %s", + pvc.Namespace, pvc.Name, rs.String()) + } + } +} + +// Execute modifies the PVC's spec to use the VolumeSnapshot object as the +// data source ensuring that the newly provisioned volume can be pre-populated +// with data from the VolumeSnapshot. +func (p *pvcRestoreItemAction) Execute( + input *velero.RestoreItemActionExecuteInput, +) (*velero.RestoreItemActionExecuteOutput, error) { + var pvc, pvcFromBackup corev1api.PersistentVolumeClaim + if err := runtime.DefaultUnstructuredConverter.FromUnstructured( + input.Item.UnstructuredContent(), &pvc); err != nil { + return nil, errors.WithStack(err) + } + if err := runtime.DefaultUnstructuredConverter.FromUnstructured( + input.ItemFromBackup.UnstructuredContent(), &pvcFromBackup); err != nil { + return nil, errors.WithStack(err) + } + + logger := p.log.WithFields(logrus.Fields{ + "Action": "PVCRestoreItemAction", + "PVC": pvc.Namespace + "/" + pvc.Name, + "Restore": input.Restore.Namespace + "/" + input.Restore.Name, + }) + logger.Info("Starting PVCRestoreItemAction for PVC") + + // If PVC already exists, returns early. + if p.isResourceExist(pvc, *input.Restore) { + logger.Warnf("PVC already exists. Skip restore this PVC.") + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: input.Item, + }, nil + } + + // remove the VolumeSnapshot name annotation as well + // clean the DataUploadNameLabel for snapshot data mover case. + removePVCAnnotations( + &pvc, + []string{ + AnnBindCompleted, + AnnBoundByController, + AnnStorageProvisioner, + AnnBetaStorageProvisioner, + AnnSelectedNode, + velerov1api.VolumeSnapshotLabel, + velerov1api.DataUploadNameAnnotation, + }, + ) + + // If cross-namespace restore is configured, change the namespace + // for PVC object to be restored + newNamespace, ok := input.Restore.Spec.NamespaceMapping[pvc.GetNamespace()] + if !ok { + // Use original namespace + newNamespace = pvc.Namespace + } + + operationID := "" + + if boolptr.IsSetToFalse(input.Restore.Spec.RestorePVs) { + logger.Info("Restore did not request for PVs to be restored from snapshot") + pvc.Spec.VolumeName = "" + pvc.Spec.DataSource = nil + pvc.Spec.DataSourceRef = nil + } else { + backup := new(velerov1api.Backup) + err := p.crClient.Get( + context.TODO(), + crclient.ObjectKey{ + Namespace: input.Restore.Namespace, + Name: input.Restore.Spec.BackupName, + }, + backup, + ) + + if err != nil { + logger.Error("Fail to get backup for restore.") + return nil, fmt.Errorf("fail to get backup for restore: %s", err.Error()) + } + + if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) { + logger.Info("Start DataMover restore.") + + // If PVC doesn't have a DataUploadNameLabel, which should be created + // during backup, then CSI cannot handle the volume during to restore, + // so return early to let Velero tries to fall back to Velero native snapshot. + if _, ok := pvcFromBackup.Annotations[velerov1api.DataUploadNameAnnotation]; !ok { + logger.Warnf("PVC doesn't have a DataUpload for data mover. Return.") + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: input.Item, + }, nil + } + + operationID = label.GetValidName( + string(velerov1api.AsyncOperationIDPrefixDataDownload) + + string(input.Restore.UID) + "." + string(pvcFromBackup.UID)) + dataDownload, err := restoreFromDataUploadResult( + context.Background(), input.Restore, backup, &pvc, newNamespace, + operationID, p.crClient) + if err != nil { + logger.Errorf("Fail to restore from DataUploadResult: %s", err.Error()) + return nil, errors.WithStack(err) + } + logger.Infof("DataDownload %s/%s is created successfully.", + dataDownload.Namespace, dataDownload.Name) + } else { + volumeSnapshotName, ok := pvcFromBackup.Annotations[velerov1api.VolumeSnapshotLabel] + if !ok { + logger.Info("Skipping PVCRestoreItemAction for PVC,", + "PVC does not have a CSI VolumeSnapshot.") + // Make no change in the input PVC. + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: input.Item, + }, nil + } + if err := restoreFromVolumeSnapshot( + &pvc, newNamespace, p.crClient, volumeSnapshotName, logger, + ); err != nil { + logger.Errorf("Failed to restore PVC from VolumeSnapshot.") + return nil, errors.WithStack(err) + } + } + } + + pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pvc) + if err != nil { + return nil, errors.WithStack(err) + } + logger.Info("Returning from PVCRestoreItemAction for PVC") + + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: &unstructured.Unstructured{Object: pvcMap}, + OperationID: operationID, + }, nil +} + +func (p *pvcRestoreItemAction) Name() string { + return "PVCRestoreItemAction" +} + +func (p *pvcRestoreItemAction) Progress( + operationID string, + restore *velerov1api.Restore, +) (velero.OperationProgress, error) { + progress := velero.OperationProgress{} + + if operationID == "" { + return progress, riav2.InvalidOperationIDError(operationID) + } + logger := p.log.WithFields(logrus.Fields{ + "Action": "PVCRestoreItemAction", + "OperationID": operationID, + "Namespace": restore.Namespace, + }) + + dataDownload, err := getDataDownload( + context.Background(), + restore.Namespace, + operationID, + p.crClient, + ) + if err != nil { + logger.Errorf("fail to get DataDownload: %s", err.Error()) + return progress, err + } + if dataDownload.Status.Phase == velerov2alpha1.DataDownloadPhaseNew || + dataDownload.Status.Phase == "" { + logger.Debugf("DataDownload is still not processed yet. Skip progress update.") + return progress, nil + } + + progress.Description = string(dataDownload.Status.Phase) + progress.OperationUnits = "Bytes" + progress.NCompleted = dataDownload.Status.Progress.BytesDone + progress.NTotal = dataDownload.Status.Progress.TotalBytes + + if dataDownload.Status.StartTimestamp != nil { + progress.Started = dataDownload.Status.StartTimestamp.Time + } + + if dataDownload.Status.CompletionTimestamp != nil { + progress.Updated = dataDownload.Status.CompletionTimestamp.Time + } + + if dataDownload.Status.Phase == velerov2alpha1.DataDownloadPhaseCompleted { + progress.Completed = true + } else if dataDownload.Status.Phase == velerov2alpha1.DataDownloadPhaseCanceled { + progress.Completed = true + progress.Err = "DataDownload is canceled" + } else if dataDownload.Status.Phase == velerov2alpha1.DataDownloadPhaseFailed { + progress.Completed = true + progress.Err = dataDownload.Status.Message + } + + return progress, nil +} + +func (p *pvcRestoreItemAction) Cancel( + operationID string, restore *velerov1api.Restore) error { + if operationID == "" { + return riav2.InvalidOperationIDError(operationID) + } + logger := p.log.WithFields(logrus.Fields{ + "Action": "PVCRestoreItemAction", + "OperationID": operationID, + "Namespace": restore.Namespace, + }) + + dataDownload, err := getDataDownload( + context.Background(), + restore.Namespace, + operationID, + p.crClient, + ) + if err != nil { + logger.Errorf("fail to get DataDownload: %s", err.Error()) + return err + } + + err = cancelDataDownload(context.Background(), p.crClient, dataDownload) + if err != nil { + logger.Errorf("fail to cancel DataDownload %s: %s", dataDownload.Name, err.Error()) + } + return err +} + +func (p *pvcRestoreItemAction) AreAdditionalItemsReady( + additionalItems []velero.ResourceIdentifier, + restore *velerov1api.Restore, +) (bool, error) { + return true, nil +} + +func getDataUploadResult( + ctx context.Context, + restore *velerov1api.Restore, + pvc *corev1api.PersistentVolumeClaim, + crClient crclient.Client, +) (*velerov2alpha1.DataUploadResult, error) { + selectorStr := fmt.Sprintf("%s=%s,%s=%s,%s=%s", + velerov1api.PVCNamespaceNameLabel, + label.GetValidName(pvc.Namespace+"."+pvc.Name), + velerov1api.RestoreUIDLabel, + label.GetValidName(string(restore.UID)), + velerov1api.ResourceUsageLabel, + label.GetValidName(string(velerov1api.VeleroResourceUsageDataUploadResult)), + ) + selector, _ := labels.Parse(selectorStr) + + cmList := new(corev1api.ConfigMapList) + if err := crClient.List( + ctx, + cmList, + &crclient.ListOptions{ + LabelSelector: selector, + Namespace: restore.Namespace, + }); err != nil { + return nil, errors.Wrapf(err, + "error to get DataUpload result cm with labels %s", selectorStr) + } + + if len(cmList.Items) == 0 { + return nil, errors.Errorf( + "no DataUpload result cm found with labels %s", selectorStr) + } + + if len(cmList.Items) > 1 { + return nil, errors.Errorf( + "multiple DataUpload result cms found with labels %s", selectorStr) + } + + jsonBytes, exist := cmList.Items[0].Data[string(restore.UID)] + if !exist { + return nil, errors.Errorf( + "no DataUpload result found with restore key %s, restore %s", + string(restore.UID), restore.Name) + } + + result := velerov2alpha1.DataUploadResult{} + if err := json.Unmarshal([]byte(jsonBytes), &result); err != nil { + return nil, errors.Errorf( + "error to unmarshal DataUploadResult, restore UID %s, restore name %s", + string(restore.UID), restore.Name) + } + + return &result, nil +} + +func getDataDownload( + ctx context.Context, + namespace string, + operationID string, + crClient crclient.Client, +) (*velerov2alpha1.DataDownload, error) { + dataDownloadList := new(velerov2alpha1.DataDownloadList) + err := crClient.List(ctx, dataDownloadList, &crclient.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{ + velerov1api.AsyncOperationIDLabel: operationID, + }), + Namespace: namespace, + }) + if err != nil { + return nil, errors.Wrap(err, "fail to list DataDownload") + } + + if len(dataDownloadList.Items) == 0 { + return nil, errors.Errorf("didn't find DataDownload") + } + + if len(dataDownloadList.Items) > 1 { + return nil, errors.Errorf("find multiple DataDownloads") + } + + return &dataDownloadList.Items[0], nil +} + +func cancelDataDownload(ctx context.Context, crClient crclient.Client, + dataDownload *velerov2alpha1.DataDownload) error { + updatedDataDownload := dataDownload.DeepCopy() + updatedDataDownload.Spec.Cancel = true + + return crClient.Patch(ctx, updatedDataDownload, crclient.MergeFrom(dataDownload)) +} + +func newDataDownload( + restore *velerov1api.Restore, + backup *velerov1api.Backup, + dataUploadResult *velerov2alpha1.DataUploadResult, + pvc *corev1api.PersistentVolumeClaim, + newNamespace, operationID string, +) *velerov2alpha1.DataDownload { + dataDownload := &velerov2alpha1.DataDownload{ + TypeMeta: metav1.TypeMeta{ + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + Kind: "DataDownload", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: restore.Namespace, + GenerateName: restore.Name + "-", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: velerov1api.SchemeGroupVersion.String(), + Kind: "Restore", + Name: restore.Name, + UID: restore.UID, + Controller: boolptr.True(), + }, + }, + Labels: map[string]string{ + velerov1api.RestoreNameLabel: label.GetValidName(restore.Name), + velerov1api.RestoreUIDLabel: string(restore.UID), + velerov1api.AsyncOperationIDLabel: operationID, + }, + }, + Spec: velerov2alpha1.DataDownloadSpec{ + TargetVolume: velerov2alpha1.TargetVolumeSpec{ + PVC: pvc.Name, + Namespace: newNamespace, + }, + BackupStorageLocation: dataUploadResult.BackupStorageLocation, + DataMover: dataUploadResult.DataMover, + SnapshotID: dataUploadResult.SnapshotID, + SourceNamespace: dataUploadResult.SourceNamespace, + OperationTimeout: backup.Spec.CSISnapshotTimeout, + }, + } + if restore.Spec.UploaderConfig != nil { + dataDownload.Spec.DataMoverConfig = make(map[string]string) + if boolptr.IsSetToTrue(restore.Spec.UploaderConfig.WriteSparseFiles) { + dataDownload.Spec.DataMoverConfig[uploaderUtil.WriteSparseFiles] = "true" + } else { + dataDownload.Spec.DataMoverConfig[uploaderUtil.WriteSparseFiles] = "false" + } + } + return dataDownload +} + +func restoreFromVolumeSnapshot( + pvc *corev1api.PersistentVolumeClaim, + newNamespace string, + crClient crclient.Client, + volumeSnapshotName string, + logger logrus.FieldLogger, +) error { + vs := new(snapshotv1api.VolumeSnapshot) + if err := crClient.Get(context.TODO(), + crclient.ObjectKey{ + Namespace: newNamespace, + Name: volumeSnapshotName, + }, + vs, + ); err != nil { + return errors.Wrapf(err, + fmt.Sprintf("Failed to get Volumesnapshot %s/%s to restore PVC %s/%s", + newNamespace, volumeSnapshotName, newNamespace, pvc.Name), + ) + } + + if _, exists := vs.Annotations[velerov1api.VolumeSnapshotRestoreSize]; exists { + restoreSize, err := resource.ParseQuantity( + vs.Annotations[velerov1api.VolumeSnapshotRestoreSize]) + if err != nil { + return errors.Wrapf(err, fmt.Sprintf( + "Failed to parse %s from annotation on Volumesnapshot %s/%s into restore size", + vs.Annotations[velerov1api.VolumeSnapshotRestoreSize], vs.Namespace, vs.Name)) + } + // It is possible that the volume provider allocated a larger + // capacity volume than what was requested in the backed up PVC. + // In this scenario the volumesnapshot of the PVC will end being + // larger than its requested storage size. Such a PVC, on restore + // as-is, will be stuck attempting to use a VolumeSnapshot as a + // data source for a PVC that is not large enough. + // To counter that, here we set the storage request on the PVC + // to the larger of the PVC's storage request and the size of the + // VolumeSnapshot + setPVCStorageResourceRequest(pvc, restoreSize, logger) + } + + resetPVCSpec(pvc, volumeSnapshotName) + + return nil +} + +func restoreFromDataUploadResult( + ctx context.Context, + restore *velerov1api.Restore, + backup *velerov1api.Backup, + pvc *corev1api.PersistentVolumeClaim, + newNamespace, operationID string, + crClient crclient.Client, +) (*velerov2alpha1.DataDownload, error) { + dataUploadResult, err := getDataUploadResult(ctx, restore, pvc, crClient) + if err != nil { + return nil, errors.Wrapf(err, "fail get DataUploadResult for restore: %s", + restore.Name) + } + pvc.Spec.VolumeName = "" + if pvc.Spec.Selector == nil { + pvc.Spec.Selector = &metav1.LabelSelector{} + } + if pvc.Spec.Selector.MatchLabels == nil { + pvc.Spec.Selector.MatchLabels = make(map[string]string) + } + pvc.Spec.Selector.MatchLabels[velerov1api.DynamicPVRestoreLabel] = label. + GetValidName(fmt.Sprintf("%s.%s.%s", newNamespace, + pvc.Name, utilrand.String(GenerateNameRandomLength))) + + dataDownload := newDataDownload( + restore, + backup, + dataUploadResult, + pvc, + newNamespace, + operationID, + ) + err = crClient.Create(ctx, dataDownload) + if err != nil { + return nil, errors.Wrapf(err, "fail to create DataDownload") + } + + return dataDownload, nil +} + +func (p *pvcRestoreItemAction) isResourceExist( + pvc corev1api.PersistentVolumeClaim, + restore velerov1api.Restore, +) bool { + // get target namespace to restore into, if different from source namespace + targetNamespace := pvc.Namespace + if target, ok := restore.Spec.NamespaceMapping[pvc.Namespace]; ok { + targetNamespace = target + } + + tmpPVC := new(corev1api.PersistentVolumeClaim) + if err := p.crClient.Get( + context.Background(), + crclient.ObjectKey{ + Name: pvc.Name, + Namespace: targetNamespace, + }, + tmpPVC, + ); err == nil { + return true + } + return false +} + +func NewPvcRestoreItemAction(f client.Factory) plugincommon.HandlerInitializer { + return func(logger logrus.FieldLogger) (interface{}, error) { + crClient, err := f.KubebuilderClient() + if err != nil { + return nil, err + } + + return &pvcRestoreItemAction{ + log: logger, + crClient: crClient, + }, nil + } +} diff --git a/pkg/restore/actions/csi/pvc_action_test.go b/pkg/restore/actions/csi/pvc_action_test.go new file mode 100644 index 000000000..4ba8ca83e --- /dev/null +++ b/pkg/restore/actions/csi/pvc_action_test.go @@ -0,0 +1,713 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1api "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + "github.com/vmware-tanzu/velero/pkg/builder" + "github.com/vmware-tanzu/velero/pkg/label" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + velerotest "github.com/vmware-tanzu/velero/pkg/test" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" +) + +func TestRemovePVCAnnotations(t *testing.T) { + testCases := []struct { + name string + pvc corev1api.PersistentVolumeClaim + removeAnnotations []string + expectedAnnotations map[string]string + }{ + { + name: "should create empty annotation map", + pvc: corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: nil, + }, + }, + removeAnnotations: []string{"foo"}, + expectedAnnotations: map[string]string{}, + }, + { + name: "should preserve all existing annotations", + pvc: corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "ann1": "ann1-val", + "ann2": "ann2-val", + "ann3": "ann3-val", + "ann4": "ann4-val", + }, + }, + }, + removeAnnotations: []string{}, + expectedAnnotations: map[string]string{ + "ann1": "ann1-val", + "ann2": "ann2-val", + "ann3": "ann3-val", + "ann4": "ann4-val", + }, + }, + { + name: "should remove all existing annotations", + pvc: corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "ann1": "ann1-val", + "ann2": "ann2-val", + "ann3": "ann3-val", + "ann4": "ann4-val", + }, + }, + }, + removeAnnotations: []string{"ann1", "ann2", "ann3", "ann4"}, + expectedAnnotations: map[string]string{}, + }, + { + name: "should preserve some existing annotations", + pvc: corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "ann1": "ann1-val", + "ann2": "ann2-val", + "ann3": "ann3-val", + "ann4": "ann4-val", + "ann5": "ann5-val", + "ann6": "ann6-val", + "ann7": "ann7-val", + "ann8": "ann8-val", + }, + }, + }, + removeAnnotations: []string{"ann1", "ann2", "ann3", "ann4"}, + expectedAnnotations: map[string]string{ + "ann5": "ann5-val", + "ann6": "ann6-val", + "ann7": "ann7-val", + "ann8": "ann8-val", + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + removePVCAnnotations(&tc.pvc, tc.removeAnnotations) + assert.Equal(t, tc.expectedAnnotations, tc.pvc.Annotations) + }) + } +} + +func TestResetPVCSpec(t *testing.T) { + fileMode := corev1api.PersistentVolumeFilesystem + blockMode := corev1api.PersistentVolumeBlock + + testCases := []struct { + name string + pvc corev1api.PersistentVolumeClaim + vsName string + }{ + { + name: "should reset expected fields in pvc using file mode volumes", + pvc: corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pvc", + Namespace: "test-ns", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadOnlyMany, corev1api.ReadWriteMany, corev1api.ReadWriteOnce}, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "foo": "bar", + "baz": "qux", + }, + }, + Resources: corev1api.VolumeResourceRequirements{ + Requests: corev1api.ResourceList{ + corev1api.ResourceCPU: resource.Quantity{ + Format: resource.DecimalExponent, + }, + }, + }, + VolumeName: "should-be-removed", + VolumeMode: &fileMode, + }, + }, + vsName: "test-vs", + }, + { + name: "should reset expected fields in pvc using block mode volumes", + pvc: corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pvc", + Namespace: "test-ns", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadOnlyMany, corev1api.ReadWriteMany, corev1api.ReadWriteOnce}, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "foo": "bar", + "baz": "qux", + }, + }, + Resources: corev1api.VolumeResourceRequirements{ + Requests: corev1api.ResourceList{ + corev1api.ResourceCPU: resource.Quantity{ + Format: resource.DecimalExponent, + }, + }, + }, + VolumeName: "should-be-removed", + VolumeMode: &blockMode, + }, + }, + vsName: "test-vs", + }, + { + name: "should overwrite existing DataSource per reset parameters", + pvc: corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pvc", + Namespace: "test-ns", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadOnlyMany, corev1api.ReadWriteMany, corev1api.ReadWriteOnce}, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "foo": "bar", + "baz": "qux", + }, + }, + Resources: corev1api.VolumeResourceRequirements{ + Requests: corev1api.ResourceList{ + corev1api.ResourceCPU: resource.Quantity{ + Format: resource.DecimalExponent, + }, + }, + }, + VolumeName: "should-be-removed", + VolumeMode: &fileMode, + DataSource: &corev1api.TypedLocalObjectReference{ + Kind: "something-that-does-not-exist", + Name: "not-found", + }, + DataSourceRef: &corev1api.TypedObjectReference{ + Kind: "something-that-does-not-exist", + Name: "not-found", + }, + }, + }, + vsName: "test-vs", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + before := tc.pvc.DeepCopy() + resetPVCSpec(&tc.pvc, tc.vsName) + + assert.Equalf(t, tc.pvc.Name, before.Name, "unexpected change to Object.Name, Want: %s; Got %s", before.Name, tc.pvc.Name) + assert.Equalf(t, tc.pvc.Namespace, before.Namespace, "unexpected change to Object.Namespace, Want: %s; Got %s", before.Namespace, tc.pvc.Namespace) + assert.Equalf(t, tc.pvc.Spec.AccessModes, before.Spec.AccessModes, "unexpected Spec.AccessModes, Want: %v; Got: %v", before.Spec.AccessModes, tc.pvc.Spec.AccessModes) + assert.Equalf(t, tc.pvc.Spec.Selector, before.Spec.Selector, "unexpected change to Spec.Selector, Want: %s; Got: %s", before.Spec.Selector.String(), tc.pvc.Spec.Selector.String()) + assert.Equalf(t, tc.pvc.Spec.Resources, before.Spec.Resources, "unexpected change to Spec.Resources, Want: %s; Got: %s", before.Spec.Resources.String(), tc.pvc.Spec.Resources.String()) + assert.Emptyf(t, tc.pvc.Spec.VolumeName, "expected change to Spec.VolumeName missing, Want: \"\"; Got: %s", tc.pvc.Spec.VolumeName) + assert.Equalf(t, *tc.pvc.Spec.VolumeMode, *before.Spec.VolumeMode, "expected change to Spec.VolumeName missing, Want: \"\"; Got: %s", tc.pvc.Spec.VolumeName) + assert.NotNil(t, tc.pvc.Spec.DataSource, "expected change to Spec.DataSource missing") + assert.Equalf(t, tc.pvc.Spec.DataSource.Kind, "VolumeSnapshot", "expected change to Spec.DataSource.Kind missing, Want: VolumeSnapshot, Got: %s", tc.pvc.Spec.DataSource.Kind) + assert.Equalf(t, tc.pvc.Spec.DataSource.Name, tc.vsName, "expected change to Spec.DataSource.Name missing, Want: %s, Got: %s", tc.vsName, tc.pvc.Spec.DataSource.Name) + }) + } +} + +func TestResetPVCResourceRequest(t *testing.T) { + var storageReq50Mi, storageReq1Gi, cpuQty resource.Quantity + + storageReq50Mi, err := resource.ParseQuantity("50Mi") + assert.NoError(t, err) + storageReq1Gi, err = resource.ParseQuantity("1Gi") + assert.NoError(t, err) + cpuQty, err = resource.ParseQuantity("100m") + assert.NoError(t, err) + + testCases := []struct { + name string + pvc corev1api.PersistentVolumeClaim + restoreSize resource.Quantity + expectedStorageRequestQty string + }{ + { + name: "should set storage resource request from volumesnapshot, pvc has nil resource requests", + pvc: corev1api.PersistentVolumeClaim{ + Spec: corev1api.PersistentVolumeClaimSpec{ + Resources: corev1api.VolumeResourceRequirements{ + Requests: nil, + }, + }, + }, + restoreSize: storageReq50Mi, + expectedStorageRequestQty: "50Mi", + }, + { + name: "should set storage resource request from volumesnapshot, pvc has empty resource requests", + pvc: corev1api.PersistentVolumeClaim{ + Spec: corev1api.PersistentVolumeClaimSpec{ + Resources: corev1api.VolumeResourceRequirements{ + Requests: corev1api.ResourceList{}, + }, + }, + }, + restoreSize: storageReq50Mi, + expectedStorageRequestQty: "50Mi", + }, + { + name: "should merge resource requests from volumesnapshot into pvc with no storage resource requests", + pvc: corev1api.PersistentVolumeClaim{ + Spec: corev1api.PersistentVolumeClaimSpec{ + Resources: corev1api.VolumeResourceRequirements{ + Requests: corev1api.ResourceList{ + corev1api.ResourceCPU: cpuQty, + }, + }, + }, + }, + restoreSize: storageReq50Mi, + expectedStorageRequestQty: "50Mi", + }, + { + name: "should set storage resource request from volumesnapshot, pvc requests less storage", + pvc: corev1api.PersistentVolumeClaim{ + Spec: corev1api.PersistentVolumeClaimSpec{ + Resources: corev1api.VolumeResourceRequirements{ + Requests: corev1api.ResourceList{ + corev1api.ResourceStorage: storageReq50Mi, + }, + }, + }, + }, + restoreSize: storageReq1Gi, + expectedStorageRequestQty: "1Gi", + }, + { + name: "should not set storage resource request from volumesnapshot, pvc requests more storage", + pvc: corev1api.PersistentVolumeClaim{ + Spec: corev1api.PersistentVolumeClaimSpec{ + Resources: corev1api.VolumeResourceRequirements{ + Requests: corev1api.ResourceList{ + corev1api.ResourceStorage: storageReq1Gi, + }, + }, + }, + }, + restoreSize: storageReq50Mi, + expectedStorageRequestQty: "1Gi", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + log := logrus.New().WithField("unit-test", tc.name) + setPVCStorageResourceRequest(&tc.pvc, tc.restoreSize, log) + expected, err := resource.ParseQuantity(tc.expectedStorageRequestQty) + assert.NoError(t, err) + assert.Equal(t, expected, tc.pvc.Spec.Resources.Requests[corev1api.ResourceStorage]) + }) + } +} + +func TestProgress(t *testing.T) { + currentTime := time.Now() + tests := []struct { + name string + restore *velerov1api.Restore + dataDownload *velerov2alpha1.DataDownload + operationID string + expectedErr string + expectedProgress velero.OperationProgress + }{ + { + name: "DataDownload cannot be found", + restore: builder.ForRestore("velero", "test").Result(), + operationID: "testing", + expectedErr: "didn't find DataDownload", + }, + { + name: "DataDownload is not in the expected namespace", + restore: builder.ForRestore("velero", "test").Result(), + dataDownload: &velerov2alpha1.DataDownload{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataUpload", + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "invalid-namespace", + Name: "testing", + Labels: map[string]string{ + velerov1api.AsyncOperationIDLabel: "testing", + }, + }, + }, + operationID: "testing", + expectedErr: "didn't find DataDownload", + }, + { + name: "DataUpload is found", + restore: builder.ForRestore("velero", "test").Result(), + dataDownload: &velerov2alpha1.DataDownload{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataUpload", + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "velero", + Name: "testing", + Labels: map[string]string{ + velerov1api.AsyncOperationIDLabel: "testing", + }, + }, + Status: velerov2alpha1.DataDownloadStatus{ + Phase: velerov2alpha1.DataDownloadPhaseFailed, + Progress: shared.DataMoveOperationProgress{ + BytesDone: 1000, + TotalBytes: 1000, + }, + StartTimestamp: &metav1.Time{Time: currentTime}, + CompletionTimestamp: &metav1.Time{Time: currentTime}, + Message: "Testing error", + }, + }, + operationID: "testing", + expectedProgress: velero.OperationProgress{ + Completed: true, + Err: "Testing error", + NCompleted: 1000, + NTotal: 1000, + OperationUnits: "Bytes", + Description: "Failed", + Started: currentTime, + Updated: currentTime, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(*testing.T) { + pvcRIA := pvcRestoreItemAction{ + log: logrus.New(), + crClient: velerotest.NewFakeControllerRuntimeClient(t), + } + if tc.dataDownload != nil { + err := pvcRIA.crClient.Create(context.Background(), tc.dataDownload) + require.NoError(t, err) + } + + progress, err := pvcRIA.Progress(tc.operationID, tc.restore) + if tc.expectedErr != "" { + require.Equal(t, tc.expectedErr, err.Error()) + return + } + + require.NoError(t, err) + require.True(t, cmp.Equal(tc.expectedProgress, progress, cmpopts.IgnoreFields(velero.OperationProgress{}, "Started", "Updated"))) + }) + } +} + +func TestCancel(t *testing.T) { + tests := []struct { + name string + restore *velerov1api.Restore + dataDownload *velerov2alpha1.DataDownload + operationID string + expectedErr string + expectedDataDownload velerov2alpha1.DataDownload + }{ + { + name: "Cancel DataUpload", + restore: builder.ForRestore("velero", "test").Result(), + dataDownload: &velerov2alpha1.DataDownload{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataDownload", + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "velero", + Name: "testing", + Labels: map[string]string{ + velerov1api.AsyncOperationIDLabel: "testing", + }, + }, + }, + operationID: "testing", + expectedErr: "", + expectedDataDownload: velerov2alpha1.DataDownload{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataDownload", + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "velero", + Name: "testing", + Labels: map[string]string{ + velerov1api.AsyncOperationIDLabel: "testing", + }, + }, + Spec: velerov2alpha1.DataDownloadSpec{ + Cancel: true, + }, + }, + }, + { + name: "Cannot find DataUpload", + restore: builder.ForRestore("velero", "test").Result(), + dataDownload: nil, + operationID: "testing", + expectedErr: "didn't find DataDownload", + expectedDataDownload: velerov2alpha1.DataDownload{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataDownload", + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "velero", + Name: "testing", + Labels: map[string]string{ + velerov1api.AsyncOperationIDLabel: "testing", + }, + }, + Spec: velerov2alpha1.DataDownloadSpec{ + Cancel: true, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(*testing.T) { + pvcRIA := pvcRestoreItemAction{ + log: logrus.New(), + crClient: velerotest.NewFakeControllerRuntimeClient(t), + } + if tc.dataDownload != nil { + err := pvcRIA.crClient.Create(context.Background(), tc.dataDownload) + require.NoError(t, err) + } + + err := pvcRIA.Cancel(tc.operationID, tc.restore) + if tc.expectedErr != "" { + require.Equal(t, tc.expectedErr, err.Error()) + return + } + require.NoError(t, err) + + resultDataDownload := new(velerov2alpha1.DataDownload) + err = pvcRIA.crClient.Get(context.Background(), crclient.ObjectKey{Namespace: tc.dataDownload.Namespace, Name: tc.dataDownload.Name}, resultDataDownload) + require.NoError(t, err) + + require.True(t, cmp.Equal(tc.expectedDataDownload, *resultDataDownload, cmpopts.IgnoreFields(velerov2alpha1.DataDownload{}, "ResourceVersion", "Name"))) + }) + } +} + +func TestExecute(t *testing.T) { + tests := []struct { + name string + backup *velerov1api.Backup + restore *velerov1api.Restore + pvc *corev1api.PersistentVolumeClaim + vs *snapshotv1api.VolumeSnapshot + dataUploadResult *corev1api.ConfigMap + expectedErr string + expectedDataDownload *velerov2alpha1.DataDownload + expectedPVC *corev1api.PersistentVolumeClaim + preCreatePVC bool + }{ + { + name: "Don't restore PV", + restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").RestorePVs(false).Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").Result(), + expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("").Result(), + }, + { + name: "restore's backup cannot be found", + restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").Result(), + expectedErr: "fail to get backup for restore: backups.velero.io \"testBackup\" not found", + }, + { + name: "VolumeSnapshot cannot be found", + backup: builder.ForBackup("velero", "testBackup").Result(), + restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "testVS")).Result(), + expectedErr: "Failed to get Volumesnapshot velero/testVS to restore PVC velero/testPVC: volumesnapshots.snapshot.storage.k8s.io \"testVS\" not found", + }, + { + name: "Restore from VolumeSnapshot", + backup: builder.ForBackup("velero", "testBackup").Result(), + restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "testVS")). + RequestResource(map[corev1api.ResourceName]resource.Quantity{corev1api.ResourceStorage: resource.MustParse("10Gi")}). + DataSource(&corev1api.TypedLocalObjectReference{APIGroup: &snapshotv1api.SchemeGroupVersion.Group, Kind: "VolumeSnapshot", Name: "testVS"}). + DataSourceRef(&corev1api.TypedObjectReference{APIGroup: &snapshotv1api.SchemeGroupVersion.Group, Kind: "VolumeSnapshot", Name: "testVS"}). + Result(), + vs: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi")).Result(), + expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").Result(), + }, + { + name: "Restore from VolumeSnapshot without volume-snapshot-name annotation", + backup: builder.ForBackup("velero", "testBackup").Result(), + restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(AnnSelectedNode, "node1")).Result(), + vs: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi")).Result(), + expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(AnnSelectedNode, "node1")).Result(), + }, + { + name: "DataUploadResult cannot be found", + backup: builder.ForBackup("velero", "testBackup").SnapshotMoveData(true).Result(), + restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(), + expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").Result(), + expectedErr: "fail get DataUploadResult for restore: testRestore: no DataUpload result cm found with labels velero.io/pvc-namespace-name=velero.testPVC,velero.io/restore-uid=,velero.io/resource-usage=DataUpload", + }, + { + name: "Restore from DataUploadResult", + backup: builder.ForBackup("velero", "testBackup").SnapshotMoveData(true).Result(), + restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").ObjectMeta(builder.WithUID("uid")).Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(), + dataUploadResult: builder.ForConfigMap("velero", "testCM").Data("uid", "{}").ObjectMeta(builder.WithLabels(velerov1api.RestoreUIDLabel, "uid", velerov1api.PVCNamespaceNameLabel, "velero.testPVC", velerov1api.ResourceUsageLabel, label.GetValidName(string(velerov1api.VeleroResourceUsageDataUploadResult)))).Result(), + expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations("velero.io/csi-volumesnapshot-restore-size", "10Gi")).Result(), + expectedDataDownload: builder.ForDataDownload("velero", "name").TargetVolume(velerov2alpha1.TargetVolumeSpec{PVC: "testPVC", Namespace: "velero"}). + ObjectMeta(builder.WithOwnerReference([]metav1.OwnerReference{{APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "Restore", Name: "testRestore", UID: "uid", Controller: boolptr.True()}}), + builder.WithLabelsMap(map[string]string{velerov1api.AsyncOperationIDLabel: "dd-uid.", velerov1api.RestoreNameLabel: "testRestore", velerov1api.RestoreUIDLabel: "uid"}), + builder.WithGenerateName("testRestore-")).Result(), + }, + { + name: "Restore from DataUploadResult with long source PVC namespace and name", + backup: builder.ForBackup("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1", "testBackup").SnapshotMoveData(true).Result(), + restore: builder.ForRestore("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1", "testRestore").Backup("testBackup").ObjectMeta(builder.WithUID("uid")).Result(), + pvc: builder.ForPersistentVolumeClaim("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1", "kibishii-data-kibishii-deployment-0").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(), + dataUploadResult: builder.ForConfigMap("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1", "testCM").Data("uid", "{}").ObjectMeta(builder.WithLabels(velerov1api.RestoreUIDLabel, "uid", velerov1api.PVCNamespaceNameLabel, "migre209d0da-49c7-45ba-8d5a-3e59fd591ec1.kibishii-data-ki152333", velerov1api.ResourceUsageLabel, label.GetValidName(string(velerov1api.VeleroResourceUsageDataUploadResult)))).Result(), + expectedPVC: builder.ForPersistentVolumeClaim("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1", "kibishii-data-kibishii-deployment-0").ObjectMeta(builder.WithAnnotations("velero.io/csi-volumesnapshot-restore-size", "10Gi")).Result(), + }, + { + name: "PVC had no DataUploadNameLabel annotation", + backup: builder.ForBackup("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1", "testBackup").SnapshotMoveData(true).Result(), + restore: builder.ForRestore("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1", "testRestore").Backup("testBackup").ObjectMeta(builder.WithUID("uid")).Result(), + pvc: builder.ForPersistentVolumeClaim("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1", "kibishii-data-kibishii-deployment-0").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi")).Result(), + }, + { + name: "Restore a PVC that already exists.", + backup: builder.ForBackup("velero", "testBackup").SnapshotMoveData(true).Result(), + restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").ObjectMeta(builder.WithUID("uid")).Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(), + preCreatePVC: true, + }, + { + name: "Restore a PVC that already exists in the mapping namespace", + backup: builder.ForBackup("velero", "testBackup").SnapshotMoveData(true).Result(), + restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").NamespaceMappings("velero", "restore").ObjectMeta(builder.WithUID("uid")).Result(), + pvc: builder.ForPersistentVolumeClaim("restore", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(), + preCreatePVC: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(*testing.T) { + object := make([]runtime.Object, 0) + if tc.backup != nil { + object = append(object, tc.backup) + } + + if tc.vs != nil { + object = append(object, tc.vs) + } + + input := new(velero.RestoreItemActionExecuteInput) + + if tc.pvc != nil { + pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pvc) + require.NoError(t, err) + + input.Item = &unstructured.Unstructured{Object: pvcMap} + input.ItemFromBackup = &unstructured.Unstructured{Object: pvcMap} + input.Restore = tc.restore + } + if tc.preCreatePVC { + object = append(object, tc.pvc) + } + + if tc.dataUploadResult != nil { + object = append(object, tc.dataUploadResult) + } + + pvcRIA := pvcRestoreItemAction{ + log: logrus.New(), + crClient: velerotest.NewFakeControllerRuntimeClient(t, object...), + } + + output, err := pvcRIA.Execute(input) + if tc.expectedErr != "" { + require.Equal(t, tc.expectedErr, err.Error()) + return + } + require.NoError(t, err) + + if tc.expectedPVC != nil { + pvc := new(corev1api.PersistentVolumeClaim) + err := runtime.DefaultUnstructuredConverter.FromUnstructured(output.UpdatedItem.UnstructuredContent(), pvc) + require.NoError(t, err) + require.Equal(t, tc.expectedPVC.GetObjectMeta(), pvc.GetObjectMeta()) + if pvc.Spec.Selector != nil && pvc.Spec.Selector.MatchLabels != nil { + // This is used for long name and namespace case. + if len(tc.pvc.Namespace+"."+tc.pvc.Name) >= validation.DNS1035LabelMaxLength { + require.Contains(t, pvc.Spec.Selector.MatchLabels[velerov1api.DynamicPVRestoreLabel], label.GetValidName(tc.pvc.Namespace + "." + tc.pvc.Name)[:56]) + } else { + require.Contains(t, pvc.Spec.Selector.MatchLabels[velerov1api.DynamicPVRestoreLabel], tc.pvc.Namespace+"."+tc.pvc.Name) + } + } + } + if tc.expectedDataDownload != nil { + dataDownloadList := new(velerov2alpha1.DataDownloadList) + err := pvcRIA.crClient.List(context.Background(), dataDownloadList, &crclient.ListOptions{ + LabelSelector: labels.SelectorFromSet(tc.expectedDataDownload.Labels), + }) + require.NoError(t, err) + require.True(t, cmp.Equal(tc.expectedDataDownload, &dataDownloadList.Items[0], cmpopts.IgnoreFields(velerov2alpha1.DataDownload{}, "ResourceVersion", "Name"))) + } + }) + } +} diff --git a/pkg/restore/actions/csi/volumesnapshot_action.go b/pkg/restore/actions/csi/volumesnapshot_action.go new file mode 100644 index 000000000..6f8326c22 --- /dev/null +++ b/pkg/restore/actions/csi/volumesnapshot_action.go @@ -0,0 +1,217 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + core_v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/client" + "github.com/vmware-tanzu/velero/pkg/label" + plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/csi" +) + +// volumeSnapshotRestoreItemAction is a Velero restore item +// action plugin for VolumeSnapshots +type volumeSnapshotRestoreItemAction struct { + log logrus.FieldLogger + crClient crclient.Client +} + +// AppliesTo returns information indicating that +// VolumeSnapshotRestoreItemAction should be invoked while +// restoring volumesnapshots.snapshot.storage.k8s.io resources. +func (p *volumeSnapshotRestoreItemAction) AppliesTo() ( + velero.ResourceSelector, error) { + return velero.ResourceSelector{ + IncludedResources: []string{"volumesnapshots.snapshot.storage.k8s.io"}, + }, nil +} + +func resetVolumeSnapshotSpecForRestore( + vs *snapshotv1api.VolumeSnapshot, vscName *string) { + // Spec of the backed-up object used the PVC as the source + // of the volumeSnapshot. Restore operation will however, + // restore the VolumeSnapshot from the VolumeSnapshotContent + vs.Spec.Source.PersistentVolumeClaimName = nil + vs.Spec.Source.VolumeSnapshotContentName = vscName +} + +func resetVolumeSnapshotAnnotation(vs *snapshotv1api.VolumeSnapshot) { + vs.ObjectMeta.Annotations[velerov1api.VSCDeletionPolicyAnnotation] = + string(snapshotv1api.VolumeSnapshotContentRetain) +} + +// Execute uses the data such as CSI driver name, storage +// snapshot handle, snapshot deletion secret (if any) from +// the annotations to recreate a VolumeSnapshotContent object +// and statically bind the VolumeSnapshot object being restored. +func (p *volumeSnapshotRestoreItemAction) Execute( + input *velero.RestoreItemActionExecuteInput, +) (*velero.RestoreItemActionExecuteOutput, error) { + p.log.Info("Starting VolumeSnapshotRestoreItemAction") + if boolptr.IsSetToFalse(input.Restore.Spec.RestorePVs) { + p.log.Infof("Restore %s/%s did not request for PVs to be restored.", + input.Restore.Namespace, input.Restore.Name) + return &velero.RestoreItemActionExecuteOutput{SkipRestore: true}, nil + } + var vs snapshotv1api.VolumeSnapshot + + if err := runtime.DefaultUnstructuredConverter.FromUnstructured( + input.Item.UnstructuredContent(), &vs); err != nil { + return &velero.RestoreItemActionExecuteOutput{}, + errors.Wrapf(err, "failed to convert input.Item from unstructured") + } + + // If cross-namespace restore is configured, change the namespace + // for VolumeSnapshot object to be restored + newNamespace, ok := input.Restore.Spec.NamespaceMapping[vs.GetNamespace()] + if !ok { + // Use original namespace + newNamespace = vs.Namespace + } + + if !csi.IsVolumeSnapshotExists(newNamespace, vs.Name, p.crClient) { + snapHandle, exists := vs.Annotations[velerov1api.VolumeSnapshotHandleAnnotation] + if !exists { + return nil, errors.Errorf( + "Volumesnapshot %s/%s does not have a %s annotation", + vs.Namespace, + vs.Name, + velerov1api.VolumeSnapshotHandleAnnotation, + ) + } + + csiDriverName, exists := vs.Annotations[velerov1api.DriverNameAnnotation] + if !exists { + return nil, errors.Errorf( + "Volumesnapshot %s/%s does not have a %s annotation", + vs.Namespace, vs.Name, velerov1api.DriverNameAnnotation) + } + + p.log.Debugf("Set VolumeSnapshotContent %s/%s DeletionPolicy", + "to Retain to make sure VS deletion in namespace will not", + "delete Snapshot on cloud provider.", + newNamespace, vs.Name) + + vsc := snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: vs.Name + "-", + Labels: map[string]string{ + velerov1api.RestoreNameLabel: label.GetValidName(input.Restore.Name), + }, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, + Driver: csiDriverName, + VolumeSnapshotRef: core_v1.ObjectReference{ + Kind: "VolumeSnapshot", + Namespace: newNamespace, + Name: vs.Name, + }, + Source: snapshotv1api.VolumeSnapshotContentSource{ + SnapshotHandle: &snapHandle, + }, + }, + } + + // we create the VolumeSnapshotContent here instead of relying on the + // restore flow because we want to statically bind this VolumeSnapshot + // with a VolumeSnapshotContent that will be used as its source for pre-populating + // the volume that will be created as a result of the restore. To perform + // this static binding, a bi-directional link between the VolumeSnapshotContent + // and VolumeSnapshot objects have to be setup. Further, it is disallowed + // to convert a dynamically created VolumeSnapshotContent for static binding. + // See: https://github.com/kubernetes-csi/external-snapshotter/issues/274 + if err := p.crClient.Create(context.TODO(), &vsc); err != nil { + return nil, errors.Wrapf(err, + "failed to create volumesnapshotcontents %s", + vsc.GenerateName) + } + p.log.Infof("Created VolumesnapshotContents %s with static", + "binding to volumesnapshot %s/%s", vsc, newNamespace, vs.Name) + + // Reset Spec to convert the VolumeSnapshot from using + // the dynamic VolumeSnapshotContent to the static one. + resetVolumeSnapshotSpecForRestore(&vs, &vsc.Name) + + // Reset VolumeSnapshot annotation. By now, only change + // DeletionPolicy to Retain. + resetVolumeSnapshotAnnotation(&vs) + } + + vsMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&vs) + if err != nil { + return nil, errors.WithStack(err) + } + + p.log.Infof("Returning from VolumeSnapshotRestoreItemAction with no additionalItems") + + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: &unstructured.Unstructured{Object: vsMap}, + AdditionalItems: []velero.ResourceIdentifier{}, + }, nil +} + +func (p *volumeSnapshotRestoreItemAction) Name() string { + return "VolumeSnapshotRestoreItemAction" +} + +func (p *volumeSnapshotRestoreItemAction) Progress( + operationID string, + restore *velerov1api.Restore, +) (velero.OperationProgress, error) { + return velero.OperationProgress{}, nil +} + +func (p *volumeSnapshotRestoreItemAction) Cancel( + operationID string, + restore *velerov1api.Restore, +) error { + return nil +} + +func (p *volumeSnapshotRestoreItemAction) AreAdditionalItemsReady( + additionalItems []velero.ResourceIdentifier, + restore *velerov1api.Restore, +) (bool, error) { + return true, nil +} + +func NewVolumeSnapshotRestoreItemAction( + f client.Factory) plugincommon.HandlerInitializer { + return func(logger logrus.FieldLogger) (interface{}, error) { + crClient, err := f.KubebuilderClient() + if err != nil { + return nil, err + } + + return &volumeSnapshotRestoreItemAction{logger, crClient}, nil + } +} diff --git a/pkg/restore/actions/csi/volumesnapshot_action_test.go b/pkg/restore/actions/csi/volumesnapshot_action_test.go new file mode 100644 index 000000000..ed425a660 --- /dev/null +++ b/pkg/restore/actions/csi/volumesnapshot_action_test.go @@ -0,0 +1,89 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "testing" + + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ( + testPVC = "test-pvc" + testSnapClass = "snap-class" + randText = "DEADFEED" +) + +func TestResetVolumeSnapshotSpecForRestore(t *testing.T) { + testCases := []struct { + name string + vs snapshotv1api.VolumeSnapshot + vscName string + }{ + { + name: "should reset spec as expected", + vs: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vs", + Namespace: "test-ns", + }, + Spec: snapshotv1api.VolumeSnapshotSpec{ + Source: snapshotv1api.VolumeSnapshotSource{ + PersistentVolumeClaimName: &testPVC, + }, + VolumeSnapshotClassName: &testSnapClass, + }, + }, + vscName: "test-vsc", + }, + { + name: "should reset spec and overwriting value for Source.VolumeSnapshotContentName", + vs: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vs", + Namespace: "test-ns", + }, + Spec: snapshotv1api.VolumeSnapshotSpec{ + Source: snapshotv1api.VolumeSnapshotSource{ + VolumeSnapshotContentName: &randText, + }, + VolumeSnapshotClassName: &testSnapClass, + }, + }, + vscName: "test-vsc", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + before := tc.vs.DeepCopy() + resetVolumeSnapshotSpecForRestore(&tc.vs, &tc.vscName) + + assert.Equalf(t, tc.vs.Name, before.Name, "unexpected change to Object.Name, Want: %s; Got %s", tc.name, before.Name, tc.vs.Name) + assert.Equal(t, tc.vs.Namespace, before.Namespace, "unexpected change to Object.Namespace, Want: %s; Got %s", tc.name, before.Namespace, tc.vs.Namespace) + assert.NotNil(t, tc.vs.Spec.Source) + assert.Nil(t, tc.vs.Spec.Source.PersistentVolumeClaimName) + assert.NotNil(t, tc.vs.Spec.Source.VolumeSnapshotContentName) + assert.Equal(t, *tc.vs.Spec.Source.VolumeSnapshotContentName, tc.vscName) + assert.Equal(t, *tc.vs.Spec.VolumeSnapshotClassName, *before.Spec.VolumeSnapshotClassName, "unexpected value for Spec.VolumeSnapshotClassName, Want: %s, Got: %s", + *tc.vs.Spec.VolumeSnapshotClassName, *before.Spec.VolumeSnapshotClassName) + assert.Nil(t, tc.vs.Status) + }) + } +} diff --git a/pkg/restore/actions/csi/volumesnapshotclass_action.go b/pkg/restore/actions/csi/volumesnapshotclass_action.go new file mode 100644 index 000000000..5afba786b --- /dev/null +++ b/pkg/restore/actions/csi/volumesnapshotclass_action.go @@ -0,0 +1,111 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/csi" +) + +// volumeSnapshotClassRestoreItemAction is a Velero restore +// item action plugin for VolumeSnapshotClass +type volumeSnapshotClassRestoreItemAction struct { + log logrus.FieldLogger +} + +// AppliesTo returns information indicating that VolumeSnapshotClassRestoreItemAction +// should be invoked while restoring volumesnapshotclass.snapshot.storage.k8s.io resources. +func (p *volumeSnapshotClassRestoreItemAction) AppliesTo() (velero.ResourceSelector, error) { + return velero.ResourceSelector{ + IncludedResources: []string{"volumesnapshotclass.snapshot.storage.k8s.io"}, + }, nil +} + +// Execute restores VolumeSnapshotClass objects returning any +// snapshotlister secret as additional items to restore +func (p *volumeSnapshotClassRestoreItemAction) Execute( + input *velero.RestoreItemActionExecuteInput, +) (*velero.RestoreItemActionExecuteOutput, error) { + p.log.Info("Starting VolumeSnapshotClassRestoreItemAction") + if boolptr.IsSetToFalse(input.Restore.Spec.RestorePVs) { + p.log.Infof("Restore did not request for PVs to be restored %s/%s", + input.Restore.Namespace, input.Restore.Name) + return &velero.RestoreItemActionExecuteOutput{SkipRestore: true}, nil + } + var snapClass snapshotv1api.VolumeSnapshotClass + + if err := runtime.DefaultUnstructuredConverter.FromUnstructured( + input.Item.UnstructuredContent(), &snapClass); err != nil { + return &velero.RestoreItemActionExecuteOutput{}, errors.Wrapf(err, + "failed to convert input.Item from unstructured") + } + + additionalItems := []velero.ResourceIdentifier{} + if csi.IsVolumeSnapshotClassHasListerSecret(&snapClass) { + additionalItems = append(additionalItems, velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{Group: "", Resource: "secrets"}, + Name: snapClass.Annotations[velerov1api.PrefixedListSecretNameAnnotation], + Namespace: snapClass.Annotations[velerov1api.PrefixedListSecretNamespaceAnnotation], + }) + } + + p.log.Infof("Returning from VolumeSnapshotClassRestoreItemAction with %d additionalItems", + len(additionalItems)) + + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: input.Item, + AdditionalItems: additionalItems, + }, nil +} + +func (p *volumeSnapshotClassRestoreItemAction) Name() string { + return "VolumeSnapshotClassRestoreItemAction" +} + +func (p *volumeSnapshotClassRestoreItemAction) Progress( + operationID string, + restore *velerov1api.Restore, +) (velero.OperationProgress, error) { + return velero.OperationProgress{}, nil +} + +func (p *volumeSnapshotClassRestoreItemAction) Cancel( + operationID string, + restore *velerov1api.Restore, +) error { + return nil +} + +func (p *volumeSnapshotClassRestoreItemAction) AreAdditionalItemsReady( + additionalItems []velero.ResourceIdentifier, + restore *velerov1api.Restore, +) (bool, error) { + return true, nil +} + +func NewVolumeSnapshotClassRestoreItemAction( + logger logrus.FieldLogger) (interface{}, error) { + return &volumeSnapshotClassRestoreItemAction{logger}, nil +} diff --git a/pkg/restore/actions/csi/volumesnapshotcontent_action.go b/pkg/restore/actions/csi/volumesnapshotcontent_action.go new file mode 100644 index 000000000..bd07fb982 --- /dev/null +++ b/pkg/restore/actions/csi/volumesnapshotcontent_action.go @@ -0,0 +1,113 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/csi" +) + +// volumeSnapshotContentRestoreItemAction is a restore item action +// plugin for Velero +type volumeSnapshotContentRestoreItemAction struct { + log logrus.FieldLogger +} + +// AppliesTo returns information indicating VolumeSnapshotContentRestoreItemAction +// action should be invoked while restoring +// volumesnapshotcontent.snapshot.storage.k8s.io resources +func (p *volumeSnapshotContentRestoreItemAction) AppliesTo() ( + velero.ResourceSelector, error, +) { + return velero.ResourceSelector{ + IncludedResources: []string{"volumesnapshotcontent.snapshot.storage.k8s.io"}, + }, nil +} + +// Execute restores a VolumeSnapshotContent object without modification +// returning the snapshot lister secret, if any, as additional items to restore. +func (p *volumeSnapshotContentRestoreItemAction) Execute( + input *velero.RestoreItemActionExecuteInput, +) (*velero.RestoreItemActionExecuteOutput, error) { + p.log.Info("Starting VolumeSnapshotContentRestoreItemAction") + var snapCont snapshotv1api.VolumeSnapshotContent + if boolptr.IsSetToFalse(input.Restore.Spec.RestorePVs) { + p.log.Infof("Restore did not request for PVs to be restored %s/%s", + input.Restore.Namespace, input.Restore.Name) + return &velero.RestoreItemActionExecuteOutput{SkipRestore: true}, nil + } + if err := runtime.DefaultUnstructuredConverter.FromUnstructured( + input.Item.UnstructuredContent(), &snapCont); err != nil { + return &velero.RestoreItemActionExecuteOutput{}, + errors.Wrapf(err, "failed to convert input.Item from unstructured") + } + + additionalItems := []velero.ResourceIdentifier{} + if csi.IsVolumeSnapshotContentHasDeleteSecret(&snapCont) { + additionalItems = append(additionalItems, + velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{Group: "", Resource: "secrets"}, + Name: snapCont.Annotations[velerov1api.DeleteSecretNameAnnotation], + Namespace: snapCont.Annotations[velerov1api.DeleteSecretNamespaceAnnotation], + }, + ) + } + + p.log.Infof("Returning from VolumeSnapshotContentRestoreItemAction", + "with %d additionalItems", len(additionalItems)) + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: input.Item, + AdditionalItems: additionalItems, + }, nil +} + +func (p *volumeSnapshotContentRestoreItemAction) Name() string { + return "VolumeSnapshotContentRestoreItemAction" +} + +func (p *volumeSnapshotContentRestoreItemAction) Progress( + operationID string, + restore *velerov1api.Restore, +) (velero.OperationProgress, error) { + return velero.OperationProgress{}, nil +} + +func (p *volumeSnapshotContentRestoreItemAction) Cancel( + operationID string, + restore *velerov1api.Restore, +) error { + return nil +} + +func (p *volumeSnapshotContentRestoreItemAction) AreAdditionalItemsReady( + additionalItems []velero.ResourceIdentifier, + restore *velerov1api.Restore, +) (bool, error) { + return true, nil +} + +func NewVolumeSnapshotContentRestoreItemAction(logger logrus.FieldLogger) (interface{}, error) { + return &volumeSnapshotContentRestoreItemAction{logger}, nil +} diff --git a/pkg/restore/actions/dataupload_retrieve_action.go b/pkg/restore/actions/dataupload_retrieve_action.go index 8194b6892..653f5e340 100644 --- a/pkg/restore/actions/dataupload_retrieve_action.go +++ b/pkg/restore/actions/dataupload_retrieve_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "context" diff --git a/pkg/restore/actions/dataupload_retrieve_action_test.go b/pkg/restore/actions/dataupload_retrieve_action_test.go index 1d14cf4c8..6bfc7aabf 100644 --- a/pkg/restore/actions/dataupload_retrieve_action_test.go +++ b/pkg/restore/actions/dataupload_retrieve_action_test.go @@ -14,11 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "context" - "fmt" "testing" "github.com/stretchr/testify/require" @@ -120,8 +119,6 @@ func TestDataUploadRetrieveActionExectue(t *testing.T) { }) require.NoError(t, err) - // debug - fmt.Printf("CM: %s\n", &cmList.Items[0]) require.Equal(t, tc.expectedDataUploadResult.Labels, cmList.Items[0].Labels) require.Equal(t, tc.expectedDataUploadResult.Data, cmList.Items[0].Data) } diff --git a/pkg/restore/actions/init_restorehook_pod_action.go b/pkg/restore/actions/init_restorehook_pod_action.go index f994d811d..7614ef085 100644 --- a/pkg/restore/actions/init_restorehook_pod_action.go +++ b/pkg/restore/actions/init_restorehook_pod_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "github.com/pkg/errors" diff --git a/pkg/restore/actions/init_restorehook_pod_action_test.go b/pkg/restore/actions/init_restorehook_pod_action_test.go index c69d3c23f..7d269b504 100644 --- a/pkg/restore/actions/init_restorehook_pod_action_test.go +++ b/pkg/restore/actions/init_restorehook_pod_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "testing" diff --git a/pkg/restore/actions/job_action.go b/pkg/restore/actions/job_action.go index 364ec79ad..1eabc208c 100644 --- a/pkg/restore/actions/job_action.go +++ b/pkg/restore/actions/job_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "github.com/pkg/errors" diff --git a/pkg/restore/actions/job_action_test.go b/pkg/restore/actions/job_action_test.go index 606dd089d..e3578bb60 100644 --- a/pkg/restore/actions/job_action_test.go +++ b/pkg/restore/actions/job_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "testing" diff --git a/pkg/restore/actions/pod_action.go b/pkg/restore/actions/pod_action.go index d4bdc1384..9a98aa88c 100644 --- a/pkg/restore/actions/pod_action.go +++ b/pkg/restore/actions/pod_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "strings" diff --git a/pkg/restore/actions/pod_action_test.go b/pkg/restore/actions/pod_action_test.go index f1aa83c1a..27ae814f0 100644 --- a/pkg/restore/actions/pod_action_test.go +++ b/pkg/restore/actions/pod_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "testing" diff --git a/pkg/restore/actions/pod_volume_restore_action.go b/pkg/restore/actions/pod_volume_restore_action.go index af5c2373c..36722b0a2 100644 --- a/pkg/restore/actions/pod_volume_restore_action.go +++ b/pkg/restore/actions/pod_volume_restore_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "context" diff --git a/pkg/restore/actions/pod_volume_restore_action_test.go b/pkg/restore/actions/pod_volume_restore_action_test.go index cba4914e5..7be617bd4 100644 --- a/pkg/restore/actions/pod_volume_restore_action_test.go +++ b/pkg/restore/actions/pod_volume_restore_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "sort" diff --git a/pkg/restore/actions/rolebinding_action.go b/pkg/restore/actions/rolebinding_action.go index c402075a6..ed345edb0 100644 --- a/pkg/restore/actions/rolebinding_action.go +++ b/pkg/restore/actions/rolebinding_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "github.com/pkg/errors" diff --git a/pkg/restore/actions/rolebinding_action_test.go b/pkg/restore/actions/rolebinding_action_test.go index 8995df8c6..5b605937e 100644 --- a/pkg/restore/actions/rolebinding_action_test.go +++ b/pkg/restore/actions/rolebinding_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "sort" diff --git a/pkg/restore/actions/secret_action.go b/pkg/restore/actions/secret_action.go index 1e63372b6..9e3d06fc8 100644 --- a/pkg/restore/actions/secret_action.go +++ b/pkg/restore/actions/secret_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "context" diff --git a/pkg/restore/actions/secret_action_test.go b/pkg/restore/actions/secret_action_test.go index a627e5296..0cb05617a 100644 --- a/pkg/restore/actions/secret_action_test.go +++ b/pkg/restore/actions/secret_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "testing" diff --git a/pkg/restore/actions/service_account_action.go b/pkg/restore/actions/service_account_action.go index 252d9fc57..85b9bc3df 100644 --- a/pkg/restore/actions/service_account_action.go +++ b/pkg/restore/actions/service_account_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "strings" diff --git a/pkg/restore/actions/service_account_action_test.go b/pkg/restore/actions/service_account_action_test.go index 0bc976a0f..273ac660f 100644 --- a/pkg/restore/actions/service_account_action_test.go +++ b/pkg/restore/actions/service_account_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "sort" diff --git a/pkg/restore/actions/service_action.go b/pkg/restore/actions/service_action.go index 6fc1b2cb4..ee392a381 100644 --- a/pkg/restore/actions/service_action.go +++ b/pkg/restore/actions/service_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "encoding/json" diff --git a/pkg/restore/actions/service_action_test.go b/pkg/restore/actions/service_action_test.go index 16115e920..e8ed85d69 100644 --- a/pkg/restore/actions/service_action_test.go +++ b/pkg/restore/actions/service_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "encoding/json" diff --git a/pkg/test/fake_controller_runtime_client.go b/pkg/test/fake_controller_runtime_client.go index 802999166..ec0432493 100644 --- a/pkg/test/fake_controller_runtime_client.go +++ b/pkg/test/fake_controller_runtime_client.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" + storagev1api "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" k8sfake "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -33,34 +34,33 @@ import ( func NewFakeControllerRuntimeClientBuilder(t *testing.T) *k8sfake.ClientBuilder { scheme := runtime.NewScheme() - err := velerov1api.AddToScheme(scheme) - require.NoError(t, err) - err = velerov2alpha1api.AddToScheme(scheme) - require.NoError(t, err) - err = corev1api.AddToScheme(scheme) - require.NoError(t, err) - err = appsv1api.AddToScheme(scheme) - require.NoError(t, err) - err = snapshotv1api.AddToScheme(scheme) - require.NoError(t, err) + + require.NoError(t, velerov1api.AddToScheme(scheme)) + require.NoError(t, velerov2alpha1api.AddToScheme(scheme)) + require.NoError(t, corev1api.AddToScheme(scheme)) + require.NoError(t, appsv1api.AddToScheme(scheme)) + require.NoError(t, snapshotv1api.AddToScheme(scheme)) + require.NoError(t, storagev1api.AddToScheme(scheme)) + return k8sfake.NewClientBuilder().WithScheme(scheme) } func NewFakeControllerRuntimeClient(t *testing.T, initObjs ...runtime.Object) client.Client { scheme := runtime.NewScheme() - err := velerov1api.AddToScheme(scheme) - require.NoError(t, err) - err = velerov2alpha1api.AddToScheme(scheme) - require.NoError(t, err) - err = corev1api.AddToScheme(scheme) - require.NoError(t, err) - err = appsv1api.AddToScheme(scheme) - require.NoError(t, err) - err = snapshotv1api.AddToScheme(scheme) - require.NoError(t, err) + + require.NoError(t, velerov1api.AddToScheme(scheme)) + require.NoError(t, velerov2alpha1api.AddToScheme(scheme)) + require.NoError(t, corev1api.AddToScheme(scheme)) + require.NoError(t, appsv1api.AddToScheme(scheme)) + require.NoError(t, snapshotv1api.AddToScheme(scheme)) + require.NoError(t, storagev1api.AddToScheme(scheme)) + return k8sfake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(initObjs...).Build() } -func NewFakeControllerRuntimeWatchClient(t *testing.T, initObjs ...runtime.Object) client.WithWatch { +func NewFakeControllerRuntimeWatchClient( + t *testing.T, + initObjs ...runtime.Object, +) client.WithWatch { return NewFakeControllerRuntimeClientBuilder(t).WithRuntimeObjects(initObjs...).Build() } diff --git a/pkg/util/csi/volume_snapshot.go b/pkg/util/csi/volume_snapshot.go index a9a88fd32..b1b7f9375 100644 --- a/pkg/util/csi/volume_snapshot.go +++ b/pkg/util/csi/volume_snapshot.go @@ -20,25 +20,26 @@ import ( "context" "encoding/json" "fmt" + "strings" "time" jsonpatch "github.com/evanphx/json-patch" + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + snapshotter "github.com/kubernetes-csi/external-snapshotter/client/v7/clientset/versioned/typed/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" + corev1api "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/stringptr" "github.com/vmware-tanzu/velero/pkg/util/stringslice" - - snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" - snapshotter "github.com/kubernetes-csi/external-snapshotter/client/v7/clientset/versioned/typed/volumesnapshot/v1" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - apierrors "k8s.io/apimachinery/pkg/api/errors" ) const ( @@ -47,51 +48,80 @@ const ( ) // WaitVolumeSnapshotReady waits a VS to become ready to use until the timeout reaches -func WaitVolumeSnapshotReady(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, - volumeSnapshot string, volumeSnapshotNS string, timeout time.Duration, log logrus.FieldLogger) (*snapshotv1api.VolumeSnapshot, error) { +func WaitVolumeSnapshotReady( + ctx context.Context, + snapshotClient snapshotter.SnapshotV1Interface, + volumeSnapshot string, + volumeSnapshotNS string, + timeout time.Duration, + log logrus.FieldLogger, +) (*snapshotv1api.VolumeSnapshot, error) { var updated *snapshotv1api.VolumeSnapshot errMessage := sets.NewString() - err := wait.PollUntilContextTimeout(ctx, waitInternal, timeout, true, func(ctx context.Context) (bool, error) { - tmpVS, err := snapshotClient.VolumeSnapshots(volumeSnapshotNS).Get(ctx, volumeSnapshot, metav1.GetOptions{}) - if err != nil { - return false, errors.Wrapf(err, fmt.Sprintf("error to get volumesnapshot %s/%s", volumeSnapshotNS, volumeSnapshot)) - } + err := wait.PollUntilContextTimeout( + ctx, + waitInternal, + timeout, + true, + func(ctx context.Context) (bool, error) { + tmpVS, err := snapshotClient.VolumeSnapshots(volumeSnapshotNS).Get( + ctx, volumeSnapshot, metav1.GetOptions{}) + if err != nil { + return false, errors.Wrapf( + err, + fmt.Sprintf("error to get VolumeSnapshot %s/%s", + volumeSnapshotNS, volumeSnapshot), + ) + } - if tmpVS.Status == nil { - return false, nil - } + if tmpVS.Status == nil { + return false, nil + } - if tmpVS.Status.Error != nil { - errMessage.Insert(stringptr.GetString(tmpVS.Status.Error.Message)) - } + if tmpVS.Status.Error != nil { + errMessage.Insert(stringptr.GetString(tmpVS.Status.Error.Message)) + } - if !boolptr.IsSetToTrue(tmpVS.Status.ReadyToUse) { - return false, nil - } + if !boolptr.IsSetToTrue(tmpVS.Status.ReadyToUse) { + return false, nil + } - updated = tmpVS - return true, nil - }) + updated = tmpVS + return true, nil + }, + ) if wait.Interrupted(err) { - err = errors.Errorf("volume snapshot is not ready until timeout, errors: %v", errMessage.List()) + err = errors.Errorf( + "volume snapshot is not ready until timeout, errors: %v", + errMessage.List(), + ) } if errMessage.Len() > 0 { - log.Warnf("Some errors happened during waiting for ready snapshot, errors: %v", errMessage.List()) + log.Warnf("Some errors happened during waiting for ready snapshot, errors: %v", + errMessage.List()) } return updated, err } -// GetVolumeSnapshotContentForVolumeSnapshot returns the volumesnapshotcontent object associated with the volumesnapshot -func GetVolumeSnapshotContentForVolumeSnapshot(volSnap *snapshotv1api.VolumeSnapshot, snapshotClient snapshotter.SnapshotV1Interface) (*snapshotv1api.VolumeSnapshotContent, error) { +// GetVolumeSnapshotContentForVolumeSnapshot returns the VolumeSnapshotContent +// object associated with the VolumeSnapshot. +func GetVolumeSnapshotContentForVolumeSnapshot( + volSnap *snapshotv1api.VolumeSnapshot, + snapshotClient snapshotter.SnapshotV1Interface, +) (*snapshotv1api.VolumeSnapshotContent, error) { if volSnap.Status == nil || volSnap.Status.BoundVolumeSnapshotContentName == nil { return nil, errors.Errorf("invalid snapshot info in volume snapshot %s", volSnap.Name) } - vsc, err := snapshotClient.VolumeSnapshotContents().Get(context.TODO(), *volSnap.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}) + vsc, err := snapshotClient.VolumeSnapshotContents().Get( + context.TODO(), + *volSnap.Status.BoundVolumeSnapshotContentName, + metav1.GetOptions{}, + ) if err != nil { return nil, errors.Wrap(err, "error getting volume snapshot content from API") } @@ -99,7 +129,8 @@ func GetVolumeSnapshotContentForVolumeSnapshot(volSnap *snapshotv1api.VolumeSnap return vsc, nil } -// RetainVSC updates the VSC's deletion policy to Retain and add a finalier and then return the update VSC +// RetainVSC updates the VSC's deletion policy to Retain and add a +// finalizer and then return the update VSC func RetainVSC(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, vsc *snapshotv1api.VolumeSnapshotContent) (*snapshotv1api.VolumeSnapshotContent, error) { if vsc.Spec.DeletionPolicy == snapshotv1api.VolumeSnapshotContentRetain { @@ -112,8 +143,13 @@ func RetainVSC(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interfa }) } -// DeleteVolumeSnapshotContentIfAny deletes a VSC by name if it exists, and log an error when the deletion fails -func DeleteVolumeSnapshotContentIfAny(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, vscName string, log logrus.FieldLogger) { +// DeleteVolumeSnapshotContentIfAny deletes a VSC by name if it exists, +// and log an error when the deletion fails. +func DeleteVolumeSnapshotContentIfAny( + ctx context.Context, + snapshotClient snapshotter.SnapshotV1Interface, + vscName string, log logrus.FieldLogger, +) { err := snapshotClient.VolumeSnapshotContents().Delete(ctx, vscName, metav1.DeleteOptions{}) if err != nil { if apierrors.IsNotFound(err) { @@ -124,7 +160,8 @@ func DeleteVolumeSnapshotContentIfAny(ctx context.Context, snapshotClient snapsh } } -// EnsureDeleteVS asserts the existence of a VS by name, deletes it and waits for its disappearance and returns errors on any failure +// EnsureDeleteVS asserts the existence of a VS by name, deletes it and waits for its +// disappearance and returns errors on any failure. func EnsureDeleteVS(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, vsName string, vsNamespace string, timeout time.Duration) error { err := snapshotClient.VolumeSnapshots(vsNamespace).Delete(ctx, vsName, metav1.DeleteOptions{}) @@ -176,7 +213,8 @@ func RemoveVSCProtect(ctx context.Context, snapshotClient snapshotter.SnapshotV1 return err } -// EnsureDeleteVSC asserts the existence of a VSC by name, deletes it and waits for its disappearance and returns errors on any failure +// EnsureDeleteVSC asserts the existence of a VSC by name, deletes it and waits for its +// disappearance and returns errors on any failure. func EnsureDeleteVSC(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, vscName string, timeout time.Duration) error { err := snapshotClient.VolumeSnapshotContents().Delete(ctx, vscName, metav1.DeleteOptions{}) @@ -203,20 +241,34 @@ func EnsureDeleteVSC(ctx context.Context, snapshotClient snapshotter.SnapshotV1I return nil } -// DeleteVolumeSnapshotIfAny deletes a VS by name if it exists, and log an error when the deletion fails -func DeleteVolumeSnapshotIfAny(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, vsName string, vsNamespace string, log logrus.FieldLogger) { +// DeleteVolumeSnapshotIfAny deletes a VS by name if it exists, +// and log an error when the deletion fails +func DeleteVolumeSnapshotIfAny( + ctx context.Context, + snapshotClient snapshotter.SnapshotV1Interface, + vsName string, + vsNamespace string, + log logrus.FieldLogger, +) { err := snapshotClient.VolumeSnapshots(vsNamespace).Delete(ctx, vsName, metav1.DeleteOptions{}) if err != nil { if apierrors.IsNotFound(err) { - log.WithError(err).Debugf("Abort deleting volume snapshot, it doesn't exist %s/%s", vsNamespace, vsName) + log.WithError(err).Debugf( + "Abort deleting volume snapshot, it doesn't exist %s/%s", + vsNamespace, vsName) } else { - log.WithError(err).Errorf("Failed to delete volume snapshot %s/%s", vsNamespace, vsName) + log.WithError(err).Errorf( + "Failed to delete volume snapshot %s/%s", vsNamespace, vsName) } } } -func patchVSC(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, - vsc *snapshotv1api.VolumeSnapshotContent, updateFunc func(*snapshotv1api.VolumeSnapshotContent)) (*snapshotv1api.VolumeSnapshotContent, error) { +func patchVSC( + ctx context.Context, + snapshotClient snapshotter.SnapshotV1Interface, + vsc *snapshotv1api.VolumeSnapshotContent, + updateFunc func(*snapshotv1api.VolumeSnapshotContent), +) (*snapshotv1api.VolumeSnapshotContent, error) { origBytes, err := json.Marshal(vsc) if err != nil { return nil, errors.Wrap(err, "error marshaling original VSC") @@ -242,3 +294,493 @@ func patchVSC(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interfac return patched, nil } + +func GetVolumeSnapshotClass( + provisioner string, + backup *velerov1api.Backup, + pvc *corev1api.PersistentVolumeClaim, + log logrus.FieldLogger, + crClient crclient.Client, +) (*snapshotv1api.VolumeSnapshotClass, error) { + snapshotClasses := new(snapshotv1api.VolumeSnapshotClassList) + err := crClient.List(context.TODO(), snapshotClasses) + if err != nil { + return nil, errors.Wrap(err, "error listing VolumeSnapshotClass") + } + // If a snapshot class is set for provider in PVC annotations, use that + snapshotClass, err := GetVolumeSnapshotClassFromPVCAnnotationsForDriver( + pvc, provisioner, snapshotClasses, + ) + if err != nil { + log.Debugf("Didn't find VolumeSnapshotClass from PVC annotations: %v", err) + } + if snapshotClass != nil { + return snapshotClass, nil + } + + // If there is no annotation in PVC, attempt to fetch it from backup annotations + snapshotClass, err = GetVolumeSnapshotClassFromBackupAnnotationsForDriver( + backup, provisioner, snapshotClasses) + if err != nil { + log.Debugf("Didn't find VolumeSnapshotClass from Backup annotations: %v", err) + } + if snapshotClass != nil { + return snapshotClass, nil + } + + // fallback to default behavior of fetching snapshot class based on label + snapshotClass, err = GetVolumeSnapshotClassForStorageClass( + provisioner, snapshotClasses) + if err != nil || snapshotClass == nil { + return nil, errors.Wrap(err, "error getting VolumeSnapshotClass") + } + + return snapshotClass, nil +} + +func GetVolumeSnapshotClassFromPVCAnnotationsForDriver( + pvc *corev1api.PersistentVolumeClaim, + provisioner string, + snapshotClasses *snapshotv1api.VolumeSnapshotClassList, +) (*snapshotv1api.VolumeSnapshotClass, error) { + annotationKey := velerov1api.VolumeSnapshotClassDriverPVCAnnotation + snapshotClassName, ok := pvc.ObjectMeta.Annotations[annotationKey] + if !ok { + return nil, nil + } + for _, sc := range snapshotClasses.Items { + if strings.EqualFold(snapshotClassName, sc.ObjectMeta.Name) { + if !strings.EqualFold(sc.Driver, provisioner) { + return nil, errors.Errorf( + "Incorrect VolumeSnapshotClass %s is not for driver %s", + sc.ObjectMeta.Name, provisioner, + ) + } + return &sc, nil + } + } + return nil, errors.Errorf( + "No CSI VolumeSnapshotClass found with name %s for provisioner %s for PVC %s", + snapshotClassName, provisioner, pvc.Name, + ) +} + +// GetVolumeSnapshotClassFromAnnotationsForDriver returns a +// VolumeSnapshotClass for the supplied volume provisioner/driver +// name from the annotation of the backup. +func GetVolumeSnapshotClassFromBackupAnnotationsForDriver( + backup *velerov1api.Backup, + provisioner string, + snapshotClasses *snapshotv1api.VolumeSnapshotClassList, +) (*snapshotv1api.VolumeSnapshotClass, error) { + annotationKey := fmt.Sprintf( + "%s_%s", + velerov1api.VolumeSnapshotClassDriverBackupAnnotationPrefix, + strings.ToLower(provisioner), + ) + snapshotClassName, ok := backup.ObjectMeta.Annotations[annotationKey] + if !ok { + return nil, nil + } + for _, sc := range snapshotClasses.Items { + if strings.EqualFold(snapshotClassName, sc.ObjectMeta.Name) { + if !strings.EqualFold(sc.Driver, provisioner) { + return nil, errors.Errorf( + "Incorrect VolumeSnapshotClass %s is not for driver %s for backup %s", + sc.ObjectMeta.Name, provisioner, backup.Name, + ) + } + return &sc, nil + } + } + return nil, errors.Errorf( + "No CSI VolumeSnapshotClass found with name %s for driver %s for backup %s", + snapshotClassName, provisioner, backup.Name, + ) +} + +// GetVolumeSnapshotClassForStorageClass returns a VolumeSnapshotClass +// for the supplied volume provisioner/ driver name. +func GetVolumeSnapshotClassForStorageClass( + provisioner string, + snapshotClasses *snapshotv1api.VolumeSnapshotClassList, +) (*snapshotv1api.VolumeSnapshotClass, error) { + n := 0 + var vsClass snapshotv1api.VolumeSnapshotClass + // We pick the VolumeSnapshotClass that matches the CSI driver name + // and has a 'velero.io/csi-volumesnapshot-class' label. This allows + // multiple VolumeSnapshotClasses for the same driver with different + // values for the other fields in the spec. + for _, sc := range snapshotClasses.Items { + _, hasLabelSelector := sc.Labels[velerov1api.VolumeSnapshotClassSelectorLabel] + if sc.Driver == provisioner { + n += 1 + vsClass = sc + if hasLabelSelector { + return &sc, nil + } + } + } + // If there's only one volumesnapshotclass for the driver, return it. + if n == 1 { + return &vsClass, nil + } + return nil, fmt.Errorf( + `failed to get VolumeSnapshotClass for provisioner %s, + ensure that the desired VolumeSnapshot class has the %s label`, + provisioner, velerov1api.VolumeSnapshotClassSelectorLabel) +} + +// IsVolumeSnapshotClassHasListerSecret returns whether a volumesnapshotclass has a snapshotlister secret +func IsVolumeSnapshotClassHasListerSecret(vc *snapshotv1api.VolumeSnapshotClass) bool { + // https://github.com/kubernetes-csi/external-snapshotter/blob/master/pkg/utils/util.go#L59-L60 + // There is no release w/ these constants exported. Using the strings for now. + _, nameExists := vc.Annotations[velerov1api.PrefixedListSecretNameAnnotation] + _, nsExists := vc.Annotations[velerov1api.PrefixedListSecretNamespaceAnnotation] + return nameExists && nsExists +} + +// IsVolumeSnapshotContentHasDeleteSecret returns whether a volumesnapshotcontent has a deletesnapshot secret +func IsVolumeSnapshotContentHasDeleteSecret(vsc *snapshotv1api.VolumeSnapshotContent) bool { + // https://github.com/kubernetes-csi/external-snapshotter/blob/master/pkg/utils/util.go#L56-L57 + // use exported constants in the next release + _, nameExists := vsc.Annotations[velerov1api.PrefixedSecretNameAnnotation] + _, nsExists := vsc.Annotations[velerov1api.PrefixedSecretNamespaceAnnotation] + return nameExists && nsExists +} + +// IsVolumeSnapshotHasVSCDeleteSecret returns whether a volumesnapshot should set the deletesnapshot secret +// for the static volumesnapshotcontent that is created on restore +func IsVolumeSnapshotHasVSCDeleteSecret(vs *snapshotv1api.VolumeSnapshot) bool { + _, nameExists := vs.Annotations[velerov1api.DeleteSecretNameAnnotation] + _, nsExists := vs.Annotations[velerov1api.DeleteSecretNamespaceAnnotation] + return nameExists && nsExists +} + +// IsVolumeSnapshotExists returns whether a specific volumesnapshot object exists. +func IsVolumeSnapshotExists( + ns, + name string, + crClient crclient.Client, +) bool { + vs := new(snapshotv1api.VolumeSnapshot) + err := crClient.Get( + context.TODO(), + crclient.ObjectKey{Namespace: ns, Name: name}, + vs, + ) + + return err == nil +} + +func SetVolumeSnapshotContentDeletionPolicy( + vscName string, + crClient crclient.Client, +) error { + vsc := new(snapshotv1api.VolumeSnapshotContent) + if err := crClient.Get(context.TODO(), crclient.ObjectKey{Name: vscName}, vsc); err != nil { + return err + } + + originVSC := vsc.DeepCopy() + vsc.Spec.DeletionPolicy = snapshotv1api.VolumeSnapshotContentDelete + + return crClient.Patch(context.TODO(), vsc, crclient.MergeFrom(originVSC)) +} + +func CleanupVolumeSnapshot( + volSnap *snapshotv1api.VolumeSnapshot, + crClient crclient.Client, + log logrus.FieldLogger, +) { + log.Infof("Deleting Volumesnapshot %s/%s", volSnap.Namespace, volSnap.Name) + vs := new(snapshotv1api.VolumeSnapshot) + err := crClient.Get( + context.TODO(), + crclient.ObjectKey{Name: volSnap.Name, Namespace: volSnap.Namespace}, + vs, + ) + if err != nil { + log.Debugf("Failed to get volumesnapshot %s/%s", volSnap.Namespace, volSnap.Name) + return + } + + if vs.Status != nil && vs.Status.BoundVolumeSnapshotContentName != nil { + // we patch the DeletionPolicy of the VolumeSnapshotContent to set it to Delete. + // This ensures that the volume snapshot in the storage provider is also deleted. + err := SetVolumeSnapshotContentDeletionPolicy( + *vs.Status.BoundVolumeSnapshotContentName, + crClient, + ) + if err != nil { + log.Debugf("Failed to patch DeletionPolicy of volume snapshot %s/%s", + vs.Namespace, vs.Name) + } + } + err = crClient.Delete(context.TODO(), vs) + if err != nil { + log.Debugf("Failed to delete volumesnapshot %s/%s: %v", vs.Namespace, vs.Name, err) + } else { + log.Infof("Deleted volumesnapshot with volumesnapshotContent %s/%s", + vs.Namespace, vs.Name) + } +} + +// DeleteVolumeSnapshot handles the VolumeSnapshot instance deletion. +func DeleteVolumeSnapshot( + vs snapshotv1api.VolumeSnapshot, + vsc snapshotv1api.VolumeSnapshotContent, + backup *velerov1api.Backup, + client crclient.Client, + logger logrus.FieldLogger, +) { + modifyVSCFlag := false + if vs.Status != nil && + vs.Status.BoundVolumeSnapshotContentName != nil && + len(*vs.Status.BoundVolumeSnapshotContentName) > 0 && + vsc.Spec.DeletionPolicy == snapshotv1api.VolumeSnapshotContentDelete { + modifyVSCFlag = true + } else { + logger.Errorf("VolumeSnapshot %s/%s is not ready. This is not expected.", + vs.Namespace, vs.Name) + } + + // Change VolumeSnapshotContent's DeletionPolicy to Retain before deleting VolumeSnapshot, + // because VolumeSnapshotContent will be deleted by deleting VolumeSnapshot, when + // DeletionPolicy is set to Delete, but Velero needs VSC for cleaning snapshot on cloud + // in backup deletion. + if modifyVSCFlag { + logger.Debugf("Patching VolumeSnapshotContent %s", vsc.Name) + originVSC := vsc.DeepCopy() + vsc.Spec.DeletionPolicy = snapshotv1api.VolumeSnapshotContentRetain + err := client.Patch( + context.Background(), + &vsc, + crclient.MergeFrom(originVSC), + ) + if err != nil { + logger.Errorf( + "fail to modify VolumeSnapshotContent %s DeletionPolicy to Retain: %s", + vsc.Name, err.Error(), + ) + return + } + + defer func() { + logger.Debugf("Start to recreate VolumeSnapshotContent %s", vsc.Name) + err := recreateVolumeSnapshotContent(vsc, backup, client, logger) + if err != nil { + logger.Errorf( + "fail to recreate VolumeSnapshotContent %s: %s", + vsc.Name, + err.Error(), + ) + } + }() + } + + // Delete VolumeSnapshot from cluster + logger.Debugf("Deleting VolumeSnapshot %s/%s", vs.Namespace, vs.Name) + err := client.Delete(context.TODO(), &vs) + if err != nil { + logger.Errorf("fail to delete VolumeSnapshot %s/%s: %s", + vs.Namespace, vs.Name, err.Error()) + } +} + +// recreateVolumeSnapshotContent will delete then re-create VolumeSnapshotContent, +// because some parameter in VolumeSnapshotContent Spec is immutable, +// e.g. VolumeSnapshotRef and Source. +// Source is updated to let csi-controller thinks the VSC is statically +// provisioned with VS. +// Set VolumeSnapshotRef's UID to nil will let the csi-controller finds out +// the related VS is gone, then VSC can be deleted. +func recreateVolumeSnapshotContent( + vsc snapshotv1api.VolumeSnapshotContent, + backup *velerov1api.Backup, + client crclient.Client, + log logrus.FieldLogger, +) error { + // Read resource timeout from backup annotation, if not set, use default value. + timeout, err := time.ParseDuration( + backup.Annotations[velerov1api.ResourceTimeoutAnnotation]) + if err != nil { + log.Warnf("fail to parse resource timeout annotation %s: %s", + backup.Annotations[velerov1api.ResourceTimeoutAnnotation], err.Error()) + timeout = 10 * time.Minute + } + log.Debugf("resource timeout is set to %s", timeout.String()) + interval := 1 * time.Second + + if err := client.Delete(context.TODO(), &vsc); err != nil { + return errors.Wrapf(err, "fail to delete VolumeSnapshotContent: %s", vsc.Name) + } + + // Check VolumeSnapshotContents is already deleted, before re-creating it. + err = wait.PollUntilContextTimeout( + context.Background(), + interval, + timeout, + true, + func(ctx context.Context) (bool, error) { + tmpVSC := new(snapshotv1api.VolumeSnapshotContent) + if err := client.Get(ctx, crclient.ObjectKeyFromObject(&vsc), tmpVSC); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, errors.Wrapf( + err, + fmt.Sprintf("failed to get VolumeSnapshotContent %s", vsc.Name), + ) + } + return false, nil + }, + ) + if err != nil { + return errors.Wrapf(err, "fail to retrieve VolumeSnapshotContent %s info", vsc.Name) + } + + // Make the VolumeSnapshotContent static + vsc.Spec.Source = snapshotv1api.VolumeSnapshotContentSource{ + SnapshotHandle: vsc.Status.SnapshotHandle, + } + // Set VolumeSnapshotRef to none exist one, because VolumeSnapshotContent + // validation webhook will check whether name and namespace are nil. + // external-snapshotter needs Source pointing to snapshot and VolumeSnapshot + // reference's UID to nil to determine the VolumeSnapshotContent is deletable. + vsc.Spec.VolumeSnapshotRef = corev1api.ObjectReference{ + APIVersion: snapshotv1api.SchemeGroupVersion.String(), + Kind: "VolumeSnapshot", + Namespace: "ns-" + string(vsc.UID), + Name: "name-" + string(vsc.UID), + } + // ResourceVersion shouldn't exist for new creation. + vsc.ResourceVersion = "" + if err := client.Create(context.TODO(), &vsc); err != nil { + return errors.Wrapf(err, "fail to create VolumeSnapshotContent %s", vsc.Name) + } + + return nil +} + +// WaitUntilVSCHandleIsReady returns the VolumeSnapshotContent +// object associated with the volumesnapshot +func WaitUntilVSCHandleIsReady( + volSnap *snapshotv1api.VolumeSnapshot, + crClient crclient.Client, + log logrus.FieldLogger, + shouldWait bool, + csiSnapshotTimeout time.Duration, +) (*snapshotv1api.VolumeSnapshotContent, error) { + if !shouldWait { + if volSnap.Status == nil || + volSnap.Status.BoundVolumeSnapshotContentName == nil { + // volumesnapshot hasn't been reconciled and we're + // not waiting for it. + return nil, nil + } + vsc := new(snapshotv1api.VolumeSnapshotContent) + err := crClient.Get( + context.TODO(), + crclient.ObjectKey{ + Name: *volSnap.Status.BoundVolumeSnapshotContentName, + }, + vsc, + ) + if err != nil { + return nil, + errors.Wrap(err, + "error getting volume snapshot content from API") + } + return vsc, nil + } + + // We'll wait 10m for the VSC to be reconciled polling + // every 5s unless backup's csiSnapshotTimeout is set + interval := 5 * time.Second + vsc := new(snapshotv1api.VolumeSnapshotContent) + + err := wait.PollUntilContextTimeout( + context.Background(), + interval, + csiSnapshotTimeout, + true, + func(ctx context.Context) (bool, error) { + vs := new(snapshotv1api.VolumeSnapshot) + if err := crClient.Get( + ctx, + crclient.ObjectKeyFromObject(volSnap), + vs, + ); err != nil { + return false, + errors.Wrapf(err, fmt.Sprintf( + "failed to get volumesnapshot %s/%s", + volSnap.Namespace, volSnap.Name), + ) + } + + if vs.Status == nil || vs.Status.BoundVolumeSnapshotContentName == nil { + log.Infof("Waiting for CSI driver to reconcile volumesnapshot %s/%s. Retrying in %ds", + volSnap.Namespace, volSnap.Name, interval/time.Second) + return false, nil + } + + if err := crClient.Get( + ctx, + crclient.ObjectKey{ + Name: *vs.Status.BoundVolumeSnapshotContentName, + }, + vsc, + ); err != nil { + return false, + errors.Wrapf( + err, + fmt.Sprintf("failed to get VolumeSnapshotContent %s for VolumeSnapshot %s/%s", + *vs.Status.BoundVolumeSnapshotContentName, vs.Namespace, vs.Name), + ) + } + + // we need to wait for the VolumeSnapshotContent + // to have a snapshot handle because during restore, + // we'll use that snapshot handle as the source for + // the VolumeSnapshotContent so it's statically + // bound to the existing snapshot. + if vsc.Status == nil || + vsc.Status.SnapshotHandle == nil { + log.Infof( + "Waiting for VolumeSnapshotContents %s to have snapshot handle. Retrying in %ds", + vsc.Name, interval/time.Second) + if vsc.Status != nil && + vsc.Status.Error != nil { + log.Warnf("VolumeSnapshotContent %s has error: %v", + vsc.Name, *vsc.Status.Error.Message) + } + return false, nil + } + + return true, nil + }, + ) + + if err != nil { + if err == wait.ErrorInterrupted(errors.New("timed out waiting for the condition")) { + if vsc != nil && + vsc.Status != nil && + vsc.Status.Error != nil { + log.Errorf( + "Timed out awaiting reconciliation of VolumeSnapshot, VolumeSnapshotContent %s has error: %v", + vsc.Name, *vsc.Status.Error.Message) + return nil, + errors.Errorf("CSI got timed out with error: %v", + *vsc.Status.Error.Message) + } else { + log.Errorf( + "Timed out awaiting reconciliation of volumesnapshot %s/%s", + volSnap.Namespace, volSnap.Name) + } + } + return nil, err + } + + return vsc, nil +} diff --git a/pkg/util/csi/volume_snapshot_test.go b/pkg/util/csi/volume_snapshot_test.go index 0b02865e2..74be22786 100644 --- a/pkg/util/csi/volume_snapshot_test.go +++ b/pkg/util/csi/volume_snapshot_test.go @@ -24,18 +24,23 @@ import ( snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" snapshotFake "github.com/kubernetes-csi/external-snapshotter/client/v7/clientset/versioned/fake" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" clientTesting "k8s.io/client-go/testing" + crclient "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/vmware-tanzu/velero/pkg/util/boolptr" - "github.com/vmware-tanzu/velero/pkg/util/stringptr" - + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/builder" velerotest "github.com/vmware-tanzu/velero/pkg/test" - - apierrors "k8s.io/apimachinery/pkg/api/errors" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/logging" + "github.com/vmware-tanzu/velero/pkg/util/stringptr" ) type reactor struct { @@ -44,8 +49,12 @@ type reactor struct { reactorFunc clientTesting.ReactionFunc } +// expected: &v1.VolumeSnapshot{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"fake-vs", GenerateName:"", Namespace:"fake-ns", SelfLink:"", UID:"", ResourceVersion:"999", Generation:0, CreationTimestamp:time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC), DeletionTimestamp:, DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ManagedFields:[]v1.ManagedFieldsEntry(nil)}, Spec:v1.VolumeSnapshotSpec{Source:v1.VolumeSnapshotSource{PersistentVolumeClaimName:(*string)(nil), VolumeSnapshotContentName:(*string)(nil)}, VolumeSnapshotClassName:(*string)(nil)}, Status:(*v1.VolumeSnapshotStatus)(0x140000af8c0)} +// actual : &v1.VolumeSnapshot{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"fake-vs", GenerateName:"", Namespace:"fake-ns", SelfLink:"", UID:"", ResourceVersion:"999", Generation:0, CreationTimestamp:time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC), DeletionTimestamp:, DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ManagedFields:[]v1.ManagedFieldsEntry(nil)}, Spec:v1.VolumeSnapshotSpec{Source:v1.VolumeSnapshotSource{PersistentVolumeClaimName:(*string)(nil), VolumeSnapshotContentName:(*string)(nil)}, VolumeSnapshotClassName:(*string)(nil)}, Status:(*v1.VolumeSnapshotStatus)(0x1400024ed20)} + func TestWaitVolumeSnapshotReady(t *testing.T) { vscName := "fake-vsc" + quantity := resource.MustParse("0") vsObj := &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vs", @@ -54,7 +63,7 @@ func TestWaitVolumeSnapshotReady(t *testing.T) { Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: &vscName, ReadyToUse: boolptr.True(), - RestoreSize: &resource.Quantity{}, + RestoreSize: &quantity, }, } @@ -72,7 +81,7 @@ func TestWaitVolumeSnapshotReady(t *testing.T) { name: "get vs error", vsName: "fake-vs-1", namespace: "fake-ns-1", - err: "error to get volumesnapshot fake-ns-1/fake-vs-1: volumesnapshots.snapshot.storage.k8s.io \"fake-vs-1\" not found", + err: "error to get VolumeSnapshot fake-ns-1/fake-vs-1: volumesnapshots.snapshot.storage.k8s.io \"fake-vs-1\" not found", }, { name: "vs status is nil", @@ -187,9 +196,9 @@ func TestWaitVolumeSnapshotReady(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - fakeSnapshotClient := snapshotFake.NewSimpleClientset(test.clientObj...) + fakeClient := snapshotFake.NewSimpleClientset(test.clientObj...) - vs, err := WaitVolumeSnapshotReady(context.Background(), fakeSnapshotClient.SnapshotV1(), test.vsName, test.namespace, time.Millisecond, velerotest.NewLogger()) + vs, err := WaitVolumeSnapshotReady(context.Background(), fakeClient.SnapshotV1(), test.vsName, test.namespace, time.Millisecond, velerotest.NewLogger()) if err != nil { assert.EqualError(t, err, test.err) } else { @@ -273,9 +282,9 @@ func TestGetVolumeSnapshotContentForVolumeSnapshot(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - fakeSnapshotClient := snapshotFake.NewSimpleClientset(test.clientObj...) + fakeClient := snapshotFake.NewSimpleClientset(test.clientObj...) - vs, err := GetVolumeSnapshotContentForVolumeSnapshot(test.snapshotObj, fakeSnapshotClient.SnapshotV1()) + vs, err := GetVolumeSnapshotContentForVolumeSnapshot(test.snapshotObj, fakeClient.SnapshotV1()) if err != nil { assert.EqualError(t, err, test.err) } else { @@ -746,3 +755,992 @@ func TestRemoveVSCProtect(t *testing.T) { }) } } + +func TestGetVolumeSnapshotClass(t *testing.T) { + // backups + backupFoo := &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Annotations: map[string]string{ + "velero.io/csi-volumesnapshot-class_foo.csi.k8s.io": "foowithoutlabel", + }, + }, + Spec: velerov1api.BackupSpec{ + IncludedNamespaces: []string{"ns1", "ns2"}, + }, + } + backupFoo2 := &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo2", + Annotations: map[string]string{ + "velero.io/csi-volumesnapshot-class_foo.csi.k8s.io": "foo2", + }, + }, + Spec: velerov1api.BackupSpec{ + IncludedNamespaces: []string{"ns1", "ns2"}, + }, + } + + backupBar2 := &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + Annotations: map[string]string{ + "velero.io/csi-volumesnapshot-class_bar.csi.k8s.io": "bar2", + }, + }, + Spec: velerov1api.BackupSpec{ + IncludedNamespaces: []string{"ns1", "ns2"}, + }, + } + + backupNone := &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "none", + }, + Spec: velerov1api.BackupSpec{ + IncludedNamespaces: []string{"ns1", "ns2"}, + }, + } + + // pvcs + pvcFoo := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Annotations: map[string]string{ + "velero.io/csi-volumesnapshot-class": "foowithoutlabel", + }, + }, + Spec: v1.PersistentVolumeClaimSpec{}, + } + pvcFoo2 := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Annotations: map[string]string{ + "velero.io/csi-volumesnapshot-class": "foo2", + }, + }, + Spec: v1.PersistentVolumeClaimSpec{}, + } + pvcNone := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "none", + }, + Spec: v1.PersistentVolumeClaimSpec{}, + } + + // vsclasses + hostpathClass := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hostpath", + Labels: map[string]string{ + velerov1api.VolumeSnapshotClassSelectorLabel: "foo", + }, + }, + Driver: "hostpath.csi.k8s.io", + } + + fooClass := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Labels: map[string]string{ + velerov1api.VolumeSnapshotClassSelectorLabel: "foo", + }, + }, + Driver: "foo.csi.k8s.io", + } + fooClassWithoutLabel := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foowithoutlabel", + }, + Driver: "foo.csi.k8s.io", + } + + barClass := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + Labels: map[string]string{ + velerov1api.VolumeSnapshotClassSelectorLabel: "true", + }, + }, + Driver: "bar.csi.k8s.io", + } + + barClass2 := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bar2", + Labels: map[string]string{ + velerov1api.VolumeSnapshotClassSelectorLabel: "true", + }, + }, + Driver: "bar.csi.k8s.io", + } + + objs := []runtime.Object{hostpathClass, fooClass, barClass, fooClassWithoutLabel, barClass2} + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) + + testCases := []struct { + name string + driverName string + pvc *v1.PersistentVolumeClaim + backup *velerov1api.Backup + expectedVSC *snapshotv1api.VolumeSnapshotClass + expectError bool + }{ + { + name: "no annotations on pvc and backup, should find hostpath volumesnapshotclass using default behavior of labels", + driverName: "hostpath.csi.k8s.io", + pvc: pvcNone, + backup: backupNone, + expectedVSC: hostpathClass, + expectError: false, + }, + { + name: "foowithoutlabel VSC annotations on pvc", + driverName: "foo.csi.k8s.io", + pvc: pvcFoo, + backup: backupNone, + expectedVSC: fooClassWithoutLabel, + expectError: false, + }, + { + name: "foowithoutlabel VSC annotations on pvc, but csi driver does not match, no annotation on backup so fallback to default behavior of labels", + driverName: "bar.csi.k8s.io", + pvc: pvcFoo, + backup: backupNone, + expectedVSC: barClass, + expectError: false, + }, + { + name: "foowithoutlabel VSC annotations on pvc, but csi driver does not match so fallback to fetch from backupAnnotations ", + driverName: "bar.csi.k8s.io", + pvc: pvcFoo, + backup: backupBar2, + expectedVSC: barClass2, + expectError: false, + }, + { + name: "foowithoutlabel VSC annotations on backup for foo.csi.k8s.io", + driverName: "foo.csi.k8s.io", + pvc: pvcNone, + backup: backupFoo, + expectedVSC: fooClassWithoutLabel, + expectError: false, + }, + { + name: "foowithoutlabel VSC annotations on backup for bar.csi.k8s.io, no annotation corresponding to foo.csi.k8s.io, so fallback to default behavior of labels", + driverName: "bar.csi.k8s.io", + pvc: pvcNone, + backup: backupFoo, + expectedVSC: barClass, + expectError: false, + }, + { + name: "no snapshotClass for given driver", + driverName: "blah.csi.k8s.io", + pvc: pvcNone, + backup: backupNone, + expectedVSC: nil, + expectError: true, + }, + { + name: "foo2 VSC annotations on pvc, but doesn't exist in cluster, fallback to default behavior of labels", + driverName: "foo.csi.k8s.io", + pvc: pvcFoo2, + backup: backupFoo2, + expectedVSC: fooClass, + expectError: false, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualSnapshotClass, actualError := GetVolumeSnapshotClass( + tc.driverName, tc.backup, tc.pvc, logrus.New(), fakeClient) + if tc.expectError { + assert.NotNil(t, actualError) + assert.Nil(t, actualSnapshotClass) + return + } + assert.Equal(t, tc.expectedVSC, actualSnapshotClass) + }) + } +} + +func TestGetVolumeSnapshotClassForStorageClass(t *testing.T) { + hostpathClass := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hostpath", + Labels: map[string]string{ + velerov1api.VolumeSnapshotClassSelectorLabel: "foo", + }, + }, + Driver: "hostpath.csi.k8s.io", + } + + fooClass := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Labels: map[string]string{ + velerov1api.VolumeSnapshotClassSelectorLabel: "foo", + }, + }, + Driver: "foo.csi.k8s.io", + } + + barClass := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + Labels: map[string]string{ + velerov1api.VolumeSnapshotClassSelectorLabel: "foo", + }, + }, + Driver: "bar.csi.k8s.io", + } + + bazClass := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "baz", + }, + Driver: "baz.csi.k8s.io", + } + + ambClass1 := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "amb1", + }, + Driver: "amb.csi.k8s.io", + } + + ambClass2 := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "amb2", + }, + Driver: "amb.csi.k8s.io", + } + + snapshotClasses := &snapshotv1api.VolumeSnapshotClassList{ + Items: []snapshotv1api.VolumeSnapshotClass{ + *hostpathClass, *fooClass, *barClass, *bazClass, *ambClass1, *ambClass2}, + } + + testCases := []struct { + name string + driverName string + expectedVSC *snapshotv1api.VolumeSnapshotClass + expectError bool + }{ + { + name: "should find hostpath volumesnapshotclass", + driverName: "hostpath.csi.k8s.io", + expectedVSC: hostpathClass, + expectError: false, + }, + { + name: "should find foo volumesnapshotclass", + driverName: "foo.csi.k8s.io", + expectedVSC: fooClass, + expectError: false, + }, + { + name: "should find bar volumesnapshotclass", + driverName: "bar.csi.k8s.io", + expectedVSC: barClass, + expectError: false, + }, + { + name: "should find baz volumesnapshotclass without \"velero.io/csi-volumesnapshot-class\" label, b/c there's only one vsclass matching the driver name", + driverName: "baz.csi.k8s.io", + expectedVSC: bazClass, + expectError: false, + }, + { + name: "should not find amb volumesnapshotclass without \"velero.io/csi-volumesnapshot-class\" label, b/c there're more than one vsclass matching the driver name", + driverName: "amb.csi.k8s.io", + expectedVSC: nil, + expectError: true, + }, + { + name: "should not find does-not-exist volumesnapshotclass", + driverName: "not-found.csi.k8s.io", + expectedVSC: nil, + expectError: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualVSC, actualError := GetVolumeSnapshotClassForStorageClass(tc.driverName, snapshotClasses) + + if tc.expectError { + assert.NotNil(t, actualError) + assert.Nil(t, actualVSC) + return + } + + assert.Equalf(t, tc.expectedVSC.Name, actualVSC.Name, "unexpected volumesnapshotclass name returned. Want: %s; Got:%s", tc.expectedVSC.Name, actualVSC.Name) + assert.Equalf(t, tc.expectedVSC.Driver, actualVSC.Driver, "unexpected driver name returned. Want: %s; Got:%s", tc.expectedVSC.Driver, actualVSC.Driver) + }) + } +} + +func TestIsVolumeSnapshotClassHasListerSecret(t *testing.T) { + testCases := []struct { + name string + snapClass snapshotv1api.VolumeSnapshotClass + expected bool + }{ + { + name: "should find both annotations", + snapClass: snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "class-1", + Annotations: map[string]string{ + velerov1api.PrefixedListSecretNameAnnotation: "snapListSecret", + velerov1api.PrefixedListSecretNamespaceAnnotation: "awesome-ns", + }, + }, + }, + expected: true, + }, + { + name: "should not find both annotations name is missing", + snapClass: snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "class-1", + Annotations: map[string]string{ + "foo": "snapListSecret", + velerov1api.PrefixedListSecretNamespaceAnnotation: "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find both annotations namespace is missing", + snapClass: snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "class-1", + Annotations: map[string]string{ + velerov1api.PrefixedListSecretNameAnnotation: "snapListSecret", + "foo": "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find expected annotation non-empty annotation", + snapClass: snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "class-2", + Annotations: map[string]string{ + "foo": "snapListSecret", + "bar": "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find expected annotation nil annotation", + snapClass: snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "class-3", + Annotations: nil, + }, + }, + expected: false, + }, + { + name: "should not find expected annotation empty annotation", + snapClass: snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "class-3", + Annotations: map[string]string{}, + }, + }, + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actual := IsVolumeSnapshotClassHasListerSecret(&tc.snapClass) + assert.Equal(t, tc.expected, actual) + }) + } +} + +func TestIsVolumeSnapshotContentHasDeleteSecret(t *testing.T) { + testCases := []struct { + name string + vsc snapshotv1api.VolumeSnapshotContent + expected bool + }{ + { + name: "should find both annotations", + vsc: snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-1", + Annotations: map[string]string{ + velerov1api.PrefixedSecretNameAnnotation: "delSnapSecret", + velerov1api.PrefixedSecretNamespaceAnnotation: "awesome-ns", + }, + }, + }, + expected: true, + }, + { + name: "should not find both annotations name is missing", + vsc: snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-2", + Annotations: map[string]string{ + "foo": "delSnapSecret", + velerov1api.PrefixedSecretNamespaceAnnotation: "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find both annotations namespace is missing", + vsc: snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-3", + Annotations: map[string]string{ + velerov1api.PrefixedSecretNameAnnotation: "delSnapSecret", + "foo": "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find expected annotation non-empty annotation", + vsc: snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-4", + Annotations: map[string]string{ + "foo": "delSnapSecret", + "bar": "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find expected annotation empty annotation", + vsc: snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-5", + Annotations: map[string]string{}, + }, + }, + expected: false, + }, + { + name: "should not find expected annotation nil annotation", + vsc: snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-6", + Annotations: nil, + }, + }, + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actual := IsVolumeSnapshotContentHasDeleteSecret(&tc.vsc) + assert.Equal(t, tc.expected, actual) + }) + } +} + +func TestIsVolumeSnapshotHasVSCDeleteSecret(t *testing.T) { + testCases := []struct { + name string + vs snapshotv1api.VolumeSnapshot + expected bool + }{ + { + name: "should find both annotations", + vs: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-1", + Annotations: map[string]string{ + "velero.io/csi-deletesnapshotsecret-name": "snapDelSecret", + "velero.io/csi-deletesnapshotsecret-namespace": "awesome-ns", + }, + }, + }, + expected: true, + }, + { + name: "should not find both annotations name is missing", + vs: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-1", + Annotations: map[string]string{ + "foo": "snapDelSecret", + "velero.io/csi-deletesnapshotsecret-namespace": "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find both annotations namespace is missing", + vs: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-1", + Annotations: map[string]string{ + "velero.io/csi-deletesnapshotsecret-name": "snapDelSecret", + "foo": "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find annotation non-empty annotation", + vs: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-1", + Annotations: map[string]string{ + "foo": "snapDelSecret", + "bar": "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find annotation empty annotation", + vs: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-1", + Annotations: map[string]string{}, + }, + }, + expected: false, + }, + { + name: "should not find annotation nil annotation", + vs: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-1", + Annotations: nil, + }, + }, + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actual := IsVolumeSnapshotHasVSCDeleteSecret(&tc.vs) + assert.Equal(t, tc.expected, actual) + }) + } +} + +func TestIsVolumeSnapshotExists(t *testing.T) { + vsExists := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-exists", + Namespace: "default", + }, + } + vsNotExists := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-does-not-exists", + Namespace: "default", + }, + } + + objs := []runtime.Object{vsExists} + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) + testCases := []struct { + name string + expected bool + vs *snapshotv1api.VolumeSnapshot + }{ + { + name: "should find existing VolumeSnapshot object", + expected: true, + vs: vsExists, + }, + { + name: "should not find non-existing VolumeSnapshot object", + expected: false, + vs: vsNotExists, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actual := IsVolumeSnapshotExists(tc.vs.Namespace, tc.vs.Name, fakeClient) + assert.Equal(t, tc.expected, actual) + }) + } +} + +func TestSetVolumeSnapshotContentDeletionPolicy(t *testing.T) { + testCases := []struct { + name string + inputVSCName string + objs []runtime.Object + expectError bool + }{ + { + name: "should update DeletionPolicy of a VSC from retain to delete", + inputVSCName: "retainVSC", + objs: []runtime.Object{ + &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "retainVSC", + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, + }, + }, + }, + expectError: false, + }, + { + name: "should be a no-op updating if DeletionPolicy of a VSC is already Delete", + inputVSCName: "deleteVSC", + objs: []runtime.Object{ + &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "deleteVSC", + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + DeletionPolicy: snapshotv1api.VolumeSnapshotContentDelete, + }, + }, + }, + expectError: false, + }, + { + name: "should update DeletionPolicy of a VSC with no DeletionPolicy", + inputVSCName: "nothingVSC", + objs: []runtime.Object{ + &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nothingVSC", + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{}, + }, + }, + expectError: false, + }, + { + name: "should return not found error if supplied VSC does not exist", + inputVSCName: "does-not-exist", + objs: []runtime.Object{}, + expectError: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, tc.objs...) + err := SetVolumeSnapshotContentDeletionPolicy(tc.inputVSCName, fakeClient) + if tc.expectError { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + actual := new(snapshotv1api.VolumeSnapshotContent) + err := fakeClient.Get( + context.TODO(), + crclient.ObjectKey{Name: tc.inputVSCName}, + actual, + ) + assert.Nil(t, err) + assert.Equal( + t, + snapshotv1api.VolumeSnapshotContentDelete, + actual.Spec.DeletionPolicy, + ) + } + }) + } +} + +func TestDeleteVolumeSnapshots(t *testing.T) { + tests := []struct { + name string + vs snapshotv1api.VolumeSnapshot + vsc snapshotv1api.VolumeSnapshotContent + expectedVS snapshotv1api.VolumeSnapshot + expectedVSC snapshotv1api.VolumeSnapshotContent + }{ + { + name: "VS is ReadyToUse, and VS has corresponding VSC. VS should be deleted.", + vs: *builder.ForVolumeSnapshot("velero", "vs1"). + ObjectMeta(builder.WithLabels("testing-vs", "vs1")). + Status().BoundVolumeSnapshotContentName("vsc1").Result(), + vsc: *builder.ForVolumeSnapshotContent("vsc1"). + DeletionPolicy(snapshotv1api.VolumeSnapshotContentDelete). + Status(&snapshotv1api.VolumeSnapshotContentStatus{}).Result(), + expectedVS: snapshotv1api.VolumeSnapshot{}, + expectedVSC: *builder.ForVolumeSnapshotContent("vsc1"). + DeletionPolicy(snapshotv1api.VolumeSnapshotContentRetain). + VolumeSnapshotRef("ns-", "name-").Result(), + }, + { + name: "VS status is nil. VSC should not be modified.", + vs: *builder.ForVolumeSnapshot("velero", "vs1"). + ObjectMeta(builder.WithLabels("testing-vs", "vs1")).Result(), + vsc: *builder.ForVolumeSnapshotContent("vsc1"). + DeletionPolicy(snapshotv1api.VolumeSnapshotContentDelete). + Status(&snapshotv1api.VolumeSnapshotContentStatus{}).Result(), + expectedVS: snapshotv1api.VolumeSnapshot{}, + expectedVSC: *builder.ForVolumeSnapshotContent("vsc1"). + DeletionPolicy(snapshotv1api.VolumeSnapshotContentDelete).Result(), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + client := velerotest.NewFakeControllerRuntimeClient( + t, + []runtime.Object{&tc.vs, &tc.vsc}..., + ) + logger := logging.DefaultLogger(logrus.DebugLevel, logging.FormatText) + backup := builder.ForBackup(velerov1api.DefaultNamespace, "backup-1"). + DefaultVolumesToFsBackup(false).Result() + + DeleteVolumeSnapshot(tc.vs, tc.vsc, backup, client, logger) + + vsList := new(snapshotv1api.VolumeSnapshotList) + err := client.List( + context.TODO(), + vsList, + &crclient.ListOptions{ + Namespace: "velero", + }, + ) + require.NoError(t, err) + if tc.expectedVS.Name == "" { + require.Empty(t, vsList.Items) + } else { + require.Equal(t, tc.expectedVS.Status, vsList.Items[0].Status) + require.Equal(t, tc.expectedVS.Spec, vsList.Items[0].Spec) + } + + vscList := new(snapshotv1api.VolumeSnapshotContentList) + err = client.List( + context.TODO(), + vscList, + ) + require.NoError(t, err) + require.Len(t, vscList.Items, 1) + require.Equal(t, tc.expectedVSC.Spec, vscList.Items[0].Spec) + }) + } +} + +func TestWaitUntilVSCHandleIsReady(t *testing.T) { + vscName := "snapcontent-7d1bdbd1-d10d-439c-8d8e-e1c2565ddc53" + snapshotHandle := "snapshot-handle" + vscObj := &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: vscName, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + VolumeSnapshotRef: v1.ObjectReference{ + Name: "vol-snap-1", + APIVersion: snapshotv1api.SchemeGroupVersion.String(), + }, + }, + Status: &snapshotv1api.VolumeSnapshotContentStatus{ + SnapshotHandle: &snapshotHandle, + }, + } + validVS := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs", + Namespace: "default", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &vscName, + }, + } + + notFound := "does-not-exist" + vsWithVSCNotFound := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: notFound, + Namespace: "default", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: ¬Found, + }, + } + + vsWithNilStatus := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nil-status-vs", + Namespace: "default", + }, + Status: nil, + } + vsWithNilStatusField := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nil-status-field-vs", + Namespace: "default", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: nil, + }, + } + + nilStatusVsc := "nil-status-vsc" + vscWithNilStatus := &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: nilStatusVsc, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + VolumeSnapshotRef: v1.ObjectReference{ + Name: "vol-snap-1", + APIVersion: snapshotv1api.SchemeGroupVersion.String(), + }, + }, + Status: nil, + } + vsForNilStatusVsc := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-for-nil-status-vsc", + Namespace: "default", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &nilStatusVsc, + }, + } + + nilStatusFieldVsc := "nil-status-field-vsc" + vscWithNilStatusField := &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: nilStatusFieldVsc, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + VolumeSnapshotRef: v1.ObjectReference{ + Name: "vol-snap-1", + APIVersion: snapshotv1api.SchemeGroupVersion.String(), + }, + }, + Status: &snapshotv1api.VolumeSnapshotContentStatus{ + SnapshotHandle: nil, + }, + } + vsForNilStatusFieldVsc := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-for-nil-status-field", + Namespace: "default", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &nilStatusFieldVsc, + }, + } + + objs := []runtime.Object{ + vscObj, + validVS, + vsWithVSCNotFound, + vsWithNilStatus, + vsWithNilStatusField, + vscWithNilStatus, + vsForNilStatusVsc, + vscWithNilStatusField, + vsForNilStatusFieldVsc, + } + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) + testCases := []struct { + name string + volSnap *snapshotv1api.VolumeSnapshot + exepctedVSC *snapshotv1api.VolumeSnapshotContent + wait bool + expectError bool + }{ + { + name: "waitEnabled should find volumesnapshotcontent for volumesnapshot", + volSnap: validVS, + exepctedVSC: vscObj, + wait: true, + expectError: false, + }, + { + name: "waitEnabled should not find volumesnapshotcontent for volumesnapshot with non-existing snapshotcontent name in status.BoundVolumeSnapshotContentName", + volSnap: vsWithVSCNotFound, + exepctedVSC: nil, + wait: true, + expectError: true, + }, + { + name: "waitEnabled should not find volumesnapshotcontent for a non-existent volumesnapshot", + wait: true, + exepctedVSC: nil, + expectError: true, + volSnap: &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "not-found", + Namespace: "default", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &nilStatusVsc, + }, + }, + }, + { + name: "waitDisabled should not find volumesnapshotcontent volumesnapshot status is nil", + wait: false, + expectError: false, + exepctedVSC: nil, + volSnap: vsWithNilStatus, + }, + { + name: "waitDisabled should not find volumesnapshotcontent volumesnapshot status.BoundVolumeSnapshotContentName is nil", + wait: false, + expectError: false, + exepctedVSC: nil, + volSnap: vsWithNilStatusField, + }, + { + name: "waitDisabled should find volumesnapshotcontent volumesnapshotcontent status is nil", + wait: false, + expectError: false, + exepctedVSC: vscWithNilStatus, + volSnap: vsForNilStatusVsc, + }, + { + name: "waitDisabled should find volumesnapshotcontent volumesnapshotcontent status.SnapshotHandle is nil", + wait: false, + expectError: false, + exepctedVSC: vscWithNilStatusField, + volSnap: vsForNilStatusFieldVsc, + }, + { + name: "waitDisabled should not find a non-existent volumesnapshotcontent", + wait: false, + exepctedVSC: nil, + expectError: true, + volSnap: vsWithVSCNotFound, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualVSC, actualError := WaitUntilVSCHandleIsReady(tc.volSnap, fakeClient, logrus.New().WithField("fake", "test"), tc.wait, 0) + if tc.expectError && actualError == nil { + assert.NotNil(t, actualError) + assert.Nil(t, actualVSC) + return + } + assert.Equal(t, tc.exepctedVSC, actualVSC) + }) + } +} diff --git a/pkg/util/kube/pvc_pv.go b/pkg/util/kube/pvc_pv.go index 32551334b..6f40d2029 100644 --- a/pkg/util/kube/pvc_pv.go +++ b/pkg/util/kube/pvc_pv.go @@ -27,11 +27,11 @@ import ( "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + crclient "sigs.k8s.io/controller-runtime/pkg/client" storagev1api "k8s.io/api/storage/v1" storagev1 "k8s.io/client-go/kubernetes/typed/storage/v1" @@ -358,3 +358,31 @@ func MakePodPVCAttachment(volumeName string, volumeMode *corev1api.PersistentVol return volumeMounts, volumeDevices } + +func GetPVForPVC( + pvc *corev1api.PersistentVolumeClaim, + crClient crclient.Client, +) (*corev1api.PersistentVolume, error) { + if pvc.Spec.VolumeName == "" { + return nil, errors.Errorf("PVC %s/%s has no volume backing this claim", + pvc.Namespace, pvc.Name) + } + if pvc.Status.Phase != corev1api.ClaimBound { + // TODO: confirm if this PVC should be snapshotted if it has no PV bound + return nil, + errors.Errorf("PVC %s/%s is in phase %v and is not bound to a volume", + pvc.Namespace, pvc.Name, pvc.Status.Phase) + } + + pv := &corev1api.PersistentVolume{} + err := crClient.Get( + context.TODO(), + crclient.ObjectKey{Name: pvc.Spec.VolumeName}, + pv, + ) + if err != nil { + return nil, errors.Wrapf(err, "failed to get PV %s for PVC %s/%s", + pvc.Spec.VolumeName, pvc.Namespace, pvc.Name) + } + return pv, nil +} diff --git a/pkg/util/kube/pvc_pv_test.go b/pkg/util/kube/pvc_pv_test.go index 660dae376..a2eeea346 100644 --- a/pkg/util/kube/pvc_pv_test.go +++ b/pkg/util/kube/pvc_pv_test.go @@ -29,6 +29,7 @@ import ( "k8s.io/client-go/kubernetes/fake" corev1api "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" storagev1api "k8s.io/api/storage/v1" clientTesting "k8s.io/client-go/testing" @@ -1119,3 +1120,139 @@ func TestIsPVCBound(t *testing.T) { }) } } + +var ( + csiStorageClass = "csi-hostpath-sc" +) + +func TestGetPVForPVC(t *testing.T) { + boundPVC := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-csi-pvc", + Namespace: "default", + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Resources: v1.VolumeResourceRequirements{ + Requests: v1.ResourceList{}, + }, + StorageClassName: &csiStorageClass, + VolumeName: "test-csi-7d28e566-ade7-4ed6-9e15-2e44d2fbcc08", + }, + Status: v1.PersistentVolumeClaimStatus{ + Phase: v1.ClaimBound, + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Capacity: v1.ResourceList{}, + }, + } + matchingPV := &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-csi-7d28e566-ade7-4ed6-9e15-2e44d2fbcc08", + }, + Spec: v1.PersistentVolumeSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Capacity: v1.ResourceList{}, + ClaimRef: &v1.ObjectReference{ + Kind: "PersistentVolumeClaim", + Name: "test-csi-pvc", + Namespace: "default", + ResourceVersion: "1027", + UID: "7d28e566-ade7-4ed6-9e15-2e44d2fbcc08", + }, + PersistentVolumeSource: v1.PersistentVolumeSource{ + CSI: &v1.CSIPersistentVolumeSource{ + Driver: "hostpath.csi.k8s.io", + FSType: "ext4", + VolumeAttributes: map[string]string{ + "storage.kubernetes.io/csiProvisionerIdentity": "1582049697841-8081-hostpath.csi.k8s.io", + }, + VolumeHandle: "e61f2b48-527a-11ea-b54f-cab6317018f1", + }, + }, + PersistentVolumeReclaimPolicy: v1.PersistentVolumeReclaimDelete, + StorageClassName: csiStorageClass, + }, + Status: v1.PersistentVolumeStatus{ + Phase: v1.VolumeBound, + }, + } + + pvcWithNoVolumeName := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "no-vol-pvc", + Namespace: "default", + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Resources: v1.VolumeResourceRequirements{ + Requests: v1.ResourceList{}, + }, + StorageClassName: &csiStorageClass, + }, + Status: v1.PersistentVolumeClaimStatus{}, + } + + unboundPVC := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "unbound-pvc", + Namespace: "default", + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Resources: v1.VolumeResourceRequirements{ + Requests: v1.ResourceList{}, + }, + StorageClassName: &csiStorageClass, + VolumeName: "test-csi-7d28e566-ade7-4ed6-9e15-2e44d2fbcc08", + }, + Status: v1.PersistentVolumeClaimStatus{ + Phase: v1.ClaimPending, + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Capacity: v1.ResourceList{}, + }, + } + + testCases := []struct { + name string + inPVC *v1.PersistentVolumeClaim + expectError bool + expectedPV *v1.PersistentVolume + }{ + { + name: "should find PV matching the PVC", + inPVC: boundPVC, + expectError: false, + expectedPV: matchingPV, + }, + { + name: "should fail to find PV for PVC with no volumeName", + inPVC: pvcWithNoVolumeName, + expectError: true, + expectedPV: nil, + }, + { + name: "should fail to find PV for PVC not in bound phase", + inPVC: unboundPVC, + expectError: true, + expectedPV: nil, + }, + } + + objs := []runtime.Object{boundPVC, matchingPV, pvcWithNoVolumeName, unboundPVC} + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualPV, actualError := GetPVForPVC(tc.inPVC, fakeClient) + + if tc.expectError { + assert.NotNil(t, actualError, "Want error; Got nil error") + assert.Nilf(t, actualPV, "Want PV: nil; Got PV: %q", actualPV) + return + } + + assert.Nilf(t, actualError, "Want: nil error; Got: %v", actualError) + assert.Equalf(t, actualPV.Name, tc.expectedPV.Name, "Want PV with name %q; Got PV with name %q", tc.expectedPV.Name, actualPV.Name) + }) + } +} diff --git a/pkg/util/kube/utils.go b/pkg/util/kube/utils.go index bda40d35c..3eb853829 100644 --- a/pkg/util/kube/utils.go +++ b/pkg/util/kube/utils.go @@ -19,6 +19,7 @@ package kube import ( "context" "fmt" + "strings" "time" "github.com/pkg/errors" @@ -35,6 +36,8 @@ import ( corev1client "k8s.io/client-go/kubernetes/typed/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) @@ -303,3 +306,30 @@ func IsCRDReady(crd *unstructured.Unstructured) (bool, error) { return false, fmt.Errorf("unable to handle CRD with version %s", ver) } } + +// AddAnnotations adds the supplied key-values to the annotations on the object +func AddAnnotations(o *metav1.ObjectMeta, vals map[string]string) { + if o.Annotations == nil { + o.Annotations = make(map[string]string) + } + for k, v := range vals { + o.Annotations[k] = v + } +} + +// AddLabels adds the supplied key-values to the labels on the object +func AddLabels(o *metav1.ObjectMeta, vals map[string]string) { + if o.Labels == nil { + o.Labels = make(map[string]string) + } + for k, v := range vals { + o.Labels[k] = label.GetValidName(v) + } +} + +func HasBackupLabel(o *metav1.ObjectMeta, backupName string) bool { + if o.Labels == nil || len(strings.TrimSpace(backupName)) == 0 { + return false + } + return o.Labels[velerov1api.BackupNameLabel] == label.GetValidName(backupName) +} diff --git a/pkg/util/kube/utils_test.go b/pkg/util/kube/utils_test.go index 141fb45ce..f811fdfcc 100644 --- a/pkg/util/kube/utils_test.go +++ b/pkg/util/kube/utils_test.go @@ -481,3 +481,187 @@ func TestSinglePathMatch(t *testing.T) { assert.NotNil(t, err) assert.Contains(t, err.Error(), "expected one matching path") } + +func TestAddAnnotations(t *testing.T) { + annotationValues := map[string]string{ + "k1": "v1", + "k2": "v2", + "k3": "v3", + "k4": "v4", + "k5": "v5", + } + testCases := []struct { + name string + o metav1.ObjectMeta + toAdd map[string]string + }{ + { + name: "should create a new annotation map when annotation is nil", + o: metav1.ObjectMeta{ + Annotations: nil, + }, + toAdd: annotationValues, + }, + { + name: "should add all supplied annotations into empty annotation", + o: metav1.ObjectMeta{ + Annotations: map[string]string{}, + }, + toAdd: annotationValues, + }, + { + name: "should add all supplied annotations to existing annotation", + o: metav1.ObjectMeta{ + Annotations: map[string]string{ + "k100": "v100", + "k200": "v200", + "k300": "v300", + }, + }, + toAdd: annotationValues, + }, + { + name: "should overwrite some existing annotations", + o: metav1.ObjectMeta{ + Annotations: map[string]string{ + "k100": "v100", + "k2": "v200", + "k300": "v300", + }, + }, + toAdd: annotationValues, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + AddAnnotations(&tc.o, tc.toAdd) + for k, v := range tc.toAdd { + actual, exists := tc.o.Annotations[k] + assert.True(t, exists) + assert.Equal(t, v, actual) + } + }) + } +} + +func TestAddLabels(t *testing.T) { + labelValues := map[string]string{ + "l1": "v1", + "l2": "v2", + "l3": "v3", + "l4": "v4", + "l5": "v5", + } + testCases := []struct { + name string + o metav1.ObjectMeta + toAdd map[string]string + }{ + { + name: "should create a new labels map when labels is nil", + o: metav1.ObjectMeta{ + Labels: nil, + }, + toAdd: labelValues, + }, + { + name: "should add all supplied labels into empty labels", + o: metav1.ObjectMeta{ + Labels: map[string]string{}, + }, + toAdd: labelValues, + }, + { + name: "should add all supplied labels to existing labels", + o: metav1.ObjectMeta{ + Labels: map[string]string{ + "l100": "v100", + "l200": "v200", + "l300": "v300", + }, + }, + toAdd: labelValues, + }, + { + name: "should overwrite some existing labels", + o: metav1.ObjectMeta{ + Labels: map[string]string{ + "l100": "v100", + "l2": "v200", + "l300": "v300", + }, + }, + toAdd: labelValues, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + AddLabels(&tc.o, tc.toAdd) + for k, v := range tc.toAdd { + actual, exists := tc.o.Labels[k] + assert.True(t, exists) + assert.Equal(t, v, actual) + } + }) + } +} + +func TestHasBackupLabel(t *testing.T) { + testCases := []struct { + name string + o metav1.ObjectMeta + backupName string + expected bool + }{ + { + name: "object has no labels", + o: metav1.ObjectMeta{}, + expected: false, + }, + { + name: "object has no velero backup label", + backupName: "csi-b1", + o: metav1.ObjectMeta{ + Labels: map[string]string{ + "l100": "v100", + "l2": "v200", + "l300": "v300", + }, + }, + expected: false, + }, + { + name: "object has velero backup label but value not equal to backup name", + backupName: "csi-b1", + o: metav1.ObjectMeta{ + Labels: map[string]string{ + "velero.io/backup-name": "does-not-match", + "l100": "v100", + "l2": "v200", + "l300": "v300", + }, + }, + expected: false, + }, + { + name: "object has backup label with matching backup name value", + backupName: "does-match", + o: metav1.ObjectMeta{ + Labels: map[string]string{ + "velero.io/backup-name": "does-match", + "l100": "v100", + "l2": "v200", + "l300": "v300", + }, + }, + expected: true, + }, + } + + for _, tc := range testCases { + actual := HasBackupLabel(&tc.o, tc.backupName) + assert.Equal(t, tc.expected, actual) + } +} diff --git a/pkg/util/podvolume/pod_volume.go b/pkg/util/podvolume/pod_volume.go index a4107e6c2..20f991930 100644 --- a/pkg/util/podvolume/pod_volume.go +++ b/pkg/util/podvolume/pod_volume.go @@ -17,12 +17,16 @@ limitations under the License. package podvolume import ( + "context" "strings" + "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + crclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/util" ) // GetVolumesByPod returns a list of volume names to backup for the provided pod. @@ -62,7 +66,7 @@ func GetVolumesByPod(pod *corev1api.Pod, defaultVolumesToFsBackup, backupExclude continue } // don't backup volumes that are included in the exclude list. - if contains(volsToExclude, pv.Name) { + if util.Contains(volsToExclude, pv.Name) { optedOutVolumes = append(optedOutVolumes, pv.Name) continue } @@ -101,11 +105,58 @@ func getVolumesToExclude(obj metav1.Object) []string { return strings.Split(annotations[velerov1api.VolumesToExcludeAnnotation], ",") } -func contains(list []string, k string) bool { - for _, i := range list { - if i == k { - return true +func IsPVCDefaultToFSBackup(pvcNamespace, pvcName string, crClient crclient.Client, defaultVolumesToFsBackup bool) (bool, error) { + pods, err := getPodsUsingPVC(pvcNamespace, pvcName, crClient) + if err != nil { + return false, errors.WithStack(err) + } + + for index := range pods { + vols, _ := GetVolumesByPod(&pods[index], defaultVolumesToFsBackup, false) + if len(vols) > 0 { + volName, err := getPodVolumeNameForPVC(pods[index], pvcName) + if err != nil { + return false, err + } + if util.Contains(vols, volName) { + return true, nil + } } } - return false + + return false, nil +} + +func getPodVolumeNameForPVC(pod corev1api.Pod, pvcName string) (string, error) { + for _, v := range pod.Spec.Volumes { + if v.PersistentVolumeClaim != nil && v.PersistentVolumeClaim.ClaimName == pvcName { + return v.Name, nil + } + } + return "", errors.Errorf("Pod %s/%s does not use PVC %s/%s", pod.Namespace, pod.Name, pod.Namespace, pvcName) +} + +func getPodsUsingPVC( + pvcNamespace, pvcName string, + crClient crclient.Client, +) ([]corev1api.Pod, error) { + podsUsingPVC := []corev1api.Pod{} + podList := new(corev1api.PodList) + if err := crClient.List( + context.TODO(), + podList, + &crclient.ListOptions{Namespace: pvcNamespace}, + ); err != nil { + return nil, err + } + + for _, p := range podList.Items { + for _, v := range p.Spec.Volumes { + if v.PersistentVolumeClaim != nil && v.PersistentVolumeClaim.ClaimName == pvcName { + podsUsingPVC = append(podsUsingPVC, p) + } + } + } + + return podsUsingPVC, nil } diff --git a/pkg/util/podvolume/pod_volume_test.go b/pkg/util/podvolume/pod_volume_test.go index 19174e434..3bebb3cf6 100644 --- a/pkg/util/podvolume/pod_volume_test.go +++ b/pkg/util/podvolume/pod_volume_test.go @@ -22,9 +22,12 @@ import ( "github.com/stretchr/testify/assert" corev1api "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestGetVolumesToBackup(t *testing.T) { @@ -378,3 +381,414 @@ func TestGetVolumesByPod(t *testing.T) { }) } } + +func TestIsPVCDefaultToFSBackup(t *testing.T) { + objs := []runtime.Object{ + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "default", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "default", + Annotations: map[string]string{ + "backup.velero.io/backup-volumes": "csi-vol1", + }, + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod3", + Namespace: "default", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "awesome-pod-1", + Namespace: "awesome-ns", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "awesome-csi-pvc1", + }, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "awesome-pod-2", + Namespace: "awesome-ns", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "awesome-csi-pvc1", + }, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "uploader-ns", + Annotations: map[string]string{ + "backup.velero.io/backup-volumes": "csi-vol1", + }, + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "uploader-ns", + Annotations: map[string]string{ + "backup.velero.io/backup-volumes": "csi-vol1", + }, + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + } + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) + + testCases := []struct { + name string + inPVCNamespace string + inPVCName string + expectedIsFSUploaderUsed bool + defaultVolumesToFSBackup bool + }{ + { + name: "2 pods using PVC, 1 pod using uploader", + inPVCNamespace: "default", + inPVCName: "csi-pvc1", + expectedIsFSUploaderUsed: true, + defaultVolumesToFSBackup: false, + }, + { + name: "2 pods using PVC, 2 pods using uploader", + inPVCNamespace: "uploader-ns", + inPVCName: "csi-pvc1", + expectedIsFSUploaderUsed: true, + defaultVolumesToFSBackup: false, + }, + { + name: "2 pods using PVC, 0 pods using uploader", + inPVCNamespace: "awesome-ns", + inPVCName: "awesome-csi-pvc1", + expectedIsFSUploaderUsed: false, + defaultVolumesToFSBackup: false, + }, + { + name: "0 pods using PVC", + inPVCNamespace: "default", + inPVCName: "does-not-exist", + expectedIsFSUploaderUsed: false, + defaultVolumesToFSBackup: false, + }, + { + name: "2 pods using PVC, using uploader by default", + inPVCNamespace: "awesome-ns", + inPVCName: "awesome-csi-pvc1", + expectedIsFSUploaderUsed: true, + defaultVolumesToFSBackup: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualIsFSUploaderUsed, _ := IsPVCDefaultToFSBackup(tc.inPVCNamespace, tc.inPVCName, fakeClient, tc.defaultVolumesToFSBackup) + assert.Equal(t, tc.expectedIsFSUploaderUsed, actualIsFSUploaderUsed) + }) + } +} + +func TestGetPodVolumeNameForPVC(t *testing.T) { + testCases := []struct { + name string + pod v1.Pod + pvcName string + expectError bool + expectedVolumeName string + }{ + { + name: "should get volume name for pod with multuple PVCs", + pod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + { + Name: "csi-vol2", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc2", + }, + }, + }, + { + Name: "csi-vol3", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc3", + }, + }, + }, + }, + }, + }, + pvcName: "csi-pvc2", + expectedVolumeName: "csi-vol2", + expectError: false, + }, + { + name: "should get volume name from pod using exactly one PVC", + pod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + pvcName: "csi-pvc1", + expectedVolumeName: "csi-vol1", + expectError: false, + }, + { + name: "should return error for pod with no PVCs", + pod: v1.Pod{ + Spec: v1.PodSpec{}, + }, + pvcName: "csi-pvc2", + expectError: true, + }, + { + name: "should return error for pod with no matching PVC", + pod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + pvcName: "mismatch-pvc", + expectError: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualVolumeName, err := getPodVolumeNameForPVC(tc.pod, tc.pvcName) + if tc.expectError && err == nil { + assert.NotNil(t, err, "Want error; Got nil error") + return + } + assert.Equalf(t, tc.expectedVolumeName, actualVolumeName, "unexpected podVolumename returned. Want %s; Got %s", tc.expectedVolumeName, actualVolumeName) + }) + } +} + +func TestGetPodsUsingPVC(t *testing.T) { + objs := []runtime.Object{ + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "default", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "default", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod3", + Namespace: "default", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "awesome-ns", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + } + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) + + testCases := []struct { + name string + pvcNamespace string + pvcName string + expectedPodCount int + }{ + { + name: "should find exactly 2 pods using the PVC", + pvcNamespace: "default", + pvcName: "csi-pvc1", + expectedPodCount: 2, + }, + { + name: "should find exactly 1 pod using the PVC", + pvcNamespace: "awesome-ns", + pvcName: "csi-pvc1", + expectedPodCount: 1, + }, + { + name: "should find 0 pods using the PVC", + pvcNamespace: "default", + pvcName: "unused-pvc", + expectedPodCount: 0, + }, + { + name: "should find 0 pods in non-existent namespace", + pvcNamespace: "does-not-exist", + pvcName: "csi-pvc1", + expectedPodCount: 0, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualPods, err := getPodsUsingPVC(tc.pvcNamespace, tc.pvcName, fakeClient) + assert.Nilf(t, err, "Want error=nil; Got error=%v", err) + assert.Lenf(t, actualPods, tc.expectedPodCount, "unexpected number of pods in result; Want: %d; Got: %d", tc.expectedPodCount, len(actualPods)) + }) + } +} diff --git a/pkg/util/scheme.go b/pkg/util/scheme.go index 7e5703fac..c398e4196 100644 --- a/pkg/util/scheme.go +++ b/pkg/util/scheme.go @@ -1,3 +1,19 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package util import ( diff --git a/pkg/util/util.go b/pkg/util/util.go new file mode 100644 index 000000000..2aad8ccb1 --- /dev/null +++ b/pkg/util/util.go @@ -0,0 +1,26 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +func Contains(slice []string, key string) bool { + for _, i := range slice { + if i == key { + return true + } + } + return false +} diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go new file mode 100644 index 000000000..5eed5195f --- /dev/null +++ b/pkg/util/util_test.go @@ -0,0 +1,64 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestContains(t *testing.T) { + testCases := []struct { + name string + inSlice []string + inKey string + expectedResult bool + }{ + { + name: "should find the key", + inSlice: []string{"key1", "key2", "key3", "key4", "key5"}, + inKey: "key3", + expectedResult: true, + }, + { + name: "should not find the key in non-empty slice", + inSlice: []string{"key1", "key2", "key3", "key4", "key5"}, + inKey: "key300", + expectedResult: false, + }, + { + name: "should not find key in empty slice", + inSlice: []string{}, + inKey: "key300", + expectedResult: false, + }, + { + name: "should not find key in nil slice", + inSlice: nil, + inKey: "key300", + expectedResult: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualResult := Contains(tc.inSlice, tc.inKey) + assert.Equal(t, tc.expectedResult, actualResult) + }) + } +} diff --git a/test/e2e/resource-filtering/exclude_label.go b/test/e2e/resource-filtering/exclude_label.go index afa3f2b0d..ecf26878e 100644 --- a/test/e2e/resource-filtering/exclude_label.go +++ b/test/e2e/resource-filtering/exclude_label.go @@ -26,6 +26,7 @@ import ( "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" . "github.com/vmware-tanzu/velero/test/e2e/test" . "github.com/vmware-tanzu/velero/test/util/k8s" ) @@ -57,9 +58,9 @@ func (e *ExcludeFromBackup) Init() error { *e.NSIncluded = append(*e.NSIncluded, createNSName) } e.labels = map[string]string{ - "velero.io/exclude-from-backup": "true", + velerov1api.ExcludeFromBackupLabel: "true", } - e.labelSelector = "velero.io/exclude-from-backup" + e.labelSelector = velerov1api.ExcludeFromBackupLabel e.BackupArgs = []string{ "create", "--namespace", e.VeleroCfg.VeleroNamespace, "backup", e.BackupName, @@ -82,7 +83,7 @@ func (e *ExcludeFromBackup) CreateResources() error { "meaningless-label-resource-to-include": "true", } label2 := map[string]string{ - "velero.io/exclude-from-backup": "false", + velerov1api.ExcludeFromBackupLabel: "false", } fmt.Printf("Creating resources in namespace ...%s\n", namespace) if err := CreateNamespace(e.Ctx, e.Client, namespace); err != nil {