From 00b98693697fb96a34aa0690f4f27c3605cc9e66 Mon Sep 17 00:00:00 2001 From: lou Date: Thu, 14 Mar 2024 15:21:00 +0800 Subject: [PATCH 01/16] do not ignore unknown gvr at the beginning and get new gr when kind is changed Signed-off-by: lou --- pkg/restore/restore.go | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/pkg/restore/restore.go b/pkg/restore/restore.go index 3c30e4cd2..7cc4fa7ac 100644 --- a/pkg/restore/restore.go +++ b/pkg/restore/restore.go @@ -1113,6 +1113,7 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso // itemExists bool is used to determine whether to include this item in the "wait for additional items" list itemExists := false resourceID := getResourceID(groupResource, namespace, obj.GetName()) + resourceKind := obj.GetKind() restoreLogger := ctx.log.WithFields(logrus.Fields{ "namespace": obj.GetNamespace(), @@ -1217,13 +1218,13 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso return warnings, errs, itemExists } - resourceClient, err := ctx.getResourceClient(groupResource, obj, namespace) - if err != nil { - errs.AddVeleroError(fmt.Errorf("error getting resource client for namespace %q, resource %q: %v", namespace, &groupResource, err)) - return warnings, errs, itemExists - } - if groupResource == kuberesource.PersistentVolumes { + resourceClient, err := ctx.getResourceClient(groupResource, obj, namespace) + if err != nil { + errs.AddVeleroError(fmt.Errorf("error getting resource client for namespace %q, resource %q: %v", namespace, &groupResource, err)) + return warnings, errs, itemExists + } + if volumeInfo, ok := ctx.volumeInfoMap[obj.GetName()]; ok { ctx.log.Infof("Find VolumeInfo for PV %s.", obj.GetName()) @@ -1504,9 +1505,18 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso // The object apiVersion might get modified by a RestorePlugin so we need to // get a new client to reflect updated resource path. newGR := schema.GroupResource{Group: obj.GroupVersionKind().Group, Resource: groupResource.Resource} - resourceClient, err = ctx.getResourceClient(newGR, obj, obj.GetNamespace()) + if obj.GetKind() != resourceKind { + ctx.log.Infof("Resource kind changed from %s to %s", resourceKind, obj.GetKind()) + gvr, _, err := ctx.discoveryHelper.KindFor(obj.GroupVersionKind()) + if err != nil { + errs.Add(namespace, fmt.Errorf("error getting GVR for %s: %v", obj.GroupVersionKind(), err)) + return warnings, errs, itemExists + } + newGR.Resource = gvr.Resource + } + resourceClient, err := ctx.getResourceClient(newGR, obj, obj.GetNamespace()) if err != nil { - errs.AddVeleroError(fmt.Errorf("error getting updated resource client for namespace %q, resource %q: %v", namespace, &groupResource, err)) + warnings.Add(namespace, fmt.Errorf("error getting updated resource client for namespace %q, resource %q: %v", namespace, &groupResource, err)) return warnings, errs, itemExists } @@ -2184,13 +2194,14 @@ func (ctx *restoreContext) getOrderedResourceCollection( resourceList = resourcePriorities.HighPriorities } for _, resource := range resourceList { + groupResource := schema.ParseGroupResource(resource) // try to resolve the resource via discovery to a complete group/version/resource - gvr, _, err := ctx.discoveryHelper.ResourceFor(schema.ParseGroupResource(resource).WithVersion("")) + gvr, _, err := ctx.discoveryHelper.ResourceFor(groupResource.WithVersion("")) if err != nil { - ctx.log.WithField("resource", resource).Infof("Skipping restore of resource because it cannot be resolved via discovery") - continue + ctx.log.WithField("resource", resource).Infof("resource cannot be resolved via discovery") + } else { + groupResource = gvr.GroupResource() } - groupResource := gvr.GroupResource() // Check if we've already restored this resource (this would happen if // the resource we're currently looking at was already restored because From 25c006f5364af8172b7147ad36bf7fb8ca501eb8 Mon Sep 17 00:00:00 2001 From: lou Date: Thu, 14 Mar 2024 15:27:44 +0800 Subject: [PATCH 02/16] add changelog Signed-off-by: lou --- changelogs/unreleased/7520-27149chen | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/unreleased/7520-27149chen diff --git a/changelogs/unreleased/7520-27149chen b/changelogs/unreleased/7520-27149chen new file mode 100644 index 000000000..4704322ed --- /dev/null +++ b/changelogs/unreleased/7520-27149chen @@ -0,0 +1 @@ +do not skip unknown gvr at the beginning and get new gr when kind is changed From f25004cd9cbfdc545742f8083d005cead0b9ec4d Mon Sep 17 00:00:00 2001 From: lou Date: Thu, 14 Mar 2024 15:29:57 +0800 Subject: [PATCH 03/16] fix changelog Signed-off-by: lou --- changelogs/unreleased/{7520-27149chen => 7523-27149chen} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelogs/unreleased/{7520-27149chen => 7523-27149chen} (100%) diff --git a/changelogs/unreleased/7520-27149chen b/changelogs/unreleased/7523-27149chen similarity index 100% rename from changelogs/unreleased/7520-27149chen rename to changelogs/unreleased/7523-27149chen From 19e5f38cbc973f2cb564e21a36fe1d04da94da04 Mon Sep 17 00:00:00 2001 From: lou Date: Mon, 25 Mar 2024 12:42:28 +0800 Subject: [PATCH 04/16] update after review Signed-off-by: lou --- pkg/restore/restore.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/restore/restore.go b/pkg/restore/restore.go index 7cc4fa7ac..054896dd5 100644 --- a/pkg/restore/restore.go +++ b/pkg/restore/restore.go @@ -1505,6 +1505,9 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso // The object apiVersion might get modified by a RestorePlugin so we need to // get a new client to reflect updated resource path. newGR := schema.GroupResource{Group: obj.GroupVersionKind().Group, Resource: groupResource.Resource} + // obj kind might change within a special RIA which is used to convert objects, + // like from Openshift DeploymentConfig to native Deployment. + // we should re-get the newGR.Resource again in such a case if obj.GetKind() != resourceKind { ctx.log.Infof("Resource kind changed from %s to %s", resourceKind, obj.GetKind()) gvr, _, err := ctx.discoveryHelper.KindFor(obj.GroupVersionKind()) @@ -2198,6 +2201,8 @@ func (ctx *restoreContext) getOrderedResourceCollection( // try to resolve the resource via discovery to a complete group/version/resource gvr, _, err := ctx.discoveryHelper.ResourceFor(groupResource.WithVersion("")) if err != nil { + // don't skip if we can't resolve the resource via discovery, log it + // the gv of this resource may be changed in a RIA, we can try to get it after that ctx.log.WithField("resource", resource).Infof("resource cannot be resolved via discovery") } else { groupResource = gvr.GroupResource() From 35d2534e192f952f5671ee2daa7a9039f0ba88df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wenkai=20Yin=28=E5=B0=B9=E6=96=87=E5=BC=80=29?= Date: Wed, 27 Mar 2024 11:23:39 +0800 Subject: [PATCH 05/16] Check the existence of the namespaces provided in the "--include-namespaces" option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Check the existence of the namespaces provided in the "--include-namespaces" opt ion and reports validation error if not found Fixes #7431 Signed-off-by: Wenkai Yin(尹文开) --- changelogs/unreleased/7569-ywk253100 | 1 + pkg/controller/backup_controller.go | 20 +++++++++- pkg/controller/backup_controller_test.go | 50 ++++++++++++++++++++++-- 3 files changed, 67 insertions(+), 4 deletions(-) create mode 100644 changelogs/unreleased/7569-ywk253100 diff --git a/changelogs/unreleased/7569-ywk253100 b/changelogs/unreleased/7569-ywk253100 new file mode 100644 index 000000000..e16d8551c --- /dev/null +++ b/changelogs/unreleased/7569-ywk253100 @@ -0,0 +1 @@ +Check the existence of the namespaces provided in the "--include-namespaces" option \ No newline at end of file diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index da915ed7e..4a6656b88 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -464,7 +464,7 @@ func (b *backupReconciler) prepareBackupRequest(backup *velerov1api.Backup, logg } // validate the included/excluded namespaces - for _, err := range collections.ValidateNamespaceIncludesExcludes(request.Spec.IncludedNamespaces, request.Spec.ExcludedNamespaces) { + for _, err := range b.validateNamespaceIncludesExcludes(request.Spec.IncludedNamespaces, request.Spec.ExcludedNamespaces) { request.Status.ValidationErrors = append(request.Status.ValidationErrors, fmt.Sprintf("Invalid included/excluded namespace lists: %v", err)) } @@ -601,6 +601,24 @@ func (b *backupReconciler) validateAndGetSnapshotLocations(backup *velerov1api.B return providerLocations, nil } +func (b *backupReconciler) validateNamespaceIncludesExcludes(includedNamespaces, excludedNamespaces []string) []error { + var errs []error + if errs = collections.ValidateNamespaceIncludesExcludes(includedNamespaces, excludedNamespaces); len(errs) > 0 { + return errs + } + + namespace := &corev1api.Namespace{} + for _, name := range collections.NewIncludesExcludes().Includes(includedNamespaces...).GetIncludes() { + if name == "" || name == "*" { + continue + } + if err := b.kbClient.Get(context.Background(), kbclient.ObjectKey{Name: name}, namespace); err != nil { + errs = append(errs, err) + } + } + return errs +} + // runBackup runs and uploads a validated backup. Any error returned from this function // causes the backup to be Failed; if no error is returned, the backup's status's Errors // field is checked to see if the backup was a partial failure. diff --git a/pkg/controller/backup_controller_test.go b/pkg/controller/backup_controller_test.go index 7dd392299..3ebd465e0 100644 --- a/pkg/controller/backup_controller_test.go +++ b/pkg/controller/backup_controller_test.go @@ -185,10 +185,16 @@ func TestProcessBackupValidationFailures(t *testing.T) { }, { name: "use old filter parameters and new filter parameters together", - backup: defaultBackup().IncludeClusterResources(true).IncludedNamespaceScopedResources("Deployment").IncludedNamespaces("default").Result(), + backup: defaultBackup().IncludeClusterResources(true).IncludedNamespaceScopedResources("Deployment").IncludedNamespaces("foo").Result(), backupLocation: defaultBackupLocation, expectedErrs: []string{"include-resources, exclude-resources and include-cluster-resources are old filter parameters.\ninclude-cluster-scoped-resources, exclude-cluster-scoped-resources, include-namespace-scoped-resources and exclude-namespace-scoped-resources are new filter parameters.\nThey cannot be used together"}, }, + { + name: "nonexisting namespace", + backup: defaultBackup().IncludedNamespaces("non-existing").Result(), + backupLocation: defaultBackupLocation, + expectedErrs: []string{"Invalid included/excluded namespace lists: namespaces \"non-existing\" not found"}, + }, } for _, test := range tests { @@ -203,10 +209,11 @@ func TestProcessBackupValidationFailures(t *testing.T) { require.NoError(t, err) var fakeClient kbclient.Client + namespace := builder.ForNamespace("foo").Result() if test.backupLocation != nil { - fakeClient = velerotest.NewFakeControllerRuntimeClient(t, test.backupLocation) + fakeClient = velerotest.NewFakeControllerRuntimeClient(t, test.backupLocation, namespace) } else { - fakeClient = velerotest.NewFakeControllerRuntimeClient(t) + fakeClient = velerotest.NewFakeControllerRuntimeClient(t, namespace) } c := &backupReconciler{ @@ -1563,6 +1570,43 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) { } } +func TestValidateNamespaceIncludesExcludes(t *testing.T) { + namespace := builder.ForNamespace("default").Result() + reconciler := &backupReconciler{ + kbClient: velerotest.NewFakeControllerRuntimeClient(t, namespace), + } + + // empty string as includedNamespaces + includedNamespaces := []string{""} + excludedNamespaces := []string{"test"} + errs := reconciler.validateNamespaceIncludesExcludes(includedNamespaces, excludedNamespaces) + assert.Empty(t, errs) + + // "*" as includedNamespaces + includedNamespaces = []string{"*"} + excludedNamespaces = []string{"test"} + errs = reconciler.validateNamespaceIncludesExcludes(includedNamespaces, excludedNamespaces) + assert.Empty(t, errs) + + // invalid namespaces + includedNamespaces = []string{"1@#"} + excludedNamespaces = []string{"2@#"} + errs = reconciler.validateNamespaceIncludesExcludes(includedNamespaces, excludedNamespaces) + assert.Len(t, errs, 2) + + // not exist namespaces + includedNamespaces = []string{"non-existing-namespace"} + excludedNamespaces = []string{} + errs = reconciler.validateNamespaceIncludesExcludes(includedNamespaces, excludedNamespaces) + assert.Len(t, errs, 1) + + // valid namespaces + includedNamespaces = []string{"default"} + excludedNamespaces = []string{} + errs = reconciler.validateNamespaceIncludesExcludes(includedNamespaces, excludedNamespaces) + assert.Empty(t, errs) +} + // Test_getLastSuccessBySchedule verifies that the getLastSuccessBySchedule helper function correctly returns // the completion timestamp of the most recent completed backup for each schedule, including an entry for ad-hoc // or non-scheduled backups. From a9085033b2c7b3aea91b3421eb840edb2b738242 Mon Sep 17 00:00:00 2001 From: Matthieu MOREL Date: Thu, 28 Mar 2024 21:15:27 +0000 Subject: [PATCH 06/16] build(deps): bump json-patch to v5.8.0 Signed-off-by: Matthieu MOREL --- changelogs/unreleased/7584-mmorel-35 | 1 + go.mod | 4 ++-- internal/resourcemodifiers/json_merge_patch.go | 2 +- internal/resourcemodifiers/json_patch.go | 2 +- internal/resourcemodifiers/resource_modifiers.go | 2 +- pkg/cmd/cli/plugin/add.go | 2 +- pkg/cmd/cli/plugin/remove.go | 2 +- pkg/controller/backup_deletion_controller.go | 2 +- pkg/restore/merge_service_account.go | 2 +- pkg/util/csi/volume_snapshot.go | 2 +- pkg/util/kube/pvc_pv.go | 2 +- 11 files changed, 12 insertions(+), 11 deletions(-) create mode 100644 changelogs/unreleased/7584-mmorel-35 diff --git a/changelogs/unreleased/7584-mmorel-35 b/changelogs/unreleased/7584-mmorel-35 new file mode 100644 index 000000000..cf76f38ac --- /dev/null +++ b/changelogs/unreleased/7584-mmorel-35 @@ -0,0 +1 @@ +build(deps): bump json-patch to v5.8.0 diff --git a/go.mod b/go.mod index 500607e57..85baa28b5 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.48.0 github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 github.com/bombsimon/logrusr/v3 v3.0.0 - github.com/evanphx/json-patch v5.6.0+incompatible + github.com/evanphx/json-patch/v5 v5.8.0 github.com/fatih/color v1.16.0 github.com/gobwas/glob v0.2.3 github.com/golang/protobuf v1.5.4 @@ -105,7 +105,7 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/edsrzf/mmap-go v1.1.0 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/evanphx/json-patch/v5 v5.8.0 // indirect + github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/logr v1.4.1 // indirect diff --git a/internal/resourcemodifiers/json_merge_patch.go b/internal/resourcemodifiers/json_merge_patch.go index 5082e2cf5..ea6882f0b 100644 --- a/internal/resourcemodifiers/json_merge_patch.go +++ b/internal/resourcemodifiers/json_merge_patch.go @@ -3,7 +3,7 @@ package resourcemodifiers import ( "fmt" - jsonpatch "github.com/evanphx/json-patch" + jsonpatch "github.com/evanphx/json-patch/v5" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "sigs.k8s.io/yaml" diff --git a/internal/resourcemodifiers/json_patch.go b/internal/resourcemodifiers/json_patch.go index d137d40be..30563f8d9 100644 --- a/internal/resourcemodifiers/json_patch.go +++ b/internal/resourcemodifiers/json_patch.go @@ -6,7 +6,7 @@ import ( "strconv" "strings" - jsonpatch "github.com/evanphx/json-patch" + jsonpatch "github.com/evanphx/json-patch/v5" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) diff --git a/internal/resourcemodifiers/resource_modifiers.go b/internal/resourcemodifiers/resource_modifiers.go index d4ae93e71..c11b99cd8 100644 --- a/internal/resourcemodifiers/resource_modifiers.go +++ b/internal/resourcemodifiers/resource_modifiers.go @@ -19,7 +19,7 @@ import ( "fmt" "regexp" - jsonpatch "github.com/evanphx/json-patch" + jsonpatch "github.com/evanphx/json-patch/v5" "github.com/gobwas/glob" "github.com/pkg/errors" "github.com/sirupsen/logrus" diff --git a/pkg/cmd/cli/plugin/add.go b/pkg/cmd/cli/plugin/add.go index 7c0b98929..4abc6c5bb 100644 --- a/pkg/cmd/cli/plugin/add.go +++ b/pkg/cmd/cli/plugin/add.go @@ -22,7 +22,7 @@ import ( "fmt" "strings" - jsonpatch "github.com/evanphx/json-patch" + jsonpatch "github.com/evanphx/json-patch/v5" "github.com/pkg/errors" "github.com/spf13/cobra" corev1api "k8s.io/api/core/v1" diff --git a/pkg/cmd/cli/plugin/remove.go b/pkg/cmd/cli/plugin/remove.go index 41543ad9b..ed25dc680 100644 --- a/pkg/cmd/cli/plugin/remove.go +++ b/pkg/cmd/cli/plugin/remove.go @@ -20,7 +20,7 @@ import ( "context" "encoding/json" - jsonpatch "github.com/evanphx/json-patch" + jsonpatch "github.com/evanphx/json-patch/v5" "github.com/pkg/errors" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/controller/backup_deletion_controller.go b/pkg/controller/backup_deletion_controller.go index 03a9e379a..a37f1f9e8 100644 --- a/pkg/controller/backup_deletion_controller.go +++ b/pkg/controller/backup_deletion_controller.go @@ -22,7 +22,7 @@ import ( "fmt" "time" - jsonpatch "github.com/evanphx/json-patch" + jsonpatch "github.com/evanphx/json-patch/v5" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" diff --git a/pkg/restore/merge_service_account.go b/pkg/restore/merge_service_account.go index a5eddb3b4..f77a7730a 100644 --- a/pkg/restore/merge_service_account.go +++ b/pkg/restore/merge_service_account.go @@ -19,7 +19,7 @@ package restore import ( "encoding/json" - jsonpatch "github.com/evanphx/json-patch" + jsonpatch "github.com/evanphx/json-patch/v5" "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" diff --git a/pkg/util/csi/volume_snapshot.go b/pkg/util/csi/volume_snapshot.go index a9a88fd32..1abafaa35 100644 --- a/pkg/util/csi/volume_snapshot.go +++ b/pkg/util/csi/volume_snapshot.go @@ -22,7 +22,7 @@ import ( "fmt" "time" - jsonpatch "github.com/evanphx/json-patch" + jsonpatch "github.com/evanphx/json-patch/v5" "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/util/kube/pvc_pv.go b/pkg/util/kube/pvc_pv.go index 32551334b..6395302d6 100644 --- a/pkg/util/kube/pvc_pv.go +++ b/pkg/util/kube/pvc_pv.go @@ -22,7 +22,7 @@ import ( "fmt" "time" - jsonpatch "github.com/evanphx/json-patch" + jsonpatch "github.com/evanphx/json-patch/v5" "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" From ab5ee7b6ff6875d91c8a44268a0c9cec6dc8003a Mon Sep 17 00:00:00 2001 From: Daniel Jiang Date: Fri, 22 Mar 2024 15:41:50 +0800 Subject: [PATCH 07/16] Add design to introduce restore volume info Signed-off-by: Daniel Jiang --- changelogs/unreleased/7610-reasonerjt | 1 + design/pv_restore_info.md | 143 ++++++++++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100644 changelogs/unreleased/7610-reasonerjt create mode 100644 design/pv_restore_info.md diff --git a/changelogs/unreleased/7610-reasonerjt b/changelogs/unreleased/7610-reasonerjt new file mode 100644 index 000000000..be5ac3949 --- /dev/null +++ b/changelogs/unreleased/7610-reasonerjt @@ -0,0 +1 @@ +Add design to introduce restore volume info \ No newline at end of file diff --git a/design/pv_restore_info.md b/design/pv_restore_info.md new file mode 100644 index 000000000..d2e49655c --- /dev/null +++ b/design/pv_restore_info.md @@ -0,0 +1,143 @@ +# Volume information for restore design + +## Background +Velero has different ways to handle data in the volumes during restore. The users want to have more clarity in terms of how +the volumes are handled in restore process via either Velero CLI or other downstream product which consumes Velero. + +## Goals +- Create new metadata to store the information of the restored volume, which will have the same life-cycle as the restore CR. +- Consume the metadata in velero CLI to enable it display more details for volumes in the output of `velero restore describe --details` + +## Non Goals +- Provide finer grained control of the volume restore process. The focus of the design is to enable displaying more details. +- Persist additional metadata like podvolume, datadownloads etc to the restore folder in backup-location. + +## Design + +### Structure of the restore volume info +The restore volume info will be stored in a file named like `${restore_name}-vol-info.json`. The content of the file will +be a list of volume info objects, each of which will map to a volume that is restored, and will contain the information +like name of the restored PV/PVC, restore method and related objects to provide details depending on the way it's restored, +it will look like this: +``` +[ + { + "pvcName": "nginx-logs-2", + "pvcNamespace": "nginx-app-restore", + "pvName": "pvc-e320d75b-a788-41a3-b6ba-267a553efa5e", + "restoreMethod": "PodVolumeRestore", + "snapshotDataMoved": false, + "pvrInfo": { + "snapshotHandle": "81973157c3a945a5229285c931b02c68", + "uploaderType": "kopia", + "volumeName": "nginx-logs", + "podName": "nginx-deployment-79b56c644b-mjdhp", + "podNamespace": "nginx-app-restore" + } + }, + { + "pvcName": "nginx-logs-1", + "pvcNamespace": "nginx-app-restore", + "pvName": "pvc-98c151f4-df47-4980-ba6d-470842f652cc", + "restoreMethod": "CSISnapshot", + "snapshotDataMoved": false, + "csiSnapshotInfo": { + "snapshotHandle": "snap-01a3b21a5e9f85528", + "size": 2147483648, + "driver": "ebs.csi.aws.com", + "vscName": "velero-velero-nginx-logs-1-jxmbg-hx9x5" + } + } +...... +] +``` +Each field will have the same meaning as the corresponding field in the backup volume info. It will not have the fields +that were introduced to help with the backup process, like `pvInfo`, `dataupload` etc. + +### How the restore volume info is generated +Two steps are involved in generating the restore volume info, the first is "collection", which is to gather the information +for restoration of the volumes, the second is "generation", which is to iterate through the data collected in the first step +and generate the volume info list as is described above. + +Unlike backup, the CR objects created during the restore process will not be persisted to the backup storage location. +Therefore, to gather the information needed to generate volume information, we either need to collect the CRs in the middle +of the restore process, or retrieve the objects based on the `resouce-list.json` of the restore via API server. +The information to be collected are: +- **PV/PVC mapping relationship:** It will be collected via the `restore-resource-list.json`, b/c at the time the json is ready, all +PVCs and PVs are already created. +- **Native snapshot information:** It will be collected in the restore workflow when each snapshot is restored. +- **podvolumerestore CRs:** It will be collected in the restore workflow after each pvr is created. +- **volumesnapshot CRs for CSI snapshot:** It will be collected in the step of collecting PVC info, by reading the `dataSource` +field in the spec of the PVC. +- **datadownload CRs** It will be collected in the phase of collecting PVC info, by querying the API-server to list the datadownload +CRs labeled with the restore name. + +After the collection step, the generation step is relatively straight-forward, as we have all the information needed in +the data structures. + +The whole collection and generation steps will be done with the "best-effort" manner, i.e. if there are any failures we +will only log the error in restore log, rather than failing the whole restore process, we will not put these errors or warnings +into the `result.json`, b/c it won't impact the restored resources. + +Depending on the number of the restored PVCs the "collection" step may involve many API calls, but it's considered acceptable +b/c at that time the resources are already created, so the actual RTO is not impacted. By using the client of controller runtime +we can make the collection step more efficient by using the cache of the API server. We may consider to make improvements if +we observe performance issues, like using multiple go-routines in the collection. + +### Implementation +Because the restore volume info shares the same data structures with the backup volume info, we will refactor the code in +package `internal/volume` to make the sub-components in backup volume info shared by both backup and restore volume info. + +We'll introduce a struct called `RestoreVolumeInfoTracker` which encapsulates the logic of collecting and generating the restore volume info: +``` +// RestoreVolumeInfoTracker is used to track the volume information during restore. +// It is used to generate the RestoreVolumeInfo array. +type RestoreVolumeInfoTracker struct { + *sync.Mutex + restore *velerov1api.Restore + log logrus.FieldLogger + client kbclient.Client + pvPvc *pvcPvMap + + // map of PV name to the NativeSnapshotInfo from which the PV is restored + pvNativeSnapshotMap map[string]NativeSnapshotInfo + // map of PV name to the CSISnapshot object from which the PV is restored + pvCSISnapshotMap map[string]snapshotv1api.VolumeSnapshot + datadownloadList *velerov2alpha1.DataDownloadList + pvrs []*velerov1api.PodVolumeRestore +} +``` +The `RestoreVolumeInfoTracker` will be created when the restore request is initialized, and it will be passed to the `restoreContext` +and carried over the whole restore process. + +The `client` in this struct is to be used to query the resources in the restored namespace, and the current client in restore +reconciler only watches the resources in the namespace where velero is installed. Therefore, we need to introduce the +`CrClient` which has the same life-cycle of velero server to the restore reconciler, because this is the client that watches all the +resources on the cluster. + +In addition to that, we will make small changes in the restore workflow to collect the information needed. We'll make the +changes un-intrusive and make sure not to change the logic of the restore to avoid break change or regression. +We'll also introduce routine changes in the package `pkg/persistence` to persist the restore volume info to the backup storage location. + +Last but not least, the `velero restore describe --details` will be updated to display the volume info in the output. + +## Alternatives Considered +There used to be suggestion that to provide more details about volume, we can query the `backup-vol-info.json` with the resource +identifier in `restore-resource-list.json`. This will not work when there're resource modifiers involved in the restore process, +which may change the metadata of PVC/PV. In addition, we may add more detailed restore-specific information about the volumes that is not available +in the `backup-vol-info.json`. Therefore, the `restore-vol-info.json` is a better approach. + +## Security Considerations +There should be no security impact introduced by this design. + +## Compatibility +The restore volume info will be consumed by Velero CLI and downstream products for displaying details. So the functionality +of backup and restore will not be impacted for restores created by older versions of Velero which do not have the restore volume info +metadata. The client should properly handle the case when the restore volume info does not exist. + +The data structures referenced by volume info is shared between both restore and backup and it's not versioned, so in the future +we must make sure there will only be incremental changes to the metadata, such that no break change will be introduced to the client. + +## Open Issues +https://github.com/vmware-tanzu/velero/issues/7546 +https://github.com/vmware-tanzu/velero/issues/6478 From 91774af54d1d2b17bb78901964ca00c5f231e8cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wenkai=20Yin=28=E5=B0=B9=E6=96=87=E5=BC=80=29?= Date: Mon, 8 Apr 2024 13:54:04 +0800 Subject: [PATCH 08/16] Empty the list before next round of listing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Empty the list before next round of listing Signed-off-by: Wenkai Yin(尹文开) --- pkg/util/kube/periodical_enqueue_source.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/util/kube/periodical_enqueue_source.go b/pkg/util/kube/periodical_enqueue_source.go index 885767f60..b4c4f27b8 100644 --- a/pkg/util/kube/periodical_enqueue_source.go +++ b/pkg/util/kube/periodical_enqueue_source.go @@ -69,6 +69,11 @@ type PeriodicalEnqueueSourceOption struct { func (p *PeriodicalEnqueueSource) Start(ctx context.Context, h handler.EventHandler, q workqueue.RateLimitingInterface, predicates ...predicate.Predicate) error { go wait.Until(func() { p.logger.Debug("enqueueing resources ...") + // empty the list otherwise the result of the new list call will be appended + if err := meta.SetList(p.objList, nil); err != nil { + p.logger.WithError(err).Error("error reset resource list") + return + } if err := p.List(ctx, p.objList); err != nil { p.logger.WithError(err).Error("error listing resources") return From c36dc9263e9b10a72e0e2b10b1127ff545970e84 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Apr 2024 02:03:09 +0000 Subject: [PATCH 09/16] Bump google-github-actions/setup-gcloud from 0 to 2 Bumps [google-github-actions/setup-gcloud](https://github.com/google-github-actions/setup-gcloud) from 0 to 2. - [Release notes](https://github.com/google-github-actions/setup-gcloud/releases) - [Changelog](https://github.com/google-github-actions/setup-gcloud/blob/main/CHANGELOG.md) - [Commits](https://github.com/google-github-actions/setup-gcloud/compare/v0...v2) --- updated-dependencies: - dependency-name: google-github-actions/setup-gcloud dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 06e74f7d0..5b876f92e 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -28,7 +28,7 @@ jobs: sudo apt-get install python2.7 export CLOUDSDK_PYTHON="/usr/bin/python2" - - uses: google-github-actions/setup-gcloud@v0 + - uses: google-github-actions/setup-gcloud@v2 with: version: '285.0.0' service_account_key: ${{ secrets.GCS_SA_KEY }} From 9274f4b664851120a22cf7533c0ca331823b5239 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 9 Apr 2024 02:03:14 +0000 Subject: [PATCH 10/16] Bump docker/setup-buildx-action from 1 to 3 Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 1 to 3. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v1...v3) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/pr-containers.yml | 2 +- .github/workflows/push.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-containers.yml b/.github/workflows/pr-containers.yml index c4d21f6d4..8dfd226b8 100644 --- a/.github/workflows/pr-containers.yml +++ b/.github/workflows/pr-containers.yml @@ -24,7 +24,7 @@ jobs: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 with: version: latest diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 06e74f7d0..484b23927 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -43,7 +43,7 @@ jobs: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 with: version: latest From 080a61b43d48ca3e61c3d8d445fba76a3bf1443e Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Wed, 3 Apr 2024 14:36:39 +0800 Subject: [PATCH 11/16] data mover node selection doc Signed-off-by: Lyndon-Li --- changelogs/unreleased/7640-Lyndon-Li | 1 + .../docs/main/csi-snapshot-data-movement.md | 19 +++++- .../data-movement-backup-node-selection.md | 60 +++++++++++++++++++ 3 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 changelogs/unreleased/7640-Lyndon-Li create mode 100644 site/content/docs/main/data-movement-backup-node-selection.md diff --git a/changelogs/unreleased/7640-Lyndon-Li b/changelogs/unreleased/7640-Lyndon-Li new file mode 100644 index 000000000..4483c4ff1 --- /dev/null +++ b/changelogs/unreleased/7640-Lyndon-Li @@ -0,0 +1 @@ +For issue #7036, add the document for data mover node selection \ No newline at end of file diff --git a/site/content/docs/main/csi-snapshot-data-movement.md b/site/content/docs/main/csi-snapshot-data-movement.md index fb113fa18..11d0763d7 100644 --- a/site/content/docs/main/csi-snapshot-data-movement.md +++ b/site/content/docs/main/csi-snapshot-data-movement.md @@ -360,8 +360,8 @@ Velero calls the CSI plugin concurrently for the volume, so `DataUpload`/`DataDo In which manner the `DataUpload`/`DataDownload` CRs are processed is totally decided by the data mover you select for the backup/restore. For Velero built-in data mover, it uses Kubernetes' scheduler to mount a snapshot volume/restore volume associated to a `DataUpload`/`DataDownload` CR into a specific node, and then the `DataUpload`/`DataDownload` controller (in node-agent daemonset) in that node will handle the `DataUpload`/`DataDownload`. -At present, a `DataUpload`/`DataDownload` controller in one node handles one request at a time. -That is to say, the snapshot volumes/restore volumes may spread in different nodes, then their associated `DataUpload`/`DataDownload` CRs will be processed in parallel; while for the snapshot volumes/restore volumes in the same node, their associated `DataUpload`/`DataDownload` CRs are processed sequentially. +By default, a `DataUpload`/`DataDownload` controller in one node handles one request at a time. You can configure more parallelism per node by [node-agent Concurrency Configuration][14]. +That is to say, the snapshot volumes/restore volumes may spread in different nodes, then their associated `DataUpload`/`DataDownload` CRs will be processed in parallel; while for the snapshot volumes/restore volumes in the same node, by default, their associated `DataUpload`/`DataDownload` CRs are processed sequentially and can be processed concurrently according to your [node-agent Concurrency Configuration][14]. You can check in which node the `DataUpload`/`DataDownload` CRs are processed and their parallelism by watching the `DataUpload`/`DataDownload` CRs: @@ -425,12 +425,23 @@ spec: ### Resource Consumption Both the uploader and repository consume remarkable CPU/memory during the backup/restore, especially for massive small files or large backup size cases. -Velero node-agent uses [BestEffort as the QoS][13] for node-agent pods (so no CPU/memory request/limit is set), so that backups/restores wouldn't fail due to resource throttling in any cases. + +For Velero built-in data mover, Velero uses [BestEffort as the QoS][13] for node-agent pods (so no CPU/memory request/limit is set), so that backups/restores wouldn't fail due to resource throttling in any cases. If you want to constraint the CPU/memory usage, you need to [customize the resource limits][11]. The CPU/memory consumption is always related to the scale of data to be backed up/restored, refer to [Performance Guidance][12] for more details, so it is highly recommended that you perform your own testing to find the best resource limits for your data. During the restore, the repository may also cache data/metadata so as to reduce the network footprint and speed up the restore. The repository uses its own policy to store and clean up the cache. For Kopia repository, the cache is stored in the node-agent pod's root file system and the cleanup is triggered for the data/metadata that are older than 10 minutes (not configurable at present). So you should prepare enough disk space, otherwise, the node-agent pod may be evicted due to running out of the ephemeral storage. +### Node Selection + +The node where a data movement backup/restore runs is decided by the data mover. + +For Velero built-in data mover, it uses Kubernetes' scheduler to mount a snapshot volume/restore volume associated to a `DataUpload`/`DataDownload` CR into a specific node, and then the data movement backup/restore will happen in that node. +For the backup, you can intervene this scheduling process through [Data Movement Backup Node Selection][15], so that you can decide which node(s) should/should not run the data movement backup for various purposes. +For the restore, this is not supported because sometimes the data movement restore must run in the same node where the restored workload pod is scheduled. + + + [1]: https://github.com/vmware-tanzu/velero/pull/5968 [2]: csi.md @@ -445,3 +456,5 @@ For Kopia repository, the cache is stored in the node-agent pod's root file syst [11]: customize-installation.md#customize-resource-requests-and-limits [12]: performance-guidance.md [13]: https://kubernetes.io/docs/concepts/workloads/pods/pod-qos/ +[14]: node-agent-concurrency.md +[15]: data-movement-backup-node-selection.md diff --git a/site/content/docs/main/data-movement-backup-node-selection.md b/site/content/docs/main/data-movement-backup-node-selection.md new file mode 100644 index 000000000..f5b20fa8a --- /dev/null +++ b/site/content/docs/main/data-movement-backup-node-selection.md @@ -0,0 +1,60 @@ +--- +title: "Node Selection for Data Movement Backup" +layout: docs +--- + +Velero node-agent is a daemonset hosting the data movement modules to complete the concrete work of backups/restores. +Varying from the data size, data complexity, resource availability, the data movement may take a long time and remarkable resources (CPU, memory, network bandwidth, etc.) during the backup and restore. + +Velero data movement backup supports to constrain the nodes where it runs. This is helpful in below scenarios: +- Prevent the data movement backup from running in specific nodes because users have more critical workloads in the nodes +- Constrain the data movement backup to run in specific nodes because these nodes have more resources than others +- Constrain the data movement backup to run in specific nodes because the storage allows volume/snapshot provisions in these nodes only + +Velero introduces a new section in ```node-agent-config``` configMap, called ```loadAffinity```, through which you can specify the nodes to/not to run data movement backups, in the affinity and anti-affinity flavors. +If it is not there, ```node-agent-config``` should be created manually. The configMap should be in the same namespace where Velero is installed. If multiple Velero instances are installed in different namespaces, there should be one configMap in each namespace which applies to node-agent in that namespace only. +Node-agent server checks these configurations at startup time. Therefore, you could edit this configMap any time, but in order to make the changes effective, node-agent server needs to be restarted. + +### Sample +Here is a sample of the ```node-agent-config``` configMap with ```loadAffinity```: +```json +{ + "loadAffinity": [ + { + "nodeSelector": { + "matchLabels": { + "beta.kubernetes.io/instance-type": "Standard_B4ms" + }, + "matchExpressions": [ + { + "key": "kubernetes.io/hostname", + "values": [ + "node-1", + "node-2", + "node-3" + ], + "operator": "In" + }, + { + "key": "xxx/critial-workload", + "operator": "DoesNotExist" + } + ] + } + } + ] +} +``` +To create the configMap, save something like the above sample to a json file and then run below command: +``` +kubectl create cm node-agent-config -n velero --from-file= +``` + +### Affinity +Affinity configuration means allowing the data movement backup to run in the nodes specified. There are two ways to define it: +- It could be defined by `MatchLabels`. The labels defined in `MatchLabels` means a `LabelSelectorOpIn` operation by default, so in the current context, they will be treated as affinity rules. In the above sample, it defines to run data movement backups in nodes with label `beta.kubernetes.io/instance-type` of value `Standard_B4ms` (Run data movement backups in `Standard_B4ms` nodes only). +- It could be defined by `MatchExpressions`. The labels are defined in `Key` and `Values` of `MatchExpressions` and the `Operator` should be defined as `LabelSelectorOpIn` or `LabelSelectorOpExists`. In the above sample, it defines to run data movement backups in nodes with label `kubernetes.io/hostname` of values `node-1`, `node-2` and `node-3` (Run data movement backups in `node-1`, `node-2` and `node-3` only). + +### Anti-affinity +Anti-affinity configuration means preventing the data movement backup from running in the nodes specified. Below is the way to define it: +- It could be defined by `MatchExpressions`. The labels are defined in `Key` and `Values` of `MatchExpressions` and the `Operator` should be defined as `LabelSelectorOpNotIn` or `LabelSelectorOpDoesNotExist`. In the above sample, it disallows data movement backups to run in nodes with label `xxx/critial-workload`. \ No newline at end of file From 836300f583f061f09ca5704a2bf19b0396099d3e Mon Sep 17 00:00:00 2001 From: Ming Qiu Date: Tue, 9 Apr 2024 06:45:27 +0000 Subject: [PATCH 12/16] Fix labeler action failure Signed-off-by: Ming Qiu --- .github/labels.yml | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/.github/labels.yml b/.github/labels.yml index 4471bc029..9dcaf0d00 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -1,41 +1,39 @@ -area: - - "Cloud/AWS" - - "Cloud/GCP" - - "Cloud/Azure" - - "Design" - - "Plugins" - -# Labels that can be applied to PRs with the /kind command -kind: - - "changelog-not-required" - - "tech-debt" - # Works with https://github.com/actions/labeler/ # Below this line, the keys are labels to be applied, and the values are the file globs to match against. # Anything in the `design` directory gets the `Design` label. Area/Design: - - design/* +- changed-files: + - any-glob-to-any-file: 'design/*' # Anything in the site directory gets the website label *EXCEPT* docs Website: - - any: ["site/**/*", "!site/content/docs/**/*"] +- any: + - changed-files: + - any-glob-to-any-file: 'site/**/*' + - any-glob-to-any-file: '!site/content/docs/**/*' Documentation: - - site/content/docs/**/* +- changed-files: + - any-glob-to-any-file: 'site/content/docs/**/*' Dependencies: - - go.mod +- changed-files: + - any-glob-to-any-file: 'go.mod' # Anything that has plugin infra will be labeled. # Individual plugins don't necessarily live here, though Area/Plugins: - - "pkg/plugins/**/*" +- changed-files: + - any-glob-to-any-file: 'pkg/plugins/**/*' has-unit-tests: - - "pkg/**/*_test.go" +- changed-files: + - any-glob-to-any-file: 'pkg/**/*_test.go' has-e2e-2tests: - - "test/e2e/**/*" +- changed-files: + - any-glob-to-any-file: 'test/e2e/**/*' has-changelog: - - "changelogs/**" +- changed-files: + - any-glob-to-any-file: 'changelogs/**' From 69def18ccf8224facb20b76d81bc47d370877e19 Mon Sep 17 00:00:00 2001 From: Ming Qiu Date: Tue, 9 Apr 2024 09:31:50 +0000 Subject: [PATCH 13/16] Modify labels config for label action Signed-off-by: Ming Qiu --- .github/labels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/labels.yml b/.github/labels.yml index 9dcaf0d00..979995f56 100644 --- a/.github/labels.yml +++ b/.github/labels.yml @@ -7,10 +7,10 @@ Area/Design: # Anything in the site directory gets the website label *EXCEPT* docs Website: -- any: +- all: - changed-files: - any-glob-to-any-file: 'site/**/*' - - any-glob-to-any-file: '!site/content/docs/**/*' + - all-globs-to-all-files: '!site/content/docs/**/*' Documentation: - changed-files: From 31e140919a62855ba6f9479cfce924e4d2e61d76 Mon Sep 17 00:00:00 2001 From: Xun Jiang Date: Fri, 29 Mar 2024 18:18:31 +0800 Subject: [PATCH 14/16] Merge CSI plugin code. Signed-off-by: Xun Jiang --- changelogs/unreleased/7609-blackpiglet | 1 + .../actions/csi/volumesnapshot_action.go | 120 ++ .../csi/volumesnapshotcontent_action.go | 126 ++ pkg/apis/velero/v1/labels_annotations.go | 40 + pkg/backup/actions/csi/pvc_action.go | 578 ++++++++++ pkg/backup/actions/csi/pvc_action_test.go | 367 ++++++ .../actions/csi/volumesnapshot_action.go | 379 ++++++ .../actions/csi/volumesnapshotclass_action.go | 124 ++ .../csi/volumesnapshotcontent_action.go | 141 +++ pkg/backup/backup_test.go | 22 +- pkg/backup/item_backupper.go | 25 +- pkg/client/factory.go | 3 + pkg/cmd/server/plugin/plugin.go | 209 +++- pkg/controller/backup_controller.go | 2 +- pkg/exposer/csi_snapshot.go | 19 +- pkg/exposer/csi_snapshot_test.go | 2 +- pkg/restore/actions/add_pv_from_pvc_action.go | 2 +- .../actions/add_pv_from_pvc_action_test.go | 2 +- .../actions/add_pvc_from_pod_action.go | 2 +- .../actions/add_pvc_from_pod_action_test.go | 2 +- .../actions/admissionwebhook_config_action.go | 2 +- .../admissionwebhook_config_action_test.go | 2 +- pkg/restore/actions/apiservice_action.go | 2 +- pkg/restore/actions/apiservice_action_test.go | 2 +- .../actions/change_image_name_action.go | 2 +- .../actions/change_image_name_action_test.go | 2 +- .../actions/change_pvc_node_selector.go | 2 +- .../actions/change_pvc_node_selector_test.go | 2 +- .../actions/change_storageclass_action.go | 2 +- .../change_storageclass_action_test.go | 2 +- .../actions/clusterrolebinding_action.go | 2 +- .../actions/clusterrolebinding_action_test.go | 2 +- .../crd_v1_preserve_unknown_fields_action.go | 2 +- ..._v1_preserve_unknown_fields_action_test.go | 2 +- pkg/restore/actions/csi/pvc_action.go | 616 ++++++++++ pkg/restore/actions/csi/pvc_action_test.go | 713 ++++++++++++ .../actions/csi/volumesnapshot_action.go | 217 ++++ .../actions/csi/volumesnapshot_action_test.go | 89 ++ .../actions/csi/volumesnapshotclass_action.go | 111 ++ .../csi/volumesnapshotcontent_action.go | 113 ++ .../actions/dataupload_retrieve_action.go | 2 +- .../dataupload_retrieve_action_test.go | 5 +- .../actions/init_restorehook_pod_action.go | 2 +- .../init_restorehook_pod_action_test.go | 2 +- pkg/restore/actions/job_action.go | 2 +- pkg/restore/actions/job_action_test.go | 2 +- pkg/restore/actions/pod_action.go | 2 +- pkg/restore/actions/pod_action_test.go | 2 +- .../actions/pod_volume_restore_action.go | 2 +- .../actions/pod_volume_restore_action_test.go | 2 +- pkg/restore/actions/rolebinding_action.go | 2 +- .../actions/rolebinding_action_test.go | 2 +- pkg/restore/actions/secret_action.go | 2 +- pkg/restore/actions/secret_action_test.go | 2 +- pkg/restore/actions/service_account_action.go | 2 +- .../actions/service_account_action_test.go | 2 +- pkg/restore/actions/service_action.go | 2 +- pkg/restore/actions/service_action_test.go | 2 +- pkg/test/fake_controller_runtime_client.go | 42 +- pkg/util/csi/volume_snapshot.go | 626 +++++++++- pkg/util/csi/volume_snapshot_test.go | 1020 ++++++++++++++++- pkg/util/kube/pvc_pv.go | 32 +- pkg/util/kube/pvc_pv_test.go | 137 +++ pkg/util/kube/utils.go | 30 + pkg/util/kube/utils_test.go | 184 +++ pkg/util/podvolume/pod_volume.go | 63 +- pkg/util/podvolume/pod_volume_test.go | 414 +++++++ pkg/util/scheme.go | 16 + pkg/util/util.go | 26 + pkg/util/util_test.go | 64 ++ test/e2e/resource-filtering/exclude_label.go | 7 +- 71 files changed, 6565 insertions(+), 188 deletions(-) create mode 100644 changelogs/unreleased/7609-blackpiglet create mode 100644 internal/delete/actions/csi/volumesnapshot_action.go create mode 100644 internal/delete/actions/csi/volumesnapshotcontent_action.go create mode 100644 pkg/backup/actions/csi/pvc_action.go create mode 100644 pkg/backup/actions/csi/pvc_action_test.go create mode 100644 pkg/backup/actions/csi/volumesnapshot_action.go create mode 100644 pkg/backup/actions/csi/volumesnapshotclass_action.go create mode 100644 pkg/backup/actions/csi/volumesnapshotcontent_action.go create mode 100644 pkg/restore/actions/csi/pvc_action.go create mode 100644 pkg/restore/actions/csi/pvc_action_test.go create mode 100644 pkg/restore/actions/csi/volumesnapshot_action.go create mode 100644 pkg/restore/actions/csi/volumesnapshot_action_test.go create mode 100644 pkg/restore/actions/csi/volumesnapshotclass_action.go create mode 100644 pkg/restore/actions/csi/volumesnapshotcontent_action.go create mode 100644 pkg/util/util.go create mode 100644 pkg/util/util_test.go diff --git a/changelogs/unreleased/7609-blackpiglet b/changelogs/unreleased/7609-blackpiglet new file mode 100644 index 000000000..48644ae05 --- /dev/null +++ b/changelogs/unreleased/7609-blackpiglet @@ -0,0 +1 @@ +Merge CSI plugin code into Velero. \ No newline at end of file diff --git a/internal/delete/actions/csi/volumesnapshot_action.go b/internal/delete/actions/csi/volumesnapshot_action.go new file mode 100644 index 000000000..bc6d9a2be --- /dev/null +++ b/internal/delete/actions/csi/volumesnapshot_action.go @@ -0,0 +1,120 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + "fmt" + + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/velero/pkg/client" + plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + "github.com/vmware-tanzu/velero/pkg/util/csi" + kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" +) + +// volumeSnapshotDeleteItemAction is a backup item action plugin for Velero. +type volumeSnapshotDeleteItemAction struct { + log logrus.FieldLogger + crClient crclient.Client +} + +// AppliesTo returns information indicating that the +// VolumeSnapshotBackupItemAction should be invoked to backup +// VolumeSnapshots. +func (p *volumeSnapshotDeleteItemAction) AppliesTo() (velero.ResourceSelector, error) { + p.log.Debug("VolumeSnapshotBackupItemAction AppliesTo") + + return velero.ResourceSelector{ + IncludedResources: []string{"volumesnapshots.snapshot.storage.k8s.io"}, + }, nil +} + +func (p *volumeSnapshotDeleteItemAction) Execute( + input *velero.DeleteItemActionExecuteInput, +) error { + p.log.Info("Starting VolumeSnapshotDeleteItemAction for volumeSnapshot") + + var vs snapshotv1api.VolumeSnapshot + + if err := runtime.DefaultUnstructuredConverter.FromUnstructured( + input.Item.UnstructuredContent(), + &vs, + ); err != nil { + return errors.Wrapf(err, "failed to convert input.Item from unstructured") + } + + // We don't want this DeleteItemAction plugin to delete VolumeSnapshot + // taken outside of Velero. So skip deleting VolumeSnapshot objects + // that were not created in the process of creating the Velero + // backup being deleted. + if !kubeutil.HasBackupLabel(&vs.ObjectMeta, input.Backup.Name) { + p.log.Info( + "VolumeSnapshot %s/%s was not taken by backup %s, skipping deletion", + vs.Namespace, vs.Name, input.Backup.Name, + ) + return nil + } + + p.log.Infof("Deleting VolumeSnapshot %s/%s", vs.Namespace, vs.Name) + if vs.Status != nil && vs.Status.BoundVolumeSnapshotContentName != nil { + // we patch the DeletionPolicy of the VolumeSnapshotContent + // to set it to Delete. This ensures that the volume snapshot + // in the storage provider is also deleted. + err := csi.SetVolumeSnapshotContentDeletionPolicy( + *vs.Status.BoundVolumeSnapshotContentName, + p.crClient, + ) + if err != nil && !apierrors.IsNotFound(err) { + return errors.Wrapf( + err, + fmt.Sprintf("failed to patch DeletionPolicy of volume snapshot %s/%s", + vs.Namespace, vs.Name), + ) + } + + if apierrors.IsNotFound(err) { + return nil + } + } + err := p.crClient.Delete(context.TODO(), &vs) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil +} + +func NewVolumeSnapshotDeleteItemAction(f client.Factory) plugincommon.HandlerInitializer { + return func(logger logrus.FieldLogger) (interface{}, error) { + crClient, err := f.KubebuilderClient() + if err != nil { + return nil, errors.WithStack(err) + } + + return &volumeSnapshotDeleteItemAction{ + log: logger, + crClient: crClient, + }, nil + } +} diff --git a/internal/delete/actions/csi/volumesnapshotcontent_action.go b/internal/delete/actions/csi/volumesnapshotcontent_action.go new file mode 100644 index 000000000..f3cfe41f7 --- /dev/null +++ b/internal/delete/actions/csi/volumesnapshotcontent_action.go @@ -0,0 +1,126 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + "fmt" + + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/velero/pkg/client" + plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + "github.com/vmware-tanzu/velero/pkg/util/csi" + kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" +) + +// volumeSnapshotContentDeleteItemAction is a restore item action plugin for Velero +type volumeSnapshotContentDeleteItemAction struct { + log logrus.FieldLogger + crClient crclient.Client +} + +// AppliesTo returns information indicating +// VolumeSnapshotContentRestoreItemAction action should be invoked +// while restoring VolumeSnapshotContent.snapshot.storage.k8s.io resources +func (p *volumeSnapshotContentDeleteItemAction) AppliesTo() (velero.ResourceSelector, error) { + return velero.ResourceSelector{ + IncludedResources: []string{"volumesnapshotcontent.snapshot.storage.k8s.io"}, + }, nil +} + +func (p *volumeSnapshotContentDeleteItemAction) Execute( + input *velero.DeleteItemActionExecuteInput, +) error { + p.log.Info("Starting VolumeSnapshotContentDeleteItemAction") + + var snapCont snapshotv1api.VolumeSnapshotContent + if err := runtime.DefaultUnstructuredConverter.FromUnstructured( + input.Item.UnstructuredContent(), + &snapCont, + ); err != nil { + return errors.Wrapf(err, "failed to convert VolumeSnapshotContent from unstructured") + } + + // We don't want this DeleteItemAction plugin to delete + // VolumeSnapshotContent taken outside of Velero. + // So skip deleting VolumeSnapshotContent not have the backup name + // in its labels. + if !kubeutil.HasBackupLabel(&snapCont.ObjectMeta, input.Backup.Name) { + p.log.Info( + "VolumeSnapshotContent %s was not taken by backup %s, skipping deletion", + snapCont.Name, + input.Backup.Name, + ) + return nil + } + + p.log.Infof("Deleting VolumeSnapshotContent %s", snapCont.Name) + + if err := csi.SetVolumeSnapshotContentDeletionPolicy( + snapCont.Name, + p.crClient, + ); err != nil { + // #4764: Leave a warning when VolumeSnapshotContent cannot be found for deletion. + // Manual deleting VolumeSnapshotContent can cause this. + // It's tricky for Velero to handle this inconsistency. + // Even if Velero restores the VolumeSnapshotContent, CSI snapshot controller + // may not delete it correctly due to the snapshot represented by VolumeSnapshotContent + // already deleted on cloud provider. + if apierrors.IsNotFound(err) { + p.log.Warnf( + "VolumeSnapshotContent %s of backup %s cannot be found. May leave orphan snapshot %s on cloud provider.", + snapCont.Name, input.Backup.Name, *snapCont.Status.SnapshotHandle) + return nil + } + return errors.Wrapf(err, fmt.Sprintf( + "failed to set DeletionPolicy on volumesnapshotcontent %s. Skipping deletion", + snapCont.Name)) + } + + if err := p.crClient.Delete( + context.TODO(), + &snapCont, + ); err != nil && !apierrors.IsNotFound(err) { + p.log.Infof("VolumeSnapshotContent %s not found", snapCont.Name) + return err + } + + return nil +} + +func NewVolumeSnapshotContentDeleteItemAction( + f client.Factory, +) plugincommon.HandlerInitializer { + return func(logger logrus.FieldLogger) (interface{}, error) { + crClient, err := f.KubebuilderClient() + if err != nil { + return nil, err + } + + return &volumeSnapshotContentDeleteItemAction{ + log: logger, + crClient: crClient, + }, nil + } +} diff --git a/pkg/apis/velero/v1/labels_annotations.go b/pkg/apis/velero/v1/labels_annotations.go index 08b6b2440..6a2b59d49 100644 --- a/pkg/apis/velero/v1/labels_annotations.go +++ b/pkg/apis/velero/v1/labels_annotations.go @@ -97,6 +97,10 @@ const ( // VolumesToExcludeAnnotation is the annotation on a pod whose mounted volumes // should be excluded from pod volume backup. VolumesToExcludeAnnotation = "backup.velero.io/backup-volumes-excludes" + + // ExcludeFromBackupLabel is the label to exclude k8s resource from backup, + // even if the resource contains a matching selector label. + ExcludeFromBackupLabel = "velero.io/exclude-from-backup" ) type AsyncOperationIDPrefix string @@ -111,3 +115,39 @@ type VeleroResourceUsage string const ( VeleroResourceUsageDataUploadResult VeleroResourceUsage = "DataUpload" ) + +// CSI related plugin actions' constant variable +const ( + VolumeSnapshotLabel = "velero.io/volume-snapshot-name" + VolumeSnapshotHandleAnnotation = "velero.io/csi-volumesnapshot-handle" + VolumeSnapshotRestoreSize = "velero.io/csi-volumesnapshot-restore-size" + DriverNameAnnotation = "velero.io/csi-driver-name" + DeleteSecretNameAnnotation = "velero.io/csi-deletesnapshotsecret-name" // #nosec G101 + DeleteSecretNamespaceAnnotation = "velero.io/csi-deletesnapshotsecret-namespace" // #nosec G101 + VSCDeletionPolicyAnnotation = "velero.io/csi-vsc-deletion-policy" + VolumeSnapshotClassSelectorLabel = "velero.io/csi-volumesnapshot-class" + VolumeSnapshotClassDriverBackupAnnotationPrefix = "velero.io/csi-volumesnapshot-class" + VolumeSnapshotClassDriverPVCAnnotation = "velero.io/csi-volumesnapshot-class" + + // There is no release w/ these constants exported. Using the strings for now. + // CSI Annotation volumesnapshotclass + // https://github.com/kubernetes-csi/external-snapshotter/blob/master/pkg/utils/util.go#L59-L60 + PrefixedListSecretNameAnnotation = "csi.storage.k8s.io/snapshotter-list-secret-name" // #nosec G101 + PrefixedListSecretNamespaceAnnotation = "csi.storage.k8s.io/snapshotter-list-secret-namespace" // #nosec G101 + + // CSI Annotation volumesnapshotcontents + PrefixedSecretNameAnnotation = "csi.storage.k8s.io/snapshotter-secret-name" // #nosec G101 + PrefixedSecretNamespaceAnnotation = "csi.storage.k8s.io/snapshotter-secret-namespace" // #nosec G101 + + // Velero checks this annotation to determine whether to skip resource excluding check. + MustIncludeAdditionalItemAnnotation = "backup.velero.io/must-include-additional-items" + // SkippedNoCSIPVAnnotation - Velero checks this annotation on processed PVC to + // find out if the snapshot was skipped b/c the PV is not provisioned via CSI + SkippedNoCSIPVAnnotation = "backup.velero.io/skipped-no-csi-pv" + + // DynamicPVRestoreLabel is the label key for dynamic PV restore + DynamicPVRestoreLabel = "velero.io/dynamic-pv-restore" + + // DataUploadNameAnnotation is the label key for the DataUpload name + DataUploadNameAnnotation = "velero.io/data-upload-name" +) diff --git a/pkg/backup/actions/csi/pvc_action.go b/pkg/backup/actions/csi/pvc_action.go new file mode 100644 index 000000000..1417e9120 --- /dev/null +++ b/pkg/backup/actions/csi/pvc_action.go @@ -0,0 +1,578 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + "fmt" + + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + corev1api "k8s.io/api/core/v1" + storagev1api "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + "github.com/vmware-tanzu/velero/pkg/client" + "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/label" + plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + biav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" + uploaderUtil "github.com/vmware-tanzu/velero/pkg/uploader/util" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/csi" + kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" + "github.com/vmware-tanzu/velero/pkg/util/podvolume" +) + +// pvcBackupItemAction is a backup item action plugin for Velero. +type pvcBackupItemAction struct { + log logrus.FieldLogger + crClient crclient.Client +} + +// AppliesTo returns information indicating that the PVCBackupItemAction +// should be invoked to backup PVCs. +func (p *pvcBackupItemAction) AppliesTo() (velero.ResourceSelector, error) { + return velero.ResourceSelector{ + IncludedResources: []string{"persistentvolumeclaims"}, + }, nil +} + +func (p *pvcBackupItemAction) validateBackup(backup velerov1api.Backup) (valid bool) { + // Do nothing if volume snapshots have not been requested in this backup + if boolptr.IsSetToFalse(backup.Spec.SnapshotVolumes) { + p.log.Infof( + "Volume snapshotting not requested for backup %s/%s", + backup.Namespace, backup.Name) + return false + } + + if backup.Status.Phase == velerov1api.BackupPhaseFinalizing || + backup.Status.Phase == velerov1api.BackupPhaseFinalizingPartiallyFailed { + p.log.WithFields( + logrus.Fields{ + "Backup": fmt.Sprintf("%s/%s", backup.Namespace, backup.Name), + "Phase": backup.Status.Phase, + }, + ).Debug("Backup is in finalizing phase. Skip this PVC.") + return false + } + + return true +} + +func (p *pvcBackupItemAction) validatePVCandPV( + pvc corev1api.PersistentVolumeClaim, + defaultVolumesToFsBackup *bool, + item runtime.Unstructured, +) ( + valid bool, + updateItem runtime.Unstructured, + err error, +) { + updateItem = item + + // no storage class: we don't know how to map to a VolumeSnapshotClass + if pvc.Spec.StorageClassName == nil { + return false, + updateItem, + errors.Errorf( + "Cannot snapshot PVC %s/%s, PVC has no storage class.", + pvc.Namespace, pvc.Name) + } + + p.log.Debugf( + "Fetching underlying PV for PVC %s", + fmt.Sprintf("%s/%s", pvc.Namespace, pvc.Name), + ) + + // Do nothing if this is not a CSI provisioned volume + pv, err := kubeutil.GetPVForPVC(&pvc, p.crClient) + if err != nil { + return false, updateItem, errors.WithStack(err) + } + + if pv.Spec.PersistentVolumeSource.CSI == nil { + p.log.Infof( + "Skipping PVC %s/%s, associated PV %s is not a CSI volume", + pvc.Namespace, pvc.Name, pv.Name) + + kubeutil.AddAnnotations( + &pvc.ObjectMeta, + map[string]string{ + velerov1api.SkippedNoCSIPVAnnotation: "true", + }) + data, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pvc) + updateItem = &unstructured.Unstructured{Object: data} + return false, updateItem, err + } + + // Do nothing if FS uploader is used to backup this PV + isFSUploaderUsed, err := podvolume.IsPVCDefaultToFSBackup( + pvc.Namespace, + pvc.Name, + p.crClient, + boolptr.IsSetToTrue(defaultVolumesToFsBackup), + ) + if err != nil { + return false, updateItem, errors.WithStack(err) + } + if isFSUploaderUsed { + p.log.Infof( + "Skipping PVC %s/%s, PV %s will be backed up using FS uploader", + pvc.Namespace, pvc.Name, pv.Name) + return false, updateItem, nil + } + + return true, updateItem, nil +} + +func (p *pvcBackupItemAction) createVolumeSnapshot( + pvc corev1api.PersistentVolumeClaim, + backup *velerov1api.Backup, +) ( + vs *snapshotv1api.VolumeSnapshot, + err error, +) { + p.log.Debugf("Fetching storage class for PV %s", *pvc.Spec.StorageClassName) + storageClass := new(storagev1api.StorageClass) + if err := p.crClient.Get( + context.TODO(), crclient.ObjectKey{Name: *pvc.Spec.StorageClassName}, + storageClass, + ); err != nil { + return nil, errors.Wrap(err, "error getting storage class") + } + + p.log.Debugf("Fetching VolumeSnapshotClass for %s", storageClass.Provisioner) + vsClass, err := csi.GetVolumeSnapshotClass( + storageClass.Provisioner, + backup, + &pvc, + p.log, + p.crClient, + ) + if err != nil { + return nil, errors.Wrapf( + err, "failed to get VolumeSnapshotClass for StorageClass %s", + storageClass.Name, + ) + } + p.log.Infof("VolumeSnapshotClass=%s", vsClass.Name) + + vsLabels := map[string]string{} + for k, v := range pvc.ObjectMeta.Labels { + vsLabels[k] = v + } + vsLabels[velerov1api.BackupNameLabel] = label.GetValidName(backup.Name) + + // Craft the vs object to be created + vs = &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "velero-" + pvc.Name + "-", + Namespace: pvc.Namespace, + Labels: vsLabels, + }, + Spec: snapshotv1api.VolumeSnapshotSpec{ + Source: snapshotv1api.VolumeSnapshotSource{ + PersistentVolumeClaimName: &pvc.Name, + }, + VolumeSnapshotClassName: &vsClass.Name, + }, + } + + if err := p.crClient.Create(context.TODO(), vs); err != nil { + return nil, errors.Wrapf( + err, "error creating volume snapshot", + ) + } + p.log.Infof( + "Created VolumeSnapshot %s", + fmt.Sprintf("%s/%s", vs.Namespace, vs.Name), + ) + + return vs, nil +} + +// Execute recognizes PVCs backed by volumes provisioned by CSI drivers +// with VolumeSnapshotting capability and creates snapshots of the +// underlying PVs by creating VolumeSnapshot CSI API objects that will +// trigger the CSI driver to perform the snapshot operation on the volume. +func (p *pvcBackupItemAction) Execute( + item runtime.Unstructured, + backup *velerov1api.Backup, +) ( + runtime.Unstructured, + []velero.ResourceIdentifier, + string, + []velero.ResourceIdentifier, + error, +) { + p.log.Info("Starting PVCBackupItemAction") + + if valid := p.validateBackup(*backup); !valid { + return item, nil, "", nil, nil + } + + var pvc corev1api.PersistentVolumeClaim + if err := runtime.DefaultUnstructuredConverter.FromUnstructured( + item.UnstructuredContent(), + &pvc, + ); err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + if valid, item, err := p.validatePVCandPV( + pvc, + backup.Spec.DefaultVolumesToFsBackup, + item, + ); !valid { + if err != nil { + return nil, nil, "", nil, err + } + return item, nil, "", nil, nil + } + + vs, err := p.createVolumeSnapshot(pvc, backup) + if err != nil { + return nil, nil, "", nil, err + } + + labels := map[string]string{ + velerov1api.VolumeSnapshotLabel: vs.Name, + velerov1api.BackupNameLabel: backup.Name, + } + + annotations := map[string]string{ + velerov1api.VolumeSnapshotLabel: vs.Name, + velerov1api.MustIncludeAdditionalItemAnnotation: "true", + } + + var additionalItems []velero.ResourceIdentifier + operationID := "" + var itemToUpdate []velero.ResourceIdentifier + + if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) { + operationID = label.GetValidName( + string( + velerov1api.AsyncOperationIDPrefixDataUpload, + ) + string(backup.UID) + "." + string(pvc.UID), + ) + dataUploadLog := p.log.WithFields(logrus.Fields{ + "Source PVC": fmt.Sprintf("%s/%s", pvc.Namespace, pvc.Name), + "VolumeSnapshot": fmt.Sprintf("%s/%s", vs.Namespace, vs.Name), + "Operation ID": operationID, + "Backup": backup.Name, + }) + + // Wait until VS associated VSC snapshot handle created before + // returning with the Async operation for data mover. + _, err := csi.WaitUntilVSCHandleIsReady( + vs, + p.crClient, + p.log, + true, + backup.Spec.CSISnapshotTimeout.Duration, + ) + if err != nil { + dataUploadLog.Errorf( + "Fail to wait VolumeSnapshot turned to ReadyToUse: %s", + err.Error(), + ) + csi.CleanupVolumeSnapshot(vs, p.crClient, p.log) + return nil, nil, "", nil, errors.WithStack(err) + } + + dataUploadLog.Info("Starting data upload of backup") + + dataUpload, err := createDataUpload( + context.Background(), + backup, + p.crClient, + vs, + &pvc, + operationID, + ) + if err != nil { + dataUploadLog.WithError(err).Error("failed to submit DataUpload") + + // TODO: need to use DeleteVolumeSnapshotIfAny, after data mover + // adopting the controller-runtime client. + if deleteErr := p.crClient.Delete(context.TODO(), vs); deleteErr != nil { + if !apierrors.IsNotFound(deleteErr) { + dataUploadLog.WithError(deleteErr).Error("fail to delete VolumeSnapshot") + } + } + + // Return without modification to not fail the backup, + // and the above error log makes the backup partially fail. + return item, nil, "", nil, nil + } else { + itemToUpdate = []velero.ResourceIdentifier{ + { + GroupResource: schema.GroupResource{ + Group: "velero.io", + Resource: "datauploads", + }, + Namespace: dataUpload.Namespace, + Name: dataUpload.Name, + }, + } + // Set the DataUploadNameLabel, which is used for restore to + // let CSI plugin check whether it should handle the volume. + // If volume is CSI migration, PVC doesn't have the annotation. + annotations[velerov1api.DataUploadNameAnnotation] = dataUpload.Namespace + "/" + dataUpload.Name + + dataUploadLog.Info("DataUpload is submitted successfully.") + } + } else { + additionalItems = []velero.ResourceIdentifier{ + { + GroupResource: kuberesource.VolumeSnapshots, + Namespace: vs.Namespace, + Name: vs.Name, + }, + } + } + + kubeutil.AddAnnotations(&pvc.ObjectMeta, annotations) + kubeutil.AddLabels(&pvc.ObjectMeta, labels) + + p.log.Infof("Returning from PVCBackupItemAction with %d additionalItems to backup", + len(additionalItems)) + for _, ai := range additionalItems { + p.log.Debugf("%s: %s", ai.GroupResource.String(), ai.Name) + } + + pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pvc) + if err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + return &unstructured.Unstructured{Object: pvcMap}, + additionalItems, operationID, itemToUpdate, nil +} + +func (p *pvcBackupItemAction) Name() string { + return "PVCBackupItemAction" +} + +func (p *pvcBackupItemAction) Progress( + operationID string, + backup *velerov1api.Backup, +) (velero.OperationProgress, error) { + progress := velero.OperationProgress{} + if operationID == "" { + return progress, biav2.InvalidOperationIDError(operationID) + } + + dataUpload, err := getDataUpload(context.Background(), p.crClient, operationID) + if err != nil { + p.log.Errorf( + "fail to get DataUpload for backup %s/%s by operation ID %s: %s", + backup.Namespace, backup.Name, operationID, err.Error(), + ) + return progress, err + } + if dataUpload.Status.Phase == velerov2alpha1.DataUploadPhaseNew || + dataUpload.Status.Phase == "" { + p.log.Debugf("DataUpload is still not processed yet. Skip progress update.") + return progress, nil + } + + progress.Description = string(dataUpload.Status.Phase) + progress.OperationUnits = "Bytes" + progress.NCompleted = dataUpload.Status.Progress.BytesDone + progress.NTotal = dataUpload.Status.Progress.TotalBytes + + if dataUpload.Status.StartTimestamp != nil { + progress.Started = dataUpload.Status.StartTimestamp.Time + } + + if dataUpload.Status.CompletionTimestamp != nil { + progress.Updated = dataUpload.Status.CompletionTimestamp.Time + } + + if dataUpload.Status.Phase == velerov2alpha1.DataUploadPhaseCompleted { + progress.Completed = true + } else if dataUpload.Status.Phase == velerov2alpha1.DataUploadPhaseFailed { + progress.Completed = true + progress.Err = dataUpload.Status.Message + } else if dataUpload.Status.Phase == velerov2alpha1.DataUploadPhaseCanceled { + progress.Completed = true + progress.Err = "DataUpload is canceled" + } + + return progress, nil +} + +func (p *pvcBackupItemAction) Cancel(operationID string, backup *velerov1api.Backup) error { + if operationID == "" { + return biav2.InvalidOperationIDError(operationID) + } + + dataUpload, err := getDataUpload(context.Background(), p.crClient, operationID) + if err != nil { + p.log.Errorf( + "fail to get DataUpload for backup %s/%s: %s", + backup.Namespace, backup.Name, err.Error(), + ) + return err + } + + return cancelDataUpload(context.Background(), p.crClient, dataUpload) +} + +func newDataUpload( + backup *velerov1api.Backup, + vs *snapshotv1api.VolumeSnapshot, + pvc *corev1api.PersistentVolumeClaim, + operationID string, +) *velerov2alpha1.DataUpload { + dataUpload := &velerov2alpha1.DataUpload{ + TypeMeta: metav1.TypeMeta{ + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + Kind: "DataUpload", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: backup.Namespace, + GenerateName: backup.Name + "-", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: velerov1api.SchemeGroupVersion.String(), + Kind: "Backup", + Name: backup.Name, + UID: backup.UID, + Controller: boolptr.True(), + }, + }, + Labels: map[string]string{ + velerov1api.BackupNameLabel: label.GetValidName(backup.Name), + velerov1api.BackupUIDLabel: string(backup.UID), + velerov1api.PVCUIDLabel: string(pvc.UID), + velerov1api.AsyncOperationIDLabel: operationID, + }, + }, + Spec: velerov2alpha1.DataUploadSpec{ + SnapshotType: velerov2alpha1.SnapshotTypeCSI, + CSISnapshot: &velerov2alpha1.CSISnapshotSpec{ + VolumeSnapshot: vs.Name, + StorageClass: *pvc.Spec.StorageClassName, + SnapshotClass: *vs.Spec.VolumeSnapshotClassName, + }, + SourcePVC: pvc.Name, + DataMover: backup.Spec.DataMover, + BackupStorageLocation: backup.Spec.StorageLocation, + SourceNamespace: pvc.Namespace, + OperationTimeout: backup.Spec.CSISnapshotTimeout, + }, + } + + if backup.Spec.UploaderConfig != nil && + backup.Spec.UploaderConfig.ParallelFilesUpload > 0 { + dataUpload.Spec.DataMoverConfig = make(map[string]string) + dataUpload.Spec.DataMoverConfig[uploaderUtil.ParallelFilesUpload] = fmt.Sprintf( + "%d", backup.Spec.UploaderConfig.ParallelFilesUpload, + ) + } + + return dataUpload +} + +func createDataUpload( + ctx context.Context, + backup *velerov1api.Backup, + crClient crclient.Client, + vs *snapshotv1api.VolumeSnapshot, + pvc *corev1api.PersistentVolumeClaim, + operationID string, +) (*velerov2alpha1.DataUpload, error) { + dataUpload := newDataUpload(backup, vs, pvc, operationID) + + err := crClient.Create(ctx, dataUpload) + if err != nil { + return nil, errors.Wrap(err, "fail to create DataUpload CR") + } + + return dataUpload, err +} + +func getDataUpload( + ctx context.Context, + crClient crclient.Client, + operationID string, +) (*velerov2alpha1.DataUpload, error) { + dataUploadList := new(velerov2alpha1.DataUploadList) + err := crClient.List(ctx, dataUploadList, &crclient.ListOptions{ + LabelSelector: labels.SelectorFromSet( + map[string]string{velerov1api.AsyncOperationIDLabel: operationID}, + ), + }) + if err != nil { + return nil, errors.Wrapf(err, "error to list DataUpload") + } + + if len(dataUploadList.Items) == 0 { + return nil, errors.Errorf("not found DataUpload for operationID %s", operationID) + } + + if len(dataUploadList.Items) > 1 { + return nil, errors.Errorf("more than one DataUpload found operationID %s", operationID) + } + + return &dataUploadList.Items[0], nil +} + +func cancelDataUpload( + ctx context.Context, + crClient crclient.Client, + dataUpload *velerov2alpha1.DataUpload, +) error { + updatedDataUpload := dataUpload.DeepCopy() + updatedDataUpload.Spec.Cancel = true + + err := crClient.Patch(ctx, updatedDataUpload, crclient.MergeFrom(dataUpload)) + if err != nil { + return errors.Wrap(err, "error patch DataUpload") + } + + return nil +} + +func NewPvcBackupItemAction(f client.Factory) plugincommon.HandlerInitializer { + return func(logger logrus.FieldLogger) (interface{}, error) { + crClient, err := f.KubebuilderClient() + if err != nil { + return nil, errors.WithStack(err) + } + + return &pvcBackupItemAction{ + log: logger, + crClient: crClient, + }, nil + } +} diff --git a/pkg/backup/actions/csi/pvc_action_test.go b/pkg/backup/actions/csi/pvc_action_test.go new file mode 100644 index 000000000..4d8a0e910 --- /dev/null +++ b/pkg/backup/actions/csi/pvc_action_test.go @@ -0,0 +1,367 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + v1 "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + "github.com/vmware-tanzu/velero/pkg/builder" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + velerotest "github.com/vmware-tanzu/velero/pkg/test" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" +) + +func TestExecute(t *testing.T) { + boolTrue := true + tests := []struct { + name string + backup *velerov1api.Backup + pvc *corev1.PersistentVolumeClaim + pv *corev1.PersistentVolume + sc *storagev1.StorageClass + vsClass *snapshotv1api.VolumeSnapshotClass + operationID string + expectedErr error + expectedBackup *velerov1api.Backup + expectedDataUpload *velerov2alpha1.DataUpload + expectedPVC *corev1.PersistentVolumeClaim + }{ + { + name: "Skip PVC handling if SnapshotVolume set to false", + backup: builder.ForBackup("velero", "test").SnapshotVolumes(false).Result(), + expectedErr: nil, + }, + { + name: "Skip PVC BIA when backup is in finalizing phase", + backup: builder.ForBackup("velero", "test").Phase(velerov1api.BackupPhaseFinalizing).Result(), + expectedErr: nil, + }, + { + name: "Test SnapshotMoveData", + backup: builder.ForBackup("velero", "test").SnapshotMoveData(true).CSISnapshotTimeout(1 * time.Minute).Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").StorageClass("testSC").Phase(corev1.ClaimBound).Result(), + pv: builder.ForPersistentVolume("testPV").CSI("hostpath", "testVolume").Result(), + sc: builder.ForStorageClass("testSC").Provisioner("hostpath").Result(), + vsClass: builder.ForVolumeSnapshotClass("testVSClass").Driver("hostpath").ObjectMeta(builder.WithLabels(velerov1api.VolumeSnapshotClassSelectorLabel, "")).Result(), + operationID: ".", + expectedErr: nil, + expectedDataUpload: &velerov2alpha1.DataUpload{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataUpload", + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-", + Namespace: "velero", + Labels: map[string]string{ + velerov1api.BackupNameLabel: "test", + velerov1api.BackupUIDLabel: "", + velerov1api.PVCUIDLabel: "", + velerov1api.AsyncOperationIDLabel: "du-.", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "velero.io/v1", + Kind: "Backup", + Name: "test", + UID: "", + Controller: &boolTrue, + }, + }, + }, + Spec: velerov2alpha1.DataUploadSpec{ + SnapshotType: velerov2alpha1.SnapshotTypeCSI, + CSISnapshot: &velerov2alpha1.CSISnapshotSpec{ + VolumeSnapshot: "", + StorageClass: "testSC", + SnapshotClass: "testVSClass", + }, + SourcePVC: "testPVC", + SourceNamespace: "velero", + OperationTimeout: metav1.Duration{Duration: 1 * time.Minute}, + }, + }, + }, + { + name: "Verify PVC is modified as expected", + backup: builder.ForBackup("velero", "test").SnapshotMoveData(true).CSISnapshotTimeout(1 * time.Minute).Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("testPV").StorageClass("testSC").Phase(corev1.ClaimBound).Result(), + pv: builder.ForPersistentVolume("testPV").CSI("hostpath", "testVolume").Result(), + sc: builder.ForStorageClass("testSC").Provisioner("hostpath").Result(), + vsClass: builder.ForVolumeSnapshotClass("tescVSClass").Driver("hostpath").ObjectMeta(builder.WithLabels(velerov1api.VolumeSnapshotClassSelectorLabel, "")).Result(), + operationID: ".", + expectedErr: nil, + expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC"). + ObjectMeta(builder.WithAnnotations(velerov1api.MustIncludeAdditionalItemAnnotation, "true", velerov1api.DataUploadNameAnnotation, "velero/", velerov1api.VolumeSnapshotLabel, ""), + builder.WithLabels(velerov1api.BackupNameLabel, "test", velerov1api.VolumeSnapshotLabel, "")). + VolumeName("testPV").StorageClass("testSC").Phase(corev1.ClaimBound).Result(), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(*testing.T) { + crClient := velerotest.NewFakeControllerRuntimeClient(t) + logger := logrus.New() + logger.Level = logrus.DebugLevel + + if tc.pvc != nil { + require.NoError(t, crClient.Create(context.Background(), tc.pvc)) + } + if tc.pv != nil { + require.NoError(t, crClient.Create(context.Background(), tc.pv)) + } + if tc.sc != nil { + require.NoError(t, crClient.Create(context.Background(), tc.sc)) + } + if tc.vsClass != nil { + require.NoError(t, crClient.Create(context.Background(), tc.vsClass)) + } + + pvcBIA := pvcBackupItemAction{ + log: logger, + crClient: crClient, + } + + pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&tc.pvc) + require.NoError(t, err) + + if boolptr.IsSetToTrue(tc.backup.Spec.SnapshotMoveData) == true { + go func() { + var vsList v1.VolumeSnapshotList + err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 10*time.Second, true, func(ctx context.Context) (bool, error) { + err = pvcBIA.crClient.List(ctx, &vsList, &crclient.ListOptions{Namespace: tc.pvc.Namespace}) + + require.NoError(t, err) + if err != nil || len(vsList.Items) == 0 { + //lint:ignore nilerr reason + return false, nil // ignore + } + return true, nil + }) + + require.NoError(t, err) + vscName := "testVSC" + readyToUse := true + vsList.Items[0].Status = &v1.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &vscName, + ReadyToUse: &readyToUse, + } + err = pvcBIA.crClient.Update(context.Background(), &vsList.Items[0]) + require.NoError(t, err) + + handleName := "testHandle" + vsc := builder.ForVolumeSnapshotContent("testVSC").Status(&snapshotv1api.VolumeSnapshotContentStatus{SnapshotHandle: &handleName}).Result() + err = pvcBIA.crClient.Create(context.Background(), vsc) + require.NoError(t, err) + }() + } + + resultUnstructed, _, _, _, err := pvcBIA.Execute(&unstructured.Unstructured{Object: pvcMap}, tc.backup) + if tc.expectedErr != nil { + require.Equal(t, err, tc.expectedErr) + } + + if tc.expectedDataUpload != nil { + dataUploadList := new(velerov2alpha1.DataUploadList) + err := crClient.List(context.Background(), dataUploadList, &crclient.ListOptions{LabelSelector: labels.SelectorFromSet(map[string]string{velerov1api.BackupNameLabel: tc.backup.Name})}) + require.NoError(t, err) + require.Len(t, dataUploadList.Items, 1) + require.True(t, cmp.Equal(tc.expectedDataUpload, &dataUploadList.Items[0], cmpopts.IgnoreFields(velerov2alpha1.DataUpload{}, "ResourceVersion", "Name", "Spec.CSISnapshot.VolumeSnapshot"))) + } + + if tc.expectedPVC != nil { + resultPVC := new(corev1.PersistentVolumeClaim) + runtime.DefaultUnstructuredConverter.FromUnstructured(resultUnstructed.UnstructuredContent(), resultPVC) + + require.True(t, cmp.Equal(tc.expectedPVC, resultPVC, cmpopts.IgnoreFields(corev1.PersistentVolumeClaim{}, "ResourceVersion", "Annotations", "Labels"))) + } + }) + } +} + +func TestProgress(t *testing.T) { + currentTime := time.Now() + tests := []struct { + name string + backup *velerov1api.Backup + dataUpload *velerov2alpha1.DataUpload + operationID string + expectedErr string + expectedProgress velero.OperationProgress + }{ + { + name: "DataUpload cannot be found", + backup: builder.ForBackup("velero", "test").Result(), + operationID: "testing", + expectedErr: "not found DataUpload for operationID testing", + }, + { + name: "DataUpload is found", + backup: builder.ForBackup("velero", "test").Result(), + dataUpload: &velerov2alpha1.DataUpload{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataUpload", + APIVersion: "v2alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "velero", + Name: "testing", + Labels: map[string]string{ + velerov1api.AsyncOperationIDLabel: "testing", + }, + }, + Status: velerov2alpha1.DataUploadStatus{ + Phase: velerov2alpha1.DataUploadPhaseFailed, + Progress: shared.DataMoveOperationProgress{ + BytesDone: 1000, + TotalBytes: 1000, + }, + StartTimestamp: &metav1.Time{Time: currentTime}, + CompletionTimestamp: &metav1.Time{Time: currentTime}, + Message: "Testing error", + }, + }, + operationID: "testing", + expectedProgress: velero.OperationProgress{ + Completed: true, + Err: "Testing error", + NCompleted: 1000, + NTotal: 1000, + OperationUnits: "Bytes", + Description: "Failed", + Started: currentTime, + Updated: currentTime, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(*testing.T) { + crClient := velerotest.NewFakeControllerRuntimeClient(t) + logger := logrus.New() + + pvcBIA := pvcBackupItemAction{ + log: logger, + crClient: crClient, + } + + if tc.dataUpload != nil { + err := crClient.Create(context.Background(), tc.dataUpload) + require.NoError(t, err) + } + + progress, err := pvcBIA.Progress(tc.operationID, tc.backup) + if tc.expectedErr != "" { + require.Equal(t, tc.expectedErr, err.Error()) + } + require.True(t, cmp.Equal(tc.expectedProgress, progress, cmpopts.IgnoreFields(velero.OperationProgress{}, "Started", "Updated"))) + }) + } +} + +func TestCancel(t *testing.T) { + tests := []struct { + name string + backup *velerov1api.Backup + dataUpload velerov2alpha1.DataUpload + operationID string + expectedErr error + expectedDataUpload velerov2alpha1.DataUpload + }{ + { + name: "Cancel DataUpload", + backup: builder.ForBackup("velero", "test").Result(), + dataUpload: velerov2alpha1.DataUpload{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataUpload", + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "velero", + Name: "testing", + Labels: map[string]string{ + velerov1api.AsyncOperationIDLabel: "testing", + }, + }, + }, + operationID: "testing", + expectedErr: nil, + expectedDataUpload: velerov2alpha1.DataUpload{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataUpload", + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "velero", + Name: "testing", + Labels: map[string]string{ + velerov1api.AsyncOperationIDLabel: "testing", + }, + }, + Spec: velerov2alpha1.DataUploadSpec{ + Cancel: true, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(*testing.T) { + crClient := velerotest.NewFakeControllerRuntimeClient(t) + logger := logrus.New() + + pvcBIA := pvcBackupItemAction{ + log: logger, + crClient: crClient, + } + + err := crClient.Create(context.Background(), &tc.dataUpload) + require.NoError(t, err) + + err = pvcBIA.Cancel(tc.operationID, tc.backup) + if tc.expectedErr != nil { + require.Equal(t, err, tc.expectedErr) + } + + du := new(velerov2alpha1.DataUpload) + err = crClient.Get(context.Background(), crclient.ObjectKey{Namespace: tc.dataUpload.Namespace, Name: tc.dataUpload.Name}, du) + require.NoError(t, err) + + require.True(t, cmp.Equal(tc.expectedDataUpload, *du, cmpopts.IgnoreFields(velerov2alpha1.DataUpload{}, "ResourceVersion"))) + }) + } +} diff --git a/pkg/backup/actions/csi/volumesnapshot_action.go b/pkg/backup/actions/csi/volumesnapshot_action.go new file mode 100644 index 000000000..64e38c3c7 --- /dev/null +++ b/pkg/backup/actions/csi/volumesnapshot_action.go @@ -0,0 +1,379 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + "fmt" + "strings" + "time" + + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/client" + "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/label" + plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + biav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/backupitemaction/v2" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/csi" + kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" +) + +// volumeSnapshotBackupItemAction is a backup item action plugin to backup +// CSI VolumeSnapshot objects using Velero +type volumeSnapshotBackupItemAction struct { + log logrus.FieldLogger + crClient crclient.Client +} + +// AppliesTo returns information indicating that the +// VolumeSnapshotBackupItemAction should be invoked to +// backup VolumeSnapshots. +func (p *volumeSnapshotBackupItemAction) AppliesTo() ( + velero.ResourceSelector, + error, +) { + return velero.ResourceSelector{ + IncludedResources: []string{"volumesnapshots.snapshot.storage.k8s.io"}, + }, nil +} + +// Execute backs up a CSI VolumeSnapshot object and captures, as labels and annotations, +// information from its associated VolumeSnapshotContents such as CSI driver name, +// storage snapshot handle and namespace and name of the snapshot delete secret, if any. +// It returns the VolumeSnapshotClass and the VolumeSnapshotContents as additional items +// to be backed up. +func (p *volumeSnapshotBackupItemAction) Execute( + item runtime.Unstructured, + backup *velerov1api.Backup, +) ( + runtime.Unstructured, + []velero.ResourceIdentifier, + string, + []velero.ResourceIdentifier, + error, +) { + p.log.Infof("Executing VolumeSnapshotBackupItemAction") + + vs := new(snapshotv1api.VolumeSnapshot) + if err := runtime.DefaultUnstructuredConverter.FromUnstructured( + item.UnstructuredContent(), vs); err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + additionalItems := []velero.ResourceIdentifier{ + { + GroupResource: kuberesource.VolumeSnapshotClasses, + Name: *vs.Spec.VolumeSnapshotClassName, + }, + } + + // determine if we are backing up a VolumeSnapshot that was created by velero while + // performing backup of a CSI backed PVC. + // For VolumeSnapshots that were created during the backup of a CSI backed PVC, + // we will wait for the VolumeSnapshotContents to be available. + // For VolumeSnapshots created outside of velero, we expect the VolumeSnapshotContent + // to be available prior to backing up the VolumeSnapshot. In case of a failure, + // backup should be re-attempted after the CSI driver has reconciled the VolumeSnapshot. + // existence of the velerov1api.BackupNameLabel indicates that the VolumeSnapshot was + // created while backing up a CSI backed PVC. + + // We want to await reconciliation of only those VolumeSnapshots created during the + // ongoing backup. For this we will wait only if the backup label exists on the + // VolumeSnapshot object and the backup name is the same as that of the value of the + // backup label. + backupOngoing := vs.Labels[velerov1api.BackupNameLabel] == label.GetValidName(backup.Name) + + p.log.Infof("Getting VolumesnapshotContent for Volumesnapshot %s/%s", + vs.Namespace, vs.Name) + + vsc, err := csi.WaitUntilVSCHandleIsReady( + vs, + p.crClient, + p.log, + backupOngoing, + backup.Spec.CSISnapshotTimeout.Duration, + ) + if err != nil { + csi.CleanupVolumeSnapshot(vs, p.crClient, p.log) + return nil, nil, "", nil, errors.WithStack(err) + } + + if backup.Status.Phase == velerov1api.BackupPhaseFinalizing || + backup.Status.Phase == velerov1api.BackupPhaseFinalizingPartiallyFailed { + p.log. + WithField("Backup", fmt.Sprintf("%s/%s", backup.Namespace, backup.Name)). + WithField("BackupPhase", backup.Status.Phase).Debugf("Clean VolumeSnapshots.") + + csi.DeleteVolumeSnapshot(*vs, *vsc, backup, p.crClient, p.log) + return item, nil, "", nil, nil + } + + annotations := make(map[string]string) + + if vsc != nil { + // when we are backing up VolumeSnapshots created outside of velero, we + // will not await VolumeSnapshot reconciliation and in this case + // GetVolumeSnapshotContentForVolumeSnapshot may not find the associated + // VolumeSnapshotContents to add to the backup. This is not an error + // encountered in the backup process. So we add the VolumeSnapshotContent + // to the backup only if one is found. + additionalItems = append(additionalItems, velero.ResourceIdentifier{ + GroupResource: kuberesource.VolumeSnapshotContents, + Name: vsc.Name, + }) + annotations[velerov1api.VSCDeletionPolicyAnnotation] = string(vsc.Spec.DeletionPolicy) + + if vsc.Status != nil { + if vsc.Status.SnapshotHandle != nil { + // Capture storage provider snapshot handle and CSI driver name + // to be used on restore to create a static VolumeSnapshotContent + // that will be the source of the VolumeSnapshot. + annotations[velerov1api.VolumeSnapshotHandleAnnotation] = *vsc.Status.SnapshotHandle + annotations[velerov1api.DriverNameAnnotation] = vsc.Spec.Driver + } + if vsc.Status.RestoreSize != nil { + annotations[velerov1api.VolumeSnapshotRestoreSize] = resource.NewQuantity( + *vsc.Status.RestoreSize, resource.BinarySI).String() + } + } + + if backupOngoing { + p.log.Infof("Patching VolumeSnapshotContent %s with velero BackupNameLabel", + vsc.Name) + // If we created the VolumeSnapshotContent object during this ongoing backup, + // we would have created it with a DeletionPolicy of Retain. + // But, we want to retain these VolumeSnapshotContent ONLY for the lifetime + // of the backup. To that effect, during velero backup + // deletion, we will update the DeletionPolicy of the VolumeSnapshotContent + // and then delete the VolumeSnapshot object which will cascade delete the + // VolumeSnapshotContent and the associated snapshot in the storage + // provider (handled by the CSI driver and the CSI common controller). + // However, in the event that the VolumeSnapshot object is deleted outside + // of the backup deletion process, it is possible that the dynamically created + // VolumeSnapshotContent object will be left as an orphaned and non-discoverable + // resource in the cluster as well as in the storage provider. To avoid piling + // up of such orphaned resources, we will want to discover and delete the + // dynamically created VolumeSnapshotContent. We do that by adding + // the "velero.io/backup-name" label on the VolumeSnapshotContent. + // Further, we want to add this label only on VolumeSnapshotContents that + // were created during an ongoing velero backup. + originVSC := vsc.DeepCopy() + kubeutil.AddLabels( + &vsc.ObjectMeta, + map[string]string{ + velerov1api.BackupNameLabel: label.GetValidName(backup.Name), + }, + ) + + if vscPatchError := p.crClient.Patch( + context.TODO(), + vsc, + crclient.MergeFrom(originVSC), + ); vscPatchError != nil { + p.log.Warnf("Failed to patch VolumeSnapshotContent %s: %v", + vsc.Name, vscPatchError) + } + } + } + + // Before applying the BIA v2, the in-cluster VS state is not persisted into backup. + // After the change, because the final state of VS will be stored in backup as the + // result of async operation result, need to patch the annotations into VS to work, + // because restore will check the annotations information. + originVS := vs.DeepCopy() + kubeutil.AddAnnotations(&vs.ObjectMeta, annotations) + if err := p.crClient.Patch( + context.TODO(), + vs, + crclient.MergeFrom(originVS), + ); err != nil { + p.log.Errorf("Fail to patch VolumeSnapshot: %s.", err.Error()) + return nil, nil, "", nil, errors.WithStack(err) + } + + annotations[velerov1api.MustIncludeAdditionalItemAnnotation] = "true" + // save newly applied annotations into the backed-up VolumeSnapshot item + kubeutil.AddAnnotations(&vs.ObjectMeta, annotations) + + vsMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(vs) + if err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + p.log.Infof("Returning from VolumeSnapshotBackupItemAction", + "with %d additionalItems to backup", len(additionalItems)) + for _, ai := range additionalItems { + p.log.Debugf("%s: %s", ai.GroupResource.String(), ai.Name) + } + + operationID := "" + var itemToUpdate []velero.ResourceIdentifier + + // Only return Async operation for VSC created for this backup. + if backupOngoing { + // The operationID is of the form // + operationID = vs.Namespace + "/" + vs.Name + "/" + time.Now().Format(time.RFC3339) + itemToUpdate = []velero.ResourceIdentifier{ + { + GroupResource: kuberesource.VolumeSnapshots, + Namespace: vs.Namespace, + Name: vs.Name, + }, + { + GroupResource: kuberesource.VolumeSnapshotContents, + Name: vsc.Name, + }, + } + } + + return &unstructured.Unstructured{Object: vsMap}, + additionalItems, operationID, itemToUpdate, nil +} + +// Name returns the plugin's name. +func (p *volumeSnapshotBackupItemAction) Name() string { + return "VolumeSnapshotBackupItemAction" +} + +func (p *volumeSnapshotBackupItemAction) Progress( + operationID string, + backup *velerov1api.Backup, +) (velero.OperationProgress, error) { + progress := velero.OperationProgress{} + if operationID == "" { + return progress, biav2.InvalidOperationIDError(operationID) + } + // The operationID is of the form // + operationIDParts := strings.Split(operationID, "/") + if len(operationIDParts) != 3 { + p.log.Errorf("invalid operation ID %s", operationID) + return progress, biav2.InvalidOperationIDError(operationID) + } + var err error + if progress.Started, err = time.Parse(time.RFC3339, operationIDParts[2]); err != nil { + p.log.Errorf("error parsing operation ID's StartedTime", + "part into time %s: %s", operationID, err.Error()) + return progress, errors.WithStack(err) + } + + vs := new(snapshotv1api.VolumeSnapshot) + if err := p.crClient.Get( + context.Background(), + crclient.ObjectKey{ + Namespace: operationIDParts[0], + Name: operationIDParts[1], + }, + vs); err != nil { + p.log.Errorf("error getting volumesnapshot %s/%s: %s", + operationIDParts[0], operationIDParts[1], err.Error()) + return progress, errors.WithStack(err) + } + + if vs.Status == nil { + p.log.Debugf("VolumeSnapshot %s/%s has an empty status.", + "Skip progress update.", vs.Namespace, vs.Name) + return progress, nil + } + + if boolptr.IsSetToTrue(vs.Status.ReadyToUse) { + p.log.Debugf("VolumeSnapshot %s/%s is ReadyToUse.", + "Continue on querying corresponding VolumeSnapshotContent.", + vs.Namespace, vs.Name) + } else if vs.Status.Error != nil { + errorMessage := "" + if vs.Status.Error.Message != nil { + errorMessage = *vs.Status.Error.Message + } + p.log.Warnf("VolumeSnapshot has a temporary error %s.", + "Snapshot controller will retry later.", errorMessage) + + return progress, nil + } + + if vs.Status != nil && vs.Status.BoundVolumeSnapshotContentName != nil { + vsc := new(snapshotv1api.VolumeSnapshotContent) + err := p.crClient.Get( + context.Background(), + crclient.ObjectKey{Name: *vs.Status.BoundVolumeSnapshotContentName}, + vsc, + ) + if err != nil { + p.log.Errorf("error getting VolumeSnapshotContent %s: %s", + *vs.Status.BoundVolumeSnapshotContentName, err.Error()) + return progress, errors.WithStack(err) + } + + if vsc.Status == nil { + p.log.Debugf("VolumeSnapshotContent %s has an empty Status.", + "Skip progress update.", vsc.Name) + return progress, nil + } + + now := time.Now() + + if boolptr.IsSetToTrue(vsc.Status.ReadyToUse) { + progress.Completed = true + progress.Updated = now + } else if vsc.Status.Error != nil { + progress.Completed = true + progress.Updated = now + if vsc.Status.Error.Message != nil { + progress.Err = *vsc.Status.Error.Message + } + p.log.Warnf("VolumeSnapshotContent meets an error %s.", progress.Err) + } + } + + return progress, nil +} + +// Cancel is not implemented for VolumeSnapshotBackupItemAction +func (p *volumeSnapshotBackupItemAction) Cancel( + operationID string, + backup *velerov1api.Backup, +) error { + // CSI Specification doesn't support canceling a snapshot creation. + return nil +} + +// NewVolumeSnapshotBackupItemAction returns +// VolumeSnapshotBackupItemAction instance. +func NewVolumeSnapshotBackupItemAction( + f client.Factory, +) plugincommon.HandlerInitializer { + return func(logger logrus.FieldLogger) (interface{}, error) { + crClient, err := f.KubebuilderClient() + if err != nil { + return nil, errors.WithStack(err) + } + + return &volumeSnapshotBackupItemAction{ + log: logger, + crClient: crClient, + }, nil + } +} diff --git a/pkg/backup/actions/csi/volumesnapshotclass_action.go b/pkg/backup/actions/csi/volumesnapshotclass_action.go new file mode 100644 index 000000000..d6f9ca4e5 --- /dev/null +++ b/pkg/backup/actions/csi/volumesnapshotclass_action.go @@ -0,0 +1,124 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + csiutil "github.com/vmware-tanzu/velero/pkg/util/csi" + kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" +) + +// volumeSnapshotClassBackupItemAction is a backup item action plugin to +// backup CSI VolumeSnapshotClass objects using Velero +type volumeSnapshotClassBackupItemAction struct { + log logrus.FieldLogger +} + +// AppliesTo returns information indicating that the +// VolumeSnapshotClassBackupItemAction action should be invoked +// to backup VolumeSnapshotClass. +func (p *volumeSnapshotClassBackupItemAction) AppliesTo() ( + velero.ResourceSelector, + error, +) { + return velero.ResourceSelector{ + IncludedResources: []string{"volumesnapshotclass.snapshot.storage.k8s.io"}, + }, nil +} + +// Execute backs up a VolumeSnapshotClass object and returns as additional +// items any snapshot lister secret that may be referenced in its annotations. +func (p *volumeSnapshotClassBackupItemAction) Execute( + item runtime.Unstructured, + backup *velerov1api.Backup, +) ( + runtime.Unstructured, + []velero.ResourceIdentifier, + string, + []velero.ResourceIdentifier, + error, +) { + p.log.Infof("Executing VolumeSnapshotClassBackupItemAction") + + var snapClass snapshotv1api.VolumeSnapshotClass + if err := runtime.DefaultUnstructuredConverter.FromUnstructured( + item.UnstructuredContent(), + &snapClass, + ); err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + additionalItems := []velero.ResourceIdentifier{} + if csiutil.IsVolumeSnapshotClassHasListerSecret(&snapClass) { + additionalItems = append(additionalItems, velero.ResourceIdentifier{ + GroupResource: kuberesource.Secrets, + Name: snapClass.Annotations[velerov1api.PrefixedListSecretNameAnnotation], + Namespace: snapClass.Annotations[velerov1api.PrefixedListSecretNamespaceAnnotation], + }) + + kubeutil.AddAnnotations(&snapClass.ObjectMeta, map[string]string{ + velerov1api.MustIncludeAdditionalItemAnnotation: "true", + }) + } + + snapClassMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&snapClass) + if err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + p.log.Infof( + "Returning from VolumeSnapshotClassBackupItemAction with %d additionalItems to backup", + len(additionalItems), + ) + return &unstructured.Unstructured{Object: snapClassMap}, additionalItems, "", nil, nil +} + +// Name returns the plugin's name. +func (p *volumeSnapshotClassBackupItemAction) Name() string { + return "VolumeSnapshotClassBackupItemAction" +} + +// Progress is not implemented for VolumeSnapshotClassBackupItemAction +func (p *volumeSnapshotClassBackupItemAction) Progress( + operationID string, + backup *velerov1api.Backup, +) (velero.OperationProgress, error) { + return velero.OperationProgress{}, nil +} + +// Cancel is not implemented for VolumeSnapshotClassBackupItemAction +func (p *volumeSnapshotClassBackupItemAction) Cancel( + operationID string, + backup *velerov1api.Backup, +) error { + return nil +} + +// NewVolumeSnapshotClassBackupItemAction returns a +// VolumeSnapshotClassBackupItemAction instance. +func NewVolumeSnapshotClassBackupItemAction(logger logrus.FieldLogger) (interface{}, error) { + return &volumeSnapshotClassBackupItemAction{log: logger}, nil +} diff --git a/pkg/backup/actions/csi/volumesnapshotcontent_action.go b/pkg/backup/actions/csi/volumesnapshotcontent_action.go new file mode 100644 index 000000000..e8e17280d --- /dev/null +++ b/pkg/backup/actions/csi/volumesnapshotcontent_action.go @@ -0,0 +1,141 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "fmt" + + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + csiutil "github.com/vmware-tanzu/velero/pkg/util/csi" + kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" +) + +// volumeSnapshotContentBackupItemAction is a backup item action plugin to backup +// CSI VolumeSnapshotContent objects using Velero +type volumeSnapshotContentBackupItemAction struct { + log logrus.FieldLogger +} + +// AppliesTo returns information indicating that the +// VolumeSnapshotContentBackupItemAction action should be invoked to +// backup VolumeSnapshotContents. +func (p *volumeSnapshotContentBackupItemAction) AppliesTo() (velero.ResourceSelector, error) { + return velero.ResourceSelector{ + IncludedResources: []string{"volumesnapshotcontent.snapshot.storage.k8s.io"}, + }, nil +} + +// Execute returns the unmodified VolumeSnapshotContent object along +// with the snapshot deletion secret, if any, from its annotation +// as additional items to backup. +func (p *volumeSnapshotContentBackupItemAction) Execute( + item runtime.Unstructured, + backup *velerov1api.Backup, +) ( + runtime.Unstructured, + []velero.ResourceIdentifier, + string, + []velero.ResourceIdentifier, + error, +) { + p.log.Infof("Executing VolumeSnapshotContentBackupItemAction") + + if backup.Status.Phase == velerov1api.BackupPhaseFinalizing || + backup.Status.Phase == velerov1api.BackupPhaseFinalizingPartiallyFailed { + p.log.WithField("Backup", fmt.Sprintf("%s/%s", backup.Namespace, backup.Name)). + WithField("BackupPhase", backup.Status.Phase). + Debug("Skipping VolumeSnapshotContentBackupItemAction", + "as backup is in finalizing phase.") + return item, nil, "", nil, nil + } + + var snapCont snapshotv1api.VolumeSnapshotContent + if err := runtime.DefaultUnstructuredConverter.FromUnstructured( + item.UnstructuredContent(), + &snapCont, + ); err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + additionalItems := []velero.ResourceIdentifier{} + + // we should backup the snapshot deletion secrets that may be referenced + // in the VolumeSnapshotContent's annotation + if csiutil.IsVolumeSnapshotContentHasDeleteSecret(&snapCont) { + additionalItems = append( + additionalItems, + velero.ResourceIdentifier{ + GroupResource: kuberesource.Secrets, + Name: snapCont.Annotations[velerov1api.PrefixedSecretNameAnnotation], + Namespace: snapCont.Annotations[velerov1api.PrefixedSecretNamespaceAnnotation], + }) + + kubeutil.AddAnnotations(&snapCont.ObjectMeta, map[string]string{ + velerov1api.MustIncludeAdditionalItemAnnotation: "true", + }) + } + + snapContMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&snapCont) + if err != nil { + return nil, nil, "", nil, errors.WithStack(err) + } + + p.log.Infof( + "Returning from VolumeSnapshotContentBackupItemAction", + "with %d additionalItems to backup", + len(additionalItems), + ) + return &unstructured.Unstructured{Object: snapContMap}, additionalItems, "", nil, nil +} + +// Name returns the plugin's name. +func (p *volumeSnapshotContentBackupItemAction) Name() string { + return "VolumeSnapshotContentBackupItemAction" +} + +// Progress is not implemented for VolumeSnapshotContentBackupItemAction. +func (p *volumeSnapshotContentBackupItemAction) Progress( + operationID string, + backup *velerov1api.Backup, +) (velero.OperationProgress, error) { + return velero.OperationProgress{}, nil +} + +// Cancel is not implemented for VolumeSnapshotContentBackupItemAction. +func (p *volumeSnapshotContentBackupItemAction) Cancel( + operationID string, + backup *velerov1api.Backup, +) error { + // CSI Specification doesn't support canceling a snapshot creation. + return nil +} + +// NewVolumeSnapshotContentBackupItemAction returns a +// VolumeSnapshotContentBackupItemAction instance. +func NewVolumeSnapshotContentBackupItemAction( + logger logrus.FieldLogger, +) (interface{}, error) { + return &volumeSnapshotContentBackupItemAction{log: logger}, nil +} diff --git a/pkg/backup/backup_test.go b/pkg/backup/backup_test.go index 0a8c7db78..142570648 100644 --- a/pkg/backup/backup_test.go +++ b/pkg/backup/backup_test.go @@ -383,16 +383,16 @@ func TestBackupOldResourceFiltering(t *testing.T) { Result(), apiResources: []*test.APIResource{ test.Pods( - builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "true")).Result(), + builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true")).Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), - builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "true")).Result(), + builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true")).Result(), ), test.PVs( builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a", "b")).Result(), - builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "true")).Result(), + builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true")).Result(), ), }, want: []string{ @@ -411,16 +411,16 @@ func TestBackupOldResourceFiltering(t *testing.T) { Result(), apiResources: []*test.APIResource{ test.Pods( - builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "true", "a", "b")).Result(), + builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true", "a", "b")).Result(), builder.ForPod("zoo", "raz").ObjectMeta(builder.WithLabels("a", "b")).Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), - builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "true", "a", "b")).Result(), + builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true", "a", "b")).Result(), ), test.PVs( builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a", "b")).Result(), - builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels("a", "b", "velero.io/exclude-from-backup", "true")).Result(), + builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels("a", "b", velerov1.ExcludeFromBackupLabel, "true")).Result(), ), }, want: []string{ @@ -436,16 +436,16 @@ func TestBackupOldResourceFiltering(t *testing.T) { Result(), apiResources: []*test.APIResource{ test.Pods( - builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "false")).Result(), + builder.ForPod("foo", "bar").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "false")).Result(), builder.ForPod("zoo", "raz").Result(), ), test.Deployments( builder.ForDeployment("foo", "bar").Result(), - builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "1")).Result(), + builder.ForDeployment("zoo", "raz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "1")).Result(), ), test.PVs( builder.ForPersistentVolume("bar").ObjectMeta(builder.WithLabels("a", "b")).Result(), - builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "")).Result(), + builder.ForPersistentVolume("baz").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "")).Result(), ), }, want: []string{ @@ -1273,7 +1273,7 @@ func (a *recordResourcesAction) Execute(item runtime.Unstructured, backup *veler a.backups = append(a.backups, *backup) if a.skippedCSISnapshot { u := &unstructured.Unstructured{Object: item.UnstructuredContent()} - u.SetAnnotations(map[string]string{skippedNoCSIPVAnnotation: "true"}) + u.SetAnnotations(map[string]string{velerov1.SkippedNoCSIPVAnnotation: "true"}) item = u a.additionalItems = nil } @@ -2028,7 +2028,7 @@ func TestBackupActionAdditionalItems(t *testing.T) { builder.ForPod("ns-1", "pod-1").Result(), ), test.PVs( - builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabels("velero.io/exclude-from-backup", "true")).Result(), + builder.ForPersistentVolume("pv-1").ObjectMeta(builder.WithLabels(velerov1.ExcludeFromBackupLabel, "true")).Result(), builder.ForPersistentVolume("pv-2").Result(), ), }, diff --git a/pkg/backup/item_backupper.go b/pkg/backup/item_backupper.go index 4b589fae3..953b3191e 100644 --- a/pkg/backup/item_backupper.go +++ b/pkg/backup/item_backupper.go @@ -24,19 +24,17 @@ import ( "strings" "time" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "github.com/pkg/errors" "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" kubeerrs "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" - kbClient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/hook" @@ -58,11 +56,8 @@ import ( ) const ( - mustIncludeAdditionalItemAnnotation = "backup.velero.io/must-include-additional-items" - skippedNoCSIPVAnnotation = "backup.velero.io/skipped-no-csi-pv" - excludeFromBackupLabel = "velero.io/exclude-from-backup" - csiBIAPluginName = "velero.io/csi-pvc-backupper" - vsphereBIAPluginName = "velero.io/vsphere-pvc-backupper" + csiBIAPluginName = "velero.io/csi-pvc-backupper" + vsphereBIAPluginName = "velero.io/vsphere-pvc-backupper" ) // itemBackupper can back up individual items to a tar writer. @@ -129,9 +124,9 @@ func (ib *itemBackupper) backupItemInternal(logger logrus.FieldLogger, obj runti if mustInclude { log.Infof("Skipping the exclusion checks for this resource") } else { - if metadata.GetLabels()[excludeFromBackupLabel] == "true" { - log.Infof("Excluding item because it has label %s=true", excludeFromBackupLabel) - ib.trackSkippedPV(obj, groupResource, "", fmt.Sprintf("item has label %s=true", excludeFromBackupLabel), log) + if metadata.GetLabels()[velerov1api.ExcludeFromBackupLabel] == "true" { + log.Infof("Excluding item because it has label %s=true", velerov1api.ExcludeFromBackupLabel) + ib.trackSkippedPV(obj, groupResource, "", fmt.Sprintf("item has label %s=true", velerov1api.ExcludeFromBackupLabel), log) return false, itemFiles, nil } // NOTE: we have to re-check namespace & resource includes/excludes because it's possible that @@ -384,18 +379,18 @@ func (ib *itemBackupper) executeActions( return nil, itemFiles, errors.Wrapf(err, "error executing custom action (groupResource=%s, namespace=%s, name=%s)", groupResource.String(), namespace, name) } u := &unstructured.Unstructured{Object: updatedItem.UnstructuredContent()} - if actionName == csiBIAPluginName && additionalItemIdentifiers == nil && u.GetAnnotations()[skippedNoCSIPVAnnotation] == "true" { + if actionName == csiBIAPluginName && additionalItemIdentifiers == nil && u.GetAnnotations()[velerov1api.SkippedNoCSIPVAnnotation] == "true" { // snapshot was skipped by CSI plugin ib.trackSkippedPV(obj, groupResource, csiSnapshotApproach, "skipped b/c it's not a CSI volume", log) - delete(u.GetAnnotations(), skippedNoCSIPVAnnotation) + delete(u.GetAnnotations(), velerov1api.SkippedNoCSIPVAnnotation) } else if (actionName == csiBIAPluginName || actionName == vsphereBIAPluginName) && !boolptr.IsSetToFalse(ib.backupRequest.Backup.Spec.SnapshotVolumes) { // the snapshot has been taken by the BIA plugin ib.unTrackSkippedPV(obj, groupResource, log) } - mustInclude := u.GetAnnotations()[mustIncludeAdditionalItemAnnotation] == "true" || finalize + mustInclude := u.GetAnnotations()[velerov1api.MustIncludeAdditionalItemAnnotation] == "true" || finalize // remove the annotation as it's for communication between BIA and velero server, // we don't want the resource be restored with this annotation. - delete(u.GetAnnotations(), mustIncludeAdditionalItemAnnotation) + delete(u.GetAnnotations(), velerov1api.MustIncludeAdditionalItemAnnotation) obj = u // If async plugin started async operation, add it to the ItemOperations list diff --git a/pkg/client/factory.go b/pkg/client/factory.go index d359ee229..c74f43b88 100644 --- a/pkg/client/factory.go +++ b/pkg/client/factory.go @@ -195,6 +195,9 @@ func (f *factory) KubebuilderWatchClient() (kbclient.WithWatch, error) { if err := apiextv1.AddToScheme(scheme); err != nil { return nil, err } + if err := snapshotv1api.AddToScheme(scheme); err != nil { + return nil, err + } kubebuilderWatchClient, err := kbclient.NewWithWatch(clientConfig, kbclient.Options{ Scheme: scheme, }) diff --git a/pkg/cmd/server/plugin/plugin.go b/pkg/cmd/server/plugin/plugin.go index c375c5437..0f729f3ac 100644 --- a/pkg/cmd/server/plugin/plugin.go +++ b/pkg/cmd/server/plugin/plugin.go @@ -23,14 +23,17 @@ import ( "github.com/vmware-tanzu/velero/pkg/datamover" + dia "github.com/vmware-tanzu/velero/internal/delete/actions/csi" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" bia "github.com/vmware-tanzu/velero/pkg/backup/actions" + csibia "github.com/vmware-tanzu/velero/pkg/backup/actions/csi" "github.com/vmware-tanzu/velero/pkg/client" velerodiscovery "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/features" veleroplugin "github.com/vmware-tanzu/velero/pkg/plugin/framework" plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" ria "github.com/vmware-tanzu/velero/pkg/restore/actions" + csiria "github.com/vmware-tanzu/velero/pkg/restore/actions/csi" ) func NewCommand(f client.Factory) *cobra.Command { @@ -41,32 +44,142 @@ func NewCommand(f client.Factory) *cobra.Command { Short: "INTERNAL COMMAND ONLY - not intended to be run directly by users", Run: func(c *cobra.Command, args []string) { pluginServer = pluginServer. - RegisterBackupItemAction("velero.io/pv", newPVBackupItemAction). - RegisterBackupItemAction("velero.io/pod", newPodBackupItemAction). - RegisterBackupItemAction("velero.io/service-account", newServiceAccountBackupItemAction(f)). - RegisterRestoreItemAction("velero.io/job", newJobRestoreItemAction). - RegisterRestoreItemAction("velero.io/pod", newPodRestoreItemAction). - RegisterRestoreItemAction("velero.io/pod-volume-restore", newPodVolumeRestoreItemAction(f)). - RegisterRestoreItemAction("velero.io/init-restore-hook", newInitRestoreHookPodAction). - RegisterRestoreItemAction("velero.io/service", newServiceRestoreItemAction). - RegisterRestoreItemAction("velero.io/service-account", newServiceAccountRestoreItemAction). - RegisterRestoreItemAction("velero.io/add-pvc-from-pod", newAddPVCFromPodRestoreItemAction). - RegisterRestoreItemAction("velero.io/add-pv-from-pvc", newAddPVFromPVCRestoreItemAction). - RegisterRestoreItemAction("velero.io/change-storage-class", newChangeStorageClassRestoreItemAction(f)). - RegisterRestoreItemAction("velero.io/change-image-name", newChangeImageNameRestoreItemAction(f)). - RegisterRestoreItemAction("velero.io/role-bindings", newRoleBindingItemAction). - RegisterRestoreItemAction("velero.io/cluster-role-bindings", newClusterRoleBindingItemAction). - RegisterRestoreItemAction("velero.io/crd-preserve-fields", newCRDV1PreserveUnknownFieldsItemAction). - RegisterRestoreItemAction("velero.io/change-pvc-node-selector", newChangePVCNodeSelectorItemAction(f)). - RegisterRestoreItemAction("velero.io/apiservice", newAPIServiceRestoreItemAction). - RegisterRestoreItemAction("velero.io/admission-webhook-configuration", newAdmissionWebhookConfigurationAction). - RegisterRestoreItemAction("velero.io/secret", newSecretRestoreItemAction(f)). - RegisterRestoreItemAction("velero.io/dataupload", newDataUploadRetrieveAction(f)). - RegisterDeleteItemAction("velero.io/dataupload-delete", newDateUploadDeleteItemAction(f)) + RegisterBackupItemAction( + "velero.io/pv", + newPVBackupItemAction, + ). + RegisterBackupItemAction( + "velero.io/pod", + newPodBackupItemAction, + ). + RegisterBackupItemAction( + "velero.io/service-account", + newServiceAccountBackupItemAction(f), + ). + RegisterRestoreItemAction( + "velero.io/job", + newJobRestoreItemAction, + ). + RegisterRestoreItemAction( + "velero.io/pod", + newPodRestoreItemAction, + ). + RegisterRestoreItemAction( + "velero.io/pod-volume-restore", + newPodVolumeRestoreItemAction(f), + ). + RegisterRestoreItemAction( + "velero.io/init-restore-hook", + newInitRestoreHookPodAction, + ). + RegisterRestoreItemAction( + "velero.io/service", + newServiceRestoreItemAction, + ). + RegisterRestoreItemAction( + "velero.io/service-account", + newServiceAccountRestoreItemAction, + ). + RegisterRestoreItemAction( + "velero.io/add-pvc-from-pod", + newAddPVCFromPodRestoreItemAction, + ). + RegisterRestoreItemAction( + "velero.io/add-pv-from-pvc", + newAddPVFromPVCRestoreItemAction, + ). + RegisterRestoreItemAction( + "velero.io/change-storage-class", + newChangeStorageClassRestoreItemAction(f), + ). + RegisterRestoreItemAction( + "velero.io/change-image-name", + newChangeImageNameRestoreItemAction(f), + ). + RegisterRestoreItemAction( + "velero.io/role-bindings", + newRoleBindingItemAction, + ). + RegisterRestoreItemAction( + "velero.io/cluster-role-bindings", + newClusterRoleBindingItemAction, + ). + RegisterRestoreItemAction( + "velero.io/crd-preserve-fields", + newCRDV1PreserveUnknownFieldsItemAction, + ). + RegisterRestoreItemAction( + "velero.io/change-pvc-node-selector", + newChangePVCNodeSelectorItemAction(f), + ). + RegisterRestoreItemAction( + "velero.io/apiservice", + newAPIServiceRestoreItemAction, + ). + RegisterRestoreItemAction( + "velero.io/admission-webhook-configuration", + newAdmissionWebhookConfigurationAction, + ). + RegisterRestoreItemAction( + "velero.io/secret", + newSecretRestoreItemAction(f), + ). + RegisterRestoreItemAction( + "velero.io/dataupload", + newDataUploadRetrieveAction(f), + ). + RegisterDeleteItemAction( + "velero.io/dataupload-delete", + newDateUploadDeleteItemAction(f), + ). + RegisterDeleteItemAction( + "velero.io/csi-volumesnapshot-delete", + newVolumeSnapshotDeleteItemAction(f), + ). + RegisterDeleteItemAction( + "velero.io/csi-volumesnapshotcontent-delete", + newVolumeSnapshotContentDeleteItemAction(f), + ). + RegisterBackupItemActionV2( + "velero.io/csi-pvc-backupper", + newPvcBackupItemAction(f), + ). + RegisterBackupItemActionV2( + "velero.io/csi-volumesnapshot-backupper", + newVolumeSnapshotBackupItemAction(f), + ). + RegisterBackupItemActionV2( + "velero.io/csi-volumesnapshotcontent-backupper", + newVolumeSnapshotContentBackupItemAction, + ). + RegisterBackupItemActionV2( + "velero.io/csi-volumesnapshotclass-backupper", + newVolumeSnapshotClassBackupItemAction, + ). + RegisterRestoreItemActionV2( + "velero.io/csi-pvc-restorer", + newPvcRestoreItemAction(f), + ). + RegisterRestoreItemActionV2( + "velero.io/csi-volumesnapshot-restorer", + newVolumeSnapshotRestoreItemAction(f), + ). + RegisterRestoreItemActionV2( + "velero.io/csi-volumesnapshotcontent-restorer", + newVolumeSnapshotContentRestoreItemAction, + ). + RegisterRestoreItemActionV2( + "velero.io/csi-volumesnapshotclass-restorer", + newVolumeSnapshotClassRestoreItemAction, + ) if !features.IsEnabled(velerov1api.APIGroupVersionsFeatureFlag) { - // Do not register crd-remap-version BIA if the API Group feature flag is enabled, so that the v1 CRD can be backed up - pluginServer = pluginServer.RegisterBackupItemAction("velero.io/crd-remap-version", newRemapCRDVersionAction(f)) + // Do not register crd-remap-version BIA if the API Group feature + // flag is enabled, so that the v1 CRD can be backed up. + pluginServer = pluginServer.RegisterBackupItemAction( + "velero.io/crd-remap-version", + newRemapCRDVersionAction(f), + ) } pluginServer.Serve() }, @@ -270,3 +383,51 @@ func newDateUploadDeleteItemAction(f client.Factory) plugincommon.HandlerInitial return datamover.NewDataUploadDeleteAction(logger, client), nil } } + +// CSI plugin init functions. + +// BackupItemAction plugins + +func newPvcBackupItemAction(f client.Factory) plugincommon.HandlerInitializer { + return csibia.NewPvcBackupItemAction(f) +} + +func newVolumeSnapshotBackupItemAction(f client.Factory) plugincommon.HandlerInitializer { + return csibia.NewVolumeSnapshotBackupItemAction(f) +} + +func newVolumeSnapshotContentBackupItemAction(logger logrus.FieldLogger) (interface{}, error) { + return csibia.NewVolumeSnapshotContentBackupItemAction(logger) +} + +func newVolumeSnapshotClassBackupItemAction(logger logrus.FieldLogger) (interface{}, error) { + return csibia.NewVolumeSnapshotClassBackupItemAction(logger) +} + +// DeleteItemAction plugins + +func newVolumeSnapshotDeleteItemAction(f client.Factory) plugincommon.HandlerInitializer { + return dia.NewVolumeSnapshotDeleteItemAction(f) +} + +func newVolumeSnapshotContentDeleteItemAction(f client.Factory) plugincommon.HandlerInitializer { + return dia.NewVolumeSnapshotContentDeleteItemAction(f) +} + +// RestoreItemAction plugins + +func newPvcRestoreItemAction(f client.Factory) plugincommon.HandlerInitializer { + return csiria.NewPvcRestoreItemAction(f) +} + +func newVolumeSnapshotRestoreItemAction(f client.Factory) plugincommon.HandlerInitializer { + return csiria.NewVolumeSnapshotRestoreItemAction(f) +} + +func newVolumeSnapshotContentRestoreItemAction(logger logrus.FieldLogger) (interface{}, error) { + return csiria.NewVolumeSnapshotContentRestoreItemAction(logger) +} + +func newVolumeSnapshotClassRestoreItemAction(logger logrus.FieldLogger) (interface{}, error) { + return csiria.NewVolumeSnapshotClassRestoreItemAction(logger) +} diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index da915ed7e..162d388f9 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -431,7 +431,7 @@ func (b *backupReconciler) prepareBackupRequest(backup *velerov1api.Backup, logg // Add namespaces with label velero.io/exclude-from-backup=true into request.Spec.ExcludedNamespaces // Essentially, adding the label velero.io/exclude-from-backup=true to a namespace would be equivalent to setting spec.ExcludedNamespaces namespaces := corev1api.NamespaceList{} - if err := b.kbClient.List(context.Background(), &namespaces, kbclient.MatchingLabels{"velero.io/exclude-from-backup": "true"}); err == nil { + if err := b.kbClient.List(context.Background(), &namespaces, kbclient.MatchingLabels{velerov1api.ExcludeFromBackupLabel: "true"}); err == nil { for _, ns := range namespaces.Items { request.Spec.ExcludedNamespaces = append(request.Spec.ExcludedNamespaces, ns.Name) } diff --git a/pkg/exposer/csi_snapshot.go b/pkg/exposer/csi_snapshot.go index 637d0805d..68dce2437 100644 --- a/pkg/exposer/csi_snapshot.go +++ b/pkg/exposer/csi_snapshot.go @@ -20,27 +20,22 @@ import ( "context" "time" + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + snapshotter "github.com/kubernetes-csi/external-snapshotter/client/v7/clientset/versioned/typed/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" - - "github.com/vmware-tanzu/velero/pkg/nodeagent" - "github.com/vmware-tanzu/velero/pkg/util/boolptr" - corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" - - snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" - - "github.com/vmware-tanzu/velero/pkg/util/csi" - "github.com/vmware-tanzu/velero/pkg/util/kube" - - snapshotter "github.com/kubernetes-csi/external-snapshotter/client/v7/clientset/versioned/typed/volumesnapshot/v1" "sigs.k8s.io/controller-runtime/pkg/client" - apierrors "k8s.io/apimachinery/pkg/api/errors" + "github.com/vmware-tanzu/velero/pkg/nodeagent" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/csi" + "github.com/vmware-tanzu/velero/pkg/util/kube" ) // CSISnapshotExposeParam define the input param for Expose of CSI snapshots diff --git a/pkg/exposer/csi_snapshot_test.go b/pkg/exposer/csi_snapshot_test.go index 6fbee7450..4bed98cd1 100644 --- a/pkg/exposer/csi_snapshot_test.go +++ b/pkg/exposer/csi_snapshot_test.go @@ -160,7 +160,7 @@ func TestExpose(t *testing.T) { OperationTimeout: time.Millisecond, ExposeTimeout: time.Millisecond, }, - err: "error wait volume snapshot ready: error to get volumesnapshot /fake-vs: volumesnapshots.snapshot.storage.k8s.io \"fake-vs\" not found", + err: "error wait volume snapshot ready: error to get VolumeSnapshot /fake-vs: volumesnapshots.snapshot.storage.k8s.io \"fake-vs\" not found", }, { name: "get vsc fail", diff --git a/pkg/restore/actions/add_pv_from_pvc_action.go b/pkg/restore/actions/add_pv_from_pvc_action.go index e9e882d07..d597a33fd 100644 --- a/pkg/restore/actions/add_pv_from_pvc_action.go +++ b/pkg/restore/actions/add_pv_from_pvc_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "github.com/pkg/errors" diff --git a/pkg/restore/actions/add_pv_from_pvc_action_test.go b/pkg/restore/actions/add_pv_from_pvc_action_test.go index 4124fb2d8..625b35dbd 100644 --- a/pkg/restore/actions/add_pv_from_pvc_action_test.go +++ b/pkg/restore/actions/add_pv_from_pvc_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "testing" diff --git a/pkg/restore/actions/add_pvc_from_pod_action.go b/pkg/restore/actions/add_pvc_from_pod_action.go index 70f33d985..3e88f796a 100644 --- a/pkg/restore/actions/add_pvc_from_pod_action.go +++ b/pkg/restore/actions/add_pvc_from_pod_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "github.com/pkg/errors" diff --git a/pkg/restore/actions/add_pvc_from_pod_action_test.go b/pkg/restore/actions/add_pvc_from_pod_action_test.go index 5e99006ad..44b5fb4ad 100644 --- a/pkg/restore/actions/add_pvc_from_pod_action_test.go +++ b/pkg/restore/actions/add_pvc_from_pod_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "testing" diff --git a/pkg/restore/actions/admissionwebhook_config_action.go b/pkg/restore/actions/admissionwebhook_config_action.go index 74fd1b3ff..c6c967699 100644 --- a/pkg/restore/actions/admissionwebhook_config_action.go +++ b/pkg/restore/actions/admissionwebhook_config_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "github.com/pkg/errors" diff --git a/pkg/restore/actions/admissionwebhook_config_action_test.go b/pkg/restore/actions/admissionwebhook_config_action_test.go index cbc019ff0..b0f24f7f5 100644 --- a/pkg/restore/actions/admissionwebhook_config_action_test.go +++ b/pkg/restore/actions/admissionwebhook_config_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "encoding/json" diff --git a/pkg/restore/actions/apiservice_action.go b/pkg/restore/actions/apiservice_action.go index 7f817a59e..f2433352b 100644 --- a/pkg/restore/actions/apiservice_action.go +++ b/pkg/restore/actions/apiservice_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "github.com/sirupsen/logrus" diff --git a/pkg/restore/actions/apiservice_action_test.go b/pkg/restore/actions/apiservice_action_test.go index 81f4a6171..0e37e8936 100644 --- a/pkg/restore/actions/apiservice_action_test.go +++ b/pkg/restore/actions/apiservice_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "testing" diff --git a/pkg/restore/actions/change_image_name_action.go b/pkg/restore/actions/change_image_name_action.go index 0bd96b0b4..66fac1f18 100644 --- a/pkg/restore/actions/change_image_name_action.go +++ b/pkg/restore/actions/change_image_name_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "context" diff --git a/pkg/restore/actions/change_image_name_action_test.go b/pkg/restore/actions/change_image_name_action_test.go index d851e7c0d..5aa409bf4 100644 --- a/pkg/restore/actions/change_image_name_action_test.go +++ b/pkg/restore/actions/change_image_name_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "context" diff --git a/pkg/restore/actions/change_pvc_node_selector.go b/pkg/restore/actions/change_pvc_node_selector.go index 5934cd266..2888b6b2d 100644 --- a/pkg/restore/actions/change_pvc_node_selector.go +++ b/pkg/restore/actions/change_pvc_node_selector.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "context" diff --git a/pkg/restore/actions/change_pvc_node_selector_test.go b/pkg/restore/actions/change_pvc_node_selector_test.go index 7cfbd170c..615f8e10d 100644 --- a/pkg/restore/actions/change_pvc_node_selector_test.go +++ b/pkg/restore/actions/change_pvc_node_selector_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "bytes" diff --git a/pkg/restore/actions/change_storageclass_action.go b/pkg/restore/actions/change_storageclass_action.go index 91384c64e..b4052eb2d 100644 --- a/pkg/restore/actions/change_storageclass_action.go +++ b/pkg/restore/actions/change_storageclass_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "context" diff --git a/pkg/restore/actions/change_storageclass_action_test.go b/pkg/restore/actions/change_storageclass_action_test.go index 7f701a0d6..4d32a6bad 100644 --- a/pkg/restore/actions/change_storageclass_action_test.go +++ b/pkg/restore/actions/change_storageclass_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "context" diff --git a/pkg/restore/actions/clusterrolebinding_action.go b/pkg/restore/actions/clusterrolebinding_action.go index 851b13f09..072d2f4a1 100644 --- a/pkg/restore/actions/clusterrolebinding_action.go +++ b/pkg/restore/actions/clusterrolebinding_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "github.com/pkg/errors" diff --git a/pkg/restore/actions/clusterrolebinding_action_test.go b/pkg/restore/actions/clusterrolebinding_action_test.go index cea1c5787..bf9284e78 100644 --- a/pkg/restore/actions/clusterrolebinding_action_test.go +++ b/pkg/restore/actions/clusterrolebinding_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "sort" diff --git a/pkg/restore/actions/crd_v1_preserve_unknown_fields_action.go b/pkg/restore/actions/crd_v1_preserve_unknown_fields_action.go index 3f7b78241..673edd1b6 100644 --- a/pkg/restore/actions/crd_v1_preserve_unknown_fields_action.go +++ b/pkg/restore/actions/crd_v1_preserve_unknown_fields_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "encoding/json" diff --git a/pkg/restore/actions/crd_v1_preserve_unknown_fields_action_test.go b/pkg/restore/actions/crd_v1_preserve_unknown_fields_action_test.go index 63e2cea94..329cfb433 100644 --- a/pkg/restore/actions/crd_v1_preserve_unknown_fields_action_test.go +++ b/pkg/restore/actions/crd_v1_preserve_unknown_fields_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "encoding/json" diff --git a/pkg/restore/actions/csi/pvc_action.go b/pkg/restore/actions/csi/pvc_action.go new file mode 100644 index 000000000..1f75889f9 --- /dev/null +++ b/pkg/restore/actions/csi/pvc_action.go @@ -0,0 +1,616 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + "encoding/json" + "fmt" + + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + corev1api "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + utilrand "k8s.io/apimachinery/pkg/util/rand" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + "github.com/vmware-tanzu/velero/pkg/client" + "github.com/vmware-tanzu/velero/pkg/label" + plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + riav2 "github.com/vmware-tanzu/velero/pkg/plugin/velero/restoreitemaction/v2" + uploaderUtil "github.com/vmware-tanzu/velero/pkg/uploader/util" + "github.com/vmware-tanzu/velero/pkg/util" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" +) + +const ( + AnnBindCompleted = "pv.kubernetes.io/bind-completed" + AnnBoundByController = "pv.kubernetes.io/bound-by-controller" + AnnStorageProvisioner = "volume.kubernetes.io/storage-provisioner" + AnnBetaStorageProvisioner = "volume.beta.kubernetes.io/storage-provisioner" + AnnSelectedNode = "volume.kubernetes.io/selected-node" +) + +const ( + GenerateNameRandomLength = 5 +) + +// pvcRestoreItemAction is a restore item action plugin for Velero +type pvcRestoreItemAction struct { + log logrus.FieldLogger + crClient crclient.Client +} + +// AppliesTo returns information indicating that the +// PVCRestoreItemAction should be run while restoring PVCs. +func (p *pvcRestoreItemAction) AppliesTo() (velero.ResourceSelector, error) { + return velero.ResourceSelector{ + IncludedResources: []string{"persistentvolumeclaims"}, + //TODO: add label selector volumeSnapshotLabel + }, nil +} + +func removePVCAnnotations(pvc *corev1api.PersistentVolumeClaim, remove []string) { + if pvc.Annotations == nil { + pvc.Annotations = make(map[string]string) + return + } + for k := range pvc.Annotations { + if util.Contains(remove, k) { + delete(pvc.Annotations, k) + } + } +} + +func resetPVCSpec(pvc *corev1api.PersistentVolumeClaim, vsName string) { + // Restore operation for the PVC will use the VolumeSnapshot as the data source. + // So clear out the volume name, which is a ref to the PV + pvc.Spec.VolumeName = "" + dataSource := &corev1api.TypedLocalObjectReference{ + APIGroup: &snapshotv1api.SchemeGroupVersion.Group, + Kind: "VolumeSnapshot", + Name: vsName, + } + pvc.Spec.DataSource = dataSource + pvc.Spec.DataSourceRef = nil +} + +func setPVCStorageResourceRequest( + pvc *corev1api.PersistentVolumeClaim, + restoreSize resource.Quantity, + log logrus.FieldLogger, +) { + { + if pvc.Spec.Resources.Requests == nil { + pvc.Spec.Resources.Requests = corev1api.ResourceList{} + } + + storageReq, exists := pvc.Spec.Resources.Requests[corev1api.ResourceStorage] + if !exists || storageReq.Cmp(restoreSize) < 0 { + pvc.Spec.Resources.Requests[corev1api.ResourceStorage] = restoreSize + rs := pvc.Spec.Resources.Requests[corev1api.ResourceStorage] + log.Infof("Resetting storage requests for PVC %s/%s to %s", + pvc.Namespace, pvc.Name, rs.String()) + } + } +} + +// Execute modifies the PVC's spec to use the VolumeSnapshot object as the +// data source ensuring that the newly provisioned volume can be pre-populated +// with data from the VolumeSnapshot. +func (p *pvcRestoreItemAction) Execute( + input *velero.RestoreItemActionExecuteInput, +) (*velero.RestoreItemActionExecuteOutput, error) { + var pvc, pvcFromBackup corev1api.PersistentVolumeClaim + if err := runtime.DefaultUnstructuredConverter.FromUnstructured( + input.Item.UnstructuredContent(), &pvc); err != nil { + return nil, errors.WithStack(err) + } + if err := runtime.DefaultUnstructuredConverter.FromUnstructured( + input.ItemFromBackup.UnstructuredContent(), &pvcFromBackup); err != nil { + return nil, errors.WithStack(err) + } + + logger := p.log.WithFields(logrus.Fields{ + "Action": "PVCRestoreItemAction", + "PVC": pvc.Namespace + "/" + pvc.Name, + "Restore": input.Restore.Namespace + "/" + input.Restore.Name, + }) + logger.Info("Starting PVCRestoreItemAction for PVC") + + // If PVC already exists, returns early. + if p.isResourceExist(pvc, *input.Restore) { + logger.Warnf("PVC already exists. Skip restore this PVC.") + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: input.Item, + }, nil + } + + // remove the VolumeSnapshot name annotation as well + // clean the DataUploadNameLabel for snapshot data mover case. + removePVCAnnotations( + &pvc, + []string{ + AnnBindCompleted, + AnnBoundByController, + AnnStorageProvisioner, + AnnBetaStorageProvisioner, + AnnSelectedNode, + velerov1api.VolumeSnapshotLabel, + velerov1api.DataUploadNameAnnotation, + }, + ) + + // If cross-namespace restore is configured, change the namespace + // for PVC object to be restored + newNamespace, ok := input.Restore.Spec.NamespaceMapping[pvc.GetNamespace()] + if !ok { + // Use original namespace + newNamespace = pvc.Namespace + } + + operationID := "" + + if boolptr.IsSetToFalse(input.Restore.Spec.RestorePVs) { + logger.Info("Restore did not request for PVs to be restored from snapshot") + pvc.Spec.VolumeName = "" + pvc.Spec.DataSource = nil + pvc.Spec.DataSourceRef = nil + } else { + backup := new(velerov1api.Backup) + err := p.crClient.Get( + context.TODO(), + crclient.ObjectKey{ + Namespace: input.Restore.Namespace, + Name: input.Restore.Spec.BackupName, + }, + backup, + ) + + if err != nil { + logger.Error("Fail to get backup for restore.") + return nil, fmt.Errorf("fail to get backup for restore: %s", err.Error()) + } + + if boolptr.IsSetToTrue(backup.Spec.SnapshotMoveData) { + logger.Info("Start DataMover restore.") + + // If PVC doesn't have a DataUploadNameLabel, which should be created + // during backup, then CSI cannot handle the volume during to restore, + // so return early to let Velero tries to fall back to Velero native snapshot. + if _, ok := pvcFromBackup.Annotations[velerov1api.DataUploadNameAnnotation]; !ok { + logger.Warnf("PVC doesn't have a DataUpload for data mover. Return.") + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: input.Item, + }, nil + } + + operationID = label.GetValidName( + string(velerov1api.AsyncOperationIDPrefixDataDownload) + + string(input.Restore.UID) + "." + string(pvcFromBackup.UID)) + dataDownload, err := restoreFromDataUploadResult( + context.Background(), input.Restore, backup, &pvc, newNamespace, + operationID, p.crClient) + if err != nil { + logger.Errorf("Fail to restore from DataUploadResult: %s", err.Error()) + return nil, errors.WithStack(err) + } + logger.Infof("DataDownload %s/%s is created successfully.", + dataDownload.Namespace, dataDownload.Name) + } else { + volumeSnapshotName, ok := pvcFromBackup.Annotations[velerov1api.VolumeSnapshotLabel] + if !ok { + logger.Info("Skipping PVCRestoreItemAction for PVC,", + "PVC does not have a CSI VolumeSnapshot.") + // Make no change in the input PVC. + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: input.Item, + }, nil + } + if err := restoreFromVolumeSnapshot( + &pvc, newNamespace, p.crClient, volumeSnapshotName, logger, + ); err != nil { + logger.Errorf("Failed to restore PVC from VolumeSnapshot.") + return nil, errors.WithStack(err) + } + } + } + + pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&pvc) + if err != nil { + return nil, errors.WithStack(err) + } + logger.Info("Returning from PVCRestoreItemAction for PVC") + + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: &unstructured.Unstructured{Object: pvcMap}, + OperationID: operationID, + }, nil +} + +func (p *pvcRestoreItemAction) Name() string { + return "PVCRestoreItemAction" +} + +func (p *pvcRestoreItemAction) Progress( + operationID string, + restore *velerov1api.Restore, +) (velero.OperationProgress, error) { + progress := velero.OperationProgress{} + + if operationID == "" { + return progress, riav2.InvalidOperationIDError(operationID) + } + logger := p.log.WithFields(logrus.Fields{ + "Action": "PVCRestoreItemAction", + "OperationID": operationID, + "Namespace": restore.Namespace, + }) + + dataDownload, err := getDataDownload( + context.Background(), + restore.Namespace, + operationID, + p.crClient, + ) + if err != nil { + logger.Errorf("fail to get DataDownload: %s", err.Error()) + return progress, err + } + if dataDownload.Status.Phase == velerov2alpha1.DataDownloadPhaseNew || + dataDownload.Status.Phase == "" { + logger.Debugf("DataDownload is still not processed yet. Skip progress update.") + return progress, nil + } + + progress.Description = string(dataDownload.Status.Phase) + progress.OperationUnits = "Bytes" + progress.NCompleted = dataDownload.Status.Progress.BytesDone + progress.NTotal = dataDownload.Status.Progress.TotalBytes + + if dataDownload.Status.StartTimestamp != nil { + progress.Started = dataDownload.Status.StartTimestamp.Time + } + + if dataDownload.Status.CompletionTimestamp != nil { + progress.Updated = dataDownload.Status.CompletionTimestamp.Time + } + + if dataDownload.Status.Phase == velerov2alpha1.DataDownloadPhaseCompleted { + progress.Completed = true + } else if dataDownload.Status.Phase == velerov2alpha1.DataDownloadPhaseCanceled { + progress.Completed = true + progress.Err = "DataDownload is canceled" + } else if dataDownload.Status.Phase == velerov2alpha1.DataDownloadPhaseFailed { + progress.Completed = true + progress.Err = dataDownload.Status.Message + } + + return progress, nil +} + +func (p *pvcRestoreItemAction) Cancel( + operationID string, restore *velerov1api.Restore) error { + if operationID == "" { + return riav2.InvalidOperationIDError(operationID) + } + logger := p.log.WithFields(logrus.Fields{ + "Action": "PVCRestoreItemAction", + "OperationID": operationID, + "Namespace": restore.Namespace, + }) + + dataDownload, err := getDataDownload( + context.Background(), + restore.Namespace, + operationID, + p.crClient, + ) + if err != nil { + logger.Errorf("fail to get DataDownload: %s", err.Error()) + return err + } + + err = cancelDataDownload(context.Background(), p.crClient, dataDownload) + if err != nil { + logger.Errorf("fail to cancel DataDownload %s: %s", dataDownload.Name, err.Error()) + } + return err +} + +func (p *pvcRestoreItemAction) AreAdditionalItemsReady( + additionalItems []velero.ResourceIdentifier, + restore *velerov1api.Restore, +) (bool, error) { + return true, nil +} + +func getDataUploadResult( + ctx context.Context, + restore *velerov1api.Restore, + pvc *corev1api.PersistentVolumeClaim, + crClient crclient.Client, +) (*velerov2alpha1.DataUploadResult, error) { + selectorStr := fmt.Sprintf("%s=%s,%s=%s,%s=%s", + velerov1api.PVCNamespaceNameLabel, + label.GetValidName(pvc.Namespace+"."+pvc.Name), + velerov1api.RestoreUIDLabel, + label.GetValidName(string(restore.UID)), + velerov1api.ResourceUsageLabel, + label.GetValidName(string(velerov1api.VeleroResourceUsageDataUploadResult)), + ) + selector, _ := labels.Parse(selectorStr) + + cmList := new(corev1api.ConfigMapList) + if err := crClient.List( + ctx, + cmList, + &crclient.ListOptions{ + LabelSelector: selector, + Namespace: restore.Namespace, + }); err != nil { + return nil, errors.Wrapf(err, + "error to get DataUpload result cm with labels %s", selectorStr) + } + + if len(cmList.Items) == 0 { + return nil, errors.Errorf( + "no DataUpload result cm found with labels %s", selectorStr) + } + + if len(cmList.Items) > 1 { + return nil, errors.Errorf( + "multiple DataUpload result cms found with labels %s", selectorStr) + } + + jsonBytes, exist := cmList.Items[0].Data[string(restore.UID)] + if !exist { + return nil, errors.Errorf( + "no DataUpload result found with restore key %s, restore %s", + string(restore.UID), restore.Name) + } + + result := velerov2alpha1.DataUploadResult{} + if err := json.Unmarshal([]byte(jsonBytes), &result); err != nil { + return nil, errors.Errorf( + "error to unmarshal DataUploadResult, restore UID %s, restore name %s", + string(restore.UID), restore.Name) + } + + return &result, nil +} + +func getDataDownload( + ctx context.Context, + namespace string, + operationID string, + crClient crclient.Client, +) (*velerov2alpha1.DataDownload, error) { + dataDownloadList := new(velerov2alpha1.DataDownloadList) + err := crClient.List(ctx, dataDownloadList, &crclient.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{ + velerov1api.AsyncOperationIDLabel: operationID, + }), + Namespace: namespace, + }) + if err != nil { + return nil, errors.Wrap(err, "fail to list DataDownload") + } + + if len(dataDownloadList.Items) == 0 { + return nil, errors.Errorf("didn't find DataDownload") + } + + if len(dataDownloadList.Items) > 1 { + return nil, errors.Errorf("find multiple DataDownloads") + } + + return &dataDownloadList.Items[0], nil +} + +func cancelDataDownload(ctx context.Context, crClient crclient.Client, + dataDownload *velerov2alpha1.DataDownload) error { + updatedDataDownload := dataDownload.DeepCopy() + updatedDataDownload.Spec.Cancel = true + + return crClient.Patch(ctx, updatedDataDownload, crclient.MergeFrom(dataDownload)) +} + +func newDataDownload( + restore *velerov1api.Restore, + backup *velerov1api.Backup, + dataUploadResult *velerov2alpha1.DataUploadResult, + pvc *corev1api.PersistentVolumeClaim, + newNamespace, operationID string, +) *velerov2alpha1.DataDownload { + dataDownload := &velerov2alpha1.DataDownload{ + TypeMeta: metav1.TypeMeta{ + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + Kind: "DataDownload", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: restore.Namespace, + GenerateName: restore.Name + "-", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: velerov1api.SchemeGroupVersion.String(), + Kind: "Restore", + Name: restore.Name, + UID: restore.UID, + Controller: boolptr.True(), + }, + }, + Labels: map[string]string{ + velerov1api.RestoreNameLabel: label.GetValidName(restore.Name), + velerov1api.RestoreUIDLabel: string(restore.UID), + velerov1api.AsyncOperationIDLabel: operationID, + }, + }, + Spec: velerov2alpha1.DataDownloadSpec{ + TargetVolume: velerov2alpha1.TargetVolumeSpec{ + PVC: pvc.Name, + Namespace: newNamespace, + }, + BackupStorageLocation: dataUploadResult.BackupStorageLocation, + DataMover: dataUploadResult.DataMover, + SnapshotID: dataUploadResult.SnapshotID, + SourceNamespace: dataUploadResult.SourceNamespace, + OperationTimeout: backup.Spec.CSISnapshotTimeout, + }, + } + if restore.Spec.UploaderConfig != nil { + dataDownload.Spec.DataMoverConfig = make(map[string]string) + if boolptr.IsSetToTrue(restore.Spec.UploaderConfig.WriteSparseFiles) { + dataDownload.Spec.DataMoverConfig[uploaderUtil.WriteSparseFiles] = "true" + } else { + dataDownload.Spec.DataMoverConfig[uploaderUtil.WriteSparseFiles] = "false" + } + } + return dataDownload +} + +func restoreFromVolumeSnapshot( + pvc *corev1api.PersistentVolumeClaim, + newNamespace string, + crClient crclient.Client, + volumeSnapshotName string, + logger logrus.FieldLogger, +) error { + vs := new(snapshotv1api.VolumeSnapshot) + if err := crClient.Get(context.TODO(), + crclient.ObjectKey{ + Namespace: newNamespace, + Name: volumeSnapshotName, + }, + vs, + ); err != nil { + return errors.Wrapf(err, + fmt.Sprintf("Failed to get Volumesnapshot %s/%s to restore PVC %s/%s", + newNamespace, volumeSnapshotName, newNamespace, pvc.Name), + ) + } + + if _, exists := vs.Annotations[velerov1api.VolumeSnapshotRestoreSize]; exists { + restoreSize, err := resource.ParseQuantity( + vs.Annotations[velerov1api.VolumeSnapshotRestoreSize]) + if err != nil { + return errors.Wrapf(err, fmt.Sprintf( + "Failed to parse %s from annotation on Volumesnapshot %s/%s into restore size", + vs.Annotations[velerov1api.VolumeSnapshotRestoreSize], vs.Namespace, vs.Name)) + } + // It is possible that the volume provider allocated a larger + // capacity volume than what was requested in the backed up PVC. + // In this scenario the volumesnapshot of the PVC will end being + // larger than its requested storage size. Such a PVC, on restore + // as-is, will be stuck attempting to use a VolumeSnapshot as a + // data source for a PVC that is not large enough. + // To counter that, here we set the storage request on the PVC + // to the larger of the PVC's storage request and the size of the + // VolumeSnapshot + setPVCStorageResourceRequest(pvc, restoreSize, logger) + } + + resetPVCSpec(pvc, volumeSnapshotName) + + return nil +} + +func restoreFromDataUploadResult( + ctx context.Context, + restore *velerov1api.Restore, + backup *velerov1api.Backup, + pvc *corev1api.PersistentVolumeClaim, + newNamespace, operationID string, + crClient crclient.Client, +) (*velerov2alpha1.DataDownload, error) { + dataUploadResult, err := getDataUploadResult(ctx, restore, pvc, crClient) + if err != nil { + return nil, errors.Wrapf(err, "fail get DataUploadResult for restore: %s", + restore.Name) + } + pvc.Spec.VolumeName = "" + if pvc.Spec.Selector == nil { + pvc.Spec.Selector = &metav1.LabelSelector{} + } + if pvc.Spec.Selector.MatchLabels == nil { + pvc.Spec.Selector.MatchLabels = make(map[string]string) + } + pvc.Spec.Selector.MatchLabels[velerov1api.DynamicPVRestoreLabel] = label. + GetValidName(fmt.Sprintf("%s.%s.%s", newNamespace, + pvc.Name, utilrand.String(GenerateNameRandomLength))) + + dataDownload := newDataDownload( + restore, + backup, + dataUploadResult, + pvc, + newNamespace, + operationID, + ) + err = crClient.Create(ctx, dataDownload) + if err != nil { + return nil, errors.Wrapf(err, "fail to create DataDownload") + } + + return dataDownload, nil +} + +func (p *pvcRestoreItemAction) isResourceExist( + pvc corev1api.PersistentVolumeClaim, + restore velerov1api.Restore, +) bool { + // get target namespace to restore into, if different from source namespace + targetNamespace := pvc.Namespace + if target, ok := restore.Spec.NamespaceMapping[pvc.Namespace]; ok { + targetNamespace = target + } + + tmpPVC := new(corev1api.PersistentVolumeClaim) + if err := p.crClient.Get( + context.Background(), + crclient.ObjectKey{ + Name: pvc.Name, + Namespace: targetNamespace, + }, + tmpPVC, + ); err == nil { + return true + } + return false +} + +func NewPvcRestoreItemAction(f client.Factory) plugincommon.HandlerInitializer { + return func(logger logrus.FieldLogger) (interface{}, error) { + crClient, err := f.KubebuilderClient() + if err != nil { + return nil, err + } + + return &pvcRestoreItemAction{ + log: logger, + crClient: crClient, + }, nil + } +} diff --git a/pkg/restore/actions/csi/pvc_action_test.go b/pkg/restore/actions/csi/pvc_action_test.go new file mode 100644 index 000000000..4ba8ca83e --- /dev/null +++ b/pkg/restore/actions/csi/pvc_action_test.go @@ -0,0 +1,713 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1api "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/velero/pkg/apis/velero/shared" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov2alpha1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + "github.com/vmware-tanzu/velero/pkg/builder" + "github.com/vmware-tanzu/velero/pkg/label" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + velerotest "github.com/vmware-tanzu/velero/pkg/test" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" +) + +func TestRemovePVCAnnotations(t *testing.T) { + testCases := []struct { + name string + pvc corev1api.PersistentVolumeClaim + removeAnnotations []string + expectedAnnotations map[string]string + }{ + { + name: "should create empty annotation map", + pvc: corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: nil, + }, + }, + removeAnnotations: []string{"foo"}, + expectedAnnotations: map[string]string{}, + }, + { + name: "should preserve all existing annotations", + pvc: corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "ann1": "ann1-val", + "ann2": "ann2-val", + "ann3": "ann3-val", + "ann4": "ann4-val", + }, + }, + }, + removeAnnotations: []string{}, + expectedAnnotations: map[string]string{ + "ann1": "ann1-val", + "ann2": "ann2-val", + "ann3": "ann3-val", + "ann4": "ann4-val", + }, + }, + { + name: "should remove all existing annotations", + pvc: corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "ann1": "ann1-val", + "ann2": "ann2-val", + "ann3": "ann3-val", + "ann4": "ann4-val", + }, + }, + }, + removeAnnotations: []string{"ann1", "ann2", "ann3", "ann4"}, + expectedAnnotations: map[string]string{}, + }, + { + name: "should preserve some existing annotations", + pvc: corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "ann1": "ann1-val", + "ann2": "ann2-val", + "ann3": "ann3-val", + "ann4": "ann4-val", + "ann5": "ann5-val", + "ann6": "ann6-val", + "ann7": "ann7-val", + "ann8": "ann8-val", + }, + }, + }, + removeAnnotations: []string{"ann1", "ann2", "ann3", "ann4"}, + expectedAnnotations: map[string]string{ + "ann5": "ann5-val", + "ann6": "ann6-val", + "ann7": "ann7-val", + "ann8": "ann8-val", + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + removePVCAnnotations(&tc.pvc, tc.removeAnnotations) + assert.Equal(t, tc.expectedAnnotations, tc.pvc.Annotations) + }) + } +} + +func TestResetPVCSpec(t *testing.T) { + fileMode := corev1api.PersistentVolumeFilesystem + blockMode := corev1api.PersistentVolumeBlock + + testCases := []struct { + name string + pvc corev1api.PersistentVolumeClaim + vsName string + }{ + { + name: "should reset expected fields in pvc using file mode volumes", + pvc: corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pvc", + Namespace: "test-ns", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadOnlyMany, corev1api.ReadWriteMany, corev1api.ReadWriteOnce}, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "foo": "bar", + "baz": "qux", + }, + }, + Resources: corev1api.VolumeResourceRequirements{ + Requests: corev1api.ResourceList{ + corev1api.ResourceCPU: resource.Quantity{ + Format: resource.DecimalExponent, + }, + }, + }, + VolumeName: "should-be-removed", + VolumeMode: &fileMode, + }, + }, + vsName: "test-vs", + }, + { + name: "should reset expected fields in pvc using block mode volumes", + pvc: corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pvc", + Namespace: "test-ns", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadOnlyMany, corev1api.ReadWriteMany, corev1api.ReadWriteOnce}, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "foo": "bar", + "baz": "qux", + }, + }, + Resources: corev1api.VolumeResourceRequirements{ + Requests: corev1api.ResourceList{ + corev1api.ResourceCPU: resource.Quantity{ + Format: resource.DecimalExponent, + }, + }, + }, + VolumeName: "should-be-removed", + VolumeMode: &blockMode, + }, + }, + vsName: "test-vs", + }, + { + name: "should overwrite existing DataSource per reset parameters", + pvc: corev1api.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pvc", + Namespace: "test-ns", + }, + Spec: corev1api.PersistentVolumeClaimSpec{ + AccessModes: []corev1api.PersistentVolumeAccessMode{corev1api.ReadOnlyMany, corev1api.ReadWriteMany, corev1api.ReadWriteOnce}, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "foo": "bar", + "baz": "qux", + }, + }, + Resources: corev1api.VolumeResourceRequirements{ + Requests: corev1api.ResourceList{ + corev1api.ResourceCPU: resource.Quantity{ + Format: resource.DecimalExponent, + }, + }, + }, + VolumeName: "should-be-removed", + VolumeMode: &fileMode, + DataSource: &corev1api.TypedLocalObjectReference{ + Kind: "something-that-does-not-exist", + Name: "not-found", + }, + DataSourceRef: &corev1api.TypedObjectReference{ + Kind: "something-that-does-not-exist", + Name: "not-found", + }, + }, + }, + vsName: "test-vs", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + before := tc.pvc.DeepCopy() + resetPVCSpec(&tc.pvc, tc.vsName) + + assert.Equalf(t, tc.pvc.Name, before.Name, "unexpected change to Object.Name, Want: %s; Got %s", before.Name, tc.pvc.Name) + assert.Equalf(t, tc.pvc.Namespace, before.Namespace, "unexpected change to Object.Namespace, Want: %s; Got %s", before.Namespace, tc.pvc.Namespace) + assert.Equalf(t, tc.pvc.Spec.AccessModes, before.Spec.AccessModes, "unexpected Spec.AccessModes, Want: %v; Got: %v", before.Spec.AccessModes, tc.pvc.Spec.AccessModes) + assert.Equalf(t, tc.pvc.Spec.Selector, before.Spec.Selector, "unexpected change to Spec.Selector, Want: %s; Got: %s", before.Spec.Selector.String(), tc.pvc.Spec.Selector.String()) + assert.Equalf(t, tc.pvc.Spec.Resources, before.Spec.Resources, "unexpected change to Spec.Resources, Want: %s; Got: %s", before.Spec.Resources.String(), tc.pvc.Spec.Resources.String()) + assert.Emptyf(t, tc.pvc.Spec.VolumeName, "expected change to Spec.VolumeName missing, Want: \"\"; Got: %s", tc.pvc.Spec.VolumeName) + assert.Equalf(t, *tc.pvc.Spec.VolumeMode, *before.Spec.VolumeMode, "expected change to Spec.VolumeName missing, Want: \"\"; Got: %s", tc.pvc.Spec.VolumeName) + assert.NotNil(t, tc.pvc.Spec.DataSource, "expected change to Spec.DataSource missing") + assert.Equalf(t, tc.pvc.Spec.DataSource.Kind, "VolumeSnapshot", "expected change to Spec.DataSource.Kind missing, Want: VolumeSnapshot, Got: %s", tc.pvc.Spec.DataSource.Kind) + assert.Equalf(t, tc.pvc.Spec.DataSource.Name, tc.vsName, "expected change to Spec.DataSource.Name missing, Want: %s, Got: %s", tc.vsName, tc.pvc.Spec.DataSource.Name) + }) + } +} + +func TestResetPVCResourceRequest(t *testing.T) { + var storageReq50Mi, storageReq1Gi, cpuQty resource.Quantity + + storageReq50Mi, err := resource.ParseQuantity("50Mi") + assert.NoError(t, err) + storageReq1Gi, err = resource.ParseQuantity("1Gi") + assert.NoError(t, err) + cpuQty, err = resource.ParseQuantity("100m") + assert.NoError(t, err) + + testCases := []struct { + name string + pvc corev1api.PersistentVolumeClaim + restoreSize resource.Quantity + expectedStorageRequestQty string + }{ + { + name: "should set storage resource request from volumesnapshot, pvc has nil resource requests", + pvc: corev1api.PersistentVolumeClaim{ + Spec: corev1api.PersistentVolumeClaimSpec{ + Resources: corev1api.VolumeResourceRequirements{ + Requests: nil, + }, + }, + }, + restoreSize: storageReq50Mi, + expectedStorageRequestQty: "50Mi", + }, + { + name: "should set storage resource request from volumesnapshot, pvc has empty resource requests", + pvc: corev1api.PersistentVolumeClaim{ + Spec: corev1api.PersistentVolumeClaimSpec{ + Resources: corev1api.VolumeResourceRequirements{ + Requests: corev1api.ResourceList{}, + }, + }, + }, + restoreSize: storageReq50Mi, + expectedStorageRequestQty: "50Mi", + }, + { + name: "should merge resource requests from volumesnapshot into pvc with no storage resource requests", + pvc: corev1api.PersistentVolumeClaim{ + Spec: corev1api.PersistentVolumeClaimSpec{ + Resources: corev1api.VolumeResourceRequirements{ + Requests: corev1api.ResourceList{ + corev1api.ResourceCPU: cpuQty, + }, + }, + }, + }, + restoreSize: storageReq50Mi, + expectedStorageRequestQty: "50Mi", + }, + { + name: "should set storage resource request from volumesnapshot, pvc requests less storage", + pvc: corev1api.PersistentVolumeClaim{ + Spec: corev1api.PersistentVolumeClaimSpec{ + Resources: corev1api.VolumeResourceRequirements{ + Requests: corev1api.ResourceList{ + corev1api.ResourceStorage: storageReq50Mi, + }, + }, + }, + }, + restoreSize: storageReq1Gi, + expectedStorageRequestQty: "1Gi", + }, + { + name: "should not set storage resource request from volumesnapshot, pvc requests more storage", + pvc: corev1api.PersistentVolumeClaim{ + Spec: corev1api.PersistentVolumeClaimSpec{ + Resources: corev1api.VolumeResourceRequirements{ + Requests: corev1api.ResourceList{ + corev1api.ResourceStorage: storageReq1Gi, + }, + }, + }, + }, + restoreSize: storageReq50Mi, + expectedStorageRequestQty: "1Gi", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + log := logrus.New().WithField("unit-test", tc.name) + setPVCStorageResourceRequest(&tc.pvc, tc.restoreSize, log) + expected, err := resource.ParseQuantity(tc.expectedStorageRequestQty) + assert.NoError(t, err) + assert.Equal(t, expected, tc.pvc.Spec.Resources.Requests[corev1api.ResourceStorage]) + }) + } +} + +func TestProgress(t *testing.T) { + currentTime := time.Now() + tests := []struct { + name string + restore *velerov1api.Restore + dataDownload *velerov2alpha1.DataDownload + operationID string + expectedErr string + expectedProgress velero.OperationProgress + }{ + { + name: "DataDownload cannot be found", + restore: builder.ForRestore("velero", "test").Result(), + operationID: "testing", + expectedErr: "didn't find DataDownload", + }, + { + name: "DataDownload is not in the expected namespace", + restore: builder.ForRestore("velero", "test").Result(), + dataDownload: &velerov2alpha1.DataDownload{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataUpload", + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "invalid-namespace", + Name: "testing", + Labels: map[string]string{ + velerov1api.AsyncOperationIDLabel: "testing", + }, + }, + }, + operationID: "testing", + expectedErr: "didn't find DataDownload", + }, + { + name: "DataUpload is found", + restore: builder.ForRestore("velero", "test").Result(), + dataDownload: &velerov2alpha1.DataDownload{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataUpload", + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "velero", + Name: "testing", + Labels: map[string]string{ + velerov1api.AsyncOperationIDLabel: "testing", + }, + }, + Status: velerov2alpha1.DataDownloadStatus{ + Phase: velerov2alpha1.DataDownloadPhaseFailed, + Progress: shared.DataMoveOperationProgress{ + BytesDone: 1000, + TotalBytes: 1000, + }, + StartTimestamp: &metav1.Time{Time: currentTime}, + CompletionTimestamp: &metav1.Time{Time: currentTime}, + Message: "Testing error", + }, + }, + operationID: "testing", + expectedProgress: velero.OperationProgress{ + Completed: true, + Err: "Testing error", + NCompleted: 1000, + NTotal: 1000, + OperationUnits: "Bytes", + Description: "Failed", + Started: currentTime, + Updated: currentTime, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(*testing.T) { + pvcRIA := pvcRestoreItemAction{ + log: logrus.New(), + crClient: velerotest.NewFakeControllerRuntimeClient(t), + } + if tc.dataDownload != nil { + err := pvcRIA.crClient.Create(context.Background(), tc.dataDownload) + require.NoError(t, err) + } + + progress, err := pvcRIA.Progress(tc.operationID, tc.restore) + if tc.expectedErr != "" { + require.Equal(t, tc.expectedErr, err.Error()) + return + } + + require.NoError(t, err) + require.True(t, cmp.Equal(tc.expectedProgress, progress, cmpopts.IgnoreFields(velero.OperationProgress{}, "Started", "Updated"))) + }) + } +} + +func TestCancel(t *testing.T) { + tests := []struct { + name string + restore *velerov1api.Restore + dataDownload *velerov2alpha1.DataDownload + operationID string + expectedErr string + expectedDataDownload velerov2alpha1.DataDownload + }{ + { + name: "Cancel DataUpload", + restore: builder.ForRestore("velero", "test").Result(), + dataDownload: &velerov2alpha1.DataDownload{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataDownload", + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "velero", + Name: "testing", + Labels: map[string]string{ + velerov1api.AsyncOperationIDLabel: "testing", + }, + }, + }, + operationID: "testing", + expectedErr: "", + expectedDataDownload: velerov2alpha1.DataDownload{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataDownload", + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "velero", + Name: "testing", + Labels: map[string]string{ + velerov1api.AsyncOperationIDLabel: "testing", + }, + }, + Spec: velerov2alpha1.DataDownloadSpec{ + Cancel: true, + }, + }, + }, + { + name: "Cannot find DataUpload", + restore: builder.ForRestore("velero", "test").Result(), + dataDownload: nil, + operationID: "testing", + expectedErr: "didn't find DataDownload", + expectedDataDownload: velerov2alpha1.DataDownload{ + TypeMeta: metav1.TypeMeta{ + Kind: "DataDownload", + APIVersion: velerov2alpha1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "velero", + Name: "testing", + Labels: map[string]string{ + velerov1api.AsyncOperationIDLabel: "testing", + }, + }, + Spec: velerov2alpha1.DataDownloadSpec{ + Cancel: true, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(*testing.T) { + pvcRIA := pvcRestoreItemAction{ + log: logrus.New(), + crClient: velerotest.NewFakeControllerRuntimeClient(t), + } + if tc.dataDownload != nil { + err := pvcRIA.crClient.Create(context.Background(), tc.dataDownload) + require.NoError(t, err) + } + + err := pvcRIA.Cancel(tc.operationID, tc.restore) + if tc.expectedErr != "" { + require.Equal(t, tc.expectedErr, err.Error()) + return + } + require.NoError(t, err) + + resultDataDownload := new(velerov2alpha1.DataDownload) + err = pvcRIA.crClient.Get(context.Background(), crclient.ObjectKey{Namespace: tc.dataDownload.Namespace, Name: tc.dataDownload.Name}, resultDataDownload) + require.NoError(t, err) + + require.True(t, cmp.Equal(tc.expectedDataDownload, *resultDataDownload, cmpopts.IgnoreFields(velerov2alpha1.DataDownload{}, "ResourceVersion", "Name"))) + }) + } +} + +func TestExecute(t *testing.T) { + tests := []struct { + name string + backup *velerov1api.Backup + restore *velerov1api.Restore + pvc *corev1api.PersistentVolumeClaim + vs *snapshotv1api.VolumeSnapshot + dataUploadResult *corev1api.ConfigMap + expectedErr string + expectedDataDownload *velerov2alpha1.DataDownload + expectedPVC *corev1api.PersistentVolumeClaim + preCreatePVC bool + }{ + { + name: "Don't restore PV", + restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").RestorePVs(false).Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").Result(), + expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").VolumeName("").Result(), + }, + { + name: "restore's backup cannot be found", + restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").Result(), + expectedErr: "fail to get backup for restore: backups.velero.io \"testBackup\" not found", + }, + { + name: "VolumeSnapshot cannot be found", + backup: builder.ForBackup("velero", "testBackup").Result(), + restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "testVS")).Result(), + expectedErr: "Failed to get Volumesnapshot velero/testVS to restore PVC velero/testPVC: volumesnapshots.snapshot.storage.k8s.io \"testVS\" not found", + }, + { + name: "Restore from VolumeSnapshot", + backup: builder.ForBackup("velero", "testBackup").Result(), + restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotLabel, "testVS")). + RequestResource(map[corev1api.ResourceName]resource.Quantity{corev1api.ResourceStorage: resource.MustParse("10Gi")}). + DataSource(&corev1api.TypedLocalObjectReference{APIGroup: &snapshotv1api.SchemeGroupVersion.Group, Kind: "VolumeSnapshot", Name: "testVS"}). + DataSourceRef(&corev1api.TypedObjectReference{APIGroup: &snapshotv1api.SchemeGroupVersion.Group, Kind: "VolumeSnapshot", Name: "testVS"}). + Result(), + vs: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi")).Result(), + expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").Result(), + }, + { + name: "Restore from VolumeSnapshot without volume-snapshot-name annotation", + backup: builder.ForBackup("velero", "testBackup").Result(), + restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(AnnSelectedNode, "node1")).Result(), + vs: builder.ForVolumeSnapshot("velero", "testVS").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi")).Result(), + expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(AnnSelectedNode, "node1")).Result(), + }, + { + name: "DataUploadResult cannot be found", + backup: builder.ForBackup("velero", "testBackup").SnapshotMoveData(true).Result(), + restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(), + expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").Result(), + expectedErr: "fail get DataUploadResult for restore: testRestore: no DataUpload result cm found with labels velero.io/pvc-namespace-name=velero.testPVC,velero.io/restore-uid=,velero.io/resource-usage=DataUpload", + }, + { + name: "Restore from DataUploadResult", + backup: builder.ForBackup("velero", "testBackup").SnapshotMoveData(true).Result(), + restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").ObjectMeta(builder.WithUID("uid")).Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(), + dataUploadResult: builder.ForConfigMap("velero", "testCM").Data("uid", "{}").ObjectMeta(builder.WithLabels(velerov1api.RestoreUIDLabel, "uid", velerov1api.PVCNamespaceNameLabel, "velero.testPVC", velerov1api.ResourceUsageLabel, label.GetValidName(string(velerov1api.VeleroResourceUsageDataUploadResult)))).Result(), + expectedPVC: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations("velero.io/csi-volumesnapshot-restore-size", "10Gi")).Result(), + expectedDataDownload: builder.ForDataDownload("velero", "name").TargetVolume(velerov2alpha1.TargetVolumeSpec{PVC: "testPVC", Namespace: "velero"}). + ObjectMeta(builder.WithOwnerReference([]metav1.OwnerReference{{APIVersion: velerov1api.SchemeGroupVersion.String(), Kind: "Restore", Name: "testRestore", UID: "uid", Controller: boolptr.True()}}), + builder.WithLabelsMap(map[string]string{velerov1api.AsyncOperationIDLabel: "dd-uid.", velerov1api.RestoreNameLabel: "testRestore", velerov1api.RestoreUIDLabel: "uid"}), + builder.WithGenerateName("testRestore-")).Result(), + }, + { + name: "Restore from DataUploadResult with long source PVC namespace and name", + backup: builder.ForBackup("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1", "testBackup").SnapshotMoveData(true).Result(), + restore: builder.ForRestore("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1", "testRestore").Backup("testBackup").ObjectMeta(builder.WithUID("uid")).Result(), + pvc: builder.ForPersistentVolumeClaim("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1", "kibishii-data-kibishii-deployment-0").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(), + dataUploadResult: builder.ForConfigMap("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1", "testCM").Data("uid", "{}").ObjectMeta(builder.WithLabels(velerov1api.RestoreUIDLabel, "uid", velerov1api.PVCNamespaceNameLabel, "migre209d0da-49c7-45ba-8d5a-3e59fd591ec1.kibishii-data-ki152333", velerov1api.ResourceUsageLabel, label.GetValidName(string(velerov1api.VeleroResourceUsageDataUploadResult)))).Result(), + expectedPVC: builder.ForPersistentVolumeClaim("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1", "kibishii-data-kibishii-deployment-0").ObjectMeta(builder.WithAnnotations("velero.io/csi-volumesnapshot-restore-size", "10Gi")).Result(), + }, + { + name: "PVC had no DataUploadNameLabel annotation", + backup: builder.ForBackup("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1", "testBackup").SnapshotMoveData(true).Result(), + restore: builder.ForRestore("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1", "testRestore").Backup("testBackup").ObjectMeta(builder.WithUID("uid")).Result(), + pvc: builder.ForPersistentVolumeClaim("migre209d0da-49c7-45ba-8d5a-3e59fd591ec1", "kibishii-data-kibishii-deployment-0").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi")).Result(), + }, + { + name: "Restore a PVC that already exists.", + backup: builder.ForBackup("velero", "testBackup").SnapshotMoveData(true).Result(), + restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").ObjectMeta(builder.WithUID("uid")).Result(), + pvc: builder.ForPersistentVolumeClaim("velero", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(), + preCreatePVC: true, + }, + { + name: "Restore a PVC that already exists in the mapping namespace", + backup: builder.ForBackup("velero", "testBackup").SnapshotMoveData(true).Result(), + restore: builder.ForRestore("velero", "testRestore").Backup("testBackup").NamespaceMappings("velero", "restore").ObjectMeta(builder.WithUID("uid")).Result(), + pvc: builder.ForPersistentVolumeClaim("restore", "testPVC").ObjectMeta(builder.WithAnnotations(velerov1api.VolumeSnapshotRestoreSize, "10Gi", velerov1api.DataUploadNameAnnotation, "velero/")).Result(), + preCreatePVC: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(*testing.T) { + object := make([]runtime.Object, 0) + if tc.backup != nil { + object = append(object, tc.backup) + } + + if tc.vs != nil { + object = append(object, tc.vs) + } + + input := new(velero.RestoreItemActionExecuteInput) + + if tc.pvc != nil { + pvcMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.pvc) + require.NoError(t, err) + + input.Item = &unstructured.Unstructured{Object: pvcMap} + input.ItemFromBackup = &unstructured.Unstructured{Object: pvcMap} + input.Restore = tc.restore + } + if tc.preCreatePVC { + object = append(object, tc.pvc) + } + + if tc.dataUploadResult != nil { + object = append(object, tc.dataUploadResult) + } + + pvcRIA := pvcRestoreItemAction{ + log: logrus.New(), + crClient: velerotest.NewFakeControllerRuntimeClient(t, object...), + } + + output, err := pvcRIA.Execute(input) + if tc.expectedErr != "" { + require.Equal(t, tc.expectedErr, err.Error()) + return + } + require.NoError(t, err) + + if tc.expectedPVC != nil { + pvc := new(corev1api.PersistentVolumeClaim) + err := runtime.DefaultUnstructuredConverter.FromUnstructured(output.UpdatedItem.UnstructuredContent(), pvc) + require.NoError(t, err) + require.Equal(t, tc.expectedPVC.GetObjectMeta(), pvc.GetObjectMeta()) + if pvc.Spec.Selector != nil && pvc.Spec.Selector.MatchLabels != nil { + // This is used for long name and namespace case. + if len(tc.pvc.Namespace+"."+tc.pvc.Name) >= validation.DNS1035LabelMaxLength { + require.Contains(t, pvc.Spec.Selector.MatchLabels[velerov1api.DynamicPVRestoreLabel], label.GetValidName(tc.pvc.Namespace + "." + tc.pvc.Name)[:56]) + } else { + require.Contains(t, pvc.Spec.Selector.MatchLabels[velerov1api.DynamicPVRestoreLabel], tc.pvc.Namespace+"."+tc.pvc.Name) + } + } + } + if tc.expectedDataDownload != nil { + dataDownloadList := new(velerov2alpha1.DataDownloadList) + err := pvcRIA.crClient.List(context.Background(), dataDownloadList, &crclient.ListOptions{ + LabelSelector: labels.SelectorFromSet(tc.expectedDataDownload.Labels), + }) + require.NoError(t, err) + require.True(t, cmp.Equal(tc.expectedDataDownload, &dataDownloadList.Items[0], cmpopts.IgnoreFields(velerov2alpha1.DataDownload{}, "ResourceVersion", "Name"))) + } + }) + } +} diff --git a/pkg/restore/actions/csi/volumesnapshot_action.go b/pkg/restore/actions/csi/volumesnapshot_action.go new file mode 100644 index 000000000..6f8326c22 --- /dev/null +++ b/pkg/restore/actions/csi/volumesnapshot_action.go @@ -0,0 +1,217 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "context" + + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + core_v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/client" + "github.com/vmware-tanzu/velero/pkg/label" + plugincommon "github.com/vmware-tanzu/velero/pkg/plugin/framework/common" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/csi" +) + +// volumeSnapshotRestoreItemAction is a Velero restore item +// action plugin for VolumeSnapshots +type volumeSnapshotRestoreItemAction struct { + log logrus.FieldLogger + crClient crclient.Client +} + +// AppliesTo returns information indicating that +// VolumeSnapshotRestoreItemAction should be invoked while +// restoring volumesnapshots.snapshot.storage.k8s.io resources. +func (p *volumeSnapshotRestoreItemAction) AppliesTo() ( + velero.ResourceSelector, error) { + return velero.ResourceSelector{ + IncludedResources: []string{"volumesnapshots.snapshot.storage.k8s.io"}, + }, nil +} + +func resetVolumeSnapshotSpecForRestore( + vs *snapshotv1api.VolumeSnapshot, vscName *string) { + // Spec of the backed-up object used the PVC as the source + // of the volumeSnapshot. Restore operation will however, + // restore the VolumeSnapshot from the VolumeSnapshotContent + vs.Spec.Source.PersistentVolumeClaimName = nil + vs.Spec.Source.VolumeSnapshotContentName = vscName +} + +func resetVolumeSnapshotAnnotation(vs *snapshotv1api.VolumeSnapshot) { + vs.ObjectMeta.Annotations[velerov1api.VSCDeletionPolicyAnnotation] = + string(snapshotv1api.VolumeSnapshotContentRetain) +} + +// Execute uses the data such as CSI driver name, storage +// snapshot handle, snapshot deletion secret (if any) from +// the annotations to recreate a VolumeSnapshotContent object +// and statically bind the VolumeSnapshot object being restored. +func (p *volumeSnapshotRestoreItemAction) Execute( + input *velero.RestoreItemActionExecuteInput, +) (*velero.RestoreItemActionExecuteOutput, error) { + p.log.Info("Starting VolumeSnapshotRestoreItemAction") + if boolptr.IsSetToFalse(input.Restore.Spec.RestorePVs) { + p.log.Infof("Restore %s/%s did not request for PVs to be restored.", + input.Restore.Namespace, input.Restore.Name) + return &velero.RestoreItemActionExecuteOutput{SkipRestore: true}, nil + } + var vs snapshotv1api.VolumeSnapshot + + if err := runtime.DefaultUnstructuredConverter.FromUnstructured( + input.Item.UnstructuredContent(), &vs); err != nil { + return &velero.RestoreItemActionExecuteOutput{}, + errors.Wrapf(err, "failed to convert input.Item from unstructured") + } + + // If cross-namespace restore is configured, change the namespace + // for VolumeSnapshot object to be restored + newNamespace, ok := input.Restore.Spec.NamespaceMapping[vs.GetNamespace()] + if !ok { + // Use original namespace + newNamespace = vs.Namespace + } + + if !csi.IsVolumeSnapshotExists(newNamespace, vs.Name, p.crClient) { + snapHandle, exists := vs.Annotations[velerov1api.VolumeSnapshotHandleAnnotation] + if !exists { + return nil, errors.Errorf( + "Volumesnapshot %s/%s does not have a %s annotation", + vs.Namespace, + vs.Name, + velerov1api.VolumeSnapshotHandleAnnotation, + ) + } + + csiDriverName, exists := vs.Annotations[velerov1api.DriverNameAnnotation] + if !exists { + return nil, errors.Errorf( + "Volumesnapshot %s/%s does not have a %s annotation", + vs.Namespace, vs.Name, velerov1api.DriverNameAnnotation) + } + + p.log.Debugf("Set VolumeSnapshotContent %s/%s DeletionPolicy", + "to Retain to make sure VS deletion in namespace will not", + "delete Snapshot on cloud provider.", + newNamespace, vs.Name) + + vsc := snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: vs.Name + "-", + Labels: map[string]string{ + velerov1api.RestoreNameLabel: label.GetValidName(input.Restore.Name), + }, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, + Driver: csiDriverName, + VolumeSnapshotRef: core_v1.ObjectReference{ + Kind: "VolumeSnapshot", + Namespace: newNamespace, + Name: vs.Name, + }, + Source: snapshotv1api.VolumeSnapshotContentSource{ + SnapshotHandle: &snapHandle, + }, + }, + } + + // we create the VolumeSnapshotContent here instead of relying on the + // restore flow because we want to statically bind this VolumeSnapshot + // with a VolumeSnapshotContent that will be used as its source for pre-populating + // the volume that will be created as a result of the restore. To perform + // this static binding, a bi-directional link between the VolumeSnapshotContent + // and VolumeSnapshot objects have to be setup. Further, it is disallowed + // to convert a dynamically created VolumeSnapshotContent for static binding. + // See: https://github.com/kubernetes-csi/external-snapshotter/issues/274 + if err := p.crClient.Create(context.TODO(), &vsc); err != nil { + return nil, errors.Wrapf(err, + "failed to create volumesnapshotcontents %s", + vsc.GenerateName) + } + p.log.Infof("Created VolumesnapshotContents %s with static", + "binding to volumesnapshot %s/%s", vsc, newNamespace, vs.Name) + + // Reset Spec to convert the VolumeSnapshot from using + // the dynamic VolumeSnapshotContent to the static one. + resetVolumeSnapshotSpecForRestore(&vs, &vsc.Name) + + // Reset VolumeSnapshot annotation. By now, only change + // DeletionPolicy to Retain. + resetVolumeSnapshotAnnotation(&vs) + } + + vsMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&vs) + if err != nil { + return nil, errors.WithStack(err) + } + + p.log.Infof("Returning from VolumeSnapshotRestoreItemAction with no additionalItems") + + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: &unstructured.Unstructured{Object: vsMap}, + AdditionalItems: []velero.ResourceIdentifier{}, + }, nil +} + +func (p *volumeSnapshotRestoreItemAction) Name() string { + return "VolumeSnapshotRestoreItemAction" +} + +func (p *volumeSnapshotRestoreItemAction) Progress( + operationID string, + restore *velerov1api.Restore, +) (velero.OperationProgress, error) { + return velero.OperationProgress{}, nil +} + +func (p *volumeSnapshotRestoreItemAction) Cancel( + operationID string, + restore *velerov1api.Restore, +) error { + return nil +} + +func (p *volumeSnapshotRestoreItemAction) AreAdditionalItemsReady( + additionalItems []velero.ResourceIdentifier, + restore *velerov1api.Restore, +) (bool, error) { + return true, nil +} + +func NewVolumeSnapshotRestoreItemAction( + f client.Factory) plugincommon.HandlerInitializer { + return func(logger logrus.FieldLogger) (interface{}, error) { + crClient, err := f.KubebuilderClient() + if err != nil { + return nil, err + } + + return &volumeSnapshotRestoreItemAction{logger, crClient}, nil + } +} diff --git a/pkg/restore/actions/csi/volumesnapshot_action_test.go b/pkg/restore/actions/csi/volumesnapshot_action_test.go new file mode 100644 index 000000000..ed425a660 --- /dev/null +++ b/pkg/restore/actions/csi/volumesnapshot_action_test.go @@ -0,0 +1,89 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + "testing" + + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ( + testPVC = "test-pvc" + testSnapClass = "snap-class" + randText = "DEADFEED" +) + +func TestResetVolumeSnapshotSpecForRestore(t *testing.T) { + testCases := []struct { + name string + vs snapshotv1api.VolumeSnapshot + vscName string + }{ + { + name: "should reset spec as expected", + vs: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vs", + Namespace: "test-ns", + }, + Spec: snapshotv1api.VolumeSnapshotSpec{ + Source: snapshotv1api.VolumeSnapshotSource{ + PersistentVolumeClaimName: &testPVC, + }, + VolumeSnapshotClassName: &testSnapClass, + }, + }, + vscName: "test-vsc", + }, + { + name: "should reset spec and overwriting value for Source.VolumeSnapshotContentName", + vs: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-vs", + Namespace: "test-ns", + }, + Spec: snapshotv1api.VolumeSnapshotSpec{ + Source: snapshotv1api.VolumeSnapshotSource{ + VolumeSnapshotContentName: &randText, + }, + VolumeSnapshotClassName: &testSnapClass, + }, + }, + vscName: "test-vsc", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + before := tc.vs.DeepCopy() + resetVolumeSnapshotSpecForRestore(&tc.vs, &tc.vscName) + + assert.Equalf(t, tc.vs.Name, before.Name, "unexpected change to Object.Name, Want: %s; Got %s", tc.name, before.Name, tc.vs.Name) + assert.Equal(t, tc.vs.Namespace, before.Namespace, "unexpected change to Object.Namespace, Want: %s; Got %s", tc.name, before.Namespace, tc.vs.Namespace) + assert.NotNil(t, tc.vs.Spec.Source) + assert.Nil(t, tc.vs.Spec.Source.PersistentVolumeClaimName) + assert.NotNil(t, tc.vs.Spec.Source.VolumeSnapshotContentName) + assert.Equal(t, *tc.vs.Spec.Source.VolumeSnapshotContentName, tc.vscName) + assert.Equal(t, *tc.vs.Spec.VolumeSnapshotClassName, *before.Spec.VolumeSnapshotClassName, "unexpected value for Spec.VolumeSnapshotClassName, Want: %s, Got: %s", + *tc.vs.Spec.VolumeSnapshotClassName, *before.Spec.VolumeSnapshotClassName) + assert.Nil(t, tc.vs.Status) + }) + } +} diff --git a/pkg/restore/actions/csi/volumesnapshotclass_action.go b/pkg/restore/actions/csi/volumesnapshotclass_action.go new file mode 100644 index 000000000..5afba786b --- /dev/null +++ b/pkg/restore/actions/csi/volumesnapshotclass_action.go @@ -0,0 +1,111 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/csi" +) + +// volumeSnapshotClassRestoreItemAction is a Velero restore +// item action plugin for VolumeSnapshotClass +type volumeSnapshotClassRestoreItemAction struct { + log logrus.FieldLogger +} + +// AppliesTo returns information indicating that VolumeSnapshotClassRestoreItemAction +// should be invoked while restoring volumesnapshotclass.snapshot.storage.k8s.io resources. +func (p *volumeSnapshotClassRestoreItemAction) AppliesTo() (velero.ResourceSelector, error) { + return velero.ResourceSelector{ + IncludedResources: []string{"volumesnapshotclass.snapshot.storage.k8s.io"}, + }, nil +} + +// Execute restores VolumeSnapshotClass objects returning any +// snapshotlister secret as additional items to restore +func (p *volumeSnapshotClassRestoreItemAction) Execute( + input *velero.RestoreItemActionExecuteInput, +) (*velero.RestoreItemActionExecuteOutput, error) { + p.log.Info("Starting VolumeSnapshotClassRestoreItemAction") + if boolptr.IsSetToFalse(input.Restore.Spec.RestorePVs) { + p.log.Infof("Restore did not request for PVs to be restored %s/%s", + input.Restore.Namespace, input.Restore.Name) + return &velero.RestoreItemActionExecuteOutput{SkipRestore: true}, nil + } + var snapClass snapshotv1api.VolumeSnapshotClass + + if err := runtime.DefaultUnstructuredConverter.FromUnstructured( + input.Item.UnstructuredContent(), &snapClass); err != nil { + return &velero.RestoreItemActionExecuteOutput{}, errors.Wrapf(err, + "failed to convert input.Item from unstructured") + } + + additionalItems := []velero.ResourceIdentifier{} + if csi.IsVolumeSnapshotClassHasListerSecret(&snapClass) { + additionalItems = append(additionalItems, velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{Group: "", Resource: "secrets"}, + Name: snapClass.Annotations[velerov1api.PrefixedListSecretNameAnnotation], + Namespace: snapClass.Annotations[velerov1api.PrefixedListSecretNamespaceAnnotation], + }) + } + + p.log.Infof("Returning from VolumeSnapshotClassRestoreItemAction with %d additionalItems", + len(additionalItems)) + + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: input.Item, + AdditionalItems: additionalItems, + }, nil +} + +func (p *volumeSnapshotClassRestoreItemAction) Name() string { + return "VolumeSnapshotClassRestoreItemAction" +} + +func (p *volumeSnapshotClassRestoreItemAction) Progress( + operationID string, + restore *velerov1api.Restore, +) (velero.OperationProgress, error) { + return velero.OperationProgress{}, nil +} + +func (p *volumeSnapshotClassRestoreItemAction) Cancel( + operationID string, + restore *velerov1api.Restore, +) error { + return nil +} + +func (p *volumeSnapshotClassRestoreItemAction) AreAdditionalItemsReady( + additionalItems []velero.ResourceIdentifier, + restore *velerov1api.Restore, +) (bool, error) { + return true, nil +} + +func NewVolumeSnapshotClassRestoreItemAction( + logger logrus.FieldLogger) (interface{}, error) { + return &volumeSnapshotClassRestoreItemAction{logger}, nil +} diff --git a/pkg/restore/actions/csi/volumesnapshotcontent_action.go b/pkg/restore/actions/csi/volumesnapshotcontent_action.go new file mode 100644 index 000000000..bd07fb982 --- /dev/null +++ b/pkg/restore/actions/csi/volumesnapshotcontent_action.go @@ -0,0 +1,113 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package csi + +import ( + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/csi" +) + +// volumeSnapshotContentRestoreItemAction is a restore item action +// plugin for Velero +type volumeSnapshotContentRestoreItemAction struct { + log logrus.FieldLogger +} + +// AppliesTo returns information indicating VolumeSnapshotContentRestoreItemAction +// action should be invoked while restoring +// volumesnapshotcontent.snapshot.storage.k8s.io resources +func (p *volumeSnapshotContentRestoreItemAction) AppliesTo() ( + velero.ResourceSelector, error, +) { + return velero.ResourceSelector{ + IncludedResources: []string{"volumesnapshotcontent.snapshot.storage.k8s.io"}, + }, nil +} + +// Execute restores a VolumeSnapshotContent object without modification +// returning the snapshot lister secret, if any, as additional items to restore. +func (p *volumeSnapshotContentRestoreItemAction) Execute( + input *velero.RestoreItemActionExecuteInput, +) (*velero.RestoreItemActionExecuteOutput, error) { + p.log.Info("Starting VolumeSnapshotContentRestoreItemAction") + var snapCont snapshotv1api.VolumeSnapshotContent + if boolptr.IsSetToFalse(input.Restore.Spec.RestorePVs) { + p.log.Infof("Restore did not request for PVs to be restored %s/%s", + input.Restore.Namespace, input.Restore.Name) + return &velero.RestoreItemActionExecuteOutput{SkipRestore: true}, nil + } + if err := runtime.DefaultUnstructuredConverter.FromUnstructured( + input.Item.UnstructuredContent(), &snapCont); err != nil { + return &velero.RestoreItemActionExecuteOutput{}, + errors.Wrapf(err, "failed to convert input.Item from unstructured") + } + + additionalItems := []velero.ResourceIdentifier{} + if csi.IsVolumeSnapshotContentHasDeleteSecret(&snapCont) { + additionalItems = append(additionalItems, + velero.ResourceIdentifier{ + GroupResource: schema.GroupResource{Group: "", Resource: "secrets"}, + Name: snapCont.Annotations[velerov1api.DeleteSecretNameAnnotation], + Namespace: snapCont.Annotations[velerov1api.DeleteSecretNamespaceAnnotation], + }, + ) + } + + p.log.Infof("Returning from VolumeSnapshotContentRestoreItemAction", + "with %d additionalItems", len(additionalItems)) + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: input.Item, + AdditionalItems: additionalItems, + }, nil +} + +func (p *volumeSnapshotContentRestoreItemAction) Name() string { + return "VolumeSnapshotContentRestoreItemAction" +} + +func (p *volumeSnapshotContentRestoreItemAction) Progress( + operationID string, + restore *velerov1api.Restore, +) (velero.OperationProgress, error) { + return velero.OperationProgress{}, nil +} + +func (p *volumeSnapshotContentRestoreItemAction) Cancel( + operationID string, + restore *velerov1api.Restore, +) error { + return nil +} + +func (p *volumeSnapshotContentRestoreItemAction) AreAdditionalItemsReady( + additionalItems []velero.ResourceIdentifier, + restore *velerov1api.Restore, +) (bool, error) { + return true, nil +} + +func NewVolumeSnapshotContentRestoreItemAction(logger logrus.FieldLogger) (interface{}, error) { + return &volumeSnapshotContentRestoreItemAction{logger}, nil +} diff --git a/pkg/restore/actions/dataupload_retrieve_action.go b/pkg/restore/actions/dataupload_retrieve_action.go index 8194b6892..653f5e340 100644 --- a/pkg/restore/actions/dataupload_retrieve_action.go +++ b/pkg/restore/actions/dataupload_retrieve_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "context" diff --git a/pkg/restore/actions/dataupload_retrieve_action_test.go b/pkg/restore/actions/dataupload_retrieve_action_test.go index 1d14cf4c8..6bfc7aabf 100644 --- a/pkg/restore/actions/dataupload_retrieve_action_test.go +++ b/pkg/restore/actions/dataupload_retrieve_action_test.go @@ -14,11 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "context" - "fmt" "testing" "github.com/stretchr/testify/require" @@ -120,8 +119,6 @@ func TestDataUploadRetrieveActionExectue(t *testing.T) { }) require.NoError(t, err) - // debug - fmt.Printf("CM: %s\n", &cmList.Items[0]) require.Equal(t, tc.expectedDataUploadResult.Labels, cmList.Items[0].Labels) require.Equal(t, tc.expectedDataUploadResult.Data, cmList.Items[0].Data) } diff --git a/pkg/restore/actions/init_restorehook_pod_action.go b/pkg/restore/actions/init_restorehook_pod_action.go index f994d811d..7614ef085 100644 --- a/pkg/restore/actions/init_restorehook_pod_action.go +++ b/pkg/restore/actions/init_restorehook_pod_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "github.com/pkg/errors" diff --git a/pkg/restore/actions/init_restorehook_pod_action_test.go b/pkg/restore/actions/init_restorehook_pod_action_test.go index c69d3c23f..7d269b504 100644 --- a/pkg/restore/actions/init_restorehook_pod_action_test.go +++ b/pkg/restore/actions/init_restorehook_pod_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "testing" diff --git a/pkg/restore/actions/job_action.go b/pkg/restore/actions/job_action.go index 364ec79ad..1eabc208c 100644 --- a/pkg/restore/actions/job_action.go +++ b/pkg/restore/actions/job_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "github.com/pkg/errors" diff --git a/pkg/restore/actions/job_action_test.go b/pkg/restore/actions/job_action_test.go index 606dd089d..e3578bb60 100644 --- a/pkg/restore/actions/job_action_test.go +++ b/pkg/restore/actions/job_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "testing" diff --git a/pkg/restore/actions/pod_action.go b/pkg/restore/actions/pod_action.go index d4bdc1384..9a98aa88c 100644 --- a/pkg/restore/actions/pod_action.go +++ b/pkg/restore/actions/pod_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "strings" diff --git a/pkg/restore/actions/pod_action_test.go b/pkg/restore/actions/pod_action_test.go index f1aa83c1a..27ae814f0 100644 --- a/pkg/restore/actions/pod_action_test.go +++ b/pkg/restore/actions/pod_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "testing" diff --git a/pkg/restore/actions/pod_volume_restore_action.go b/pkg/restore/actions/pod_volume_restore_action.go index af5c2373c..36722b0a2 100644 --- a/pkg/restore/actions/pod_volume_restore_action.go +++ b/pkg/restore/actions/pod_volume_restore_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "context" diff --git a/pkg/restore/actions/pod_volume_restore_action_test.go b/pkg/restore/actions/pod_volume_restore_action_test.go index cba4914e5..7be617bd4 100644 --- a/pkg/restore/actions/pod_volume_restore_action_test.go +++ b/pkg/restore/actions/pod_volume_restore_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "sort" diff --git a/pkg/restore/actions/rolebinding_action.go b/pkg/restore/actions/rolebinding_action.go index c402075a6..ed345edb0 100644 --- a/pkg/restore/actions/rolebinding_action.go +++ b/pkg/restore/actions/rolebinding_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "github.com/pkg/errors" diff --git a/pkg/restore/actions/rolebinding_action_test.go b/pkg/restore/actions/rolebinding_action_test.go index 8995df8c6..5b605937e 100644 --- a/pkg/restore/actions/rolebinding_action_test.go +++ b/pkg/restore/actions/rolebinding_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "sort" diff --git a/pkg/restore/actions/secret_action.go b/pkg/restore/actions/secret_action.go index 1e63372b6..9e3d06fc8 100644 --- a/pkg/restore/actions/secret_action.go +++ b/pkg/restore/actions/secret_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "context" diff --git a/pkg/restore/actions/secret_action_test.go b/pkg/restore/actions/secret_action_test.go index a627e5296..0cb05617a 100644 --- a/pkg/restore/actions/secret_action_test.go +++ b/pkg/restore/actions/secret_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "testing" diff --git a/pkg/restore/actions/service_account_action.go b/pkg/restore/actions/service_account_action.go index 252d9fc57..85b9bc3df 100644 --- a/pkg/restore/actions/service_account_action.go +++ b/pkg/restore/actions/service_account_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "strings" diff --git a/pkg/restore/actions/service_account_action_test.go b/pkg/restore/actions/service_account_action_test.go index 0bc976a0f..273ac660f 100644 --- a/pkg/restore/actions/service_account_action_test.go +++ b/pkg/restore/actions/service_account_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "sort" diff --git a/pkg/restore/actions/service_action.go b/pkg/restore/actions/service_action.go index 6fc1b2cb4..ee392a381 100644 --- a/pkg/restore/actions/service_action.go +++ b/pkg/restore/actions/service_action.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "encoding/json" diff --git a/pkg/restore/actions/service_action_test.go b/pkg/restore/actions/service_action_test.go index 16115e920..e8ed85d69 100644 --- a/pkg/restore/actions/service_action_test.go +++ b/pkg/restore/actions/service_action_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package restore +package actions import ( "encoding/json" diff --git a/pkg/test/fake_controller_runtime_client.go b/pkg/test/fake_controller_runtime_client.go index 802999166..ec0432493 100644 --- a/pkg/test/fake_controller_runtime_client.go +++ b/pkg/test/fake_controller_runtime_client.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/require" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" + storagev1api "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" k8sfake "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -33,34 +34,33 @@ import ( func NewFakeControllerRuntimeClientBuilder(t *testing.T) *k8sfake.ClientBuilder { scheme := runtime.NewScheme() - err := velerov1api.AddToScheme(scheme) - require.NoError(t, err) - err = velerov2alpha1api.AddToScheme(scheme) - require.NoError(t, err) - err = corev1api.AddToScheme(scheme) - require.NoError(t, err) - err = appsv1api.AddToScheme(scheme) - require.NoError(t, err) - err = snapshotv1api.AddToScheme(scheme) - require.NoError(t, err) + + require.NoError(t, velerov1api.AddToScheme(scheme)) + require.NoError(t, velerov2alpha1api.AddToScheme(scheme)) + require.NoError(t, corev1api.AddToScheme(scheme)) + require.NoError(t, appsv1api.AddToScheme(scheme)) + require.NoError(t, snapshotv1api.AddToScheme(scheme)) + require.NoError(t, storagev1api.AddToScheme(scheme)) + return k8sfake.NewClientBuilder().WithScheme(scheme) } func NewFakeControllerRuntimeClient(t *testing.T, initObjs ...runtime.Object) client.Client { scheme := runtime.NewScheme() - err := velerov1api.AddToScheme(scheme) - require.NoError(t, err) - err = velerov2alpha1api.AddToScheme(scheme) - require.NoError(t, err) - err = corev1api.AddToScheme(scheme) - require.NoError(t, err) - err = appsv1api.AddToScheme(scheme) - require.NoError(t, err) - err = snapshotv1api.AddToScheme(scheme) - require.NoError(t, err) + + require.NoError(t, velerov1api.AddToScheme(scheme)) + require.NoError(t, velerov2alpha1api.AddToScheme(scheme)) + require.NoError(t, corev1api.AddToScheme(scheme)) + require.NoError(t, appsv1api.AddToScheme(scheme)) + require.NoError(t, snapshotv1api.AddToScheme(scheme)) + require.NoError(t, storagev1api.AddToScheme(scheme)) + return k8sfake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(initObjs...).Build() } -func NewFakeControllerRuntimeWatchClient(t *testing.T, initObjs ...runtime.Object) client.WithWatch { +func NewFakeControllerRuntimeWatchClient( + t *testing.T, + initObjs ...runtime.Object, +) client.WithWatch { return NewFakeControllerRuntimeClientBuilder(t).WithRuntimeObjects(initObjs...).Build() } diff --git a/pkg/util/csi/volume_snapshot.go b/pkg/util/csi/volume_snapshot.go index a9a88fd32..b1b7f9375 100644 --- a/pkg/util/csi/volume_snapshot.go +++ b/pkg/util/csi/volume_snapshot.go @@ -20,25 +20,26 @@ import ( "context" "encoding/json" "fmt" + "strings" "time" jsonpatch "github.com/evanphx/json-patch" + snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" + snapshotter "github.com/kubernetes-csi/external-snapshotter/client/v7/clientset/versioned/typed/volumesnapshot/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" + corev1api "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/stringptr" "github.com/vmware-tanzu/velero/pkg/util/stringslice" - - snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" - snapshotter "github.com/kubernetes-csi/external-snapshotter/client/v7/clientset/versioned/typed/volumesnapshot/v1" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - apierrors "k8s.io/apimachinery/pkg/api/errors" ) const ( @@ -47,51 +48,80 @@ const ( ) // WaitVolumeSnapshotReady waits a VS to become ready to use until the timeout reaches -func WaitVolumeSnapshotReady(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, - volumeSnapshot string, volumeSnapshotNS string, timeout time.Duration, log logrus.FieldLogger) (*snapshotv1api.VolumeSnapshot, error) { +func WaitVolumeSnapshotReady( + ctx context.Context, + snapshotClient snapshotter.SnapshotV1Interface, + volumeSnapshot string, + volumeSnapshotNS string, + timeout time.Duration, + log logrus.FieldLogger, +) (*snapshotv1api.VolumeSnapshot, error) { var updated *snapshotv1api.VolumeSnapshot errMessage := sets.NewString() - err := wait.PollUntilContextTimeout(ctx, waitInternal, timeout, true, func(ctx context.Context) (bool, error) { - tmpVS, err := snapshotClient.VolumeSnapshots(volumeSnapshotNS).Get(ctx, volumeSnapshot, metav1.GetOptions{}) - if err != nil { - return false, errors.Wrapf(err, fmt.Sprintf("error to get volumesnapshot %s/%s", volumeSnapshotNS, volumeSnapshot)) - } + err := wait.PollUntilContextTimeout( + ctx, + waitInternal, + timeout, + true, + func(ctx context.Context) (bool, error) { + tmpVS, err := snapshotClient.VolumeSnapshots(volumeSnapshotNS).Get( + ctx, volumeSnapshot, metav1.GetOptions{}) + if err != nil { + return false, errors.Wrapf( + err, + fmt.Sprintf("error to get VolumeSnapshot %s/%s", + volumeSnapshotNS, volumeSnapshot), + ) + } - if tmpVS.Status == nil { - return false, nil - } + if tmpVS.Status == nil { + return false, nil + } - if tmpVS.Status.Error != nil { - errMessage.Insert(stringptr.GetString(tmpVS.Status.Error.Message)) - } + if tmpVS.Status.Error != nil { + errMessage.Insert(stringptr.GetString(tmpVS.Status.Error.Message)) + } - if !boolptr.IsSetToTrue(tmpVS.Status.ReadyToUse) { - return false, nil - } + if !boolptr.IsSetToTrue(tmpVS.Status.ReadyToUse) { + return false, nil + } - updated = tmpVS - return true, nil - }) + updated = tmpVS + return true, nil + }, + ) if wait.Interrupted(err) { - err = errors.Errorf("volume snapshot is not ready until timeout, errors: %v", errMessage.List()) + err = errors.Errorf( + "volume snapshot is not ready until timeout, errors: %v", + errMessage.List(), + ) } if errMessage.Len() > 0 { - log.Warnf("Some errors happened during waiting for ready snapshot, errors: %v", errMessage.List()) + log.Warnf("Some errors happened during waiting for ready snapshot, errors: %v", + errMessage.List()) } return updated, err } -// GetVolumeSnapshotContentForVolumeSnapshot returns the volumesnapshotcontent object associated with the volumesnapshot -func GetVolumeSnapshotContentForVolumeSnapshot(volSnap *snapshotv1api.VolumeSnapshot, snapshotClient snapshotter.SnapshotV1Interface) (*snapshotv1api.VolumeSnapshotContent, error) { +// GetVolumeSnapshotContentForVolumeSnapshot returns the VolumeSnapshotContent +// object associated with the VolumeSnapshot. +func GetVolumeSnapshotContentForVolumeSnapshot( + volSnap *snapshotv1api.VolumeSnapshot, + snapshotClient snapshotter.SnapshotV1Interface, +) (*snapshotv1api.VolumeSnapshotContent, error) { if volSnap.Status == nil || volSnap.Status.BoundVolumeSnapshotContentName == nil { return nil, errors.Errorf("invalid snapshot info in volume snapshot %s", volSnap.Name) } - vsc, err := snapshotClient.VolumeSnapshotContents().Get(context.TODO(), *volSnap.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{}) + vsc, err := snapshotClient.VolumeSnapshotContents().Get( + context.TODO(), + *volSnap.Status.BoundVolumeSnapshotContentName, + metav1.GetOptions{}, + ) if err != nil { return nil, errors.Wrap(err, "error getting volume snapshot content from API") } @@ -99,7 +129,8 @@ func GetVolumeSnapshotContentForVolumeSnapshot(volSnap *snapshotv1api.VolumeSnap return vsc, nil } -// RetainVSC updates the VSC's deletion policy to Retain and add a finalier and then return the update VSC +// RetainVSC updates the VSC's deletion policy to Retain and add a +// finalizer and then return the update VSC func RetainVSC(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, vsc *snapshotv1api.VolumeSnapshotContent) (*snapshotv1api.VolumeSnapshotContent, error) { if vsc.Spec.DeletionPolicy == snapshotv1api.VolumeSnapshotContentRetain { @@ -112,8 +143,13 @@ func RetainVSC(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interfa }) } -// DeleteVolumeSnapshotContentIfAny deletes a VSC by name if it exists, and log an error when the deletion fails -func DeleteVolumeSnapshotContentIfAny(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, vscName string, log logrus.FieldLogger) { +// DeleteVolumeSnapshotContentIfAny deletes a VSC by name if it exists, +// and log an error when the deletion fails. +func DeleteVolumeSnapshotContentIfAny( + ctx context.Context, + snapshotClient snapshotter.SnapshotV1Interface, + vscName string, log logrus.FieldLogger, +) { err := snapshotClient.VolumeSnapshotContents().Delete(ctx, vscName, metav1.DeleteOptions{}) if err != nil { if apierrors.IsNotFound(err) { @@ -124,7 +160,8 @@ func DeleteVolumeSnapshotContentIfAny(ctx context.Context, snapshotClient snapsh } } -// EnsureDeleteVS asserts the existence of a VS by name, deletes it and waits for its disappearance and returns errors on any failure +// EnsureDeleteVS asserts the existence of a VS by name, deletes it and waits for its +// disappearance and returns errors on any failure. func EnsureDeleteVS(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, vsName string, vsNamespace string, timeout time.Duration) error { err := snapshotClient.VolumeSnapshots(vsNamespace).Delete(ctx, vsName, metav1.DeleteOptions{}) @@ -176,7 +213,8 @@ func RemoveVSCProtect(ctx context.Context, snapshotClient snapshotter.SnapshotV1 return err } -// EnsureDeleteVSC asserts the existence of a VSC by name, deletes it and waits for its disappearance and returns errors on any failure +// EnsureDeleteVSC asserts the existence of a VSC by name, deletes it and waits for its +// disappearance and returns errors on any failure. func EnsureDeleteVSC(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, vscName string, timeout time.Duration) error { err := snapshotClient.VolumeSnapshotContents().Delete(ctx, vscName, metav1.DeleteOptions{}) @@ -203,20 +241,34 @@ func EnsureDeleteVSC(ctx context.Context, snapshotClient snapshotter.SnapshotV1I return nil } -// DeleteVolumeSnapshotIfAny deletes a VS by name if it exists, and log an error when the deletion fails -func DeleteVolumeSnapshotIfAny(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, vsName string, vsNamespace string, log logrus.FieldLogger) { +// DeleteVolumeSnapshotIfAny deletes a VS by name if it exists, +// and log an error when the deletion fails +func DeleteVolumeSnapshotIfAny( + ctx context.Context, + snapshotClient snapshotter.SnapshotV1Interface, + vsName string, + vsNamespace string, + log logrus.FieldLogger, +) { err := snapshotClient.VolumeSnapshots(vsNamespace).Delete(ctx, vsName, metav1.DeleteOptions{}) if err != nil { if apierrors.IsNotFound(err) { - log.WithError(err).Debugf("Abort deleting volume snapshot, it doesn't exist %s/%s", vsNamespace, vsName) + log.WithError(err).Debugf( + "Abort deleting volume snapshot, it doesn't exist %s/%s", + vsNamespace, vsName) } else { - log.WithError(err).Errorf("Failed to delete volume snapshot %s/%s", vsNamespace, vsName) + log.WithError(err).Errorf( + "Failed to delete volume snapshot %s/%s", vsNamespace, vsName) } } } -func patchVSC(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interface, - vsc *snapshotv1api.VolumeSnapshotContent, updateFunc func(*snapshotv1api.VolumeSnapshotContent)) (*snapshotv1api.VolumeSnapshotContent, error) { +func patchVSC( + ctx context.Context, + snapshotClient snapshotter.SnapshotV1Interface, + vsc *snapshotv1api.VolumeSnapshotContent, + updateFunc func(*snapshotv1api.VolumeSnapshotContent), +) (*snapshotv1api.VolumeSnapshotContent, error) { origBytes, err := json.Marshal(vsc) if err != nil { return nil, errors.Wrap(err, "error marshaling original VSC") @@ -242,3 +294,493 @@ func patchVSC(ctx context.Context, snapshotClient snapshotter.SnapshotV1Interfac return patched, nil } + +func GetVolumeSnapshotClass( + provisioner string, + backup *velerov1api.Backup, + pvc *corev1api.PersistentVolumeClaim, + log logrus.FieldLogger, + crClient crclient.Client, +) (*snapshotv1api.VolumeSnapshotClass, error) { + snapshotClasses := new(snapshotv1api.VolumeSnapshotClassList) + err := crClient.List(context.TODO(), snapshotClasses) + if err != nil { + return nil, errors.Wrap(err, "error listing VolumeSnapshotClass") + } + // If a snapshot class is set for provider in PVC annotations, use that + snapshotClass, err := GetVolumeSnapshotClassFromPVCAnnotationsForDriver( + pvc, provisioner, snapshotClasses, + ) + if err != nil { + log.Debugf("Didn't find VolumeSnapshotClass from PVC annotations: %v", err) + } + if snapshotClass != nil { + return snapshotClass, nil + } + + // If there is no annotation in PVC, attempt to fetch it from backup annotations + snapshotClass, err = GetVolumeSnapshotClassFromBackupAnnotationsForDriver( + backup, provisioner, snapshotClasses) + if err != nil { + log.Debugf("Didn't find VolumeSnapshotClass from Backup annotations: %v", err) + } + if snapshotClass != nil { + return snapshotClass, nil + } + + // fallback to default behavior of fetching snapshot class based on label + snapshotClass, err = GetVolumeSnapshotClassForStorageClass( + provisioner, snapshotClasses) + if err != nil || snapshotClass == nil { + return nil, errors.Wrap(err, "error getting VolumeSnapshotClass") + } + + return snapshotClass, nil +} + +func GetVolumeSnapshotClassFromPVCAnnotationsForDriver( + pvc *corev1api.PersistentVolumeClaim, + provisioner string, + snapshotClasses *snapshotv1api.VolumeSnapshotClassList, +) (*snapshotv1api.VolumeSnapshotClass, error) { + annotationKey := velerov1api.VolumeSnapshotClassDriverPVCAnnotation + snapshotClassName, ok := pvc.ObjectMeta.Annotations[annotationKey] + if !ok { + return nil, nil + } + for _, sc := range snapshotClasses.Items { + if strings.EqualFold(snapshotClassName, sc.ObjectMeta.Name) { + if !strings.EqualFold(sc.Driver, provisioner) { + return nil, errors.Errorf( + "Incorrect VolumeSnapshotClass %s is not for driver %s", + sc.ObjectMeta.Name, provisioner, + ) + } + return &sc, nil + } + } + return nil, errors.Errorf( + "No CSI VolumeSnapshotClass found with name %s for provisioner %s for PVC %s", + snapshotClassName, provisioner, pvc.Name, + ) +} + +// GetVolumeSnapshotClassFromAnnotationsForDriver returns a +// VolumeSnapshotClass for the supplied volume provisioner/driver +// name from the annotation of the backup. +func GetVolumeSnapshotClassFromBackupAnnotationsForDriver( + backup *velerov1api.Backup, + provisioner string, + snapshotClasses *snapshotv1api.VolumeSnapshotClassList, +) (*snapshotv1api.VolumeSnapshotClass, error) { + annotationKey := fmt.Sprintf( + "%s_%s", + velerov1api.VolumeSnapshotClassDriverBackupAnnotationPrefix, + strings.ToLower(provisioner), + ) + snapshotClassName, ok := backup.ObjectMeta.Annotations[annotationKey] + if !ok { + return nil, nil + } + for _, sc := range snapshotClasses.Items { + if strings.EqualFold(snapshotClassName, sc.ObjectMeta.Name) { + if !strings.EqualFold(sc.Driver, provisioner) { + return nil, errors.Errorf( + "Incorrect VolumeSnapshotClass %s is not for driver %s for backup %s", + sc.ObjectMeta.Name, provisioner, backup.Name, + ) + } + return &sc, nil + } + } + return nil, errors.Errorf( + "No CSI VolumeSnapshotClass found with name %s for driver %s for backup %s", + snapshotClassName, provisioner, backup.Name, + ) +} + +// GetVolumeSnapshotClassForStorageClass returns a VolumeSnapshotClass +// for the supplied volume provisioner/ driver name. +func GetVolumeSnapshotClassForStorageClass( + provisioner string, + snapshotClasses *snapshotv1api.VolumeSnapshotClassList, +) (*snapshotv1api.VolumeSnapshotClass, error) { + n := 0 + var vsClass snapshotv1api.VolumeSnapshotClass + // We pick the VolumeSnapshotClass that matches the CSI driver name + // and has a 'velero.io/csi-volumesnapshot-class' label. This allows + // multiple VolumeSnapshotClasses for the same driver with different + // values for the other fields in the spec. + for _, sc := range snapshotClasses.Items { + _, hasLabelSelector := sc.Labels[velerov1api.VolumeSnapshotClassSelectorLabel] + if sc.Driver == provisioner { + n += 1 + vsClass = sc + if hasLabelSelector { + return &sc, nil + } + } + } + // If there's only one volumesnapshotclass for the driver, return it. + if n == 1 { + return &vsClass, nil + } + return nil, fmt.Errorf( + `failed to get VolumeSnapshotClass for provisioner %s, + ensure that the desired VolumeSnapshot class has the %s label`, + provisioner, velerov1api.VolumeSnapshotClassSelectorLabel) +} + +// IsVolumeSnapshotClassHasListerSecret returns whether a volumesnapshotclass has a snapshotlister secret +func IsVolumeSnapshotClassHasListerSecret(vc *snapshotv1api.VolumeSnapshotClass) bool { + // https://github.com/kubernetes-csi/external-snapshotter/blob/master/pkg/utils/util.go#L59-L60 + // There is no release w/ these constants exported. Using the strings for now. + _, nameExists := vc.Annotations[velerov1api.PrefixedListSecretNameAnnotation] + _, nsExists := vc.Annotations[velerov1api.PrefixedListSecretNamespaceAnnotation] + return nameExists && nsExists +} + +// IsVolumeSnapshotContentHasDeleteSecret returns whether a volumesnapshotcontent has a deletesnapshot secret +func IsVolumeSnapshotContentHasDeleteSecret(vsc *snapshotv1api.VolumeSnapshotContent) bool { + // https://github.com/kubernetes-csi/external-snapshotter/blob/master/pkg/utils/util.go#L56-L57 + // use exported constants in the next release + _, nameExists := vsc.Annotations[velerov1api.PrefixedSecretNameAnnotation] + _, nsExists := vsc.Annotations[velerov1api.PrefixedSecretNamespaceAnnotation] + return nameExists && nsExists +} + +// IsVolumeSnapshotHasVSCDeleteSecret returns whether a volumesnapshot should set the deletesnapshot secret +// for the static volumesnapshotcontent that is created on restore +func IsVolumeSnapshotHasVSCDeleteSecret(vs *snapshotv1api.VolumeSnapshot) bool { + _, nameExists := vs.Annotations[velerov1api.DeleteSecretNameAnnotation] + _, nsExists := vs.Annotations[velerov1api.DeleteSecretNamespaceAnnotation] + return nameExists && nsExists +} + +// IsVolumeSnapshotExists returns whether a specific volumesnapshot object exists. +func IsVolumeSnapshotExists( + ns, + name string, + crClient crclient.Client, +) bool { + vs := new(snapshotv1api.VolumeSnapshot) + err := crClient.Get( + context.TODO(), + crclient.ObjectKey{Namespace: ns, Name: name}, + vs, + ) + + return err == nil +} + +func SetVolumeSnapshotContentDeletionPolicy( + vscName string, + crClient crclient.Client, +) error { + vsc := new(snapshotv1api.VolumeSnapshotContent) + if err := crClient.Get(context.TODO(), crclient.ObjectKey{Name: vscName}, vsc); err != nil { + return err + } + + originVSC := vsc.DeepCopy() + vsc.Spec.DeletionPolicy = snapshotv1api.VolumeSnapshotContentDelete + + return crClient.Patch(context.TODO(), vsc, crclient.MergeFrom(originVSC)) +} + +func CleanupVolumeSnapshot( + volSnap *snapshotv1api.VolumeSnapshot, + crClient crclient.Client, + log logrus.FieldLogger, +) { + log.Infof("Deleting Volumesnapshot %s/%s", volSnap.Namespace, volSnap.Name) + vs := new(snapshotv1api.VolumeSnapshot) + err := crClient.Get( + context.TODO(), + crclient.ObjectKey{Name: volSnap.Name, Namespace: volSnap.Namespace}, + vs, + ) + if err != nil { + log.Debugf("Failed to get volumesnapshot %s/%s", volSnap.Namespace, volSnap.Name) + return + } + + if vs.Status != nil && vs.Status.BoundVolumeSnapshotContentName != nil { + // we patch the DeletionPolicy of the VolumeSnapshotContent to set it to Delete. + // This ensures that the volume snapshot in the storage provider is also deleted. + err := SetVolumeSnapshotContentDeletionPolicy( + *vs.Status.BoundVolumeSnapshotContentName, + crClient, + ) + if err != nil { + log.Debugf("Failed to patch DeletionPolicy of volume snapshot %s/%s", + vs.Namespace, vs.Name) + } + } + err = crClient.Delete(context.TODO(), vs) + if err != nil { + log.Debugf("Failed to delete volumesnapshot %s/%s: %v", vs.Namespace, vs.Name, err) + } else { + log.Infof("Deleted volumesnapshot with volumesnapshotContent %s/%s", + vs.Namespace, vs.Name) + } +} + +// DeleteVolumeSnapshot handles the VolumeSnapshot instance deletion. +func DeleteVolumeSnapshot( + vs snapshotv1api.VolumeSnapshot, + vsc snapshotv1api.VolumeSnapshotContent, + backup *velerov1api.Backup, + client crclient.Client, + logger logrus.FieldLogger, +) { + modifyVSCFlag := false + if vs.Status != nil && + vs.Status.BoundVolumeSnapshotContentName != nil && + len(*vs.Status.BoundVolumeSnapshotContentName) > 0 && + vsc.Spec.DeletionPolicy == snapshotv1api.VolumeSnapshotContentDelete { + modifyVSCFlag = true + } else { + logger.Errorf("VolumeSnapshot %s/%s is not ready. This is not expected.", + vs.Namespace, vs.Name) + } + + // Change VolumeSnapshotContent's DeletionPolicy to Retain before deleting VolumeSnapshot, + // because VolumeSnapshotContent will be deleted by deleting VolumeSnapshot, when + // DeletionPolicy is set to Delete, but Velero needs VSC for cleaning snapshot on cloud + // in backup deletion. + if modifyVSCFlag { + logger.Debugf("Patching VolumeSnapshotContent %s", vsc.Name) + originVSC := vsc.DeepCopy() + vsc.Spec.DeletionPolicy = snapshotv1api.VolumeSnapshotContentRetain + err := client.Patch( + context.Background(), + &vsc, + crclient.MergeFrom(originVSC), + ) + if err != nil { + logger.Errorf( + "fail to modify VolumeSnapshotContent %s DeletionPolicy to Retain: %s", + vsc.Name, err.Error(), + ) + return + } + + defer func() { + logger.Debugf("Start to recreate VolumeSnapshotContent %s", vsc.Name) + err := recreateVolumeSnapshotContent(vsc, backup, client, logger) + if err != nil { + logger.Errorf( + "fail to recreate VolumeSnapshotContent %s: %s", + vsc.Name, + err.Error(), + ) + } + }() + } + + // Delete VolumeSnapshot from cluster + logger.Debugf("Deleting VolumeSnapshot %s/%s", vs.Namespace, vs.Name) + err := client.Delete(context.TODO(), &vs) + if err != nil { + logger.Errorf("fail to delete VolumeSnapshot %s/%s: %s", + vs.Namespace, vs.Name, err.Error()) + } +} + +// recreateVolumeSnapshotContent will delete then re-create VolumeSnapshotContent, +// because some parameter in VolumeSnapshotContent Spec is immutable, +// e.g. VolumeSnapshotRef and Source. +// Source is updated to let csi-controller thinks the VSC is statically +// provisioned with VS. +// Set VolumeSnapshotRef's UID to nil will let the csi-controller finds out +// the related VS is gone, then VSC can be deleted. +func recreateVolumeSnapshotContent( + vsc snapshotv1api.VolumeSnapshotContent, + backup *velerov1api.Backup, + client crclient.Client, + log logrus.FieldLogger, +) error { + // Read resource timeout from backup annotation, if not set, use default value. + timeout, err := time.ParseDuration( + backup.Annotations[velerov1api.ResourceTimeoutAnnotation]) + if err != nil { + log.Warnf("fail to parse resource timeout annotation %s: %s", + backup.Annotations[velerov1api.ResourceTimeoutAnnotation], err.Error()) + timeout = 10 * time.Minute + } + log.Debugf("resource timeout is set to %s", timeout.String()) + interval := 1 * time.Second + + if err := client.Delete(context.TODO(), &vsc); err != nil { + return errors.Wrapf(err, "fail to delete VolumeSnapshotContent: %s", vsc.Name) + } + + // Check VolumeSnapshotContents is already deleted, before re-creating it. + err = wait.PollUntilContextTimeout( + context.Background(), + interval, + timeout, + true, + func(ctx context.Context) (bool, error) { + tmpVSC := new(snapshotv1api.VolumeSnapshotContent) + if err := client.Get(ctx, crclient.ObjectKeyFromObject(&vsc), tmpVSC); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, errors.Wrapf( + err, + fmt.Sprintf("failed to get VolumeSnapshotContent %s", vsc.Name), + ) + } + return false, nil + }, + ) + if err != nil { + return errors.Wrapf(err, "fail to retrieve VolumeSnapshotContent %s info", vsc.Name) + } + + // Make the VolumeSnapshotContent static + vsc.Spec.Source = snapshotv1api.VolumeSnapshotContentSource{ + SnapshotHandle: vsc.Status.SnapshotHandle, + } + // Set VolumeSnapshotRef to none exist one, because VolumeSnapshotContent + // validation webhook will check whether name and namespace are nil. + // external-snapshotter needs Source pointing to snapshot and VolumeSnapshot + // reference's UID to nil to determine the VolumeSnapshotContent is deletable. + vsc.Spec.VolumeSnapshotRef = corev1api.ObjectReference{ + APIVersion: snapshotv1api.SchemeGroupVersion.String(), + Kind: "VolumeSnapshot", + Namespace: "ns-" + string(vsc.UID), + Name: "name-" + string(vsc.UID), + } + // ResourceVersion shouldn't exist for new creation. + vsc.ResourceVersion = "" + if err := client.Create(context.TODO(), &vsc); err != nil { + return errors.Wrapf(err, "fail to create VolumeSnapshotContent %s", vsc.Name) + } + + return nil +} + +// WaitUntilVSCHandleIsReady returns the VolumeSnapshotContent +// object associated with the volumesnapshot +func WaitUntilVSCHandleIsReady( + volSnap *snapshotv1api.VolumeSnapshot, + crClient crclient.Client, + log logrus.FieldLogger, + shouldWait bool, + csiSnapshotTimeout time.Duration, +) (*snapshotv1api.VolumeSnapshotContent, error) { + if !shouldWait { + if volSnap.Status == nil || + volSnap.Status.BoundVolumeSnapshotContentName == nil { + // volumesnapshot hasn't been reconciled and we're + // not waiting for it. + return nil, nil + } + vsc := new(snapshotv1api.VolumeSnapshotContent) + err := crClient.Get( + context.TODO(), + crclient.ObjectKey{ + Name: *volSnap.Status.BoundVolumeSnapshotContentName, + }, + vsc, + ) + if err != nil { + return nil, + errors.Wrap(err, + "error getting volume snapshot content from API") + } + return vsc, nil + } + + // We'll wait 10m for the VSC to be reconciled polling + // every 5s unless backup's csiSnapshotTimeout is set + interval := 5 * time.Second + vsc := new(snapshotv1api.VolumeSnapshotContent) + + err := wait.PollUntilContextTimeout( + context.Background(), + interval, + csiSnapshotTimeout, + true, + func(ctx context.Context) (bool, error) { + vs := new(snapshotv1api.VolumeSnapshot) + if err := crClient.Get( + ctx, + crclient.ObjectKeyFromObject(volSnap), + vs, + ); err != nil { + return false, + errors.Wrapf(err, fmt.Sprintf( + "failed to get volumesnapshot %s/%s", + volSnap.Namespace, volSnap.Name), + ) + } + + if vs.Status == nil || vs.Status.BoundVolumeSnapshotContentName == nil { + log.Infof("Waiting for CSI driver to reconcile volumesnapshot %s/%s. Retrying in %ds", + volSnap.Namespace, volSnap.Name, interval/time.Second) + return false, nil + } + + if err := crClient.Get( + ctx, + crclient.ObjectKey{ + Name: *vs.Status.BoundVolumeSnapshotContentName, + }, + vsc, + ); err != nil { + return false, + errors.Wrapf( + err, + fmt.Sprintf("failed to get VolumeSnapshotContent %s for VolumeSnapshot %s/%s", + *vs.Status.BoundVolumeSnapshotContentName, vs.Namespace, vs.Name), + ) + } + + // we need to wait for the VolumeSnapshotContent + // to have a snapshot handle because during restore, + // we'll use that snapshot handle as the source for + // the VolumeSnapshotContent so it's statically + // bound to the existing snapshot. + if vsc.Status == nil || + vsc.Status.SnapshotHandle == nil { + log.Infof( + "Waiting for VolumeSnapshotContents %s to have snapshot handle. Retrying in %ds", + vsc.Name, interval/time.Second) + if vsc.Status != nil && + vsc.Status.Error != nil { + log.Warnf("VolumeSnapshotContent %s has error: %v", + vsc.Name, *vsc.Status.Error.Message) + } + return false, nil + } + + return true, nil + }, + ) + + if err != nil { + if err == wait.ErrorInterrupted(errors.New("timed out waiting for the condition")) { + if vsc != nil && + vsc.Status != nil && + vsc.Status.Error != nil { + log.Errorf( + "Timed out awaiting reconciliation of VolumeSnapshot, VolumeSnapshotContent %s has error: %v", + vsc.Name, *vsc.Status.Error.Message) + return nil, + errors.Errorf("CSI got timed out with error: %v", + *vsc.Status.Error.Message) + } else { + log.Errorf( + "Timed out awaiting reconciliation of volumesnapshot %s/%s", + volSnap.Namespace, volSnap.Name) + } + } + return nil, err + } + + return vsc, nil +} diff --git a/pkg/util/csi/volume_snapshot_test.go b/pkg/util/csi/volume_snapshot_test.go index 0b02865e2..74be22786 100644 --- a/pkg/util/csi/volume_snapshot_test.go +++ b/pkg/util/csi/volume_snapshot_test.go @@ -24,18 +24,23 @@ import ( snapshotv1api "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" snapshotFake "github.com/kubernetes-csi/external-snapshotter/client/v7/clientset/versioned/fake" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" clientTesting "k8s.io/client-go/testing" + crclient "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/vmware-tanzu/velero/pkg/util/boolptr" - "github.com/vmware-tanzu/velero/pkg/util/stringptr" - + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/builder" velerotest "github.com/vmware-tanzu/velero/pkg/test" - - apierrors "k8s.io/apimachinery/pkg/api/errors" + "github.com/vmware-tanzu/velero/pkg/util/boolptr" + "github.com/vmware-tanzu/velero/pkg/util/logging" + "github.com/vmware-tanzu/velero/pkg/util/stringptr" ) type reactor struct { @@ -44,8 +49,12 @@ type reactor struct { reactorFunc clientTesting.ReactionFunc } +// expected: &v1.VolumeSnapshot{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"fake-vs", GenerateName:"", Namespace:"fake-ns", SelfLink:"", UID:"", ResourceVersion:"999", Generation:0, CreationTimestamp:time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC), DeletionTimestamp:, DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ManagedFields:[]v1.ManagedFieldsEntry(nil)}, Spec:v1.VolumeSnapshotSpec{Source:v1.VolumeSnapshotSource{PersistentVolumeClaimName:(*string)(nil), VolumeSnapshotContentName:(*string)(nil)}, VolumeSnapshotClassName:(*string)(nil)}, Status:(*v1.VolumeSnapshotStatus)(0x140000af8c0)} +// actual : &v1.VolumeSnapshot{TypeMeta:v1.TypeMeta{Kind:"", APIVersion:""}, ObjectMeta:v1.ObjectMeta{Name:"fake-vs", GenerateName:"", Namespace:"fake-ns", SelfLink:"", UID:"", ResourceVersion:"999", Generation:0, CreationTimestamp:time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC), DeletionTimestamp:, DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string(nil), Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ManagedFields:[]v1.ManagedFieldsEntry(nil)}, Spec:v1.VolumeSnapshotSpec{Source:v1.VolumeSnapshotSource{PersistentVolumeClaimName:(*string)(nil), VolumeSnapshotContentName:(*string)(nil)}, VolumeSnapshotClassName:(*string)(nil)}, Status:(*v1.VolumeSnapshotStatus)(0x1400024ed20)} + func TestWaitVolumeSnapshotReady(t *testing.T) { vscName := "fake-vsc" + quantity := resource.MustParse("0") vsObj := &snapshotv1api.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{ Name: "fake-vs", @@ -54,7 +63,7 @@ func TestWaitVolumeSnapshotReady(t *testing.T) { Status: &snapshotv1api.VolumeSnapshotStatus{ BoundVolumeSnapshotContentName: &vscName, ReadyToUse: boolptr.True(), - RestoreSize: &resource.Quantity{}, + RestoreSize: &quantity, }, } @@ -72,7 +81,7 @@ func TestWaitVolumeSnapshotReady(t *testing.T) { name: "get vs error", vsName: "fake-vs-1", namespace: "fake-ns-1", - err: "error to get volumesnapshot fake-ns-1/fake-vs-1: volumesnapshots.snapshot.storage.k8s.io \"fake-vs-1\" not found", + err: "error to get VolumeSnapshot fake-ns-1/fake-vs-1: volumesnapshots.snapshot.storage.k8s.io \"fake-vs-1\" not found", }, { name: "vs status is nil", @@ -187,9 +196,9 @@ func TestWaitVolumeSnapshotReady(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - fakeSnapshotClient := snapshotFake.NewSimpleClientset(test.clientObj...) + fakeClient := snapshotFake.NewSimpleClientset(test.clientObj...) - vs, err := WaitVolumeSnapshotReady(context.Background(), fakeSnapshotClient.SnapshotV1(), test.vsName, test.namespace, time.Millisecond, velerotest.NewLogger()) + vs, err := WaitVolumeSnapshotReady(context.Background(), fakeClient.SnapshotV1(), test.vsName, test.namespace, time.Millisecond, velerotest.NewLogger()) if err != nil { assert.EqualError(t, err, test.err) } else { @@ -273,9 +282,9 @@ func TestGetVolumeSnapshotContentForVolumeSnapshot(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - fakeSnapshotClient := snapshotFake.NewSimpleClientset(test.clientObj...) + fakeClient := snapshotFake.NewSimpleClientset(test.clientObj...) - vs, err := GetVolumeSnapshotContentForVolumeSnapshot(test.snapshotObj, fakeSnapshotClient.SnapshotV1()) + vs, err := GetVolumeSnapshotContentForVolumeSnapshot(test.snapshotObj, fakeClient.SnapshotV1()) if err != nil { assert.EqualError(t, err, test.err) } else { @@ -746,3 +755,992 @@ func TestRemoveVSCProtect(t *testing.T) { }) } } + +func TestGetVolumeSnapshotClass(t *testing.T) { + // backups + backupFoo := &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Annotations: map[string]string{ + "velero.io/csi-volumesnapshot-class_foo.csi.k8s.io": "foowithoutlabel", + }, + }, + Spec: velerov1api.BackupSpec{ + IncludedNamespaces: []string{"ns1", "ns2"}, + }, + } + backupFoo2 := &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo2", + Annotations: map[string]string{ + "velero.io/csi-volumesnapshot-class_foo.csi.k8s.io": "foo2", + }, + }, + Spec: velerov1api.BackupSpec{ + IncludedNamespaces: []string{"ns1", "ns2"}, + }, + } + + backupBar2 := &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + Annotations: map[string]string{ + "velero.io/csi-volumesnapshot-class_bar.csi.k8s.io": "bar2", + }, + }, + Spec: velerov1api.BackupSpec{ + IncludedNamespaces: []string{"ns1", "ns2"}, + }, + } + + backupNone := &velerov1api.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "none", + }, + Spec: velerov1api.BackupSpec{ + IncludedNamespaces: []string{"ns1", "ns2"}, + }, + } + + // pvcs + pvcFoo := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Annotations: map[string]string{ + "velero.io/csi-volumesnapshot-class": "foowithoutlabel", + }, + }, + Spec: v1.PersistentVolumeClaimSpec{}, + } + pvcFoo2 := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Annotations: map[string]string{ + "velero.io/csi-volumesnapshot-class": "foo2", + }, + }, + Spec: v1.PersistentVolumeClaimSpec{}, + } + pvcNone := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "none", + }, + Spec: v1.PersistentVolumeClaimSpec{}, + } + + // vsclasses + hostpathClass := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hostpath", + Labels: map[string]string{ + velerov1api.VolumeSnapshotClassSelectorLabel: "foo", + }, + }, + Driver: "hostpath.csi.k8s.io", + } + + fooClass := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Labels: map[string]string{ + velerov1api.VolumeSnapshotClassSelectorLabel: "foo", + }, + }, + Driver: "foo.csi.k8s.io", + } + fooClassWithoutLabel := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foowithoutlabel", + }, + Driver: "foo.csi.k8s.io", + } + + barClass := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + Labels: map[string]string{ + velerov1api.VolumeSnapshotClassSelectorLabel: "true", + }, + }, + Driver: "bar.csi.k8s.io", + } + + barClass2 := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bar2", + Labels: map[string]string{ + velerov1api.VolumeSnapshotClassSelectorLabel: "true", + }, + }, + Driver: "bar.csi.k8s.io", + } + + objs := []runtime.Object{hostpathClass, fooClass, barClass, fooClassWithoutLabel, barClass2} + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) + + testCases := []struct { + name string + driverName string + pvc *v1.PersistentVolumeClaim + backup *velerov1api.Backup + expectedVSC *snapshotv1api.VolumeSnapshotClass + expectError bool + }{ + { + name: "no annotations on pvc and backup, should find hostpath volumesnapshotclass using default behavior of labels", + driverName: "hostpath.csi.k8s.io", + pvc: pvcNone, + backup: backupNone, + expectedVSC: hostpathClass, + expectError: false, + }, + { + name: "foowithoutlabel VSC annotations on pvc", + driverName: "foo.csi.k8s.io", + pvc: pvcFoo, + backup: backupNone, + expectedVSC: fooClassWithoutLabel, + expectError: false, + }, + { + name: "foowithoutlabel VSC annotations on pvc, but csi driver does not match, no annotation on backup so fallback to default behavior of labels", + driverName: "bar.csi.k8s.io", + pvc: pvcFoo, + backup: backupNone, + expectedVSC: barClass, + expectError: false, + }, + { + name: "foowithoutlabel VSC annotations on pvc, but csi driver does not match so fallback to fetch from backupAnnotations ", + driverName: "bar.csi.k8s.io", + pvc: pvcFoo, + backup: backupBar2, + expectedVSC: barClass2, + expectError: false, + }, + { + name: "foowithoutlabel VSC annotations on backup for foo.csi.k8s.io", + driverName: "foo.csi.k8s.io", + pvc: pvcNone, + backup: backupFoo, + expectedVSC: fooClassWithoutLabel, + expectError: false, + }, + { + name: "foowithoutlabel VSC annotations on backup for bar.csi.k8s.io, no annotation corresponding to foo.csi.k8s.io, so fallback to default behavior of labels", + driverName: "bar.csi.k8s.io", + pvc: pvcNone, + backup: backupFoo, + expectedVSC: barClass, + expectError: false, + }, + { + name: "no snapshotClass for given driver", + driverName: "blah.csi.k8s.io", + pvc: pvcNone, + backup: backupNone, + expectedVSC: nil, + expectError: true, + }, + { + name: "foo2 VSC annotations on pvc, but doesn't exist in cluster, fallback to default behavior of labels", + driverName: "foo.csi.k8s.io", + pvc: pvcFoo2, + backup: backupFoo2, + expectedVSC: fooClass, + expectError: false, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualSnapshotClass, actualError := GetVolumeSnapshotClass( + tc.driverName, tc.backup, tc.pvc, logrus.New(), fakeClient) + if tc.expectError { + assert.NotNil(t, actualError) + assert.Nil(t, actualSnapshotClass) + return + } + assert.Equal(t, tc.expectedVSC, actualSnapshotClass) + }) + } +} + +func TestGetVolumeSnapshotClassForStorageClass(t *testing.T) { + hostpathClass := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hostpath", + Labels: map[string]string{ + velerov1api.VolumeSnapshotClassSelectorLabel: "foo", + }, + }, + Driver: "hostpath.csi.k8s.io", + } + + fooClass := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Labels: map[string]string{ + velerov1api.VolumeSnapshotClassSelectorLabel: "foo", + }, + }, + Driver: "foo.csi.k8s.io", + } + + barClass := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + Labels: map[string]string{ + velerov1api.VolumeSnapshotClassSelectorLabel: "foo", + }, + }, + Driver: "bar.csi.k8s.io", + } + + bazClass := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "baz", + }, + Driver: "baz.csi.k8s.io", + } + + ambClass1 := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "amb1", + }, + Driver: "amb.csi.k8s.io", + } + + ambClass2 := &snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "amb2", + }, + Driver: "amb.csi.k8s.io", + } + + snapshotClasses := &snapshotv1api.VolumeSnapshotClassList{ + Items: []snapshotv1api.VolumeSnapshotClass{ + *hostpathClass, *fooClass, *barClass, *bazClass, *ambClass1, *ambClass2}, + } + + testCases := []struct { + name string + driverName string + expectedVSC *snapshotv1api.VolumeSnapshotClass + expectError bool + }{ + { + name: "should find hostpath volumesnapshotclass", + driverName: "hostpath.csi.k8s.io", + expectedVSC: hostpathClass, + expectError: false, + }, + { + name: "should find foo volumesnapshotclass", + driverName: "foo.csi.k8s.io", + expectedVSC: fooClass, + expectError: false, + }, + { + name: "should find bar volumesnapshotclass", + driverName: "bar.csi.k8s.io", + expectedVSC: barClass, + expectError: false, + }, + { + name: "should find baz volumesnapshotclass without \"velero.io/csi-volumesnapshot-class\" label, b/c there's only one vsclass matching the driver name", + driverName: "baz.csi.k8s.io", + expectedVSC: bazClass, + expectError: false, + }, + { + name: "should not find amb volumesnapshotclass without \"velero.io/csi-volumesnapshot-class\" label, b/c there're more than one vsclass matching the driver name", + driverName: "amb.csi.k8s.io", + expectedVSC: nil, + expectError: true, + }, + { + name: "should not find does-not-exist volumesnapshotclass", + driverName: "not-found.csi.k8s.io", + expectedVSC: nil, + expectError: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualVSC, actualError := GetVolumeSnapshotClassForStorageClass(tc.driverName, snapshotClasses) + + if tc.expectError { + assert.NotNil(t, actualError) + assert.Nil(t, actualVSC) + return + } + + assert.Equalf(t, tc.expectedVSC.Name, actualVSC.Name, "unexpected volumesnapshotclass name returned. Want: %s; Got:%s", tc.expectedVSC.Name, actualVSC.Name) + assert.Equalf(t, tc.expectedVSC.Driver, actualVSC.Driver, "unexpected driver name returned. Want: %s; Got:%s", tc.expectedVSC.Driver, actualVSC.Driver) + }) + } +} + +func TestIsVolumeSnapshotClassHasListerSecret(t *testing.T) { + testCases := []struct { + name string + snapClass snapshotv1api.VolumeSnapshotClass + expected bool + }{ + { + name: "should find both annotations", + snapClass: snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "class-1", + Annotations: map[string]string{ + velerov1api.PrefixedListSecretNameAnnotation: "snapListSecret", + velerov1api.PrefixedListSecretNamespaceAnnotation: "awesome-ns", + }, + }, + }, + expected: true, + }, + { + name: "should not find both annotations name is missing", + snapClass: snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "class-1", + Annotations: map[string]string{ + "foo": "snapListSecret", + velerov1api.PrefixedListSecretNamespaceAnnotation: "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find both annotations namespace is missing", + snapClass: snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "class-1", + Annotations: map[string]string{ + velerov1api.PrefixedListSecretNameAnnotation: "snapListSecret", + "foo": "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find expected annotation non-empty annotation", + snapClass: snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "class-2", + Annotations: map[string]string{ + "foo": "snapListSecret", + "bar": "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find expected annotation nil annotation", + snapClass: snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "class-3", + Annotations: nil, + }, + }, + expected: false, + }, + { + name: "should not find expected annotation empty annotation", + snapClass: snapshotv1api.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "class-3", + Annotations: map[string]string{}, + }, + }, + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actual := IsVolumeSnapshotClassHasListerSecret(&tc.snapClass) + assert.Equal(t, tc.expected, actual) + }) + } +} + +func TestIsVolumeSnapshotContentHasDeleteSecret(t *testing.T) { + testCases := []struct { + name string + vsc snapshotv1api.VolumeSnapshotContent + expected bool + }{ + { + name: "should find both annotations", + vsc: snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-1", + Annotations: map[string]string{ + velerov1api.PrefixedSecretNameAnnotation: "delSnapSecret", + velerov1api.PrefixedSecretNamespaceAnnotation: "awesome-ns", + }, + }, + }, + expected: true, + }, + { + name: "should not find both annotations name is missing", + vsc: snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-2", + Annotations: map[string]string{ + "foo": "delSnapSecret", + velerov1api.PrefixedSecretNamespaceAnnotation: "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find both annotations namespace is missing", + vsc: snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-3", + Annotations: map[string]string{ + velerov1api.PrefixedSecretNameAnnotation: "delSnapSecret", + "foo": "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find expected annotation non-empty annotation", + vsc: snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-4", + Annotations: map[string]string{ + "foo": "delSnapSecret", + "bar": "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find expected annotation empty annotation", + vsc: snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-5", + Annotations: map[string]string{}, + }, + }, + expected: false, + }, + { + name: "should not find expected annotation nil annotation", + vsc: snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vsc-6", + Annotations: nil, + }, + }, + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actual := IsVolumeSnapshotContentHasDeleteSecret(&tc.vsc) + assert.Equal(t, tc.expected, actual) + }) + } +} + +func TestIsVolumeSnapshotHasVSCDeleteSecret(t *testing.T) { + testCases := []struct { + name string + vs snapshotv1api.VolumeSnapshot + expected bool + }{ + { + name: "should find both annotations", + vs: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-1", + Annotations: map[string]string{ + "velero.io/csi-deletesnapshotsecret-name": "snapDelSecret", + "velero.io/csi-deletesnapshotsecret-namespace": "awesome-ns", + }, + }, + }, + expected: true, + }, + { + name: "should not find both annotations name is missing", + vs: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-1", + Annotations: map[string]string{ + "foo": "snapDelSecret", + "velero.io/csi-deletesnapshotsecret-namespace": "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find both annotations namespace is missing", + vs: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-1", + Annotations: map[string]string{ + "velero.io/csi-deletesnapshotsecret-name": "snapDelSecret", + "foo": "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find annotation non-empty annotation", + vs: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-1", + Annotations: map[string]string{ + "foo": "snapDelSecret", + "bar": "awesome-ns", + }, + }, + }, + expected: false, + }, + { + name: "should not find annotation empty annotation", + vs: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-1", + Annotations: map[string]string{}, + }, + }, + expected: false, + }, + { + name: "should not find annotation nil annotation", + vs: snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-1", + Annotations: nil, + }, + }, + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actual := IsVolumeSnapshotHasVSCDeleteSecret(&tc.vs) + assert.Equal(t, tc.expected, actual) + }) + } +} + +func TestIsVolumeSnapshotExists(t *testing.T) { + vsExists := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-exists", + Namespace: "default", + }, + } + vsNotExists := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-does-not-exists", + Namespace: "default", + }, + } + + objs := []runtime.Object{vsExists} + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) + testCases := []struct { + name string + expected bool + vs *snapshotv1api.VolumeSnapshot + }{ + { + name: "should find existing VolumeSnapshot object", + expected: true, + vs: vsExists, + }, + { + name: "should not find non-existing VolumeSnapshot object", + expected: false, + vs: vsNotExists, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actual := IsVolumeSnapshotExists(tc.vs.Namespace, tc.vs.Name, fakeClient) + assert.Equal(t, tc.expected, actual) + }) + } +} + +func TestSetVolumeSnapshotContentDeletionPolicy(t *testing.T) { + testCases := []struct { + name string + inputVSCName string + objs []runtime.Object + expectError bool + }{ + { + name: "should update DeletionPolicy of a VSC from retain to delete", + inputVSCName: "retainVSC", + objs: []runtime.Object{ + &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "retainVSC", + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + DeletionPolicy: snapshotv1api.VolumeSnapshotContentRetain, + }, + }, + }, + expectError: false, + }, + { + name: "should be a no-op updating if DeletionPolicy of a VSC is already Delete", + inputVSCName: "deleteVSC", + objs: []runtime.Object{ + &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "deleteVSC", + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + DeletionPolicy: snapshotv1api.VolumeSnapshotContentDelete, + }, + }, + }, + expectError: false, + }, + { + name: "should update DeletionPolicy of a VSC with no DeletionPolicy", + inputVSCName: "nothingVSC", + objs: []runtime.Object{ + &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nothingVSC", + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{}, + }, + }, + expectError: false, + }, + { + name: "should return not found error if supplied VSC does not exist", + inputVSCName: "does-not-exist", + objs: []runtime.Object{}, + expectError: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, tc.objs...) + err := SetVolumeSnapshotContentDeletionPolicy(tc.inputVSCName, fakeClient) + if tc.expectError { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + actual := new(snapshotv1api.VolumeSnapshotContent) + err := fakeClient.Get( + context.TODO(), + crclient.ObjectKey{Name: tc.inputVSCName}, + actual, + ) + assert.Nil(t, err) + assert.Equal( + t, + snapshotv1api.VolumeSnapshotContentDelete, + actual.Spec.DeletionPolicy, + ) + } + }) + } +} + +func TestDeleteVolumeSnapshots(t *testing.T) { + tests := []struct { + name string + vs snapshotv1api.VolumeSnapshot + vsc snapshotv1api.VolumeSnapshotContent + expectedVS snapshotv1api.VolumeSnapshot + expectedVSC snapshotv1api.VolumeSnapshotContent + }{ + { + name: "VS is ReadyToUse, and VS has corresponding VSC. VS should be deleted.", + vs: *builder.ForVolumeSnapshot("velero", "vs1"). + ObjectMeta(builder.WithLabels("testing-vs", "vs1")). + Status().BoundVolumeSnapshotContentName("vsc1").Result(), + vsc: *builder.ForVolumeSnapshotContent("vsc1"). + DeletionPolicy(snapshotv1api.VolumeSnapshotContentDelete). + Status(&snapshotv1api.VolumeSnapshotContentStatus{}).Result(), + expectedVS: snapshotv1api.VolumeSnapshot{}, + expectedVSC: *builder.ForVolumeSnapshotContent("vsc1"). + DeletionPolicy(snapshotv1api.VolumeSnapshotContentRetain). + VolumeSnapshotRef("ns-", "name-").Result(), + }, + { + name: "VS status is nil. VSC should not be modified.", + vs: *builder.ForVolumeSnapshot("velero", "vs1"). + ObjectMeta(builder.WithLabels("testing-vs", "vs1")).Result(), + vsc: *builder.ForVolumeSnapshotContent("vsc1"). + DeletionPolicy(snapshotv1api.VolumeSnapshotContentDelete). + Status(&snapshotv1api.VolumeSnapshotContentStatus{}).Result(), + expectedVS: snapshotv1api.VolumeSnapshot{}, + expectedVSC: *builder.ForVolumeSnapshotContent("vsc1"). + DeletionPolicy(snapshotv1api.VolumeSnapshotContentDelete).Result(), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + client := velerotest.NewFakeControllerRuntimeClient( + t, + []runtime.Object{&tc.vs, &tc.vsc}..., + ) + logger := logging.DefaultLogger(logrus.DebugLevel, logging.FormatText) + backup := builder.ForBackup(velerov1api.DefaultNamespace, "backup-1"). + DefaultVolumesToFsBackup(false).Result() + + DeleteVolumeSnapshot(tc.vs, tc.vsc, backup, client, logger) + + vsList := new(snapshotv1api.VolumeSnapshotList) + err := client.List( + context.TODO(), + vsList, + &crclient.ListOptions{ + Namespace: "velero", + }, + ) + require.NoError(t, err) + if tc.expectedVS.Name == "" { + require.Empty(t, vsList.Items) + } else { + require.Equal(t, tc.expectedVS.Status, vsList.Items[0].Status) + require.Equal(t, tc.expectedVS.Spec, vsList.Items[0].Spec) + } + + vscList := new(snapshotv1api.VolumeSnapshotContentList) + err = client.List( + context.TODO(), + vscList, + ) + require.NoError(t, err) + require.Len(t, vscList.Items, 1) + require.Equal(t, tc.expectedVSC.Spec, vscList.Items[0].Spec) + }) + } +} + +func TestWaitUntilVSCHandleIsReady(t *testing.T) { + vscName := "snapcontent-7d1bdbd1-d10d-439c-8d8e-e1c2565ddc53" + snapshotHandle := "snapshot-handle" + vscObj := &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: vscName, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + VolumeSnapshotRef: v1.ObjectReference{ + Name: "vol-snap-1", + APIVersion: snapshotv1api.SchemeGroupVersion.String(), + }, + }, + Status: &snapshotv1api.VolumeSnapshotContentStatus{ + SnapshotHandle: &snapshotHandle, + }, + } + validVS := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs", + Namespace: "default", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &vscName, + }, + } + + notFound := "does-not-exist" + vsWithVSCNotFound := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: notFound, + Namespace: "default", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: ¬Found, + }, + } + + vsWithNilStatus := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nil-status-vs", + Namespace: "default", + }, + Status: nil, + } + vsWithNilStatusField := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nil-status-field-vs", + Namespace: "default", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: nil, + }, + } + + nilStatusVsc := "nil-status-vsc" + vscWithNilStatus := &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: nilStatusVsc, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + VolumeSnapshotRef: v1.ObjectReference{ + Name: "vol-snap-1", + APIVersion: snapshotv1api.SchemeGroupVersion.String(), + }, + }, + Status: nil, + } + vsForNilStatusVsc := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-for-nil-status-vsc", + Namespace: "default", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &nilStatusVsc, + }, + } + + nilStatusFieldVsc := "nil-status-field-vsc" + vscWithNilStatusField := &snapshotv1api.VolumeSnapshotContent{ + ObjectMeta: metav1.ObjectMeta{ + Name: nilStatusFieldVsc, + }, + Spec: snapshotv1api.VolumeSnapshotContentSpec{ + VolumeSnapshotRef: v1.ObjectReference{ + Name: "vol-snap-1", + APIVersion: snapshotv1api.SchemeGroupVersion.String(), + }, + }, + Status: &snapshotv1api.VolumeSnapshotContentStatus{ + SnapshotHandle: nil, + }, + } + vsForNilStatusFieldVsc := &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "vs-for-nil-status-field", + Namespace: "default", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &nilStatusFieldVsc, + }, + } + + objs := []runtime.Object{ + vscObj, + validVS, + vsWithVSCNotFound, + vsWithNilStatus, + vsWithNilStatusField, + vscWithNilStatus, + vsForNilStatusVsc, + vscWithNilStatusField, + vsForNilStatusFieldVsc, + } + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) + testCases := []struct { + name string + volSnap *snapshotv1api.VolumeSnapshot + exepctedVSC *snapshotv1api.VolumeSnapshotContent + wait bool + expectError bool + }{ + { + name: "waitEnabled should find volumesnapshotcontent for volumesnapshot", + volSnap: validVS, + exepctedVSC: vscObj, + wait: true, + expectError: false, + }, + { + name: "waitEnabled should not find volumesnapshotcontent for volumesnapshot with non-existing snapshotcontent name in status.BoundVolumeSnapshotContentName", + volSnap: vsWithVSCNotFound, + exepctedVSC: nil, + wait: true, + expectError: true, + }, + { + name: "waitEnabled should not find volumesnapshotcontent for a non-existent volumesnapshot", + wait: true, + exepctedVSC: nil, + expectError: true, + volSnap: &snapshotv1api.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "not-found", + Namespace: "default", + }, + Status: &snapshotv1api.VolumeSnapshotStatus{ + BoundVolumeSnapshotContentName: &nilStatusVsc, + }, + }, + }, + { + name: "waitDisabled should not find volumesnapshotcontent volumesnapshot status is nil", + wait: false, + expectError: false, + exepctedVSC: nil, + volSnap: vsWithNilStatus, + }, + { + name: "waitDisabled should not find volumesnapshotcontent volumesnapshot status.BoundVolumeSnapshotContentName is nil", + wait: false, + expectError: false, + exepctedVSC: nil, + volSnap: vsWithNilStatusField, + }, + { + name: "waitDisabled should find volumesnapshotcontent volumesnapshotcontent status is nil", + wait: false, + expectError: false, + exepctedVSC: vscWithNilStatus, + volSnap: vsForNilStatusVsc, + }, + { + name: "waitDisabled should find volumesnapshotcontent volumesnapshotcontent status.SnapshotHandle is nil", + wait: false, + expectError: false, + exepctedVSC: vscWithNilStatusField, + volSnap: vsForNilStatusFieldVsc, + }, + { + name: "waitDisabled should not find a non-existent volumesnapshotcontent", + wait: false, + exepctedVSC: nil, + expectError: true, + volSnap: vsWithVSCNotFound, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualVSC, actualError := WaitUntilVSCHandleIsReady(tc.volSnap, fakeClient, logrus.New().WithField("fake", "test"), tc.wait, 0) + if tc.expectError && actualError == nil { + assert.NotNil(t, actualError) + assert.Nil(t, actualVSC) + return + } + assert.Equal(t, tc.exepctedVSC, actualVSC) + }) + } +} diff --git a/pkg/util/kube/pvc_pv.go b/pkg/util/kube/pvc_pv.go index 32551334b..6f40d2029 100644 --- a/pkg/util/kube/pvc_pv.go +++ b/pkg/util/kube/pvc_pv.go @@ -27,11 +27,11 @@ import ( "github.com/sirupsen/logrus" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + crclient "sigs.k8s.io/controller-runtime/pkg/client" storagev1api "k8s.io/api/storage/v1" storagev1 "k8s.io/client-go/kubernetes/typed/storage/v1" @@ -358,3 +358,31 @@ func MakePodPVCAttachment(volumeName string, volumeMode *corev1api.PersistentVol return volumeMounts, volumeDevices } + +func GetPVForPVC( + pvc *corev1api.PersistentVolumeClaim, + crClient crclient.Client, +) (*corev1api.PersistentVolume, error) { + if pvc.Spec.VolumeName == "" { + return nil, errors.Errorf("PVC %s/%s has no volume backing this claim", + pvc.Namespace, pvc.Name) + } + if pvc.Status.Phase != corev1api.ClaimBound { + // TODO: confirm if this PVC should be snapshotted if it has no PV bound + return nil, + errors.Errorf("PVC %s/%s is in phase %v and is not bound to a volume", + pvc.Namespace, pvc.Name, pvc.Status.Phase) + } + + pv := &corev1api.PersistentVolume{} + err := crClient.Get( + context.TODO(), + crclient.ObjectKey{Name: pvc.Spec.VolumeName}, + pv, + ) + if err != nil { + return nil, errors.Wrapf(err, "failed to get PV %s for PVC %s/%s", + pvc.Spec.VolumeName, pvc.Namespace, pvc.Name) + } + return pv, nil +} diff --git a/pkg/util/kube/pvc_pv_test.go b/pkg/util/kube/pvc_pv_test.go index 660dae376..a2eeea346 100644 --- a/pkg/util/kube/pvc_pv_test.go +++ b/pkg/util/kube/pvc_pv_test.go @@ -29,6 +29,7 @@ import ( "k8s.io/client-go/kubernetes/fake" corev1api "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" storagev1api "k8s.io/api/storage/v1" clientTesting "k8s.io/client-go/testing" @@ -1119,3 +1120,139 @@ func TestIsPVCBound(t *testing.T) { }) } } + +var ( + csiStorageClass = "csi-hostpath-sc" +) + +func TestGetPVForPVC(t *testing.T) { + boundPVC := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-csi-pvc", + Namespace: "default", + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Resources: v1.VolumeResourceRequirements{ + Requests: v1.ResourceList{}, + }, + StorageClassName: &csiStorageClass, + VolumeName: "test-csi-7d28e566-ade7-4ed6-9e15-2e44d2fbcc08", + }, + Status: v1.PersistentVolumeClaimStatus{ + Phase: v1.ClaimBound, + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Capacity: v1.ResourceList{}, + }, + } + matchingPV := &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-csi-7d28e566-ade7-4ed6-9e15-2e44d2fbcc08", + }, + Spec: v1.PersistentVolumeSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Capacity: v1.ResourceList{}, + ClaimRef: &v1.ObjectReference{ + Kind: "PersistentVolumeClaim", + Name: "test-csi-pvc", + Namespace: "default", + ResourceVersion: "1027", + UID: "7d28e566-ade7-4ed6-9e15-2e44d2fbcc08", + }, + PersistentVolumeSource: v1.PersistentVolumeSource{ + CSI: &v1.CSIPersistentVolumeSource{ + Driver: "hostpath.csi.k8s.io", + FSType: "ext4", + VolumeAttributes: map[string]string{ + "storage.kubernetes.io/csiProvisionerIdentity": "1582049697841-8081-hostpath.csi.k8s.io", + }, + VolumeHandle: "e61f2b48-527a-11ea-b54f-cab6317018f1", + }, + }, + PersistentVolumeReclaimPolicy: v1.PersistentVolumeReclaimDelete, + StorageClassName: csiStorageClass, + }, + Status: v1.PersistentVolumeStatus{ + Phase: v1.VolumeBound, + }, + } + + pvcWithNoVolumeName := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "no-vol-pvc", + Namespace: "default", + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Resources: v1.VolumeResourceRequirements{ + Requests: v1.ResourceList{}, + }, + StorageClassName: &csiStorageClass, + }, + Status: v1.PersistentVolumeClaimStatus{}, + } + + unboundPVC := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "unbound-pvc", + Namespace: "default", + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Resources: v1.VolumeResourceRequirements{ + Requests: v1.ResourceList{}, + }, + StorageClassName: &csiStorageClass, + VolumeName: "test-csi-7d28e566-ade7-4ed6-9e15-2e44d2fbcc08", + }, + Status: v1.PersistentVolumeClaimStatus{ + Phase: v1.ClaimPending, + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Capacity: v1.ResourceList{}, + }, + } + + testCases := []struct { + name string + inPVC *v1.PersistentVolumeClaim + expectError bool + expectedPV *v1.PersistentVolume + }{ + { + name: "should find PV matching the PVC", + inPVC: boundPVC, + expectError: false, + expectedPV: matchingPV, + }, + { + name: "should fail to find PV for PVC with no volumeName", + inPVC: pvcWithNoVolumeName, + expectError: true, + expectedPV: nil, + }, + { + name: "should fail to find PV for PVC not in bound phase", + inPVC: unboundPVC, + expectError: true, + expectedPV: nil, + }, + } + + objs := []runtime.Object{boundPVC, matchingPV, pvcWithNoVolumeName, unboundPVC} + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualPV, actualError := GetPVForPVC(tc.inPVC, fakeClient) + + if tc.expectError { + assert.NotNil(t, actualError, "Want error; Got nil error") + assert.Nilf(t, actualPV, "Want PV: nil; Got PV: %q", actualPV) + return + } + + assert.Nilf(t, actualError, "Want: nil error; Got: %v", actualError) + assert.Equalf(t, actualPV.Name, tc.expectedPV.Name, "Want PV with name %q; Got PV with name %q", tc.expectedPV.Name, actualPV.Name) + }) + } +} diff --git a/pkg/util/kube/utils.go b/pkg/util/kube/utils.go index bda40d35c..3eb853829 100644 --- a/pkg/util/kube/utils.go +++ b/pkg/util/kube/utils.go @@ -19,6 +19,7 @@ package kube import ( "context" "fmt" + "strings" "time" "github.com/pkg/errors" @@ -35,6 +36,8 @@ import ( corev1client "k8s.io/client-go/kubernetes/typed/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) @@ -303,3 +306,30 @@ func IsCRDReady(crd *unstructured.Unstructured) (bool, error) { return false, fmt.Errorf("unable to handle CRD with version %s", ver) } } + +// AddAnnotations adds the supplied key-values to the annotations on the object +func AddAnnotations(o *metav1.ObjectMeta, vals map[string]string) { + if o.Annotations == nil { + o.Annotations = make(map[string]string) + } + for k, v := range vals { + o.Annotations[k] = v + } +} + +// AddLabels adds the supplied key-values to the labels on the object +func AddLabels(o *metav1.ObjectMeta, vals map[string]string) { + if o.Labels == nil { + o.Labels = make(map[string]string) + } + for k, v := range vals { + o.Labels[k] = label.GetValidName(v) + } +} + +func HasBackupLabel(o *metav1.ObjectMeta, backupName string) bool { + if o.Labels == nil || len(strings.TrimSpace(backupName)) == 0 { + return false + } + return o.Labels[velerov1api.BackupNameLabel] == label.GetValidName(backupName) +} diff --git a/pkg/util/kube/utils_test.go b/pkg/util/kube/utils_test.go index 141fb45ce..f811fdfcc 100644 --- a/pkg/util/kube/utils_test.go +++ b/pkg/util/kube/utils_test.go @@ -481,3 +481,187 @@ func TestSinglePathMatch(t *testing.T) { assert.NotNil(t, err) assert.Contains(t, err.Error(), "expected one matching path") } + +func TestAddAnnotations(t *testing.T) { + annotationValues := map[string]string{ + "k1": "v1", + "k2": "v2", + "k3": "v3", + "k4": "v4", + "k5": "v5", + } + testCases := []struct { + name string + o metav1.ObjectMeta + toAdd map[string]string + }{ + { + name: "should create a new annotation map when annotation is nil", + o: metav1.ObjectMeta{ + Annotations: nil, + }, + toAdd: annotationValues, + }, + { + name: "should add all supplied annotations into empty annotation", + o: metav1.ObjectMeta{ + Annotations: map[string]string{}, + }, + toAdd: annotationValues, + }, + { + name: "should add all supplied annotations to existing annotation", + o: metav1.ObjectMeta{ + Annotations: map[string]string{ + "k100": "v100", + "k200": "v200", + "k300": "v300", + }, + }, + toAdd: annotationValues, + }, + { + name: "should overwrite some existing annotations", + o: metav1.ObjectMeta{ + Annotations: map[string]string{ + "k100": "v100", + "k2": "v200", + "k300": "v300", + }, + }, + toAdd: annotationValues, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + AddAnnotations(&tc.o, tc.toAdd) + for k, v := range tc.toAdd { + actual, exists := tc.o.Annotations[k] + assert.True(t, exists) + assert.Equal(t, v, actual) + } + }) + } +} + +func TestAddLabels(t *testing.T) { + labelValues := map[string]string{ + "l1": "v1", + "l2": "v2", + "l3": "v3", + "l4": "v4", + "l5": "v5", + } + testCases := []struct { + name string + o metav1.ObjectMeta + toAdd map[string]string + }{ + { + name: "should create a new labels map when labels is nil", + o: metav1.ObjectMeta{ + Labels: nil, + }, + toAdd: labelValues, + }, + { + name: "should add all supplied labels into empty labels", + o: metav1.ObjectMeta{ + Labels: map[string]string{}, + }, + toAdd: labelValues, + }, + { + name: "should add all supplied labels to existing labels", + o: metav1.ObjectMeta{ + Labels: map[string]string{ + "l100": "v100", + "l200": "v200", + "l300": "v300", + }, + }, + toAdd: labelValues, + }, + { + name: "should overwrite some existing labels", + o: metav1.ObjectMeta{ + Labels: map[string]string{ + "l100": "v100", + "l2": "v200", + "l300": "v300", + }, + }, + toAdd: labelValues, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + AddLabels(&tc.o, tc.toAdd) + for k, v := range tc.toAdd { + actual, exists := tc.o.Labels[k] + assert.True(t, exists) + assert.Equal(t, v, actual) + } + }) + } +} + +func TestHasBackupLabel(t *testing.T) { + testCases := []struct { + name string + o metav1.ObjectMeta + backupName string + expected bool + }{ + { + name: "object has no labels", + o: metav1.ObjectMeta{}, + expected: false, + }, + { + name: "object has no velero backup label", + backupName: "csi-b1", + o: metav1.ObjectMeta{ + Labels: map[string]string{ + "l100": "v100", + "l2": "v200", + "l300": "v300", + }, + }, + expected: false, + }, + { + name: "object has velero backup label but value not equal to backup name", + backupName: "csi-b1", + o: metav1.ObjectMeta{ + Labels: map[string]string{ + "velero.io/backup-name": "does-not-match", + "l100": "v100", + "l2": "v200", + "l300": "v300", + }, + }, + expected: false, + }, + { + name: "object has backup label with matching backup name value", + backupName: "does-match", + o: metav1.ObjectMeta{ + Labels: map[string]string{ + "velero.io/backup-name": "does-match", + "l100": "v100", + "l2": "v200", + "l300": "v300", + }, + }, + expected: true, + }, + } + + for _, tc := range testCases { + actual := HasBackupLabel(&tc.o, tc.backupName) + assert.Equal(t, tc.expected, actual) + } +} diff --git a/pkg/util/podvolume/pod_volume.go b/pkg/util/podvolume/pod_volume.go index a4107e6c2..20f991930 100644 --- a/pkg/util/podvolume/pod_volume.go +++ b/pkg/util/podvolume/pod_volume.go @@ -17,12 +17,16 @@ limitations under the License. package podvolume import ( + "context" "strings" + "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + crclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/util" ) // GetVolumesByPod returns a list of volume names to backup for the provided pod. @@ -62,7 +66,7 @@ func GetVolumesByPod(pod *corev1api.Pod, defaultVolumesToFsBackup, backupExclude continue } // don't backup volumes that are included in the exclude list. - if contains(volsToExclude, pv.Name) { + if util.Contains(volsToExclude, pv.Name) { optedOutVolumes = append(optedOutVolumes, pv.Name) continue } @@ -101,11 +105,58 @@ func getVolumesToExclude(obj metav1.Object) []string { return strings.Split(annotations[velerov1api.VolumesToExcludeAnnotation], ",") } -func contains(list []string, k string) bool { - for _, i := range list { - if i == k { - return true +func IsPVCDefaultToFSBackup(pvcNamespace, pvcName string, crClient crclient.Client, defaultVolumesToFsBackup bool) (bool, error) { + pods, err := getPodsUsingPVC(pvcNamespace, pvcName, crClient) + if err != nil { + return false, errors.WithStack(err) + } + + for index := range pods { + vols, _ := GetVolumesByPod(&pods[index], defaultVolumesToFsBackup, false) + if len(vols) > 0 { + volName, err := getPodVolumeNameForPVC(pods[index], pvcName) + if err != nil { + return false, err + } + if util.Contains(vols, volName) { + return true, nil + } } } - return false + + return false, nil +} + +func getPodVolumeNameForPVC(pod corev1api.Pod, pvcName string) (string, error) { + for _, v := range pod.Spec.Volumes { + if v.PersistentVolumeClaim != nil && v.PersistentVolumeClaim.ClaimName == pvcName { + return v.Name, nil + } + } + return "", errors.Errorf("Pod %s/%s does not use PVC %s/%s", pod.Namespace, pod.Name, pod.Namespace, pvcName) +} + +func getPodsUsingPVC( + pvcNamespace, pvcName string, + crClient crclient.Client, +) ([]corev1api.Pod, error) { + podsUsingPVC := []corev1api.Pod{} + podList := new(corev1api.PodList) + if err := crClient.List( + context.TODO(), + podList, + &crclient.ListOptions{Namespace: pvcNamespace}, + ); err != nil { + return nil, err + } + + for _, p := range podList.Items { + for _, v := range p.Spec.Volumes { + if v.PersistentVolumeClaim != nil && v.PersistentVolumeClaim.ClaimName == pvcName { + podsUsingPVC = append(podsUsingPVC, p) + } + } + } + + return podsUsingPVC, nil } diff --git a/pkg/util/podvolume/pod_volume_test.go b/pkg/util/podvolume/pod_volume_test.go index 19174e434..3bebb3cf6 100644 --- a/pkg/util/podvolume/pod_volume_test.go +++ b/pkg/util/podvolume/pod_volume_test.go @@ -22,9 +22,12 @@ import ( "github.com/stretchr/testify/assert" corev1api "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerotest "github.com/vmware-tanzu/velero/pkg/test" ) func TestGetVolumesToBackup(t *testing.T) { @@ -378,3 +381,414 @@ func TestGetVolumesByPod(t *testing.T) { }) } } + +func TestIsPVCDefaultToFSBackup(t *testing.T) { + objs := []runtime.Object{ + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "default", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "default", + Annotations: map[string]string{ + "backup.velero.io/backup-volumes": "csi-vol1", + }, + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod3", + Namespace: "default", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "awesome-pod-1", + Namespace: "awesome-ns", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "awesome-csi-pvc1", + }, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "awesome-pod-2", + Namespace: "awesome-ns", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "awesome-csi-pvc1", + }, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "uploader-ns", + Annotations: map[string]string{ + "backup.velero.io/backup-volumes": "csi-vol1", + }, + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "uploader-ns", + Annotations: map[string]string{ + "backup.velero.io/backup-volumes": "csi-vol1", + }, + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + } + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) + + testCases := []struct { + name string + inPVCNamespace string + inPVCName string + expectedIsFSUploaderUsed bool + defaultVolumesToFSBackup bool + }{ + { + name: "2 pods using PVC, 1 pod using uploader", + inPVCNamespace: "default", + inPVCName: "csi-pvc1", + expectedIsFSUploaderUsed: true, + defaultVolumesToFSBackup: false, + }, + { + name: "2 pods using PVC, 2 pods using uploader", + inPVCNamespace: "uploader-ns", + inPVCName: "csi-pvc1", + expectedIsFSUploaderUsed: true, + defaultVolumesToFSBackup: false, + }, + { + name: "2 pods using PVC, 0 pods using uploader", + inPVCNamespace: "awesome-ns", + inPVCName: "awesome-csi-pvc1", + expectedIsFSUploaderUsed: false, + defaultVolumesToFSBackup: false, + }, + { + name: "0 pods using PVC", + inPVCNamespace: "default", + inPVCName: "does-not-exist", + expectedIsFSUploaderUsed: false, + defaultVolumesToFSBackup: false, + }, + { + name: "2 pods using PVC, using uploader by default", + inPVCNamespace: "awesome-ns", + inPVCName: "awesome-csi-pvc1", + expectedIsFSUploaderUsed: true, + defaultVolumesToFSBackup: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualIsFSUploaderUsed, _ := IsPVCDefaultToFSBackup(tc.inPVCNamespace, tc.inPVCName, fakeClient, tc.defaultVolumesToFSBackup) + assert.Equal(t, tc.expectedIsFSUploaderUsed, actualIsFSUploaderUsed) + }) + } +} + +func TestGetPodVolumeNameForPVC(t *testing.T) { + testCases := []struct { + name string + pod v1.Pod + pvcName string + expectError bool + expectedVolumeName string + }{ + { + name: "should get volume name for pod with multuple PVCs", + pod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + { + Name: "csi-vol2", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc2", + }, + }, + }, + { + Name: "csi-vol3", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc3", + }, + }, + }, + }, + }, + }, + pvcName: "csi-pvc2", + expectedVolumeName: "csi-vol2", + expectError: false, + }, + { + name: "should get volume name from pod using exactly one PVC", + pod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + pvcName: "csi-pvc1", + expectedVolumeName: "csi-vol1", + expectError: false, + }, + { + name: "should return error for pod with no PVCs", + pod: v1.Pod{ + Spec: v1.PodSpec{}, + }, + pvcName: "csi-pvc2", + expectError: true, + }, + { + name: "should return error for pod with no matching PVC", + pod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + pvcName: "mismatch-pvc", + expectError: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualVolumeName, err := getPodVolumeNameForPVC(tc.pod, tc.pvcName) + if tc.expectError && err == nil { + assert.NotNil(t, err, "Want error; Got nil error") + return + } + assert.Equalf(t, tc.expectedVolumeName, actualVolumeName, "unexpected podVolumename returned. Want %s; Got %s", tc.expectedVolumeName, actualVolumeName) + }) + } +} + +func TestGetPodsUsingPVC(t *testing.T) { + objs := []runtime.Object{ + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "default", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: "default", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod3", + Namespace: "default", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + }, + }, + }, + &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: "awesome-ns", + }, + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + { + Name: "csi-vol1", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "csi-pvc1", + }, + }, + }, + }, + }, + }, + } + fakeClient := velerotest.NewFakeControllerRuntimeClient(t, objs...) + + testCases := []struct { + name string + pvcNamespace string + pvcName string + expectedPodCount int + }{ + { + name: "should find exactly 2 pods using the PVC", + pvcNamespace: "default", + pvcName: "csi-pvc1", + expectedPodCount: 2, + }, + { + name: "should find exactly 1 pod using the PVC", + pvcNamespace: "awesome-ns", + pvcName: "csi-pvc1", + expectedPodCount: 1, + }, + { + name: "should find 0 pods using the PVC", + pvcNamespace: "default", + pvcName: "unused-pvc", + expectedPodCount: 0, + }, + { + name: "should find 0 pods in non-existent namespace", + pvcNamespace: "does-not-exist", + pvcName: "csi-pvc1", + expectedPodCount: 0, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualPods, err := getPodsUsingPVC(tc.pvcNamespace, tc.pvcName, fakeClient) + assert.Nilf(t, err, "Want error=nil; Got error=%v", err) + assert.Lenf(t, actualPods, tc.expectedPodCount, "unexpected number of pods in result; Want: %d; Got: %d", tc.expectedPodCount, len(actualPods)) + }) + } +} diff --git a/pkg/util/scheme.go b/pkg/util/scheme.go index 7e5703fac..c398e4196 100644 --- a/pkg/util/scheme.go +++ b/pkg/util/scheme.go @@ -1,3 +1,19 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package util import ( diff --git a/pkg/util/util.go b/pkg/util/util.go new file mode 100644 index 000000000..2aad8ccb1 --- /dev/null +++ b/pkg/util/util.go @@ -0,0 +1,26 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +func Contains(slice []string, key string) bool { + for _, i := range slice { + if i == key { + return true + } + } + return false +} diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go new file mode 100644 index 000000000..5eed5195f --- /dev/null +++ b/pkg/util/util_test.go @@ -0,0 +1,64 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestContains(t *testing.T) { + testCases := []struct { + name string + inSlice []string + inKey string + expectedResult bool + }{ + { + name: "should find the key", + inSlice: []string{"key1", "key2", "key3", "key4", "key5"}, + inKey: "key3", + expectedResult: true, + }, + { + name: "should not find the key in non-empty slice", + inSlice: []string{"key1", "key2", "key3", "key4", "key5"}, + inKey: "key300", + expectedResult: false, + }, + { + name: "should not find key in empty slice", + inSlice: []string{}, + inKey: "key300", + expectedResult: false, + }, + { + name: "should not find key in nil slice", + inSlice: nil, + inKey: "key300", + expectedResult: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actualResult := Contains(tc.inSlice, tc.inKey) + assert.Equal(t, tc.expectedResult, actualResult) + }) + } +} diff --git a/test/e2e/resource-filtering/exclude_label.go b/test/e2e/resource-filtering/exclude_label.go index afa3f2b0d..ecf26878e 100644 --- a/test/e2e/resource-filtering/exclude_label.go +++ b/test/e2e/resource-filtering/exclude_label.go @@ -26,6 +26,7 @@ import ( "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" . "github.com/vmware-tanzu/velero/test/e2e/test" . "github.com/vmware-tanzu/velero/test/util/k8s" ) @@ -57,9 +58,9 @@ func (e *ExcludeFromBackup) Init() error { *e.NSIncluded = append(*e.NSIncluded, createNSName) } e.labels = map[string]string{ - "velero.io/exclude-from-backup": "true", + velerov1api.ExcludeFromBackupLabel: "true", } - e.labelSelector = "velero.io/exclude-from-backup" + e.labelSelector = velerov1api.ExcludeFromBackupLabel e.BackupArgs = []string{ "create", "--namespace", e.VeleroCfg.VeleroNamespace, "backup", e.BackupName, @@ -82,7 +83,7 @@ func (e *ExcludeFromBackup) CreateResources() error { "meaningless-label-resource-to-include": "true", } label2 := map[string]string{ - "velero.io/exclude-from-backup": "false", + velerov1api.ExcludeFromBackupLabel: "false", } fmt.Printf("Creating resources in namespace ...%s\n", namespace) if err := CreateNamespace(e.Ctx, e.Client, namespace); err != nil { From 474dc824e76643bc94c1c254f46df30baa2e2edd Mon Sep 17 00:00:00 2001 From: clonefetch <166312854+clonefetch@users.noreply.github.com> Date: Wed, 10 Apr 2024 17:45:29 +0800 Subject: [PATCH 15/16] chore: fix function names in comment (#7633) Signed-off-by: clonefetch --- pkg/backup/backup_test.go | 4 ++-- pkg/builder/v1_customresourcedefinition_builder.go | 2 +- pkg/cmd/util/output/backup_structured_describer.go | 2 +- pkg/restore/actions/change_image_name_action_test.go | 2 +- pkg/restore/restore_test.go | 2 +- pkg/test/fake_credential_file_store.go | 2 +- pkg/uploader/kopia/shim.go | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/backup/backup_test.go b/pkg/backup/backup_test.go index 0a8c7db78..6be7c9422 100644 --- a/pkg/backup/backup_test.go +++ b/pkg/backup/backup_test.go @@ -164,7 +164,7 @@ func TestBackupProgressIsUpdated(t *testing.T) { assert.Len(t, req.BackedUpItems, req.Status.Progress.ItemsBackedUp) } -// TestBackupResourceFiltering runs backups with different combinations +// TestBackupOldResourceFiltering runs backups with different combinations // of resource filters (included/excluded resources, included/excluded // namespaces, label selectors, "include cluster resources" flag), and // verifies that the set of items written to the backup tarball are @@ -1430,7 +1430,7 @@ func TestBackupItemActionsForSkippedPV(t *testing.T) { } } -// TestBackupActionsRunsForCorrectItems runs backups with backup item actions, and +// TestBackupActionsRunForCorrectItems runs backups with backup item actions, and // verifies that each backup item action is run for the correct set of resources based on its // AppliesTo() resource selector. Verification is done by using the recordResourcesAction struct, // which records which resources it's executed for. diff --git a/pkg/builder/v1_customresourcedefinition_builder.go b/pkg/builder/v1_customresourcedefinition_builder.go index 4bb20590a..1962bf309 100644 --- a/pkg/builder/v1_customresourcedefinition_builder.go +++ b/pkg/builder/v1_customresourcedefinition_builder.go @@ -78,7 +78,7 @@ type V1CustomResourceDefinitionConditionBuilder struct { object apiextv1.CustomResourceDefinitionCondition } -// ForV1V1CustomResourceDefinitionConditionBuilder is the constructor for a V1CustomResourceDefinitionConditionBuilder. +// ForV1CustomResourceDefinitionCondition is the constructor for a V1CustomResourceDefinitionConditionBuilder. func ForV1CustomResourceDefinitionCondition() *V1CustomResourceDefinitionConditionBuilder { return &V1CustomResourceDefinitionConditionBuilder{ object: apiextv1.CustomResourceDefinitionCondition{}, diff --git a/pkg/cmd/util/output/backup_structured_describer.go b/pkg/cmd/util/output/backup_structured_describer.go index 751991356..20d6371c0 100644 --- a/pkg/cmd/util/output/backup_structured_describer.go +++ b/pkg/cmd/util/output/backup_structured_describer.go @@ -410,7 +410,7 @@ func describeCSISnapshotInSF(details bool, info *volume.VolumeInfo, snapshotDeta snapshotDetails[fmt.Sprintf("%s/%s", info.PVCNamespace, info.PVCName)] = snapshotDetail } -// describeVSCInSF describes CSI volume snapshot contents in structured format. +// describeLocalSnapshotInSF describes CSI volume snapshot contents in structured format. func describeLocalSnapshotInSF(details bool, info *volume.VolumeInfo, snapshotDetail map[string]interface{}) { if !info.PreserveLocalSnapshot { return diff --git a/pkg/restore/actions/change_image_name_action_test.go b/pkg/restore/actions/change_image_name_action_test.go index d851e7c0d..37130006a 100644 --- a/pkg/restore/actions/change_image_name_action_test.go +++ b/pkg/restore/actions/change_image_name_action_test.go @@ -33,7 +33,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/plugin/velero" ) -// TestChangeImageNameActionExecute runs the ChangeImageNameAction's Execute +// TestChangeImageRepositoryActionExecute runs the ChangeImageNameAction's Execute // method and validates that the item's image name is modified (or not) as expected. // Validation is done by comparing the result of the Execute method to the test case's // desired result. diff --git a/pkg/restore/restore_test.go b/pkg/restore/restore_test.go index dd9138237..f098537e2 100644 --- a/pkg/restore/restore_test.go +++ b/pkg/restore/restore_test.go @@ -1491,7 +1491,7 @@ func (a *recordResourcesAction) WithAdditionalItems(items []velero.ResourceIdent return a } -// TestRestoreActionsRunsForCorrectItems runs restores with restore item actions, and +// TestRestoreActionsRunForCorrectItems runs restores with restore item actions, and // verifies that each restore item action is run for the correct set of resources based on its // AppliesTo() resource selector. Verification is done by using the recordResourcesAction struct, // which records which resources it's executed for. diff --git a/pkg/test/fake_credential_file_store.go b/pkg/test/fake_credential_file_store.go index 1534a7115..af6618521 100644 --- a/pkg/test/fake_credential_file_store.go +++ b/pkg/test/fake_credential_file_store.go @@ -39,7 +39,7 @@ func (f *fakeCredentialsFileStore) Path(*corev1api.SecretKeySelector) (string, e return f.path, f.err } -// NewFakeCredentialFileStore creates a FileStore which will return the given path +// NewFakeCredentialsFileStore creates a FileStore which will return the given path // and error when Path is called. func NewFakeCredentialsFileStore(path string, err error) FileStore { return &fakeCredentialsFileStore{ diff --git a/pkg/uploader/kopia/shim.go b/pkg/uploader/kopia/shim.go index b2e5e942c..b90837d19 100644 --- a/pkg/uploader/kopia/shim.go +++ b/pkg/uploader/kopia/shim.go @@ -95,7 +95,7 @@ func (sr *shimRepository) FindManifests(ctx context.Context, labels map[string]s return GetKopiaManifestEntries(metadata), nil } -// GetKopiaManifestEntries get metadata from specific ManifestEntryMetadata +// GetKopiaManifestEntry get metadata from specific ManifestEntryMetadata func GetKopiaManifestEntry(uMani *udmrepo.ManifestEntryMetadata) *manifest.EntryMetadata { var ret manifest.EntryMetadata From 89967c1cb65953fad8b21a5f9d952e022e45b7dc Mon Sep 17 00:00:00 2001 From: Ming Qiu Date: Thu, 11 Apr 2024 02:53:29 +0000 Subject: [PATCH 16/16] Add data download parallel files download configuration Signed-off-by: Ming Qiu --- pkg/restore/actions/csi/pvc_action.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pkg/restore/actions/csi/pvc_action.go b/pkg/restore/actions/csi/pvc_action.go index 1f75889f9..19c687e8b 100644 --- a/pkg/restore/actions/csi/pvc_action.go +++ b/pkg/restore/actions/csi/pvc_action.go @@ -481,12 +481,7 @@ func newDataDownload( }, } if restore.Spec.UploaderConfig != nil { - dataDownload.Spec.DataMoverConfig = make(map[string]string) - if boolptr.IsSetToTrue(restore.Spec.UploaderConfig.WriteSparseFiles) { - dataDownload.Spec.DataMoverConfig[uploaderUtil.WriteSparseFiles] = "true" - } else { - dataDownload.Spec.DataMoverConfig[uploaderUtil.WriteSparseFiles] = "false" - } + dataDownload.Spec.DataMoverConfig = uploaderUtil.StoreRestoreConfig(restore.Spec.UploaderConfig) } return dataDownload }