From 5677e04bb19ffd3547f1cdcc8c250e9ab59afa9a Mon Sep 17 00:00:00 2001 From: "David L. Smith-Uchida" Date: Fri, 10 Dec 2021 09:53:47 -0800 Subject: [PATCH] Consolidated code for resolving actions and plugins into ActionResolver (#4410) * Consolidated code for resolving actions and plugins into ActionResolver. Added BackupWithResolvers and RestoreWithResolvers. Introduces ItemSnapshooterResolver to bring ItemSnapshotter plugins into backup and restore. ItemSnapshotters are not used yet. Added action_resolver_test Signed-off-by: Dave Smith-Uchida * Addressed review comments Signed-off-by: Dave Smith-Uchida --- changelogs/unreleased/4410-dsu | 2 + internal/delete/delete_item_action_handler.go | 75 +----- pkg/backup/backup.go | 67 ++--- pkg/backup/item_backupper.go | 22 +- pkg/backup/request.go | 11 +- pkg/controller/backup_controller.go | 11 +- pkg/controller/backup_controller_test.go | 10 + pkg/controller/restore_controller.go | 11 +- pkg/controller/restore_controller_test.go | 20 +- pkg/plugin/framework/action_resolver.go | 242 ++++++++++++++++++ pkg/plugin/framework/action_resolver_test.go | 93 +++++++ pkg/plugin/velero/shared.go | 8 +- pkg/restore/restore.go | 104 ++++---- pkg/restore/restore_test.go | 14 +- pkg/test/fake_mapper.go | 10 + 15 files changed, 498 insertions(+), 202 deletions(-) create mode 100644 changelogs/unreleased/4410-dsu create mode 100644 pkg/plugin/framework/action_resolver.go create mode 100644 pkg/plugin/framework/action_resolver_test.go diff --git a/changelogs/unreleased/4410-dsu b/changelogs/unreleased/4410-dsu new file mode 100644 index 000000000..7762509c0 --- /dev/null +++ b/changelogs/unreleased/4410-dsu @@ -0,0 +1,2 @@ +Added BackupWithResolvers and RestoreWithResolvers calls. Will eventually replace Backup and Restore methods. +Adds ItemSnapshotters to Backup and Restore workflows. diff --git a/internal/delete/delete_item_action_handler.go b/internal/delete/delete_item_action_handler.go index c8545c345..b2ab39ead 100644 --- a/internal/delete/delete_item_action_handler.go +++ b/internal/delete/delete_item_action_handler.go @@ -19,6 +19,8 @@ package delete import ( "io" + "github.com/vmware-tanzu/velero/pkg/plugin/framework" + "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/labels" @@ -29,7 +31,6 @@ import ( "github.com/vmware-tanzu/velero/pkg/archive" "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/plugin/velero" - "github.com/vmware-tanzu/velero/pkg/util/collections" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) @@ -41,14 +42,13 @@ type Context struct { Filesystem filesystem.Interface Log logrus.FieldLogger DiscoveryHelper discovery.Helper - - resolvedActions []resolvedAction + resolvedActions []framework.DeleteItemResolvedAction } func InvokeDeleteActions(ctx *Context) error { var err error - ctx.resolvedActions, err = resolveActions(ctx.Actions, ctx.DiscoveryHelper) - + resolver := framework.NewDeleteItemActionResolver(ctx.Actions) + ctx.resolvedActions, err = resolver.ResolveActions(ctx.DiscoveryHelper) // No actions installed and no error means we don't have to continue; // just do the backup deletion without worrying about plugins. if len(ctx.resolvedActions) == 0 && err == nil { @@ -119,10 +119,10 @@ func InvokeDeleteActions(ctx *Context) error { itemLog.Infof("invoking DeleteItemAction plugins") for _, action := range actions { - if !action.selector.Matches(labels.Set(obj.GetLabels())) { + if !action.Selector.Matches(labels.Set(obj.GetLabels())) { continue } - err = action.Execute(&velero.DeleteItemActionExecuteInput{ + err = action.DeleteItemAction.Execute(&velero.DeleteItemActionExecuteInput{ Item: obj, Backup: ctx.Backup, }) @@ -139,65 +139,12 @@ func InvokeDeleteActions(ctx *Context) error { } // getApplicableActions takes resolved DeleteItemActions and filters them for a given group/resource and namespace. -func (ctx *Context) getApplicableActions(groupResource schema.GroupResource, namespace string) []resolvedAction { - var actions []resolvedAction - +func (ctx *Context) getApplicableActions(groupResource schema.GroupResource, namespace string) []framework.DeleteItemResolvedAction { + var actions []framework.DeleteItemResolvedAction for _, action := range ctx.resolvedActions { - if !action.resourceIncludesExcludes.ShouldInclude(groupResource.String()) { - continue + if action.ShouldUse(groupResource, namespace, nil, ctx.Log) { + actions = append(actions, action) } - - if namespace != "" && !action.namespaceIncludesExcludes.ShouldInclude(namespace) { - continue - } - - if namespace == "" && !action.namespaceIncludesExcludes.IncludeEverything() { - continue - } - - actions = append(actions, action) } - return actions } - -// resolvedActions are DeleteItemActions decorated with resource/namespace include/exclude collections, as well as label selectors for easy comparison. -type resolvedAction struct { - velero.DeleteItemAction - - resourceIncludesExcludes *collections.IncludesExcludes - namespaceIncludesExcludes *collections.IncludesExcludes - selector labels.Selector -} - -// resolveActions resolves the AppliesTo ResourceSelectors of DeleteItemActions plugins against the Kubernetes discovery API for fully-qualified names. -func resolveActions(actions []velero.DeleteItemAction, helper discovery.Helper) ([]resolvedAction, error) { - var resolved []resolvedAction - - for _, action := range actions { - resourceSelector, err := action.AppliesTo() - if err != nil { - return nil, err - } - - resources := collections.GetResourceIncludesExcludes(helper, resourceSelector.IncludedResources, resourceSelector.ExcludedResources) - namespaces := collections.NewIncludesExcludes().Includes(resourceSelector.IncludedNamespaces...).Excludes(resourceSelector.ExcludedNamespaces...) - - selector := labels.Everything() - if resourceSelector.LabelSelector != "" { - if selector, err = labels.Parse(resourceSelector.LabelSelector); err != nil { - return nil, err - } - } - - res := resolvedAction{ - DeleteItemAction: action, - resourceIncludesExcludes: resources, - namespaceIncludesExcludes: namespaces, - selector: selector, - } - resolved = append(resolved, res) - } - - return resolved, nil -} diff --git a/pkg/backup/backup.go b/pkg/backup/backup.go index 746695e00..ef03e78f7 100644 --- a/pkg/backup/backup.go +++ b/pkg/backup/backup.go @@ -33,7 +33,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" kubeerrs "k8s.io/apimachinery/pkg/util/errors" @@ -44,6 +43,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/discovery" velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" "github.com/vmware-tanzu/velero/pkg/kuberesource" + "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/podexec" "github.com/vmware-tanzu/velero/pkg/restic" @@ -62,6 +62,9 @@ type Backupper interface { // Backup takes a backup using the specification in the velerov1api.Backup and writes backup and log data // to the given writers. Backup(logger logrus.FieldLogger, backup *Request, backupFile io.Writer, actions []velero.BackupItemAction, volumeSnapshotterGetter VolumeSnapshotterGetter) error + BackupWithResolvers(log logrus.FieldLogger, backupRequest *Request, backupFile io.Writer, + backupItemActionResolver framework.BackupItemActionResolver, itemSnapshotterResolver framework.ItemSnapshotterResolver, + volumeSnapshotterGetter VolumeSnapshotterGetter) error } // kubernetesBackupper implements Backupper. @@ -76,14 +79,6 @@ type kubernetesBackupper struct { clientPageSize int } -type resolvedAction struct { - velero.BackupItemAction - - resourceIncludesExcludes *collections.IncludesExcludes - namespaceIncludesExcludes *collections.IncludesExcludes - selector labels.Selector -} - func (i *itemKey) String() string { return fmt.Sprintf("resource=%s,namespace=%s,name=%s", i.resource, i.namespace, i.name) } @@ -121,38 +116,6 @@ func NewKubernetesBackupper( }, nil } -func resolveActions(actions []velero.BackupItemAction, helper discovery.Helper) ([]resolvedAction, error) { - var resolved []resolvedAction - - for _, action := range actions { - resourceSelector, err := action.AppliesTo() - if err != nil { - return nil, err - } - - resources := collections.GetResourceIncludesExcludes(helper, resourceSelector.IncludedResources, resourceSelector.ExcludedResources) - namespaces := collections.NewIncludesExcludes().Includes(resourceSelector.IncludedNamespaces...).Excludes(resourceSelector.ExcludedNamespaces...) - - selector := labels.Everything() - if resourceSelector.LabelSelector != "" { - if selector, err = labels.Parse(resourceSelector.LabelSelector); err != nil { - return nil, err - } - } - - res := resolvedAction{ - BackupItemAction: action, - resourceIncludesExcludes: resources, - namespaceIncludesExcludes: namespaces, - selector: selector, - } - - resolved = append(resolved, res) - } - - return resolved, nil -} - // getNamespaceIncludesExcludes returns an IncludesExcludes list containing which namespaces to // include and exclude from the backup. func getNamespaceIncludesExcludes(backup *velerov1api.Backup) *collections.IncludesExcludes { @@ -205,7 +168,20 @@ type VolumeSnapshotterGetter interface { // a complete backup failure is returned. Errors that constitute partial failures (i.e. failures to // back up individual resources that don't prevent the backup from continuing to be processed) are logged // to the backup log. -func (kb *kubernetesBackupper) Backup(log logrus.FieldLogger, backupRequest *Request, backupFile io.Writer, actions []velero.BackupItemAction, volumeSnapshotterGetter VolumeSnapshotterGetter) error { +func (kb *kubernetesBackupper) Backup(log logrus.FieldLogger, backupRequest *Request, backupFile io.Writer, + actions []velero.BackupItemAction, volumeSnapshotterGetter VolumeSnapshotterGetter) error { + backupItemActions := framework.NewBackupItemActionResolver(actions) + itemSnapshotters := framework.NewItemSnapshotterResolver(nil) + return kb.BackupWithResolvers(log, backupRequest, backupFile, backupItemActions, itemSnapshotters, + volumeSnapshotterGetter) +} + +func (kb *kubernetesBackupper) BackupWithResolvers(log logrus.FieldLogger, + backupRequest *Request, + backupFile io.Writer, + backupItemActionResolver framework.BackupItemActionResolver, + itemSnapshotterResolver framework.ItemSnapshotterResolver, + volumeSnapshotterGetter VolumeSnapshotterGetter) error { gzippedData := gzip.NewWriter(backupFile) defer gzippedData.Close() @@ -232,7 +208,12 @@ func (kb *kubernetesBackupper) Backup(log logrus.FieldLogger, backupRequest *Req return err } - backupRequest.ResolvedActions, err = resolveActions(actions, kb.discoveryHelper) + backupRequest.ResolvedActions, err = backupItemActionResolver.ResolveActions(kb.discoveryHelper) + if err != nil { + return err + } + + backupRequest.ResolvedItemSnapshotters, err = itemSnapshotterResolver.ResolveActions(kb.discoveryHelper) if err != nil { return err } diff --git a/pkg/backup/item_backupper.go b/pkg/backup/item_backupper.go index e02920c20..a2830484a 100644 --- a/pkg/backup/item_backupper.go +++ b/pkg/backup/item_backupper.go @@ -1,5 +1,5 @@ /* -Copyright 2020 the Velero contributors. +Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -29,7 +29,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" kubeerrs "k8s.io/apimachinery/pkg/util/errors" @@ -305,26 +304,9 @@ func (ib *itemBackupper) executeActions( metadata metav1.Object, ) (runtime.Unstructured, error) { for _, action := range ib.backupRequest.ResolvedActions { - if !action.resourceIncludesExcludes.ShouldInclude(groupResource.String()) { - log.Debug("Skipping action because it does not apply to this resource") + if !action.ShouldUse(groupResource, namespace, metadata, log) { continue } - - if namespace != "" && !action.namespaceIncludesExcludes.ShouldInclude(namespace) { - log.Debug("Skipping action because it does not apply to this namespace") - continue - } - - if namespace == "" && !action.namespaceIncludesExcludes.IncludeEverything() { - log.Debug("Skipping action because resource is cluster-scoped and action only applies to specific namespaces") - continue - } - - if !action.selector.Matches(labels.Set(metadata.GetLabels())) { - log.Debug("Skipping action because label selector does not match") - continue - } - log.Info("Executing custom action") updatedItem, additionalItemIdentifiers, err := action.Execute(obj, ib.backupRequest.Backup) diff --git a/pkg/backup/request.go b/pkg/backup/request.go index 0b4a63611..bc483ef97 100644 --- a/pkg/backup/request.go +++ b/pkg/backup/request.go @@ -22,6 +22,7 @@ import ( "github.com/vmware-tanzu/velero/internal/hook" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/util/collections" "github.com/vmware-tanzu/velero/pkg/volume" ) @@ -42,11 +43,11 @@ type Request struct { NamespaceIncludesExcludes *collections.IncludesExcludes ResourceIncludesExcludes *collections.IncludesExcludes ResourceHooks []hook.ResourceHook - ResolvedActions []resolvedAction - - VolumeSnapshots []*volume.Snapshot - PodVolumeBackups []*velerov1api.PodVolumeBackup - BackedUpItems map[itemKey]struct{} + ResolvedActions []framework.BackupItemResolvedAction + ResolvedItemSnapshotters []framework.ItemSnapshotterResolvedAction + VolumeSnapshots []*volume.Snapshot + PodVolumeBackups []*velerov1api.PodVolumeBackup + BackedUpItems map[itemKey]struct{} } // BackupResourceList returns the list of backed up resources grouped by the API diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 034f56d43..6adfeeec7 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -53,6 +53,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" + "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/util/boolptr" "github.com/vmware-tanzu/velero/pkg/util/collections" "github.com/vmware-tanzu/velero/pkg/util/encode" @@ -569,6 +570,10 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error { if err != nil { return err } + itemSnapshotters, err := pluginManager.GetItemSnapshotters() + if err != nil { + return err + } backupLog.Info("Setting up backup store to check for backup existence") backupStore, err := c.backupStoreGetter.Get(backup.StorageLocation, pluginManager, backupLog) @@ -586,8 +591,12 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error { return errors.Errorf("backup already exists in object storage") } + backupItemActionsResolver := framework.NewBackupItemActionResolver(actions) + itemSnapshottersResolver := framework.NewItemSnapshotterResolver(itemSnapshotters) + var fatalErrs []error - if err := c.backupper.Backup(backupLog, backup, backupFile, actions, pluginManager); err != nil { + if err := c.backupper.BackupWithResolvers(backupLog, backup, backupFile, backupItemActionsResolver, + itemSnapshottersResolver, pluginManager); err != nil { fatalErrs = append(fatalErrs, err) } diff --git a/pkg/controller/backup_controller_test.go b/pkg/controller/backup_controller_test.go index 16fc6c1a0..85c581a81 100644 --- a/pkg/controller/backup_controller_test.go +++ b/pkg/controller/backup_controller_test.go @@ -47,6 +47,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/persistence" persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" + "github.com/vmware-tanzu/velero/pkg/plugin/framework" pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/velero" velerotest "github.com/vmware-tanzu/velero/pkg/test" @@ -63,6 +64,13 @@ func (b *fakeBackupper) Backup(logger logrus.FieldLogger, backup *pkgbackup.Requ return args.Error(0) } +func (b *fakeBackupper) BackupWithResolvers(logger logrus.FieldLogger, backup *pkgbackup.Request, backupFile io.Writer, + backupItemActionResolver framework.BackupItemActionResolver, itemSnapshotterResolver framework.ItemSnapshotterResolver, + volumeSnapshotterGetter pkgbackup.VolumeSnapshotterGetter) error { + args := b.Called(logger, backup, backupFile, backupItemActionResolver, itemSnapshotterResolver, volumeSnapshotterGetter) + return args.Error(0) +} + func defaultBackup() *builder.BackupBuilder { return builder.ForBackup(velerov1api.DefaultNamespace, "backup-1") } @@ -825,7 +833,9 @@ func TestProcessBackupCompletions(t *testing.T) { pluginManager.On("GetBackupItemActions").Return(nil, nil) pluginManager.On("CleanupClients").Return(nil) + pluginManager.On("GetItemSnapshotters").Return(nil, nil) backupper.On("Backup", mock.Anything, mock.Anything, mock.Anything, []velero.BackupItemAction(nil), pluginManager).Return(nil) + backupper.On("BackupWithResolvers", mock.Anything, mock.Anything, mock.Anything, framework.BackupItemActionResolver{}, framework.ItemSnapshotterResolver{}, pluginManager).Return(nil) backupStore.On("BackupExists", test.backupLocation.Spec.StorageType.ObjectStorage.Bucket, test.backup.Name).Return(test.backupExists, test.existenceCheckError) // Ensure we have a CompletionTimestamp when uploading and that the backup name matches the backup in the object store. diff --git a/pkg/controller/restore_controller.go b/pkg/controller/restore_controller.go index 78472df4d..5b83e5a13 100644 --- a/pkg/controller/restore_controller.go +++ b/pkg/controller/restore_controller.go @@ -47,6 +47,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/metrics" "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" + "github.com/vmware-tanzu/velero/pkg/plugin/framework" pkgrestore "github.com/vmware-tanzu/velero/pkg/restore" "github.com/vmware-tanzu/velero/pkg/util/collections" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" @@ -443,6 +444,13 @@ func (c *restoreController) runValidatedRestore(restore *api.Restore, info backu if err != nil { return errors.Wrap(err, "error getting restore item actions") } + actionsResolver := framework.NewRestoreItemActionResolver(actions) + + itemSnapshotters, err := pluginManager.GetItemSnapshotters() + if err != nil { + return errors.Wrap(err, "error getting item snapshotters") + } + snapshotItemResolver := framework.NewItemSnapshotterResolver(itemSnapshotters) backupFile, err := downloadToTempFile(restore.Spec.BackupName, info.backupStore, restoreLog) if err != nil { @@ -476,7 +484,8 @@ func (c *restoreController) runValidatedRestore(restore *api.Restore, info backu VolumeSnapshots: volumeSnapshots, BackupReader: backupFile, } - restoreWarnings, restoreErrors := c.restorer.Restore(restoreReq, actions, c.snapshotLocationLister, pluginManager) + restoreWarnings, restoreErrors := c.restorer.RestoreWithResolvers(restoreReq, actionsResolver, snapshotItemResolver, + c.snapshotLocationLister, pluginManager) restoreLog.Info("restore completed") // re-instantiate the backup store because credentials could have changed since the original diff --git a/pkg/controller/restore_controller_test.go b/pkg/controller/restore_controller_test.go index d9f2a181c..1641b4f85 100644 --- a/pkg/controller/restore_controller_test.go +++ b/pkg/controller/restore_controller_test.go @@ -44,8 +44,10 @@ import ( "github.com/vmware-tanzu/velero/pkg/metrics" persistencemocks "github.com/vmware-tanzu/velero/pkg/persistence/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" + "github.com/vmware-tanzu/velero/pkg/plugin/framework" pluginmocks "github.com/vmware-tanzu/velero/pkg/plugin/mocks" "github.com/vmware-tanzu/velero/pkg/plugin/velero" + isv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/item_snapshotter/v1" pkgrestore "github.com/vmware-tanzu/velero/pkg/restore" velerotest "github.com/vmware-tanzu/velero/pkg/test" "github.com/vmware-tanzu/velero/pkg/util/logging" @@ -505,7 +507,8 @@ func TestProcessQueueItem(t *testing.T) { if test.expectedRestorerCall != nil { backupStore.On("GetBackupContents", test.backup.Name).Return(ioutil.NopCloser(bytes.NewReader([]byte("hello world"))), nil) - restorer.On("Restore", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(warnings, errors) + restorer.On("RestoreWithResolvers", mock.Anything, mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(warnings, errors) backupStore.On("PutRestoreLog", test.backup.Name, test.restore.Name, mock.Anything).Return(test.putRestoreLogErr) @@ -545,6 +548,7 @@ func TestProcessQueueItem(t *testing.T) { if test.restore != nil { pluginManager.On("GetRestoreItemActions").Return(nil, nil) + pluginManager.On("GetItemSnapshotters").Return([]isv1.ItemSnapshotter{}, nil) pluginManager.On("CleanupClients") } @@ -858,3 +862,17 @@ func (r *fakeRestorer) Restore( return res.Get(0).(pkgrestore.Result), res.Get(1).(pkgrestore.Result) } + +func (r *fakeRestorer) RestoreWithResolvers(req pkgrestore.Request, + resolver framework.RestoreItemActionResolver, + itemSnapshotterResolver framework.ItemSnapshotterResolver, + snapshotLocationLister listers.VolumeSnapshotLocationLister, + volumeSnapshotterGetter pkgrestore.VolumeSnapshotterGetter, +) (pkgrestore.Result, pkgrestore.Result) { + res := r.Called(req.Log, req.Restore, req.Backup, req.BackupReader, resolver, itemSnapshotterResolver, + snapshotLocationLister, volumeSnapshotterGetter) + + r.calledWithArg = *req.Restore + + return res.Get(0).(pkgrestore.Result), res.Get(1).(pkgrestore.Result) +} diff --git a/pkg/plugin/framework/action_resolver.go b/pkg/plugin/framework/action_resolver.go new file mode 100644 index 000000000..9797ba526 --- /dev/null +++ b/pkg/plugin/framework/action_resolver.go @@ -0,0 +1,242 @@ +/* +Copyright the Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + isv1 "github.com/vmware-tanzu/velero/pkg/plugin/velero/item_snapshotter/v1" + + "github.com/vmware-tanzu/velero/pkg/discovery" + "github.com/vmware-tanzu/velero/pkg/util/collections" +) + +/* +Velero has a variety of Actions that can be executed on Kubernetes resources. The Actions (BackupItemAction, RestoreItemAction +and others) implement the Applicable interface which returns a ResourceSelector for the Action. The ResourceSelector +can specify namespaces, resource names and labels to include or exclude. The ResourceSelector is resolved into lists +of namespaces and resources present in the backup to be matched against. These lists and the label selector are then used to +decide whether or not the ResolvedAction should be used for a particular resource. +*/ + +// ResolvedAction is an action that has had the namespaces, resources names and labels to include or exclude resolved +type ResolvedAction interface { + // ShouldUse returns true if the resolved namespaces, resource names and labels match those passed in the parameters. + // metadata is optional and may be nil + ShouldUse(groupResource schema.GroupResource, namespace string, metadata metav1.Object, + log logrus.FieldLogger) bool +} + +// resolvedAction is a core struct that holds the resolved namespaces, resource names and labels +type resolvedAction struct { + ResourceIncludesExcludes *collections.IncludesExcludes + NamespaceIncludesExcludes *collections.IncludesExcludes + Selector labels.Selector +} + +func (recv resolvedAction) ShouldUse(groupResource schema.GroupResource, namespace string, metadata metav1.Object, + log logrus.FieldLogger) bool { + if !recv.ResourceIncludesExcludes.ShouldInclude(groupResource.String()) { + log.Debug("Skipping action because it does not apply to this resource") + return false + } + + if namespace != "" && !recv.NamespaceIncludesExcludes.ShouldInclude(namespace) { + log.Debug("Skipping action because it does not apply to this namespace") + return false + } + + if namespace == "" && !recv.NamespaceIncludesExcludes.IncludeEverything() { + log.Debug("Skipping action because resource is cluster-scoped and action only applies to specific namespaces") + return false + } + + if metadata != nil && !recv.Selector.Matches(labels.Set(metadata.GetLabels())) { + log.Debug("Skipping action because label selector does not match") + return false + } + return true +} + +// resolveAction resolves the resources, namespaces and selector into fully-qualified versions +func resolveAction(helper discovery.Helper, action velero.Applicable) (resources *collections.IncludesExcludes, + namespaces *collections.IncludesExcludes, selector labels.Selector, err error) { + resourceSelector, err := action.AppliesTo() + if err != nil { + return nil, nil, nil, err + } + + resources = collections.GetResourceIncludesExcludes(helper, resourceSelector.IncludedResources, resourceSelector.ExcludedResources) + namespaces = collections.NewIncludesExcludes().Includes(resourceSelector.IncludedNamespaces...).Excludes(resourceSelector.ExcludedNamespaces...) + + selector = labels.Everything() + if resourceSelector.LabelSelector != "" { + if selector, err = labels.Parse(resourceSelector.LabelSelector); err != nil { + return nil, nil, nil, err + } + } + + return +} + +type BackupItemResolvedAction struct { + velero.BackupItemAction + resolvedAction +} + +func NewBackupItemActionResolver(actions []velero.BackupItemAction) BackupItemActionResolver { + return BackupItemActionResolver{ + actions: actions, + } +} + +func NewRestoreItemActionResolver(actions []velero.RestoreItemAction) RestoreItemActionResolver { + return RestoreItemActionResolver{ + actions: actions, + } +} + +func NewDeleteItemActionResolver(actions []velero.DeleteItemAction) DeleteItemActionResolver { + return DeleteItemActionResolver{ + actions: actions, + } +} + +func NewItemSnapshotterResolver(actions []isv1.ItemSnapshotter) ItemSnapshotterResolver { + return ItemSnapshotterResolver{ + actions: actions, + } +} + +type ActionResolver interface { + ResolveAction(helper discovery.Helper, action velero.Applicable) (ResolvedAction, error) +} + +type BackupItemActionResolver struct { + actions []velero.BackupItemAction +} + +func (recv BackupItemActionResolver) ResolveActions(helper discovery.Helper) ([]BackupItemResolvedAction, error) { + var resolved []BackupItemResolvedAction + for _, action := range recv.actions { + resources, namespaces, selector, err := resolveAction(helper, action) + if err != nil { + return nil, err + } + res := BackupItemResolvedAction{ + BackupItemAction: action, + resolvedAction: resolvedAction{ + ResourceIncludesExcludes: resources, + NamespaceIncludesExcludes: namespaces, + Selector: selector, + }, + } + resolved = append(resolved, res) + } + return resolved, nil +} + +type RestoreItemResolvedAction struct { + velero.RestoreItemAction + resolvedAction +} + +type RestoreItemActionResolver struct { + actions []velero.RestoreItemAction +} + +func (recv RestoreItemActionResolver) ResolveActions(helper discovery.Helper) ([]RestoreItemResolvedAction, error) { + var resolved []RestoreItemResolvedAction + for _, action := range recv.actions { + resources, namespaces, selector, err := resolveAction(helper, action) + if err != nil { + return nil, err + } + res := RestoreItemResolvedAction{ + RestoreItemAction: action, + resolvedAction: resolvedAction{ + ResourceIncludesExcludes: resources, + NamespaceIncludesExcludes: namespaces, + Selector: selector, + }, + } + resolved = append(resolved, res) + } + return resolved, nil +} + +type DeleteItemResolvedAction struct { + velero.DeleteItemAction + resolvedAction +} + +type DeleteItemActionResolver struct { + actions []velero.DeleteItemAction +} + +func (recv DeleteItemActionResolver) ResolveActions(helper discovery.Helper) ([]DeleteItemResolvedAction, error) { + var resolved []DeleteItemResolvedAction + for _, action := range recv.actions { + resources, namespaces, selector, err := resolveAction(helper, action) + if err != nil { + return nil, err + } + res := DeleteItemResolvedAction{ + DeleteItemAction: action, + resolvedAction: resolvedAction{ + ResourceIncludesExcludes: resources, + NamespaceIncludesExcludes: namespaces, + Selector: selector, + }, + } + resolved = append(resolved, res) + } + return resolved, nil +} + +type ItemSnapshotterResolvedAction struct { + isv1.ItemSnapshotter + resolvedAction +} + +type ItemSnapshotterResolver struct { + actions []isv1.ItemSnapshotter +} + +func (recv ItemSnapshotterResolver) ResolveActions(helper discovery.Helper) ([]ItemSnapshotterResolvedAction, error) { + var resolved []ItemSnapshotterResolvedAction + for _, action := range recv.actions { + resources, namespaces, selector, err := resolveAction(helper, action) + if err != nil { + return nil, err + } + res := ItemSnapshotterResolvedAction{ + ItemSnapshotter: action, + resolvedAction: resolvedAction{ + ResourceIncludesExcludes: resources, + NamespaceIncludesExcludes: namespaces, + Selector: selector, + }, + } + resolved = append(resolved, res) + } + return resolved, nil +} diff --git a/pkg/plugin/framework/action_resolver_test.go b/pkg/plugin/framework/action_resolver_test.go new file mode 100644 index 000000000..cf0e411da --- /dev/null +++ b/pkg/plugin/framework/action_resolver_test.go @@ -0,0 +1,93 @@ +/* +Copyright the Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "testing" + + "k8s.io/apimachinery/pkg/labels" + + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + velerotest "github.com/vmware-tanzu/velero/pkg/test" +) + +type mockApplicable struct { + selector velero.ResourceSelector +} + +func (recv mockApplicable) AppliesTo() (velero.ResourceSelector, error) { + return recv.selector, nil +} + +func TestActionResolverNamespace(t *testing.T) { + discoveryHelper := velerotest.NewFakeDiscoveryHelper(false, map[schema.GroupVersionResource]schema.GroupVersionResource{}) + namespaceMatchApplicable := mockApplicable{ + selector: velero.ResourceSelector{ + IncludedNamespaces: []string{"default"}, + }, + } + resources, namespaces, selector, err := resolveAction(discoveryHelper, namespaceMatchApplicable) + require.NoError(t, err) + require.Equal(t, []string{"default"}, namespaces.GetIncludes()) + require.Empty(t, namespaces.GetExcludes()) + require.Empty(t, resources.GetIncludes()) + require.Empty(t, resources.GetExcludes()) + require.True(t, selector.Empty()) +} + +func TestActionResolverResource(t *testing.T) { + pvGVR := schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "persistentvolumes", + } + discoveryHelper := velerotest.NewFakeDiscoveryHelper(false, map[schema.GroupVersionResource]schema.GroupVersionResource{pvGVR: pvGVR}) + namespaceMatchApplicable := mockApplicable{ + selector: velero.ResourceSelector{ + IncludedResources: []string{"persistentvolumes"}, + }, + } + resources, namespaces, selector, err := resolveAction(discoveryHelper, namespaceMatchApplicable) + require.NoError(t, err) + require.Empty(t, namespaces.GetIncludes()) + require.Empty(t, namespaces.GetExcludes()) + require.True(t, resources.ShouldInclude("persistentvolumes")) + require.Empty(t, resources.GetExcludes()) + require.True(t, selector.Empty()) +} + +func TestActionResolverLabel(t *testing.T) { + discoveryHelper := velerotest.NewFakeDiscoveryHelper(false, map[schema.GroupVersionResource]schema.GroupVersionResource{}) + namespaceMatchApplicable := mockApplicable{ + selector: velero.ResourceSelector{ + LabelSelector: "myLabel=true", + }, + } + checkLabel, err := labels.ConvertSelectorToLabelsMap("myLabel=true") + require.NoError(t, err) + + resources, namespaces, selector, err := resolveAction(discoveryHelper, namespaceMatchApplicable) + require.NoError(t, err) + require.Empty(t, namespaces.GetIncludes()) + require.Empty(t, namespaces.GetExcludes()) + require.Empty(t, resources.GetIncludes()) + require.Empty(t, resources.GetExcludes()) + require.True(t, selector.Matches(checkLabel)) +} diff --git a/pkg/plugin/velero/shared.go b/pkg/plugin/velero/shared.go index 138de4886..76c7464ad 100644 --- a/pkg/plugin/velero/shared.go +++ b/pkg/plugin/velero/shared.go @@ -1,5 +1,5 @@ /* -Copyright 2019 the Velero contributors. +Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -48,3 +48,9 @@ type ResourceSelector struct { // for details on syntax. LabelSelector string } + +// Applicable allows actions and plugins to specify which resources they should be invoked for +type Applicable interface { + // AppliesTo returns information about which resources this Responder should be invoked for. + AppliesTo() (ResourceSelector, error) +} diff --git a/pkg/restore/restore.go b/pkg/restore/restore.go index 846d71c78..19e500269 100644 --- a/pkg/restore/restore.go +++ b/pkg/restore/restore.go @@ -56,6 +56,7 @@ import ( listers "github.com/vmware-tanzu/velero/pkg/generated/listers/velero/v1" "github.com/vmware-tanzu/velero/pkg/kuberesource" "github.com/vmware-tanzu/velero/pkg/label" + "github.com/vmware-tanzu/velero/pkg/plugin/framework" "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/podexec" "github.com/vmware-tanzu/velero/pkg/restic" @@ -96,6 +97,13 @@ type Restorer interface { snapshotLocationLister listers.VolumeSnapshotLocationLister, volumeSnapshotterGetter VolumeSnapshotterGetter, ) (Result, Result) + RestoreWithResolvers( + req Request, + restoreItemActionResolver framework.RestoreItemActionResolver, + itemSnapshotterResolver framework.ItemSnapshotterResolver, + snapshotLocationLister listers.VolumeSnapshotLocationLister, + volumeSnapshotterGetter VolumeSnapshotterGetter, + ) (Result, Result) } // kubernetesRestorer implements Restorer for restoring into a Kubernetes cluster. @@ -161,6 +169,18 @@ func (kr *kubernetesRestorer) Restore( actions []velero.RestoreItemAction, snapshotLocationLister listers.VolumeSnapshotLocationLister, volumeSnapshotterGetter VolumeSnapshotterGetter, +) (Result, Result) { + resolver := framework.NewRestoreItemActionResolver(actions) + snapshotItemResolver := framework.NewItemSnapshotterResolver(nil) + return kr.RestoreWithResolvers(req, resolver, snapshotItemResolver, snapshotLocationLister, volumeSnapshotterGetter) +} + +func (kr *kubernetesRestorer) RestoreWithResolvers( + req Request, + restoreItemActionResolver framework.RestoreItemActionResolver, + itemSnapshotterResolver framework.ItemSnapshotterResolver, + snapshotLocationLister listers.VolumeSnapshotLocationLister, + volumeSnapshotterGetter VolumeSnapshotterGetter, ) (Result, Result) { // metav1.LabelSelectorAsSelector converts a nil LabelSelector to a // Nothing Selector, i.e. a selector that matches nothing. We want @@ -188,7 +208,12 @@ func (kr *kubernetesRestorer) Restore( Includes(req.Restore.Spec.IncludedNamespaces...). Excludes(req.Restore.Spec.ExcludedNamespaces...) - resolvedActions, err := resolveActions(actions, kr.discoveryHelper) + resolvedActions, err := restoreItemActionResolver.ResolveActions(kr.discoveryHelper) + if err != nil { + return Result{}, Result{Velero: []string{err.Error()}} + } + + resolvedItemSnapshotterActions, err := itemSnapshotterResolver.ResolveActions(kr.discoveryHelper) if err != nil { return Result{}, Result{Velero: []string{err.Error()}} } @@ -251,7 +276,8 @@ func (kr *kubernetesRestorer) Restore( dynamicFactory: kr.dynamicFactory, fileSystem: kr.fileSystem, namespaceClient: kr.namespaceClient, - actions: resolvedActions, + restoreItemActions: resolvedActions, + itemSnapshotterActions: resolvedItemSnapshotterActions, volumeSnapshotterGetter: volumeSnapshotterGetter, resticRestorer: resticRestorer, resticErrs: make(chan error), @@ -277,46 +303,6 @@ func (kr *kubernetesRestorer) Restore( return restoreCtx.execute() } -type resolvedAction struct { - velero.RestoreItemAction - - resourceIncludesExcludes *collections.IncludesExcludes - namespaceIncludesExcludes *collections.IncludesExcludes - selector labels.Selector -} - -func resolveActions(actions []velero.RestoreItemAction, helper discovery.Helper) ([]resolvedAction, error) { - var resolved []resolvedAction - - for _, action := range actions { - resourceSelector, err := action.AppliesTo() - if err != nil { - return nil, err - } - - resources := collections.GetResourceIncludesExcludes(helper, resourceSelector.IncludedResources, resourceSelector.ExcludedResources) - namespaces := collections.NewIncludesExcludes().Includes(resourceSelector.IncludedNamespaces...).Excludes(resourceSelector.ExcludedNamespaces...) - - selector := labels.Everything() - if resourceSelector.LabelSelector != "" { - if selector, err = labels.Parse(resourceSelector.LabelSelector); err != nil { - return nil, err - } - } - - res := resolvedAction{ - RestoreItemAction: action, - resourceIncludesExcludes: resources, - namespaceIncludesExcludes: namespaces, - selector: selector, - } - - resolved = append(resolved, res) - } - - return resolved, nil -} - type restoreContext struct { backup *velerov1api.Backup backupReader io.Reader @@ -331,7 +317,8 @@ type restoreContext struct { dynamicFactory client.DynamicFactory fileSystem filesystem.Interface namespaceClient corev1.NamespaceInterface - actions []resolvedAction + restoreItemActions []framework.RestoreItemResolvedAction + itemSnapshotterActions []framework.ItemSnapshotterResolvedAction volumeSnapshotterGetter VolumeSnapshotterGetter resticRestorer restic.Restorer resticWaitGroup sync.WaitGroup @@ -713,23 +700,22 @@ func getNamespace(logger logrus.FieldLogger, path, remappedName string) *v1.Name } } -// TODO: this should be combined with DeleteItemActions at some point. -func (ctx *restoreContext) getApplicableActions(groupResource schema.GroupResource, namespace string) []resolvedAction { - var actions []resolvedAction - for _, action := range ctx.actions { - if !action.resourceIncludesExcludes.ShouldInclude(groupResource.String()) { - continue +func (ctx *restoreContext) getApplicableActions(groupResource schema.GroupResource, namespace string) []framework.RestoreItemResolvedAction { + var actions []framework.RestoreItemResolvedAction + for _, action := range ctx.restoreItemActions { + if action.ShouldUse(groupResource, namespace, nil, ctx.log) { + actions = append(actions, action) } + } + return actions +} - if namespace != "" && !action.namespaceIncludesExcludes.ShouldInclude(namespace) { - continue +func (ctx *restoreContext) getApplicableItemSnapshotters(groupResource schema.GroupResource, namespace string) []framework.ItemSnapshotterResolvedAction { + var actions []framework.ItemSnapshotterResolvedAction + for _, action := range ctx.itemSnapshotterActions { + if action.ShouldUse(groupResource, namespace, nil, ctx.log) { + actions = append(actions, action) } - - if namespace == "" && !action.namespaceIncludesExcludes.IncludeEverything() { - continue - } - - actions = append(actions, action) } return actions @@ -1127,13 +1113,13 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso } for _, action := range ctx.getApplicableActions(groupResource, namespace) { - if !action.selector.Matches(labels.Set(obj.GetLabels())) { + if !action.Selector.Matches(labels.Set(obj.GetLabels())) { return warnings, errs } ctx.log.Infof("Executing item action for %v", &groupResource) - executeOutput, err := action.Execute(&velero.RestoreItemActionExecuteInput{ + executeOutput, err := action.RestoreItemAction.Execute(&velero.RestoreItemActionExecuteInput{ Item: obj, ItemFromBackup: itemFromBackup, Restore: ctx.restore, diff --git a/pkg/restore/restore_test.go b/pkg/restore/restore_test.go index 35d72cfff..faada13f9 100644 --- a/pkg/restore/restore_test.go +++ b/pkg/restore/restore_test.go @@ -545,7 +545,7 @@ func TestRestoreResourceFiltering(t *testing.T) { } warnings, errs := h.restorer.Restore( data, - nil, // actions + nil, // restoreItemActions nil, // snapshot location lister nil, // volume snapshotter getter ) @@ -626,7 +626,7 @@ func TestRestoreNamespaceMapping(t *testing.T) { } warnings, errs := h.restorer.Restore( data, - nil, // actions + nil, // restoreItemActions nil, // snapshot location lister nil, // volume snapshotter getter ) @@ -708,7 +708,7 @@ func TestRestoreResourcePriorities(t *testing.T) { } warnings, errs := h.restorer.Restore( data, - nil, // actions + nil, // restoreItemActions nil, // snapshot location lister nil, // volume snapshotter getter ) @@ -785,7 +785,7 @@ func TestInvalidTarballContents(t *testing.T) { } warnings, errs := h.restorer.Restore( data, - nil, // actions + nil, // restoreItemActions nil, // snapshot location lister nil, // volume snapshotter getter ) @@ -1000,7 +1000,7 @@ func TestRestoreItems(t *testing.T) { } warnings, errs := h.restorer.Restore( data, - nil, // actions + nil, // restoreItemActions nil, // snapshot location lister nil, // volume snapshotter getter ) @@ -2510,7 +2510,7 @@ func TestRestorePersistentVolumes(t *testing.T) { } warnings, errs := h.restorer.Restore( data, - nil, // actions + nil, // restoreItemActions vslInformer.Lister(), tc.volumeSnapshotterGetter, ) @@ -2646,7 +2646,7 @@ func TestRestoreWithRestic(t *testing.T) { warnings, errs := h.restorer.Restore( data, - nil, // actions + nil, // restoreItemActions nil, // snapshot location lister nil, // volume snapshotter getter ) diff --git a/pkg/test/fake_mapper.go b/pkg/test/fake_mapper.go index 1b430c5aa..210d25747 100644 --- a/pkg/test/fake_mapper.go +++ b/pkg/test/fake_mapper.go @@ -43,6 +43,16 @@ func (m *FakeMapper) ResourceFor(input schema.GroupVersionResource) (schema.Grou if gr, found := m.Resources[input]; found { return gr, nil } + if input.Version == "" { + input.Version = "v1" + if gr, found := m.Resources[input]; found { + return gr, nil + } + input.Version = "v1beta1" + if gr, found := m.Resources[input]; found { + return gr, nil + } + } return schema.GroupVersionResource{}, errors.Errorf("invalid resource %q", input.String()) }