From 984176f156f43dcb01ff10c829078141ccb7e178 Mon Sep 17 00:00:00 2001 From: Bridget McErlean Date: Tue, 10 Aug 2021 18:12:17 -0400 Subject: [PATCH] Skip restore of APIServices managed by Kubernetes It was discovered during Velero 1.6.3 upgrade testing that Velero was restoring `APIService` objects for APIs that are no longer being served by Kubernetes 1.22. If these items were restored, it would break the behaviour of discovery within the cluster. This change introduces a new RestoreItemAction plugin that skips the restore of any `APIService` object which is managed by Kubernetes such as those for built-in APIs or CRDs. The `APIService`s for these will be created when the Kubernetes API server starts or when new CRDs are registered. These objects are identified by looking for the `kube-aggregator.kubernetes.io/automanaged` label. Signed-off-by: Bridget McErlean --- changelogs/unreleased/4028-zubron | 1 + go.mod | 1 + go.sum | 2 + pkg/cmd/server/plugin/plugin.go | 5 + pkg/restore/apiservice_action.go | 60 +++++++++++ pkg/restore/apiservice_action_test.go | 147 ++++++++++++++++++++++++++ 6 files changed, 216 insertions(+) create mode 100644 changelogs/unreleased/4028-zubron create mode 100644 pkg/restore/apiservice_action.go create mode 100644 pkg/restore/apiservice_action_test.go diff --git a/changelogs/unreleased/4028-zubron b/changelogs/unreleased/4028-zubron new file mode 100644 index 000000000..79a32656d --- /dev/null +++ b/changelogs/unreleased/4028-zubron @@ -0,0 +1 @@ +Add a RestoreItemAction plugin (`velero.io/apiservice`) which skips the restore of any `APIService` which is managed by Kubernetes. These are identified using the `kube-aggregator.kubernetes.io/automanaged` label. \ No newline at end of file diff --git a/go.mod b/go.mod index a4145144c..0efd00054 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,7 @@ require ( k8s.io/client-go v0.19.12 k8s.io/klog v1.0.0 k8s.io/klog/v2 v2.3.0 // indirect + k8s.io/kube-aggregator v0.19.12 k8s.io/utils v0.0.0-20201005171033-6301aaf42dc7 // indirect sigs.k8s.io/cluster-api v0.3.11-0.20210106212952-b6c1b5b3db3d sigs.k8s.io/controller-runtime v0.7.1-0.20201215171748-096b2e07c091 diff --git a/go.sum b/go.sum index 25e91ae97..019a32396 100644 --- a/go.sum +++ b/go.sum @@ -992,6 +992,8 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.3.0 h1:WmkrnW7fdrm0/DMClc+HIxtftvxVIPAhlVwMQo5yLco= k8s.io/klog/v2 v2.3.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-aggregator v0.19.12 h1:OwyNUe/7/gxzEnaLd3sC9Yrpx0fZAERzvFslX5Qq5g8= +k8s.io/kube-aggregator v0.19.12/go.mod h1:K76wPd03pSHEmS1FgJOcpryac5C3va4cbCvSu+4EmE0= k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6 h1:+WnxoVtG8TMiudHBSEtrVL1egv36TkkJm+bA8AxicmQ= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= diff --git a/pkg/cmd/server/plugin/plugin.go b/pkg/cmd/server/plugin/plugin.go index 58e257268..1a79b6dd6 100644 --- a/pkg/cmd/server/plugin/plugin.go +++ b/pkg/cmd/server/plugin/plugin.go @@ -54,6 +54,7 @@ func NewCommand(f client.Factory) *cobra.Command { RegisterRestoreItemAction("velero.io/cluster-role-bindings", newClusterRoleBindingItemAction). RegisterRestoreItemAction("velero.io/crd-preserve-fields", newCRDV1PreserveUnknownFieldsItemAction). RegisterRestoreItemAction("velero.io/change-pvc-node-selector", newChangePVCNodeSelectorItemAction(f)). + RegisterRestoreItemAction("velero.io/apiservice", newAPIServiceRestoreItemAction). Serve() }, } @@ -197,3 +198,7 @@ func newChangePVCNodeSelectorItemAction(f client.Factory) veleroplugin.HandlerIn ), nil } } + +func newAPIServiceRestoreItemAction(logger logrus.FieldLogger) (interface{}, error) { + return restore.NewAPIServiceAction(logger), nil +} diff --git a/pkg/restore/apiservice_action.go b/pkg/restore/apiservice_action.go new file mode 100644 index 000000000..97c4dc808 --- /dev/null +++ b/pkg/restore/apiservice_action.go @@ -0,0 +1,60 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package restore + +import ( + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/runtime" + apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" + "k8s.io/kube-aggregator/pkg/controllers/autoregister" + + "github.com/vmware-tanzu/velero/pkg/plugin/velero" +) + +type APIServiceAction struct { + logger logrus.FieldLogger +} + +// NewAPIServiceAction returns an APIServiceAction which is a RestoreItemAction plugin +// that will skip the restore of any APIServices which are managed by Kubernetes. This +// is determined by looking for the "kube-aggregator.kubernetes.io/automanaged" label on +// the APIService. +func NewAPIServiceAction(logger logrus.FieldLogger) *APIServiceAction { + return &APIServiceAction{logger: logger} +} + +func (a *APIServiceAction) AppliesTo() (velero.ResourceSelector, error) { + return velero.ResourceSelector{ + IncludedResources: []string{"apiservices"}, + }, nil +} + +func (a *APIServiceAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + apiService := new(apiregistrationv1.APIService) + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(input.Item.UnstructuredContent(), apiService); err != nil { + return nil, errors.WithStack(err) + } + + output := velero.NewRestoreItemActionExecuteOutput(input.Item) + + if _, ok := apiService.Labels[autoregister.AutoRegisterManagedLabel]; ok { + output = output.WithoutRestore() + } + + return output, nil +} diff --git a/pkg/restore/apiservice_action_test.go b/pkg/restore/apiservice_action_test.go new file mode 100644 index 000000000..3cf2fce8b --- /dev/null +++ b/pkg/restore/apiservice_action_test.go @@ -0,0 +1,147 @@ +/* +Copyright 2017 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package restore + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" + "k8s.io/kube-aggregator/pkg/controllers/autoregister" + + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + velerotest "github.com/vmware-tanzu/velero/pkg/test" +) + +func TestAPIServiceActionExecute(t *testing.T) { + tests := []struct { + name string + obj apiregistrationv1.APIService + skipRestore bool + }{ + { + name: "APIService with no labels should be restored without modification", + obj: apiregistrationv1.APIService{ + ObjectMeta: metav1.ObjectMeta{ + Name: "v1.foo.velero.io", + }, + }, + skipRestore: false, + }, + { + name: "Non-Local APIService without Kubernetes managed label should be restored without modification", + obj: apiregistrationv1.APIService{ + ObjectMeta: metav1.ObjectMeta{ + Name: "v1.foo.velero.io", + Labels: map[string]string{ + "component": "velero", + }, + }, + Spec: apiregistrationv1.APIServiceSpec{ + Group: "velero.io", + Version: "v1", + Service: &apiregistrationv1.ServiceReference{ + Namespace: "velero", + Name: "velero-aggregated-api-server", + }, + }, + }, + skipRestore: false, + }, + { + name: "APIService with Kubernetes managed label with 'true' value should not be restored", + obj: apiregistrationv1.APIService{ + ObjectMeta: metav1.ObjectMeta{ + Name: "v1.foo.velero.io", + Labels: map[string]string{ + autoregister.AutoRegisterManagedLabel: "true", + }, + }, + }, + skipRestore: true, + }, + { + name: "APIService with Kubernetes managed label with 'onstart' value should not be restored", + obj: apiregistrationv1.APIService{ + ObjectMeta: metav1.ObjectMeta{ + Name: "v1.foo.velero.io", + Labels: map[string]string{ + autoregister.AutoRegisterManagedLabel: "onstart", + }, + }, + }, + skipRestore: true, + }, + { + name: "APIService with Kubernetes managed label with any value should not be restored", + obj: apiregistrationv1.APIService{ + ObjectMeta: metav1.ObjectMeta{ + Name: "v1.foo.velero.io", + Labels: map[string]string{ + autoregister.AutoRegisterManagedLabel: "randomvalue", + }, + }, + }, + skipRestore: true, + }, + { + name: "Non-Local APIService with Kubernetes managed label should not be restored", + obj: apiregistrationv1.APIService{ + ObjectMeta: metav1.ObjectMeta{ + Name: "v1.foo.velero.io", + Labels: map[string]string{ + autoregister.AutoRegisterManagedLabel: "onstart", + }, + }, + Spec: apiregistrationv1.APIServiceSpec{ + Group: "velero.io", + Version: "v1", + Service: &apiregistrationv1.ServiceReference{ + Namespace: "velero", + Name: "velero-aggregated-api-server", + }, + }, + }, + skipRestore: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + action := NewAPIServiceAction(velerotest.NewLogger()) + + unstructuredAPIService, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&test.obj) + require.NoError(t, err) + + res, err := action.Execute(&velero.RestoreItemActionExecuteInput{ + Item: &unstructured.Unstructured{Object: unstructuredAPIService}, + ItemFromBackup: &unstructured.Unstructured{Object: unstructuredAPIService}, + }) + + require.NoError(t, err) + + var apiService apiregistrationv1.APIService + require.NoError(t, runtime.DefaultUnstructuredConverter.FromUnstructured(res.UpdatedItem.UnstructuredContent(), &apiService)) + assert.Equal(t, test.obj, apiService) + assert.Equal(t, test.skipRestore, res.SkipRestore) + }) + } +}