From 499631ba8ecc3f82bf591347be74d904d8d1fd0f Mon Sep 17 00:00:00 2001 From: Bridget McErlean Date: Thu, 29 Jul 2021 19:31:48 -0400 Subject: [PATCH] Cherry pick changes for 1.6.3 and add changelog (#4006) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Merge pull request #3941 from sseago/e2e-crdversion enable e2e tests to choose crd apiVersion * Updated uninstall to remove both v1beta1 and v1 CRDs if present (#3997) * Add changelog for v1.6.3 Signed-off-by: Bridget McErlean Co-authored-by: Wenkai Yin(尹文开) Co-authored-by: David L. Smith-Uchida --- changelogs/CHANGELOG-1.6.md | 36 ++++++++++++++++ changelogs/unreleased/3995-jenting | 1 - changelogs/unreleased/3999-jenting | 1 - pkg/client/factory.go | 2 + pkg/cmd/cli/uninstall/uninstall.go | 50 ++++++++++++---------- test/e2e/Makefile | 2 + test/e2e/backup_test.go | 2 +- test/e2e/e2e_suite_test.go | 7 +-- test/e2e/enable_api_group_versions_test.go | 1 + test/e2e/velero_utils.go | 3 +- 10 files changed, 76 insertions(+), 29 deletions(-) delete mode 100644 changelogs/unreleased/3995-jenting delete mode 100644 changelogs/unreleased/3999-jenting diff --git a/changelogs/CHANGELOG-1.6.md b/changelogs/CHANGELOG-1.6.md index 7e9fc65c9..ee713ad1a 100644 --- a/changelogs/CHANGELOG-1.6.md +++ b/changelogs/CHANGELOG-1.6.md @@ -1,3 +1,39 @@ +## v1.6.3 +### 2021-07-30 + +### Download +https://github.com/vmware-tanzu/velero/releases/tag/v1.6.3 + +### Container Image +`velero/velero:v1.6.3` + +### Documentation +https://velero.io/docs/v1.6/ + +### Upgrading +https://velero.io/docs/v1.6/upgrade-to-1.6/ + +### Highlights + +This release introduces changes to provide compatibility with Kubernetes v1.22. + +The `apiextensions.k8s.io/v1beta1` API version of `CustomResourceDefinition` will no longer be served in Kubernetes v1.22. +Velero will now use the cluster preferred API version for the `CustomResourceDefinition`s that it creates. + +If you are using Kubernetes v1.15 or earlier, the `apiextensions.k8s.io/v1beta1` API version will be used. +If you are using Kubernetes v1.22 or later, the `apiextensions.k8s.io/v1` API version will be used. +For clusters between these versions, the cluster preferred API version will be used. + +The `rbac.authorization.k8s.io/v1beta1` API version of `ClusterRoleBinding` will no longer be served in Kubernetes v1.22. +Velero will now use the `rbac.authorization.k8s.io/v1` API version for the `ClusterRoleBinding`s that it creates. +This API version was introduced in Kubernetes v1.8. + +### All Changes + + * enable e2e tests to choose crd apiVersion (#3941, @sseago) + * Upgrade Velero ClusterRoleBinding to use v1 API (#3995, @jenting) + * Install Kubernetes preferred CRDs API version (v1beta1/v1). (#3999, @jenting) + ## v1.6.2 ### 2021-07-16 diff --git a/changelogs/unreleased/3995-jenting b/changelogs/unreleased/3995-jenting deleted file mode 100644 index fb2e2d791..000000000 --- a/changelogs/unreleased/3995-jenting +++ /dev/null @@ -1 +0,0 @@ -Upgrade Velero ClusterRoleBinding to use v1 API \ No newline at end of file diff --git a/changelogs/unreleased/3999-jenting b/changelogs/unreleased/3999-jenting deleted file mode 100644 index a2479943c..000000000 --- a/changelogs/unreleased/3999-jenting +++ /dev/null @@ -1 +0,0 @@ -Install Kubernetes preferred CRDs API version (v1beta1/v1). \ No newline at end of file diff --git a/pkg/client/factory.go b/pkg/client/factory.go index 5274345c5..73b7d3751 100644 --- a/pkg/client/factory.go +++ b/pkg/client/factory.go @@ -19,6 +19,7 @@ package client import ( "os" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" k8scheme "k8s.io/client-go/kubernetes/scheme" kbclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -156,6 +157,7 @@ func (f *factory) KubebuilderClient() (kbclient.Client, error) { velerov1api.AddToScheme(scheme) k8scheme.AddToScheme(scheme) apiextv1beta1.AddToScheme(scheme) + apiextv1.AddToScheme(scheme) kubebuilderClient, err := kbclient.New(clientConfig, kbclient.Options{ Scheme: scheme, }) diff --git a/pkg/cmd/cli/uninstall/uninstall.go b/pkg/cmd/cli/uninstall/uninstall.go index c8d56d53a..5c61142ca 100644 --- a/pkg/cmd/cli/uninstall/uninstall.go +++ b/pkg/cmd/cli/uninstall/uninstall.go @@ -26,9 +26,10 @@ import ( "github.com/spf13/pflag" corev1 "k8s.io/api/core/v1" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/labels" kubeerrs "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/wait" @@ -125,31 +126,36 @@ func Run(ctx context.Context, kbClient kbclient.Client, namespace string, waitTo } // CRDs - veleroLabels := labels.FormatLabels(install.Labels()) - crdList := apiextv1beta1.CustomResourceDefinitionList{} - opts := kbclient.ListOptions{ - Namespace: namespace, - Raw: &metav1.ListOptions{ - LabelSelector: veleroLabels, + + veleroLabelSelector := labels.SelectorFromSet(install.Labels()) + opts := []kbclient.DeleteAllOfOption{ + kbclient.InNamespace(namespace), + kbclient.MatchingLabelsSelector{ + Selector: veleroLabelSelector, }, } - if err := kbClient.List(context.Background(), &crdList, &opts); err != nil { - errs = append(errs, errors.WithStack(err)) - } else { - if len(crdList.Items) == 0 { - fmt.Print("Velero CRDs do not exist, skipping.\n") + v1CRDsRemoved := false + v1crd := &apiextv1.CustomResourceDefinition{} + if err := kbClient.DeleteAllOf(ctx, v1crd, opts...); err != nil { + if meta.IsNoMatchError(err) { + fmt.Println("V1 Velero CRDs not found, skipping...") } else { - veleroLabelSelector := labels.SelectorFromSet(install.Labels()) - opts := []kbclient.DeleteAllOfOption{ - kbclient.InNamespace(namespace), - kbclient.MatchingLabelsSelector{ - Selector: veleroLabelSelector, - }, - } - crd := &apiextv1beta1.CustomResourceDefinition{} - if err := kbClient.DeleteAllOf(ctx, crd, opts...); err != nil { - errs = append(errs, errors.WithStack(err)) + errs = append(errs, errors.WithStack(err)) + } + } else { + v1CRDsRemoved = true + } + + // Remove any old Velero v1beta1 CRDs hanging around. + v1beta1crd := &apiextv1beta1.CustomResourceDefinition{} + if err := kbClient.DeleteAllOf(ctx, v1beta1crd, opts...); err != nil { + if meta.IsNoMatchError(err) { + if !v1CRDsRemoved { + // Only mention this if there were no V1 CRDs removed + fmt.Println("V1Beta1 Velero CRDs not found, skipping...") } + } else { + errs = append(errs, errors.WithStack(err)) } } diff --git a/test/e2e/Makefile b/test/e2e/Makefile index fb85a824b..2fdba0449 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -48,6 +48,7 @@ OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin GINKGO_FOCUS ?= VELERO_CLI ?=$$(pwd)/../../_output/bin/$(GOOS)/$(GOARCH)/velero VELERO_IMAGE ?= velero/velero:main +CRDS_VERSION ?= v1 VELERO_NAMESPACE ?= velero CREDS_FILE ?= BSL_BUCKET ?= @@ -79,6 +80,7 @@ run: ginkgo @$(GINKGO) -v -focus="$(GINKGO_FOCUS)" . -- -velerocli=$(VELERO_CLI) \ -velero-image=$(VELERO_IMAGE) \ -velero-namespace=$(VELERO_NAMESPACE) \ + -crds-version=$(CRDS_VERSION) \ -credentials-file=$(CREDS_FILE) \ -bucket=$(BSL_BUCKET) \ -prefix=$(BSL_PREFIX) \ diff --git a/test/e2e/backup_test.go b/test/e2e/backup_test.go index 7474679e1..db60cb571 100644 --- a/test/e2e/backup_test.go +++ b/test/e2e/backup_test.go @@ -60,7 +60,7 @@ func backup_restore_test(useVolumeSnapshots bool) { Expect(err).To(Succeed()) if installVelero { Expect(veleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, - cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, "")).To(Succeed()) + cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "")).To(Succeed()) } }) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 0429070ec..b2103c685 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -25,9 +25,9 @@ import ( ) var ( - veleroCLI, veleroImage, cloudCredentialsFile, bslConfig, bslBucket, bslPrefix, vslConfig, cloudProvider, objectStoreProvider, veleroNamespace string - additionalBSLProvider, additionalBSLBucket, additionalBSLPrefix, additionalBSLConfig, additionalBSLCredentials string - installVelero bool + veleroCLI, veleroImage, cloudCredentialsFile, bslConfig, bslBucket, bslPrefix, vslConfig, cloudProvider, objectStoreProvider, veleroNamespace, crdsVersion string + additionalBSLProvider, additionalBSLBucket, additionalBSLPrefix, additionalBSLConfig, additionalBSLCredentials string + installVelero bool ) func init() { @@ -49,6 +49,7 @@ func init() { flag.StringVar(&additionalBSLPrefix, "additional-bsl-prefix", "", "prefix under which all Velero data should be stored within the bucket for additional backup storage location. Optional.") flag.StringVar(&additionalBSLConfig, "additional-bsl-config", "", "configuration to use for the additional backup storage location. Format is key1=value1,key2=value2") flag.StringVar(&additionalBSLCredentials, "additional-bsl-credentials-file", "", "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.") + flag.StringVar(&crdsVersion, "crds-version", "v1", "CRD apiVersion for velero CRD creation.") } func TestE2e(t *testing.T) { diff --git a/test/e2e/enable_api_group_versions_test.go b/test/e2e/enable_api_group_versions_test.go index 12ec6d2f3..442d911cd 100644 --- a/test/e2e/enable_api_group_versions_test.go +++ b/test/e2e/enable_api_group_versions_test.go @@ -68,6 +68,7 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", bslPrefix, bslConfig, vslConfig, + crdsVersion, "EnableAPIGroupVersions", // TODO: remove when feature flag is removed ) Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e/velero_utils.go b/test/e2e/velero_utils.go index ca4c10958..72d7f58fe 100644 --- a/test/e2e/velero_utils.go +++ b/test/e2e/velero_utils.go @@ -268,7 +268,7 @@ func veleroRestore(ctx context.Context, veleroCLI string, veleroNamespace string func veleroInstall(ctx context.Context, veleroImage string, veleroNamespace string, cloudProvider string, objectStoreProvider string, useVolumeSnapshots bool, cloudCredentialsFile string, bslBucket string, bslPrefix string, bslConfig string, vslConfig string, - features string) error { + crdsVersion string, features string) error { if cloudProvider != "kind" { if objectStoreProvider != "" { @@ -312,6 +312,7 @@ func veleroInstall(ctx context.Context, veleroImage string, veleroNamespace stri } veleroInstallOptions.UseRestic = !useVolumeSnapshots veleroInstallOptions.Image = veleroImage + veleroInstallOptions.CRDsVersion = crdsVersion veleroInstallOptions.Namespace = veleroNamespace err = installVeleroServer(veleroInstallOptions)