diff --git a/go.mod b/go.mod index ca5af08fe..fd02ae00b 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/hashicorp/go-plugin v0.0.0-20190610192547-a1bc61569a26 github.com/joho/godotenv v1.3.0 github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0 - github.com/onsi/ginkgo v1.16.1 + github.com/onsi/ginkgo v1.16.2 github.com/onsi/gomega v1.10.2 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.7.1 diff --git a/go.sum b/go.sum index d10afce7e..73239795f 100644 --- a/go.sum +++ b/go.sum @@ -433,8 +433,8 @@ github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.1 h1:foqVmeWDD6yYpK+Yz3fHyNIxFYNxswxqNFjSKe+vI54= -github.com/onsi/ginkgo v1.16.1/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= +github.com/onsi/ginkgo v1.16.2 h1:HFB2fbVIlhIfCfOW81bZFbiC/RvnpXSdhbF2/DJr134= +github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= diff --git a/pkg/client/factory.go b/pkg/client/factory.go index 6ecc63be7..5274345c5 100644 --- a/pkg/client/factory.go +++ b/pkg/client/factory.go @@ -19,6 +19,8 @@ package client import ( "os" + apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + k8scheme "k8s.io/client-go/kubernetes/scheme" kbclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/pkg/errors" @@ -45,7 +47,8 @@ type Factory interface { // DynamicClient returns a Kubernetes dynamic client. It uses the following priority to specify the cluster // configuration: --kubeconfig flag, KUBECONFIG environment variable, in-cluster configuration. DynamicClient() (dynamic.Interface, error) - // KubebuilderClient returns a Kubernetes dynamic client. It uses the following priority to specify the cluster + // KubebuilderClient returns a client for the controller runtime framework. It adds Kubernetes and Velero + // types to its scheme. It uses the following priority to specify the cluster // configuration: --kubeconfig flag, KUBECONFIG environment variable, in-cluster configuration. KubebuilderClient() (kbclient.Client, error) // SetBasename changes the basename for an already-constructed client. @@ -151,6 +154,8 @@ func (f *factory) KubebuilderClient() (kbclient.Client, error) { scheme := runtime.NewScheme() velerov1api.AddToScheme(scheme) + k8scheme.AddToScheme(scheme) + apiextv1beta1.AddToScheme(scheme) kubebuilderClient, err := kbclient.New(clientConfig, kbclient.Options{ Scheme: scheme, }) diff --git a/pkg/cmd/cli/uninstall/uninstall.go b/pkg/cmd/cli/uninstall/uninstall.go index a4226609e..c8d56d53a 100644 --- a/pkg/cmd/cli/uninstall/uninstall.go +++ b/pkg/cmd/cli/uninstall/uninstall.go @@ -26,20 +26,18 @@ import ( "github.com/spf13/pflag" corev1 "k8s.io/api/core/v1" + apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" kubeerrs "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" - - apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/cli" "github.com/vmware-tanzu/velero/pkg/install" - "github.com/vmware-tanzu/velero/pkg/util/kube" ) // uninstallOptions collects all the options for uninstalling Velero from a Kubernetes cluster. @@ -78,9 +76,9 @@ Use '--force' to skip the prompt confirming if you want to uninstall Velero. } } - client, extCl, err := kube.GetClusterClient() + kbClient, err := f.KubebuilderClient() cmd.CheckError(err) - cmd.CheckError(Run(context.Background(), client, extCl, f.Namespace(), o.wait)) + cmd.CheckError(Run(context.Background(), kbClient, f.Namespace(), o.wait)) }, } @@ -89,53 +87,68 @@ Use '--force' to skip the prompt confirming if you want to uninstall Velero. } // Run removes all components that were deployed using the Velero install command -func Run(ctx context.Context, client *kubernetes.Clientset, extensionsClient *apiextensionsclientset.Clientset, namespace string, waitToTerminate bool) error { +func Run(ctx context.Context, kbClient kbclient.Client, namespace string, waitToTerminate bool) error { var errs []error // namespace - ns, err := client.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) - if err != nil { + ns := &corev1.Namespace{} + key := kbclient.ObjectKey{Name: namespace} + if err := kbClient.Get(ctx, key, ns); err != nil { if apierrors.IsNotFound(err) { - fmt.Printf("Velero installation namespace %q does not exist, skipping.\n", namespace) + fmt.Printf("Velero namespace %q does not exist, skipping.\n", namespace) } else { errs = append(errs, errors.WithStack(err)) } } else { if ns.Status.Phase == corev1.NamespaceTerminating { - fmt.Printf("Velero installation namespace %q is terminating.\n", namespace) + fmt.Printf("Velero namespace %q is terminating.\n", namespace) } else { - err = client.CoreV1().Namespaces().Delete(ctx, ns.Name, metav1.DeleteOptions{}) - if err != nil { + if err := kbClient.Delete(ctx, ns); err != nil { errs = append(errs, errors.WithStack(err)) } } } - // rolebinding + // ClusterRoleBinding crb := install.ClusterRoleBinding(namespace) - if err := client.RbacV1().ClusterRoleBindings().Delete(ctx, crb.Name, metav1.DeleteOptions{}); err != nil { + key = kbclient.ObjectKey{Name: crb.Name} + if err := kbClient.Get(ctx, key, crb); err != nil { if apierrors.IsNotFound(err) { - fmt.Printf("Velero installation clusterrolebinding %q does not exist, skipping.\n", crb.Name) + fmt.Printf("Velero ClusterRoleBinding %q does not exist, skipping.\n", crb.Name) } else { errs = append(errs, errors.WithStack(err)) } + } else { + if err := kbClient.Delete(ctx, crb); err != nil { + errs = append(errs, errors.WithStack(err)) + } } // CRDs veleroLabels := labels.FormatLabels(install.Labels()) - crds, err := extensionsClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{ - LabelSelector: veleroLabels, - }) - if err != nil { - errs = append(errs, errors.WithStack(err)) + crdList := apiextv1beta1.CustomResourceDefinitionList{} + opts := kbclient.ListOptions{ + Namespace: namespace, + Raw: &metav1.ListOptions{ + LabelSelector: veleroLabels, + }, } - if len(crds.Items) == 0 { - fmt.Print("Velero CRDs do not exist, skipping.\n") + if err := kbClient.List(context.Background(), &crdList, &opts); err != nil { + errs = append(errs, errors.WithStack(err)) } else { - for _, removeCRD := range crds.Items { - if err = extensionsClient.ApiextensionsV1().CustomResourceDefinitions().Delete(ctx, removeCRD.ObjectMeta.Name, metav1.DeleteOptions{}); err != nil { - err2 := errors.WithMessagef(err, "Uninstall failed removing CRD %s", removeCRD.ObjectMeta.Name) - errs = append(errs, errors.WithStack(err2)) + if len(crdList.Items) == 0 { + fmt.Print("Velero CRDs do not exist, skipping.\n") + } else { + veleroLabelSelector := labels.SelectorFromSet(install.Labels()) + opts := []kbclient.DeleteAllOfOption{ + kbclient.InNamespace(namespace), + kbclient.MatchingLabelsSelector{ + Selector: veleroLabelSelector, + }, + } + crd := &apiextv1beta1.CustomResourceDefinition{} + if err := kbClient.DeleteAllOf(ctx, crd, opts...); err != nil { + errs = append(errs, errors.WithStack(err)) } } } @@ -147,7 +160,7 @@ func Run(ctx context.Context, client *kubernetes.Clientset, extensionsClient *ap defer cancel() checkFunc := func() { - _, err := client.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + err := kbClient.Get(ctx, key, ns) if err != nil { if apierrors.IsNotFound(err) { fmt.Print("\n") diff --git a/pkg/util/kube/utils.go b/pkg/util/kube/utils.go index c8262080d..994e8f370 100644 --- a/pkg/util/kube/utils.go +++ b/pkg/util/kube/utils.go @@ -24,15 +24,12 @@ import ( "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" corev1listers "k8s.io/client-go/listers/core/v1" - "k8s.io/client-go/tools/clientcmd" ) // NamespaceAndName returns a string in the format / @@ -220,26 +217,3 @@ func IsUnstructuredCRDReady(crd *unstructured.Unstructured) (bool, error) { return (isEstablished && namesAccepted), nil } - -// GetClusterClient instantiates and returns a client for the cluster. -func GetClusterClient() (*kubernetes.Clientset, *apiextensionsclientset.Clientset, error) { - loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() - configOverrides := &clientcmd.ConfigOverrides{} - kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) - clientConfig, err := kubeConfig.ClientConfig() - if err != nil { - return nil, nil, errors.WithStack(err) - } - - client, err := kubernetes.NewForConfig(clientConfig) - if err != nil { - return nil, nil, errors.WithStack(err) - } - - extensionClientSet, err := apiextensionsclientset.NewForConfig(clientConfig) - if err != nil { - return nil, nil, errors.WithStack(err) - } - - return client, extensionClientSet, nil -} diff --git a/test/e2e/Makefile b/test/e2e/Makefile index 2d43d85ae..fb85a824b 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -48,7 +48,7 @@ OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin GINKGO_FOCUS ?= VELERO_CLI ?=$$(pwd)/../../_output/bin/$(GOOS)/$(GOARCH)/velero VELERO_IMAGE ?= velero/velero:main -VELERO_NAMESPACE ?= +VELERO_NAMESPACE ?= velero CREDS_FILE ?= BSL_BUCKET ?= BSL_PREFIX ?= diff --git a/test/e2e/README.md b/test/e2e/README.md index 08eeb40c3..4f147b97b 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -105,3 +105,11 @@ For example, E2E tests can be run from Velero repository roots using the command Velero E2E tests uses [Ginkgo](https://onsi.github.io/ginkgo/) testing framework which allows a subset of the tests to be run using the [`-focus` and `-skip`](https://onsi.github.io/ginkgo/#focused-specs) flags to ginkgo. The `-focus` flag is passed to ginkgo using the `GINKGO_FOCUS` make variable. This can be used to focus on specific tests. + +## Adding tests + +### API clients +When adding a test, aim to instantiate an API client only once at the beginning of the test. There is a constructor `newTestClient` that facilitates the configuration and instantiation of clients. Also, please use the `kubebuilder` runtime controller client for any new test, as we will phase out usage of `client-go` API clients. + +### Tips +Look for the ⛵ emoji printed at the end of each install and uninstall log. There should not be two install/unintall in a row, and there should be tests between an install and an uninstall. \ No newline at end of file diff --git a/test/e2e/backup_test.go b/test/e2e/backup_test.go index 5de97a7c1..7474679e1 100644 --- a/test/e2e/backup_test.go +++ b/test/e2e/backup_test.go @@ -1,31 +1,28 @@ /* Copyright the Velero contributors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - package e2e import ( "context" "flag" "fmt" - "time" "github.com/google/uuid" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" - "k8s.io/client-go/kubernetes" - - "github.com/vmware-tanzu/velero/pkg/util/kube" ) var ( @@ -47,12 +44,12 @@ func backup_restore_with_restic() { func backup_restore_test(useVolumeSnapshots bool) { var ( - client *kubernetes.Clientset - extensionsClient *apiextensionsclientset.Clientset - backupName string - restoreName string + backupName, restoreName string ) + client, err := newTestClient() + Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") + BeforeEach(func() { if useVolumeSnapshots && cloudProvider == "kind" { Skip("Volume snapshots not supported on kind") @@ -64,19 +61,14 @@ func backup_restore_test(useVolumeSnapshots bool) { if installVelero { Expect(veleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, "")).To(Succeed()) - } - client, extensionsClient, err = kube.GetClusterClient() - Expect(err).To(Succeed(), "Failed to instantiate cluster client") }) AfterEach(func() { if installVelero { - timeoutCTX, _ := context.WithTimeout(context.Background(), time.Minute) - err := veleroUninstall(timeoutCTX, client, extensionsClient, veleroNamespace) + err = veleroUninstall(context.Background(), client.kubebuilder, installVelero, veleroNamespace) Expect(err).To(Succeed()) } - }) When("kibishii is the sample workload", func() { diff --git a/test/e2e/client.go b/test/e2e/client.go new file mode 100644 index 000000000..0b1993934 --- /dev/null +++ b/test/e2e/client.go @@ -0,0 +1,79 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "k8s.io/client-go/kubernetes" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/velero/pkg/client" +) + +// testClient contains different API clients that are in use throughout +// the e2e tests. + +type testClient struct { + kubebuilder kbclient.Client + + // clientGo returns a client-go API client. + // + // Deprecated, TODO(2.0): presuming all controllers and resources are converted to the + // controller runtime framework by v2.0, it is the intent to remove all + // client-go API clients. Please use the controller runtime to make API calls for tests. + clientGo kubernetes.Interface + + // dynamicFactory returns a client-go API client for retrieving dynamic clients + // for GroupVersionResources and GroupVersionKinds. + // + // Deprecated, TODO(2.0): presuming all controllers and resources are converted to the + // controller runtime framework by v2.0, it is the intent to remove all + // client-go API clients. Please use the controller runtime to make API calls for tests. + dynamicFactory client.DynamicFactory +} + +// newTestClient returns a set of ready-to-use API clients. +func newTestClient() (testClient, error) { + config, err := client.LoadConfig() + if err != nil { + return testClient{}, err + } + + f := client.NewFactory("e2e", config) + + clientGo, err := f.KubeClient() + if err != nil { + return testClient{}, err + } + + kb, err := f.KubebuilderClient() + if err != nil { + return testClient{}, err + } + + dynamicClient, err := f.DynamicClient() + if err != nil { + return testClient{}, err + } + + factory := client.NewDynamicFactory(dynamicClient) + + return testClient{ + kubebuilder: kb, + clientGo: clientGo, + dynamicFactory: factory, + }, nil +} diff --git a/test/e2e/common.go b/test/e2e/common.go index d080e3746..ce553d815 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -1,9 +1,12 @@ /* Copyright the Velero contributors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,7 +29,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" "github.com/vmware-tanzu/velero/pkg/builder" ) @@ -37,9 +39,9 @@ func ensureClusterExists(ctx context.Context) error { } // createNamespace creates a kubernetes namespace -func createNamespace(ctx context.Context, client *kubernetes.Clientset, namespace string) error { +func createNamespace(ctx context.Context, client testClient, namespace string) error { ns := builder.ForNamespace(namespace).Result() - _, err := client.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + _, err := client.clientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { return nil } @@ -47,9 +49,9 @@ func createNamespace(ctx context.Context, client *kubernetes.Clientset, namespac } // waitForNamespaceDeletion waits for namespace to be deleted. -func waitForNamespaceDeletion(interval, timeout time.Duration, client *kubernetes.Clientset, ns string) error { +func waitForNamespaceDeletion(interval, timeout time.Duration, client testClient, ns string) error { err := wait.PollImmediate(interval, timeout, func() (bool, error) { - _, err := client.CoreV1().Namespaces().Get(context.TODO(), ns, metav1.GetOptions{}) + _, err := client.clientGo.CoreV1().Namespaces().Get(context.TODO(), ns, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { return true, nil @@ -62,7 +64,7 @@ func waitForNamespaceDeletion(interval, timeout time.Duration, client *kubernete return err } -func createSecretFromFiles(ctx context.Context, client *kubernetes.Clientset, namespace string, name string, files map[string]string) error { +func createSecretFromFiles(ctx context.Context, client testClient, namespace string, name string, files map[string]string) error { data := make(map[string][]byte) for key, filePath := range files { @@ -75,17 +77,17 @@ func createSecretFromFiles(ctx context.Context, client *kubernetes.Clientset, na } secret := builder.ForSecret(namespace, name).Data(data).Result() - _, err := client.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) + _, err := client.clientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) return err } // waitForPods waits until all of the pods have gone to PodRunning state -func waitForPods(ctx context.Context, client *kubernetes.Clientset, namespace string, pods []string) error { +func waitForPods(ctx context.Context, client testClient, namespace string, pods []string) error { timeout := 10 * time.Minute interval := 5 * time.Second err := wait.PollImmediate(interval, timeout, func() (bool, error) { for _, podName := range pods { - checkPod, err := client.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + checkPod, err := client.clientGo.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) if err != nil { return false, errors.WithMessage(err, fmt.Sprintf("Failed to verify pod %s/%s is %s", namespace, podName, corev1api.PodRunning)) } diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index e00969283..0429070ec 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -1,9 +1,12 @@ /* Copyright the Velero contributors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/test/e2e/enable_api_group_versions_test.go b/test/e2e/enable_api_group_versions_test.go index 7198f0e4a..c06659eb7 100644 --- a/test/e2e/enable_api_group_versions_test.go +++ b/test/e2e/enable_api_group_versions_test.go @@ -1,9 +1,12 @@ /* Copyright the Velero contributors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,17 +28,12 @@ import ( "github.com/google/uuid" - "github.com/vmware-tanzu/velero/pkg/util/kube" - - apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" "github.com/vmware-tanzu/velero/pkg/builder" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" @@ -43,14 +41,15 @@ import ( var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", func() { var ( - resource, group string - certMgrCRD map[string]string - client *kubernetes.Clientset - extensionsClient *apiextensionsclient.Clientset - err error - ctx = context.Background() + resource, group string + certMgrCRD map[string]string + err error + ctx = context.Background() ) + client, err := newTestClient() + Expect(err).To(Succeed(), "Failed to instantiate cluster client for group version tests") + BeforeEach(func() { resource = "rockbands" group = "music.example.io" @@ -59,9 +58,6 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", "namespace": "cert-manager", } - client, extensionsClient, err = kube.GetClusterClient() // Currently we ignore the API extensions client - Expect(err).NotTo(HaveOccurred()) - err = installCRD(ctx, certMgrCRD["url"], certMgrCRD["namespace"]) Expect(err).NotTo(HaveOccurred()) @@ -83,17 +79,15 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", It("Should back up API group version and restore by version priority", func() { Expect(runEnableAPIGroupVersionsTests( ctx, + client, resource, group, - client, - extensionsClient, )).To(Succeed(), "Failed to successfully backup and restore multiple API Groups") }) }) }) -func runEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, client *kubernetes.Clientset, - extensionsClient *apiextensionsclient.Clientset) error { +func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, resource, group string) error { tests := []struct { name string namespaces []string @@ -257,7 +251,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, } for _, ns := range tc.namespaces { - if err := client.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}); err != nil { + if err := client.clientGo.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}); err != nil { return errors.Wrapf(err, "deleting %s namespace from source cluster", ns) } @@ -273,7 +267,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, // Apply config map if there is one. if tc.cm != nil { - _, err := client.CoreV1().ConfigMaps(veleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{}) + _, err := client.clientGo.CoreV1().ConfigMaps(veleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{}) if err != nil { return errors.Wrap(err, "creating config map with user version priorities") } @@ -338,7 +332,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, // Delete namespaces created for CRs for _, ns := range tc.namespaces { fmt.Println("Delete namespace", ns) - _ = client.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}) + _ = client.clientGo.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}) _ = waitNamespaceDelete(ctx, ns) } @@ -356,12 +350,9 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, tc.srcCRD["namespace"], ) - // Uninstall Velero - if installVelero { - err = veleroUninstall(ctx, client, extensionsClient, veleroNamespace) - if err != nil { - return err - } + err = veleroUninstall(context.Background(), client.kubebuilder, installVelero, veleroNamespace) + if err != nil { + return err } } diff --git a/test/e2e/kibishii_tests.go b/test/e2e/kibishii_tests.go index 6e2b58862..c2313105c 100644 --- a/test/e2e/kibishii_tests.go +++ b/test/e2e/kibishii_tests.go @@ -1,9 +1,12 @@ /* Copyright the Velero contributors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,7 +25,6 @@ import ( "github.com/pkg/errors" "golang.org/x/net/context" - "k8s.io/client-go/kubernetes" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -96,7 +98,7 @@ func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel } // runKibishiiTests runs kibishii tests on the provider. -func runKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string, +func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string, useVolumeSnapshots bool) error { fiveMinTimeout, _ := context.WithTimeout(context.Background(), 5*time.Minute) oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) @@ -136,7 +138,7 @@ func runKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, vel } } fmt.Printf("Simulating a disaster by removing namespace %s\n", kibishiiNamespace) - if err := client.CoreV1().Namespaces().Delete(oneHourTimeout, kibishiiNamespace, metav1.DeleteOptions{}); err != nil { + if err := client.clientGo.CoreV1().Namespaces().Delete(oneHourTimeout, kibishiiNamespace, metav1.DeleteOptions{}); err != nil { return errors.Wrap(err, "Failed to simulate a disaster") } // wait for ns delete @@ -163,7 +165,7 @@ func runKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, vel return errors.Wrap(err, "Failed to verify data generated by kibishii") } - if err := client.CoreV1().Namespaces().Delete(oneHourTimeout, kibishiiNamespace, metav1.DeleteOptions{}); err != nil { + if err := client.clientGo.CoreV1().Namespaces().Delete(oneHourTimeout, kibishiiNamespace, metav1.DeleteOptions{}); err != nil { return errors.Wrapf(err, "Failed to cleanup %s wrokload namespace", kibishiiNamespace) } // wait for ns delete @@ -174,6 +176,6 @@ func runKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, vel return nil } -func waitForKibishiiPods(ctx context.Context, client *kubernetes.Clientset, kibishiiNamespace string) error { +func waitForKibishiiPods(ctx context.Context, client testClient, kibishiiNamespace string) error { return waitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"}) } diff --git a/test/e2e/velero_utils.go b/test/e2e/velero_utils.go index 9b6fba708..6da152aab 100644 --- a/test/e2e/velero_utils.go +++ b/test/e2e/velero_utils.go @@ -1,9 +1,12 @@ /* Copyright the Velero contributors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,21 +28,16 @@ import ( "strings" "time" - "k8s.io/apimachinery/pkg/util/wait" - - veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" - "github.com/pkg/errors" - "k8s.io/client-go/kubernetes" - - apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + "k8s.io/apimachinery/pkg/util/wait" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - "github.com/vmware-tanzu/velero/pkg/client" cliinstall "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" "github.com/vmware-tanzu/velero/pkg/cmd/cli/uninstall" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" "github.com/vmware-tanzu/velero/pkg/install" + veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" ) func getProviderPlugins(providerName string) []string { @@ -99,45 +97,37 @@ func getProviderVeleroInstallOptions( // installVeleroServer installs velero in the cluster. func installVeleroServer(io *cliinstall.InstallOptions) error { - config, err := client.LoadConfig() - if err != nil { - return err - } - vo, err := io.AsVeleroOptions() if err != nil { return errors.Wrap(err, "Failed to translate InstallOptions to VeleroOptions for Velero") } - f := client.NewFactory("e2e", config) - resources := install.AllResources(vo) + client, err := newTestClient() if err != nil { - return errors.Wrap(err, "Failed to install Velero in the cluster") + return errors.Wrap(err, "Failed to instantiate cluster client for installing Velero") } - dynamicClient, err := f.DynamicClient() - if err != nil { - return err - } - factory := client.NewDynamicFactory(dynamicClient) errorMsg := "\n\nError installing Velero. Use `kubectl logs deploy/velero -n velero` to check the deploy logs" - err = install.Install(factory, resources, os.Stdout) + resources := install.AllResources(vo) + err = install.Install(client.dynamicFactory, resources, os.Stdout) if err != nil { return errors.Wrap(err, errorMsg) } fmt.Println("Waiting for Velero deployment to be ready.") - if _, err = install.DeploymentIsReady(factory, io.Namespace); err != nil { + if _, err = install.DeploymentIsReady(client.dynamicFactory, io.Namespace); err != nil { return errors.Wrap(err, errorMsg) } if io.UseRestic { fmt.Println("Waiting for Velero restic daemonset to be ready.") - if _, err = install.DaemonSetIsReady(factory, io.Namespace); err != nil { + if _, err = install.DaemonSetIsReady(client.dynamicFactory, io.Namespace); err != nil { return errors.Wrap(err, errorMsg) } } + fmt.Printf("Velero is installed and ready to be tested in the %s namespace! ⛵ \n", io.Namespace) + return nil } @@ -291,6 +281,7 @@ func veleroInstall(ctx context.Context, veleroImage string, veleroNamespace stri } } + // Fetch the plugins for the provider before checking for the object store provider below. providerPlugins := getProviderPlugins(objectStoreProvider) // TODO - handle this better @@ -302,10 +293,14 @@ func veleroInstall(ctx context.Context, veleroImage string, veleroNamespace stri } err := ensureClusterExists(ctx) if err != nil { - return errors.WithMessage(err, "Failed to ensure kubernetes cluster exists") + return errors.WithMessage(err, "Failed to ensure Kubernetes cluster exists") } + veleroInstallOptions, err := getProviderVeleroInstallOptions(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, providerPlugins, features) + if err != nil { + return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", objectStoreProvider) + } if useVolumeSnapshots { if cloudProvider != "vsphere" { veleroInstallOptions.UseVolumeSnapshots = true @@ -315,22 +310,20 @@ func veleroInstall(ctx context.Context, veleroImage string, veleroNamespace stri // being an AWS VSL which causes problems) } } - if err != nil { - return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", objectStoreProvider) - } veleroInstallOptions.UseRestic = !useVolumeSnapshots - veleroInstallOptions.Image = veleroImage veleroInstallOptions.Namespace = veleroNamespace + err = installVeleroServer(veleroInstallOptions) if err != nil { - return errors.WithMessagef(err, "Failed to install Velero in cluster") + return errors.WithMessagef(err, "Failed to install Velero in the cluster") } + return nil } -func veleroUninstall(ctx context.Context, client *kubernetes.Clientset, extensionsClient *apiextensionsclient.Clientset, veleroNamespace string) error { - return uninstall.Run(ctx, client, extensionsClient, veleroNamespace, true) +func veleroUninstall(ctx context.Context, client kbclient.Client, installVelero bool, veleroNamespace string) error { + return uninstall.Run(ctx, client, veleroNamespace, true) } func veleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error {