From ff1a31db4a05b3bd8166b068ad50071f71f80be5 Mon Sep 17 00:00:00 2001 From: Suraj Banakar <34534103+vadasambar@users.noreply.github.com> Date: Fri, 5 Mar 2021 00:46:40 +0530 Subject: [PATCH] Support cli uninstall (#3399) * Add uninstall cmd - init fn to uninstall velero - abstract dynamic client creation to a separate fn - creates a separate client per unstructured resource - add delete client for CRDs - export appendUnstructured - add uninstall command to main cmd - export `podTemplateOption` - uninstall resources in the reverse order of installation - fallback to `velero` if no ns is provided during uninstall - skip deletion if the resource doesn't exist - handle resource not found error - match log formatting with cli install logs - add Delete fn to fake client - fix import order - add changelog - add comment doc for CreateClient fn Signed-off-by: Suraj Banakar * Re-use uninstall code from test suite - move helper functions out of test suite - this is to prevent cyclic imports - move uninstall helpers to uninstall cmd - call them from test suite - revert export of variables/fns from install code - because not required anymore Signed-off-by: Suraj Banakar * Revert `PodTemplateOption` -> `podTemplateOption` Signed-off-by: Suraj Banakar * Use uninstall helper under VeleroUninstall - as a wrapper - fix import related errors in test suite Signed-off-by: Suraj Banakar --- changelogs/unreleased/3399-vadasambar | 1 + pkg/client/dynamic.go | 12 +++ pkg/cmd/cli/uninstall/uninstall.go | 114 +++++++++++++++++++++ pkg/cmd/velero/velero.go | 2 + pkg/install/install.go | 37 +++++-- pkg/test/fake_dynamic.go | 5 + pkg/util/kube/utils.go | 26 +++++ test/e2e/backup_test.go | 4 +- test/e2e/common.go | 45 -------- test/e2e/enable_api_group_versions_test.go | 3 +- test/e2e/velero_utils.go | 36 +------ 11 files changed, 196 insertions(+), 89 deletions(-) create mode 100644 changelogs/unreleased/3399-vadasambar create mode 100644 pkg/cmd/cli/uninstall/uninstall.go diff --git a/changelogs/unreleased/3399-vadasambar b/changelogs/unreleased/3399-vadasambar new file mode 100644 index 000000000..dc6ba54d5 --- /dev/null +++ b/changelogs/unreleased/3399-vadasambar @@ -0,0 +1 @@ +Add uninstall option for velero cli \ No newline at end of file diff --git a/pkg/client/dynamic.go b/pkg/client/dynamic.go index 63b41b5b6..114841805 100644 --- a/pkg/client/dynamic.go +++ b/pkg/client/dynamic.go @@ -82,6 +82,13 @@ type Patcher interface { Patch(name string, data []byte) (*unstructured.Unstructured, error) } +// Deletor deletes an object. +type Deletor interface { + //Patch patches the named object using the provided patch bytes, which are expected to be in JSON merge patch format. The patched object is returned. + + Delete(name string, opts metav1.DeleteOptions) error +} + // Dynamic contains client methods that Velero needs for backing up and restoring resources. type Dynamic interface { Creator @@ -89,6 +96,7 @@ type Dynamic interface { Watcher Getter Patcher + Deletor } // dynamicResourceClient implements Dynamic. @@ -117,3 +125,7 @@ func (d *dynamicResourceClient) Get(name string, opts metav1.GetOptions) (*unstr func (d *dynamicResourceClient) Patch(name string, data []byte) (*unstructured.Unstructured, error) { return d.resourceClient.Patch(context.TODO(), name, types.MergePatchType, data, metav1.PatchOptions{}) } + +func (d *dynamicResourceClient) Delete(name string, opts metav1.DeleteOptions) error { + return d.resourceClient.Delete(context.TODO(), name, opts) +} diff --git a/pkg/cmd/cli/uninstall/uninstall.go b/pkg/cmd/cli/uninstall/uninstall.go new file mode 100644 index 000000000..f7ce824ca --- /dev/null +++ b/pkg/cmd/cli/uninstall/uninstall.go @@ -0,0 +1,114 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package uninstall + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/pkg/errors" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + + "github.com/vmware-tanzu/velero/pkg/client" + "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/util/output" + "github.com/vmware-tanzu/velero/pkg/install" + "github.com/vmware-tanzu/velero/pkg/util/kube" +) + +// Uninstall uninstalls all components deployed using velero install command +func Uninstall(ctx context.Context, client *kubernetes.Clientset, extensionsClient *apiextensionsclientset.Clientset, veleroNamespace string) error { + if veleroNamespace == "" { + veleroNamespace = "velero" + } + err := DeleteNamespace(ctx, client, veleroNamespace) + if err != nil { + return errors.WithMessagef(err, "Uninstall failed removing Velero namespace %s", veleroNamespace) + } + + rolebinding := install.ClusterRoleBinding(veleroNamespace) + + err = client.RbacV1().ClusterRoleBindings().Delete(ctx, rolebinding.Name, metav1.DeleteOptions{}) + if err != nil { + return errors.WithMessagef(err, "Uninstall failed removing Velero cluster role binding %s", rolebinding) + } + veleroLabels := labels.FormatLabels(install.Labels()) + + crds, err := extensionsClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{ + LabelSelector: veleroLabels, + }) + if err != nil { + return errors.WithMessagef(err, "Uninstall failed listing Velero crds") + } + for _, removeCRD := range crds.Items { + err = extensionsClient.ApiextensionsV1().CustomResourceDefinitions().Delete(ctx, removeCRD.ObjectMeta.Name, metav1.DeleteOptions{}) + if err != nil { + return errors.WithMessagef(err, "Uninstall failed removing CRD %s", removeCRD.ObjectMeta.Name) + } + } + + fmt.Println("Uninstalled Velero") + return nil +} + +func DeleteNamespace(ctx context.Context, client *kubernetes.Clientset, namespace string) error { + err := client.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}) + if err != nil { + return errors.WithMessagef(err, "Delete namespace failed removing namespace %s", namespace) + } + return wait.Poll(1*time.Second, 3*time.Minute, func() (bool, error) { + _, err := client.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + if err != nil { + // Commented this out because not sure if removing this is okay + // Printing this on Uninstall will lead to confusion + // fmt.Printf("Namespaces.Get after delete return err %v\n", err) + return true, nil // Assume any error means the delete was successful + } + return false, nil + }) +} + +// NewCommand creates a cobra command. +func NewCommand(f client.Factory) *cobra.Command { + c := &cobra.Command{ + Use: "uninstall", + Short: "Uninstall Velero", + Long: `Uninstall Velero along with the CRDs. + +The '--namespace' flag can be used to specify the namespace where velero is installed (default: velero). + `, + Example: `# velero uninstall -n backup`, + Run: func(c *cobra.Command, args []string) { + veleroNs := strings.TrimSpace(f.Namespace()) + cl, extCl, err := kube.GetClusterClient() + cmd.CheckError(err) + cmd.CheckError(Uninstall(context.Background(), cl, extCl, veleroNs)) + }, + } + + output.BindFlags(c.Flags()) + output.ClearOutputFlagDefault(c) + return c +} diff --git a/pkg/cmd/velero/velero.go b/pkg/cmd/velero/velero.go index bbabf9ce4..3c83e80c3 100644 --- a/pkg/cmd/velero/velero.go +++ b/pkg/cmd/velero/velero.go @@ -41,6 +41,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/cmd/cli/restore" "github.com/vmware-tanzu/velero/pkg/cmd/cli/schedule" "github.com/vmware-tanzu/velero/pkg/cmd/cli/snapshotlocation" + "github.com/vmware-tanzu/velero/pkg/cmd/cli/uninstall" "github.com/vmware-tanzu/velero/pkg/cmd/cli/version" "github.com/vmware-tanzu/velero/pkg/cmd/server" runplugin "github.com/vmware-tanzu/velero/pkg/cmd/server/plugin" @@ -103,6 +104,7 @@ operations can also be performed as 'velero backup get' and 'velero schedule cre version.NewCommand(f), get.NewCommand(f), install.NewCommand(f), + uninstall.NewCommand(f), describe.NewCommand(f), create.NewCommand(f), runplugin.NewCommand(f), diff --git a/pkg/install/install.go b/pkg/install/install.go index acccd7f51..88e12a71a 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -236,16 +236,9 @@ func createResource(r *unstructured.Unstructured, factory client.DynamicFactory, } log("attempting to create resource") - gvk := schema.FromAPIVersionAndKind(r.GetAPIVersion(), r.GetKind()) - - apiResource := metav1.APIResource{ - Name: kindToResource[r.GetKind()], - Namespaced: (r.GetNamespace() != ""), - } - - c, err := factory.ClientForGroupVersionResource(gvk.GroupVersion(), apiResource, r.GetNamespace()) + c, err := CreateClient(r, factory, w) if err != nil { - return errors.Wrapf(err, "Error creating client for resource %s", id) + return err } if _, err := c.Create(r); apierrors.IsAlreadyExists(err) { @@ -258,6 +251,32 @@ func createResource(r *unstructured.Unstructured, factory client.DynamicFactory, return nil } +// CreateClient creates a client for an unstructured resource +func CreateClient(r *unstructured.Unstructured, factory client.DynamicFactory, w io.Writer) (client.Dynamic, error) { + id := fmt.Sprintf("%s/%s", r.GetKind(), r.GetName()) + + // Helper to reduce boilerplate message about the same object + log := func(f string, a ...interface{}) { + format := strings.Join([]string{id, ": ", f, "\n"}, "") + fmt.Fprintf(w, format, a...) + } + log("attempting to create resource client") + + gvk := schema.FromAPIVersionAndKind(r.GetAPIVersion(), r.GetKind()) + + apiResource := metav1.APIResource{ + Name: kindToResource[r.GetKind()], + Namespaced: (r.GetNamespace() != ""), + } + + c, err := factory.ClientForGroupVersionResource(gvk.GroupVersion(), apiResource, r.GetNamespace()) + if err != nil { + return nil, errors.Wrapf(err, "Error creating client for resource %s", id) + } + + return c, nil +} + // Install creates resources on the Kubernetes cluster. // An unstructured list of resources is sent, one at a time, to the server. These are assumed to be in the preferred order already. // Resources will be sorted into CustomResourceDefinitions and any other resource type, and the function will wait up to 1 minute diff --git a/pkg/test/fake_dynamic.go b/pkg/test/fake_dynamic.go index 04c26719a..2a90323d4 100644 --- a/pkg/test/fake_dynamic.go +++ b/pkg/test/fake_dynamic.go @@ -67,3 +67,8 @@ func (c *FakeDynamicClient) Patch(name string, data []byte) (*unstructured.Unstr args := c.Called(name, data) return args.Get(0).(*unstructured.Unstructured), args.Error(1) } + +func (c *FakeDynamicClient) Delete(name string, opts metav1.DeleteOptions) error { + args := c.Called(name, opts) + return args.Error(1) +} diff --git a/pkg/util/kube/utils.go b/pkg/util/kube/utils.go index 56bd33dba..e91c90b1d 100644 --- a/pkg/util/kube/utils.go +++ b/pkg/util/kube/utils.go @@ -24,12 +24,15 @@ import ( "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" corev1listers "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/clientcmd" ) // NamespaceAndName returns a string in the format / @@ -211,3 +214,26 @@ func IsUnstructuredCRDReady(crd *unstructured.Unstructured) (bool, error) { return (isEstablished && namesAccepted), nil } + +// GetClusterClient instantiates and returns a client for the cluster. +func GetClusterClient() (*kubernetes.Clientset, *apiextensionsclientset.Clientset, error) { + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + configOverrides := &clientcmd.ConfigOverrides{} + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) + clientConfig, err := kubeConfig.ClientConfig() + if err != nil { + return nil, nil, errors.WithStack(err) + } + + client, err := kubernetes.NewForConfig(clientConfig) + if err != nil { + return nil, nil, errors.WithStack(err) + } + + extensionClientSet, err := apiextensionsclientset.NewForConfig(clientConfig) + if err != nil { + return nil, nil, errors.WithStack(err) + } + + return client, extensionClientSet, nil +} diff --git a/test/e2e/backup_test.go b/test/e2e/backup_test.go index 22a676faa..fe6359974 100644 --- a/test/e2e/backup_test.go +++ b/test/e2e/backup_test.go @@ -10,6 +10,8 @@ import ( . "github.com/onsi/gomega" apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/client-go/kubernetes" + + "github.com/vmware-tanzu/velero/pkg/util/kube" ) var ( @@ -34,7 +36,7 @@ var _ = Describe("[Restic] Velero tests on cluster using the plugin provider for VeleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, "") } - client, extensionsClient, err = GetClusterClient() + client, extensionsClient, err = kube.GetClusterClient() Expect(err).To(Succeed(), "Failed to instantiate cluster client") }) diff --git a/test/e2e/common.go b/test/e2e/common.go index e3cbf8fec..c479bc159 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -1,19 +1,12 @@ package e2e import ( - "fmt" "os/exec" - "time" - "k8s.io/apimachinery/pkg/util/wait" - - "github.com/pkg/errors" "golang.org/x/net/context" - apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" "github.com/vmware-tanzu/velero/pkg/builder" ) @@ -23,29 +16,6 @@ func EnsureClusterExists(ctx context.Context) error { return exec.CommandContext(ctx, "kubectl", "cluster-info").Run() } -// GetClusterClient instantiates and returns a client for the cluster. -func GetClusterClient() (*kubernetes.Clientset, *apiextensionsclientset.Clientset, error) { - loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() - configOverrides := &clientcmd.ConfigOverrides{} - kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) - clientConfig, err := kubeConfig.ClientConfig() - if err != nil { - return nil, nil, errors.WithStack(err) - } - - client, err := kubernetes.NewForConfig(clientConfig) - if err != nil { - return nil, nil, errors.WithStack(err) - } - - extensionClientSet, err := apiextensionsclientset.NewForConfig(clientConfig) - if err != nil { - return nil, nil, errors.WithStack(err) - } - - return client, extensionClientSet, nil -} - // CreateNamespace creates a kubernetes namespace func CreateNamespace(ctx context.Context, client *kubernetes.Clientset, namespace string) error { ns := builder.ForNamespace(namespace).Result() @@ -55,18 +25,3 @@ func CreateNamespace(ctx context.Context, client *kubernetes.Clientset, namespac } return err } - -func DeleteNamespace(ctx context.Context, client *kubernetes.Clientset, namespace string) error { - err := client.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}) - if err != nil { - return errors.WithMessagef(err, "Delete namespace failed removing namespace %s", namespace) - } - return wait.Poll(1*time.Second, 3*time.Minute, func() (bool, error) { - _, err := client.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) - if err != nil { - fmt.Printf("Namespaces.Get after delete return err %v\n", err) - return true, nil // Assume any error means the delete was successful - } - return false, nil - }) -} diff --git a/test/e2e/enable_api_group_versions_test.go b/test/e2e/enable_api_group_versions_test.go index b75a508fe..29e463e48 100644 --- a/test/e2e/enable_api_group_versions_test.go +++ b/test/e2e/enable_api_group_versions_test.go @@ -23,6 +23,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/builder" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" + "github.com/vmware-tanzu/velero/pkg/util/kube" ) var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", func() { @@ -43,7 +44,7 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", "namespace": "cert-manager", } - client, extensionsClient, err = GetClusterClient() // Currently we ignore the API extensions client + client, extensionsClient, err = kube.GetClusterClient() // Currently we ignore the API extensions client Expect(err).NotTo(HaveOccurred()) err = InstallCRD(ctx, certMgrCRD["url"], certMgrCRD["namespace"]) diff --git a/test/e2e/velero_utils.go b/test/e2e/velero_utils.go index 47bc1897e..b120b6b81 100644 --- a/test/e2e/velero_utils.go +++ b/test/e2e/velero_utils.go @@ -9,10 +9,7 @@ import ( "os/exec" "path/filepath" - "k8s.io/apimachinery/pkg/labels" - "github.com/pkg/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" @@ -20,6 +17,7 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" cliinstall "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" + "github.com/vmware-tanzu/velero/pkg/cmd/cli/uninstall" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" "github.com/vmware-tanzu/velero/pkg/install" ) @@ -277,36 +275,8 @@ func VeleroInstall(ctx context.Context, veleroImage string, veleroNamespace stri return nil } -func VeleroUninstall(ctx context.Context, client *kubernetes.Clientset, extensionsClient *apiextensionsclient.Clientset, - veleroNamespace string) error { - // TODO - replace with invocation of "velero uninstall" when that becomes available - err := DeleteNamespace(ctx, client, veleroNamespace) - if err != nil { - return errors.WithMessagef(err, "Uninstall failed removing Velero namespace %s", veleroNamespace) - } - - return err - rolebinding := install.ClusterRoleBinding(veleroNamespace) - - err = client.RbacV1().ClusterRoleBindings().Delete(ctx, rolebinding.Name, metav1.DeleteOptions{}) - if err != nil { - return errors.WithMessagef(err, "Uninstall failed removing Velero cluster role binding %s", rolebinding) - } - veleroLabels := labels.FormatLabels(install.Labels()) - - crds, err := extensionsClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{ - LabelSelector: veleroLabels, - }) - if err != nil { - return errors.WithMessagef(err, "Uninstall failed listing Velero crds") - } - for _, removeCRD := range crds.Items { - err = extensionsClient.ApiextensionsV1().CustomResourceDefinitions().Delete(ctx, removeCRD.ObjectMeta.Name, metav1.DeleteOptions{}) - if err != nil { - return errors.WithMessagef(err, "Uninstall failed removing CRD %s", removeCRD.ObjectMeta.Name) - } - } - return nil +func VeleroUninstall(ctx context.Context, client *kubernetes.Clientset, extensionsClient *apiextensionsclient.Clientset, veleroNamespace string) error { + return uninstall.Uninstall(ctx, client, extensionsClient, veleroNamespace) } func VeleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error {