Support cli uninstall (#3399)

* Add uninstall cmd
- init fn to uninstall velero
- abstract dynamic client creation to a separate fn
- creates a separate client per unstructured resource
- add delete client for CRDs
- export appendUnstructured
- add uninstall command to main cmd
- export `podTemplateOption`
- uninstall resources in the reverse order of installation
- fallback to `velero` if no ns is provided during uninstall
- skip deletion if the resource doesn't exist
- handle resource not found error
- match log formatting with cli install logs
- add Delete fn to fake client
- fix import order
- add changelog
- add comment doc for CreateClient fn

Signed-off-by: Suraj Banakar <suraj@infracloud.io>

* Re-use uninstall code from test suite
- move helper functions out of test suite
- this is to prevent cyclic imports
- move uninstall helpers to uninstall cmd
- call them from test suite
- revert export of variables/fns from install code
- because not required anymore

Signed-off-by: Suraj Banakar <suraj@infracloud.io>

* Revert `PodTemplateOption` -> `podTemplateOption`

Signed-off-by: Suraj Banakar <suraj@infracloud.io>

* Use uninstall helper under VeleroUninstall
- as a wrapper
- fix import related errors in test suite

Signed-off-by: Suraj Banakar <suraj@infracloud.io>

pkg/client/dynamic.go

@@ -82,6 +82,13 @@ type Patcher interface {
 	Patch(name string, data []byte) (*unstructured.Unstructured, error)
 }
 
+// Deletor deletes an object.
+type Deletor interface {
+	//Patch patches the named object using the provided patch bytes, which are expected to be in JSON merge patch format. The patched object is returned.
+
+	Delete(name string, opts metav1.DeleteOptions) error
+}
+
 // Dynamic contains client methods that Velero needs for backing up and restoring resources.
 type Dynamic interface {
 	Creator
@@ -89,6 +96,7 @@ type Dynamic interface {
 	Watcher
 	Getter
 	Patcher
+	Deletor
 }
 
 // dynamicResourceClient implements Dynamic.
@@ -117,3 +125,7 @@ func (d *dynamicResourceClient) Get(name string, opts metav1.GetOptions) (*unstr
 func (d *dynamicResourceClient) Patch(name string, data []byte) (*unstructured.Unstructured, error) {
 	return d.resourceClient.Patch(context.TODO(), name, types.MergePatchType, data, metav1.PatchOptions{})
 }
+
+func (d *dynamicResourceClient) Delete(name string, opts metav1.DeleteOptions) error {
+	return d.resourceClient.Delete(context.TODO(), name, opts)
+}

pkg/cmd/cli/uninstall/uninstall.go

@@ -0,0 +1,114 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package uninstall
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+
+	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+
+	"github.com/vmware-tanzu/velero/pkg/client"
+	"github.com/vmware-tanzu/velero/pkg/cmd"
+	"github.com/vmware-tanzu/velero/pkg/cmd/util/output"
+	"github.com/vmware-tanzu/velero/pkg/install"
+	"github.com/vmware-tanzu/velero/pkg/util/kube"
+)
+
+// Uninstall uninstalls all components deployed using velero install command
+func Uninstall(ctx context.Context, client *kubernetes.Clientset, extensionsClient *apiextensionsclientset.Clientset, veleroNamespace string) error {
+	if veleroNamespace == "" {
+		veleroNamespace = "velero"
+	}
+	err := DeleteNamespace(ctx, client, veleroNamespace)
+	if err != nil {
+		return errors.WithMessagef(err, "Uninstall failed removing Velero namespace %s", veleroNamespace)
+	}
+
+	rolebinding := install.ClusterRoleBinding(veleroNamespace)
+
+	err = client.RbacV1().ClusterRoleBindings().Delete(ctx, rolebinding.Name, metav1.DeleteOptions{})
+	if err != nil {
+		return errors.WithMessagef(err, "Uninstall failed removing Velero cluster role binding %s", rolebinding)
+	}
+	veleroLabels := labels.FormatLabels(install.Labels())
+
+	crds, err := extensionsClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{
+		LabelSelector: veleroLabels,
+	})
+	if err != nil {
+		return errors.WithMessagef(err, "Uninstall failed listing Velero crds")
+	}
+	for _, removeCRD := range crds.Items {
+		err = extensionsClient.ApiextensionsV1().CustomResourceDefinitions().Delete(ctx, removeCRD.ObjectMeta.Name, metav1.DeleteOptions{})
+		if err != nil {
+			return errors.WithMessagef(err, "Uninstall failed removing CRD %s", removeCRD.ObjectMeta.Name)
+		}
+	}
+
+	fmt.Println("Uninstalled Velero")
+	return nil
+}
+
+func DeleteNamespace(ctx context.Context, client *kubernetes.Clientset, namespace string) error {
+	err := client.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{})
+	if err != nil {
+		return errors.WithMessagef(err, "Delete namespace failed removing namespace %s", namespace)
+	}
+	return wait.Poll(1*time.Second, 3*time.Minute, func() (bool, error) {
+		_, err := client.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{})
+		if err != nil {
+			// Commented this out because not sure if removing this is okay
+			// Printing this on Uninstall will lead to confusion
+			// fmt.Printf("Namespaces.Get after delete return err %v\n", err)
+			return true, nil // Assume any error means the delete was successful
+		}
+		return false, nil
+	})
+}
+
+// NewCommand creates a cobra command.
+func NewCommand(f client.Factory) *cobra.Command {
+	c := &cobra.Command{
+		Use:   "uninstall",
+		Short: "Uninstall Velero",
+		Long: `Uninstall Velero along with the CRDs.
+
+The '--namespace' flag can be used to specify the namespace where velero is installed (default: velero).
+		`,
+		Example: `# velero uninstall -n backup`,
+		Run: func(c *cobra.Command, args []string) {
+			veleroNs := strings.TrimSpace(f.Namespace())
+			cl, extCl, err := kube.GetClusterClient()
+			cmd.CheckError(err)
+			cmd.CheckError(Uninstall(context.Background(), cl, extCl, veleroNs))
+		},
+	}
+
+	output.BindFlags(c.Flags())
+	output.ClearOutputFlagDefault(c)
+	return c
+}

pkg/cmd/velero/velero.go

@@ -41,6 +41,7 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/cmd/cli/restore"
 	"github.com/vmware-tanzu/velero/pkg/cmd/cli/schedule"
 	"github.com/vmware-tanzu/velero/pkg/cmd/cli/snapshotlocation"
+	"github.com/vmware-tanzu/velero/pkg/cmd/cli/uninstall"
 	"github.com/vmware-tanzu/velero/pkg/cmd/cli/version"
 	"github.com/vmware-tanzu/velero/pkg/cmd/server"
 	runplugin "github.com/vmware-tanzu/velero/pkg/cmd/server/plugin"
@@ -103,6 +104,7 @@ operations can also be performed as 'velero backup get' and 'velero schedule cre
 		version.NewCommand(f),
 		get.NewCommand(f),
 		install.NewCommand(f),
+		uninstall.NewCommand(f),
 		describe.NewCommand(f),
 		create.NewCommand(f),
 		runplugin.NewCommand(f),

pkg/install/install.go

@@ -236,16 +236,9 @@ func createResource(r *unstructured.Unstructured, factory client.DynamicFactory,
 	}
 	log("attempting to create resource")
 
-	gvk := schema.FromAPIVersionAndKind(r.GetAPIVersion(), r.GetKind())
-
-	apiResource := metav1.APIResource{
-		Name:       kindToResource[r.GetKind()],
-		Namespaced: (r.GetNamespace() != ""),
-	}
-
-	c, err := factory.ClientForGroupVersionResource(gvk.GroupVersion(), apiResource, r.GetNamespace())
+	c, err := CreateClient(r, factory, w)
 	if err != nil {
-		return errors.Wrapf(err, "Error creating client for resource %s", id)
+		return err
 	}
 
 	if _, err := c.Create(r); apierrors.IsAlreadyExists(err) {
@@ -258,6 +251,32 @@ func createResource(r *unstructured.Unstructured, factory client.DynamicFactory,
 	return nil
 }
 
+// CreateClient creates a client for an unstructured resource
+func CreateClient(r *unstructured.Unstructured, factory client.DynamicFactory, w io.Writer) (client.Dynamic, error) {
+	id := fmt.Sprintf("%s/%s", r.GetKind(), r.GetName())
+
+	// Helper to reduce boilerplate message about the same object
+	log := func(f string, a ...interface{}) {
+		format := strings.Join([]string{id, ": ", f, "\n"}, "")
+		fmt.Fprintf(w, format, a...)
+	}
+	log("attempting to create resource client")
+
+	gvk := schema.FromAPIVersionAndKind(r.GetAPIVersion(), r.GetKind())
+
+	apiResource := metav1.APIResource{
+		Name:       kindToResource[r.GetKind()],
+		Namespaced: (r.GetNamespace() != ""),
+	}
+
+	c, err := factory.ClientForGroupVersionResource(gvk.GroupVersion(), apiResource, r.GetNamespace())
+	if err != nil {
+		return nil, errors.Wrapf(err, "Error creating client for resource %s", id)
+	}
+
+	return c, nil
+}
+
 // Install creates resources on the Kubernetes cluster.
 // An unstructured list of resources is sent, one at a time, to the server. These are assumed to be in the preferred order already.
 // Resources will be sorted into CustomResourceDefinitions and any other resource type, and the function will wait up to 1 minute

pkg/test/fake_dynamic.go

@@ -67,3 +67,8 @@ func (c *FakeDynamicClient) Patch(name string, data []byte) (*unstructured.Unstr
 	args := c.Called(name, data)
 	return args.Get(0).(*unstructured.Unstructured), args.Error(1)
 }
+
+func (c *FakeDynamicClient) Delete(name string, opts metav1.DeleteOptions) error {
+	args := c.Called(name, opts)
+	return args.Error(1)
+}

pkg/util/kube/utils.go

@@ -24,12 +24,15 @@ import (
 	"github.com/pkg/errors"
 	corev1api "k8s.io/api/core/v1"
 	apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
 	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
 	corev1listers "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/tools/clientcmd"
 )
 
 // NamespaceAndName returns a string in the format <namespace>/<name>
@@ -211,3 +214,26 @@ func IsUnstructuredCRDReady(crd *unstructured.Unstructured) (bool, error) {
 
 	return (isEstablished && namesAccepted), nil
 }
+
+// GetClusterClient instantiates and returns a client for the cluster.
+func GetClusterClient() (*kubernetes.Clientset, *apiextensionsclientset.Clientset, error) {
+	loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
+	configOverrides := &clientcmd.ConfigOverrides{}
+	kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides)
+	clientConfig, err := kubeConfig.ClientConfig()
+	if err != nil {
+		return nil, nil, errors.WithStack(err)
+	}
+
+	client, err := kubernetes.NewForConfig(clientConfig)
+	if err != nil {
+		return nil, nil, errors.WithStack(err)
+	}
+
+	extensionClientSet, err := apiextensionsclientset.NewForConfig(clientConfig)
+	if err != nil {
+		return nil, nil, errors.WithStack(err)
+	}
+
+	return client, extensionClientSet, nil
+}