Allow multiple Pinnipeds to work on same cluster

Yes, this is a huge commit.

The middleware allows you to customize the API groups of all of the
*.pinniped.dev API groups.

Some notes about other small things in this commit:
- We removed the internal/client package in favor of pkg/conciergeclient. The
  two packages do basically the same thing. I don't think we use the former
  anymore.
- We re-enabled cluster-scoped owner assertions in the integration tests.
  This code was added in internal/ownerref. See a0546942 for when this
  assertion was removed.
- Note: the middlware code is in charge of restoring the GV of a request object,
  so we should never need to write mutations that do that.
- We updated the supervisor secret generation to no longer manually set an owner
  reference to the deployment since the middleware code now does this. I think we
  still need some way to make an initial event for the secret generator
  controller, which involves knowing the namespace and the name of the generated
  secret, so I still wired the deployment through. We could use a namespace/name
  tuple here, but I was lazy.

Signed-off-by: Andrew Keesler <akeesler@vmware.com>
Co-authored-by: Ryan Richard <richardry@vmware.com>

internal/ownerref/ownerref.go

@@ -4,36 +4,68 @@
 package ownerref
 
 import (
-	"net/http"
+	"context"
 
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"go.pinniped.dev/internal/kubeclient"
 )
 
-func New(ref metav1.OwnerReference) kubeclient.Middleware {
-	return ownerRefMiddleware(ref)
-}
+func New(refObj kubeclient.Object) kubeclient.Middleware {
+	ref := metav1.OwnerReference{
+		Name: refObj.GetName(),
+		UID:  refObj.GetUID(),
+	}
+	ref.APIVersion, ref.Kind = refObj.GetObjectKind().GroupVersionKind().ToAPIVersionAndKind()
+	refNamespace := refObj.GetNamespace()
 
-var _ kubeclient.Middleware = ownerRefMiddleware(metav1.OwnerReference{})
+	// if refNamespace is empty, we assume the owner ref is to a cluster scoped object which can own any object
+	refIsNamespaced := len(refNamespace) != 0
 
-type ownerRefMiddleware metav1.OwnerReference
-
-func (o ownerRefMiddleware) Handles(httpMethod string) bool {
-	return httpMethod == http.MethodPost // only handle create requests
-}
-
-// TODO this func assumes all objects are namespace scoped and are in the same namespace.
-//  i.e. it assumes all objects are safe to set an owner ref on
-//  i.e. the owner could be namespace scoped and thus cannot own a cluster scoped object
-//  this could be fixed by using a rest mapper to confirm the REST scoping
-//  or we could always use an owner ref to a cluster scoped object
-func (o ownerRefMiddleware) Mutate(obj metav1.Object) (mutated bool) {
-	// we only want to set the owner ref on create and when one is not already present
-	if len(obj.GetOwnerReferences()) != 0 {
-		return false
+	// special handling of namespaces to treat them as namespace scoped to themselves
+	if isNamespace(refObj) {
+		refNamespace = refObj.GetName()
+		refIsNamespaced = true
 	}
 
-	obj.SetOwnerReferences([]metav1.OwnerReference{metav1.OwnerReference(o)})
-	return true
+	return kubeclient.MiddlewareFunc(func(_ context.Context, rt kubeclient.RoundTrip) {
+		// we should not mess with owner refs on things we did not create
+		if rt.Verb() != kubeclient.VerbCreate {
+			return
+		}
+
+		// we probably do not want to set an owner ref on a subresource
+		if len(rt.Subresource()) != 0 {
+			return
+		}
+
+		// when ref is not cluster scoped, we ignore cluster scoped resources
+		if refIsNamespaced && !rt.NamespaceScoped() {
+			return
+		}
+
+		// when ref is not cluster scoped, we require refNamespace to match
+		// the request namespace since cross namespace ownership is disallowed
+		if refIsNamespaced && refNamespace != rt.Namespace() {
+			return
+		}
+
+		rt.MutateRequest(func(obj kubeclient.Object) {
+			// we only want to set the owner ref on create and when one is not already present
+			if len(obj.GetOwnerReferences()) != 0 {
+				return
+			}
+
+			obj.SetOwnerReferences([]metav1.OwnerReference{ref})
+		})
+	})
+}
+
+//nolint: gochecknoglobals
+var namespaceGVK = corev1.SchemeGroupVersion.WithKind("Namespace")
+
+func isNamespace(obj kubeclient.Object) bool {
+	_, ok := obj.(*corev1.Namespace)
+	return ok || obj.GetObjectKind().GroupVersionKind() == namespaceGVK
 }