Allow multiple Pinnipeds to work on same cluster

Yes, this is a huge commit.

The middleware allows you to customize the API groups of all of the
*.pinniped.dev API groups.

Some notes about other small things in this commit:
- We removed the internal/client package in favor of pkg/conciergeclient. The
  two packages do basically the same thing. I don't think we use the former
  anymore.
- We re-enabled cluster-scoped owner assertions in the integration tests.
  This code was added in internal/ownerref. See a0546942 for when this
  assertion was removed.
- Note: the middlware code is in charge of restoring the GV of a request object,
  so we should never need to write mutations that do that.
- We updated the supervisor secret generation to no longer manually set an owner
  reference to the deployment since the middleware code now does this. I think we
  still need some way to make an initial event for the secret generator
  controller, which involves knowing the namespace and the name of the generated
  secret, so I still wired the deployment through. We could use a namespace/name
  tuple here, but I was lazy.

Signed-off-by: Andrew Keesler <akeesler@vmware.com>
Co-authored-by: Ryan Richard <richardry@vmware.com>

internal/controller/supervisorconfig/generator/supervisor_secrets.go

@@ -28,7 +28,6 @@ import (
 var generateKey = generateSymmetricKey
 
 type supervisorSecretsController struct {
-	owner          *appsv1.Deployment
 	labels         map[string]string
 	kubeClient     kubernetes.Interface
 	secretInformer corev1informers.SecretInformer
@@ -46,7 +45,6 @@ func NewSupervisorSecretsController(
 	initialEventFunc pinnipedcontroller.WithInitialEventOptionFunc,
 ) controllerlib.Controller {
 	c := supervisorSecretsController{
-		owner:          owner,
 		labels:         labels,
 		kubeClient:     kubeClient,
 		secretInformer: secretInformer,
@@ -64,13 +62,7 @@ func NewSupervisorSecretsController(
 				if secret.Type != SupervisorCSRFSigningKeySecretType {
 					return false
 				}
-				ownerReferences := secret.GetOwnerReferences()
-				for i := range secret.GetOwnerReferences() {
-					if ownerReferences[i].UID == owner.GetUID() {
-						return true
-					}
-				}
-				return false
+				return true
 			}, nil),
 			controllerlib.InformerOption{},
 		),
@@ -96,7 +88,7 @@ func (c *supervisorSecretsController) Sync(ctx controllerlib.Context) error {
 		return nil
 	}
 
-	newSecret, err := generateSecret(ctx.Key.Namespace, ctx.Key.Name, c.labels, secretDataFunc, c.owner)
+	newSecret, err := generateSecret(ctx.Key.Namespace, ctx.Key.Name, c.labels, secretDataFunc)
 	if err != nil {
 		return fmt.Errorf("failed to generate secret: %w", err)
 	}
@@ -193,27 +185,17 @@ func secretDataFunc() (map[string][]byte, error) {
 	}, nil
 }
 
-func generateSecret(namespace, name string, labels map[string]string, secretDataFunc func() (map[string][]byte, error), owner metav1.Object) (*corev1.Secret, error) {
+func generateSecret(namespace, name string, labels map[string]string, secretDataFunc func() (map[string][]byte, error)) (*corev1.Secret, error) {
 	secretData, err := secretDataFunc()
 	if err != nil {
 		return nil, err
 	}
 
-	deploymentGVK := appsv1.SchemeGroupVersion.WithKind("Deployment")
-
 	return &corev1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
 			Namespace: namespace,
-			OwnerReferences: []metav1.OwnerReference{
-				{
-					APIVersion: deploymentGVK.GroupVersion().String(),
-					Kind:       deploymentGVK.Kind,
-					Name:       owner.GetName(),
-					UID:        owner.GetUID(),
-				},
-			},
-			Labels: labels,
+			Labels:    labels,
 		},
 		Type: SupervisorCSRFSigningKeySecretType,
 		Data: secretData,

internal/controller/supervisorconfig/generator/supervisor_secrets_test.go

@@ -34,12 +34,6 @@ var (
 		},
 	}
 
-	ownerGVK = schema.GroupVersionKind{
-		Group:   appsv1.SchemeGroupVersion.Group,
-		Version: appsv1.SchemeGroupVersion.Version,
-		Kind:    "Deployment",
-	}
-
 	labels = map[string]string{
 		"some-label-key-1": "some-label-value-1",
 		"some-label-key-2": "some-label-value-2",
@@ -57,89 +51,13 @@ func TestSupervisorSecretsControllerFilterSecret(t *testing.T) {
 		wantDelete bool
 	}{
 		{
-			name: "owner reference is missing",
+			name: "owner reference is missing but Secret type is correct",
 			secret: &corev1.Secret{
 				Type: "secrets.pinniped.dev/supervisor-csrf-signing-key",
 				ObjectMeta: metav1.ObjectMeta{
 					Namespace: "some-namespace",
 				},
 			},
-		},
-		{
-			name: "owner reference with incorrect `APIVersion`",
-			secret: &corev1.Secret{
-				Type: "secrets.pinniped.dev/supervisor-csrf-signing-key",
-				ObjectMeta: metav1.ObjectMeta{
-					Namespace: "some-namespace",
-					OwnerReferences: []metav1.OwnerReference{
-						{
-							Name: owner.GetName(),
-							Kind: ownerGVK.Kind,
-							UID:  owner.GetUID(),
-						},
-					},
-				},
-			},
-			wantAdd:    true,
-			wantUpdate: true,
-			wantDelete: true,
-		},
-		{
-			name: "owner reference with incorrect `Kind`",
-			secret: &corev1.Secret{
-				Type: "secrets.pinniped.dev/supervisor-csrf-signing-key",
-				ObjectMeta: metav1.ObjectMeta{
-					Namespace: "some-namespace",
-					OwnerReferences: []metav1.OwnerReference{
-						{
-							APIVersion: ownerGVK.String(),
-							Name:       owner.GetName(),
-							Kind:       "IncorrectKind",
-							UID:        owner.GetUID(),
-						},
-					},
-				},
-			},
-			wantAdd:    true,
-			wantUpdate: true,
-			wantDelete: true,
-		},
-		{
-			name: "expected owner reference with incorrect `UID`",
-			secret: &corev1.Secret{
-				Type: "secrets.pinniped.dev/supervisor-csrf-signing-key",
-				ObjectMeta: metav1.ObjectMeta{
-					Namespace: "some-namespace",
-					OwnerReferences: []metav1.OwnerReference{
-						{
-							APIVersion: ownerGVK.String(),
-							Name:       owner.GetName(),
-							Kind:       ownerGVK.Kind,
-							UID:        "DOES_NOT_MATCH",
-						},
-					},
-				},
-			},
-		},
-		{
-			name: "multiple owner references (expected owner reference, and one more)",
-			secret: &corev1.Secret{
-				Type: "secrets.pinniped.dev/supervisor-csrf-signing-key",
-				ObjectMeta: metav1.ObjectMeta{
-					Namespace: "some-namespace",
-					OwnerReferences: []metav1.OwnerReference{
-						{
-							Kind: "UnrelatedKind",
-						},
-						{
-							APIVersion: ownerGVK.String(),
-							Name:       owner.GetName(),
-							Kind:       ownerGVK.Kind,
-							UID:        owner.GetUID(),
-						},
-					},
-				},
-			},
 			wantAdd:    true,
 			wantUpdate: true,
 			wantDelete: true,
@@ -152,10 +70,8 @@ func TestSupervisorSecretsControllerFilterSecret(t *testing.T) {
 					Namespace: "some-namespace",
 					OwnerReferences: []metav1.OwnerReference{
 						{
-							APIVersion: ownerGVK.String(),
-							Name:       owner.GetName(),
-							Kind:       ownerGVK.Kind,
-							UID:        owner.GetUID(),
+							Name: owner.GetName(),
+							UID:  owner.GetUID(),
 						},
 					},
 				},
@@ -166,32 +82,15 @@ func TestSupervisorSecretsControllerFilterSecret(t *testing.T) {
 			secret: &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "some-namespace"}},
 		},
 		{
-			name: "owner reference with `Controller`: true",
-			secret: &corev1.Secret{
-				Type: "secrets.pinniped.dev/supervisor-csrf-signing-key",
-				ObjectMeta: metav1.ObjectMeta{
-					Namespace: "some-namespace",
-					OwnerReferences: []metav1.OwnerReference{
-						*metav1.NewControllerRef(owner, ownerGVK),
-					},
-				},
-			},
-			wantAdd:    true,
-			wantUpdate: true,
-			wantDelete: true,
-		},
-		{
-			name: "expected owner reference - where `Controller`: false",
+			name: "realistic owner reference and correct Secret type",
 			secret: &corev1.Secret{
 				Type: "secrets.pinniped.dev/supervisor-csrf-signing-key",
 				ObjectMeta: metav1.ObjectMeta{
 					Namespace: "some-namespace",
 					OwnerReferences: []metav1.OwnerReference{
 						{
-							APIVersion: ownerGVK.String(),
-							Name:       owner.GetName(),
-							Kind:       ownerGVK.Kind,
-							UID:        owner.GetUID(),
+							Name: owner.GetName(),
+							UID:  owner.GetUID(),
 						},
 					},
 				},
@@ -272,15 +171,7 @@ func TestSupervisorSecretsControllerSync(t *testing.T) {
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      generatedSecretName,
 				Namespace: generatedSecretNamespace,
-				OwnerReferences: []metav1.OwnerReference{
-					{
-						APIVersion: ownerGVK.GroupVersion().String(),
-						Kind:       ownerGVK.Kind,
-						Name:       owner.GetName(),
-						UID:        owner.GetUID(),
-					},
-				},
-				Labels: labels,
+				Labels:    labels,
 			},
 			Type: "secrets.pinniped.dev/supervisor-csrf-signing-key",
 			Data: map[string][]byte{
@@ -292,15 +183,7 @@ func TestSupervisorSecretsControllerSync(t *testing.T) {
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      generatedSecretName,
 				Namespace: generatedSecretNamespace,
-				OwnerReferences: []metav1.OwnerReference{
-					{
-						APIVersion: ownerGVK.GroupVersion().String(),
-						Kind:       ownerGVK.Kind,
-						Name:       owner.GetName(),
-						UID:        owner.GetUID(),
-					},
-				},
-				Labels: labels,
+				Labels:    labels,
 			},
 			Type: "secrets.pinniped.dev/supervisor-csrf-signing-key",
 			Data: map[string][]byte{