Allow multiple Pinnipeds to work on same cluster

Yes, this is a huge commit.

The middleware allows you to customize the API groups of all of the
*.pinniped.dev API groups.

Some notes about other small things in this commit:
- We removed the internal/client package in favor of pkg/conciergeclient. The
  two packages do basically the same thing. I don't think we use the former
  anymore.
- We re-enabled cluster-scoped owner assertions in the integration tests.
  This code was added in internal/ownerref. See a0546942 for when this
  assertion was removed.
- Note: the middlware code is in charge of restoring the GV of a request object,
  so we should never need to write mutations that do that.
- We updated the supervisor secret generation to no longer manually set an owner
  reference to the deployment since the middleware code now does this. I think we
  still need some way to make an initial event for the secret generator
  controller, which involves knowing the namespace and the name of the generated
  secret, so I still wired the deployment through. We could use a namespace/name
  tuple here, but I was lazy.

Signed-off-by: Andrew Keesler <akeesler@vmware.com>
Co-authored-by: Ryan Richard <richardry@vmware.com>

test/library/access.go

@@ -6,7 +6,6 @@ package library
 import (
 	"context"
 	"io/ioutil"
-	"net/http"
 	"os"
 	"os/exec"
 	"testing"
@@ -16,7 +15,6 @@ import (
 	"github.com/stretchr/testify/require"
 	v1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 )
@@ -33,12 +31,11 @@ const (
 // performing a Pinniped credential exchange.
 func AccessAsUserTest(
 	ctx context.Context,
-	adminClient kubernetes.Interface,
 	testUsername string,
 	clientUnderTest kubernetes.Interface,
 ) func(t *testing.T) {
 	return func(t *testing.T) {
-		addTestClusterUserCanViewEverythingRoleBinding(ctx, t, adminClient, testUsername)
+		addTestClusterUserCanViewEverythingRoleBinding(t, testUsername)
 
 		// Use the client which is authenticated as the test user to list namespaces
 		var listNamespaceResponse *v1.NamespaceList
@@ -55,14 +52,12 @@ func AccessAsUserTest(
 }
 
 func AccessAsUserWithKubectlTest(
-	ctx context.Context,
-	adminClient kubernetes.Interface,
 	testKubeConfigYAML string,
 	testUsername string,
 	expectedNamespace string,
 ) func(t *testing.T) {
 	return func(t *testing.T) {
-		addTestClusterUserCanViewEverythingRoleBinding(ctx, t, adminClient, testUsername)
+		addTestClusterUserCanViewEverythingRoleBinding(t, testUsername)
 
 		// Use the given kubeconfig with kubectl to list namespaces as the test user
 		var kubectlCommandOutput string
@@ -85,12 +80,11 @@ func AccessAsUserWithKubectlTest(
 // a group membership) after performing a Pinniped credential exchange.
 func AccessAsGroupTest(
 	ctx context.Context,
-	adminClient kubernetes.Interface,
 	testGroup string,
 	clientUnderTest kubernetes.Interface,
 ) func(t *testing.T) {
 	return func(t *testing.T) {
-		addTestClusterGroupCanViewEverythingRoleBinding(ctx, t, adminClient, testGroup)
+		addTestClusterGroupCanViewEverythingRoleBinding(t, testGroup)
 
 		// Use the client which is authenticated as the test user to list namespaces
 		var listNamespaceResponse *v1.NamespaceList
@@ -107,14 +101,12 @@ func AccessAsGroupTest(
 }
 
 func AccessAsGroupWithKubectlTest(
-	ctx context.Context,
-	adminClient kubernetes.Interface,
 	testKubeConfigYAML string,
 	testGroup string,
 	expectedNamespace string,
 ) func(t *testing.T) {
 	return func(t *testing.T) {
-		addTestClusterGroupCanViewEverythingRoleBinding(ctx, t, adminClient, testGroup)
+		addTestClusterGroupCanViewEverythingRoleBinding(t, testGroup)
 
 		// Use the given kubeconfig with kubectl to list namespaces as the test user
 		var kubectlCommandOutput string
@@ -130,67 +122,38 @@ func AccessAsGroupWithKubectlTest(
 	}
 }
 
-func addTestClusterUserCanViewEverythingRoleBinding(ctx context.Context, t *testing.T, adminClient kubernetes.Interface, testUsername string) {
+func addTestClusterUserCanViewEverythingRoleBinding(t *testing.T, testUsername string) {
 	t.Helper()
 
-	addTestClusterRoleBinding(ctx, t, adminClient, &rbacv1.ClusterRoleBinding{
-		TypeMeta: metav1.TypeMeta{},
-		ObjectMeta: metav1.ObjectMeta{
-			Name: "integration-test-user-readonly-role-binding",
-		},
-		Subjects: []rbacv1.Subject{{
+	CreateTestClusterRoleBinding(t,
+		rbacv1.Subject{
 			Kind:     rbacv1.UserKind,
 			APIGroup: rbacv1.GroupName,
 			Name:     testUsername,
-		}},
-		RoleRef: rbacv1.RoleRef{
+		},
+		rbacv1.RoleRef{
 			Kind:     "ClusterRole",
 			APIGroup: rbacv1.GroupName,
 			Name:     "view",
 		},
-	})
+	)
 }
 
-func addTestClusterGroupCanViewEverythingRoleBinding(ctx context.Context, t *testing.T, adminClient kubernetes.Interface, testGroup string) {
+func addTestClusterGroupCanViewEverythingRoleBinding(t *testing.T, testGroup string) {
 	t.Helper()
 
-	addTestClusterRoleBinding(ctx, t, adminClient, &rbacv1.ClusterRoleBinding{
-		TypeMeta: metav1.TypeMeta{},
-		ObjectMeta: metav1.ObjectMeta{
-			Name: "integration-test-group-readonly-role-binding",
-		},
-		Subjects: []rbacv1.Subject{{
+	CreateTestClusterRoleBinding(t,
+		rbacv1.Subject{
 			Kind:     rbacv1.GroupKind,
 			APIGroup: rbacv1.GroupName,
 			Name:     testGroup,
-		}},
-		RoleRef: rbacv1.RoleRef{
+		},
+		rbacv1.RoleRef{
 			Kind:     "ClusterRole",
 			APIGroup: rbacv1.GroupName,
 			Name:     "view",
 		},
-	})
-}
-
-func addTestClusterRoleBinding(ctx context.Context, t *testing.T, adminClient kubernetes.Interface, binding *rbacv1.ClusterRoleBinding) {
-	t.Helper()
-
-	_, err := adminClient.RbacV1().ClusterRoleBindings().Get(ctx, binding.Name, metav1.GetOptions{})
-	if err != nil {
-		statusError, isStatus := err.(*errors.StatusError)
-		require.True(t, isStatus, "Only StatusNotFound error would be acceptable, but error was: ", err.Error())
-		require.Equal(t, http.StatusNotFound, int(statusError.Status().Code))
-
-		_, err = adminClient.RbacV1().ClusterRoleBindings().Create(ctx, binding, metav1.CreateOptions{})
-		require.NoError(t, err)
-	}
-
-	t.Cleanup(func() {
-		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-		defer cancel()
-		err = adminClient.RbacV1().ClusterRoleBindings().Delete(ctx, binding.Name, metav1.DeleteOptions{})
-		require.NoError(t, err, "Test failed to clean up after itself")
-	})
+	)
 }
 
 func runKubectlGetNamespaces(t *testing.T, kubeConfigYAML string) (string, error) {

test/library/client.go

@@ -31,6 +31,7 @@ import (
 	idpv1alpha1 "go.pinniped.dev/generated/1.20/apis/supervisor/idp/v1alpha1"
 	conciergeclientset "go.pinniped.dev/generated/1.20/client/concierge/clientset/versioned"
 	supervisorclientset "go.pinniped.dev/generated/1.20/client/supervisor/clientset/versioned"
+	"go.pinniped.dev/internal/groupsuffix"
 	"go.pinniped.dev/internal/kubeclient"
 
 	// Import to initialize client auth plugins - the kubeconfig that we use for
@@ -44,12 +45,6 @@ func NewClientConfig(t *testing.T) *rest.Config {
 	return newClientConfigWithOverrides(t, &clientcmd.ConfigOverrides{})
 }
 
-func NewClientset(t *testing.T) kubernetes.Interface {
-	t.Helper()
-
-	return newClientsetWithConfig(t, NewClientConfig(t))
-}
-
 func NewClientsetForKubeConfig(t *testing.T, kubeConfig string) kubernetes.Interface {
 	t.Helper()
 	return newClientsetWithConfig(t, NewRestConfigFromKubeconfig(t, kubeConfig))
@@ -74,6 +69,12 @@ func NewClientsetWithCertAndKey(t *testing.T, clientCertificateData, clientKeyDa
 	return newClientsetWithConfig(t, newAnonymousClientRestConfigWithCertAndKeyAdded(t, clientCertificateData, clientKeyData))
 }
 
+func NewKubernetesClientset(t *testing.T) kubernetes.Interface {
+	t.Helper()
+
+	return newKubeclient(t, NewClientConfig(t)).Kubernetes
+}
+
 func NewSupervisorClientset(t *testing.T) supervisorclientset.Interface {
 	t.Helper()
 
@@ -135,8 +136,11 @@ func newAnonymousClientRestConfigWithCertAndKeyAdded(t *testing.T, clientCertifi
 
 func newKubeclient(t *testing.T, config *rest.Config) *kubeclient.Client {
 	t.Helper()
-	_ = IntegrationEnv(t).APIGroupSuffix // TODO: wire API group into kubeclient.
-	client, err := kubeclient.New(kubeclient.WithConfig(config))
+	env := IntegrationEnv(t)
+	client, err := kubeclient.New(
+		kubeclient.WithConfig(config),
+		kubeclient.WithMiddleware(groupsuffix.New(env.APIGroupSuffix)),
+	)
 	require.NoError(t, err)
 	return client
 }
@@ -163,6 +167,12 @@ func CreateTestWebhookAuthenticator(ctx context.Context, t *testing.T) corev1.Ty
 
 	t.Cleanup(func() {
 		t.Helper()
+
+		if t.Failed() {
+			t.Logf("skipping deletion of test WebhookAuthenticator %s/%s", webhook.Namespace, webhook.Name)
+			return
+		}
+
 		t.Logf("cleaning up test WebhookAuthenticator %s/%s", webhook.Namespace, webhook.Name)
 		deleteCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 		defer cancel()
@@ -227,6 +237,12 @@ func CreateTestJWTAuthenticator(ctx context.Context, t *testing.T, spec auth1alp
 
 	t.Cleanup(func() {
 		t.Helper()
+
+		if t.Failed() {
+			t.Logf("skipping deletion of test JWTAuthenticator %s/%s", jwtAuthenticator.Namespace, jwtAuthenticator.Name)
+			return
+		}
+
 		t.Logf("cleaning up test JWTAuthenticator %s/%s", jwtAuthenticator.Namespace, jwtAuthenticator.Name)
 		deleteCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 		defer cancel()
@@ -325,7 +341,7 @@ func RandHex(t *testing.T, numBytes int) string {
 
 func CreateTestSecret(t *testing.T, namespace string, baseName string, secretType corev1.SecretType, stringData map[string]string) *corev1.Secret {
 	t.Helper()
-	client := NewClientset(t)
+	client := NewKubernetesClientset(t)
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 
@@ -396,7 +412,7 @@ func CreateTestOIDCIdentityProvider(t *testing.T, spec idpv1alpha1.OIDCIdentityP
 
 func CreateTestClusterRoleBinding(t *testing.T, subject rbacv1.Subject, roleRef rbacv1.RoleRef) *rbacv1.ClusterRoleBinding {
 	t.Helper()
-	client := NewClientset(t)
+	client := NewKubernetesClientset(t)
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 

test/library/dumplogs.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Pinniped contributors. All Rights Reserved.
+// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
 package library
@@ -21,7 +21,7 @@ func DumpLogs(t *testing.T, namespace string, labelSelector string) {
 		return
 	}
 
-	kubeClient := NewClientset(t)
+	kubeClient := NewKubernetesClientset(t)
 	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
 	defer cancel()