Allow multiple Pinnipeds to work on same cluster

Yes, this is a huge commit.

The middleware allows you to customize the API groups of all of the
*.pinniped.dev API groups.

Some notes about other small things in this commit:
- We removed the internal/client package in favor of pkg/conciergeclient. The
  two packages do basically the same thing. I don't think we use the former
  anymore.
- We re-enabled cluster-scoped owner assertions in the integration tests.
  This code was added in internal/ownerref. See a0546942 for when this
  assertion was removed.
- Note: the middlware code is in charge of restoring the GV of a request object,
  so we should never need to write mutations that do that.
- We updated the supervisor secret generation to no longer manually set an owner
  reference to the deployment since the middleware code now does this. I think we
  still need some way to make an initial event for the secret generator
  controller, which involves knowing the namespace and the name of the generated
  secret, so I still wired the deployment through. We could use a namespace/name
  tuple here, but I was lazy.

Signed-off-by: Andrew Keesler <akeesler@vmware.com>
Co-authored-by: Ryan Richard <richardry@vmware.com>

test/integration/cli_test.go

@@ -57,6 +57,7 @@ func TestCLIGetKubeconfigStaticToken(t *testing.T) {
 				"--pinniped-namespace", env.ConciergeNamespace,
 				"--authenticator-type", "webhook",
 				"--authenticator-name", authenticator.Name,
+				"--api-group-suffix", env.APIGroupSuffix,
 			},
 			expectStderr: "Command \"get-kubeconfig\" is deprecated, Please use `pinniped get kubeconfig` instead.\n",
 		},
@@ -84,16 +85,15 @@ func TestCLIGetKubeconfigStaticToken(t *testing.T) {
 
 			// In addition to the client-go based testing below, also try the kubeconfig
 			// with kubectl to validate that it works.
-			adminClient := library.NewClientset(t)
 			t.Run(
 				"access as user with kubectl",
-				library.AccessAsUserWithKubectlTest(ctx, adminClient, stdout, env.TestUser.ExpectedUsername, env.ConciergeNamespace),
+				library.AccessAsUserWithKubectlTest(stdout, env.TestUser.ExpectedUsername, env.ConciergeNamespace),
 			)
 			for _, group := range env.TestUser.ExpectedGroups {
 				group := group
 				t.Run(
 					"access as group "+group+" with kubectl",
-					library.AccessAsGroupWithKubectlTest(ctx, adminClient, stdout, group, env.ConciergeNamespace),
+					library.AccessAsGroupWithKubectlTest(stdout, group, env.ConciergeNamespace),
 				)
 			}
 
@@ -101,10 +101,10 @@ func TestCLIGetKubeconfigStaticToken(t *testing.T) {
 			kubeClient := library.NewClientsetForKubeConfig(t, stdout)
 
 			// Validate that we can auth to the API via our user.
-			t.Run("access as user with client-go", library.AccessAsUserTest(ctx, adminClient, env.TestUser.ExpectedUsername, kubeClient))
+			t.Run("access as user with client-go", library.AccessAsUserTest(ctx, env.TestUser.ExpectedUsername, kubeClient))
 			for _, group := range env.TestUser.ExpectedGroups {
 				group := group
-				t.Run("access as group "+group+" with client-go", library.AccessAsGroupTest(ctx, adminClient, group, kubeClient))
+				t.Run("access as group "+group+" with client-go", library.AccessAsGroupTest(ctx, group, kubeClient))
 			}
 		})
 	}

test/integration/concierge_api_serving_certs_test.go

@@ -73,7 +73,7 @@ func TestAPIServingCertificateAutoCreationAndRotation(t *testing.T) {
 	for _, test := range tests {
 		test := test
 		t.Run(test.name, func(t *testing.T) {
-			kubeClient := library.NewClientset(t)
+			kubeClient := library.NewKubernetesClientset(t)
 			aggregatedClient := library.NewAggregatedClientset(t)
 			conciergeClient := library.NewConciergeClientset(t)
 			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)

test/integration/concierge_availability_test.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Pinniped contributors. All Rights Reserved.
+// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
 package integration
@@ -18,7 +18,7 @@ import (
 
 func TestGetDeployment(t *testing.T) {
 	env := library.IntegrationEnv(t)
-	client := library.NewClientset(t)
+	client := library.NewKubernetesClientset(t)
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()

test/integration/concierge_credentialrequest_test.go

@@ -104,9 +104,6 @@ func TestSuccessfulCredentialRequest(t *testing.T) {
 			require.NotNil(t, response.Status.Credential.ExpirationTimestamp)
 			require.InDelta(t, 5*time.Minute, time.Until(response.Status.Credential.ExpirationTimestamp.Time), float64(time.Minute))
 
-			// Create a client using the admin kubeconfig.
-			adminClient := library.NewClientset(t)
-
 			// Create a client using the certificate from the CredentialRequest.
 			clientWithCertFromCredentialRequest := library.NewClientsetWithCertAndKey(
 				t,
@@ -116,13 +113,13 @@ func TestSuccessfulCredentialRequest(t *testing.T) {
 
 			t.Run(
 				"access as user",
-				library.AccessAsUserTest(ctx, adminClient, username, clientWithCertFromCredentialRequest),
+				library.AccessAsUserTest(ctx, username, clientWithCertFromCredentialRequest),
 			)
 			for _, group := range groups {
 				group := group
 				t.Run(
 					"access as group "+group,
-					library.AccessAsGroupTest(ctx, adminClient, group, clientWithCertFromCredentialRequest),
+					library.AccessAsGroupTest(ctx, group, clientWithCertFromCredentialRequest),
 				)
 			}
 		})

test/integration/concierge_kubecertagent_test.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Pinniped contributors. All Rights Reserved.
+// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
 package integration
@@ -30,7 +30,7 @@ func TestKubeCertAgent(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
 	defer cancel()
 
-	kubeClient := library.NewClientset(t)
+	kubeClient := library.NewKubernetesClientset(t)
 
 	// Get the current number of kube-cert-agent pods.
 	//

test/integration/kube_api_discovery_test.go

@@ -4,6 +4,7 @@
 package integration
 
 import (
+	"errors"
 	"fmt"
 	"strings"
 	"testing"
@@ -12,6 +13,7 @@ import (
 	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/discovery"
 
 	"go.pinniped.dev/test/library"
 )
@@ -19,9 +21,21 @@ import (
 func TestGetAPIResourceList(t *testing.T) {
 	env := library.IntegrationEnv(t)
 
-	client := library.NewClientset(t)
+	client := library.NewKubernetesClientset(t)
 	groups, resources, err := client.Discovery().ServerGroupsAndResources()
-	require.NoError(t, err)
+
+	// discovery can have partial failures when an API service is unavailable (i.e. because of TestAPIServingCertificateAutoCreationAndRotation)
+	// we ignore failures for groups that are not relevant to this test
+	if err != nil {
+		discoveryFailed := &discovery.ErrGroupDiscoveryFailed{}
+		isDiscoveryFailed := errors.As(err, &discoveryFailed)
+		require.True(t, isDiscoveryFailed, err)
+		for gv, gvErr := range discoveryFailed.Groups {
+			if strings.HasSuffix(gv.Group, "."+env.APIGroupSuffix) {
+				require.NoError(t, gvErr)
+			}
+		}
+	}
 
 	makeGV := func(firstSegment, secondSegment string) schema.GroupVersion {
 		return schema.GroupVersion{
@@ -195,12 +209,15 @@ func TestGetAPIResourceList(t *testing.T) {
 		for _, tt := range tests {
 			testedGroups[tt.group.Name] = true
 		}
+		foundPinnipedGroups := 0
 		for _, g := range groups {
 			if !strings.Contains(g.Name, env.APIGroupSuffix) {
 				continue
 			}
+			foundPinnipedGroups++
 			assert.Truef(t, testedGroups[g.Name], "expected group %q to have assertions defined", g.Name)
 		}
+		require.Equal(t, len(testedGroups), foundPinnipedGroups)
 	})
 
 	t.Run("every API categorized appropriately", func(t *testing.T) {

test/integration/kubeclient_test.go

@@ -17,6 +17,7 @@ import (
 
 	conciergeconfigv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/config/v1alpha1"
 	supervisorconfigv1alpha1 "go.pinniped.dev/generated/1.20/apis/supervisor/config/v1alpha1"
+	"go.pinniped.dev/internal/groupsuffix"
 	"go.pinniped.dev/internal/kubeclient"
 	"go.pinniped.dev/internal/ownerref"
 	"go.pinniped.dev/test/library"
@@ -25,7 +26,7 @@ import (
 func TestKubeClientOwnerRef(t *testing.T) {
 	env := library.IntegrationEnv(t)
 
-	regularClient := library.NewClientset(t)
+	regularClient := library.NewKubernetesClientset(t)
 
 	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
 	defer cancel()
@@ -62,16 +63,20 @@ func TestKubeClientOwnerRef(t *testing.T) {
 	require.NoError(t, err)
 	require.Len(t, parentSecret.OwnerReferences, 0)
 
-	// create a client that should set an owner ref back to parent on create
+	// work around stupid behavior of WithoutVersionDecoder.Decode
+	parentSecret.APIVersion, parentSecret.Kind = corev1.SchemeGroupVersion.WithKind("Secret").ToAPIVersionAndKind()
+
 	ref := metav1.OwnerReference{
-		APIVersion: "v1",
-		Kind:       "Secret",
+		APIVersion: parentSecret.APIVersion,
+		Kind:       parentSecret.Kind,
 		Name:       parentSecret.Name,
 		UID:        parentSecret.UID,
 	}
-	_ = env.APIGroupSuffix // TODO: wire API group into kubeclient.
+
+	// create a client that should set an owner ref back to parent on create
 	ownerRefClient, err := kubeclient.New(
-		kubeclient.WithMiddleware(ownerref.New(ref)),
+		kubeclient.WithMiddleware(ownerref.New(parentSecret)),
+		kubeclient.WithMiddleware(groupsuffix.New(env.APIGroupSuffix)),
 		kubeclient.WithConfig(library.NewClientConfig(t)),
 	)
 	require.NoError(t, err)
@@ -137,44 +142,29 @@ func TestKubeClientOwnerRef(t *testing.T) {
 		return err
 	})
 
-	// TODO: update middleware code to not set owner references on cluster-scoped objects.
-	//
-	// The Kube 1.20 garbage collector asserts some new behavior in regards to invalid owner
-	// references (i.e., when you have a namespace-scoped owner references for a cluster-scoped
-	// dependent, the cluster-scoped dependent is not removed). We also found a bug in the 1.20
-	// garbage collector where namespace-scoped dependents are not garbage collected if their owner
-	// had been used as an invalid owner reference before - this bug causes our test to fallover
-	// because we are setting a namespace-scoped owner ref on this APIService.
-	//
-	// We believe that the best way to get around this problem is to update our kubeclient code to
-	// never set owner references on cluster-scoped objects. After we do that, we will uncomment this
-	// part of the test.
-	if false {
-		// sanity check API service client
-		apiService, err := ownerRefClient.Aggregation.ApiregistrationV1().APIServices().Create(
-			ctx,
-			&apiregistrationv1.APIService{
-				ObjectMeta: metav1.ObjectMeta{
-					Name:            "v1.pandas.dev",
-					OwnerReferences: nil, // no owner refs set
-				},
-				Spec: apiregistrationv1.APIServiceSpec{
-					Version:              "v1",
-					Group:                "pandas.dev",
-					GroupPriorityMinimum: 10_000,
-					VersionPriority:      500,
-				},
+	// sanity check API service client - the middleware code shouldn't add an owner reference to this
+	// APIService because the APIService is cluster-scoped and the parent object is namespace-scoped,
+	// which is invalid in Kubernetes
+	apiService, err := ownerRefClient.Aggregation.ApiregistrationV1().APIServices().Create(
+		ctx,
+		&apiregistrationv1.APIService{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:            "v1.pandas.dev",
+				OwnerReferences: nil, // no owner refs set
 			},
-			metav1.CreateOptions{},
-		)
-		require.NoError(t, err)
-		hasOwnerRef(t, apiService, ref)
-		// this owner ref is invalid for an API service so it should be immediately deleted
-		isEventuallyDeleted(t, func() error {
-			_, err := ownerRefClient.Aggregation.ApiregistrationV1().APIServices().Get(ctx, apiService.Name, metav1.GetOptions{})
-			return err
-		})
-	}
+			Spec: apiregistrationv1.APIServiceSpec{
+				Version:              "v1",
+				Group:                "pandas.dev",
+				GroupPriorityMinimum: 10_000,
+				VersionPriority:      500,
+			},
+		},
+		metav1.CreateOptions{},
+	)
+	require.NoError(t, err)
+	hasNoOwnerRef(t, apiService)
+	err = ownerRefClient.Aggregation.ApiregistrationV1().APIServices().Delete(ctx, apiService.Name, metav1.DeleteOptions{})
+	require.NoError(t, err)
 
 	// sanity check concierge client
 	credentialIssuer, err := ownerRefClient.PinnipedConcierge.ConfigV1alpha1().CredentialIssuers(namespace.Name).Create(
@@ -256,6 +246,13 @@ func hasOwnerRef(t *testing.T, obj metav1.Object, ref metav1.OwnerReference) {
 	require.Equal(t, ref, ownerReferences[0])
 }
 
+func hasNoOwnerRef(t *testing.T, obj metav1.Object) {
+	t.Helper()
+
+	ownerReferences := obj.GetOwnerReferences()
+	require.Len(t, ownerReferences, 0)
+}
+
 func isEventuallyDeleted(t *testing.T, f func() error) {
 	t.Helper()
 

test/integration/supervisor_discovery_test.go

@@ -48,7 +48,7 @@ func TestSupervisorOIDCDiscovery(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
 	defer cancel()
 
-	temporarilyRemoveAllFederationDomainsAndDefaultTLSCertSecret(ctx, t, ns, defaultTLSCertSecretName(env), client, library.NewClientset(t))
+	temporarilyRemoveAllFederationDomainsAndDefaultTLSCertSecret(ctx, t, ns, defaultTLSCertSecretName(env), client, library.NewKubernetesClientset(t))
 
 	tests := []struct {
 		Scheme   string
@@ -146,7 +146,7 @@ func TestSupervisorOIDCDiscovery(t *testing.T) {
 func TestSupervisorTLSTerminationWithSNI(t *testing.T) {
 	env := library.IntegrationEnv(t)
 	pinnipedClient := library.NewSupervisorClientset(t)
-	kubeClient := library.NewClientset(t)
+	kubeClient := library.NewKubernetesClientset(t)
 
 	ns := env.SupervisorNamespace
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
@@ -213,7 +213,7 @@ func TestSupervisorTLSTerminationWithSNI(t *testing.T) {
 func TestSupervisorTLSTerminationWithDefaultCerts(t *testing.T) {
 	env := library.IntegrationEnv(t)
 	pinnipedClient := library.NewSupervisorClientset(t)
-	kubeClient := library.NewClientset(t)
+	kubeClient := library.NewKubernetesClientset(t)
 
 	ns := env.SupervisorNamespace
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)

test/integration/supervisor_secrets_test.go

@@ -21,7 +21,7 @@ import (
 
 func TestSupervisorSecrets(t *testing.T) {
 	env := library.IntegrationEnv(t)
-	kubeClient := library.NewClientset(t)
+	kubeClient := library.NewKubernetesClientset(t)
 	supervisorClient := library.NewSupervisorClientset(t)
 
 	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)

test/integration/supervisor_storage_garbage_collection_test.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Pinniped contributors. All Rights Reserved.
+// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
 package integration
@@ -26,7 +26,7 @@ func TestStorageGarbageCollection(t *testing.T) {
 	t.Parallel()
 
 	env := library.IntegrationEnv(t)
-	client := library.NewClientset(t)
+	client := library.NewKubernetesClientset(t)
 	secrets := client.CoreV1().Secrets(env.SupervisorNamespace)
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
 	defer cancel()

test/integration/supervisor_storage_test.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Pinniped contributors. All Rights Reserved.
+// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
 package integration
@@ -25,7 +25,7 @@ import (
 
 func TestAuthorizeCodeStorage(t *testing.T) {
 	env := library.IntegrationEnv(t)
-	client := library.NewClientset(t)
+	client := library.NewKubernetesClientset(t)
 
 	const (
 		// randomly generated HMAC authorization code (see below)

test/library/access.go

@@ -6,7 +6,6 @@ package library
 import (
 	"context"
 	"io/ioutil"
-	"net/http"
 	"os"
 	"os/exec"
 	"testing"
@@ -16,7 +15,6 @@ import (
 	"github.com/stretchr/testify/require"
 	v1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 )
@@ -33,12 +31,11 @@ const (
 // performing a Pinniped credential exchange.
 func AccessAsUserTest(
 	ctx context.Context,
-	adminClient kubernetes.Interface,
 	testUsername string,
 	clientUnderTest kubernetes.Interface,
 ) func(t *testing.T) {
 	return func(t *testing.T) {
-		addTestClusterUserCanViewEverythingRoleBinding(ctx, t, adminClient, testUsername)
+		addTestClusterUserCanViewEverythingRoleBinding(t, testUsername)
 
 		// Use the client which is authenticated as the test user to list namespaces
 		var listNamespaceResponse *v1.NamespaceList
@@ -55,14 +52,12 @@ func AccessAsUserTest(
 }
 
 func AccessAsUserWithKubectlTest(
-	ctx context.Context,
-	adminClient kubernetes.Interface,
 	testKubeConfigYAML string,
 	testUsername string,
 	expectedNamespace string,
 ) func(t *testing.T) {
 	return func(t *testing.T) {
-		addTestClusterUserCanViewEverythingRoleBinding(ctx, t, adminClient, testUsername)
+		addTestClusterUserCanViewEverythingRoleBinding(t, testUsername)
 
 		// Use the given kubeconfig with kubectl to list namespaces as the test user
 		var kubectlCommandOutput string
@@ -85,12 +80,11 @@ func AccessAsUserWithKubectlTest(
 // a group membership) after performing a Pinniped credential exchange.
 func AccessAsGroupTest(
 	ctx context.Context,
-	adminClient kubernetes.Interface,
 	testGroup string,
 	clientUnderTest kubernetes.Interface,
 ) func(t *testing.T) {
 	return func(t *testing.T) {
-		addTestClusterGroupCanViewEverythingRoleBinding(ctx, t, adminClient, testGroup)
+		addTestClusterGroupCanViewEverythingRoleBinding(t, testGroup)
 
 		// Use the client which is authenticated as the test user to list namespaces
 		var listNamespaceResponse *v1.NamespaceList
@@ -107,14 +101,12 @@ func AccessAsGroupTest(
 }
 
 func AccessAsGroupWithKubectlTest(
-	ctx context.Context,
-	adminClient kubernetes.Interface,
 	testKubeConfigYAML string,
 	testGroup string,
 	expectedNamespace string,
 ) func(t *testing.T) {
 	return func(t *testing.T) {
-		addTestClusterGroupCanViewEverythingRoleBinding(ctx, t, adminClient, testGroup)
+		addTestClusterGroupCanViewEverythingRoleBinding(t, testGroup)
 
 		// Use the given kubeconfig with kubectl to list namespaces as the test user
 		var kubectlCommandOutput string
@@ -130,67 +122,38 @@ func AccessAsGroupWithKubectlTest(
 	}
 }
 
-func addTestClusterUserCanViewEverythingRoleBinding(ctx context.Context, t *testing.T, adminClient kubernetes.Interface, testUsername string) {
+func addTestClusterUserCanViewEverythingRoleBinding(t *testing.T, testUsername string) {
 	t.Helper()
 
-	addTestClusterRoleBinding(ctx, t, adminClient, &rbacv1.ClusterRoleBinding{
-		TypeMeta: metav1.TypeMeta{},
-		ObjectMeta: metav1.ObjectMeta{
-			Name: "integration-test-user-readonly-role-binding",
-		},
-		Subjects: []rbacv1.Subject{{
+	CreateTestClusterRoleBinding(t,
+		rbacv1.Subject{
 			Kind:     rbacv1.UserKind,
 			APIGroup: rbacv1.GroupName,
 			Name:     testUsername,
-		}},
-		RoleRef: rbacv1.RoleRef{
+		},
+		rbacv1.RoleRef{
 			Kind:     "ClusterRole",
 			APIGroup: rbacv1.GroupName,
 			Name:     "view",
 		},
-	})
+	)
 }
 
-func addTestClusterGroupCanViewEverythingRoleBinding(ctx context.Context, t *testing.T, adminClient kubernetes.Interface, testGroup string) {
+func addTestClusterGroupCanViewEverythingRoleBinding(t *testing.T, testGroup string) {
 	t.Helper()
 
-	addTestClusterRoleBinding(ctx, t, adminClient, &rbacv1.ClusterRoleBinding{
-		TypeMeta: metav1.TypeMeta{},
-		ObjectMeta: metav1.ObjectMeta{
-			Name: "integration-test-group-readonly-role-binding",
-		},
-		Subjects: []rbacv1.Subject{{
+	CreateTestClusterRoleBinding(t,
+		rbacv1.Subject{
 			Kind:     rbacv1.GroupKind,
 			APIGroup: rbacv1.GroupName,
 			Name:     testGroup,
-		}},
-		RoleRef: rbacv1.RoleRef{
+		},
+		rbacv1.RoleRef{
 			Kind:     "ClusterRole",
 			APIGroup: rbacv1.GroupName,
 			Name:     "view",
 		},
-	})
-}
-
-func addTestClusterRoleBinding(ctx context.Context, t *testing.T, adminClient kubernetes.Interface, binding *rbacv1.ClusterRoleBinding) {
-	t.Helper()
-
-	_, err := adminClient.RbacV1().ClusterRoleBindings().Get(ctx, binding.Name, metav1.GetOptions{})
-	if err != nil {
-		statusError, isStatus := err.(*errors.StatusError)
-		require.True(t, isStatus, "Only StatusNotFound error would be acceptable, but error was: ", err.Error())
-		require.Equal(t, http.StatusNotFound, int(statusError.Status().Code))
-
-		_, err = adminClient.RbacV1().ClusterRoleBindings().Create(ctx, binding, metav1.CreateOptions{})
-		require.NoError(t, err)
-	}
-
-	t.Cleanup(func() {
-		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-		defer cancel()
-		err = adminClient.RbacV1().ClusterRoleBindings().Delete(ctx, binding.Name, metav1.DeleteOptions{})
-		require.NoError(t, err, "Test failed to clean up after itself")
-	})
+	)
 }
 
 func runKubectlGetNamespaces(t *testing.T, kubeConfigYAML string) (string, error) {

test/library/client.go

@@ -31,6 +31,7 @@ import (
 	idpv1alpha1 "go.pinniped.dev/generated/1.20/apis/supervisor/idp/v1alpha1"
 	conciergeclientset "go.pinniped.dev/generated/1.20/client/concierge/clientset/versioned"
 	supervisorclientset "go.pinniped.dev/generated/1.20/client/supervisor/clientset/versioned"
+	"go.pinniped.dev/internal/groupsuffix"
 	"go.pinniped.dev/internal/kubeclient"
 
 	// Import to initialize client auth plugins - the kubeconfig that we use for
@@ -44,12 +45,6 @@ func NewClientConfig(t *testing.T) *rest.Config {
 	return newClientConfigWithOverrides(t, &clientcmd.ConfigOverrides{})
 }
 
-func NewClientset(t *testing.T) kubernetes.Interface {
-	t.Helper()
-
-	return newClientsetWithConfig(t, NewClientConfig(t))
-}
-
 func NewClientsetForKubeConfig(t *testing.T, kubeConfig string) kubernetes.Interface {
 	t.Helper()
 	return newClientsetWithConfig(t, NewRestConfigFromKubeconfig(t, kubeConfig))
@@ -74,6 +69,12 @@ func NewClientsetWithCertAndKey(t *testing.T, clientCertificateData, clientKeyDa
 	return newClientsetWithConfig(t, newAnonymousClientRestConfigWithCertAndKeyAdded(t, clientCertificateData, clientKeyData))
 }
 
+func NewKubernetesClientset(t *testing.T) kubernetes.Interface {
+	t.Helper()
+
+	return newKubeclient(t, NewClientConfig(t)).Kubernetes
+}
+
 func NewSupervisorClientset(t *testing.T) supervisorclientset.Interface {
 	t.Helper()
 
@@ -135,8 +136,11 @@ func newAnonymousClientRestConfigWithCertAndKeyAdded(t *testing.T, clientCertifi
 
 func newKubeclient(t *testing.T, config *rest.Config) *kubeclient.Client {
 	t.Helper()
-	_ = IntegrationEnv(t).APIGroupSuffix // TODO: wire API group into kubeclient.
-	client, err := kubeclient.New(kubeclient.WithConfig(config))
+	env := IntegrationEnv(t)
+	client, err := kubeclient.New(
+		kubeclient.WithConfig(config),
+		kubeclient.WithMiddleware(groupsuffix.New(env.APIGroupSuffix)),
+	)
 	require.NoError(t, err)
 	return client
 }
@@ -163,6 +167,12 @@ func CreateTestWebhookAuthenticator(ctx context.Context, t *testing.T) corev1.Ty
 
 	t.Cleanup(func() {
 		t.Helper()
+
+		if t.Failed() {
+			t.Logf("skipping deletion of test WebhookAuthenticator %s/%s", webhook.Namespace, webhook.Name)
+			return
+		}
+
 		t.Logf("cleaning up test WebhookAuthenticator %s/%s", webhook.Namespace, webhook.Name)
 		deleteCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 		defer cancel()
@@ -227,6 +237,12 @@ func CreateTestJWTAuthenticator(ctx context.Context, t *testing.T, spec auth1alp
 
 	t.Cleanup(func() {
 		t.Helper()
+
+		if t.Failed() {
+			t.Logf("skipping deletion of test JWTAuthenticator %s/%s", jwtAuthenticator.Namespace, jwtAuthenticator.Name)
+			return
+		}
+
 		t.Logf("cleaning up test JWTAuthenticator %s/%s", jwtAuthenticator.Namespace, jwtAuthenticator.Name)
 		deleteCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 		defer cancel()
@@ -325,7 +341,7 @@ func RandHex(t *testing.T, numBytes int) string {
 
 func CreateTestSecret(t *testing.T, namespace string, baseName string, secretType corev1.SecretType, stringData map[string]string) *corev1.Secret {
 	t.Helper()
-	client := NewClientset(t)
+	client := NewKubernetesClientset(t)
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 
@@ -396,7 +412,7 @@ func CreateTestOIDCIdentityProvider(t *testing.T, spec idpv1alpha1.OIDCIdentityP
 
 func CreateTestClusterRoleBinding(t *testing.T, subject rbacv1.Subject, roleRef rbacv1.RoleRef) *rbacv1.ClusterRoleBinding {
 	t.Helper()
-	client := NewClientset(t)
+	client := NewKubernetesClientset(t)
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 

test/library/dumplogs.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Pinniped contributors. All Rights Reserved.
+// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
 package library
@@ -21,7 +21,7 @@ func DumpLogs(t *testing.T, namespace string, labelSelector string) {
 		return
 	}
 
-	kubeClient := NewClientset(t)
+	kubeClient := NewKubernetesClientset(t)
 	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
 	defer cancel()