Switch to a slimmer distroless base image.

At a high level, it switches us to a distroless base container image, but that also includes several related bits:

- Add a writable /tmp but make the rest of our filesystems read-only at runtime.

- Condense our main server binaries into a single pinniped-server binary. This saves a bunch of space in
  the image due to duplicated library code. The correct behavior is dispatched based on `os.Args[0]`, and
  the `pinniped-server` binary is symlinked to `pinniped-concierge` and `pinniped-supervisor`.

- Strip debug symbols from our binaries. These aren't really useful in a distroless image anyway and all the
  normal stuff you'd expect to work, such as stack traces, still does.

- Add a separate `pinniped-concierge-kube-cert-agent` binary with "sleep" and "print" functionality instead of
  using builtin /bin/sleep and /bin/cat for the kube-cert-agent. This is split from the main server binary
  because the loading/init time of the main server binary was too large for the tiny resource footprint we
  established in our kube-cert-agent PodSpec. Using a separate binary eliminates this issue and the extra
  binary adds only around 1.5MiB of image size.

- Switch the kube-cert-agent code to use a JSON `{"tls.crt": "<b64 cert>", "tls.key": "<b64 key>"}` format.
  This is more robust to unexpected input formatting than the old code, which simply concatenated the files
  with some extra newlines and split on whitespace.

- Update integration tests that made now-invalid assumptions about the `pinniped-server` image.

Signed-off-by: Matt Moyer <moyerm@vmware.com>

internal/concierge/server/server.go

@@ -8,6 +8,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"os"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -16,6 +17,10 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	genericoptions "k8s.io/apiserver/pkg/server/options"
+	"k8s.io/client-go/pkg/version"
+	"k8s.io/client-go/rest"
+	"k8s.io/component-base/logs"
+	"k8s.io/klog/v2"
 
 	"go.pinniped.dev/internal/certauthority/dynamiccertauthority"
 	"go.pinniped.dev/internal/concierge/apiserver"
@@ -231,3 +236,21 @@ func getAggregatedAPIServerConfig(
 	}
 	return apiServerConfig, nil
 }
+
+func Main() {
+	logs.InitLogs()
+	defer logs.FlushLogs()
+
+	// Dump out the time since compile (mostly useful for benchmarking our local development cycle latency).
+	var timeSinceCompile time.Duration
+	if buildDate, err := time.Parse(time.RFC3339, version.Get().BuildDate); err == nil {
+		timeSinceCompile = time.Since(buildDate).Round(time.Second)
+	}
+	klog.Infof("Running %s at %#v (%s since build)", rest.DefaultKubernetesUserAgent(), version.Get(), timeSinceCompile)
+
+	ctx := genericapiserver.SetupSignalContext()
+
+	if err := New(ctx, os.Args[1:], os.Stdout, os.Stderr).Run(); err != nil {
+		klog.Fatal(err)
+	}
+}

internal/controller/kubecertagent/kubecertagent.go

@@ -8,6 +8,7 @@ package kubecertagent
 import (
 	"context"
 	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"strings"
 	"time"
@@ -309,22 +310,30 @@ func (c *agentController) loadSigningKey(agentPod *corev1.Pod) error {
 	}
 
 	// Exec into the agent pod and cat out the certificate and the key.
-	combinedPEM, err := c.executor.Exec(
-		agentPod.Namespace, agentPod.Name,
-		"sh", "-c", "cat ${CERT_PATH}; echo; echo; cat ${KEY_PATH}",
-	)
+	outputJSON, err := c.executor.Exec(agentPod.Namespace, agentPod.Name, "pinniped-concierge-kube-cert-agent", "print")
 	if err != nil {
 		return fmt.Errorf("could not exec into agent pod %s/%s: %w", agentPod.Namespace, agentPod.Name, err)
 	}
 
-	// Split up the output by looking for the block of newlines.
-	var certPEM, keyPEM string
-	if parts := strings.Split(combinedPEM, "\n\n\n"); len(parts) == 2 {
-		certPEM, keyPEM = parts[0], parts[1]
+	// Parse and decode the JSON output from the "pinniped-concierge-kube-cert-agent print" command.
+	var output struct {
+		Cert string `json:"tls.crt"`
+		Key  string `json:"tls.key"`
+	}
+	if err := json.Unmarshal([]byte(outputJSON), &output); err != nil {
+		return fmt.Errorf("failed to decode signing cert/key JSON from agent pod %s/%s: %w", agentPod.Namespace, agentPod.Name, err)
+	}
+	certPEM, err := base64.StdEncoding.DecodeString(output.Cert)
+	if err != nil {
+		return fmt.Errorf("failed to decode signing cert base64 from agent pod %s/%s: %w", agentPod.Namespace, agentPod.Name, err)
+	}
+	keyPEM, err := base64.StdEncoding.DecodeString(output.Key)
+	if err != nil {
+		return fmt.Errorf("failed to decode signing key base64 from agent pod %s/%s: %w", agentPod.Namespace, agentPod.Name, err)
 	}
 
 	// Load the certificate and key into the dynamic signer.
-	if err := c.dynamicCertProvider.SetCertKeyContent([]byte(certPEM), []byte(keyPEM)); err != nil {
+	if err := c.dynamicCertProvider.SetCertKeyContent(certPEM, keyPEM); err != nil {
 		return fmt.Errorf("failed to set signing cert/key content from agent pod %s/%s: %w", agentPod.Namespace, agentPod.Name, err)
 	}
 
@@ -461,7 +470,7 @@ func (c *agentController) newAgentDeployment(controllerManagerPod *corev1.Pod) *
 							Name:            "sleeper",
 							Image:           c.cfg.ContainerImage,
 							ImagePullPolicy: corev1.PullIfNotPresent,
-							Command:         []string{"/bin/sleep", "infinity"},
+							Command:         []string{"pinniped-concierge-kube-cert-agent", "sleep"},
 							VolumeMounts:    volumeMounts,
 							Env: []corev1.EnvVar{
 								{Name: "CERT_PATH", Value: getContainerArgByName(controllerManagerPod, "cluster-signing-cert-file", "/etc/kubernetes/ca/ca.pem")},

internal/controller/kubecertagent/kubecertagent_test.go

@@ -104,7 +104,7 @@ func TestAgentController(t *testing.T) {
 					Containers: []corev1.Container{{
 						Name:    "sleeper",
 						Image:   "pinniped-server-image",
-						Command: []string{"/bin/sleep", "infinity"},
+						Command: []string{"pinniped-concierge-kube-cert-agent", "sleep"},
 						Env: []corev1.EnvVar{
 							{Name: "CERT_PATH", Value: "/path/to/signing.crt"},
 							{Name: "KEY_PATH", Value: "/path/to/signing.key"},
@@ -200,8 +200,8 @@ func TestAgentController(t *testing.T) {
 	}
 
 	mockExecSucceeds := func(t *testing.T, executor *mocks.MockPodCommandExecutorMockRecorder, dynamicCert *mocks.MockDynamicCertPrivateMockRecorder, execCache *cache.Expiring) {
-		executor.Exec("concierge", "pinniped-concierge-kube-cert-agent-xyz-1234", "sh", "-c", "cat ${CERT_PATH}; echo; echo; cat ${KEY_PATH}").
-			Return("test-cert\n\n\ntest-key", nil)
+		executor.Exec("concierge", "pinniped-concierge-kube-cert-agent-xyz-1234", "pinniped-concierge-kube-cert-agent", "print").
+			Return(`{"tls.crt": "dGVzdC1jZXJ0", "tls.key": "dGVzdC1rZXk="}`, nil) // "test-cert" / "test-key"
 		dynamicCert.SetCertKeyContent([]byte("test-cert"), []byte("test-key")).
 			Return(nil)
 	}
@@ -573,7 +573,7 @@ func TestAgentController(t *testing.T) {
 				validClusterInfoConfigMap,
 			},
 			mocks: func(t *testing.T, executor *mocks.MockPodCommandExecutorMockRecorder, dynamicCert *mocks.MockDynamicCertPrivateMockRecorder, execCache *cache.Expiring) {
-				executor.Exec("concierge", "pinniped-concierge-kube-cert-agent-xyz-1234", "sh", "-c", "cat ${CERT_PATH}; echo; echo; cat ${KEY_PATH}").
+				executor.Exec("concierge", "pinniped-concierge-kube-cert-agent-xyz-1234", "pinniped-concierge-kube-cert-agent", "print").
 					Return("", fmt.Errorf("some exec error")).
 					AnyTimes()
 			},
@@ -589,6 +589,90 @@ func TestAgentController(t *testing.T) {
 				LastUpdateTime: metav1.NewTime(now),
 			},
 		},
+		{
+			name: "deployment exists, configmap is valid, exec into agent pod returns invalid JSON",
+			pinnipedObjects: []runtime.Object{
+				initialCredentialIssuer,
+			},
+			kubeObjects: []runtime.Object{
+				healthyKubeControllerManagerPod,
+				healthyAgentDeployment,
+				healthyAgentPod,
+				validClusterInfoConfigMap,
+			},
+			mocks: func(t *testing.T, executor *mocks.MockPodCommandExecutorMockRecorder, dynamicCert *mocks.MockDynamicCertPrivateMockRecorder, execCache *cache.Expiring) {
+				executor.Exec("concierge", "pinniped-concierge-kube-cert-agent-xyz-1234", "pinniped-concierge-kube-cert-agent", "print").
+					Return("bogus-data", nil).
+					AnyTimes()
+			},
+			wantDistinctErrors: []string{
+				`failed to decode signing cert/key JSON from agent pod concierge/pinniped-concierge-kube-cert-agent-xyz-1234: invalid character 'b' looking for beginning of value`,
+			},
+			wantAgentDeployment: healthyAgentDeployment,
+			wantStrategy: &configv1alpha1.CredentialIssuerStrategy{
+				Type:           configv1alpha1.KubeClusterSigningCertificateStrategyType,
+				Status:         configv1alpha1.ErrorStrategyStatus,
+				Reason:         configv1alpha1.CouldNotFetchKeyStrategyReason,
+				Message:        `failed to decode signing cert/key JSON from agent pod concierge/pinniped-concierge-kube-cert-agent-xyz-1234: invalid character 'b' looking for beginning of value`,
+				LastUpdateTime: metav1.NewTime(now),
+			},
+		},
+		{
+			name: "deployment exists, configmap is valid, exec into agent pod returns invalid cert base64",
+			pinnipedObjects: []runtime.Object{
+				initialCredentialIssuer,
+			},
+			kubeObjects: []runtime.Object{
+				healthyKubeControllerManagerPod,
+				healthyAgentDeployment,
+				healthyAgentPod,
+				validClusterInfoConfigMap,
+			},
+			mocks: func(t *testing.T, executor *mocks.MockPodCommandExecutorMockRecorder, dynamicCert *mocks.MockDynamicCertPrivateMockRecorder, execCache *cache.Expiring) {
+				executor.Exec("concierge", "pinniped-concierge-kube-cert-agent-xyz-1234", "pinniped-concierge-kube-cert-agent", "print").
+					Return(`{"tls.crt": "invalid"}`, nil).
+					AnyTimes()
+			},
+			wantDistinctErrors: []string{
+				`failed to decode signing cert base64 from agent pod concierge/pinniped-concierge-kube-cert-agent-xyz-1234: illegal base64 data at input byte 4`,
+			},
+			wantAgentDeployment: healthyAgentDeployment,
+			wantStrategy: &configv1alpha1.CredentialIssuerStrategy{
+				Type:           configv1alpha1.KubeClusterSigningCertificateStrategyType,
+				Status:         configv1alpha1.ErrorStrategyStatus,
+				Reason:         configv1alpha1.CouldNotFetchKeyStrategyReason,
+				Message:        `failed to decode signing cert base64 from agent pod concierge/pinniped-concierge-kube-cert-agent-xyz-1234: illegal base64 data at input byte 4`,
+				LastUpdateTime: metav1.NewTime(now),
+			},
+		},
+		{
+			name: "deployment exists, configmap is valid, exec into agent pod returns invalid key base64",
+			pinnipedObjects: []runtime.Object{
+				initialCredentialIssuer,
+			},
+			kubeObjects: []runtime.Object{
+				healthyKubeControllerManagerPod,
+				healthyAgentDeployment,
+				healthyAgentPod,
+				validClusterInfoConfigMap,
+			},
+			mocks: func(t *testing.T, executor *mocks.MockPodCommandExecutorMockRecorder, dynamicCert *mocks.MockDynamicCertPrivateMockRecorder, execCache *cache.Expiring) {
+				executor.Exec("concierge", "pinniped-concierge-kube-cert-agent-xyz-1234", "pinniped-concierge-kube-cert-agent", "print").
+					Return(`{"tls.crt": "dGVzdAo=", "tls.key": "invalid"}`, nil).
+					AnyTimes()
+			},
+			wantDistinctErrors: []string{
+				`failed to decode signing key base64 from agent pod concierge/pinniped-concierge-kube-cert-agent-xyz-1234: illegal base64 data at input byte 4`,
+			},
+			wantAgentDeployment: healthyAgentDeployment,
+			wantStrategy: &configv1alpha1.CredentialIssuerStrategy{
+				Type:           configv1alpha1.KubeClusterSigningCertificateStrategyType,
+				Status:         configv1alpha1.ErrorStrategyStatus,
+				Reason:         configv1alpha1.CouldNotFetchKeyStrategyReason,
+				Message:        `failed to decode signing key base64 from agent pod concierge/pinniped-concierge-kube-cert-agent-xyz-1234: illegal base64 data at input byte 4`,
+				LastUpdateTime: metav1.NewTime(now),
+			},
+		},
 		{
 			name: "deployment exists, configmap is valid, exec into agent pod returns bogus certs",
 			pinnipedObjects: []runtime.Object{
@@ -601,10 +685,10 @@ func TestAgentController(t *testing.T) {
 				validClusterInfoConfigMap,
 			},
 			mocks: func(t *testing.T, executor *mocks.MockPodCommandExecutorMockRecorder, dynamicCert *mocks.MockDynamicCertPrivateMockRecorder, execCache *cache.Expiring) {
-				executor.Exec("concierge", "pinniped-concierge-kube-cert-agent-xyz-1234", "sh", "-c", "cat ${CERT_PATH}; echo; echo; cat ${KEY_PATH}").
-					Return("bogus-data", nil).
+				executor.Exec("concierge", "pinniped-concierge-kube-cert-agent-xyz-1234", "pinniped-concierge-kube-cert-agent", "print").
+					Return(`{"tls.crt": "dGVzdC1jZXJ0", "tls.key": "dGVzdC1rZXk="}`, nil). // "test-cert" / "test-key"
 					AnyTimes()
-				dynamicCert.SetCertKeyContent([]byte(""), []byte("")).
+				dynamicCert.SetCertKeyContent([]byte("test-cert"), []byte("test-key")).
 					Return(fmt.Errorf("some dynamic cert error")).
 					AnyTimes()
 			},

internal/localuserauthenticator/localuserauthenticator.go

@@ -0,0 +1,390 @@
+// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package localuserauthenticator provides a authentication webhook program.
+//
+// This webhook is meant to be used in demo settings to play around with
+// Pinniped. As well, it can come in handy in integration tests.
+//
+// This webhook is NOT meant for use in production systems.
+package localuserauthenticator
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"encoding/csv"
+	"encoding/json"
+	"fmt"
+	"mime"
+	"net"
+	"net/http"
+	"os"
+	"os/signal"
+	"strings"
+	"time"
+
+	"golang.org/x/crypto/bcrypt"
+	authenticationv1beta1 "k8s.io/api/authentication/v1beta1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	kubeinformers "k8s.io/client-go/informers"
+	corev1informers "k8s.io/client-go/informers/core/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
+
+	"go.pinniped.dev/internal/constable"
+	"go.pinniped.dev/internal/controller/apicerts"
+	"go.pinniped.dev/internal/controllerlib"
+	"go.pinniped.dev/internal/dynamiccert"
+	"go.pinniped.dev/internal/kubeclient"
+	"go.pinniped.dev/internal/plog"
+)
+
+const (
+	// This string must match the name of the Namespace declared in the deployment yaml.
+	namespace = "local-user-authenticator"
+	// This string must match the name of the Service declared in the deployment yaml.
+	serviceName = "local-user-authenticator"
+
+	singletonWorker       = 1
+	defaultResyncInterval = 3 * time.Minute
+
+	invalidRequest = constable.Error("invalid request")
+)
+
+type webhook struct {
+	certProvider   dynamiccert.Private
+	secretInformer corev1informers.SecretInformer
+}
+
+func newWebhook(
+	certProvider dynamiccert.Private,
+	secretInformer corev1informers.SecretInformer,
+) *webhook {
+	return &webhook{
+		certProvider:   certProvider,
+		secretInformer: secretInformer,
+	}
+}
+
+// start runs the webhook in a separate goroutine and returns whether or not the
+// webhook was started successfully.
+func (w *webhook) start(ctx context.Context, l net.Listener) error {
+	server := http.Server{
+		Handler: w,
+		TLSConfig: &tls.Config{
+			MinVersion: tls.VersionTLS13,
+			GetCertificate: func(_ *tls.ClientHelloInfo) (*tls.Certificate, error) {
+				certPEM, keyPEM := w.certProvider.CurrentCertKeyContent()
+				cert, err := tls.X509KeyPair(certPEM, keyPEM)
+				return &cert, err
+			},
+		},
+	}
+
+	errCh := make(chan error)
+	go func() {
+		// Per ListenAndServeTLS doc, the {cert,key}File parameters can be empty
+		// since we want to use the certs from http.Server.TLSConfig.
+		errCh <- server.ServeTLS(l, "", "")
+	}()
+
+	go func() {
+		select {
+		case err := <-errCh:
+			plog.Debug("server exited", "err", err)
+		case <-ctx.Done():
+			plog.Debug("server context cancelled", "err", ctx.Err())
+			if err := server.Shutdown(context.Background()); err != nil {
+				plog.Debug("server shutdown failed", "err", err)
+			}
+		}
+	}()
+
+	return nil
+}
+
+func (w *webhook) ServeHTTP(rsp http.ResponseWriter, req *http.Request) {
+	username, password, err := getUsernameAndPasswordFromRequest(rsp, req)
+	if err != nil {
+		return
+	}
+	defer func() { _ = req.Body.Close() }()
+
+	secret, err := w.secretInformer.Lister().Secrets(namespace).Get(username)
+	notFound := k8serrors.IsNotFound(err)
+	if err != nil && !notFound {
+		plog.Debug("could not get secret", "err", err)
+		rsp.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	if notFound {
+		plog.Debug("user not found")
+		respondWithUnauthenticated(rsp)
+		return
+	}
+
+	passwordMatches := bcrypt.CompareHashAndPassword(
+		secret.Data["passwordHash"],
+		[]byte(password),
+	) == nil
+	if !passwordMatches {
+		plog.Debug("authentication failed: wrong password")
+		respondWithUnauthenticated(rsp)
+		return
+	}
+
+	groups := []string{}
+	groupsBuf := bytes.NewBuffer(secret.Data["groups"])
+	if groupsBuf.Len() > 0 {
+		groupsCSVReader := csv.NewReader(groupsBuf)
+		groups, err = groupsCSVReader.Read()
+		if err != nil {
+			plog.Debug("could not read groups", "err", err)
+			rsp.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		trimLeadingAndTrailingWhitespace(groups)
+	}
+
+	plog.Debug("successful authentication")
+	respondWithAuthenticated(rsp, secret.ObjectMeta.Name, groups)
+}
+
+func getUsernameAndPasswordFromRequest(rsp http.ResponseWriter, req *http.Request) (string, string, error) {
+	if req.URL.Path != "/authenticate" {
+		plog.Debug("received request path other than /authenticate", "path", req.URL.Path)
+		rsp.WriteHeader(http.StatusNotFound)
+		return "", "", invalidRequest
+	}
+
+	if req.Method != http.MethodPost {
+		plog.Debug("received request method other than post", "method", req.Method)
+		rsp.WriteHeader(http.StatusMethodNotAllowed)
+		return "", "", invalidRequest
+	}
+
+	if !headerContains(req, "Content-Type", "application/json") {
+		plog.Debug("content type is not application/json", "Content-Type", req.Header.Values("Content-Type"))
+		rsp.WriteHeader(http.StatusUnsupportedMediaType)
+		return "", "", invalidRequest
+	}
+
+	if !headerContains(req, "Accept", "application/json") &&
+		!headerContains(req, "Accept", "application/*") &&
+		!headerContains(req, "Accept", "*/*") {
+		plog.Debug("client does not accept application/json", "Accept", req.Header.Values("Accept"))
+		rsp.WriteHeader(http.StatusUnsupportedMediaType)
+		return "", "", invalidRequest
+	}
+
+	if req.Body == nil {
+		plog.Debug("invalid nil body")
+		rsp.WriteHeader(http.StatusBadRequest)
+		return "", "", invalidRequest
+	}
+
+	var body authenticationv1beta1.TokenReview
+	if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
+		plog.Debug("failed to decode body", "err", err)
+		rsp.WriteHeader(http.StatusBadRequest)
+		return "", "", invalidRequest
+	}
+
+	if body.APIVersion != authenticationv1beta1.SchemeGroupVersion.String() {
+		plog.Debug("invalid TokenReview apiVersion", "apiVersion", body.APIVersion)
+		rsp.WriteHeader(http.StatusBadRequest)
+		return "", "", invalidRequest
+	}
+
+	if body.Kind != "TokenReview" {
+		plog.Debug("invalid TokenReview kind", "kind", body.Kind)
+		rsp.WriteHeader(http.StatusBadRequest)
+		return "", "", invalidRequest
+	}
+
+	tokenSegments := strings.SplitN(body.Spec.Token, ":", 2)
+	if len(tokenSegments) != 2 {
+		plog.Debug("bad token format in request")
+		rsp.WriteHeader(http.StatusBadRequest)
+		return "", "", invalidRequest
+	}
+
+	return tokenSegments[0], tokenSegments[1], nil
+}
+
+func headerContains(req *http.Request, headerName, s string) bool {
+	headerValues := req.Header.Values(headerName)
+	for i := range headerValues {
+		mimeTypes := strings.Split(headerValues[i], ",")
+		for _, mimeType := range mimeTypes {
+			mediaType, _, _ := mime.ParseMediaType(mimeType)
+			if mediaType == s {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func trimLeadingAndTrailingWhitespace(ss []string) {
+	for i := range ss {
+		ss[i] = strings.TrimSpace(ss[i])
+	}
+}
+
+func respondWithUnauthenticated(rsp http.ResponseWriter) {
+	rsp.Header().Add("Content-Type", "application/json")
+
+	body := authenticationv1beta1.TokenReview{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "TokenReview",
+			APIVersion: authenticationv1beta1.SchemeGroupVersion.String(),
+		},
+		Status: authenticationv1beta1.TokenReviewStatus{
+			Authenticated: false,
+		},
+	}
+	if err := json.NewEncoder(rsp).Encode(body); err != nil {
+		plog.Debug("could not encode response", "err", err)
+		rsp.WriteHeader(http.StatusInternalServerError)
+	}
+}
+
+func respondWithAuthenticated(
+	rsp http.ResponseWriter,
+	username string,
+	groups []string,
+) {
+	rsp.Header().Add("Content-Type", "application/json")
+	body := authenticationv1beta1.TokenReview{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "TokenReview",
+			APIVersion: authenticationv1beta1.SchemeGroupVersion.String(),
+		},
+		Status: authenticationv1beta1.TokenReviewStatus{
+			Authenticated: true,
+			User: authenticationv1beta1.UserInfo{
+				Username: username,
+				Groups:   groups,
+			},
+		},
+	}
+	if err := json.NewEncoder(rsp).Encode(body); err != nil {
+		plog.Debug("could not encode response", "err", err)
+		rsp.WriteHeader(http.StatusInternalServerError)
+	}
+}
+
+func startControllers(
+	ctx context.Context,
+	dynamicCertProvider dynamiccert.Private,
+	kubeClient kubernetes.Interface,
+	kubeInformers kubeinformers.SharedInformerFactory,
+) {
+	aVeryLongTime := time.Hour * 24 * 365 * 100
+
+	const certsSecretResourceName = "local-user-authenticator-tls-serving-certificate"
+
+	// Create controller manager.
+	controllerManager := controllerlib.
+		NewManager().
+		WithController(
+			apicerts.NewCertsManagerController(
+				namespace,
+				certsSecretResourceName,
+				map[string]string{
+					"app": "local-user-authenticator",
+				},
+				kubeClient,
+				kubeInformers.Core().V1().Secrets(),
+				controllerlib.WithInformer,
+				controllerlib.WithInitialEvent,
+				aVeryLongTime,
+				"local-user-authenticator CA",
+				serviceName,
+			),
+			singletonWorker,
+		).
+		WithController(
+			apicerts.NewCertsObserverController(
+				namespace,
+				certsSecretResourceName,
+				dynamicCertProvider,
+				kubeInformers.Core().V1().Secrets(),
+				controllerlib.WithInformer,
+			),
+			singletonWorker,
+		)
+
+	kubeInformers.Start(ctx.Done())
+
+	go controllerManager.Start(ctx)
+}
+
+func startWebhook(
+	ctx context.Context,
+	l net.Listener,
+	dynamicCertProvider dynamiccert.Private,
+	secretInformer corev1informers.SecretInformer,
+) error {
+	return newWebhook(dynamicCertProvider, secretInformer).start(ctx, l)
+}
+
+func waitForSignal() os.Signal {
+	signalCh := make(chan os.Signal, 1)
+	signal.Notify(signalCh, os.Interrupt)
+	return <-signalCh
+}
+
+func run() error {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	client, err := kubeclient.New()
+	if err != nil {
+		return fmt.Errorf("cannot create k8s client: %w", err)
+	}
+
+	kubeInformers := kubeinformers.NewSharedInformerFactoryWithOptions(
+		client.Kubernetes,
+		defaultResyncInterval,
+		kubeinformers.WithNamespace(namespace),
+	)
+
+	dynamicCertProvider := dynamiccert.NewServingCert("local-user-authenticator-tls-serving-certificate")
+
+	startControllers(ctx, dynamicCertProvider, client.Kubernetes, kubeInformers)
+	plog.Debug("controllers are ready")
+
+	//nolint: gosec // Intentionally binding to all network interfaces.
+	l, err := net.Listen("tcp", ":8443")
+	if err != nil {
+		return fmt.Errorf("cannot create listener: %w", err)
+	}
+	defer func() { _ = l.Close() }()
+
+	err = startWebhook(ctx, l, dynamicCertProvider, kubeInformers.Core().V1().Secrets())
+	if err != nil {
+		return fmt.Errorf("cannot start webhook: %w", err)
+	}
+	plog.Debug("webhook is ready", "address", l.Addr().String())
+
+	gotSignal := waitForSignal()
+	plog.Debug("webhook exiting", "signal", gotSignal)
+
+	return nil
+}
+
+func Main() {
+	// Hardcode the logging level to debug, since this is a test app and it is very helpful to have
+	// verbose logs to debug test failures.
+	if err := plog.ValidateAndSetLogLevelGlobally(plog.LevelDebug); err != nil {
+		klog.Fatal(err)
+	}
+	if err := run(); err != nil {
+		klog.Fatal(err)
+	}
+}

internal/localuserauthenticator/localuserauthenticator_test.go

@@ -0,0 +1,567 @@
+// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package localuserauthenticator
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"golang.org/x/crypto/bcrypt"
+	authenticationv1beta1 "k8s.io/api/authentication/v1beta1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	kubeinformers "k8s.io/client-go/informers"
+	corev1informers "k8s.io/client-go/informers/core/v1"
+	"k8s.io/client-go/kubernetes"
+	kubernetesfake "k8s.io/client-go/kubernetes/fake"
+
+	"go.pinniped.dev/internal/certauthority"
+	"go.pinniped.dev/internal/dynamiccert"
+)
+
+func TestWebhook(t *testing.T) {
+	t.Parallel()
+
+	const namespace = "local-user-authenticator"
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	user, otherUser, colonUser, noGroupUser, oneGroupUser, passwordUndefinedUser, emptyPasswordUser, invalidPasswordHashUser, undefinedGroupsUser :=
+		"some-user", "other-user", "colon-user", "no-group-user", "one-group-user", "password-undefined-user", "empty-password-user", "invalid-password-hash-user", "undefined-groups-user"
+	password, otherPassword, colonPassword, noGroupPassword, oneGroupPassword, undefinedGroupsPassword :=
+		"some-password", "other-password", "some-:-password", "no-group-password", "one-group-password", "undefined-groups-password"
+
+	group0, group1 := "some-group-0", "some-group-1"
+	groups := group0 + " , " + group1
+
+	kubeClient := kubernetesfake.NewSimpleClientset()
+	addSecretToFakeClientTracker(t, kubeClient, user, password, groups)
+	addSecretToFakeClientTracker(t, kubeClient, otherUser, otherPassword, groups)
+	addSecretToFakeClientTracker(t, kubeClient, colonUser, colonPassword, groups)
+	addSecretToFakeClientTracker(t, kubeClient, noGroupUser, noGroupPassword, "")
+	addSecretToFakeClientTracker(t, kubeClient, oneGroupUser, oneGroupPassword, group0)
+	addSecretToFakeClientTracker(t, kubeClient, emptyPasswordUser, "", groups)
+
+	require.NoError(t, kubeClient.Tracker().Add(&corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      passwordUndefinedUser,
+			Namespace: namespace,
+		},
+		Data: map[string][]byte{
+			"groups": []byte(groups),
+		},
+	}))
+
+	undefinedGroupsUserPasswordHash, err := bcrypt.GenerateFromPassword([]byte(undefinedGroupsPassword), bcrypt.MinCost)
+	require.NoError(t, err)
+
+	require.NoError(t, kubeClient.Tracker().Add(&corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      undefinedGroupsUser,
+			Namespace: namespace,
+		},
+		Data: map[string][]byte{
+			"passwordHash": undefinedGroupsUserPasswordHash,
+		},
+	}))
+
+	require.NoError(t, kubeClient.Tracker().Add(&corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      invalidPasswordHashUser,
+			Namespace: namespace,
+		},
+		Data: map[string][]byte{
+			"groups":       []byte(groups),
+			"passwordHash": []byte("not a valid password hash"),
+		},
+	}))
+
+	secretInformer := createSecretInformer(ctx, t, kubeClient)
+
+	certProvider, caBundle, serverName := newCertProvider(t)
+	w := newWebhook(certProvider, secretInformer)
+
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	defer func() { _ = l.Close() }()
+	require.NoError(t, w.start(ctx, l))
+
+	client := newClient(caBundle, serverName)
+
+	goodURL := fmt.Sprintf("https://%s/authenticate", l.Addr().String())
+	goodRequestHeaders := map[string][]string{
+		"Content-Type": {"application/json; charset=UTF-8"},
+		"Accept":       {"application/json, */*"},
+	}
+
+	tests := []struct {
+		name    string
+		url     string
+		method  string
+		headers map[string][]string
+		body    func() (io.ReadCloser, error)
+
+		wantStatus  int
+		wantHeaders map[string][]string
+		wantBody    *authenticationv1beta1.TokenReview
+	}{
+		{
+			name:        "success for a user who belongs to multiple groups",
+			url:         goodURL,
+			method:      http.MethodPost,
+			headers:     goodRequestHeaders,
+			body:        func() (io.ReadCloser, error) { return newTokenReviewBody(user + ":" + password) },
+			wantStatus:  http.StatusOK,
+			wantHeaders: map[string][]string{"Content-Type": {"application/json"}},
+			wantBody:    authenticatedResponseJSON(user, []string{group0, group1}),
+		},
+		{
+			name:        "success for a user who belongs to one groups",
+			url:         goodURL,
+			method:      http.MethodPost,
+			headers:     goodRequestHeaders,
+			body:        func() (io.ReadCloser, error) { return newTokenReviewBody(oneGroupUser + ":" + oneGroupPassword) },
+			wantStatus:  http.StatusOK,
+			wantHeaders: map[string][]string{"Content-Type": {"application/json"}},
+			wantBody:    authenticatedResponseJSON(oneGroupUser, []string{group0}),
+		},
+		{
+			name:        "success for a user who belongs to no groups",
+			url:         goodURL,
+			method:      http.MethodPost,
+			headers:     goodRequestHeaders,
+			body:        func() (io.ReadCloser, error) { return newTokenReviewBody(noGroupUser + ":" + noGroupPassword) },
+			wantStatus:  http.StatusOK,
+			wantHeaders: map[string][]string{"Content-Type": {"application/json"}},
+			wantBody:    authenticatedResponseJSON(noGroupUser, nil),
+		},
+		{
+			name:        "wrong username for password",
+			url:         goodURL,
+			method:      http.MethodPost,
+			headers:     goodRequestHeaders,
+			body:        func() (io.ReadCloser, error) { return newTokenReviewBody(otherUser + ":" + password) },
+			wantStatus:  http.StatusOK,
+			wantHeaders: map[string][]string{"Content-Type": {"application/json"}},
+			wantBody:    unauthenticatedResponseJSON(),
+		},
+		{
+			name:        "when a user has no password hash in the secret",
+			url:         goodURL,
+			method:      http.MethodPost,
+			headers:     goodRequestHeaders,
+			body:        func() (io.ReadCloser, error) { return newTokenReviewBody(passwordUndefinedUser + ":foo") },
+			wantStatus:  http.StatusOK,
+			wantHeaders: map[string][]string{"Content-Type": {"application/json"}},
+			wantBody:    unauthenticatedResponseJSON(),
+		},
+		{
+			name:        "when a user has an invalid password hash in the secret",
+			url:         goodURL,
+			method:      http.MethodPost,
+			headers:     goodRequestHeaders,
+			body:        func() (io.ReadCloser, error) { return newTokenReviewBody(invalidPasswordHashUser + ":foo") },
+			wantStatus:  http.StatusOK,
+			wantHeaders: map[string][]string{"Content-Type": {"application/json"}},
+			wantBody:    unauthenticatedResponseJSON(),
+		},
+		{
+			name:    "success for a user has no groups defined in the secret",
+			url:     goodURL,
+			method:  http.MethodPost,
+			headers: goodRequestHeaders,
+			body: func() (io.ReadCloser, error) {
+				return newTokenReviewBody(undefinedGroupsUser + ":" + undefinedGroupsPassword)
+			},
+			wantStatus:  http.StatusOK,
+			wantHeaders: map[string][]string{"Content-Type": {"application/json"}},
+			wantBody:    authenticatedResponseJSON(undefinedGroupsUser, nil),
+		},
+		{
+			name:        "when a user has empty string as their password",
+			url:         goodURL,
+			method:      http.MethodPost,
+			headers:     goodRequestHeaders,
+			body:        func() (io.ReadCloser, error) { return newTokenReviewBody(passwordUndefinedUser + ":foo") },
+			wantStatus:  http.StatusOK,
+			wantHeaders: map[string][]string{"Content-Type": {"application/json"}},
+			wantBody:    unauthenticatedResponseJSON(),
+		},
+		{
+			name:        "wrong password for username",
+			url:         goodURL,
+			method:      http.MethodPost,
+			headers:     goodRequestHeaders,
+			body:        func() (io.ReadCloser, error) { return newTokenReviewBody(user + ":" + otherPassword) },
+			wantStatus:  http.StatusOK,
+			wantHeaders: map[string][]string{"Content-Type": {"application/json"}},
+			wantBody:    unauthenticatedResponseJSON(),
+		},
+		{
+			name:        "non-existent password for username",
+			url:         goodURL,
+			method:      http.MethodPost,
+			headers:     goodRequestHeaders,
+			body:        func() (io.ReadCloser, error) { return newTokenReviewBody(user + ":" + "some-non-existent-password") },
+			wantStatus:  http.StatusOK,
+			wantHeaders: map[string][]string{"Content-Type": {"application/json"}},
+			wantBody:    unauthenticatedResponseJSON(),
+		},
+		{
+			name:        "non-existent username",
+			url:         goodURL,
+			method:      http.MethodPost,
+			headers:     goodRequestHeaders,
+			body:        func() (io.ReadCloser, error) { return newTokenReviewBody("some-non-existent-user" + ":" + password) },
+			wantStatus:  http.StatusOK,
+			wantHeaders: map[string][]string{"Content-Type": {"application/json"}},
+			wantBody:    unauthenticatedResponseJSON(),
+		},
+		{
+			name:       "bad token format (missing colon)",
+			url:        goodURL,
+			method:     http.MethodPost,
+			headers:    goodRequestHeaders,
+			body:       func() (io.ReadCloser, error) { return newTokenReviewBody(user) },
+			wantStatus: http.StatusBadRequest,
+		},
+		{
+			name:        "password contains colon",
+			url:         goodURL,
+			method:      http.MethodPost,
+			headers:     goodRequestHeaders,
+			body:        func() (io.ReadCloser, error) { return newTokenReviewBody(colonUser + ":" + colonPassword) },
+			wantStatus:  http.StatusOK,
+			wantHeaders: map[string][]string{"Content-Type": {"application/json"}},
+			wantBody:    authenticatedResponseJSON(colonUser, []string{group0, group1}),
+		},
+		{
+			name:    "bad TokenReview group",
+			url:     goodURL,
+			method:  http.MethodPost,
+			headers: goodRequestHeaders,
+			body: func() (io.ReadCloser, error) {
+				return newTokenReviewBodyWithGVK(
+					user+":"+password,
+					&schema.GroupVersionKind{
+						Group:   "bad group",
+						Version: authenticationv1beta1.SchemeGroupVersion.Version,
+						Kind:    "TokenReview",
+					},
+				)
+			},
+			wantStatus: http.StatusBadRequest,
+		},
+		{
+			name:    "bad TokenReview version",
+			url:     goodURL,
+			method:  http.MethodPost,
+			headers: goodRequestHeaders,
+			body: func() (io.ReadCloser, error) {
+				return newTokenReviewBodyWithGVK(
+					user+":"+password,
+					&schema.GroupVersionKind{
+						Group:   authenticationv1beta1.SchemeGroupVersion.Group,
+						Version: "bad version",
+						Kind:    "TokenReview",
+					},
+				)
+			},
+			wantStatus: http.StatusBadRequest,
+		},
+		{
+			name:    "bad TokenReview kind",
+			url:     goodURL,
+			method:  http.MethodPost,
+			headers: goodRequestHeaders,
+			body: func() (io.ReadCloser, error) {
+				return newTokenReviewBodyWithGVK(
+					user+":"+password,
+					&schema.GroupVersionKind{
+						Group:   authenticationv1beta1.SchemeGroupVersion.Group,
+						Version: authenticationv1beta1.SchemeGroupVersion.Version,
+						Kind:    "wrong-kind",
+					},
+				)
+			},
+			wantStatus: http.StatusBadRequest,
+		},
+		{
+			name:       "bad path",
+			url:        fmt.Sprintf("https://%s/tuna", l.Addr().String()),
+			method:     http.MethodPost,
+			headers:    goodRequestHeaders,
+			body:       func() (io.ReadCloser, error) { return newTokenReviewBody("some-token") },
+			wantStatus: http.StatusNotFound,
+		},
+		{
+			name:       "bad method",
+			url:        goodURL,
+			method:     http.MethodGet,
+			headers:    goodRequestHeaders,
+			body:       func() (io.ReadCloser, error) { return newTokenReviewBody("some-token") },
+			wantStatus: http.StatusMethodNotAllowed,
+		},
+		{
+			name:   "bad content type",
+			url:    goodURL,
+			method: http.MethodPost,
+			headers: map[string][]string{
+				"Content-Type": {"application/xml"},
+				"Accept":       {"application/json"},
+			},
+			body:       func() (io.ReadCloser, error) { return newTokenReviewBody("some-token") },
+			wantStatus: http.StatusUnsupportedMediaType,
+		},
+		{
+			name:   "bad accept",
+			url:    goodURL,
+			method: http.MethodPost,
+			headers: map[string][]string{
+				"Content-Type": {"application/json"},
+				"Accept":       {"application/xml"},
+			},
+			body:       func() (io.ReadCloser, error) { return newTokenReviewBody("some-token") },
+			wantStatus: http.StatusUnsupportedMediaType,
+		},
+		{
+			name:   "success when there are multiple accepts and one of them is json",
+			url:    goodURL,
+			method: http.MethodPost,
+			headers: map[string][]string{
+				"Content-Type": {"application/json"},
+				"Accept":       {"something/else, application/xml, application/json"},
+			},
+			body:        func() (io.ReadCloser, error) { return newTokenReviewBody(user + ":" + password) },
+			wantStatus:  http.StatusOK,
+			wantHeaders: map[string][]string{"Content-Type": {"application/json"}},
+			wantBody:    authenticatedResponseJSON(user, []string{group0, group1}),
+		},
+		{
+			name:   "success when there are multiple accepts and one of them is */*",
+			url:    goodURL,
+			method: http.MethodPost,
+			headers: map[string][]string{
+				"Content-Type": {"application/json"},
+				"Accept":       {"something/else, */*, application/foo"},
+			},
+			body:        func() (io.ReadCloser, error) { return newTokenReviewBody(user + ":" + password) },
+			wantStatus:  http.StatusOK,
+			wantHeaders: map[string][]string{"Content-Type": {"application/json"}},
+			wantBody:    authenticatedResponseJSON(user, []string{group0, group1}),
+		},
+		{
+			name:   "success when there are multiple accepts and one of them is application/*",
+			url:    goodURL,
+			method: http.MethodPost,
+			headers: map[string][]string{
+				"Content-Type": {"application/json"},
+				"Accept":       {"something/else, application/*, application/foo"},
+			},
+			body:        func() (io.ReadCloser, error) { return newTokenReviewBody(user + ":" + password) },
+			wantStatus:  http.StatusOK,
+			wantHeaders: map[string][]string{"Content-Type": {"application/json"}},
+			wantBody:    authenticatedResponseJSON(user, []string{group0, group1}),
+		},
+		{
+			name:       "bad body",
+			url:        goodURL,
+			method:     http.MethodPost,
+			headers:    goodRequestHeaders,
+			body:       func() (io.ReadCloser, error) { return ioutil.NopCloser(bytes.NewBuffer([]byte("invalid body"))), nil },
+			wantStatus: http.StatusBadRequest,
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			parsedURL, err := url.Parse(test.url)
+			require.NoError(t, err)
+
+			body, err := test.body()
+			require.NoError(t, err)
+
+			rsp, err := client.Do(&http.Request{
+				Method: test.method,
+				URL:    parsedURL,
+				Header: test.headers,
+				Body:   body,
+			})
+			require.NoError(t, err)
+			defer func() { _ = rsp.Body.Close() }()
+
+			require.Equal(t, test.wantStatus, rsp.StatusCode)
+
+			if test.wantHeaders != nil {
+				for k, v := range test.wantHeaders {
+					require.Equal(t, v, rsp.Header.Values(k))
+				}
+			}
+
+			responseBody, err := ioutil.ReadAll(rsp.Body)
+			require.NoError(t, err)
+			if test.wantBody != nil {
+				require.NoError(t, err)
+
+				var tr authenticationv1beta1.TokenReview
+				require.NoError(t, json.Unmarshal(responseBody, &tr))
+				require.Equal(t, test.wantBody, &tr)
+			} else {
+				require.Empty(t, responseBody)
+			}
+		})
+	}
+}
+
+func createSecretInformer(ctx context.Context, t *testing.T, kubeClient kubernetes.Interface) corev1informers.SecretInformer {
+	t.Helper()
+
+	kubeInformers := kubeinformers.NewSharedInformerFactory(kubeClient, 0)
+
+	secretInformer := kubeInformers.Core().V1().Secrets()
+
+	// We need to call Informer() on the secretInformer to lazily instantiate the
+	// informer factory before syncing it.
+	secretInformer.Informer()
+
+	kubeInformers.Start(ctx.Done())
+
+	informerTypesSynced := kubeInformers.WaitForCacheSync(ctx.Done())
+	require.True(t, informerTypesSynced[reflect.TypeOf(&corev1.Secret{})])
+
+	return secretInformer
+}
+
+// newClientProvider returns a dynamiccert.Provider configured
+// with valid serving cert, the CA bundle that can be used to verify the serving
+// cert, and the server name that can be used to verify the TLS peer.
+func newCertProvider(t *testing.T) (dynamiccert.Private, []byte, string) {
+	t.Helper()
+
+	serverName := "local-user-authenticator"
+
+	ca, err := certauthority.New(serverName+" CA", time.Hour*24)
+	require.NoError(t, err)
+
+	cert, err := ca.IssueServerCert([]string{serverName}, nil, time.Hour*24)
+	require.NoError(t, err)
+
+	certPEM, keyPEM, err := certauthority.ToPEM(cert)
+	require.NoError(t, err)
+
+	certProvider := dynamiccert.NewServingCert(t.Name())
+	err = certProvider.SetCertKeyContent(certPEM, keyPEM)
+	require.NoError(t, err)
+
+	return certProvider, ca.Bundle(), serverName
+}
+
+// newClient creates an http.Client that can be used to make an HTTPS call to a
+// service whose serving certs can be verified by the provided CA bundle.
+func newClient(caBundle []byte, serverName string) *http.Client {
+	rootCAs := x509.NewCertPool()
+	rootCAs.AppendCertsFromPEM(caBundle)
+	return &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				MinVersion: tls.VersionTLS13,
+				RootCAs:    rootCAs,
+				ServerName: serverName,
+			},
+		},
+	}
+}
+
+// newTokenReviewBody creates an io.ReadCloser that contains a JSON-encodeed
+// TokenReview request with expected APIVersion and Kind fields.
+func newTokenReviewBody(token string) (io.ReadCloser, error) {
+	return newTokenReviewBodyWithGVK(
+		token,
+		&schema.GroupVersionKind{
+			Group:   authenticationv1beta1.SchemeGroupVersion.Group,
+			Version: authenticationv1beta1.SchemeGroupVersion.Version,
+			Kind:    "TokenReview",
+		},
+	)
+}
+
+// newTokenReviewBodyWithGVK creates an io.ReadCloser that contains a
+// JSON-encoded TokenReview request. The TypeMeta fields of the TokenReview are
+// filled in with the provided gvk.
+func newTokenReviewBodyWithGVK(token string, gvk *schema.GroupVersionKind) (io.ReadCloser, error) {
+	buf := bytes.NewBuffer([]byte{})
+	tr := authenticationv1beta1.TokenReview{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: gvk.GroupVersion().String(),
+			Kind:       gvk.Kind,
+		},
+		Spec: authenticationv1beta1.TokenReviewSpec{
+			Token: token,
+		},
+	}
+	err := json.NewEncoder(buf).Encode(&tr)
+	return ioutil.NopCloser(buf), err
+}
+
+func unauthenticatedResponseJSON() *authenticationv1beta1.TokenReview {
+	return &authenticationv1beta1.TokenReview{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "TokenReview",
+			APIVersion: "authentication.k8s.io/v1beta1",
+		},
+		Status: authenticationv1beta1.TokenReviewStatus{
+			Authenticated: false,
+		},
+	}
+}
+
+func authenticatedResponseJSON(user string, groups []string) *authenticationv1beta1.TokenReview {
+	return &authenticationv1beta1.TokenReview{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "TokenReview",
+			APIVersion: "authentication.k8s.io/v1beta1",
+		},
+		Status: authenticationv1beta1.TokenReviewStatus{
+			Authenticated: true,
+			User: authenticationv1beta1.UserInfo{
+				Username: user,
+				Groups:   groups,
+			},
+		},
+	}
+}
+
+func addSecretToFakeClientTracker(t *testing.T, kubeClient *kubernetesfake.Clientset, username, password, groups string) {
+	passwordHash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.MinCost)
+	require.NoError(t, err)
+
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      username,
+			Namespace: namespace,
+		},
+		Data: map[string][]byte{
+			"passwordHash": passwordHash,
+			"groups":       []byte(groups),
+		},
+	}
+
+	require.NoError(t, kubeClient.Tracker().Add(secret))
+}

internal/supervisor/server/server.go

@@ -0,0 +1,398 @@
+// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package server defines the entrypoint for the Pinniped Supervisor server.
+package server
+
+import (
+	"context"
+	"crypto/rand"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"os/signal"
+	"strings"
+	"time"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/clock"
+	kubeinformers "k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/pkg/version"
+	"k8s.io/client-go/rest"
+	"k8s.io/component-base/logs"
+	"k8s.io/klog/v2"
+	"k8s.io/klog/v2/klogr"
+
+	configv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/config/v1alpha1"
+	pinnipedclientset "go.pinniped.dev/generated/latest/client/supervisor/clientset/versioned"
+	pinnipedinformers "go.pinniped.dev/generated/latest/client/supervisor/informers/externalversions"
+	"go.pinniped.dev/internal/config/supervisor"
+	"go.pinniped.dev/internal/controller/supervisorconfig"
+	"go.pinniped.dev/internal/controller/supervisorconfig/generator"
+	"go.pinniped.dev/internal/controller/supervisorconfig/ldapupstreamwatcher"
+	"go.pinniped.dev/internal/controller/supervisorconfig/oidcupstreamwatcher"
+	"go.pinniped.dev/internal/controller/supervisorstorage"
+	"go.pinniped.dev/internal/controllerlib"
+	"go.pinniped.dev/internal/deploymentref"
+	"go.pinniped.dev/internal/downward"
+	"go.pinniped.dev/internal/groupsuffix"
+	"go.pinniped.dev/internal/kubeclient"
+	"go.pinniped.dev/internal/oidc/jwks"
+	"go.pinniped.dev/internal/oidc/provider"
+	"go.pinniped.dev/internal/oidc/provider/manager"
+	"go.pinniped.dev/internal/plog"
+	"go.pinniped.dev/internal/secret"
+)
+
+const (
+	singletonWorker       = 1
+	defaultResyncInterval = 3 * time.Minute
+)
+
+func start(ctx context.Context, l net.Listener, handler http.Handler) {
+	server := http.Server{Handler: handler}
+
+	errCh := make(chan error)
+	go func() {
+		errCh <- server.Serve(l)
+	}()
+
+	go func() {
+		select {
+		case err := <-errCh:
+			plog.Debug("server exited", "err", err)
+		case <-ctx.Done():
+			plog.Debug("server context cancelled", "err", ctx.Err())
+			if err := server.Shutdown(context.Background()); err != nil {
+				plog.Debug("server shutdown failed", "err", err)
+			}
+		}
+	}()
+}
+
+func waitForSignal() os.Signal {
+	signalCh := make(chan os.Signal, 1)
+	signal.Notify(signalCh, os.Interrupt)
+	return <-signalCh
+}
+
+//nolint:funlen
+func startControllers(
+	ctx context.Context,
+	cfg *supervisor.Config,
+	issuerManager *manager.Manager,
+	dynamicJWKSProvider jwks.DynamicJWKSProvider,
+	dynamicTLSCertProvider provider.DynamicTLSCertProvider,
+	dynamicUpstreamIDPProvider provider.DynamicUpstreamIDPProvider,
+	secretCache *secret.Cache,
+	supervisorDeployment *appsv1.Deployment,
+	kubeClient kubernetes.Interface,
+	pinnipedClient pinnipedclientset.Interface,
+	kubeInformers kubeinformers.SharedInformerFactory,
+	pinnipedInformers pinnipedinformers.SharedInformerFactory,
+) {
+	federationDomainInformer := pinnipedInformers.Config().V1alpha1().FederationDomains()
+	secretInformer := kubeInformers.Core().V1().Secrets()
+
+	// Create controller manager.
+	controllerManager := controllerlib.
+		NewManager().
+		WithController(
+			supervisorstorage.GarbageCollectorController(
+				clock.RealClock{},
+				kubeClient,
+				secretInformer,
+				controllerlib.WithInformer,
+			),
+			singletonWorker,
+		).
+		WithController(
+			supervisorconfig.NewFederationDomainWatcherController(
+				issuerManager,
+				clock.RealClock{},
+				pinnipedClient,
+				federationDomainInformer,
+				controllerlib.WithInformer,
+			),
+			singletonWorker,
+		).
+		WithController(
+			supervisorconfig.NewJWKSWriterController(
+				cfg.Labels,
+				kubeClient,
+				pinnipedClient,
+				secretInformer,
+				federationDomainInformer,
+				controllerlib.WithInformer,
+			),
+			singletonWorker,
+		).
+		WithController(
+			supervisorconfig.NewJWKSObserverController(
+				dynamicJWKSProvider,
+				secretInformer,
+				federationDomainInformer,
+				controllerlib.WithInformer,
+			),
+			singletonWorker,
+		).
+		WithController(
+			supervisorconfig.NewTLSCertObserverController(
+				dynamicTLSCertProvider,
+				cfg.NamesConfig.DefaultTLSCertificateSecret,
+				secretInformer,
+				federationDomainInformer,
+				controllerlib.WithInformer,
+			),
+			singletonWorker,
+		).
+		WithController(
+			generator.NewSupervisorSecretsController(
+				supervisorDeployment,
+				cfg.Labels,
+				kubeClient,
+				secretInformer,
+				func(secret []byte) {
+					plog.Debug("setting csrf cookie secret")
+					secretCache.SetCSRFCookieEncoderHashKey(secret)
+				},
+				controllerlib.WithInformer,
+				controllerlib.WithInitialEvent,
+			),
+			singletonWorker,
+		).
+		WithController(
+			generator.NewFederationDomainSecretsController(
+				generator.NewSymmetricSecretHelper(
+					"pinniped-oidc-provider-hmac-key-",
+					cfg.Labels,
+					rand.Reader,
+					generator.SecretUsageTokenSigningKey,
+					func(federationDomainIssuer string, symmetricKey []byte) {
+						plog.Debug("setting hmac secret", "issuer", federationDomainIssuer)
+						secretCache.SetTokenHMACKey(federationDomainIssuer, symmetricKey)
+					},
+				),
+				func(fd *configv1alpha1.FederationDomainStatus) *corev1.LocalObjectReference {
+					return &fd.Secrets.TokenSigningKey
+				},
+				kubeClient,
+				pinnipedClient,
+				secretInformer,
+				federationDomainInformer,
+				controllerlib.WithInformer,
+			),
+			singletonWorker,
+		).
+		WithController(
+			generator.NewFederationDomainSecretsController(
+				generator.NewSymmetricSecretHelper(
+					"pinniped-oidc-provider-upstream-state-signature-key-",
+					cfg.Labels,
+					rand.Reader,
+					generator.SecretUsageStateSigningKey,
+					func(federationDomainIssuer string, symmetricKey []byte) {
+						plog.Debug("setting state signature key", "issuer", federationDomainIssuer)
+						secretCache.SetStateEncoderHashKey(federationDomainIssuer, symmetricKey)
+					},
+				),
+				func(fd *configv1alpha1.FederationDomainStatus) *corev1.LocalObjectReference {
+					return &fd.Secrets.StateSigningKey
+				},
+				kubeClient,
+				pinnipedClient,
+				secretInformer,
+				federationDomainInformer,
+				controllerlib.WithInformer,
+			),
+			singletonWorker,
+		).
+		WithController(
+			generator.NewFederationDomainSecretsController(
+				generator.NewSymmetricSecretHelper(
+					"pinniped-oidc-provider-upstream-state-encryption-key-",
+					cfg.Labels,
+					rand.Reader,
+					generator.SecretUsageStateEncryptionKey,
+					func(federationDomainIssuer string, symmetricKey []byte) {
+						plog.Debug("setting state encryption key", "issuer", federationDomainIssuer)
+						secretCache.SetStateEncoderBlockKey(federationDomainIssuer, symmetricKey)
+					},
+				),
+				func(fd *configv1alpha1.FederationDomainStatus) *corev1.LocalObjectReference {
+					return &fd.Secrets.StateEncryptionKey
+				},
+				kubeClient,
+				pinnipedClient,
+				secretInformer,
+				federationDomainInformer,
+				controllerlib.WithInformer,
+			),
+			singletonWorker,
+		).
+		WithController(
+			oidcupstreamwatcher.New(
+				dynamicUpstreamIDPProvider,
+				pinnipedClient,
+				pinnipedInformers.IDP().V1alpha1().OIDCIdentityProviders(),
+				secretInformer,
+				klogr.New(),
+				controllerlib.WithInformer,
+			),
+			singletonWorker).
+		WithController(
+			ldapupstreamwatcher.New(
+				dynamicUpstreamIDPProvider,
+				pinnipedClient,
+				pinnipedInformers.IDP().V1alpha1().LDAPIdentityProviders(),
+				secretInformer,
+				controllerlib.WithInformer,
+			),
+			singletonWorker)
+
+	kubeInformers.Start(ctx.Done())
+	pinnipedInformers.Start(ctx.Done())
+
+	// Wait until the caches are synced before returning.
+	kubeInformers.WaitForCacheSync(ctx.Done())
+	pinnipedInformers.WaitForCacheSync(ctx.Done())
+
+	go controllerManager.Start(ctx)
+}
+
+func run(podInfo *downward.PodInfo, cfg *supervisor.Config) error {
+	serverInstallationNamespace := podInfo.Namespace
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	dref, supervisorDeployment, err := deploymentref.New(podInfo)
+	if err != nil {
+		return fmt.Errorf("cannot create deployment ref: %w", err)
+	}
+
+	client, err := kubeclient.New(
+		dref,
+		kubeclient.WithMiddleware(groupsuffix.New(*cfg.APIGroupSuffix)),
+	)
+	if err != nil {
+		return fmt.Errorf("cannot create k8s client: %w", err)
+	}
+
+	kubeInformers := kubeinformers.NewSharedInformerFactoryWithOptions(
+		client.Kubernetes,
+		defaultResyncInterval,
+		kubeinformers.WithNamespace(serverInstallationNamespace),
+	)
+
+	pinnipedInformers := pinnipedinformers.NewSharedInformerFactoryWithOptions(
+		client.PinnipedSupervisor,
+		defaultResyncInterval,
+		pinnipedinformers.WithNamespace(serverInstallationNamespace),
+	)
+
+	// Serve the /healthz endpoint and make all other paths result in 404.
+	healthMux := http.NewServeMux()
+	healthMux.Handle("/healthz", http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+		_, _ = writer.Write([]byte("ok"))
+	}))
+
+	dynamicJWKSProvider := jwks.NewDynamicJWKSProvider()
+	dynamicTLSCertProvider := provider.NewDynamicTLSCertProvider()
+	dynamicUpstreamIDPProvider := provider.NewDynamicUpstreamIDPProvider()
+	secretCache := secret.Cache{}
+
+	// OIDC endpoints will be served by the oidProvidersManager, and any non-OIDC paths will fallback to the healthMux.
+	oidProvidersManager := manager.NewManager(
+		healthMux,
+		dynamicJWKSProvider,
+		dynamicUpstreamIDPProvider,
+		&secretCache,
+		client.Kubernetes.CoreV1().Secrets(serverInstallationNamespace),
+	)
+
+	startControllers(
+		ctx,
+		cfg,
+		oidProvidersManager,
+		dynamicJWKSProvider,
+		dynamicTLSCertProvider,
+		dynamicUpstreamIDPProvider,
+		&secretCache,
+		supervisorDeployment,
+		client.Kubernetes,
+		client.PinnipedSupervisor,
+		kubeInformers,
+		pinnipedInformers,
+	)
+
+	//nolint: gosec // Intentionally binding to all network interfaces.
+	httpListener, err := net.Listen("tcp", ":8080")
+	if err != nil {
+		return fmt.Errorf("cannot create listener: %w", err)
+	}
+	defer func() { _ = httpListener.Close() }()
+	start(ctx, httpListener, oidProvidersManager)
+
+	//nolint: gosec // Intentionally binding to all network interfaces.
+	httpsListener, err := tls.Listen("tcp", ":8443", &tls.Config{
+		MinVersion: tls.VersionTLS12, // Allow v1.2 because clients like the default `curl` on MacOS don't support 1.3 yet.
+		GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
+			cert := dynamicTLSCertProvider.GetTLSCert(strings.ToLower(info.ServerName))
+			defaultCert := dynamicTLSCertProvider.GetDefaultTLSCert()
+			plog.Debug("GetCertificate called for port 8443",
+				"info.ServerName", info.ServerName,
+				"foundSNICert", cert != nil,
+				"foundDefaultCert", defaultCert != nil,
+			)
+			if cert == nil {
+				cert = defaultCert
+			}
+			return cert, nil
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("cannot create listener: %w", err)
+	}
+	defer func() { _ = httpsListener.Close() }()
+	start(ctx, httpsListener, oidProvidersManager)
+
+	plog.Debug("supervisor is ready",
+		"httpAddress", httpListener.Addr().String(),
+		"httpsAddress", httpsListener.Addr().String(),
+	)
+
+	gotSignal := waitForSignal()
+	plog.Debug("supervisor exiting", "signal", gotSignal)
+
+	return nil
+}
+
+func Main() {
+	logs.InitLogs()
+	defer logs.FlushLogs()
+	plog.RemoveKlogGlobalFlags() // move this whenever the below code gets refactored to use cobra
+
+	klog.Infof("Running %s at %#v", rest.DefaultKubernetesUserAgent(), version.Get())
+	klog.Infof("Command-line arguments were: %s %s %s", os.Args[0], os.Args[1], os.Args[2])
+
+	// Discover in which namespace we are installed.
+	podInfo, err := downward.Load(os.Args[1])
+	if err != nil {
+		klog.Fatal(fmt.Errorf("could not read pod metadata: %w", err))
+	}
+
+	// Read the server config file.
+	cfg, err := supervisor.FromPath(os.Args[2])
+	if err != nil {
+		klog.Fatal(fmt.Errorf("could not load config: %w", err))
+	}
+
+	if err := run(podInfo, cfg); err != nil {
+		klog.Fatal(err)
+	}
+}