Merge pull request #2009 from vmware-tanzu/audit_logging

Add audit logging for Supervisor and Concierge

cmd/pinniped/cmd/audit_id.go

@@ -0,0 +1,46 @@
+// Copyright 2024 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package cmd
+
+import (
+	"net/http"
+
+	"go.pinniped.dev/internal/httputil/roundtripper"
+	"go.pinniped.dev/internal/plog"
+)
+
+type auditIDLoggerFunc func(path string, statusCode int, auditID string)
+
+func logAuditID(path string, statusCode int, auditID string) {
+	plog.Info("Received auditID for failed request",
+		"path", path,
+		"statusCode", statusCode,
+		"auditID", auditID)
+}
+
+func LogAuditIDTransportWrapper(rt http.RoundTripper) http.RoundTripper {
+	return logAuditIDTransportWrapper(rt, logAuditID)
+}
+
+func logAuditIDTransportWrapper(rt http.RoundTripper, auditIDLoggerFunc auditIDLoggerFunc) http.RoundTripper {
+	return roundtripper.WrapFunc(rt, func(r *http.Request) (*http.Response, error) {
+		response, responseErr := rt.RoundTrip(r)
+
+		if responseErr != nil ||
+			response == nil ||
+			response.Header.Get("audit-ID") == "" ||
+			response.Request == nil ||
+			response.Request.URL == nil {
+			return response, responseErr
+		}
+
+		// Use the request path from the response's request, in case the
+		// original request was modified by any other roudtrippers in the chain.
+		auditIDLoggerFunc(response.Request.URL.Path,
+			response.StatusCode,
+			response.Header.Get("audit-ID"))
+
+		return response, responseErr
+	})
+}

cmd/pinniped/cmd/audit_id_test.go

@@ -0,0 +1,116 @@
+// Copyright 2024 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package cmd
+
+import (
+	"errors"
+	"net/http"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"go.pinniped.dev/internal/httputil/roundtripper"
+)
+
+func TestLogAuditIDTransportWrapper(t *testing.T) {
+	canonicalAuditIdHeaderName := "Audit-Id"
+
+	tests := []struct {
+		name        string
+		response    *http.Response
+		responseErr error
+		want        func(t *testing.T, called func()) auditIDLoggerFunc
+		wantCalled  bool
+	}{
+		{
+			name: "happy HTTP response - no error and no log",
+			response: &http.Response{ // no headers
+				StatusCode: http.StatusOK,
+				Request: &http.Request{
+					URL: &url.URL{
+						Path: "some-path-from-response-request",
+					},
+				},
+			},
+			responseErr: nil,
+			want: func(t *testing.T, called func()) auditIDLoggerFunc {
+				return func(_ string, _ int, _ string) {
+					called()
+				}
+			},
+			wantCalled: false, // make it obvious
+		},
+		{
+			name:        "nil HTTP response - no error and no log",
+			response:    nil,
+			responseErr: nil,
+			want: func(t *testing.T, called func()) auditIDLoggerFunc {
+				return func(_ string, _ int, _ string) {
+					called()
+				}
+			},
+			wantCalled: false, // make it obvious
+		},
+		{
+			name:        "err HTTP response - no error and no log",
+			response:    nil,
+			responseErr: errors.New("some error"),
+			want: func(t *testing.T, called func()) auditIDLoggerFunc {
+				return func(_ string, _ int, _ string) {
+					called()
+				}
+			},
+			wantCalled: false, // make it obvious
+		},
+		{
+			name: "happy HTTP response with audit-ID - logs",
+			response: &http.Response{
+				Header: http.Header{
+					canonicalAuditIdHeaderName: []string{"some-audit-id", "some-other-audit-id-that-will-never-be-seen"},
+				},
+				StatusCode: http.StatusBadGateway, // statusCode does not matter
+				Request: &http.Request{
+					URL: &url.URL{
+						Path: "some-path-from-response-request",
+					},
+				},
+			},
+			want: func(t *testing.T, called func()) auditIDLoggerFunc {
+				return func(path string, statusCode int, auditID string) {
+					called()
+					require.Equal(t, "some-path-from-response-request", path)
+					require.Equal(t, http.StatusBadGateway, statusCode)
+					require.Equal(t, "some-audit-id", auditID)
+				}
+			},
+			wantCalled: true, // make it obvious
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			require.NotNil(t, test.want)
+
+			mockRequest := &http.Request{
+				URL: &url.URL{
+					Path: "should-never-use-this-path",
+				},
+			}
+			var mockRt roundtripper.Func = func(r *http.Request) (*http.Response, error) {
+				require.Equal(t, mockRequest, r)
+				return test.response, test.responseErr
+			}
+			called := false
+			subjectRt := logAuditIDTransportWrapper(mockRt, test.want(t, func() {
+				called = true
+			}))
+			actualResponse, err := subjectRt.RoundTrip(mockRequest) //nolint:bodyclose // there is no Body.
+			require.Equal(t, test.responseErr, err)                 // This roundtripper only returns mocked errors.
+			require.Equal(t, test.response, actualResponse)
+			require.Equal(t, test.wantCalled, called,
+				"want logFunc to be called: %t, actually was called: %t", test.wantCalled, called)
+		})
+	}
+}

cmd/pinniped/cmd/kubeconfig_test.go

@@ -11,7 +11,6 @@ import (
 	"os"
 	"path/filepath"
 	"slices"
-	"strings"
 	"testing"
 	"time"
 
@@ -3293,14 +3292,10 @@ func TestGetKubeconfig(t *testing.T) {
 				require.NoError(t, err)
 			}
 
-			var expectedLogs string
 			if tt.wantLogs != nil {
-				temp := tt.wantLogs(string(testServerCA), testServer.URL)
-				if len(temp) > 0 {
-					expectedLogs = strings.Join(tt.wantLogs(string(testServerCA), testServer.URL), "\n") + "\n"
-				}
+				wantLogs := tt.wantLogs(string(testServerCA), testServer.URL)
+				testutil.RequireLogLines(t, wantLogs, &log)
 			}
-			require.Equal(t, expectedLogs, log.String())
 
 			expectedStdout := ""
 			if tt.wantStdout != nil {

cmd/pinniped/cmd/login_oidc.go

@@ -224,6 +224,7 @@ func runOIDCLogin(cmd *cobra.Command, deps oidcLoginCommandDeps, flags oidcLogin
 			conciergeclient.WithBase64CABundle(flags.conciergeCABundle),
 			conciergeclient.WithAuthenticator(flags.conciergeAuthenticatorType, flags.conciergeAuthenticatorName),
 			conciergeclient.WithAPIGroupSuffix(flags.conciergeAPIGroupSuffix),
+			conciergeclient.WithTransportWrapper(LogAuditIDTransportWrapper),
 		)
 		if err != nil {
 			return fmt.Errorf("invalid Concierge parameters: %w", err)

cmd/pinniped/cmd/login_oidc_test.go

@@ -274,8 +274,8 @@ func TestLoginOIDCCommand(t *testing.T) {
 			wantOptionsCount: 4,
 			wantStdout:       `{"kind":"ExecCredential","apiVersion":"client.authentication.k8s.io/v1beta1","spec":{"interactive":false},"status":{"expirationTimestamp":"3020-10-12T13:14:15Z","token":"test-id-token"}}` + "\n",
 			wantLogs: []string{
-				nowStr + `  cmd/login_oidc.go:267  Performing OIDC login  {"issuer": "test-issuer", "client id": "test-client-id"}`,
-				nowStr + `  cmd/login_oidc.go:287  No concierge configured, skipping token credential exchange`,
+				nowStr + `  cmd/login_oidc.go:268  Performing OIDC login  {"issuer": "test-issuer", "client id": "test-client-id"}`,
+				nowStr + `  cmd/login_oidc.go:288  No concierge configured, skipping token credential exchange`,
 			},
 		},
 		{
@@ -319,10 +319,10 @@ func TestLoginOIDCCommand(t *testing.T) {
 			wantOptionsCount: 12,
 			wantStdout:       `{"kind":"ExecCredential","apiVersion":"client.authentication.k8s.io/v1beta1","spec":{"interactive":false},"status":{"token":"exchanged-token"}}` + "\n",
 			wantLogs: []string{
-				nowStr + `  cmd/login_oidc.go:267  Performing OIDC login  {"issuer": "test-issuer", "client id": "test-client-id"}`,
-				nowStr + `  cmd/login_oidc.go:277  Exchanging token for cluster credential  {"endpoint": "https://127.0.0.1:1234/", "authenticator type": "webhook", "authenticator name": "test-authenticator"}`,
-				nowStr + `  cmd/login_oidc.go:285  Successfully exchanged token for cluster credential.`,
-				nowStr + `  cmd/login_oidc.go:292  caching cluster credential for future use.`,
+				nowStr + `  cmd/login_oidc.go:268  Performing OIDC login  {"issuer": "test-issuer", "client id": "test-client-id"}`,
+				nowStr + `  cmd/login_oidc.go:278  Exchanging token for cluster credential  {"endpoint": "https://127.0.0.1:1234/", "authenticator type": "webhook", "authenticator name": "test-authenticator"}`,
+				nowStr + `  cmd/login_oidc.go:286  Successfully exchanged token for cluster credential.`,
+				nowStr + `  cmd/login_oidc.go:293  caching cluster credential for future use.`,
 			},
 		},
 	}

cmd/pinniped/cmd/login_static.go

@@ -113,6 +113,7 @@ func runStaticLogin(cmd *cobra.Command, deps staticLoginDeps, flags staticLoginP
 			conciergeclient.WithBase64CABundle(flags.conciergeCABundle),
 			conciergeclient.WithAuthenticator(flags.conciergeAuthenticatorType, flags.conciergeAuthenticatorName),
 			conciergeclient.WithAPIGroupSuffix(flags.conciergeAPIGroupSuffix),
+			conciergeclient.WithTransportWrapper(LogAuditIDTransportWrapper),
 		)
 		if err != nil {
 			return fmt.Errorf("invalid Concierge parameters: %w", err)

cmd/pinniped/cmd/login_static_test.go

@@ -147,7 +147,7 @@ func TestLoginStaticCommand(t *testing.T) {
 				Error: could not complete Concierge credential exchange: some concierge error
 			`),
 			wantLogs: []string{
-				nowStr + `  cmd/login_static.go:159  exchanging static token for cluster credential  {"endpoint": "https://127.0.0.1/", "authenticator type": "webhook", "authenticator name": "test-authenticator"}`,
+				nowStr + `  cmd/login_static.go:160  exchanging static token for cluster credential  {"endpoint": "https://127.0.0.1/", "authenticator type": "webhook", "authenticator name": "test-authenticator"}`,
 			},
 		},
 		{

deploy/concierge/deployment.yaml

@@ -103,6 +103,8 @@ data:
     tls:
       onedottwo:
         allowedCiphers: (@= str(data.values.allowed_ciphers_for_tls_onedottwo) @)
+    audit:
+      logUsernamesAndGroups: (@= data.values.audit.log_usernames_and_groups @)
 ---
 #@ if data.values.image_pull_dockerconfigjson and data.values.image_pull_dockerconfigjson != "":
 apiVersion: v1

deploy/concierge/values.yaml

@@ -231,3 +231,15 @@ no_proxy: "$(KUBERNETES_SERVICE_HOST),169.254.169.254,127.0.0.1,localhost,.svc,.
 #! An empty array is perfectly valid, as is any array of strings.
 allowed_ciphers_for_tls_onedottwo:
 - ""
+
+#@schema/title "Audit logging configuration"
+#@schema/desc "Customize the content of audit log events."
+audit:
+
+  #@schema/title "Log usernames and groups"
+  #@ log_usernames_and_groups_desc = "Enables or disables printing usernames and group names in audit logs. Options are 'enabled' or 'disabled'. \
+  #@ If enabled, usernames are group names may be printed in audit log events. \
+  #@ If disabled, usernames and group names will be redacted from audit logs because they might contain personally identifiable information."
+  #@schema/desc log_usernames_and_groups_desc
+  #@schema/validation one_of=["enabled", "disabled"]
+  log_usernames_and_groups: disabled

deploy/supervisor/helpers.lib.yaml

@@ -57,6 +57,10 @@ _: #@ template.replace(data.values.custom_labels)
 #@       "onedottwo": {
 #@         "allowedCiphers": data.values.allowed_ciphers_for_tls_onedottwo
 #@       }
+#@     },
+#@     "audit": {
+#@       "logUsernamesAndGroups": data.values.audit.log_usernames_and_groups,
+#@       "logInternalPaths": data.values.audit.log_internal_paths
 #@     }
 #@   }
 #@   if data.values.log_level:

deploy/supervisor/values.yaml

@@ -220,3 +220,23 @@ endpoints: { }
 #! An empty array is perfectly valid, as is any array of strings.
 allowed_ciphers_for_tls_onedottwo:
 - ""
+
+#@schema/title "Audit logging configuration"
+#@schema/desc "Customize the content of audit log events."
+audit:
+
+  #@schema/title "Log usernames and groups"
+  #@ log_usernames_and_groups_desc = "Enables or disables printing usernames and group names in audit logs. Options are 'enabled' or 'disabled'. \
+  #@ If enabled, usernames are group names may be printed in audit log events. \
+  #@ If disabled, usernames and group names will be redacted from audit logs because they might contain personally identifiable information."
+  #@schema/desc log_usernames_and_groups_desc
+  #@schema/validation one_of=["enabled", "disabled"]
+  log_usernames_and_groups: disabled
+
+  #@schema/title "Log HTTPS requests for internal paths"
+  #@ log_internal_paths = "Enables or disables request logging for internal paths in audit logs. Options are 'enabled' or 'disabled'. \
+  #@ If enabled, requests to certain paths that are typically only used internal to the cluster (e.g. /healthz) will be enabled, which can be very verbose. \
+  #@ If disabled, requests to those paths will not be audit logged."
+  #@schema/desc log_internal_paths
+  #@schema/validation one_of=["enabled", "disabled"]
+  log_internal_paths: disabled

hack/kind-up.sh

@@ -37,8 +37,12 @@ if [[ "${PINNIPED_USE_LOCAL_KIND_REGISTRY:-}" != "" ]]; then
   use_kind_registry="--file=${ROOT}/hack/lib/kind-config/kind-registry-overlay.yaml"
 fi
 
+cp "${ROOT}/hack/lib/kind-config/metadata-audit-policy.yaml" /tmp/metadata-audit-policy.yaml
+
 # Do not quote ${use_kind_registry} ${use_contour_registry} in this command because they might be empty.
-ytt ${use_kind_registry} ${use_contour_registry} --file="${ROOT}"/hack/lib/kind-config/single-node.yaml >/tmp/kind-config.yaml
+ytt ${use_kind_registry} ${use_contour_registry} \
+  --data-value-yaml enable_audit_logs=${ENABLE_KIND_AUDIT_LOGS:-false} \
+  --file="${ROOT}"/hack/lib/kind-config/single-node.yaml >/tmp/kind-config.yaml
 
 # To choose a specific version of kube, add this option to the command below: `--image kindest/node:v1.28.0`.
 # To use the "latest-main" version of kubernetes builds by the pipeline, use `--image ghcr.io/pinniped-ci-bot/kind-node-image:latest`

hack/lib/kind-config/metadata-audit-policy.yaml

@@ -0,0 +1,4 @@
+apiVersion: audit.k8s.io/v1
+kind: Policy
+rules:
+- level: Metadata

hack/lib/kind-config/single-node.yaml

@@ -1,46 +1,64 @@
+#@ load("@ytt:data", "data")
+
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 nodes:
   - role: control-plane
     extraPortMappings:
     - protocol: TCP
-      # This same port number is hardcoded in the integration test setup
-      # when creating a Service on a kind cluster. It is used to talk to
-      # the supervisor app via HTTPS.
+      #! This same port number is hardcoded in the integration test setup
+      #! when creating a Service on a kind cluster. It is used to talk to
+      #! the supervisor app via HTTPS.
       containerPort: 31243
       hostPort: 12344
       listenAddress: 127.0.0.1
     - protocol: TCP
-      # This same port number is hardcoded in the integration test setup
-      # when creating a Service on a kind cluster. It is used to talk to
-      # the Dex app.
+      #! This same port number is hardcoded in the integration test setup
+      #! when creating a Service on a kind cluster. It is used to talk to
+      #! the Dex app.
       containerPort: 31235
       hostPort: 12346
       listenAddress: 127.0.0.1
-# Kind v0.12.0 ignores kubeadm.k8s.io/v1beta2 for Kube v1.23+ but uses it for older versions of Kube.
-# Previous versions of Kind would use kubeadm.k8s.io/v1beta2 for all versions of Kube including 1.23.
-# To try to maximize compatibility with various versions of Kind and Kube, define this
-# ClusterConfiguration twice and hope that Kind will use the one that it likes for the given version
-# of Kube, and ignore the one that it doesn't like. This seems to work, at least for Kind v0.12.0.
+    #@ if data.values.enable_audit_logs:
+    #! mount the local file on the control plane
+    extraMounts:
+      - hostPath: /tmp/metadata-audit-policy.yaml
+        containerPath: /etc/kubernetes/policies/audit-policy.yaml
+        readOnly: true
+    #@ end
+#! Apply these patches to all nodes.
 kubeadmConfigPatches:
 - |
-  apiVersion: kubeadm.k8s.io/v1beta2
   kind: ClusterConfiguration
   apiServer:
     extraArgs:
-      # To make sure the endpoints on our service are correct (this mostly matters for kubectl based
-      # installs where kapp is not doing magic changes to the deployment and service selectors).
-      # Setting this field to true makes it so that the API service will do the service cluster IP
-      # to endpoint IP translations internally instead of relying on the network stack (i.e. kube-proxy).
-      # The logic inside the API server is very straightforward - randomly pick an IP from the list
-      # of available endpoints.  This means that over time, all endpoints associated with the service
-      # are exercised.  For whatever reason, leaving this as false (i.e. use kube-proxy) appears to
-      # hide some network misconfigurations when used internally by the API server aggregation layer.
+      #! To make sure the endpoints on our service are correct (this mostly matters for kubectl based
+      #! installs where kapp is not doing magic changes to the deployment and service selectors).
+      #! Setting this field to true makes it so that the API service will do the service cluster IP
+      #! to endpoint IP translations internally instead of relying on the network stack (i.e. kube-proxy).
+      #! The logic inside the API server is very straightforward - randomly pick an IP from the list
+      #! of available endpoints. This means that over time, all endpoints associated with the service
+      #! are exercised. For whatever reason, leaving this as false (i.e. use kube-proxy) appears to
+      #! hide some network misconfigurations when used internally by the API server aggregation layer.
       enable-aggregator-routing: "true"
+#@ if data.values.enable_audit_logs:
 - |
-  apiVersion: kubeadm.k8s.io/v1beta3
   kind: ClusterConfiguration
   apiServer:
-    extraArgs:
-      # See comment above.
-      enable-aggregator-routing: "true"
+      #! enable auditing flags on the API server
+      extraArgs:
+        audit-log-path: /var/log/kubernetes/kube-apiserver-audit.log
+        audit-policy-file: /etc/kubernetes/policies/audit-policy.yaml
+      #! mount new files / directories on the control plane
+      extraVolumes:
+        - name: audit-policies
+          hostPath: /etc/kubernetes/policies
+          mountPath: /etc/kubernetes/policies
+          readOnly: true
+          pathType: "DirectoryOrCreate"
+        - name: "audit-logs"
+          hostPath: "/var/log/kubernetes"
+          mountPath: "/var/log/kubernetes"
+          readOnly: false
+          pathType: DirectoryOrCreate
+#@ end

hack/prepare-for-integration-tests.sh

@@ -317,6 +317,8 @@ custom_labels: $supervisor_custom_labels
 service_https_nodeport_port: $service_https_nodeport_port
 service_https_nodeport_nodeport: $service_https_nodeport_nodeport
 service_https_clusterip_port: $service_https_clusterip_port
+audit:
+  log_usernames_and_groups: ${LOG_USERNAMES_AND_GROUPS:-disabled}
 EOF
 
 if [[ "${FIREWALL_IDPS:-no}" == "yes" ]]; then
@@ -361,6 +363,8 @@ custom_labels: $concierge_custom_labels
 image_repo: $registry_repo
 image_tag: $tag
 discovery_url: $discovery_url
+audit:
+  log_usernames_and_groups: ${LOG_USERNAMES_AND_GROUPS:-disabled}
 EOF
 
 if [[ "${FIREWALL_IDPS:-no}" == "yes" ]]; then

hack/prepare-supervisor-on-kind.sh

@@ -44,8 +44,20 @@ use_ldap_upstream=no
 use_ad_upstream=no
 use_github_upstream=no
 use_flow=""
+api_group_suffix="pinniped.dev" # same default as in the values.yaml ytt file
+
 while (("$#")); do
   case "$1" in
+  -g | --api-group-suffix)
+    shift
+    # If there are no more command line arguments, or there is another command line argument but it starts with a dash, then error
+    if [[ "$#" == "0" || "$1" == -* ]]; then
+      log_error "-g|--api-group-suffix requires a group name to be specified"
+      exit 1
+    fi
+    api_group_suffix=$1
+    shift
+    ;;
   --flow)
     shift
     # If there are no more command line arguments, or there is another command line argument but it starts with a dash, then error
@@ -183,7 +195,7 @@ fi
 if [[ "$use_oidc_upstream" == "yes" ]]; then
   # Make an OIDCIdentityProvider which uses Dex to provide identity.
   cat <<EOF | kubectl apply --namespace "$PINNIPED_TEST_SUPERVISOR_NAMESPACE" -f -
-apiVersion: idp.supervisor.pinniped.dev/v1alpha1
+apiVersion: idp.supervisor.${api_group_suffix}/v1alpha1
 kind: OIDCIdentityProvider
 metadata:
   name: my-oidc-provider
@@ -222,7 +234,7 @@ fi
 if [[ "$use_ldap_upstream" == "yes" ]]; then
   # Make an LDAPIdentityProvider which uses OpenLDAP to provide identity.
   cat <<EOF | kubectl apply --namespace "$PINNIPED_TEST_SUPERVISOR_NAMESPACE" -f -
-apiVersion: idp.supervisor.pinniped.dev/v1alpha1
+apiVersion: idp.supervisor.${api_group_suffix}/v1alpha1
 kind: LDAPIdentityProvider
 metadata:
   name: my-ldap-provider
@@ -265,7 +277,7 @@ fi
 if [[ "$use_ad_upstream" == "yes" ]]; then
   # Make an ActiveDirectoryIdentityProvider. Needs to be pointed to a real AD server by env vars.
   cat <<EOF | kubectl apply --namespace "$PINNIPED_TEST_SUPERVISOR_NAMESPACE" -f -
-apiVersion: idp.supervisor.pinniped.dev/v1alpha1
+apiVersion: idp.supervisor.${api_group_suffix}/v1alpha1
 kind: ActiveDirectoryIdentityProvider
 metadata:
   name: my-ad-provider
@@ -298,7 +310,7 @@ fi
 if [[ "$use_github_upstream" == "yes" ]]; then
   # Make an GitHubIdentityProvider.  Needs to be configured with an actual GitHub App or GitHub OAuth App.
   cat <<EOF | kubectl apply --namespace "$PINNIPED_TEST_SUPERVISOR_NAMESPACE" -f -
-apiVersion: idp.supervisor.pinniped.dev/v1alpha1
+apiVersion: idp.supervisor.${api_group_suffix}/v1alpha1
 kind: GitHubIdentityProvider
 metadata:
   name: my-github-provider
@@ -351,7 +363,7 @@ kubectl create secret tls -n "$PINNIPED_TEST_SUPERVISOR_NAMESPACE" my-federation
 # Make a FederationDomain using the TLS Secret and identity providers from above in a temp file.
 fd_file="/tmp/federationdomain.yaml"
 cat <<EOF >$fd_file
-apiVersion: config.supervisor.pinniped.dev/v1alpha1
+apiVersion: config.supervisor.${api_group_suffix}/v1alpha1
 kind: FederationDomain
 metadata:
   name: my-federation-domain
@@ -368,7 +380,7 @@ if [[ "$use_oidc_upstream" == "yes" ]]; then
 
     - displayName: "My OIDC IDP 🚀"
       objectRef:
-        apiGroup: idp.supervisor.pinniped.dev
+        apiGroup: idp.supervisor.${api_group_suffix}
         kind: OIDCIdentityProvider
         name: my-oidc-provider
       transforms:
@@ -392,7 +404,7 @@ if [[ "$use_ldap_upstream" == "yes" ]]; then
 
     - displayName: "My LDAP IDP 🚀"
       objectRef:
-        apiGroup: idp.supervisor.pinniped.dev
+        apiGroup: idp.supervisor.${api_group_suffix}
         kind: LDAPIdentityProvider
         name: my-ldap-provider
       transforms: # these are contrived to exercise all the available features
@@ -446,7 +458,7 @@ if [[ "$use_ad_upstream" == "yes" ]]; then
 
     - displayName: "My AD IDP 🚀"
       objectRef:
-        apiGroup: idp.supervisor.pinniped.dev
+        apiGroup: idp.supervisor.${api_group_suffix}
         kind: ActiveDirectoryIdentityProvider
         name: my-ad-provider
 EOF
@@ -458,7 +470,7 @@ if [[ "$use_github_upstream" == "yes" ]]; then
 
     - displayName: "My GitHub IDP 🚀"
       objectRef:
-        apiGroup: idp.supervisor.pinniped.dev
+        apiGroup: idp.supervisor.${api_group_suffix}
         kind: GitHubIdentityProvider
         name: my-github-provider
 EOF
@@ -501,7 +513,7 @@ fi
 # The issuer URL must be accessible from within the cluster for OIDC discovery.
 echo "Creating JWTAuthenticator..."
 cat <<EOF | kubectl apply -f -
-apiVersion: authentication.concierge.pinniped.dev/v1alpha1
+apiVersion: authentication.concierge.${api_group_suffix}/v1alpha1
 kind: JWTAuthenticator
 metadata:
   name: my-jwt-authenticator
@@ -534,22 +546,26 @@ fi
 if [[ "$use_oidc_upstream" == "yes" ]]; then
   echo "Generating OIDC kubeconfig..."
   https_proxy="$proxy_server" no_proxy="$proxy_except" \
-    ./pinniped get kubeconfig --oidc-skip-browser $flow_arg --upstream-identity-provider-type oidc >kubeconfig-oidc.yaml
+    ./pinniped get kubeconfig --concierge-api-group-suffix "$api_group_suffix" \
+    --oidc-skip-browser $flow_arg --upstream-identity-provider-type oidc >kubeconfig-oidc.yaml
 fi
 if [[ "$use_ldap_upstream" == "yes" ]]; then
   echo "Generating LDAP kubeconfig..."
   https_proxy="$proxy_server" no_proxy="$proxy_except" \
-    ./pinniped get kubeconfig --oidc-skip-browser $flow_arg --upstream-identity-provider-type ldap >kubeconfig-ldap.yaml
+    ./pinniped get kubeconfig --concierge-api-group-suffix "$api_group_suffix" \
+    --oidc-skip-browser $flow_arg --upstream-identity-provider-type ldap >kubeconfig-ldap.yaml
 fi
 if [[ "$use_ad_upstream" == "yes" ]]; then
   echo "Generating AD kubeconfig..."
   https_proxy="$proxy_server" no_proxy="$proxy_except" \
-    ./pinniped get kubeconfig --oidc-skip-browser $flow_arg --upstream-identity-provider-type activedirectory >kubeconfig-ad.yaml
+    ./pinniped get kubeconfig --concierge-api-group-suffix "$api_group_suffix" \
+    --oidc-skip-browser $flow_arg --upstream-identity-provider-type activedirectory >kubeconfig-ad.yaml
 fi
 if [[ "$use_github_upstream" == "yes" ]]; then
   echo "Generating GitHub kubeconfig..."
   https_proxy="$proxy_server" no_proxy="$proxy_except" \
-    ./pinniped get kubeconfig --oidc-skip-browser $flow_arg --upstream-identity-provider-type github >kubeconfig-github.yaml
+    ./pinniped get kubeconfig --concierge-api-group-suffix "$api_group_suffix" \
+    --oidc-skip-browser $flow_arg --upstream-identity-provider-type github >kubeconfig-github.yaml
 fi
 
 # Clear the local CLI cache to ensure that the kubectl command below will need to perform a fresh login.
@@ -559,6 +575,12 @@ rm -f "$HOME/.config/pinniped/credentials.yaml"
 echo
 echo "Ready! 🚀"
 
+if [[ "$api_group_suffix" == "pinniped.dev" ]]; then
+  api_group_flag=""
+else
+  api_group_flag=" --api-group-suffix $api_group_suffix"
+fi
+
 # These instructions only apply when you are not using Contour and you will need a browser to log in.
 if [[ "${PINNIPED_USE_CONTOUR:-}" == "" && ("$use_oidc_upstream" == "yes" || "$use_flow" == "browser_authcode") ]]; then
   echo
@@ -601,21 +623,21 @@ fi
 # they expire, so you should not be prompted to log in again for the rest of the day.
 if [[ "$use_oidc_upstream" == "yes" ]]; then
   echo "To log in using OIDC:"
-  echo "PINNIPED_DEBUG=true ${proxy_env_vars}./pinniped whoami --kubeconfig ./kubeconfig-oidc.yaml"
+  echo "PINNIPED_DEBUG=true ${proxy_env_vars}./pinniped whoami --kubeconfig ./kubeconfig-oidc.yaml${api_group_flag}"
   echo
 fi
 if [[ "$use_ldap_upstream" == "yes" ]]; then
   echo "To log in using LDAP:"
-  echo "PINNIPED_DEBUG=true ${proxy_env_vars}./pinniped whoami --kubeconfig ./kubeconfig-ldap.yaml"
+  echo "PINNIPED_DEBUG=true ${proxy_env_vars}./pinniped whoami --kubeconfig ./kubeconfig-ldap.yaml${api_group_flag}"
   echo
 fi
 if [[ "$use_ad_upstream" == "yes" ]]; then
   echo "To log in using AD:"
-  echo "PINNIPED_DEBUG=true ${proxy_env_vars}./pinniped whoami --kubeconfig ./kubeconfig-ad.yaml"
+  echo "PINNIPED_DEBUG=true ${proxy_env_vars}./pinniped whoami --kubeconfig ./kubeconfig-ad.yaml${api_group_flag}"
   echo
 fi
 if [[ "$use_github_upstream" == "yes" ]]; then
   echo "To log in using GitHub:"
-  echo "PINNIPED_DEBUG=true ${proxy_env_vars}./pinniped whoami --kubeconfig ./kubeconfig-github.yaml"
+  echo "PINNIPED_DEBUG=true ${proxy_env_vars}./pinniped whoami --kubeconfig ./kubeconfig-github.yaml${api_group_flag}"
   echo
 fi

internal/auditevent/audit_event.go

@@ -0,0 +1,47 @@
+// Copyright 2024 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package auditevent
+
+type Message string
+
+const (
+	// Supervisor request logging.
+
+	HTTPRequestReceived          Message = "HTTP Request Received"
+	HTTPRequestCompleted         Message = "HTTP Request Completed"
+	HTTPRequestParameters        Message = "HTTP Request Parameters"
+	HTTPRequestCustomHeadersUsed Message = "HTTP Request Custom Headers Used"
+	HTTPRequestBasicAuthUsed     Message = "HTTP Request Basic Auth"
+
+	// Supervisor authentication logging.
+
+	UsingUpstreamIDP                   Message = "Using Upstream IDP"
+	AuthorizeIDFromParameters          Message = "AuthorizeID From Parameters"
+	IdentityFromUpstreamIDP            Message = "Identity From Upstream IDP"
+	UpstreamAuthorizeRedirect          Message = "Upstream Authorize Redirect"
+	IdentityRefreshedFromUpstreamIDP   Message = "Identity Refreshed From Upstream IDP"
+	IDTokenIssued                      Message = "ID Token Issued" //nolint:gosec // this is not a credential
+	SessionStarted                     Message = "Session Started"
+	SessionRefreshed                   Message = "Session Refreshed"
+	SessionFound                       Message = "Session Found"
+	AuthenticationRejectedByTransforms Message = "Authentication Rejected By Transforms"
+	IncorrectUsernameOrPassword        Message = "Incorrect Username Or Password"
+
+	// Supervisor session ending logging.
+
+	UpstreamOIDCTokenRevoked Message = "Upstream OIDC Token Revoked" //nolint:gosec // this is not a credential
+	SessionGarbageCollected  Message = "Session Garbage Collected"
+
+	// Supervisor aggregated APIs logging.
+
+	OIDCClientSecretRequestUpdatedSecrets Message = "OIDCClientSecretRequest Updated Secrets"
+
+	// Concierge aggregated APIs logging.
+
+	TokenCredentialRequestTokenReceived        Message = "TokenCredentialRequest Token Received"        //nolint:gosec // this is not a credential
+	TokenCredentialRequestAuthenticatedUser    Message = "TokenCredentialRequest Authenticated User"    //nolint:gosec // this is not a credential
+	TokenCredentialRequestAuthenticationFailed Message = "TokenCredentialRequest Authentication Failed" //nolint:gosec // this is not a credential
+	TokenCredentialRequestUnexpectedError      Message = "TokenCredentialRequest Unexpected Error"      //nolint:gosec // this is not a credential
+	TokenCredentialRequestUnsupportedUserInfo  Message = "TokenCredentialRequest Unsupported UserInfo"  //nolint:gosec // this is not a credential
+)

internal/auditid/auditid.go

@@ -0,0 +1,38 @@
+// Copyright 2024 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package auditid
+
+import (
+	"net/http"
+
+	"github.com/google/uuid"
+	"k8s.io/apimachinery/pkg/types"
+	apiserveraudit "k8s.io/apiserver/pkg/apis/audit"
+	"k8s.io/apiserver/pkg/audit"
+)
+
+// NewRequestWithAuditID is public for use in unit tests. Production code should use WithAuditID().
+func NewRequestWithAuditID(r *http.Request, newAuditIDFunc func() string) (*http.Request, string) {
+	ctx := audit.WithAuditContext(r.Context())
+	r = r.WithContext(ctx)
+
+	auditID := newAuditIDFunc()
+	audit.WithAuditID(ctx, types.UID(auditID))
+
+	return r, auditID
+}
+
+func WithAuditID(handler http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Add a randomly generated request ID to the context for this request.
+		r, auditID := NewRequestWithAuditID(r, func() string {
+			return uuid.New().String()
+		})
+
+		// Send the Audit-ID response header.
+		w.Header().Set(apiserveraudit.HeaderAuditID, auditID)
+
+		handler.ServeHTTP(w, r)
+	})
+}

internal/cert/pem.go

@@ -0,0 +1,13 @@
+// Copyright 2024 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package cert
+
+import "time"
+
+type PEM struct {
+	CertPEM   []byte
+	KeyPEM    []byte
+	NotBefore time.Time
+	NotAfter  time.Time
+}

internal/certauthority/certauthority.go

@@ -1,4 +1,4 @@
-// Copyright 2020-2023 the Pinniped contributors. All Rights Reserved.
+// Copyright 2020-2024 the Pinniped contributors. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
 // Package certauthority implements a simple x509 certificate authority suitable for use in an aggregated API service.
@@ -20,6 +20,7 @@ import (
 	"net"
 	"time"
 
+	"go.pinniped.dev/internal/cert"
 	"go.pinniped.dev/internal/constable"
 )
 
@@ -38,7 +39,7 @@ type env struct {
 	// clock tells the current time (usually time.Now(), but broken out here for tests).
 	clock func() time.Time
 
-	// parse function to parse an ASN.1 byte slice into an x509 struct (normally x509.ParseCertificate)
+	// parse function to parse an ASN.1 byte slice into a x509 struct (normally x509.ParseCertificate)
 	parseCert func([]byte) (*x509.Certificate, error)
 }
 
@@ -180,19 +181,19 @@ func (c *CA) IssueServerCert(dnsNames []string, ips []net.IP, ttl time.Duration)
 }
 
 // IssueClientCertPEM is similar to IssueClientCert, but returns the new cert as a pair of PEM-formatted byte slices
-// for the certificate and private key.
-func (c *CA) IssueClientCertPEM(username string, groups []string, ttl time.Duration) ([]byte, []byte, error) {
+// for the certificate and private key, along with the notBefore and notAfter values.
+func (c *CA) IssueClientCertPEM(username string, groups []string, ttl time.Duration) (*cert.PEM, error) {
 	return toPEM(c.IssueClientCert(username, groups, ttl))
 }
 
 // IssueServerCertPEM is similar to IssueServerCert, but returns the new cert as a pair of PEM-formatted byte slices
-// for the certificate and private key.
-func (c *CA) IssueServerCertPEM(dnsNames []string, ips []net.IP, ttl time.Duration) ([]byte, []byte, error) {
+// for the certificate and private key, along with the notBefore and notAfter values.
+func (c *CA) IssueServerCertPEM(dnsNames []string, ips []net.IP, ttl time.Duration) (*cert.PEM, error) {
 	return toPEM(c.IssueServerCert(dnsNames, ips, ttl))
 }
 
 func (c *CA) issueCert(extKeyUsage x509.ExtKeyUsage, subject pkix.Name, dnsNames []string, ips []net.IP, ttl time.Duration) (*tls.Certificate, error) {
-	// Choose a random 128 bit serial number.
+	// Choose a random 128-bit serial number.
 	serialNumber, err := randomSerial(c.env.serialRNG)
 	if err != nil {
 		return nil, fmt.Errorf("could not generate serial number for certificate: %w", err)
@@ -209,7 +210,7 @@ func (c *CA) issueCert(extKeyUsage x509.ExtKeyUsage, subject pkix.Name, dnsNames
 	notBefore := now.Add(-certBackdate)
 	notAfter := now.Add(ttl)
 
-	// Parse the DER encoded certificate to get an x509.Certificate.
+	// Parse the DER encoded certificate to get a x509.Certificate.
 	caCert, err := x509.ParseCertificate(c.caCertBytes)
 	if err != nil {
 		return nil, fmt.Errorf("could not parse CA certificate: %w", err)
@@ -246,18 +247,23 @@ func (c *CA) issueCert(extKeyUsage x509.ExtKeyUsage, subject pkix.Name, dnsNames
 	}, nil
 }
 
-func toPEM(cert *tls.Certificate, err error) ([]byte, []byte, error) {
+func toPEM(certificate *tls.Certificate, err error) (*cert.PEM, error) {
 	// If the wrapped IssueServerCert() returned an error, pass it back.
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
-	certPEM, keyPEM, err := ToPEM(cert)
+	certPEM, keyPEM, err := ToPEM(certificate)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
-	return certPEM, keyPEM, nil
+	return &cert.PEM{
+		CertPEM:   certPEM,
+		KeyPEM:    keyPEM,
+		NotBefore: certificate.Leaf.NotBefore,
+		NotAfter:  certificate.Leaf.NotAfter,
+	}, nil
 }
 
 // ToPEM encodes a tls.Certificate into a private key PEM and a cert chain PEM.
@@ -279,7 +285,7 @@ func ToPEM(cert *tls.Certificate) ([]byte, []byte, error) {
 	return certPEM, keyPEM, nil
 }
 
-// randomSerial generates a random 128 bit serial number.
+// randomSerial generates a random 128-bit serial number.
 func randomSerial(rng io.Reader) (*big.Int, error) {
 	return rand.Int(rng, new(big.Int).Lsh(big.NewInt(1), 128))
 }

internal/certauthority/certauthority_test.go

@@ -228,7 +228,7 @@ func (e *errSigner) Sign(_ io.Reader, _ []byte, _ crypto.SignerOpts) ([]byte, er
 func TestIssue(t *testing.T) {
 	const numRandBytes = 64 * 2 // each call to issue a cert will consume 64 bytes from the reader
 
-	now := time.Date(2020, 7, 10, 12, 41, 12, 1234, time.UTC)
+	now := time.Date(2020, 7, 10, 12, 41, 12, 0, time.UTC)
 
 	realCA, err := Load(testCert, testKey)
 	require.NoError(t, err)
@@ -323,6 +323,8 @@ func TestIssue(t *testing.T) {
 			} else {
 				require.NoError(t, err)
 				require.NotNil(t, got)
+				require.Equal(t, now.Add(-5*time.Minute), got.Leaf.NotBefore) // always back-dated
+				require.Equal(t, now.Add(10*time.Minute), got.Leaf.NotAfter)
 			}
 			got, err = tt.ca.IssueClientCert("test-user", []string{"group1", "group2"}, 10*time.Minute)
 			if tt.wantErr != "" {
@@ -331,6 +333,8 @@ func TestIssue(t *testing.T) {
 			} else {
 				require.NoError(t, err)
 				require.NotNil(t, got)
+				require.Equal(t, now.Add(-5*time.Minute), got.Leaf.NotBefore) // always back-dated
+				require.Equal(t, now.Add(10*time.Minute), got.Leaf.NotAfter)
 			}
 		})
 	}
@@ -341,26 +345,26 @@ func TestToPEM(t *testing.T) {
 	require.NoError(t, err)
 
 	t.Run("error from input", func(t *testing.T) {
-		certPEM, keyPEM, err := toPEM(nil, fmt.Errorf("some error"))
+		pem, err := toPEM(nil, fmt.Errorf("some error"))
 		require.EqualError(t, err, "some error")
-		require.Nil(t, certPEM)
-		require.Nil(t, keyPEM)
+		require.Nil(t, pem)
 	})
 
 	t.Run("invalid private key", func(t *testing.T) {
 		cert := realCert
 		cert.PrivateKey = nil
-		certPEM, keyPEM, err := toPEM(&cert, nil)
+		pem, err := toPEM(&cert, nil)
 		require.EqualError(t, err, "failed to marshal private key into PKCS8: x509: unknown key type while marshaling PKCS#8: <nil>")
-		require.Nil(t, certPEM)
-		require.Nil(t, keyPEM)
+		require.Nil(t, pem)
 	})
 
 	t.Run("success", func(t *testing.T) {
-		certPEM, keyPEM, err := toPEM(&realCert, nil)
+		pem, err := toPEM(&realCert, nil)
 		require.NoError(t, err)
-		require.NotEmpty(t, certPEM)
-		require.NotEmpty(t, keyPEM)
+		require.NotEmpty(t, pem.CertPEM)
+		require.NotEmpty(t, pem.KeyPEM)
+		require.Equal(t, time.Date(2020, time.July, 25, 21, 4, 18, 0, time.UTC), pem.NotBefore)
+		require.Equal(t, time.Date(2030, time.July, 23, 21, 4, 18, 0, time.UTC), pem.NotAfter)
 	})
 }
 
@@ -381,21 +385,21 @@ func TestIssueMethods(t *testing.T) {
 		require.NoError(t, err)
 		validateClientCert(t, ca.Bundle(), certPEM, keyPEM, user, groups, ttl)
 
-		certPEM, keyPEM, err = ca.IssueClientCertPEM(user, groups, ttl)
+		pem, err := ca.IssueClientCertPEM(user, groups, ttl)
 		require.NoError(t, err)
-		validateClientCert(t, ca.Bundle(), certPEM, keyPEM, user, groups, ttl)
+		validateClientCert(t, ca.Bundle(), pem.CertPEM, pem.KeyPEM, user, groups, ttl)
 
-		certPEM, keyPEM, err = ca.IssueClientCertPEM(user, nil, ttl)
+		pem, err = ca.IssueClientCertPEM(user, nil, ttl)
 		require.NoError(t, err)
-		validateClientCert(t, ca.Bundle(), certPEM, keyPEM, user, nil, ttl)
+		validateClientCert(t, ca.Bundle(), pem.CertPEM, pem.KeyPEM, user, nil, ttl)
 
-		certPEM, keyPEM, err = ca.IssueClientCertPEM(user, []string{}, ttl)
+		pem, err = ca.IssueClientCertPEM(user, []string{}, ttl)
 		require.NoError(t, err)
-		validateClientCert(t, ca.Bundle(), certPEM, keyPEM, user, nil, ttl)
+		validateClientCert(t, ca.Bundle(), pem.CertPEM, pem.KeyPEM, user, nil, ttl)
 
-		certPEM, keyPEM, err = ca.IssueClientCertPEM("", []string{}, ttl)
+		pem, err = ca.IssueClientCertPEM("", []string{}, ttl)
 		require.NoError(t, err)
-		validateClientCert(t, ca.Bundle(), certPEM, keyPEM, "", nil, ttl)
+		validateClientCert(t, ca.Bundle(), pem.CertPEM, pem.KeyPEM, "", nil, ttl)
 	})
 
 	t.Run("server certs", func(t *testing.T) {
@@ -408,25 +412,25 @@ func TestIssueMethods(t *testing.T) {
 		require.NoError(t, err)
 		validateServerCert(t, ca.Bundle(), certPEM, keyPEM, dnsNames, ips, ttl)
 
-		certPEM, keyPEM, err = ca.IssueServerCertPEM(dnsNames, ips, ttl)
+		pem, err := ca.IssueServerCertPEM(dnsNames, ips, ttl)
 		require.NoError(t, err)
-		validateServerCert(t, ca.Bundle(), certPEM, keyPEM, dnsNames, ips, ttl)
+		validateServerCert(t, ca.Bundle(), pem.CertPEM, pem.KeyPEM, dnsNames, ips, ttl)
 
-		certPEM, keyPEM, err = ca.IssueServerCertPEM(nil, ips, ttl)
+		pem, err = ca.IssueServerCertPEM(nil, ips, ttl)
 		require.NoError(t, err)
-		validateServerCert(t, ca.Bundle(), certPEM, keyPEM, nil, ips, ttl)
+		validateServerCert(t, ca.Bundle(), pem.CertPEM, pem.KeyPEM, nil, ips, ttl)
 
-		certPEM, keyPEM, err = ca.IssueServerCertPEM(dnsNames, nil, ttl)
+		pem, err = ca.IssueServerCertPEM(dnsNames, nil, ttl)
 		require.NoError(t, err)
-		validateServerCert(t, ca.Bundle(), certPEM, keyPEM, dnsNames, nil, ttl)
+		validateServerCert(t, ca.Bundle(), pem.CertPEM, pem.KeyPEM, dnsNames, nil, ttl)
 
-		certPEM, keyPEM, err = ca.IssueServerCertPEM([]string{}, ips, ttl)
+		pem, err = ca.IssueServerCertPEM([]string{}, ips, ttl)
 		require.NoError(t, err)
-		validateServerCert(t, ca.Bundle(), certPEM, keyPEM, nil, ips, ttl)
+		validateServerCert(t, ca.Bundle(), pem.CertPEM, pem.KeyPEM, nil, ips, ttl)
 
-		certPEM, keyPEM, err = ca.IssueServerCertPEM(dnsNames, []net.IP{}, ttl)
+		pem, err = ca.IssueServerCertPEM(dnsNames, []net.IP{}, ttl)
 		require.NoError(t, err)
-		validateServerCert(t, ca.Bundle(), certPEM, keyPEM, dnsNames, nil, ttl)
+		validateServerCert(t, ca.Bundle(), pem.CertPEM, pem.KeyPEM, dnsNames, nil, ttl)
 	})
 }
 

internal/certauthority/dynamiccertauthority/dynamiccertauthority.go

@@ -1,4 +1,4 @@
-// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved.
+// Copyright 2020-2024 the Pinniped contributors. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
 // Package dynamiccertauthority implements a x509 certificate authority capable of issuing
@@ -10,6 +10,7 @@ import (
 
 	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 
+	"go.pinniped.dev/internal/cert"
 	"go.pinniped.dev/internal/certauthority"
 	"go.pinniped.dev/internal/clientcertissuer"
 )
@@ -32,15 +33,15 @@ func (c *ca) Name() string {
 }
 
 // IssueClientCertPEM issues a new client certificate for the given identity and duration, returning it as a
-// pair of PEM-formatted byte slices for the certificate and private key.
-func (c *ca) IssueClientCertPEM(username string, groups []string, ttl time.Duration) ([]byte, []byte, error) {
+// pair of PEM-formatted byte slices for the certificate and private key, along with the notBefore and notAfter values.
+func (c *ca) IssueClientCertPEM(username string, groups []string, ttl time.Duration) (*cert.PEM, error) {
 	caCrtPEM, caKeyPEM := c.provider.CurrentCertKeyContent()
 	// in the future we could split dynamiccert.Private into two interfaces (Private and PrivateRead)
 	// and have this code take PrivateRead as input.  We would then add ourselves as a listener to
 	// the PrivateRead.  This would allow us to only reload the CA contents when they actually change.
 	ca, err := certauthority.Load(string(caCrtPEM), string(caKeyPEM))
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
 	return ca.IssueClientCertPEM(username, groups, ttl)

internal/certauthority/dynamiccertauthority/dynamiccertauthority_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	"go.pinniped.dev/internal/cert"
 	"go.pinniped.dev/internal/clientcertissuer"
 	"go.pinniped.dev/internal/dynamiccert"
 	"go.pinniped.dev/internal/testutil"
@@ -93,36 +94,35 @@ func TestCAIssuePEM(t *testing.T) {
 			// Can't run these steps in parallel, because each one depends on the previous steps being
 			// run.
 
-			crtPEM, keyPEM, err := issuePEM(provider, ca, step.caCrtPEM, step.caKeyPEM)
+			pem, err := issuePEM(provider, ca, step.caCrtPEM, step.caKeyPEM)
 
 			if step.wantError != "" {
 				require.EqualError(t, err, step.wantError)
-				require.Empty(t, crtPEM)
-				require.Empty(t, keyPEM)
+				require.Nil(t, pem)
 			} else {
 				require.NoError(t, err)
-				require.NotEmpty(t, crtPEM)
-				require.NotEmpty(t, keyPEM)
+				require.NotEmpty(t, pem.CertPEM)
+				require.NotEmpty(t, pem.KeyPEM)
 
 				caCrtPEM, _ := provider.CurrentCertKeyContent()
-				crtAssertions := testutil.ValidateClientCertificate(t, string(caCrtPEM), string(crtPEM))
+				crtAssertions := testutil.ValidateClientCertificate(t, string(caCrtPEM), string(pem.CertPEM))
 				crtAssertions.RequireCommonName("some-username")
 				crtAssertions.RequireOrganizations([]string{"some-group1", "some-group2"})
 				crtAssertions.RequireLifetime(time.Now(), time.Now().Add(time.Hour*24), time.Minute*10)
-				crtAssertions.RequireMatchesPrivateKey(string(keyPEM))
+				crtAssertions.RequireMatchesPrivateKey(string(pem.KeyPEM))
 			}
 		})
 	}
 }
 
-func issuePEM(provider dynamiccert.Provider, ca clientcertissuer.ClientCertIssuer, caCrt, caKey []byte) ([]byte, []byte, error) {
+func issuePEM(provider dynamiccert.Provider, ca clientcertissuer.ClientCertIssuer, caCrt, caKey []byte) (*cert.PEM, error) {
 	// if setting fails, look at that error
 	if caCrt != nil || caKey != nil {
 		if err := provider.SetCertKeyContent(caCrt, caKey); err != nil {
-			return nil, nil, err
+			return nil, err
 		}
 	}
 
-	// otherwise check to see if their is an issuing error
+	// otherwise check to see if there is an issuing error
 	return ca.IssueClientCertPEM("some-username", []string{"some-group1", "some-group2"}, time.Hour*24)
 }

internal/clientcertissuer/issuer.go

@@ -10,6 +10,7 @@ import (
 
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 
+	"go.pinniped.dev/internal/cert"
 	"go.pinniped.dev/internal/constable"
 )
 
@@ -17,7 +18,7 @@ const defaultCertIssuerErr = constable.Error("failed to issue cert")
 
 type ClientCertIssuer interface {
 	Name() string
-	IssueClientCertPEM(username string, groups []string, ttl time.Duration) (certPEM, keyPEM []byte, err error)
+	IssueClientCertPEM(username string, groups []string, ttl time.Duration) (pem *cert.PEM, err error)
 }
 
 var _ ClientCertIssuer = ClientCertIssuers{}
@@ -37,20 +38,20 @@ func (c ClientCertIssuers) Name() string {
 	return strings.Join(names, ",")
 }
 
-func (c ClientCertIssuers) IssueClientCertPEM(username string, groups []string, ttl time.Duration) ([]byte, []byte, error) {
+func (c ClientCertIssuers) IssueClientCertPEM(username string, groups []string, ttl time.Duration) (*cert.PEM, error) {
 	errs := make([]error, 0, len(c))
 
 	for _, issuer := range c {
-		certPEM, keyPEM, err := issuer.IssueClientCertPEM(username, groups, ttl)
+		pem, err := issuer.IssueClientCertPEM(username, groups, ttl)
 		if err == nil {
-			return certPEM, keyPEM, nil
+			return pem, nil
 		}
 		errs = append(errs, fmt.Errorf("%s failed to issue client cert: %w", issuer.Name(), err))
 	}
 
 	if err := utilerrors.NewAggregate(errs); err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
-	return nil, nil, defaultCertIssuerErr
+	return nil, defaultCertIssuerErr
 }

internal/clientcertissuer/issuer_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.uber.org/mock/gomock"
 
+	"go.pinniped.dev/internal/cert"
 	"go.pinniped.dev/internal/mocks/mockissuer"
 )
 
@@ -85,7 +86,7 @@ func TestIssueClientCertPEM(t *testing.T) {
 				errClientCertIssuer.EXPECT().Name().Return("error cert issuer")
 				errClientCertIssuer.EXPECT().
 					IssueClientCertPEM("username", []string{"group1", "group2"}, 32*time.Second).
-					Return(nil, nil, errors.New("error from wrapped cert issuer"))
+					Return(nil, errors.New("error from wrapped cert issuer"))
 				return ClientCertIssuers{errClientCertIssuer}
 			},
 			wantErrorMessage: "error cert issuer failed to issue client cert: error from wrapped cert issuer",
@@ -96,7 +97,7 @@ func TestIssueClientCertPEM(t *testing.T) {
 				validClientCertIssuer := mockissuer.NewMockClientCertIssuer(ctrl)
 				validClientCertIssuer.EXPECT().
 					IssueClientCertPEM("username", []string{"group1", "group2"}, 32*time.Second).
-					Return([]byte("cert"), []byte("key"), nil)
+					Return(&cert.PEM{CertPEM: []byte("cert"), KeyPEM: []byte("key")}, nil)
 				return ClientCertIssuers{validClientCertIssuer}
 			},
 			wantCert: []byte("cert"),
@@ -109,12 +110,12 @@ func TestIssueClientCertPEM(t *testing.T) {
 				errClientCertIssuer.EXPECT().Name().Return("error cert issuer")
 				errClientCertIssuer.EXPECT().
 					IssueClientCertPEM("username", []string{"group1", "group2"}, 32*time.Second).
-					Return(nil, nil, errors.New("error from wrapped cert issuer"))
+					Return(nil, errors.New("error from wrapped cert issuer"))
 
 				validClientCertIssuer := mockissuer.NewMockClientCertIssuer(ctrl)
 				validClientCertIssuer.EXPECT().
 					IssueClientCertPEM("username", []string{"group1", "group2"}, 32*time.Second).
-					Return([]byte("cert"), []byte("key"), nil)
+					Return(&cert.PEM{CertPEM: []byte("cert"), KeyPEM: []byte("key")}, nil)
 				return ClientCertIssuers{
 					errClientCertIssuer,
 					validClientCertIssuer,
@@ -130,13 +131,13 @@ func TestIssueClientCertPEM(t *testing.T) {
 				err1ClientCertIssuer.EXPECT().Name().Return("error1 cert issuer")
 				err1ClientCertIssuer.EXPECT().
 					IssueClientCertPEM("username", []string{"group1", "group2"}, 32*time.Second).
-					Return(nil, nil, errors.New("error1 from wrapped cert issuer"))
+					Return(nil, errors.New("error1 from wrapped cert issuer"))
 
 				err2ClientCertIssuer := mockissuer.NewMockClientCertIssuer(ctrl)
 				err2ClientCertIssuer.EXPECT().Name().Return("error2 cert issuer")
 				err2ClientCertIssuer.EXPECT().
 					IssueClientCertPEM("username", []string{"group1", "group2"}, 32*time.Second).
-					Return(nil, nil, errors.New("error2 from wrapped cert issuer"))
+					Return(nil, errors.New("error2 from wrapped cert issuer"))
 
 				return ClientCertIssuers{
 					err1ClientCertIssuer,
@@ -152,17 +153,16 @@ func TestIssueClientCertPEM(t *testing.T) {
 		t.Run(testcase.name, func(t *testing.T) {
 			t.Parallel()
 
-			certPEM, keyPEM, err := testcase.buildIssuerMocks().
+			pem, err := testcase.buildIssuerMocks().
 				IssueClientCertPEM("username", []string{"group1", "group2"}, 32*time.Second)
 
 			if testcase.wantErrorMessage != "" {
 				require.ErrorContains(t, err, testcase.wantErrorMessage)
-				require.Empty(t, certPEM)
-				require.Empty(t, keyPEM)
+				require.Nil(t, pem)
 			} else {
 				require.NoError(t, err)
-				require.Equal(t, testcase.wantCert, certPEM)
-				require.Equal(t, testcase.wantKey, keyPEM)
+				require.Equal(t, testcase.wantCert, pem.CertPEM)
+				require.Equal(t, testcase.wantKey, pem.KeyPEM)
 			}
 		})
 	}

internal/concierge/apiserver/apiserver.go

@@ -39,6 +39,7 @@ type ExtraConfig struct {
 	LoginConciergeGroupVersion    schema.GroupVersion
 	IdentityConciergeGroupVersion schema.GroupVersion
 	TokenClient                   *tokenclient.TokenClient
+	AuditLogger                   plog.AuditLogger
 }
 
 type PinnipedServer struct {
@@ -82,7 +83,12 @@ func (c completedConfig) New() (*PinnipedServer, error) {
 	for _, f := range []func() (schema.GroupVersionResource, rest.Storage){
 		func() (schema.GroupVersionResource, rest.Storage) {
 			tokenCredReqGVR := c.ExtraConfig.LoginConciergeGroupVersion.WithResource("tokencredentialrequests")
-			tokenCredStorage := credentialrequest.NewREST(c.ExtraConfig.Authenticator, c.ExtraConfig.Issuer, tokenCredReqGVR.GroupResource())
+			tokenCredStorage := credentialrequest.NewREST(
+				c.ExtraConfig.Authenticator,
+				c.ExtraConfig.Issuer,
+				tokenCredReqGVR.GroupResource(),
+				c.ExtraConfig.AuditLogger,
+			)
 			return tokenCredReqGVR, tokenCredStorage
 		},
 		func() (schema.GroupVersionResource, rest.Storage) {

internal/concierge/impersonator/impersonator_test.go

@@ -70,10 +70,10 @@ func TestImpersonator(t *testing.T) {
 	err = caContent.SetCertKeyContent(ca.Bundle(), caKey)
 	require.NoError(t, err)
 
-	cert, key, err := ca.IssueServerCertPEM(nil, []net.IP{net.ParseIP("127.0.0.1")}, time.Hour)
+	pem, err := ca.IssueServerCertPEM(nil, []net.IP{net.ParseIP("127.0.0.1")}, time.Hour)
 	require.NoError(t, err)
 	certKeyContent := dynamiccert.NewServingCert("cert-key")
-	err = certKeyContent.SetCertKeyContent(cert, key)
+	err = certKeyContent.SetCertKeyContent(pem.CertPEM, pem.KeyPEM)
 	require.NoError(t, err)
 
 	unrelatedCA, err := certauthority.New("ca", time.Hour)
@@ -1997,11 +1997,11 @@ type clientCert struct {
 
 func newClientCert(t *testing.T, ca *certauthority.CA, username string, groups []string) *clientCert {
 	t.Helper()
-	certPEM, keyPEM, err := ca.IssueClientCertPEM(username, groups, time.Hour)
+	pem, err := ca.IssueClientCertPEM(username, groups, time.Hour)
 	require.NoError(t, err)
 	return &clientCert{
-		certPEM: certPEM,
-		keyPEM:  keyPEM,
+		certPEM: pem.CertPEM,
+		keyPEM:  pem.KeyPEM,
 	}
 }
 

internal/concierge/server/server.go

@@ -180,21 +180,9 @@ func (a *App) runServer(ctx context.Context) error {
 		dynamiccertauthority.New(impersonationProxySigningCertProvider), // fallback to our internal CA if we need to
 	}
 
-	// Get the aggregated API server config.
-	aggregatedAPIServerConfig, err := getAggregatedAPIServerConfig(
-		dynamicServingCertProvider,
-		authenticators,
-		certIssuer,
-		buildControllers,
-		*cfg.APIGroupSuffix,
-		*cfg.AggregatedAPIServerPort,
-		scheme,
-		loginGV,
-		identityGV,
-	)
-	if err != nil {
-		return fmt.Errorf("could not configure aggregated API server: %w", err)
-	}
+	auditLogger := plog.NewAuditLogger(plog.AuditLogConfig{
+		LogUsernamesAndGroupNames: cfg.Audit.LogUsernamesAndGroups.Enabled(),
+	})
 
 	// Configure a token client that retrieves relatively short-lived tokens from the API server.
 	// It uses a k8s client without leader election because all pods need tokens.
@@ -206,13 +194,31 @@ func (a *App) runServer(ctx context.Context) error {
 	if err != nil {
 		return fmt.Errorf("could not create default kubernetes client: %w", err)
 	}
-	aggregatedAPIServerConfig.ExtraConfig.TokenClient = tokenclient.New(
+	tokenClient := tokenclient.New(
 		cfg.NamesConfig.ImpersonationProxyServiceAccount,
 		k8sClient.Kubernetes.CoreV1().ServiceAccounts(podInfo.Namespace),
 		impersonationProxyTokenCache.Set,
 		plog.New(),
 		tokenclient.WithExpirationSeconds(oneDayInSeconds))
 
+	// Get the aggregated API server config.
+	aggregatedAPIServerConfig, err := getAggregatedAPIServerConfig(
+		dynamicServingCertProvider,
+		authenticators,
+		certIssuer,
+		buildControllers,
+		*cfg.APIGroupSuffix,
+		*cfg.AggregatedAPIServerPort,
+		scheme,
+		loginGV,
+		identityGV,
+		auditLogger,
+		tokenClient,
+	)
+	if err != nil {
+		return fmt.Errorf("could not configure aggregated API server: %w", err)
+	}
+
 	// Complete the aggregated API server config and make a server instance.
 	server, err := aggregatedAPIServerConfig.Complete().New()
 	if err != nil {
@@ -235,6 +241,8 @@ func getAggregatedAPIServerConfig(
 	aggregatedAPIServerPort int64,
 	scheme *runtime.Scheme,
 	loginConciergeGroupVersion, identityConciergeGroupVersion schema.GroupVersion,
+	auditLogger plog.AuditLogger,
+	tokenClient *tokenclient.TokenClient,
 ) (*apiserver.Config, error) {
 	codecs := serializer.NewCodecFactory(scheme)
 
@@ -301,6 +309,8 @@ func getAggregatedAPIServerConfig(
 			NegotiatedSerializer:          codecs,
 			LoginConciergeGroupVersion:    loginConciergeGroupVersion,
 			IdentityConciergeGroupVersion: identityConciergeGroupVersion,
+			TokenClient:                   tokenClient,
+			AuditLogger:                   auditLogger,
 		},
 	}
 	return apiServerConfig, nil

internal/config/concierge/config.go

@@ -88,6 +88,10 @@ func FromPath(ctx context.Context, path string, setAllowedCiphers ptls.SetAllowe
 		return nil, fmt.Errorf("validate tls: %w", err)
 	}
 
+	if err := validateAudit(&config.Audit); err != nil {
+		return nil, fmt.Errorf("validate audit: %w", err)
+	}
+
 	if config.Labels == nil {
 		config.Labels = make(map[string]string)
 	}
@@ -200,3 +204,11 @@ func validateServerPort(port *int64) error {
 	}
 	return nil
 }
+
+func validateAudit(auditConfig *AuditSpec) error {
+	v := auditConfig.LogUsernamesAndGroups
+	if v != "" && v != Enabled && v != Disabled {
+		return constable.Error("invalid logUsernamesAndGroups format, valid choices are 'enabled', 'disabled', or empty string (equivalent to 'disabled')")
+	}
+	return nil
+}

internal/config/concierge/config_test.go

@@ -67,6 +67,8 @@ func TestFromPath(t *testing.T) {
 				    - foo
 				    - bar
 					- TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
+				audit:
+				  logUsernamesAndGroups: enabled
 			`),
 			wantConfig: &Config{
 				DiscoveryInfo: DiscoveryInfoSpec{
@@ -115,6 +117,9 @@ func TestFromPath(t *testing.T) {
 						},
 					},
 				},
+				Audit: AuditSpec{
+					LogUsernamesAndGroups: "enabled",
+				},
 			},
 		},
 		{
@@ -155,6 +160,8 @@ func TestFromPath(t *testing.T) {
 				log:
 				  level: all
 				  format: json
+				audit:
+				  logUsernamesAndGroups: disabled
 			`),
 			wantConfig: &Config{
 				DiscoveryInfo: DiscoveryInfoSpec{
@@ -195,6 +202,9 @@ func TestFromPath(t *testing.T) {
 					Level:  plog.LevelAll,
 					Format: plog.FormatJSON,
 				},
+				Audit: AuditSpec{
+					LogUsernamesAndGroups: "disabled",
+				},
 			},
 		},
 		{
@@ -287,6 +297,7 @@ func TestFromPath(t *testing.T) {
 					NamePrefix: ptr.To("pinniped-kube-cert-agent-"),
 					Image:      ptr.To("debian:latest"),
 				},
+				Audit: AuditSpec{LogUsernamesAndGroups: ""},
 			},
 		},
 		{
@@ -629,6 +640,28 @@ func TestFromPath(t *testing.T) {
 			allowedCiphersError: fmt.Errorf("some error from setAllowedCiphers"),
 			wantError:           "validate tls: some error from setAllowedCiphers",
 		},
+		{
+			name: "invalid audit.logUsernamesAndGroups format",
+			yaml: here.Doc(`
+				---
+				names:
+				  servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate
+				  credentialIssuer: pinniped-config
+				  apiService: pinniped-api
+				  impersonationLoadBalancerService: impersonationLoadBalancerService-value
+				  impersonationClusterIPService: impersonationClusterIPService-value
+				  impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value
+				  impersonationCACertificateSecret: impersonationCACertificateSecret-value
+				  impersonationSignerSecret: impersonationSignerSecret-value
+				  impersonationSignerSecret: impersonationSignerSecret-value
+				  agentServiceAccount: agentServiceAccount-value
+				  impersonationProxyServiceAccount: impersonationProxyServiceAccount-value
+				  impersonationProxyLegacySecret: impersonationProxyLegacySecret-value
+				audit:
+				  logUsernamesAndGroups: this-value-is-not-allowed
+			`),
+			wantError: "validate audit: invalid logUsernamesAndGroups format, valid choices are 'enabled', 'disabled', or empty string (equivalent to 'disabled')",
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {

internal/config/concierge/types.go

@@ -5,6 +5,11 @@ package concierge
 
 import "go.pinniped.dev/internal/plog"
 
+const (
+	Enabled  = "enabled"
+	Disabled = "disabled"
+)
+
 // Config contains knobs to set up an instance of the Pinniped Concierge.
 type Config struct {
 	DiscoveryInfo                DiscoveryInfoSpec `json:"discovery"`
@@ -17,6 +22,17 @@ type Config struct {
 	Labels                       map[string]string `json:"labels"`
 	Log                          plog.LogSpec      `json:"log"`
 	TLS                          TLSSpec           `json:"tls"`
+	Audit                        AuditSpec         `json:"audit"`
+}
+
+type AuditUsernamesAndGroups string
+
+func (l AuditUsernamesAndGroups) Enabled() bool {
+	return l == Enabled
+}
+
+type AuditSpec struct {
+	LogUsernamesAndGroups AuditUsernamesAndGroups `json:"logUsernamesAndGroups"`
 }
 
 type TLSSpec struct {

internal/config/supervisor/config.go

@@ -100,6 +100,10 @@ func FromPath(ctx context.Context, path string, setAllowedCiphers ptls.SetAllowe
 		return nil, fmt.Errorf("validate tls: %w", err)
 	}
 
+	if err := validateAudit(&config.Audit); err != nil {
+		return nil, fmt.Errorf("validate audit: %w", err)
+	}
+
 	return &config, nil
 }
 
@@ -214,3 +218,23 @@ func validateServerPort(port *int64) error {
 	}
 	return nil
 }
+
+func validateAudit(auditConfig *AuditSpec) error {
+	const errFmt = "invalid %s format, valid choices are 'enabled', 'disabled', or empty string (equivalent to 'disabled')"
+
+	switch auditConfig.LogUsernamesAndGroups {
+	case Enabled, Disabled, "":
+		// no-op
+	default:
+		return fmt.Errorf(errFmt, "logUsernamesAndGroups")
+	}
+
+	switch auditConfig.LogInternalPaths {
+	case Enabled, Disabled, "":
+		// no-op
+	default:
+		return fmt.Errorf(errFmt, "logInternalPaths")
+	}
+
+	return nil
+}

internal/config/supervisor/config_test.go

@@ -52,6 +52,9 @@ func TestFromPath(t *testing.T) {
 				    - foo
 				    - bar
 				    - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
+				audit:
+				  logUsernamesAndGroups: enabled
+				  logInternalPaths: enabled
 			`),
 			wantConfig: &Config{
 				APIGroupSuffix: ptr.To("some.suffix.com"),
@@ -86,6 +89,10 @@ func TestFromPath(t *testing.T) {
 						},
 					},
 				},
+				Audit: AuditSpec{
+					LogUsernamesAndGroups: "enabled",
+					LogInternalPaths:      "enabled",
+				},
 			},
 		},
 		{
@@ -123,6 +130,42 @@ func TestFromPath(t *testing.T) {
 					},
 				},
 				AggregatedAPIServerPort: ptr.To[int64](10250),
+				Audit: AuditSpec{
+					LogInternalPaths:      "",
+					LogUsernamesAndGroups: "",
+				},
+			},
+		},
+		{
+			name: "audit settings can be disabled explicitly",
+			yaml: here.Doc(`
+				---
+				names:
+				  defaultTLSCertificateSecret: my-secret-name
+				audit:
+				  logInternalPaths: disabled
+				  logUsernamesAndGroups: disabled
+			`),
+			wantConfig: &Config{
+				APIGroupSuffix: ptr.To("pinniped.dev"),
+				Labels:         map[string]string{},
+				NamesConfig: NamesConfigSpec{
+					DefaultTLSCertificateSecret: "my-secret-name",
+				},
+				Endpoints: &Endpoints{
+					HTTPS: &Endpoint{
+						Network: "tcp",
+						Address: ":8443",
+					},
+					HTTP: &Endpoint{
+						Network: "disabled",
+					},
+				},
+				AggregatedAPIServerPort: ptr.To[int64](10250),
+				Audit: AuditSpec{
+					LogInternalPaths:      "disabled",
+					LogUsernamesAndGroups: "disabled",
+				},
 			},
 		},
 		{
@@ -267,6 +310,28 @@ func TestFromPath(t *testing.T) {
 			`),
 			wantError: "validate aggregatedAPIServerPort: must be within range 1024 to 65535",
 		},
+		{
+			name: "invalid audit.logUsernamesAndGroups format",
+			yaml: here.Doc(`
+				---
+				names:
+				  defaultTLSCertificateSecret: my-secret-name
+				audit:
+				  logUsernamesAndGroups: this-is-not-a-valid-value
+			`),
+			wantError: "validate audit: invalid logUsernamesAndGroups format, valid choices are 'enabled', 'disabled', or empty string (equivalent to 'disabled')",
+		},
+		{
+			name: "invalid audit.logInternalPaths format",
+			yaml: here.Doc(`
+				---
+				names:
+				  defaultTLSCertificateSecret: my-secret-name
+				audit:
+				  logInternalPaths: this-is-not-a-valid-value
+			`),
+			wantError: "validate audit: invalid logInternalPaths format, valid choices are 'enabled', 'disabled', or empty string (equivalent to 'disabled')",
+		},
 		{
 			name: "returns setAllowedCiphers errors",
 			yaml: here.Doc(`

internal/config/supervisor/types.go

@@ -7,7 +7,12 @@ import (
 	"go.pinniped.dev/internal/plog"
 )
 
-// Config contains knobs to setup an instance of the Pinniped Supervisor.
+const (
+	Enabled  = "enabled"
+	Disabled = "disabled"
+)
+
+// Config contains knobs to set up an instance of the Pinniped Supervisor.
 type Config struct {
 	APIGroupSuffix          *string           `json:"apiGroupSuffix,omitempty"`
 	Labels                  map[string]string `json:"labels"`
@@ -16,6 +21,22 @@ type Config struct {
 	Endpoints               *Endpoints        `json:"endpoints"`
 	AggregatedAPIServerPort *int64            `json:"aggregatedAPIServerPort"`
 	TLS                     TLSSpec           `json:"tls"`
+	Audit                   AuditSpec         `json:"audit"`
+}
+
+type AuditInternalPaths string
+type AuditUsernamesAndGroups string
+
+func (l AuditInternalPaths) Enabled() bool {
+	return l == Enabled
+}
+func (l AuditUsernamesAndGroups) Enabled() bool {
+	return l == Enabled
+}
+
+type AuditSpec struct {
+	LogInternalPaths      AuditInternalPaths      `json:"logInternalPaths"`
+	LogUsernamesAndGroups AuditUsernamesAndGroups `json:"logUsernamesAndGroups"`
 }
 
 type TLSSpec struct {

internal/controller/apicerts/certs_observer_test.go

@@ -1,4 +1,4 @@
-// Copyright 2020-2023 the Pinniped contributors. All Rights Reserved.
+// Copyright 2020-2024 the Pinniped contributors. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
 package apicerts
@@ -173,10 +173,10 @@ func TestObserverControllerSync(t *testing.T) {
 				ca, err := certauthority.Load(string(caCrt), string(caKey))
 				require.NoError(t, err)
 
-				crt, key, err := ca.IssueServerCertPEM(nil, nil, time.Hour)
+				pem, err := ca.IssueServerCertPEM(nil, nil, time.Hour)
 				require.NoError(t, err)
 
-				err = dynamicCertProvider.SetCertKeyContent(crt, key)
+				err = dynamicCertProvider.SetCertKeyContent(pem.CertPEM, pem.KeyPEM)
 				r.NoError(err)
 			})
 
@@ -202,7 +202,7 @@ func TestObserverControllerSync(t *testing.T) {
 				ca, err := certauthority.Load(string(caCrt), string(caKey))
 				require.NoError(t, err)
 
-				crt, key, err := ca.IssueServerCertPEM(nil, nil, time.Hour)
+				pem, err := ca.IssueServerCertPEM(nil, nil, time.Hour)
 				require.NoError(t, err)
 
 				apiServingCertSecret := &corev1.Secret{
@@ -212,8 +212,8 @@ func TestObserverControllerSync(t *testing.T) {
 					},
 					Data: map[string][]byte{
 						"caCertificate":       []byte("fake cert"),
-						"tlsPrivateKey":       key,
-						"tlsCertificateChain": crt,
+						"tlsPrivateKey":       pem.KeyPEM,
+						"tlsCertificateChain": pem.CertPEM,
 					},
 				}
 				err = kubeInformerClient.Tracker().Add(apiServingCertSecret)

internal/controller/authenticator/webhookcachefiller/webhookcachefiller_test.go

@@ -79,7 +79,7 @@ func TestController(t *testing.T) {
 	require.NoError(t, err)
 	someUnknownHostNames := []string{"some-dns-name", "some-other-dns-name"}
 	someLocalIPAddress := []net.IP{net.ParseIP("10.2.3.4")}
-	pemServerCertForUnknownServer, _, err := caForUnknownServer.IssueServerCertPEM(
+	pemServerCertForUnknownServer, err := caForUnknownServer.IssueServerCertPEM(
 		someUnknownHostNames,
 		someLocalIPAddress,
 		time.Hour,
@@ -216,7 +216,7 @@ func TestController(t *testing.T) {
 	badWebhookAuthenticatorSpecGoodEndpointButUnknownCA := authenticationv1alpha1.WebhookAuthenticatorSpec{
 		Endpoint: goodWebhookDefaultServingCertEndpoint,
 		TLS: &authenticationv1alpha1.TLSSpec{
-			CertificateAuthorityData: base64.StdEncoding.EncodeToString(pemServerCertForUnknownServer),
+			CertificateAuthorityData: base64.StdEncoding.EncodeToString(pemServerCertForUnknownServer.CertPEM),
 		},
 	}
 

internal/controller/supervisorconfig/githubupstreamwatcher/github_upstream_watcher_test.go

@@ -127,7 +127,7 @@ func TestController(t *testing.T) {
 
 	caForUnknownServer, err := certauthority.New("Some Unknown CA", time.Hour)
 	require.NoError(t, err)
-	unknownServerCABytes, _, err := caForUnknownServer.IssueServerCertPEM(
+	unknownServerPEM, err := caForUnknownServer.IssueServerCertPEM(
 		[]string{"some-dns-name", "some-other-dns-name"},
 		[]net.IP{net.ParseIP("10.2.3.4")},
 		time.Hour,
@@ -1849,7 +1849,7 @@ func TestController(t *testing.T) {
 				func() runtime.Object {
 					badIDP := validFilledOutIDP.DeepCopy()
 					badIDP.Spec.GitHubAPI.TLS = &idpv1alpha1.TLSSpec{
-						CertificateAuthorityData: base64.StdEncoding.EncodeToString(unknownServerCABytes),
+						CertificateAuthorityData: base64.StdEncoding.EncodeToString(unknownServerPEM.CertPEM),
 					}
 					return badIDP
 				}(),
@@ -1861,7 +1861,7 @@ func TestController(t *testing.T) {
 					Spec: func() idpv1alpha1.GitHubIdentityProviderSpec {
 						badSpec := validFilledOutIDP.Spec.DeepCopy()
 						badSpec.GitHubAPI.TLS = &idpv1alpha1.TLSSpec{
-							CertificateAuthorityData: base64.StdEncoding.EncodeToString(unknownServerCABytes),
+							CertificateAuthorityData: base64.StdEncoding.EncodeToString(unknownServerPEM.CertPEM),
 						}
 						return *badSpec
 					}(),
@@ -2555,12 +2555,7 @@ func TestController(t *testing.T) {
 				require.Len(t, actualIDP.Status.Conditions, countExpectedConditions)
 				require.Equal(t, tt.wantResultingUpstreams[i], *actualIDP)
 			}
-
-			expectedLogs := ""
-			if len(tt.wantLogs) > 0 {
-				expectedLogs = strings.Join(tt.wantLogs, "\n") + "\n"
-			}
-			require.Equal(t, expectedLogs, log.String())
+			testutil.RequireLogLines(t, tt.wantLogs, log)
 
 			// This needs to happen after the expected condition LastTransitionTime has been updated.
 			wantActions := make([]coretesting.Action, 3+len(tt.wantResultingUpstreams))

internal/controller/supervisorstorage/garbage_collector.go

@@ -7,8 +7,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"slices"
+	"strings"
 	"time"
 
+	"github.com/ory/fosite"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
@@ -18,6 +21,7 @@ import (
 	clocktesting "k8s.io/utils/clock/testing"
 
 	oidcapi "go.pinniped.dev/generated/latest/apis/supervisor/oidc"
+	"go.pinniped.dev/internal/auditevent"
 	pinnipedcontroller "go.pinniped.dev/internal/controller"
 	"go.pinniped.dev/internal/controllerlib"
 	"go.pinniped.dev/internal/crud"
@@ -35,10 +39,12 @@ import (
 const minimumRepeatInterval = 30 * time.Second
 
 type garbageCollectorController struct {
-	idpCache              UpstreamOIDCIdentityProviderICache
-	secretInformer        corev1informers.SecretInformer
-	kubeClient            kubernetes.Interface
-	clock                 clock.Clock
+	idpCache       UpstreamOIDCIdentityProviderICache
+	secretInformer corev1informers.SecretInformer
+	kubeClient     kubernetes.Interface
+	clock          clock.Clock
+	auditLogger    plog.AuditLogger
+
 	timeOfMostRecentSweep time.Time
 }
 
@@ -53,6 +59,7 @@ func GarbageCollectorController(
 	kubeClient kubernetes.Interface,
 	secretInformer corev1informers.SecretInformer,
 	withInformer pinnipedcontroller.WithInformerOptionFunc,
+	auditLogger plog.AuditLogger,
 ) controllerlib.Controller {
 	isSecretWithGCAnnotation := func(obj metav1.Object) bool {
 		secret, ok := obj.(*corev1.Secret)
@@ -70,6 +77,7 @@ func GarbageCollectorController(
 				secretInformer: secretInformer,
 				kubeClient:     kubeClient,
 				clock:          clock,
+				auditLogger:    auditLogger,
 			},
 		},
 		withInformer(
@@ -108,6 +116,11 @@ func (c *garbageCollectorController) Sync(ctx controllerlib.Context) error {
 		return err
 	}
 
+	// Sort secrets by name so that audit log tests are deterministic
+	slices.SortStableFunc(listOfSecrets, func(a, b *corev1.Secret) int {
+		return strings.Compare(a.ObjectMeta.Name, b.ObjectMeta.Name)
+	})
+
 	for i := range listOfSecrets {
 		secret := listOfSecrets[i]
 
@@ -163,6 +176,7 @@ func (c *garbageCollectorController) Sync(ctx controllerlib.Context) error {
 			plog.WarningErr("failed to garbage collect resource", err, logKV(secret)...)
 			continue
 		}
+		c.maybeAuditLogGC(storageType, secret)
 		plog.Info("storage garbage collector deleted resource", logKV(secret)...)
 	}
 
@@ -192,7 +206,10 @@ func (c *garbageCollectorController) maybeRevokeUpstreamOIDCToken(ctx context.Co
 			return nil
 		}
 		// When the downstream authcode was never used, then its storage must contain the latest upstream token.
-		return c.tryRevokeUpstreamOIDCToken(ctx, authorizeCodeSession.Request.Session.(*psession.PinnipedSession).Custom, secret)
+		return c.tryRevokeUpstreamOIDCToken(ctx,
+			authorizeCodeSession.Request.Session.(*psession.PinnipedSession).Custom,
+			authorizeCodeSession.Request,
+			secret)
 
 	case accesstoken.TypeLabelValue:
 		// For access token storage, check if the "offline_access" scope was granted on the downstream session.
@@ -203,11 +220,13 @@ func (c *garbageCollectorController) maybeRevokeUpstreamOIDCToken(ctx context.Co
 		if err != nil {
 			return err
 		}
-		pinnipedSession := accessTokenSession.Request.Session.(*psession.PinnipedSession)
 		if accessTokenSession.Request.GetGrantedScopes().Has(oidcapi.ScopeOfflineAccess) {
 			return nil
 		}
-		return c.tryRevokeUpstreamOIDCToken(ctx, pinnipedSession.Custom, secret)
+		return c.tryRevokeUpstreamOIDCToken(ctx,
+			accessTokenSession.Request.Session.(*psession.PinnipedSession).Custom,
+			accessTokenSession.Request,
+			secret)
 
 	case refreshtoken.TypeLabelValue:
 		// For refresh token storage, always revoke its upstream token. This refresh token storage could be
@@ -217,7 +236,10 @@ func (c *garbageCollectorController) maybeRevokeUpstreamOIDCToken(ctx context.Co
 		if err != nil {
 			return err
 		}
-		return c.tryRevokeUpstreamOIDCToken(ctx, refreshTokenSession.Request.Session.(*psession.PinnipedSession).Custom, secret)
+		return c.tryRevokeUpstreamOIDCToken(ctx,
+			refreshTokenSession.Request.Session.(*psession.PinnipedSession).Custom,
+			refreshTokenSession.Request,
+			secret)
 
 	case pkce.TypeLabelValue:
 		// For PKCE storage, its very existence means that the downstream authcode was never exchanged, because
@@ -237,7 +259,12 @@ func (c *garbageCollectorController) maybeRevokeUpstreamOIDCToken(ctx context.Co
 	}
 }
 
-func (c *garbageCollectorController) tryRevokeUpstreamOIDCToken(ctx context.Context, customSessionData *psession.CustomSessionData, secret *corev1.Secret) error {
+func (c *garbageCollectorController) tryRevokeUpstreamOIDCToken(
+	ctx context.Context,
+	customSessionData *psession.CustomSessionData,
+	request *fosite.Request,
+	secret *corev1.Secret,
+) error {
 	// When session was for another upstream IDP type, e.g. LDAP, there is no upstream OIDC token involved.
 	if customSessionData.ProviderType != psession.ProviderTypeOIDC {
 		return nil
@@ -264,6 +291,10 @@ func (c *garbageCollectorController) tryRevokeUpstreamOIDCToken(ctx context.Cont
 		if err != nil {
 			return err
 		}
+		c.auditLogger.Audit(auditevent.UpstreamOIDCTokenRevoked, &plog.AuditParams{
+			Session:       request,
+			KeysAndValues: []any{"type", upstreamprovider.RefreshTokenType},
+		})
 		plog.Trace("garbage collector successfully revoked upstream OIDC refresh token (or provider has no revocation endpoint)", logKV(secret)...)
 	}
 
@@ -272,12 +303,61 @@ func (c *garbageCollectorController) tryRevokeUpstreamOIDCToken(ctx context.Cont
 		if err != nil {
 			return err
 		}
+		c.auditLogger.Audit(auditevent.UpstreamOIDCTokenRevoked, &plog.AuditParams{
+			Session:       request,
+			KeysAndValues: []any{"type", upstreamprovider.AccessTokenType},
+		})
 		plog.Trace("garbage collector successfully revoked upstream OIDC access token (or provider has no revocation endpoint)", logKV(secret)...)
 	}
 
 	return nil
 }
 
+func (c *garbageCollectorController) maybeAuditLogGC(storageType string, secret *corev1.Secret) {
+	r, err := c.requestFromSecret(storageType, secret)
+	if err == nil && r != nil {
+		c.auditLogger.Audit(auditevent.SessionGarbageCollected, &plog.AuditParams{
+			Session:       r,
+			KeysAndValues: []any{"storageType", storageType},
+		})
+	}
+}
+
+func (c *garbageCollectorController) requestFromSecret(storageType string, secret *corev1.Secret) (*fosite.Request, error) {
+	switch storageType {
+	case authorizationcode.TypeLabelValue:
+		authorizeCodeSession, err := authorizationcode.ReadFromSecret(secret)
+		if err != nil {
+			return nil, err
+		}
+		return authorizeCodeSession.Request, nil
+
+	case accesstoken.TypeLabelValue:
+		accessTokenSession, err := accesstoken.ReadFromSecret(secret)
+		if err != nil {
+			return nil, err
+		}
+		return accessTokenSession.Request, nil
+
+	case refreshtoken.TypeLabelValue:
+		refreshTokenSession, err := refreshtoken.ReadFromSecret(secret)
+		if err != nil {
+			return nil, err
+		}
+		return refreshTokenSession.Request, nil
+
+	case pkce.TypeLabelValue:
+		return nil, nil // if this still exists, then it means that the user never exchanged their authcode
+
+	case openidconnect.TypeLabelValue:
+		return nil, nil // if this still exists, then it means that the user never exchanged their authcode
+
+	default:
+		// There are no other storage types, so this should never happen in practice.
+		return nil, errors.New("garbage collector saw invalid label on Secret when trying to determine session ID")
+	}
+}
+
 func logKV(secret *corev1.Secret) []any {
 	return []any{
 		"secretName", secret.Name,

internal/controller/supervisorstorage/garbage_collector_test.go

@@ -4,6 +4,7 @@
 package supervisorstorage
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
@@ -31,6 +32,7 @@ import (
 	"go.pinniped.dev/internal/fositestorage/accesstoken"
 	"go.pinniped.dev/internal/fositestorage/authorizationcode"
 	"go.pinniped.dev/internal/fositestorage/refreshtoken"
+	"go.pinniped.dev/internal/plog"
 	"go.pinniped.dev/internal/psession"
 	"go.pinniped.dev/internal/testutil"
 	"go.pinniped.dev/internal/testutil/oidctestutil"
@@ -55,6 +57,7 @@ func TestGarbageCollectorControllerInformerFilters(t *testing.T) {
 				nil,
 				secretsInformer,
 				observableWithInformerOption.WithInformer, // make it possible to observe the behavior of the Filters
+				nil,
 			)
 			secretsInformerFilter = observableWithInformerOption.GetFilterForInformer(secretsInformer)
 		})
@@ -136,18 +139,23 @@ func TestGarbageCollectorControllerSync(t *testing.T) {
 			syncContext             *controllerlib.Context
 			fakeClock               *clocktesting.FakeClock
 			frozenNow               time.Time
+			actualAuditLog          *bytes.Buffer
+			wantAuditLogs           []testutil.WantedAuditLog
 		)
 
 		// Defer starting the informers until the last possible moment so that the
 		// nested Before's can keep adding things to the informer caches.
 		var startInformersAndController = func(idpCache dynamicupstreamprovider.DynamicUpstreamIDPProvider) {
 			// Set this at the last second to allow for injection of server override.
+			var auditLogger plog.AuditLogger
+			auditLogger, actualAuditLog = plog.TestAuditLogger(t)
 			subject = GarbageCollectorController(
 				idpCache,
 				fakeClock,
 				kubeClient,
 				kubeInformers.Core().V1().Secrets(),
 				controllerlib.WithInformer,
+				auditLogger,
 			)
 
 			// Set this at the last second to support calling subject.Name().
@@ -189,6 +197,8 @@ func TestGarbageCollectorControllerSync(t *testing.T) {
 
 		it.After(func() {
 			cancelContextCancelFunc()
+
+			testutil.CompareAuditLogs(t, wantAuditLogs, actualAuditLog.String())
 		})
 
 		when("there are secrets without the garbage-collect-after annotation", func() {
@@ -384,6 +394,27 @@ func TestGarbageCollectorControllerSync(t *testing.T) {
 					},
 					kubeClient.Actions(),
 				)
+
+				wantAuditLogs = []testutil.WantedAuditLog{
+					testutil.WantAuditLog("Upstream OIDC Token Revoked",
+						map[string]any{
+							"sessionID": "request-id-1",
+							"type":      "refresh_token",
+						},
+					),
+					testutil.WantAuditLog("Session Garbage Collected",
+						map[string]any{
+							"sessionID":   "request-id-1",
+							"storageType": "authcode",
+						},
+					),
+					testutil.WantAuditLog("Session Garbage Collected",
+						map[string]any{
+							"sessionID":   "request-id-2",
+							"storageType": "authcode",
+						},
+					),
+				}
 			})
 		})
 
@@ -508,6 +539,27 @@ func TestGarbageCollectorControllerSync(t *testing.T) {
 					},
 					kubeClient.Actions(),
 				)
+
+				wantAuditLogs = []testutil.WantedAuditLog{
+					testutil.WantAuditLog("Upstream OIDC Token Revoked",
+						map[string]any{
+							"sessionID": "request-id-1",
+							"type":      "access_token",
+						},
+					),
+					testutil.WantAuditLog("Session Garbage Collected",
+						map[string]any{
+							"sessionID":   "request-id-1",
+							"storageType": "authcode",
+						},
+					),
+					testutil.WantAuditLog("Session Garbage Collected",
+						map[string]any{
+							"sessionID":   "request-id-2",
+							"storageType": "authcode",
+						},
+					),
+				}
 			})
 		})
 
@@ -648,6 +700,15 @@ func TestGarbageCollectorControllerSync(t *testing.T) {
 					},
 					kubeClient.Actions(),
 				)
+
+				wantAuditLogs = []testutil.WantedAuditLog{
+					testutil.WantAuditLog("Session Garbage Collected",
+						map[string]any{
+							"sessionID":   "request-id-1",
+							"storageType": "authcode",
+						},
+					),
+				}
 			})
 		})
 
@@ -719,6 +780,15 @@ func TestGarbageCollectorControllerSync(t *testing.T) {
 					},
 					kubeClient.Actions(),
 				)
+
+				wantAuditLogs = []testutil.WantedAuditLog{
+					testutil.WantAuditLog("Session Garbage Collected",
+						map[string]any{
+							"sessionID":   "request-id-1",
+							"storageType": "authcode",
+						},
+					),
+				}
 			})
 		})
 
@@ -824,6 +894,15 @@ func TestGarbageCollectorControllerSync(t *testing.T) {
 					},
 					kubeClient.Actions(),
 				)
+
+				wantAuditLogs = []testutil.WantedAuditLog{
+					testutil.WantAuditLog("Session Garbage Collected",
+						map[string]any{
+							"sessionID":   "request-id-1",
+							"storageType": "authcode",
+						},
+					),
+				}
 			})
 		})
 
@@ -903,6 +982,15 @@ func TestGarbageCollectorControllerSync(t *testing.T) {
 					},
 					kubeClient.Actions(),
 				)
+
+				wantAuditLogs = []testutil.WantedAuditLog{
+					testutil.WantAuditLog("Session Garbage Collected",
+						map[string]any{
+							"sessionID":   "request-id-1",
+							"storageType": "authcode",
+						},
+					),
+				}
 			})
 		})
 
@@ -1027,6 +1115,27 @@ func TestGarbageCollectorControllerSync(t *testing.T) {
 					},
 					kubeClient.Actions(),
 				)
+
+				wantAuditLogs = []testutil.WantedAuditLog{
+					testutil.WantAuditLog("Session Garbage Collected",
+						map[string]any{
+							"sessionID":   "request-id-1",
+							"storageType": "access-token",
+						},
+					),
+					testutil.WantAuditLog("Upstream OIDC Token Revoked",
+						map[string]any{
+							"sessionID": "request-id-2",
+							"type":      "refresh_token",
+						},
+					),
+					testutil.WantAuditLog("Session Garbage Collected",
+						map[string]any{
+							"sessionID":   "request-id-2",
+							"storageType": "access-token",
+						},
+					),
+				}
 			})
 		})
 
@@ -1151,6 +1260,27 @@ func TestGarbageCollectorControllerSync(t *testing.T) {
 					},
 					kubeClient.Actions(),
 				)
+
+				wantAuditLogs = []testutil.WantedAuditLog{
+					testutil.WantAuditLog("Session Garbage Collected",
+						map[string]any{
+							"sessionID":   "request-id-1",
+							"storageType": "access-token",
+						},
+					),
+					testutil.WantAuditLog("Upstream OIDC Token Revoked",
+						map[string]any{
+							"sessionID": "request-id-2",
+							"type":      "access_token",
+						},
+					),
+					testutil.WantAuditLog("Session Garbage Collected",
+						map[string]any{
+							"sessionID":   "request-id-2",
+							"storageType": "access-token",
+						},
+					),
+				}
 			})
 		})
 
@@ -1228,6 +1358,21 @@ func TestGarbageCollectorControllerSync(t *testing.T) {
 					},
 					kubeClient.Actions(),
 				)
+
+				wantAuditLogs = []testutil.WantedAuditLog{
+					testutil.WantAuditLog("Upstream OIDC Token Revoked",
+						map[string]any{
+							"sessionID": "request-id-1",
+							"type":      "refresh_token",
+						},
+					),
+					testutil.WantAuditLog("Session Garbage Collected",
+						map[string]any{
+							"sessionID":   "request-id-1",
+							"storageType": "refresh-token",
+						},
+					),
+				}
 			})
 		})
 
@@ -1305,6 +1450,21 @@ func TestGarbageCollectorControllerSync(t *testing.T) {
 					},
 					kubeClient.Actions(),
 				)
+
+				wantAuditLogs = []testutil.WantedAuditLog{
+					testutil.WantAuditLog("Upstream OIDC Token Revoked",
+						map[string]any{
+							"sessionID": "request-id-1",
+							"type":      "access_token",
+						},
+					),
+					testutil.WantAuditLog("Session Garbage Collected",
+						map[string]any{
+							"sessionID":   "request-id-1",
+							"storageType": "refresh-token",
+						},
+					),
+				}
 			})
 		})
 

internal/crypto/ptls/log_profiles_test.go

@@ -1,28 +1,25 @@
 // Copyright 2024 the Pinniped contributors. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-package ptls
+package ptls_test
 
 import (
-	"strings"
 	"testing"
 
-	"github.com/stretchr/testify/require"
-
+	"go.pinniped.dev/internal/crypto/ptls"
 	"go.pinniped.dev/internal/plog"
+	"go.pinniped.dev/internal/testutil"
 )
 
 func TestLogAllProfiles(t *testing.T) {
 	logger, log := plog.TestLogger(t)
 
-	LogAllProfiles(logger)
+	ptls.LogAllProfiles(logger)
 
 	expectedLines := []string{
 		`{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"ptls/log_profiles.go:<line>$ptls.logProfile","message":"tls configuration","profile name":"Default","MinVersion":"TLS 1.2","MaxVersion":"NONE","CipherSuites":["TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256","TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384","TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384","TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256","TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256"],"NextProtos":["h2","http/1.1"]}`,
 		`{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"ptls/log_profiles.go:<line>$ptls.logProfile","message":"tls configuration","profile name":"DefaultLDAP","MinVersion":"TLS 1.2","MaxVersion":"NONE","CipherSuites":["TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256","TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384","TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384","TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256","TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256","TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA","TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA","TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA"],"NextProtos":["h2","http/1.1"]}`,
 		`{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"ptls/log_profiles.go:<line>$ptls.logProfile","message":"tls configuration","profile name":"Secure","MinVersion":"TLS 1.3","MaxVersion":"NONE","CipherSuites":[],"NextProtos":["h2","http/1.1"]}`,
 	}
-	expectedOutput := strings.Join(expectedLines, "\n") + "\n"
-
-	require.Equal(t, expectedOutput, log.String())
+	testutil.RequireLogLines(t, expectedLines, log)
 }

internal/dynamiccert/provider_test.go

@@ -114,13 +114,13 @@ func TestProviderWithDynamicServingCertificateController(t *testing.T) {
 				newCA, err := certauthority.New(names.SimpleNameGenerator.GenerateName("new-ca"), time.Hour)
 				require.NoError(t, err)
 
-				certPEM, keyPEM, err := newCA.IssueServerCertPEM(nil, []net.IP{net.ParseIP("127.0.0.2")}, time.Hour)
+				pem, err := newCA.IssueServerCertPEM(nil, []net.IP{net.ParseIP("127.0.0.2")}, time.Hour)
 				require.NoError(t, err)
 
-				err = certKey.SetCertKeyContent(certPEM, keyPEM)
+				err = certKey.SetCertKeyContent(pem.CertPEM, pem.KeyPEM)
 				require.NoError(t, err)
 
-				cert, err := tls.X509KeyPair(certPEM, keyPEM)
+				cert, err := tls.X509KeyPair(pem.CertPEM, pem.KeyPEM)
 				require.NoError(t, err)
 
 				return []tls.Certificate{cert}
@@ -144,13 +144,13 @@ func TestProviderWithDynamicServingCertificateController(t *testing.T) {
 				newCA, err := certauthority.New(names.SimpleNameGenerator.GenerateName("new-ca"), time.Hour)
 				require.NoError(t, err)
 
-				certPEM, keyPEM, err := newCA.IssueServerCertPEM(nil, []net.IP{net.ParseIP("127.0.0.3")}, time.Hour)
+				pem, err := newCA.IssueServerCertPEM(nil, []net.IP{net.ParseIP("127.0.0.3")}, time.Hour)
 				require.NoError(t, err)
 
-				err = certKey.SetCertKeyContent(certPEM, keyPEM)
+				err = certKey.SetCertKeyContent(pem.CertPEM, pem.KeyPEM)
 				require.NoError(t, err)
 
-				cert, err := tls.X509KeyPair(certPEM, keyPEM)
+				cert, err := tls.X509KeyPair(pem.CertPEM, pem.KeyPEM)
 				require.NoError(t, err)
 
 				return []tls.Certificate{cert}
@@ -170,10 +170,10 @@ func TestProviderWithDynamicServingCertificateController(t *testing.T) {
 			err = caContent.SetCertKeyContent(ca.Bundle(), caKey)
 			require.NoError(t, err)
 
-			cert, key, err := ca.IssueServerCertPEM(nil, []net.IP{net.ParseIP("127.0.0.1")}, time.Hour)
+			pem, err := ca.IssueServerCertPEM(nil, []net.IP{net.ParseIP("127.0.0.1")}, time.Hour)
 			require.NoError(t, err)
 			certKeyContent := NewServingCert("cert-key")
-			err = certKeyContent.SetCertKeyContent(cert, key)
+			err = certKeyContent.SetCertKeyContent(pem.CertPEM, pem.KeyPEM)
 			require.NoError(t, err)
 
 			tlsConfig := ptls.Default(nil)

internal/federationdomain/downstreamsession/downstream_session.go

@@ -15,6 +15,7 @@ import (
 	fositejwt "github.com/ory/fosite/token/jwt"
 
 	oidcapi "go.pinniped.dev/generated/latest/apis/supervisor/oidc"
+	"go.pinniped.dev/internal/auditevent"
 	"go.pinniped.dev/internal/constable"
 	"go.pinniped.dev/internal/federationdomain/oidc"
 	"go.pinniped.dev/internal/federationdomain/resolvedprovider"
@@ -34,20 +35,42 @@ type SessionConfig struct {
 	ClientID string
 	// The scopes that were granted for the new downstream session.
 	GrantedScopes []string
+	// The identity provider used to authenticate the user.
+	IdentityProvider resolvedprovider.FederationDomainResolvedIdentityProvider
+	// The fosite Requester that is starting this session.
+	SessionIDGetter plog.SessionIDGetter
 }
 
 // NewPinnipedSession applies the configured FederationDomain identity transformations
 // and creates a downstream Pinniped session.
 func NewPinnipedSession(
 	ctx context.Context,
-	idp resolvedprovider.FederationDomainResolvedIdentityProvider,
+	auditLogger plog.AuditLogger,
 	c *SessionConfig,
 ) (*psession.PinnipedSession, error) {
 	now := time.Now().UTC()
 
+	auditLogger.Audit(auditevent.IdentityFromUpstreamIDP, &plog.AuditParams{
+		ReqCtx: ctx,
+		PIIKeysAndValues: []any{
+			"upstreamUsername", c.UpstreamIdentity.UpstreamUsername,
+			"upstreamGroups", c.UpstreamIdentity.UpstreamGroups,
+		},
+		KeysAndValues: []any{
+			"upstreamIDPDisplayName", c.IdentityProvider.GetDisplayName(),
+			"upstreamIDPType", c.IdentityProvider.GetSessionProviderType(),
+			"upstreamIDPResourceName", c.IdentityProvider.GetProvider().GetResourceName(),
+			"upstreamIDPResourceUID", c.IdentityProvider.GetProvider().GetResourceUID(),
+		},
+	})
+
 	downstreamUsername, downstreamGroups, err := applyIdentityTransformations(ctx,
-		idp.GetTransforms(), c.UpstreamIdentity.UpstreamUsername, c.UpstreamIdentity.UpstreamGroups)
+		c.IdentityProvider.GetTransforms(), c.UpstreamIdentity.UpstreamUsername, c.UpstreamIdentity.UpstreamGroups)
 	if err != nil {
+		auditLogger.Audit(auditevent.AuthenticationRejectedByTransforms, &plog.AuditParams{
+			ReqCtx:        ctx,
+			KeysAndValues: []any{"reason", err},
+		})
 		return nil, err
 	}
 
@@ -55,12 +78,12 @@ func NewPinnipedSession(
 		Username:         downstreamUsername,
 		UpstreamUsername: c.UpstreamIdentity.UpstreamUsername,
 		UpstreamGroups:   c.UpstreamIdentity.UpstreamGroups,
-		ProviderUID:      idp.GetProvider().GetResourceUID(),
-		ProviderName:     idp.GetProvider().GetResourceName(),
-		ProviderType:     idp.GetSessionProviderType(),
+		ProviderUID:      c.IdentityProvider.GetProvider().GetResourceUID(),
+		ProviderName:     c.IdentityProvider.GetProvider().GetResourceName(),
+		ProviderType:     c.IdentityProvider.GetSessionProviderType(),
 		Warnings:         c.UpstreamLoginExtras.Warnings,
 	}
-	idp.ApplyIDPSpecificSessionDataToSession(customSessionData, c.UpstreamIdentity.IDPSpecificSessionData)
+	c.IdentityProvider.ApplyIDPSpecificSessionDataToSession(customSessionData, c.UpstreamIdentity.IDPSpecificSessionData)
 
 	pinnipedSession := &psession.PinnipedSession{
 		Fosite: &openid.DefaultSession{
@@ -94,6 +117,20 @@ func NewPinnipedSession(
 
 	pinnipedSession.IDTokenClaims().Extra = extras
 
+	auditLogger.Audit(auditevent.SessionStarted, &plog.AuditParams{
+		ReqCtx:  ctx,
+		Session: c.SessionIDGetter,
+		PIIKeysAndValues: []any{
+			"username", downstreamUsername,
+			"groups", downstreamGroups,
+			"subject", c.UpstreamIdentity.DownstreamSubject,
+			"additionalClaims", c.UpstreamLoginExtras.DownstreamAdditionalClaims,
+		},
+		KeysAndValues: []any{
+			"warnings", c.UpstreamLoginExtras.Warnings,
+		},
+	})
+
 	return pinnipedSession, nil
 }
 

internal/federationdomain/endpoints/auth/auth_handler.go

@@ -13,14 +13,18 @@ import (
 	"github.com/ory/fosite"
 	"github.com/ory/fosite/handler/openid"
 	fositejwt "github.com/ory/fosite/token/jwt"
+	"k8s.io/apimachinery/pkg/util/sets"
 
 	oidcapi "go.pinniped.dev/generated/latest/apis/supervisor/oidc"
+	"go.pinniped.dev/internal/auditevent"
 	"go.pinniped.dev/internal/federationdomain/csrftoken"
 	"go.pinniped.dev/internal/federationdomain/downstreamsession"
 	"go.pinniped.dev/internal/federationdomain/federationdomainproviders"
 	"go.pinniped.dev/internal/federationdomain/formposthtml"
 	"go.pinniped.dev/internal/federationdomain/oidc"
 	"go.pinniped.dev/internal/federationdomain/resolvedprovider"
+	"go.pinniped.dev/internal/federationdomain/resolvedprovider/resolvedldap"
+	"go.pinniped.dev/internal/federationdomain/stateparam"
 	"go.pinniped.dev/internal/httputil/responseutil"
 	"go.pinniped.dev/internal/httputil/securityheader"
 	"go.pinniped.dev/internal/plog"
@@ -34,6 +38,22 @@ const (
 	promptParamNone = "none"
 )
 
+func paramsSafeToLog() sets.Set[string] {
+	return sets.New[string](
+		// Standard params from https://openid.net/specs/openid-connect-core-1_0.html, some of which are ignored.
+		// Redacting state and nonce params, in case they contain any info that the client considers sensitive.
+		"scope", "response_type", "client_id", "redirect_uri", "response_mode", "display", "prompt",
+		"max_age", "ui_locales", "id_token_hint", "login_hint", "acr_values", "claims_locales", "claims",
+		"request", "request_uri", "registration",
+		// PKCE params from https://datatracker.ietf.org/doc/html/rfc7636. Let code_challenge be redacted.
+		"code_challenge_method",
+		// Custom Pinniped authorization params.
+		oidcapi.AuthorizeUpstreamIDPNameParamName, oidcapi.AuthorizeUpstreamIDPTypeParamName,
+		// Google-specific param that some client libraries will send anyway. Ignored by Pinniped but safe to log.
+		"access_type",
+	)
+}
+
 type authorizeHandler struct {
 	downstreamIssuerURL       string
 	idpFinder                 federationdomainproviders.FederationDomainIdentityProvidersFinderI
@@ -44,6 +64,7 @@ type authorizeHandler struct {
 	generateNonce             func() (nonce.Nonce, error)
 	upstreamStateEncoder      oidc.Encoder
 	cookieCodec               oidc.Codec
+	auditLogger               plog.AuditLogger
 }
 
 func NewHandler(
@@ -56,6 +77,7 @@ func NewHandler(
 	generateNonce func() (nonce.Nonce, error),
 	upstreamStateEncoder oidc.Encoder,
 	cookieCodec oidc.Codec,
+	auditLogger plog.AuditLogger,
 ) http.Handler {
 	h := &authorizeHandler{
 		downstreamIssuerURL:       downstreamIssuerURL,
@@ -67,6 +89,7 @@ func NewHandler(
 		generateNonce:             generateNonce,
 		upstreamStateEncoder:      upstreamStateEncoder,
 		cookieCodec:               cookieCodec,
+		auditLogger:               auditLogger,
 	}
 	// During a response_mode=form_post auth request using the browser flow, the custom form_post html page may
 	// be used to post certain errors back to the CLI from this handler's response, so allow the form_post
@@ -75,6 +98,29 @@ func NewHandler(
 }
 
 func (h *authorizeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// If the client set a username or password header, they are trying to log in without using a browser.
+	hadUsernameHeader := len(r.Header.Values(oidcapi.AuthorizeUsernameHeaderName)) > 0
+	hadPasswordHeader := len(r.Header.Values(oidcapi.AuthorizePasswordHeaderName)) > 0
+	requestedBrowserlessFlow := hadUsernameHeader || hadPasswordHeader
+
+	// Audit the request params. Also gives us access to the IDP name param for use below,
+	// before fosite would normally parse the params.
+	if err := h.auditLogger.AuditRequestParams(r, paramsSafeToLog()); err != nil {
+		oidc.WriteAuthorizeError(r, w,
+			h.oauthHelperWithoutStorage, fosite.NewAuthorizeRequest(), err, requestedBrowserlessFlow)
+		return
+	}
+
+	// Log if these headers were present, but don't log the actual values. The password is obviously sensitive,
+	// and sometimes users use their password as their username by mistake.
+	h.auditLogger.Audit(auditevent.HTTPRequestCustomHeadersUsed, &plog.AuditParams{
+		ReqCtx: r.Context(),
+		KeysAndValues: []any{
+			oidcapi.AuthorizeUsernameHeaderName, hadUsernameHeader,
+			oidcapi.AuthorizePasswordHeaderName, hadPasswordHeader,
+		},
+	})
+
 	if r.Method != http.MethodPost && r.Method != http.MethodGet {
 		// https://openid.net/specs/openid-connect-core-1_0.html#AuthRequest
 		// Authorization Servers MUST support the use of the HTTP GET and POST methods defined in
@@ -83,35 +129,6 @@ func (h *authorizeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// The client set a username or password header, so they are trying to log in without using a browser.
-	requestedBrowserlessFlow := len(r.Header.Values(oidcapi.AuthorizeUsernameHeaderName)) > 0 ||
-		len(r.Header.Values(oidcapi.AuthorizePasswordHeaderName)) > 0
-
-	// Need to parse the request params, so we can get the IDP name. The style and text of the error is inspired by
-	// fosite's implementation of NewAuthorizeRequest(). Fosite only calls ParseMultipartForm() there. However,
-	// although ParseMultipartForm() calls ParseForm(), it swallows errors from ParseForm() sometimes. To avoid
-	// having any errors swallowed, we call both. When fosite calls ParseMultipartForm() later, it will be a noop.
-	if err := r.ParseForm(); err != nil {
-		oidc.WriteAuthorizeError(r, w,
-			h.oauthHelperWithoutStorage,
-			fosite.NewAuthorizeRequest(),
-			fosite.ErrInvalidRequest.
-				WithHint("Unable to parse form params, make sure to send a properly formatted query params or form request body.").
-				WithWrap(err).WithDebug(err.Error()),
-			requestedBrowserlessFlow)
-		return
-	}
-	if err := r.ParseMultipartForm(1 << 20); err != nil && err != http.ErrNotMultipart {
-		oidc.WriteAuthorizeError(r, w,
-			h.oauthHelperWithoutStorage,
-			fosite.NewAuthorizeRequest(),
-			fosite.ErrInvalidRequest.
-				WithHint("Unable to parse multipart HTTP body, make sure to send a properly formatted form request body.").
-				WithWrap(err).WithDebug(err.Error()),
-			requestedBrowserlessFlow)
-		return
-	}
-
 	// Note that the client might have used oidcapi.AuthorizeUpstreamIDPNameParamName and
 	// oidcapi.AuthorizeUpstreamIDPTypeParamName query (or form) params to request a certain upstream IDP.
 	// The Pinniped CLI has been sending these params since v0.9.0.
@@ -141,6 +158,16 @@ func (h *authorizeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	h.auditLogger.Audit(auditevent.UsingUpstreamIDP, &plog.AuditParams{
+		ReqCtx: r.Context(),
+		KeysAndValues: []any{
+			"displayName", idp.GetDisplayName(),
+			"resourceName", idp.GetProvider().GetResourceName(),
+			"resourceUID", idp.GetProvider().GetResourceUID(),
+			"type", idp.GetSessionProviderType(),
+		},
+	})
+
 	h.authorize(w, r, requestedBrowserlessFlow, idp)
 }
 
@@ -175,9 +202,19 @@ func (h *authorizeHandler) authorize(
 	if requestedBrowserlessFlow {
 		err = h.authorizeWithoutBrowser(r, w, oauthHelper, authorizeRequester, idp)
 	} else {
-		err = h.authorizeWithBrowser(r, w, oauthHelper, authorizeRequester, idp)
+		var authorizeID string
+		authorizeID, err = h.authorizeWithBrowser(r, w, oauthHelper, authorizeRequester, idp)
+
+		if err == nil {
+			h.auditLogger.Audit(auditevent.UpstreamAuthorizeRedirect, &plog.AuditParams{
+				ReqCtx:        r.Context(),
+				KeysAndValues: []any{"authorizeID", authorizeID},
+			})
+		}
 	}
 	if err != nil {
+		// No specific audit event is emitted here in the case of an authorization error.
+		// Rely on the "HTTP Request Completed" audit event with an error and error_description to indicate what went wrong.
 		oidc.WriteAuthorizeError(r, w, oauthHelper, authorizeRequester, err, requestedBrowserlessFlow)
 	}
 }
@@ -200,14 +237,21 @@ func (h *authorizeHandler) authorizeWithoutBrowser(
 
 	identity, loginExtras, err := idp.Login(r.Context(), submittedUsername, submittedPassword)
 	if err != nil {
+		if err == resolvedldap.ErrAccessDeniedDueToUsernamePasswordNotAccepted {
+			h.auditLogger.Audit(auditevent.IncorrectUsernameOrPassword, &plog.AuditParams{
+				ReqCtx: r.Context(),
+			})
+		}
 		return err
 	}
 
-	session, err := downstreamsession.NewPinnipedSession(r.Context(), idp, &downstreamsession.SessionConfig{
+	session, err := downstreamsession.NewPinnipedSession(r.Context(), h.auditLogger, &downstreamsession.SessionConfig{
 		UpstreamIdentity:    identity,
 		UpstreamLoginExtras: loginExtras,
 		ClientID:            authorizeRequester.GetClient().GetID(),
 		GrantedScopes:       authorizeRequester.GetGrantedScopes(),
+		IdentityProvider:    idp,
+		SessionIDGetter:     authorizeRequester,
 	})
 	if err != nil {
 		return fosite.ErrAccessDenied.WithHintf("Reason: %s.", err.Error())
@@ -224,7 +268,7 @@ func (h *authorizeHandler) authorizeWithBrowser(
 	oauthHelper fosite.OAuth2Provider,
 	authorizeRequester fosite.AuthorizeRequester,
 	idp resolvedprovider.FederationDomainResolvedIdentityProvider,
-) error {
+) (string, error) {
 	authRequestState, err := generateUpstreamAuthorizeRequestState(r, w,
 		authorizeRequester,
 		oauthHelper,
@@ -237,19 +281,19 @@ func (h *authorizeHandler) authorizeWithBrowser(
 		h.upstreamStateEncoder,
 	)
 	if err != nil {
-		return err
+		return "", err
 	}
 
 	redirectURL, err := idp.UpstreamAuthorizeRedirectURL(authRequestState, h.downstreamIssuerURL)
 	if err != nil {
-		return err
+		return "", err
 	}
 
 	http.Redirect(w, r, redirectURL,
 		http.StatusSeeOther, // match fosite and https://tools.ietf.org/id/draft-ietf-oauth-security-topics-18.html#section-4.11
 	)
 
-	return nil
+	return authRequestState.EncodedStateParam.AuthorizeID(), nil
 }
 
 func shouldShowIDPChooser(
@@ -425,7 +469,7 @@ func upstreamStateParam(
 	csrfValue csrftoken.CSRFToken,
 	pkceValue pkce.Code,
 	encoder oidc.Encoder,
-) (string, error) {
+) (stateparam.Encoded, error) {
 	stateParamData := oidc.UpstreamStateParamData{
 		// The auth params might have included oidcapi.AuthorizeUpstreamIDPNameParamName and
 		// oidcapi.AuthorizeUpstreamIDPTypeParamName, but those can be ignored by other handlers
@@ -444,7 +488,7 @@ func upstreamStateParam(
 	if err != nil {
 		return "", fmt.Errorf("error encoding upstream state param: %w", err)
 	}
-	return encodedStateParamValue, nil
+	return stateparam.Encoded(encodedStateParamValue), nil
 }
 
 func removeCustomIDPParams(params url.Values) url.Values {

internal/federationdomain/endpoints/callback/callback_handler.go

@@ -7,43 +7,78 @@ package callback
 import (
 	"net/http"
 	"net/url"
+	"strings"
 
 	"github.com/ory/fosite"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apiserver/pkg/audit"
 
+	"go.pinniped.dev/internal/auditevent"
 	"go.pinniped.dev/internal/federationdomain/downstreamsession"
 	"go.pinniped.dev/internal/federationdomain/federationdomainproviders"
 	"go.pinniped.dev/internal/federationdomain/formposthtml"
 	"go.pinniped.dev/internal/federationdomain/oidc"
+	"go.pinniped.dev/internal/federationdomain/stateparam"
 	"go.pinniped.dev/internal/httputil/httperr"
 	"go.pinniped.dev/internal/httputil/securityheader"
 	"go.pinniped.dev/internal/plog"
 )
 
+func paramsSafeToLog() sets.Set[string] {
+	return sets.New[string](
+		// Due to https://datatracker.ietf.org/doc/html/rfc6749#section-4.1.2.1,
+		// authorize errors can have these parameters, which should not contain PII or secrets and are safe to log.
+		"error", "error_description", "error_uri",
+		// Note that this endpoint also receives 'code' and 'state' params, which are not safe to log.
+	)
+}
+
 func NewHandler(
 	upstreamIDPs federationdomainproviders.FederationDomainIdentityProvidersFinderI,
 	oauthHelper fosite.OAuth2Provider,
 	stateDecoder, cookieDecoder oidc.Decoder,
 	redirectURI string,
+	auditLogger plog.AuditLogger,
 ) http.Handler {
 	handler := httperr.HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
-		state, err := validateRequest(r, stateDecoder, cookieDecoder)
+		if err := auditLogger.AuditRequestParams(r, paramsSafeToLog()); err != nil {
+			plog.DebugErr("error parsing callback request params", err)
+			return httperr.New(http.StatusBadRequest, "error parsing request params")
+		}
+
+		encodedState, decodedState, err := validateRequest(r, stateDecoder, cookieDecoder)
 		if err != nil {
 			return err
 		}
 
-		idp, err := upstreamIDPs.FindUpstreamIDPByDisplayName(state.UpstreamName)
+		auditLogger.Audit(auditevent.AuthorizeIDFromParameters, &plog.AuditParams{
+			ReqCtx:        r.Context(),
+			KeysAndValues: []any{"authorizeID", encodedState.AuthorizeID()},
+		})
+
+		idp, err := upstreamIDPs.FindUpstreamIDPByDisplayName(decodedState.UpstreamName)
 		if err != nil || idp == nil {
 			plog.Warning("upstream provider not found")
 			return httperr.New(http.StatusUnprocessableEntity, "upstream provider not found")
 		}
 
-		downstreamAuthParams, err := url.ParseQuery(state.AuthParams)
+		auditLogger.Audit(auditevent.UsingUpstreamIDP, &plog.AuditParams{
+			ReqCtx: r.Context(),
+			KeysAndValues: []any{
+				"displayName", idp.GetDisplayName(),
+				"resourceName", idp.GetProvider().GetResourceName(),
+				"resourceUID", idp.GetProvider().GetResourceUID(),
+				"type", idp.GetSessionProviderType(),
+			},
+		})
+
+		downstreamAuthParams, err := url.ParseQuery(decodedState.AuthParams)
 		if err != nil {
 			plog.Error("error reading state downstream auth params", err)
 			return httperr.New(http.StatusBadRequest, "error reading state downstream auth params")
 		}
 
-		// Recreate enough of the original authorize request so we can pass it to NewAuthorizeRequest().
+		// Recreate enough of the original authorize request, so we can pass it to NewAuthorizeRequest().
 		reconstitutedAuthRequest := &http.Request{Form: downstreamAuthParams}
 		authorizeRequester, err := oauthHelper.NewAuthorizeRequest(r.Context(), reconstitutedAuthRequest)
 		if err != nil {
@@ -60,7 +95,7 @@ func NewHandler(
 		// an error if the client requested a scope that they are not allowed to request, so we don't need to worry about that here.
 		downstreamsession.AutoApproveScopes(authorizeRequester)
 
-		identity, loginExtras, err := idp.LoginFromCallback(r.Context(), authcode(r), state.PKCECode, state.Nonce, redirectURI)
+		identity, loginExtras, err := idp.LoginFromCallback(r.Context(), authcode(r), decodedState.PKCECode, decodedState.Nonce, redirectURI)
 		if err != nil {
 			plog.WarningErr("unable to complete login from callback", err,
 				"identityProviderDisplayName", idp.GetDisplayName(),
@@ -69,11 +104,13 @@ func NewHandler(
 			return err
 		}
 
-		session, err := downstreamsession.NewPinnipedSession(r.Context(), idp, &downstreamsession.SessionConfig{
+		session, err := downstreamsession.NewPinnipedSession(r.Context(), auditLogger, &downstreamsession.SessionConfig{
 			UpstreamIdentity:    identity,
 			UpstreamLoginExtras: loginExtras,
 			ClientID:            authorizeRequester.GetClient().GetID(),
 			GrantedScopes:       authorizeRequester.GetGrantedScopes(),
+			IdentityProvider:    idp,
+			SessionIDGetter:     authorizeRequester,
 		})
 		if err != nil {
 			plog.WarningErr("unable to create a Pinniped session", err,
@@ -104,21 +141,56 @@ func authcode(r *http.Request) string {
 	return r.FormValue("code")
 }
 
-func validateRequest(r *http.Request, stateDecoder, cookieDecoder oidc.Decoder) (*oidc.UpstreamStateParamData, error) {
+func validateRequest(r *http.Request, stateDecoder, cookieDecoder oidc.Decoder) (stateparam.Encoded, *oidc.UpstreamStateParamData, error) {
 	if r.Method != http.MethodGet {
-		return nil, httperr.Newf(http.StatusMethodNotAllowed, "%s (try GET)", r.Method)
+		return "", nil, httperr.Newf(http.StatusMethodNotAllowed, "%s (try GET)", r.Method)
 	}
 
-	_, decodedState, err := oidc.ReadStateParamAndValidateCSRFCookie(r, cookieDecoder, stateDecoder)
+	encodedState, decodedState, err := oidc.ReadStateParamAndValidateCSRFCookie(r, cookieDecoder, stateDecoder)
 	if err != nil {
 		plog.InfoErr("state or CSRF error", err)
-		return nil, err
+		return "", nil, err
 	}
 
 	if authcode(r) == "" {
 		plog.Info("code param not found")
-		return nil, httperr.New(http.StatusBadRequest, "code param not found")
+		return "", nil, httperr.New(http.StatusBadRequest, errorMsgForNoCodeParam(r))
 	}
 
-	return decodedState, nil
+	return encodedState, decodedState, nil
+}
+
+func errorMsgForNoCodeParam(r *http.Request) string {
+	msg := strings.Builder{}
+
+	msg.WriteString("code param not found\n\n")
+
+	errorParam, hasError := r.Form["error"]
+	errorDescParam, hasErrorDesc := r.Form["error_description"]
+	errorURIParam, hasErrorURI := r.Form["error_uri"]
+
+	if hasError {
+		msg.WriteString("error from external identity provider: ")
+		msg.WriteString(errorParam[0])
+		msg.WriteByte('\n')
+	}
+	if hasErrorDesc {
+		msg.WriteString("error_description from external identity provider: ")
+		msg.WriteString(errorDescParam[0])
+		msg.WriteByte('\n')
+	}
+	if hasErrorURI {
+		msg.WriteString("error_uri from external identity provider: ")
+		msg.WriteString(errorURIParam[0])
+		msg.WriteByte('\n')
+	}
+	if !hasError && !hasErrorDesc && !hasErrorURI {
+		msg.WriteString("Something went wrong with your authentication attempt at your external identity provider.\n")
+	}
+
+	msg.WriteByte('\n')
+	msg.WriteString("Pinniped AuditID: ")
+	msg.WriteString(audit.GetAuditIDTruncated(r.Context()))
+
+	return msg.String()
 }

internal/federationdomain/endpoints/callback/callback_handler_test.go

@@ -22,11 +22,15 @@ import (
 
 	supervisorconfigv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/config/v1alpha1"
 	supervisorfake "go.pinniped.dev/generated/latest/client/supervisor/clientset/versioned/fake"
+	"go.pinniped.dev/internal/auditid"
 	"go.pinniped.dev/internal/federationdomain/endpoints/jwks"
 	"go.pinniped.dev/internal/federationdomain/oidc"
 	"go.pinniped.dev/internal/federationdomain/oidcclientvalidator"
+	"go.pinniped.dev/internal/federationdomain/stateparam"
 	"go.pinniped.dev/internal/federationdomain/storage"
 	"go.pinniped.dev/internal/federationdomain/upstreamprovider"
+	"go.pinniped.dev/internal/here"
+	"go.pinniped.dev/internal/plog"
 	"go.pinniped.dev/internal/psession"
 	"go.pinniped.dev/internal/testutil"
 	"go.pinniped.dev/internal/testutil/oidctestutil"
@@ -245,6 +249,7 @@ func TestCallbackEndpoint(t *testing.T) {
 		wantDownstreamAdditionalClaims    map[string]any
 		wantOIDCAuthcodeExchangeCall      *expectedOIDCAuthcodeExchange
 		wantGitHubAuthcodeExchangeCall    *expectedGitHubAuthcodeExchange
+		wantAuditLogs                     func(encodedStateParam stateparam.Encoded, sessionID string) []testutil.WantedAuditLog
 	}{
 		{
 			name:   "OIDC: GET with good state and cookie and successful upstream token exchange with response_mode=form_post returns 200 with HTML+JS form",
@@ -276,6 +281,42 @@ func TestCallbackEndpoint(t *testing.T) {
 				performedByUpstreamName: happyOIDCUpstreamIDPName,
 				args:                    happyOIDCUpstreamExchangeAuthcodeAndValidateTokenArgs,
 			},
+			wantAuditLogs: func(encodedStateParam stateparam.Encoded, sessionID string) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+						"params": map[string]any{"code": "redacted", "state": "redacted"},
+					}),
+					testutil.WantAuditLog("AuthorizeID From Parameters", map[string]any{
+						"authorizeID": encodedStateParam.AuthorizeID(),
+					}),
+					testutil.WantAuditLog("Using Upstream IDP", map[string]any{
+						"displayName":  "upstream-oidc-idp-name",
+						"resourceName": "upstream-oidc-idp-name",
+						"resourceUID":  "upstream-oidc-resource-uid",
+						"type":         "oidc",
+					}),
+					testutil.WantAuditLog("Identity From Upstream IDP", map[string]any{
+						"upstreamIDPDisplayName":  "upstream-oidc-idp-name",
+						"upstreamIDPType":         "oidc",
+						"upstreamIDPResourceName": "upstream-oidc-idp-name",
+						"upstreamIDPResourceUID":  "upstream-oidc-resource-uid",
+						"personalInfo": map[string]any{
+							"upstreamUsername": "test-pinniped-username",
+							"upstreamGroups":   []any{"test-pinniped-group-0", "test-pinniped-group-1"},
+						},
+					}),
+					testutil.WantAuditLog("Session Started", map[string]any{
+						"sessionID": sessionID,
+						"warnings":  []any{}, // json: []
+						"personalInfo": map[string]any{
+							"username":         "test-pinniped-username",
+							"groups":           []any{"test-pinniped-group-0", "test-pinniped-group-1"},
+							"subject":          "https://my-upstream-issuer.com?idpName=upstream-oidc-idp-name&sub=abc123-some+guid",
+							"additionalClaims": map[string]any{}, // json: {}
+						},
+					}),
+				}
+			},
 		},
 		{
 			name:   "GitHub: GET with good state and cookie and successful upstream token exchange with response_mode=form_post returns 200 with HTML+JS form",
@@ -307,6 +348,42 @@ func TestCallbackEndpoint(t *testing.T) {
 				performedByUpstreamName: happyGithubIDPName,
 				args:                    happyGitHubUpstreamExchangeAuthcodeArgs,
 			},
+			wantAuditLogs: func(encodedStateParam stateparam.Encoded, sessionID string) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+						"params": map[string]any{"code": "redacted", "state": "redacted"},
+					}),
+					testutil.WantAuditLog("AuthorizeID From Parameters", map[string]any{
+						"authorizeID": encodedStateParam.AuthorizeID(),
+					}),
+					testutil.WantAuditLog("Using Upstream IDP", map[string]any{
+						"displayName":  "upstream-github-idp-name",
+						"resourceName": "upstream-github-idp-name",
+						"resourceUID":  "upstream-github-idp-resource-uid",
+						"type":         "github",
+					}),
+					testutil.WantAuditLog("Identity From Upstream IDP", map[string]any{
+						"upstreamIDPDisplayName":  "upstream-github-idp-name",
+						"upstreamIDPType":         "github",
+						"upstreamIDPResourceName": "upstream-github-idp-name",
+						"upstreamIDPResourceUID":  "upstream-github-idp-resource-uid",
+						"personalInfo": map[string]any{
+							"upstreamUsername": "some-github-login",
+							"upstreamGroups":   []any{"org1/team1", "org2/team2"},
+						},
+					}),
+					testutil.WantAuditLog("Session Started", map[string]any{
+						"sessionID": sessionID,
+						"warnings":  []any{}, // json: []
+						"personalInfo": map[string]any{
+							"username":         "some-github-login",
+							"groups":           []any{"org1/team1", "org2/team2"},
+							"subject":          "https://github.com?idpName=upstream-github-idp-name&sub=some-github-login",
+							"additionalClaims": map[string]any{}, // json: {}
+						},
+					}),
+				}
+			},
 		},
 		{
 			name: "GET with good state and cookie with additional params",
@@ -655,6 +732,22 @@ func TestCallbackEndpoint(t *testing.T) {
 				performedByUpstreamName: happyOIDCUpstreamIDPName,
 				args:                    happyOIDCUpstreamExchangeAuthcodeAndValidateTokenArgs,
 			},
+			wantAuditLogs: func(encodedStateParam stateparam.Encoded, _ string) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+						"params": map[string]any{"code": "redacted", "state": "redacted"},
+					}),
+					testutil.WantAuditLog("AuthorizeID From Parameters", map[string]any{
+						"authorizeID": encodedStateParam.AuthorizeID(),
+					}),
+					testutil.WantAuditLog("Using Upstream IDP", map[string]any{
+						"displayName":  "upstream-oidc-idp-name",
+						"resourceName": "upstream-oidc-idp-name",
+						"resourceUID":  "upstream-oidc-resource-uid",
+						"type":         "oidc",
+					}),
+				}
+			},
 		},
 		{
 			name:            "return an error when upstream IDP returned no refresh token and no access token",
@@ -1039,6 +1132,13 @@ func TestCallbackEndpoint(t *testing.T) {
 			wantStatus:      http.StatusMethodNotAllowed,
 			wantContentType: htmlContentType,
 			wantBody:        "Method Not Allowed: PUT (try GET)\n",
+			wantAuditLogs: func(encodedStateParam stateparam.Encoded, sessionID string) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+						"params": map[string]any{"code": "redacted", "state": "redacted"},
+					}),
+				}
+			},
 		},
 		{
 			name:            "POST method is invalid",
@@ -1068,14 +1168,95 @@ func TestCallbackEndpoint(t *testing.T) {
 			wantBody:        "Method Not Allowed: DELETE (try GET)\n",
 		},
 		{
-			name:            "code param was not included on request",
+			name:            "params cannot be parsed",
+			idps:            testidplister.NewUpstreamIDPListerBuilder().WithOIDC(happyOIDCUpstream().Build()),
+			method:          http.MethodGet,
+			path:            newRequestPath().String() + "&invalid;;param",
+			csrfCookie:      happyCSRFCookie,
+			wantStatus:      http.StatusBadRequest,
+			wantContentType: htmlContentType,
+			wantBody:        "Bad Request: error parsing request params\n",
+			wantAuditLogs: func(encodedStateParam stateparam.Encoded, sessionID string) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{}
+			},
+		},
+		{
+			name:            "error redirect from upstream IDP audit logs all the error params from the OAuth2 spec",
+			idps:            testidplister.NewUpstreamIDPListerBuilder().WithOIDC(happyOIDCUpstream().Build()),
+			method:          http.MethodGet,
+			path:            newRequestPath().WithState(happyOIDCState).WithoutCode().String() + "&error=some%20error&error_description=some%20description&error_uri=some%20uri",
+			csrfCookie:      happyCSRFCookie,
+			wantStatus:      http.StatusBadRequest,
+			wantContentType: htmlContentType,
+			wantBody: here.Doc(`Bad Request: code param not found
+
+				error from external identity provider: some error
+				error_description from external identity provider: some description
+				error_uri from external identity provider: some uri
+
+				Pinniped AuditID: fake-audit-id
+			`),
+			wantAuditLogs: func(encodedStateParam stateparam.Encoded, sessionID string) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+						"params": map[string]any{
+							"state":             "redacted",
+							"error":             "some error",
+							"error_description": "some description",
+							"error_uri":         "some uri",
+						},
+					}),
+				}
+			},
+		},
+		{
+			name:            "error redirect from upstream IDP when only some of the error params from the OAuth2 spec are included on the URL",
+			idps:            testidplister.NewUpstreamIDPListerBuilder().WithOIDC(happyOIDCUpstream().Build()),
+			method:          http.MethodGet,
+			path:            newRequestPath().WithState(happyOIDCState).WithoutCode().String() + "&error=some%20error&error_description=some%20description",
+			csrfCookie:      happyCSRFCookie,
+			wantStatus:      http.StatusBadRequest,
+			wantContentType: htmlContentType,
+			wantBody: here.Doc(`Bad Request: code param not found
+
+				error from external identity provider: some error
+				error_description from external identity provider: some description
+
+				Pinniped AuditID: fake-audit-id
+			`),
+			wantAuditLogs: func(encodedStateParam stateparam.Encoded, sessionID string) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+						"params": map[string]any{
+							"state":             "redacted",
+							"error":             "some error",
+							"error_description": "some description",
+						},
+					}),
+				}
+			},
+		},
+		{
+			name:            "code param was not included on request and there is no error param",
 			idps:            testidplister.NewUpstreamIDPListerBuilder().WithOIDC(happyOIDCUpstream().Build()),
 			method:          http.MethodGet,
 			path:            newRequestPath().WithState(happyOIDCState).WithoutCode().String(),
 			csrfCookie:      happyCSRFCookie,
 			wantStatus:      http.StatusBadRequest,
 			wantContentType: htmlContentType,
-			wantBody:        "Bad Request: code param not found\n",
+			wantBody: here.Doc(`Bad Request: code param not found
+
+				Something went wrong with your authentication attempt at your external identity provider.
+
+				Pinniped AuditID: fake-audit-id
+			`),
+			wantAuditLogs: func(encodedStateParam stateparam.Encoded, sessionID string) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+						"params": map[string]any{"state": "redacted"},
+					}),
+				}
+			},
 		},
 		{
 			name:            "state param was not included on request",
@@ -1086,6 +1267,13 @@ func TestCallbackEndpoint(t *testing.T) {
 			wantStatus:      http.StatusBadRequest,
 			wantContentType: htmlContentType,
 			wantBody:        "Bad Request: state param not found\n",
+			wantAuditLogs: func(encodedStateParam stateparam.Encoded, sessionID string) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+						"params": map[string]any{"code": "redacted"},
+					}),
+				}
+			},
 		},
 		{
 			name:            "state param was not signed correctly, has expired, or otherwise cannot be decoded for any reason",
@@ -1718,6 +1906,35 @@ func TestCallbackEndpoint(t *testing.T) {
 				performedByUpstreamName: happyOIDCUpstreamIDPName,
 				args:                    happyOIDCUpstreamExchangeAuthcodeAndValidateTokenArgs,
 			},
+			wantAuditLogs: func(encodedStateParam stateparam.Encoded, sessionID string) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+						"params": map[string]any{"code": "redacted", "state": "redacted"},
+					}),
+					testutil.WantAuditLog("AuthorizeID From Parameters", map[string]any{
+						"authorizeID": encodedStateParam.AuthorizeID(),
+					}),
+					testutil.WantAuditLog("Using Upstream IDP", map[string]any{
+						"displayName":  "upstream-oidc-idp-name",
+						"resourceName": "upstream-oidc-idp-name",
+						"resourceUID":  "upstream-oidc-resource-uid",
+						"type":         "oidc",
+					}),
+					testutil.WantAuditLog("Identity From Upstream IDP", map[string]any{
+						"upstreamIDPDisplayName":  "upstream-oidc-idp-name",
+						"upstreamIDPType":         "oidc",
+						"upstreamIDPResourceName": "upstream-oidc-idp-name",
+						"upstreamIDPResourceUID":  "upstream-oidc-resource-uid",
+						"personalInfo": map[string]any{
+							"upstreamUsername": "test-pinniped-username",
+							"upstreamGroups":   []any{"test-pinniped-group-0", "test-pinniped-group-1"},
+						},
+					}),
+					testutil.WantAuditLog("Authentication Rejected By Transforms", map[string]any{
+						"reason": "configured identity policy rejected this authentication: authentication was rejected by a configured policy",
+					}),
+				}
+			},
 		},
 		{
 			name: "GitHub: using identity transformations which reject the authentication",
@@ -1733,6 +1950,35 @@ func TestCallbackEndpoint(t *testing.T) {
 				performedByUpstreamName: happyGithubIDPName,
 				args:                    happyGitHubUpstreamExchangeAuthcodeArgs,
 			},
+			wantAuditLogs: func(encodedStateParam stateparam.Encoded, sessionID string) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+						"params": map[string]any{"code": "redacted", "state": "redacted"},
+					}),
+					testutil.WantAuditLog("AuthorizeID From Parameters", map[string]any{
+						"authorizeID": encodedStateParam.AuthorizeID(),
+					}),
+					testutil.WantAuditLog("Using Upstream IDP", map[string]any{
+						"displayName":  "upstream-github-idp-name",
+						"resourceName": "upstream-github-idp-name",
+						"resourceUID":  "upstream-github-idp-resource-uid",
+						"type":         "github",
+					}),
+					testutil.WantAuditLog("Identity From Upstream IDP", map[string]any{
+						"upstreamIDPDisplayName":  "upstream-github-idp-name",
+						"upstreamIDPType":         "github",
+						"upstreamIDPResourceName": "upstream-github-idp-name",
+						"upstreamIDPResourceUID":  "upstream-github-idp-resource-uid",
+						"personalInfo": map[string]any{
+							"upstreamUsername": "some-github-login",
+							"upstreamGroups":   []any{"org1/team1", "org2/team2"},
+						},
+					}),
+					testutil.WantAuditLog("Authentication Rejected By Transforms", map[string]any{
+						"reason": "configured identity policy rejected this authentication: authentication was rejected by a configured policy",
+					}),
+				}
+			},
 		},
 	}
 
@@ -1757,12 +2003,23 @@ func TestCallbackEndpoint(t *testing.T) {
 			jwksProviderIsUnused := jwks.NewDynamicJWKSProvider()
 			oauthHelper := oidc.FositeOauth2Helper(oauthStore, downstreamIssuer, hmacSecretFunc, jwksProviderIsUnused, timeoutsConfiguration)
 
-			subject := NewHandler(test.idps.BuildFederationDomainIdentityProvidersListerFinder(), oauthHelper, happyStateCodec, happyCookieCodec, happyUpstreamRedirectURI)
+			auditLogger, actualAuditLog := plog.TestAuditLogger(t)
+
+			subject := NewHandler(
+				test.idps.BuildFederationDomainIdentityProvidersListerFinder(),
+				oauthHelper,
+				happyStateCodec,
+				happyCookieCodec,
+				happyUpstreamRedirectURI,
+				auditLogger,
+			)
+
 			reqContext := context.WithValue(context.Background(), struct{ name string }{name: "test"}, "request-context")
 			req := httptest.NewRequest(test.method, test.path, nil).WithContext(reqContext)
 			if test.csrfCookie != "" {
 				req.Header.Set("Cookie", test.csrfCookie)
 			}
+			req, _ = auditid.NewRequestWithAuditID(req, func() string { return "fake-audit-id" })
 			rsp := httptest.NewRecorder()
 			subject.ServeHTTP(rsp, req)
 			t.Logf("response: %#v", rsp)
@@ -1772,13 +2029,13 @@ func TestCallbackEndpoint(t *testing.T) {
 
 			switch {
 			case test.wantOIDCAuthcodeExchangeCall != nil:
-				test.wantOIDCAuthcodeExchangeCall.args.Ctx = reqContext
+				test.wantOIDCAuthcodeExchangeCall.args.Ctx = req.Context()
 				test.idps.RequireExactlyOneOIDCAuthcodeExchange(t,
 					test.wantOIDCAuthcodeExchangeCall.performedByUpstreamName,
 					test.wantOIDCAuthcodeExchangeCall.args,
 				)
 			case test.wantGitHubAuthcodeExchangeCall != nil:
-				test.wantGitHubAuthcodeExchangeCall.args.Ctx = reqContext
+				test.wantGitHubAuthcodeExchangeCall.args.Ctx = req.Context()
 				test.idps.RequireExactlyOneGitHubAuthcodeExchange(t,
 					test.wantGitHubAuthcodeExchangeCall.performedByUpstreamName,
 					test.wantGitHubAuthcodeExchangeCall.args,
@@ -1790,6 +2047,8 @@ func TestCallbackEndpoint(t *testing.T) {
 			require.Equal(t, test.wantStatus, rsp.Code)
 			testutil.RequireEqualContentType(t, rsp.Header().Get("Content-Type"), test.wantContentType)
 
+			sessionID := ""
+
 			switch {
 			// If we want a specific static response body, assert that.
 			case test.wantBody != "":
@@ -1797,7 +2056,7 @@ func TestCallbackEndpoint(t *testing.T) {
 
 			// Else if we want a body that contains a regex-matched auth code, assert that (for "response_mode=form_post").
 			case test.wantBodyFormResponseRegexp != "":
-				oidctestutil.RequireAuthCodeRegexpMatch(
+				sessionID = oidctestutil.RequireAuthCodeRegexpMatch(
 					t,
 					rsp.Body.String(),
 					test.wantBodyFormResponseRegexp,
@@ -1825,7 +2084,7 @@ func TestCallbackEndpoint(t *testing.T) {
 
 			if test.wantRedirectLocationRegexp != "" {
 				require.Len(t, rsp.Header().Values("Location"), 1)
-				oidctestutil.RequireAuthCodeRegexpMatch(
+				sessionID = oidctestutil.RequireAuthCodeRegexpMatch(
 					t,
 					rsp.Header().Get("Location"),
 					test.wantRedirectLocationRegexp,
@@ -1846,6 +2105,12 @@ func TestCallbackEndpoint(t *testing.T) {
 					test.wantDownstreamAdditionalClaims,
 				)
 			}
+
+			if test.wantAuditLogs != nil {
+				wantAuditLogs := test.wantAuditLogs(testutil.GetStateParam(t, test.path), sessionID)
+				testutil.WantAuditIDOnEveryAuditLog(wantAuditLogs, "fake-audit-id")
+				testutil.CompareAuditLogs(t, wantAuditLogs, actualAuditLog.String())
+			}
 		})
 	}
 }
@@ -1861,12 +2126,13 @@ type expectedGitHubAuthcodeExchange struct {
 }
 
 type requestPath struct {
-	code, state *string
+	code  *string
+	state *stateparam.Encoded
 }
 
 func newRequestPath() *requestPath {
 	c := happyUpstreamAuthcode
-	s := "4321"
+	s := stateparam.Encoded("4321")
 	return &requestPath{
 		code:  &c,
 		state: &s,
@@ -1883,7 +2149,7 @@ func (r *requestPath) WithoutCode() *requestPath {
 	return r
 }
 
-func (r *requestPath) WithState(state string) *requestPath {
+func (r *requestPath) WithState(state stateparam.Encoded) *requestPath {
 	r.state = &state
 	return r
 }
@@ -1900,7 +2166,7 @@ func (r *requestPath) String() string {
 		params.Add("code", *r.code)
 	}
 	if r.state != nil {
-		params.Add("state", *r.state)
+		params.Add("state", r.state.String())
 	}
 	return path + params.Encode()
 }
@@ -1980,3 +2246,14 @@ func shallowCopyAndModifyQuery(query url.Values, modifications map[string]string
 	}
 	return copied
 }
+
+// TestParamsSafeToLog only exists to ensure that paramsSafeToLog will not be accidentally updated.
+func TestParamsSafeToLog(t *testing.T) {
+	wantParams := []string{
+		"error",
+		"error_description",
+		"error_uri",
+	}
+
+	require.ElementsMatch(t, wantParams, paramsSafeToLog().UnsortedList())
+}

internal/federationdomain/endpoints/login/get_login_handler.go

@@ -9,6 +9,7 @@ import (
 	"go.pinniped.dev/internal/federationdomain/endpoints/login/loginhtml"
 	"go.pinniped.dev/internal/federationdomain/endpoints/loginurl"
 	"go.pinniped.dev/internal/federationdomain/oidc"
+	"go.pinniped.dev/internal/federationdomain/stateparam"
 )
 
 const (
@@ -17,12 +18,12 @@ const (
 )
 
 func NewGetHandler(loginPath string) HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request, encodedState string, decodedState *oidc.UpstreamStateParamData) error {
+	return func(w http.ResponseWriter, r *http.Request, encodedState stateparam.Encoded, decodedState *oidc.UpstreamStateParamData) error {
 		alertMessage, hasAlert := getAlert(r)
 
 		pageInputs := &loginhtml.PageData{
 			PostPath:      loginPath,
-			State:         encodedState,
+			State:         encodedState.String(),
 			IDPName:       decodedState.UpstreamName,
 			HasAlertError: hasAlert,
 			AlertMessage:  alertMessage,

internal/federationdomain/endpoints/login/get_login_handler_test.go

@@ -1,4 +1,4 @@
-// Copyright 2022-2023 the Pinniped contributors. All Rights Reserved.
+// Copyright 2022-2024 the Pinniped contributors. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
 package login
@@ -13,6 +13,7 @@ import (
 	"go.pinniped.dev/internal/federationdomain/endpoints/login/loginhtml"
 	"go.pinniped.dev/internal/federationdomain/idplister"
 	"go.pinniped.dev/internal/federationdomain/oidc"
+	"go.pinniped.dev/internal/federationdomain/stateparam"
 	"go.pinniped.dev/internal/testutil"
 )
 
@@ -27,7 +28,7 @@ func TestGetLogin(t *testing.T) {
 	tests := []struct {
 		name            string
 		decodedState    *oidc.UpstreamStateParamData
-		encodedState    string
+		encodedState    stateparam.Encoded
 		errParam        string
 		idps            idplister.UpstreamIdentityProvidersLister
 		wantStatus      int
@@ -46,13 +47,13 @@ func TestGetLogin(t *testing.T) {
 			wantBody:        testutil.ExpectedLoginPageHTML(loginhtml.CSS(), testUpstreamName, testPath, testEncodedState, ""), // no alert message
 		},
 		{
-			name: "displays error banner when err=login_error param is sent",
+			name: "displays error banner when err=incorrect_username_or_password param is sent",
 			decodedState: &oidc.UpstreamStateParamData{
 				UpstreamName: testUpstreamName,
 				UpstreamType: testUpstreamType,
 			},
 			encodedState:    testEncodedState,
-			errParam:        "login_error",
+			errParam:        "incorrect_username_or_password",
 			wantStatus:      http.StatusOK,
 			wantContentType: htmlContentType,
 			wantBody: testutil.ExpectedLoginPageHTML(loginhtml.CSS(), testUpstreamName, testPath, testEncodedState,
@@ -98,7 +99,7 @@ func TestGetLogin(t *testing.T) {
 			t.Parallel()
 
 			handler := NewGetHandler(testPath)
-			target := testPath + "?state=" + tt.encodedState
+			target := testPath + "?state=" + tt.encodedState.String()
 			if tt.errParam != "" {
 				target += "&err=" + tt.errParam
 			}

internal/federationdomain/endpoints/login/login_handler.go

@@ -6,10 +6,14 @@ package login
 import (
 	"net/http"
 
+	"k8s.io/apimachinery/pkg/util/sets"
+
 	idpdiscoveryv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/idpdiscovery/v1alpha1"
+	"go.pinniped.dev/internal/auditevent"
 	"go.pinniped.dev/internal/federationdomain/endpoints/login/loginhtml"
 	"go.pinniped.dev/internal/federationdomain/formposthtml"
 	"go.pinniped.dev/internal/federationdomain/oidc"
+	"go.pinniped.dev/internal/federationdomain/stateparam"
 	"go.pinniped.dev/internal/httputil/httperr"
 	"go.pinniped.dev/internal/httputil/securityheader"
 	"go.pinniped.dev/internal/plog"
@@ -19,10 +23,17 @@ import (
 type HandlerFunc func(
 	w http.ResponseWriter,
 	r *http.Request,
-	encodedState string,
+	encodedState stateparam.Encoded,
 	decodedState *oidc.UpstreamStateParamData,
 ) error
 
+func paramsSafeToLog() sets.Set[string] {
+	return sets.New[string](
+		// This param is sometimes added by the POST login handler when redirecting back to the GET login handler.
+		"err",
+	)
+}
+
 // NewHandler returns a http.Handler that serves the login endpoint for IDPs that don't have their own web UI for login.
 //
 // This handler takes care of the shared concerns between the GET and POST methods of the login endpoint:
@@ -38,8 +49,14 @@ func NewHandler(
 	cookieDecoder oidc.Decoder,
 	getHandler HandlerFunc, // use NewGetHandler() for production
 	postHandler HandlerFunc, // use NewPostHandler() for production
+	auditLogger plog.AuditLogger,
 ) http.Handler {
 	loginHandler := httperr.HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
+		if err := auditLogger.AuditRequestParams(r, paramsSafeToLog()); err != nil {
+			plog.DebugErr("error parsing callback request params", err)
+			return httperr.New(http.StatusBadRequest, "error parsing request params")
+		}
+
 		var handler HandlerFunc
 		switch r.Method {
 		case http.MethodGet:
@@ -56,6 +73,11 @@ func NewHandler(
 			return err
 		}
 
+		auditLogger.Audit(auditevent.AuthorizeIDFromParameters, &plog.AuditParams{
+			ReqCtx:        r.Context(),
+			KeysAndValues: []any{"authorizeID", encodedState.AuthorizeID()},
+		})
+
 		switch decodedState.UpstreamType {
 		case string(idpdiscoveryv1alpha1.IDPTypeLDAP), string(idpdiscoveryv1alpha1.IDPTypeActiveDirectory):
 			// these are the types supported by this endpoint, so no error here

internal/federationdomain/endpoints/login/login_handler_test.go

@@ -1,4 +1,4 @@
-// Copyright 2022-2023 the Pinniped contributors. All Rights Reserved.
+// Copyright 2022-2024 the Pinniped contributors. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
 package login
@@ -13,8 +13,12 @@ import (
 	"github.com/gorilla/securecookie"
 	"github.com/stretchr/testify/require"
 
+	"go.pinniped.dev/internal/auditid"
+	"go.pinniped.dev/internal/federationdomain/endpoints/loginurl"
 	"go.pinniped.dev/internal/federationdomain/oidc"
+	"go.pinniped.dev/internal/federationdomain/stateparam"
 	"go.pinniped.dev/internal/httputil/httperr"
+	"go.pinniped.dev/internal/plog"
 	"go.pinniped.dev/internal/testutil"
 	"go.pinniped.dev/internal/testutil/oidctestutil"
 )
@@ -118,8 +122,9 @@ func TestLoginEndpoint(t *testing.T) {
 		wantStatus       int
 		wantContentType  string
 		wantBody         string
-		wantEncodedState string
+		wantEncodedState stateparam.Encoded
 		wantDecodedState *oidc.UpstreamStateParamData
+		wantAuditLogs    func(encodedStateParam stateparam.Encoded) []testutil.WantedAuditLog
 	}{
 		{
 			name:            "PUT method is invalid",
@@ -129,6 +134,13 @@ func TestLoginEndpoint(t *testing.T) {
 			wantStatus:      http.StatusMethodNotAllowed,
 			wantContentType: htmlContentType,
 			wantBody:        "Method Not Allowed: PUT (try GET or POST)\n",
+			wantAuditLogs: func(encodedStateParam stateparam.Encoded) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+						"params": map[string]any{"state": "redacted"},
+					}),
+				}
+			},
 		},
 		{
 			name:            "PATCH method is invalid",
@@ -192,6 +204,13 @@ func TestLoginEndpoint(t *testing.T) {
 			wantStatus:      http.StatusBadRequest,
 			wantContentType: htmlContentType,
 			wantBody:        "Bad Request: state param not found\n",
+			wantAuditLogs: func(encodedStateParam stateparam.Encoded) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+						"params": map[string]any{},
+					}),
+				}
+			},
 		},
 		{
 			name:            "state param was not included on POST request",
@@ -285,6 +304,17 @@ func TestLoginEndpoint(t *testing.T) {
 			wantContentType: htmlContentType,
 			wantBody:        "Bad Request: not a supported upstream IDP type for this endpoint: \"oidc\"\n",
 		},
+		{
+			name:   "GET request with invalid form",
+			method: http.MethodGet,
+			path: newRequestPath().WithState(
+				happyUpstreamStateParam().WithUpstreamIDPType("oidc").Build(t, happyStateCodec),
+			).String() + "&invalid;;param",
+			csrfCookie:      happyCSRFCookie,
+			wantStatus:      http.StatusBadRequest,
+			wantContentType: htmlContentType,
+			wantBody:        "Bad Request: error parsing request params\n",
+		},
 		{
 			name:   "POST request when upstream IDP type in state param is not supported by this endpoint",
 			method: http.MethodPost,
@@ -320,6 +350,27 @@ func TestLoginEndpoint(t *testing.T) {
 			wantEncodedState: happyState,
 			wantDecodedState: expectedHappyDecodedUpstreamStateParam(),
 		},
+		{
+			name:             "happy GET request with err param which can be set by the real POST handler on redirects back to the GET handler",
+			method:           http.MethodGet,
+			path:             happyPathWithState + "&" + loginurl.ErrParamName + "=" + string(loginurl.ShowBadUserPassErr),
+			csrfCookie:       happyCSRFCookie,
+			wantStatus:       http.StatusOK,
+			wantContentType:  htmlContentType,
+			wantBody:         happyGetResult,
+			wantEncodedState: happyState,
+			wantDecodedState: expectedHappyDecodedUpstreamStateParam(),
+			wantAuditLogs: func(encodedStateParam stateparam.Encoded) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+						"params": map[string]any{"state": "redacted", "err": "incorrect_username_or_password"},
+					}),
+					testutil.WantAuditLog("AuthorizeID From Parameters", map[string]any{
+						"authorizeID": encodedStateParam.AuthorizeID(),
+					}),
+				}
+			},
+		},
 		{
 			name:             "happy GET request for LDAP upstream",
 			method:           http.MethodGet,
@@ -330,6 +381,16 @@ func TestLoginEndpoint(t *testing.T) {
 			wantBody:         happyGetResult,
 			wantEncodedState: happyState,
 			wantDecodedState: expectedHappyDecodedUpstreamStateParam(),
+			wantAuditLogs: func(encodedStateParam stateparam.Encoded) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+						"params": map[string]any{"state": "redacted"},
+					}),
+					testutil.WantAuditLog("AuthorizeID From Parameters", map[string]any{
+						"authorizeID": encodedStateParam.AuthorizeID(),
+					}),
+				}
+			},
 		},
 		{
 			name:             "happy POST request for LDAP upstream",
@@ -341,6 +402,16 @@ func TestLoginEndpoint(t *testing.T) {
 			wantBody:         happyPostResult,
 			wantEncodedState: happyState,
 			wantDecodedState: expectedHappyDecodedUpstreamStateParam(),
+			wantAuditLogs: func(encodedStateParam stateparam.Encoded) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+						"params": map[string]any{"state": "redacted"},
+					}),
+					testutil.WantAuditLog("AuthorizeID From Parameters", map[string]any{
+						"authorizeID": encodedStateParam.AuthorizeID(),
+					}),
+				}
+			},
 		},
 		{
 			name:             "happy GET request for ActiveDirectory upstream",
@@ -352,6 +423,16 @@ func TestLoginEndpoint(t *testing.T) {
 			wantBody:         happyGetResult,
 			wantEncodedState: happyActiveDirectoryState,
 			wantDecodedState: expectedHappyDecodedUpstreamStateParamForActiveDirectory(),
+			wantAuditLogs: func(encodedStateParam stateparam.Encoded) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+						"params": map[string]any{"state": "redacted"},
+					}),
+					testutil.WantAuditLog("AuthorizeID From Parameters", map[string]any{
+						"authorizeID": encodedStateParam.AuthorizeID(),
+					}),
+				}
+			},
 		},
 		{
 			name:             "happy POST request for ActiveDirectory upstream",
@@ -363,81 +444,98 @@ func TestLoginEndpoint(t *testing.T) {
 			wantBody:         happyPostResult,
 			wantEncodedState: happyActiveDirectoryState,
 			wantDecodedState: expectedHappyDecodedUpstreamStateParamForActiveDirectory(),
+			wantAuditLogs: func(encodedStateParam stateparam.Encoded) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+						"params": map[string]any{"state": "redacted"},
+					}),
+					testutil.WantAuditLog("AuthorizeID From Parameters", map[string]any{
+						"authorizeID": encodedStateParam.AuthorizeID(),
+					}),
+				}
+			},
 		},
 	}
 
 	for _, test := range tests {
-		tt := test
-
-		t.Run(tt.name, func(t *testing.T) {
+		t.Run(test.name, func(t *testing.T) {
 			t.Parallel()
 
-			req := httptest.NewRequest(tt.method, tt.path, nil)
-			if tt.csrfCookie != "" {
-				req.Header.Set("Cookie", tt.csrfCookie)
+			req := httptest.NewRequest(test.method, test.path, nil)
+			if test.csrfCookie != "" {
+				req.Header.Set("Cookie", test.csrfCookie)
 			}
+			req, _ = auditid.NewRequestWithAuditID(req, func() string { return "fake-audit-id" })
 			rsp := httptest.NewRecorder()
 
 			testGetHandler := func(
 				w http.ResponseWriter,
 				r *http.Request,
-				encodedState string,
+				encodedState stateparam.Encoded,
 				decodedState *oidc.UpstreamStateParamData,
 			) error {
 				require.Equal(t, req, r)
 				require.Equal(t, rsp, w)
-				require.Equal(t, tt.wantEncodedState, encodedState)
-				require.Equal(t, tt.wantDecodedState, decodedState)
-				if tt.getHandlerErr == nil {
+				require.Equal(t, test.wantEncodedState, encodedState)
+				require.Equal(t, test.wantDecodedState, decodedState)
+				if test.getHandlerErr == nil {
 					_, err := w.Write([]byte(happyGetResult))
 					require.NoError(t, err)
 				}
-				return tt.getHandlerErr
+				return test.getHandlerErr
 			}
 
 			testPostHandler := func(
 				w http.ResponseWriter,
 				r *http.Request,
-				encodedState string,
+				encodedState stateparam.Encoded,
 				decodedState *oidc.UpstreamStateParamData,
 			) error {
 				require.Equal(t, req, r)
 				require.Equal(t, rsp, w)
-				require.Equal(t, tt.wantEncodedState, encodedState)
-				require.Equal(t, tt.wantDecodedState, decodedState)
-				if tt.postHandlerErr == nil {
+				require.Equal(t, test.wantEncodedState, encodedState)
+				require.Equal(t, test.wantDecodedState, decodedState)
+				if test.postHandlerErr == nil {
 					_, err := w.Write([]byte(happyPostResult))
 					require.NoError(t, err)
 				}
-				return tt.postHandlerErr
+				return test.postHandlerErr
 			}
 
-			subject := NewHandler(happyStateCodec, happyCookieCodec, testGetHandler, testPostHandler)
+			auditLogger, actualAuditLog := plog.TestAuditLogger(t)
+
+			subject := NewHandler(happyStateCodec, happyCookieCodec, testGetHandler, testPostHandler, auditLogger)
 
 			subject.ServeHTTP(rsp, req)
 
-			if tt.method == http.MethodPost {
+			if test.method == http.MethodPost {
 				testutil.RequireSecurityHeadersWithFormPostPageCSPs(t, rsp)
 			} else {
 				testutil.RequireSecurityHeadersWithLoginPageCSPs(t, rsp)
 			}
 
-			require.Equal(t, tt.wantStatus, rsp.Code)
-			testutil.RequireEqualContentType(t, rsp.Header().Get("Content-Type"), tt.wantContentType)
-			require.Equal(t, tt.wantBody, rsp.Body.String())
+			require.Equal(t, test.wantStatus, rsp.Code)
+			testutil.RequireEqualContentType(t, rsp.Header().Get("Content-Type"), test.wantContentType)
+			require.Equal(t, test.wantBody, rsp.Body.String())
+
+			if test.wantAuditLogs != nil {
+				wantAuditLogs := test.wantAuditLogs(testutil.GetStateParam(t, test.path))
+				testutil.WantAuditIDOnEveryAuditLog(wantAuditLogs, "fake-audit-id")
+				testutil.CompareAuditLogs(t, wantAuditLogs, actualAuditLog.String())
+			}
 		})
 	}
 }
 
 type requestPath struct {
-	state *string
+	state *stateparam.Encoded
 }
 
 func newRequestPath() *requestPath {
 	return &requestPath{}
 }
 
-func (r *requestPath) WithState(state string) *requestPath {
+func (r *requestPath) WithState(state stateparam.Encoded) *requestPath {
 	r.state = &state
 	return r
 }
@@ -451,7 +549,16 @@ func (r *requestPath) String() string {
 	path := "/login?"
 	params := url.Values{}
 	if r.state != nil {
-		params.Add("state", *r.state)
+		params.Add("state", r.state.String())
 	}
 	return path + params.Encode()
 }
+
+// TestParamsSafeToLog only exists to ensure that paramsSafeToLog will not be accidentally updated.
+func TestParamsSafeToLog(t *testing.T) {
+	wantParams := []string{
+		"err",
+	}
+
+	require.ElementsMatch(t, wantParams, paramsSafeToLog().UnsortedList())
+}

internal/federationdomain/endpoints/login/post_login_handler.go

@@ -10,17 +10,24 @@ import (
 
 	"github.com/ory/fosite"
 
+	"go.pinniped.dev/internal/auditevent"
 	"go.pinniped.dev/internal/federationdomain/downstreamsession"
 	"go.pinniped.dev/internal/federationdomain/endpoints/loginurl"
 	"go.pinniped.dev/internal/federationdomain/federationdomainproviders"
 	"go.pinniped.dev/internal/federationdomain/oidc"
 	"go.pinniped.dev/internal/federationdomain/resolvedprovider/resolvedldap"
+	"go.pinniped.dev/internal/federationdomain/stateparam"
 	"go.pinniped.dev/internal/httputil/httperr"
 	"go.pinniped.dev/internal/plog"
 )
 
-func NewPostHandler(issuerURL string, upstreamIDPs federationdomainproviders.FederationDomainIdentityProvidersFinderI, oauthHelper fosite.OAuth2Provider) HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request, encodedState string, decodedState *oidc.UpstreamStateParamData) error {
+func NewPostHandler(
+	issuerURL string,
+	upstreamIDPs federationdomainproviders.FederationDomainIdentityProvidersFinderI,
+	oauthHelper fosite.OAuth2Provider,
+	auditLogger plog.AuditLogger,
+) HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request, encodedState stateparam.Encoded, decodedState *oidc.UpstreamStateParamData) error {
 		// Note that the login handler prevents this handler from being called with OIDC upstreams.
 		idp, err := upstreamIDPs.FindUpstreamIDPByDisplayName(decodedState.UpstreamName)
 		if err != nil {
@@ -30,6 +37,16 @@ func NewPostHandler(issuerURL string, upstreamIDPs federationdomainproviders.Fed
 			return httperr.Wrap(http.StatusUnprocessableEntity, "error finding upstream provider", err)
 		}
 
+		auditLogger.Audit(auditevent.UsingUpstreamIDP, &plog.AuditParams{
+			ReqCtx: r.Context(),
+			KeysAndValues: []any{
+				"displayName", idp.GetDisplayName(),
+				"resourceName", idp.GetProvider().GetResourceName(),
+				"resourceUID", idp.GetProvider().GetResourceUID(),
+				"type", idp.GetSessionProviderType(),
+			},
+		})
+
 		// Get the original params that were used at the authorization endpoint.
 		downstreamAuthParams, err := url.ParseQuery(decodedState.AuthParams)
 		if err != nil {
@@ -74,6 +91,10 @@ func NewPostHandler(issuerURL string, upstreamIDPs federationdomainproviders.Fed
 				// The user may try to log in again if they'd like, so redirect back to the login page with an error.
 				return redirectToLoginPage(r, w, issuerURL, encodedState, loginurl.ShowInternalError)
 			case err == resolvedldap.ErrAccessDeniedDueToUsernamePasswordNotAccepted:
+				auditLogger.Audit(auditevent.IncorrectUsernameOrPassword, &plog.AuditParams{
+					ReqCtx: r.Context(),
+				})
+
 				// The upstream did not accept the username/password combination.
 				// The user may try to log in again if they'd like, so redirect back to the login page with an error.
 				return redirectToLoginPage(r, w, issuerURL, encodedState, loginurl.ShowBadUserPassErr)
@@ -84,11 +105,13 @@ func NewPostHandler(issuerURL string, upstreamIDPs federationdomainproviders.Fed
 			}
 		}
 
-		session, err := downstreamsession.NewPinnipedSession(r.Context(), idp, &downstreamsession.SessionConfig{
+		session, err := downstreamsession.NewPinnipedSession(r.Context(), auditLogger, &downstreamsession.SessionConfig{
 			UpstreamIdentity:    identity,
 			UpstreamLoginExtras: loginExtras,
 			ClientID:            authorizeRequester.GetClient().GetID(),
 			GrantedScopes:       authorizeRequester.GetGrantedScopes(),
+			IdentityProvider:    idp,
+			SessionIDGetter:     authorizeRequester,
 		})
 		if err != nil {
 			err = fosite.ErrAccessDenied.WithHintf("Reason: %s.", err.Error())
@@ -107,7 +130,7 @@ func redirectToLoginPage(
 	r *http.Request,
 	w http.ResponseWriter,
 	downstreamIssuer string,
-	encodedStateParamValue string,
+	encodedStateParamValue stateparam.Encoded,
 	errToDisplay loginurl.ErrorParamValue,
 ) error {
 	loginURL, err := loginurl.URL(downstreamIssuer, encodedStateParamValue, errToDisplay)

internal/federationdomain/endpoints/login/post_login_handler_test.go

@@ -19,12 +19,14 @@ import (
 
 	supervisorconfigv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/config/v1alpha1"
 	supervisorfake "go.pinniped.dev/generated/latest/client/supervisor/clientset/versioned/fake"
+	"go.pinniped.dev/internal/auditid"
 	"go.pinniped.dev/internal/authenticators"
 	"go.pinniped.dev/internal/celtransformer"
 	"go.pinniped.dev/internal/federationdomain/endpoints/jwks"
 	"go.pinniped.dev/internal/federationdomain/oidc"
 	"go.pinniped.dev/internal/federationdomain/oidcclientvalidator"
 	"go.pinniped.dev/internal/federationdomain/storage"
+	"go.pinniped.dev/internal/plog"
 	"go.pinniped.dev/internal/psession"
 	"go.pinniped.dev/internal/testutil"
 	"go.pinniped.dev/internal/testutil/oidctestutil"
@@ -62,7 +64,7 @@ func TestPostLoginEndpoint(t *testing.T) {
 
 		userParam                = "username"
 		passParam                = "password"
-		badUserPassErrParamValue = "login_error"
+		badUserPassErrParamValue = "incorrect_username_or_password"
 		internalErrParamValue    = "internal_error"
 
 		transformationUsernamePrefix = "username_prefix:"
@@ -289,6 +291,10 @@ func TestPostLoginEndpoint(t *testing.T) {
 	prefixUsernameAndGroupsPipeline := transformtestutil.NewPrefixingPipeline(t, transformationUsernamePrefix, transformationGroupsPrefix)
 	rejectAuthPipeline := transformtestutil.NewRejectAllAuthPipeline(t)
 
+	noAuditLogsWanted := func(_ string) []testutil.WantedAuditLog {
+		return nil
+	}
+
 	tests := []struct {
 		name          string
 		idps          *testidplister.UpstreamIDPListerBuilder
@@ -328,6 +334,7 @@ func TestPostLoginEndpoint(t *testing.T) {
 		// is stored, so it is possible with an LDAP upstream to store objects and then return an error to
 		// the client anyway (which makes the stored objects useless, but oh well).
 		wantUnnecessaryStoredRecords int
+		wantAuditLogs                func(sessionID string) []testutil.WantedAuditLog
 	}{
 		{
 			name: "happy LDAP login",
@@ -351,6 +358,36 @@ func TestPostLoginEndpoint(t *testing.T) {
 			wantDownstreamPKCEChallenge:       downstreamPKCEChallenge,
 			wantDownstreamPKCEChallengeMethod: downstreamPKCEChallengeMethod,
 			wantDownstreamCustomSessionData:   expectedHappyLDAPUpstreamCustomSession,
+			wantAuditLogs: func(sessionID string) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("Using Upstream IDP", map[string]any{
+						"displayName":  "some-ldap-idp",
+						"resourceName": "some-ldap-idp",
+						"resourceUID":  "ldap-resource-uid",
+						"type":         "ldap",
+					}),
+					testutil.WantAuditLog("Identity From Upstream IDP", map[string]any{
+						"upstreamIDPDisplayName":  "some-ldap-idp",
+						"upstreamIDPType":         "ldap",
+						"upstreamIDPResourceName": "some-ldap-idp",
+						"upstreamIDPResourceUID":  "ldap-resource-uid",
+						"personalInfo": map[string]any{
+							"upstreamUsername": "some-mapped-ldap-username",
+							"upstreamGroups":   []any{"group1", "group2", "group3"},
+						},
+					}),
+					testutil.WantAuditLog("Session Started", map[string]any{
+						"sessionID": sessionID,
+						"warnings":  []any{}, // json: []
+						"personalInfo": map[string]any{
+							"username":         "some-mapped-ldap-username",
+							"groups":           []any{"group1", "group2", "group3"},
+							"subject":          "ldaps://some-ldap-host:123?base=ou%3Dusers%2Cdc%3Dpinniped%2Cdc%3Ddev&idpName=some-ldap-idp&sub=some-ldap-uid",
+							"additionalClaims": map[string]any{}, // json: {}
+						},
+					}),
+				}
+			},
 		},
 		{
 			name: "happy LDAP login with identity transformations which modify the username and group names",
@@ -379,6 +416,36 @@ func TestPostLoginEndpoint(t *testing.T) {
 				happyLDAPUsernameFromAuthenticator,
 				happyLDAPGroups,
 			),
+			wantAuditLogs: func(sessionID string) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("Using Upstream IDP", map[string]any{
+						"displayName":  "some-ldap-idp",
+						"resourceName": "some-ldap-idp",
+						"resourceUID":  "ldap-resource-uid",
+						"type":         "ldap",
+					}),
+					testutil.WantAuditLog("Identity From Upstream IDP", map[string]any{
+						"upstreamIDPDisplayName":  "some-ldap-idp",
+						"upstreamIDPType":         "ldap",
+						"upstreamIDPResourceName": "some-ldap-idp",
+						"upstreamIDPResourceUID":  "ldap-resource-uid",
+						"personalInfo": map[string]any{
+							"upstreamUsername": "some-mapped-ldap-username",
+							"upstreamGroups":   []any{"group1", "group2", "group3"},
+						},
+					}),
+					testutil.WantAuditLog("Session Started", map[string]any{
+						"sessionID": sessionID,
+						"warnings":  []any{}, // json: []
+						"personalInfo": map[string]any{
+							"username":         "username_prefix:some-mapped-ldap-username",
+							"groups":           []any{"groups_prefix:group1", "groups_prefix:group2", "groups_prefix:group3"},
+							"subject":          "ldaps://some-ldap-host:123?base=ou%3Dusers%2Cdc%3Dpinniped%2Cdc%3Ddev&idpName=some-ldap-idp&sub=some-ldap-uid",
+							"additionalClaims": map[string]any{}, // json: {}
+						},
+					}),
+				}
+			},
 		},
 		{
 			name: "happy LDAP login with dynamic client",
@@ -426,6 +493,36 @@ func TestPostLoginEndpoint(t *testing.T) {
 			wantDownstreamPKCEChallenge:       downstreamPKCEChallenge,
 			wantDownstreamPKCEChallengeMethod: downstreamPKCEChallengeMethod,
 			wantDownstreamCustomSessionData:   expectedHappyActiveDirectoryUpstreamCustomSession,
+			wantAuditLogs: func(sessionID string) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("Using Upstream IDP", map[string]any{
+						"displayName":  "some-active-directory-idp",
+						"resourceName": "some-active-directory-idp",
+						"resourceUID":  "active-directory-resource-uid",
+						"type":         "activedirectory",
+					}),
+					testutil.WantAuditLog("Identity From Upstream IDP", map[string]any{
+						"upstreamIDPDisplayName":  "some-active-directory-idp",
+						"upstreamIDPType":         "activedirectory",
+						"upstreamIDPResourceName": "some-active-directory-idp",
+						"upstreamIDPResourceUID":  "active-directory-resource-uid",
+						"personalInfo": map[string]any{
+							"upstreamUsername": "some-mapped-ldap-username",
+							"upstreamGroups":   []any{"group1", "group2", "group3"},
+						},
+					}),
+					testutil.WantAuditLog("Session Started", map[string]any{
+						"sessionID": sessionID,
+						"warnings":  []any{}, // json: []
+						"personalInfo": map[string]any{
+							"username":         "some-mapped-ldap-username",
+							"groups":           []any{"group1", "group2", "group3"},
+							"subject":          "ldaps://some-ldap-host:123?base=ou%3Dusers%2Cdc%3Dpinniped%2Cdc%3Ddev&idpName=some-active-directory-idp&sub=some-ldap-uid",
+							"additionalClaims": map[string]any{}, // json: {}
+						},
+					}),
+				}
+			},
 		},
 		{
 			name: "happy AD login with identity transformations which modify the username and group names",
@@ -712,6 +809,29 @@ func TestPostLoginEndpoint(t *testing.T) {
 				"error_description": "The resource owner or authorization server denied the request. Reason: configured identity policy rejected this authentication: users who belong to certain upstream group are not allowed.",
 				"state":             happyDownstreamState,
 			}),
+			wantAuditLogs: func(sessionID string) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("Using Upstream IDP", map[string]any{
+						"displayName":  "some-ldap-idp",
+						"resourceName": "some-ldap-idp",
+						"resourceUID":  "ldap-resource-uid",
+						"type":         "ldap",
+					}),
+					testutil.WantAuditLog("Identity From Upstream IDP", map[string]any{
+						"upstreamIDPDisplayName":  "some-ldap-idp",
+						"upstreamIDPType":         "ldap",
+						"upstreamIDPResourceName": "some-ldap-idp",
+						"upstreamIDPResourceUID":  "ldap-resource-uid",
+						"personalInfo": map[string]any{
+							"upstreamUsername": "some-mapped-ldap-username",
+							"upstreamGroups":   []any{"group1", "group2", "group3"},
+						},
+					}),
+					testutil.WantAuditLog("Authentication Rejected By Transforms", map[string]any{
+						"reason": "configured identity policy rejected this authentication: users who belong to certain upstream group are not allowed",
+					}),
+				}
+			},
 		},
 		{
 			name: "happy LDAP when downstream OIDC validations are skipped because the openid scope was not requested",
@@ -780,6 +900,17 @@ func TestPostLoginEndpoint(t *testing.T) {
 			wantContentType:              htmlContentType,
 			wantBodyString:               "",
 			wantRedirectToLoginPageError: badUserPassErrParamValue,
+			wantAuditLogs: func(sessionID string) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("Using Upstream IDP", map[string]any{
+						"displayName":  "some-ldap-idp",
+						"resourceName": "some-ldap-idp",
+						"resourceUID":  "ldap-resource-uid",
+						"type":         "ldap",
+					}),
+					testutil.WantAuditLog("Incorrect Username Or Password", map[string]any{}),
+				}
+			},
 		},
 		{
 			name:                         "bad password LDAP login",
@@ -790,6 +921,17 @@ func TestPostLoginEndpoint(t *testing.T) {
 			wantContentType:              htmlContentType,
 			wantBodyString:               "",
 			wantRedirectToLoginPageError: badUserPassErrParamValue,
+			wantAuditLogs: func(sessionID string) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("Using Upstream IDP", map[string]any{
+						"displayName":  "some-ldap-idp",
+						"resourceName": "some-ldap-idp",
+						"resourceUID":  "ldap-resource-uid",
+						"type":         "ldap",
+					}),
+					testutil.WantAuditLog("Incorrect Username Or Password", map[string]any{}),
+				}
+			},
 		},
 		{
 			name:                         "blank username LDAP login",
@@ -830,6 +972,16 @@ func TestPostLoginEndpoint(t *testing.T) {
 			wantContentType:              htmlContentType,
 			wantBodyString:               "",
 			wantRedirectToLoginPageError: internalErrParamValue,
+			wantAuditLogs: func(sessionID string) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("Using Upstream IDP", map[string]any{
+						"displayName":  "some-ldap-idp",
+						"resourceName": "some-ldap-idp",
+						"resourceUID":  "ldap-resource-uid",
+						"type":         "ldap",
+					}),
+				}
+			},
 		},
 		{
 			name: "downstream redirect uri does not match what is configured for client",
@@ -841,6 +993,30 @@ func TestPostLoginEndpoint(t *testing.T) {
 			}),
 			formParams: happyUsernamePasswordFormParams,
 			wantErr:    "error using state downstream auth params",
+			wantAuditLogs: func(sessionID string) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("Identity From Upstream IDP", map[string]any{
+						"upstreamIDPDisplayName":  "some-ldap-idp",
+						"upstreamIDPType":         "ldap",
+						"upstreamIDPResourceName": "some-ldap-idp",
+						"upstreamIDPResourceUID":  "ldap-resource-uid",
+						"personalInfo": map[string]any{
+							"upstreamUsername": "some-mapped-ldap-username",
+							"upstreamGroups":   []any{"group1", "group2", "group3"},
+						},
+					}),
+					testutil.WantAuditLog("Session Started", map[string]any{
+						"sessionID": sessionID,
+						"warnings":  []any{}, // json: []
+						"personalInfo": map[string]any{
+							"username":         "some-mapped-ldap-username",
+							"groups":           []any{"group1", "group2", "group3"},
+							"subject":          "ldaps://some-ldap-host:123?base=ou%3Dusers%2Cdc%3Dpinniped%2Cdc%3Ddev&idpName=some-ldap-idp&sub=some-ldap-uid",
+							"additionalClaims": map[string]any{}, // json: {}
+						},
+					}),
+				}
+			},
 		},
 		{
 			name:          "downstream redirect uri does not match what is configured for client with dynamic client",
@@ -1042,8 +1218,9 @@ func TestPostLoginEndpoint(t *testing.T) {
 					map[string]string{"scope": "openid offline_access pinniped:request-audience scope_not_allowed"},
 				).Encode()
 			}),
-			formParams: happyUsernamePasswordFormParams,
-			wantErr:    "error using state downstream auth params",
+			formParams:    happyUsernamePasswordFormParams,
+			wantErr:       "error using state downstream auth params",
+			wantAuditLogs: noAuditLogsWanted,
 		},
 		{
 			name: "using dynamic client which is not allowed to request username scope in authorize request but requests it anyway",
@@ -1143,10 +1320,13 @@ func TestPostLoginEndpoint(t *testing.T) {
 			if tt.reqURIQuery != nil {
 				req.URL.RawQuery = tt.reqURIQuery.Encode()
 			}
+			req, _ = auditid.NewRequestWithAuditID(req, func() string { return "some-audit-id" })
 
 			rsp := httptest.NewRecorder()
 
-			subject := NewPostHandler(downstreamIssuer, tt.idps.BuildFederationDomainIdentityProvidersListerFinder(), oauthHelper)
+			auditLogger, actualAuditLog := plog.TestAuditLogger(t)
+
+			subject := NewPostHandler(downstreamIssuer, tt.idps.BuildFederationDomainIdentityProvidersListerFinder(), oauthHelper, auditLogger)
 
 			err := subject(rsp, req, happyEncodedUpstreamState, tt.decodedState)
 			if tt.wantErr != "" {
@@ -1162,12 +1342,13 @@ func TestPostLoginEndpoint(t *testing.T) {
 
 			actualLocation := rsp.Header().Get("Location")
 
+			var sessionID string
 			switch {
 			case tt.wantRedirectLocationRegexp != "":
 				// Expecting a success redirect to the client.
 				require.Equal(t, tt.wantBodyString, rsp.Body.String())
 				require.Len(t, rsp.Header().Values("Location"), 1)
-				oidctestutil.RequireAuthCodeRegexpMatch(
+				sessionID = oidctestutil.RequireAuthCodeRegexpMatch(
 					t,
 					actualLocation,
 					tt.wantRedirectLocationRegexp,
@@ -1203,7 +1384,7 @@ func TestPostLoginEndpoint(t *testing.T) {
 				// Expecting the body of the response to be a html page with a form (for "response_mode=form_post").
 				_, hasLocationHeader := rsp.Header()["Location"]
 				require.False(t, hasLocationHeader)
-				oidctestutil.RequireAuthCodeRegexpMatch(
+				sessionID = oidctestutil.RequireAuthCodeRegexpMatch(
 					t,
 					rsp.Body.String(),
 					tt.wantBodyFormResponseRegexp,
@@ -1227,6 +1408,12 @@ func TestPostLoginEndpoint(t *testing.T) {
 				require.Failf(t, "test should have expected a redirect or form body",
 					"actual location was %q", actualLocation)
 			}
+
+			if test.wantAuditLogs != nil {
+				wantAuditLogs := test.wantAuditLogs(sessionID)
+				testutil.WantAuditIDOnEveryAuditLog(wantAuditLogs, "some-audit-id")
+				testutil.CompareAuditLogs(t, wantAuditLogs, actualAuditLog.String())
+			}
 		})
 	}
 }

internal/federationdomain/endpoints/loginurl/login_url.go

@@ -7,6 +7,7 @@ import (
 	"net/url"
 
 	"go.pinniped.dev/internal/federationdomain/oidc"
+	"go.pinniped.dev/internal/federationdomain/stateparam"
 )
 
 const (
@@ -17,7 +18,7 @@ const (
 
 	ShowNoError        ErrorParamValue = ""
 	ShowInternalError  ErrorParamValue = "internal_error"
-	ShowBadUserPassErr ErrorParamValue = "login_error"
+	ShowBadUserPassErr ErrorParamValue = "incorrect_username_or_password"
 )
 
 type ErrorParamValue string
@@ -27,7 +28,7 @@ type ErrorParamValue string
 // provider.FederationDomainIssuer when the issuer string comes from that type.
 func URL(
 	downstreamIssuer string,
-	encodedStateParamValue string,
+	encodedStateParamValue stateparam.Encoded,
 	errToDisplay ErrorParamValue,
 ) (string, error) {
 	loginURL, err := url.Parse(downstreamIssuer + oidc.PinnipedLoginPath)
@@ -36,7 +37,7 @@ func URL(
 	}
 
 	q := loginURL.Query()
-	q.Set(StateParamName, encodedStateParamValue)
+	q.Set(StateParamName, encodedStateParamValue.String())
 	if errToDisplay != ShowNoError {
 		q.Set(ErrParamName, string(errToDisplay))
 	}

internal/federationdomain/endpoints/token/token_handler.go

@@ -6,6 +6,7 @@ package token
 
 import (
 	"context"
+	"crypto/sha256"
 	"errors"
 	"fmt"
 	"net/http"
@@ -19,6 +20,7 @@ import (
 	"k8s.io/apiserver/pkg/warning"
 
 	oidcapi "go.pinniped.dev/generated/latest/apis/supervisor/oidc"
+	"go.pinniped.dev/internal/auditevent"
 	"go.pinniped.dev/internal/federationdomain/federationdomainproviders"
 	"go.pinniped.dev/internal/federationdomain/idtokenlifespan"
 	"go.pinniped.dev/internal/federationdomain/oidc"
@@ -30,13 +32,33 @@ import (
 	"go.pinniped.dev/internal/psession"
 )
 
+func paramsSafeToLog() sets.Set[string] {
+	return sets.New(
+		// Standard params from https://openid.net/specs/openid-connect-core-1_0.html for authcode and refresh grants.
+		// Redacting code, client_secret, refresh_token, and PKCE code_verifier params.
+		"grant_type", "client_id", "redirect_uri", "scope",
+		// Token exchange params from https://datatracker.ietf.org/doc/html/rfc8693#section-2.1.
+		// Redact subject_token and actor_token.
+		// We don't allow all of these, but they should be safe to log.
+		// "scope" is already included from the authcode grant.
+		"audience", "resource", "requested_token_type", "actor_token_type", "subject_token_type",
+	)
+}
+
 func NewHandler(
 	idpLister federationdomainproviders.FederationDomainIdentityProvidersListerI,
 	oauthHelper fosite.OAuth2Provider,
 	overrideAccessTokenLifespan timeouts.OverrideLifespan,
 	overrideIDTokenLifespan timeouts.OverrideLifespan,
+	auditLogger plog.AuditLogger,
 ) http.Handler {
 	return httperr.HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
+		if err := auditLogger.AuditRequestParams(r, paramsSafeToLog()); err != nil {
+			oauthHelper.WriteAccessError(r.Context(), w, nil, err)
+			return nil
+		}
+		auditLogBasicAuthClientID(r, auditLogger)
+
 		session := psession.NewPinnipedSession()
 		accessRequest, err := oauthHelper.NewAccessRequest(r.Context(), r, session)
 		if err != nil {
@@ -45,13 +67,19 @@ func NewHandler(
 			return nil
 		}
 
+		// Log sessionID for cross-request correlation purposes.
+		auditLogger.Audit(auditevent.SessionFound, &plog.AuditParams{
+			ReqCtx:  r.Context(),
+			Session: accessRequest,
+		})
+
 		// Check if we are performing a refresh grant.
 		if accessRequest.GetGrantTypes().ExactOne(oidcapi.GrantTypeRefreshToken) {
 			// The above call to NewAccessRequest has loaded the session from storage into the accessRequest variable.
 			// The session, requested scopes, and requested audience from the original authorize request was retrieved
 			// from the Kube storage layer and added to the accessRequest. Additionally, the audience and scopes may
 			// have already been granted on the accessRequest.
-			err = upstreamRefresh(r.Context(), accessRequest, idpLister)
+			err = upstreamRefresh(r.Context(), accessRequest, idpLister, auditLogger)
 			if err != nil {
 				plog.Info("upstream refresh error", oidc.FositeErrorForLog(err)...)
 				oauthHelper.WriteAccessError(r.Context(), w, accessRequest, err)
@@ -88,6 +116,9 @@ func NewHandler(
 			return nil
 		}
 
+		// Allow cross-referencing the token with the Concierge's audit logs.
+		auditLogIDToken(r.Context(), auditLogger, accessRequest, accessResponse)
+
 		oauthHelper.WriteAccessResponse(r.Context(), w, accessRequest, accessResponse)
 
 		return nil
@@ -128,6 +159,7 @@ func upstreamRefresh(
 	ctx context.Context,
 	accessRequest fosite.AccessRequester,
 	idpLister federationdomainproviders.FederationDomainIdentityProvidersListerI,
+	auditLogger plog.AuditLogger,
 ) error {
 	session := accessRequest.GetSession().(*psession.PinnipedSession)
 
@@ -136,6 +168,7 @@ func upstreamRefresh(
 		return errorsx.WithStack(errMissingUpstreamSessionInternalError())
 	}
 	providerName := customSessionData.ProviderName
+	providerType := customSessionData.ProviderType
 	providerUID := customSessionData.ProviderUID
 	if providerUID == "" || providerName == "" {
 		return errorsx.WithStack(errMissingUpstreamSessionInternalError())
@@ -188,6 +221,15 @@ func upstreamRefresh(
 		return err
 	}
 
+	auditLogger.Audit(auditevent.IdentityRefreshedFromUpstreamIDP, &plog.AuditParams{
+		ReqCtx:  ctx,
+		Session: accessRequest,
+		PIIKeysAndValues: []any{
+			"upstreamUsername", refreshedIdentity.UpstreamUsername,
+			"upstreamGroups", refreshedIdentity.UpstreamGroups,
+		},
+	})
+
 	// If the idp wants to update the session with new information from the refresh, then update it.
 	if refreshedIdentity.IDPSpecificSessionData != nil {
 		idp.ApplyIDPSpecificSessionDataToSession(session.Custom, refreshedIdentity.IDPSpecificSessionData)
@@ -203,16 +245,29 @@ func upstreamRefresh(
 		refreshedIdentity.UpstreamGroups = oldUntransformedGroups
 	}
 
-	refreshedTransformedGroups, err := applyIdentityTransformationsDuringRefresh(ctx,
+	refreshedTransformedUsername, refreshedTransformedGroups, fositeErr := applyIdentityTransformationsDuringRefresh(ctx,
 		idp.GetTransforms(),
-		oldTransformedUsername, // this function validates that the old and new transformed usernames match
 		refreshedIdentity.UpstreamUsername,
 		refreshedIdentity.UpstreamGroups,
-		session.Custom.ProviderName,
-		session.Custom.ProviderType,
+		providerName,
+		providerType,
 	)
-	if err != nil {
-		return err
+	if fositeErr != nil {
+		// The HintField is always populated by applyIdentityTransformationsDuringRefresh,
+		// and more descriptive than fositeErr.Error() which is just "error".
+		auditLogger.Audit(auditevent.AuthenticationRejectedByTransforms, &plog.AuditParams{
+			ReqCtx:        ctx,
+			Session:       accessRequest,
+			KeysAndValues: []any{"reason", fositeErr.HintField},
+		})
+		return fositeErr
+	}
+
+	if oldTransformedUsername != refreshedTransformedUsername {
+		return errUpstreamRefreshError().WithHintf(
+			"Upstream refresh failed.").
+			WithTrace(errors.New("username in upstream refresh does not match previous value")).
+			WithDebugf("provider name: %q, provider type: %q", providerName, providerType)
 	}
 
 	if !skipGroups {
@@ -221,6 +276,15 @@ func upstreamRefresh(
 		session.Fosite.Claims.Extra[oidcapi.IDTokenClaimGroups] = refreshedTransformedGroups
 	}
 
+	auditLogger.Audit(auditevent.SessionRefreshed, &plog.AuditParams{
+		ReqCtx:  ctx,
+		Session: accessRequest,
+		PIIKeysAndValues: []any{
+			"username", oldTransformedUsername, // not allowed to change above so must be the same as old
+			"groups", refreshedTransformedGroups,
+			"subject", previousIdentity.DownstreamSubject},
+	})
+
 	return nil
 }
 
@@ -255,38 +319,30 @@ func validateSessionHasUsername(session *psession.PinnipedSession) error {
 }
 
 // applyIdentityTransformationsDuringRefresh is similar to downstreamsession.applyIdentityTransformations
-// but with validation that the username has not changed, and with slightly different error messaging.
+// but with slightly different error messaging.
 func applyIdentityTransformationsDuringRefresh(
 	ctx context.Context,
 	transforms *idtransform.TransformationPipeline,
-	oldTransformedUsername string,
 	upstreamUsername string,
 	upstreamGroups []string,
 	providerName string,
 	providerType psession.ProviderType,
-) ([]string, error) {
+) (string, []string, *fosite.RFC6749Error) {
 	transformationResult, err := transforms.Evaluate(ctx, upstreamUsername, upstreamGroups)
 	if err != nil {
-		return nil, errUpstreamRefreshError().WithHintf(
+		return "", nil, errUpstreamRefreshError().WithHintf(
 			"Upstream refresh error while applying configured identity transformations.").
 			WithTrace(err).
 			WithDebugf("provider name: %q, provider type: %q", providerName, providerType)
 	}
 
 	if !transformationResult.AuthenticationAllowed {
-		return nil, errUpstreamRefreshError().WithHintf(
+		return "", nil, errUpstreamRefreshError().WithHintf(
 			"Upstream refresh rejected by configured identity policy: %s.", transformationResult.RejectedAuthenticationMessage).
 			WithDebugf("provider name: %q, provider type: %q", providerName, providerType)
 	}
 
-	if oldTransformedUsername != transformationResult.Username {
-		return nil, errUpstreamRefreshError().WithHintf(
-			"Upstream refresh failed.").
-			WithTrace(errors.New("username in upstream refresh does not match previous value")).
-			WithDebugf("provider name: %q, provider type: %q", providerName, providerType)
-	}
-
-	return transformationResult.Groups, nil
+	return transformationResult.Username, transformationResult.Groups, nil
 }
 
 func validateAndGetDownstreamGroupsFromSession(session *psession.PinnipedSession) ([]string, error) {
@@ -340,3 +396,50 @@ func diffSortedGroups(oldGroups, newGroups []string) ([]string, []string) {
 	removed := oldGroupsAsSet.Difference(newGroupsAsSet) // groups in oldGroups that are not in newGroups i.e. removed
 	return added.List(), removed.List()
 }
+
+func auditLogBasicAuthClientID(r *http.Request, auditLogger plog.AuditLogger) {
+	// For dynamic clients, the client ID is from basic auth, not from the request parameters.
+	clientIDFromBasicAuth, _, basicAuthUsed := r.BasicAuth()
+	if basicAuthUsed {
+		auditLogger.Audit(auditevent.HTTPRequestBasicAuthUsed, &plog.AuditParams{
+			ReqCtx:        r.Context(),
+			KeysAndValues: []any{"clientID", clientIDFromBasicAuth},
+		})
+	}
+}
+
+func auditLogIDToken(
+	reqCtx context.Context,
+	auditLogger plog.AuditLogger,
+	accessRequest fosite.AccessRequester,
+	accessResponse fosite.AccessResponder,
+) {
+	var idToken string
+
+	if accessRequest.GetGrantTypes().ExactOne(oidcapi.GrantTypeTokenExchange) {
+		// Token exchanges return the ID token in the access token field of the response.
+		idToken = accessResponse.GetAccessToken()
+	} else {
+		// For other grant types, there may not be an access token, e.g. when the openid scope was not granted.
+		tok := accessResponse.GetExtra("id_token")
+		if tok != nil {
+			// This should always be a string. Checking just to be safe.
+			tokAsStr, ok := tok.(string)
+			if ok {
+				idToken = tokAsStr
+			}
+		}
+	}
+
+	if len(idToken) == 0 {
+		return
+	}
+
+	auditLogger.Audit(auditevent.IDTokenIssued, &plog.AuditParams{
+		ReqCtx:  reqCtx,
+		Session: accessRequest,
+		KeysAndValues: []any{
+			"tokenID", fmt.Sprintf("%x", sha256.Sum256([]byte(idToken))),
+		},
+	})
+}

internal/federationdomain/endpoints/token/token_handler_test.go

@@ -4,6 +4,7 @@
 package token
 
 import (
+	"bytes"
 	"context"
 	"crypto/ecdsa"
 	"crypto/elliptic"
@@ -43,6 +44,7 @@ import (
 
 	supervisorconfigv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/config/v1alpha1"
 	supervisorfake "go.pinniped.dev/generated/latest/client/supervisor/clientset/versioned/fake"
+	"go.pinniped.dev/internal/auditid"
 	"go.pinniped.dev/internal/celtransformer"
 	"go.pinniped.dev/internal/crud"
 	"go.pinniped.dev/internal/federationdomain/clientregistry"
@@ -61,6 +63,7 @@ import (
 	"go.pinniped.dev/internal/here"
 	"go.pinniped.dev/internal/httputil/httperr"
 	"go.pinniped.dev/internal/oidcclientsecretstorage"
+	"go.pinniped.dev/internal/plog"
 	"go.pinniped.dev/internal/psession"
 	"go.pinniped.dev/internal/testutil"
 	"go.pinniped.dev/internal/testutil/oidctestutil"
@@ -126,7 +129,7 @@ var (
 	fositeInvalidPayloadErrorBody = here.Doc(`
 		{
 			"error":             "invalid_request",
-			"error_description": "The request is missing a required parameter, includes an invalid parameter value, includes a parameter more than once, or is otherwise malformed. Unable to parse HTTP body, make sure to send a properly formatted form request body."
+			"error_description": "The request is missing a required parameter, includes an invalid parameter value, includes a parameter more than once, or is otherwise malformed. Unable to parse form params, make sure to send a properly formatted query params or form request body."
 		}
 	`)
 
@@ -309,6 +312,7 @@ type tokenEndpointResponseExpectedValues struct {
 	// The expected lifetime of the ID tokens issued by authcode exchange and refresh, but not token exchange.
 	// When zero, will assume that the test wants the default value for ID token lifetime.
 	wantIDTokenLifetimeSeconds int
+	wantAuditLogs              func(sessionID string, idToken string) []testutil.WantedAuditLog
 }
 
 func withWantCustomIDTokenLifetime(wantIDTokenLifetimeSeconds int, w tokenEndpointResponseExpectedValues) tokenEndpointResponseExpectedValues {
@@ -364,6 +368,10 @@ func addDynamicClientIDToFormPostBody(r *http.Request) {
 	r.Form.Set("client_id", dynamicClientID)
 }
 
+func idTokenToHash(tok string) string {
+	return fmt.Sprintf("%x", sha256.Sum256([]byte(tok)))
+}
+
 func TestTokenEndpointAuthcodeExchange(t *testing.T) {
 	tests := []struct {
 		name             string
@@ -383,6 +391,24 @@ func TestTokenEndpointAuthcodeExchange(t *testing.T) {
 					wantGrantedScopes:     []string{"openid", "username", "groups"},
 					wantUsername:          goodUsername,
 					wantGroups:            goodGroups,
+					wantAuditLogs: func(sessionID string, idToken string) []testutil.WantedAuditLog {
+						return []testutil.WantedAuditLog{
+							testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+								"params": map[string]any{
+									"client_id":     "pinniped-cli",
+									"code":          "redacted",
+									"code_verifier": "redacted",
+									"grant_type":    "authorization_code",
+									"redirect_uri":  "http://127.0.0.1/callback",
+								},
+							}),
+							testutil.WantAuditLog("Session Found", map[string]any{"sessionID": sessionID}),
+							testutil.WantAuditLog("ID Token Issued", map[string]any{
+								"sessionID": sessionID,
+								"tokenID":   idTokenToHash(idToken),
+							}),
+						}
+					},
 				},
 			},
 		},
@@ -440,6 +466,24 @@ func TestTokenEndpointAuthcodeExchange(t *testing.T) {
 					wantGrantedScopes:     []string{"openid", "pinniped:request-audience", "username", "groups"},
 					wantUsername:          goodUsername,
 					wantGroups:            goodGroups,
+					wantAuditLogs: func(sessionID string, idToken string) []testutil.WantedAuditLog {
+						return []testutil.WantedAuditLog{
+							testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+								"params": map[string]any{
+									"code":          "redacted",
+									"code_verifier": "redacted",
+									"grant_type":    "authorization_code",
+									"redirect_uri":  "http://127.0.0.1/callback",
+								},
+							}),
+							testutil.WantAuditLog("HTTP Request Basic Auth", map[string]any{"clientID": dynamicClientID}),
+							testutil.WantAuditLog("Session Found", map[string]any{"sessionID": sessionID}),
+							testutil.WantAuditLog("ID Token Issued", map[string]any{
+								"sessionID": sessionID,
+								"tokenID":   idTokenToHash(idToken),
+							}),
+						}
+					},
 				},
 			},
 		},
@@ -518,6 +562,20 @@ func TestTokenEndpointAuthcodeExchange(t *testing.T) {
 					wantGrantedScopes:     []string{"username", "groups"}, // username and groups were not requested, but granted anyway for backwards compatibility
 					wantUsername:          goodUsername,
 					wantGroups:            goodGroups,
+					wantAuditLogs: func(sessionID string, idToken string) []testutil.WantedAuditLog {
+						return []testutil.WantedAuditLog{
+							testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+								"params": map[string]any{
+									"client_id":     "pinniped-cli",
+									"code":          "redacted",
+									"code_verifier": "redacted",
+									"grant_type":    "authorization_code",
+									"redirect_uri":  "http://127.0.0.1/callback",
+								},
+							}),
+							testutil.WantAuditLog("Session Found", map[string]any{"sessionID": sessionID}),
+						}
+					},
 				},
 			},
 		},
@@ -538,6 +596,21 @@ func TestTokenEndpointAuthcodeExchange(t *testing.T) {
 					wantGrantedScopes:     []string{"pinniped:request-audience", "username", "groups"},
 					wantUsername:          goodUsername,
 					wantGroups:            goodGroups,
+					wantAuditLogs: func(sessionID string, idToken string) []testutil.WantedAuditLog {
+						return []testutil.WantedAuditLog{
+							testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+								"params": map[string]any{
+									"code":          "redacted",
+									"code_verifier": "redacted",
+									"grant_type":    "authorization_code",
+									"redirect_uri":  "http://127.0.0.1/callback",
+								},
+							}),
+							testutil.WantAuditLog("HTTP Request Basic Auth", map[string]any{"clientID": dynamicClientID}),
+							testutil.WantAuditLog("Session Found", map[string]any{"sessionID": sessionID}),
+							// Note that there was no ID token issued, so there is no "ID Token Issued" audit log.
+						}
+					},
 				},
 			},
 		},
@@ -910,6 +983,18 @@ func TestTokenEndpointAuthcodeExchange(t *testing.T) {
 				want: tokenEndpointResponseExpectedValues{
 					wantStatus:            http.StatusBadRequest,
 					wantErrorResponseBody: fositeMissingPKCEVerifierErrorBody,
+					wantAuditLogs: func(sessionID string, idToken string) []testutil.WantedAuditLog {
+						return []testutil.WantedAuditLog{
+							testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+								"params": map[string]any{
+									"client_id":    "pinniped-cli",
+									"code":         "redacted",
+									"grant_type":   "authorization_code",
+									"redirect_uri": "http://127.0.0.1/callback",
+								},
+							}),
+						}
+					},
 				},
 			},
 		},
@@ -924,6 +1009,19 @@ func TestTokenEndpointAuthcodeExchange(t *testing.T) {
 				want: tokenEndpointResponseExpectedValues{
 					wantStatus:            http.StatusBadRequest,
 					wantErrorResponseBody: fositeWrongPKCEVerifierErrorBody,
+					wantAuditLogs: func(sessionID string, idToken string) []testutil.WantedAuditLog {
+						return []testutil.WantedAuditLog{
+							testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+								"params": map[string]any{
+									"client_id":     "pinniped-cli",
+									"code":          "redacted",
+									"code_verifier": "redacted",
+									"grant_type":    "authorization_code",
+									"redirect_uri":  "http://127.0.0.1/callback",
+								},
+							}),
+						}
+					},
 				},
 			},
 		},
@@ -979,7 +1077,7 @@ func TestTokenEndpointWhenAuthcodeIsUsedTwice(t *testing.T) {
 
 			// First call - should be successful.
 			// Authcode exchange doesn't use the upstream provider cache, so just pass an empty cache.
-			subject, rsp, authCode, _, secrets, oauthStore := exchangeAuthcodeForTokens(t,
+			subject, rsp, authCode, _, secrets, oauthStore, _, _ := exchangeAuthcodeForTokens(t,
 				test.authcodeExchange, testidplister.NewUpstreamIDPListerBuilder().BuildFederationDomainIdentityProvidersListerFinder(), test.kubeResources)
 			var parsedResponseBody map[string]any
 			require.NoError(t, json.Unmarshal(rsp.Body.Bytes(), &parsedResponseBody))
@@ -1074,6 +1172,7 @@ func TestTokenEndpointTokenExchange(t *testing.T) { // tests for grant_type "urn
 		wantStatus            int
 		wantErrorType         string
 		wantErrorDescContains string
+		wantAuditLogs         func(sessionID string, idToken string) []testutil.WantedAuditLog
 	}{
 		{
 			name:              "happy path",
@@ -1117,10 +1216,47 @@ func TestTokenEndpointTokenExchange(t *testing.T) { // tests for grant_type "urn
 							"name": "value",
 						},
 					},
+					wantAuditLogs: func(sessionID string, idToken string) []testutil.WantedAuditLog {
+						return []testutil.WantedAuditLog{
+							testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+								"params": map[string]any{
+									"client_id":     "pinniped-cli",
+									"code":          "redacted",
+									"code_verifier": "redacted",
+									"grant_type":    "authorization_code",
+									"redirect_uri":  "http://127.0.0.1/callback",
+								},
+							}),
+							testutil.WantAuditLog("Session Found", map[string]any{"sessionID": sessionID}),
+							testutil.WantAuditLog("ID Token Issued", map[string]any{
+								"sessionID": sessionID,
+								"tokenID":   idTokenToHash(idToken),
+							}),
+						}
+					},
 				},
 			},
 			requestedAudience: "some-workload-cluster",
 			wantStatus:        http.StatusOK,
+			wantAuditLogs: func(sessionID string, idToken string) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+						"params": map[string]any{
+							"audience":             "some-workload-cluster",
+							"client_id":            "pinniped-cli",
+							"grant_type":           "urn:ietf:params:oauth:grant-type:token-exchange",
+							"requested_token_type": "urn:ietf:params:oauth:token-type:jwt",
+							"subject_token":        "redacted",
+							"subject_token_type":   "urn:ietf:params:oauth:token-type:access_token",
+						},
+					}),
+					testutil.WantAuditLog("Session Found", map[string]any{"sessionID": sessionID}),
+					testutil.WantAuditLog("ID Token Issued", map[string]any{
+						"sessionID": sessionID,
+						"tokenID":   idTokenToHash(idToken),
+					}),
+				}
+			},
 		},
 		{
 			name: "happy path without requesting username and groups scopes",
@@ -1294,6 +1430,20 @@ func TestTokenEndpointTokenExchange(t *testing.T) { // tests for grant_type "urn
 			wantStatus:            http.StatusBadRequest,
 			wantErrorType:         "unauthorized_client",
 			wantErrorDescContains: `The client is not authorized to request a token using this method. The OAuth 2.0 Client is not allowed to use token exchange grant 'urn:ietf:params:oauth:grant-type:token-exchange'.`,
+			wantAuditLogs: func(sessionID string, idToken string) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+						"params": map[string]any{
+							"audience":             "some-workload-cluster",
+							"grant_type":           "urn:ietf:params:oauth:grant-type:token-exchange",
+							"requested_token_type": "urn:ietf:params:oauth:token-type:jwt",
+							"subject_token":        "redacted",
+							"subject_token_type":   "urn:ietf:params:oauth:token-type:access_token",
+						},
+					}),
+					testutil.WantAuditLog("HTTP Request Basic Auth", map[string]any{"clientID": dynamicClientID}),
+				}
+			},
 		},
 		{
 			name:          "dynamic client did not ask for the pinniped:request-audience scope in the original authorization request, so the access token submitted during token exchange lacks the scope",
@@ -1392,6 +1542,20 @@ func TestTokenEndpointTokenExchange(t *testing.T) { // tests for grant_type "urn
 			wantStatus:            http.StatusBadRequest,
 			wantErrorType:         "invalid_request",
 			wantErrorDescContains: "Missing 'audience' parameter.",
+			wantAuditLogs: func(sessionID string, idToken string) []testutil.WantedAuditLog {
+				return []testutil.WantedAuditLog{
+					testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+						"params": map[string]any{
+							"audience":             "", // make it obvious
+							"client_id":            "pinniped-cli",
+							"grant_type":           "urn:ietf:params:oauth:grant-type:token-exchange",
+							"requested_token_type": "urn:ietf:params:oauth:token-type:jwt",
+							"subject_token":        "redacted",
+							"subject_token_type":   "urn:ietf:params:oauth:token-type:access_token",
+						},
+					}),
+				}
+			},
 		},
 		{
 			name:                  "bad requested audience when it looks like the name of an OIDCClient CR",
@@ -1657,14 +1821,14 @@ func TestTokenEndpointTokenExchange(t *testing.T) { // tests for grant_type "urn
 			t.Parallel()
 
 			// Authcode exchange doesn't use the upstream provider cache, so just pass an empty cache.
-			subject, rsp, _, _, secrets, storage := exchangeAuthcodeForTokens(t,
+			subject, rsp, _, _, secrets, oauthStore, actualAuditLog, actualSessionID := exchangeAuthcodeForTokens(t,
 				test.authcodeExchange, testidplister.NewUpstreamIDPListerBuilder().BuildFederationDomainIdentityProvidersListerFinder(), test.kubeResources)
 			var parsedAuthcodeExchangeResponseBody map[string]any
 			require.NoError(t, json.Unmarshal(rsp.Body.Bytes(), &parsedAuthcodeExchangeResponseBody))
 
 			request := happyTokenExchangeRequest(test.requestedAudience, parsedAuthcodeExchangeResponseBody["access_token"].(string))
 			if test.modifyStorage != nil {
-				test.modifyStorage(t, storage, secrets, request)
+				test.modifyStorage(t, oauthStore, secrets, request)
 			}
 			if test.modifyRequestParams != nil {
 				test.modifyRequestParams(t, request.Form)
@@ -1672,6 +1836,7 @@ func TestTokenEndpointTokenExchange(t *testing.T) { // tests for grant_type "urn
 
 			req := httptest.NewRequest("POST", "/token/exchange/path/shouldn't/matter", body(request.Form).ReadCloser())
 			req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+			req, _ = auditid.NewRequestWithAuditID(req, func() string { return "fake-token-exchange-audit-id" })
 			rsp = httptest.NewRecorder()
 
 			if test.modifyRequestHeaders != nil {
@@ -1688,6 +1853,7 @@ func TestTokenEndpointTokenExchange(t *testing.T) { // tests for grant_type "urn
 
 			// Perform the token exchange.
 			approxRequestTime := time.Now()
+			actualAuditLog.Reset() // Clear audit logs from the authcode exchange
 			subject.ServeHTTP(rsp, req)
 			t.Logf("response: %#v", rsp)
 			t.Logf("response body: %q", rsp.Body.String())
@@ -1710,6 +1876,13 @@ func TestTokenEndpointTokenExchange(t *testing.T) { // tests for grant_type "urn
 				require.NotEmpty(t, errorDesc)
 				require.Contains(t, errorDesc, test.wantErrorDescContains)
 
+				// Even in the error case, make assertions about audit logs, but without an ID token.
+				if test.wantAuditLogs != nil {
+					wantAuditLogs := test.wantAuditLogs(actualSessionID, "")
+					testutil.WantAuditIDOnEveryAuditLog(wantAuditLogs, "fake-token-exchange-audit-id")
+					testutil.CompareAuditLogs(t, wantAuditLogs, actualAuditLog.String())
+				}
+
 				// The remaining assertions apply only to the happy path.
 				return
 			}
@@ -1725,7 +1898,8 @@ func TestTokenEndpointTokenExchange(t *testing.T) { // tests for grant_type "urn
 			require.Equal(t, "urn:ietf:params:oauth:token-type:jwt", parsedResponseBody["issued_token_type"])
 
 			// Parse the returned token.
-			parsedJWT, err := jose.ParseSigned(parsedResponseBody["access_token"].(string), []jose.SignatureAlgorithm{jose.ES256})
+			actualIDToken := parsedResponseBody["access_token"].(string)
+			parsedJWT, err := jose.ParseSigned(actualIDToken, []jose.SignatureAlgorithm{jose.ES256})
 			require.NoError(t, err)
 			var tokenClaims map[string]any
 			require.NoError(t, json.Unmarshal(parsedJWT.UnsafePayloadWithoutVerification(), &tokenClaims))
@@ -1813,6 +1987,12 @@ func TestTokenEndpointTokenExchange(t *testing.T) { // tests for grant_type "urn
 			newSecrets, err := secrets.List(context.Background(), metav1.ListOptions{})
 			require.NoError(t, err)
 			require.ElementsMatch(t, existingSecrets.Items, newSecrets.Items)
+
+			if test.wantAuditLogs != nil {
+				wantAuditLogs := test.wantAuditLogs(actualSessionID, actualIDToken)
+				testutil.WantAuditIDOnEveryAuditLog(wantAuditLogs, "fake-token-exchange-audit-id")
+				testutil.CompareAuditLogs(t, wantAuditLogs, actualAuditLog.String())
+			}
 		})
 	}
 }
@@ -2049,6 +2229,11 @@ func TestRefreshGrant(t *testing.T) {
 		return want
 	}
 
+	refreshResponseWithAuditLogs := func(expectedValues tokenEndpointResponseExpectedValues, wantAuditLogs func(sessionID string, idToken string) []testutil.WantedAuditLog) tokenEndpointResponseExpectedValues {
+		expectedValues.wantAuditLogs = wantAuditLogs
+		return expectedValues
+	}
+
 	happyRefreshTokenResponseForOpenIDAndOfflineAccessWithUsernameAndGroups := func(wantCustomSessionDataStored *psession.CustomSessionData, expectToValidateToken *oauth2.Token, wantDownstreamUsername string, wantDownstreamGroups []string) tokenEndpointResponseExpectedValues {
 		// Should always have some custom session data stored. The other expectations happens to be the
 		// same as the same values as the authcode exchange case.
@@ -2193,9 +2378,46 @@ func TestRefreshGrant(t *testing.T) {
 				}).WithRefreshedTokens(refreshedUpstreamTokensWithIDAndRefreshTokens()).Build()),
 			authcodeExchange: happyAuthcodeExchangeInputsForOIDCUpstream,
 			refreshRequest: refreshRequestInputs{
-				want: happyRefreshTokenResponseForOpenIDAndOfflineAccess(
-					upstreamOIDCCustomSessionDataWithNewRefreshToken(oidcUpstreamRefreshedRefreshToken),
-					refreshedUpstreamTokensWithIDAndRefreshTokens(),
+				want: refreshResponseWithAuditLogs(
+					happyRefreshTokenResponseForOpenIDAndOfflineAccess(
+						upstreamOIDCCustomSessionDataWithNewRefreshToken(oidcUpstreamRefreshedRefreshToken),
+						refreshedUpstreamTokensWithIDAndRefreshTokens(),
+					),
+					func(sessionID string, idToken string) []testutil.WantedAuditLog {
+						return []testutil.WantedAuditLog{
+							testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+								"params": map[string]any{
+									"client_id":     "pinniped-cli",
+									"grant_type":    "refresh_token",
+									"refresh_token": "redacted",
+									"scope":         "openid",
+								},
+							}),
+							testutil.WantAuditLog("Session Found", map[string]any{"sessionID": sessionID}),
+							testutil.WantAuditLog("Identity Refreshed From Upstream IDP", map[string]any{
+								"sessionID": sessionID,
+								"personalInfo": map[string]any{
+									"upstreamGroups":   []any{},
+									"upstreamUsername": "some-username",
+								},
+							}),
+							testutil.WantAuditLog("Session Refreshed", map[string]any{
+								"sessionID": sessionID,
+								"personalInfo": map[string]any{
+									"username": "some-username",
+									"groups": []any{
+										"group1",
+										"groups2",
+									},
+									"subject": "https://issuer?sub=some-subject",
+								},
+							}),
+							testutil.WantAuditLog("ID Token Issued", map[string]any{
+								"sessionID": sessionID,
+								"tokenID":   idTokenToHash(idToken),
+							}),
+						}
+					},
 				),
 			},
 		},
@@ -2363,6 +2585,30 @@ func TestRefreshGrant(t *testing.T) {
 							"error_description": "Error during upstream refresh. Upstream refresh rejected by configured identity policy: authentication was rejected by a configured policy."
 						}
 					`),
+					wantAuditLogs: func(sessionID string, idToken string) []testutil.WantedAuditLog {
+						return []testutil.WantedAuditLog{
+							testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+								"params": map[string]any{
+									"client_id":     "pinniped-cli",
+									"grant_type":    "refresh_token",
+									"refresh_token": "redacted",
+									"scope":         "openid",
+								},
+							}),
+							testutil.WantAuditLog("Session Found", map[string]any{"sessionID": sessionID}),
+							testutil.WantAuditLog("Identity Refreshed From Upstream IDP", map[string]any{
+								"sessionID": sessionID,
+								"personalInfo": map[string]any{
+									"upstreamGroups":   []any{},
+									"upstreamUsername": "some-username",
+								},
+							}),
+							testutil.WantAuditLog("Authentication Rejected By Transforms", map[string]any{
+								"sessionID": sessionID,
+								"reason":    "Upstream refresh rejected by configured identity policy: authentication was rejected by a configured policy.",
+							}),
+						}
+					},
 				},
 			},
 		},
@@ -2410,6 +2656,24 @@ func TestRefreshGrant(t *testing.T) {
 							"name": "value",
 						},
 					},
+					wantAuditLogs: func(sessionID string, idToken string) []testutil.WantedAuditLog {
+						return []testutil.WantedAuditLog{
+							testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+								"params": map[string]any{
+									"client_id":     "pinniped-cli",
+									"code":          "redacted",
+									"code_verifier": "redacted",
+									"grant_type":    "authorization_code",
+									"redirect_uri":  "http://127.0.0.1/callback",
+								},
+							}),
+							testutil.WantAuditLog("Session Found", map[string]any{"sessionID": sessionID}),
+							testutil.WantAuditLog("ID Token Issued", map[string]any{
+								"sessionID": sessionID,
+								"tokenID":   idTokenToHash(idToken),
+							}),
+						}
+					},
 				},
 			},
 			refreshRequest: refreshRequestInputs{
@@ -2762,6 +3026,46 @@ func TestRefreshGrant(t *testing.T) {
 						{Text: `User "some-username" has been added to the following groups: ["new-group1" "new-group2" "new-group3"]`},
 						{Text: `User "some-username" has been removed from the following groups: ["group1" "groups2"]`},
 					},
+					wantAuditLogs: func(sessionID string, idToken string) []testutil.WantedAuditLog {
+						return []testutil.WantedAuditLog{
+							testutil.WantAuditLog("HTTP Request Parameters", map[string]any{
+								"params": map[string]any{
+									"client_id":     "pinniped-cli",
+									"grant_type":    "refresh_token",
+									"refresh_token": "redacted",
+									"scope":         "openid",
+								},
+							}),
+							testutil.WantAuditLog("Session Found", map[string]any{"sessionID": sessionID}),
+							testutil.WantAuditLog("Identity Refreshed From Upstream IDP", map[string]any{
+								"sessionID": sessionID,
+								"personalInfo": map[string]any{
+									"upstreamGroups": []any{
+										"new-group1",
+										"new-group2",
+										"new-group3",
+									},
+									"upstreamUsername": "some-username",
+								},
+							}),
+							testutil.WantAuditLog("Session Refreshed", map[string]any{
+								"sessionID": sessionID,
+								"personalInfo": map[string]any{
+									"username": "some-username",
+									"groups": []any{
+										"new-group1",
+										"new-group2",
+										"new-group3",
+									},
+									"subject": "https://issuer?sub=some-subject",
+								},
+							}),
+							testutil.WantAuditLog("ID Token Issued", map[string]any{
+								"sessionID": sessionID,
+								"tokenID":   idTokenToHash(idToken),
+							}),
+						}
+					},
 				},
 			},
 		},
@@ -4706,9 +5010,9 @@ func TestRefreshGrant(t *testing.T) {
 			t.Parallel()
 
 			// First exchange the authcode for tokens, including a refresh token.
-			// its actually fine to use this function even when simulating ldap (which uses a different flow) because it's
+			// It's actually fine to use this function even when simulating LDAP (which uses a different flow) because it's
 			// just populating a secret in storage.
-			subject, rsp, authCode, jwtSigningKey, secrets, oauthStore := exchangeAuthcodeForTokens(t,
+			subject, rsp, authCode, jwtSigningKey, secrets, oauthStore, actualAuditLog, actualSessionID := exchangeAuthcodeForTokens(t,
 				test.authcodeExchange, test.idps.BuildFederationDomainIdentityProvidersListerFinder(), test.kubeResources)
 			var parsedAuthcodeExchangeResponseBody map[string]any
 			require.NoError(t, json.Unmarshal(rsp.Body.Bytes(), &parsedAuthcodeExchangeResponseBody))
@@ -4733,14 +5037,19 @@ func TestRefreshGrant(t *testing.T) {
 			}
 
 			reqContextWarningRecorder := &TestWarningRecorder{}
-			reqContext := warning.WithWarningRecorder(context.WithValue(context.Background(), struct{ name string }{name: "test"}, "request-context"), reqContextWarningRecorder)
 			req := httptest.NewRequest("POST", "/path/shouldn't/matter",
-				happyRefreshRequestBody(firstRefreshToken).ReadCloser()).WithContext(reqContext)
+				happyRefreshRequestBody(firstRefreshToken).ReadCloser()).
+				WithContext(warning.WithWarningRecorder(
+					context.WithValue(context.Background(), struct{ name string }{name: "test"}, "request-context"),
+					reqContextWarningRecorder,
+				))
 			req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+			req, _ = auditid.NewRequestWithAuditID(req, func() string { return "fake-refresh-grant-audit-id" })
 			if test.refreshRequest.modifyTokenRequest != nil {
 				test.refreshRequest.modifyTokenRequest(req, firstRefreshToken, parsedAuthcodeExchangeResponseBody["access_token"].(string))
 			}
 
+			actualAuditLog.Reset() // Clear audit logs from the authcode exchange
 			refreshResponse := httptest.NewRecorder()
 			approxRequestTime := time.Now()
 			subject.ServeHTTP(refreshResponse, req)
@@ -4750,25 +5059,25 @@ func TestRefreshGrant(t *testing.T) {
 			// Test that we did or did not make a call to the upstream provider's interface to perform refresh.
 			switch {
 			case test.refreshRequest.want.wantOIDCUpstreamRefreshCall != nil:
-				test.refreshRequest.want.wantOIDCUpstreamRefreshCall.args.Ctx = reqContext
+				test.refreshRequest.want.wantOIDCUpstreamRefreshCall.args.Ctx = req.Context()
 				test.idps.RequireExactlyOneCallToOIDCPerformRefresh(t,
 					test.refreshRequest.want.wantOIDCUpstreamRefreshCall.performedByUpstreamName,
 					test.refreshRequest.want.wantOIDCUpstreamRefreshCall.args,
 				)
 			case test.refreshRequest.want.wantLDAPUpstreamRefreshCall != nil:
-				test.refreshRequest.want.wantLDAPUpstreamRefreshCall.args.Ctx = reqContext
+				test.refreshRequest.want.wantLDAPUpstreamRefreshCall.args.Ctx = req.Context()
 				test.idps.RequireExactlyOneCallToLDAPPerformRefresh(t,
 					test.refreshRequest.want.wantLDAPUpstreamRefreshCall.performedByUpstreamName,
 					test.refreshRequest.want.wantLDAPUpstreamRefreshCall.args,
 				)
 			case test.refreshRequest.want.wantActiveDirectoryUpstreamRefreshCall != nil:
-				test.refreshRequest.want.wantActiveDirectoryUpstreamRefreshCall.args.Ctx = reqContext
+				test.refreshRequest.want.wantActiveDirectoryUpstreamRefreshCall.args.Ctx = req.Context()
 				test.idps.RequireExactlyOneCallToActiveDirectoryPerformRefresh(t,
 					test.refreshRequest.want.wantActiveDirectoryUpstreamRefreshCall.performedByUpstreamName,
 					test.refreshRequest.want.wantActiveDirectoryUpstreamRefreshCall.args,
 				)
 			case test.refreshRequest.want.wantGithubUpstreamRefreshCall != nil:
-				test.refreshRequest.want.wantGithubUpstreamRefreshCall.args.Ctx = reqContext
+				test.refreshRequest.want.wantGithubUpstreamRefreshCall.args.Ctx = req.Context()
 				test.idps.RequireExactlyOneCallToGithubGetUser(t,
 					test.refreshRequest.want.wantGithubUpstreamRefreshCall.performedByUpstreamName,
 					test.refreshRequest.want.wantGithubUpstreamRefreshCall.args,
@@ -4780,7 +5089,7 @@ func TestRefreshGrant(t *testing.T) {
 			// Test that we did or did not make a call to the upstream OIDC provider interface to validate the
 			// new ID token that was returned by the upstream refresh, in the case of an OIDC upstream.
 			if test.refreshRequest.want.wantUpstreamOIDCValidateTokenCall != nil {
-				test.refreshRequest.want.wantUpstreamOIDCValidateTokenCall.args.Ctx = reqContext
+				test.refreshRequest.want.wantUpstreamOIDCValidateTokenCall.args.Ctx = req.Context()
 				test.idps.RequireExactlyOneCallToValidateToken(t,
 					test.refreshRequest.want.wantUpstreamOIDCValidateTokenCall.performedByUpstreamName,
 					test.refreshRequest.want.wantUpstreamOIDCValidateTokenCall.args,
@@ -4809,6 +5118,9 @@ func TestRefreshGrant(t *testing.T) {
 				jwtSigningKey,
 				secrets,
 				approxRequestTime,
+				actualSessionID,
+				"fake-refresh-grant-audit-id",
+				actualAuditLog,
 			)
 
 			if test.refreshRequest.want.wantStatus == http.StatusOK {
@@ -4882,6 +5194,8 @@ func exchangeAuthcodeForTokens(
 	jwtSigningKey *ecdsa.PrivateKey,
 	secrets v1.SecretInterface,
 	oauthStore *storage.KubeStorage,
+	actualAuditLog *bytes.Buffer,
+	actualSessionID string,
 ) {
 	authRequest := deepCopyRequestForm(happyAuthRequest)
 	if test.modifyAuthRequest != nil {
@@ -4907,6 +5221,8 @@ func exchangeAuthcodeForTokens(
 		test.makeJwksSigningKeyAndProvider = generateJWTSigningKeyAndJWKSProvider
 	}
 
+	auditLogger, actualAuditLog := plog.TestAuditLogger(t)
+
 	var oauthHelper fosite.OAuth2Provider
 	// Note that makeHappyOauthHelper() calls simulateAuthEndpointHavingAlreadyRun() to preload the session storage.
 	oauthHelper, authCode, jwtSigningKey = makeHappyOauthHelper(t, authRequest, oauthStore, test.makeJwksSigningKeyAndProvider, test.customSessionData, test.modifySession)
@@ -4916,6 +5232,7 @@ func exchangeAuthcodeForTokens(
 		oauthHelper,
 		timeoutsConfiguration.OverrideDefaultAccessTokenLifespan,
 		timeoutsConfiguration.OverrideDefaultIDTokenLifespan,
+		auditLogger,
 	)
 
 	authorizeEndpointGrantedOpenIDScope := strings.Contains(authRequest.Form.Get("scope"), "openid")
@@ -4935,6 +5252,7 @@ func exchangeAuthcodeForTokens(
 	if test.modifyTokenRequest != nil {
 		test.modifyTokenRequest(req, authCode)
 	}
+	req, _ = auditid.NewRequestWithAuditID(req, func() string { return "fake-code-grant-audit-id" })
 	rsp = httptest.NewRecorder()
 
 	approxRequestTime := time.Now()
@@ -4942,6 +5260,8 @@ func exchangeAuthcodeForTokens(
 	t.Logf("response: %#v", rsp)
 	t.Logf("response body: %q", rsp.Body.String())
 
+	actualSessionID = getSessionID(t, secrets)
+
 	wantNonceValueInIDToken := true // ID tokens returned by the authcode exchange must include the nonce from the auth request (unlike refreshed ID tokens)
 
 	requireTokenEndpointBehavior(
@@ -4954,9 +5274,26 @@ func exchangeAuthcodeForTokens(
 		jwtSigningKey,
 		secrets,
 		approxRequestTime,
+		actualSessionID,
+		"fake-code-grant-audit-id",
+		actualAuditLog,
 	)
 
-	return subject, rsp, authCode, jwtSigningKey, secrets, oauthStore
+	return subject, rsp, authCode, jwtSigningKey, secrets, oauthStore, actualAuditLog, actualSessionID
+}
+
+func getSessionID(t *testing.T, secrets v1.SecretInterface) string {
+	t.Helper()
+
+	authCodeLabelSelector := fmt.Sprintf("%s=%s", crud.SecretLabelKey, authorizationcode.TypeLabelValue)
+	allAuthCodeSecrets, _ := secrets.List(context.Background(), metav1.ListOptions{
+		LabelSelector: authCodeLabelSelector,
+	})
+	require.NotNil(t, allAuthCodeSecrets)
+	require.Len(t, allAuthCodeSecrets.Items, 1, "expected exactly one secret with label %s", authCodeLabelSelector)
+	session, err := authorizationcode.ReadFromSecret(&allAuthCodeSecrets.Items[0])
+	require.NoError(t, err)
+	return session.Request.GetID()
 }
 
 func requireTokenEndpointBehavior(
@@ -4969,10 +5306,14 @@ func requireTokenEndpointBehavior(
 	jwtSigningKey *ecdsa.PrivateKey,
 	secrets v1.SecretInterface,
 	requestTime time.Time,
+	actualSessionID string,
+	wantAuditID string,
+	actualAuditLog *bytes.Buffer,
 ) {
 	testutil.RequireEqualContentType(t, tokenEndpointResponse.Header().Get("Content-Type"), "application/json")
 	require.Equal(t, test.wantStatus, tokenEndpointResponse.Code)
 
+	var actualIDToken string
 	if test.wantStatus == http.StatusOK {
 		require.NotNil(t, test.wantSuccessBodyFields, "problem with test table setup: wanted success but did not specify expected response body")
 
@@ -4993,7 +5334,7 @@ func requireTokenEndpointBehavior(
 			expectedNumberOfRefreshTokenSessionsStored = 1
 		}
 		if wantIDToken {
-			requireValidIDToken(t, parsedResponseBody, jwtSigningKey, test.wantClientID, wantNonceValueInIDToken, test.wantUsername, test.wantGroups, test.wantAdditionalClaims, test.wantIDTokenLifetimeSeconds, parsedResponseBody["access_token"].(string), requestTime)
+			actualIDToken = requireValidIDToken(t, parsedResponseBody, jwtSigningKey, test.wantClientID, wantNonceValueInIDToken, test.wantUsername, test.wantGroups, test.wantAdditionalClaims, test.wantIDTokenLifetimeSeconds, parsedResponseBody["access_token"].(string), requestTime)
 		}
 		if wantRefreshToken {
 			requireValidRefreshTokenStorage(t, parsedResponseBody, oauthStore, test.wantClientID, test.wantRequestedScopes, test.wantGrantedScopes, test.wantUsername, test.wantGroups, test.wantCustomSessionDataStored, test.wantAdditionalClaims, secrets, requestTime)
@@ -5011,6 +5352,12 @@ func requireTokenEndpointBehavior(
 
 		require.JSONEq(t, test.wantErrorResponseBody, tokenEndpointResponse.Body.String())
 	}
+
+	if test.wantAuditLogs != nil {
+		wantAuditLogs := test.wantAuditLogs(actualSessionID, actualIDToken)
+		testutil.WantAuditIDOnEveryAuditLog(wantAuditLogs, wantAuditID)
+		testutil.CompareAuditLogs(t, wantAuditLogs, actualAuditLog.String())
+	}
 }
 
 func hashAccessToken(accessToken string) string {
@@ -5520,7 +5867,7 @@ func requireValidIDToken(
 	wantIDTokenLifetimeSeconds int,
 	actualAccessToken string,
 	requestTime time.Time,
-) {
+) string {
 	t.Helper()
 
 	idToken, ok := body["id_token"]
@@ -5605,6 +5952,8 @@ func requireValidIDToken(
 
 	require.NotEmpty(t, actualAccessToken)
 	require.Equal(t, hashAccessToken(actualAccessToken), claims.AccessTokenHash)
+
+	return idTokenString
 }
 
 func deepCopyRequestForm(r *http.Request) *http.Request {
@@ -5710,3 +6059,20 @@ func getSecretNameFromSignature(t *testing.T, signature string, typeLabel string
 	signatureAsValidName := strings.ToLower(b32.EncodeToString(signatureBytes))
 	return fmt.Sprintf("pinniped-storage-%s-%s", typeLabel, signatureAsValidName)
 }
+
+// TestParamsSafeToLog only exists to ensure that paramsSafeToLog will not be accidentally updated.
+func TestParamsSafeToLog(t *testing.T) {
+	wantParams := []string{
+		"actor_token_type",
+		"audience",
+		"client_id",
+		"grant_type",
+		"redirect_uri",
+		"requested_token_type",
+		"resource",
+		"scope",
+		"subject_token_type",
+	}
+
+	require.ElementsMatch(t, wantParams, paramsSafeToLog().UnsortedList())
+}

internal/federationdomain/endpoints/tokenexchange/token_exchange.go

@@ -46,17 +46,10 @@ type tokenExchangeHandler struct {
 var _ fosite.TokenEndpointHandler = (*tokenExchangeHandler)(nil)
 
 func (t *tokenExchangeHandler) HandleTokenEndpointRequest(ctx context.Context, requester fosite.AccessRequester) error {
+	// Skip this request if it's for a different grant type.
 	if !t.CanHandleTokenEndpointRequest(ctx, requester) {
 		return errors.WithStack(fosite.ErrUnknownRequest)
 	}
-	return nil
-}
-
-func (t *tokenExchangeHandler) PopulateTokenEndpointResponse(ctx context.Context, requester fosite.AccessRequester, responder fosite.AccessResponder) error {
-	// Skip this request if it's for a different grant type.
-	if err := t.HandleTokenEndpointRequest(ctx, requester); err != nil {
-		return errors.WithStack(err)
-	}
 
 	// Validate the basic RFC8693 parameters we support.
 	params, err := t.validateParams(requester.GetRequestForm())
@@ -64,7 +57,7 @@ func (t *tokenExchangeHandler) PopulateTokenEndpointResponse(ctx context.Context
 		return errors.WithStack(err)
 	}
 
-	// Validate the incoming access token and lookup the information about the original authorize request.
+	// Validate the incoming access token and lookup the information about the original authorize request from storage.
 	originalRequester, err := t.validateAccessToken(ctx, requester, params.subjectAccessToken)
 	if err != nil {
 		return errors.WithStack(err)
@@ -95,8 +88,28 @@ func (t *tokenExchangeHandler) PopulateTokenEndpointResponse(ctx context.Context
 		return errors.WithStack(err)
 	}
 
+	// Copy the original session ID from storage.
+	requester.SetID(originalRequester.GetID())
+	// Copy the original session details from storage, which will be used by PopulateTokenEndpointResponse() to mint a token.
+	requester.SetSession(originalRequester.GetSession().Clone())
+	// Maybe not needed, but just to be safe, copy these too, similar to how flow_refresh.go copies them.
+	requester.SetRequestedScopes(originalRequester.GetRequestedScopes())
+	requester.SetRequestedAudience(originalRequester.GetRequestedAudience())
+
+	return nil
+}
+
+func (t *tokenExchangeHandler) PopulateTokenEndpointResponse(ctx context.Context, requester fosite.AccessRequester, responder fosite.AccessResponder) error {
+	// Skip this request if it's for a different grant type.
+	if !t.CanHandleTokenEndpointRequest(ctx, requester) {
+		return errors.WithStack(fosite.ErrUnknownRequest)
+	}
+
+	// Get the requested audience parameter again, which was already validated by HandleTokenEndpointRequest() above.
+	requestedNewAudience := requester.GetRequestForm().Get("audience")
+
 	// Use the original authorize request information, along with the requested audience, to mint a new JWT.
-	responseToken, err := t.mintJWT(ctx, originalRequester, params.requestedAudience)
+	responseToken, err := t.mintJWT(ctx, requester, requestedNewAudience)
 	if err != nil {
 		return errors.WithStack(err)
 	}
@@ -108,15 +121,15 @@ func (t *tokenExchangeHandler) PopulateTokenEndpointResponse(ctx context.Context
 	return nil
 }
 
-func (t *tokenExchangeHandler) mintJWT(ctx context.Context, requester fosite.Requester, audience string) (string, error) {
-	downscoped := fosite.NewAccessRequest(requester.GetSession())
-	downscoped.Client.(*fosite.DefaultClient).ID = audience
+func (t *tokenExchangeHandler) mintJWT(ctx context.Context, requester fosite.Requester, newAudience string) (string, error) {
+	requestWithNewAudience := fosite.NewAccessRequest(requester.GetSession())
+	requestWithNewAudience.Client.(*fosite.DefaultClient).ID = newAudience
 
 	// Note: if we wanted to support clients with custom token lifespans, then we would need to call
 	// fosite.GetEffectiveLifespan() to determine the lifespan here.
 	idTokenLifespan := t.fositeConfig.GetIDTokenLifespan(ctx)
 
-	return t.idTokenStrategy.GenerateIDToken(ctx, idTokenLifespan, downscoped)
+	return t.idTokenStrategy.GenerateIDToken(ctx, idTokenLifespan, requestWithNewAudience)
 }
 
 func (t *tokenExchangeHandler) validateSession(requester fosite.Requester) error {

internal/federationdomain/endpointsmanager/manager.go

@@ -11,6 +11,8 @@ import (
 	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
 
 	"go.pinniped.dev/generated/latest/client/supervisor/clientset/versioned/typed/config/v1alpha1"
+	"go.pinniped.dev/internal/auditid"
+	"go.pinniped.dev/internal/config/supervisor"
 	"go.pinniped.dev/internal/federationdomain/csrftoken"
 	"go.pinniped.dev/internal/federationdomain/dynamiccodec"
 	"go.pinniped.dev/internal/federationdomain/endpoints/auth"
@@ -25,8 +27,8 @@ import (
 	"go.pinniped.dev/internal/federationdomain/idplister"
 	"go.pinniped.dev/internal/federationdomain/oidc"
 	"go.pinniped.dev/internal/federationdomain/oidcclientvalidator"
+	"go.pinniped.dev/internal/federationdomain/requestlogger"
 	"go.pinniped.dev/internal/federationdomain/storage"
-	"go.pinniped.dev/internal/httputil/requestutil"
 	"go.pinniped.dev/internal/plog"
 	"go.pinniped.dev/internal/secret"
 	"go.pinniped.dev/pkg/oidcclient/nonce"
@@ -40,12 +42,13 @@ type Manager struct {
 	mu                  sync.RWMutex
 	providers           []*federationdomainproviders.FederationDomainIssuer
 	providerHandlers    map[string]http.Handler                   // map of all routes for all providers
-	nextHandler         http.Handler                              // the next handler in a chain, called when this manager didn't know how to handle a request
+	handlerChain        http.Handler                              // http handlers
 	dynamicJWKSProvider jwks.DynamicJWKSProvider                  // in-memory cache of per-issuer JWKS data
 	upstreamIDPs        idplister.UpstreamIdentityProvidersLister // in-memory cache of upstream IDPs
 	secretCache         *secret.Cache                             // in-memory cache of cryptographic material
 	secretsClient       corev1client.SecretInterface
 	oidcClientsClient   v1alpha1.OIDCClientInterface
+	auditLogger         plog.AuditLogger
 }
 
 // NewManager returns an empty Manager.
@@ -59,16 +62,25 @@ func NewManager(
 	secretCache *secret.Cache,
 	secretsClient corev1client.SecretInterface,
 	oidcClientsClient v1alpha1.OIDCClientInterface,
+	auditLogger plog.AuditLogger,
+	auditInternalPathsCfg supervisor.AuditInternalPaths,
 ) *Manager {
-	return &Manager{
+	m := &Manager{
 		providerHandlers:    make(map[string]http.Handler),
-		nextHandler:         nextHandler,
 		dynamicJWKSProvider: dynamicJWKSProvider,
 		upstreamIDPs:        upstreamIDPs,
 		secretCache:         secretCache,
 		secretsClient:       secretsClient,
 		oidcClientsClient:   oidcClientsClient,
+		auditLogger:         auditLogger,
 	}
+	// nextHandler is the next handler in the chain, called when this manager didn't know how to handle a request
+	m.buildHandlerChain(nextHandler, auditInternalPathsCfg)
+	return m
+}
+
+func (m *Manager) HandlerChain() http.Handler {
+	return m.handlerChain
 }
 
 // SetFederationDomains adds or updates all the given providerHandlers using each provider's issuer string
@@ -77,7 +89,7 @@ func NewManager(
 // It also removes any providerHandlers that were previously added but were not passed in to
 // the current invocation.
 //
-// This method assumes that all of the FederationDomainIssuer arguments have already been validated
+// This method assumes that all the FederationDomainIssuer arguments have already been validated
 // by someone else before they are passed to this method.
 func (m *Manager) SetFederationDomains(federationDomains ...*federationdomainproviders.FederationDomainIssuer) {
 	m.mu.Lock()
@@ -143,6 +155,7 @@ func (m *Manager) SetFederationDomains(federationDomains ...*federationdomainpro
 			nonce.Generate,
 			upstreamStateEncoder,
 			csrfCookieEncoder,
+			m.auditLogger,
 		)
 
 		m.providerHandlers[(issuerHostWithPath + oidc.CallbackEndpointPath)] = callback.NewHandler(
@@ -151,6 +164,7 @@ func (m *Manager) SetFederationDomains(federationDomains ...*federationdomainpro
 			upstreamStateEncoder,
 			csrfCookieEncoder,
 			issuerURL+oidc.CallbackEndpointPath,
+			m.auditLogger,
 		)
 
 		m.providerHandlers[(issuerHostWithPath + oidc.ChooseIDPEndpointPath)] = chooseidp.NewHandler(
@@ -163,38 +177,43 @@ func (m *Manager) SetFederationDomains(federationDomains ...*federationdomainpro
 			oauthHelperWithKubeStorage,
 			timeoutsConfiguration.OverrideDefaultAccessTokenLifespan,
 			timeoutsConfiguration.OverrideDefaultIDTokenLifespan,
+			m.auditLogger,
 		)
 
 		m.providerHandlers[(issuerHostWithPath + oidc.PinnipedLoginPath)] = login.NewHandler(
 			upstreamStateEncoder,
 			csrfCookieEncoder,
 			login.NewGetHandler(incomingFederationDomain.IssuerPath()+oidc.PinnipedLoginPath),
-			login.NewPostHandler(issuerURL, idpLister, oauthHelperWithKubeStorage),
+			login.NewPostHandler(issuerURL, idpLister, oauthHelperWithKubeStorage, m.auditLogger),
+			m.auditLogger,
 		)
 
 		plog.Debug("oidc provider manager added or updated issuer", "issuer", issuerURL)
 	}
 }
 
-// ServeHTTP implements the http.Handler interface.
-func (m *Manager) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
-	requestHandler := m.findHandler(req)
+func (m *Manager) buildHandlerChain(nextHandler http.Handler, auditInternalPathsCfg supervisor.AuditInternalPaths) {
+	// Build the basic handler for FederationDomain endpoints.
+	handler := m.buildManagerHandler(nextHandler)
+	// Log all requests, including audit ID.
+	handler = requestlogger.WithHTTPRequestAuditLogging(handler, m.auditLogger, auditInternalPathsCfg)
+	// Add random audit ID to request context and response headers.
+	handler = auditid.WithAuditID(handler)
+	m.handlerChain = handler
+}
 
-	// Using Info level so the user can safely configure a production Supervisor to show this message if they choose.
-	plog.Info("received incoming request",
-		"proto", req.Proto,
-		"method", req.Method,
-		"host", req.Host,
-		"requestSNIServerName", requestutil.SNIServerName(req),
-		"path", req.URL.Path,
-		"remoteAddr", req.RemoteAddr,
-		"foundFederationDomainRequestHandler", requestHandler != nil,
-	)
-
-	if requestHandler == nil {
-		requestHandler = m.nextHandler // couldn't find an issuer to handle the request
-	}
-	requestHandler.ServeHTTP(resp, req)
+func (m *Manager) buildManagerHandler(nextHandler http.Handler) http.Handler {
+	return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+		requestHandler := m.findHandler(req)
+		if requestHandler == nil {
+			// Couldn't find any FederationDomain to handle this request based on the request's host and path.
+			// It could be a bad request to a path that does not exist. Or it could be because something in
+			// front of the Supervisor is not passing the SNI of the original request through to the Supervisor.
+			// All 404's will be logged by request_logger.go, so we don't need to log it here.
+			requestHandler = nextHandler
+		}
+		requestHandler.ServeHTTP(resp, req)
+	})
 }
 
 func (m *Manager) findHandler(req *http.Request) http.Handler {

internal/federationdomain/endpointsmanager/manager_test.go

@@ -20,12 +20,14 @@ import (
 	"k8s.io/client-go/kubernetes/fake"
 
 	supervisorfake "go.pinniped.dev/generated/latest/client/supervisor/clientset/versioned/fake"
+	"go.pinniped.dev/internal/config/supervisor"
 	"go.pinniped.dev/internal/federationdomain/endpoints/discovery"
 	"go.pinniped.dev/internal/federationdomain/endpoints/jwks"
 	"go.pinniped.dev/internal/federationdomain/federationdomainproviders"
 	"go.pinniped.dev/internal/federationdomain/oidc"
 	"go.pinniped.dev/internal/here"
 	"go.pinniped.dev/internal/idtransform"
+	"go.pinniped.dev/internal/plog"
 	"go.pinniped.dev/internal/secret"
 	"go.pinniped.dev/internal/testutil"
 	"go.pinniped.dev/internal/testutil/oidctestutil"
@@ -83,7 +85,7 @@ func TestManager(t *testing.T) {
 		requireDiscoveryRequestToBeHandled := func(requestIssuer, requestURLSuffix, expectedIssuer string) {
 			recorder := httptest.NewRecorder()
 
-			subject.ServeHTTP(recorder, newGetRequest(requestIssuer+oidc.WellKnownEndpointPath+requestURLSuffix))
+			subject.HandlerChain().ServeHTTP(recorder, newGetRequest(requestIssuer+oidc.WellKnownEndpointPath+requestURLSuffix))
 
 			r.False(fallbackHandlerWasCalled)
 
@@ -101,7 +103,7 @@ func TestManager(t *testing.T) {
 		requirePinnipedIDPsDiscoveryRequestToBeHandled := func(requestIssuer, requestURLSuffix string, expectedIDPNames []string, expectedIDPTypes string, expectedFlows []string) {
 			recorder := httptest.NewRecorder()
 
-			subject.ServeHTTP(recorder, newGetRequest(requestIssuer+oidc.PinnipedIDPsPathV1Alpha1+requestURLSuffix))
+			subject.HandlerChain().ServeHTTP(recorder, newGetRequest(requestIssuer+oidc.PinnipedIDPsPathV1Alpha1+requestURLSuffix))
 
 			r.False(fallbackHandlerWasCalled)
 
@@ -145,7 +147,7 @@ func TestManager(t *testing.T) {
 				"response_type": []string{"bat"},
 			}
 
-			subject.ServeHTTP(recorder, newGetRequest(requestIssuer+oidc.ChooseIDPEndpointPath+"?"+requiredParams.Encode()))
+			subject.HandlerChain().ServeHTTP(recorder, newGetRequest(requestIssuer+oidc.ChooseIDPEndpointPath+"?"+requiredParams.Encode()))
 
 			r.False(fallbackHandlerWasCalled)
 
@@ -164,7 +166,7 @@ func TestManager(t *testing.T) {
 		requireAuthorizationRequestToBeHandled := func(requestIssuer, requestURLSuffix, expectedRedirectLocationPrefix string) (string, string) {
 			recorder := httptest.NewRecorder()
 
-			subject.ServeHTTP(recorder, newGetRequest(requestIssuer+oidc.AuthorizationEndpointPath+requestURLSuffix))
+			subject.HandlerChain().ServeHTTP(recorder, newGetRequest(requestIssuer+oidc.AuthorizationEndpointPath+requestURLSuffix))
 
 			r.False(fallbackHandlerWasCalled)
 
@@ -202,7 +204,7 @@ func TestManager(t *testing.T) {
 				Name:  "__Host-pinniped-csrf",
 				Value: csrfCookieValue,
 			})
-			subject.ServeHTTP(recorder, getRequest)
+			subject.HandlerChain().ServeHTTP(recorder, getRequest)
 
 			r.False(fallbackHandlerWasCalled)
 
@@ -242,7 +244,7 @@ func TestManager(t *testing.T) {
 				"code_verifier": []string{downstreamPKCECodeVerifier},
 				"grant_type":    []string{"authorization_code"},
 			}.Encode()
-			subject.ServeHTTP(recorder, newPostRequest(requestIssuer+oidc.TokenEndpointPath, tokenRequestBody))
+			subject.HandlerChain().ServeHTTP(recorder, newPostRequest(requestIssuer+oidc.TokenEndpointPath, tokenRequestBody))
 
 			r.False(fallbackHandlerWasCalled)
 
@@ -272,7 +274,7 @@ func TestManager(t *testing.T) {
 		requireJWKSRequestToBeHandled := func(requestIssuer, requestURLSuffix, expectedJWKKeyID string) *jose.JSONWebKeySet {
 			recorder := httptest.NewRecorder()
 
-			subject.ServeHTTP(recorder, newGetRequest(requestIssuer+oidc.JWKSEndpointPath+requestURLSuffix))
+			subject.HandlerChain().ServeHTTP(recorder, newGetRequest(requestIssuer+oidc.JWKSEndpointPath+requestURLSuffix))
 
 			r.False(fallbackHandlerWasCalled)
 
@@ -358,13 +360,24 @@ func TestManager(t *testing.T) {
 			cache.SetStateEncoderHashKey(issuer2, []byte("some-state-encoder-hash-key-2"))
 			cache.SetStateEncoderBlockKey(issuer2, []byte("16-bytes-STATE02"))
 
-			subject = NewManager(nextHandler, dynamicJWKSProvider, idpLister, &cache, secretsClient, oidcClientsClient)
+			auditLogger, _ := plog.TestAuditLogger(t)
+
+			subject = NewManager(
+				nextHandler,
+				dynamicJWKSProvider,
+				idpLister,
+				&cache,
+				secretsClient,
+				oidcClientsClient,
+				auditLogger,
+				supervisor.Enabled,
+			)
 		})
 
 		when("given no providers via SetFederationDomains()", func() {
 			it("sends all requests to the nextHandler", func() {
 				r.False(fallbackHandlerWasCalled)
-				subject.ServeHTTP(httptest.NewRecorder(), newGetRequest("/anything"))
+				subject.HandlerChain().ServeHTTP(httptest.NewRecorder(), newGetRequest("/anything"))
 				r.True(fallbackHandlerWasCalled)
 			})
 		})
@@ -507,19 +520,19 @@ func TestManager(t *testing.T) {
 			it("sends all non-matching host requests to the nextHandler", func() {
 				r.False(fallbackHandlerWasCalled)
 				wrongHostURL := strings.ReplaceAll(issuer1+oidc.WellKnownEndpointPath, "example.com", "wrong-host.com")
-				subject.ServeHTTP(httptest.NewRecorder(), newGetRequest(wrongHostURL))
+				subject.HandlerChain().ServeHTTP(httptest.NewRecorder(), newGetRequest(wrongHostURL))
 				r.True(fallbackHandlerWasCalled)
 			})
 
 			it("sends all non-matching path requests to the nextHandler", func() {
 				r.False(fallbackHandlerWasCalled)
-				subject.ServeHTTP(httptest.NewRecorder(), newGetRequest("https://example.com/path-does-not-match-any-provider"))
+				subject.HandlerChain().ServeHTTP(httptest.NewRecorder(), newGetRequest("https://example.com/path-does-not-match-any-provider"))
 				r.True(fallbackHandlerWasCalled)
 			})
 
 			it("sends requests which match the issuer prefix but do not match any of that provider's known paths to the nextHandler", func() {
 				r.False(fallbackHandlerWasCalled)
-				subject.ServeHTTP(httptest.NewRecorder(), newGetRequest(issuer1+"/unhandled-sub-path"))
+				subject.HandlerChain().ServeHTTP(httptest.NewRecorder(), newGetRequest(issuer1+"/unhandled-sub-path"))
 				r.True(fallbackHandlerWasCalled)
 			})
 

internal/federationdomain/oidc/oidc.go

@@ -25,6 +25,7 @@ import (
 	"go.pinniped.dev/internal/federationdomain/endpoints/tokenexchange"
 	"go.pinniped.dev/internal/federationdomain/formposthtml"
 	"go.pinniped.dev/internal/federationdomain/idtokenlifespan"
+	"go.pinniped.dev/internal/federationdomain/stateparam"
 	"go.pinniped.dev/internal/federationdomain/strategy"
 	"go.pinniped.dev/internal/federationdomain/timeouts"
 	"go.pinniped.dev/internal/httputil/httperr"
@@ -326,7 +327,7 @@ func ScopeWasRequested(authorizeRequester fosite.AuthorizeRequester, scopeName s
 	return false
 }
 
-func ReadStateParamAndValidateCSRFCookie(r *http.Request, cookieDecoder Decoder, stateDecoder Decoder) (string, *UpstreamStateParamData, error) {
+func ReadStateParamAndValidateCSRFCookie(r *http.Request, cookieDecoder Decoder, stateDecoder Decoder) (stateparam.Encoded, *UpstreamStateParamData, error) {
 	csrfValue, err := readCSRFCookie(r, cookieDecoder)
 	if err != nil {
 		return "", nil, err
@@ -342,7 +343,7 @@ func ReadStateParamAndValidateCSRFCookie(r *http.Request, cookieDecoder Decoder,
 		return "", nil, err
 	}
 
-	return encodedState, decodedState, nil
+	return stateparam.Encoded(encodedState), decodedState, nil
 }
 
 func readCSRFCookie(r *http.Request, cookieDecoder Decoder) (csrftoken.CSRFToken, error) {

internal/federationdomain/requestlogger/request_logger.go

@@ -0,0 +1,180 @@
+// Copyright 2024 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package requestlogger
+
+import (
+	"bufio"
+	"net"
+	"net/http"
+	"net/url"
+	"slices"
+	"time"
+
+	"k8s.io/apiserver/pkg/endpoints/responsewriter"
+	"k8s.io/utils/clock"
+
+	"go.pinniped.dev/internal/auditevent"
+	"go.pinniped.dev/internal/config/supervisor"
+	"go.pinniped.dev/internal/httputil/requestutil"
+	"go.pinniped.dev/internal/plog"
+)
+
+func WithHTTPRequestAuditLogging(handler http.Handler, auditLogger plog.AuditLogger, auditInternalPathsCfg supervisor.AuditInternalPaths) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		rl := newRequestLogger(req, w, auditLogger, time.Now(), auditInternalPathsCfg)
+
+		rl.logRequestReceived()
+		defer rl.logRequestComplete()
+
+		statusCodeCapturingResponseWriter := responsewriter.WrapForHTTP1Or2(rl)
+		handler.ServeHTTP(statusCodeCapturingResponseWriter, req)
+	})
+}
+
+type requestLogger struct {
+	startTime time.Time
+	clock     clock.Clock // clock is used to calculate the response latency, and useful for unit tests.
+
+	hijacked       bool
+	statusRecorded bool
+	status         int
+
+	req       *http.Request
+	userAgent string
+	w         http.ResponseWriter
+
+	auditLogger        plog.AuditLogger
+	auditInternalPaths bool
+}
+
+func newRequestLogger(
+	req *http.Request,
+	w http.ResponseWriter,
+	auditLogger plog.AuditLogger,
+	startTime time.Time,
+	auditInternalPathsCfg supervisor.AuditInternalPaths,
+) *requestLogger {
+	return &requestLogger{
+		req:                req,
+		w:                  w,
+		startTime:          startTime,
+		clock:              clock.RealClock{},
+		userAgent:          req.UserAgent(), // cache this from the req to avoid any possibility of concurrent read/write problems with headers map
+		auditLogger:        auditLogger,
+		auditInternalPaths: auditInternalPathsCfg.Enabled(),
+	}
+}
+
+func internalPaths() []string {
+	return []string{
+		"/healthz",
+	}
+}
+
+func (rl *requestLogger) logRequestReceived() {
+	r := rl.req
+
+	if !rl.auditInternalPaths && slices.Contains(internalPaths(), r.URL.Path) {
+		return
+	}
+
+	// Always log all other requests, including 404's caused by bad paths, for debugging purposes.
+	rl.auditLogger.Audit(auditevent.HTTPRequestReceived, &plog.AuditParams{
+		ReqCtx: r.Context(),
+		KeysAndValues: []any{
+			"proto", r.Proto,
+			"method", r.Method,
+			"host", r.Host, // The "Host" header is promoted to this field.
+			"serverName", requestutil.SNIServerName(r),
+			"path", r.URL.Path,
+			"userAgent", rl.userAgent,
+			"remoteAddr", r.RemoteAddr,
+		},
+	})
+}
+
+func getLocationForAuditLogs(location string) string {
+	if location == "" {
+		return "no location header"
+	}
+
+	parsedLocation, err := url.Parse(location)
+	if err != nil {
+		return "unparsable location header"
+	}
+
+	// We don't know what this `Location` header is used for, so redact nearly all query parameters
+	redactedParams := parsedLocation.Query()
+	for k, v := range redactedParams {
+		// Due to https://datatracker.ietf.org/doc/html/rfc6749#section-4.1.2.1,
+		// authorize errors can have an 'error' and an 'error_description' parameter
+		// which should never contain PII and is safe to log.
+		// The 'err' parameter may be populated by the post_login_handler to indicate issues
+		// when using Supervisor's built-in login page.
+		if k == "error" || k == "error_description" || k == "err" {
+			continue
+		}
+		for i := range v {
+			redactedParams[k][i] = "redacted"
+		}
+	}
+	parsedLocation.RawQuery = redactedParams.Encode()
+	return parsedLocation.String()
+}
+
+func (rl *requestLogger) logRequestComplete() {
+	r := rl.req
+
+	if !rl.auditInternalPaths && slices.Contains(internalPaths(), r.URL.Path) {
+		return
+	}
+
+	rl.auditLogger.Audit(auditevent.HTTPRequestCompleted, &plog.AuditParams{
+		ReqCtx: r.Context(),
+		KeysAndValues: []any{
+			"path", r.URL.Path,
+			"latency", rl.clock.Since(rl.startTime),
+			"responseStatus", rl.status,
+			"location", getLocationForAuditLogs(rl.Header().Get("Location")),
+		},
+	})
+}
+
+// Unwrap implements responsewriter.UserProvidedDecorator.
+func (rl *requestLogger) Unwrap() http.ResponseWriter {
+	return rl.w
+}
+
+// Header implements http.ResponseWriter.
+func (rl *requestLogger) Header() http.Header {
+	return rl.w.Header()
+}
+
+// Write implements http.ResponseWriter.
+func (rl *requestLogger) Write(b []byte) (int, error) {
+	if !rl.statusRecorded {
+		rl.recordStatus(http.StatusOK) // Default if WriteHeader hasn't been called
+	}
+	return rl.w.Write(b)
+}
+
+// WriteHeader implements http.ResponseWriter.
+func (rl *requestLogger) WriteHeader(status int) {
+	rl.recordStatus(status)
+	rl.w.WriteHeader(status)
+}
+
+// Hijack implements http.Hijacker.
+func (rl *requestLogger) Hijack() (net.Conn, *bufio.ReadWriter, error) {
+	rl.hijacked = true
+
+	// the outer ResponseWriter object returned by WrapForHTTP1Or2 implements
+	// http.Hijacker if the inner object (rl.w) implements http.Hijacker.
+	return rl.w.(http.Hijacker).Hijack()
+}
+
+func (rl *requestLogger) recordStatus(status int) {
+	rl.status = status
+	rl.statusRecorded = true
+}

internal/federationdomain/requestlogger/request_logger_test.go

@@ -0,0 +1,210 @@
+// Copyright 2024 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package requestlogger
+
+import (
+	"crypto/tls"
+	"net/http"
+	"net/url"
+	"testing"
+	"time"
+
+	"go.uber.org/mock/gomock"
+	clocktesting "k8s.io/utils/clock/testing"
+
+	"go.pinniped.dev/internal/mocks/mockresponsewriter"
+	"go.pinniped.dev/internal/plog"
+	"go.pinniped.dev/internal/testutil"
+)
+
+func TestLogRequestReceived(t *testing.T) {
+	var noAuditEventsWanted []testutil.WantedAuditLog
+
+	happyAuditEventWanted := func(path string) []testutil.WantedAuditLog {
+		return []testutil.WantedAuditLog{
+			testutil.WantAuditLog("HTTP Request Received",
+				map[string]any{
+					"proto":      "some-proto",
+					"method":     "some-method",
+					"host":       "some-host",
+					"serverName": "some-sni-server-name",
+					"path":       path,
+					"userAgent":  "some-user-agent",
+					"remoteAddr": "some-remote-addr",
+				},
+			),
+		}
+	}
+
+	tests := []struct {
+		name               string
+		path               string
+		auditInternalPaths bool
+		wantAuditLogs      []testutil.WantedAuditLog
+	}{
+		{
+			name:               "when internal paths are not enabled, ignores internal paths",
+			path:               "/healthz",
+			auditInternalPaths: false,
+			wantAuditLogs:      noAuditEventsWanted,
+		},
+		{
+			name:               "when internal paths are not enabled, audits external path",
+			path:               "/pretend-to-login",
+			auditInternalPaths: false,
+			wantAuditLogs:      happyAuditEventWanted("/pretend-to-login"),
+		},
+		{
+			name:               "when internal paths are enabled, audits internal paths",
+			path:               "/healthz",
+			auditInternalPaths: true,
+			wantAuditLogs:      happyAuditEventWanted("/healthz"),
+		},
+		{
+			name:               "when internal paths are enabled, audits external paths",
+			path:               "/pretend-to-login",
+			auditInternalPaths: true,
+			wantAuditLogs:      happyAuditEventWanted("/pretend-to-login"),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+			auditLogger, actualAuditLog := plog.TestAuditLogger(t)
+
+			subject := requestLogger{
+				auditLogger: auditLogger,
+				req: &http.Request{
+					Method: "some-method",
+					Proto:  "some-proto",
+					Host:   "some-host",
+					URL: &url.URL{
+						Path: test.path,
+					},
+					RemoteAddr: "some-remote-addr",
+					TLS: &tls.ConnectionState{
+						ServerName: "some-sni-server-name",
+					},
+				},
+				userAgent:          "some-user-agent",
+				auditInternalPaths: test.auditInternalPaths,
+			}
+
+			subject.logRequestReceived()
+
+			testutil.CompareAuditLogs(t, test.wantAuditLogs, actualAuditLog.String())
+		})
+	}
+}
+
+func TestLogRequestComplete(t *testing.T) {
+	wantLatency := time.Minute + 2*time.Second + 345*time.Millisecond
+
+	var noAuditEventsWanted []testutil.WantedAuditLog
+
+	happyAuditEventWanted := func(path, location string) []testutil.WantedAuditLog {
+		return []testutil.WantedAuditLog{
+			testutil.WantAuditLog("HTTP Request Completed",
+				map[string]any{
+					"path":           path,
+					"latency":        "1m2.345s",
+					"responseStatus": 777.0, // JSON serializes this as a float
+					"location":       location,
+				},
+			),
+		}
+	}
+
+	tests := []struct {
+		name               string
+		path               string
+		location           string
+		auditInternalPaths bool
+		wantAuditLogs      []testutil.WantedAuditLog
+	}{
+		{
+			name:               "when internal paths are not enabled, ignores internal paths",
+			path:               "/healthz",
+			auditInternalPaths: false,
+			wantAuditLogs:      noAuditEventsWanted,
+		},
+		{
+			name:               "when internal paths are not enabled, audits external path with location (redacting unknown query params)",
+			path:               "/pretend-to-login",
+			location:           "http://127.0.0.1?foo=bar&foo=quz&lorem=ipsum",
+			auditInternalPaths: false,
+			wantAuditLogs:      happyAuditEventWanted("/pretend-to-login", "http://127.0.0.1?foo=redacted&foo=redacted&lorem=redacted"),
+		},
+		{
+			name:               "when internal paths are enabled, audits internal paths",
+			path:               "/healthz",
+			auditInternalPaths: true,
+			wantAuditLogs:      happyAuditEventWanted("/healthz", "no location header"),
+		},
+		{
+			name:               "when internal paths are enabled, audits external paths",
+			path:               "/pretend-to-login",
+			location:           "some-location",
+			auditInternalPaths: true,
+			wantAuditLogs:      happyAuditEventWanted("/pretend-to-login", "some-location"),
+		},
+		{
+			name:          "audits path without location",
+			path:          "/pretend-to-login",
+			location:      "", // make it obvious
+			wantAuditLogs: happyAuditEventWanted("/pretend-to-login", "no location header"),
+		},
+		{
+			name:          "audits path with invalid location",
+			path:          "/pretend-to-login",
+			location:      "http://e x a m p l e.com",
+			wantAuditLogs: happyAuditEventWanted("/pretend-to-login", "unparsable location header"),
+		},
+		{
+			name:          "audits path with location redacting all query params except err, error, and error_description",
+			path:          "/pretend-to-login",
+			location:      "http://127.0.0.1:1234?code=pin_ac_FAKE&foo=bar&foo=quz&lorem=ipsum&err=some-err&error=some-error&error_description=some-error-description&zzlast=some-value",
+			wantAuditLogs: happyAuditEventWanted("/pretend-to-login", "http://127.0.0.1:1234?code=redacted&err=some-err&error=some-error&error_description=some-error-description&foo=redacted&foo=redacted&lorem=redacted&zzlast=redacted"),
+		},
+	}
+
+	nowDoesntMatter := time.Date(1122, time.September, 33, 4, 55, 56, 778899, time.Local)
+	startTime := nowDoesntMatter.Add(-wantLatency)
+	frozenClock := clocktesting.NewFakeClock(nowDoesntMatter)
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+
+			ctrl := gomock.NewController(t)
+			mockResponseWriter := mockresponsewriter.NewMockResponseWriter(ctrl)
+			if len(test.wantAuditLogs) > 0 {
+				mockResponseWriter.EXPECT().Header().Return(http.Header{
+					"Location": []string{test.location},
+				})
+			}
+
+			auditLogger, actualAuditLog := plog.TestAuditLogger(t)
+
+			subject := requestLogger{
+				auditLogger: auditLogger,
+				startTime:   startTime,
+				clock:       frozenClock,
+				req: &http.Request{
+					URL: &url.URL{
+						Path: test.path,
+					},
+				},
+				status:             777,
+				w:                  mockResponseWriter,
+				auditInternalPaths: test.auditInternalPaths,
+			}
+
+			subject.logRequestComplete()
+
+			testutil.CompareAuditLogs(t, test.wantAuditLogs, actualAuditLog.String())
+		})
+	}
+}

internal/federationdomain/resolvedprovider/resolved_provider.go

@@ -10,6 +10,7 @@ import (
 	"github.com/ory/fosite"
 
 	"go.pinniped.dev/generated/latest/apis/supervisor/idpdiscovery/v1alpha1"
+	"go.pinniped.dev/internal/federationdomain/stateparam"
 	"go.pinniped.dev/internal/federationdomain/upstreamprovider"
 	"go.pinniped.dev/internal/idtransform"
 	"go.pinniped.dev/internal/psession"
@@ -86,7 +87,7 @@ type RefreshedIdentity struct {
 // upstream authorization request does not allow PKCE, then implementations of
 // FederationDomainResolvedIdentityProvider.UpstreamAuthorizeRedirectURL may choose to ignore that struct field.
 type UpstreamAuthorizeRequestState struct {
-	EncodedStateParam string
+	EncodedStateParam stateparam.Encoded
 	PKCE              pkce.Code
 	Nonce             nonce.Nonce
 }

internal/federationdomain/resolvedprovider/resolvedgithub/resolved_github_provider.go

@@ -81,7 +81,7 @@ func (p *FederationDomainResolvedGitHubIdentityProvider) UpstreamAuthorizeRedire
 		RedirectURL: fmt.Sprintf("%s/callback", downstreamIssuerURL),
 		Scopes:      p.Provider.GetScopes(),
 	}
-	redirectURL := upstreamOAuthConfig.AuthCodeURL(state.EncodedStateParam)
+	redirectURL := upstreamOAuthConfig.AuthCodeURL(state.EncodedStateParam.String())
 	return redirectURL, nil
 }
 

internal/federationdomain/resolvedprovider/resolvedoidc/resolved_oidc_provider.go

@@ -118,7 +118,7 @@ func (p *FederationDomainResolvedOIDCIdentityProvider) UpstreamAuthorizeRedirect
 	}
 
 	redirectURL := upstreamOAuthConfig.AuthCodeURL(
-		state.EncodedStateParam,
+		state.EncodedStateParam.String(),
 		authCodeOptions...,
 	)
 

internal/federationdomain/stateparam/encoded.go

@@ -0,0 +1,19 @@
+// Copyright 2024 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package stateparam
+
+import (
+	"crypto/sha256"
+	"fmt"
+)
+
+type Encoded string
+
+func (e Encoded) String() string {
+	return string(e)
+}
+
+func (e Encoded) AuthorizeID() string {
+	return fmt.Sprintf("%x", sha256.Sum256([]byte(e)))
+}

internal/federationdomain/stateparam/encoded_test.go

@@ -0,0 +1,22 @@
+// Copyright 2024 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package stateparam
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestAuthorizeID(t *testing.T) {
+	// $ echo -n "foo" | shasum -a 256
+	// 2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae
+	require.Equal(t, "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae",
+		Encoded("foo").AuthorizeID())
+
+	// $ echo -n "" | shasum -a 256
+	// e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+	require.Equal(t, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+		Encoded("").AuthorizeID())
+}

internal/kubeclient/roundtrip.go

@@ -31,25 +31,25 @@ func configWithWrapper(config *restclient.Config, scheme *runtime.Scheme, negoti
 		return config // invalid input config, will fail existing client-go validation
 	}
 
-	// no need for any wrapping when we have no middleware to inject
-	if len(middlewares) == 0 {
-		return config
+	var middlewareWrapper transport.WrapperFunc
+	if len(middlewares) > 0 {
+		info, ok := runtime.SerializerInfoForMediaType(negotiatedSerializer.SupportedMediaTypes(), config.ContentType)
+		if !ok {
+			panic(fmt.Errorf("unknown content type: %s ", config.ContentType)) // static input, programmer error
+		}
+		regSerializer := info.Serializer // should perform no conversion
+
+		resolver := server.NewRequestInfoResolver(server.NewConfig(serializer.CodecFactory{}))
+
+		schemeRestMapperFunc := schemeRestMapper(scheme)
+
+		middlewareWrapper = newWrapper(hostURL, apiPathPrefix, config, resolver, regSerializer, negotiatedSerializer, schemeRestMapperFunc, middlewares)
 	}
 
-	info, ok := runtime.SerializerInfoForMediaType(negotiatedSerializer.SupportedMediaTypes(), config.ContentType)
-	if !ok {
-		panic(fmt.Errorf("unknown content type: %s ", config.ContentType)) // static input, programmer error
-	}
-	regSerializer := info.Serializer // should perform no conversion
-
-	resolver := server.NewRequestInfoResolver(server.NewConfig(serializer.CodecFactory{}))
-
-	schemeRestMapperFunc := schemeRestMapper(scheme)
-
-	f := newWrapper(hostURL, apiPathPrefix, config, resolver, regSerializer, negotiatedSerializer, schemeRestMapperFunc, middlewares)
-
 	cc := restclient.CopyConfig(config)
-	cc.Wrap(f)
+	if middlewareWrapper != nil {
+		cc.Wrap(middlewareWrapper)
+	}
 	if wrapper != nil {
 		cc.Wrap(wrapper)
 	}
@@ -204,7 +204,7 @@ func handleCreateOrUpdate(
 	negotiatedSerializer runtime.NegotiatedSerializer,
 ) (bool, *http.Response, error) {
 	if req.GetBody == nil {
-		return true, nil, fmt.Errorf("unreadible body for request: %#v", middlewareReq) // this should never happen
+		return true, nil, fmt.Errorf("unreadable body for request: %#v", middlewareReq) // this should never happen
 	}
 
 	body, err := req.GetBody()

internal/mocks/mockissuer/mockissuer.go

@@ -17,6 +17,7 @@ import (
 	reflect "reflect"
 	time "time"
 
+	cert "go.pinniped.dev/internal/cert"
 	gomock "go.uber.org/mock/gomock"
 )
 
@@ -45,13 +46,12 @@ func (m *MockClientCertIssuer) EXPECT() *MockClientCertIssuerMockRecorder {
 }
 
 // IssueClientCertPEM mocks base method.
-func (m *MockClientCertIssuer) IssueClientCertPEM(username string, groups []string, ttl time.Duration) ([]byte, []byte, error) {
+func (m *MockClientCertIssuer) IssueClientCertPEM(username string, groups []string, ttl time.Duration) (*cert.PEM, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "IssueClientCertPEM", username, groups, ttl)
-	ret0, _ := ret[0].([]byte)
-	ret1, _ := ret[1].([]byte)
-	ret2, _ := ret[2].(error)
-	return ret0, ret1, ret2
+	ret0, _ := ret[0].(*cert.PEM)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
 }
 
 // IssueClientCertPEM indicates an expected call of IssueClientCertPEM.

internal/mocks/mockresponsewriter/generate.go

@@ -0,0 +1,6 @@
+// Copyright 2024 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package mockresponsewriter
+
+//go:generate go run -v go.uber.org/mock/mockgen  -destination=mockresponsewriter.go -package=mockresponsewriter -copyright_file=../../../hack/header.txt net/http ResponseWriter

internal/mocks/mockresponsewriter/mockresponsewriter.go

@@ -0,0 +1,86 @@
+// Copyright 2020-2024 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+//
+
+// Code generated by MockGen. DO NOT EDIT.
+// Source: net/http (interfaces: ResponseWriter)
+//
+// Generated by this command:
+//
+//	mockgen -destination=mockresponsewriter.go -package=mockresponsewriter -copyright_file=../../../hack/header.txt net/http ResponseWriter
+//
+
+// Package mockresponsewriter is a generated GoMock package.
+package mockresponsewriter
+
+import (
+	http "net/http"
+	reflect "reflect"
+
+	gomock "go.uber.org/mock/gomock"
+)
+
+// MockResponseWriter is a mock of ResponseWriter interface.
+type MockResponseWriter struct {
+	ctrl     *gomock.Controller
+	recorder *MockResponseWriterMockRecorder
+	isgomock struct{}
+}
+
+// MockResponseWriterMockRecorder is the mock recorder for MockResponseWriter.
+type MockResponseWriterMockRecorder struct {
+	mock *MockResponseWriter
+}
+
+// NewMockResponseWriter creates a new mock instance.
+func NewMockResponseWriter(ctrl *gomock.Controller) *MockResponseWriter {
+	mock := &MockResponseWriter{ctrl: ctrl}
+	mock.recorder = &MockResponseWriterMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockResponseWriter) EXPECT() *MockResponseWriterMockRecorder {
+	return m.recorder
+}
+
+// Header mocks base method.
+func (m *MockResponseWriter) Header() http.Header {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Header")
+	ret0, _ := ret[0].(http.Header)
+	return ret0
+}
+
+// Header indicates an expected call of Header.
+func (mr *MockResponseWriterMockRecorder) Header() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Header", reflect.TypeOf((*MockResponseWriter)(nil).Header))
+}
+
+// Write mocks base method.
+func (m *MockResponseWriter) Write(arg0 []byte) (int, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Write", arg0)
+	ret0, _ := ret[0].(int)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Write indicates an expected call of Write.
+func (mr *MockResponseWriterMockRecorder) Write(arg0 any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Write", reflect.TypeOf((*MockResponseWriter)(nil).Write), arg0)
+}
+
+// WriteHeader mocks base method.
+func (m *MockResponseWriter) WriteHeader(statusCode int) {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "WriteHeader", statusCode)
+}
+
+// WriteHeader indicates an expected call of WriteHeader.
+func (mr *MockResponseWriterMockRecorder) WriteHeader(statusCode any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WriteHeader", reflect.TypeOf((*MockResponseWriter)(nil).WriteHeader), statusCode)
+}

internal/plog/plog.go

@@ -28,14 +28,58 @@
 package plog
 
 import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
 	"os"
+	"reflect"
 	"slices"
 
 	"github.com/go-logr/logr"
+	"github.com/ory/fosite"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apiserver/pkg/audit"
+
+	"go.pinniped.dev/internal/auditevent"
 )
 
 const errorKey = "error" // this matches zapr's default for .Error calls (which is asserted via tests)
 
+type SessionIDGetter interface {
+	GetID() string
+}
+
+type AuditParams struct {
+	// ReqCtx may be nil. When possible, pass the http request's context as ReqCtx,
+	// so we may read the audit ID from the context.
+	ReqCtx context.Context
+
+	// Session may be nil. When possible, pass the fosite.Requester or fosite.Request as the Session,
+	// so we can log the session ID.
+	Session SessionIDGetter
+
+	// PIIKeysAndValues can optionally be used to pass along more keys are values.
+	// Use these when the values might contain personally identifiable information (PII).
+	// These values may be redacted by configuration.
+	// They must come in alternating pairs of string keys and any values.
+	PIIKeysAndValues []any
+
+	// KeysAndValues can optionally be used to pass along more keys are values.
+	// These values are never redacted and therefore should never contain PII.
+	// They must come in alternating pairs of string keys and any values.
+	KeysAndValues []any
+}
+
+// AuditLogger is the interface for audit logging. There is no global function for Audit because
+// that would make unit testing of audit logs harder.
+type AuditLogger interface {
+	Audit(msg auditevent.Message, p *AuditParams)
+	AuditRequestParams(r *http.Request, reqParamsSafeToLog sets.Set[string]) error
+}
+
 // Logger implements the plog logging convention described above.  The global functions in this package
 // such as Info should be used when one does not intend to write tests assertions for specific log messages.
 // If test assertions are desired, Logger should be passed in as an input.  New should be used as the
@@ -60,6 +104,7 @@ type Logger interface {
 	// for internal and test use only
 	withDepth(d int) Logger
 	withLogrMod(mod func(logr.Logger) logr.Logger) Logger
+	audit(msg string, keysAndValues ...any)
 }
 
 // MinLogger is the overlap between Logger and logr.Logger.
@@ -75,10 +120,110 @@ type pLogger struct {
 	depth int
 }
 
+type AuditLogConfig struct {
+	LogUsernamesAndGroupNames bool
+}
+
+type auditLogger struct {
+	cfg    AuditLogConfig
+	logger Logger
+}
+
 func New() Logger {
 	return pLogger{}
 }
 
+func NewAuditLogger(cfg AuditLogConfig) AuditLogger {
+	return &auditLogger{
+		cfg:    cfg,
+		logger: New(),
+	}
+}
+
+// Audit logs show in the pod log output as `"level":"info","message":"some msg","auditEvent":true`
+// where the message text comes from the msg parameter.
+// They also contain the standard `timestamp` and `caller` keys, along with any other keysAndValues.
+// Only when the global log level is configured to "trace" or "all", then they will also include a `stacktrace` key.
+// Audit logs cannot be suppressed by the global log level configuration. This is because Audit logs should always
+// be printed, regardless of global log level. Audit logs offer their own configuration options, such as a way to
+// avoid potential PII (e.g. usernames and group names) in their pod logs.
+// msg is required. All fields of p, and p itself, are optional.
+func (a *auditLogger) Audit(msg auditevent.Message, p *AuditParams) {
+	// Always add a key/value auditEvent=true.
+	allKV := []any{"auditEvent", true}
+
+	var auditID string
+	if p != nil && p.ReqCtx != nil {
+		auditID = audit.GetAuditIDTruncated(p.ReqCtx)
+	}
+	if len(auditID) > 0 {
+		allKV = slices.Concat(allKV, []any{"auditID", auditID})
+	}
+
+	var sessionID string
+	if p != nil && p.Session != nil {
+		sessionID = p.Session.GetID()
+	}
+	if len(sessionID) > 0 {
+		allKV = slices.Concat(allKV, []any{"sessionID", sessionID})
+	}
+
+	if p != nil && len(p.PIIKeysAndValues) > 1 {
+		allKV = slices.Concat(allKV, []any{
+			"personalInfo", &piiKeysAndValues{
+				kv:                        p.PIIKeysAndValues,
+				logUsernamesAndGroupNames: a.cfg.LogUsernamesAndGroupNames,
+			},
+		})
+	}
+
+	if p != nil && p.KeysAndValues != nil {
+		allKV = slices.Concat(allKV, p.KeysAndValues)
+	}
+
+	a.logger.audit(string(msg), allKV...)
+}
+
+// AuditRequestParams parses the URL's query params and/or POST body form params, and then audit logs them with
+// redaction as specified by reqParamsSafeToLog. It can return an error, or in the case of success it has the side
+// effect of leaving the parsed form params on the http.Request in the Form field. Note that request body parameters
+// take precedence over URL query values.
+func (a *auditLogger) AuditRequestParams(r *http.Request, reqParamsSafeToLog sets.Set[string]) error {
+	// The style of form parsing and the text of the error is inspired by fosite's implementation of NewAuthorizeRequest().
+	// Fosite only calls ParseMultipartForm() there. However, although ParseMultipartForm() calls ParseForm(),
+	// it swallows errors from ParseForm() sometimes. To avoid having any errors swallowed, we call both.
+	// When fosite calls ParseMultipartForm() later, it will be a noop.
+	if err := r.ParseForm(); err != nil {
+		return fosite.ErrInvalidRequest.
+			WithHint("Unable to parse form params, make sure to send a properly formatted query params or form request body.").
+			WithWrap(err).WithDebug(err.Error())
+	}
+	if err := r.ParseMultipartForm(1 << 20); err != nil && !errors.Is(err, http.ErrNotMultipart) {
+		return fosite.ErrInvalidRequest.
+			WithHint("Unable to parse multipart HTTP body, make sure to send a properly formatted form request body.").
+			WithWrap(err).WithDebug(err.Error())
+	}
+
+	a.Audit(auditevent.HTTPRequestParameters, &AuditParams{
+		ReqCtx:        r.Context(),
+		KeysAndValues: sanitizeRequestParams(r.Form, reqParamsSafeToLog),
+	})
+
+	return nil
+}
+
+// audit is used internally by AuditLogger to print an audit log event to the pLogger's output.
+func (p pLogger) audit(msg string, keysAndValues ...any) {
+	// Always print log message (klogLevelWarning cannot be suppressed by configuration),
+	// and always use the Info function because audit logs are not warnings or errors.
+	p.logr().V(klogLevelWarning).WithCallDepth(p.depth+2).Info(msg, keysAndValues...)
+}
+
+// Error logs show in the pod log output as `"level":"error","message":"some error msg"`
+// where the message text comes from the err parameter.
+// They also contain the standard `timestamp` and `caller` keys, along with any other keysAndValues.
+// Only when the global log level is configured to "trace" or "all", then they will also include a `stacktrace` key.
+// Error logs cannot be suppressed by the global log level configuration.
 func (p pLogger) Error(msg string, err error, keysAndValues ...any) {
 	p.logr().WithCallDepth(p.depth+1).Error(err, msg, keysAndValues...)
 }
@@ -94,10 +239,20 @@ func (p pLogger) warningDepth(msg string, depth int, keysAndValues ...any) {
 	}
 }
 
+// Warning logs show in the pod log output as `"level":"info","message":"some msg","warning":true`
+// where the message text comes from the msg parameter.
+// They also contain the standard `timestamp` and `caller` keys, along with any other keysAndValues.
+// Only when the global log level is configured to "trace" or "all", then they will also include a `stacktrace` key.
+// Warning logs cannot be suppressed by the global log level configuration.
 func (p pLogger) Warning(msg string, keysAndValues ...any) {
 	p.warningDepth(msg, p.depth+1, keysAndValues...)
 }
 
+// WarningErr logs show in the pod log output as `"level":"info","message":"some msg","warning":true,"error":"some error msg"`
+// where the message text comes from the msg parameter and the error text comes from the err parameter.
+// They also contain the standard `timestamp` and `caller` keys, along with any other keysAndValues.
+// Only when the global log level is configured to "trace" or "all", then they will also include a `stacktrace` key.
+// WarningErr logs cannot be suppressed by the global log level configuration.
 func (p pLogger) WarningErr(msg string, err error, keysAndValues ...any) {
 	p.warningDepth(msg, p.depth+1, slices.Concat([]any{errorKey, err}, keysAndValues)...)
 }
@@ -108,10 +263,20 @@ func (p pLogger) infoDepth(msg string, depth int, keysAndValues ...any) {
 	}
 }
 
+// Info logs show in the pod log output as `"level":"info","message":"some msg"`
+// where the message text comes from the msg parameter.
+// They also contain the standard `timestamp` and `caller` keys, along with any other keysAndValues.
+// Only when the global log level is configured to "trace" or "all", then they will also include a `stacktrace` key.
+// Info logs are suppressed by the global log level configuration, unless it is set to "info" or above.
 func (p pLogger) Info(msg string, keysAndValues ...any) {
 	p.infoDepth(msg, p.depth+1, keysAndValues...)
 }
 
+// InfoErr logs show in the pod log output as `"level":"info","message":"some msg","error":"some error msg"`
+// where the message text comes from the msg parameter and the error text comes from the err parameter.
+// They also contain the standard `timestamp` and `caller` keys, along with any other keysAndValues.
+// Only when the global log level is configured to "trace" or "all", then they will also include a `stacktrace` key.
+// InfoErr logs are suppressed by the global log level configuration, unless it is set to "info" or above.
 func (p pLogger) InfoErr(msg string, err error, keysAndValues ...any) {
 	p.infoDepth(msg, p.depth+1, slices.Concat([]any{errorKey, err}, keysAndValues)...)
 }
@@ -122,10 +287,20 @@ func (p pLogger) debugDepth(msg string, depth int, keysAndValues ...any) {
 	}
 }
 
+// Debug logs show in the pod log output as `"level":"debug","message":"some msg"`
+// where the message text comes from the msg parameter.
+// They also contain the standard `timestamp` and `caller` keys, along with any other keysAndValues.
+// Only when the global log level is configured to "trace" or "all", then they will also include a `stacktrace` key.
+// Debug logs are suppressed by the global log level configuration, unless it is set to "debug" or above.
 func (p pLogger) Debug(msg string, keysAndValues ...any) {
 	p.debugDepth(msg, p.depth+1, keysAndValues...)
 }
 
+// DebugErr logs show in the pod log output as `"level":"debug","message":"some msg","error":"some error msg"`
+// where the message text comes from the msg parameter and the error text comes from the err parameter.
+// They also contain the standard `timestamp` and `caller` keys, along with any other keysAndValues.
+// Only when the global log level is configured to "trace" or "all", then they will also include a `stacktrace` key.
+// DebugErr logs are suppressed by the global log level configuration, unless it is set to "debug" or above.
 func (p pLogger) DebugErr(msg string, err error, keysAndValues ...any) {
 	p.debugDepth(msg, p.depth+1, slices.Concat([]any{errorKey, err}, keysAndValues)...)
 }
@@ -136,20 +311,39 @@ func (p pLogger) traceDepth(msg string, depth int, keysAndValues ...any) {
 	}
 }
 
+// Trace logs show in the pod log output as `"level":"trace","message":"some msg"`
+// where the message text comes from the msg parameter.
+// They also contain the standard `timestamp` and `caller` keys, along with any other keysAndValues.
+// Only when the global log level is configured to "trace" or "all", then they will also include a `stacktrace` key.
+// Trace logs are suppressed by the global log level configuration, unless it is set to "trace" or above.
 func (p pLogger) Trace(msg string, keysAndValues ...any) {
 	p.traceDepth(msg, p.depth+1, keysAndValues...)
 }
 
+// TraceErr logs show in the pod log output as `"level":"trace","message":"some msg","error":"some error msg"`
+// where the message text comes from the msg parameter and the error text comes from the err parameter.
+// They also contain the standard `timestamp` and `caller` keys, along with any other keysAndValues.
+// Only when the global log level is configured to "trace" or "all", then they will also include a `stacktrace` key.
+// TraceErr logs are suppressed by the global log level configuration, unless it is set to "trace" or above.
 func (p pLogger) TraceErr(msg string, err error, keysAndValues ...any) {
 	p.traceDepth(msg, p.depth+1, slices.Concat([]any{errorKey, err}, keysAndValues)...)
 }
 
+// All logs show in the pod log output as `"level":"all","message":"some msg"`
+// where the message text comes from the msg parameter.
+// They also contain the standard `timestamp` and `caller` keys, along with any other keysAndValues.
+// Only when the global log level is configured to "trace" or "all", then they will also include a `stacktrace` key.
+// All logs are suppressed by the global log level configuration, unless it is set to "all" or above.
 func (p pLogger) All(msg string, keysAndValues ...any) {
 	if p.logr().V(klogLevelAll).Enabled() {
 		p.logr().V(klogLevelAll).WithCallDepth(p.depth+1).Info(msg, keysAndValues...)
 	}
 }
 
+// Always logs show in the pod log output exactly the same as an Info() message,
+// except Always logs are always logged regardless of log level configuration.
+// Only when the global log level is configured to "trace" or "all", then they will also include a `stacktrace` key.
+// Always logs cannot be suppressed by the global log level configuration.
 func (p pLogger) Always(msg string, keysAndValues ...any) {
 	p.logr().WithCallDepth(p.depth+1).Info(msg, keysAndValues...)
 }
@@ -258,3 +452,129 @@ func Fatal(err error, keysAndValues ...any) {
 	globalFlush()
 	os.Exit(1)
 }
+
+// piiKeysAndValues can be used to serialize the keys and values as a JSON map without losing their order,
+// and optionally redacting values.
+type piiKeysAndValues struct {
+	kv                        []any
+	logUsernamesAndGroupNames bool
+}
+
+func (p *piiKeysAndValues) MarshalJSON() ([]byte, error) {
+	var buf []byte
+	var k string
+	var wroteOne bool
+
+	buf = append(buf, '{')
+
+	for i, v := range p.kv {
+		if i%2 == 0 {
+			// Interpret even indices (0, 2, 4, etc.) as a key.
+			// Remember its value for the next loop iteration.
+			k = p.asJSONKey(v)
+		} else {
+			// Interpret odd indices as a value.
+			// Write it using the key from the previous loop iteration.
+			// First write a comma if needed.
+			if wroteOne {
+				buf = append(buf, ',')
+			}
+			buf = append(buf, []byte(k)...)
+			buf = append(buf, ':')
+			buf = append(buf, []byte(p.asJSONValue(v))...)
+			wroteOne = true
+		}
+	}
+
+	buf = append(buf, '}')
+
+	return buf, nil
+}
+
+func (p *piiKeysAndValues) asJSONKey(v any) string {
+	vStr, ok := v.(string)
+	if !ok {
+		// Indicates programmer error that will hopefully be caught by unit tests of usages of Audit().
+		return `"cannotCastKeyNameToString"`
+	}
+	// Encode the string to get proper JSON escaping if needed.
+	b, err := json.Marshal(vStr)
+	if err != nil {
+		// Shouldn't really happen because the argument to Marshal was a string.
+		return `"cannotMarshalKeyNameToJSON"`
+	}
+	return string(b)
+}
+
+func (p *piiKeysAndValues) asJSONValue(v any) string {
+	rt := reflect.TypeOf(v) // note that rt can be nil when v is nil
+	if p.logUsernamesAndGroupNames {
+		// Encode the original value without redacting.
+		switch {
+		case v != nil && rt.Kind() == reflect.Slice && reflect.ValueOf(v).IsNil():
+			// Handle the special case where v is a nil slice by showing it as an empty slice.
+			return "[]"
+		case v != nil && rt.Kind() == reflect.Map && reflect.ValueOf(v).IsNil():
+			// Handle the special case where v is a nil map by showing it as an empty map.
+			return "{}"
+		default:
+			b, err := json.Marshal(v)
+			if err != nil {
+				// Hopefully this would be caught by unit tests of usages of Audit().
+				return `"cannotMarshalValueToJSON"`
+			}
+			return string(b)
+		}
+	} else {
+		// Redact the value.
+		switch {
+		case rt != nil && rt.Kind() == reflect.Slice:
+			// Handle the special case where v is a slice by showing it as a slice with redacted values.
+			return fmt.Sprintf(`["redacted %d values"]`, reflect.ValueOf(v).Len())
+		case rt != nil && rt.Kind() == reflect.Map:
+			// Handle the special case where v is a map by showing it as a map with redacted values.
+			return fmt.Sprintf(`{"redacted": "redacted %d keys"}`, reflect.ValueOf(v).Len())
+		default:
+			// For anything else, just redact it without worrying about the original type.
+			return `"redacted"`
+		}
+	}
+}
+
+// sanitizeRequestParams can be used to redact all params not included in the allowedKeys set.
+// Useful when audit logging HTTPRequestParameters events.
+func sanitizeRequestParams(inputParams url.Values, allowedKeys sets.Set[string]) []any {
+	params := make(map[string]string)
+	multiValueParams := make(url.Values)
+
+	transform := func(key, value string) string {
+		if !allowedKeys.Has(key) {
+			return "redacted"
+		}
+
+		unescape, err := url.QueryUnescape(value)
+		if err != nil {
+			// ignore these errors and just use the original query parameter
+			unescape = value
+		}
+		return unescape
+	}
+
+	for key := range inputParams {
+		for i, p := range inputParams[key] {
+			transformed := transform(key, p)
+			if i == 0 {
+				params[key] = transformed
+			}
+
+			if len(inputParams[key]) > 1 {
+				multiValueParams[key] = append(multiValueParams[key], transformed)
+			}
+		}
+	}
+
+	if len(multiValueParams) > 0 {
+		return []any{"params", params, "multiValueParams", multiValueParams}
+	}
+	return []any{"params", params}
+}

internal/plog/plog_test.go

@@ -4,16 +4,321 @@
 package plog
 
 import (
+	"context"
 	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
 	"runtime"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/coreos/go-semver/semver"
+	"github.com/ory/fosite"
 	"github.com/stretchr/testify/require"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apiserver/pkg/audit"
+
+	"go.pinniped.dev/internal/auditid"
+	"go.pinniped.dev/internal/here"
 )
 
+type fakeSessionGetter struct{}
+
+func (f fakeSessionGetter) GetID() string {
+	return "fake-session-id"
+}
+
+func TestAudit(t *testing.T) {
+	fakeReqContext := audit.WithAuditContext(context.Background())
+	audit.WithAuditID(fakeReqContext, "fake-audit-id")
+
+	tests := []struct {
+		name      string
+		redactPII bool
+		run       func(AuditLogger)
+		want      string
+	}{
+		{
+			name: "only message, with both nil and empty audit params",
+			run: func(a AuditLogger) {
+				a.Audit("fake event type 1", nil)
+				a.Audit("fake event type 2", &AuditParams{})
+			},
+			want: here.Doc(`
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestAudit.func1","message":"fake event type 1","auditEvent":true}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestAudit.func1","message":"fake event type 2","auditEvent":true}
+			`),
+		},
+		{
+			name: "with request context which has no audit ID",
+			run: func(a AuditLogger) {
+				a.Audit("fake event type", &AuditParams{ReqCtx: context.Background()})
+			},
+			want: here.Doc(`
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestAudit.func2","message":"fake event type","auditEvent":true}
+			`),
+		},
+		{
+			name: "with request context which has audit ID",
+			run: func(a AuditLogger) {
+				a.Audit("fake event type", &AuditParams{ReqCtx: fakeReqContext})
+			},
+			want: here.Doc(`
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestAudit.func3","message":"fake event type","auditEvent":true,"auditID":"fake-audit-id"}
+			`),
+		},
+		{
+			name: "with session getter",
+			run: func(a AuditLogger) {
+				a.Audit("fake event type", &AuditParams{Session: &fakeSessionGetter{}})
+			},
+			want: here.Doc(`
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestAudit.func4","message":"fake event type","auditEvent":true,"sessionID":"fake-session-id"}
+			`),
+		},
+		{
+			name: "with an even number of PII keys and values, nests them under personalInfo and preserves their original order, without redacting PII",
+			run: func(a AuditLogger) {
+				a.Audit("fake event type", &AuditParams{PIIKeysAndValues: []any{
+					"username", "ryan",
+					"groups", []string{"g1", "g2"},
+					"int", 42,
+					"float", 42.75,
+					`specialJSONChars"👋\`, `hi"👋\`,
+					"map", map[string]int{"k1": 1, "k2": 2},
+					"empty_list", []any{},
+					"empty_map", map[string]any{},
+					"nil_list", []any(nil),
+					"nil_map", map[string]any(nil),
+					"nil_ptr", (*int)(nil),
+					"nil", nil,
+				}})
+			},
+			want: here.Doc(`
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestAudit.func5","message":"fake event type","auditEvent":true,"personalInfo":{"username":"ryan","groups":["g1","g2"],"int":42,"float":42.75,"specialJSONChars\"👋\\":"hi\"👋\\","map":{"k1":1,"k2":2},"empty_list":[],"empty_map":{},"nil_list":[],"nil_map":{},"nil_ptr":null,"nil":null}}
+			`),
+		},
+		{
+			name:      "with an even number of PII keys and values and PII configured to be redacted",
+			redactPII: true,
+			run: func(a AuditLogger) {
+				a.Audit("fake event type", &AuditParams{PIIKeysAndValues: []any{
+					"username", "ryan",
+					"groups", []string{"g1", "g2"},
+					"int", 42,
+					"float", 42.75,
+					`specialJSONChars"👋\`, `hi"👋\`,
+					"map", map[string]int{"k1": 1, "k2": 2},
+					"empty_list", []any{},
+					"empty_map", map[string]any{},
+					"nil_list", []any(nil),
+					"nil_map", map[string]any(nil),
+					"nil_ptr", (*int)(nil),
+					"nil", nil,
+				}})
+			},
+			want: here.Doc(`
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestAudit.func6","message":"fake event type","auditEvent":true,"personalInfo":{"username":"redacted","groups":["redacted 2 values"],"int":"redacted","float":"redacted","specialJSONChars\"👋\\":"redacted","map":{"redacted":"redacted 2 keys"},"empty_list":["redacted 0 values"],"empty_map":{"redacted":"redacted 0 keys"},"nil_list":["redacted 0 values"],"nil_map":{"redacted":"redacted 0 keys"},"nil_ptr":"redacted","nil":"redacted"}}
+			`),
+		},
+		{
+			name: "with an illegal single PII keys and values, quietly ignores it",
+			run: func(a AuditLogger) {
+				a.Audit("fake event type", &AuditParams{PIIKeysAndValues: []any{"foo"}})
+			},
+			want: here.Doc(`
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestAudit.func7","message":"fake event type","auditEvent":true}
+			`),
+		},
+		{
+			name: "with an illegal odd number of PII keys and values, quietly ignores the last one",
+			run: func(a AuditLogger) {
+				a.Audit("fake event type", &AuditParams{PIIKeysAndValues: []any{"foo", 42, "bar"}})
+			},
+			want: here.Doc(`
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestAudit.func8","message":"fake event type","auditEvent":true,"personalInfo":{"foo":42}}
+			`),
+		},
+		{
+			name: "with a PII keys that is not a string, converts it to an error-looking key name rather than having the function return errors or panic",
+			run: func(a AuditLogger) {
+				a.Audit("fake event type", &AuditParams{PIIKeysAndValues: []any{42, "foo", "bar", "baz"}})
+			},
+			want: here.Doc(`
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestAudit.func9","message":"fake event type","auditEvent":true,"personalInfo":{"cannotCastKeyNameToString":"foo","bar":"baz"}}
+			`),
+		},
+		{
+			name: "with arbitrary keys and values",
+			run: func(a AuditLogger) {
+				a.Audit("fake event type", &AuditParams{KeysAndValues: []any{"foo", 42, "bar", "baz"}})
+			},
+			want: here.Doc(`
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestAudit.func10","message":"fake event type","auditEvent":true,"foo":42,"bar":"baz"}
+			`),
+		},
+		{
+			name: "with everything, showing order of keys printed in log",
+			run: func(a AuditLogger) {
+				a.Audit("fake event type", &AuditParams{
+					ReqCtx:           fakeReqContext,
+					Session:          &fakeSessionGetter{},
+					PIIKeysAndValues: []any{"username", "ryan", "groups", []string{"g1", "g2"}, "bat", 14},
+					KeysAndValues:    []any{"foo", 42, "bar", "baz"},
+				})
+			},
+			want: here.Doc(`
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestAudit.func11","message":"fake event type","auditEvent":true,"auditID":"fake-audit-id","sessionID":"fake-session-id","personalInfo":{"username":"ryan","groups":["g1","g2"],"bat":14},"foo":42,"bar":"baz"}
+			`),
+		},
+		{
+			name:      "with everything, when PII is redacted, showing order of keys printed in log",
+			redactPII: true,
+			run: func(a AuditLogger) {
+				a.Audit("fake event type", &AuditParams{
+					ReqCtx:           fakeReqContext,
+					Session:          &fakeSessionGetter{},
+					PIIKeysAndValues: []any{"username", "ryan", "groups", []string{"g1", "g2"}, "bat", 14},
+					KeysAndValues:    []any{"foo", 42, "bar", "baz"},
+				})
+			},
+			want: here.Doc(`
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestAudit.func12","message":"fake event type","auditEvent":true,"auditID":"fake-audit-id","sessionID":"fake-session-id","personalInfo":{"username":"redacted","groups":["redacted 2 values"],"bat":"redacted"},"foo":42,"bar":"baz"}
+			`),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+
+			l, actualAuditLogs := TestAuditLoggerWithConfig(t, AuditLogConfig{LogUsernamesAndGroupNames: !test.redactPII})
+			test.run(l)
+
+			require.Equal(t, strings.TrimSpace(test.want), strings.TrimSpace(actualAuditLogs.String()))
+		})
+	}
+}
+
+func TestAuditRequestParams(t *testing.T) {
+	tests := []struct {
+		name            string
+		req             func() *http.Request
+		paramsSafeToLog sets.Set[string]
+		want            string
+		wantErr         *fosite.RFC6749Error
+	}{
+		{
+			name: "get request",
+			req: func() *http.Request {
+				params := url.Values{
+					"foo": []string{"bar1", "bar2"},
+					"baz": []string{"baz1", "baz2"},
+				}
+				req := httptest.NewRequestWithContext(context.Background(), "GET", "/?"+params.Encode(), nil)
+				return req
+			},
+			paramsSafeToLog: sets.New("foo"),
+			want: here.Doc(`
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.(*auditLogger).AuditRequestParams","message":"HTTP Request Parameters","auditEvent":true,"auditID":"some-audit-id","params":{"baz":"redacted","foo":"bar1"},"multiValueParams":{"baz":["redacted","redacted"],"foo":["bar1","bar2"]}}
+			`),
+		},
+		{
+			name: "post request with urlencoded form in body",
+			req: func() *http.Request {
+				params := url.Values{
+					"foo": []string{"bar1", "bar2"},
+					"baz": []string{"baz1", "baz2"},
+				}
+				req := httptest.NewRequestWithContext(context.Background(), "POST", "/", strings.NewReader(params.Encode()))
+				req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+				return req
+			},
+			paramsSafeToLog: sets.New("foo"),
+			want: here.Doc(`
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.(*auditLogger).AuditRequestParams","message":"HTTP Request Parameters","auditEvent":true,"auditID":"some-audit-id","params":{"baz":"redacted","foo":"bar1"},"multiValueParams":{"baz":["redacted","redacted"],"foo":["bar1","bar2"]}}
+			`),
+		},
+		{
+			name: "get request with bad form",
+			req: func() *http.Request {
+				req := httptest.NewRequestWithContext(context.Background(), "GET", "/?invalid;;;form", nil)
+				return req
+			},
+			paramsSafeToLog: sets.New("foo"),
+			wantErr: &fosite.RFC6749Error{
+				CodeField:        fosite.ErrInvalidRequest.CodeField,
+				ErrorField:       fosite.ErrInvalidRequest.ErrorField,
+				DescriptionField: fosite.ErrInvalidRequest.DescriptionField,
+				HintField:        "Unable to parse form params, make sure to send a properly formatted query params or form request body.",
+				DebugField:       "invalid semicolon separator in query",
+			},
+		},
+		{
+			name: "post request with bad urlencoded form in body",
+			req: func() *http.Request {
+				req := httptest.NewRequestWithContext(context.Background(), "POST", "/", strings.NewReader("invalid;;;form"))
+				req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+				return req
+			},
+			paramsSafeToLog: sets.New("foo"),
+			wantErr: &fosite.RFC6749Error{
+				CodeField:        fosite.ErrInvalidRequest.CodeField,
+				ErrorField:       fosite.ErrInvalidRequest.ErrorField,
+				DescriptionField: fosite.ErrInvalidRequest.DescriptionField,
+				HintField:        "Unable to parse form params, make sure to send a properly formatted query params or form request body.",
+				DebugField:       "invalid semicolon separator in query",
+			},
+		},
+		{
+			name: "post request with bad multipart form in body",
+			req: func() *http.Request {
+				req := httptest.NewRequestWithContext(context.Background(), "POST", "/", strings.NewReader("this is not a valid multipart form"))
+				req.Header.Set("Content-Type", "multipart/form-data")
+				return req
+			},
+			paramsSafeToLog: sets.New("foo"),
+			wantErr: &fosite.RFC6749Error{
+				CodeField:        fosite.ErrInvalidRequest.CodeField,
+				ErrorField:       fosite.ErrInvalidRequest.ErrorField,
+				DescriptionField: fosite.ErrInvalidRequest.DescriptionField,
+				HintField:        "Unable to parse multipart HTTP body, make sure to send a properly formatted form request body.",
+				DebugField:       "no multipart boundary param in Content-Type",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+
+			l, actualAuditLogs := TestAuditLogger(t)
+
+			req := test.req()
+			req, _ = auditid.NewRequestWithAuditID(req, func() string { return "some-audit-id" })
+
+			rawErr := l.AuditRequestParams(req, test.paramsSafeToLog)
+
+			if test.wantErr == nil {
+				require.NoError(t, rawErr)
+			} else {
+				require.Error(t, rawErr)
+				err, ok := rawErr.(*fosite.RFC6749Error)
+				require.True(t, ok)
+				require.Equal(t, test.wantErr.CodeField, err.CodeField)
+				require.Equal(t, test.wantErr.ErrorField, err.ErrorField)
+				require.Equal(t, test.wantErr.DescriptionField, err.DescriptionField)
+				require.Equal(t, test.wantErr.HintField, err.HintField)
+				require.Equal(t, test.wantErr.DebugField, err.DebugField)
+			}
+
+			require.Equal(t, strings.TrimSpace(test.want), strings.TrimSpace(actualAuditLogs.String()))
+		})
+	}
+}
+
 func TestPlog(t *testing.T) {
 	runtimeVersion := runtime.Version()
 	if strings.HasPrefix(runtimeVersion, "go") {
@@ -30,246 +335,247 @@ func TestPlog(t *testing.T) {
 		{
 			name: "basic",
 			run:  testAllPlogMethods,
-			want: `
-{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"e","panda":2,"error":"some err"}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"w","warning":true,"panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"we","warning":true,"error":"some err","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"i","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"ie","error":"some err","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"d","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"de","error":"some err","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"t","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"te","error":"some err","panda":2}
-{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"all","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"always","panda":2}
-`,
+			want: here.Doc(`
+				{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"e","panda":2,"error":"some err"}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"w","warning":true,"panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"we","warning":true,"error":"some err","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"i","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"ie","error":"some err","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"d","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"de","error":"some err","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"t","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"te","error":"some err","panda":2}
+				{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"all","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"always","panda":2}
+			`),
 		},
 		{
 			name: "with values",
 			run: func(l Logger) {
 				testAllPlogMethods(l.WithValues("hi", 42))
 			},
-			want: `
-{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"e","hi":42,"panda":2,"error":"some err"}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"w","hi":42,"warning":true,"panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"we","hi":42,"warning":true,"error":"some err","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"i","hi":42,"panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"ie","hi":42,"error":"some err","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"d","hi":42,"panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"de","hi":42,"error":"some err","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"t","hi":42,"panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"te","hi":42,"error":"some err","panda":2}
-{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"all","hi":42,"panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"always","hi":42,"panda":2}
-`,
+			want: here.Doc(`
+				{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"e","hi":42,"panda":2,"error":"some err"}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"w","hi":42,"warning":true,"panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"we","hi":42,"warning":true,"error":"some err","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"i","hi":42,"panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"ie","hi":42,"error":"some err","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"d","hi":42,"panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"de","hi":42,"error":"some err","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"t","hi":42,"panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"te","hi":42,"error":"some err","panda":2}
+				{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"all","hi":42,"panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"always","hi":42,"panda":2}
+			`),
 		},
 		{
 			name: "with values conflict", // duplicate key is included twice ...
 			run: func(l Logger) {
 				testAllPlogMethods(l.WithValues("panda", false))
 			},
-			want: `
-{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"e","panda":false,"panda":2,"error":"some err"}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"w","panda":false,"warning":true,"panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"we","panda":false,"warning":true,"error":"some err","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"i","panda":false,"panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"ie","panda":false,"error":"some err","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"d","panda":false,"panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"de","panda":false,"error":"some err","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"t","panda":false,"panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"te","panda":false,"error":"some err","panda":2}
-{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"all","panda":false,"panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"always","panda":false,"panda":2}
-`,
+			want: here.Doc(`
+				{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"e","panda":false,"panda":2,"error":"some err"}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"w","panda":false,"warning":true,"panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"we","panda":false,"warning":true,"error":"some err","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"i","panda":false,"panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"ie","panda":false,"error":"some err","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"d","panda":false,"panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"de","panda":false,"error":"some err","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"t","panda":false,"panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"te","panda":false,"error":"some err","panda":2}
+				{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"all","panda":false,"panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"always","panda":false,"panda":2}
+			`),
 		},
 		{
 			name: "with values nested",
 			run: func(l Logger) {
 				testAllPlogMethods(l.WithValues("hi", 42).WithValues("not", time.Hour))
 			},
-			want: `
-{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"e","hi":42,"not":"1h0m0s","panda":2,"error":"some err"}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"w","hi":42,"not":"1h0m0s","warning":true,"panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"we","hi":42,"not":"1h0m0s","warning":true,"error":"some err","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"i","hi":42,"not":"1h0m0s","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"ie","hi":42,"not":"1h0m0s","error":"some err","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"d","hi":42,"not":"1h0m0s","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"de","hi":42,"not":"1h0m0s","error":"some err","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"t","hi":42,"not":"1h0m0s","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"te","hi":42,"not":"1h0m0s","error":"some err","panda":2}
-{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"all","hi":42,"not":"1h0m0s","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"always","hi":42,"not":"1h0m0s","panda":2}
-`,
+			want: here.Doc(`
+				{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"e","hi":42,"not":"1h0m0s","panda":2,"error":"some err"}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"w","hi":42,"not":"1h0m0s","warning":true,"panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"we","hi":42,"not":"1h0m0s","warning":true,"error":"some err","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"i","hi":42,"not":"1h0m0s","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"ie","hi":42,"not":"1h0m0s","error":"some err","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"d","hi":42,"not":"1h0m0s","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"de","hi":42,"not":"1h0m0s","error":"some err","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"t","hi":42,"not":"1h0m0s","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"te","hi":42,"not":"1h0m0s","error":"some err","panda":2}
+				{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"all","hi":42,"not":"1h0m0s","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"always","hi":42,"not":"1h0m0s","panda":2}
+			`),
 		},
 		{
 			name: "with name",
 			run: func(l Logger) {
 				testAllPlogMethods(l.WithName("yoyo"))
 			},
-			want: `
-{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"e","panda":2,"error":"some err"}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"w","warning":true,"panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"we","warning":true,"error":"some err","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"i","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"ie","error":"some err","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"d","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"de","error":"some err","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"t","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"te","error":"some err","panda":2}
-{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"all","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"always","panda":2}
-`,
+			want: here.Doc(`
+				{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"e","panda":2,"error":"some err"}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"w","warning":true,"panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"we","warning":true,"error":"some err","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"i","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"ie","error":"some err","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"d","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"de","error":"some err","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"t","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"te","error":"some err","panda":2}
+				{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"all","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"always","panda":2}
+			`),
 		},
 		{
 			name: "with name nested",
 			run: func(l Logger) {
 				testAllPlogMethods(l.WithName("yoyo").WithName("gold"))
 			},
-			want: `
-{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo.gold","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"e","panda":2,"error":"some err"}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo.gold","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"w","warning":true,"panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo.gold","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"we","warning":true,"error":"some err","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo.gold","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"i","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo.gold","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"ie","error":"some err","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo.gold","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"d","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo.gold","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"de","error":"some err","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo.gold","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"t","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo.gold","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"te","error":"some err","panda":2}
-{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo.gold","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"all","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo.gold","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"always","panda":2}
-`,
+			want: here.Doc(`
+				{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo.gold","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"e","panda":2,"error":"some err"}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo.gold","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"w","warning":true,"panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo.gold","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"we","warning":true,"error":"some err","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo.gold","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"i","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo.gold","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"ie","error":"some err","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo.gold","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"d","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo.gold","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"de","error":"some err","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo.gold","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"t","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo.gold","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"te","error":"some err","panda":2}
+				{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo.gold","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"all","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","logger":"yoyo.gold","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"always","panda":2}
+			`),
 		},
 		{
 			name: "depth 3",
 			run: func(l Logger) {
 				testAllPlogMethods(l.withDepth(3))
 			},
-			want: `
-{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"testing/testing.go:<line>$testing.tRunner","message":"e","panda":2,"error":"some err"}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"testing/testing.go:<line>$testing.tRunner","message":"w","warning":true,"panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"testing/testing.go:<line>$testing.tRunner","message":"we","warning":true,"error":"some err","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"testing/testing.go:<line>$testing.tRunner","message":"i","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"testing/testing.go:<line>$testing.tRunner","message":"ie","error":"some err","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"testing/testing.go:<line>$testing.tRunner","message":"d","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"testing/testing.go:<line>$testing.tRunner","message":"de","error":"some err","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"testing/testing.go:<line>$testing.tRunner","message":"t","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"testing/testing.go:<line>$testing.tRunner","message":"te","error":"some err","panda":2}
-{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"testing/testing.go:<line>$testing.tRunner","message":"all","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"testing/testing.go:<line>$testing.tRunner","message":"always","panda":2}
-`,
+			want: here.Doc(`
+				{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"testing/testing.go:<line>$testing.tRunner","message":"e","panda":2,"error":"some err"}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"testing/testing.go:<line>$testing.tRunner","message":"w","warning":true,"panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"testing/testing.go:<line>$testing.tRunner","message":"we","warning":true,"error":"some err","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"testing/testing.go:<line>$testing.tRunner","message":"i","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"testing/testing.go:<line>$testing.tRunner","message":"ie","error":"some err","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"testing/testing.go:<line>$testing.tRunner","message":"d","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"testing/testing.go:<line>$testing.tRunner","message":"de","error":"some err","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"testing/testing.go:<line>$testing.tRunner","message":"t","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"testing/testing.go:<line>$testing.tRunner","message":"te","error":"some err","panda":2}
+				{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"testing/testing.go:<line>$testing.tRunner","message":"all","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"testing/testing.go:<line>$testing.tRunner","message":"always","panda":2}
+			`),
 		},
 		{
 			name: "depth 2",
 			run: func(l Logger) {
 				testAllPlogMethods(l.withDepth(2))
 			},
-			want: `
-{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func16","message":"e","panda":2,"error":"some err"}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func16","message":"w","warning":true,"panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func16","message":"we","warning":true,"error":"some err","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func16","message":"i","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func16","message":"ie","error":"some err","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func16","message":"d","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func16","message":"de","error":"some err","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func16","message":"t","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func16","message":"te","error":"some err","panda":2}
-{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func16","message":"all","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func16","message":"always","panda":2}
-`,
+			want: here.Doc(`
+				{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func16","message":"e","panda":2,"error":"some err"}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func16","message":"w","warning":true,"panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func16","message":"we","warning":true,"error":"some err","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func16","message":"i","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func16","message":"ie","error":"some err","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func16","message":"d","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func16","message":"de","error":"some err","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func16","message":"t","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func16","message":"te","error":"some err","panda":2}
+				{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func16","message":"all","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func16","message":"always","panda":2}
+			`),
 		},
 		{
 			name: "depth 1",
 			run: func(l Logger) {
 				testAllPlogMethods(l.withDepth(1))
 			},
-			want: `
-{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func8","message":"e","panda":2,"error":"some err"}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func8","message":"w","warning":true,"panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func8","message":"we","warning":true,"error":"some err","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func8","message":"i","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func8","message":"ie","error":"some err","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func8","message":"d","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func8","message":"de","error":"some err","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func8","message":"t","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func8","message":"te","error":"some err","panda":2}
-{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func8","message":"all","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func8","message":"always","panda":2}
-`,
+			want: here.Doc(`
+				{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func8","message":"e","panda":2,"error":"some err"}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func8","message":"w","warning":true,"panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func8","message":"we","warning":true,"error":"some err","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func8","message":"i","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func8","message":"ie","error":"some err","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func8","message":"d","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func8","message":"de","error":"some err","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func8","message":"t","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func8","message":"te","error":"some err","panda":2}
+				{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func8","message":"all","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.func8","message":"always","panda":2}
+			`),
 		},
 		{
 			name: "depth 0",
 			run: func(l Logger) {
 				testAllPlogMethods(l.withDepth(0))
 			},
-			want: `
-{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"e","panda":2,"error":"some err"}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"w","warning":true,"panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"we","warning":true,"error":"some err","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"i","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"ie","error":"some err","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"d","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"de","error":"some err","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"t","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"te","error":"some err","panda":2}
-{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"all","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"always","panda":2}
-`,
+			want: here.Doc(`
+				{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"e","panda":2,"error":"some err"}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"w","warning":true,"panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"we","warning":true,"error":"some err","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"i","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"ie","error":"some err","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"d","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"de","error":"some err","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"t","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"te","error":"some err","panda":2}
+				{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"all","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.testAllPlogMethods","message":"always","panda":2}
+			`),
 		},
 		{
 			name: "depth -1",
 			run: func(l Logger) {
 				testAllPlogMethods(l.withDepth(-1))
 			},
-			want: `
-{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.Error","message":"e","panda":2,"error":"some err"}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.Warning","message":"w","warning":true,"panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.WarningErr","message":"we","warning":true,"error":"some err","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.Info","message":"i","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.InfoErr","message":"ie","error":"some err","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.Debug","message":"d","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.DebugErr","message":"de","error":"some err","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.Trace","message":"t","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.TraceErr","message":"te","error":"some err","panda":2}
-{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.All","message":"all","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.Always","message":"always","panda":2}
-`,
+			want: here.Doc(`
+				{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.Error","message":"e","panda":2,"error":"some err"}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.Warning","message":"w","warning":true,"panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.WarningErr","message":"we","warning":true,"error":"some err","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.Info","message":"i","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.InfoErr","message":"ie","error":"some err","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.Debug","message":"d","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.DebugErr","message":"de","error":"some err","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.Trace","message":"t","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.TraceErr","message":"te","error":"some err","panda":2}
+				{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.All","message":"all","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.Always","message":"always","panda":2}
+			`),
 		},
 		{
 			name: "depth -2",
 			run: func(l Logger) {
 				testAllPlogMethods(l.withDepth(-2))
 			},
-			want: `
-{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"logr@v1.4.2/logr.go:<line>$logr.Logger.Error","message":"e","panda":2,"error":"some err"}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.warningDepth","message":"w","warning":true,"panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.warningDepth","message":"we","warning":true,"error":"some err","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.infoDepth","message":"i","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.infoDepth","message":"ie","error":"some err","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.debugDepth","message":"d","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.debugDepth","message":"de","error":"some err","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.traceDepth","message":"t","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.traceDepth","message":"te","error":"some err","panda":2}
-{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"logr@v1.4.2/logr.go:<line>$logr.Logger.Info","message":"all","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"logr@v1.4.2/logr.go:<line>$logr.Logger.Info","message":"always","panda":2}
-`,
+			want: here.Doc(`
+				{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"logr@v1.4.2/logr.go:<line>$logr.Logger.Error","message":"e","panda":2,"error":"some err"}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.warningDepth","message":"w","warning":true,"panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.warningDepth","message":"we","warning":true,"error":"some err","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.infoDepth","message":"i","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.infoDepth","message":"ie","error":"some err","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.debugDepth","message":"d","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.debugDepth","message":"de","error":"some err","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.traceDepth","message":"t","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.traceDepth","message":"te","error":"some err","panda":2}
+				{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"logr@v1.4.2/logr.go:<line>$logr.Logger.Info","message":"all","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"logr@v1.4.2/logr.go:<line>$logr.Logger.Info","message":"always","panda":2}
+			`),
 		},
 		{
 			name: "depth -3",
 			run: func(l Logger) {
 				testAllPlogMethods(l.withDepth(-3))
 			},
-			want: `
-{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"zapr@v1.3.0/zapr.go:<line>$zapr.(*zapLogger).Error","message":"e","panda":2,"error":"some err"}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"logr@v1.4.2/logr.go:<line>$logr.Logger.Info","message":"w","warning":true,"panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"logr@v1.4.2/logr.go:<line>$logr.Logger.Info","message":"we","warning":true,"error":"some err","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"logr@v1.4.2/logr.go:<line>$logr.Logger.Info","message":"i","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"logr@v1.4.2/logr.go:<line>$logr.Logger.Info","message":"ie","error":"some err","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"logr@v1.4.2/logr.go:<line>$logr.Logger.Info","message":"d","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"logr@v1.4.2/logr.go:<line>$logr.Logger.Info","message":"de","error":"some err","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"logr@v1.4.2/logr.go:<line>$logr.Logger.Info","message":"t","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"logr@v1.4.2/logr.go:<line>$logr.Logger.Info","message":"te","error":"some err","panda":2}
-{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"zapr@v1.3.0/zapr.go:<line>$zapr.(*zapLogger).Info","message":"all","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"zapr@v1.3.0/zapr.go:<line>$zapr.(*zapLogger).Info","message":"always","panda":2}`,
+			want: here.Doc(`
+				{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"zapr@v1.3.0/zapr.go:<line>$zapr.(*zapLogger).Error","message":"e","panda":2,"error":"some err"}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"logr@v1.4.2/logr.go:<line>$logr.Logger.Info","message":"w","warning":true,"panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"logr@v1.4.2/logr.go:<line>$logr.Logger.Info","message":"we","warning":true,"error":"some err","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"logr@v1.4.2/logr.go:<line>$logr.Logger.Info","message":"i","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"logr@v1.4.2/logr.go:<line>$logr.Logger.Info","message":"ie","error":"some err","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"logr@v1.4.2/logr.go:<line>$logr.Logger.Info","message":"d","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"logr@v1.4.2/logr.go:<line>$logr.Logger.Info","message":"de","error":"some err","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"logr@v1.4.2/logr.go:<line>$logr.Logger.Info","message":"t","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"logr@v1.4.2/logr.go:<line>$logr.Logger.Info","message":"te","error":"some err","panda":2}
+				{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"zapr@v1.3.0/zapr.go:<line>$zapr.(*zapLogger).Info","message":"all","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"zapr@v1.3.0/zapr.go:<line>$zapr.(*zapLogger).Info","message":"always","panda":2}
+			`),
 		},
 		{
 			name: "closure",
@@ -292,19 +598,19 @@ func TestPlog(t *testing.T) {
 					}()
 				}()
 			},
-			want: fmt.Sprintf(`
-{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.%[1]s","message":"e","panda":2,"error":"some err"}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.%[1]s","message":"w","warning":true,"panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.%[1]s","message":"we","warning":true,"error":"some err","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.%[1]s","message":"i","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.%[1]s","message":"ie","error":"some err","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.%[1]s","message":"d","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.%[1]s","message":"de","error":"some err","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.%[1]s","message":"t","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.%[1]s","message":"te","error":"some err","panda":2}
-{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.%[1]s","message":"all","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.%[1]s","message":"always","panda":2}
-`, func() string {
+			want: here.Docf(`
+				{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.%[1]s","message":"e","panda":2,"error":"some err"}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.%[1]s","message":"w","warning":true,"panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.%[1]s","message":"we","warning":true,"error":"some err","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.%[1]s","message":"i","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.%[1]s","message":"ie","error":"some err","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.%[1]s","message":"d","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.%[1]s","message":"de","error":"some err","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.%[1]s","message":"t","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.%[1]s","message":"te","error":"some err","panda":2}
+				{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.%[1]s","message":"all","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog_test.go:<line>$plog.TestPlog.%[1]s","message":"always","panda":2}
+			`, func() string {
 				switch {
 				case runtimeVersionSemver.Major == 1 && runtimeVersionSemver.Minor == 21:
 					// Format of string for Go 1.21
@@ -340,29 +646,29 @@ func TestPlog(t *testing.T) {
 					}()
 				}()
 			},
-			want: `
-{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.Error","message":"e","panda":2,"error":"some err"}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.Warning","message":"w","warning":true,"panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.WarningErr","message":"we","warning":true,"error":"some err","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.Info","message":"i","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.InfoErr","message":"ie","error":"some err","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.Debug","message":"d","panda":2}
-{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.DebugErr","message":"de","error":"some err","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.Trace","message":"t","panda":2}
-{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.TraceErr","message":"te","error":"some err","panda":2}
-{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.All","message":"all","panda":2}
-{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.Always","message":"always","panda":2}
-`,
+			want: here.Doc(`
+				{"level":"error","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.Error","message":"e","panda":2,"error":"some err"}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.Warning","message":"w","warning":true,"panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.WarningErr","message":"we","warning":true,"error":"some err","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.Info","message":"i","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.InfoErr","message":"ie","error":"some err","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.Debug","message":"d","panda":2}
+				{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.DebugErr","message":"de","error":"some err","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.Trace","message":"t","panda":2}
+				{"level":"trace","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.TraceErr","message":"te","error":"some err","panda":2}
+				{"level":"all","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.All","message":"all","panda":2}
+				{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","caller":"plog/plog.go:<line>$plog.pLogger.Always","message":"always","panda":2}
+			`),
 		},
 	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
 			t.Parallel()
 
-			subjectLogger, log := TestLogger(t)
-			tt.run(subjectLogger)
+			testLogger, log := TestLogger(t)
+			test.run(testLogger)
 
-			require.Equal(t, strings.TrimSpace(tt.want), strings.TrimSpace(log.String()))
+			require.Equal(t, strings.TrimSpace(test.want), strings.TrimSpace(log.String()))
 		})
 	}
 }
@@ -382,3 +688,162 @@ func testAllPlogMethods(l Logger) {
 	l.All("all", "panda", 2)
 	l.Always("always", "panda", 2)
 }
+
+func TestSanitizeRequestParams(t *testing.T) {
+	tests := []struct {
+		name        string
+		params      url.Values
+		allowedKeys sets.Set[string]
+		want        []any
+	}{
+		{
+			name:        "nil values",
+			params:      nil,
+			allowedKeys: nil,
+			want: []any{
+				"params",
+				map[string]string{},
+			},
+		},
+		{
+			name:        "empty values",
+			params:      url.Values{},
+			allowedKeys: nil,
+			want: []any{
+				"params",
+				map[string]string{},
+			},
+		},
+		{
+			name:        "all allowed values",
+			params:      url.Values{"foo": []string{"a", "b", "c"}, "bar": []string{"d", "e", "f"}},
+			allowedKeys: sets.New("foo", "bar"),
+			want: []any{
+				"params",
+				map[string]string{
+					"bar": "d",
+					"foo": "a",
+				},
+				"multiValueParams",
+				url.Values{
+					"bar": []string{"d", "e", "f"},
+					"foo": []string{"a", "b", "c"},
+				},
+			},
+		},
+		{
+			name:        "all allowed values with single values",
+			params:      url.Values{"foo": []string{"a"}, "bar": []string{"d"}},
+			allowedKeys: sets.New("foo", "bar"),
+			want: []any{
+				"params",
+				map[string]string{
+					"foo": "a",
+					"bar": "d",
+				},
+			},
+		},
+		{
+			name:        "some allowed values",
+			params:      url.Values{"foo": []string{"a", "b", "c"}, "bar": []string{"d", "e", "f"}},
+			allowedKeys: sets.New("foo"),
+			want: []any{
+				"params",
+				map[string]string{
+					"bar": "redacted",
+					"foo": "a",
+				},
+				"multiValueParams",
+				url.Values{
+					"bar": []string{"redacted", "redacted", "redacted"},
+					"foo": []string{"a", "b", "c"},
+				},
+			},
+		},
+		{
+			name:        "some allowed values with single values",
+			params:      url.Values{"foo": []string{"a"}, "bar": []string{"d"}},
+			allowedKeys: sets.New("foo"),
+			want: []any{
+				"params",
+				map[string]string{
+					"bar": "redacted",
+					"foo": "a",
+				},
+			},
+		},
+		{
+			name:        "no allowed values",
+			params:      url.Values{"foo": []string{"a", "b", "c"}, "bar": []string{"d", "e", "f"}},
+			allowedKeys: sets.New[string](),
+			want: []any{
+				"params",
+				map[string]string{
+					"bar": "redacted",
+					"foo": "redacted",
+				},
+				"multiValueParams",
+				url.Values{
+					"bar": {"redacted", "redacted", "redacted"},
+					"foo": {"redacted", "redacted", "redacted"},
+				},
+			},
+		},
+		{
+			name:        "nil allowed values",
+			params:      url.Values{"foo": []string{"a", "b", "c"}, "bar": []string{"d", "e", "f"}},
+			allowedKeys: nil,
+			want: []any{
+				"params",
+				map[string]string{
+					"bar": "redacted",
+					"foo": "redacted",
+				},
+				"multiValueParams",
+				url.Values{
+					"bar": {"redacted", "redacted", "redacted"},
+					"foo": {"redacted", "redacted", "redacted"},
+				},
+			},
+		},
+		{
+			name: "url decodes allowed values",
+			params: url.Values{
+				"foo": []string{"a%3Ab", "c", "urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Atoken-exchange"},
+				"bar": []string{"d", "e", "f"},
+			},
+			allowedKeys: sets.New("foo"),
+			want: []any{
+				"params",
+				map[string]string{
+					"bar": "redacted",
+					"foo": "a:b",
+				},
+				"multiValueParams",
+				url.Values{
+					"bar": {"redacted", "redacted", "redacted"},
+					"foo": {"a:b", "c", "urn:ietf:params:oauth:grant-type:token-exchange"},
+				},
+			},
+		},
+		{
+			name: "ignores url decode errors",
+			params: url.Values{
+				"bad_encoding": []string{"%.."},
+			},
+			allowedKeys: sets.New("bad_encoding"),
+			want: []any{
+				"params",
+				map[string]string{
+					"bad_encoding": "%..",
+				},
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			// This comparison should require the exact order
+			require.Equal(t, test.want, sanitizeRequestParams(test.params, test.allowedKeys))
+		})
+	}
+}

internal/plog/testing.go

@@ -71,6 +71,17 @@ func TestLogger(t *testing.T) (Logger, *bytes.Buffer) {
 		&log
 }
 
+func TestAuditLogger(t *testing.T) (AuditLogger, *bytes.Buffer) {
+	return TestAuditLoggerWithConfig(t, AuditLogConfig{LogUsernamesAndGroupNames: true})
+}
+
+func TestAuditLoggerWithConfig(t *testing.T, cfg AuditLogConfig) (AuditLogger, *bytes.Buffer) {
+	t.Helper()
+
+	underlyingLogger, logBuf := TestLogger(t)
+	return &auditLogger{logger: underlyingLogger, cfg: cfg}, logBuf
+}
+
 func TestConsoleLogger(t *testing.T, w io.Writer) Logger {
 	t.Helper()
 

internal/registry/clientsecretrequest/rest.go

@@ -28,8 +28,11 @@ import (
 	"k8s.io/utils/trace"
 
 	clientsecretapi "go.pinniped.dev/generated/latest/apis/supervisor/clientsecret"
+	supervisorconfigv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/config/v1alpha1"
 	configv1alpha1clientset "go.pinniped.dev/generated/latest/client/supervisor/clientset/versioned/typed/config/v1alpha1"
+	"go.pinniped.dev/internal/auditevent"
 	"go.pinniped.dev/internal/oidcclientsecretstorage"
+	"go.pinniped.dev/internal/plog"
 )
 
 // Cost is a good bcrypt cost for 2022, should take about 250 ms to validate.
@@ -48,6 +51,7 @@ func NewREST(
 	randByteGenerator io.Reader,
 	byteHasher byteHasher,
 	timeNowFunc timeNowFunc,
+	auditLogger plog.AuditLogger,
 ) *REST {
 	return &REST{
 		secretStorage:     oidcclientsecretstorage.New(secretsClient),
@@ -58,6 +62,7 @@ func NewREST(
 		byteHasher:        byteHasher,
 		tableConvertor:    rest.NewDefaultTableConvertor(resource),
 		timeNowFunc:       timeNowFunc,
+		auditLogger:       auditLogger,
 	}
 }
 
@@ -70,6 +75,7 @@ type REST struct {
 	byteHasher        byteHasher
 	tableConvertor    rest.TableConvertor
 	timeNowFunc       timeNowFunc
+	auditLogger       plog.AuditLogger
 }
 
 // Assert that our *REST implements all the optional interfaces that we expect it to implement.
@@ -121,7 +127,12 @@ func (*REST) GetSingularName() string {
 	return "oidcclientsecretrequest"
 }
 
-func (r *REST) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
+func (r *REST) Create(
+	ctx context.Context,
+	obj runtime.Object,
+	createValidation rest.ValidateObjectFunc,
+	options *metav1.CreateOptions,
+) (runtime.Object, error) {
 	t := trace.FromContext(ctx).Nest("create",
 		trace.Field{Key: "kind", Value: "OIDCClientSecretRequest"},
 		trace.Field{Key: "metadata.name", Value: name(obj)},
@@ -137,14 +148,9 @@ func (r *REST) Create(ctx context.Context, obj runtime.Object, createValidation
 	t.Step("validateRequest")
 
 	// Find the specified OIDCClient.
-	oidcClient, err := r.oidcClientsClient.Get(ctx, req.Name, metav1.GetOptions{})
+	oidcClient, err := r.findClient(ctx, req.Name, t)
 	if err != nil {
-		traceFailureWithError(t, "oidcClientsClient.Get", err)
-		if apierrors.IsNotFound(err) {
-			errs := field.ErrorList{field.NotFound(field.NewPath("metadata", "name"), req.Name)}
-			return nil, apierrors.NewInvalid(kindFromContext(ctx), req.Name, errs)
-		}
-		return nil, apierrors.NewInternalError(fmt.Errorf("getting client %q failed", req.Name))
+		return nil, err
 	}
 	t.Step("oidcClientsClient.Get")
 
@@ -155,19 +161,20 @@ func (r *REST) Create(ctx context.Context, obj runtime.Object, createValidation
 		traceFailureWithError(t, "secretStorage.Get", err)
 		return nil, apierrors.NewInternalError(fmt.Errorf("getting secret for client %q failed", req.Name))
 	}
+	numPreviouslyStoredHashes := len(hashes)
 	t.Step("secretStorage.Get")
 
 	// If requested, generate a new client secret and add it to the list.
-	var secret string
+	var generatedSecret string
 	if req.Spec.GenerateNewSecret {
-		secret, err = generateSecret(r.randByteGenerator)
+		generatedSecret, err = generateSecret(r.randByteGenerator)
 		if err != nil {
 			traceFailureWithError(t, "generateSecret", err)
 			return nil, apierrors.NewInternalError(fmt.Errorf("client secret generation failed"))
 		}
 		t.Step("generateSecret")
 
-		hash, err := r.byteHasher([]byte(secret), r.cost)
+		hash, err := r.byteHasher([]byte(generatedSecret), r.cost)
 		if err != nil {
 			traceFailureWithError(t, "bcrypt.GenerateFromPassword", err)
 			return nil, apierrors.NewInternalError(fmt.Errorf("hash generation failed"))
@@ -179,8 +186,16 @@ func (r *REST) Create(ctx context.Context, obj runtime.Object, createValidation
 
 	// If requested, remove all client secrets except for the most recent one.
 	needsRevoke := req.Spec.RevokeOldSecrets && len(hashes) > 0
+	numRevokedHashes := 0
 	if needsRevoke {
 		hashes = []string{hashes[0]}
+		if generatedSecret == "" {
+			// There is no newly generated secret, so one old hash is retained and all others are revoked.
+			numRevokedHashes = numPreviouslyStoredHashes - 1
+		} else {
+			// The newly generated secret was added to the list, and all old hashes are revoked.
+			numRevokedHashes = numPreviouslyStoredHashes
+		}
 	}
 
 	// If anything was requested to change...
@@ -204,6 +219,16 @@ func (r *REST) Create(ctx context.Context, obj runtime.Object, createValidation
 			return nil, apierrors.NewInternalError(fmt.Errorf("setting client secret failed"))
 		}
 		t.Step("secretStorage.Set")
+
+		r.auditLogger.Audit(auditevent.OIDCClientSecretRequestUpdatedSecrets, &plog.AuditParams{
+			ReqCtx: ctx,
+			KeysAndValues: []any{
+				"clientID", req.Name,
+				"generatedSecret", len(generatedSecret) > 0,
+				"revokedSecrets", numRevokedHashes,
+				"totalSecrets", len(hashes),
+			},
+		})
 	}
 
 	// Return the new secret in plaintext, if one was generated, along with the total number of secrets.
@@ -218,12 +243,25 @@ func (r *REST) Create(ctx context.Context, obj runtime.Object, createValidation
 			RevokeOldSecrets:  req.Spec.RevokeOldSecrets,
 		},
 		Status: clientsecretapi.OIDCClientSecretRequestStatus{
-			GeneratedSecret:    secret,
+			GeneratedSecret:    generatedSecret,
 			TotalClientSecrets: len(hashes),
 		},
 	}, nil
 }
 
+func (r *REST) findClient(ctx context.Context, clientName string, tracer *trace.Trace) (*supervisorconfigv1alpha1.OIDCClient, error) {
+	oidcClient, err := r.oidcClientsClient.Get(ctx, clientName, metav1.GetOptions{})
+	if err != nil {
+		traceFailureWithError(tracer, "oidcClientsClient.Get", err)
+		if apierrors.IsNotFound(err) {
+			errs := field.ErrorList{field.NotFound(field.NewPath("metadata", "name"), clientName)}
+			return nil, apierrors.NewInvalid(kindFromContext(ctx), clientName, errs)
+		}
+		return nil, apierrors.NewInternalError(fmt.Errorf("getting client %q failed", clientName))
+	}
+	return oidcClient, nil
+}
+
 func (r *REST) validateRequest(
 	ctx context.Context,
 	obj runtime.Object,

internal/registry/clientsecretrequest/rest_test.go

@@ -21,6 +21,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apiserver/pkg/audit"
 	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/rest"
 	kubefake "k8s.io/client-go/kubernetes/fake"
@@ -31,6 +32,7 @@ import (
 	supervisorconfigv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/config/v1alpha1"
 	supervisorfake "go.pinniped.dev/generated/latest/client/supervisor/clientset/versioned/fake"
 	"go.pinniped.dev/internal/oidcclientsecretstorage"
+	"go.pinniped.dev/internal/plog"
 	"go.pinniped.dev/internal/testutil"
 )
 
@@ -44,6 +46,7 @@ func TestNew(t *testing.T) {
 		nil,
 		nil,
 		nil,
+		nil,
 	)
 
 	require.NotNil(t, r)
@@ -120,6 +123,7 @@ func TestCreate(t *testing.T) {
 		wantErrStatus         *metav1.Status
 		wantHashes            *wantHashes
 		wantLogStepSubstrings []string
+		wantAuditLog          []testutil.WantedAuditLog
 	}{
 		{
 			name: "wrong type of request object provided",
@@ -714,6 +718,14 @@ func TestCreate(t *testing.T) {
 				`secretStorage.Set`,
 				`END`,
 			},
+			wantAuditLog: []testutil.WantedAuditLog{
+				testutil.WantAuditLog("OIDCClientSecretRequest Updated Secrets", map[string]any{
+					"clientID":        "client.oauth.pinniped.dev-happy-new-secret",
+					"generatedSecret": true,
+					"revokedSecrets":  float64(0),
+					"totalSecrets":    float64(1),
+				}),
+			},
 		},
 		{
 			name: "happy path: secret exists, prepend new secret hash to secret to the list of hashes for found oidcclient",
@@ -783,6 +795,14 @@ func TestCreate(t *testing.T) {
 				`secretStorage.Set`,
 				`END`,
 			},
+			wantAuditLog: []testutil.WantedAuditLog{
+				testutil.WantAuditLog("OIDCClientSecretRequest Updated Secrets", map[string]any{
+					"clientID":        "client.oauth.pinniped.dev-append-new-secret-hash",
+					"generatedSecret": true,
+					"revokedSecrets":  float64(0),
+					"totalSecrets":    float64(3),
+				}),
+			},
 		},
 		{
 			name: "happy path: secret exists, append new secret hash to secret and revoke old for found oidcclient",
@@ -849,9 +869,17 @@ func TestCreate(t *testing.T) {
 				`secretStorage.Set`,
 				`END`,
 			},
+			wantAuditLog: []testutil.WantedAuditLog{
+				testutil.WantAuditLog("OIDCClientSecretRequest Updated Secrets", map[string]any{
+					"clientID":        "client.oauth.pinniped.dev-append-new-secret-hash",
+					"generatedSecret": true,
+					"revokedSecrets":  float64(2),
+					"totalSecrets":    float64(1),
+				}),
+			},
 		},
 		{
-			name: "happy path: secret exists, revoke old secrets but retain latest for found oidcclient",
+			name: "happy path: secret exists, revoke oldest secrets but retain latest old secret for found oidcclient",
 			args: args{
 				ctx: namespacedContext,
 				obj: &clientsecretapi.OIDCClientSecretRequest{
@@ -874,6 +902,7 @@ func TestCreate(t *testing.T) {
 						[]string{
 							"hashed-password-1",
 							"hashed-password-2",
+							"hashed-password-3",
 						},
 					))
 			},
@@ -913,6 +942,14 @@ func TestCreate(t *testing.T) {
 				`secretStorage.Set`,
 				`END`,
 			},
+			wantAuditLog: []testutil.WantedAuditLog{
+				testutil.WantAuditLog("OIDCClientSecretRequest Updated Secrets", map[string]any{
+					"clientID":        "client.oauth.pinniped.dev-some-client",
+					"generatedSecret": false,
+					"revokedSecrets":  float64(2),
+					"totalSecrets":    float64(1),
+				}),
+			},
 		},
 		{
 			name: "secret exists but oidcclient secret has too many hashes, fails to create when RevokeOldSecrets:false (max 5), secret is not updated",
@@ -1413,6 +1450,14 @@ func TestCreate(t *testing.T) {
 				`secretStorage.Set`,
 				`END`,
 			},
+			wantAuditLog: []testutil.WantedAuditLog{
+				testutil.WantAuditLog("OIDCClientSecretRequest Updated Secrets", map[string]any{
+					"clientID":        "client.oauth.pinniped.dev-some-client",
+					"generatedSecret": true,
+					"revokedSecrets":  float64(1),
+					"totalSecrets":    float64(1),
+				}),
+			},
 		},
 		{
 			name: "happy path: generate new secret when existing secrets is max (5)",
@@ -1482,6 +1527,14 @@ func TestCreate(t *testing.T) {
 				`secretStorage.Set`,
 				`END`,
 			},
+			wantAuditLog: []testutil.WantedAuditLog{
+				testutil.WantAuditLog("OIDCClientSecretRequest Updated Secrets", map[string]any{
+					"clientID":        "client.oauth.pinniped.dev-some-client",
+					"generatedSecret": true,
+					"revokedSecrets":  float64(5),
+					"totalSecrets":    float64(1),
+				}),
+			},
 		},
 		{
 			name: "happy path: generate new secret when existing secrets exceeds maximum (5)",
@@ -1552,6 +1605,14 @@ func TestCreate(t *testing.T) {
 				`secretStorage.Set`,
 				`END`,
 			},
+			wantAuditLog: []testutil.WantedAuditLog{
+				testutil.WantAuditLog("OIDCClientSecretRequest Updated Secrets", map[string]any{
+					"clientID":        "client.oauth.pinniped.dev-some-client",
+					"generatedSecret": true,
+					"revokedSecrets":  float64(6),
+					"totalSecrets":    float64(1),
+				}),
+			},
 		},
 	}
 	for _, tt := range tests {
@@ -1606,6 +1667,10 @@ func TestCreate(t *testing.T) {
 				fakeByteGenerator = strings.NewReader(fakeRandomBytes + "these extra bytes should be ignored since we only read 32 bytes")
 			}
 
+			auditLogger, actualAuditLog := plog.TestAuditLogger(t)
+			ctx := audit.WithAuditContext(tt.args.ctx)
+			audit.WithAuditID(ctx, "fake-audit-id")
+
 			r := NewREST(
 				schema.GroupResource{Group: "bears", Resource: "panda"},
 				secretsClient,
@@ -1615,9 +1680,10 @@ func TestCreate(t *testing.T) {
 				fakeByteGenerator,
 				fakeHasher,
 				fakeTimeNowFunc,
+				auditLogger,
 			)
 
-			got, err := r.Create(tt.args.ctx, tt.args.obj, tt.args.createValidation, tt.args.options)
+			got, err := r.Create(ctx, tt.args.obj, tt.args.createValidation, tt.args.options)
 
 			require.Equal(t, tt.want, got)
 			if tt.wantErrStatus != nil {
@@ -1646,6 +1712,9 @@ func TestCreate(t *testing.T) {
 			}
 
 			requireExactlyOneLogLineWithMultipleSteps(t, logger, tt.wantLogStepSubstrings)
+
+			testutil.WantAuditIDOnEveryAuditLog(tt.wantAuditLog, "fake-audit-id")
+			testutil.CompareAuditLogs(t, tt.wantAuditLog, actualAuditLog.String())
 		})
 	}
 }

internal/registry/credentialrequest/rest.go

@@ -6,6 +6,8 @@ package credentialrequest
 
 import (
 	"context"
+	"crypto/sha256"
+	"errors"
 	"fmt"
 	"time"
 
@@ -18,10 +20,11 @@ import (
 	"k8s.io/apiserver/pkg/authentication/user"
 	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/rest"
-	"k8s.io/utils/trace"
 
 	loginapi "go.pinniped.dev/generated/latest/apis/concierge/login"
+	"go.pinniped.dev/internal/auditevent"
 	"go.pinniped.dev/internal/clientcertissuer"
+	"go.pinniped.dev/internal/plog"
 )
 
 // clientCertificateTTL is the TTL for short-lived client certificates returned by this API.
@@ -31,11 +34,17 @@ type TokenCredentialRequestAuthenticator interface {
 	AuthenticateTokenCredentialRequest(ctx context.Context, req *loginapi.TokenCredentialRequest) (user.Info, error)
 }
 
-func NewREST(authenticator TokenCredentialRequestAuthenticator, issuer clientcertissuer.ClientCertIssuer, resource schema.GroupResource) *REST {
+func NewREST(
+	authenticator TokenCredentialRequestAuthenticator,
+	issuer clientcertissuer.ClientCertIssuer,
+	resource schema.GroupResource,
+	auditLogger plog.AuditLogger,
+) *REST {
 	return &REST{
 		authenticator:  authenticator,
 		issuer:         issuer,
 		tableConvertor: rest.NewDefaultTableConvertor(resource),
+		auditLogger:    auditLogger,
 	}
 }
 
@@ -43,6 +52,7 @@ type REST struct {
 	authenticator  TokenCredentialRequestAuthenticator
 	issuer         clientcertissuer.ClientCertIssuer
 	tableConvertor rest.TableConvertor
+	auditLogger    plog.AuditLogger
 }
 
 // Assert that our *REST implements all the optional interfaces that we expect it to implement.
@@ -92,57 +102,111 @@ func (*REST) GetSingularName() string {
 }
 
 func (r *REST) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
-	t := trace.FromContext(ctx).Nest("create", trace.Field{
-		Key:   "kind",
-		Value: "TokenCredentialRequest",
-	})
-	defer t.Log()
-
-	credentialRequest, err := validateRequest(ctx, obj, createValidation, options, t)
+	credentialRequest, err := validateRequest(ctx, obj, createValidation, options)
 	if err != nil {
+		// Bad requests are not audit logged because the Kubernetes audit log will show the response's status error code.
+		plog.DebugErr("TokenCredentialRequest request object validation error", err)
 		return nil, err
 	}
 
+	// Allow cross-referencing the token with the Supervisor's audit logs.
+	r.auditLogger.Audit(auditevent.TokenCredentialRequestTokenReceived, &plog.AuditParams{
+		ReqCtx: ctx,
+		KeysAndValues: []any{
+			"tokenID", fmt.Sprintf("%x", sha256.Sum256([]byte(credentialRequest.Spec.Token))),
+		},
+	})
+
 	userInfo, err := r.authenticator.AuthenticateTokenCredentialRequest(ctx, credentialRequest)
 	if err != nil {
-		traceFailureWithError(t, "token authentication", err)
-		return failureResponse(), nil
-	}
-	if ok := isUserInfoValid(userInfo); !ok {
-		traceSuccess(t, userInfo, false)
-		return failureResponse(), nil
+		r.auditLogger.Audit(auditevent.TokenCredentialRequestUnexpectedError, &plog.AuditParams{
+			ReqCtx: ctx,
+			KeysAndValues: []any{
+				"reason", "authenticator returned an error",
+				"err", err.Error(),
+				"authenticator", credentialRequest.Spec.Authenticator,
+			},
+		})
+		return authenticationFailedResponse(), nil
 	}
 
-	// this timestamp should be returned from IssueClientCertPEM but this is a safe approximation
-	expires := metav1.NewTime(time.Now().UTC().Add(clientCertificateTTL))
-	certPEM, keyPEM, err := r.issuer.IssueClientCertPEM(userInfo.GetName(), userInfo.GetGroups(), clientCertificateTTL)
+	if userInfo == nil {
+		r.auditLogger.Audit(auditevent.TokenCredentialRequestAuthenticationFailed, &plog.AuditParams{
+			ReqCtx: ctx,
+			KeysAndValues: []any{
+				"reason", "auth rejected by authenticator",
+				"authenticator", credentialRequest.Spec.Authenticator,
+			},
+		})
+		return authenticationFailedResponse(), nil
+	}
+
+	if err = validateUserInfo(userInfo); err != nil {
+		r.auditLogger.Audit(auditevent.TokenCredentialRequestUnsupportedUserInfo, &plog.AuditParams{
+			ReqCtx: ctx,
+			PIIKeysAndValues: []any{
+				"userInfoName", userInfo.GetName(),
+				"userInfoUID", userInfo.GetUID(),
+			},
+			KeysAndValues: []any{
+				"userInfoExtrasCount", len(userInfo.GetExtra()),
+				"reason", "unsupported value in userInfo returned by authenticator",
+				"err", err.Error(),
+				"authenticator", credentialRequest.Spec.Authenticator,
+			},
+		})
+		return authenticationFailedResponse(), nil
+	}
+
+	pem, err := r.issuer.IssueClientCertPEM(userInfo.GetName(), userInfo.GetGroups(), clientCertificateTTL)
 	if err != nil {
-		traceFailureWithError(t, "cert issuer", err)
-		return failureResponse(), nil
+		r.auditLogger.Audit(auditevent.TokenCredentialRequestUnexpectedError, &plog.AuditParams{
+			ReqCtx: ctx,
+			KeysAndValues: []any{
+				"reason", "cert issuer returned an error",
+				"err", err.Error(),
+				"authenticator", credentialRequest.Spec.Authenticator,
+			},
+		})
+		return authenticationFailedResponse(), nil
 	}
 
-	traceSuccess(t, userInfo, true)
+	notBefore := metav1.NewTime(pem.NotBefore)
+	notAfter := metav1.NewTime(pem.NotAfter)
+
+	r.auditLogger.Audit(auditevent.TokenCredentialRequestAuthenticatedUser, &plog.AuditParams{
+		ReqCtx: ctx,
+		PIIKeysAndValues: []any{
+			"username", userInfo.GetName(),
+			"groups", userInfo.GetGroups(),
+		},
+		KeysAndValues: []any{
+			"issuedClientCert", map[string]string{
+				"notBefore": notBefore.Format(time.RFC3339),
+				"notAfter":  notAfter.Format(time.RFC3339),
+			},
+			"authenticator", credentialRequest.Spec.Authenticator,
+		},
+	})
 
 	return &loginapi.TokenCredentialRequest{
 		Status: loginapi.TokenCredentialRequestStatus{
 			Credential: &loginapi.ClusterCredential{
-				ExpirationTimestamp:   expires,
-				ClientCertificateData: string(certPEM),
-				ClientKeyData:         string(keyPEM),
+				ExpirationTimestamp:   notAfter,
+				ClientCertificateData: string(pem.CertPEM),
+				ClientKeyData:         string(pem.KeyPEM),
 			},
 		},
 	}, nil
 }
 
-func validateRequest(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions, t *trace.Trace) (*loginapi.TokenCredentialRequest, error) {
+func validateRequest(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (*loginapi.TokenCredentialRequest, error) {
 	credentialRequest, ok := obj.(*loginapi.TokenCredentialRequest)
 	if !ok {
-		traceValidationFailure(t, "not a TokenCredentialRequest")
 		return nil, apierrors.NewBadRequest(fmt.Sprintf("not a TokenCredentialRequest: %#v", obj))
 	}
 
 	if len(credentialRequest.Spec.Token) == 0 {
-		traceValidationFailure(t, "token must be supplied")
 		errs := field.ErrorList{field.Required(field.NewPath("spec", "token", "value"), "token must be supplied")}
 		return nil, apierrors.NewInvalid(loginapi.Kind(credentialRequest.Kind), credentialRequest.Name, errs)
 	}
@@ -150,14 +214,12 @@ func validateRequest(ctx context.Context, obj runtime.Object, createValidation r
 	// just a sanity check, not sure how to honor a dry run on a virtual API
 	if options != nil {
 		if len(options.DryRun) != 0 {
-			traceValidationFailure(t, "dryRun not supported")
 			errs := field.ErrorList{field.NotSupported(field.NewPath("dryRun"), options.DryRun, []string(nil))}
 			return nil, apierrors.NewInvalid(loginapi.Kind(credentialRequest.Kind), credentialRequest.Name, errs)
 		}
 	}
 
 	if namespace := genericapirequest.NamespaceValue(ctx); len(namespace) != 0 {
-		traceValidationFailure(t, "namespace is not allowed")
 		return nil, apierrors.NewBadRequest(fmt.Sprintf("namespace is not allowed on TokenCredentialRequest: %v", namespace))
 	}
 
@@ -170,7 +232,6 @@ func validateRequest(ctx context.Context, obj runtime.Object, createValidation r
 		requestForValidation := obj.DeepCopyObject()
 		requestForValidation.(*loginapi.TokenCredentialRequest).Spec.Token = ""
 		if err := createValidation(ctx, requestForValidation); err != nil {
-			traceFailureWithError(t, "validation webhook", err)
 			return nil, err
 		}
 	}
@@ -178,48 +239,20 @@ func validateRequest(ctx context.Context, obj runtime.Object, createValidation r
 	return credentialRequest, nil
 }
 
-func isUserInfoValid(userInfo user.Info) bool {
+func validateUserInfo(userInfo user.Info) error {
 	switch {
-	case userInfo == nil, // must be non-nil
-		len(userInfo.GetName()) == 0,  // must have a username, groups are optional
-		len(userInfo.GetUID()) != 0,   // certs cannot assert UID
-		len(userInfo.GetExtra()) != 0: // certs cannot assert extra
-		return false
-
+	case len(userInfo.GetName()) == 0:
+		return errors.New("empty username is not allowed")
+	case len(userInfo.GetUID()) != 0:
+		return errors.New("UIDs are not supported") // certs cannot assert UID
+	case len(userInfo.GetExtra()) != 0:
+		return errors.New("extras are not supported") // certs cannot assert extra
 	default:
-		return true
+		return nil
 	}
 }
 
-func traceSuccess(t *trace.Trace, userInfo user.Info, authenticated bool) {
-	userID := "<none>"
-	hasExtra := false
-	if userInfo != nil {
-		userID = userInfo.GetUID()
-		hasExtra = len(userInfo.GetExtra()) > 0
-	}
-	t.Step("success",
-		trace.Field{Key: "userID", Value: userID},
-		trace.Field{Key: "hasExtra", Value: hasExtra},
-		trace.Field{Key: "authenticated", Value: authenticated},
-	)
-}
-
-func traceValidationFailure(t *trace.Trace, msg string) {
-	t.Step("failure",
-		trace.Field{Key: "failureType", Value: "request validation"},
-		trace.Field{Key: "msg", Value: msg},
-	)
-}
-
-func traceFailureWithError(t *trace.Trace, failureType string, err error) {
-	t.Step("failure",
-		trace.Field{Key: "failureType", Value: failureType},
-		trace.Field{Key: "msg", Value: err.Error()},
-	)
-}
-
-func failureResponse() *loginapi.TokenCredentialRequest {
+func authenticationFailedResponse() *loginapi.TokenCredentialRequest {
 	m := "authentication failed"
 	return &loginapi.TokenCredentialRequest{
 		Status: loginapi.TokenCredentialRequestStatus{

internal/registry/credentialrequest/rest_test.go

@@ -4,35 +4,39 @@
 package credentialrequest
 
 import (
+	"bytes"
 	"context"
+	"crypto/sha256"
 	"errors"
 	"fmt"
 	"testing"
 	"time"
 
-	"github.com/go-logr/logr"
 	"github.com/sclevine/spec"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/mock/gomock"
+	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/audit"
 	"k8s.io/apiserver/pkg/authentication/user"
 	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/rest"
-	"k8s.io/klog/v2"
 	"k8s.io/utils/ptr"
 
 	loginapi "go.pinniped.dev/generated/latest/apis/concierge/login"
+	"go.pinniped.dev/internal/cert"
 	"go.pinniped.dev/internal/clientcertissuer"
 	"go.pinniped.dev/internal/mocks/mockcredentialrequest"
 	"go.pinniped.dev/internal/mocks/mockissuer"
+	"go.pinniped.dev/internal/plog"
 	"go.pinniped.dev/internal/testutil"
 )
 
 func TestNew(t *testing.T) {
-	r := NewREST(nil, nil, schema.GroupResource{Group: "bears", Resource: "panda"})
+	r := NewREST(nil, nil, schema.GroupResource{Group: "bears", Resource: "panda"}, nil)
 	require.NotNil(t, r)
 	require.False(t, r.NamespaceScoped())
 	require.Equal(t, []string{"pinniped"}, r.Categories())
@@ -63,26 +67,28 @@ func TestNew(t *testing.T) {
 	require.Error(t, err, "the resource panda.bears does not support being converted to a Table")
 }
 
+func tokenToHash(tok string) string {
+	return fmt.Sprintf("%x", sha256.Sum256([]byte(tok)))
+}
+
 func TestCreate(t *testing.T) {
 	spec.Run(t, "create", func(t *testing.T, when spec.G, it spec.S) {
 		var r *require.Assertions
 		var ctrl *gomock.Controller
-		var logger *testutil.TranscriptLogger
-		var originalKLogLevel klog.Level
+		var auditLogger plog.AuditLogger
+		var actualAuditLog *bytes.Buffer
+		var fakeNow time.Time
+		var wantAuditLog []testutil.WantedAuditLog
 
 		it.Before(func() {
 			r = require.New(t)
 			ctrl = gomock.NewController(t)
-			logger = testutil.NewTranscriptLogger(t) //nolint:staticcheck  // old test with lots of log statements
-			klog.SetLogger(logr.New(logger))         // this is unfortunately a global logger, so can't run these tests in parallel :(
-			originalKLogLevel = testutil.GetGlobalKlogLevel()
-			// trace.Log() utility will only log at level 2 or above, so set that for this test.
-			testutil.SetGlobalKlogLevel(t, 2) //nolint:staticcheck // old test of code using trace.Log()
+			auditLogger, actualAuditLog = plog.TestAuditLogger(t)
+			fakeNow = time.Date(2024, time.September, 12, 4, 25, 56, 778899, time.UTC)
 		})
 
 		it.After(func() {
-			klog.ClearLogger()
-			testutil.SetGlobalKlogLevel(t, originalKLogLevel) //nolint:staticcheck // old test of code using trace.Log()
+			testutil.CompareAuditLogs(t, wantAuditLog, actualAuditLog.String())
 			ctrl.Finish()
 		})
 
@@ -101,30 +107,52 @@ func TestCreate(t *testing.T) {
 				"test-user",
 				[]string{"test-group-1", "test-group-2"},
 				5*time.Minute,
-			).Return([]byte("test-cert"), []byte("test-key"), nil)
+			).Return(&cert.PEM{
+				CertPEM:   []byte("test-cert"),
+				KeyPEM:    []byte("test-key"),
+				NotBefore: fakeNow.Add(-5 * time.Minute),
+				NotAfter:  fakeNow.Add(5 * time.Minute),
+			}, nil)
 
-			storage := NewREST(requestAuthenticator, clientCertIssuer, schema.GroupResource{})
+			storage := NewREST(requestAuthenticator, clientCertIssuer, schema.GroupResource{}, auditLogger)
 
-			response, err := callCreate(context.Background(), storage, req)
+			response, err := callCreate(storage, req)
 
 			r.NoError(err)
 			r.IsType(&loginapi.TokenCredentialRequest{}, response)
 
-			expires := response.(*loginapi.TokenCredentialRequest).Status.Credential.ExpirationTimestamp
-			r.NotNil(expires)
-			r.InDelta(time.Now().Add(5*time.Minute).Unix(), expires.Unix(), 5)
-			response.(*loginapi.TokenCredentialRequest).Status.Credential.ExpirationTimestamp = metav1.Time{}
-
 			r.Equal(response, &loginapi.TokenCredentialRequest{
 				Status: loginapi.TokenCredentialRequestStatus{
 					Credential: &loginapi.ClusterCredential{
-						ExpirationTimestamp:   metav1.Time{},
+						ExpirationTimestamp:   metav1.NewTime(fakeNow.Add(5 * time.Minute).UTC()),
 						ClientCertificateData: "test-cert",
 						ClientKeyData:         "test-key",
 					},
 				},
 			})
-			requireOneLogStatement(r, logger, `"success" userID:,hasExtra:false,authenticated:true`)
+
+			wantAuditLog = []testutil.WantedAuditLog{
+				testutil.WantAuditLog("TokenCredentialRequest Token Received", map[string]any{
+					"auditID": "fake-audit-id",
+					"tokenID": tokenToHash(req.Spec.Token),
+				}),
+				testutil.WantAuditLog("TokenCredentialRequest Authenticated User", map[string]any{
+					"auditID": "fake-audit-id",
+					"authenticator": map[string]any{
+						"apiGroup": "fake-api-group.com",
+						"kind":     "FakeAuthenticatorKind",
+						"name":     "fake-authenticator-name",
+					},
+					"issuedClientCert": map[string]any{
+						"notBefore": "2024-09-12T04:20:56Z", // this is fakeNow - 5 minutes in UTC
+						"notAfter":  "2024-09-12T04:30:56Z", // this is fakeNow + 5 minutes in UTC
+					},
+					"personalInfo": map[string]any{
+						"username": "test-user",
+						"groups":   []any{"test-group-1", "test-group-2"},
+					},
+				}),
+			}
 		})
 
 		it("CreateFailsWithValidTokenWhenCertIssuerFails", func() {
@@ -140,13 +168,29 @@ func TestCreate(t *testing.T) {
 			clientCertIssuer := mockissuer.NewMockClientCertIssuer(ctrl)
 			clientCertIssuer.EXPECT().
 				IssueClientCertPEM(gomock.Any(), gomock.Any(), gomock.Any()).
-				Return(nil, nil, fmt.Errorf("some certificate authority error"))
+				Return(nil, fmt.Errorf("some certificate authority error"))
 
-			storage := NewREST(requestAuthenticator, clientCertIssuer, schema.GroupResource{})
+			storage := NewREST(requestAuthenticator, clientCertIssuer, schema.GroupResource{}, auditLogger)
 
-			response, err := callCreate(context.Background(), storage, req)
+			response, err := callCreate(storage, req)
 			requireSuccessfulResponseWithAuthenticationFailureMessage(t, err, response)
-			requireOneLogStatement(r, logger, `"failure" failureType:cert issuer,msg:some certificate authority error`)
+
+			wantAuditLog = []testutil.WantedAuditLog{
+				testutil.WantAuditLog("TokenCredentialRequest Token Received", map[string]any{
+					"auditID": "fake-audit-id",
+					"tokenID": tokenToHash(req.Spec.Token),
+				}),
+				testutil.WantAuditLog("TokenCredentialRequest Unexpected Error", map[string]any{
+					"auditID": "fake-audit-id",
+					"authenticator": map[string]any{
+						"apiGroup": "fake-api-group.com",
+						"kind":     "FakeAuthenticatorKind",
+						"name":     "fake-authenticator-name",
+					},
+					"reason": "cert issuer returned an error",
+					"err":    "some certificate authority error",
+				}),
+			}
 		})
 
 		it("CreateSucceedsWithAnUnauthenticatedStatusWhenGivenATokenAndTheWebhookReturnsNilUser", func() {
@@ -155,12 +199,27 @@ func TestCreate(t *testing.T) {
 			requestAuthenticator := mockcredentialrequest.NewMockTokenCredentialRequestAuthenticator(ctrl)
 			requestAuthenticator.EXPECT().AuthenticateTokenCredentialRequest(gomock.Any(), req).Return(nil, nil)
 
-			storage := NewREST(requestAuthenticator, nil, schema.GroupResource{})
+			storage := NewREST(requestAuthenticator, nil, schema.GroupResource{}, auditLogger)
 
-			response, err := callCreate(context.Background(), storage, req)
+			response, err := callCreate(storage, req)
 
 			requireSuccessfulResponseWithAuthenticationFailureMessage(t, err, response)
-			requireOneLogStatement(r, logger, `"success" userID:<none>,hasExtra:false,authenticated:false`)
+
+			wantAuditLog = []testutil.WantedAuditLog{
+				testutil.WantAuditLog("TokenCredentialRequest Token Received", map[string]any{
+					"auditID": "fake-audit-id",
+					"tokenID": tokenToHash(req.Spec.Token),
+				}),
+				testutil.WantAuditLog("TokenCredentialRequest Authentication Failed", map[string]any{
+					"auditID": "fake-audit-id",
+					"authenticator": map[string]any{
+						"apiGroup": "fake-api-group.com",
+						"kind":     "FakeAuthenticatorKind",
+						"name":     "fake-authenticator-name",
+					},
+					"reason": "auth rejected by authenticator",
+				}),
+			}
 		})
 
 		it("CreateSucceedsWithAnUnauthenticatedStatusWhenWebhookFails", func() {
@@ -170,12 +229,28 @@ func TestCreate(t *testing.T) {
 			requestAuthenticator.EXPECT().AuthenticateTokenCredentialRequest(gomock.Any(), req).
 				Return(nil, errors.New("some webhook error"))
 
-			storage := NewREST(requestAuthenticator, nil, schema.GroupResource{})
+			storage := NewREST(requestAuthenticator, nil, schema.GroupResource{}, auditLogger)
 
-			response, err := callCreate(context.Background(), storage, req)
+			response, err := callCreate(storage, req)
 
 			requireSuccessfulResponseWithAuthenticationFailureMessage(t, err, response)
-			requireOneLogStatement(r, logger, `"failure" failureType:token authentication,msg:some webhook error`)
+
+			wantAuditLog = []testutil.WantedAuditLog{
+				testutil.WantAuditLog("TokenCredentialRequest Token Received", map[string]any{
+					"auditID": "fake-audit-id",
+					"tokenID": tokenToHash(req.Spec.Token),
+				}),
+				testutil.WantAuditLog("TokenCredentialRequest Unexpected Error", map[string]any{
+					"auditID": "fake-audit-id",
+					"authenticator": map[string]any{
+						"apiGroup": "fake-api-group.com",
+						"kind":     "FakeAuthenticatorKind",
+						"name":     "fake-authenticator-name",
+					},
+					"reason": "authenticator returned an error",
+					"err":    "some webhook error",
+				}),
+			}
 		})
 
 		it("CreateSucceedsWithAnUnauthenticatedStatusWhenWebhookReturnsAnEmptyUsername", func() {
@@ -183,14 +258,35 @@ func TestCreate(t *testing.T) {
 
 			requestAuthenticator := mockcredentialrequest.NewMockTokenCredentialRequestAuthenticator(ctrl)
 			requestAuthenticator.EXPECT().AuthenticateTokenCredentialRequest(gomock.Any(), req).
-				Return(&user.DefaultInfo{Name: ""}, nil)
+				Return(&user.DefaultInfo{Name: "", UID: "test-uid"}, nil)
 
-			storage := NewREST(requestAuthenticator, nil, schema.GroupResource{})
+			storage := NewREST(requestAuthenticator, nil, schema.GroupResource{}, auditLogger)
 
-			response, err := callCreate(context.Background(), storage, req)
+			response, err := callCreate(storage, req)
 
 			requireSuccessfulResponseWithAuthenticationFailureMessage(t, err, response)
-			requireOneLogStatement(r, logger, `"success" userID:,hasExtra:false,authenticated:false`)
+
+			wantAuditLog = []testutil.WantedAuditLog{
+				testutil.WantAuditLog("TokenCredentialRequest Token Received", map[string]any{
+					"auditID": "fake-audit-id",
+					"tokenID": tokenToHash(req.Spec.Token),
+				}),
+				testutil.WantAuditLog("TokenCredentialRequest Unsupported UserInfo", map[string]any{
+					"auditID": "fake-audit-id",
+					"authenticator": map[string]any{
+						"apiGroup": "fake-api-group.com",
+						"kind":     "FakeAuthenticatorKind",
+						"name":     "fake-authenticator-name",
+					},
+					"reason":              "unsupported value in userInfo returned by authenticator",
+					"err":                 "empty username is not allowed",
+					"userInfoExtrasCount": float64(0),
+					"personalInfo": map[string]any{
+						"userInfoName": "",
+						"userInfoUID":  "test-uid",
+					},
+				}),
+			}
 		})
 
 		it("CreateSucceedsWithAnUnauthenticatedStatusWhenWebhookReturnsAUserWithUID", func() {
@@ -204,12 +300,33 @@ func TestCreate(t *testing.T) {
 					Groups: []string{"test-group-1", "test-group-2"},
 				}, nil)
 
-			storage := NewREST(requestAuthenticator, nil, schema.GroupResource{})
+			storage := NewREST(requestAuthenticator, nil, schema.GroupResource{}, auditLogger)
 
-			response, err := callCreate(context.Background(), storage, req)
+			response, err := callCreate(storage, req)
 
 			requireSuccessfulResponseWithAuthenticationFailureMessage(t, err, response)
-			requireOneLogStatement(r, logger, `"success" userID:test-uid,hasExtra:false,authenticated:false`)
+
+			wantAuditLog = []testutil.WantedAuditLog{
+				testutil.WantAuditLog("TokenCredentialRequest Token Received", map[string]any{
+					"auditID": "fake-audit-id",
+					"tokenID": tokenToHash(req.Spec.Token),
+				}),
+				testutil.WantAuditLog("TokenCredentialRequest Unsupported UserInfo", map[string]any{
+					"auditID": "fake-audit-id",
+					"authenticator": map[string]any{
+						"apiGroup": "fake-api-group.com",
+						"kind":     "FakeAuthenticatorKind",
+						"name":     "fake-authenticator-name",
+					},
+					"reason":              "unsupported value in userInfo returned by authenticator",
+					"err":                 "UIDs are not supported",
+					"userInfoExtrasCount": float64(0),
+					"personalInfo": map[string]any{
+						"userInfoName": "test-user",
+						"userInfoUID":  "test-uid",
+					},
+				}),
+			}
 		})
 
 		it("CreateSucceedsWithAnUnauthenticatedStatusWhenWebhookReturnsAUserWithExtra", func() {
@@ -223,39 +340,58 @@ func TestCreate(t *testing.T) {
 					Extra:  map[string][]string{"test-key": {"test-val-1", "test-val-2"}},
 				}, nil)
 
-			storage := NewREST(requestAuthenticator, nil, schema.GroupResource{})
+			storage := NewREST(requestAuthenticator, nil, schema.GroupResource{}, auditLogger)
 
-			response, err := callCreate(context.Background(), storage, req)
+			response, err := callCreate(storage, req)
 
 			requireSuccessfulResponseWithAuthenticationFailureMessage(t, err, response)
-			requireOneLogStatement(r, logger, `"success" userID:,hasExtra:true,authenticated:false`)
+
+			wantAuditLog = []testutil.WantedAuditLog{
+				testutil.WantAuditLog("TokenCredentialRequest Token Received", map[string]any{
+					"auditID": "fake-audit-id",
+					"tokenID": tokenToHash(req.Spec.Token),
+				}),
+				testutil.WantAuditLog("TokenCredentialRequest Unsupported UserInfo", map[string]any{
+					"auditID": "fake-audit-id",
+					"authenticator": map[string]any{
+						"apiGroup": "fake-api-group.com",
+						"kind":     "FakeAuthenticatorKind",
+						"name":     "fake-authenticator-name",
+					},
+					"reason":              "unsupported value in userInfo returned by authenticator",
+					"err":                 "extras are not supported",
+					"userInfoExtrasCount": float64(1),
+					"personalInfo": map[string]any{
+						"userInfoName": "test-user",
+						"userInfoUID":  "",
+					},
+				}),
+			}
 		})
 
 		it("CreateFailsWhenGivenTheWrongInputType", func() {
 			notACredentialRequest := runtime.Unknown{}
-			response, err := NewREST(nil, nil, schema.GroupResource{}).Create(
+			response, err := NewREST(nil, nil, schema.GroupResource{}, auditLogger).Create(
 				genericapirequest.NewContext(),
 				&notACredentialRequest,
 				rest.ValidateAllObjectFunc,
 				&metav1.CreateOptions{})
 
 			requireAPIError(t, response, err, apierrors.IsBadRequest, "not a TokenCredentialRequest")
-			requireOneLogStatement(r, logger, `"failure" failureType:request validation,msg:not a TokenCredentialRequest`)
 		})
 
 		it("CreateFailsWhenTokenValueIsEmptyInRequest", func() {
-			storage := NewREST(nil, nil, schema.GroupResource{})
-			response, err := callCreate(context.Background(), storage, credentialRequest(loginapi.TokenCredentialRequestSpec{
+			storage := NewREST(nil, nil, schema.GroupResource{}, auditLogger)
+			response, err := callCreate(storage, credentialRequest(loginapi.TokenCredentialRequestSpec{
 				Token: "",
 			}))
 
 			requireAPIError(t, response, err, apierrors.IsInvalid,
 				`.pinniped.dev "request name" is invalid: spec.token.value: Required value: token must be supplied`)
-			requireOneLogStatement(r, logger, `"failure" failureType:request validation,msg:token must be supplied`)
 		})
 
 		it("CreateFailsWhenValidationFails", func() {
-			storage := NewREST(nil, nil, schema.GroupResource{})
+			storage := NewREST(nil, nil, schema.GroupResource{}, auditLogger)
 			response, err := storage.Create(
 				context.Background(),
 				validCredentialRequest(),
@@ -265,7 +401,6 @@ func TestCreate(t *testing.T) {
 				&metav1.CreateOptions{})
 			r.Nil(response)
 			r.EqualError(err, "some validation error")
-			requireOneLogStatement(r, logger, `"failure" failureType:validation webhook,msg:some validation error`)
 		})
 
 		it("CreateDoesNotAllowValidationFunctionToMutateRequest", func() {
@@ -275,9 +410,12 @@ func TestCreate(t *testing.T) {
 			requestAuthenticator.EXPECT().AuthenticateTokenCredentialRequest(gomock.Any(), req.DeepCopy()).
 				Return(&user.DefaultInfo{Name: "test-user"}, nil)
 
-			storage := NewREST(requestAuthenticator, successfulIssuer(ctrl), schema.GroupResource{})
+			fakeReqContext := audit.WithAuditContext(context.Background())
+			audit.WithAuditID(fakeReqContext, "fake-audit-id")
+
+			storage := NewREST(requestAuthenticator, successfulIssuer(ctrl, fakeNow), schema.GroupResource{}, auditLogger)
 			response, err := storage.Create(
-				context.Background(),
+				fakeReqContext,
 				req,
 				func(ctx context.Context, obj runtime.Object) error {
 					credentialRequest, _ := obj.(*loginapi.TokenCredentialRequest)
@@ -287,6 +425,29 @@ func TestCreate(t *testing.T) {
 				&metav1.CreateOptions{})
 			r.NoError(err)
 			r.NotEmpty(response)
+
+			wantAuditLog = []testutil.WantedAuditLog{
+				testutil.WantAuditLog("TokenCredentialRequest Token Received", map[string]any{
+					"auditID": "fake-audit-id",
+					"tokenID": tokenToHash(req.Spec.Token),
+				}),
+				testutil.WantAuditLog("TokenCredentialRequest Authenticated User", map[string]any{
+					"auditID": "fake-audit-id",
+					"authenticator": map[string]any{
+						"apiGroup": "fake-api-group.com",
+						"kind":     "FakeAuthenticatorKind",
+						"name":     "fake-authenticator-name",
+					},
+					"issuedClientCert": map[string]any{
+						"notBefore": "2024-09-12T04:20:56Z", // this is fakeNow - 5 minutes in UTC
+						"notAfter":  "2024-09-12T04:30:56Z", // this is fakeNow + 5 minutes in UTC
+					},
+					"personalInfo": map[string]any{
+						"username": "test-user",
+						"groups":   []any{},
+					},
+				}),
+			}
 		})
 
 		it("CreateDoesNotAllowValidationFunctionToSeeTheActualRequestToken", func() {
@@ -296,11 +457,16 @@ func TestCreate(t *testing.T) {
 			requestAuthenticator.EXPECT().AuthenticateTokenCredentialRequest(gomock.Any(), req.DeepCopy()).
 				Return(&user.DefaultInfo{Name: "test-user"}, nil)
 
-			storage := NewREST(requestAuthenticator, successfulIssuer(ctrl), schema.GroupResource{})
+			storage := NewREST(requestAuthenticator, successfulIssuer(ctrl, fakeNow), schema.GroupResource{}, auditLogger)
+
+			fakeReqContext := audit.WithAuditContext(context.Background())
+			audit.WithAuditID(fakeReqContext, "fake-audit-id")
+
 			validationFunctionWasCalled := false
 			var validationFunctionSawTokenValue string
+
 			response, err := storage.Create(
-				context.Background(),
+				fakeReqContext,
 				req,
 				func(ctx context.Context, obj runtime.Object) error {
 					credentialRequest, _ := obj.(*loginapi.TokenCredentialRequest)
@@ -313,10 +479,33 @@ func TestCreate(t *testing.T) {
 			r.NotEmpty(response)
 			r.True(validationFunctionWasCalled)
 			r.Empty(validationFunctionSawTokenValue)
+
+			wantAuditLog = []testutil.WantedAuditLog{
+				testutil.WantAuditLog("TokenCredentialRequest Token Received", map[string]any{
+					"auditID": "fake-audit-id",
+					"tokenID": tokenToHash(req.Spec.Token),
+				}),
+				testutil.WantAuditLog("TokenCredentialRequest Authenticated User", map[string]any{
+					"auditID": "fake-audit-id",
+					"authenticator": map[string]any{
+						"apiGroup": "fake-api-group.com",
+						"kind":     "FakeAuthenticatorKind",
+						"name":     "fake-authenticator-name",
+					},
+					"issuedClientCert": map[string]any{
+						"notBefore": "2024-09-12T04:20:56Z", // this is fakeNow - 5 minutes in UTC
+						"notAfter":  "2024-09-12T04:30:56Z", // this is fakeNow + 5 minutes in UTC
+					},
+					"personalInfo": map[string]any{
+						"username": "test-user",
+						"groups":   []any{},
+					},
+				}),
+			}
 		})
 
 		it("CreateFailsWhenRequestOptionsDryRunIsNotEmpty", func() {
-			response, err := NewREST(nil, nil, schema.GroupResource{}).Create(
+			response, err := NewREST(nil, nil, schema.GroupResource{}, auditLogger).Create(
 				genericapirequest.NewContext(),
 				validCredentialRequest(),
 				rest.ValidateAllObjectFunc,
@@ -326,32 +515,26 @@ func TestCreate(t *testing.T) {
 
 			requireAPIError(t, response, err, apierrors.IsInvalid,
 				`.pinniped.dev "request name" is invalid: dryRun: Unsupported value: []string{"some dry run flag"}`)
-			requireOneLogStatement(r, logger, `"failure" failureType:request validation,msg:dryRun not supported`)
 		})
 
 		it("CreateFailsWhenNamespaceIsNotEmpty", func() {
-			response, err := NewREST(nil, nil, schema.GroupResource{}).Create(
+			response, err := NewREST(nil, nil, schema.GroupResource{}, auditLogger).Create(
 				genericapirequest.WithNamespace(genericapirequest.NewContext(), "some-ns"),
 				validCredentialRequest(),
 				rest.ValidateAllObjectFunc,
 				&metav1.CreateOptions{})
 
 			requireAPIError(t, response, err, apierrors.IsBadRequest, `namespace is not allowed on TokenCredentialRequest: some-ns`)
-			requireOneLogStatement(r, logger, `"failure" failureType:request validation,msg:namespace is not allowed`)
 		})
 	}, spec.Sequential())
 }
 
-func requireOneLogStatement(r *require.Assertions, logger *testutil.TranscriptLogger, messageContains string) {
-	transcript := logger.Transcript()
-	r.Len(transcript, 1)
-	r.Equal("info", transcript[0].Level)
-	r.Contains(transcript[0].Message, messageContains)
-}
+func callCreate(storage *REST, obj runtime.Object) (runtime.Object, error) {
+	fakeReqContext := audit.WithAuditContext(context.Background())
+	audit.WithAuditID(fakeReqContext, "fake-audit-id")
 
-func callCreate(ctx context.Context, storage *REST, obj runtime.Object) (runtime.Object, error) {
 	return storage.Create(
-		ctx,
+		fakeReqContext,
 		obj,
 		rest.ValidateAllObjectFunc,
 		&metav1.CreateOptions{
@@ -364,7 +547,14 @@ func validCredentialRequest() *loginapi.TokenCredentialRequest {
 }
 
 func validCredentialRequestWithToken(token string) *loginapi.TokenCredentialRequest {
-	return credentialRequest(loginapi.TokenCredentialRequestSpec{Token: token})
+	return credentialRequest(loginapi.TokenCredentialRequestSpec{
+		Token: token,
+		Authenticator: corev1.TypedLocalObjectReference{
+			APIGroup: ptr.To("fake-api-group.com"),
+			Kind:     "FakeAuthenticatorKind",
+			Name:     "fake-authenticator-name",
+		},
+	})
 }
 
 func credentialRequest(spec loginapi.TokenCredentialRequestSpec) *loginapi.TokenCredentialRequest {
@@ -397,10 +587,15 @@ func requireSuccessfulResponseWithAuthenticationFailureMessage(t *testing.T, err
 	})
 }
 
-func successfulIssuer(ctrl *gomock.Controller) clientcertissuer.ClientCertIssuer {
+func successfulIssuer(ctrl *gomock.Controller, fakeNow time.Time) clientcertissuer.ClientCertIssuer {
 	clientCertIssuer := mockissuer.NewMockClientCertIssuer(ctrl)
 	clientCertIssuer.EXPECT().
 		IssueClientCertPEM(gomock.Any(), gomock.Any(), gomock.Any()).
-		Return([]byte("test-cert"), []byte("test-key"), nil)
+		Return(&cert.PEM{
+			CertPEM:   []byte("test-cert"),
+			KeyPEM:    []byte("test-key"),
+			NotBefore: fakeNow.Add(-5 * time.Minute),
+			NotAfter:  fakeNow.Add(5 * time.Minute),
+		}, nil)
 	return clientCertIssuer
 }

internal/supervisor/apiserver/apiserver.go

@@ -39,6 +39,7 @@ type ExtraConfig struct {
 	Secrets                            corev1client.SecretInterface
 	OIDCClients                        configv1alpha1clientset.OIDCClientInterface
 	Namespace                          string
+	AuditLogger                        plog.AuditLogger
 }
 
 type PinnipedServer struct {
@@ -92,6 +93,7 @@ func (c completedConfig) New() (*PinnipedServer, error) {
 				rand.Reader,
 				bcrypt.GenerateFromPassword,
 				metav1.Now,
+				c.ExtraConfig.AuditLogger,
 			)
 			return clientSecretReqGVR, clientSecretReqStorage
 		},

internal/supervisor/server/server.go

@@ -149,6 +149,7 @@ func prepareControllers(
 	pinnipedInformers supervisorinformers.SharedInformerFactory,
 	leaderElector controllerinit.RunnerWrapper,
 	podInfo *downward.PodInfo,
+	auditLogger plog.AuditLogger,
 ) controllerinit.RunnerBuilder {
 	const certificateName string = "pinniped-supervisor-api-tls-serving-certificate"
 	clientSecretSupervisorGroupData := groupsuffix.SupervisorAggregatedGroups(*cfg.APIGroupSuffix)
@@ -167,6 +168,7 @@ func prepareControllers(
 				kubeClient,
 				secretInformer,
 				controllerlib.WithInformer,
+				auditLogger,
 			),
 			singletonWorker,
 		).
@@ -450,6 +452,10 @@ func runSupervisor(ctx context.Context, podInfo *downward.PodInfo, cfg *supervis
 		return fmt.Errorf("cannot create k8s client without leader election: %w", err)
 	}
 
+	auditLogger := plog.NewAuditLogger(plog.AuditLogConfig{
+		LogUsernamesAndGroupNames: cfg.Audit.LogUsernamesAndGroups.Enabled(),
+	})
+
 	kubeInformers := k8sinformers.NewSharedInformerFactoryWithOptions(
 		client.Kubernetes,
 		defaultResyncInterval,
@@ -483,6 +489,8 @@ func runSupervisor(ctx context.Context, podInfo *downward.PodInfo, cfg *supervis
 		&secretCache,
 		clientWithoutLeaderElection.Kubernetes.CoreV1().Secrets(serverInstallationNamespace), // writes to kube storage are allowed for non-leaders
 		client.PinnipedSupervisor.ConfigV1alpha1().OIDCClients(serverInstallationNamespace),
+		auditLogger,
+		cfg.Audit.LogInternalPaths,
 	)
 
 	// Get the "real" name of the client secret supervisor API group (i.e., the API group name with the
@@ -505,6 +513,7 @@ func runSupervisor(ctx context.Context, podInfo *downward.PodInfo, cfg *supervis
 		pinnipedInformers,
 		leaderElector,
 		podInfo,
+		auditLogger,
 	)
 
 	shutdown := &sync.WaitGroup{}
@@ -520,6 +529,7 @@ func runSupervisor(ctx context.Context, podInfo *downward.PodInfo, cfg *supervis
 		clientWithoutLeaderElection.Kubernetes.CoreV1().Secrets(serverInstallationNamespace),
 		client.PinnipedSupervisor.ConfigV1alpha1().OIDCClients(serverInstallationNamespace),
 		serverInstallationNamespace,
+		auditLogger,
 	)
 	if err != nil {
 		return fmt.Errorf("could not configure aggregated API server: %w", err)
@@ -544,7 +554,7 @@ func runSupervisor(ctx context.Context, podInfo *downward.PodInfo, cfg *supervis
 		}
 
 		defer func() { _ = httpListener.Close() }()
-		startServer(ctx, shutdown, httpListener, oidProvidersManager)
+		startServer(ctx, shutdown, httpListener, oidProvidersManager.HandlerChain())
 		plog.Debug("supervisor http listener started", "address", httpListener.Addr().String())
 	}
 
@@ -601,7 +611,7 @@ func runSupervisor(ctx context.Context, podInfo *downward.PodInfo, cfg *supervis
 		}
 
 		defer func() { _ = httpsListener.Close() }()
-		startServer(ctx, shutdown, httpsListener, oidProvidersManager)
+		startServer(ctx, shutdown, httpsListener, oidProvidersManager.HandlerChain())
 		plog.Debug("supervisor https listener started", "address", httpsListener.Addr().String())
 	}
 
@@ -630,6 +640,7 @@ func getAggregatedAPIServerConfig(
 	secrets corev1client.SecretInterface,
 	oidcClients v1alpha1.OIDCClientInterface,
 	serverInstallationNamespace string,
+	auditLogger plog.AuditLogger,
 ) (*apiserver.Config, error) {
 	codecs := serializer.NewCodecFactory(scheme)
 
@@ -696,6 +707,7 @@ func getAggregatedAPIServerConfig(
 			Secrets:                            secrets,
 			OIDCClients:                        oidcClients,
 			Namespace:                          serverInstallationNamespace,
+			AuditLogger:                        auditLogger,
 		},
 	}
 	return apiServerConfig, nil

internal/testutil/log_lines.go

@@ -0,0 +1,118 @@
+// Copyright 2024 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package testutil
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"go.pinniped.dev/internal/federationdomain/stateparam"
+)
+
+func RequireLogLines(t *testing.T, wantLogs []string, log *bytes.Buffer) {
+	t.Helper()
+
+	expectedLogs := ""
+	if len(wantLogs) > 0 {
+		expectedLogs = strings.Join(wantLogs, "\n") + "\n"
+	}
+	require.Equal(t, expectedLogs, log.String())
+}
+
+type WantedAuditLog struct {
+	Message string
+	Params  map[string]any
+}
+
+func WantAuditLog(message string, params map[string]any) WantedAuditLog {
+	result := WantedAuditLog{
+		Message: message,
+		Params:  params,
+	}
+	return result
+}
+
+func WantAuditIDOnEveryAuditLog(wantedAuditLogs []WantedAuditLog, wantAuditID string) {
+	for _, wantedAuditLog := range wantedAuditLogs {
+		wantedAuditLog.Params["auditID"] = wantAuditID
+	}
+}
+
+func GetStateParam(t *testing.T, fullURL string) stateparam.Encoded {
+	if fullURL == "" {
+		var empty stateparam.Encoded
+		return empty
+	}
+
+	path, err := url.Parse(fullURL)
+	require.NoError(t, err)
+	return stateparam.Encoded(path.Query().Get("state"))
+}
+
+func CompareAuditLogs(t *testing.T, wantAuditLogs []WantedAuditLog, actualAuditLogsOneLiner string) {
+	t.Helper()
+
+	// There are tests that verify that no audit events were emitted
+	if len(wantAuditLogs) == 0 {
+		require.Empty(t, actualAuditLogsOneLiner, "no audit events were expected, but some were found")
+		return
+	}
+
+	wantJsonAuditLogs := make([]map[string]any, 0)
+	wantMessages := make([]string, 0)
+	for _, wantAuditLog := range wantAuditLogs {
+		wantJsonAuditLog := make(map[string]any)
+		require.Empty(t, wantAuditLog.Params["level"], "do not specify level in audit log expectations")
+		wantJsonAuditLog["level"] = "info"
+		require.Empty(t, wantAuditLog.Params["message"], "do not specify message in audit log expectations")
+		wantJsonAuditLog["message"] = wantAuditLog.Message
+		wantMessages = append(wantMessages, wantAuditLog.Message)
+		wantJsonAuditLog["auditEvent"] = true
+		require.Empty(t, wantAuditLog.Params["timestamp"], "do not specify timestamp in audit log expectations")
+		wantJsonAuditLog["timestamp"] = "2099-08-08T13:57:36.123456Z"
+		for k, v := range wantAuditLog.Params {
+			wantJsonAuditLog[k] = v
+		}
+		wantJsonAuditLogs = append(wantJsonAuditLogs, wantJsonAuditLog)
+	}
+
+	actualJsonAuditLogs := make([]map[string]any, 0)
+	actualMessages := make([]string, 0)
+	actualAuditLogs := strings.Split(actualAuditLogsOneLiner, "\n")
+	require.GreaterOrEqual(t, len(actualAuditLogs), 2,
+		"expected %d log lines, found %d", len(wantAuditLogs), len(actualAuditLogs)-1)
+	actualAuditLogs = actualAuditLogs[:len(actualAuditLogs)-1] // trim off the last ""
+	for _, actualAuditLog := range actualAuditLogs {
+		actualJsonAuditLog := make(map[string]any)
+		err := json.Unmarshal([]byte(actualAuditLog), &actualJsonAuditLog)
+		require.NoError(t, err)
+
+		// we don't care to test exact equality on the caller - just make sure it is a non-empty string
+		caller, ok := actualJsonAuditLog["caller"]
+		require.True(t, ok)
+		require.NotEmpty(t, caller, "caller for message %q must not be empty", actualJsonAuditLog["message"])
+		delete(actualJsonAuditLog, "caller")
+		actualJsonAuditLogs = append(actualJsonAuditLogs, actualJsonAuditLog)
+
+		actualMessage, ok := actualJsonAuditLog["message"].(string)
+		require.True(t, ok, "actual message is not a string, instead %+v", actualJsonAuditLog["message"])
+		actualMessages = append(actualMessages, actualMessage)
+	}
+
+	// We should check array indices first so that we don't exceed any boundaries.
+	// But we also want to be sure to indicate to the caller what went wrong, so compare the messages.
+	require.Equal(t, wantMessages, actualMessages)
+
+	// We can expect the audit logs to be ordered deterministically.
+	for i := range wantJsonAuditLogs {
+		// compare each item individually so we know which message it is
+		require.Equal(t, wantJsonAuditLogs[i], actualJsonAuditLogs[i],
+			"audit event for message %q does not match", wantJsonAuditLogs[i]["message"])
+	}
+}

internal/testutil/oidctestutil/expected_upstream_state_param.go

@@ -10,6 +10,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	idpdiscoveryv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/idpdiscovery/v1alpha1"
+	"go.pinniped.dev/internal/federationdomain/stateparam"
 )
 
 // ExpectedUpstreamStateParamFormat is a separate type from the production code to ensure that the state
@@ -28,10 +29,10 @@ type ExpectedUpstreamStateParamFormat struct {
 
 type UpstreamStateParamBuilder ExpectedUpstreamStateParamFormat
 
-func (b *UpstreamStateParamBuilder) Build(t *testing.T, stateEncoder *securecookie.SecureCookie) string {
+func (b *UpstreamStateParamBuilder) Build(t *testing.T, stateEncoder *securecookie.SecureCookie) stateparam.Encoded {
 	state, err := stateEncoder.Encode("s", b)
 	require.NoError(t, err)
-	return state
+	return stateparam.Encoded(state)
 }
 
 func (b *UpstreamStateParamBuilder) WithAuthorizeRequestParams(params string) *UpstreamStateParamBuilder {

internal/testutil/oidctestutil/session_storage_assertions.go

@@ -47,7 +47,7 @@ func RequireAuthCodeRegexpMatch(
 	wantDownstreamRedirectURI string,
 	wantCustomSessionData *psession.CustomSessionData,
 	wantDownstreamAdditionalClaims map[string]any,
-) {
+) string {
 	t.Helper()
 
 	// Assert that Location header matches regular expression.
@@ -73,7 +73,7 @@ func RequireAuthCodeRegexpMatch(
 	// One authcode should have been stored.
 	testutil.RequireNumberOfSecretsMatchingLabelSelector(t, secretsClient, labels.Set{crud.SecretLabelKey: authorizationcode.TypeLabelValue}, 1)
 
-	storedRequestFromAuthcode, storedSessionFromAuthcode := validateAuthcodeStorage(
+	sessionID, storedRequestFromAuthcode, storedSessionFromAuthcode := validateAuthcodeStorage(
 		t,
 		oauthStore,
 		authcodeDataAndSignature[1], // Authcode store key is authcode signature
@@ -114,6 +114,8 @@ func RequireAuthCodeRegexpMatch(
 			wantDownstreamNonce,
 		)
 	}
+
+	return sessionID
 }
 
 func includesOpenIDScope(scopes []string) bool {
@@ -139,7 +141,7 @@ func validateAuthcodeStorage(
 	wantDownstreamRedirectURI string,
 	wantCustomSessionData *psession.CustomSessionData,
 	wantDownstreamAdditionalClaims map[string]any,
-) (*fosite.Request, *psession.PinnipedSession) {
+) (string, *fosite.Request, *psession.PinnipedSession) {
 	t.Helper()
 
 	const (
@@ -151,6 +153,8 @@ func validateAuthcodeStorage(
 	storedAuthorizeRequestFromAuthcode, err := oauthStore.GetAuthorizeCodeSession(context.Background(), storeKey, nil)
 	require.NoError(t, err)
 
+	sessionID := storedAuthorizeRequestFromAuthcode.GetID()
+
 	// Check that storage returned the expected concrete data types.
 	storedRequestFromAuthcode, storedSessionFromAuthcode := castStoredAuthorizeRequest(t, storedAuthorizeRequestFromAuthcode)
 
@@ -258,7 +262,7 @@ func validateAuthcodeStorage(
 	// Check that the custom Pinniped session data matches.
 	require.Equal(t, wantCustomSessionData, storedSessionFromAuthcode.Custom)
 
-	return storedRequestFromAuthcode, storedSessionFromAuthcode
+	return sessionID, storedRequestFromAuthcode, storedSessionFromAuthcode
 }
 
 func validatePKCEStorage(

pkg/conciergeclient/conciergeclient.go

@@ -17,6 +17,7 @@ import (
 	clientauthenticationv1beta1 "k8s.io/client-go/pkg/apis/clientauthentication/v1beta1"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	"k8s.io/client-go/transport"
 
 	authenticationv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/authentication/v1alpha1"
 	loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1"
@@ -34,10 +35,11 @@ type Option func(*Client) error
 
 // Client is a configuration for talking to the Pinniped concierge.
 type Client struct {
-	authenticator  *corev1.TypedLocalObjectReference
-	caBundle       string
-	endpoint       *url.URL
-	apiGroupSuffix string
+	authenticator    *corev1.TypedLocalObjectReference
+	caBundle         string
+	endpoint         *url.URL
+	apiGroupSuffix   string
+	transportWrapper transport.WrapperFunc
 }
 
 // WithAuthenticator configures the authenticator reference (spec.authenticator) of the TokenCredentialRequests.
@@ -116,6 +118,16 @@ func WithAPIGroupSuffix(apiGroupSuffix string) Option {
 	}
 }
 
+func WithTransportWrapper(wrapper transport.WrapperFunc) Option {
+	return func(c *Client) error {
+		if wrapper == nil {
+			return fmt.Errorf("transport wrapper cannot be nil")
+		}
+		c.transportWrapper = wrapper
+		return nil
+	}
+}
+
 // New validates the specified options and returns a newly initialized *Client.
 func New(opts ...Option) (*Client, error) {
 	c := Client{apiGroupSuffix: groupsuffix.PinnipedDefaultSuffix}
@@ -158,6 +170,7 @@ func (c *Client) clientset() (conciergeclientset.Interface, error) {
 	client, err := kubeclient.New(
 		kubeclient.WithConfig(cfg),
 		kubeclient.WithMiddleware(groupsuffix.New(c.apiGroupSuffix)),
+		kubeclient.WithTransportWrapper(c.transportWrapper),
 	)
 	if err != nil {
 		return nil, err

pkg/oidcclient/login.go

@@ -33,6 +33,7 @@ import (
 	oidcapi "go.pinniped.dev/generated/latest/apis/supervisor/oidc"
 	"go.pinniped.dev/internal/federationdomain/upstreamprovider"
 	"go.pinniped.dev/internal/httputil/httperr"
+	"go.pinniped.dev/internal/httputil/roundtripper"
 	"go.pinniped.dev/internal/httputil/securityheader"
 	"go.pinniped.dev/internal/net/phttp"
 	"go.pinniped.dev/internal/plog"
@@ -357,6 +358,54 @@ type nopCache struct{}
 func (*nopCache) GetToken(SessionCacheKey) *oidctypes.Token  { return nil }
 func (*nopCache) PutToken(SessionCacheKey, *oidctypes.Token) {}
 
+type auditIDLoggerFunc func(path string, statusCode int, auditID string)
+
+func logFailedRequest(path string, statusCode int, auditID string) {
+	plog.Info("Received auditID for failed request",
+		"path", path,
+		"statusCode", statusCode,
+		"auditID", auditID)
+}
+
+// maybePrintAuditID will choose to log the auditID when certain failure cases are detected,
+// to give a breadcrumb for an admin to follow.
+// Older Supervisors and other OIDC identity providers may not provide this header.
+func maybePrintAuditID(rt http.RoundTripper, logFunc auditIDLoggerFunc) http.RoundTripper {
+	return roundtripper.WrapFunc(rt, func(r *http.Request) (*http.Response, error) {
+		response, responseErr := rt.RoundTrip(r)
+
+		if response == nil ||
+			responseErr != nil ||
+			response.Header.Get("audit-ID") == "" ||
+			response.Request == nil ||
+			response.Request.URL == nil {
+			return response, responseErr
+		}
+
+		auditID := response.Header.Get("audit-ID")
+		// Use the request from the response in case other round-trippers modified the request
+		path := response.Request.URL.Path
+
+		switch statusCode := response.StatusCode; {
+		case statusCode < http.StatusMultipleChoices: // (-inf,300)
+			break // noop
+		case response.StatusCode < http.StatusBadRequest: // [300,400)
+			// Rejected oauth2/authorize redirects from audit-enabled Supervisors will ALWAYS include
+			// the "error" parameter since it is required.
+			// See https://datatracker.ietf.org/doc/html/rfc6749#section-4.1.2.1 for more details.
+			location, err := url.Parse(response.Header.Get(httpLocationHeaderName))
+			if err != nil || location == nil || location.Query().Get("error") == "" {
+				break
+			}
+			logFunc(path, statusCode, auditID)
+		default: // [400,inf)
+			// failing discovery, oauth2/authorize, or oauth2/token responses from audit-enabled Supervisors.
+			logFunc(path, statusCode, auditID)
+		}
+		return response, responseErr
+	})
+}
+
 // Login performs an OAuth2/OIDC authorization code login using a localhost listener.
 func Login(issuer string, clientID string, opts ...Option) (*oidctypes.Token, error) {
 	h := handlerState{
@@ -379,7 +428,15 @@ func Login(issuer string, clientID string, opts ...Option) (*oidctypes.Token, er
 		getEnv:        os.Getenv,
 		listen:        net.Listen,
 		stdinIsTTY:    func() bool { return term.IsTerminal(stdin()) },
-		getProvider:   upstreamoidc.New,
+		getProvider: func(config *oauth2.Config, provider *coreosoidc.Provider, client *http.Client) upstreamprovider.UpstreamOIDCIdentityProviderI {
+			// can't use upstreamoidc.New here since it does not set the Name
+			return &upstreamoidc.ProviderConfig{
+				Name:     issuer, // use the issuer as the Name
+				Config:   config,
+				Provider: provider,
+				Client:   client,
+			}
+		},
 		validateIDToken: func(ctx context.Context, provider *coreosoidc.Provider, audience string, token string) (*coreosoidc.IDToken, error) {
 			return provider.Verifier(&coreosoidc.Config{ClientID: audience}).Verify(ctx, token)
 		},
@@ -393,6 +450,8 @@ func Login(issuer string, clientID string, opts ...Option) (*oidctypes.Token, er
 		}
 	}
 
+	h.httpClient.Transport = maybePrintAuditID(h.httpClient.Transport, logFailedRequest)
+
 	if h.cliToSendCredentials {
 		if h.loginFlow != "" {
 			return nil, fmt.Errorf("do not use deprecated option WithCLISendingCredentials when using option WithLoginFlow")

pkg/oidcclient/login_test.go

@@ -3911,3 +3911,155 @@ func TestLoggers(t *testing.T) {
 
 	// NOTE: We can't really test logs with the default (e.g. no logger option specified)
 }
+
+func TestMaybePrintAuditID(t *testing.T) {
+	canonicalAuditIdHeaderName := "Audit-Id"
+
+	buildResponse := func(statusCode int) *http.Response {
+		return &http.Response{
+			Header: http.Header{
+				canonicalAuditIdHeaderName: []string{"some-audit-id", "some-other-audit-id-that-will-never-be-seen"},
+			},
+			StatusCode: statusCode,
+			Request: &http.Request{
+				URL: &url.URL{
+					Path: "some-path-from-response-request",
+				},
+			},
+		}
+	}
+	tests := []struct {
+		name        string
+		response    *http.Response
+		responseErr error
+		want        func(t *testing.T, called func()) auditIDLoggerFunc
+		wantCalled  bool
+	}{
+		{
+			name:        "happy HTTP response - no error",
+			response:    buildResponse(http.StatusOK), //nolint:bodyclose // there is no Body.
+			responseErr: nil,
+			want: func(t *testing.T, called func()) auditIDLoggerFunc {
+				return func(_ string, _ int, _ string) {
+					called()
+				}
+			},
+			wantCalled: false, // make it obvious
+		},
+		{
+			name: "HTTP response with no response.request.url will not log",
+			response: func() *http.Response {
+				response := buildResponse(http.StatusOK)
+				response.Request.URL = nil
+				return response
+			}(), //nolint:bodyclose // there is no Body.
+			responseErr: nil,
+			want: func(t *testing.T, called func()) auditIDLoggerFunc {
+				return func(_ string, _ int, _ string) {
+					called()
+				}
+			},
+			wantCalled: false, // make it obvious
+		},
+		{
+			name: "302 with error parameter in location and audit-ID will log",
+			response: func() *http.Response {
+				response := buildResponse(http.StatusFound)
+				response.Header.Set("Location", "https://example.com?error=some-error")
+				return response
+			}(), //nolint:bodyclose // there is no Body.
+			responseErr: nil,
+			want: func(t *testing.T, called func()) auditIDLoggerFunc {
+				return func(path string, statusCode int, auditID string) {
+					called()
+					require.Equal(t, "some-path-from-response-request", path)
+					require.Equal(t, http.StatusFound, statusCode)
+					require.Equal(t, "some-audit-id", auditID)
+				}
+			},
+			wantCalled: true,
+		},
+		{
+			name: "303 with error parameter in location and audit-ID will log",
+			response: func() *http.Response {
+				response := buildResponse(http.StatusSeeOther)
+				response.Header.Set("Location", "https://example.com?error=some-error")
+				return response
+			}(), //nolint:bodyclose // there is no Body.
+			responseErr: nil,
+			want: func(t *testing.T, called func()) auditIDLoggerFunc {
+				return func(path string, statusCode int, auditID string) {
+					called()
+					require.Equal(t, "some-path-from-response-request", path)
+					require.Equal(t, http.StatusSeeOther, statusCode)
+					require.Equal(t, "some-audit-id", auditID)
+				}
+			},
+			wantCalled: true,
+		},
+		{
+			name: "303 without error parameter in location and audit-ID will not log",
+			response: func() *http.Response {
+				response := buildResponse(http.StatusSeeOther)
+				response.Header.Set("Location", "https://example.com?foo=bar")
+				return response
+			}(), //nolint:bodyclose // there is no Body.
+			responseErr: nil,
+			want: func(t *testing.T, called func()) auditIDLoggerFunc {
+				return func(path string, statusCode int, auditID string) {
+					called()
+				}
+			},
+			wantCalled: false, // make it obvious
+		},
+		{
+			name:        "404 with error parameter in location and audit-ID will log",
+			response:    buildResponse(http.StatusNotFound), //nolint:bodyclose // there is no Body.
+			responseErr: nil,
+			want: func(t *testing.T, called func()) auditIDLoggerFunc {
+				return func(path string, statusCode int, auditID string) {
+					called()
+					require.Equal(t, "some-path-from-response-request", path)
+					require.Equal(t, http.StatusNotFound, statusCode)
+					require.Equal(t, "some-audit-id", auditID)
+				}
+			},
+			wantCalled: true,
+		},
+		{
+			name:        "when the roundtrip returns an error, will not log",
+			responseErr: errors.New("some error"),
+			want: func(t *testing.T, called func()) auditIDLoggerFunc {
+				return func(path string, statusCode int, auditID string) {
+					called()
+				}
+			},
+			wantCalled: false, // make it obvious
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			require.NotNil(t, test.want)
+
+			mockRequest := &http.Request{
+				URL: &url.URL{
+					Path: "should-never-use-this-path",
+				},
+			}
+			var mockRt roundtripper.Func = func(r *http.Request) (*http.Response, error) {
+				require.Equal(t, mockRequest, r)
+				return test.response, test.responseErr
+			}
+			called := false
+			subjectRt := maybePrintAuditID(mockRt, test.want(t, func() {
+				called = true
+			}))
+			actualResponse, err := subjectRt.RoundTrip(mockRequest) //nolint:bodyclose // there is no Body.
+			require.Equal(t, test.responseErr, err)                 // This roundtripper only returns mocked errors.
+			require.Equal(t, test.response, actualResponse)
+			require.Equal(t, test.wantCalled, called,
+				"want logFunc to be called: %t, actually was called: %t", test.wantCalled, called)
+		})
+	}
+}

proposals/1141_audit-logging/README.md

@@ -1,11 +1,18 @@
 ---
 title: "Audit Logging"
 authors: [ "@cfryanr" ]
-status: "accepted"
+status: "implemented"
 sponsor: [ ]
-approval_date: ""
+approval_date: "June 28, 2022"
 ---
 
+*IMPORTANT NOTE*: This proposal was written in May 2022 and implemented much later in November 2024.
+Due to changes in the Kubernetes ecosystem in the intervening years, this design underwent some
+redesign before implementation. Please see the
+[audit logging documentation](https://pinniped.dev/docs/reference/audit-logging/)
+for a more accurate and up-to-date description of how audit logging
+works. The document below is retained only for historical purposes.
+
 *Disclaimer*: Proposals are point-in-time designs and decisions. Once approved and implemented, they become historical
 documents. If you are reading an old proposal, please be aware that the features described herein might have continued
 to evolve since.

site/content/docs/reference/audit-logging.md

@@ -0,0 +1,537 @@
+---
+title: Supervisor and Concierge Audit Logging
+description: Reference for audit log statements in Pinniped pod logs
+cascade:
+  layout: docs
+menu:
+  docs:
+    name: Audit Logging
+    weight: 40
+    parent: reference
+---
+
+The Pinniped Supervisor and Pinniped Concierge components provide audit logging capabilities
+to help you meet your security and compliance standards.
+
+The configuration of the Pinniped Supervisor and Pinniped Concierge is managed by Kubernetes
+custom resources. These resources are protected by the
+[standard Kubernetes authorization controls](https://kubernetes.io/docs/reference/access-authn-authz/rbac/)
+and audited by the
+[standard Kubernetes audit logging](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/)
+capabilities.
+
+Pinniped also offers additional audit logging capabilities. These additional audit logs appear in
+the pod logs of the Supervisor and Concierge pods. Each line of the pod logs is a JSON object.
+Although these audit events are interleaved with other pod log messages, they are identifiable by always
+having an `"auditEvent":true` key-value pair.
+
+## APIs that can emit Pinniped audit events to the pod logs
+
+Both the Supervisor and the Concierge offer several custom Kubernetes resources for configuration,
+which are protected by Kubernetes RBAC and by default are only available for administrators to use.
+These APIs are not part of the authentication flows for end users.
+Changes to these resources are audited by the standard Kubernetes audit logging.
+Of these resources, only two will emit additional audit events into the Supervisor or Concierge pod logs.
+These audit events can be cross-referenced to the standard Kubernetes audit logs using the value at the `auditID`
+key, which will be the same value in the Supervisor or Concierge pod logs and in the Kubernetes audit logs for
+a particular request to the resource. These resources are:
+- The Supervisor's `OIDCClientSecretRequest` resource. This is used create client secrets for `OIDCClient` resources.
+  It will emit audit events into the Supervisor pod logs to describe the changes to client secrets saved by the request.
+- The Concierge's `TokenCredendtialRequest` resource. This is used to authenticate a user and return a temporary
+  cluster credential for that user. It will emit audit events into the Concierge pod logs to describe the authentication
+  success or authentication failure of the request.
+
+Additionally, the Pinniped Supervisor offers several public APIs for end-user authentication for each
+configured `FederationDomain`. These REST APIs are not represented as Kubernetes resources,
+so they are not audited by the standard Kubernetes audit logging. These APIs will emit Pinniped audit events
+into the Supervisor pod logs. Each request may emit several audit events. These APIs include:
+- `<issuer_path>/.well-known/openid-configuration` is the standard OIDC discovery endpoint, which can be used to discover all the other endpoints listed here.
+- `<issuer_path>/jwks.json` is the standard OIDC JWKS discovery endpoint.
+- `<issuer_path>/v1alpha1/pinniped_identity_providers` is a custom discovery endpoint for clients to learn about available upstream identity providers.
+- `<issuer_path>/oauth2/authorize` is the standard OIDC authorize endpoint.
+- `<issuer_path>/oauth2/token` is the standard OIDC token endpoint.
+  The token endpoint can handle the standard OIDC `authorization_code` and `refresh_token` grant types, and has also been
+  extended to handle an additional grant type for [RFC 8693](https://datatracker.ietf.org/doc/html/rfc8693) token exchanges to
+  reduce the applicable scope (technically, the `aud` claim) of ID tokens.
+- `<issuer_path>/callback` is a special endpoint that is used as the redirect URL when performing an OAuth 2.0 or OIDC authcode flow against an upstream OIDC identity provider as configured by an `OIDCIdentityProvider` or `GitHubIdentityProvider` custom resource.
+- `<issuer_path>/choose_identity_provider` is a UI page which allows users to choose which identity provider they would like to use during a browser-based login flow.
+- `<issuer_path>/login` is a UI page which prompts for username and password to support the optional browser-based login flow for LDAP and Active Directory identity providers.
+
+## Structure of an audit event
+
+Every log line in a Supervisor or Concierge pod log is a JSON object. Only those log lines that include the
+key-value pair `"auditEvent":true` are audit events. Other lines are for errors, warnings, and
+debugging information.
+
+Every line in the pod logs contains the following common keys and values, including audit event log lines:
+
+- `timestamp`, whose value is in UTC time, e.g. `2024-07-10T20:03:26.164470Z`
+- `level`, which for an audit event will always have the value `info`
+- `message`, which for audit events is effectively the audit event type, whose
+  value will always be one of the messages declared as an enum value in
+  [`audit_event.go`](https://github.com/vmware-tanzu/pinniped/blob/main/internal/auditevent/audit_event.go),
+  which is effectively a catalog of all possible audit event types
+- `caller`, which is the line of Go code which caused the log
+- `stacktrace`, which is only included when the global log level is configured to `trace` or `all`,
+  in which case the value shows a full Go stacktrace for the caller
+
+Some audit event log lines may also have the following keys and values, which are specifically designed to help
+correlate an audit event log line to other logs. The values for these keys are opaque and only used for correlation.
+
+- When applicable, audit logs have an `auditID` which is a unique ID for every HTTP request, to allow multiple
+  lines of audit events to be correlated when they came from a single HTTP request. This `auditID` is also returned
+  to the client as an HTTP response header to allow for correlation between the request as observed by the client
+  and the logs as observed by the administrator. Only for `TokenCredendtialRequest` and `OIDCClientSecretRequest`,
+  the `auditID` can also be used to correlate Pinniped audit events with Kubernetes audit logs, which will use the
+  same `auditID` value for a particular request.
+- When applicable, audit logs have a `sessionID` which is the unique ID of a stored Pinniped Supervisor user session,
+  to allow audit events to be correlated which relate to a single session even when they are caused by different
+  HTTP requests or controllers. The same `sessionID` can help you observe all the actions performed during a single user's
+  session across multiple HTTP requests that make up a login, token exchanges, session refreshes, and session
+  expiration (garbage collection).
+- When applicable, audit logs have an `authorizeID` which is a unique ID to allow audit events to be correlated
+  across some of the browser redirects which relate to a single login attempt by an end user. This is only applicable
+  to those browser-based login flows which use redirects to identity providers and/or interstitial pages in the login flow.
+- When applicable, audit logs have a `tokenID` which is a unique ID of a token to allow audit events to be correlated
+  between where a token is issued to an end user in the Supervisor and where a token is used to gain access to a
+  Kubernetes cluster in the Concierge.
+
+Each audit event may also have more key-value pairs specific to the event's type.
+
+## Configuration options for audit events
+
+Logging of audit events is always enabled. There are two configuration options available:
+
+1. By default, usernames and group names are not included in the audit events. This is because these names may
+   include personally identifiable information (PII) which you may wish to avoid sending to your pod logs.
+   However, authentication audit logs can be more useful when this information is included, so there is a
+   configuration option to enable it.
+2. By default, the Supervisor does not audit log requests made to the `/healthz` endpoint, which is used for
+   pod liveness and readiness probes, because it is called so often and it has no behavior other than returning OK.
+
+Both of these can be optionally enabled in the ConfigMaps which hold the pod startup settings for the Supervisor
+and Concierge deployments. When these ConfigMaps are changed, the corresponding Supervisor or Concierge pods must
+be restarted for the new settings to be picked up by the pods. You can find these ConfigMaps by looking at which
+ConfigMap is volume-mounted by the Supervisor or Concierge Deployment.
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata: # ...
+data:
+  pinniped.yaml: |
+    # ...other settings
+
+    audit:
+
+      # This setting is available in both the Supervisor and Concierge ConfigMaps.
+      # When enabled, usernames and group names determined during end-user auth
+      # will be audit logged.
+      logUsernamesAndGroups: enabled
+
+      # This setting is only available in the Supervisor's ConfigMap.
+      # Enables audit logging of the /healthz endpoint.
+      logInternalPaths: enabled
+```
+
+## Exporting Pinniped audit events off-cluster
+
+There are several tools to help cluster administrators export pod logs off-cluster for safe keeping. Because Pinniped
+audit events appear in the pod logs, they will be exported along with the rest of the lines in the pod logs.
+Popular tools, like [Fluentbit](https://fluentbit.io), allow configuration options that could let you
+export only the audit event lines, or export the audit event lines separately from the other log lines.
+This can be achieved by configuring Fluentbit `FILTER`s to evaluate each Supervisor or Concierge pod log line
+based on the presence or absence of the `"auditEvent":true` key-value pair.
+
+## Example of audit event logs
+
+The follow example shows several audit event logs from the Supervisor's pod logs during an end user's browser-based
+login using an OIDC identity provider.
+
+For this example, the `logUsernamesAndGroups` setting is enabled. If it were disabled,
+all values in the `personalInfo` maps shown below would be redacted. The pod logs contain one JSON object per line.
+For readability, we have pretty-printed each line. Also for readability, we have removed the `caller` key
+in the example logs below. In the pod logs, every line includes `caller` and the value identifies the line of
+code which caused the message to be logged.
+
+The login flow starts with the client calling several discovery endpoints.
+We will skip showing those audit logs here for brevity.
+
+Next, the client calls the authorize endpoint to start the login flow.
+A single call to the authorize endpoint causes several audit log events,
+which can be correlated using the `auditID` (request ID) to find all logs related to that single HTTPS request.
+Note that potentially sensitive values such as credentials are automatically redacted in the logs.
+The logs from the authorize endpoint are shown below.
+
+```json lines
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:48:43.566433Z",
+  "message": "HTTP Request Received",
+  "auditEvent": true,
+  "auditID": "2d979b88-0e1e-46d4-8c64-44a0bfa1af17",
+  "proto": "HTTP/2.0",
+  "method": "GET",
+  "host": "example-supervisor.pinniped.dev",
+  "serverName": "example-supervisor.pinniped.dev",
+  "path": "/oauth2/authorize",
+  "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1.1 Safari/605.1.15",
+  "remoteAddr": "1.2.3.4:58586"
+}
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:48:43.566519Z",
+  "message": "HTTP Request Parameters",
+  "auditEvent": true,
+  "auditID": "2d979b88-0e1e-46d4-8c64-44a0bfa1af17",
+  "params": {
+    "access_type": "offline",
+    "client_id": "pinniped-cli",
+    "code_challenge": "redacted",
+    "code_challenge_method": "S256",
+    "nonce": "redacted",
+    "pinniped_idp_name": "My OIDC IDP",
+    "pinniped_idp_type": "oidc",
+    "redirect_uri": "http://127.0.0.1:55379/callback",
+    "response_mode": "form_post",
+    "response_type": "code",
+    "scope": "groups offline_access openid pinniped:request-audience username",
+    "state": "redacted"
+  }
+}
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:48:43.567086Z",
+  "message": "HTTP Request Custom Headers Used",
+  "auditEvent": true,
+  "auditID": "2d979b88-0e1e-46d4-8c64-44a0bfa1af17",
+  "Pinniped-Username": false,
+  "Pinniped-Password": false
+}
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:48:43.567133Z",
+  "message": "Using Upstream IDP",
+  "auditEvent": true,
+  "auditID": "2d979b88-0e1e-46d4-8c64-44a0bfa1af17",
+  "displayName": "My OIDC IDP",
+  "resourceName": "my-oidc-provider",
+  "resourceUID": "754c1c2f-84a4-4e79-981c-8d8ff9da42df",
+  "type": "oidc"
+}
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:48:43.567548Z",
+  "message": "Upstream Authorize Redirect",
+  "auditEvent": true,
+  "auditID": "2d979b88-0e1e-46d4-8c64-44a0bfa1af17",
+  "authorizeID": "fe25634e5094b7f74e4666166f1520436d95bbeeea5109744ca5ad163217a08b"
+}
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:48:43.567576Z",
+  "message": "HTTP Request Completed",
+  "auditEvent": true,
+  "auditID": "2d979b88-0e1e-46d4-8c64-44a0bfa1af17",
+  "path": "/oauth2/authorize",
+  "latency": "1.173084ms",
+  "responseStatus": 303,
+  "location": "https://example-external-oidc.pinniped.dev/auth?client_id=redacted&code_challenge=redacted&code_challenge_method=redacted&nonce=redacted&redirect_uri=redacted&response_type=redacted&scope=redacted&state=redacted"
+}
+```
+
+As shown by the logs above, the authorize endpoint has redirected the user's browser to the external OIDC identity provider
+for authentication. After the user authenticates there, the OIDC provider redirects back to the Supervisor's callback
+endpoint. The `authorizeID` can be used to correlate the logs from the original authorize request, shown above,
+with the logs from this callback request, shown below.
+
+```json lines
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:49:07.764567Z",
+  "message": "HTTP Request Received",
+  "auditEvent": true,
+  "auditID": "1697bdfd-ccdc-4f22-9f30-9b9b8acf964a",
+  "proto": "HTTP/2.0",
+  "method": "GET",
+  "host": "example-supervisor.pinniped.dev",
+  "serverName": "example-supervisor.pinniped.dev",
+  "path": "/callback",
+  "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1.1 Safari/605.1.15",
+  "remoteAddr": "1.2.3.4:58586"
+}
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:49:07.764626Z",
+  "message": "HTTP Request Parameters",
+  "auditEvent": true,
+  "auditID": "1697bdfd-ccdc-4f22-9f30-9b9b8acf964a",
+  "params": {
+    "code": "redacted",
+    "state": "redacted"
+  }
+}
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:49:07.764707Z",
+  "message": "AuthorizeID From Parameters",
+  "auditEvent": true,
+  "auditID": "1697bdfd-ccdc-4f22-9f30-9b9b8acf964a",
+  "authorizeID": "fe25634e5094b7f74e4666166f1520436d95bbeeea5109744ca5ad163217a08b"
+}
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:49:07.764734Z",
+  "message": "Using Upstream IDP",
+  "auditEvent": true,
+  "auditID": "1697bdfd-ccdc-4f22-9f30-9b9b8acf964a",
+  "displayName": "My OIDC IDP",
+  "resourceName": "my-oidc-provider",
+  "resourceUID": "754c1c2f-84a4-4e79-981c-8d8ff9da42df",
+  "type": "oidc"
+}
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:49:07.775753Z",
+  "message": "Identity From Upstream IDP",
+  "auditEvent": true,
+  "auditID": "1697bdfd-ccdc-4f22-9f30-9b9b8acf964a",
+  "personalInfo": {
+    "upstreamUsername": "pinny@example.com",
+    "upstreamGroups": ["developers", "auditors"]
+  },
+  "upstreamIDPDisplayName": "My OIDC IDP",
+  "upstreamIDPType": "oidc",
+  "upstreamIDPResourceName": "my-oidc-provider",
+  "upstreamIDPResourceUID": "754c1c2f-84a4-4e79-981c-8d8ff9da42df"
+}
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:49:07.775859Z",
+  "message": "Session Started",
+  "auditEvent": true,
+  "auditID": "1697bdfd-ccdc-4f22-9f30-9b9b8acf964a",
+  "sessionID": "316fa17f-2ea3-47fd-b7b0-2b02097d8c87",
+  "personalInfo": {
+    "username": "pinny@example.com",
+    "groups": ["developers", "auditors"],
+    "subject": "https://example-external-oidc.pinniped.dev?idpName=My+OIDC+IDP&sub=CiQwNjFkMjNkMS1mZTFlLTQ3NzctOWFlOS01OWNkMTJhYmVhYWESBWxvY2Fs",
+    "additionalClaims": {}
+  },
+  "warnings": []
+}
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:49:07.786155Z",
+  "message": "HTTP Request Completed",
+  "auditEvent": true,
+  "auditID": "1697bdfd-ccdc-4f22-9f30-9b9b8acf964a",
+  "path": "/callback",
+  "latency": "21.603667ms",
+  "responseStatus": 200,
+  "location": "no location header"
+}
+```
+
+The callback endpoint started a Supervisor session for the user and sent an authorization code to the client.
+Note that it logged a new unique `sessionID` for this user session.
+Next, the client will call the token endpoint to exchange that authorization code for tokens. The requests to the
+callback endpoint and the token endpoint can be correlated using the `sessionID`.
+Additionally, all future activity related to this user session can also be correlated using the `sessionID`,
+including session refreshes, token exchanges, and session expiration.
+The logs from the token endpoint are shown below.
+
+```json lines
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:49:11.359739Z",
+  "message": "HTTP Request Received",
+  "auditEvent": true,
+  "auditID": "4effaac3-3f56-4133-9fa8-15104a3022c9",
+  "proto": "HTTP/2.0",
+  "method": "POST",
+  "host": "example-supervisor.pinniped.dev",
+  "serverName": "example-supervisor.pinniped.dev",
+  "path": "/oauth2/token",
+  "userAgent": "pinniped/v0.0.0 (darwin/arm64) kubernetes/$Format",
+  "remoteAddr": "1.2.3.4:59420"
+}
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:49:11.359905Z",
+  "message": "HTTP Request Parameters",
+  "auditEvent": true,
+  "auditID": "4effaac3-3f56-4133-9fa8-15104a3022c9",
+  "params": {
+    "code": "redacted",
+    "code_verifier": "redacted",
+    "grant_type": "authorization_code",
+    "redirect_uri": "http://127.0.0.1:55379/callback"
+  }
+}
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:49:11.359954Z",
+  "message": "HTTP Request Basic Auth",
+  "auditEvent": true,
+  "auditID": "4effaac3-3f56-4133-9fa8-15104a3022c9",
+  "clientID": "pinniped-cli"
+}
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:49:11.372646Z",
+  "message": "Session Found",
+  "auditEvent": true,
+  "auditID": "4effaac3-3f56-4133-9fa8-15104a3022c9",
+  "sessionID": "316fa17f-2ea3-47fd-b7b0-2b02097d8c87"
+}
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:49:11.576172Z",
+  "message": "ID Token Issued",
+  "auditEvent": true,
+  "auditID": "4effaac3-3f56-4133-9fa8-15104a3022c9",
+  "sessionID": "316fa17f-2ea3-47fd-b7b0-2b02097d8c87",
+  "tokenID": "255b785220fe841e950aaf2f78df167991f2b38d2f0b25cc4449301e91d63913"
+}
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:49:11.576319Z",
+  "message": "HTTP Request Completed",
+  "auditEvent": true,
+  "auditID": "4effaac3-3f56-4133-9fa8-15104a3022c9",
+  "path": "/oauth2/token",
+  "latency": "216.627292ms",
+  "responseStatus": 200,
+  "location": "no location header"
+}
+```
+
+Next, the token endpoint is called again to request a new ID token with reduced scope which will only work
+for the target workload cluster (technically, an ID token with a different `aud` claim). These logs are shown below.
+
+```json lines
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:49:11.585635Z",
+  "message": "HTTP Request Received",
+  "auditEvent": true,
+  "auditID": "b49b0a29-b1af-4902-a4fc-bea2c851fcb6",
+  "proto": "HTTP/2.0",
+  "method": "POST",
+  "host": "example-supervisor.pinniped.dev",
+  "serverName": "example-supervisor.pinniped.dev",
+  "path": "/oauth2/token",
+  "userAgent": "pinniped/v0.0.0 (darwin/arm64) kubernetes/$Format",
+  "remoteAddr": "1.2.3.4:59420"
+}
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:49:11.585748Z",
+  "message": "HTTP Request Parameters",
+  "auditEvent": true,
+  "auditID": "b49b0a29-b1af-4902-a4fc-bea2c851fcb6",
+  "params": {
+    "audience": "my-workload-cluster-1f4757da",
+    "client_id": "pinniped-cli",
+    "grant_type": "urn:ietf:params:oauth:grant-type:token-exchange",
+    "requested_token_type": "urn:ietf:params:oauth:token-type:jwt",
+    "subject_token": "redacted",
+    "subject_token_type": "urn:ietf:params:oauth:token-type:access_token"
+  }
+}
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:49:11.766796Z",
+  "message": "Session Found",
+  "auditEvent": true,
+  "auditID": "b49b0a29-b1af-4902-a4fc-bea2c851fcb6",
+  "sessionID": "316fa17f-2ea3-47fd-b7b0-2b02097d8c87"
+}
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:49:11.767113Z",
+  "message": "ID Token Issued",
+  "auditEvent": true,
+  "auditID": "b49b0a29-b1af-4902-a4fc-bea2c851fcb6",
+  "sessionID": "316fa17f-2ea3-47fd-b7b0-2b02097d8c87",
+  "tokenID": "931aabb59f2ecedb1ae9ed1d3c94dd37d169aecce5cbd3dd2096295d3b409720"
+}
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:49:11.767198Z",
+  "message": "HTTP Request Completed",
+  "auditEvent": true,
+  "auditID": "b49b0a29-b1af-4902-a4fc-bea2c851fcb6",
+  "path": "/oauth2/token",
+  "latency": "181.197416ms",
+  "responseStatus": 200,
+  "location": "no location header"
+}
+```
+
+Note that when the ID token is issued, it prints a `tokenID` which is a unique identifier for that
+specific token. Technically, it is a sha256sum of the token. This can be used to cross-reference the usage
+of this specific token to other systems.
+
+Finally, that ID token is submitted to the workload cluster's Concierge to get a temporary credential which
+grants access to that workload cluster. In those logs below, you can see how the `tokenID` can be used
+to follow the user's session to another cluster by following the token. This `TokenCredentialRequest` endpoint
+is a Kubernetes API, so the `auditID` value from the Concierge pod logs will match the `auditID` value in
+the Kubernetes audit logs for the same request, allowing them to be correlated.
+
+```json lines
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:49:11.783402Z",
+  "message": "TokenCredentialRequest Token Received",
+  "auditEvent": true,
+  "auditID": "6776ad70-b587-4bfd-ae41-74ab5e3e00f5",
+  "tokenID": "931aabb59f2ecedb1ae9ed1d3c94dd37d169aecce5cbd3dd2096295d3b409720"
+}
+{
+  "level": "info",
+  "timestamp": "2024-11-21T17:49:11.786405Z",
+  "message": "TokenCredentialRequest Authenticated User",
+  "auditEvent": true,
+  "auditID": "6776ad70-b587-4bfd-ae41-74ab5e3e00f5",
+  "personalInfo": {
+    "username": "pinny@example.com",
+    "groups": ["developers", "auditors"]
+  },
+  "issuedClientCert": {
+    "notAfter": "2024-11-21T17:54:11Z",
+    "notBefore": "2024-11-21T17:44:11Z"
+  },
+  "authenticator": {
+    "apiGroup": "authentication.concierge.pinniped.dev",
+    "kind": "JWTAuthenticator",
+    "name": "my-jwt-authenticator"
+  }
+}
+```
+
+As we've seen, a user's entire authentication journey across clusters can be followed by using the
+`auditID`, `authorizeID`, `sessionID`, and `tokenID` correlation values to find related audit log events.
+The same correlation values could be used to trace a user's journey both forwards and backwards in time
+through the logs.
+
+## Watching the audit logs
+
+Here is a handy command to watch the audit logs from a Supervisor's pod logs which pretty-prints the logs and
+removes keys to make them more terse. A similar command would work for the Concierge's pod logs.
+
+```shell
+kubectl logs --follow --selector=app=pinniped-supervisor -n pinniped-supervisor \
+  | jq --unbuffered -r '. | select(.auditEvent == true) | del(.caller) | del(.level) | del(.auditEvent)'
+```
+
+## End users getting auditIDs
+
+The `auditID` of each request is returned on an HTTP response header to clients.
+
+If an end user encounters an authentication problem, they can get the `auditID` of the failed request to share
+with their Pinniped administrator, who can then search the pod logs to find the audit logs associated with that
+particular request. This may aid in debugging the problem. The end user can set the environment variable
+`PINNIPED_DEBUG=true` while using `kubectl` and other similar tools with their Pinniped-compatible kubeconfig.
+The extra console output caused by that environment variable will include the `auditID` of any failed requests.

site/content/docs/reference/code-walkthrough.md

@@ -166,7 +166,7 @@ as aggregated API endpoints, which makes them appear to a client almost as if th
   as that user.
   It is in [internal/registry/credentialrequest/rest.go](https://github.com/vmware-tanzu/pinniped/blob/main/internal/registry/credentialrequest/rest.go).
 
-- `WhoAmI` will return basic details about the currently authenticated user.
+- `WhoAmIRequest` will return basic details about the currently authenticated user.
   It is in [internal/registry/whoamirequest/rest.go](https://github.com/vmware-tanzu/pinniped/blob/main/internal/registry/whoamirequest/rest.go).
 
 The Concierge may also run an impersonation proxy service. This is not an aggregated API endpoint, so it needs to be
@@ -200,6 +200,8 @@ The per-FederationDomain endpoints are:
   See [internal/federationdomain/endpoints/discovery/discovery_handler.go](https://github.com/vmware-tanzu/pinniped/blob/main/internal/federationdomain/endpoints/discovery/discovery_handler.go).
 - `<issuer_path>/jwks.json` is the standard OIDC JWKS discovery endpoint.
   See [internal/federationdomain/endpoints/jwks/jwks_handler.go](https://github.com/vmware-tanzu/pinniped/blob/main/internal/federationdomain/endpoints/jwks/jwks_handler.go).
+- `<issuer_path>/v1alpha1/pinniped_identity_providers` is a custom discovery endpoint for clients to learn about available upstream identity providers.
+  See [internal/federationdomain/endpoints/idpdiscovery/idp_discovery_handler.go](https://github.com/vmware-tanzu/pinniped/blob/main/internal/federationdomain/endpoints/idpdiscovery/idp_discovery_handler.go).
 - `<issuer_path>/oauth2/authorize` is the standard OIDC authorize endpoint.
   See [internal/federationdomain/endpoints/auth/auth_handler.go](https://github.com/vmware-tanzu/pinniped/blob/main/internal/federationdomain/endpoints/auth/auth_handler.go).
 - `<issuer_path>/oauth2/token` is the standard OIDC token endpoint.
@@ -210,9 +212,9 @@ The per-FederationDomain endpoints are:
   reduce the applicable scope (technically, the `aud` claim) of ID tokens.
 - `<issuer_path>/callback` is a special endpoint that is used as the redirect URL when performing an OAuth 2.0 or OIDC authcode flow against an upstream OIDC identity provider as configured by an OIDCIdentityProvider or GitHubIdentityProvider custom resource.
   See [internal/federationdomain/endpoints/callback/callback_handler.go](https://github.com/vmware-tanzu/pinniped/blob/main/internal/federationdomain/endpoints/callback/callback_handler.go).
-- `<issuer_path>/v1alpha1/pinniped_identity_providers` is a custom discovery endpoint for clients to learn about available upstream identity providers.
-  See [internal/federationdomain/endpoints/idpdiscovery/idp_discovery_handler.go](https://github.com/vmware-tanzu/pinniped/blob/main/internal/federationdomain/endpoints/idpdiscovery/idp_discovery_handler.go).
-- `<issuer_path>/login` is a login UI page to support the optional browser-based login flow for LDAP and Active Directory identity providers.
+- `<issuer_path>/choose_identity_provider` is a UI page which allows users to choose which identity provider they would like to use during a browser-based login flow.
+  See [internal/federationdomain/endpoints/chooseidp/choose_idp_handler.go](https://github.com/vmware-tanzu/pinniped/blob/main/internal/federationdomain/endpoints/chooseidp/choose_idp_handler.go).
+- `<issuer_path>/login` is a UI page which prompts for username and password to support the optional browser-based login flow for LDAP and Active Directory identity providers.
   See [internal/federationdomain/endpoints/login/login_handler.go](https://github.com/vmware-tanzu/pinniped/blob/main/internal/federationdomain/endpoints/login/login_handler.go).
 
 The OIDC specifications implemented by the Supervisor can be found at [openid.net](https://openid.net/connect).

site/content/docs/reference/fips.md

@@ -13,15 +13,14 @@ By default, the Pinniped supervisor and concierge use ciphers that
 are not supported by FIPS 140-2. If you are deploying Pinniped in an
 environment with FIPS compliance requirements, you will have to build
 the binaries yourself using the `fips_strict` build tag and Golang's
-`go-boringcrypto` fork.
+`GOEXPERIMENT=boringcrypto` compiler option.
 
 The Pinniped team provides an [example Dockerfile](https://github.com/vmware-tanzu/pinniped/blob/main/hack/Dockerfile_fips)
 demonstrating how you can build Pinniped images in a FIPS compatible way.
-However, we do not provide official support for FIPS configuration, and we may not
-respond to GitHub issues opened related to FIPS support.
+However, we do not provide official support for FIPS configuration.
 We provide this for informational purposes only.
 
-To build Pinniped use our example fips Dockerfile, you can run:
+To build Pinniped use our example FIPS Dockerfile, you can run:
 ```bash
 $ git clone git@github.com:vmware-tanzu/pinniped.git
 $ cd pinniped

test/integration/audit_test.go

@@ -0,0 +1,784 @@
+// Copyright 2024 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package integration
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/utils/ptr"
+	"sigs.k8s.io/yaml"
+
+	authenticationv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/authentication/v1alpha1"
+	supervisorconfigv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/config/v1alpha1"
+	idpv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/idp/v1alpha1"
+	"go.pinniped.dev/internal/auditevent"
+	"go.pinniped.dev/internal/certauthority"
+	"go.pinniped.dev/internal/config/concierge"
+	"go.pinniped.dev/internal/config/supervisor"
+	"go.pinniped.dev/internal/kubeclient"
+	"go.pinniped.dev/test/testlib"
+)
+
+// kubeClientWithoutPinnipedAPISuffix is much like testlib.NewKubernetesClientset but does not
+// use middleware to change the Pinniped API suffix (kubeclient.WithMiddleware).
+//
+// The returned kubeclient is only for interacting with K8s-native objects, not Pinniped objects,
+// so it does not need to be aware of Pinniped's API suffix.
+func kubeClientWithoutPinnipedAPISuffix(t *testing.T) kubernetes.Interface {
+	t.Helper()
+
+	client, err := kubeclient.New(kubeclient.WithConfig(testlib.NewClientConfig(t)))
+	require.NoError(t, err)
+
+	return client.Kubernetes
+}
+
+// TestAuditLogsDuringLogin is an end-to-end login test which cares more about making audit log
+// assertions than assertions about the login itself. Much of how this test performs a login was
+// inspired by a test case from TestE2EFullIntegration_Browser. This test is Disruptive because
+// it restarts the Supervisor and Concierge to reconfigure audit logging, and then restarts them
+// again to put back the original configuration.
+func TestAuditLogsDuringLogin_Disruptive(t *testing.T) {
+	env := testEnvForPodShutdownTests(t)
+
+	testStartTime := metav1.Now()
+
+	ctx, cancelFunc := context.WithTimeout(context.Background(), 10*time.Minute)
+	defer cancelFunc()
+
+	kubeClient := testlib.NewKubernetesClientset(t)
+	kubeClientForK8sResourcesOnly := kubeClientWithoutPinnipedAPISuffix(t)
+
+	// Build pinniped CLI.
+	pinnipedExe := testlib.PinnipedCLIPath(t)
+
+	supervisorIssuer := env.InferSupervisorIssuerURL(t)
+
+	// Generate a CA bundle with which to serve this provider.
+	t.Logf("generating test CA")
+	tlsServingCertForSupervisorSecretName := "federation-domain-serving-cert-" + testlib.RandHex(t, 8)
+
+	federationDomainSelfSignedCA := createTLSServingCertSecretForSupervisor(
+		ctx,
+		t,
+		env,
+		supervisorIssuer,
+		tlsServingCertForSupervisorSecretName,
+		kubeClient,
+	)
+
+	// Save that bundle plus the one that signs the upstream issuer, for test purposes.
+	federationDomainCABundlePath := filepath.Join(t.TempDir(), "test-ca.pem")
+	federationDomainCABundlePEM := federationDomainSelfSignedCA.Bundle()
+	require.NoError(t, os.WriteFile(federationDomainCABundlePath, federationDomainCABundlePEM, 0600))
+
+	// Create the downstream FederationDomain.
+	// This helper function will nil out spec.TLS if spec.Issuer is an IP address.
+	federationDomain := testlib.CreateTestFederationDomain(ctx, t,
+		supervisorconfigv1alpha1.FederationDomainSpec{
+			Issuer: supervisorIssuer.Issuer(),
+			TLS:    &supervisorconfigv1alpha1.FederationDomainTLSSpec{SecretName: tlsServingCertForSupervisorSecretName},
+		},
+		supervisorconfigv1alpha1.FederationDomainPhaseError, // in phase error until there is an IDP created
+	)
+
+	expectedUsername := env.SupervisorUpstreamLDAP.TestUserMailAttributeValue
+	expectedGroups := make([]any, len(env.SupervisorUpstreamLDAP.TestUserDirectGroupsDNs))
+	for i, g := range env.SupervisorUpstreamLDAP.TestUserDirectGroupsDNs {
+		expectedGroups[i] = g
+	}
+
+	// Create a JWTAuthenticator that will validate the tokens from the downstream issuer.
+	// If the FederationDomain is not Ready, the JWTAuthenticator cannot be ready, either.
+	clusterAudience := "test-cluster-" + testlib.RandHex(t, 8)
+	defaultJWTAuthenticatorSpec := authenticationv1alpha1.JWTAuthenticatorSpec{
+		Issuer:   federationDomain.Spec.Issuer,
+		Audience: clusterAudience,
+		TLS:      &authenticationv1alpha1.TLSSpec{CertificateAuthorityData: base64.StdEncoding.EncodeToString(federationDomainCABundlePEM)},
+	}
+	authenticator := testlib.CreateTestJWTAuthenticator(ctx, t, defaultJWTAuthenticatorSpec, authenticationv1alpha1.JWTAuthenticatorPhaseError)
+	setupClusterForEndToEndLDAPTest(t, expectedUsername, env)
+	testlib.WaitForFederationDomainStatusPhase(ctx, t, federationDomain.Name, supervisorconfigv1alpha1.FederationDomainPhaseReady)
+	testlib.WaitForJWTAuthenticatorStatusPhase(ctx, t, authenticator.Name, authenticationv1alpha1.JWTAuthenticatorPhaseReady)
+
+	tempDir := t.TempDir() // per-test tmp dir to avoid sharing files between tests
+	// Use a specific session cache for this test.
+	sessionCachePath := tempDir + "/test-sessions.yaml"
+	credentialCachePath := tempDir + "/test-credentials.yaml"
+
+	pinnipedStyleKubeconfigPath := runPinnipedGetKubeconfig(t, env, pinnipedExe, tempDir, []string{
+		"get", "kubeconfig",
+		"--concierge-api-group-suffix", env.APIGroupSuffix,
+		"--concierge-authenticator-type", "jwt",
+		"--concierge-authenticator-name", authenticator.Name,
+		"--oidc-session-cache", sessionCachePath,
+		"--credential-cache", credentialCachePath,
+		// use default for --oidc-scopes, which is to request all relevant scopes
+	})
+
+	t.Setenv("PINNIPED_USERNAME", expectedUsername)
+	t.Setenv("PINNIPED_PASSWORD", env.SupervisorUpstreamLDAP.TestUserPassword)
+
+	timeBeforeLogin := metav1.Now()
+
+	// Run kubectl command which should run an LDAP-style login without interactive prompts for username and password.
+	// We'd prefer to use "kubectl auth whoami" but that's only available in recent K8s.
+	// Generally on a kind cluster there is a clusterrolebinding "system:basic-user" and a clusterrole "system:basic-user"
+	// that allows those in group "system:authenticated" to call this API, so it does prove that we authenticated.
+	kubectlCmd := exec.CommandContext(ctx, "kubectl", "auth", "can-i", "create", "selfsubjectaccessreviews",
+		"--kubeconfig", pinnipedStyleKubeconfigPath)
+	kubectlCmd.Env = slices.Concat(os.Environ(), env.ProxyEnv())
+	kubectlOutput, err := kubectlCmd.CombinedOutput()
+	require.NoErrorf(t, err,
+		"expected no error but got error, combined stdout/stderr was:\n----start of output\n%s\n----end of output", kubectlOutput)
+
+	allSupervisorSessionStartedLogs := getFilteredAuditLogs(t, ctx,
+		func(log map[string]any) bool {
+			return log["message"] == string(auditevent.SessionStarted)
+		},
+		kubeClientForK8sResourcesOnly,
+		env.SupervisorNamespace,
+		env.SupervisorAppName,
+		timeBeforeLogin,
+	)
+	removeSomeKeysFromEachAuditLogEvent(allSupervisorSessionStartedLogs)
+	// Also remove sessionID, which is a UUID that we can't predict for the assertions below.
+	for _, log := range allSupervisorSessionStartedLogs {
+		require.NotEmpty(t, log["sessionID"])
+		delete(log, "sessionID")
+	}
+
+	// All values in the personalInfo map should be redacted by default.
+	require.Equal(t, []map[string]any{
+		{
+			"message": "Session Started",
+			"personalInfo": map[string]any{
+				"username":         "redacted",
+				"groups":           []any{"redacted 2 values"},
+				"subject":          "redacted",
+				"additionalClaims": map[string]any{"redacted": "redacted 0 keys"},
+			},
+			"warnings": []any{},
+		},
+	}, allSupervisorSessionStartedLogs)
+
+	allConciergeTCRLogs := getFilteredAuditLogs(t, ctx,
+		func(log map[string]any) bool {
+			return log["message"] == string(auditevent.TokenCredentialRequestAuthenticatedUser)
+		},
+		kubeClientForK8sResourcesOnly,
+		env.ConciergeNamespace,
+		env.ConciergeAppName,
+		timeBeforeLogin,
+	)
+	removeSomeKeysFromEachAuditLogEvent(allConciergeTCRLogs)
+	// Also remove issuedClientCert, which contains timestamps that we can't easily predict for the assertions below.
+	for _, log := range allConciergeTCRLogs {
+		require.NotEmpty(t, log["issuedClientCert"])
+		delete(log, "issuedClientCert")
+	}
+
+	// All values in the personalInfo map should be redacted by default.
+	require.Equal(t, []map[string]any{
+		{
+			"message": "TokenCredentialRequest Authenticated User",
+			"authenticator": map[string]any{
+				// this is always pinniped.dev even when the API group suffix was customized because of the way that the production code works
+				"apiGroup": "authentication.concierge.pinniped.dev",
+				"kind":     "JWTAuthenticator",
+				"name":     authenticator.Name,
+			},
+			"personalInfo": map[string]any{
+				"username": "redacted",
+				"groups":   []any{"redacted 2 values"},
+			},
+		},
+	}, allConciergeTCRLogs)
+
+	allSupervisorHealthzLogs := getFilteredAuditLogs(t, ctx,
+		func(log map[string]any) bool {
+			return log["path"] == "/healthz"
+		},
+		kubeClientForK8sResourcesOnly,
+		env.SupervisorNamespace,
+		env.SupervisorAppName,
+		testStartTime,
+	)
+	// There should be none, because /healthz audit logs are disabled by default.
+	require.Empty(t, allSupervisorHealthzLogs)
+
+	t.Log("updating Supervisor's static ConfigMap and restarting the pods")
+	updateStaticConfigMapAndRestartApp(t,
+		ctx,
+		env.SupervisorNamespace,
+		env.SupervisorAppName+"-static-config",
+		env.SupervisorAppName,
+		false,
+		func(t *testing.T, configMapData string) string {
+			t.Helper()
+
+			var config supervisor.Config
+			err := yaml.Unmarshal([]byte(configMapData), &config)
+			require.NoError(t, err)
+
+			// The Supervisor has two audit configuration options. Enable both.
+			config.Audit.LogUsernamesAndGroups = "enabled"
+			config.Audit.LogInternalPaths = "enabled"
+
+			updatedConfig, err := yaml.Marshal(config)
+			require.NoError(t, err)
+			return string(updatedConfig)
+		},
+	)
+
+	t.Log("updating Concierge's static ConfigMap and restarting the pods")
+	updateStaticConfigMapAndRestartApp(t,
+		ctx,
+		env.ConciergeNamespace,
+		env.ConciergeAppName+"-config",
+		env.ConciergeAppName,
+		true,
+		func(t *testing.T, configMapData string) string {
+			t.Helper()
+
+			var config concierge.Config
+			err := yaml.Unmarshal([]byte(configMapData), &config)
+			require.NoError(t, err)
+
+			// The Concierge has only one audit configuration option. Enable it.
+			config.Audit.LogUsernamesAndGroups = "enabled"
+
+			updatedConfig, err := yaml.Marshal(config)
+			require.NoError(t, err)
+			return string(updatedConfig)
+		},
+	)
+
+	// Force a fresh login for the next kubectl command by removing the local caches.
+	require.NoError(t, os.Remove(sessionCachePath))
+	require.NoError(t, os.Remove(credentialCachePath))
+
+	// Reset the start time before we do a second login.
+	timeBeforeLogin = metav1.Now()
+
+	// Do a second login, which should cause audit logs with non-redacted personal info.
+	// Run kubectl command which should run an LDAP-style login without interactive prompts for username and password.
+	// We'd prefer to use "kubectl auth whoami" but that's only available in recent K8s.
+	// Generally on a kind cluster there is a clusterrolebinding "system:basic-user" and a clusterrole "system:basic-user"
+	// that allows those in group "system:authenticated" to call this API, so it does prove that we authenticated.
+	kubectlCmd = exec.CommandContext(ctx, "kubectl", "auth", "can-i", "create", "selfsubjectaccessreviews",
+		"--kubeconfig", pinnipedStyleKubeconfigPath)
+	kubectlCmd.Env = slices.Concat(os.Environ(), env.ProxyEnv())
+	kubectlOutput, err = kubectlCmd.CombinedOutput()
+	require.NoErrorf(t, err,
+		"expected no error but got error, combined stdout/stderr was:\n----start of output\n%s\n----end of output", kubectlOutput)
+
+	allSupervisorSessionStartedLogs = getFilteredAuditLogs(t, ctx,
+		func(log map[string]any) bool {
+			return log["message"] == string(auditevent.SessionStarted)
+		},
+		kubeClientForK8sResourcesOnly,
+		env.SupervisorNamespace,
+		env.SupervisorAppName,
+		timeBeforeLogin,
+	)
+	removeSomeKeysFromEachAuditLogEvent(allSupervisorSessionStartedLogs)
+	// Also remove sessionID, which is a UUID that we can't predict for the assertions below.
+	for _, log := range allSupervisorSessionStartedLogs {
+		require.NotEmpty(t, log["sessionID"])
+		delete(log, "sessionID")
+	}
+	// Now that "subject" should not be redacted, remove it too because it also contains values that are hard to predict.
+	for _, log := range allSupervisorSessionStartedLogs {
+		p := log["personalInfo"].(map[string]any)
+		require.NotEmpty(t, p)
+		require.Contains(t, p["subject"], "ldaps://"+env.SupervisorUpstreamLDAP.Host+"?")
+		delete(p, "subject")
+	}
+
+	// All values in the personalInfo map should not be redacted anymore.
+	require.Equal(t, []map[string]any{
+		{
+			"message": "Session Started",
+			"personalInfo": map[string]any{
+				"username": expectedUsername,
+				"groups":   expectedGroups,
+				// note that we removed "subject" above
+				"additionalClaims": map[string]any{},
+			},
+			"warnings": []any{},
+		},
+	}, allSupervisorSessionStartedLogs)
+
+	allConciergeTCRLogs = getFilteredAuditLogs(t, ctx,
+		func(log map[string]any) bool {
+			return log["message"] == string(auditevent.TokenCredentialRequestAuthenticatedUser)
+		},
+		kubeClientForK8sResourcesOnly,
+		env.ConciergeNamespace,
+		env.ConciergeAppName,
+		timeBeforeLogin,
+	)
+	removeSomeKeysFromEachAuditLogEvent(allConciergeTCRLogs)
+	// Also remove issuedClientCert, which contains timestamps that we can't easily predict for the assertions below.
+	for _, log := range allConciergeTCRLogs {
+		require.NotEmpty(t, log["issuedClientCert"])
+		delete(log, "issuedClientCert")
+	}
+
+	// All values in the personalInfo map should not be redacted anymore.
+	require.Equal(t, []map[string]any{
+		{
+			"message": "TokenCredentialRequest Authenticated User",
+			"authenticator": map[string]any{
+				// this is always pinniped.dev even when the API group suffix was customized because of the way that the production code works
+				"apiGroup": "authentication.concierge.pinniped.dev",
+				"kind":     "JWTAuthenticator",
+				"name":     authenticator.Name,
+			},
+			"personalInfo": map[string]any{
+				"username": expectedUsername,
+				"groups":   expectedGroups,
+			},
+		},
+	}, allConciergeTCRLogs)
+
+	allSupervisorHealthzLogs = getFilteredAuditLogs(t, ctx,
+		func(log map[string]any) bool {
+			return log["path"] == "/healthz"
+		},
+		kubeClientForK8sResourcesOnly,
+		env.SupervisorNamespace,
+		env.SupervisorAppName,
+		testStartTime,
+	)
+	// There should be some, because we reconfigured the setting to enable them.
+	t.Logf("saw %d audit logs where path=/healthz in Supervisor pod logs", len(allSupervisorHealthzLogs))
+	require.NotEmpty(t, allSupervisorHealthzLogs)
+}
+
+func TestAuditLogsEmittedForDiscoveryEndpoints_Parallel(t *testing.T) {
+	ctx, cancelFunc := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancelFunc()
+
+	env, kubeClientForK8sResourcesOnly, fakeIssuerForDisplayPurposes, ca, dnsOverrides := auditSetup(t, ctx)
+
+	startTime := metav1.Now()
+	//nolint:bodyclose // this is closed in the helper function
+	_, _, auditID := requireSuccessEndpointResponse(t,
+		fakeIssuerForDisplayPurposes.Issuer()+"/.well-known/openid-configuration",
+		fakeIssuerForDisplayPurposes.Issuer(),
+		ca.Bundle(),
+		dnsOverrides,
+	)
+
+	allSupervisorPodLogsWithAuditID := getFilteredAuditLogs(t, ctx,
+		func(log map[string]any) bool {
+			return log["auditID"] == auditID
+		},
+		kubeClientForK8sResourcesOnly,
+		env.SupervisorNamespace,
+		env.SupervisorAppName,
+		startTime,
+	)
+	removeSomeKeysFromEachAuditLogEvent(allSupervisorPodLogsWithAuditID)
+
+	require.Equal(t, 2, len(allSupervisorPodLogsWithAuditID),
+		"expected exactly two log lines with auditID=%s", auditID)
+
+	require.Equal(t, []map[string]any{
+		{
+			"message":    "HTTP Request Received",
+			"proto":      "HTTP/1.1",
+			"method":     "GET",
+			"host":       fakeIssuerForDisplayPurposes.Address(),
+			"serverName": fakeIssuerForDisplayPurposes.Address(),
+			"path":       "/federation/domain/for/auditing/.well-known/openid-configuration",
+		},
+		{
+			"message":        "HTTP Request Completed",
+			"path":           "/federation/domain/for/auditing/.well-known/openid-configuration",
+			"responseStatus": float64(200),
+			"location":       "no location header",
+		},
+	}, allSupervisorPodLogsWithAuditID)
+}
+
+// Certain endpoints will log their parameters with an "HTTP Request Parameters" audit event,
+// although most values are redacted. This test sets up a failing call to each of the following:
+// /oauth2/authorize, /callback, /login, and /oauth2/token.
+func TestAuditLogsEmittedForEndpointsEvenWhenTheCallsAreInvalid_Parallel(t *testing.T) {
+	ctx, cancelFunc := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancelFunc()
+
+	env, kubeClientForK8sResourcesOnly, fakeIssuerForDisplayPurposes, ca, dnsOverrides := auditSetup(t, ctx)
+
+	// Call the /oauth2/authorize endpoint
+	startTime := metav1.Now()
+	//nolint:bodyclose // this is closed in the helper function
+	_, _, auditID := requireEndpointResponse(t,
+		fakeIssuerForDisplayPurposes.Issuer()+"/oauth2/authorize?foo=bar&foo=bar&scope=safe-to-log",
+		fakeIssuerForDisplayPurposes.Issuer(),
+		ca.Bundle(),
+		dnsOverrides,
+		http.StatusBadRequest,
+	)
+
+	allSupervisorPodLogsWithAuditID := getFilteredAuditLogs(t, ctx,
+		func(log map[string]any) bool {
+			return log["auditID"] == auditID
+		},
+		kubeClientForK8sResourcesOnly,
+		env.SupervisorNamespace,
+		env.SupervisorAppName,
+		startTime,
+	)
+	removeSomeKeysFromEachAuditLogEvent(allSupervisorPodLogsWithAuditID)
+
+	require.Equal(t, []map[string]any{
+		{
+			"message":    "HTTP Request Received",
+			"proto":      "HTTP/1.1",
+			"method":     "GET",
+			"host":       fakeIssuerForDisplayPurposes.Address(),
+			"serverName": fakeIssuerForDisplayPurposes.Address(),
+			"path":       "/federation/domain/for/auditing/oauth2/authorize",
+		},
+		{
+			"message": "HTTP Request Parameters",
+			"multiValueParams": map[string]any{
+				"foo": []any{"redacted", "redacted"},
+			},
+			"params": map[string]any{
+				"scope": "safe-to-log",
+				"foo":   "redacted",
+			},
+		},
+		{
+			"message":           "HTTP Request Custom Headers Used",
+			"Pinniped-Password": false,
+			"Pinniped-Username": false,
+		},
+		{
+			"message":        "HTTP Request Completed",
+			"path":           "/federation/domain/for/auditing/oauth2/authorize",
+			"responseStatus": float64(http.StatusBadRequest),
+			"location":       "no location header",
+		},
+	}, allSupervisorPodLogsWithAuditID)
+
+	// Call the /callback endpoint
+	startTime = metav1.Now()
+	//nolint:bodyclose // this is closed in the helper function
+	_, _, auditID = requireEndpointResponse(t,
+		fakeIssuerForDisplayPurposes.Issuer()+"/callback?foo=bar&foo=bar&error=safe-to-log",
+		fakeIssuerForDisplayPurposes.Issuer(),
+		ca.Bundle(),
+		dnsOverrides,
+		http.StatusForbidden,
+	)
+
+	allSupervisorPodLogsWithAuditID = getFilteredAuditLogs(t, ctx,
+		func(log map[string]any) bool {
+			return log["auditID"] == auditID
+		},
+		kubeClientForK8sResourcesOnly,
+		env.SupervisorNamespace,
+		env.SupervisorAppName,
+		startTime,
+	)
+	removeSomeKeysFromEachAuditLogEvent(allSupervisorPodLogsWithAuditID)
+
+	require.Equal(t, []map[string]any{
+		{
+			"message":    "HTTP Request Received",
+			"proto":      "HTTP/1.1",
+			"method":     "GET",
+			"host":       fakeIssuerForDisplayPurposes.Address(),
+			"serverName": fakeIssuerForDisplayPurposes.Address(),
+			"path":       "/federation/domain/for/auditing/callback",
+		},
+		{
+			"message": "HTTP Request Parameters",
+			"multiValueParams": map[string]any{
+				"foo": []any{"redacted", "redacted"},
+			},
+			"params": map[string]any{
+				"error": "safe-to-log",
+				"foo":   "redacted",
+			},
+		},
+		{
+			"message":        "HTTP Request Completed",
+			"path":           "/federation/domain/for/auditing/callback",
+			"responseStatus": float64(http.StatusForbidden),
+			"location":       "no location header",
+		},
+	}, allSupervisorPodLogsWithAuditID)
+
+	// Call the /login endpoint
+	startTime = metav1.Now()
+	//nolint:bodyclose // this is closed in the helper function
+	_, _, auditID = requireEndpointResponse(t,
+		fakeIssuerForDisplayPurposes.Issuer()+"/login?foo=bar&foo=bar&err=safe-to-log",
+		fakeIssuerForDisplayPurposes.Issuer(),
+		ca.Bundle(),
+		dnsOverrides,
+		http.StatusForbidden,
+	)
+
+	allSupervisorPodLogsWithAuditID = getFilteredAuditLogs(t, ctx,
+		func(log map[string]any) bool {
+			return log["auditID"] == auditID
+		},
+		kubeClientForK8sResourcesOnly,
+		env.SupervisorNamespace,
+		env.SupervisorAppName,
+		startTime,
+	)
+	removeSomeKeysFromEachAuditLogEvent(allSupervisorPodLogsWithAuditID)
+
+	require.Equal(t, []map[string]any{
+		{
+			"message":    "HTTP Request Received",
+			"proto":      "HTTP/1.1",
+			"method":     "GET",
+			"host":       fakeIssuerForDisplayPurposes.Address(),
+			"serverName": fakeIssuerForDisplayPurposes.Address(),
+			"path":       "/federation/domain/for/auditing/login",
+		},
+		{
+			"message": "HTTP Request Parameters",
+			"multiValueParams": map[string]any{
+				"foo": []any{"redacted", "redacted"},
+			},
+			"params": map[string]any{
+				"err": "safe-to-log",
+				"foo": "redacted",
+			},
+		},
+		{
+			"message":        "HTTP Request Completed",
+			"path":           "/federation/domain/for/auditing/login",
+			"responseStatus": float64(http.StatusForbidden),
+			"location":       "no location header",
+		},
+	}, allSupervisorPodLogsWithAuditID)
+
+	// Call the /oauth2/token endpoint
+	startTime = metav1.Now()
+	//nolint:bodyclose // this is closed in the helper function
+	_, _, auditID = requireEndpointResponse(t,
+		fakeIssuerForDisplayPurposes.Issuer()+"/oauth2/token?foo=bar&foo=bar&grant_type=safe-to-log",
+		fakeIssuerForDisplayPurposes.Issuer(),
+		ca.Bundle(),
+		dnsOverrides,
+		http.StatusBadRequest,
+	)
+
+	allSupervisorPodLogsWithAuditID = getFilteredAuditLogs(t, ctx,
+		func(log map[string]any) bool {
+			return log["auditID"] == auditID
+		},
+		kubeClientForK8sResourcesOnly,
+		env.SupervisorNamespace,
+		env.SupervisorAppName,
+		startTime,
+	)
+	removeSomeKeysFromEachAuditLogEvent(allSupervisorPodLogsWithAuditID)
+
+	require.Equal(t, []map[string]any{
+		{
+			"message":    "HTTP Request Received",
+			"proto":      "HTTP/1.1",
+			"method":     "GET",
+			"host":       fakeIssuerForDisplayPurposes.Address(),
+			"serverName": fakeIssuerForDisplayPurposes.Address(),
+			"path":       "/federation/domain/for/auditing/oauth2/token",
+		},
+		{
+			"message": "HTTP Request Parameters",
+			"multiValueParams": map[string]any{
+				"foo": []any{"redacted", "redacted"},
+			},
+			"params": map[string]any{
+				"grant_type": "safe-to-log",
+				"foo":        "redacted",
+			},
+		},
+		{
+			"message":        "HTTP Request Completed",
+			"path":           "/federation/domain/for/auditing/oauth2/token",
+			"responseStatus": float64(http.StatusBadRequest),
+			"location":       "no location header",
+		},
+	}, allSupervisorPodLogsWithAuditID)
+}
+
+func auditSetup(t *testing.T, ctx context.Context) (
+	*testlib.TestEnv,
+	kubernetes.Interface,
+	*testlib.SupervisorIssuer,
+	*certauthority.CA,
+	map[string]string,
+) {
+	env := testlib.IntegrationEnv(t).WithKubeDistribution(testlib.KindDistro)
+
+	kubeClientForK8sResourcesOnly := kubeClientWithoutPinnipedAPISuffix(t)
+
+	// Use a unique hostname so that it won't interfere with any other FederationDomain,
+	// which means this test can be run in _Parallel.
+	fakeHostname := "pinniped-" + strings.ToLower(testlib.RandHex(t, 8)) + ".example.com"
+	fakeIssuerForDisplayPurposes := testlib.NewSupervisorIssuer(t, "https://"+fakeHostname+"/federation/domain/for/auditing")
+
+	// Generate a CA bundle with which to serve this provider.
+	t.Logf("generating test CA")
+	tlsServingCertForSupervisorSecretName := "federation-domain-serving-cert-" + testlib.RandHex(t, 8)
+
+	ca := createTLSServingCertSecretForSupervisor(
+		ctx,
+		t,
+		env,
+		fakeIssuerForDisplayPurposes,
+		tlsServingCertForSupervisorSecretName,
+		kubeClientForK8sResourcesOnly,
+	)
+
+	// Create any IDP so that any FederationDomain created later by this test will see that exactly one IDP exists.
+	idp := testlib.CreateTestOIDCIdentityProvider(t, idpv1alpha1.OIDCIdentityProviderSpec{
+		Issuer: "https://example.cluster.local/fake-issuer-url-does-not-matter",
+		Client: idpv1alpha1.OIDCClient{SecretName: "this-will-not-exist-but-does-not-matter"},
+	}, idpv1alpha1.PhaseError)
+
+	_ = testlib.CreateTestFederationDomain(ctx, t,
+		supervisorconfigv1alpha1.FederationDomainSpec{
+			Issuer: fakeIssuerForDisplayPurposes.Issuer(),
+			TLS: &supervisorconfigv1alpha1.FederationDomainTLSSpec{
+				SecretName: tlsServingCertForSupervisorSecretName,
+			},
+			IdentityProviders: []supervisorconfigv1alpha1.FederationDomainIdentityProvider{
+				{
+					DisplayName: idp.GetName(),
+					ObjectRef: corev1.TypedLocalObjectReference{
+						APIGroup: ptr.To("idp.supervisor." + env.APIGroupSuffix),
+						Kind:     "OIDCIdentityProvider",
+						Name:     idp.GetName(),
+					},
+				},
+			},
+		},
+		supervisorconfigv1alpha1.FederationDomainPhaseReady,
+	)
+
+	// hostname and port WITHOUT SCHEME for direct access to the supervisor's port 8443
+	physicalAddress := testlib.NewSupervisorIssuer(t, env.SupervisorHTTPSAddress).Address()
+
+	dnsOverrides := map[string]string{
+		fakeHostname + ":443": physicalAddress,
+	}
+	return env, kubeClientForK8sResourcesOnly, fakeIssuerForDisplayPurposes, ca, dnsOverrides
+}
+
+func removeSomeKeysFromEachAuditLogEvent(logs []map[string]any) {
+	for _, log := range logs {
+		delete(log, "level")
+		delete(log, "auditEvent")
+		delete(log, "caller")
+		delete(log, "remoteAddr")
+		delete(log, "userAgent")
+		delete(log, "timestamp")
+		delete(log, "latency")
+		delete(log, "auditID")
+	}
+}
+
+func getFilteredAuditLogs(
+	t *testing.T,
+	ctx context.Context,
+	filterAuditLogEvent func(log map[string]any) bool,
+	kubeClient kubernetes.Interface,
+	namespace string,
+	appName string,
+	startTime metav1.Time,
+) []map[string]any {
+	t.Helper()
+
+	ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
+	defer cancel()
+
+	pods, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
+		LabelSelector: labels.Set{"app": appName}.String(),
+	})
+	require.NoError(t, err)
+
+	var allPodLogsBuffer bytes.Buffer
+	for _, pod := range pods.Items {
+		_, err = io.Copy(&allPodLogsBuffer, getLogsForPodSince(t, ctx, kubeClient, pod, startTime))
+		require.NoError(t, err)
+	}
+
+	allPodLogs := strings.Split(allPodLogsBuffer.String(), "\n")
+	var filteredAuditLogs []map[string]any
+	for _, podLog := range allPodLogs {
+		if len(podLog) == 0 {
+			continue
+		}
+		var deserializedPodLog map[string]any
+		err = json.Unmarshal([]byte(podLog), &deserializedPodLog)
+		require.NoErrorf(t, err, "error parsing line of pod log: %s", podLog)
+		isAuditEventBool, hasAuditEvent := deserializedPodLog["auditEvent"]
+		if hasAuditEvent {
+			require.Equal(t, true, isAuditEventBool)
+			require.Equal(t, "info", deserializedPodLog["level"])
+		}
+		if hasAuditEvent && filterAuditLogEvent(deserializedPodLog) {
+			filteredAuditLogs = append(filteredAuditLogs, deserializedPodLog)
+		}
+	}
+
+	return filteredAuditLogs
+}
+
+func getLogsForPodSince(
+	t *testing.T,
+	ctx context.Context,
+	kubeClient kubernetes.Interface,
+	pod corev1.Pod,
+	startTime metav1.Time,
+) *bytes.Buffer {
+	t.Helper()
+
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
+	req := kubeClient.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &corev1.PodLogOptions{
+		SinceTime: &startTime,
+	})
+	body, err := req.Stream(ctx)
+	require.NoError(t, err)
+
+	var buf bytes.Buffer
+	_, err = io.Copy(&buf, body)
+	require.NoError(t, err)
+	require.NoError(t, body.Close())
+
+	return &buf
+}

test/integration/concierge_impersonation_proxy_test.go

@@ -1783,8 +1783,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl
 		externallyProvidedCA, err = certauthority.New("Impersonation Proxy Integration Test CA", 1*time.Hour)
 		require.NoError(t, err)
 
-		var externallyProvidedTLSServingCertPEM, externallyProvidedTLSServingKeyPEM []byte
-		externallyProvidedTLSServingCertPEM, externallyProvidedTLSServingKeyPEM, err = externallyProvidedCA.IssueServerCertPEM([]string{proxyServiceEndpoint}, nil, 1*time.Hour)
+		externallyProvidedTLSServingCertPEM, err := externallyProvidedCA.IssueServerCertPEM([]string{proxyServiceEndpoint}, nil, 1*time.Hour)
 		require.NoError(t, err)
 
 		// Specifically use corev1.Secret.StringData
@@ -1796,8 +1795,8 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl
 			corev1.SecretTypeTLS,
 			map[string]string{
 				"ca.crt":                string(externallyProvidedCA.Bundle()),
-				corev1.TLSCertKey:       string(externallyProvidedTLSServingCertPEM),
-				corev1.TLSPrivateKeyKey: string(externallyProvidedTLSServingKeyPEM),
+				corev1.TLSCertKey:       string(externallyProvidedTLSServingCertPEM.CertPEM),
+				corev1.TLSPrivateKeyKey: string(externallyProvidedTLSServingCertPEM.KeyPEM),
 			})
 
 		_, originalInternallyGeneratedCAPEM := performImpersonatorDiscoveryURL(ctx, t, env, adminConciergeClient)
@@ -1855,8 +1854,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl
 		externallyProvidedCA, err = certauthority.New("Impersonation Proxy Integration Test CA", 1*time.Hour)
 		require.NoError(t, err)
 
-		var externallyProvidedTLSServingCertPEM, externallyProvidedTLSServingKeyPEM []byte
-		externallyProvidedTLSServingCertPEM, externallyProvidedTLSServingKeyPEM, err = externallyProvidedCA.IssueServerCertPEM([]string{proxyServiceEndpoint}, nil, 1*time.Hour)
+		externallyProvidedTLSServingCertPEM, err := externallyProvidedCA.IssueServerCertPEM([]string{proxyServiceEndpoint}, nil, 1*time.Hour)
 		require.NoError(t, err)
 
 		// Specifically use corev1.Secret.Data
@@ -1868,8 +1866,8 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl
 			corev1.SecretTypeTLS,
 			map[string][]byte{
 				"ca.crt":                externallyProvidedCA.Bundle(),
-				corev1.TLSCertKey:       externallyProvidedTLSServingCertPEM,
-				corev1.TLSPrivateKeyKey: externallyProvidedTLSServingKeyPEM,
+				corev1.TLSCertKey:       externallyProvidedTLSServingCertPEM.CertPEM,
+				corev1.TLSPrivateKeyKey: externallyProvidedTLSServingCertPEM.KeyPEM,
 			})
 
 		_, originalInternallyGeneratedCAPEM := performImpersonatorDiscoveryURL(ctx, t, env, adminConciergeClient)

test/integration/formposthtml_test.go

@@ -1,4 +1,4 @@
-// Copyright 2021-2023 the Pinniped contributors. All Rights Reserved.
+// Copyright 2021-2024 the Pinniped contributors. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
 package integration
@@ -76,15 +76,16 @@ func TestFormPostHTML_Browser_Parallel(t *testing.T) {
 		require.Equal(t, responseParams.Get("code"), actualCode)
 	})
 
-	t.Run("timeout", func(t *testing.T) {
+	t.Run("timeout followed by eventual success", func(t *testing.T) {
 		browser := browsertest.OpenBrowser(t)
 
 		// Serve the form_post template with successful parameters.
 		responseParams := formpostRandomParams(t)
 		formpostInitiate(t, browser, formpostTemplateServer(t, callbackURL, responseParams))
 
-		// Sleep for longer than the two second timeout.
-		// During this sleep we are blocking the callback from returning.
+		// Sleep for longer than the two-second timeout hardcoded in form_post.js.
+		// During this sleep we are blocking the callback from returning because we
+		// have not yet called expectCallback().
 		time.Sleep(3 * time.Second)
 
 		// Assert that the timeout fires and we see the manual instructions.
@@ -92,7 +93,7 @@ func TestFormPostHTML_Browser_Parallel(t *testing.T) {
 		require.Equal(t, responseParams.Get("code"), actualCode)
 
 		// Now simulate the callback finally succeeding, in which case
-		// the manual instructions should disappear and we should see the success
+		// the manual instructions should disappear, and we should see the success
 		// div instead.
 		expectCallback(t, responseParams)
 		formpostExpectSuccessState(t, browser)
@@ -117,9 +118,7 @@ func formpostCallbackServer(t *testing.T) (string, func(*testing.T, url.Values))
 			return
 		}
 
-		// Allow CORS requests. This will be needed for this test in the future if we change
-		// the Javascript code from using mode 'no-cors' to instead use mode 'cors'. At the
-		// moment it should be ignored by the browser.
+		// Allow CORS requests.
 		w.Header().Set("Access-Control-Allow-Origin", "*")
 
 		assert.NoError(t, r.ParseForm())
@@ -132,8 +131,9 @@ func formpostCallbackServer(t *testing.T) (string, func(*testing.T, url.Values))
 			}
 		}
 
-		// Send the form parameters back on the results channel, giving up if the
-		// request context is cancelled (such as if the client disconnects).
+		// Send the form parameters back on the results channel, blocking until the test calls
+		// the function returned by formpostCallbackServer() to read this message, but also
+		// giving up if the request context is cancelled (such as if the client disconnects).
 		select {
 		case results <- postParams:
 		case <-r.Context().Done():
@@ -235,9 +235,24 @@ func formpostExpectFavicon(t *testing.T, b *browsertest.Browser, expected string
 // loading animation to be shown.
 func formpostInitiate(t *testing.T, b *browsertest.Browser, url string) {
 	t.Helper()
+
 	t.Logf("navigating to mock form_post template URL %s...", url)
+	navigationStartTime := time.Now()
 	b.Navigate(t, url)
 
+	// There is a race here, because the JS code will only show this loading animation
+	// for two seconds, and then will automatically hide it and instead show the manual
+	// copy/paste UI. So if this test runs on a very busy/slow machine that takes more
+	// than two seconds to start waiting for the loading div after opening the page,
+	// then it would fail. This is rare but does happen occasionally, so just skip these
+	// assertions in that case.
+	if time.Since(navigationStartTime) > 1500*time.Millisecond {
+		// Took too long to navigate to the page to be able to consistently see the
+		// loading animation, which is only supposed to last for 2 seconds.
+		t.Logf("skipping loading animation assertions because test was too slow...")
+		return
+	}
+
 	t.Logf("expecting to see loading animation...")
 	b.WaitForVisibleElements(t, "div#loading")
 	require.Equal(t, "Logging in...", b.Title(t))

test/integration/limited_ciphers_utils_test.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/yaml"
 
@@ -173,7 +174,6 @@ func updateStaticConfigMapAndRestartApp(
 }
 
 // restartAllPodsOfApp will immediately scale to 0 and then scale back.
-// There are no uses of t.Cleanup since these actions need to happen immediately.
 func restartAllPodsOfApp(
 	t *testing.T,
 	namespace string,
@@ -195,17 +195,39 @@ func restartAllPodsOfApp(
 	originalScale := updateDeploymentScale(t, namespace, appName, 0)
 	require.Greater(t, int(originalScale), 0)
 
+	scaleDeploymentBackToOriginalScale := func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
+		defer cancel()
+		client := testlib.NewKubernetesClientset(t)
+
+		currentScale, err := client.AppsV1().Deployments(namespace).GetScale(ctx, appName, metav1.GetOptions{})
+		require.NoError(t, err)
+		if currentScale.Spec.Replicas == originalScale {
+			// Already scaled appropriately. No need to change the scale.
+			return
+		}
+
+		updateDeploymentScale(t, namespace, appName, originalScale)
+
+		// Wait for all the new pods to be running and ready.
+		var newPods []corev1.Pod
+		testlib.RequireEventually(t, func(requireEventually *require.Assertions) {
+			newPods = getRunningPodsByNamePrefix(t, namespace, appName+"-", ignorePodsWithNameSubstring)
+			requireEventually.Equal(len(newPods), int(originalScale), "wanted pods to return to original scale")
+			requireEventually.True(allPodsReady(newPods), "wanted all new pods to be ready")
+		}, 2*time.Minute, 200*time.Millisecond)
+	}
+
+	// Even if the test fails due to the below assertions, still try to scale back to original scale,
+	// to avoid polluting other tests.
+	t.Cleanup(scaleDeploymentBackToOriginalScale)
+
+	// Now that we have adjusted the scale to 0, the pods should go away.
 	testlib.RequireEventually(t, func(requireEventually *require.Assertions) {
 		newPods := getRunningPodsByNamePrefix(t, namespace, appName+"-", ignorePodsWithNameSubstring)
 		requireEventually.Len(newPods, 0, "wanted zero pods")
 	}, 2*time.Minute, 200*time.Millisecond)
 
-	// Reset the application to its original scale.
-	updateDeploymentScale(t, namespace, appName, originalScale)
-
-	testlib.RequireEventually(t, func(requireEventually *require.Assertions) {
-		newPods := getRunningPodsByNamePrefix(t, namespace, appName+"-", ignorePodsWithNameSubstring)
-		requireEventually.Equal(len(newPods), int(originalScale), "wanted %d pods", originalScale)
-		requireEventually.True(allPodsReady(newPods), "wanted all new pods to be ready")
-	}, 2*time.Minute, 200*time.Millisecond)
+	// Scale back to original scale immediately.
+	scaleDeploymentBackToOriginalScale()
 }

test/integration/supervisor_discovery_test.go

@@ -73,10 +73,10 @@ func TestSupervisorOIDCDiscovery_Disruptive(t *testing.T) {
 		Name     string
 		Scheme   string
 		Address  string
-		CABundle string
+		CABundle []byte
 	}{
-		{Name: "direct https", Scheme: "https", Address: env.SupervisorHTTPSAddress, CABundle: string(defaultCA.Bundle())},
-		{Name: "ingress https", Scheme: "https", Address: env.SupervisorHTTPSIngressAddress, CABundle: env.SupervisorHTTPSIngressCABundle},
+		{Name: "direct https", Scheme: "https", Address: env.SupervisorHTTPSAddress, CABundle: defaultCA.Bundle()},
+		{Name: "ingress https", Scheme: "https", Address: env.SupervisorHTTPSIngressAddress, CABundle: []byte(env.SupervisorHTTPSIngressCABundle)},
 	}
 
 	for _, test := range tests {
@@ -219,7 +219,7 @@ func TestSupervisorTLSTerminationWithSNI_Disruptive(t *testing.T) {
 	)
 
 	// Now that the Secret exists, we should be able to access the endpoints by hostname using the CA.
-	_ = requireStandardDiscoveryEndpointsAreWorking(t, scheme, address, string(ca1.Bundle()), issuer1, nil)
+	_ = requireStandardDiscoveryEndpointsAreWorking(t, scheme, address, ca1.Bundle(), issuer1, nil)
 
 	// Delete the default TLS secret as well
 	err := kubeClient.CoreV1().Secrets(env.SupervisorNamespace).Delete(ctx, env.DefaultTLSCertSecretName(), metav1.DeleteOptions{})
@@ -251,7 +251,7 @@ func TestSupervisorTLSTerminationWithSNI_Disruptive(t *testing.T) {
 	)
 
 	// Now that the Secret exists at the new name, we should be able to access the endpoints by hostname using the CA.
-	_ = requireStandardDiscoveryEndpointsAreWorking(t, scheme, address, string(ca1update.Bundle()), issuer1, nil)
+	_ = requireStandardDiscoveryEndpointsAreWorking(t, scheme, address, ca1update.Bundle(), issuer1, nil)
 
 	// To test SNI virtual hosting, send requests to discovery endpoints when the public address is different from the issuer name.
 	hostname2 := "some-issuer-host-and-port-that-doesnt-match-public-supervisor-address.com"
@@ -278,7 +278,7 @@ func TestSupervisorTLSTerminationWithSNI_Disruptive(t *testing.T) {
 	)
 
 	// Now that the Secret exists, we should be able to access the endpoints by hostname using the CA.
-	_ = requireStandardDiscoveryEndpointsAreWorking(t, scheme, hostname2+":"+hostnamePort2, string(ca2.Bundle()), issuer2, map[string]string{
+	_ = requireStandardDiscoveryEndpointsAreWorking(t, scheme, hostname2+":"+hostnamePort2, ca2.Bundle(), issuer2, map[string]string{
 		hostname2 + ":" + hostnamePort2: address,
 	})
 }
@@ -336,7 +336,7 @@ func TestSupervisorTLSTerminationWithDefaultCerts_Disruptive(t *testing.T) {
 	)
 
 	// Now that the Secret exists, we should be able to access the endpoints by IP address using the CA.
-	_ = requireStandardDiscoveryEndpointsAreWorking(t, scheme, ipWithPort, string(defaultCA.Bundle()), issuerUsingIPAddress, nil)
+	_ = requireStandardDiscoveryEndpointsAreWorking(t, scheme, ipWithPort, defaultCA.Bundle(), issuerUsingIPAddress, nil)
 
 	// Create an FederationDomain with a spec.tls.secretName.
 	certSecretName := "integration-test-cert-1"
@@ -360,12 +360,12 @@ func TestSupervisorTLSTerminationWithDefaultCerts_Disruptive(t *testing.T) {
 	// Now that the Secret exists, we should be able to access the endpoints by hostname using the CA from the SNI cert.
 	// Hostnames are case-insensitive, so the request should still work even if the case of the hostname is different
 	// from the case of the issuer URL's hostname.
-	_ = requireStandardDiscoveryEndpointsAreWorking(t, scheme, strings.ToUpper(hostname)+":"+port, string(certCA.Bundle()), issuerUsingHostname, nil)
+	_ = requireStandardDiscoveryEndpointsAreWorking(t, scheme, strings.ToUpper(hostname)+":"+port, certCA.Bundle(), issuerUsingHostname, nil)
 
 	if !supervisorIssuer.IsIPAddress() {
 		// And we can still access the other issuer using the default cert,
 		// except when we have an IP address, because in that case we just overwrote the default cert
-		_ = requireStandardDiscoveryEndpointsAreWorking(t, scheme, ipWithPort, string(defaultCA.Bundle()), issuerUsingIPAddress, nil)
+		_ = requireStandardDiscoveryEndpointsAreWorking(t, scheme, ipWithPort, defaultCA.Bundle(), issuerUsingIPAddress, nil)
 	}
 }
 
@@ -492,7 +492,7 @@ func wellKnownURLForIssuer(scheme, host, path string) string {
 	return fmt.Sprintf("%s://%s/%s/.well-known/openid-configuration", scheme, host, strings.TrimPrefix(path, "/"))
 }
 
-func requireDiscoveryEndpointsAreNotFound(t *testing.T, supervisorScheme, supervisorAddress, supervisorCABundle, issuerName string) {
+func requireDiscoveryEndpointsAreNotFound(t *testing.T, supervisorScheme, supervisorAddress string, supervisorCABundle []byte, issuerName string) {
 	t.Helper()
 	issuerURL, err := url.Parse(issuerName)
 	require.NoError(t, err)
@@ -500,7 +500,7 @@ func requireDiscoveryEndpointsAreNotFound(t *testing.T, supervisorScheme, superv
 	requireEndpointNotFound(t, jwksURLForIssuer(supervisorScheme, supervisorAddress, issuerURL.Path), issuerURL.Host, supervisorCABundle)
 }
 
-func requireEndpointNotFound(t *testing.T, url, host, caBundle string) {
+func requireEndpointNotFound(t *testing.T, url, host string, caBundle []byte) {
 	t.Helper()
 	httpClient := newHTTPClient(t, caBundle, nil)
 
@@ -555,7 +555,8 @@ func requireEndpointHasBootstrapTLSErrorBecauseCertificatesAreNotReady(t *testin
 func requireCreatingFederationDomainCausesDiscoveryEndpointsToAppear(
 	ctx context.Context,
 	t *testing.T,
-	supervisorScheme, supervisorAddress, supervisorCABundle string,
+	supervisorScheme, supervisorAddress string,
+	supervisorCABundle []byte,
 	issuerName string,
 	client supervisorclientset.Interface,
 ) (*supervisorconfigv1alpha1.FederationDomain, *ExpectedJWKSResponseFormat) {
@@ -566,7 +567,7 @@ func requireCreatingFederationDomainCausesDiscoveryEndpointsToAppear(
 	return newFederationDomain, jwksResult
 }
 
-func requireStandardDiscoveryEndpointsAreWorking(t *testing.T, supervisorScheme, supervisorAddress, supervisorCABundle, issuerName string, dnsOverrides map[string]string) *ExpectedJWKSResponseFormat {
+func requireStandardDiscoveryEndpointsAreWorking(t *testing.T, supervisorScheme, supervisorAddress string, supervisorCABundle []byte, issuerName string, dnsOverrides map[string]string) *ExpectedJWKSResponseFormat {
 	requireWellKnownEndpointIsWorking(t, supervisorScheme, supervisorAddress, supervisorCABundle, issuerName, dnsOverrides)
 	jwksResult := requireJWKSEndpointIsWorking(t, supervisorScheme, supervisorAddress, supervisorCABundle, issuerName, dnsOverrides)
 	return jwksResult
@@ -577,7 +578,8 @@ func requireDeletingFederationDomainCausesDiscoveryEndpointsToDisappear(
 	existingFederationDomain *supervisorconfigv1alpha1.FederationDomain,
 	client supervisorclientset.Interface,
 	ns string,
-	supervisorScheme, supervisorAddress, supervisorCABundle string,
+	supervisorScheme, supervisorAddress string,
+	supervisorCABundle []byte,
 	issuerName string,
 ) {
 	t.Helper()
@@ -592,11 +594,11 @@ func requireDeletingFederationDomainCausesDiscoveryEndpointsToDisappear(
 	requireDiscoveryEndpointsAreNotFound(t, supervisorScheme, supervisorAddress, supervisorCABundle, issuerName)
 }
 
-func requireWellKnownEndpointIsWorking(t *testing.T, supervisorScheme, supervisorAddress, supervisorCABundle, issuerName string, dnsOverrides map[string]string) {
+func requireWellKnownEndpointIsWorking(t *testing.T, supervisorScheme, supervisorAddress string, supervisorCABundle []byte, issuerName string, dnsOverrides map[string]string) {
 	t.Helper()
 	issuerURL, err := url.Parse(issuerName)
 	require.NoError(t, err)
-	response, responseBody := requireSuccessEndpointResponse(t, wellKnownURLForIssuer(supervisorScheme, supervisorAddress, issuerURL.Path), issuerName, supervisorCABundle, dnsOverrides) //nolint:bodyclose
+	response, responseBody, _ := requireSuccessEndpointResponse(t, wellKnownURLForIssuer(supervisorScheme, supervisorAddress, issuerURL.Path), issuerName, supervisorCABundle, dnsOverrides) //nolint:bodyclose
 
 	// Check that the response matches our expectations.
 	expectedResultTemplate := here.Doc(`{
@@ -624,12 +626,12 @@ type ExpectedJWKSResponseFormat struct {
 	Keys []map[string]string
 }
 
-func requireJWKSEndpointIsWorking(t *testing.T, supervisorScheme, supervisorAddress, supervisorCABundle, issuerName string, dnsOverrides map[string]string) *ExpectedJWKSResponseFormat {
+func requireJWKSEndpointIsWorking(t *testing.T, supervisorScheme, supervisorAddress string, supervisorCABundle []byte, issuerName string, dnsOverrides map[string]string) *ExpectedJWKSResponseFormat {
 	t.Helper()
 
 	issuerURL, err := url.Parse(issuerName)
 	require.NoError(t, err)
-	response, responseBody := requireSuccessEndpointResponse(t, //nolint:bodyclose
+	response, responseBody, _ := requireSuccessEndpointResponse(t, //nolint:bodyclose
 		jwksURLForIssuer(supervisorScheme, supervisorAddress, issuerURL.Path),
 		issuerName,
 		supervisorCABundle,
@@ -664,14 +666,18 @@ func printServerCert(t *testing.T, address string, dnsOverrides map[string]strin
 	addressURL, err := url.Parse(address)
 	require.NoError(t, err)
 
-	host := addressURL.Host
-	if _, ok := dnsOverrides[host]; ok {
-		host = dnsOverrides[host]
+	require.Equal(t, "https", addressURL.Scheme,
+		"can only print server certificates for TLS-enabled endpoints")
+
+	if !strings.Contains(addressURL.Host, ":") {
+		// tls.Dial() requires a port number, but there was no port number in the host, so assume 443.
+		addressURL.Host += ":443"
 	}
 
-	if !strings.Contains(host, ":") {
-		// tls.Dial() requires a port number, but there was no port number in the host, so assume 443.
-		host += ":443"
+	host := addressURL.Host
+	if _, ok := dnsOverrides[host]; ok {
+		t.Logf("printServerCert replacing addr %s with %s", host, dnsOverrides[host])
+		host = dnsOverrides[host]
 	}
 
 	conn, err := tls.Dial("tcp", host, conf)
@@ -688,7 +694,13 @@ func printServerCert(t *testing.T, address string, dnsOverrides map[string]strin
 	}
 }
 
-func requireSuccessEndpointResponse(t *testing.T, endpointURL, issuer, caBundle string, dnsOverrides map[string]string) (*http.Response, string) {
+func requireEndpointResponse(
+	t *testing.T,
+	endpointURL, issuer string,
+	caBundle []byte,
+	dnsOverrides map[string]string,
+	wantStatusCode int,
+) (*http.Response, string, string) {
 	t.Helper()
 	httpClient := newHTTPClient(t, caBundle, dnsOverrides)
 
@@ -714,6 +726,7 @@ func requireSuccessEndpointResponse(t *testing.T, endpointURL, issuer, caBundle
 		// Set the host header on the request to match the issuer's hostname, which could potentially be different
 		// from the public ingress address, e.g. when a load balancer is used, so we want to test here that the host
 		// header is respected by the supervisor server.
+		// TODO: Why is this set?
 		requestDiscoveryEndpoint.Host = issuerURL.Host
 
 		printServerCert(t, endpointURL, dnsOverrides)
@@ -722,8 +735,9 @@ func requireSuccessEndpointResponse(t *testing.T, endpointURL, issuer, caBundle
 		requireEventually.NoError(err)
 		defer func() { _ = response.Body.Close() }()
 
-		t.Logf("successful GET requestDiscoveryEndpoint=%q, found serverName=%s, with %d certificates",
+		t.Logf("GET requestDiscoveryEndpoint=%q, statusCode=%d, found serverName=%s, with %d certificates",
 			requestDiscoveryEndpoint.URL.String(),
+			response.StatusCode,
 			response.TLS.ServerName,
 			len(response.TLS.PeerCertificates))
 		for _, peerCertificate := range response.TLS.PeerCertificates {
@@ -732,13 +746,21 @@ func requireSuccessEndpointResponse(t *testing.T, endpointURL, issuer, caBundle
 				peerCertificate.IPAddresses)
 		}
 
-		requireEventually.Equal(http.StatusOK, response.StatusCode)
+		requireEventually.Equal(wantStatusCode, response.StatusCode)
 
 		responseBody, err = io.ReadAll(response.Body)
 		requireEventually.NoError(err)
 	}, 2*time.Minute, 200*time.Millisecond)
 
-	return response, string(responseBody)
+	require.NotNil(t, response)
+	auditID := response.Header.Get("Audit-Id")
+	require.NotEmpty(t, auditID)
+
+	return response, string(responseBody), auditID
+}
+
+func requireSuccessEndpointResponse(t *testing.T, endpointURL, issuer string, caBundle []byte, dnsOverrides map[string]string) (*http.Response, string, string) {
+	return requireEndpointResponse(t, endpointURL, issuer, caBundle, dnsOverrides, http.StatusOK)
 }
 
 func editFederationDomainIssuerName(
@@ -824,7 +846,7 @@ func requireStatus(t *testing.T, client supervisorclientset.Interface, ns, name
 	}, 5*time.Minute, 200*time.Millisecond)
 }
 
-func newHTTPClient(t *testing.T, caBundle string, dnsOverrides map[string]string) *http.Client {
+func newHTTPClient(t *testing.T, caBundle []byte, dnsOverrides map[string]string) *http.Client {
 	c := &http.Client{}
 
 	realDialer := &net.Dialer{}
@@ -834,14 +856,14 @@ func newHTTPClient(t *testing.T, caBundle string, dnsOverrides map[string]string
 			t.Logf("DialContext replacing addr %s with %s", addr, replacementAddr)
 			addr = replacementAddr
 		} else if dnsOverrides != nil {
-			t.Fatal("dnsOverrides was provided but not used, which was probably a mistake")
+			t.Fatalf("dnsOverrides was provided but not used, which was probably a mistake. addr %s", addr)
 		}
 		return realDialer.DialContext(ctx, network, addr)
 	}
 
-	if caBundle != "" { // CA bundle is optional
+	if len(caBundle) > 0 { // CA bundle is optional
 		caCertPool := x509.NewCertPool()
-		caCertPool.AppendCertsFromPEM([]byte(caBundle))
+		caCertPool.AppendCertsFromPEM(caBundle)
 		c.Transport = &http.Transport{
 			DialContext:     overrideDialContext,
 			TLSClientConfig: &tls.Config{MinVersion: ptls.SecureTLSConfigMinTLSVersion, RootCAs: caCertPool}, //nolint:gosec // this seems to be a false flag, min tls version is 1.3 in normal mode or 1.2 in fips mode
@@ -860,7 +882,9 @@ func requireIDPsListedByIDPDiscoveryEndpoint(
 	env *testlib.TestEnv,
 	ctx context.Context,
 	kubeClient kubernetes.Interface,
-	ns, supervisorScheme, supervisorAddress, supervisorCABundle, issuerName string) *supervisorconfigv1alpha1.FederationDomain {
+	ns, supervisorScheme, supervisorAddress string,
+	supervisorCABundle []byte,
+	issuerName string) *supervisorconfigv1alpha1.FederationDomain {
 	// github
 	gitHubIDPSecretName := "github-idp-secret" //nolint:gosec // this is not a credential
 	_, err := kubeClient.CoreV1().Secrets(ns).Create(ctx, &corev1.Secret{
@@ -999,7 +1023,7 @@ func requireIDPsListedByIDPDiscoveryEndpoint(
 	issuer8URL, err := url.Parse(issuerName)
 	require.NoError(t, err)
 	wellKnownURL := wellKnownURLForIssuer(supervisorScheme, supervisorAddress, issuer8URL.Path)
-	_, wellKnownResponseBody := requireSuccessEndpointResponse(t, wellKnownURL, issuerName, supervisorCABundle, nil) //nolint:bodyclose
+	_, wellKnownResponseBody, _ := requireSuccessEndpointResponse(t, wellKnownURL, issuerName, supervisorCABundle, nil) //nolint:bodyclose
 
 	type WellKnownResponse struct {
 		Issuer                string `json:"issuer"`
@@ -1014,7 +1038,7 @@ func requireIDPsListedByIDPDiscoveryEndpoint(
 	err = json.Unmarshal([]byte(wellKnownResponseBody), &wellKnownResponse)
 	require.NoError(t, err)
 	discoveryIDPEndpoint := wellKnownResponse.DiscoverySupervisor.IdentityProvidersEndpoint
-	_, discoveryIDPResponseBody := requireSuccessEndpointResponse(t, discoveryIDPEndpoint, issuerName, supervisorCABundle, nil) //nolint:bodyclose
+	_, discoveryIDPResponseBody, _ := requireSuccessEndpointResponse(t, discoveryIDPEndpoint, issuerName, supervisorCABundle, nil) //nolint:bodyclose
 	type IdentityProviderListResponse struct {
 		IdentityProviders []struct {
 			Name string `json:"name"`

test/testlib/browsertest/browsertest.go

@@ -584,7 +584,7 @@ func WaitForUpstreamLDAPLoginPageWithError(t *testing.T, b *Browser, issuer stri
 
 	// Wait for redirect back to the login page again with an error.
 	t.Logf("waiting for redirect to back to login page with error message")
-	loginURLRegexp, err := regexp.Compile(`\A` + regexp.QuoteMeta(issuer+"/login") + `\?err=login_error&state=.+\z`)
+	loginURLRegexp, err := regexp.Compile(`\A` + regexp.QuoteMeta(issuer+"/login") + `\?err=incorrect_username_or_password&state=.+\z`)
 	require.NoError(t, err)
 	b.WaitForURL(t, loginURLRegexp)
 

test/testlib/client.go

@@ -374,7 +374,7 @@ func CreateTestFederationDomain(
 
 	federationDomainsClient := NewSupervisorClientset(t).ConfigV1alpha1().FederationDomains(testEnv.SupervisorNamespace)
 	federationDomain, err := federationDomainsClient.Create(createContext, &supervisorconfigv1alpha1.FederationDomain{
-		ObjectMeta: TestObjectMeta(t, "oidc-provider"),
+		ObjectMeta: TestObjectMeta(t, "federation-domain"),
 		Spec:       spec,
 	}, metav1.CreateOptions{})
 	require.NoError(t, err, "could not create test FederationDomain")

test/testlib/env.go

@@ -401,7 +401,7 @@ func (e *TestEnv) WithoutCapability(cap Capability) *TestEnv {
 // Please use this sparingly. We would prefer that a test run on every cluster type where it can possibly run, so
 // prefer to run everywhere when possible or use cluster capabilities when needed, rather than looking at the
 // type of cluster to decide to skip a test. However, there are some tests that do not depend on or interact with
-// Kubernetes itself which really only need to run on on a single platform to give us the coverage that we desire.
+// Kubernetes itself which really only need to run on a single platform to give us the coverage that we desire.
 func (e *TestEnv) WithKubeDistribution(distro KubeDistro) *TestEnv {
 	e.t.Helper()
 	if e.KubernetesDistribution != distro {

test/testlib/supervisor_issuer.go

@@ -37,6 +37,10 @@ func NewSupervisorIssuer(t *testing.T, issuer string) *SupervisorIssuer {
 	}
 }
 
+func (s *SupervisorIssuer) AddPathSuffix(path string) {
+	s.issuerURL.Path += path
+}
+
 // AddAlternativeName adds a SAN for the cert. It is not intended to take an IP address as its argument.
 func (s *SupervisorIssuer) AddAlternativeName(san string) {
 	s.alternativeNames = append(s.alternativeNames, san)