mirror of
https://github.com/vmware-tanzu/pinniped.git
synced 2025-12-23 06:15:47 +00:00
2e3e0eed8e
"defer cancelFunc()" causes the context to be cancelled already when the t.Cleanup's are called, which causes strange test results if those t.Cleanup's try to use that cancelled context to perform operations.
791 lines
27 KiB
Go
791 lines
27 KiB
Go
// Copyright 2024-2025 the Pinniped contributors. All Rights Reserved.
|
|
// SPDX-License-Identifier: Apache-2.0
|
|
|
|
package integration
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
corev1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/labels"
|
|
"k8s.io/client-go/kubernetes"
|
|
"k8s.io/utils/ptr"
|
|
"sigs.k8s.io/yaml"
|
|
|
|
authenticationv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/authentication/v1alpha1"
|
|
supervisorconfigv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/config/v1alpha1"
|
|
idpv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/idp/v1alpha1"
|
|
"go.pinniped.dev/internal/auditevent"
|
|
"go.pinniped.dev/internal/certauthority"
|
|
"go.pinniped.dev/internal/config/concierge"
|
|
"go.pinniped.dev/internal/config/supervisor"
|
|
"go.pinniped.dev/internal/kubeclient"
|
|
"go.pinniped.dev/test/testlib"
|
|
)
|
|
|
|
// kubeClientWithoutPinnipedAPISuffix is much like testlib.NewKubernetesClientset but does not
|
|
// use middleware to change the Pinniped API suffix (kubeclient.WithMiddleware).
|
|
//
|
|
// The returned kubeclient is only for interacting with K8s-native objects, not Pinniped objects,
|
|
// so it does not need to be aware of Pinniped's API suffix.
|
|
func kubeClientWithoutPinnipedAPISuffix(t *testing.T) kubernetes.Interface {
|
|
t.Helper()
|
|
|
|
client, err := kubeclient.New(kubeclient.WithConfig(testlib.NewClientConfig(t)))
|
|
require.NoError(t, err)
|
|
|
|
return client.Kubernetes
|
|
}
|
|
|
|
// TestAuditLogsDuringLogin is an end-to-end login test which cares more about making audit log
|
|
// assertions than assertions about the login itself. Much of how this test performs a login was
|
|
// inspired by a test case from TestE2EFullIntegration_Browser. This test is Disruptive because
|
|
// it restarts the Supervisor and Concierge to reconfigure audit logging, and then restarts them
|
|
// again to put back the original configuration.
|
|
func TestAuditLogsDuringLogin_Disruptive(t *testing.T) {
|
|
env := testEnvForPodShutdownTests(t)
|
|
|
|
testStartTime := metav1.Now()
|
|
|
|
ctx, cancelFunc := context.WithTimeout(context.Background(), 10*time.Minute)
|
|
t.Cleanup(cancelFunc)
|
|
|
|
kubeClient := testlib.NewKubernetesClientset(t)
|
|
kubeClientForK8sResourcesOnly := kubeClientWithoutPinnipedAPISuffix(t)
|
|
|
|
// Build pinniped CLI.
|
|
pinnipedExe := testlib.PinnipedCLIPath(t)
|
|
|
|
supervisorIssuer := env.InferSupervisorIssuerURL(t)
|
|
|
|
// Generate a CA bundle with which to serve this provider.
|
|
t.Logf("generating test CA")
|
|
tlsServingCertForSupervisorSecretName := "federation-domain-serving-cert-" + testlib.RandHex(t, 8)
|
|
|
|
federationDomainSelfSignedCA := createTLSServingCertSecretForSupervisor(
|
|
ctx,
|
|
t,
|
|
env,
|
|
supervisorIssuer,
|
|
tlsServingCertForSupervisorSecretName,
|
|
kubeClient,
|
|
)
|
|
|
|
// Save that bundle plus the one that signs the upstream issuer, for test purposes.
|
|
federationDomainCABundlePath := filepath.Join(t.TempDir(), "test-ca.pem")
|
|
federationDomainCABundlePEM := federationDomainSelfSignedCA.Bundle()
|
|
require.NoError(t, os.WriteFile(federationDomainCABundlePath, federationDomainCABundlePEM, 0600))
|
|
|
|
// Create the downstream FederationDomain.
|
|
// This helper function will nil out spec.TLS if spec.Issuer is an IP address.
|
|
federationDomain := testlib.CreateTestFederationDomain(ctx, t,
|
|
supervisorconfigv1alpha1.FederationDomainSpec{
|
|
Issuer: supervisorIssuer.Issuer(),
|
|
TLS: &supervisorconfigv1alpha1.FederationDomainTLSSpec{SecretName: tlsServingCertForSupervisorSecretName},
|
|
},
|
|
supervisorconfigv1alpha1.FederationDomainPhaseError, // in phase error until there is an IDP created
|
|
)
|
|
|
|
expectedUsername := env.SupervisorUpstreamLDAP.TestUserMailAttributeValue
|
|
expectedGroups := make([]any, len(env.SupervisorUpstreamLDAP.TestUserDirectGroupsDNs))
|
|
for i, g := range env.SupervisorUpstreamLDAP.TestUserDirectGroupsDNs {
|
|
expectedGroups[i] = g
|
|
}
|
|
|
|
// Create a JWTAuthenticator that will validate the tokens from the downstream issuer.
|
|
// If the FederationDomain is not Ready, the JWTAuthenticator cannot be ready, either.
|
|
clusterAudience := "test-cluster-" + testlib.RandHex(t, 8)
|
|
defaultJWTAuthenticatorSpec := authenticationv1alpha1.JWTAuthenticatorSpec{
|
|
Issuer: federationDomain.Spec.Issuer,
|
|
Audience: clusterAudience,
|
|
TLS: &authenticationv1alpha1.TLSSpec{CertificateAuthorityData: base64.StdEncoding.EncodeToString(federationDomainCABundlePEM)},
|
|
}
|
|
authenticator := testlib.CreateTestJWTAuthenticator(ctx, t, defaultJWTAuthenticatorSpec, authenticationv1alpha1.JWTAuthenticatorPhaseError)
|
|
setupClusterForEndToEndLDAPTest(t, expectedUsername, env)
|
|
testlib.WaitForFederationDomainStatusPhase(ctx, t, federationDomain.Name, supervisorconfigv1alpha1.FederationDomainPhaseReady)
|
|
testlib.WaitForJWTAuthenticatorStatusPhase(ctx, t, authenticator.Name, authenticationv1alpha1.JWTAuthenticatorPhaseReady)
|
|
|
|
tempDir := t.TempDir() // per-test tmp dir to avoid sharing files between tests
|
|
// Use a specific session cache for this test.
|
|
sessionCachePath := tempDir + "/test-sessions.yaml"
|
|
credentialCachePath := tempDir + "/test-credentials.yaml"
|
|
|
|
pinnipedStyleKubeconfigPath := runPinnipedGetKubeconfig(t, env, pinnipedExe, tempDir, []string{
|
|
"get", "kubeconfig",
|
|
"--concierge-api-group-suffix", env.APIGroupSuffix,
|
|
"--concierge-authenticator-type", "jwt",
|
|
"--concierge-authenticator-name", authenticator.Name,
|
|
"--oidc-session-cache", sessionCachePath,
|
|
"--credential-cache", credentialCachePath,
|
|
// use default for --oidc-scopes, which is to request all relevant scopes
|
|
})
|
|
|
|
t.Setenv("PINNIPED_USERNAME", expectedUsername)
|
|
t.Setenv("PINNIPED_PASSWORD", env.SupervisorUpstreamLDAP.TestUserPassword)
|
|
|
|
timeBeforeLogin := metav1.Now()
|
|
|
|
// Run kubectl command which should run an LDAP-style login without interactive prompts for username and password.
|
|
// We'd prefer to use "kubectl auth whoami" but that's only available in recent K8s.
|
|
// Generally on a kind cluster there is a clusterrolebinding "system:basic-user" and a clusterrole "system:basic-user"
|
|
// that allows those in group "system:authenticated" to call this API, so it does prove that we authenticated.
|
|
kubectlCmd := exec.CommandContext(ctx, "kubectl", "auth", "can-i", "create", "selfsubjectaccessreviews",
|
|
"--kubeconfig", pinnipedStyleKubeconfigPath)
|
|
kubectlCmd.Env = slices.Concat(os.Environ(), env.ProxyEnv())
|
|
kubectlOutput, err := kubectlCmd.CombinedOutput()
|
|
require.NoErrorf(t, err,
|
|
"expected no error but got error, combined stdout/stderr was:\n----start of output\n%s\n----end of output", kubectlOutput)
|
|
|
|
allSupervisorSessionStartedLogs := getFilteredAuditLogs(t, ctx,
|
|
func(log map[string]any) bool {
|
|
return log["message"] == string(auditevent.SessionStarted)
|
|
},
|
|
kubeClientForK8sResourcesOnly,
|
|
env.SupervisorNamespace,
|
|
env.SupervisorAppName,
|
|
timeBeforeLogin,
|
|
)
|
|
removeSomeKeysFromEachAuditLogEvent(allSupervisorSessionStartedLogs)
|
|
// Also remove sessionID, which is a UUID that we can't predict for the assertions below.
|
|
for _, log := range allSupervisorSessionStartedLogs {
|
|
require.NotEmpty(t, log["sessionID"])
|
|
delete(log, "sessionID")
|
|
}
|
|
|
|
// All values in the personalInfo map should be redacted by default.
|
|
require.Equal(t, []map[string]any{
|
|
{
|
|
"message": "Session Started",
|
|
"personalInfo": map[string]any{
|
|
"username": "redacted",
|
|
"groups": []any{"redacted 2 values"},
|
|
"subject": "redacted",
|
|
"additionalClaims": map[string]any{"redacted": "redacted 0 keys"},
|
|
},
|
|
"warnings": []any{},
|
|
},
|
|
}, allSupervisorSessionStartedLogs)
|
|
|
|
allConciergeTCRLogs := getFilteredAuditLogs(t, ctx,
|
|
func(log map[string]any) bool {
|
|
return log["message"] == string(auditevent.TokenCredentialRequestAuthenticatedUser)
|
|
},
|
|
kubeClientForK8sResourcesOnly,
|
|
env.ConciergeNamespace,
|
|
env.ConciergeAppName,
|
|
timeBeforeLogin,
|
|
)
|
|
removeSomeKeysFromEachAuditLogEvent(allConciergeTCRLogs)
|
|
// Also remove issuedClientCert, which contains timestamps that we can't easily predict for the assertions below.
|
|
for _, log := range allConciergeTCRLogs {
|
|
require.NotEmpty(t, log["issuedClientCert"])
|
|
delete(log, "issuedClientCert")
|
|
}
|
|
|
|
// All values in the personalInfo map should be redacted by default.
|
|
require.Equal(t, []map[string]any{
|
|
{
|
|
"message": "TokenCredentialRequest Authenticated User",
|
|
"authenticator": map[string]any{
|
|
// this is always pinniped.dev even when the API group suffix was customized because of the way that the production code works
|
|
"apiGroup": "authentication.concierge.pinniped.dev",
|
|
"kind": "JWTAuthenticator",
|
|
"name": authenticator.Name,
|
|
},
|
|
"personalInfo": map[string]any{
|
|
"username": "redacted",
|
|
"groups": []any{"redacted 2 values"},
|
|
},
|
|
},
|
|
}, allConciergeTCRLogs)
|
|
|
|
allSupervisorHealthzLogs := getFilteredAuditLogs(t, ctx,
|
|
func(log map[string]any) bool {
|
|
return log["path"] == "/healthz"
|
|
},
|
|
kubeClientForK8sResourcesOnly,
|
|
env.SupervisorNamespace,
|
|
env.SupervisorAppName,
|
|
testStartTime,
|
|
)
|
|
// There should be none, because /healthz audit logs are disabled by default.
|
|
require.Empty(t, allSupervisorHealthzLogs)
|
|
|
|
t.Log("updating Supervisor's static ConfigMap and restarting the pods")
|
|
updateStaticConfigMapAndRestartApp(t,
|
|
ctx,
|
|
env.SupervisorNamespace,
|
|
env.SupervisorAppName+"-static-config",
|
|
env.SupervisorAppName,
|
|
false,
|
|
func(t *testing.T, configMapData string) string {
|
|
t.Helper()
|
|
|
|
var config supervisor.Config
|
|
err := yaml.Unmarshal([]byte(configMapData), &config)
|
|
require.NoError(t, err)
|
|
|
|
// The Supervisor has two audit configuration options. Enable both.
|
|
config.Audit.LogUsernamesAndGroups = "enabled"
|
|
config.Audit.LogInternalPaths = "enabled"
|
|
|
|
updatedConfig, err := yaml.Marshal(config)
|
|
require.NoError(t, err)
|
|
return string(updatedConfig)
|
|
},
|
|
)
|
|
|
|
t.Log("updating Concierge's static ConfigMap and restarting the pods")
|
|
updateStaticConfigMapAndRestartApp(t,
|
|
ctx,
|
|
env.ConciergeNamespace,
|
|
env.ConciergeAppName+"-config",
|
|
env.ConciergeAppName,
|
|
true,
|
|
func(t *testing.T, configMapData string) string {
|
|
t.Helper()
|
|
|
|
var config concierge.Config
|
|
err := yaml.Unmarshal([]byte(configMapData), &config)
|
|
require.NoError(t, err)
|
|
|
|
// The Concierge has only one audit configuration option. Enable it.
|
|
config.Audit.LogUsernamesAndGroups = "enabled"
|
|
|
|
updatedConfig, err := yaml.Marshal(config)
|
|
require.NoError(t, err)
|
|
return string(updatedConfig)
|
|
},
|
|
)
|
|
|
|
// Force a fresh login for the next kubectl command by removing the local caches.
|
|
require.NoError(t, os.Remove(sessionCachePath))
|
|
require.NoError(t, os.Remove(credentialCachePath))
|
|
|
|
// Reset the start time before we do a second login.
|
|
timeBeforeLogin = metav1.Now()
|
|
|
|
// Do a second login, which should cause audit logs with non-redacted personal info.
|
|
// Run kubectl command which should run an LDAP-style login without interactive prompts for username and password.
|
|
// We'd prefer to use "kubectl auth whoami" but that's only available in recent K8s.
|
|
// Generally on a kind cluster there is a clusterrolebinding "system:basic-user" and a clusterrole "system:basic-user"
|
|
// that allows those in group "system:authenticated" to call this API, so it does prove that we authenticated.
|
|
kubectlCmd = exec.CommandContext(ctx, "kubectl", "auth", "can-i", "create", "selfsubjectaccessreviews",
|
|
"--kubeconfig", pinnipedStyleKubeconfigPath)
|
|
kubectlCmd.Env = slices.Concat(os.Environ(), env.ProxyEnv())
|
|
kubectlOutput, err = kubectlCmd.CombinedOutput()
|
|
require.NoErrorf(t, err,
|
|
"expected no error but got error, combined stdout/stderr was:\n----start of output\n%s\n----end of output", kubectlOutput)
|
|
|
|
allSupervisorSessionStartedLogs = getFilteredAuditLogs(t, ctx,
|
|
func(log map[string]any) bool {
|
|
return log["message"] == string(auditevent.SessionStarted)
|
|
},
|
|
kubeClientForK8sResourcesOnly,
|
|
env.SupervisorNamespace,
|
|
env.SupervisorAppName,
|
|
timeBeforeLogin,
|
|
)
|
|
removeSomeKeysFromEachAuditLogEvent(allSupervisorSessionStartedLogs)
|
|
// Also remove sessionID, which is a UUID that we can't predict for the assertions below.
|
|
for _, log := range allSupervisorSessionStartedLogs {
|
|
require.NotEmpty(t, log["sessionID"])
|
|
delete(log, "sessionID")
|
|
}
|
|
// Now that "subject" should not be redacted, remove it too because it also contains values that are hard to predict.
|
|
for _, log := range allSupervisorSessionStartedLogs {
|
|
p := log["personalInfo"].(map[string]any)
|
|
require.NotEmpty(t, p)
|
|
require.Contains(t, p["subject"], "ldaps://"+env.SupervisorUpstreamLDAP.Host+"?")
|
|
delete(p, "subject")
|
|
}
|
|
|
|
// All values in the personalInfo map should not be redacted anymore.
|
|
require.Equal(t, []map[string]any{
|
|
{
|
|
"message": "Session Started",
|
|
"personalInfo": map[string]any{
|
|
"username": expectedUsername,
|
|
"groups": expectedGroups,
|
|
// note that we removed "subject" above
|
|
"additionalClaims": map[string]any{},
|
|
},
|
|
"warnings": []any{},
|
|
},
|
|
}, allSupervisorSessionStartedLogs)
|
|
|
|
allConciergeTCRLogs = getFilteredAuditLogs(t, ctx,
|
|
func(log map[string]any) bool {
|
|
return log["message"] == string(auditevent.TokenCredentialRequestAuthenticatedUser)
|
|
},
|
|
kubeClientForK8sResourcesOnly,
|
|
env.ConciergeNamespace,
|
|
env.ConciergeAppName,
|
|
timeBeforeLogin,
|
|
)
|
|
removeSomeKeysFromEachAuditLogEvent(allConciergeTCRLogs)
|
|
// Also remove issuedClientCert, which contains timestamps that we can't easily predict for the assertions below.
|
|
for _, log := range allConciergeTCRLogs {
|
|
require.NotEmpty(t, log["issuedClientCert"])
|
|
delete(log, "issuedClientCert")
|
|
}
|
|
|
|
// All values in the personalInfo map should not be redacted anymore.
|
|
require.Equal(t, []map[string]any{
|
|
{
|
|
"message": "TokenCredentialRequest Authenticated User",
|
|
"authenticator": map[string]any{
|
|
// this is always pinniped.dev even when the API group suffix was customized because of the way that the production code works
|
|
"apiGroup": "authentication.concierge.pinniped.dev",
|
|
"kind": "JWTAuthenticator",
|
|
"name": authenticator.Name,
|
|
},
|
|
"personalInfo": map[string]any{
|
|
"username": expectedUsername,
|
|
"groups": expectedGroups,
|
|
},
|
|
},
|
|
}, allConciergeTCRLogs)
|
|
|
|
allSupervisorHealthzLogs = getFilteredAuditLogs(t, ctx,
|
|
func(log map[string]any) bool {
|
|
return log["path"] == "/healthz"
|
|
},
|
|
kubeClientForK8sResourcesOnly,
|
|
env.SupervisorNamespace,
|
|
env.SupervisorAppName,
|
|
testStartTime,
|
|
)
|
|
// There should be some, because we reconfigured the setting to enable them.
|
|
t.Logf("saw %d audit logs where path=/healthz in Supervisor pod logs", len(allSupervisorHealthzLogs))
|
|
require.NotEmpty(t, allSupervisorHealthzLogs)
|
|
}
|
|
|
|
func TestAuditLogsEmittedForDiscoveryEndpoints_Parallel(t *testing.T) {
|
|
ctx, cancelFunc := context.WithTimeout(context.Background(), 2*time.Minute)
|
|
t.Cleanup(cancelFunc)
|
|
|
|
env, kubeClientForK8sResourcesOnly, fakeIssuerForDisplayPurposes, ca, dnsOverrides := auditSetup(t, ctx)
|
|
|
|
startTime := metav1.Now()
|
|
//nolint:bodyclose // this is closed in the helper function
|
|
_, _, auditID := requireSuccessEndpointResponse(t,
|
|
fakeIssuerForDisplayPurposes.Issuer()+"/.well-known/openid-configuration",
|
|
fakeIssuerForDisplayPurposes.Issuer(),
|
|
ca.Bundle(),
|
|
dnsOverrides,
|
|
)
|
|
|
|
allSupervisorPodLogsWithAuditID := getFilteredAuditLogs(t, ctx,
|
|
func(log map[string]any) bool {
|
|
return log["auditID"] == auditID
|
|
},
|
|
kubeClientForK8sResourcesOnly,
|
|
env.SupervisorNamespace,
|
|
env.SupervisorAppName,
|
|
startTime,
|
|
)
|
|
removeSomeKeysFromEachAuditLogEvent(allSupervisorPodLogsWithAuditID)
|
|
|
|
require.Equal(t, 2, len(allSupervisorPodLogsWithAuditID),
|
|
"expected exactly two log lines with auditID=%s", auditID)
|
|
|
|
require.Equal(t, []map[string]any{
|
|
{
|
|
"message": "HTTP Request Received",
|
|
"proto": "HTTP/1.1",
|
|
"method": "GET",
|
|
"host": fakeIssuerForDisplayPurposes.Address(),
|
|
"serverName": fakeIssuerForDisplayPurposes.Address(),
|
|
"path": "/federation/domain/for/auditing/.well-known/openid-configuration",
|
|
},
|
|
{
|
|
"message": "HTTP Request Completed",
|
|
"path": "/federation/domain/for/auditing/.well-known/openid-configuration",
|
|
"responseStatus": float64(200),
|
|
"location": "no location header",
|
|
},
|
|
}, allSupervisorPodLogsWithAuditID)
|
|
}
|
|
|
|
// Certain endpoints will log their parameters with an "HTTP Request Parameters" audit event,
|
|
// although most values are redacted. This test sets up a failing call to each of the following:
|
|
// /oauth2/authorize, /callback, /login, and /oauth2/token.
|
|
func TestAuditLogsEmittedForEndpointsEvenWhenTheCallsAreInvalid_Parallel(t *testing.T) {
|
|
ctx, cancelFunc := context.WithTimeout(context.Background(), 2*time.Minute)
|
|
t.Cleanup(cancelFunc)
|
|
|
|
env, kubeClientForK8sResourcesOnly, fakeIssuerForDisplayPurposes, ca, dnsOverrides := auditSetup(t, ctx)
|
|
|
|
// Call the /oauth2/authorize endpoint
|
|
startTime := metav1.Now()
|
|
//nolint:bodyclose // this is closed in the helper function
|
|
_, _, auditID := requireEndpointResponse(t,
|
|
fakeIssuerForDisplayPurposes.Issuer()+"/oauth2/authorize?foo=bar&foo=bar&scope=safe-to-log",
|
|
fakeIssuerForDisplayPurposes.Issuer(),
|
|
ca.Bundle(),
|
|
dnsOverrides,
|
|
http.StatusBadRequest,
|
|
)
|
|
|
|
allSupervisorPodLogsWithAuditID := getFilteredAuditLogs(t, ctx,
|
|
func(log map[string]any) bool {
|
|
return log["auditID"] == auditID
|
|
},
|
|
kubeClientForK8sResourcesOnly,
|
|
env.SupervisorNamespace,
|
|
env.SupervisorAppName,
|
|
startTime,
|
|
)
|
|
removeSomeKeysFromEachAuditLogEvent(allSupervisorPodLogsWithAuditID)
|
|
|
|
require.Equal(t, []map[string]any{
|
|
{
|
|
"message": "HTTP Request Received",
|
|
"proto": "HTTP/1.1",
|
|
"method": "GET",
|
|
"host": fakeIssuerForDisplayPurposes.Address(),
|
|
"serverName": fakeIssuerForDisplayPurposes.Address(),
|
|
"path": "/federation/domain/for/auditing/oauth2/authorize",
|
|
},
|
|
{
|
|
"message": "HTTP Request Parameters",
|
|
"multiValueParams": map[string]any{
|
|
"foo": []any{"redacted", "redacted"},
|
|
},
|
|
"params": map[string]any{
|
|
"scope": "safe-to-log",
|
|
"foo": "redacted",
|
|
},
|
|
},
|
|
{
|
|
"message": "HTTP Request Custom Headers Used",
|
|
"Pinniped-Password": false,
|
|
"Pinniped-Username": false,
|
|
},
|
|
{
|
|
"message": "HTTP Request Completed",
|
|
"path": "/federation/domain/for/auditing/oauth2/authorize",
|
|
"responseStatus": float64(http.StatusBadRequest),
|
|
"location": "no location header",
|
|
},
|
|
}, allSupervisorPodLogsWithAuditID)
|
|
|
|
// Call the /callback endpoint
|
|
startTime = metav1.Now()
|
|
//nolint:bodyclose // this is closed in the helper function
|
|
_, _, auditID = requireEndpointResponse(t,
|
|
fakeIssuerForDisplayPurposes.Issuer()+"/callback?foo=bar&foo=bar&error=safe-to-log",
|
|
fakeIssuerForDisplayPurposes.Issuer(),
|
|
ca.Bundle(),
|
|
dnsOverrides,
|
|
http.StatusForbidden,
|
|
)
|
|
|
|
allSupervisorPodLogsWithAuditID = getFilteredAuditLogs(t, ctx,
|
|
func(log map[string]any) bool {
|
|
return log["auditID"] == auditID
|
|
},
|
|
kubeClientForK8sResourcesOnly,
|
|
env.SupervisorNamespace,
|
|
env.SupervisorAppName,
|
|
startTime,
|
|
)
|
|
removeSomeKeysFromEachAuditLogEvent(allSupervisorPodLogsWithAuditID)
|
|
|
|
require.Equal(t, []map[string]any{
|
|
{
|
|
"message": "HTTP Request Received",
|
|
"proto": "HTTP/1.1",
|
|
"method": "GET",
|
|
"host": fakeIssuerForDisplayPurposes.Address(),
|
|
"serverName": fakeIssuerForDisplayPurposes.Address(),
|
|
"path": "/federation/domain/for/auditing/callback",
|
|
},
|
|
{
|
|
"message": "HTTP Request Parameters",
|
|
"multiValueParams": map[string]any{
|
|
"foo": []any{"redacted", "redacted"},
|
|
},
|
|
"params": map[string]any{
|
|
"error": "safe-to-log",
|
|
"foo": "redacted",
|
|
},
|
|
},
|
|
{
|
|
"message": "HTTP Request Completed",
|
|
"path": "/federation/domain/for/auditing/callback",
|
|
"responseStatus": float64(http.StatusForbidden),
|
|
"location": "no location header",
|
|
},
|
|
}, allSupervisorPodLogsWithAuditID)
|
|
|
|
// Call the /login endpoint
|
|
startTime = metav1.Now()
|
|
//nolint:bodyclose // this is closed in the helper function
|
|
_, _, auditID = requireEndpointResponse(t,
|
|
fakeIssuerForDisplayPurposes.Issuer()+"/login?foo=bar&foo=bar&err=safe-to-log",
|
|
fakeIssuerForDisplayPurposes.Issuer(),
|
|
ca.Bundle(),
|
|
dnsOverrides,
|
|
http.StatusForbidden,
|
|
)
|
|
|
|
allSupervisorPodLogsWithAuditID = getFilteredAuditLogs(t, ctx,
|
|
func(log map[string]any) bool {
|
|
return log["auditID"] == auditID
|
|
},
|
|
kubeClientForK8sResourcesOnly,
|
|
env.SupervisorNamespace,
|
|
env.SupervisorAppName,
|
|
startTime,
|
|
)
|
|
removeSomeKeysFromEachAuditLogEvent(allSupervisorPodLogsWithAuditID)
|
|
|
|
require.Equal(t, []map[string]any{
|
|
{
|
|
"message": "HTTP Request Received",
|
|
"proto": "HTTP/1.1",
|
|
"method": "GET",
|
|
"host": fakeIssuerForDisplayPurposes.Address(),
|
|
"serverName": fakeIssuerForDisplayPurposes.Address(),
|
|
"path": "/federation/domain/for/auditing/login",
|
|
},
|
|
{
|
|
"message": "HTTP Request Parameters",
|
|
"multiValueParams": map[string]any{
|
|
"foo": []any{"redacted", "redacted"},
|
|
},
|
|
"params": map[string]any{
|
|
"err": "safe-to-log",
|
|
"foo": "redacted",
|
|
},
|
|
},
|
|
{
|
|
"message": "HTTP Request Completed",
|
|
"path": "/federation/domain/for/auditing/login",
|
|
"responseStatus": float64(http.StatusForbidden),
|
|
"location": "no location header",
|
|
},
|
|
}, allSupervisorPodLogsWithAuditID)
|
|
|
|
// Call the /oauth2/token endpoint
|
|
startTime = metav1.Now()
|
|
//nolint:bodyclose // this is closed in the helper function
|
|
_, _, auditID = requireEndpointResponse(t,
|
|
fakeIssuerForDisplayPurposes.Issuer()+"/oauth2/token?foo=bar&foo=bar&grant_type=safe-to-log",
|
|
fakeIssuerForDisplayPurposes.Issuer(),
|
|
ca.Bundle(),
|
|
dnsOverrides,
|
|
http.StatusBadRequest,
|
|
)
|
|
|
|
allSupervisorPodLogsWithAuditID = getFilteredAuditLogs(t, ctx,
|
|
func(log map[string]any) bool {
|
|
return log["auditID"] == auditID
|
|
},
|
|
kubeClientForK8sResourcesOnly,
|
|
env.SupervisorNamespace,
|
|
env.SupervisorAppName,
|
|
startTime,
|
|
)
|
|
removeSomeKeysFromEachAuditLogEvent(allSupervisorPodLogsWithAuditID)
|
|
|
|
require.Equal(t, []map[string]any{
|
|
{
|
|
"message": "HTTP Request Received",
|
|
"proto": "HTTP/1.1",
|
|
"method": "GET",
|
|
"host": fakeIssuerForDisplayPurposes.Address(),
|
|
"serverName": fakeIssuerForDisplayPurposes.Address(),
|
|
"path": "/federation/domain/for/auditing/oauth2/token",
|
|
},
|
|
{
|
|
"message": "HTTP Request Parameters",
|
|
"multiValueParams": map[string]any{
|
|
"foo": []any{"redacted", "redacted"},
|
|
},
|
|
"params": map[string]any{
|
|
"grant_type": "safe-to-log",
|
|
"foo": "redacted",
|
|
},
|
|
},
|
|
{
|
|
"message": "HTTP Request Completed",
|
|
"path": "/federation/domain/for/auditing/oauth2/token",
|
|
"responseStatus": float64(http.StatusBadRequest),
|
|
"location": "no location header",
|
|
},
|
|
}, allSupervisorPodLogsWithAuditID)
|
|
}
|
|
|
|
func auditSetup(t *testing.T, ctx context.Context) (
|
|
*testlib.TestEnv,
|
|
kubernetes.Interface,
|
|
*testlib.SupervisorIssuer,
|
|
*certauthority.CA,
|
|
map[string]string,
|
|
) {
|
|
env := testlib.IntegrationEnv(t).WithKubeDistribution(testlib.KindDistro)
|
|
|
|
kubeClientForK8sResourcesOnly := kubeClientWithoutPinnipedAPISuffix(t)
|
|
|
|
// Use a unique hostname so that it won't interfere with any other FederationDomain,
|
|
// which means this test can be run in _Parallel.
|
|
fakeHostname := "pinniped-" + strings.ToLower(testlib.RandHex(t, 8)) + ".example.com"
|
|
fakeIssuerForDisplayPurposes := testlib.NewSupervisorIssuer(t, "https://"+fakeHostname+"/federation/domain/for/auditing")
|
|
|
|
// Generate a CA bundle with which to serve this provider.
|
|
t.Logf("generating test CA")
|
|
tlsServingCertForSupervisorSecretName := "federation-domain-serving-cert-" + testlib.RandHex(t, 8)
|
|
|
|
ca := createTLSServingCertSecretForSupervisor(
|
|
ctx,
|
|
t,
|
|
env,
|
|
fakeIssuerForDisplayPurposes,
|
|
tlsServingCertForSupervisorSecretName,
|
|
kubeClientForK8sResourcesOnly,
|
|
)
|
|
|
|
// Create any IDP so that any FederationDomain created later by this test will see that exactly one IDP exists.
|
|
idp := testlib.CreateTestOIDCIdentityProvider(t, idpv1alpha1.OIDCIdentityProviderSpec{
|
|
Issuer: "https://example.cluster.local/fake-issuer-url-does-not-matter",
|
|
Client: idpv1alpha1.OIDCClient{SecretName: "this-will-not-exist-but-does-not-matter"},
|
|
}, idpv1alpha1.PhaseError)
|
|
|
|
_ = testlib.CreateTestFederationDomain(ctx, t,
|
|
supervisorconfigv1alpha1.FederationDomainSpec{
|
|
Issuer: fakeIssuerForDisplayPurposes.Issuer(),
|
|
TLS: &supervisorconfigv1alpha1.FederationDomainTLSSpec{
|
|
SecretName: tlsServingCertForSupervisorSecretName,
|
|
},
|
|
IdentityProviders: []supervisorconfigv1alpha1.FederationDomainIdentityProvider{
|
|
{
|
|
DisplayName: idp.GetName(),
|
|
ObjectRef: corev1.TypedLocalObjectReference{
|
|
APIGroup: ptr.To("idp.supervisor." + env.APIGroupSuffix),
|
|
Kind: "OIDCIdentityProvider",
|
|
Name: idp.GetName(),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
supervisorconfigv1alpha1.FederationDomainPhaseReady,
|
|
)
|
|
|
|
// hostname and port WITHOUT SCHEME for direct access to the supervisor's port 8443
|
|
physicalAddress := testlib.NewSupervisorIssuer(t, env.SupervisorHTTPSAddress).Address()
|
|
|
|
dnsOverrides := map[string]string{
|
|
fakeHostname + ":443": physicalAddress,
|
|
}
|
|
return env, kubeClientForK8sResourcesOnly, fakeIssuerForDisplayPurposes, ca, dnsOverrides
|
|
}
|
|
|
|
func removeSomeKeysFromEachAuditLogEvent(logs []map[string]any) {
|
|
for _, log := range logs {
|
|
delete(log, "level")
|
|
delete(log, "auditEvent")
|
|
delete(log, "caller")
|
|
delete(log, "sourceIPs")
|
|
delete(log, "userAgent")
|
|
delete(log, "timestamp")
|
|
delete(log, "latency")
|
|
delete(log, "auditID")
|
|
}
|
|
}
|
|
|
|
func getFilteredAuditLogs(
|
|
t *testing.T,
|
|
ctx context.Context,
|
|
filterAuditLogEvent func(log map[string]any) bool,
|
|
kubeClient kubernetes.Interface,
|
|
namespace string,
|
|
appName string,
|
|
startTime metav1.Time,
|
|
) []map[string]any {
|
|
t.Helper()
|
|
|
|
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
|
defer cancel()
|
|
|
|
pods, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
|
|
LabelSelector: labels.Set{"app": appName}.String(),
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
var allPodLogsBuffer bytes.Buffer
|
|
for _, pod := range pods.Items {
|
|
_, err = io.Copy(&allPodLogsBuffer, getLogsForPodSince(t, ctx, kubeClient, pod, startTime))
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
allPodLogs := strings.Split(allPodLogsBuffer.String(), "\n")
|
|
var filteredAuditLogs []map[string]any
|
|
for _, podLog := range allPodLogs {
|
|
if len(podLog) == 0 {
|
|
continue
|
|
}
|
|
if !strings.HasPrefix(podLog, "{") {
|
|
// For some reason that needs investigation, an error message can appear in the pod log without
|
|
// being formatted as JSON. It starts with "http2: server: error reading preface from client".
|
|
// For now, just ignore it for the purposes of this test.
|
|
continue
|
|
}
|
|
var deserializedPodLog map[string]any
|
|
err = json.Unmarshal([]byte(podLog), &deserializedPodLog)
|
|
require.NoErrorf(t, err, "error parsing line of pod log: %s", podLog)
|
|
isAuditEventBool, hasAuditEvent := deserializedPodLog["auditEvent"]
|
|
if hasAuditEvent {
|
|
require.Equal(t, true, isAuditEventBool)
|
|
require.Equal(t, "info", deserializedPodLog["level"])
|
|
}
|
|
if hasAuditEvent && filterAuditLogEvent(deserializedPodLog) {
|
|
filteredAuditLogs = append(filteredAuditLogs, deserializedPodLog)
|
|
}
|
|
}
|
|
|
|
return filteredAuditLogs
|
|
}
|
|
|
|
func getLogsForPodSince(
|
|
t *testing.T,
|
|
ctx context.Context,
|
|
kubeClient kubernetes.Interface,
|
|
pod corev1.Pod,
|
|
startTime metav1.Time,
|
|
) *bytes.Buffer {
|
|
t.Helper()
|
|
|
|
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
defer cancel()
|
|
|
|
req := kubeClient.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &corev1.PodLogOptions{
|
|
SinceTime: &startTime,
|
|
})
|
|
body, err := req.Stream(ctx)
|
|
require.NoError(t, err)
|
|
|
|
var buf bytes.Buffer
|
|
_, err = io.Copy(&buf, body)
|
|
require.NoError(t, err)
|
|
require.NoError(t, body.Close())
|
|
|
|
return &buf
|
|
}
|