Merge remote-tracking branch 'upstream/main' into impersonation-proxy

Signed-off-by: Andrew Keesler <akeesler@vmware.com>

test/integration/cli_test.go

@@ -24,8 +24,19 @@ import (
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/errgroup"
 	"gopkg.in/square/go-jose.v2"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientauthenticationv1beta1 "k8s.io/client-go/pkg/apis/clientauthentication/v1beta1"
 
+	identityapi "go.pinniped.dev/generated/latest/apis/concierge/identity"
+	identityv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/identity/v1alpha1"
+	loginapi "go.pinniped.dev/generated/latest/apis/concierge/login"
+	loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1"
+	"go.pinniped.dev/internal/groupsuffix"
+	"go.pinniped.dev/internal/plog"
 	"go.pinniped.dev/internal/testutil"
 	"go.pinniped.dev/pkg/oidcclient"
 	"go.pinniped.dev/pkg/oidcclient/filesession"
@@ -36,8 +47,6 @@ import (
 func TestCLIGetKubeconfigStaticToken(t *testing.T) {
 	env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable)
 
-	library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "")
-
 	// Create a test webhook configuration to use with the CLI.
 	ctx, cancelFunc := context.WithTimeout(context.Background(), 5*time.Minute)
 	defer cancelFunc()
@@ -104,6 +113,19 @@ func TestCLIGetKubeconfigStaticToken(t *testing.T) {
 				group := group
 				t.Run("access as group "+group+" with client-go", library.AccessAsGroupTest(ctx, group, kubeClient))
 			}
+
+			// Validate that `pinniped whoami` returns the correct identity.
+			kubeconfigPath := filepath.Join(testutil.TempDir(t), "whoami-kubeconfig")
+			require.NoError(t, ioutil.WriteFile(kubeconfigPath, []byte(stdout), 0600))
+			assertWhoami(
+				ctx,
+				t,
+				false,
+				pinnipedExe,
+				kubeconfigPath,
+				env.TestUser.ExpectedUsername,
+				append(env.TestUser.ExpectedGroups, "system:authenticated"),
+			)
 		})
 	}
 }
@@ -119,6 +141,49 @@ func runPinnipedCLI(t *testing.T, envVars []string, pinnipedExe string, args ...
 	return stdout.String(), stderr.String()
 }
 
+func assertWhoami(ctx context.Context, t *testing.T, useProxy bool, pinnipedExe, kubeconfigPath, wantUsername string, wantGroups []string) {
+	t.Helper()
+
+	apiGroupSuffix := library.IntegrationEnv(t).APIGroupSuffix
+
+	var stdout, stderr bytes.Buffer
+	cmd := exec.CommandContext(
+		ctx,
+		pinnipedExe,
+		"whoami",
+		"--kubeconfig",
+		kubeconfigPath,
+		"--output",
+		"yaml",
+		"--api-group-suffix",
+		apiGroupSuffix,
+	)
+	if useProxy {
+		cmd.Env = append(os.Environ(), library.IntegrationEnv(t).ProxyEnv()...)
+	}
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	require.NoErrorf(t, cmd.Run(), "stderr:\n%s\n\nstdout:\n%s\n\n", stderr.String(), stdout.String())
+
+	whoAmI := deserializeWhoAmIRequest(t, stdout.String(), apiGroupSuffix)
+	require.Equal(t, wantUsername, whoAmI.Status.KubernetesUserInfo.User.Username)
+	require.ElementsMatch(t, wantGroups, whoAmI.Status.KubernetesUserInfo.User.Groups)
+}
+
+func deserializeWhoAmIRequest(t *testing.T, data string, apiGroupSuffix string) *identityv1alpha1.WhoAmIRequest {
+	t.Helper()
+
+	scheme, _, _ := conciergeschemeNew(apiGroupSuffix)
+	codecs := serializer.NewCodecFactory(scheme)
+	respInfo, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), runtime.ContentTypeYAML)
+	require.True(t, ok)
+
+	obj, err := runtime.Decode(respInfo.Serializer, []byte(data))
+	require.NoError(t, err)
+
+	return obj.(*identityv1alpha1.WhoAmIRequest)
+}
+
 func TestCLILoginOIDC(t *testing.T) {
 	env := library.IntegrationEnv(t)
 
@@ -355,3 +420,110 @@ func oidcLoginCommand(ctx context.Context, t *testing.T, pinnipedExe string, ses
 	cmd.Env = append(os.Environ(), env.ProxyEnv()...)
 	return cmd
 }
+
+// conciergeschemeNew is a temporary private function to stand in place for
+// "go.pinniped.dev/internal/concierge/scheme".New until the later function is merged to main.
+func conciergeschemeNew(apiGroupSuffix string) (_ *runtime.Scheme, login, identity schema.GroupVersion) {
+	// standard set up of the server side scheme
+	scheme := runtime.NewScheme()
+
+	// add the options to empty v1
+	metav1.AddToGroupVersion(scheme, metav1.Unversioned)
+
+	// nothing fancy is required if using the standard group suffix
+	if apiGroupSuffix == groupsuffix.PinnipedDefaultSuffix {
+		schemeBuilder := runtime.NewSchemeBuilder(
+			loginv1alpha1.AddToScheme,
+			loginapi.AddToScheme,
+			identityv1alpha1.AddToScheme,
+			identityapi.AddToScheme,
+		)
+		utilruntime.Must(schemeBuilder.AddToScheme(scheme))
+		return scheme, loginv1alpha1.SchemeGroupVersion, identityv1alpha1.SchemeGroupVersion
+	}
+
+	loginConciergeGroupData, identityConciergeGroupData := groupsuffix.ConciergeAggregatedGroups(apiGroupSuffix)
+
+	addToSchemeAtNewGroup(scheme, loginv1alpha1.GroupName, loginConciergeGroupData.Group, loginv1alpha1.AddToScheme, loginapi.AddToScheme)
+	addToSchemeAtNewGroup(scheme, identityv1alpha1.GroupName, identityConciergeGroupData.Group, identityv1alpha1.AddToScheme, identityapi.AddToScheme)
+
+	// manually register conversions and defaulting into the correct scheme since we cannot directly call AddToScheme
+	schemeBuilder := runtime.NewSchemeBuilder(
+		loginv1alpha1.RegisterConversions,
+		loginv1alpha1.RegisterDefaults,
+		identityv1alpha1.RegisterConversions,
+		identityv1alpha1.RegisterDefaults,
+	)
+	utilruntime.Must(schemeBuilder.AddToScheme(scheme))
+
+	// we do not want to return errors from the scheme and instead would prefer to defer
+	// to the REST storage layer for consistency.  The simplest way to do this is to force
+	// a cache miss from the authenticator cache.  Kube API groups are validated via the
+	// IsDNS1123Subdomain func thus we can easily create a group that is guaranteed never
+	// to be in the authenticator cache.  Add a timestamp just to be extra sure.
+	const authenticatorCacheMissPrefix = "_INVALID_API_GROUP_"
+	authenticatorCacheMiss := authenticatorCacheMissPrefix + time.Now().UTC().String()
+
+	// we do not have any defaulting functions for *loginv1alpha1.TokenCredentialRequest
+	// today, but we may have some in the future.  Calling AddTypeDefaultingFunc overwrites
+	// any previously registered defaulting function.  Thus to make sure that we catch
+	// a situation where we add a defaulting func, we attempt to call it here with a nil
+	// *loginv1alpha1.TokenCredentialRequest.  This will do nothing when there is no
+	// defaulting func registered, but it will almost certainly panic if one is added.
+	scheme.Default((*loginv1alpha1.TokenCredentialRequest)(nil))
+
+	// on incoming requests, restore the authenticator API group to the standard group
+	// note that we are responsible for duplicating this logic for every external API version
+	scheme.AddTypeDefaultingFunc(&loginv1alpha1.TokenCredentialRequest{}, func(obj interface{}) {
+		credentialRequest := obj.(*loginv1alpha1.TokenCredentialRequest)
+
+		if credentialRequest.Spec.Authenticator.APIGroup == nil {
+			// force a cache miss because this is an invalid request
+			plog.Debug("invalid token credential request, nil group", "authenticator", credentialRequest.Spec.Authenticator)
+			credentialRequest.Spec.Authenticator.APIGroup = &authenticatorCacheMiss
+			return
+		}
+
+		restoredGroup, ok := groupsuffix.Unreplace(*credentialRequest.Spec.Authenticator.APIGroup, apiGroupSuffix)
+		if !ok {
+			// force a cache miss because this is an invalid request
+			plog.Debug("invalid token credential request, wrong group", "authenticator", credentialRequest.Spec.Authenticator)
+			credentialRequest.Spec.Authenticator.APIGroup = &authenticatorCacheMiss
+			return
+		}
+
+		credentialRequest.Spec.Authenticator.APIGroup = &restoredGroup
+	})
+
+	return scheme, schema.GroupVersion(loginConciergeGroupData), schema.GroupVersion(identityConciergeGroupData)
+}
+
+func addToSchemeAtNewGroup(scheme *runtime.Scheme, oldGroup, newGroup string, funcs ...func(*runtime.Scheme) error) {
+	// we need a temporary place to register our types to avoid double registering them
+	tmpScheme := runtime.NewScheme()
+	schemeBuilder := runtime.NewSchemeBuilder(funcs...)
+	utilruntime.Must(schemeBuilder.AddToScheme(tmpScheme))
+
+	for gvk := range tmpScheme.AllKnownTypes() {
+		if gvk.GroupVersion() == metav1.Unversioned {
+			continue // metav1.AddToGroupVersion registers types outside of our aggregated API group that we need to ignore
+		}
+
+		if gvk.Group != oldGroup {
+			panic(fmt.Errorf("tmp scheme has type not in the old aggregated API group %s: %s", oldGroup, gvk)) // programmer error
+		}
+
+		obj, err := tmpScheme.New(gvk)
+		if err != nil {
+			panic(err) // programmer error, scheme internal code is broken
+		}
+		newGVK := schema.GroupVersionKind{
+			Group:   newGroup,
+			Version: gvk.Version,
+			Kind:    gvk.Kind,
+		}
+
+		// register the existing type but with the new group in the correct scheme
+		scheme.AddKnownTypeWithName(newGVK, obj)
+	}
+}

test/integration/concierge_api_serving_certs_test.go

@@ -23,8 +23,6 @@ func TestAPIServingCertificateAutoCreationAndRotation(t *testing.T) {
 	env := library.IntegrationEnv(t)
 	defaultServingCertResourceName := env.ConciergeAppName + "-api-tls-serving-certificate"
 
-	library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "")
-
 	tests := []struct {
 		name          string
 		forceRotation func(context.Context, kubernetes.Interface, string) error

test/integration/concierge_client_test.go

@@ -57,8 +57,6 @@ var maskKey = func(s string) string { return strings.ReplaceAll(s, "TESTING KEY"
 func TestClient(t *testing.T) {
 	env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable)
 
-	library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "")
-
 	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
 	defer cancel()
 

test/integration/concierge_credentialissuer_test.go

@@ -23,8 +23,6 @@ func TestCredentialIssuer(t *testing.T) {
 	client := library.NewConciergeClientset(t)
 	aggregatedClientset := library.NewAggregatedClientset(t)
 
-	library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "")
-
 	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
 	defer cancel()
 

test/integration/concierge_credentialrequest_test.go

@@ -25,8 +25,6 @@ import (
 func TestUnsuccessfulCredentialRequest(t *testing.T) {
 	env := library.IntegrationEnv(t).WithCapability(library.AnonymousAuthenticationSupported)
 
-	library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "")
-
 	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
 	defer cancel()
 
@@ -49,8 +47,6 @@ func TestUnsuccessfulCredentialRequest(t *testing.T) {
 func TestSuccessfulCredentialRequest(t *testing.T) {
 	env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable)
 
-	library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "")
-
 	ctx, cancel := context.WithTimeout(context.Background(), 6*time.Minute)
 	defer cancel()
 
@@ -135,9 +131,7 @@ func TestSuccessfulCredentialRequest(t *testing.T) {
 }
 
 func TestFailedCredentialRequestWhenTheRequestIsValidButTheTokenDoesNotAuthenticateTheUser(t *testing.T) {
-	env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable)
-
-	library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "")
+	_ = library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable)
 
 	// Create a testWebhook so we have a legitimate authenticator to pass to the
 	// TokenCredentialRequest API.
@@ -157,9 +151,7 @@ func TestFailedCredentialRequestWhenTheRequestIsValidButTheTokenDoesNotAuthentic
 }
 
 func TestCredentialRequest_ShouldFailWhenRequestDoesNotIncludeToken(t *testing.T) {
-	env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable)
-
-	library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "")
+	_ = library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable)
 
 	// Create a testWebhook so we have a legitimate authenticator to pass to the
 	// TokenCredentialRequest API.

test/integration/concierge_kubecertagent_test.go

@@ -29,8 +29,6 @@ const (
 func TestKubeCertAgent(t *testing.T) {
 	env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable)
 
-	library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "")
-
 	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
 	defer cancel()
 

test/integration/e2e_test.go

@@ -46,9 +46,6 @@ func TestE2EFullIntegration(t *testing.T) {
 	defer library.DumpLogs(t, env.SupervisorNamespace, "")
 	defer library.DumpLogs(t, "dex", "app=proxy")
 
-	library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "")
-	library.AssertNoRestartsDuringTest(t, env.SupervisorNamespace, "")
-
 	ctx, cancelFunc := context.WithTimeout(context.Background(), 5*time.Minute)
 	defer cancelFunc()
 
@@ -338,4 +335,15 @@ status:
       username: `+env.SupervisorTestUpstream.Username+`
 `,
 		string(kubectlOutput3))
+
+	// Validate that `pinniped whoami` returns the correct identity.
+	assertWhoami(
+		ctx,
+		t,
+		true,
+		pinnipedExe,
+		kubeconfigPath,
+		env.SupervisorTestUpstream.Username,
+		append(env.SupervisorTestUpstream.ExpectedGroups, "system:authenticated"),
+	)
 }

test/integration/supervisor_discovery_test.go

@@ -44,8 +44,6 @@ func TestSupervisorOIDCDiscovery(t *testing.T) {
 	env := library.IntegrationEnv(t)
 	client := library.NewSupervisorClientset(t)
 
-	library.AssertNoRestartsDuringTest(t, env.SupervisorNamespace, "")
-
 	ns := env.SupervisorNamespace
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
@@ -151,8 +149,6 @@ func TestSupervisorTLSTerminationWithSNI(t *testing.T) {
 	pinnipedClient := library.NewSupervisorClientset(t)
 	kubeClient := library.NewKubernetesClientset(t)
 
-	library.AssertNoRestartsDuringTest(t, env.SupervisorNamespace, "")
-
 	ns := env.SupervisorNamespace
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
 	defer cancel()
@@ -224,8 +220,6 @@ func TestSupervisorTLSTerminationWithDefaultCerts(t *testing.T) {
 	pinnipedClient := library.NewSupervisorClientset(t)
 	kubeClient := library.NewKubernetesClientset(t)
 
-	library.AssertNoRestartsDuringTest(t, env.SupervisorNamespace, "")
-
 	ns := env.SupervisorNamespace
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
 	defer cancel()

test/integration/supervisor_healthz_test.go

@@ -29,8 +29,6 @@ func TestSupervisorHealthz(t *testing.T) {
 		t.Skip("PINNIPED_TEST_SUPERVISOR_HTTP_ADDRESS not defined")
 	}
 
-	library.AssertNoRestartsDuringTest(t, env.SupervisorNamespace, "")
-
 	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
 	defer cancel()
 

test/integration/supervisor_login_test.go

@@ -43,8 +43,6 @@ func TestSupervisorLogin(t *testing.T) {
 	defer library.DumpLogs(t, env.SupervisorNamespace, "")
 	defer library.DumpLogs(t, "dex", "app=proxy")
 
-	library.AssertNoRestartsDuringTest(t, env.SupervisorNamespace, "")
-
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
 	defer cancel()
 

test/integration/supervisor_secrets_test.go

@@ -24,8 +24,6 @@ func TestSupervisorSecrets(t *testing.T) {
 	kubeClient := library.NewKubernetesClientset(t)
 	supervisorClient := library.NewSupervisorClientset(t)
 
-	library.AssertNoRestartsDuringTest(t, env.SupervisorNamespace, "")
-
 	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
 	defer cancel()
 

test/integration/supervisor_upstream_test.go

@@ -17,8 +17,6 @@ import (
 func TestSupervisorUpstreamOIDCDiscovery(t *testing.T) {
 	env := library.IntegrationEnv(t)
 
-	library.AssertNoRestartsDuringTest(t, env.SupervisorNamespace, "")
-
 	t.Run("invalid missing secret and bad issuer", func(t *testing.T) {
 		t.Parallel()
 		spec := v1alpha1.OIDCIdentityProviderSpec{

test/library/assertions.go

@@ -14,6 +14,7 @@ import (
 	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
 )
 
 // RequireEventuallyWithoutError is similar to require.Eventually() except that it also allows the caller to
@@ -52,52 +53,77 @@ func RequireNeverWithoutError(
 	}
 }
 
-// NewRestartAssertion allows a caller to assert that there were no restarts for a Pod in the
+// assertNoRestartsDuringTest allows a caller to assert that there were no restarts for a Pod in the
 // provided namespace with the provided labelSelector during the lifetime of a test.
-func AssertNoRestartsDuringTest(t *testing.T, namespace, labelSelector string) {
+func assertNoRestartsDuringTest(t *testing.T, namespace, labelSelector string) {
 	t.Helper()
+	kubeClient := NewKubernetesClientset(t)
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
 
-	previousRestartCounts := getRestartCounts(t, namespace, labelSelector)
+	previousRestartCounts := getRestartCounts(ctx, t, kubeClient, namespace, labelSelector)
 
 	t.Cleanup(func() {
-		currentRestartCounts := getRestartCounts(t, namespace, labelSelector)
+		ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+		defer cancel()
+		currentRestartCounts := getRestartCounts(ctx, t, kubeClient, namespace, labelSelector)
 
 		for key, previousRestartCount := range previousRestartCounts {
 			currentRestartCount, ok := currentRestartCounts[key]
-			if assert.Truef(
+
+			// If the container no longer exists, that's a test failure.
+			if !assert.Truef(
 				t,
 				ok,
-				"pod namespace/name/container %s existed at beginning of the test, but not the end",
-				key,
+				"container %s existed at beginning of the test, but not the end",
+				key.String(),
 			) {
-				assert.Equal(
-					t,
-					previousRestartCount,
-					currentRestartCount,
-					"pod namespace/name/container %s has restarted %d times (original count was %d)",
-					key,
-					currentRestartCount,
-					previousRestartCount,
-				)
+				continue
+			}
+
+			// Expect the restart count to be the same as it was before the test.
+			if !assert.Equal(
+				t,
+				previousRestartCount,
+				currentRestartCount,
+				"container %s has restarted %d times (original count was %d)",
+				key.String(),
+				currentRestartCount,
+				previousRestartCount,
+			) {
+				// Attempt to dump the logs from the previous container that crashed.
+				dumpContainerLogs(ctx, t, kubeClient, key.namespace, key.pod, key.container, true)
 			}
 		}
 	})
 }
 
-func getRestartCounts(t *testing.T, namespace, labelSelector string) map[string]int32 {
-	t.Helper()
+type containerRestartKey struct {
+	namespace string
+	pod       string
+	container string
+}
 
-	kubeClient := NewKubernetesClientset(t)
-	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
-	defer cancel()
+func (k containerRestartKey) String() string {
+	return fmt.Sprintf("%s/%s/%s", k.namespace, k.pod, k.container)
+}
+
+type containerRestartMap map[containerRestartKey]int32
+
+func getRestartCounts(ctx context.Context, t *testing.T, kubeClient kubernetes.Interface, namespace, labelSelector string) containerRestartMap {
+	t.Helper()
 
 	pods, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector})
 	require.NoError(t, err)
 
-	restartCounts := make(map[string]int32)
+	restartCounts := make(containerRestartMap)
 	for _, pod := range pods.Items {
 		for _, container := range pod.Status.ContainerStatuses {
-			key := fmt.Sprintf("%s/%s/%s", pod.Namespace, pod.Name, container.Name)
+			key := containerRestartKey{
+				namespace: pod.Namespace,
+				pod:       pod.Name,
+				container: container.Name,
+			}
 			restartCounts[key] = container.RestartCount
 		}
 	}

test/library/dumplogs.go

@@ -6,12 +6,15 @@ package library
 import (
 	"bufio"
 	"context"
+	"fmt"
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
 )
 
 // DumpLogs is meant to be called in a `defer` to dump the logs of components in the cluster on a test failure.
@@ -25,25 +28,37 @@ func DumpLogs(t *testing.T, namespace string, labelSelector string) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
 	defer cancel()
 
-	logTailLines := int64(40)
 	pods, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector})
 	require.NoError(t, err)
 
 	for _, pod := range pods.Items {
 		for _, container := range pod.Status.ContainerStatuses {
-			t.Logf("pod %s/%s container %s restarted %d times:", pod.Namespace, pod.Name, container.Name, container.RestartCount)
-			req := kubeClient.CoreV1().Pods(namespace).GetLogs(pod.Name, &corev1.PodLogOptions{
-				Container: container.Name,
-				TailLines: &logTailLines,
-			})
-			logReader, err := req.Stream(ctx)
-			require.NoError(t, err)
-
-			scanner := bufio.NewScanner(logReader)
-			for scanner.Scan() {
-				t.Logf("%s/%s/%s > %s", pod.Namespace, pod.Name, container.Name, scanner.Text())
+			if container.RestartCount > 0 {
+				dumpContainerLogs(ctx, t, kubeClient, pod.Namespace, pod.Name, container.Name, true)
 			}
-			require.NoError(t, scanner.Err())
+			dumpContainerLogs(ctx, t, kubeClient, pod.Namespace, pod.Name, container.Name, false)
 		}
 	}
 }
+
+func dumpContainerLogs(ctx context.Context, t *testing.T, kubeClient kubernetes.Interface, namespace, pod, container string, prev bool) {
+	logTailLines := int64(40)
+	shortName := fmt.Sprintf("%s/%s/%s", namespace, pod, container)
+	logReader, err := kubeClient.CoreV1().Pods(namespace).GetLogs(pod, &corev1.PodLogOptions{
+		Container: container,
+		TailLines: &logTailLines,
+		Previous:  prev,
+	}).Stream(ctx)
+	if !assert.NoErrorf(t, err, "failed to stream logs for container %s", shortName) {
+		return
+	}
+	scanner := bufio.NewScanner(logReader)
+	for scanner.Scan() {
+		prefix := shortName
+		if prev {
+			prefix += " (previous)"
+		}
+		t.Logf("%s > %s", prefix, scanner.Text())
+	}
+	assert.NoError(t, scanner.Err(), "failed to read logs from container %s", shortName)
+}

test/library/env.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"os"
 	"strings"
+	"sync"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -74,9 +75,17 @@ func (e *TestEnv) ProxyEnv() []string {
 	return []string{"http_proxy=" + e.Proxy, "https_proxy=" + e.Proxy, "no_proxy=127.0.0.1"}
 }
 
+// memoizedTestEnvsByTest maps *testing.T pointers to *TestEnv. It exists so that we don't do all the
+// environment parsing N times per test and so that any implicit assertions happen only once.
+var memoizedTestEnvsByTest sync.Map //nolint: gochecknoglobals
+
 // IntegrationEnv gets the integration test environment from OS environment variables. This
 // method also implies SkipUnlessIntegration().
 func IntegrationEnv(t *testing.T) *TestEnv {
+	if existing, exists := memoizedTestEnvsByTest.Load(t); exists {
+		return existing.(*TestEnv)
+	}
+
 	t.Helper()
 	SkipUnlessIntegration(t)
 
@@ -97,8 +106,12 @@ func IntegrationEnv(t *testing.T) *TestEnv {
 	require.NoErrorf(t, err, "capabilities specification was invalid YAML")
 
 	loadEnvVars(t, &result)
-
 	result.t = t
+	memoizedTestEnvsByTest.Store(t, &result)
+
+	// In every integration test, assert that no pods in our namespaces restart during the test.
+	assertNoRestartsDuringTest(t, result.ConciergeNamespace, "")
+	assertNoRestartsDuringTest(t, result.SupervisorNamespace, "")
 	return &result
 }