diff --git a/.gitignore b/.gitignore index 3a5c3201d..88dd571f5 100644 --- a/.gitignore +++ b/.gitignore @@ -14,8 +14,11 @@ # Dependency directories (remove the comment below to include it) # vendor/ -# goland +# GoLand .idea # Intermediate files used by Tilt /hack/lib/tilt/build + +# MacOS Desktop Services Store +.DS_Store diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c00094b4b..5157c86cf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -96,36 +96,29 @@ docker build . - [`kapp`](https://carvel.dev/#getting-started) - [`kind`](https://kind.sigs.k8s.io/docs/user/quick-start) - [`kubectl`](https://kubernetes.io/docs/tasks/tools/install-kubectl/) - - [`tilt`](https://docs.tilt.dev/install.html) - [`ytt`](https://carvel.dev/#getting-started) On macOS, these tools can be installed with [Homebrew](https://brew.sh/) (assuming you have Chrome installed already): ```bash - brew install kind tilt-dev/tap/tilt k14s/tap/ytt k14s/tap/kapp kubectl chromedriver && brew cask install docker + brew install kind k14s/tap/ytt k14s/tap/kapp kubectl chromedriver && brew cask install docker ``` -1. Create a local Kubernetes cluster using `kind`: +1. Create a kind cluster, compile, create container images, and install Pinniped and supporting dependencies using: ```bash - ./hack/kind-up.sh + ./hack/prepare-for-integration-tests.sh ``` -1. Install Pinniped and supporting dependencies using `tilt`: - - ```bash - ./hack/tilt-up.sh - ``` - - Tilt will continue running and live-updating the Pinniped deployment whenever the code changes. - 1. Run the Pinniped integration tests: ```bash - source /tmp/integration-test-env && go test -v -count 1 ./test/integration + source /tmp/integration-test-env && go test -v -count 1 -timeout 0 ./test/integration ``` -To uninstall the test environment, run `./hack/tilt-down.sh`. +1. After making production code changes, recompile, redeploy, and run tests again by repeating the same + commands described above. If there are only test code changes, then simply run the tests again. + To destroy the local Kubernetes cluster, run `./hack/kind-down.sh`. ### Observing Tests on the Continuous Integration Environment diff --git a/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl b/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl index 399895015..3c0c8ba03 100644 --- a/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl +++ b/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl @@ -1,30 +1,36 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package v1alpha1 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// +kubebuilder:validation:Enum=KubeClusterSigningCertificate +// +kubebuilder:validation:Enum=KubeClusterSigningCertificate;ImpersonationProxy type StrategyType string -// +kubebuilder:validation:Enum=TokenCredentialRequestAPI +// +kubebuilder:validation:Enum=TokenCredentialRequestAPI;ImpersonationProxy type FrontendType string // +kubebuilder:validation:Enum=Success;Error type StrategyStatus string -// +kubebuilder:validation:Enum=FetchedKey;CouldNotFetchKey +// +kubebuilder:validation:Enum=Listening;Pending;Disabled;ErrorDuringSetup;CouldNotFetchKey;CouldNotGetClusterInfo;FetchedKey type StrategyReason string const ( KubeClusterSigningCertificateStrategyType = StrategyType("KubeClusterSigningCertificate") + ImpersonationProxyStrategyType = StrategyType("ImpersonationProxy") TokenCredentialRequestAPIFrontendType = FrontendType("TokenCredentialRequestAPI") + ImpersonationProxyFrontendType = FrontendType("ImpersonationProxy") SuccessStrategyStatus = StrategyStatus("Success") ErrorStrategyStatus = StrategyStatus("Error") + ListeningStrategyReason = StrategyReason("Listening") + PendingStrategyReason = StrategyReason("Pending") + DisabledStrategyReason = StrategyReason("Disabled") + ErrorDuringSetupStrategyReason = StrategyReason("ErrorDuringSetup") CouldNotFetchKeyStrategyReason = StrategyReason("CouldNotFetchKey") CouldNotGetClusterInfoStrategyReason = StrategyReason("CouldNotGetClusterInfo") FetchedKeyStrategyReason = StrategyReason("FetchedKey") @@ -82,6 +88,10 @@ type CredentialIssuerFrontend struct { // TokenCredentialRequestAPIInfo describes the parameters for the TokenCredentialRequest API on this Concierge. // This field is only set when Type is "TokenCredentialRequestAPI". TokenCredentialRequestAPIInfo *TokenCredentialRequestAPIInfo `json:"tokenCredentialRequestInfo,omitempty"` + + // ImpersonationProxyInfo describes the parameters for the impersonation proxy on this Concierge. + // This field is only set when Type is "ImpersonationProxy". + ImpersonationProxyInfo *ImpersonationProxyInfo `json:"impersonationProxyInfo,omitempty"` } // TokenCredentialRequestAPIInfo describes the parameters for the TokenCredentialRequest API on this Concierge. @@ -91,7 +101,19 @@ type TokenCredentialRequestAPIInfo struct { // +kubebuilder:validation:Pattern=`^https://|^http://` Server string `json:"server"` - // CertificateAuthorityData is the Kubernetes API server CA bundle. + // CertificateAuthorityData is the base64-encoded Kubernetes API server CA bundle. + // +kubebuilder:validation:MinLength=1 + CertificateAuthorityData string `json:"certificateAuthorityData"` +} + +// ImpersonationProxyInfo describes the parameters for the impersonation proxy on this Concierge. +type ImpersonationProxyInfo struct { + // Endpoint is the HTTPS endpoint of the impersonation proxy. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:Pattern=`^https://` + Endpoint string `json:"endpoint"` + + // CertificateAuthorityData is the base64-encoded PEM CA bundle of the impersonation proxy. // +kubebuilder:validation:MinLength=1 CertificateAuthorityData string `json:"certificateAuthorityData"` } diff --git a/cmd/local-user-authenticator/main.go b/cmd/local-user-authenticator/main.go index 379cde217..7c1833ff0 100644 --- a/cmd/local-user-authenticator/main.go +++ b/cmd/local-user-authenticator/main.go @@ -54,12 +54,12 @@ const ( ) type webhook struct { - certProvider dynamiccert.Provider + certProvider dynamiccert.Private secretInformer corev1informers.SecretInformer } func newWebhook( - certProvider dynamiccert.Provider, + certProvider dynamiccert.Private, secretInformer corev1informers.SecretInformer, ) *webhook { return &webhook{ @@ -281,7 +281,7 @@ func respondWithAuthenticated( func startControllers( ctx context.Context, - dynamicCertProvider dynamiccert.Provider, + dynamicCertProvider dynamiccert.Private, kubeClient kubernetes.Interface, kubeInformers kubeinformers.SharedInformerFactory, ) { @@ -328,7 +328,7 @@ func startControllers( func startWebhook( ctx context.Context, l net.Listener, - dynamicCertProvider dynamiccert.Provider, + dynamicCertProvider dynamiccert.Private, secretInformer corev1informers.SecretInformer, ) error { return newWebhook(dynamicCertProvider, secretInformer).start(ctx, l) @@ -355,7 +355,7 @@ func run() error { kubeinformers.WithNamespace(namespace), ) - dynamicCertProvider := dynamiccert.New() + dynamicCertProvider := dynamiccert.NewServingCert("local-user-authenticator-tls-serving-certificate") startControllers(ctx, dynamicCertProvider, client.Kubernetes, kubeInformers) plog.Debug("controllers are ready") diff --git a/cmd/local-user-authenticator/main_test.go b/cmd/local-user-authenticator/main_test.go index 07771971f..cee265ee7 100644 --- a/cmd/local-user-authenticator/main_test.go +++ b/cmd/local-user-authenticator/main_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package main @@ -8,7 +8,6 @@ import ( "context" "crypto/tls" "crypto/x509" - "crypto/x509/pkix" "encoding/json" "fmt" "io" @@ -99,7 +98,7 @@ func TestWebhook(t *testing.T) { }, })) - secretInformer := createSecretInformer(t, kubeClient) + secretInformer := createSecretInformer(ctx, t, kubeClient) certProvider, caBundle, serverName := newCertProvider(t) w := newWebhook(certProvider, secretInformer) @@ -437,7 +436,7 @@ func TestWebhook(t *testing.T) { } } -func createSecretInformer(t *testing.T, kubeClient kubernetes.Interface) corev1informers.SecretInformer { +func createSecretInformer(ctx context.Context, t *testing.T, kubeClient kubernetes.Interface) corev1informers.SecretInformer { t.Helper() kubeInformers := kubeinformers.NewSharedInformerFactory(kubeClient, 0) @@ -448,9 +447,6 @@ func createSecretInformer(t *testing.T, kubeClient kubernetes.Interface) corev1i // informer factory before syncing it. secretInformer.Informer() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) - defer cancel() - kubeInformers.Start(ctx.Done()) informerTypesSynced := kubeInformers.WaitForCacheSync(ctx.Done()) @@ -462,22 +458,23 @@ func createSecretInformer(t *testing.T, kubeClient kubernetes.Interface) corev1i // newClientProvider returns a dynamiccert.Provider configured // with valid serving cert, the CA bundle that can be used to verify the serving // cert, and the server name that can be used to verify the TLS peer. -func newCertProvider(t *testing.T) (dynamiccert.Provider, []byte, string) { +func newCertProvider(t *testing.T) (dynamiccert.Private, []byte, string) { t.Helper() serverName := "local-user-authenticator" - ca, err := certauthority.New(pkix.Name{CommonName: serverName + " CA"}, time.Hour*24) + ca, err := certauthority.New(serverName+" CA", time.Hour*24) require.NoError(t, err) - cert, err := ca.Issue(pkix.Name{CommonName: serverName}, []string{serverName}, nil, time.Hour*24) + cert, err := ca.IssueServerCert([]string{serverName}, nil, time.Hour*24) require.NoError(t, err) certPEM, keyPEM, err := certauthority.ToPEM(cert) require.NoError(t, err) - certProvider := dynamiccert.New() - certProvider.Set(certPEM, keyPEM) + certProvider := dynamiccert.NewServingCert(t.Name()) + err = certProvider.SetCertKeyContent(certPEM, keyPEM) + require.NoError(t, err) return certProvider, ca.Bundle(), serverName } diff --git a/cmd/pinniped/cmd/flag_types.go b/cmd/pinniped/cmd/flag_types.go new file mode 100644 index 000000000..2bc55e06f --- /dev/null +++ b/cmd/pinniped/cmd/flag_types.go @@ -0,0 +1,106 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "bytes" + "crypto/x509" + "flag" + "fmt" + "io/ioutil" + "strings" + + "github.com/spf13/pflag" + + configv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" +) + +// conciergeModeFlag represents the method by which we should connect to the Concierge on a cluster during login. +// this is meant to be a valid flag.Value implementation. +type conciergeModeFlag int + +var _ flag.Value = new(conciergeModeFlag) + +const ( + modeUnknown conciergeModeFlag = iota + modeTokenCredentialRequestAPI + modeImpersonationProxy +) + +func (f *conciergeModeFlag) String() string { + switch *f { + case modeImpersonationProxy: + return "ImpersonationProxy" + case modeTokenCredentialRequestAPI: + return "TokenCredentialRequestAPI" + case modeUnknown: + fallthrough + default: + return "TokenCredentialRequestAPI" + } +} + +func (f *conciergeModeFlag) Set(s string) error { + if strings.EqualFold(s, "") { + *f = modeUnknown + return nil + } + if strings.EqualFold(s, "TokenCredentialRequestAPI") { + *f = modeTokenCredentialRequestAPI + return nil + } + if strings.EqualFold(s, "ImpersonationProxy") { + *f = modeImpersonationProxy + return nil + } + return fmt.Errorf("invalid mode %q, valid modes are TokenCredentialRequestAPI and ImpersonationProxy", s) +} + +func (f *conciergeModeFlag) Type() string { + return "mode" +} + +// MatchesFrontend returns true iff the flag matches the type of the provided frontend. +func (f *conciergeModeFlag) MatchesFrontend(frontend *configv1alpha1.CredentialIssuerFrontend) bool { + switch *f { + case modeImpersonationProxy: + return frontend.Type == configv1alpha1.ImpersonationProxyFrontendType + case modeTokenCredentialRequestAPI: + return frontend.Type == configv1alpha1.TokenCredentialRequestAPIFrontendType + case modeUnknown: + fallthrough + default: + return true + } +} + +// caBundlePathsVar represents a list of CA bundle paths, which load from disk when the flag is populated. +type caBundleFlag []byte + +var _ pflag.Value = new(caBundleFlag) + +func (f *caBundleFlag) String() string { + return string(*f) +} + +func (f *caBundleFlag) Set(path string) error { + pem, err := ioutil.ReadFile(path) + if err != nil { + return fmt.Errorf("could not read CA bundle path: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return fmt.Errorf("failed to load any CA certificates from %q", path) + } + if len(*f) == 0 { + *f = pem + return nil + } + *f = bytes.Join([][]byte{*f, pem}, []byte("\n")) + return nil +} + +func (f *caBundleFlag) Type() string { + return "path" +} diff --git a/cmd/pinniped/cmd/flag_types_test.go b/cmd/pinniped/cmd/flag_types_test.go new file mode 100644 index 000000000..101191d54 --- /dev/null +++ b/cmd/pinniped/cmd/flag_types_test.go @@ -0,0 +1,73 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "bytes" + "fmt" + "io/ioutil" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + configv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" + "go.pinniped.dev/internal/certauthority" + "go.pinniped.dev/internal/testutil" +) + +func TestConciergeModeFlag(t *testing.T) { + var f conciergeModeFlag + require.Equal(t, "mode", f.Type()) + require.Equal(t, modeUnknown, f) + require.NoError(t, f.Set("")) + require.Equal(t, modeUnknown, f) + require.EqualError(t, f.Set("foo"), `invalid mode "foo", valid modes are TokenCredentialRequestAPI and ImpersonationProxy`) + require.True(t, f.MatchesFrontend(&configv1alpha1.CredentialIssuerFrontend{Type: configv1alpha1.TokenCredentialRequestAPIFrontendType})) + require.True(t, f.MatchesFrontend(&configv1alpha1.CredentialIssuerFrontend{Type: configv1alpha1.ImpersonationProxyFrontendType})) + + require.NoError(t, f.Set("TokenCredentialRequestAPI")) + require.Equal(t, modeTokenCredentialRequestAPI, f) + require.Equal(t, "TokenCredentialRequestAPI", f.String()) + require.True(t, f.MatchesFrontend(&configv1alpha1.CredentialIssuerFrontend{Type: configv1alpha1.TokenCredentialRequestAPIFrontendType})) + require.False(t, f.MatchesFrontend(&configv1alpha1.CredentialIssuerFrontend{Type: configv1alpha1.ImpersonationProxyFrontendType})) + + require.NoError(t, f.Set("tokencredentialrequestapi")) + require.Equal(t, modeTokenCredentialRequestAPI, f) + require.Equal(t, "TokenCredentialRequestAPI", f.String()) + + require.NoError(t, f.Set("ImpersonationProxy")) + require.Equal(t, modeImpersonationProxy, f) + require.Equal(t, "ImpersonationProxy", f.String()) + require.False(t, f.MatchesFrontend(&configv1alpha1.CredentialIssuerFrontend{Type: configv1alpha1.TokenCredentialRequestAPIFrontendType})) + require.True(t, f.MatchesFrontend(&configv1alpha1.CredentialIssuerFrontend{Type: configv1alpha1.ImpersonationProxyFrontendType})) + + require.NoError(t, f.Set("impersonationproxy")) + require.Equal(t, modeImpersonationProxy, f) + require.Equal(t, "ImpersonationProxy", f.String()) +} + +func TestCABundleFlag(t *testing.T) { + testCA, err := certauthority.New("Test CA", 1*time.Hour) + require.NoError(t, err) + tmpdir := testutil.TempDir(t) + emptyFilePath := filepath.Join(tmpdir, "empty") + require.NoError(t, ioutil.WriteFile(emptyFilePath, []byte{}, 0600)) + + testCAPath := filepath.Join(tmpdir, "testca.pem") + require.NoError(t, ioutil.WriteFile(testCAPath, testCA.Bundle(), 0600)) + + f := caBundleFlag{} + require.Equal(t, "path", f.Type()) + require.Equal(t, "", f.String()) + require.EqualError(t, f.Set("./does/not/exist"), "could not read CA bundle path: open ./does/not/exist: no such file or directory") + require.EqualError(t, f.Set(emptyFilePath), fmt.Sprintf("failed to load any CA certificates from %q", emptyFilePath)) + + require.NoError(t, f.Set(testCAPath)) + require.Equal(t, 1, bytes.Count(f, []byte("BEGIN CERTIFICATE"))) + + require.NoError(t, f.Set(testCAPath)) + require.Equal(t, 2, bytes.Count(f, []byte("BEGIN CERTIFICATE"))) +} diff --git a/cmd/pinniped/cmd/kubeconfig.go b/cmd/pinniped/cmd/kubeconfig.go index 0e9e2b6a4..151e04dfd 100644 --- a/cmd/pinniped/cmd/kubeconfig.go +++ b/cmd/pinniped/cmd/kubeconfig.go @@ -4,27 +4,31 @@ package cmd import ( - "bytes" "context" + "crypto/tls" + "crypto/x509" "encoding/base64" "fmt" "io" - "io/ioutil" + "log" + "net/http" "os" "strconv" "strings" "time" "github.com/coreos/go-oidc/v3/oidc" + "github.com/go-logr/logr" + "github.com/go-logr/stdr" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientauthenticationv1beta1 "k8s.io/client-go/pkg/apis/clientauthentication/v1beta1" + _ "k8s.io/client-go/plugin/pkg/client/auth" // Adds handlers for various dynamic auth plugins in client-go "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - _ "k8s.io/client-go/plugin/pkg/client/auth" // Adds handlers for various dynamic auth plugins in client-go - conciergev1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/authentication/v1alpha1" + configv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" conciergeclientset "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned" "go.pinniped.dev/internal/groupsuffix" ) @@ -32,12 +36,14 @@ import ( type kubeconfigDeps struct { getPathToSelf func() (string, error) getClientset getConciergeClientsetFunc + log logr.Logger } func kubeconfigRealDeps() kubeconfigDeps { return kubeconfigDeps{ getPathToSelf: os.Executable, getClientset: getRealConciergeClientset, + log: stdr.New(log.New(os.Stderr, "", 0)), } } @@ -54,20 +60,28 @@ type getKubeconfigOIDCParams struct { skipBrowser bool sessionCachePath string debugSessionCache bool - caBundlePaths []string + caBundle caBundleFlag requestAudience string } type getKubeconfigConciergeParams struct { disabled bool + credentialIssuer string authenticatorName string authenticatorType string apiGroupSuffix string + caBundle caBundleFlag + endpoint string + mode conciergeModeFlag + skipWait bool } type getKubeconfigParams struct { kubeconfigPath string kubeconfigContextOverride string + skipValidate bool + timeout time.Duration + outputPath string staticToken string staticTokenEnvName string oidc getKubeconfigOIDCParams @@ -90,11 +104,17 @@ func kubeconfigCommand(deps kubeconfigDeps) *cobra.Command { f.StringVar(&flags.staticToken, "static-token", "", "Instead of doing an OIDC-based login, specify a static token") f.StringVar(&flags.staticTokenEnvName, "static-token-env", "", "Instead of doing an OIDC-based login, read a static token from the environment") - f.BoolVar(&flags.concierge.disabled, "no-concierge", false, "Generate a configuration which does not use the concierge, but sends the credential to the cluster directly") - f.StringVar(&namespace, "concierge-namespace", "pinniped-concierge", "Namespace in which the concierge was installed") + f.BoolVar(&flags.concierge.disabled, "no-concierge", false, "Generate a configuration which does not use the Concierge, but sends the credential to the cluster directly") + f.StringVar(&namespace, "concierge-namespace", "pinniped-concierge", "Namespace in which the Concierge was installed") + f.StringVar(&flags.concierge.credentialIssuer, "concierge-credential-issuer", "", "Concierge CredentialIssuer object to use for autodiscovery (default: autodiscover)") f.StringVar(&flags.concierge.authenticatorType, "concierge-authenticator-type", "", "Concierge authenticator type (e.g., 'webhook', 'jwt') (default: autodiscover)") f.StringVar(&flags.concierge.authenticatorName, "concierge-authenticator-name", "", "Concierge authenticator name (default: autodiscover)") f.StringVar(&flags.concierge.apiGroupSuffix, "concierge-api-group-suffix", groupsuffix.PinnipedDefaultSuffix, "Concierge API group suffix") + f.BoolVar(&flags.concierge.skipWait, "concierge-skip-wait", false, "Skip waiting for any pending Concierge strategies to become ready (default: false)") + + f.Var(&flags.concierge.caBundle, "concierge-ca-bundle", "Path to TLS certificate authority bundle (PEM format, optional, can be repeated) to use when connecting to the Concierge") + f.StringVar(&flags.concierge.endpoint, "concierge-endpoint", "", "API base for the Concierge endpoint") + f.Var(&flags.concierge.mode, "concierge-mode", "Concierge mode of operation") f.StringVar(&flags.oidc.issuer, "oidc-issuer", "", "OpenID Connect issuer URL (default: autodiscover)") f.StringVar(&flags.oidc.clientID, "oidc-client-id", "pinniped-cli", "OpenID Connect client ID (default: autodiscover)") @@ -102,26 +122,42 @@ func kubeconfigCommand(deps kubeconfigDeps) *cobra.Command { f.StringSliceVar(&flags.oidc.scopes, "oidc-scopes", []string{oidc.ScopeOfflineAccess, oidc.ScopeOpenID, "pinniped:request-audience"}, "OpenID Connect scopes to request during login") f.BoolVar(&flags.oidc.skipBrowser, "oidc-skip-browser", false, "During OpenID Connect login, skip opening the browser (just print the URL)") f.StringVar(&flags.oidc.sessionCachePath, "oidc-session-cache", "", "Path to OpenID Connect session cache file") - f.StringSliceVar(&flags.oidc.caBundlePaths, "oidc-ca-bundle", nil, "Path to TLS certificate authority bundle (PEM format, optional, can be repeated)") + f.Var(&flags.oidc.caBundle, "oidc-ca-bundle", "Path to TLS certificate authority bundle (PEM format, optional, can be repeated)") f.BoolVar(&flags.oidc.debugSessionCache, "oidc-debug-session-cache", false, "Print debug logs related to the OpenID Connect session cache") f.StringVar(&flags.oidc.requestAudience, "oidc-request-audience", "", "Request a token with an alternate audience using RFC8693 token exchange") f.StringVar(&flags.kubeconfigPath, "kubeconfig", os.Getenv("KUBECONFIG"), "Path to kubeconfig file") f.StringVar(&flags.kubeconfigContextOverride, "kubeconfig-context", "", "Kubeconfig context name (default: current active context)") + f.BoolVar(&flags.skipValidate, "skip-validation", false, "Skip final validation of the kubeconfig (default: false)") + f.DurationVar(&flags.timeout, "timeout", 10*time.Minute, "Timeout for autodiscovery and validation") + f.StringVarP(&flags.outputPath, "output", "o", "", "Output file path (default: stdout)") mustMarkHidden(cmd, "oidc-debug-session-cache") mustMarkDeprecated(cmd, "concierge-namespace", "not needed anymore") mustMarkHidden(cmd, "concierge-namespace") - cmd.RunE = func(cmd *cobra.Command, args []string) error { return runGetKubeconfig(cmd.OutOrStdout(), deps, flags) } + cmd.RunE = func(cmd *cobra.Command, args []string) error { + if flags.outputPath != "" { + out, err := os.Create(flags.outputPath) + if err != nil { + return fmt.Errorf("could not open output file: %w", err) + } + defer func() { _ = out.Close() }() + cmd.SetOut(out) + } + return runGetKubeconfig(cmd.Context(), cmd.OutOrStdout(), deps, flags) + } return cmd } //nolint:funlen -func runGetKubeconfig(out io.Writer, deps kubeconfigDeps, flags getKubeconfigParams) error { +func runGetKubeconfig(ctx context.Context, out io.Writer, deps kubeconfigDeps, flags getKubeconfigParams) error { + ctx, cancel := context.WithTimeout(ctx, flags.timeout) + defer cancel() + // Validate api group suffix and immediately return an error if it is invalid. if err := groupsuffix.Validate(flags.concierge.apiGroupSuffix); err != nil { - return fmt.Errorf("invalid api group suffix: %w", err) + return fmt.Errorf("invalid API group suffix: %w", err) } execConfig := clientcmdapi.ExecConfig{ @@ -137,11 +173,6 @@ func runGetKubeconfig(out io.Writer, deps kubeconfigDeps, flags getKubeconfigPar } execConfig.ProvideClusterInfo = true - oidcCABundle, err := loadCABundlePaths(flags.oidc.caBundlePaths) - if err != nil { - return fmt.Errorf("could not read --oidc-ca-bundle: %w", err) - } - clientConfig := newClientConfig(flags.kubeconfigPath, flags.kubeconfigContextOverride) currentKubeConfig, err := clientConfig.RawConfig() if err != nil { @@ -157,17 +188,39 @@ func runGetKubeconfig(out io.Writer, deps kubeconfigDeps, flags getKubeconfigPar } if !flags.concierge.disabled { + credentialIssuer, err := waitForCredentialIssuer(ctx, clientset, flags, deps) + if err != nil { + return err + } + authenticator, err := lookupAuthenticator( clientset, flags.concierge.authenticatorType, flags.concierge.authenticatorName, + deps.log, ) if err != nil { return err } - if err := configureConcierge(authenticator, &flags, cluster, &oidcCABundle, &execConfig); err != nil { + if err := discoverConciergeParams(credentialIssuer, &flags, cluster, deps.log); err != nil { return err } + if err := discoverAuthenticatorParams(authenticator, &flags, deps.log); err != nil { + return err + } + // Append the flags to configure the Concierge credential exchange at runtime. + execConfig.Args = append(execConfig.Args, + "--enable-concierge", + "--concierge-api-group-suffix="+flags.concierge.apiGroupSuffix, + "--concierge-authenticator-name="+flags.concierge.authenticatorName, + "--concierge-authenticator-type="+flags.concierge.authenticatorType, + "--concierge-endpoint="+flags.concierge.endpoint, + "--concierge-ca-bundle-data="+base64.StdEncoding.EncodeToString(flags.concierge.caBundle), + ) + + // Point kubectl at the concierge endpoint. + cluster.Server = flags.concierge.endpoint + cluster.CertificateAuthorityData = flags.concierge.caBundle } // If one of the --static-* flags was passed, output a config that runs `pinniped login static`. @@ -182,13 +235,18 @@ func runGetKubeconfig(out io.Writer, deps kubeconfigDeps, flags getKubeconfigPar if flags.staticTokenEnvName != "" { execConfig.Args = append(execConfig.Args, "--token-env="+flags.staticTokenEnvName) } - return writeConfigAsYAML(out, newExecKubeconfig(cluster, &execConfig)) + + kubeconfig := newExecKubeconfig(cluster, &execConfig) + if err := validateKubeconfig(ctx, flags, kubeconfig, deps.log); err != nil { + return err + } + return writeConfigAsYAML(out, kubeconfig) } // Otherwise continue to parse the OIDC-related flags and output a config that runs `pinniped login oidc`. execConfig.Args = append([]string{"login", "oidc"}, execConfig.Args...) if flags.oidc.issuer == "" { - return fmt.Errorf("could not autodiscover --oidc-issuer, and none was provided") + return fmt.Errorf("could not autodiscover --oidc-issuer and none was provided") } execConfig.Args = append(execConfig.Args, "--issuer="+flags.oidc.issuer, @@ -201,8 +259,8 @@ func runGetKubeconfig(out io.Writer, deps kubeconfigDeps, flags getKubeconfigPar if flags.oidc.listenPort != 0 { execConfig.Args = append(execConfig.Args, "--listen-port="+strconv.Itoa(int(flags.oidc.listenPort))) } - if oidcCABundle != "" { - execConfig.Args = append(execConfig.Args, "--ca-bundle-data="+base64.StdEncoding.EncodeToString([]byte(oidcCABundle))) + if len(flags.oidc.caBundle) != 0 { + execConfig.Args = append(execConfig.Args, "--ca-bundle-data="+base64.StdEncoding.EncodeToString(flags.oidc.caBundle)) } if flags.oidc.sessionCachePath != "" { execConfig.Args = append(execConfig.Args, "--session-cache="+flags.oidc.sessionCachePath) @@ -213,15 +271,115 @@ func runGetKubeconfig(out io.Writer, deps kubeconfigDeps, flags getKubeconfigPar if flags.oidc.requestAudience != "" { execConfig.Args = append(execConfig.Args, "--request-audience="+flags.oidc.requestAudience) } - return writeConfigAsYAML(out, newExecKubeconfig(cluster, &execConfig)) + kubeconfig := newExecKubeconfig(cluster, &execConfig) + if err := validateKubeconfig(ctx, flags, kubeconfig, deps.log); err != nil { + return err + } + return writeConfigAsYAML(out, kubeconfig) } -func configureConcierge(authenticator metav1.Object, flags *getKubeconfigParams, v1Cluster *clientcmdapi.Cluster, oidcCABundle *string, execConfig *clientcmdapi.ExecConfig) error { +func waitForCredentialIssuer(ctx context.Context, clientset conciergeclientset.Interface, flags getKubeconfigParams, deps kubeconfigDeps) (*configv1alpha1.CredentialIssuer, error) { + credentialIssuer, err := lookupCredentialIssuer(clientset, flags.concierge.credentialIssuer, deps.log) + if err != nil { + return nil, err + } + + if !flags.concierge.skipWait { + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + deadline, _ := ctx.Deadline() + attempts := 1 + + for { + if !hasPendingStrategy(credentialIssuer) { + break + } + logStrategies(credentialIssuer, deps.log) + deps.log.Info("waiting for CredentialIssuer pending strategies to finish", + "attempts", attempts, + "remaining", time.Until(deadline).Round(time.Second).String(), + ) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-ticker.C: + credentialIssuer, err = lookupCredentialIssuer(clientset, flags.concierge.credentialIssuer, deps.log) + if err != nil { + return nil, err + } + } + } + } + return credentialIssuer, nil +} + +func discoverConciergeParams(credentialIssuer *configv1alpha1.CredentialIssuer, flags *getKubeconfigParams, v1Cluster *clientcmdapi.Cluster, log logr.Logger) error { + // Autodiscover the --concierge-mode. + frontend, err := getConciergeFrontend(credentialIssuer, flags.concierge.mode) + if err != nil { + logStrategies(credentialIssuer, log) + return err + } + + // Auto-set --concierge-mode if it wasn't explicitly set. + if flags.concierge.mode == modeUnknown { + switch frontend.Type { + case configv1alpha1.TokenCredentialRequestAPIFrontendType: + log.Info("discovered Concierge operating in TokenCredentialRequest API mode") + flags.concierge.mode = modeTokenCredentialRequestAPI + case configv1alpha1.ImpersonationProxyFrontendType: + log.Info("discovered Concierge operating in impersonation proxy mode") + flags.concierge.mode = modeImpersonationProxy + } + } + + // Auto-set --concierge-endpoint if it wasn't explicitly set. + if flags.concierge.endpoint == "" { + switch frontend.Type { + case configv1alpha1.TokenCredentialRequestAPIFrontendType: + flags.concierge.endpoint = v1Cluster.Server + case configv1alpha1.ImpersonationProxyFrontendType: + flags.concierge.endpoint = frontend.ImpersonationProxyInfo.Endpoint + } + log.Info("discovered Concierge endpoint", "endpoint", flags.concierge.endpoint) + } + + // Auto-set --concierge-ca-bundle if it wasn't explicitly set.. + if len(flags.concierge.caBundle) == 0 { + switch frontend.Type { + case configv1alpha1.TokenCredentialRequestAPIFrontendType: + flags.concierge.caBundle = v1Cluster.CertificateAuthorityData + case configv1alpha1.ImpersonationProxyFrontendType: + data, err := base64.StdEncoding.DecodeString(frontend.ImpersonationProxyInfo.CertificateAuthorityData) + if err != nil { + return fmt.Errorf("autodiscovered Concierge CA bundle is invalid: %w", err) + } + flags.concierge.caBundle = data + } + log.Info("discovered Concierge certificate authority bundle", "roots", countCACerts(flags.concierge.caBundle)) + } + return nil +} + +func logStrategies(credentialIssuer *configv1alpha1.CredentialIssuer, log logr.Logger) { + for _, strategy := range credentialIssuer.Status.Strategies { + log.Info("found CredentialIssuer strategy", + "type", strategy.Type, + "status", strategy.Status, + "reason", strategy.Reason, + "message", strategy.Message, + ) + } +} + +func discoverAuthenticatorParams(authenticator metav1.Object, flags *getKubeconfigParams, log logr.Logger) error { switch auth := authenticator.(type) { case *conciergev1alpha1.WebhookAuthenticator: // If the --concierge-authenticator-type/--concierge-authenticator-name flags were not set explicitly, set // them to point at the discovered WebhookAuthenticator. if flags.concierge.authenticatorType == "" && flags.concierge.authenticatorName == "" { + log.Info("discovered WebhookAuthenticator", "name", auth.Name) flags.concierge.authenticatorType = "webhook" flags.concierge.authenticatorName = auth.Name } @@ -229,56 +387,78 @@ func configureConcierge(authenticator metav1.Object, flags *getKubeconfigParams, // If the --concierge-authenticator-type/--concierge-authenticator-name flags were not set explicitly, set // them to point at the discovered JWTAuthenticator. if flags.concierge.authenticatorType == "" && flags.concierge.authenticatorName == "" { + log.Info("discovered JWTAuthenticator", "name", auth.Name) flags.concierge.authenticatorType = "jwt" flags.concierge.authenticatorName = auth.Name } // If the --oidc-issuer flag was not set explicitly, default it to the spec.issuer field of the JWTAuthenticator. if flags.oidc.issuer == "" { + log.Info("discovered OIDC issuer", "issuer", auth.Spec.Issuer) flags.oidc.issuer = auth.Spec.Issuer } // If the --oidc-request-audience flag was not set explicitly, default it to the spec.audience field of the JWTAuthenticator. if flags.oidc.requestAudience == "" { + log.Info("discovered OIDC audience", "audience", auth.Spec.Audience) flags.oidc.requestAudience = auth.Spec.Audience } // If the --oidc-ca-bundle flags was not set explicitly, default it to the // spec.tls.certificateAuthorityData field of the JWTAuthenticator. - if *oidcCABundle == "" && auth.Spec.TLS != nil && auth.Spec.TLS.CertificateAuthorityData != "" { + if len(flags.oidc.caBundle) == 0 && auth.Spec.TLS != nil && auth.Spec.TLS.CertificateAuthorityData != "" { decoded, err := base64.StdEncoding.DecodeString(auth.Spec.TLS.CertificateAuthorityData) if err != nil { return fmt.Errorf("tried to autodiscover --oidc-ca-bundle, but JWTAuthenticator %s has invalid spec.tls.certificateAuthorityData: %w", auth.Name, err) } - *oidcCABundle = string(decoded) + log.Info("discovered OIDC CA bundle", "roots", countCACerts(decoded)) + flags.oidc.caBundle = decoded } } - - // Append the flags to configure the Concierge credential exchange at runtime. - execConfig.Args = append(execConfig.Args, - "--enable-concierge", - "--concierge-api-group-suffix="+flags.concierge.apiGroupSuffix, - "--concierge-authenticator-name="+flags.concierge.authenticatorName, - "--concierge-authenticator-type="+flags.concierge.authenticatorType, - "--concierge-endpoint="+v1Cluster.Server, - "--concierge-ca-bundle-data="+base64.StdEncoding.EncodeToString(v1Cluster.CertificateAuthorityData), - ) return nil } -func loadCABundlePaths(paths []string) (string, error) { - if len(paths) == 0 { - return "", nil - } - blobs := make([][]byte, 0, len(paths)) - for _, p := range paths { - pem, err := ioutil.ReadFile(p) - if err != nil { - return "", err +func getConciergeFrontend(credentialIssuer *configv1alpha1.CredentialIssuer, mode conciergeModeFlag) (*configv1alpha1.CredentialIssuerFrontend, error) { + for _, strategy := range credentialIssuer.Status.Strategies { + // Skip unhealthy strategies. + if strategy.Status != configv1alpha1.SuccessStrategyStatus { + continue } - blobs = append(blobs, pem) + + // Backfill the .status.strategies[].frontend field from .status.kubeConfigInfo for backwards compatibility. + if strategy.Type == configv1alpha1.KubeClusterSigningCertificateStrategyType && strategy.Frontend == nil && credentialIssuer.Status.KubeConfigInfo != nil { + strategy = *strategy.DeepCopy() + strategy.Frontend = &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.TokenCredentialRequestAPIFrontendType, + TokenCredentialRequestAPIInfo: &configv1alpha1.TokenCredentialRequestAPIInfo{ + Server: credentialIssuer.Status.KubeConfigInfo.Server, + CertificateAuthorityData: credentialIssuer.Status.KubeConfigInfo.CertificateAuthorityData, + }, + } + } + + // If the strategy frontend is still nil, skip. + if strategy.Frontend == nil { + continue + } + + // Skip any unknown frontend types. + switch strategy.Frontend.Type { + case configv1alpha1.TokenCredentialRequestAPIFrontendType, configv1alpha1.ImpersonationProxyFrontendType: + default: + continue + } + // Skip strategies that don't match --concierge-mode. + if !mode.MatchesFrontend(strategy.Frontend) { + continue + } + return strategy.Frontend, nil } - return string(bytes.Join(blobs, []byte("\n"))), nil + + if mode == modeUnknown { + return nil, fmt.Errorf("could not autodiscover --concierge-mode") + } + return nil, fmt.Errorf("could not find successful Concierge strategy matching --concierge-mode=%s", mode.String()) } func newExecKubeconfig(cluster *clientcmdapi.Cluster, execConfig *clientcmdapi.ExecConfig) clientcmdapi.Config { @@ -293,7 +473,33 @@ func newExecKubeconfig(cluster *clientcmdapi.Cluster, execConfig *clientcmdapi.E } } -func lookupAuthenticator(clientset conciergeclientset.Interface, authType, authName string) (metav1.Object, error) { +func lookupCredentialIssuer(clientset conciergeclientset.Interface, name string, log logr.Logger) (*configv1alpha1.CredentialIssuer, error) { + ctx, cancelFunc := context.WithTimeout(context.Background(), time.Second*20) + defer cancelFunc() + + // If the name is specified, get that object. + if name != "" { + return clientset.ConfigV1alpha1().CredentialIssuers().Get(ctx, name, metav1.GetOptions{}) + } + + // Otherwise list all the available CredentialIssuers and hope there's just a single one + results, err := clientset.ConfigV1alpha1().CredentialIssuers().List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to list CredentialIssuer objects for autodiscovery: %w", err) + } + if len(results.Items) == 0 { + return nil, fmt.Errorf("no CredentialIssuers were found") + } + if len(results.Items) > 1 { + return nil, fmt.Errorf("multiple CredentialIssuers were found, so the --concierge-credential-issuer flag must be specified") + } + + result := &results.Items[0] + log.Info("discovered CredentialIssuer", "name", result.Name) + return result, nil +} + +func lookupAuthenticator(clientset conciergeclientset.Interface, authType, authName string, log logr.Logger) (metav1.Object, error) { ctx, cancelFunc := context.WithTimeout(context.Background(), time.Second*20) defer cancelFunc() @@ -331,6 +537,12 @@ func lookupAuthenticator(clientset conciergeclientset.Interface, authType, authN return nil, fmt.Errorf("no authenticators were found") } if len(results) > 1 { + for _, jwtAuth := range jwtAuths.Items { + log.Info("found JWTAuthenticator", "name", jwtAuth.Name) + } + for _, webhook := range webhooks.Items { + log.Info("found WebhookAuthenticator", "name", webhook.Name) + } return nil, fmt.Errorf("multiple authenticators were found, so the --concierge-authenticator-type/--concierge-authenticator-name flags must be specified") } return results[0], nil @@ -353,9 +565,101 @@ func copyCurrentClusterFromExistingKubeConfig(currentKubeConfig clientcmdapi.Con if currentContextNameOverride != "" { contextName = currentContextNameOverride } - context := currentKubeConfig.Contexts[contextName] - if context == nil { + ctx := currentKubeConfig.Contexts[contextName] + if ctx == nil { return nil, fmt.Errorf("no such context %q", contextName) } - return currentKubeConfig.Clusters[context.Cluster], nil + return currentKubeConfig.Clusters[ctx.Cluster], nil +} + +func validateKubeconfig(ctx context.Context, flags getKubeconfigParams, kubeconfig clientcmdapi.Config, log logr.Logger) error { + if flags.skipValidate { + return nil + } + + kubeContext := kubeconfig.Contexts[kubeconfig.CurrentContext] + if kubeContext == nil { + return fmt.Errorf("invalid kubeconfig (no context)") + } + cluster := kubeconfig.Clusters[kubeContext.Cluster] + if cluster == nil { + return fmt.Errorf("invalid kubeconfig (no cluster)") + } + + kubeconfigCA := x509.NewCertPool() + if !kubeconfigCA.AppendCertsFromPEM(cluster.CertificateAuthorityData) { + return fmt.Errorf("invalid kubeconfig (no certificateAuthorityData)") + } + + httpClient := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: kubeconfigCA, + }, + Proxy: http.ProxyFromEnvironment, + TLSHandshakeTimeout: 10 * time.Second, + }, + Timeout: 10 * time.Second, + } + + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + pingCluster := func() error { + reqCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, cluster.Server, nil) + if err != nil { + return fmt.Errorf("could not form request to validate cluster: %w", err) + } + resp, err := httpClient.Do(req) + if err != nil { + return err + } + _ = resp.Body.Close() + if resp.StatusCode >= 500 { + return fmt.Errorf("unexpected status code %d", resp.StatusCode) + } + return nil + } + + err := pingCluster() + if err == nil { + log.Info("validated connection to the cluster") + return nil + } + + log.Info("could not immediately connect to the cluster but it may be initializing, will retry until timeout") + deadline, _ := ctx.Deadline() + attempts := 0 + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + attempts++ + err := pingCluster() + if err == nil { + log.Info("validated connection to the cluster", "attempts", attempts) + return nil + } + log.Error(err, "could not connect to cluster, retrying...", "attempts", attempts, "remaining", time.Until(deadline).Round(time.Second).String()) + } + } +} + +func countCACerts(pemData []byte) int { + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(pemData) + return len(pool.Subjects()) +} + +func hasPendingStrategy(credentialIssuer *configv1alpha1.CredentialIssuer) bool { + for _, strategy := range credentialIssuer.Status.Strategies { + if strategy.Reason == configv1alpha1.PendingStrategyReason { + return true + } + } + return false } diff --git a/cmd/pinniped/cmd/kubeconfig_test.go b/cmd/pinniped/cmd/kubeconfig_test.go index 540f77115..879e2d713 100644 --- a/cmd/pinniped/cmd/kubeconfig_test.go +++ b/cmd/pinniped/cmd/kubeconfig_test.go @@ -5,7 +5,6 @@ package cmd import ( "bytes" - "crypto/x509/pkix" "encoding/base64" "fmt" "io/ioutil" @@ -20,19 +19,26 @@ import ( "k8s.io/client-go/tools/clientcmd" conciergev1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/authentication/v1alpha1" + configv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" conciergeclientset "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned" fakeconciergeclientset "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned/fake" "go.pinniped.dev/internal/certauthority" "go.pinniped.dev/internal/here" "go.pinniped.dev/internal/testutil" + "go.pinniped.dev/internal/testutil/testlogger" ) func TestGetKubeconfig(t *testing.T) { - testCA, err := certauthority.New(pkix.Name{CommonName: "Test CA"}, 1*time.Hour) + testOIDCCA, err := certauthority.New("Test CA", 1*time.Hour) require.NoError(t, err) tmpdir := testutil.TempDir(t) - testCABundlePath := filepath.Join(tmpdir, "testca.pem") - require.NoError(t, ioutil.WriteFile(testCABundlePath, testCA.Bundle(), 0600)) + testOIDCCABundlePath := filepath.Join(tmpdir, "testca.pem") + require.NoError(t, ioutil.WriteFile(testOIDCCABundlePath, testOIDCCA.Bundle(), 0600)) + + testConciergeCA, err := certauthority.New("Test Concierge CA", 1*time.Hour) + require.NoError(t, err) + testConciergeCABundlePath := filepath.Join(tmpdir, "testconciergeca.pem") + require.NoError(t, ioutil.WriteFile(testConciergeCABundlePath, testConciergeCA.Bundle(), 0600)) tests := []struct { name string @@ -42,6 +48,7 @@ func TestGetKubeconfig(t *testing.T) { getClientsetErr error conciergeObjects []runtime.Object conciergeReactions []kubetesting.Reactor + wantLogs []string wantError bool wantStdout string wantStderr string @@ -61,11 +68,16 @@ func TestGetKubeconfig(t *testing.T) { --concierge-api-group-suffix string Concierge API group suffix (default "pinniped.dev") --concierge-authenticator-name string Concierge authenticator name (default: autodiscover) --concierge-authenticator-type string Concierge authenticator type (e.g., 'webhook', 'jwt') (default: autodiscover) + --concierge-ca-bundle path Path to TLS certificate authority bundle (PEM format, optional, can be repeated) to use when connecting to the Concierge + --concierge-credential-issuer string Concierge CredentialIssuer object to use for autodiscovery (default: autodiscover) + --concierge-endpoint string API base for the Concierge endpoint + --concierge-mode mode Concierge mode of operation (default TokenCredentialRequestAPI) + --concierge-skip-wait Skip waiting for any pending Concierge strategies to become ready (default: false) -h, --help help for kubeconfig --kubeconfig string Path to kubeconfig file --kubeconfig-context string Kubeconfig context name (default: current active context) - --no-concierge Generate a configuration which does not use the concierge, but sends the credential to the cluster directly - --oidc-ca-bundle strings Path to TLS certificate authority bundle (PEM format, optional, can be repeated) + --no-concierge Generate a configuration which does not use the Concierge, but sends the credential to the cluster directly + --oidc-ca-bundle path Path to TLS certificate authority bundle (PEM format, optional, can be repeated) --oidc-client-id string OpenID Connect client ID (default: autodiscover) (default "pinniped-cli") --oidc-issuer string OpenID Connect issuer URL (default: autodiscover) --oidc-listen-port uint16 TCP port for localhost listener (authorization code flow only) @@ -73,8 +85,11 @@ func TestGetKubeconfig(t *testing.T) { --oidc-scopes strings OpenID Connect scopes to request during login (default [offline_access,openid,pinniped:request-audience]) --oidc-session-cache string Path to OpenID Connect session cache file --oidc-skip-browser During OpenID Connect login, skip opening the browser (just print the URL) + -o, --output string Output file path (default: stdout) + --skip-validation Skip final validation of the kubeconfig (default: false) --static-token string Instead of doing an OIDC-based login, specify a static token --static-token-env string Instead of doing an OIDC-based login, read a static token from the environment + --timeout duration Timeout for autodiscovery and validation (default 10m0s) `), }, { @@ -87,13 +102,24 @@ func TestGetKubeconfig(t *testing.T) { `), }, { - name: "invalid CA bundle paths", + name: "invalid OIDC CA bundle path", args: []string{ "--oidc-ca-bundle", "./does/not/exist", }, wantError: true, wantStderr: here.Doc(` - Error: could not read --oidc-ca-bundle: open ./does/not/exist: no such file or directory + Error: invalid argument "./does/not/exist" for "--oidc-ca-bundle" flag: could not read CA bundle path: open ./does/not/exist: no such file or directory + `), + }, + { + name: "invalid Concierge CA bundle", + args: []string{ + "--kubeconfig", "./testdata/kubeconfig.yaml", + "--concierge-ca-bundle", "./does/not/exist", + }, + wantError: true, + wantStderr: here.Doc(` + Error: invalid argument "./does/not/exist" for "--concierge-ca-bundle" flag: could not read CA bundle path: open ./does/not/exist: no such file or directory `), }, { @@ -128,6 +154,31 @@ func TestGetKubeconfig(t *testing.T) { Error: could not configure Kubernetes client: some kube error `), }, + { + name: "no credentialissuers", + args: []string{ + "--kubeconfig", "./testdata/kubeconfig.yaml", + }, + wantError: true, + wantStderr: here.Doc(` + Error: no CredentialIssuers were found + `), + }, + + { + name: "credentialissuer not found", + args: []string{ + "--kubeconfig", "./testdata/kubeconfig.yaml", + "--concierge-credential-issuer", "does-not-exist", + }, + conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}}, + }, + wantError: true, + wantStderr: here.Doc(` + Error: credentialissuers.config.concierge.pinniped.dev "does-not-exist" not found + `), + }, { name: "webhook authenticator not found", args: []string{ @@ -135,6 +186,12 @@ func TestGetKubeconfig(t *testing.T) { "--concierge-authenticator-type", "webhook", "--concierge-authenticator-name", "test-authenticator", }, + conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}}, + }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + }, wantError: true, wantStderr: here.Doc(` Error: webhookauthenticators.authentication.concierge.pinniped.dev "test-authenticator" not found @@ -147,6 +204,12 @@ func TestGetKubeconfig(t *testing.T) { "--concierge-authenticator-type", "jwt", "--concierge-authenticator-name", "test-authenticator", }, + conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}}, + }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + }, wantError: true, wantStderr: here.Doc(` Error: jwtauthenticators.authentication.concierge.pinniped.dev "test-authenticator" not found @@ -159,6 +222,12 @@ func TestGetKubeconfig(t *testing.T) { "--concierge-authenticator-type", "invalid", "--concierge-authenticator-name", "test-authenticator", }, + conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}}, + }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + }, wantError: true, wantStderr: here.Doc(` Error: invalid authenticator type "invalid", supported values are "webhook" and "jwt" @@ -169,6 +238,12 @@ func TestGetKubeconfig(t *testing.T) { args: []string{ "--kubeconfig", "./testdata/kubeconfig.yaml", }, + conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}}, + }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + }, conciergeReactions: []kubetesting.Reactor{ &kubetesting.SimpleReactor{ Verb: "*", @@ -188,6 +263,9 @@ func TestGetKubeconfig(t *testing.T) { args: []string{ "--kubeconfig", "./testdata/kubeconfig.yaml", }, + conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}}, + }, conciergeReactions: []kubetesting.Reactor{ &kubetesting.SimpleReactor{ Verb: "*", @@ -197,6 +275,9 @@ func TestGetKubeconfig(t *testing.T) { }, }, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + }, wantError: true, wantStderr: here.Doc(` Error: failed to list WebhookAuthenticator objects for autodiscovery: some list error @@ -207,6 +288,12 @@ func TestGetKubeconfig(t *testing.T) { args: []string{ "--kubeconfig", "./testdata/kubeconfig.yaml", }, + conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}}, + }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + }, wantError: true, wantStderr: here.Doc(` Error: no authenticators were found @@ -218,27 +305,145 @@ func TestGetKubeconfig(t *testing.T) { "--kubeconfig", "./testdata/kubeconfig.yaml", }, conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}}, &conciergev1alpha1.JWTAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator-1"}}, &conciergev1alpha1.JWTAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator-2"}}, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator-3"}}, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator-4"}}, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="found JWTAuthenticator" "name"="test-authenticator-1"`, + `"level"=0 "msg"="found JWTAuthenticator" "name"="test-authenticator-2"`, + `"level"=0 "msg"="found WebhookAuthenticator" "name"="test-authenticator-3"`, + `"level"=0 "msg"="found WebhookAuthenticator" "name"="test-authenticator-4"`, + }, wantError: true, wantStderr: here.Doc(` Error: multiple authenticators were found, so the --concierge-authenticator-type/--concierge-authenticator-name flags must be specified `), }, + { + name: "autodetect webhook authenticator, bad credential issuer with only failing strategy", + args: []string{ + "--kubeconfig", "./testdata/kubeconfig.yaml", + }, + conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, + Status: configv1alpha1.CredentialIssuerStatus{ + Strategies: []configv1alpha1.CredentialIssuerStrategy{{ + Type: "SomeType", + Status: configv1alpha1.ErrorStrategyStatus, + Reason: "SomeReason", + Message: "Some message", + }}, + }, + }, + &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, + }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="found CredentialIssuer strategy" "message"="Some message" "reason"="SomeReason" "status"="Error" "type"="SomeType"`, + }, + wantError: true, + wantStderr: here.Doc(` + Error: could not autodiscover --concierge-mode + `), + }, + { + name: "autodetect webhook authenticator, bad credential issuer with invalid impersonation CA", + args: []string{ + "--kubeconfig", "./testdata/kubeconfig.yaml", + }, + conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, + Status: configv1alpha1.CredentialIssuerStatus{ + Strategies: []configv1alpha1.CredentialIssuerStrategy{ + { + Type: "SomeBrokenType", + Status: configv1alpha1.ErrorStrategyStatus, + Reason: "SomeFailureReason", + Message: "Some error message", + LastUpdateTime: metav1.Now(), + }, + { + Type: "SomeUnknownType", + Status: configv1alpha1.SuccessStrategyStatus, + Reason: "SomeReason", + Message: "Some error message", + LastUpdateTime: metav1.Now(), + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: "SomeUnknownFrontendType", + }, + }, + { + Type: "SomeType", + Status: configv1alpha1.SuccessStrategyStatus, + Reason: "SomeReason", + Message: "Some message", + LastUpdateTime: metav1.Now(), + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.ImpersonationProxyFrontendType, + ImpersonationProxyInfo: &configv1alpha1.ImpersonationProxyInfo{ + Endpoint: "https://impersonation-endpoint", + CertificateAuthorityData: "invalid-base-64", + }, + }, + }, + }, + }, + }, + &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, + }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="discovered Concierge operating in impersonation proxy mode"`, + `"level"=0 "msg"="discovered Concierge endpoint" "endpoint"="https://impersonation-endpoint"`, + }, + wantError: true, + wantStderr: here.Doc(` + Error: autodiscovered Concierge CA bundle is invalid: illegal base64 data at input byte 7 + `), + }, { name: "autodetect webhook authenticator, missing --oidc-issuer", args: []string{ "--kubeconfig", "./testdata/kubeconfig.yaml", }, conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, + Status: configv1alpha1.CredentialIssuerStatus{ + Strategies: []configv1alpha1.CredentialIssuerStrategy{{ + Type: "SomeType", + Status: configv1alpha1.SuccessStrategyStatus, + Reason: "SomeReason", + Message: "Some message", + LastUpdateTime: metav1.Now(), + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.TokenCredentialRequestAPIFrontendType, + TokenCredentialRequestAPIInfo: &configv1alpha1.TokenCredentialRequestAPIInfo{ + Server: "https://concierge-endpoint", + CertificateAuthorityData: "ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ==", + }, + }, + }}, + }, + }, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="discovered Concierge operating in TokenCredentialRequest API mode"`, + `"level"=0 "msg"="discovered Concierge endpoint" "endpoint"="https://fake-server-url-value"`, + `"level"=0 "msg"="discovered Concierge certificate authority bundle" "roots"=0`, + `"level"=0 "msg"="discovered WebhookAuthenticator" "name"="test-authenticator"`, + }, wantError: true, wantStderr: here.Doc(` - Error: could not autodiscover --oidc-issuer, and none was provided + Error: could not autodiscover --oidc-issuer and none was provided `), }, { @@ -247,15 +452,44 @@ func TestGetKubeconfig(t *testing.T) { "--kubeconfig", "./testdata/kubeconfig.yaml", }, conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, + Status: configv1alpha1.CredentialIssuerStatus{ + KubeConfigInfo: &configv1alpha1.CredentialIssuerKubeConfigInfo{ + Server: "https://concierge-endpoint", + CertificateAuthorityData: "ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ==", + }, + Strategies: []configv1alpha1.CredentialIssuerStrategy{{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.SuccessStrategyStatus, + Reason: configv1alpha1.FetchedKeyStrategyReason, + Message: "Successfully fetched key", + LastUpdateTime: metav1.Now(), + // Simulate a previous version of CredentialIssuer that's missing this Frontend field. + Frontend: nil, + }}, + }, + }, &conciergev1alpha1.JWTAuthenticator{ ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}, Spec: conciergev1alpha1.JWTAuthenticatorSpec{ + Issuer: "https://test-issuer.example.com", + Audience: "some-test-audience", TLS: &conciergev1alpha1.TLSSpec{ CertificateAuthorityData: "invalid-base64", }, }, }, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="discovered Concierge operating in TokenCredentialRequest API mode"`, + `"level"=0 "msg"="discovered Concierge endpoint" "endpoint"="https://fake-server-url-value"`, + `"level"=0 "msg"="discovered Concierge certificate authority bundle" "roots"=0`, + `"level"=0 "msg"="discovered JWTAuthenticator" "name"="test-authenticator"`, + `"level"=0 "msg"="discovered OIDC issuer" "issuer"="https://test-issuer.example.com"`, + `"level"=0 "msg"="discovered OIDC audience" "audience"="some-test-audience"`, + }, wantError: true, wantStderr: here.Doc(` Error: tried to autodiscover --oidc-ca-bundle, but JWTAuthenticator test-authenticator has invalid spec.tls.certificateAuthorityData: illegal base64 data at input byte 7 @@ -269,21 +503,45 @@ func TestGetKubeconfig(t *testing.T) { "--static-token-env", "TEST_TOKEN", }, conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, + Status: configv1alpha1.CredentialIssuerStatus{ + Strategies: []configv1alpha1.CredentialIssuerStrategy{{ + Type: configv1alpha1.ImpersonationProxyStrategyType, + Status: configv1alpha1.SuccessStrategyStatus, + Reason: configv1alpha1.ListeningStrategyReason, + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.ImpersonationProxyFrontendType, + ImpersonationProxyInfo: &configv1alpha1.ImpersonationProxyInfo{ + Endpoint: "https://impersonation-proxy-endpoint.example.com", + CertificateAuthorityData: base64.StdEncoding.EncodeToString(testConciergeCA.Bundle()), + }, + }, + }}, + }, + }, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="discovered Concierge operating in impersonation proxy mode"`, + `"level"=0 "msg"="discovered Concierge endpoint" "endpoint"="https://impersonation-proxy-endpoint.example.com"`, + `"level"=0 "msg"="discovered Concierge certificate authority bundle" "roots"=1`, + `"level"=0 "msg"="discovered WebhookAuthenticator" "name"="test-authenticator"`, + }, wantError: true, wantStderr: here.Doc(` Error: only one of --static-token and --static-token-env can be specified `), }, { - name: "invalid api group suffix", + name: "invalid API group suffix", args: []string{ "--concierge-api-group-suffix", ".starts.with.dot", }, wantError: true, wantStderr: here.Doc(` - Error: invalid api group suffix: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*') + Error: invalid API group suffix: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*') `), }, { @@ -291,10 +549,35 @@ func TestGetKubeconfig(t *testing.T) { args: []string{ "--kubeconfig", "./testdata/kubeconfig.yaml", "--static-token", "test-token", + "--skip-validation", }, conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, + Status: configv1alpha1.CredentialIssuerStatus{ + Strategies: []configv1alpha1.CredentialIssuerStrategy{{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.SuccessStrategyStatus, + Reason: configv1alpha1.FetchedKeyStrategyReason, + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.TokenCredentialRequestAPIFrontendType, + TokenCredentialRequestAPIInfo: &configv1alpha1.TokenCredentialRequestAPIInfo{ + Server: "https://concierge-endpoint.example.com", + CertificateAuthorityData: base64.StdEncoding.EncodeToString(testConciergeCA.Bundle()), + }, + }, + }}, + }, + }, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="discovered Concierge operating in TokenCredentialRequest API mode"`, + `"level"=0 "msg"="discovered Concierge endpoint" "endpoint"="https://fake-server-url-value"`, + `"level"=0 "msg"="discovered Concierge certificate authority bundle" "roots"=0`, + `"level"=0 "msg"="discovered WebhookAuthenticator" "name"="test-authenticator"`, + }, wantStdout: here.Doc(` apiVersion: v1 clusters: @@ -335,10 +618,35 @@ func TestGetKubeconfig(t *testing.T) { args: []string{ "--kubeconfig", "./testdata/kubeconfig.yaml", "--static-token-env", "TEST_TOKEN", + "--skip-validation", }, conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, + Status: configv1alpha1.CredentialIssuerStatus{ + Strategies: []configv1alpha1.CredentialIssuerStrategy{{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.SuccessStrategyStatus, + Reason: configv1alpha1.FetchedKeyStrategyReason, + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.TokenCredentialRequestAPIFrontendType, + TokenCredentialRequestAPIInfo: &configv1alpha1.TokenCredentialRequestAPIInfo{ + Server: "https://concierge-endpoint.example.com", + CertificateAuthorityData: base64.StdEncoding.EncodeToString(testConciergeCA.Bundle()), + }, + }, + }}, + }, + }, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="discovered Concierge operating in TokenCredentialRequest API mode"`, + `"level"=0 "msg"="discovered Concierge endpoint" "endpoint"="https://fake-server-url-value"`, + `"level"=0 "msg"="discovered Concierge certificate authority bundle" "roots"=0`, + `"level"=0 "msg"="discovered WebhookAuthenticator" "name"="test-authenticator"`, + }, wantStdout: here.Doc(` apiVersion: v1 clusters: @@ -378,19 +686,47 @@ func TestGetKubeconfig(t *testing.T) { name: "autodetect JWT authenticator", args: []string{ "--kubeconfig", "./testdata/kubeconfig.yaml", + "--skip-validation", }, conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, + Status: configv1alpha1.CredentialIssuerStatus{ + Strategies: []configv1alpha1.CredentialIssuerStrategy{{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.SuccessStrategyStatus, + Reason: configv1alpha1.FetchedKeyStrategyReason, + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.TokenCredentialRequestAPIFrontendType, + TokenCredentialRequestAPIInfo: &configv1alpha1.TokenCredentialRequestAPIInfo{ + Server: "https://concierge-endpoint.example.com", + CertificateAuthorityData: base64.StdEncoding.EncodeToString(testConciergeCA.Bundle()), + }, + }, + }}, + }, + }, &conciergev1alpha1.JWTAuthenticator{ ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}, Spec: conciergev1alpha1.JWTAuthenticatorSpec{ Issuer: "https://example.com/issuer", Audience: "test-audience", TLS: &conciergev1alpha1.TLSSpec{ - CertificateAuthorityData: base64.StdEncoding.EncodeToString(testCA.Bundle()), + CertificateAuthorityData: base64.StdEncoding.EncodeToString(testOIDCCA.Bundle()), }, }, }, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="discovered Concierge operating in TokenCredentialRequest API mode"`, + `"level"=0 "msg"="discovered Concierge endpoint" "endpoint"="https://fake-server-url-value"`, + `"level"=0 "msg"="discovered Concierge certificate authority bundle" "roots"=0`, + `"level"=0 "msg"="discovered JWTAuthenticator" "name"="test-authenticator"`, + `"level"=0 "msg"="discovered OIDC issuer" "issuer"="https://example.com/issuer"`, + `"level"=0 "msg"="discovered OIDC audience" "audience"="test-audience"`, + `"level"=0 "msg"="discovered OIDC CA bundle" "roots"=1`, + }, wantStdout: here.Docf(` apiVersion: v1 clusters: @@ -428,34 +764,57 @@ func TestGetKubeconfig(t *testing.T) { command: '.../path/to/pinniped' env: [] provideClusterInfo: true - `, base64.StdEncoding.EncodeToString(testCA.Bundle())), + `, base64.StdEncoding.EncodeToString(testOIDCCA.Bundle())), }, { name: "autodetect nothing, set a bunch of options", args: []string{ "--kubeconfig", "./testdata/kubeconfig.yaml", + "--concierge-credential-issuer", "test-credential-issuer", "--concierge-api-group-suffix", "tuna.io", "--concierge-authenticator-type", "webhook", "--concierge-authenticator-name", "test-authenticator", + "--concierge-mode", "TokenCredentialRequestAPI", + "--concierge-endpoint", "https://explicit-concierge-endpoint.example.com", + "--concierge-ca-bundle", testConciergeCABundlePath, "--oidc-issuer", "https://example.com/issuer", "--oidc-skip-browser", "--oidc-listen-port", "1234", - "--oidc-ca-bundle", testCABundlePath, + "--oidc-ca-bundle", testOIDCCABundlePath, "--oidc-session-cache", "/path/to/cache/dir/sessions.yaml", "--oidc-debug-session-cache", "--oidc-request-audience", "test-audience", + "--skip-validation", }, conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, + Status: configv1alpha1.CredentialIssuerStatus{ + Strategies: []configv1alpha1.CredentialIssuerStrategy{{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.SuccessStrategyStatus, + Reason: configv1alpha1.FetchedKeyStrategyReason, + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.TokenCredentialRequestAPIFrontendType, + TokenCredentialRequestAPIInfo: &configv1alpha1.TokenCredentialRequestAPIInfo{ + Server: "https://concierge-endpoint.example.com", + CertificateAuthorityData: "dGVzdC10Y3ItYXBpLWNh", + }, + }, + }}, + }, + }, &conciergev1alpha1.WebhookAuthenticator{ ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}, }, }, + wantLogs: nil, wantStdout: here.Docf(` apiVersion: v1 clusters: - cluster: - certificate-authority-data: ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ== - server: https://fake-server-url-value + certificate-authority-data: %s + server: https://explicit-concierge-endpoint.example.com name: pinniped contexts: - context: @@ -477,8 +836,8 @@ func TestGetKubeconfig(t *testing.T) { - --concierge-api-group-suffix=tuna.io - --concierge-authenticator-name=test-authenticator - --concierge-authenticator-type=webhook - - --concierge-endpoint=https://fake-server-url-value - - --concierge-ca-bundle-data=ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ== + - --concierge-endpoint=https://explicit-concierge-endpoint.example.com + - --concierge-ca-bundle-data=%s - --issuer=https://example.com/issuer - --client-id=pinniped-cli - --scopes=offline_access,openid,pinniped:request-audience @@ -491,13 +850,229 @@ func TestGetKubeconfig(t *testing.T) { command: '.../path/to/pinniped' env: [] provideClusterInfo: true - `, base64.StdEncoding.EncodeToString(testCA.Bundle())), + `, + base64.StdEncoding.EncodeToString(testConciergeCA.Bundle()), + base64.StdEncoding.EncodeToString(testConciergeCA.Bundle()), + base64.StdEncoding.EncodeToString(testOIDCCA.Bundle()), + ), wantAPIGroupSuffix: "tuna.io", }, + { + name: "configure impersonation proxy with autodiscovered JWT authenticator", + args: []string{ + "--kubeconfig", "./testdata/kubeconfig.yaml", + "--concierge-mode", "ImpersonationProxy", + "--skip-validation", + }, + conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, + Status: configv1alpha1.CredentialIssuerStatus{ + Strategies: []configv1alpha1.CredentialIssuerStrategy{ + // This TokenCredentialRequestAPI strategy would normally be chosen, but + // --concierge-mode=ImpersonationProxy should force it to be skipped. + { + Type: "SomeType", + Status: configv1alpha1.SuccessStrategyStatus, + Reason: "SomeReason", + Message: "Some message", + LastUpdateTime: metav1.Now(), + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.TokenCredentialRequestAPIFrontendType, + TokenCredentialRequestAPIInfo: &configv1alpha1.TokenCredentialRequestAPIInfo{ + Server: "https://token-credential-request-api-endpoint.test", + CertificateAuthorityData: "dGVzdC10Y3ItYXBpLWNh", + }, + }, + }, + // The endpoint and CA from this impersonation proxy strategy should be autodiscovered. + { + Type: "SomeOtherType", + Status: configv1alpha1.SuccessStrategyStatus, + Reason: "SomeOtherReason", + Message: "Some other message", + LastUpdateTime: metav1.Now(), + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.ImpersonationProxyFrontendType, + ImpersonationProxyInfo: &configv1alpha1.ImpersonationProxyInfo{ + Endpoint: "https://impersonation-proxy-endpoint.test", + CertificateAuthorityData: base64.StdEncoding.EncodeToString(testConciergeCA.Bundle()), + }, + }, + }, + }, + }, + }, + &conciergev1alpha1.JWTAuthenticator{ + ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}, + Spec: conciergev1alpha1.JWTAuthenticatorSpec{ + Issuer: "https://example.com/issuer", + Audience: "test-audience", + TLS: &conciergev1alpha1.TLSSpec{ + CertificateAuthorityData: base64.StdEncoding.EncodeToString(testOIDCCA.Bundle()), + }, + }, + }, + }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="discovered Concierge endpoint" "endpoint"="https://impersonation-proxy-endpoint.test"`, + `"level"=0 "msg"="discovered Concierge certificate authority bundle" "roots"=1`, + `"level"=0 "msg"="discovered JWTAuthenticator" "name"="test-authenticator"`, + `"level"=0 "msg"="discovered OIDC issuer" "issuer"="https://example.com/issuer"`, + `"level"=0 "msg"="discovered OIDC audience" "audience"="test-audience"`, + `"level"=0 "msg"="discovered OIDC CA bundle" "roots"=1`, + }, + wantStdout: here.Docf(` + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: %s + server: https://impersonation-proxy-endpoint.test + name: pinniped + contexts: + - context: + cluster: pinniped + user: pinniped + name: pinniped + current-context: pinniped + kind: Config + preferences: {} + users: + - name: pinniped + user: + exec: + apiVersion: client.authentication.k8s.io/v1beta1 + args: + - login + - oidc + - --enable-concierge + - --concierge-api-group-suffix=pinniped.dev + - --concierge-authenticator-name=test-authenticator + - --concierge-authenticator-type=jwt + - --concierge-endpoint=https://impersonation-proxy-endpoint.test + - --concierge-ca-bundle-data=%s + - --issuer=https://example.com/issuer + - --client-id=pinniped-cli + - --scopes=offline_access,openid,pinniped:request-audience + - --ca-bundle-data=%s + - --request-audience=test-audience + command: '.../path/to/pinniped' + env: [] + provideClusterInfo: true + `, + base64.StdEncoding.EncodeToString(testConciergeCA.Bundle()), + base64.StdEncoding.EncodeToString(testConciergeCA.Bundle()), + base64.StdEncoding.EncodeToString(testOIDCCA.Bundle()), + ), + }, + { + name: "autodetect impersonation proxy with autodiscovered JWT authenticator", + args: []string{ + "--kubeconfig", "./testdata/kubeconfig.yaml", + "--skip-validation", + }, + conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, + Status: configv1alpha1.CredentialIssuerStatus{ + Strategies: []configv1alpha1.CredentialIssuerStrategy{ + { + Type: "SomeType", + Status: configv1alpha1.SuccessStrategyStatus, + Reason: "SomeReason", + Message: "Some message", + LastUpdateTime: metav1.Now(), + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.ImpersonationProxyFrontendType, + ImpersonationProxyInfo: &configv1alpha1.ImpersonationProxyInfo{ + Endpoint: "https://impersonation-proxy-endpoint.test", + CertificateAuthorityData: "dGVzdC1jb25jaWVyZ2UtY2E=", + }, + }, + }, + { + Type: "SomeOtherType", + Status: configv1alpha1.SuccessStrategyStatus, + Reason: "SomeOtherReason", + Message: "Some other message", + LastUpdateTime: metav1.Now(), + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.ImpersonationProxyFrontendType, + ImpersonationProxyInfo: &configv1alpha1.ImpersonationProxyInfo{ + Endpoint: "https://some-other-impersonation-endpoint", + CertificateAuthorityData: "dGVzdC1jb25jaWVyZ2UtY2E=", + }, + }, + }, + }, + }, + }, + &conciergev1alpha1.JWTAuthenticator{ + ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}, + Spec: conciergev1alpha1.JWTAuthenticatorSpec{ + Issuer: "https://example.com/issuer", + Audience: "test-audience", + TLS: &conciergev1alpha1.TLSSpec{ + CertificateAuthorityData: base64.StdEncoding.EncodeToString(testOIDCCA.Bundle()), + }, + }, + }, + }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="discovered Concierge operating in impersonation proxy mode"`, + `"level"=0 "msg"="discovered Concierge endpoint" "endpoint"="https://impersonation-proxy-endpoint.test"`, + `"level"=0 "msg"="discovered Concierge certificate authority bundle" "roots"=0`, + `"level"=0 "msg"="discovered JWTAuthenticator" "name"="test-authenticator"`, + `"level"=0 "msg"="discovered OIDC issuer" "issuer"="https://example.com/issuer"`, + `"level"=0 "msg"="discovered OIDC audience" "audience"="test-audience"`, + `"level"=0 "msg"="discovered OIDC CA bundle" "roots"=1`, + }, + wantStdout: here.Docf(` + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: dGVzdC1jb25jaWVyZ2UtY2E= + server: https://impersonation-proxy-endpoint.test + name: pinniped + contexts: + - context: + cluster: pinniped + user: pinniped + name: pinniped + current-context: pinniped + kind: Config + preferences: {} + users: + - name: pinniped + user: + exec: + apiVersion: client.authentication.k8s.io/v1beta1 + args: + - login + - oidc + - --enable-concierge + - --concierge-api-group-suffix=pinniped.dev + - --concierge-authenticator-name=test-authenticator + - --concierge-authenticator-type=jwt + - --concierge-endpoint=https://impersonation-proxy-endpoint.test + - --concierge-ca-bundle-data=dGVzdC1jb25jaWVyZ2UtY2E= + - --issuer=https://example.com/issuer + - --client-id=pinniped-cli + - --scopes=offline_access,openid,pinniped:request-audience + - --ca-bundle-data=%s + - --request-audience=test-audience + command: '.../path/to/pinniped' + env: [] + provideClusterInfo: true + `, base64.StdEncoding.EncodeToString(testOIDCCA.Bundle())), + }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { + testLog := testlogger.New(t) cmd := kubeconfigCommand(kubeconfigDeps{ getPathToSelf: func() (string, error) { if tt.getPathToSelfErr != nil { @@ -516,10 +1091,11 @@ func TestGetKubeconfig(t *testing.T) { } fake := fakeconciergeclientset.NewSimpleClientset(tt.conciergeObjects...) if len(tt.conciergeReactions) > 0 { - fake.ReactionChain = tt.conciergeReactions + fake.ReactionChain = append(tt.conciergeReactions, fake.ReactionChain...) } return fake, nil }, + log: testLog, }) require.NotNil(t, cmd) @@ -533,6 +1109,7 @@ func TestGetKubeconfig(t *testing.T) { } else { require.NoError(t, err) } + testLog.Expect(tt.wantLogs) require.Equal(t, tt.wantStdout, stdout.String(), "unexpected stdout") require.Equal(t, tt.wantStderr, stderr.String(), "unexpected stderr") }) diff --git a/cmd/pinniped/cmd/login_oidc.go b/cmd/pinniped/cmd/login_oidc.go index f9f258b26..a31254756 100644 --- a/cmd/pinniped/cmd/login_oidc.go +++ b/cmd/pinniped/cmd/login_oidc.go @@ -85,15 +85,15 @@ func oidcLoginCommand(deps oidcLoginCommandDeps) *cobra.Command { cmd.Flags().BoolVar(&flags.skipBrowser, "skip-browser", false, "Skip opening the browser (just print the URL)") cmd.Flags().StringVar(&flags.sessionCachePath, "session-cache", filepath.Join(mustGetConfigDir(), "sessions.yaml"), "Path to session cache file") cmd.Flags().StringSliceVar(&flags.caBundlePaths, "ca-bundle", nil, "Path to TLS certificate authority bundle (PEM format, optional, can be repeated)") - cmd.Flags().StringSliceVar(&flags.caBundleData, "ca-bundle-data", nil, "Base64 endcoded TLS certificate authority bundle (base64 encoded PEM format, optional, can be repeated)") + cmd.Flags().StringSliceVar(&flags.caBundleData, "ca-bundle-data", nil, "Base64 encoded TLS certificate authority bundle (base64 encoded PEM format, optional, can be repeated)") cmd.Flags().BoolVar(&flags.debugSessionCache, "debug-session-cache", false, "Print debug logs related to the session cache") cmd.Flags().StringVar(&flags.requestAudience, "request-audience", "", "Request a token with an alternate audience using RFC8693 token exchange") - cmd.Flags().BoolVar(&flags.conciergeEnabled, "enable-concierge", false, "Exchange the OIDC ID token with the Pinniped concierge during login") - cmd.Flags().StringVar(&conciergeNamespace, "concierge-namespace", "pinniped-concierge", "Namespace in which the concierge was installed") + cmd.Flags().BoolVar(&flags.conciergeEnabled, "enable-concierge", false, "Use the Concierge to login") + cmd.Flags().StringVar(&conciergeNamespace, "concierge-namespace", "pinniped-concierge", "Namespace in which the Concierge was installed") cmd.Flags().StringVar(&flags.conciergeAuthenticatorType, "concierge-authenticator-type", "", "Concierge authenticator type (e.g., 'webhook', 'jwt')") cmd.Flags().StringVar(&flags.conciergeAuthenticatorName, "concierge-authenticator-name", "", "Concierge authenticator name") - cmd.Flags().StringVar(&flags.conciergeEndpoint, "concierge-endpoint", "", "API base for the Pinniped concierge endpoint") - cmd.Flags().StringVar(&flags.conciergeCABundle, "concierge-ca-bundle-data", "", "CA bundle to use when connecting to the concierge") + cmd.Flags().StringVar(&flags.conciergeEndpoint, "concierge-endpoint", "", "API base for the Concierge endpoint") + cmd.Flags().StringVar(&flags.conciergeCABundle, "concierge-ca-bundle-data", "", "CA bundle to use when connecting to the Concierge") cmd.Flags().StringVar(&flags.conciergeAPIGroupSuffix, "concierge-api-group-suffix", groupsuffix.PinnipedDefaultSuffix, "Concierge API group suffix") mustMarkHidden(cmd, "debug-session-cache") @@ -144,7 +144,7 @@ func runOIDCLogin(cmd *cobra.Command, deps oidcLoginCommandDeps, flags oidcLogin conciergeclient.WithAPIGroupSuffix(flags.conciergeAPIGroupSuffix), ) if err != nil { - return fmt.Errorf("invalid concierge parameters: %w", err) + return fmt.Errorf("invalid Concierge parameters: %w", err) } } @@ -172,17 +172,18 @@ func runOIDCLogin(cmd *cobra.Command, deps oidcLoginCommandDeps, flags oidcLogin cred := tokenCredential(token) // If the concierge was configured, exchange the credential for a separate short-lived, cluster-specific credential. - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - if concierge != nil { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cred, err = deps.exchangeToken(ctx, concierge, token.IDToken.Token) if err != nil { - return fmt.Errorf("could not complete concierge credential exchange: %w", err) + return fmt.Errorf("could not complete Concierge credential exchange: %w", err) } } return json.NewEncoder(cmd.OutOrStdout()).Encode(cred) } + func makeClient(caBundlePaths []string, caBundleData []string) (*http.Client, error) { pool := x509.NewCertPool() for _, p := range caBundlePaths { diff --git a/cmd/pinniped/cmd/login_oidc_test.go b/cmd/pinniped/cmd/login_oidc_test.go index 9a30de4ac..2b1e84698 100644 --- a/cmd/pinniped/cmd/login_oidc_test.go +++ b/cmd/pinniped/cmd/login_oidc_test.go @@ -6,7 +6,6 @@ package cmd import ( "bytes" "context" - "crypto/x509/pkix" "encoding/base64" "fmt" "io/ioutil" @@ -29,7 +28,7 @@ import ( func TestLoginOIDCCommand(t *testing.T) { cfgDir := mustGetConfigDir() - testCA, err := certauthority.New(pkix.Name{CommonName: "Test CA"}, 1*time.Hour) + testCA, err := certauthority.New("Test CA", 1*time.Hour) require.NoError(t, err) tmpdir := testutil.TempDir(t) testCABundlePath := filepath.Join(tmpdir, "testca.pem") @@ -58,14 +57,14 @@ func TestLoginOIDCCommand(t *testing.T) { Flags: --ca-bundle strings Path to TLS certificate authority bundle (PEM format, optional, can be repeated) - --ca-bundle-data strings Base64 endcoded TLS certificate authority bundle (base64 encoded PEM format, optional, can be repeated) + --ca-bundle-data strings Base64 encoded TLS certificate authority bundle (base64 encoded PEM format, optional, can be repeated) --client-id string OpenID Connect client ID (default "pinniped-cli") --concierge-api-group-suffix string Concierge API group suffix (default "pinniped.dev") --concierge-authenticator-name string Concierge authenticator name --concierge-authenticator-type string Concierge authenticator type (e.g., 'webhook', 'jwt') - --concierge-ca-bundle-data string CA bundle to use when connecting to the concierge - --concierge-endpoint string API base for the Pinniped concierge endpoint - --enable-concierge Exchange the OIDC ID token with the Pinniped concierge during login + --concierge-ca-bundle-data string CA bundle to use when connecting to the Concierge + --concierge-endpoint string API base for the Concierge endpoint + --enable-concierge Use the Concierge to login -h, --help help for oidc --issuer string OpenID Connect issuer URL --listen-port uint16 TCP port for localhost listener (authorization code flow only) @@ -92,7 +91,7 @@ func TestLoginOIDCCommand(t *testing.T) { }, wantError: true, wantStderr: here.Doc(` - Error: invalid concierge parameters: endpoint must not be empty + Error: invalid Concierge parameters: endpoint must not be empty `), }, { @@ -120,7 +119,7 @@ func TestLoginOIDCCommand(t *testing.T) { `), }, { - name: "invalid api group suffix", + name: "invalid API group suffix", args: []string{ "--issuer", "test-issuer", "--enable-concierge", @@ -131,7 +130,7 @@ func TestLoginOIDCCommand(t *testing.T) { }, wantError: true, wantStderr: here.Doc(` - Error: invalid concierge parameters: invalid api group suffix: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*') + Error: invalid Concierge parameters: invalid API group suffix: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*') `), }, { @@ -161,7 +160,7 @@ func TestLoginOIDCCommand(t *testing.T) { wantOptionsCount: 3, wantError: true, wantStderr: here.Doc(` - Error: could not complete concierge credential exchange: some concierge error + Error: could not complete Concierge credential exchange: some concierge error `), }, { diff --git a/cmd/pinniped/cmd/login_static.go b/cmd/pinniped/cmd/login_static.go index 6b391d199..4aa6d5eb6 100644 --- a/cmd/pinniped/cmd/login_static.go +++ b/cmd/pinniped/cmd/login_static.go @@ -62,13 +62,14 @@ func staticLoginCommand(deps staticLoginDeps) *cobra.Command { ) cmd.Flags().StringVar(&flags.staticToken, "token", "", "Static token to present during login") cmd.Flags().StringVar(&flags.staticTokenEnvName, "token-env", "", "Environment variable containing a static token") - cmd.Flags().BoolVar(&flags.conciergeEnabled, "enable-concierge", false, "Exchange the token with the Pinniped concierge during login") - cmd.Flags().StringVar(&conciergeNamespace, "concierge-namespace", "pinniped-concierge", "Namespace in which the concierge was installed") + cmd.Flags().BoolVar(&flags.conciergeEnabled, "enable-concierge", false, "Use the Concierge to login") + cmd.Flags().StringVar(&conciergeNamespace, "concierge-namespace", "pinniped-concierge", "Namespace in which the Concierge was installed") cmd.Flags().StringVar(&flags.conciergeAuthenticatorType, "concierge-authenticator-type", "", "Concierge authenticator type (e.g., 'webhook', 'jwt')") cmd.Flags().StringVar(&flags.conciergeAuthenticatorName, "concierge-authenticator-name", "", "Concierge authenticator name") - cmd.Flags().StringVar(&flags.conciergeEndpoint, "concierge-endpoint", "", "API base for the Pinniped concierge endpoint") - cmd.Flags().StringVar(&flags.conciergeCABundle, "concierge-ca-bundle-data", "", "CA bundle to use when connecting to the concierge") + cmd.Flags().StringVar(&flags.conciergeEndpoint, "concierge-endpoint", "", "API base for the Concierge endpoint") + cmd.Flags().StringVar(&flags.conciergeCABundle, "concierge-ca-bundle-data", "", "CA bundle to use when connecting to the Concierge") cmd.Flags().StringVar(&flags.conciergeAPIGroupSuffix, "concierge-api-group-suffix", groupsuffix.PinnipedDefaultSuffix, "Concierge API group suffix") + cmd.RunE = func(cmd *cobra.Command, args []string) error { return runStaticLogin(cmd.OutOrStdout(), deps, flags) } mustMarkDeprecated(cmd, "concierge-namespace", "not needed anymore") @@ -92,7 +93,7 @@ func runStaticLogin(out io.Writer, deps staticLoginDeps, flags staticLoginParams conciergeclient.WithAPIGroupSuffix(flags.conciergeAPIGroupSuffix), ) if err != nil { - return fmt.Errorf("invalid concierge parameters: %w", err) + return fmt.Errorf("invalid Concierge parameters: %w", err) } } @@ -112,7 +113,7 @@ func runStaticLogin(out io.Writer, deps staticLoginDeps, flags staticLoginParams } cred := tokenCredential(&oidctypes.Token{IDToken: &oidctypes.IDToken{Token: token}}) - // Exchange that token with the concierge, if configured. + // If the concierge was configured, exchange the credential for a separate short-lived, cluster-specific credential. if concierge != nil { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -120,7 +121,7 @@ func runStaticLogin(out io.Writer, deps staticLoginDeps, flags staticLoginParams var err error cred, err = deps.exchangeToken(ctx, concierge, token) if err != nil { - return fmt.Errorf("could not complete concierge credential exchange: %w", err) + return fmt.Errorf("could not complete Concierge credential exchange: %w", err) } } return json.NewEncoder(out).Encode(cred) diff --git a/cmd/pinniped/cmd/login_static_test.go b/cmd/pinniped/cmd/login_static_test.go index 6f9ced9ec..caf41df50 100644 --- a/cmd/pinniped/cmd/login_static_test.go +++ b/cmd/pinniped/cmd/login_static_test.go @@ -6,7 +6,6 @@ package cmd import ( "bytes" "context" - "crypto/x509/pkix" "fmt" "io/ioutil" "path/filepath" @@ -24,7 +23,7 @@ import ( ) func TestLoginStaticCommand(t *testing.T) { - testCA, err := certauthority.New(pkix.Name{CommonName: "Test CA"}, 1*time.Hour) + testCA, err := certauthority.New("Test CA", 1*time.Hour) require.NoError(t, err) tmpdir := testutil.TempDir(t) testCABundlePath := filepath.Join(tmpdir, "testca.pem") @@ -54,9 +53,9 @@ func TestLoginStaticCommand(t *testing.T) { --concierge-api-group-suffix string Concierge API group suffix (default "pinniped.dev") --concierge-authenticator-name string Concierge authenticator name --concierge-authenticator-type string Concierge authenticator type (e.g., 'webhook', 'jwt') - --concierge-ca-bundle-data string CA bundle to use when connecting to the concierge - --concierge-endpoint string API base for the Pinniped concierge endpoint - --enable-concierge Exchange the token with the Pinniped concierge during login + --concierge-ca-bundle-data string CA bundle to use when connecting to the Concierge + --concierge-endpoint string API base for the Concierge endpoint + --enable-concierge Use the Concierge to login -h, --help help for static --token string Static token to present during login --token-env string Environment variable containing a static token @@ -78,7 +77,7 @@ func TestLoginStaticCommand(t *testing.T) { }, wantError: true, wantStderr: here.Doc(` - Error: invalid concierge parameters: endpoint must not be empty + Error: invalid Concierge parameters: endpoint must not be empty `), }, { @@ -126,11 +125,11 @@ func TestLoginStaticCommand(t *testing.T) { conciergeErr: fmt.Errorf("some concierge error"), wantError: true, wantStderr: here.Doc(` - Error: could not complete concierge credential exchange: some concierge error + Error: could not complete Concierge credential exchange: some concierge error `), }, { - name: "invalid api group suffix", + name: "invalid API group suffix", args: []string{ "--token", "test-token", "--enable-concierge", @@ -141,7 +140,7 @@ func TestLoginStaticCommand(t *testing.T) { }, wantError: true, wantStderr: here.Doc(` - Error: invalid concierge parameters: invalid api group suffix: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*') + Error: invalid Concierge parameters: invalid API group suffix: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*') `), }, { diff --git a/cmd/pinniped/cmd/whoami.go b/cmd/pinniped/cmd/whoami.go index e0e83c01f..cfacdd6ac 100644 --- a/cmd/pinniped/cmd/whoami.go +++ b/cmd/pinniped/cmd/whoami.go @@ -15,18 +15,13 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/tools/clientcmd" - identityapi "go.pinniped.dev/generated/latest/apis/concierge/identity" identityv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/identity/v1alpha1" - loginapi "go.pinniped.dev/generated/latest/apis/concierge/login" - loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1" + conciergescheme "go.pinniped.dev/internal/concierge/scheme" "go.pinniped.dev/internal/groupsuffix" "go.pinniped.dev/internal/here" - "go.pinniped.dev/internal/plog" ) //nolint: gochecknoinits @@ -163,7 +158,7 @@ func writeWhoamiOutputYAML(output io.Writer, apiGroupSuffix string, whoAmI *iden } func serialize(output io.Writer, apiGroupSuffix string, whoAmI *identityv1alpha1.WhoAmIRequest, contentType string) error { - scheme, _, identityGV := conciergeschemeNew(apiGroupSuffix) + scheme, _, identityGV := conciergescheme.New(apiGroupSuffix) codecs := serializer.NewCodecFactory(scheme) respInfo, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), contentType) if !ok { @@ -194,110 +189,3 @@ func prettyStrings(ss []string) string { } return b.String() } - -// conciergeschemeNew is a temporary private function to stand in place for -// "go.pinniped.dev/internal/concierge/scheme".New until the later function is merged to main. -func conciergeschemeNew(apiGroupSuffix string) (_ *runtime.Scheme, login, identity schema.GroupVersion) { - // standard set up of the server side scheme - scheme := runtime.NewScheme() - - // add the options to empty v1 - metav1.AddToGroupVersion(scheme, metav1.Unversioned) - - // nothing fancy is required if using the standard group suffix - if apiGroupSuffix == groupsuffix.PinnipedDefaultSuffix { - schemeBuilder := runtime.NewSchemeBuilder( - loginv1alpha1.AddToScheme, - loginapi.AddToScheme, - identityv1alpha1.AddToScheme, - identityapi.AddToScheme, - ) - utilruntime.Must(schemeBuilder.AddToScheme(scheme)) - return scheme, loginv1alpha1.SchemeGroupVersion, identityv1alpha1.SchemeGroupVersion - } - - loginConciergeGroupData, identityConciergeGroupData := groupsuffix.ConciergeAggregatedGroups(apiGroupSuffix) - - addToSchemeAtNewGroup(scheme, loginv1alpha1.GroupName, loginConciergeGroupData.Group, loginv1alpha1.AddToScheme, loginapi.AddToScheme) - addToSchemeAtNewGroup(scheme, identityv1alpha1.GroupName, identityConciergeGroupData.Group, identityv1alpha1.AddToScheme, identityapi.AddToScheme) - - // manually register conversions and defaulting into the correct scheme since we cannot directly call AddToScheme - schemeBuilder := runtime.NewSchemeBuilder( - loginv1alpha1.RegisterConversions, - loginv1alpha1.RegisterDefaults, - identityv1alpha1.RegisterConversions, - identityv1alpha1.RegisterDefaults, - ) - utilruntime.Must(schemeBuilder.AddToScheme(scheme)) - - // we do not want to return errors from the scheme and instead would prefer to defer - // to the REST storage layer for consistency. The simplest way to do this is to force - // a cache miss from the authenticator cache. Kube API groups are validated via the - // IsDNS1123Subdomain func thus we can easily create a group that is guaranteed never - // to be in the authenticator cache. Add a timestamp just to be extra sure. - const authenticatorCacheMissPrefix = "_INVALID_API_GROUP_" - authenticatorCacheMiss := authenticatorCacheMissPrefix + time.Now().UTC().String() - - // we do not have any defaulting functions for *loginv1alpha1.TokenCredentialRequest - // today, but we may have some in the future. Calling AddTypeDefaultingFunc overwrites - // any previously registered defaulting function. Thus to make sure that we catch - // a situation where we add a defaulting func, we attempt to call it here with a nil - // *loginv1alpha1.TokenCredentialRequest. This will do nothing when there is no - // defaulting func registered, but it will almost certainly panic if one is added. - scheme.Default((*loginv1alpha1.TokenCredentialRequest)(nil)) - - // on incoming requests, restore the authenticator API group to the standard group - // note that we are responsible for duplicating this logic for every external API version - scheme.AddTypeDefaultingFunc(&loginv1alpha1.TokenCredentialRequest{}, func(obj interface{}) { - credentialRequest := obj.(*loginv1alpha1.TokenCredentialRequest) - - if credentialRequest.Spec.Authenticator.APIGroup == nil { - // force a cache miss because this is an invalid request - plog.Debug("invalid token credential request, nil group", "authenticator", credentialRequest.Spec.Authenticator) - credentialRequest.Spec.Authenticator.APIGroup = &authenticatorCacheMiss - return - } - - restoredGroup, ok := groupsuffix.Unreplace(*credentialRequest.Spec.Authenticator.APIGroup, apiGroupSuffix) - if !ok { - // force a cache miss because this is an invalid request - plog.Debug("invalid token credential request, wrong group", "authenticator", credentialRequest.Spec.Authenticator) - credentialRequest.Spec.Authenticator.APIGroup = &authenticatorCacheMiss - return - } - - credentialRequest.Spec.Authenticator.APIGroup = &restoredGroup - }) - - return scheme, schema.GroupVersion(loginConciergeGroupData), schema.GroupVersion(identityConciergeGroupData) -} - -func addToSchemeAtNewGroup(scheme *runtime.Scheme, oldGroup, newGroup string, funcs ...func(*runtime.Scheme) error) { - // we need a temporary place to register our types to avoid double registering them - tmpScheme := runtime.NewScheme() - schemeBuilder := runtime.NewSchemeBuilder(funcs...) - utilruntime.Must(schemeBuilder.AddToScheme(tmpScheme)) - - for gvk := range tmpScheme.AllKnownTypes() { - if gvk.GroupVersion() == metav1.Unversioned { - continue // metav1.AddToGroupVersion registers types outside of our aggregated API group that we need to ignore - } - - if gvk.Group != oldGroup { - panic(fmt.Errorf("tmp scheme has type not in the old aggregated API group %s: %s", oldGroup, gvk)) // programmer error - } - - obj, err := tmpScheme.New(gvk) - if err != nil { - panic(err) // programmer error, scheme internal code is broken - } - newGVK := schema.GroupVersionKind{ - Group: newGroup, - Version: gvk.Version, - Kind: gvk.Kind, - } - - // register the existing type but with the new group in the correct scheme - scheme.AddKnownTypeWithName(newGVK, obj) - } -} diff --git a/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml b/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml index d61a78350..8123f2385 100644 --- a/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml @@ -67,14 +67,34 @@ spec: description: Frontend describes how clients can connect using this strategy. properties: + impersonationProxyInfo: + description: ImpersonationProxyInfo describes the parameters + for the impersonation proxy on this Concierge. This field + is only set when Type is "ImpersonationProxy". + properties: + certificateAuthorityData: + description: CertificateAuthorityData is the base64-encoded + PEM CA bundle of the impersonation proxy. + minLength: 1 + type: string + endpoint: + description: Endpoint is the HTTPS endpoint of the impersonation + proxy. + minLength: 1 + pattern: ^https:// + type: string + required: + - certificateAuthorityData + - endpoint + type: object tokenCredentialRequestInfo: description: TokenCredentialRequestAPIInfo describes the parameters for the TokenCredentialRequest API on this Concierge. This field is only set when Type is "TokenCredentialRequestAPI". properties: certificateAuthorityData: - description: CertificateAuthorityData is the Kubernetes - API server CA bundle. + description: CertificateAuthorityData is the base64-encoded + Kubernetes API server CA bundle. minLength: 1 type: string server: @@ -91,6 +111,7 @@ spec: can use with a strategy. enum: - TokenCredentialRequestAPI + - ImpersonationProxy type: string required: - type @@ -106,8 +127,13 @@ spec: reason: description: Reason for the current status. enum: - - FetchedKey + - Listening + - Pending + - Disabled + - ErrorDuringSetup - CouldNotFetchKey + - CouldNotGetClusterInfo + - FetchedKey type: string status: description: Status of the attempted integration strategy. @@ -119,6 +145,7 @@ spec: description: Type of integration attempted. enum: - KubeClusterSigningCertificate + - ImpersonationProxy type: string required: - lastUpdateTime diff --git a/deploy/concierge/deployment.yaml b/deploy/concierge/deployment.yaml index 58a5098be..223b7fa80 100644 --- a/deploy/concierge/deployment.yaml +++ b/deploy/concierge/deployment.yaml @@ -42,6 +42,11 @@ data: servingCertificateSecret: (@= defaultResourceNameWithSuffix("api-tls-serving-certificate") @) credentialIssuer: (@= defaultResourceNameWithSuffix("config") @) apiService: (@= defaultResourceNameWithSuffix("api") @) + impersonationConfigMap: (@= defaultResourceNameWithSuffix("impersonation-proxy-config") @) + impersonationLoadBalancerService: (@= defaultResourceNameWithSuffix("impersonation-proxy-load-balancer") @) + impersonationTLSCertificateSecret: (@= defaultResourceNameWithSuffix("impersonation-proxy-tls-serving-certificate") @) + impersonationCACertificateSecret: (@= defaultResourceNameWithSuffix("impersonation-proxy-ca-certificate") @) + impersonationSignerSecret: (@= defaultResourceNameWithSuffix("impersonation-proxy-signer-ca-certificate") @) labels: (@= json.encode(labels()).rstrip() @) kubeCertAgent: namePrefix: (@= defaultResourceNameWithSuffix("kube-cert-agent-") @) @@ -189,6 +194,20 @@ spec: port: 443 targetPort: 8443 --- +apiVersion: v1 +kind: Service +metadata: + name: #@ defaultResourceNameWithSuffix("proxy") + namespace: #@ namespace() + labels: #@ labels() +spec: + type: ClusterIP + selector: #@ defaultLabel() + ports: + - protocol: TCP + port: 443 + targetPort: 8444 +--- apiVersion: apiregistration.k8s.io/v1 kind: APIService metadata: diff --git a/deploy/concierge/rbac.yaml b/deploy/concierge/rbac.yaml index 427af70e3..3f8817314 100644 --- a/deploy/concierge/rbac.yaml +++ b/deploy/concierge/rbac.yaml @@ -31,6 +31,12 @@ rules: resources: [ securitycontextconstraints ] verbs: [ use ] resourceNames: [ nonroot ] + - apiGroups: [ "" ] + resources: [ "users", "groups" ] + verbs: [ "impersonate" ] + - apiGroups: [ "" ] + resources: [ nodes ] + verbs: [ list ] - apiGroups: - #@ pinnipedDevAPIGroupWithPrefix("config.concierge") resources: [ credentialissuers ] @@ -69,7 +75,7 @@ metadata: rules: - apiGroups: [ "" ] resources: [ services ] - verbs: [ create, get, list, patch, update, watch ] + verbs: [ create, get, list, patch, update, watch, delete ] - apiGroups: [ "" ] resources: [ secrets ] verbs: [ create, get, list, patch, update, watch, delete ] @@ -81,9 +87,12 @@ rules: - apiGroups: [ "" ] resources: [ pods/exec ] verbs: [ create ] - - apiGroups: [apps] - resources: [replicasets,deployments] - verbs: [get] + - apiGroups: [ apps ] + resources: [ replicasets,deployments ] + verbs: [ get ] + - apiGroups: [ "" ] + resources: [ configmaps ] + verbs: [ list, get, watch ] --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/generated/1.17/README.adoc b/generated/1.17/README.adoc index c2848d450..25267e8c0 100644 --- a/generated/1.17/README.adoc +++ b/generated/1.17/README.adoc @@ -251,6 +251,7 @@ Describes the configuration status of a Pinniped credential issuer. | Field | Description | *`type`* __FrontendType__ | Type describes which frontend mechanism clients can use with a strategy. | *`tokenCredentialRequestInfo`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-tokencredentialrequestapiinfo[$$TokenCredentialRequestAPIInfo$$]__ | TokenCredentialRequestAPIInfo describes the parameters for the TokenCredentialRequest API on this Concierge. This field is only set when Type is "TokenCredentialRequestAPI". +| *`impersonationProxyInfo`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-impersonationproxyinfo[$$ImpersonationProxyInfo$$]__ | ImpersonationProxyInfo describes the parameters for the impersonation proxy on this Concierge. This field is only set when Type is "ImpersonationProxy". |=== @@ -314,6 +315,24 @@ Status of a credential issuer. |=== +[id="{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-impersonationproxyinfo"] +==== ImpersonationProxyInfo + + + +.Appears In: +**** +- xref:{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-credentialissuerfrontend[$$CredentialIssuerFrontend$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`endpoint`* __string__ | Endpoint is the HTTPS endpoint of the impersonation proxy. +| *`certificateAuthorityData`* __string__ | CertificateAuthorityData is the base64-encoded PEM CA bundle of the impersonation proxy. +|=== + + [id="{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-tokencredentialrequestapiinfo"] ==== TokenCredentialRequestAPIInfo @@ -328,7 +347,7 @@ Status of a credential issuer. |=== | Field | Description | *`server`* __string__ | Server is the Kubernetes API server URL. -| *`certificateAuthorityData`* __string__ | CertificateAuthorityData is the Kubernetes API server CA bundle. +| *`certificateAuthorityData`* __string__ | CertificateAuthorityData is the base64-encoded Kubernetes API server CA bundle. |=== diff --git a/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go index 399895015..3c0c8ba03 100644 --- a/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -1,30 +1,36 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package v1alpha1 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// +kubebuilder:validation:Enum=KubeClusterSigningCertificate +// +kubebuilder:validation:Enum=KubeClusterSigningCertificate;ImpersonationProxy type StrategyType string -// +kubebuilder:validation:Enum=TokenCredentialRequestAPI +// +kubebuilder:validation:Enum=TokenCredentialRequestAPI;ImpersonationProxy type FrontendType string // +kubebuilder:validation:Enum=Success;Error type StrategyStatus string -// +kubebuilder:validation:Enum=FetchedKey;CouldNotFetchKey +// +kubebuilder:validation:Enum=Listening;Pending;Disabled;ErrorDuringSetup;CouldNotFetchKey;CouldNotGetClusterInfo;FetchedKey type StrategyReason string const ( KubeClusterSigningCertificateStrategyType = StrategyType("KubeClusterSigningCertificate") + ImpersonationProxyStrategyType = StrategyType("ImpersonationProxy") TokenCredentialRequestAPIFrontendType = FrontendType("TokenCredentialRequestAPI") + ImpersonationProxyFrontendType = FrontendType("ImpersonationProxy") SuccessStrategyStatus = StrategyStatus("Success") ErrorStrategyStatus = StrategyStatus("Error") + ListeningStrategyReason = StrategyReason("Listening") + PendingStrategyReason = StrategyReason("Pending") + DisabledStrategyReason = StrategyReason("Disabled") + ErrorDuringSetupStrategyReason = StrategyReason("ErrorDuringSetup") CouldNotFetchKeyStrategyReason = StrategyReason("CouldNotFetchKey") CouldNotGetClusterInfoStrategyReason = StrategyReason("CouldNotGetClusterInfo") FetchedKeyStrategyReason = StrategyReason("FetchedKey") @@ -82,6 +88,10 @@ type CredentialIssuerFrontend struct { // TokenCredentialRequestAPIInfo describes the parameters for the TokenCredentialRequest API on this Concierge. // This field is only set when Type is "TokenCredentialRequestAPI". TokenCredentialRequestAPIInfo *TokenCredentialRequestAPIInfo `json:"tokenCredentialRequestInfo,omitempty"` + + // ImpersonationProxyInfo describes the parameters for the impersonation proxy on this Concierge. + // This field is only set when Type is "ImpersonationProxy". + ImpersonationProxyInfo *ImpersonationProxyInfo `json:"impersonationProxyInfo,omitempty"` } // TokenCredentialRequestAPIInfo describes the parameters for the TokenCredentialRequest API on this Concierge. @@ -91,7 +101,19 @@ type TokenCredentialRequestAPIInfo struct { // +kubebuilder:validation:Pattern=`^https://|^http://` Server string `json:"server"` - // CertificateAuthorityData is the Kubernetes API server CA bundle. + // CertificateAuthorityData is the base64-encoded Kubernetes API server CA bundle. + // +kubebuilder:validation:MinLength=1 + CertificateAuthorityData string `json:"certificateAuthorityData"` +} + +// ImpersonationProxyInfo describes the parameters for the impersonation proxy on this Concierge. +type ImpersonationProxyInfo struct { + // Endpoint is the HTTPS endpoint of the impersonation proxy. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:Pattern=`^https://` + Endpoint string `json:"endpoint"` + + // CertificateAuthorityData is the base64-encoded PEM CA bundle of the impersonation proxy. // +kubebuilder:validation:MinLength=1 CertificateAuthorityData string `json:"certificateAuthorityData"` } diff --git a/generated/1.17/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go b/generated/1.17/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go index ad9a9f66c..4a3ad605e 100644 --- a/generated/1.17/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go +++ b/generated/1.17/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go @@ -46,6 +46,11 @@ func (in *CredentialIssuerFrontend) DeepCopyInto(out *CredentialIssuerFrontend) *out = new(TokenCredentialRequestAPIInfo) **out = **in } + if in.ImpersonationProxyInfo != nil { + in, out := &in.ImpersonationProxyInfo, &out.ImpersonationProxyInfo + *out = new(ImpersonationProxyInfo) + **out = **in + } return } @@ -158,6 +163,22 @@ func (in *CredentialIssuerStrategy) DeepCopy() *CredentialIssuerStrategy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImpersonationProxyInfo) DeepCopyInto(out *ImpersonationProxyInfo) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImpersonationProxyInfo. +func (in *ImpersonationProxyInfo) DeepCopy() *ImpersonationProxyInfo { + if in == nil { + return nil + } + out := new(ImpersonationProxyInfo) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TokenCredentialRequestAPIInfo) DeepCopyInto(out *TokenCredentialRequestAPIInfo) { *out = *in diff --git a/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml index d61a78350..8123f2385 100644 --- a/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -67,14 +67,34 @@ spec: description: Frontend describes how clients can connect using this strategy. properties: + impersonationProxyInfo: + description: ImpersonationProxyInfo describes the parameters + for the impersonation proxy on this Concierge. This field + is only set when Type is "ImpersonationProxy". + properties: + certificateAuthorityData: + description: CertificateAuthorityData is the base64-encoded + PEM CA bundle of the impersonation proxy. + minLength: 1 + type: string + endpoint: + description: Endpoint is the HTTPS endpoint of the impersonation + proxy. + minLength: 1 + pattern: ^https:// + type: string + required: + - certificateAuthorityData + - endpoint + type: object tokenCredentialRequestInfo: description: TokenCredentialRequestAPIInfo describes the parameters for the TokenCredentialRequest API on this Concierge. This field is only set when Type is "TokenCredentialRequestAPI". properties: certificateAuthorityData: - description: CertificateAuthorityData is the Kubernetes - API server CA bundle. + description: CertificateAuthorityData is the base64-encoded + Kubernetes API server CA bundle. minLength: 1 type: string server: @@ -91,6 +111,7 @@ spec: can use with a strategy. enum: - TokenCredentialRequestAPI + - ImpersonationProxy type: string required: - type @@ -106,8 +127,13 @@ spec: reason: description: Reason for the current status. enum: - - FetchedKey + - Listening + - Pending + - Disabled + - ErrorDuringSetup - CouldNotFetchKey + - CouldNotGetClusterInfo + - FetchedKey type: string status: description: Status of the attempted integration strategy. @@ -119,6 +145,7 @@ spec: description: Type of integration attempted. enum: - KubeClusterSigningCertificate + - ImpersonationProxy type: string required: - lastUpdateTime diff --git a/generated/1.18/README.adoc b/generated/1.18/README.adoc index 4a1580f2b..2c01f9277 100644 --- a/generated/1.18/README.adoc +++ b/generated/1.18/README.adoc @@ -251,6 +251,7 @@ Describes the configuration status of a Pinniped credential issuer. | Field | Description | *`type`* __FrontendType__ | Type describes which frontend mechanism clients can use with a strategy. | *`tokenCredentialRequestInfo`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-tokencredentialrequestapiinfo[$$TokenCredentialRequestAPIInfo$$]__ | TokenCredentialRequestAPIInfo describes the parameters for the TokenCredentialRequest API on this Concierge. This field is only set when Type is "TokenCredentialRequestAPI". +| *`impersonationProxyInfo`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-impersonationproxyinfo[$$ImpersonationProxyInfo$$]__ | ImpersonationProxyInfo describes the parameters for the impersonation proxy on this Concierge. This field is only set when Type is "ImpersonationProxy". |=== @@ -314,6 +315,24 @@ Status of a credential issuer. |=== +[id="{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-impersonationproxyinfo"] +==== ImpersonationProxyInfo + + + +.Appears In: +**** +- xref:{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-credentialissuerfrontend[$$CredentialIssuerFrontend$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`endpoint`* __string__ | Endpoint is the HTTPS endpoint of the impersonation proxy. +| *`certificateAuthorityData`* __string__ | CertificateAuthorityData is the base64-encoded PEM CA bundle of the impersonation proxy. +|=== + + [id="{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-tokencredentialrequestapiinfo"] ==== TokenCredentialRequestAPIInfo @@ -328,7 +347,7 @@ Status of a credential issuer. |=== | Field | Description | *`server`* __string__ | Server is the Kubernetes API server URL. -| *`certificateAuthorityData`* __string__ | CertificateAuthorityData is the Kubernetes API server CA bundle. +| *`certificateAuthorityData`* __string__ | CertificateAuthorityData is the base64-encoded Kubernetes API server CA bundle. |=== diff --git a/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go index 399895015..3c0c8ba03 100644 --- a/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -1,30 +1,36 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package v1alpha1 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// +kubebuilder:validation:Enum=KubeClusterSigningCertificate +// +kubebuilder:validation:Enum=KubeClusterSigningCertificate;ImpersonationProxy type StrategyType string -// +kubebuilder:validation:Enum=TokenCredentialRequestAPI +// +kubebuilder:validation:Enum=TokenCredentialRequestAPI;ImpersonationProxy type FrontendType string // +kubebuilder:validation:Enum=Success;Error type StrategyStatus string -// +kubebuilder:validation:Enum=FetchedKey;CouldNotFetchKey +// +kubebuilder:validation:Enum=Listening;Pending;Disabled;ErrorDuringSetup;CouldNotFetchKey;CouldNotGetClusterInfo;FetchedKey type StrategyReason string const ( KubeClusterSigningCertificateStrategyType = StrategyType("KubeClusterSigningCertificate") + ImpersonationProxyStrategyType = StrategyType("ImpersonationProxy") TokenCredentialRequestAPIFrontendType = FrontendType("TokenCredentialRequestAPI") + ImpersonationProxyFrontendType = FrontendType("ImpersonationProxy") SuccessStrategyStatus = StrategyStatus("Success") ErrorStrategyStatus = StrategyStatus("Error") + ListeningStrategyReason = StrategyReason("Listening") + PendingStrategyReason = StrategyReason("Pending") + DisabledStrategyReason = StrategyReason("Disabled") + ErrorDuringSetupStrategyReason = StrategyReason("ErrorDuringSetup") CouldNotFetchKeyStrategyReason = StrategyReason("CouldNotFetchKey") CouldNotGetClusterInfoStrategyReason = StrategyReason("CouldNotGetClusterInfo") FetchedKeyStrategyReason = StrategyReason("FetchedKey") @@ -82,6 +88,10 @@ type CredentialIssuerFrontend struct { // TokenCredentialRequestAPIInfo describes the parameters for the TokenCredentialRequest API on this Concierge. // This field is only set when Type is "TokenCredentialRequestAPI". TokenCredentialRequestAPIInfo *TokenCredentialRequestAPIInfo `json:"tokenCredentialRequestInfo,omitempty"` + + // ImpersonationProxyInfo describes the parameters for the impersonation proxy on this Concierge. + // This field is only set when Type is "ImpersonationProxy". + ImpersonationProxyInfo *ImpersonationProxyInfo `json:"impersonationProxyInfo,omitempty"` } // TokenCredentialRequestAPIInfo describes the parameters for the TokenCredentialRequest API on this Concierge. @@ -91,7 +101,19 @@ type TokenCredentialRequestAPIInfo struct { // +kubebuilder:validation:Pattern=`^https://|^http://` Server string `json:"server"` - // CertificateAuthorityData is the Kubernetes API server CA bundle. + // CertificateAuthorityData is the base64-encoded Kubernetes API server CA bundle. + // +kubebuilder:validation:MinLength=1 + CertificateAuthorityData string `json:"certificateAuthorityData"` +} + +// ImpersonationProxyInfo describes the parameters for the impersonation proxy on this Concierge. +type ImpersonationProxyInfo struct { + // Endpoint is the HTTPS endpoint of the impersonation proxy. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:Pattern=`^https://` + Endpoint string `json:"endpoint"` + + // CertificateAuthorityData is the base64-encoded PEM CA bundle of the impersonation proxy. // +kubebuilder:validation:MinLength=1 CertificateAuthorityData string `json:"certificateAuthorityData"` } diff --git a/generated/1.18/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go b/generated/1.18/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go index ad9a9f66c..4a3ad605e 100644 --- a/generated/1.18/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go +++ b/generated/1.18/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go @@ -46,6 +46,11 @@ func (in *CredentialIssuerFrontend) DeepCopyInto(out *CredentialIssuerFrontend) *out = new(TokenCredentialRequestAPIInfo) **out = **in } + if in.ImpersonationProxyInfo != nil { + in, out := &in.ImpersonationProxyInfo, &out.ImpersonationProxyInfo + *out = new(ImpersonationProxyInfo) + **out = **in + } return } @@ -158,6 +163,22 @@ func (in *CredentialIssuerStrategy) DeepCopy() *CredentialIssuerStrategy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImpersonationProxyInfo) DeepCopyInto(out *ImpersonationProxyInfo) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImpersonationProxyInfo. +func (in *ImpersonationProxyInfo) DeepCopy() *ImpersonationProxyInfo { + if in == nil { + return nil + } + out := new(ImpersonationProxyInfo) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TokenCredentialRequestAPIInfo) DeepCopyInto(out *TokenCredentialRequestAPIInfo) { *out = *in diff --git a/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml index d61a78350..8123f2385 100644 --- a/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -67,14 +67,34 @@ spec: description: Frontend describes how clients can connect using this strategy. properties: + impersonationProxyInfo: + description: ImpersonationProxyInfo describes the parameters + for the impersonation proxy on this Concierge. This field + is only set when Type is "ImpersonationProxy". + properties: + certificateAuthorityData: + description: CertificateAuthorityData is the base64-encoded + PEM CA bundle of the impersonation proxy. + minLength: 1 + type: string + endpoint: + description: Endpoint is the HTTPS endpoint of the impersonation + proxy. + minLength: 1 + pattern: ^https:// + type: string + required: + - certificateAuthorityData + - endpoint + type: object tokenCredentialRequestInfo: description: TokenCredentialRequestAPIInfo describes the parameters for the TokenCredentialRequest API on this Concierge. This field is only set when Type is "TokenCredentialRequestAPI". properties: certificateAuthorityData: - description: CertificateAuthorityData is the Kubernetes - API server CA bundle. + description: CertificateAuthorityData is the base64-encoded + Kubernetes API server CA bundle. minLength: 1 type: string server: @@ -91,6 +111,7 @@ spec: can use with a strategy. enum: - TokenCredentialRequestAPI + - ImpersonationProxy type: string required: - type @@ -106,8 +127,13 @@ spec: reason: description: Reason for the current status. enum: - - FetchedKey + - Listening + - Pending + - Disabled + - ErrorDuringSetup - CouldNotFetchKey + - CouldNotGetClusterInfo + - FetchedKey type: string status: description: Status of the attempted integration strategy. @@ -119,6 +145,7 @@ spec: description: Type of integration attempted. enum: - KubeClusterSigningCertificate + - ImpersonationProxy type: string required: - lastUpdateTime diff --git a/generated/1.19/README.adoc b/generated/1.19/README.adoc index a77f3f184..0c2527aea 100644 --- a/generated/1.19/README.adoc +++ b/generated/1.19/README.adoc @@ -251,6 +251,7 @@ Describes the configuration status of a Pinniped credential issuer. | Field | Description | *`type`* __FrontendType__ | Type describes which frontend mechanism clients can use with a strategy. | *`tokenCredentialRequestInfo`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-tokencredentialrequestapiinfo[$$TokenCredentialRequestAPIInfo$$]__ | TokenCredentialRequestAPIInfo describes the parameters for the TokenCredentialRequest API on this Concierge. This field is only set when Type is "TokenCredentialRequestAPI". +| *`impersonationProxyInfo`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-impersonationproxyinfo[$$ImpersonationProxyInfo$$]__ | ImpersonationProxyInfo describes the parameters for the impersonation proxy on this Concierge. This field is only set when Type is "ImpersonationProxy". |=== @@ -314,6 +315,24 @@ Status of a credential issuer. |=== +[id="{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-impersonationproxyinfo"] +==== ImpersonationProxyInfo + + + +.Appears In: +**** +- xref:{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-credentialissuerfrontend[$$CredentialIssuerFrontend$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`endpoint`* __string__ | Endpoint is the HTTPS endpoint of the impersonation proxy. +| *`certificateAuthorityData`* __string__ | CertificateAuthorityData is the base64-encoded PEM CA bundle of the impersonation proxy. +|=== + + [id="{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-tokencredentialrequestapiinfo"] ==== TokenCredentialRequestAPIInfo @@ -328,7 +347,7 @@ Status of a credential issuer. |=== | Field | Description | *`server`* __string__ | Server is the Kubernetes API server URL. -| *`certificateAuthorityData`* __string__ | CertificateAuthorityData is the Kubernetes API server CA bundle. +| *`certificateAuthorityData`* __string__ | CertificateAuthorityData is the base64-encoded Kubernetes API server CA bundle. |=== diff --git a/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go index 399895015..3c0c8ba03 100644 --- a/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -1,30 +1,36 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package v1alpha1 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// +kubebuilder:validation:Enum=KubeClusterSigningCertificate +// +kubebuilder:validation:Enum=KubeClusterSigningCertificate;ImpersonationProxy type StrategyType string -// +kubebuilder:validation:Enum=TokenCredentialRequestAPI +// +kubebuilder:validation:Enum=TokenCredentialRequestAPI;ImpersonationProxy type FrontendType string // +kubebuilder:validation:Enum=Success;Error type StrategyStatus string -// +kubebuilder:validation:Enum=FetchedKey;CouldNotFetchKey +// +kubebuilder:validation:Enum=Listening;Pending;Disabled;ErrorDuringSetup;CouldNotFetchKey;CouldNotGetClusterInfo;FetchedKey type StrategyReason string const ( KubeClusterSigningCertificateStrategyType = StrategyType("KubeClusterSigningCertificate") + ImpersonationProxyStrategyType = StrategyType("ImpersonationProxy") TokenCredentialRequestAPIFrontendType = FrontendType("TokenCredentialRequestAPI") + ImpersonationProxyFrontendType = FrontendType("ImpersonationProxy") SuccessStrategyStatus = StrategyStatus("Success") ErrorStrategyStatus = StrategyStatus("Error") + ListeningStrategyReason = StrategyReason("Listening") + PendingStrategyReason = StrategyReason("Pending") + DisabledStrategyReason = StrategyReason("Disabled") + ErrorDuringSetupStrategyReason = StrategyReason("ErrorDuringSetup") CouldNotFetchKeyStrategyReason = StrategyReason("CouldNotFetchKey") CouldNotGetClusterInfoStrategyReason = StrategyReason("CouldNotGetClusterInfo") FetchedKeyStrategyReason = StrategyReason("FetchedKey") @@ -82,6 +88,10 @@ type CredentialIssuerFrontend struct { // TokenCredentialRequestAPIInfo describes the parameters for the TokenCredentialRequest API on this Concierge. // This field is only set when Type is "TokenCredentialRequestAPI". TokenCredentialRequestAPIInfo *TokenCredentialRequestAPIInfo `json:"tokenCredentialRequestInfo,omitempty"` + + // ImpersonationProxyInfo describes the parameters for the impersonation proxy on this Concierge. + // This field is only set when Type is "ImpersonationProxy". + ImpersonationProxyInfo *ImpersonationProxyInfo `json:"impersonationProxyInfo,omitempty"` } // TokenCredentialRequestAPIInfo describes the parameters for the TokenCredentialRequest API on this Concierge. @@ -91,7 +101,19 @@ type TokenCredentialRequestAPIInfo struct { // +kubebuilder:validation:Pattern=`^https://|^http://` Server string `json:"server"` - // CertificateAuthorityData is the Kubernetes API server CA bundle. + // CertificateAuthorityData is the base64-encoded Kubernetes API server CA bundle. + // +kubebuilder:validation:MinLength=1 + CertificateAuthorityData string `json:"certificateAuthorityData"` +} + +// ImpersonationProxyInfo describes the parameters for the impersonation proxy on this Concierge. +type ImpersonationProxyInfo struct { + // Endpoint is the HTTPS endpoint of the impersonation proxy. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:Pattern=`^https://` + Endpoint string `json:"endpoint"` + + // CertificateAuthorityData is the base64-encoded PEM CA bundle of the impersonation proxy. // +kubebuilder:validation:MinLength=1 CertificateAuthorityData string `json:"certificateAuthorityData"` } diff --git a/generated/1.19/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go b/generated/1.19/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go index ad9a9f66c..4a3ad605e 100644 --- a/generated/1.19/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go +++ b/generated/1.19/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go @@ -46,6 +46,11 @@ func (in *CredentialIssuerFrontend) DeepCopyInto(out *CredentialIssuerFrontend) *out = new(TokenCredentialRequestAPIInfo) **out = **in } + if in.ImpersonationProxyInfo != nil { + in, out := &in.ImpersonationProxyInfo, &out.ImpersonationProxyInfo + *out = new(ImpersonationProxyInfo) + **out = **in + } return } @@ -158,6 +163,22 @@ func (in *CredentialIssuerStrategy) DeepCopy() *CredentialIssuerStrategy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImpersonationProxyInfo) DeepCopyInto(out *ImpersonationProxyInfo) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImpersonationProxyInfo. +func (in *ImpersonationProxyInfo) DeepCopy() *ImpersonationProxyInfo { + if in == nil { + return nil + } + out := new(ImpersonationProxyInfo) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TokenCredentialRequestAPIInfo) DeepCopyInto(out *TokenCredentialRequestAPIInfo) { *out = *in diff --git a/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml index d61a78350..8123f2385 100644 --- a/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -67,14 +67,34 @@ spec: description: Frontend describes how clients can connect using this strategy. properties: + impersonationProxyInfo: + description: ImpersonationProxyInfo describes the parameters + for the impersonation proxy on this Concierge. This field + is only set when Type is "ImpersonationProxy". + properties: + certificateAuthorityData: + description: CertificateAuthorityData is the base64-encoded + PEM CA bundle of the impersonation proxy. + minLength: 1 + type: string + endpoint: + description: Endpoint is the HTTPS endpoint of the impersonation + proxy. + minLength: 1 + pattern: ^https:// + type: string + required: + - certificateAuthorityData + - endpoint + type: object tokenCredentialRequestInfo: description: TokenCredentialRequestAPIInfo describes the parameters for the TokenCredentialRequest API on this Concierge. This field is only set when Type is "TokenCredentialRequestAPI". properties: certificateAuthorityData: - description: CertificateAuthorityData is the Kubernetes - API server CA bundle. + description: CertificateAuthorityData is the base64-encoded + Kubernetes API server CA bundle. minLength: 1 type: string server: @@ -91,6 +111,7 @@ spec: can use with a strategy. enum: - TokenCredentialRequestAPI + - ImpersonationProxy type: string required: - type @@ -106,8 +127,13 @@ spec: reason: description: Reason for the current status. enum: - - FetchedKey + - Listening + - Pending + - Disabled + - ErrorDuringSetup - CouldNotFetchKey + - CouldNotGetClusterInfo + - FetchedKey type: string status: description: Status of the attempted integration strategy. @@ -119,6 +145,7 @@ spec: description: Type of integration attempted. enum: - KubeClusterSigningCertificate + - ImpersonationProxy type: string required: - lastUpdateTime diff --git a/generated/1.20/README.adoc b/generated/1.20/README.adoc index 7b35be866..91b4eef60 100644 --- a/generated/1.20/README.adoc +++ b/generated/1.20/README.adoc @@ -251,6 +251,7 @@ Describes the configuration status of a Pinniped credential issuer. | Field | Description | *`type`* __FrontendType__ | Type describes which frontend mechanism clients can use with a strategy. | *`tokenCredentialRequestInfo`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-tokencredentialrequestapiinfo[$$TokenCredentialRequestAPIInfo$$]__ | TokenCredentialRequestAPIInfo describes the parameters for the TokenCredentialRequest API on this Concierge. This field is only set when Type is "TokenCredentialRequestAPI". +| *`impersonationProxyInfo`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-impersonationproxyinfo[$$ImpersonationProxyInfo$$]__ | ImpersonationProxyInfo describes the parameters for the impersonation proxy on this Concierge. This field is only set when Type is "ImpersonationProxy". |=== @@ -314,6 +315,24 @@ Status of a credential issuer. |=== +[id="{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-impersonationproxyinfo"] +==== ImpersonationProxyInfo + + + +.Appears In: +**** +- xref:{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-credentialissuerfrontend[$$CredentialIssuerFrontend$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`endpoint`* __string__ | Endpoint is the HTTPS endpoint of the impersonation proxy. +| *`certificateAuthorityData`* __string__ | CertificateAuthorityData is the base64-encoded PEM CA bundle of the impersonation proxy. +|=== + + [id="{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-tokencredentialrequestapiinfo"] ==== TokenCredentialRequestAPIInfo @@ -328,7 +347,7 @@ Status of a credential issuer. |=== | Field | Description | *`server`* __string__ | Server is the Kubernetes API server URL. -| *`certificateAuthorityData`* __string__ | CertificateAuthorityData is the Kubernetes API server CA bundle. +| *`certificateAuthorityData`* __string__ | CertificateAuthorityData is the base64-encoded Kubernetes API server CA bundle. |=== diff --git a/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go index 399895015..3c0c8ba03 100644 --- a/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -1,30 +1,36 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package v1alpha1 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// +kubebuilder:validation:Enum=KubeClusterSigningCertificate +// +kubebuilder:validation:Enum=KubeClusterSigningCertificate;ImpersonationProxy type StrategyType string -// +kubebuilder:validation:Enum=TokenCredentialRequestAPI +// +kubebuilder:validation:Enum=TokenCredentialRequestAPI;ImpersonationProxy type FrontendType string // +kubebuilder:validation:Enum=Success;Error type StrategyStatus string -// +kubebuilder:validation:Enum=FetchedKey;CouldNotFetchKey +// +kubebuilder:validation:Enum=Listening;Pending;Disabled;ErrorDuringSetup;CouldNotFetchKey;CouldNotGetClusterInfo;FetchedKey type StrategyReason string const ( KubeClusterSigningCertificateStrategyType = StrategyType("KubeClusterSigningCertificate") + ImpersonationProxyStrategyType = StrategyType("ImpersonationProxy") TokenCredentialRequestAPIFrontendType = FrontendType("TokenCredentialRequestAPI") + ImpersonationProxyFrontendType = FrontendType("ImpersonationProxy") SuccessStrategyStatus = StrategyStatus("Success") ErrorStrategyStatus = StrategyStatus("Error") + ListeningStrategyReason = StrategyReason("Listening") + PendingStrategyReason = StrategyReason("Pending") + DisabledStrategyReason = StrategyReason("Disabled") + ErrorDuringSetupStrategyReason = StrategyReason("ErrorDuringSetup") CouldNotFetchKeyStrategyReason = StrategyReason("CouldNotFetchKey") CouldNotGetClusterInfoStrategyReason = StrategyReason("CouldNotGetClusterInfo") FetchedKeyStrategyReason = StrategyReason("FetchedKey") @@ -82,6 +88,10 @@ type CredentialIssuerFrontend struct { // TokenCredentialRequestAPIInfo describes the parameters for the TokenCredentialRequest API on this Concierge. // This field is only set when Type is "TokenCredentialRequestAPI". TokenCredentialRequestAPIInfo *TokenCredentialRequestAPIInfo `json:"tokenCredentialRequestInfo,omitempty"` + + // ImpersonationProxyInfo describes the parameters for the impersonation proxy on this Concierge. + // This field is only set when Type is "ImpersonationProxy". + ImpersonationProxyInfo *ImpersonationProxyInfo `json:"impersonationProxyInfo,omitempty"` } // TokenCredentialRequestAPIInfo describes the parameters for the TokenCredentialRequest API on this Concierge. @@ -91,7 +101,19 @@ type TokenCredentialRequestAPIInfo struct { // +kubebuilder:validation:Pattern=`^https://|^http://` Server string `json:"server"` - // CertificateAuthorityData is the Kubernetes API server CA bundle. + // CertificateAuthorityData is the base64-encoded Kubernetes API server CA bundle. + // +kubebuilder:validation:MinLength=1 + CertificateAuthorityData string `json:"certificateAuthorityData"` +} + +// ImpersonationProxyInfo describes the parameters for the impersonation proxy on this Concierge. +type ImpersonationProxyInfo struct { + // Endpoint is the HTTPS endpoint of the impersonation proxy. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:Pattern=`^https://` + Endpoint string `json:"endpoint"` + + // CertificateAuthorityData is the base64-encoded PEM CA bundle of the impersonation proxy. // +kubebuilder:validation:MinLength=1 CertificateAuthorityData string `json:"certificateAuthorityData"` } diff --git a/generated/1.20/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go b/generated/1.20/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go index ad9a9f66c..4a3ad605e 100644 --- a/generated/1.20/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go +++ b/generated/1.20/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go @@ -46,6 +46,11 @@ func (in *CredentialIssuerFrontend) DeepCopyInto(out *CredentialIssuerFrontend) *out = new(TokenCredentialRequestAPIInfo) **out = **in } + if in.ImpersonationProxyInfo != nil { + in, out := &in.ImpersonationProxyInfo, &out.ImpersonationProxyInfo + *out = new(ImpersonationProxyInfo) + **out = **in + } return } @@ -158,6 +163,22 @@ func (in *CredentialIssuerStrategy) DeepCopy() *CredentialIssuerStrategy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImpersonationProxyInfo) DeepCopyInto(out *ImpersonationProxyInfo) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImpersonationProxyInfo. +func (in *ImpersonationProxyInfo) DeepCopy() *ImpersonationProxyInfo { + if in == nil { + return nil + } + out := new(ImpersonationProxyInfo) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TokenCredentialRequestAPIInfo) DeepCopyInto(out *TokenCredentialRequestAPIInfo) { *out = *in diff --git a/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml index d61a78350..8123f2385 100644 --- a/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -67,14 +67,34 @@ spec: description: Frontend describes how clients can connect using this strategy. properties: + impersonationProxyInfo: + description: ImpersonationProxyInfo describes the parameters + for the impersonation proxy on this Concierge. This field + is only set when Type is "ImpersonationProxy". + properties: + certificateAuthorityData: + description: CertificateAuthorityData is the base64-encoded + PEM CA bundle of the impersonation proxy. + minLength: 1 + type: string + endpoint: + description: Endpoint is the HTTPS endpoint of the impersonation + proxy. + minLength: 1 + pattern: ^https:// + type: string + required: + - certificateAuthorityData + - endpoint + type: object tokenCredentialRequestInfo: description: TokenCredentialRequestAPIInfo describes the parameters for the TokenCredentialRequest API on this Concierge. This field is only set when Type is "TokenCredentialRequestAPI". properties: certificateAuthorityData: - description: CertificateAuthorityData is the Kubernetes - API server CA bundle. + description: CertificateAuthorityData is the base64-encoded + Kubernetes API server CA bundle. minLength: 1 type: string server: @@ -91,6 +111,7 @@ spec: can use with a strategy. enum: - TokenCredentialRequestAPI + - ImpersonationProxy type: string required: - type @@ -106,8 +127,13 @@ spec: reason: description: Reason for the current status. enum: - - FetchedKey + - Listening + - Pending + - Disabled + - ErrorDuringSetup - CouldNotFetchKey + - CouldNotGetClusterInfo + - FetchedKey type: string status: description: Status of the attempted integration strategy. @@ -119,6 +145,7 @@ spec: description: Type of integration attempted. enum: - KubeClusterSigningCertificate + - ImpersonationProxy type: string required: - lastUpdateTime diff --git a/generated/latest/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/latest/apis/concierge/config/v1alpha1/types_credentialissuer.go index 399895015..3c0c8ba03 100644 --- a/generated/latest/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/latest/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -1,30 +1,36 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package v1alpha1 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// +kubebuilder:validation:Enum=KubeClusterSigningCertificate +// +kubebuilder:validation:Enum=KubeClusterSigningCertificate;ImpersonationProxy type StrategyType string -// +kubebuilder:validation:Enum=TokenCredentialRequestAPI +// +kubebuilder:validation:Enum=TokenCredentialRequestAPI;ImpersonationProxy type FrontendType string // +kubebuilder:validation:Enum=Success;Error type StrategyStatus string -// +kubebuilder:validation:Enum=FetchedKey;CouldNotFetchKey +// +kubebuilder:validation:Enum=Listening;Pending;Disabled;ErrorDuringSetup;CouldNotFetchKey;CouldNotGetClusterInfo;FetchedKey type StrategyReason string const ( KubeClusterSigningCertificateStrategyType = StrategyType("KubeClusterSigningCertificate") + ImpersonationProxyStrategyType = StrategyType("ImpersonationProxy") TokenCredentialRequestAPIFrontendType = FrontendType("TokenCredentialRequestAPI") + ImpersonationProxyFrontendType = FrontendType("ImpersonationProxy") SuccessStrategyStatus = StrategyStatus("Success") ErrorStrategyStatus = StrategyStatus("Error") + ListeningStrategyReason = StrategyReason("Listening") + PendingStrategyReason = StrategyReason("Pending") + DisabledStrategyReason = StrategyReason("Disabled") + ErrorDuringSetupStrategyReason = StrategyReason("ErrorDuringSetup") CouldNotFetchKeyStrategyReason = StrategyReason("CouldNotFetchKey") CouldNotGetClusterInfoStrategyReason = StrategyReason("CouldNotGetClusterInfo") FetchedKeyStrategyReason = StrategyReason("FetchedKey") @@ -82,6 +88,10 @@ type CredentialIssuerFrontend struct { // TokenCredentialRequestAPIInfo describes the parameters for the TokenCredentialRequest API on this Concierge. // This field is only set when Type is "TokenCredentialRequestAPI". TokenCredentialRequestAPIInfo *TokenCredentialRequestAPIInfo `json:"tokenCredentialRequestInfo,omitempty"` + + // ImpersonationProxyInfo describes the parameters for the impersonation proxy on this Concierge. + // This field is only set when Type is "ImpersonationProxy". + ImpersonationProxyInfo *ImpersonationProxyInfo `json:"impersonationProxyInfo,omitempty"` } // TokenCredentialRequestAPIInfo describes the parameters for the TokenCredentialRequest API on this Concierge. @@ -91,7 +101,19 @@ type TokenCredentialRequestAPIInfo struct { // +kubebuilder:validation:Pattern=`^https://|^http://` Server string `json:"server"` - // CertificateAuthorityData is the Kubernetes API server CA bundle. + // CertificateAuthorityData is the base64-encoded Kubernetes API server CA bundle. + // +kubebuilder:validation:MinLength=1 + CertificateAuthorityData string `json:"certificateAuthorityData"` +} + +// ImpersonationProxyInfo describes the parameters for the impersonation proxy on this Concierge. +type ImpersonationProxyInfo struct { + // Endpoint is the HTTPS endpoint of the impersonation proxy. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:Pattern=`^https://` + Endpoint string `json:"endpoint"` + + // CertificateAuthorityData is the base64-encoded PEM CA bundle of the impersonation proxy. // +kubebuilder:validation:MinLength=1 CertificateAuthorityData string `json:"certificateAuthorityData"` } diff --git a/generated/latest/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go b/generated/latest/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go index ad9a9f66c..4a3ad605e 100644 --- a/generated/latest/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go +++ b/generated/latest/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go @@ -46,6 +46,11 @@ func (in *CredentialIssuerFrontend) DeepCopyInto(out *CredentialIssuerFrontend) *out = new(TokenCredentialRequestAPIInfo) **out = **in } + if in.ImpersonationProxyInfo != nil { + in, out := &in.ImpersonationProxyInfo, &out.ImpersonationProxyInfo + *out = new(ImpersonationProxyInfo) + **out = **in + } return } @@ -158,6 +163,22 @@ func (in *CredentialIssuerStrategy) DeepCopy() *CredentialIssuerStrategy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImpersonationProxyInfo) DeepCopyInto(out *ImpersonationProxyInfo) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImpersonationProxyInfo. +func (in *ImpersonationProxyInfo) DeepCopy() *ImpersonationProxyInfo { + if in == nil { + return nil + } + out := new(ImpersonationProxyInfo) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TokenCredentialRequestAPIInfo) DeepCopyInto(out *TokenCredentialRequestAPIInfo) { *out = *in diff --git a/go.mod b/go.mod index fbdd5ea00..290b9fd10 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/google/go-cmp v0.5.5 github.com/google/gofuzz v1.2.0 github.com/gorilla/securecookie v1.1.1 + github.com/gorilla/websocket v1.4.2 github.com/oleiade/reflections v1.0.1 // indirect github.com/onsi/ginkgo v1.13.0 // indirect github.com/ory/fosite v0.39.0 @@ -26,6 +27,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 golang.org/x/crypto v0.0.0-20201217014255-9d1352758620 + golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/sync v0.0.0-20201207232520-09787c993a3a golang.org/x/tools v0.0.0-20200825202427-b303f430e36d // indirect diff --git a/go.sum b/go.sum index f7564f36e..ce9d5089c 100644 --- a/go.sum +++ b/go.sum @@ -1181,8 +1181,9 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181003184128-c57b0facaced/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/hack/prepare-for-integration-tests.sh b/hack/prepare-for-integration-tests.sh index 44e298fdf..e317c8f94 100755 --- a/hack/prepare-for-integration-tests.sh +++ b/hack/prepare-for-integration-tests.sh @@ -116,6 +116,22 @@ check_dependency htpasswd "Please install htpasswd. Should be pre-installed on M check_dependency openssl "Please install openssl. Should be pre-installed on MacOS." check_dependency chromedriver "Please install chromedriver. e.g. 'brew install chromedriver' for MacOS" +# Check that Chrome and chromedriver versions match. If chromedriver falls a couple versions behind +# then usually tests start to fail with strange error messages. +if [[ "$OSTYPE" == "darwin"* ]]; then + chrome_version=$(/Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome --version | cut -d ' ' -f3 | cut -d '.' -f1) +else + chrome_version=$(google-chrome --version | cut -d ' ' -f3 | cut -d '.' -f1) +fi +chromedriver_version=$(chromedriver --version | cut -d ' ' -f2 | cut -d '.' -f1) +if [[ "$chrome_version" != "$chromedriver_version" ]]; then + log_error "It appears that you are using Chrome $chrome_version with chromedriver $chromedriver_version." + log_error "Please use the same version of chromedriver as Chrome." + log_error "If you are using the latest version of Chrome, then you can upgrade" + log_error "to the latest chromedriver, e.g. 'brew upgrade chromedriver' on MacOS." + exit 1 +fi + # Require kubectl >= 1.18.x if [ "$(kubectl version --client=true --short | cut -d '.' -f 2)" -lt 18 ]; then log_error "kubectl >= 1.18.x is required, you have $(kubectl version --client=true --short | cut -d ':' -f2)" @@ -292,7 +308,7 @@ kind_capabilities_file="$pinniped_path/test/cluster_capabilities/kind.yaml" pinniped_cluster_capability_file_content=$(cat "$kind_capabilities_file") cat </tmp/integration-test-env -# The following env vars should be set before running 'go test -v -count 1 ./test/integration' +# The following env vars should be set before running 'go test -v -count 1 -timeout 0 ./test/integration' export PINNIPED_TEST_CONCIERGE_NAMESPACE=${concierge_namespace} export PINNIPED_TEST_CONCIERGE_APP_NAME=${concierge_app_name} export PINNIPED_TEST_CONCIERGE_CUSTOM_LABELS='${concierge_custom_labels}' @@ -341,7 +357,7 @@ goland_vars=$(grep -v '^#' /tmp/integration-test-env | grep -E '^export .+=' | s log_note log_note "🚀 Ready to run integration tests! For example..." log_note " cd $pinniped_path" -log_note ' source /tmp/integration-test-env && go test -v -race -count 1 ./test/integration' +log_note ' source /tmp/integration-test-env && go test -v -race -count 1 -timeout 0 ./test/integration' log_note log_note 'Want to run integration tests in GoLand? Copy/paste this "Environment" value for GoLand run configurations:' log_note " ${goland_vars}PINNIPED_TEST_CLUSTER_CAPABILITY_FILE=${kind_capabilities_file}" diff --git a/hack/prepare-impersonator-on-kind.sh b/hack/prepare-impersonator-on-kind.sh new file mode 100755 index 000000000..aae4f4599 --- /dev/null +++ b/hack/prepare-impersonator-on-kind.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash + +# A script to perform the setup required to manually test using the impersonation proxy on a kind cluster. +# Assumes that you installed the apps already using hack/prepare-for-integration-tests.sh. + +set -euo pipefail + +# The name of the namespace in which the concierge is installed. +CONCIERGE_NAMESPACE=concierge +# The name of the concierge app's Deployment. +CONCIERGE_DEPLOYMENT=pinniped-concierge +# The namespace in which the local-user-authenticator app is installed. +LOCAL_USER_AUTHENTICATOR_NAMESPACE=local-user-authenticator +# The port on which the impersonation proxy runs in the concierge pods. +IMPERSONATION_PROXY_PORT=8444 +# The port that we will use to access the impersonator from outside the cluster via `kubectl port-forward`. +LOCAL_PORT=8777 +LOCAL_HOST="127.0.0.1:${LOCAL_PORT}" + +# Change working directory to the top of the repo. +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +# Build the CLI for use later in the script. +go build ./cmd/pinniped + +# Create a test user and password. +if ! kubectl get secret pinny-the-seal --namespace $LOCAL_USER_AUTHENTICATOR_NAMESPACE; then + kubectl create secret generic pinny-the-seal --namespace $LOCAL_USER_AUTHENTICATOR_NAMESPACE \ + --from-literal=groups=group1,group2 \ + --from-literal=passwordHash="$(htpasswd -nbBC 10 x password123 | sed -e "s/^x://")" +fi + +# Get the CA of the local-user-authenticator. +LOCAL_USER_AUTHENTICATOR_CA=$(kubectl get secret local-user-authenticator-tls-serving-certificate \ + --namespace $LOCAL_USER_AUTHENTICATOR_NAMESPACE \ + -o jsonpath=\{.data.caCertificate\}) + +# Create a WebhookAuthenticator which points at the local-user-authenticator. +cat < /dev/null || true +} +trap cleanup EXIT + +# Get a working kubeconfig that will send requests through the impersonation proxy. +./pinniped get kubeconfig \ + --static-token "pinny-the-seal:password123" \ + --concierge-mode ImpersonationProxy >/tmp/kubeconfig + +echo +echo 'Ready. In another tab, use "kubectl --kubeconfig /tmp/kubeconfig " to make requests through the impersonation proxy.' +echo "When done, cancel with ctrl-C to clean up." +wait $port_forward_pid diff --git a/internal/certauthority/certauthority.go b/internal/certauthority/certauthority.go index 87bdd7845..60aed89fc 100644 --- a/internal/certauthority/certauthority.go +++ b/internal/certauthority/certauthority.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 // Package certauthority implements a simple x509 certificate authority suitable for use in an aggregated API service. @@ -19,6 +19,8 @@ import ( "math/big" "net" "time" + + "go.pinniped.dev/internal/constable" ) // certBackdate is the amount of time before time.Now() that will be used to set @@ -44,12 +46,17 @@ type env struct { // CA holds the state for a simple x509 certificate authority suitable for use in an aggregated API service. type CA struct { - // caCert is the DER-encoded certificate for the current CA. + // caCertBytes is the DER-encoded certificate for the current CA. caCertBytes []byte // signer is the private key for the current CA. signer crypto.Signer + // privateKey is the same private key represented by signer, but in a format which allows export. + // It is only set by New, not by Load, since Load can handle various types of PrivateKey but New + // only needs to create keys of type ecdsa.PrivateKey. + privateKey *ecdsa.PrivateKey + // env is our reference to the outside world (clocks and random number generation). env env } @@ -66,7 +73,7 @@ func secureEnv() env { } // ErrInvalidCACertificate is returned when the contents of the loaded CA certificate do not meet our assumptions. -var ErrInvalidCACertificate = fmt.Errorf("invalid CA certificate") +const ErrInvalidCACertificate = constable.Error("invalid CA certificate") // Load a certificate authority from an existing certificate and private key (in PEM format). func Load(certPEM string, keyPEM string) (*CA, error) { @@ -77,6 +84,13 @@ func Load(certPEM string, keyPEM string) (*CA, error) { if certCount := len(cert.Certificate); certCount != 1 { return nil, fmt.Errorf("%w: expected a single certificate, found %d certificates", ErrInvalidCACertificate, certCount) } + x509Cert, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + return nil, fmt.Errorf("failed to parse key pair as x509 cert: %w", err) + } + if !x509Cert.IsCA { + return nil, fmt.Errorf("%w: passed in key pair is not a CA", ErrInvalidCACertificate) + } return &CA{ caCertBytes: cert.Certificate[0], signer: cert.PrivateKey.(crypto.Signer), @@ -84,13 +98,13 @@ func Load(certPEM string, keyPEM string) (*CA, error) { }, nil } -// New generates a fresh certificate authority with the given subject and ttl. -func New(subject pkix.Name, ttl time.Duration) (*CA, error) { - return newInternal(subject, ttl, secureEnv()) +// New generates a fresh certificate authority with the given Common Name and TTL. +func New(commonName string, ttl time.Duration) (*CA, error) { + return newInternal(commonName, ttl, secureEnv()) } // newInternal is the internal guts of New, broken out for easier testing. -func newInternal(subject pkix.Name, ttl time.Duration, env env) (*CA, error) { +func newInternal(commonName string, ttl time.Duration, env env) (*CA, error) { ca := CA{env: env} // Generate a random serial for the CA serialNumber, err := randomSerial(env.serialRNG) @@ -99,11 +113,11 @@ func newInternal(subject pkix.Name, ttl time.Duration, env env) (*CA, error) { } // Generate a new P256 keypair. - privateKey, err := ecdsa.GenerateKey(elliptic.P256(), env.keygenRNG) + ca.privateKey, err = ecdsa.GenerateKey(elliptic.P256(), env.keygenRNG) if err != nil { return nil, fmt.Errorf("could not generate CA private key: %w", err) } - ca.signer = privateKey + ca.signer = ca.privateKey // Make a CA certificate valid for some ttl and backdated by some amount. now := env.clock() @@ -113,7 +127,7 @@ func newInternal(subject pkix.Name, ttl time.Duration, env env) (*CA, error) { // Create CA cert template caTemplate := x509.Certificate{ SerialNumber: serialNumber, - Subject: subject, + Subject: pkix.Name{CommonName: commonName}, NotBefore: notBefore, NotAfter: notAfter, IsCA: true, @@ -123,7 +137,7 @@ func newInternal(subject pkix.Name, ttl time.Duration, env env) (*CA, error) { } // Self-sign the CA to get the DER certificate. - caCertBytes, err := x509.CreateCertificate(env.signingRNG, &caTemplate, &caTemplate, &privateKey.PublicKey, privateKey) + caCertBytes, err := x509.CreateCertificate(env.signingRNG, &caTemplate, &caTemplate, &ca.privateKey.PublicKey, ca.privateKey) if err != nil { return nil, fmt.Errorf("could not issue CA certificate: %w", err) } @@ -136,6 +150,18 @@ func (c *CA) Bundle() []byte { return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: c.caCertBytes}) } +// PrivateKeyToPEM returns the current CA private key in PEM format, if this CA was constructed by New. +func (c *CA) PrivateKeyToPEM() ([]byte, error) { + if c.privateKey == nil { + return nil, fmt.Errorf("no private key data (did you try to use this after Load?)") + } + derKey, err := x509.MarshalECPrivateKey(c.privateKey) + if err != nil { + return nil, err + } + return pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: derKey}), nil +} + // Pool returns the current CA signing bundle as a *x509.CertPool. func (c *CA) Pool() *x509.CertPool { pool := x509.NewCertPool() @@ -143,8 +169,31 @@ func (c *CA) Pool() *x509.CertPool { return pool } -// Issue a new server certificate for the given identity and duration. -func (c *CA) Issue(subject pkix.Name, dnsNames []string, ips []net.IP, ttl time.Duration) (*tls.Certificate, error) { +// IssueClientCert issues a new client certificate with username and groups included in the Kube-style +// certificate subject for the given identity and duration. +func (c *CA) IssueClientCert(username string, groups []string, ttl time.Duration) (*tls.Certificate, error) { + return c.issueCert(x509.ExtKeyUsageClientAuth, pkix.Name{CommonName: username, Organization: groups}, nil, nil, ttl) +} + +// IssueServerCert issues a new server certificate for the given identity and duration. +// The dnsNames and ips are each optional, but at least one of them should be specified. +func (c *CA) IssueServerCert(dnsNames []string, ips []net.IP, ttl time.Duration) (*tls.Certificate, error) { + return c.issueCert(x509.ExtKeyUsageServerAuth, pkix.Name{}, dnsNames, ips, ttl) +} + +// Similar to IssueClientCert, but returning the new cert as a pair of PEM-formatted byte slices +// for the certificate and private key. +func (c *CA) IssueClientCertPEM(username string, groups []string, ttl time.Duration) ([]byte, []byte, error) { + return toPEM(c.IssueClientCert(username, groups, ttl)) +} + +// Similar to IssueServerCert, but returning the new cert as a pair of PEM-formatted byte slices +// for the certificate and private key. +func (c *CA) IssueServerCertPEM(dnsNames []string, ips []net.IP, ttl time.Duration) ([]byte, []byte, error) { + return toPEM(c.IssueServerCert(dnsNames, ips, ttl)) +} + +func (c *CA) issueCert(extKeyUsage x509.ExtKeyUsage, subject pkix.Name, dnsNames []string, ips []net.IP, ttl time.Duration) (*tls.Certificate, error) { // Choose a random 128 bit serial number. serialNumber, err := randomSerial(c.env.serialRNG) if err != nil { @@ -174,8 +223,7 @@ func (c *CA) Issue(subject pkix.Name, dnsNames []string, ips []net.IP, ttl time. Subject: subject, NotBefore: notBefore, NotAfter: notAfter, - KeyUsage: x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + ExtKeyUsage: []x509.ExtKeyUsage{extKeyUsage}, BasicConstraintsValid: true, IsCA: false, DNSNames: dnsNames, @@ -200,14 +248,8 @@ func (c *CA) Issue(subject pkix.Name, dnsNames []string, ips []net.IP, ttl time. }, nil } -// IssuePEM issues a new server certificate for the given identity and duration, returning it as a pair of -// PEM-formatted byte slices for the certificate and private key. -func (c *CA) IssuePEM(subject pkix.Name, dnsNames []string, ttl time.Duration) ([]byte, []byte, error) { - return toPEM(c.Issue(subject, dnsNames, nil, ttl)) -} - func toPEM(cert *tls.Certificate, err error) ([]byte, []byte, error) { - // If the wrapped Issue() returned an error, pass it back. + // If the wrapped IssueServerCert() returned an error, pass it back. if err != nil { return nil, nil, err } diff --git a/internal/certauthority/certauthority_test.go b/internal/certauthority/certauthority_test.go index 4c1fdf8ed..80948a779 100644 --- a/internal/certauthority/certauthority_test.go +++ b/internal/certauthority/certauthority_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package certauthority @@ -7,7 +7,6 @@ import ( "crypto" "crypto/tls" "crypto/x509" - "crypto/x509/pkix" "fmt" "io" "io/ioutil" @@ -17,6 +16,8 @@ import ( "time" "github.com/stretchr/testify/require" + + "go.pinniped.dev/internal/testutil" ) func loadFromFiles(t *testing.T, certPath string, keyPath string) (*CA, error) { @@ -80,22 +81,25 @@ func TestLoad(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, ca.caCertBytes) require.NotNil(t, ca.signer) + require.Nil(t, ca.privateKey) // this struct field is only used for CA's created by New() }) } } func TestNew(t *testing.T) { now := time.Now() - got, err := New(pkix.Name{CommonName: "Test CA"}, time.Minute) + ca, err := New("Test CA", time.Minute) require.NoError(t, err) - require.NotNil(t, got) + require.NotNil(t, ca) // Make sure the CA certificate looks roughly like what we expect. - caCert, err := x509.ParseCertificate(got.caCertBytes) + caCert, err := x509.ParseCertificate(ca.caCertBytes) require.NoError(t, err) require.Equal(t, "Test CA", caCert.Subject.CommonName) require.WithinDuration(t, now.Add(-10*time.Second), caCert.NotBefore, 10*time.Second) require.WithinDuration(t, now.Add(time.Minute), caCert.NotAfter, 10*time.Second) + + require.NotNil(t, ca.privateKey) } func TestNewInternal(t *testing.T) { @@ -155,7 +159,7 @@ func TestNewInternal(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - got, err := newInternal(pkix.Name{CommonName: "Test CA"}, tt.ttl, tt.env) + got, err := newInternal("Test CA", tt.ttl, tt.env) if tt.wantErr != "" { require.EqualError(t, err, tt.wantErr) require.Nil(t, got) @@ -175,21 +179,34 @@ func TestNewInternal(t *testing.T) { } func TestBundle(t *testing.T) { - t.Run("success", func(t *testing.T) { - ca := CA{caCertBytes: []byte{1, 2, 3, 4, 5, 6, 7, 8}} - got := ca.Bundle() - require.Equal(t, "-----BEGIN CERTIFICATE-----\nAQIDBAUGBwg=\n-----END CERTIFICATE-----\n", string(got)) - }) + ca := CA{caCertBytes: []byte{1, 2, 3, 4, 5, 6, 7, 8}} + certPEM := ca.Bundle() + require.Equal(t, "-----BEGIN CERTIFICATE-----\nAQIDBAUGBwg=\n-----END CERTIFICATE-----\n", string(certPEM)) +} + +func TestPrivateKeyToPEM(t *testing.T) { + ca, err := New("Test CA", time.Hour) + require.NoError(t, err) + keyPEM, err := ca.PrivateKeyToPEM() + require.NoError(t, err) + require.Regexp(t, "(?s)-----BEGIN EC "+"PRIVATE KEY-----\n.*\n-----END EC PRIVATE KEY-----", string(keyPEM)) + certPEM := ca.Bundle() + // Check that the public and private keys work together. + _, err = tls.X509KeyPair(certPEM, keyPEM) + require.NoError(t, err) + + reloaded, err := Load(string(certPEM), string(keyPEM)) + require.NoError(t, err) + _, err = reloaded.PrivateKeyToPEM() + require.EqualError(t, err, "no private key data (did you try to use this after Load?)") } func TestPool(t *testing.T) { - t.Run("success", func(t *testing.T) { - ca, err := New(pkix.Name{CommonName: "test"}, 1*time.Hour) - require.NoError(t, err) + ca, err := New("test", 1*time.Hour) + require.NoError(t, err) - got := ca.Pool() - require.Len(t, got.Subjects(), 1) - }) + pool := ca.Pool() + require.Len(t, pool.Subjects(), 1) } type errSigner struct { @@ -204,6 +221,8 @@ func (e *errSigner) Sign(_ io.Reader, _ []byte, _ crypto.SignerOpts) ([]byte, er } func TestIssue(t *testing.T) { + const numRandBytes = 64 * 2 // each call to issue a cert will consume 64 bytes from the reader + now := time.Date(2020, 7, 10, 12, 41, 12, 1234, time.UTC) realCA, err := loadFromFiles(t, "./testdata/test.crt", "./testdata/test.key") @@ -227,7 +246,7 @@ func TestIssue(t *testing.T) { name: "failed to generate keypair", ca: CA{ env: env{ - serialRNG: strings.NewReader(strings.Repeat("x", 64)), + serialRNG: strings.NewReader(strings.Repeat("x", numRandBytes)), keygenRNG: strings.NewReader(""), }, }, @@ -237,8 +256,8 @@ func TestIssue(t *testing.T) { name: "invalid CA certificate", ca: CA{ env: env{ - serialRNG: strings.NewReader(strings.Repeat("x", 64)), - keygenRNG: strings.NewReader(strings.Repeat("x", 64)), + serialRNG: strings.NewReader(strings.Repeat("x", numRandBytes)), + keygenRNG: strings.NewReader(strings.Repeat("x", numRandBytes)), clock: func() time.Time { return now }, }, }, @@ -248,8 +267,8 @@ func TestIssue(t *testing.T) { name: "signing error", ca: CA{ env: env{ - serialRNG: strings.NewReader(strings.Repeat("x", 64)), - keygenRNG: strings.NewReader(strings.Repeat("x", 64)), + serialRNG: strings.NewReader(strings.Repeat("x", numRandBytes)), + keygenRNG: strings.NewReader(strings.Repeat("x", numRandBytes)), clock: func() time.Time { return now }, }, caCertBytes: realCA.caCertBytes, @@ -261,11 +280,11 @@ func TestIssue(t *testing.T) { wantErr: "could not sign certificate: some signer error", }, { - name: "success", + name: "parse certificate error", ca: CA{ env: env{ - serialRNG: strings.NewReader(strings.Repeat("x", 64)), - keygenRNG: strings.NewReader(strings.Repeat("x", 64)), + serialRNG: strings.NewReader(strings.Repeat("x", numRandBytes)), + keygenRNG: strings.NewReader(strings.Repeat("x", numRandBytes)), clock: func() time.Time { return now }, parseCert: func(_ []byte) (*x509.Certificate, error) { return nil, fmt.Errorf("some parse certificate error") @@ -280,8 +299,8 @@ func TestIssue(t *testing.T) { name: "success", ca: CA{ env: env{ - serialRNG: strings.NewReader(strings.Repeat("x", 64)), - keygenRNG: strings.NewReader(strings.Repeat("x", 64)), + serialRNG: strings.NewReader(strings.Repeat("x", numRandBytes)), + keygenRNG: strings.NewReader(strings.Repeat("x", numRandBytes)), clock: func() time.Time { return now }, parseCert: x509.ParseCertificate, }, @@ -293,28 +312,26 @@ func TestIssue(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - got, err := tt.ca.Issue(pkix.Name{CommonName: "Test Server"}, []string{"example.com"}, []net.IP{net.IPv4(1, 2, 3, 4)}, 10*time.Minute) + got, err := tt.ca.IssueServerCert([]string{"example.com"}, []net.IP{net.IPv4(1, 2, 3, 4)}, 10*time.Minute) if tt.wantErr != "" { require.EqualError(t, err, tt.wantErr) require.Nil(t, got) - return + } else { + require.NoError(t, err) + require.NotNil(t, got) + } + got, err = tt.ca.IssueClientCert("test-user", []string{"group1", "group2"}, 10*time.Minute) + if tt.wantErr != "" { + require.EqualError(t, err, tt.wantErr) + require.Nil(t, got) + } else { + require.NoError(t, err) + require.NotNil(t, got) } - require.NoError(t, err) - require.NotNil(t, got) }) } } -func TestIssuePEM(t *testing.T) { - realCA, err := loadFromFiles(t, "./testdata/test.crt", "./testdata/test.key") - require.NoError(t, err) - - certPEM, keyPEM, err := realCA.IssuePEM(pkix.Name{CommonName: "Test Server"}, []string{"example.com"}, 10*time.Minute) - require.NoError(t, err) - require.NotEmpty(t, certPEM) - require.NotEmpty(t, keyPEM) -} - func TestToPEM(t *testing.T) { realCert, err := tls.LoadX509KeyPair("./testdata/test.crt", "./testdata/test.key") require.NoError(t, err) @@ -342,3 +359,90 @@ func TestToPEM(t *testing.T) { require.NotEmpty(t, keyPEM) }) } + +func TestIssueMethods(t *testing.T) { + // One CA can be used to issue both kinds of certs. + ca, err := New("Test CA", time.Hour) + require.NoError(t, err) + + ttl := 121 * time.Hour + + t.Run("client certs", func(t *testing.T) { + user := "test-username" + groups := []string{"group1", "group2"} + + clientCert, err := ca.IssueClientCert(user, groups, ttl) + require.NoError(t, err) + certPEM, keyPEM, err := ToPEM(clientCert) + require.NoError(t, err) + validateClientCert(t, ca.Bundle(), certPEM, keyPEM, user, groups, ttl) + + certPEM, keyPEM, err = ca.IssueClientCertPEM(user, groups, ttl) + require.NoError(t, err) + validateClientCert(t, ca.Bundle(), certPEM, keyPEM, user, groups, ttl) + + certPEM, keyPEM, err = ca.IssueClientCertPEM(user, nil, ttl) + require.NoError(t, err) + validateClientCert(t, ca.Bundle(), certPEM, keyPEM, user, nil, ttl) + + certPEM, keyPEM, err = ca.IssueClientCertPEM(user, []string{}, ttl) + require.NoError(t, err) + validateClientCert(t, ca.Bundle(), certPEM, keyPEM, user, nil, ttl) + + certPEM, keyPEM, err = ca.IssueClientCertPEM("", []string{}, ttl) + require.NoError(t, err) + validateClientCert(t, ca.Bundle(), certPEM, keyPEM, "", nil, ttl) + }) + + t.Run("server certs", func(t *testing.T) { + dnsNames := []string{"example.com", "pinniped.dev"} + ips := []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("1.2.3.4")} + + serverCert, err := ca.IssueServerCert(dnsNames, ips, ttl) + require.NoError(t, err) + certPEM, keyPEM, err := ToPEM(serverCert) + require.NoError(t, err) + validateServerCert(t, ca.Bundle(), certPEM, keyPEM, dnsNames, ips, ttl) + + certPEM, keyPEM, err = ca.IssueServerCertPEM(dnsNames, ips, ttl) + require.NoError(t, err) + validateServerCert(t, ca.Bundle(), certPEM, keyPEM, dnsNames, ips, ttl) + + certPEM, keyPEM, err = ca.IssueServerCertPEM(nil, ips, ttl) + require.NoError(t, err) + validateServerCert(t, ca.Bundle(), certPEM, keyPEM, nil, ips, ttl) + + certPEM, keyPEM, err = ca.IssueServerCertPEM(dnsNames, nil, ttl) + require.NoError(t, err) + validateServerCert(t, ca.Bundle(), certPEM, keyPEM, dnsNames, nil, ttl) + + certPEM, keyPEM, err = ca.IssueServerCertPEM([]string{}, ips, ttl) + require.NoError(t, err) + validateServerCert(t, ca.Bundle(), certPEM, keyPEM, nil, ips, ttl) + + certPEM, keyPEM, err = ca.IssueServerCertPEM(dnsNames, []net.IP{}, ttl) + require.NoError(t, err) + validateServerCert(t, ca.Bundle(), certPEM, keyPEM, dnsNames, nil, ttl) + }) +} + +func validateClientCert(t *testing.T, caBundle []byte, certPEM []byte, keyPEM []byte, expectedUser string, expectedGroups []string, expectedTTL time.Duration) { + const fudgeFactor = 10 * time.Second + v := testutil.ValidateClientCertificate(t, string(caBundle), string(certPEM)) + v.RequireLifetime(time.Now(), time.Now().Add(expectedTTL), certBackdate+fudgeFactor) + v.RequireMatchesPrivateKey(string(keyPEM)) + v.RequireCommonName(expectedUser) + v.RequireOrganizations(expectedGroups) + v.RequireEmptyDNSNames() + v.RequireEmptyIPs() +} + +func validateServerCert(t *testing.T, caBundle []byte, certPEM []byte, keyPEM []byte, expectedDNSNames []string, expectedIPs []net.IP, expectedTTL time.Duration) { + const fudgeFactor = 10 * time.Second + v := testutil.ValidateServerCertificate(t, string(caBundle), string(certPEM)) + v.RequireLifetime(time.Now(), time.Now().Add(expectedTTL), certBackdate+fudgeFactor) + v.RequireMatchesPrivateKey(string(keyPEM)) + v.RequireCommonName("") + v.RequireDNSNames(expectedDNSNames) + v.RequireIPs(expectedIPs) +} diff --git a/internal/certauthority/dynamiccertauthority/dynamiccertauthority.go b/internal/certauthority/dynamiccertauthority/dynamiccertauthority.go index a2c978983..42dbf7d5a 100644 --- a/internal/certauthority/dynamiccertauthority/dynamiccertauthority.go +++ b/internal/certauthority/dynamiccertauthority/dynamiccertauthority.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 // Package dynamiccertauthority implements a x509 certificate authority capable of issuing @@ -6,34 +6,42 @@ package dynamiccertauthority import ( - "crypto/x509/pkix" "time" + "k8s.io/apiserver/pkg/server/dynamiccertificates" + "go.pinniped.dev/internal/certauthority" - "go.pinniped.dev/internal/dynamiccert" + "go.pinniped.dev/internal/issuer" ) -// CA is a type capable of issuing certificates. -type CA struct { - provider dynamiccert.Provider +// ca is a type capable of issuing certificates. +type ca struct { + provider dynamiccertificates.CertKeyContentProvider } -// New creates a new CA, ready to issue certs whenever the provided provider has a keypair to -// provide. -func New(provider dynamiccert.Provider) *CA { - return &CA{ +// New creates a ClientCertIssuer, ready to issue certs whenever +// the given CertKeyContentProvider has a keypair to provide. +func New(provider dynamiccertificates.CertKeyContentProvider) issuer.ClientCertIssuer { + return &ca{ provider: provider, } } -// IssuePEM issues a new server certificate for the given identity and duration, returning it as a +func (c *ca) Name() string { + return c.provider.Name() +} + +// IssueClientCertPEM issues a new client certificate for the given identity and duration, returning it as a // pair of PEM-formatted byte slices for the certificate and private key. -func (c *CA) IssuePEM(subject pkix.Name, dnsNames []string, ttl time.Duration) ([]byte, []byte, error) { +func (c *ca) IssueClientCertPEM(username string, groups []string, ttl time.Duration) ([]byte, []byte, error) { caCrtPEM, caKeyPEM := c.provider.CurrentCertKeyContent() + // in the future we could split dynamiccert.Private into two interfaces (Private and PrivateRead) + // and have this code take PrivateRead as input. We would then add ourselves as a listener to + // the PrivateRead. This would allow us to only reload the CA contents when they actually change. ca, err := certauthority.Load(string(caCrtPEM), string(caKeyPEM)) if err != nil { return nil, nil, err } - return ca.IssuePEM(subject, dnsNames, ttl) + return ca.IssueClientCertPEM(username, groups, ttl) } diff --git a/internal/certauthority/dynamiccertauthority/dynamiccertauthority_test.go b/internal/certauthority/dynamiccertauthority/dynamiccertauthority_test.go index 5482af71d..b33cb9dd0 100644 --- a/internal/certauthority/dynamiccertauthority/dynamiccertauthority_test.go +++ b/internal/certauthority/dynamiccertauthority/dynamiccertauthority_test.go @@ -1,23 +1,23 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package dynamiccertauthority import ( - "crypto/x509/pkix" "testing" "time" "github.com/stretchr/testify/require" "go.pinniped.dev/internal/dynamiccert" + "go.pinniped.dev/internal/issuer" "go.pinniped.dev/internal/testutil" ) func TestCAIssuePEM(t *testing.T) { t.Parallel() - provider := dynamiccert.New() + provider := dynamiccert.NewCA(t.Name()) ca := New(provider) goodCACrtPEM0, goodCAKeyPEM0, err := testutil.CreateCertificate( @@ -44,12 +44,12 @@ func TestCAIssuePEM(t *testing.T) { { name: "only cert", caCrtPEM: goodCACrtPEM0, - wantError: "could not load CA: tls: failed to find any PEM data in key input", + wantError: "TestCAIssuePEM: attempt to set invalid key pair: tls: failed to find any PEM data in key input", }, { name: "only key", caKeyPEM: goodCAKeyPEM0, - wantError: "could not load CA: tls: failed to find any PEM data in certificate input", + wantError: "TestCAIssuePEM: attempt to set invalid key pair: tls: failed to find any PEM data in certificate input", }, { name: "new cert+key", @@ -68,19 +68,19 @@ func TestCAIssuePEM(t *testing.T) { name: "bad cert", caCrtPEM: []byte("this is not a cert"), caKeyPEM: goodCAKeyPEM0, - wantError: "could not load CA: tls: failed to find any PEM data in certificate input", + wantError: "TestCAIssuePEM: attempt to set invalid key pair: tls: failed to find any PEM data in certificate input", }, { name: "bad key", caCrtPEM: goodCACrtPEM0, caKeyPEM: []byte("this is not a key"), - wantError: "could not load CA: tls: failed to find any PEM data in key input", + wantError: "TestCAIssuePEM: attempt to set invalid key pair: tls: failed to find any PEM data in key input", }, { name: "mismatch cert+key", caCrtPEM: goodCACrtPEM0, caKeyPEM: goodCAKeyPEM1, - wantError: "could not load CA: tls: private key does not match public key", + wantError: "TestCAIssuePEM: attempt to set invalid key pair: tls: private key does not match public key", }, { name: "good cert+key again", @@ -94,17 +94,7 @@ func TestCAIssuePEM(t *testing.T) { // Can't run these steps in parallel, because each one depends on the previous steps being // run. - if step.caCrtPEM != nil || step.caKeyPEM != nil { - provider.Set(step.caCrtPEM, step.caKeyPEM) - } - - crtPEM, keyPEM, err := ca.IssuePEM( - pkix.Name{ - CommonName: "some-common-name", - }, - []string{"some-dns-name", "some-other-dns-name"}, - time.Hour*24, - ) + crtPEM, keyPEM, err := issuePEM(provider, ca, step.caCrtPEM, step.caKeyPEM) if step.wantError != "" { require.EqualError(t, err, step.wantError) @@ -116,13 +106,24 @@ func TestCAIssuePEM(t *testing.T) { require.NotEmpty(t, keyPEM) caCrtPEM, _ := provider.CurrentCertKeyContent() - crtAssertions := testutil.ValidateCertificate(t, string(caCrtPEM), string(crtPEM)) - crtAssertions.RequireCommonName("some-common-name") - crtAssertions.RequireDNSName("some-dns-name") - crtAssertions.RequireDNSName("some-other-dns-name") + crtAssertions := testutil.ValidateClientCertificate(t, string(caCrtPEM), string(crtPEM)) + crtAssertions.RequireCommonName("some-username") + crtAssertions.RequireOrganizations([]string{"some-group1", "some-group2"}) crtAssertions.RequireLifetime(time.Now(), time.Now().Add(time.Hour*24), time.Minute*10) crtAssertions.RequireMatchesPrivateKey(string(keyPEM)) } }) } } + +func issuePEM(provider dynamiccert.Provider, ca issuer.ClientCertIssuer, caCrt, caKey []byte) ([]byte, []byte, error) { + // if setting fails, look at that error + if caCrt != nil || caKey != nil { + if err := provider.SetCertKeyContent(caCrt, caKey); err != nil { + return nil, nil, err + } + } + + // otherwise check to see if their is an issuing error + return ca.IssueClientCertPEM("some-username", []string{"some-group1", "some-group2"}, time.Hour*24) +} diff --git a/internal/clusterhost/clusterhost.go b/internal/clusterhost/clusterhost.go new file mode 100644 index 000000000..bfdbd6c79 --- /dev/null +++ b/internal/clusterhost/clusterhost.go @@ -0,0 +1,63 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package clusterhost + +import ( + "context" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + labelNodeRolePrefix = "node-role.kubernetes.io/" + nodeLabelRole = "kubernetes.io/node-role" + controlPlaneNodeRole = "control-plane" + // this role was deprecated by kubernetes 1.20. + masterNodeRole = "master" +) + +type ClusterHost struct { + client kubernetes.Interface +} + +func New(client kubernetes.Interface) *ClusterHost { + return &ClusterHost{client: client} +} + +func (c *ClusterHost) HasControlPlaneNodes(ctx context.Context) (bool, error) { + nodes, err := c.client.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + return false, fmt.Errorf("error fetching nodes: %v", err) + } + if len(nodes.Items) == 0 { + return false, fmt.Errorf("no nodes found") + } + for _, node := range nodes.Items { + for k, v := range node.Labels { + if isControlPlaneNodeRole(k, v) { + return true, nil + } + } + } + + return false, nil +} + +func isControlPlaneNodeRole(k string, v string) bool { + if k == labelNodeRolePrefix+controlPlaneNodeRole { + return true + } + if k == labelNodeRolePrefix+masterNodeRole { + return true + } + if k == nodeLabelRole && v == controlPlaneNodeRole { + return true + } + if k == nodeLabelRole && v == masterNodeRole { + return true + } + return false +} diff --git a/internal/clusterhost/clusterhost_test.go b/internal/clusterhost/clusterhost_test.go new file mode 100644 index 000000000..114050967 --- /dev/null +++ b/internal/clusterhost/clusterhost_test.go @@ -0,0 +1,169 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package clusterhost + +import ( + "context" + "errors" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime" + kubernetesfake "k8s.io/client-go/kubernetes/fake" + coretesting "k8s.io/client-go/testing" + + v1 "k8s.io/api/core/v1" +) + +func TestHasControlPlaneNodes(t *testing.T) { + tests := []struct { + name string + nodes []*v1.Node + listNodesErr error + wantErr error + wantReturnValue bool + }{ + { + name: "Fetching nodes returns an error", + listNodesErr: errors.New("couldn't get nodes"), + wantErr: errors.New("error fetching nodes: couldn't get nodes"), + }, + { + name: "Fetching nodes returns an empty array", + nodes: []*v1.Node{}, + wantErr: errors.New("no nodes found"), + }, + { + name: "Nodes found, but not control plane nodes", + nodes: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + Labels: map[string]string{ + "not-control-plane-label": "some-value", + "kubernetes.io/node-role": "worker", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-2", + Labels: map[string]string{"node-role.kubernetes.io/worker": ""}, + }, + }, + }, + wantReturnValue: false, + }, + { + name: "Nodes found, including a control-plane role in node-role.kubernetes.io/ format", + nodes: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + Labels: map[string]string{"unrelated-label": "some-value"}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-2", + Labels: map[string]string{ + "some-other-label": "some-value", + "node-role.kubernetes.io/control-plane": "", + }, + }, + }, + }, + wantReturnValue: true, + }, + { + name: "Nodes found, including a master role in node-role.kubernetes.io/ format", + nodes: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + Labels: map[string]string{"unrelated-label": "some-value"}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-2", + Labels: map[string]string{ + "some-other-label": "some-value", + "node-role.kubernetes.io/master": "", + }, + }, + }, + }, + wantReturnValue: true, + }, + { + name: "Nodes found, including a control-plane role in kubernetes.io/node-role= format", + nodes: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + Labels: map[string]string{"unrelated-label": "some-value"}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-2", + Labels: map[string]string{ + "some-other-label": "some-value", + "kubernetes.io/node-role": "control-plane", + }, + }, + }, + }, + wantReturnValue: true, + }, + { + name: "Nodes found, including a master role in kubernetes.io/node-role= format", + nodes: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + Labels: map[string]string{"unrelated-label": "some-value"}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-2", + Labels: map[string]string{ + "some-other-label": "some-value", + "kubernetes.io/node-role": "master", + }, + }, + }, + }, + wantReturnValue: true, + }, + } + for _, tt := range tests { + test := tt + t.Run(test.name, func(t *testing.T) { + kubeClient := kubernetesfake.NewSimpleClientset() + if test.listNodesErr != nil { + listNodesErr := test.listNodesErr + kubeClient.PrependReactor( + "list", + "nodes", + func(_ coretesting.Action) (bool, runtime.Object, error) { + return true, nil, listNodesErr + }, + ) + } + for _, node := range test.nodes { + err := kubeClient.Tracker().Add(node) + require.NoError(t, err) + } + clusterHost := New(kubeClient) + hasControlPlaneNodes, err := clusterHost.HasControlPlaneNodes(context.Background()) + require.Equal(t, test.wantErr, err) + require.Equal(t, test.wantReturnValue, hasControlPlaneNodes) + }) + } +} diff --git a/internal/concierge/apiserver/apiserver.go b/internal/concierge/apiserver/apiserver.go index e5fc8da9e..f1bce95b3 100644 --- a/internal/concierge/apiserver/apiserver.go +++ b/internal/concierge/apiserver/apiserver.go @@ -15,6 +15,7 @@ import ( genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/client-go/pkg/version" + "go.pinniped.dev/internal/issuer" "go.pinniped.dev/internal/plog" "go.pinniped.dev/internal/registry/credentialrequest" "go.pinniped.dev/internal/registry/whoamirequest" @@ -27,7 +28,7 @@ type Config struct { type ExtraConfig struct { Authenticator credentialrequest.TokenCredentialRequestAuthenticator - Issuer credentialrequest.CertIssuer + Issuer issuer.ClientCertIssuer StartControllersPostStartHook func(ctx context.Context) Scheme *runtime.Scheme NegotiatedSerializer runtime.NegotiatedSerializer diff --git a/internal/concierge/impersonator/config.go b/internal/concierge/impersonator/config.go new file mode 100644 index 000000000..60ab59bd6 --- /dev/null +++ b/internal/concierge/impersonator/config.go @@ -0,0 +1,65 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package impersonator + +import ( + "fmt" + + v1 "k8s.io/api/core/v1" + "sigs.k8s.io/yaml" +) + +type Mode string + +const ( + // Explicitly enable the impersonation proxy. + ModeEnabled Mode = "enabled" + + // Explicitly disable the impersonation proxy. + ModeDisabled Mode = "disabled" + + // Allow the proxy to decide if it should be enabled or disabled based upon the cluster in which it is running. + ModeAuto Mode = "auto" +) + +const ( + ConfigMapDataKey = "config.yaml" +) + +type Config struct { + // Enable or disable the impersonation proxy. Optional. Defaults to ModeAuto. + Mode Mode `json:"mode,omitempty"` + + // Used when creating TLS certificates and for clients to discover the endpoint. Optional. When not specified, if the + // impersonation proxy is started, then it will automatically create a LoadBalancer Service and use its ingress as the + // endpoint. + // + // When specified, it may be a hostname or IP address, optionally with a port number, of the impersonation proxy + // for clients to use from outside the cluster. E.g. myhost.mycompany.com:8443. Clients should assume that they should + // connect via HTTPS to this service. + Endpoint string `json:"endpoint,omitempty"` +} + +func (c *Config) HasEndpoint() bool { + return c.Endpoint != "" +} + +func NewConfig() *Config { + return &Config{Mode: ModeAuto} +} + +func ConfigFromConfigMap(configMap *v1.ConfigMap) (*Config, error) { + stringConfig, ok := configMap.Data[ConfigMapDataKey] + if !ok { + return nil, fmt.Errorf(`ConfigMap is missing expected key "%s"`, ConfigMapDataKey) + } + config := NewConfig() + if err := yaml.Unmarshal([]byte(stringConfig), config); err != nil { + return nil, fmt.Errorf("decode yaml: %w", err) + } + if config.Mode != ModeAuto && config.Mode != ModeEnabled && config.Mode != ModeDisabled { + return nil, fmt.Errorf(`illegal value for "mode": %s`, config.Mode) + } + return config, nil +} diff --git a/internal/concierge/impersonator/config_test.go b/internal/concierge/impersonator/config_test.go new file mode 100644 index 000000000..b734dba91 --- /dev/null +++ b/internal/concierge/impersonator/config_test.go @@ -0,0 +1,155 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package impersonator + +import ( + "testing" + + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "go.pinniped.dev/internal/here" +) + +func TestNewConfig(t *testing.T) { + // It defaults the mode. + require.Equal(t, &Config{Mode: ModeAuto}, NewConfig()) +} + +func TestHasEndpoint(t *testing.T) { + configWithoutEndpoint := Config{} + configWithEndpoint := Config{Endpoint: "something"} + require.False(t, configWithoutEndpoint.HasEndpoint()) + require.True(t, configWithEndpoint.HasEndpoint()) +} + +func TestConfigFromConfigMap(t *testing.T) { + tests := []struct { + name string + configMap *v1.ConfigMap + wantConfig *Config + wantError string + }{ + { + name: "fully configured, valid config", + configMap: &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Data: map[string]string{ + "config.yaml": here.Doc(` + mode: enabled + endpoint: proxy.example.com:8443 + `), + }, + }, + wantConfig: &Config{ + Mode: "enabled", + Endpoint: "proxy.example.com:8443", + }, + }, + { + name: "empty, valid config", + configMap: &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Data: map[string]string{ + "config.yaml": "", + }, + }, + wantConfig: &Config{ + Mode: "auto", + Endpoint: "", + }, + }, + { + name: "valid config with mode enabled", + configMap: &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Data: map[string]string{ + "config.yaml": "mode: enabled", + }, + }, + wantConfig: &Config{ + Mode: "enabled", + Endpoint: "", + }, + }, + { + name: "valid config with mode disabled", + configMap: &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Data: map[string]string{ + "config.yaml": "mode: disabled", + }, + }, + wantConfig: &Config{ + Mode: "disabled", + Endpoint: "", + }, + }, + { + name: "valid config with mode auto", + configMap: &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Data: map[string]string{ + "config.yaml": "mode: auto", + }, + }, + wantConfig: &Config{ + Mode: "auto", + Endpoint: "", + }, + }, + { + name: "wrong key in configmap", + configMap: &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Data: map[string]string{ + "wrong-key": "", + }, + }, + wantError: `ConfigMap is missing expected key "config.yaml"`, + }, + { + name: "illegal yaml in configmap", + configMap: &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Data: map[string]string{ + "config.yaml": "this is not yaml", + }, + }, + wantError: "decode yaml: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type impersonator.Config", + }, + { + name: "illegal value for mode in configmap", + configMap: &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Data: map[string]string{ + "config.yaml": "mode: unexpected-value", + }, + }, + wantError: `illegal value for "mode": unexpected-value`, + }, + } + + for _, tt := range tests { + test := tt + t.Run(test.name, func(t *testing.T) { + config, err := ConfigFromConfigMap(test.configMap) + require.Equal(t, test.wantConfig, config) + if test.wantError != "" { + require.EqualError(t, err, test.wantError) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go new file mode 100644 index 000000000..f3a7372d7 --- /dev/null +++ b/internal/concierge/impersonator/impersonator.go @@ -0,0 +1,391 @@ +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package impersonator + +import ( + "fmt" + "net" + "net/http" + "net/http/httputil" + "net/url" + "strings" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/httpstream" + utilnet "k8s.io/apimachinery/pkg/util/net" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/apiserver/pkg/authorization/authorizer" + "k8s.io/apiserver/pkg/endpoints/filterlatency" + "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" + "k8s.io/apiserver/pkg/endpoints/request" + genericapirequest "k8s.io/apiserver/pkg/endpoints/request" + genericapiserver "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/dynamiccertificates" + "k8s.io/apiserver/pkg/server/filters" + genericoptions "k8s.io/apiserver/pkg/server/options" + "k8s.io/client-go/rest" + "k8s.io/client-go/transport" + + "go.pinniped.dev/internal/constable" + "go.pinniped.dev/internal/dynamiccert" + "go.pinniped.dev/internal/httputil/securityheader" + "go.pinniped.dev/internal/kubeclient" + "go.pinniped.dev/internal/plog" +) + +// FactoryFunc is a function which can create an impersonator server. +// It returns a function which will start the impersonator server. +// That start function takes a stopCh which can be used to stop the server. +// Once a server has been stopped, don't start it again using the start function. +// Instead, call the factory function again to get a new start function. +type FactoryFunc func( + port int, + dynamicCertProvider dynamiccert.Private, + impersonationProxySignerCA dynamiccert.Public, +) (func(stopCh <-chan struct{}) error, error) + +func New( + port int, + dynamicCertProvider dynamiccert.Private, + impersonationProxySignerCA dynamiccert.Public, +) (func(stopCh <-chan struct{}) error, error) { + return newInternal(port, dynamicCertProvider, impersonationProxySignerCA, nil, nil) +} + +func newInternal( //nolint:funlen // yeah, it's kind of long. + port int, + dynamicCertProvider dynamiccert.Private, + impersonationProxySignerCA dynamiccert.Public, + clientOpts []kubeclient.Option, // for unit testing, should always be nil in production + recOpts func(*genericoptions.RecommendedOptions), // for unit testing, should always be nil in production +) (func(stopCh <-chan struct{}) error, error) { + var listener net.Listener + + constructServer := func() (func(stopCh <-chan struct{}) error, error) { + // Bare minimum server side scheme to allow for status messages to be encoded. + scheme := runtime.NewScheme() + metav1.AddToGroupVersion(scheme, metav1.Unversioned) + codecs := serializer.NewCodecFactory(scheme) + + // This is unused for now but it is a safe value that we could use in the future. + defaultEtcdPathPrefix := "/pinniped-impersonation-proxy-registry" + + recommendedOptions := genericoptions.NewRecommendedOptions( + defaultEtcdPathPrefix, + codecs.LegacyCodec(), + ) + recommendedOptions.Etcd = nil // turn off etcd storage because we don't need it yet + recommendedOptions.SecureServing.ServerCert.GeneratedCert = dynamicCertProvider // serving certs (end user facing) + recommendedOptions.SecureServing.BindPort = port + + // Wire up the impersonation proxy signer CA as another valid authenticator for client cert auth, + // along with the Kube API server's CA. + // Note: any changes to the the Authentication stack need to be kept in sync with any assumptions made + // by getTransportForUser, especially if we ever update the TCR API to start returning bearer tokens. + kubeClient, err := kubeclient.New(clientOpts...) + if err != nil { + return nil, err + } + kubeClientCA, err := dynamiccertificates.NewDynamicCAFromConfigMapController( + "client-ca", metav1.NamespaceSystem, "extension-apiserver-authentication", "client-ca-file", kubeClient.Kubernetes, + ) + if err != nil { + return nil, err + } + recommendedOptions.Authentication.ClientCert.ClientCA = "---irrelevant-but-needs-to-be-non-empty---" // drop when we pick up https://github.com/kubernetes/kubernetes/pull/100055 + recommendedOptions.Authentication.ClientCert.CAContentProvider = dynamiccertificates.NewUnionCAContentProvider( + impersonationProxySignerCA, kubeClientCA, + ) + + if recOpts != nil { + recOpts(recommendedOptions) + } + + serverConfig := genericapiserver.NewRecommendedConfig(codecs) + + // Note that ApplyTo is going to create a network listener and bind to the requested port. + // It puts this listener into serverConfig.SecureServing.Listener. + err = recommendedOptions.ApplyTo(serverConfig) + if serverConfig.SecureServing != nil { + // Set the pointer from the outer function to allow the outer function to close the listener in case + // this function returns an error for any reason anywhere below here. + listener = serverConfig.SecureServing.Listener + } + if err != nil { + return nil, err + } + + // Loopback authentication to this server does not really make sense since we just proxy everything to + // the Kube API server, thus we replace loopback connection config with one that does direct connections + // the Kube API server. Loopback config is mainly used by post start hooks, so this is mostly future proofing. + serverConfig.LoopbackClientConfig = rest.CopyConfig(kubeClient.ProtoConfig) // assume proto is safe (hooks can override) + // Remove the bearer token so our authorizer does not get stomped on by AuthorizeClientBearerToken. + // See sanity checks at the end of this function. + serverConfig.LoopbackClientConfig.BearerToken = "" + + // match KAS exactly since our long running operations are just a proxy to it + // this must be kept in sync with github.com/kubernetes/kubernetes/cmd/kube-apiserver/app/server.go + // this is nothing to stress about - it has not changed since the beginning of Kube: + // v1.6 no-op move away from regex to request info https://github.com/kubernetes/kubernetes/pull/38119 + // v1.1 added pods/attach to the list https://github.com/kubernetes/kubernetes/pull/13705 + serverConfig.LongRunningFunc = filters.BasicLongRunningRequestCheck( + sets.NewString("watch", "proxy"), + sets.NewString("attach", "exec", "proxy", "log", "portforward"), + ) + + // Assume proto config is safe because transport level configs do not use rest.ContentConfig. + // Thus if we are interacting with actual APIs, they should be using pre-built clients. + impersonationProxyFunc, err := newImpersonationReverseProxyFunc(rest.CopyConfig(kubeClient.ProtoConfig)) + if err != nil { + return nil, err + } + + defaultBuildHandlerChainFunc := serverConfig.BuildHandlerChainFunc + serverConfig.BuildHandlerChainFunc = func(_ http.Handler, c *genericapiserver.Config) http.Handler { + // We ignore the passed in handler because we never have any REST APIs to delegate to. + // This means we are ignoring the admission, discovery, REST storage, etc layers. + doNotDelegate := http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {}) + + // Impersonation proxy business logic with timing information. + impersonationProxyCompleted := filterlatency.TrackCompleted(doNotDelegate) + impersonationProxy := impersonationProxyFunc(c) + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer impersonationProxyCompleted.ServeHTTP(w, r) + impersonationProxy.ServeHTTP(w, r) + })) + handler = filterlatency.TrackStarted(handler, "impersonationproxy") + + // The standard Kube handler chain (authn, authz, impersonation, audit, etc). + // See the genericapiserver.DefaultBuildHandlerChain func for details. + handler = defaultBuildHandlerChainFunc(handler, c) + + // Always set security headers so browsers do the right thing. + handler = securityheader.Wrap(handler) + + return handler + } + + // Overwrite the delegating authorizer with one that only cares about impersonation. + // Empty string is disallowed because request info has had bugs in the past where it would leave it empty. + disallowedVerbs := sets.NewString("", "impersonate") + noImpersonationAuthorizer := &comparableAuthorizer{ + AuthorizerFunc: func(a authorizer.Attributes) (authorizer.Decision, string, error) { + // Supporting impersonation is not hard, it would just require a bunch of testing + // and configuring the audit layer (to preserve the caller) which we can do later. + // We would also want to delete the incoming impersonation headers + // instead of overwriting the delegating authorizer, we would + // actually use it to make the impersonation authorization checks. + if disallowedVerbs.Has(a.GetVerb()) { + return authorizer.DecisionDeny, "impersonation is not allowed or invalid verb", nil + } + + return authorizer.DecisionAllow, "deferring authorization to kube API server", nil + }, + } + // Set our custom authorizer before calling Compete(), which will use it. + serverConfig.Authorization.Authorizer = noImpersonationAuthorizer + + impersonationProxyServer, err := serverConfig.Complete().New("impersonation-proxy", genericapiserver.NewEmptyDelegate()) + if err != nil { + return nil, err + } + + preparedRun := impersonationProxyServer.PrepareRun() + + // Sanity check. Make sure that our custom authorizer is still in place and did not get changed or wrapped. + if preparedRun.Authorizer != noImpersonationAuthorizer { + return nil, constable.Error("invalid mutation of impersonation authorizer detected") + } + + // Sanity check. Assert that we have a functioning token file to use and no bearer token. + if len(preparedRun.LoopbackClientConfig.BearerToken) != 0 || len(preparedRun.LoopbackClientConfig.BearerTokenFile) == 0 { + return nil, constable.Error("invalid impersonator loopback rest config has wrong bearer token semantics") + } + + return preparedRun.Run, nil + } + + result, err := constructServer() + // If there was any error during construction, then we would like to close the listener to free up the port. + if err != nil { + errs := []error{err} + if listener != nil { + errs = append(errs, listener.Close()) + } + return nil, errors.NewAggregate(errs) + } + return result, nil +} + +// No-op wrapping around AuthorizerFunc to allow for comparisons. +type comparableAuthorizer struct { + authorizer.AuthorizerFunc +} + +func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapiserver.Config) http.Handler, error) { + serverURL, err := url.Parse(restConfig.Host) + if err != nil { + return nil, fmt.Errorf("could not parse host URL from in-cluster config: %w", err) + } + + http1RoundTripper, err := getTransportForProtocol(restConfig, "http/1.1") + if err != nil { + return nil, fmt.Errorf("could not get http/1.1 round tripper: %w", err) + } + + http2RoundTripper, err := getTransportForProtocol(restConfig, "h2") + if err != nil { + return nil, fmt.Errorf("could not get http/2.0 round tripper: %w", err) + } + + return func(c *genericapiserver.Config) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if len(r.Header.Values("Authorization")) != 0 { + plog.Warning("aggregated API server logic did not delete authorization header but it is always supposed to do so", + "url", r.URL.String(), + "method", r.Method, + ) + newInternalErrResponse(w, r, c.Serializer, "invalid authorization header") + return + } + + if err := ensureNoImpersonationHeaders(r); err != nil { + plog.Error("noImpersonationAuthorizer logic did not prevent nested impersonation but it is always supposed to do so", + err, + "url", r.URL.String(), + "method", r.Method, + ) + newInternalErrResponse(w, r, c.Serializer, "invalid impersonation") + return + } + + userInfo, ok := request.UserFrom(r.Context()) + if !ok { + plog.Warning("aggregated API server logic did not set user info but it is always supposed to do so", + "url", r.URL.String(), + "method", r.Method, + ) + newInternalErrResponse(w, r, c.Serializer, "invalid user") + return + } + + // KAS only supports upgrades via http/1.1 to websockets/SPDY (upgrades never use http/2.0) + // Thus we default to using http/2.0 when the request is not an upgrade, otherwise we use http/1.1 + baseRT := http2RoundTripper + isUpgradeRequest := httpstream.IsUpgradeRequest(r) + if isUpgradeRequest { + baseRT = http1RoundTripper + } + + rt, err := getTransportForUser(userInfo, baseRT) + if err != nil { + plog.WarningErr("rejecting request as we cannot act as the current user", err, + "url", r.URL.String(), + "method", r.Method, + "isUpgradeRequest", isUpgradeRequest, + ) + newInternalErrResponse(w, r, c.Serializer, "unimplemented functionality - unable to act as current user") + return + } + + plog.Debug("impersonation proxy servicing request", + "url", r.URL.String(), + "method", r.Method, + "isUpgradeRequest", isUpgradeRequest, + ) + plog.Trace("impersonation proxy servicing request was for user", + "url", r.URL.String(), + "method", r.Method, + "isUpgradeRequest", isUpgradeRequest, + "username", userInfo.GetName(), // this info leak seems fine for trace level logs + ) + + // The proxy library used below will panic when the client disconnects abruptly, so in order to + // assure that this log message is always printed at the end of this func, it must be deferred. + defer plog.Debug("impersonation proxy finished servicing request", + "url", r.URL.String(), + "method", r.Method, + "isUpgradeRequest", isUpgradeRequest, + ) + + // do not allow the client to cause log confusion by spoofing this header + if len(r.Header.Values("X-Forwarded-For")) > 0 { + r = utilnet.CloneRequest(r) + r.Header.Del("X-Forwarded-For") + } + + reverseProxy := httputil.NewSingleHostReverseProxy(serverURL) + reverseProxy.Transport = rt + reverseProxy.FlushInterval = 200 * time.Millisecond // the "watch" verb will not work without this line + reverseProxy.ServeHTTP(w, r) + }) + }, nil +} + +func ensureNoImpersonationHeaders(r *http.Request) error { + for key := range r.Header { + if strings.HasPrefix(key, "Impersonate") { + return fmt.Errorf("%q header already exists", key) + } + } + + return nil +} + +func getTransportForUser(userInfo user.Info, delegate http.RoundTripper) (http.RoundTripper, error) { + if len(userInfo.GetUID()) == 0 { + impersonateConfig := transport.ImpersonationConfig{ + UserName: userInfo.GetName(), + Groups: userInfo.GetGroups(), + Extra: userInfo.GetExtra(), + } + // transport.NewImpersonatingRoundTripper clones the request before setting headers + // thus it will not accidentally mutate the input request (see http.Handler docs) + return transport.NewImpersonatingRoundTripper(impersonateConfig, delegate), nil + } + + // 0. in the case of a request that is not attempting to do nested impersonation + // 1. if we make the assumption that the TCR API does not issue tokens (or pass the TCR API bearer token + // authenticator into this func - we need to know the authentication cred is something KAS would honor) + // 2. then if preserve the incoming authorization header into the request's context + // 3. we could reauthenticate it here (it would be a free cache hit) + // 4. confirm that it matches the passed in user info (i.e. it was actually the cred used to authenticate and not a client cert) + // 5. then we could issue a reverse proxy request using an anonymous rest config and the bearer token + // 6. thus instead of impersonating the user, we would just be passing their request through + // 7. this would preserve the UID info and thus allow us to safely support all token based auth + // 8. the above would be safe even if in the future Kube started supporting UIDs asserted by client certs + return nil, constable.Error("unexpected uid") +} + +func newInternalErrResponse(w http.ResponseWriter, r *http.Request, s runtime.NegotiatedSerializer, msg string) { + newStatusErrResponse(w, r, s, apierrors.NewInternalError(constable.Error(msg))) +} + +func newStatusErrResponse(w http.ResponseWriter, r *http.Request, s runtime.NegotiatedSerializer, err *apierrors.StatusError) { + requestInfo, ok := genericapirequest.RequestInfoFrom(r.Context()) + if !ok { + responsewriters.InternalError(w, r, constable.Error("no RequestInfo found in the context")) + return + } + + gv := schema.GroupVersion{Group: requestInfo.APIGroup, Version: requestInfo.APIVersion} + responsewriters.ErrorNegotiated(err, s, gv, w, r) +} + +func getTransportForProtocol(restConfig *rest.Config, protocol string) (http.RoundTripper, error) { + transportConfig, err := restConfig.TransportConfig() + if err != nil { + return nil, fmt.Errorf("could not get in-cluster transport config: %w", err) + } + transportConfig.TLS.NextProtos = []string{protocol} + + return transport.New(transportConfig) +} diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go new file mode 100644 index 000000000..7633c9dff --- /dev/null +++ b/internal/concierge/impersonator/impersonator_test.go @@ -0,0 +1,638 @@ +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package impersonator + +import ( + "context" + "math/rand" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/util/httpstream" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/apiserver/pkg/features" + genericapiserver "k8s.io/apiserver/pkg/server" + genericoptions "k8s.io/apiserver/pkg/server/options" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd/api" + featuregatetesting "k8s.io/component-base/featuregate/testing" + + "go.pinniped.dev/internal/certauthority" + "go.pinniped.dev/internal/dynamiccert" + "go.pinniped.dev/internal/here" + "go.pinniped.dev/internal/httputil/roundtripper" + "go.pinniped.dev/internal/kubeclient" + "go.pinniped.dev/internal/testutil" +) + +func TestImpersonator(t *testing.T) { + const port = 9444 + + ca, err := certauthority.New("ca", time.Hour) + require.NoError(t, err) + caKey, err := ca.PrivateKeyToPEM() + require.NoError(t, err) + caContent := dynamiccert.NewCA("ca") + err = caContent.SetCertKeyContent(ca.Bundle(), caKey) + require.NoError(t, err) + + cert, key, err := ca.IssueServerCertPEM(nil, []net.IP{net.ParseIP("127.0.0.1")}, time.Hour) + require.NoError(t, err) + certKeyContent := dynamiccert.NewServingCert("cert-key") + err = certKeyContent.SetCertKeyContent(cert, key) + require.NoError(t, err) + + unrelatedCA, err := certauthority.New("ca", time.Hour) + require.NoError(t, err) + + // Punch out just enough stuff to make New actually run without error. + recOpts := func(options *genericoptions.RecommendedOptions) { + options.Authentication.RemoteKubeConfigFileOptional = true + options.Authorization.RemoteKubeConfigFileOptional = true + options.CoreAPI = nil + options.Admission = nil + } + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIPriorityAndFairness, false)() + + tests := []struct { + name string + clientCert *clientCert + clientImpersonateUser rest.ImpersonationConfig + clientMutateHeaders func(http.Header) + clientNextProtos []string + kubeAPIServerClientBearerTokenFile string + kubeAPIServerStatusCode int + wantKubeAPIServerRequestHeaders http.Header + wantError string + wantConstructionError string + }{ + { + name: "happy path", + clientCert: newClientCert(t, ca, "test-username", []string{"test-group1", "test-group2"}), + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + wantKubeAPIServerRequestHeaders: http.Header{ + "Impersonate-User": {"test-username"}, + "Impersonate-Group": {"test-group1", "test-group2", "system:authenticated"}, + "Authorization": {"Bearer some-service-account-token"}, + "User-Agent": {"test-agent"}, + "Accept": {"application/vnd.kubernetes.protobuf,application/json"}, + "Accept-Encoding": {"gzip"}, + "X-Forwarded-For": {"127.0.0.1"}, + }, + }, + { + name: "happy path with upgrade", + clientCert: newClientCert(t, ca, "test-username2", []string{"test-group3", "test-group4"}), + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + clientMutateHeaders: func(header http.Header) { + header.Add("Connection", "Upgrade") + header.Add("Upgrade", "spdy/3.1") + + if ok := httpstream.IsUpgradeRequest(&http.Request{Header: header}); !ok { + panic("request must be upgrade in this test") + } + }, + clientNextProtos: []string{"http/1.1"}, // we need to use http1 as http2 does not support upgrades, see http2checkConnHeaders + wantKubeAPIServerRequestHeaders: http.Header{ + "Impersonate-User": {"test-username2"}, + "Impersonate-Group": {"test-group3", "test-group4", "system:authenticated"}, + "Authorization": {"Bearer some-service-account-token"}, + "User-Agent": {"test-agent"}, + "Accept": {"application/vnd.kubernetes.protobuf,application/json"}, + "Accept-Encoding": {"gzip"}, + "X-Forwarded-For": {"127.0.0.1"}, + "Connection": {"Upgrade"}, + "Upgrade": {"spdy/3.1"}, + }, + }, + { + name: "happy path ignores forwarded header", + clientCert: newClientCert(t, ca, "test-username2", []string{"test-group3", "test-group4"}), + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + clientMutateHeaders: func(header http.Header) { + header.Add("X-Forwarded-For", "example.com") + }, + wantKubeAPIServerRequestHeaders: http.Header{ + "Impersonate-User": {"test-username2"}, + "Impersonate-Group": {"test-group3", "test-group4", "system:authenticated"}, + "Authorization": {"Bearer some-service-account-token"}, + "User-Agent": {"test-agent"}, + "Accept": {"application/vnd.kubernetes.protobuf,application/json"}, + "Accept-Encoding": {"gzip"}, + "X-Forwarded-For": {"127.0.0.1"}, + }, + }, + { + name: "happy path ignores forwarded header canonicalization", + clientCert: newClientCert(t, ca, "test-username2", []string{"test-group3", "test-group4"}), + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + clientMutateHeaders: func(header http.Header) { + header.Add("x-FORWARDED-for", "example.com") + }, + wantKubeAPIServerRequestHeaders: http.Header{ + "Impersonate-User": {"test-username2"}, + "Impersonate-Group": {"test-group3", "test-group4", "system:authenticated"}, + "Authorization": {"Bearer some-service-account-token"}, + "User-Agent": {"test-agent"}, + "Accept": {"application/vnd.kubernetes.protobuf,application/json"}, + "Accept-Encoding": {"gzip"}, + "X-Forwarded-For": {"127.0.0.1"}, + }, + }, + { + name: "user is authenticated but the kube API request returns an error", + kubeAPIServerStatusCode: http.StatusNotFound, + clientCert: newClientCert(t, ca, "test-username", []string{"test-group1", "test-group2"}), + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + wantError: `the server could not find the requested resource (get namespaces)`, + wantKubeAPIServerRequestHeaders: http.Header{ + "Impersonate-User": {"test-username"}, + "Impersonate-Group": {"test-group1", "test-group2", "system:authenticated"}, + "Authorization": {"Bearer some-service-account-token"}, + "User-Agent": {"test-agent"}, + "Accept": {"application/vnd.kubernetes.protobuf,application/json"}, + "Accept-Encoding": {"gzip"}, + "X-Forwarded-For": {"127.0.0.1"}, + }, + }, + { + name: "when there is no client cert on request, it is an anonymous request", + clientCert: &clientCert{}, + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + wantKubeAPIServerRequestHeaders: http.Header{ + "Impersonate-User": {"system:anonymous"}, + "Impersonate-Group": {"system:unauthenticated"}, + "Authorization": {"Bearer some-service-account-token"}, + "User-Agent": {"test-agent"}, + "Accept": {"application/vnd.kubernetes.protobuf,application/json"}, + "Accept-Encoding": {"gzip"}, + "X-Forwarded-For": {"127.0.0.1"}, + }, + }, + { + name: "failed client cert authentication", + clientCert: newClientCert(t, unrelatedCA, "test-username", []string{"test-group1"}), + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + wantError: "Unauthorized", + }, + { + name: "double impersonation is not allowed by regular users", + clientCert: newClientCert(t, ca, "test-username", []string{"test-group1", "test-group2"}), + clientImpersonateUser: rest.ImpersonationConfig{UserName: "some-other-username"}, + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + wantError: `users "some-other-username" is forbidden: User "test-username" ` + + `cannot impersonate resource "users" in API group "" at the cluster scope: impersonation is not allowed or invalid verb`, + }, + { + name: "double impersonation is not allowed by admin users", + clientCert: newClientCert(t, ca, "test-admin", []string{"system:masters", "test-group2"}), + clientImpersonateUser: rest.ImpersonationConfig{UserName: "some-other-username"}, + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + wantError: `users "some-other-username" is forbidden: User "test-admin" ` + + `cannot impersonate resource "users" in API group "" at the cluster scope: impersonation is not allowed or invalid verb`, + }, + { + name: "no bearer token file in Kube API server client config", + wantConstructionError: "invalid impersonator loopback rest config has wrong bearer token semantics", + }, + { + name: "header canonicalization user header", + clientCert: newClientCert(t, ca, "test-username", []string{"test-group1", "test-group2"}), + clientMutateHeaders: func(header http.Header) { + header.Set("imPerSonaTE-USer", "PANDA") + }, + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + wantError: `users "PANDA" is forbidden: User "test-username" ` + + `cannot impersonate resource "users" in API group "" at the cluster scope: impersonation is not allowed or invalid verb`, + }, + { + name: "header canonicalization future UID header", + clientCert: newClientCert(t, ca, "test-username", []string{"test-group1", "test-group2"}), + clientMutateHeaders: func(header http.Header) { + header.Set("imPerSonaTE-uid", "007") + }, + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + wantError: "Internal error occurred: invalid impersonation", + }, + { + name: "future UID header", + clientCert: newClientCert(t, ca, "test-username", []string{"test-group1", "test-group2"}), + clientMutateHeaders: func(header http.Header) { + header.Set("Impersonate-Uid", "008") + }, + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + wantError: "Internal error occurred: invalid impersonation", + }, + } + for _, tt := range tests { + tt := tt + // This is a serial test because the production code binds to the port. + t.Run(tt.name, func(t *testing.T) { + // After failing to start and after shutdown, the impersonator port should be available again. + defer requireCanBindToPort(t, port) + + if tt.kubeAPIServerStatusCode == 0 { + tt.kubeAPIServerStatusCode = http.StatusOK + } + + // Set up a fake Kube API server which will stand in for the real one. The impersonator + // will proxy incoming calls to this fake server. + testKubeAPIServerWasCalled := false + var testKubeAPIServerSawHeaders http.Header + testKubeAPIServerCA, testKubeAPIServerURL := testutil.TLSTestServer(t, func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, http.MethodGet, r.Method) + switch r.URL.Path { + case "/api/v1/namespaces/kube-system/configmaps": + // The production code uses NewDynamicCAFromConfigMapController which fetches a ConfigMap, + // so treat that differently. It wants to read the Kube API server CA from that ConfigMap + // to use it to validate client certs. We don't need it for this test, so return NotFound. + http.NotFound(w, r) + return + case "/api/v1/namespaces": + testKubeAPIServerWasCalled = true + testKubeAPIServerSawHeaders = r.Header + if tt.kubeAPIServerStatusCode != http.StatusOK { + w.WriteHeader(tt.kubeAPIServerStatusCode) + } else { + w.Header().Add("Content-Type", "application/json; charset=UTF-8") + _, _ = w.Write([]byte(here.Doc(` + { + "kind": "NamespaceList", + "apiVersion":"v1", + "items": [ + {"metadata":{"name": "namespace1"}}, + {"metadata":{"name": "namespace2"}} + ] + } + `))) + } + default: + require.Fail(t, "fake Kube API server got an unexpected request") + } + }) + + // Create the client config that the impersonation server should use to talk to the Kube API server. + testKubeAPIServerKubeconfig := rest.Config{ + Host: testKubeAPIServerURL, + BearerToken: "some-service-account-token", + TLSClientConfig: rest.TLSClientConfig{CAData: []byte(testKubeAPIServerCA)}, + BearerTokenFile: tt.kubeAPIServerClientBearerTokenFile, + } + clientOpts := []kubeclient.Option{kubeclient.WithConfig(&testKubeAPIServerKubeconfig)} + + // Create an impersonator. + runner, constructionErr := newInternal(port, certKeyContent, caContent, clientOpts, recOpts) + if len(tt.wantConstructionError) > 0 { + require.EqualError(t, constructionErr, tt.wantConstructionError) + require.Nil(t, runner) + // The rest of the test doesn't make sense when you expect a construction error, so stop here. + return + } + require.NoError(t, constructionErr) + require.NotNil(t, runner) + + // Start the impersonator. + stopCh := make(chan struct{}) + errCh := make(chan error) + go func() { + stopErr := runner(stopCh) + errCh <- stopErr + }() + + // Create a kubeconfig to talk to the impersonator as a client. + clientKubeconfig := &rest.Config{ + Host: "https://127.0.0.1:" + strconv.Itoa(port), + TLSClientConfig: rest.TLSClientConfig{ + CAData: ca.Bundle(), + CertData: tt.clientCert.certPEM, + KeyData: tt.clientCert.keyPEM, + NextProtos: tt.clientNextProtos, + }, + UserAgent: "test-agent", + // BearerToken should be ignored during auth when there are valid client certs, + // and it should not passed into the impersonator handler func as an authorization header. + BearerToken: "must-be-ignored", + Impersonate: tt.clientImpersonateUser, + WrapTransport: func(rt http.RoundTripper) http.RoundTripper { + if tt.clientMutateHeaders == nil { + return rt + } + + return roundtripper.Func(func(req *http.Request) (*http.Response, error) { + req = req.Clone(req.Context()) + tt.clientMutateHeaders(req.Header) + return rt.RoundTrip(req) + }) + }, + } + + // Create a real Kube client to make API requests to the impersonator. + client, err := kubeclient.New(kubeclient.WithConfig(clientKubeconfig)) + require.NoError(t, err) + + // The fake Kube API server knows how to to list namespaces, so make that request using the client + // through the impersonator. + listResponse, err := client.Kubernetes.CoreV1().Namespaces().List(context.Background(), metav1.ListOptions{}) + if len(tt.wantError) > 0 { + require.EqualError(t, err, tt.wantError) + require.Equal(t, &corev1.NamespaceList{}, listResponse) + } else { + require.NoError(t, err) + require.Equal(t, &corev1.NamespaceList{ + Items: []corev1.Namespace{ + {ObjectMeta: metav1.ObjectMeta{Name: "namespace1"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "namespace2"}}, + }, + }, listResponse) + } + + // If we expect to see some headers, then the fake KAS should have been called. + require.Equal(t, len(tt.wantKubeAPIServerRequestHeaders) != 0, testKubeAPIServerWasCalled) + // If the impersonator proxied the request to the fake Kube API server, we should see the headers + // of the original request mutated by the impersonator. Otherwise the headers should be nil. + require.Equal(t, tt.wantKubeAPIServerRequestHeaders, testKubeAPIServerSawHeaders) + + // Stop the impersonator server. + close(stopCh) + exitErr := <-errCh + require.NoError(t, exitErr) + }) + } +} + +func TestImpersonatorHTTPHandler(t *testing.T) { + const testUser = "test-user" + + testGroups := []string{"test-group-1", "test-group-2"} + testExtra := map[string][]string{ + "extra-1": {"some", "extra", "stuff"}, + "extra-2": {"some", "more", "extra", "stuff"}, + } + + validURL, _ := url.Parse("http://pinniped.dev/blah") + newRequest := func(h http.Header, userInfo user.Info) *http.Request { + ctx := context.Background() + if userInfo != nil { + ctx = request.WithUser(ctx, userInfo) + } + r, err := http.NewRequestWithContext(ctx, http.MethodGet, validURL.String(), nil) + require.NoError(t, err) + r.Header = h + reqInfo := &request.RequestInfo{ + IsResourceRequest: false, + Path: validURL.Path, + Verb: "get", + } + r = r.WithContext(request.WithRequestInfo(ctx, reqInfo)) + return r + } + + tests := []struct { + name string + restConfig *rest.Config + wantCreationErr string + request *http.Request + wantHTTPBody string + wantHTTPStatus int + wantKubeAPIServerRequestHeaders http.Header + kubeAPIServerStatusCode int + }{ + { + name: "invalid kubeconfig host", + restConfig: &rest.Config{Host: ":"}, + wantCreationErr: "could not parse host URL from in-cluster config: parse \":\": missing protocol scheme", + }, + { + name: "invalid transport config", + restConfig: &rest.Config{ + Host: "pinniped.dev/blah", + ExecProvider: &api.ExecConfig{}, + AuthProvider: &api.AuthProviderConfig{}, + }, + wantCreationErr: "could not get http/1.1 round tripper: could not get in-cluster transport config: execProvider and authProvider cannot be used in combination", + }, + { + name: "fail to get transport from config", + restConfig: &rest.Config{ + Host: "pinniped.dev/blah", + BearerToken: "test-bearer-token", + Transport: http.DefaultTransport, + TLSClientConfig: rest.TLSClientConfig{Insecure: true}, + }, + wantCreationErr: "could not get http/1.1 round tripper: using a custom transport with TLS certificate options or the insecure flag is not allowed", + }, + { + name: "Impersonate-User header already in request", + request: newRequest(map[string][]string{"Impersonate-User": {"some-user"}}, nil), + wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: invalid impersonation","reason":"InternalError","details":{"causes":[{"message":"invalid impersonation"}]},"code":500}` + "\n", + wantHTTPStatus: http.StatusInternalServerError, + }, + { + name: "Impersonate-Group header already in request", + request: newRequest(map[string][]string{"Impersonate-Group": {"some-group"}}, nil), + wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: invalid impersonation","reason":"InternalError","details":{"causes":[{"message":"invalid impersonation"}]},"code":500}` + "\n", + wantHTTPStatus: http.StatusInternalServerError, + }, + { + name: "Impersonate-Extra header already in request", + request: newRequest(map[string][]string{"Impersonate-Extra-something": {"something"}}, nil), + wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: invalid impersonation","reason":"InternalError","details":{"causes":[{"message":"invalid impersonation"}]},"code":500}` + "\n", + wantHTTPStatus: http.StatusInternalServerError, + }, + { + name: "Impersonate-* header already in request", + request: newRequest(map[string][]string{"Impersonate-Something": {"some-newfangled-impersonate-header"}}, nil), + wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: invalid impersonation","reason":"InternalError","details":{"causes":[{"message":"invalid impersonation"}]},"code":500}` + "\n", + wantHTTPStatus: http.StatusInternalServerError, + }, + { + name: "unexpected authorization header", + request: newRequest(map[string][]string{"Authorization": {"panda"}}, nil), + wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: invalid authorization header","reason":"InternalError","details":{"causes":[{"message":"invalid authorization header"}]},"code":500}` + "\n", + wantHTTPStatus: http.StatusInternalServerError, + }, + { + name: "missing user", + request: newRequest(map[string][]string{}, nil), + wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: invalid user","reason":"InternalError","details":{"causes":[{"message":"invalid user"}]},"code":500}` + "\n", + wantHTTPStatus: http.StatusInternalServerError, + }, + { + name: "unexpected UID", + request: newRequest(map[string][]string{}, &user.DefaultInfo{UID: "007"}), + wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: unimplemented functionality - unable to act as current user","reason":"InternalError","details":{"causes":[{"message":"unimplemented functionality - unable to act as current user"}]},"code":500}` + "\n", + wantHTTPStatus: http.StatusInternalServerError, + }, + // happy path + { + name: "authenticated user", + request: newRequest(map[string][]string{ + "User-Agent": {"test-user-agent"}, + "Accept": {"some-accepted-format"}, + "Accept-Encoding": {"some-accepted-encoding"}, + "Connection": {"Upgrade"}, // the value "Upgrade" is handled in a special way by `httputil.NewSingleHostReverseProxy` + "Upgrade": {"some-upgrade"}, + "Content-Type": {"some-type"}, + "Content-Length": {"some-length"}, + "Other-Header": {"test-header-value-1"}, // this header will be passed through + }, &user.DefaultInfo{ + Name: testUser, + Groups: testGroups, + Extra: testExtra, + }), + wantKubeAPIServerRequestHeaders: map[string][]string{ + "Authorization": {"Bearer some-service-account-token"}, + "Impersonate-Extra-Extra-1": {"some", "extra", "stuff"}, + "Impersonate-Extra-Extra-2": {"some", "more", "extra", "stuff"}, + "Impersonate-Group": {"test-group-1", "test-group-2"}, + "Impersonate-User": {"test-user"}, + "User-Agent": {"test-user-agent"}, + "Accept": {"some-accepted-format"}, + "Accept-Encoding": {"some-accepted-encoding"}, + "Connection": {"Upgrade"}, + "Upgrade": {"some-upgrade"}, + "Content-Type": {"some-type"}, + "Other-Header": {"test-header-value-1"}, + }, + wantHTTPBody: "successful proxied response", + wantHTTPStatus: http.StatusOK, + }, + { + name: "user is authenticated but the kube API request returns an error", + request: newRequest(map[string][]string{ + "User-Agent": {"test-user-agent"}, + }, &user.DefaultInfo{ + Name: testUser, + Groups: testGroups, + Extra: testExtra, + }), + kubeAPIServerStatusCode: http.StatusNotFound, + wantKubeAPIServerRequestHeaders: map[string][]string{ + "Accept-Encoding": {"gzip"}, // because the rest client used in this test does not disable compression + "Authorization": {"Bearer some-service-account-token"}, + "Impersonate-Extra-Extra-1": {"some", "extra", "stuff"}, + "Impersonate-Extra-Extra-2": {"some", "more", "extra", "stuff"}, + "Impersonate-Group": {"test-group-1", "test-group-2"}, + "Impersonate-User": {"test-user"}, + "User-Agent": {"test-user-agent"}, + }, + wantHTTPStatus: http.StatusNotFound, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if tt.kubeAPIServerStatusCode == 0 { + tt.kubeAPIServerStatusCode = http.StatusOK + } + + testKubeAPIServerWasCalled := false + testKubeAPIServerSawHeaders := http.Header{} + testKubeAPIServerCA, testKubeAPIServerURL := testutil.TLSTestServer(t, func(w http.ResponseWriter, r *http.Request) { + testKubeAPIServerWasCalled = true + testKubeAPIServerSawHeaders = r.Header + if tt.kubeAPIServerStatusCode != http.StatusOK { + w.WriteHeader(tt.kubeAPIServerStatusCode) + } else { + _, _ = w.Write([]byte("successful proxied response")) + } + }) + testKubeAPIServerKubeconfig := rest.Config{ + Host: testKubeAPIServerURL, + BearerToken: "some-service-account-token", + TLSClientConfig: rest.TLSClientConfig{CAData: []byte(testKubeAPIServerCA)}, + } + if tt.restConfig == nil { + tt.restConfig = &testKubeAPIServerKubeconfig + } + + impersonatorHTTPHandlerFunc, err := newImpersonationReverseProxyFunc(tt.restConfig) + if tt.wantCreationErr != "" { + require.EqualError(t, err, tt.wantCreationErr) + require.Nil(t, impersonatorHTTPHandlerFunc) + return + } + require.NoError(t, err) + require.NotNil(t, impersonatorHTTPHandlerFunc) + + // this is not a valid way to get a server config, but it is good enough for a unit test + scheme := runtime.NewScheme() + metav1.AddToGroupVersion(scheme, metav1.Unversioned) + codecs := serializer.NewCodecFactory(scheme) + serverConfig := genericapiserver.NewRecommendedConfig(codecs) + + w := httptest.NewRecorder() + + r := tt.request + wantKubeAPIServerRequestHeaders := tt.wantKubeAPIServerRequestHeaders + + // take the isUpgradeRequest branch randomly to make sure we exercise both branches + forceUpgradeRequest := rand.Int()%2 == 0 //nolint:gosec // we do not care if this is cryptographically secure + if forceUpgradeRequest && len(r.Header.Get("Upgrade")) == 0 { + r = r.Clone(r.Context()) + r.Header.Add("Connection", "Upgrade") + r.Header.Add("Upgrade", "spdy/3.1") + + wantKubeAPIServerRequestHeaders = wantKubeAPIServerRequestHeaders.Clone() + if wantKubeAPIServerRequestHeaders == nil { + wantKubeAPIServerRequestHeaders = http.Header{} + } + wantKubeAPIServerRequestHeaders.Add("Connection", "Upgrade") + wantKubeAPIServerRequestHeaders.Add("Upgrade", "spdy/3.1") + } + + requestBeforeServe := r.Clone(r.Context()) + impersonatorHTTPHandlerFunc(&serverConfig.Config).ServeHTTP(w, r) + + require.Equal(t, requestBeforeServe, r, "ServeHTTP() mutated the request, and it should not per http.Handler docs") + if tt.wantHTTPStatus != 0 { + require.Equalf(t, tt.wantHTTPStatus, w.Code, "fyi, response body was %q", w.Body.String()) + } + if tt.wantHTTPBody != "" { + require.Equal(t, tt.wantHTTPBody, w.Body.String()) + } + + if tt.wantHTTPStatus == http.StatusOK || tt.kubeAPIServerStatusCode != http.StatusOK { + require.True(t, testKubeAPIServerWasCalled, "Should have proxied the request to the Kube API server, but didn't") + require.Equal(t, wantKubeAPIServerRequestHeaders, testKubeAPIServerSawHeaders) + } else { + require.False(t, testKubeAPIServerWasCalled, "Should not have proxied the request to the Kube API server, but did") + } + }) + } +} + +type clientCert struct { + certPEM, keyPEM []byte +} + +func newClientCert(t *testing.T, ca *certauthority.CA, username string, groups []string) *clientCert { + certPEM, keyPEM, err := ca.IssueClientCertPEM(username, groups, time.Hour) + require.NoError(t, err) + return &clientCert{ + certPEM: certPEM, + keyPEM: keyPEM, + } +} + +func requireCanBindToPort(t *testing.T, port int) { + ln, _, listenErr := genericoptions.CreateListener("", "0.0.0.0:"+strconv.Itoa(port), net.ListenConfig{}) + require.NoError(t, listenErr) + require.NoError(t, ln.Close()) +} diff --git a/internal/concierge/scheme/scheme.go b/internal/concierge/scheme/scheme.go new file mode 100644 index 000000000..658b888f8 --- /dev/null +++ b/internal/concierge/scheme/scheme.go @@ -0,0 +1,130 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package scheme contains code to construct a proper runtime.Scheme for the Concierge aggregated +// API. +package scheme + +import ( + "fmt" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + + identityapi "go.pinniped.dev/generated/latest/apis/concierge/identity" + identityv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/identity/v1alpha1" + loginapi "go.pinniped.dev/generated/latest/apis/concierge/login" + loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1" + "go.pinniped.dev/internal/groupsuffix" + "go.pinniped.dev/internal/plog" +) + +// New returns a runtime.Scheme for use by the Concierge aggregated API running with the provided +// apiGroupSuffix. +func New(apiGroupSuffix string) (_ *runtime.Scheme, login, identity schema.GroupVersion) { + // standard set up of the server side scheme + scheme := runtime.NewScheme() + + // add the options to empty v1 + metav1.AddToGroupVersion(scheme, metav1.Unversioned) + + // nothing fancy is required if using the standard group suffix + if apiGroupSuffix == groupsuffix.PinnipedDefaultSuffix { + schemeBuilder := runtime.NewSchemeBuilder( + loginv1alpha1.AddToScheme, + loginapi.AddToScheme, + identityv1alpha1.AddToScheme, + identityapi.AddToScheme, + ) + utilruntime.Must(schemeBuilder.AddToScheme(scheme)) + return scheme, loginv1alpha1.SchemeGroupVersion, identityv1alpha1.SchemeGroupVersion + } + + loginConciergeGroupData, identityConciergeGroupData := groupsuffix.ConciergeAggregatedGroups(apiGroupSuffix) + + addToSchemeAtNewGroup(scheme, loginv1alpha1.GroupName, loginConciergeGroupData.Group, loginv1alpha1.AddToScheme, loginapi.AddToScheme) + addToSchemeAtNewGroup(scheme, identityv1alpha1.GroupName, identityConciergeGroupData.Group, identityv1alpha1.AddToScheme, identityapi.AddToScheme) + + // manually register conversions and defaulting into the correct scheme since we cannot directly call AddToScheme + schemeBuilder := runtime.NewSchemeBuilder( + loginv1alpha1.RegisterConversions, + loginv1alpha1.RegisterDefaults, + identityv1alpha1.RegisterConversions, + identityv1alpha1.RegisterDefaults, + ) + utilruntime.Must(schemeBuilder.AddToScheme(scheme)) + + // we do not want to return errors from the scheme and instead would prefer to defer + // to the REST storage layer for consistency. The simplest way to do this is to force + // a cache miss from the authenticator cache. Kube API groups are validated via the + // IsDNS1123Subdomain func thus we can easily create a group that is guaranteed never + // to be in the authenticator cache. Add a timestamp just to be extra sure. + const authenticatorCacheMissPrefix = "_INVALID_API_GROUP_" + authenticatorCacheMiss := authenticatorCacheMissPrefix + time.Now().UTC().String() + + // we do not have any defaulting functions for *loginv1alpha1.TokenCredentialRequest + // today, but we may have some in the future. Calling AddTypeDefaultingFunc overwrites + // any previously registered defaulting function. Thus to make sure that we catch + // a situation where we add a defaulting func, we attempt to call it here with a nil + // *loginv1alpha1.TokenCredentialRequest. This will do nothing when there is no + // defaulting func registered, but it will almost certainly panic if one is added. + scheme.Default((*loginv1alpha1.TokenCredentialRequest)(nil)) + + // on incoming requests, restore the authenticator API group to the standard group + // note that we are responsible for duplicating this logic for every external API version + scheme.AddTypeDefaultingFunc(&loginv1alpha1.TokenCredentialRequest{}, func(obj interface{}) { + credentialRequest := obj.(*loginv1alpha1.TokenCredentialRequest) + + if credentialRequest.Spec.Authenticator.APIGroup == nil { + // force a cache miss because this is an invalid request + plog.Debug("invalid token credential request, nil group", "authenticator", credentialRequest.Spec.Authenticator) + credentialRequest.Spec.Authenticator.APIGroup = &authenticatorCacheMiss + return + } + + restoredGroup, ok := groupsuffix.Unreplace(*credentialRequest.Spec.Authenticator.APIGroup, apiGroupSuffix) + if !ok { + // force a cache miss because this is an invalid request + plog.Debug("invalid token credential request, wrong group", "authenticator", credentialRequest.Spec.Authenticator) + credentialRequest.Spec.Authenticator.APIGroup = &authenticatorCacheMiss + return + } + + credentialRequest.Spec.Authenticator.APIGroup = &restoredGroup + }) + + return scheme, schema.GroupVersion(loginConciergeGroupData), schema.GroupVersion(identityConciergeGroupData) +} + +func addToSchemeAtNewGroup(scheme *runtime.Scheme, oldGroup, newGroup string, funcs ...func(*runtime.Scheme) error) { + // we need a temporary place to register our types to avoid double registering them + tmpScheme := runtime.NewScheme() + schemeBuilder := runtime.NewSchemeBuilder(funcs...) + utilruntime.Must(schemeBuilder.AddToScheme(tmpScheme)) + + for gvk := range tmpScheme.AllKnownTypes() { + if gvk.GroupVersion() == metav1.Unversioned { + continue // metav1.AddToGroupVersion registers types outside of our aggregated API group that we need to ignore + } + + if gvk.Group != oldGroup { + panic(fmt.Errorf("tmp scheme has type not in the old aggregated API group %s: %s", oldGroup, gvk)) // programmer error + } + + obj, err := tmpScheme.New(gvk) + if err != nil { + panic(err) // programmer error, scheme internal code is broken + } + newGVK := schema.GroupVersionKind{ + Group: newGroup, + Version: gvk.Version, + Kind: gvk.Kind, + } + + // register the existing type but with the new group in the correct scheme + scheme.AddKnownTypeWithName(newGVK, obj) + } +} diff --git a/internal/concierge/scheme/scheme_test.go b/internal/concierge/scheme/scheme_test.go new file mode 100644 index 000000000..dc50c2352 --- /dev/null +++ b/internal/concierge/scheme/scheme_test.go @@ -0,0 +1,240 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package scheme + +import ( + "reflect" + "strings" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + identityapi "go.pinniped.dev/generated/latest/apis/concierge/identity" + identityv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/identity/v1alpha1" + loginapi "go.pinniped.dev/generated/latest/apis/concierge/login" + loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1" +) + +func TestNew(t *testing.T) { + // the standard group + regularLoginGV := schema.GroupVersion{ + Group: "login.concierge.pinniped.dev", + Version: "v1alpha1", + } + regularLoginGVInternal := schema.GroupVersion{ + Group: "login.concierge.pinniped.dev", + Version: runtime.APIVersionInternal, + } + regularIdentityGV := schema.GroupVersion{ + Group: "identity.concierge.pinniped.dev", + Version: "v1alpha1", + } + regularIdentityGVInternal := schema.GroupVersion{ + Group: "identity.concierge.pinniped.dev", + Version: runtime.APIVersionInternal, + } + + // the canonical other group + otherLoginGV := schema.GroupVersion{ + Group: "login.concierge.walrus.tld", + Version: "v1alpha1", + } + otherLoginGVInternal := schema.GroupVersion{ + Group: "login.concierge.walrus.tld", + Version: runtime.APIVersionInternal, + } + otherIdentityGV := schema.GroupVersion{ + Group: "identity.concierge.walrus.tld", + Version: "v1alpha1", + } + otherIdentityGVInternal := schema.GroupVersion{ + Group: "identity.concierge.walrus.tld", + Version: runtime.APIVersionInternal, + } + + // kube's core internal + internalGV := schema.GroupVersion{ + Group: "", + Version: runtime.APIVersionInternal, + } + + tests := []struct { + name string + apiGroupSuffix string + want map[schema.GroupVersionKind]reflect.Type + wantLoginGroupVersion schema.GroupVersion + wantIdentityGroupVersion schema.GroupVersion + }{ + { + name: "regular api group", + apiGroupSuffix: "pinniped.dev", + want: map[schema.GroupVersionKind]reflect.Type{ + // all the types that are in the aggregated API group + + regularLoginGV.WithKind("TokenCredentialRequest"): reflect.TypeOf(&loginv1alpha1.TokenCredentialRequest{}).Elem(), + regularLoginGV.WithKind("TokenCredentialRequestList"): reflect.TypeOf(&loginv1alpha1.TokenCredentialRequestList{}).Elem(), + + regularLoginGVInternal.WithKind("TokenCredentialRequest"): reflect.TypeOf(&loginapi.TokenCredentialRequest{}).Elem(), + regularLoginGVInternal.WithKind("TokenCredentialRequestList"): reflect.TypeOf(&loginapi.TokenCredentialRequestList{}).Elem(), + + regularIdentityGV.WithKind("WhoAmIRequest"): reflect.TypeOf(&identityv1alpha1.WhoAmIRequest{}).Elem(), + regularIdentityGV.WithKind("WhoAmIRequestList"): reflect.TypeOf(&identityv1alpha1.WhoAmIRequestList{}).Elem(), + + regularIdentityGVInternal.WithKind("WhoAmIRequest"): reflect.TypeOf(&identityapi.WhoAmIRequest{}).Elem(), + regularIdentityGVInternal.WithKind("WhoAmIRequestList"): reflect.TypeOf(&identityapi.WhoAmIRequestList{}).Elem(), + + regularLoginGV.WithKind("CreateOptions"): reflect.TypeOf(&metav1.CreateOptions{}).Elem(), + regularLoginGV.WithKind("DeleteOptions"): reflect.TypeOf(&metav1.DeleteOptions{}).Elem(), + regularLoginGV.WithKind("ExportOptions"): reflect.TypeOf(&metav1.ExportOptions{}).Elem(), + regularLoginGV.WithKind("GetOptions"): reflect.TypeOf(&metav1.GetOptions{}).Elem(), + regularLoginGV.WithKind("ListOptions"): reflect.TypeOf(&metav1.ListOptions{}).Elem(), + regularLoginGV.WithKind("PatchOptions"): reflect.TypeOf(&metav1.PatchOptions{}).Elem(), + regularLoginGV.WithKind("UpdateOptions"): reflect.TypeOf(&metav1.UpdateOptions{}).Elem(), + regularLoginGV.WithKind("WatchEvent"): reflect.TypeOf(&metav1.WatchEvent{}).Elem(), + + regularIdentityGV.WithKind("CreateOptions"): reflect.TypeOf(&metav1.CreateOptions{}).Elem(), + regularIdentityGV.WithKind("DeleteOptions"): reflect.TypeOf(&metav1.DeleteOptions{}).Elem(), + regularIdentityGV.WithKind("ExportOptions"): reflect.TypeOf(&metav1.ExportOptions{}).Elem(), + regularIdentityGV.WithKind("GetOptions"): reflect.TypeOf(&metav1.GetOptions{}).Elem(), + regularIdentityGV.WithKind("ListOptions"): reflect.TypeOf(&metav1.ListOptions{}).Elem(), + regularIdentityGV.WithKind("PatchOptions"): reflect.TypeOf(&metav1.PatchOptions{}).Elem(), + regularIdentityGV.WithKind("UpdateOptions"): reflect.TypeOf(&metav1.UpdateOptions{}).Elem(), + regularIdentityGV.WithKind("WatchEvent"): reflect.TypeOf(&metav1.WatchEvent{}).Elem(), + + regularLoginGVInternal.WithKind("WatchEvent"): reflect.TypeOf(&metav1.InternalEvent{}).Elem(), + + regularIdentityGVInternal.WithKind("WatchEvent"): reflect.TypeOf(&metav1.InternalEvent{}).Elem(), + + // the types below this line do not really matter to us because they are in the core group + + internalGV.WithKind("WatchEvent"): reflect.TypeOf(&metav1.InternalEvent{}).Elem(), + + metav1.Unversioned.WithKind("APIGroup"): reflect.TypeOf(&metav1.APIGroup{}).Elem(), + metav1.Unversioned.WithKind("APIGroupList"): reflect.TypeOf(&metav1.APIGroupList{}).Elem(), + metav1.Unversioned.WithKind("APIResourceList"): reflect.TypeOf(&metav1.APIResourceList{}).Elem(), + metav1.Unversioned.WithKind("APIVersions"): reflect.TypeOf(&metav1.APIVersions{}).Elem(), + metav1.Unversioned.WithKind("CreateOptions"): reflect.TypeOf(&metav1.CreateOptions{}).Elem(), + metav1.Unversioned.WithKind("DeleteOptions"): reflect.TypeOf(&metav1.DeleteOptions{}).Elem(), + metav1.Unversioned.WithKind("ExportOptions"): reflect.TypeOf(&metav1.ExportOptions{}).Elem(), + metav1.Unversioned.WithKind("GetOptions"): reflect.TypeOf(&metav1.GetOptions{}).Elem(), + metav1.Unversioned.WithKind("ListOptions"): reflect.TypeOf(&metav1.ListOptions{}).Elem(), + metav1.Unversioned.WithKind("PatchOptions"): reflect.TypeOf(&metav1.PatchOptions{}).Elem(), + metav1.Unversioned.WithKind("Status"): reflect.TypeOf(&metav1.Status{}).Elem(), + metav1.Unversioned.WithKind("UpdateOptions"): reflect.TypeOf(&metav1.UpdateOptions{}).Elem(), + metav1.Unversioned.WithKind("WatchEvent"): reflect.TypeOf(&metav1.WatchEvent{}).Elem(), + }, + wantLoginGroupVersion: regularLoginGV, + wantIdentityGroupVersion: regularIdentityGV, + }, + { + name: "other api group", + apiGroupSuffix: "walrus.tld", + want: map[schema.GroupVersionKind]reflect.Type{ + // all the types that are in the aggregated API group + + otherLoginGV.WithKind("TokenCredentialRequest"): reflect.TypeOf(&loginv1alpha1.TokenCredentialRequest{}).Elem(), + otherLoginGV.WithKind("TokenCredentialRequestList"): reflect.TypeOf(&loginv1alpha1.TokenCredentialRequestList{}).Elem(), + + otherLoginGVInternal.WithKind("TokenCredentialRequest"): reflect.TypeOf(&loginapi.TokenCredentialRequest{}).Elem(), + otherLoginGVInternal.WithKind("TokenCredentialRequestList"): reflect.TypeOf(&loginapi.TokenCredentialRequestList{}).Elem(), + + otherIdentityGV.WithKind("WhoAmIRequest"): reflect.TypeOf(&identityv1alpha1.WhoAmIRequest{}).Elem(), + otherIdentityGV.WithKind("WhoAmIRequestList"): reflect.TypeOf(&identityv1alpha1.WhoAmIRequestList{}).Elem(), + + otherIdentityGVInternal.WithKind("WhoAmIRequest"): reflect.TypeOf(&identityapi.WhoAmIRequest{}).Elem(), + otherIdentityGVInternal.WithKind("WhoAmIRequestList"): reflect.TypeOf(&identityapi.WhoAmIRequestList{}).Elem(), + + otherLoginGV.WithKind("CreateOptions"): reflect.TypeOf(&metav1.CreateOptions{}).Elem(), + otherLoginGV.WithKind("DeleteOptions"): reflect.TypeOf(&metav1.DeleteOptions{}).Elem(), + otherLoginGV.WithKind("ExportOptions"): reflect.TypeOf(&metav1.ExportOptions{}).Elem(), + otherLoginGV.WithKind("GetOptions"): reflect.TypeOf(&metav1.GetOptions{}).Elem(), + otherLoginGV.WithKind("ListOptions"): reflect.TypeOf(&metav1.ListOptions{}).Elem(), + otherLoginGV.WithKind("PatchOptions"): reflect.TypeOf(&metav1.PatchOptions{}).Elem(), + otherLoginGV.WithKind("UpdateOptions"): reflect.TypeOf(&metav1.UpdateOptions{}).Elem(), + otherLoginGV.WithKind("WatchEvent"): reflect.TypeOf(&metav1.WatchEvent{}).Elem(), + + otherIdentityGV.WithKind("CreateOptions"): reflect.TypeOf(&metav1.CreateOptions{}).Elem(), + otherIdentityGV.WithKind("DeleteOptions"): reflect.TypeOf(&metav1.DeleteOptions{}).Elem(), + otherIdentityGV.WithKind("ExportOptions"): reflect.TypeOf(&metav1.ExportOptions{}).Elem(), + otherIdentityGV.WithKind("GetOptions"): reflect.TypeOf(&metav1.GetOptions{}).Elem(), + otherIdentityGV.WithKind("ListOptions"): reflect.TypeOf(&metav1.ListOptions{}).Elem(), + otherIdentityGV.WithKind("PatchOptions"): reflect.TypeOf(&metav1.PatchOptions{}).Elem(), + otherIdentityGV.WithKind("UpdateOptions"): reflect.TypeOf(&metav1.UpdateOptions{}).Elem(), + otherIdentityGV.WithKind("WatchEvent"): reflect.TypeOf(&metav1.WatchEvent{}).Elem(), + + otherLoginGVInternal.WithKind("WatchEvent"): reflect.TypeOf(&metav1.InternalEvent{}).Elem(), + + otherIdentityGVInternal.WithKind("WatchEvent"): reflect.TypeOf(&metav1.InternalEvent{}).Elem(), + + // the types below this line do not really matter to us because they are in the core group + + internalGV.WithKind("WatchEvent"): reflect.TypeOf(&metav1.InternalEvent{}).Elem(), + + metav1.Unversioned.WithKind("APIGroup"): reflect.TypeOf(&metav1.APIGroup{}).Elem(), + metav1.Unversioned.WithKind("APIGroupList"): reflect.TypeOf(&metav1.APIGroupList{}).Elem(), + metav1.Unversioned.WithKind("APIResourceList"): reflect.TypeOf(&metav1.APIResourceList{}).Elem(), + metav1.Unversioned.WithKind("APIVersions"): reflect.TypeOf(&metav1.APIVersions{}).Elem(), + metav1.Unversioned.WithKind("CreateOptions"): reflect.TypeOf(&metav1.CreateOptions{}).Elem(), + metav1.Unversioned.WithKind("DeleteOptions"): reflect.TypeOf(&metav1.DeleteOptions{}).Elem(), + metav1.Unversioned.WithKind("ExportOptions"): reflect.TypeOf(&metav1.ExportOptions{}).Elem(), + metav1.Unversioned.WithKind("GetOptions"): reflect.TypeOf(&metav1.GetOptions{}).Elem(), + metav1.Unversioned.WithKind("ListOptions"): reflect.TypeOf(&metav1.ListOptions{}).Elem(), + metav1.Unversioned.WithKind("PatchOptions"): reflect.TypeOf(&metav1.PatchOptions{}).Elem(), + metav1.Unversioned.WithKind("Status"): reflect.TypeOf(&metav1.Status{}).Elem(), + metav1.Unversioned.WithKind("UpdateOptions"): reflect.TypeOf(&metav1.UpdateOptions{}).Elem(), + metav1.Unversioned.WithKind("WatchEvent"): reflect.TypeOf(&metav1.WatchEvent{}).Elem(), + }, + wantLoginGroupVersion: otherLoginGV, + wantIdentityGroupVersion: otherIdentityGV, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + scheme, loginGV, identityGV := New(tt.apiGroupSuffix) + require.Equal(t, tt.want, scheme.AllKnownTypes()) + require.Equal(t, tt.wantLoginGroupVersion, loginGV) + require.Equal(t, tt.wantIdentityGroupVersion, identityGV) + + // make a credential request like a client would send + authenticationConciergeAPIGroup := "authentication.concierge." + tt.apiGroupSuffix + credentialRequest := &loginv1alpha1.TokenCredentialRequest{ + Spec: loginv1alpha1.TokenCredentialRequestSpec{ + Authenticator: corev1.TypedLocalObjectReference{ + APIGroup: &authenticationConciergeAPIGroup, + }, + }, + } + + // run defaulting on it + scheme.Default(credentialRequest) + + // make sure the group is restored if needed + require.Equal(t, "authentication.concierge.pinniped.dev", *credentialRequest.Spec.Authenticator.APIGroup) + + // make a credential request in the standard group + defaultAuthenticationConciergeAPIGroup := "authentication.concierge.pinniped.dev" + defaultCredentialRequest := &loginv1alpha1.TokenCredentialRequest{ + Spec: loginv1alpha1.TokenCredentialRequestSpec{ + Authenticator: corev1.TypedLocalObjectReference{ + APIGroup: &defaultAuthenticationConciergeAPIGroup, + }, + }, + } + + // run defaulting on it + scheme.Default(defaultCredentialRequest) + + if tt.apiGroupSuffix == "pinniped.dev" { // when using the standard group, this should just work + require.Equal(t, "authentication.concierge.pinniped.dev", *defaultCredentialRequest.Spec.Authenticator.APIGroup) + } else { // when using any other group, this should always be a cache miss + require.True(t, strings.HasPrefix(*defaultCredentialRequest.Spec.Authenticator.APIGroup, "_INVALID_API_GROUP_2")) + } + }) + } +} diff --git a/internal/concierge/server/server.go b/internal/concierge/server/server.go index 77769fca1..d5e51379b 100644 --- a/internal/concierge/server/server.go +++ b/internal/concierge/server/server.go @@ -11,27 +11,22 @@ import ( "time" "github.com/spf13/cobra" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" - identityapi "go.pinniped.dev/generated/latest/apis/concierge/identity" - identityv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/identity/v1alpha1" - loginapi "go.pinniped.dev/generated/latest/apis/concierge/login" - loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1" "go.pinniped.dev/internal/certauthority/dynamiccertauthority" "go.pinniped.dev/internal/concierge/apiserver" + conciergescheme "go.pinniped.dev/internal/concierge/scheme" "go.pinniped.dev/internal/config/concierge" "go.pinniped.dev/internal/controller/authenticator/authncache" "go.pinniped.dev/internal/controllermanager" "go.pinniped.dev/internal/downward" "go.pinniped.dev/internal/dynamiccert" - "go.pinniped.dev/internal/groupsuffix" "go.pinniped.dev/internal/here" + "go.pinniped.dev/internal/issuer" "go.pinniped.dev/internal/plog" "go.pinniped.dev/internal/registry/credentialrequest" ) @@ -119,40 +114,57 @@ func (a *App) runServer(ctx context.Context) error { // is stored in a k8s Secret. Therefore it also effectively acting as // an in-memory cache of what is stored in the k8s Secret, helping to // keep incoming requests fast. - dynamicServingCertProvider := dynamiccert.New() + dynamicServingCertProvider := dynamiccert.NewServingCert("concierge-serving-cert") - // This cert provider will be used to provide a signing key to the + // This cert provider will be used to provide the Kube signing key to the // cert issuer used to issue certs to Pinniped clients wishing to login. - dynamicSigningCertProvider := dynamiccert.New() + dynamicSigningCertProvider := dynamiccert.NewCA("concierge-kube-signing-cert") + + // This cert provider will be used to provide the impersonation proxy signing key to the + // cert issuer used to issue certs to Pinniped clients wishing to login. + impersonationProxySigningCertProvider := dynamiccert.NewCA("impersonation-proxy-signing-cert") + + // Get the "real" name of the login concierge API group (i.e., the API group name with the + // injected suffix). + scheme, loginGV, identityGV := conciergescheme.New(*cfg.APIGroupSuffix) // Prepare to start the controllers, but defer actually starting them until the // post start hook of the aggregated API server. startControllersFunc, err := controllermanager.PrepareControllers( &controllermanager.Config{ - ServerInstallationInfo: podInfo, - APIGroupSuffix: *cfg.APIGroupSuffix, - NamesConfig: &cfg.NamesConfig, - Labels: cfg.Labels, - KubeCertAgentConfig: &cfg.KubeCertAgentConfig, - DiscoveryURLOverride: cfg.DiscoveryInfo.URL, - DynamicServingCertProvider: dynamicServingCertProvider, - DynamicSigningCertProvider: dynamicSigningCertProvider, - ServingCertDuration: time.Duration(*cfg.APIConfig.ServingCertificateConfig.DurationSeconds) * time.Second, - ServingCertRenewBefore: time.Duration(*cfg.APIConfig.ServingCertificateConfig.RenewBeforeSeconds) * time.Second, - AuthenticatorCache: authenticators, + ServerInstallationInfo: podInfo, + APIGroupSuffix: *cfg.APIGroupSuffix, + NamesConfig: &cfg.NamesConfig, + Labels: cfg.Labels, + KubeCertAgentConfig: &cfg.KubeCertAgentConfig, + DiscoveryURLOverride: cfg.DiscoveryInfo.URL, + DynamicServingCertProvider: dynamicServingCertProvider, + DynamicSigningCertProvider: dynamicSigningCertProvider, + ImpersonationSigningCertProvider: impersonationProxySigningCertProvider, + ServingCertDuration: time.Duration(*cfg.APIConfig.ServingCertificateConfig.DurationSeconds) * time.Second, + ServingCertRenewBefore: time.Duration(*cfg.APIConfig.ServingCertificateConfig.RenewBeforeSeconds) * time.Second, + AuthenticatorCache: authenticators, }, ) if err != nil { return fmt.Errorf("could not prepare controllers: %w", err) } + certIssuer := issuer.ClientCertIssuers{ + dynamiccertauthority.New(dynamicSigningCertProvider), // attempt to use the real Kube CA if possible + dynamiccertauthority.New(impersonationProxySigningCertProvider), // fallback to our internal CA if we need to + } + // Get the aggregated API server config. aggregatedAPIServerConfig, err := getAggregatedAPIServerConfig( dynamicServingCertProvider, authenticators, - dynamiccertauthority.New(dynamicSigningCertProvider), + certIssuer, startControllersFunc, *cfg.APIGroupSuffix, + scheme, + loginGV, + identityGV, ) if err != nil { return fmt.Errorf("could not configure aggregated API server: %w", err) @@ -170,13 +182,14 @@ func (a *App) runServer(ctx context.Context) error { // Create a configuration for the aggregated API server. func getAggregatedAPIServerConfig( - dynamicCertProvider dynamiccert.Provider, + dynamicCertProvider dynamiccert.Private, authenticator credentialrequest.TokenCredentialRequestAuthenticator, - issuer credentialrequest.CertIssuer, + issuer issuer.ClientCertIssuer, startControllersPostStartHook func(context.Context), apiGroupSuffix string, + scheme *runtime.Scheme, + loginConciergeGroupVersion, identityConciergeGroupVersion schema.GroupVersion, ) (*apiserver.Config, error) { - scheme, loginConciergeGroupVersion, identityConciergeGroupVersion := getAggregatedAPIServerScheme(apiGroupSuffix) codecs := serializer.NewCodecFactory(scheme) // this is unused for now but it is a safe value that we could use in the future @@ -218,108 +231,3 @@ func getAggregatedAPIServerConfig( } return apiServerConfig, nil } - -func getAggregatedAPIServerScheme(apiGroupSuffix string) (_ *runtime.Scheme, login, identity schema.GroupVersion) { - // standard set up of the server side scheme - scheme := runtime.NewScheme() - - // add the options to empty v1 - metav1.AddToGroupVersion(scheme, metav1.Unversioned) - - // nothing fancy is required if using the standard group suffix - if apiGroupSuffix == groupsuffix.PinnipedDefaultSuffix { - schemeBuilder := runtime.NewSchemeBuilder( - loginv1alpha1.AddToScheme, - loginapi.AddToScheme, - identityv1alpha1.AddToScheme, - identityapi.AddToScheme, - ) - utilruntime.Must(schemeBuilder.AddToScheme(scheme)) - return scheme, loginv1alpha1.SchemeGroupVersion, identityv1alpha1.SchemeGroupVersion - } - - loginConciergeGroupData, identityConciergeGroupData := groupsuffix.ConciergeAggregatedGroups(apiGroupSuffix) - - addToSchemeAtNewGroup(scheme, loginv1alpha1.GroupName, loginConciergeGroupData.Group, loginv1alpha1.AddToScheme, loginapi.AddToScheme) - addToSchemeAtNewGroup(scheme, identityv1alpha1.GroupName, identityConciergeGroupData.Group, identityv1alpha1.AddToScheme, identityapi.AddToScheme) - - // manually register conversions and defaulting into the correct scheme since we cannot directly call AddToScheme - schemeBuilder := runtime.NewSchemeBuilder( - loginv1alpha1.RegisterConversions, - loginv1alpha1.RegisterDefaults, - identityv1alpha1.RegisterConversions, - identityv1alpha1.RegisterDefaults, - ) - utilruntime.Must(schemeBuilder.AddToScheme(scheme)) - - // we do not want to return errors from the scheme and instead would prefer to defer - // to the REST storage layer for consistency. The simplest way to do this is to force - // a cache miss from the authenticator cache. Kube API groups are validated via the - // IsDNS1123Subdomain func thus we can easily create a group that is guaranteed never - // to be in the authenticator cache. Add a timestamp just to be extra sure. - const authenticatorCacheMissPrefix = "_INVALID_API_GROUP_" - authenticatorCacheMiss := authenticatorCacheMissPrefix + time.Now().UTC().String() - - // we do not have any defaulting functions for *loginv1alpha1.TokenCredentialRequest - // today, but we may have some in the future. Calling AddTypeDefaultingFunc overwrites - // any previously registered defaulting function. Thus to make sure that we catch - // a situation where we add a defaulting func, we attempt to call it here with a nil - // *loginv1alpha1.TokenCredentialRequest. This will do nothing when there is no - // defaulting func registered, but it will almost certainly panic if one is added. - scheme.Default((*loginv1alpha1.TokenCredentialRequest)(nil)) - - // on incoming requests, restore the authenticator API group to the standard group - // note that we are responsible for duplicating this logic for every external API version - scheme.AddTypeDefaultingFunc(&loginv1alpha1.TokenCredentialRequest{}, func(obj interface{}) { - credentialRequest := obj.(*loginv1alpha1.TokenCredentialRequest) - - if credentialRequest.Spec.Authenticator.APIGroup == nil { - // force a cache miss because this is an invalid request - plog.Debug("invalid token credential request, nil group", "authenticator", credentialRequest.Spec.Authenticator) - credentialRequest.Spec.Authenticator.APIGroup = &authenticatorCacheMiss - return - } - - restoredGroup, ok := groupsuffix.Unreplace(*credentialRequest.Spec.Authenticator.APIGroup, apiGroupSuffix) - if !ok { - // force a cache miss because this is an invalid request - plog.Debug("invalid token credential request, wrong group", "authenticator", credentialRequest.Spec.Authenticator) - credentialRequest.Spec.Authenticator.APIGroup = &authenticatorCacheMiss - return - } - - credentialRequest.Spec.Authenticator.APIGroup = &restoredGroup - }) - - return scheme, schema.GroupVersion(loginConciergeGroupData), schema.GroupVersion(identityConciergeGroupData) -} - -func addToSchemeAtNewGroup(scheme *runtime.Scheme, oldGroup, newGroup string, funcs ...func(*runtime.Scheme) error) { - // we need a temporary place to register our types to avoid double registering them - tmpScheme := runtime.NewScheme() - schemeBuilder := runtime.NewSchemeBuilder(funcs...) - utilruntime.Must(schemeBuilder.AddToScheme(tmpScheme)) - - for gvk := range tmpScheme.AllKnownTypes() { - if gvk.GroupVersion() == metav1.Unversioned { - continue // metav1.AddToGroupVersion registers types outside of our aggregated API group that we need to ignore - } - - if gvk.Group != oldGroup { - panic(fmt.Errorf("tmp scheme has type not in the old aggregated API group %s: %s", oldGroup, gvk)) // programmer error - } - - obj, err := tmpScheme.New(gvk) - if err != nil { - panic(err) // programmer error, scheme internal code is broken - } - newGVK := schema.GroupVersionKind{ - Group: newGroup, - Version: gvk.Version, - Kind: gvk.Kind, - } - - // register the existing type but with the new group in the correct scheme - scheme.AddKnownTypeWithName(newGVK, obj) - } -} diff --git a/internal/concierge/server/server_test.go b/internal/concierge/server/server_test.go index 4eab0245f..0825cf5ba 100644 --- a/internal/concierge/server/server_test.go +++ b/internal/concierge/server/server_test.go @@ -6,22 +6,12 @@ package server import ( "bytes" "context" - "reflect" "strings" "testing" "github.com/google/go-cmp/cmp" "github.com/spf13/cobra" "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - - identityapi "go.pinniped.dev/generated/latest/apis/concierge/identity" - identityv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/identity/v1alpha1" - loginapi "go.pinniped.dev/generated/latest/apis/concierge/login" - loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1" ) const knownGoodUsage = ` @@ -98,222 +88,3 @@ func TestCommand(t *testing.T) { }) } } - -func Test_getAggregatedAPIServerScheme(t *testing.T) { - // the standard group - regularLoginGV := schema.GroupVersion{ - Group: "login.concierge.pinniped.dev", - Version: "v1alpha1", - } - regularLoginGVInternal := schema.GroupVersion{ - Group: "login.concierge.pinniped.dev", - Version: runtime.APIVersionInternal, - } - regularIdentityGV := schema.GroupVersion{ - Group: "identity.concierge.pinniped.dev", - Version: "v1alpha1", - } - regularIdentityGVInternal := schema.GroupVersion{ - Group: "identity.concierge.pinniped.dev", - Version: runtime.APIVersionInternal, - } - - // the canonical other group - otherLoginGV := schema.GroupVersion{ - Group: "login.concierge.walrus.tld", - Version: "v1alpha1", - } - otherLoginGVInternal := schema.GroupVersion{ - Group: "login.concierge.walrus.tld", - Version: runtime.APIVersionInternal, - } - otherIdentityGV := schema.GroupVersion{ - Group: "identity.concierge.walrus.tld", - Version: "v1alpha1", - } - otherIdentityGVInternal := schema.GroupVersion{ - Group: "identity.concierge.walrus.tld", - Version: runtime.APIVersionInternal, - } - - // kube's core internal - internalGV := schema.GroupVersion{ - Group: "", - Version: runtime.APIVersionInternal, - } - - tests := []struct { - name string - apiGroupSuffix string - want map[schema.GroupVersionKind]reflect.Type - wantLoginGroupVersion schema.GroupVersion - wantIdentityGroupVersion schema.GroupVersion - }{ - { - name: "regular api group", - apiGroupSuffix: "pinniped.dev", - want: map[schema.GroupVersionKind]reflect.Type{ - // all the types that are in the aggregated API group - - regularLoginGV.WithKind("TokenCredentialRequest"): reflect.TypeOf(&loginv1alpha1.TokenCredentialRequest{}).Elem(), - regularLoginGV.WithKind("TokenCredentialRequestList"): reflect.TypeOf(&loginv1alpha1.TokenCredentialRequestList{}).Elem(), - - regularLoginGVInternal.WithKind("TokenCredentialRequest"): reflect.TypeOf(&loginapi.TokenCredentialRequest{}).Elem(), - regularLoginGVInternal.WithKind("TokenCredentialRequestList"): reflect.TypeOf(&loginapi.TokenCredentialRequestList{}).Elem(), - - regularIdentityGV.WithKind("WhoAmIRequest"): reflect.TypeOf(&identityv1alpha1.WhoAmIRequest{}).Elem(), - regularIdentityGV.WithKind("WhoAmIRequestList"): reflect.TypeOf(&identityv1alpha1.WhoAmIRequestList{}).Elem(), - - regularIdentityGVInternal.WithKind("WhoAmIRequest"): reflect.TypeOf(&identityapi.WhoAmIRequest{}).Elem(), - regularIdentityGVInternal.WithKind("WhoAmIRequestList"): reflect.TypeOf(&identityapi.WhoAmIRequestList{}).Elem(), - - regularLoginGV.WithKind("CreateOptions"): reflect.TypeOf(&metav1.CreateOptions{}).Elem(), - regularLoginGV.WithKind("DeleteOptions"): reflect.TypeOf(&metav1.DeleteOptions{}).Elem(), - regularLoginGV.WithKind("ExportOptions"): reflect.TypeOf(&metav1.ExportOptions{}).Elem(), - regularLoginGV.WithKind("GetOptions"): reflect.TypeOf(&metav1.GetOptions{}).Elem(), - regularLoginGV.WithKind("ListOptions"): reflect.TypeOf(&metav1.ListOptions{}).Elem(), - regularLoginGV.WithKind("PatchOptions"): reflect.TypeOf(&metav1.PatchOptions{}).Elem(), - regularLoginGV.WithKind("UpdateOptions"): reflect.TypeOf(&metav1.UpdateOptions{}).Elem(), - regularLoginGV.WithKind("WatchEvent"): reflect.TypeOf(&metav1.WatchEvent{}).Elem(), - - regularIdentityGV.WithKind("CreateOptions"): reflect.TypeOf(&metav1.CreateOptions{}).Elem(), - regularIdentityGV.WithKind("DeleteOptions"): reflect.TypeOf(&metav1.DeleteOptions{}).Elem(), - regularIdentityGV.WithKind("ExportOptions"): reflect.TypeOf(&metav1.ExportOptions{}).Elem(), - regularIdentityGV.WithKind("GetOptions"): reflect.TypeOf(&metav1.GetOptions{}).Elem(), - regularIdentityGV.WithKind("ListOptions"): reflect.TypeOf(&metav1.ListOptions{}).Elem(), - regularIdentityGV.WithKind("PatchOptions"): reflect.TypeOf(&metav1.PatchOptions{}).Elem(), - regularIdentityGV.WithKind("UpdateOptions"): reflect.TypeOf(&metav1.UpdateOptions{}).Elem(), - regularIdentityGV.WithKind("WatchEvent"): reflect.TypeOf(&metav1.WatchEvent{}).Elem(), - - regularLoginGVInternal.WithKind("WatchEvent"): reflect.TypeOf(&metav1.InternalEvent{}).Elem(), - - regularIdentityGVInternal.WithKind("WatchEvent"): reflect.TypeOf(&metav1.InternalEvent{}).Elem(), - - // the types below this line do not really matter to us because they are in the core group - - internalGV.WithKind("WatchEvent"): reflect.TypeOf(&metav1.InternalEvent{}).Elem(), - - metav1.Unversioned.WithKind("APIGroup"): reflect.TypeOf(&metav1.APIGroup{}).Elem(), - metav1.Unversioned.WithKind("APIGroupList"): reflect.TypeOf(&metav1.APIGroupList{}).Elem(), - metav1.Unversioned.WithKind("APIResourceList"): reflect.TypeOf(&metav1.APIResourceList{}).Elem(), - metav1.Unversioned.WithKind("APIVersions"): reflect.TypeOf(&metav1.APIVersions{}).Elem(), - metav1.Unversioned.WithKind("CreateOptions"): reflect.TypeOf(&metav1.CreateOptions{}).Elem(), - metav1.Unversioned.WithKind("DeleteOptions"): reflect.TypeOf(&metav1.DeleteOptions{}).Elem(), - metav1.Unversioned.WithKind("ExportOptions"): reflect.TypeOf(&metav1.ExportOptions{}).Elem(), - metav1.Unversioned.WithKind("GetOptions"): reflect.TypeOf(&metav1.GetOptions{}).Elem(), - metav1.Unversioned.WithKind("ListOptions"): reflect.TypeOf(&metav1.ListOptions{}).Elem(), - metav1.Unversioned.WithKind("PatchOptions"): reflect.TypeOf(&metav1.PatchOptions{}).Elem(), - metav1.Unversioned.WithKind("Status"): reflect.TypeOf(&metav1.Status{}).Elem(), - metav1.Unversioned.WithKind("UpdateOptions"): reflect.TypeOf(&metav1.UpdateOptions{}).Elem(), - metav1.Unversioned.WithKind("WatchEvent"): reflect.TypeOf(&metav1.WatchEvent{}).Elem(), - }, - wantLoginGroupVersion: regularLoginGV, - wantIdentityGroupVersion: regularIdentityGV, - }, - { - name: "other api group", - apiGroupSuffix: "walrus.tld", - want: map[schema.GroupVersionKind]reflect.Type{ - // all the types that are in the aggregated API group - - otherLoginGV.WithKind("TokenCredentialRequest"): reflect.TypeOf(&loginv1alpha1.TokenCredentialRequest{}).Elem(), - otherLoginGV.WithKind("TokenCredentialRequestList"): reflect.TypeOf(&loginv1alpha1.TokenCredentialRequestList{}).Elem(), - - otherLoginGVInternal.WithKind("TokenCredentialRequest"): reflect.TypeOf(&loginapi.TokenCredentialRequest{}).Elem(), - otherLoginGVInternal.WithKind("TokenCredentialRequestList"): reflect.TypeOf(&loginapi.TokenCredentialRequestList{}).Elem(), - - otherIdentityGV.WithKind("WhoAmIRequest"): reflect.TypeOf(&identityv1alpha1.WhoAmIRequest{}).Elem(), - otherIdentityGV.WithKind("WhoAmIRequestList"): reflect.TypeOf(&identityv1alpha1.WhoAmIRequestList{}).Elem(), - - otherIdentityGVInternal.WithKind("WhoAmIRequest"): reflect.TypeOf(&identityapi.WhoAmIRequest{}).Elem(), - otherIdentityGVInternal.WithKind("WhoAmIRequestList"): reflect.TypeOf(&identityapi.WhoAmIRequestList{}).Elem(), - - otherLoginGV.WithKind("CreateOptions"): reflect.TypeOf(&metav1.CreateOptions{}).Elem(), - otherLoginGV.WithKind("DeleteOptions"): reflect.TypeOf(&metav1.DeleteOptions{}).Elem(), - otherLoginGV.WithKind("ExportOptions"): reflect.TypeOf(&metav1.ExportOptions{}).Elem(), - otherLoginGV.WithKind("GetOptions"): reflect.TypeOf(&metav1.GetOptions{}).Elem(), - otherLoginGV.WithKind("ListOptions"): reflect.TypeOf(&metav1.ListOptions{}).Elem(), - otherLoginGV.WithKind("PatchOptions"): reflect.TypeOf(&metav1.PatchOptions{}).Elem(), - otherLoginGV.WithKind("UpdateOptions"): reflect.TypeOf(&metav1.UpdateOptions{}).Elem(), - otherLoginGV.WithKind("WatchEvent"): reflect.TypeOf(&metav1.WatchEvent{}).Elem(), - - otherIdentityGV.WithKind("CreateOptions"): reflect.TypeOf(&metav1.CreateOptions{}).Elem(), - otherIdentityGV.WithKind("DeleteOptions"): reflect.TypeOf(&metav1.DeleteOptions{}).Elem(), - otherIdentityGV.WithKind("ExportOptions"): reflect.TypeOf(&metav1.ExportOptions{}).Elem(), - otherIdentityGV.WithKind("GetOptions"): reflect.TypeOf(&metav1.GetOptions{}).Elem(), - otherIdentityGV.WithKind("ListOptions"): reflect.TypeOf(&metav1.ListOptions{}).Elem(), - otherIdentityGV.WithKind("PatchOptions"): reflect.TypeOf(&metav1.PatchOptions{}).Elem(), - otherIdentityGV.WithKind("UpdateOptions"): reflect.TypeOf(&metav1.UpdateOptions{}).Elem(), - otherIdentityGV.WithKind("WatchEvent"): reflect.TypeOf(&metav1.WatchEvent{}).Elem(), - - otherLoginGVInternal.WithKind("WatchEvent"): reflect.TypeOf(&metav1.InternalEvent{}).Elem(), - - otherIdentityGVInternal.WithKind("WatchEvent"): reflect.TypeOf(&metav1.InternalEvent{}).Elem(), - - // the types below this line do not really matter to us because they are in the core group - - internalGV.WithKind("WatchEvent"): reflect.TypeOf(&metav1.InternalEvent{}).Elem(), - - metav1.Unversioned.WithKind("APIGroup"): reflect.TypeOf(&metav1.APIGroup{}).Elem(), - metav1.Unversioned.WithKind("APIGroupList"): reflect.TypeOf(&metav1.APIGroupList{}).Elem(), - metav1.Unversioned.WithKind("APIResourceList"): reflect.TypeOf(&metav1.APIResourceList{}).Elem(), - metav1.Unversioned.WithKind("APIVersions"): reflect.TypeOf(&metav1.APIVersions{}).Elem(), - metav1.Unversioned.WithKind("CreateOptions"): reflect.TypeOf(&metav1.CreateOptions{}).Elem(), - metav1.Unversioned.WithKind("DeleteOptions"): reflect.TypeOf(&metav1.DeleteOptions{}).Elem(), - metav1.Unversioned.WithKind("ExportOptions"): reflect.TypeOf(&metav1.ExportOptions{}).Elem(), - metav1.Unversioned.WithKind("GetOptions"): reflect.TypeOf(&metav1.GetOptions{}).Elem(), - metav1.Unversioned.WithKind("ListOptions"): reflect.TypeOf(&metav1.ListOptions{}).Elem(), - metav1.Unversioned.WithKind("PatchOptions"): reflect.TypeOf(&metav1.PatchOptions{}).Elem(), - metav1.Unversioned.WithKind("Status"): reflect.TypeOf(&metav1.Status{}).Elem(), - metav1.Unversioned.WithKind("UpdateOptions"): reflect.TypeOf(&metav1.UpdateOptions{}).Elem(), - metav1.Unversioned.WithKind("WatchEvent"): reflect.TypeOf(&metav1.WatchEvent{}).Elem(), - }, - wantLoginGroupVersion: otherLoginGV, - wantIdentityGroupVersion: otherIdentityGV, - }, - } - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - scheme, loginGV, identityGV := getAggregatedAPIServerScheme(tt.apiGroupSuffix) - require.Equal(t, tt.want, scheme.AllKnownTypes()) - require.Equal(t, tt.wantLoginGroupVersion, loginGV) - require.Equal(t, tt.wantIdentityGroupVersion, identityGV) - - // make a credential request like a client would send - authenticationConciergeAPIGroup := "authentication.concierge." + tt.apiGroupSuffix - credentialRequest := &loginv1alpha1.TokenCredentialRequest{ - Spec: loginv1alpha1.TokenCredentialRequestSpec{ - Authenticator: corev1.TypedLocalObjectReference{ - APIGroup: &authenticationConciergeAPIGroup, - }, - }, - } - - // run defaulting on it - scheme.Default(credentialRequest) - - // make sure the group is restored if needed - require.Equal(t, "authentication.concierge.pinniped.dev", *credentialRequest.Spec.Authenticator.APIGroup) - - // make a credential request in the standard group - defaultAuthenticationConciergeAPIGroup := "authentication.concierge.pinniped.dev" - defaultCredentialRequest := &loginv1alpha1.TokenCredentialRequest{ - Spec: loginv1alpha1.TokenCredentialRequestSpec{ - Authenticator: corev1.TypedLocalObjectReference{ - APIGroup: &defaultAuthenticationConciergeAPIGroup, - }, - }, - } - - // run defaulting on it - scheme.Default(defaultCredentialRequest) - - if tt.apiGroupSuffix == "pinniped.dev" { // when using the standard group, this should just work - require.Equal(t, "authentication.concierge.pinniped.dev", *defaultCredentialRequest.Spec.Authenticator.APIGroup) - } else { // when using any other group, this should always be a cache miss - require.True(t, strings.HasPrefix(*defaultCredentialRequest.Spec.Authenticator.APIGroup, "_INVALID_API_GROUP_2")) - } - }) - } -} diff --git a/internal/config/concierge/config.go b/internal/config/concierge/config.go index 88d8fe06b..b4d4c9a5e 100644 --- a/internal/config/concierge/config.go +++ b/internal/config/concierge/config.go @@ -96,17 +96,31 @@ func maybeSetKubeCertAgentDefaults(cfg *KubeCertAgentSpec) { func validateNames(names *NamesConfigSpec) error { missingNames := []string{} if names == nil { - missingNames = append(missingNames, "servingCertificateSecret", "credentialIssuer", "apiService") - } else { - if names.ServingCertificateSecret == "" { - missingNames = append(missingNames, "servingCertificateSecret") - } - if names.CredentialIssuer == "" { - missingNames = append(missingNames, "credentialIssuer") - } - if names.APIService == "" { - missingNames = append(missingNames, "apiService") - } + names = &NamesConfigSpec{} + } + if names.ServingCertificateSecret == "" { + missingNames = append(missingNames, "servingCertificateSecret") + } + if names.CredentialIssuer == "" { + missingNames = append(missingNames, "credentialIssuer") + } + if names.APIService == "" { + missingNames = append(missingNames, "apiService") + } + if names.ImpersonationConfigMap == "" { + missingNames = append(missingNames, "impersonationConfigMap") + } + if names.ImpersonationLoadBalancerService == "" { + missingNames = append(missingNames, "impersonationLoadBalancerService") + } + if names.ImpersonationTLSCertificateSecret == "" { + missingNames = append(missingNames, "impersonationTLSCertificateSecret") + } + if names.ImpersonationCACertificateSecret == "" { + missingNames = append(missingNames, "impersonationCACertificateSecret") + } + if names.ImpersonationSignerSecret == "" { + missingNames = append(missingNames, "impersonationSignerSecret") } if len(missingNames) > 0 { return constable.Error("missing required names: " + strings.Join(missingNames, ", ")) diff --git a/internal/config/concierge/config_test.go b/internal/config/concierge/config_test.go index 4073d96d6..0034d06b4 100644 --- a/internal/config/concierge/config_test.go +++ b/internal/config/concierge/config_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/require" "go.pinniped.dev/internal/here" + "go.pinniped.dev/internal/plog" ) func TestFromPath(t *testing.T) { @@ -21,7 +22,7 @@ func TestFromPath(t *testing.T) { wantError string }{ { - name: "Happy", + name: "Fully filled out", yaml: here.Doc(` --- discovery: @@ -36,13 +37,20 @@ func TestFromPath(t *testing.T) { credentialIssuer: pinniped-config apiService: pinniped-api kubeCertAgentPrefix: kube-cert-agent-prefix + impersonationConfigMap: impersonationConfigMap-value + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value + impersonationSignerSecret: impersonationSignerSecret-value labels: myLabelKey1: myLabelValue1 myLabelKey2: myLabelValue2 - KubeCertAgent: + kubeCertAgent: namePrefix: kube-cert-agent-name-prefix- image: kube-cert-agent-image imagePullSecrets: [kube-cert-agent-image-pull-secret] + logLevel: debug `), wantConfig: &Config{ DiscoveryInfo: DiscoveryInfoSpec{ @@ -56,9 +64,14 @@ func TestFromPath(t *testing.T) { }, APIGroupSuffix: stringPtr("some.suffix.com"), NamesConfig: NamesConfigSpec{ - ServingCertificateSecret: "pinniped-concierge-api-tls-serving-certificate", - CredentialIssuer: "pinniped-config", - APIService: "pinniped-api", + ServingCertificateSecret: "pinniped-concierge-api-tls-serving-certificate", + CredentialIssuer: "pinniped-config", + APIService: "pinniped-api", + ImpersonationConfigMap: "impersonationConfigMap-value", + ImpersonationLoadBalancerService: "impersonationLoadBalancerService-value", + ImpersonationTLSCertificateSecret: "impersonationTLSCertificateSecret-value", + ImpersonationCACertificateSecret: "impersonationCACertificateSecret-value", + ImpersonationSignerSecret: "impersonationSignerSecret-value", }, Labels: map[string]string{ "myLabelKey1": "myLabelValue1", @@ -69,6 +82,7 @@ func TestFromPath(t *testing.T) { Image: stringPtr("kube-cert-agent-image"), ImagePullSecrets: []string{"kube-cert-agent-image-pull-secret"}, }, + LogLevel: plog.LevelDebug, }, }, { @@ -79,6 +93,11 @@ func TestFromPath(t *testing.T) { servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate credentialIssuer: pinniped-config apiService: pinniped-api + impersonationConfigMap: impersonationConfigMap-value + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value `), wantConfig: &Config{ DiscoveryInfo: DiscoveryInfoSpec{ @@ -92,9 +111,14 @@ func TestFromPath(t *testing.T) { }, }, NamesConfig: NamesConfigSpec{ - ServingCertificateSecret: "pinniped-concierge-api-tls-serving-certificate", - CredentialIssuer: "pinniped-config", - APIService: "pinniped-api", + ServingCertificateSecret: "pinniped-concierge-api-tls-serving-certificate", + CredentialIssuer: "pinniped-config", + APIService: "pinniped-api", + ImpersonationConfigMap: "impersonationConfigMap-value", + ImpersonationLoadBalancerService: "impersonationLoadBalancerService-value", + ImpersonationTLSCertificateSecret: "impersonationTLSCertificateSecret-value", + ImpersonationCACertificateSecret: "impersonationCACertificateSecret-value", + ImpersonationSignerSecret: "impersonationSignerSecret-value", }, Labels: map[string]string{}, KubeCertAgentConfig: KubeCertAgentSpec{ @@ -104,9 +128,12 @@ func TestFromPath(t *testing.T) { }, }, { - name: "Empty", - yaml: here.Doc(``), - wantError: "validate names: missing required names: servingCertificateSecret, credentialIssuer, apiService", + name: "Empty", + yaml: here.Doc(``), + wantError: "validate names: missing required names: servingCertificateSecret, credentialIssuer, " + + "apiService, impersonationConfigMap, impersonationLoadBalancerService, " + + "impersonationTLSCertificateSecret, impersonationCACertificateSecret, " + + "impersonationSignerSecret", }, { name: "Missing apiService name", @@ -115,6 +142,11 @@ func TestFromPath(t *testing.T) { names: servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate credentialIssuer: pinniped-config + impersonationConfigMap: impersonationConfigMap-value + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value `), wantError: "validate names: missing required names: apiService", }, @@ -125,6 +157,11 @@ func TestFromPath(t *testing.T) { names: servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate apiService: pinniped-api + impersonationConfigMap: impersonationConfigMap-value + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value `), wantError: "validate names: missing required names: credentialIssuer", }, @@ -135,9 +172,103 @@ func TestFromPath(t *testing.T) { names: credentialIssuer: pinniped-config apiService: pinniped-api + impersonationConfigMap: impersonationConfigMap-value + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value `), wantError: "validate names: missing required names: servingCertificateSecret", }, + { + name: "Missing impersonationConfigMap name", + yaml: here.Doc(` + --- + names: + servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate + credentialIssuer: pinniped-config + apiService: pinniped-api + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value + `), + wantError: "validate names: missing required names: impersonationConfigMap", + }, + { + name: "Missing impersonationLoadBalancerService name", + yaml: here.Doc(` + --- + names: + servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate + credentialIssuer: pinniped-config + apiService: pinniped-api + impersonationConfigMap: impersonationConfigMap-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value + `), + wantError: "validate names: missing required names: impersonationLoadBalancerService", + }, + { + name: "Missing impersonationTLSCertificateSecret name", + yaml: here.Doc(` + --- + names: + servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate + credentialIssuer: pinniped-config + apiService: pinniped-api + impersonationConfigMap: impersonationConfigMap-value + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value + `), + wantError: "validate names: missing required names: impersonationTLSCertificateSecret", + }, + { + name: "Missing impersonationCACertificateSecret name", + yaml: here.Doc(` + --- + names: + servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate + credentialIssuer: pinniped-config + apiService: pinniped-api + impersonationConfigMap: impersonationConfigMap-value + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value + `), + wantError: "validate names: missing required names: impersonationCACertificateSecret", + }, + { + name: "Missing impersonationSignerSecret name", + yaml: here.Doc(` + --- + names: + servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate + credentialIssuer: pinniped-config + apiService: pinniped-api + impersonationConfigMap: impersonationConfigMap-value + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value + `), + wantError: "validate names: missing required names: impersonationSignerSecret", + }, + { + name: "Missing several required names", + yaml: here.Doc(` + --- + names: + servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate + credentialIssuer: pinniped-config + apiService: pinniped-api + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationSignerSecret: impersonationSignerSecret-value + `), + wantError: "validate names: missing required names: impersonationConfigMap, " + + "impersonationTLSCertificateSecret, impersonationCACertificateSecret", + }, { name: "InvalidDurationRenewBefore", yaml: here.Doc(` @@ -150,6 +281,11 @@ func TestFromPath(t *testing.T) { servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate credentialIssuer: pinniped-config apiService: pinniped-api + impersonationConfigMap: impersonationConfigMap-value + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value `), wantError: "validate api: durationSeconds cannot be smaller than renewBeforeSeconds", }, @@ -165,6 +301,11 @@ func TestFromPath(t *testing.T) { servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate credentialIssuer: pinniped-config apiService: pinniped-api + impersonationConfigMap: impersonationConfigMap-value + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value `), wantError: "validate api: renewBefore must be positive", }, @@ -180,6 +321,11 @@ func TestFromPath(t *testing.T) { servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate credentialIssuer: pinniped-config apiService: pinniped-api + impersonationConfigMap: impersonationConfigMap-value + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value `), wantError: "validate api: renewBefore must be positive", }, @@ -196,6 +342,11 @@ func TestFromPath(t *testing.T) { servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate credentialIssuer: pinniped-config apiService: pinniped-api + impersonationConfigMap: impersonationConfigMap-value + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value `), wantError: "validate apiGroupSuffix: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')", }, diff --git a/internal/config/concierge/types.go b/internal/config/concierge/types.go index 1f402a34d..cfec621ef 100644 --- a/internal/config/concierge/types.go +++ b/internal/config/concierge/types.go @@ -33,9 +33,14 @@ type APIConfigSpec struct { // NamesConfigSpec configures the names of some Kubernetes resources for the Concierge. type NamesConfigSpec struct { - ServingCertificateSecret string `json:"servingCertificateSecret"` - CredentialIssuer string `json:"credentialIssuer"` - APIService string `json:"apiService"` + ServingCertificateSecret string `json:"servingCertificateSecret"` + CredentialIssuer string `json:"credentialIssuer"` + APIService string `json:"apiService"` + ImpersonationConfigMap string `json:"impersonationConfigMap"` + ImpersonationLoadBalancerService string `json:"impersonationLoadBalancerService"` + ImpersonationTLSCertificateSecret string `json:"impersonationTLSCertificateSecret"` + ImpersonationCACertificateSecret string `json:"impersonationCACertificateSecret"` + ImpersonationSignerSecret string `json:"impersonationSignerSecret"` } // ServingCertificateConfigSpec contains the configuration knobs for the API's diff --git a/internal/controller/apicerts/apiservice_updater.go b/internal/controller/apicerts/apiservice_updater.go index 4e28e519f..3cfe4fcc6 100644 --- a/internal/controller/apicerts/apiservice_updater.go +++ b/internal/controller/apicerts/apiservice_updater.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package apicerts @@ -64,7 +64,7 @@ func (c *apiServiceUpdaterController) Sync(ctx controllerlib.Context) error { } // Update the APIService to give it the new CA bundle. - if err := UpdateAPIService(ctx.Context, c.aggregatorClient, c.apiServiceName, c.namespace, certSecret.Data[caCertificateSecretKey]); err != nil { + if err := UpdateAPIService(ctx.Context, c.aggregatorClient, c.apiServiceName, c.namespace, certSecret.Data[CACertificateSecretKey]); err != nil { return fmt.Errorf("could not update the API service: %w", err) } diff --git a/internal/controller/apicerts/apiservice_updater_test.go b/internal/controller/apicerts/apiservice_updater_test.go index 8454170fa..643142159 100644 --- a/internal/controller/apicerts/apiservice_updater_test.go +++ b/internal/controller/apicerts/apiservice_updater_test.go @@ -7,7 +7,6 @@ import ( "context" "errors" "testing" - "time" "github.com/sclevine/spec" "github.com/sclevine/spec/report" @@ -112,8 +111,8 @@ func TestAPIServiceUpdaterControllerSync(t *testing.T) { var aggregatorAPIClient *aggregatorfake.Clientset var kubeInformerClient *kubernetesfake.Clientset var kubeInformers kubeinformers.SharedInformerFactory - var timeoutContext context.Context - var timeoutContextCancel context.CancelFunc + var cancelContext context.Context + var cancelContextCancelFunc context.CancelFunc var syncContext *controllerlib.Context // Defer starting the informers until the last possible moment so that the @@ -131,7 +130,7 @@ func TestAPIServiceUpdaterControllerSync(t *testing.T) { // Set this at the last second to support calling subject.Name(). syncContext = &controllerlib.Context{ - Context: timeoutContext, + Context: cancelContext, Name: subject.Name(), Key: controllerlib.Key{ Namespace: installedInNamespace, @@ -140,14 +139,14 @@ func TestAPIServiceUpdaterControllerSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - kubeInformers.Start(timeoutContext.Done()) + kubeInformers.Start(cancelContext.Done()) controllerlib.TestRunSynchronously(t, subject) } it.Before(func() { r = require.New(t) - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) kubeInformerClient = kubernetesfake.NewSimpleClientset() kubeInformers = kubeinformers.NewSharedInformerFactory(kubeInformerClient, 0) @@ -155,7 +154,7 @@ func TestAPIServiceUpdaterControllerSync(t *testing.T) { }) it.After(func() { - timeoutContextCancel() + cancelContextCancelFunc() }) when("there is not yet a serving cert Secret in the installation namespace or it was deleted", func() { diff --git a/internal/controller/apicerts/certs_expirer.go b/internal/controller/apicerts/certs_expirer.go index 336f619f7..2b643c3e3 100644 --- a/internal/controller/apicerts/certs_expirer.go +++ b/internal/controller/apicerts/certs_expirer.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package apicerts @@ -30,6 +30,8 @@ type certsExpirerController struct { // renewBefore is the amount of time after the cert's issuance where // this controller will start to try to rotate it. renewBefore time.Duration + + secretKey string } // NewCertsExpirerController returns a controllerlib.Controller that will delete a @@ -42,6 +44,7 @@ func NewCertsExpirerController( secretInformer corev1informers.SecretInformer, withInformer pinnipedcontroller.WithInformerOptionFunc, renewBefore time.Duration, + secretKey string, ) controllerlib.Controller { return controllerlib.New( controllerlib.Config{ @@ -52,6 +55,7 @@ func NewCertsExpirerController( k8sClient: k8sClient, secretInformer: secretInformer, renewBefore: renewBefore, + secretKey: secretKey, }, }, withInformer( @@ -74,13 +78,9 @@ func (c *certsExpirerController) Sync(ctx controllerlib.Context) error { return nil } - notBefore, notAfter, err := getCertBounds(secret) + notBefore, notAfter, err := c.getCertBounds(secret) if err != nil { - // If we can't read the cert, then really all we can do is log something, - // since if we returned an error then the controller lib would just call us - // again and again, which would probably yield the same results. - klog.Warningf("certsExpirerController Sync found that the secret is malformed: %s", err.Error()) - return nil + return fmt.Errorf("failed to get cert bounds for secret %q with key %q: %w", secret.Name, c.secretKey, err) } certAge := time.Since(notBefore) @@ -105,8 +105,8 @@ func (c *certsExpirerController) Sync(ctx controllerlib.Context) error { // certificate in the provided secret, or an error. Not that it expects the // provided secret to contain the well-known data keys from this package (see // certs_manager.go). -func getCertBounds(secret *corev1.Secret) (time.Time, time.Time, error) { - certPEM := secret.Data[tlsCertificateChainSecretKey] +func (c *certsExpirerController) getCertBounds(secret *corev1.Secret) (time.Time, time.Time, error) { + certPEM := secret.Data[c.secretKey] if certPEM == nil { return time.Time{}, time.Time{}, constable.Error("failed to find certificate") } diff --git a/internal/controller/apicerts/certs_expirer_test.go b/internal/controller/apicerts/certs_expirer_test.go index 8ba490fa5..7e82819a8 100644 --- a/internal/controller/apicerts/certs_expirer_test.go +++ b/internal/controller/apicerts/certs_expirer_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package apicerts @@ -98,7 +98,8 @@ func TestExpirerControllerFilters(t *testing.T) { nil, // k8sClient, not needed secretsInformer, withInformer.WithInformer, - 0, // renewBefore, not needed + 0, // renewBefore, not needed + "", // not needed ) unrelated := corev1.Secret{} @@ -115,6 +116,7 @@ func TestExpirerControllerSync(t *testing.T) { t.Parallel() const certsSecretResourceName = "some-resource-name" + const fakeTestKey = "some-awesome-key" tests := []struct { name string @@ -132,6 +134,7 @@ func TestExpirerControllerSync(t *testing.T) { name: "secret missing key", fillSecretData: func(t *testing.T, m map[string][]byte) {}, wantDelete: false, + wantError: `failed to get cert bounds for secret "some-resource-name" with key "some-awesome-key": failed to find certificate`, }, { name: "lifetime below threshold", @@ -143,8 +146,7 @@ func TestExpirerControllerSync(t *testing.T) { ) require.NoError(t, err) - // See certs_manager.go for this constant. - m["tlsCertificateChain"] = certPEM + m[fakeTestKey] = certPEM }, wantDelete: false, }, @@ -158,8 +160,7 @@ func TestExpirerControllerSync(t *testing.T) { ) require.NoError(t, err) - // See certs_manager.go for this constant. - m["tlsCertificateChain"] = certPEM + m[fakeTestKey] = certPEM }, wantDelete: true, }, @@ -173,8 +174,7 @@ func TestExpirerControllerSync(t *testing.T) { ) require.NoError(t, err) - // See certs_manager.go for this constant. - m["tlsCertificateChain"] = certPEM + m[fakeTestKey] = certPEM }, wantDelete: true, }, @@ -188,8 +188,7 @@ func TestExpirerControllerSync(t *testing.T) { ) require.NoError(t, err) - // See certs_manager.go for this constant. - m["tlsCertificateChain"] = certPEM + m[fakeTestKey] = certPEM }, configKubeAPIClient: func(c *kubernetesfake.Clientset) { c.PrependReactor("delete", "secrets", func(_ kubetesting.Action) (bool, runtime.Object, error) { @@ -204,11 +203,11 @@ func TestExpirerControllerSync(t *testing.T) { privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) require.NoError(t, err) - // See certs_manager.go for this constant. - m["tlsCertificateChain"], err = x509.MarshalPKCS8PrivateKey(privateKey) + m[fakeTestKey], err = x509.MarshalPKCS8PrivateKey(privateKey) require.NoError(t, err) }, wantDelete: false, + wantError: `failed to get cert bounds for secret "some-resource-name" with key "some-awesome-key": failed to decode certificate PEM`, }, } for _, test := range tests { @@ -216,7 +215,7 @@ func TestExpirerControllerSync(t *testing.T) { t.Run(test.name, func(t *testing.T) { t.Parallel() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() kubeAPIClient := kubernetesfake.NewSimpleClientset() @@ -253,6 +252,7 @@ func TestExpirerControllerSync(t *testing.T) { kubeInformers.Core().V1().Secrets(), controllerlib.WithInformer, test.renewBefore, + fakeTestKey, ) // Must start informers before calling TestRunSynchronously(). diff --git a/internal/controller/apicerts/certs_manager.go b/internal/controller/apicerts/certs_manager.go index c0be78731..c3d8bf361 100644 --- a/internal/controller/apicerts/certs_manager.go +++ b/internal/controller/apicerts/certs_manager.go @@ -1,10 +1,9 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package apicerts import ( - "crypto/x509/pkix" "fmt" "time" @@ -21,9 +20,10 @@ import ( ) const ( - caCertificateSecretKey = "caCertificate" - tlsPrivateKeySecretKey = "tlsPrivateKey" - tlsCertificateChainSecretKey = "tlsCertificateChain" + CACertificateSecretKey = "caCertificate" + CACertificatePrivateKeySecretKey = "caCertificatePrivateKey" + tlsPrivateKeySecretKey = "tlsPrivateKey" + TLSCertificateChainSecretKey = "tlsCertificateChain" ) type certsManagerController struct { @@ -93,28 +93,16 @@ func (c *certsManagerController) Sync(ctx controllerlib.Context) error { } // Create a CA. - aggregatedAPIServerCA, err := certauthority.New(pkix.Name{CommonName: c.generatedCACommonName}, c.certDuration) + ca, err := certauthority.New(c.generatedCACommonName, c.certDuration) if err != nil { return fmt.Errorf("could not initialize CA: %w", err) } - // Using the CA from above, create a TLS server cert for the aggregated API server to use. - serviceEndpoint := c.serviceNameForGeneratedCertCommonName + "." + c.namespace + ".svc" - aggregatedAPIServerTLSCert, err := aggregatedAPIServerCA.Issue( - pkix.Name{CommonName: serviceEndpoint}, - []string{serviceEndpoint}, - nil, - c.certDuration, - ) + caPrivateKeyPEM, err := ca.PrivateKeyToPEM() if err != nil { - return fmt.Errorf("could not issue serving certificate: %w", err) + return fmt.Errorf("could not get CA private key: %w", err) } - // Write the CA's public key bundle and the serving certs to a secret. - tlsCertChainPEM, tlsPrivateKeyPEM, err := certauthority.ToPEM(aggregatedAPIServerTLSCert) - if err != nil { - return fmt.Errorf("could not PEM encode serving certificate: %w", err) - } secret := corev1.Secret{ TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{ @@ -123,11 +111,29 @@ func (c *certsManagerController) Sync(ctx controllerlib.Context) error { Labels: c.certsSecretLabels, }, StringData: map[string]string{ - caCertificateSecretKey: string(aggregatedAPIServerCA.Bundle()), - tlsPrivateKeySecretKey: string(tlsPrivateKeyPEM), - tlsCertificateChainSecretKey: string(tlsCertChainPEM), + CACertificateSecretKey: string(ca.Bundle()), + CACertificatePrivateKeySecretKey: string(caPrivateKeyPEM), }, } + + // Using the CA from above, create a TLS server cert if we have service name. + if len(c.serviceNameForGeneratedCertCommonName) != 0 { + serviceEndpoint := c.serviceNameForGeneratedCertCommonName + "." + c.namespace + ".svc" + tlsCert, err := ca.IssueServerCert([]string{serviceEndpoint}, nil, c.certDuration) + if err != nil { + return fmt.Errorf("could not issue serving certificate: %w", err) + } + + // Write the CA's public key bundle and the serving certs to a secret. + tlsCertChainPEM, tlsPrivateKeyPEM, err := certauthority.ToPEM(tlsCert) + if err != nil { + return fmt.Errorf("could not PEM encode serving certificate: %w", err) + } + + secret.StringData[tlsPrivateKeySecretKey] = string(tlsPrivateKeyPEM) + secret.StringData[TLSCertificateChainSecretKey] = string(tlsCertChainPEM) + } + _, err = c.k8sClient.CoreV1().Secrets(c.namespace).Create(ctx.Context, &secret, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("could not create secret: %w", err) diff --git a/internal/controller/apicerts/certs_manager_test.go b/internal/controller/apicerts/certs_manager_test.go index d43babf6f..092f504bb 100644 --- a/internal/controller/apicerts/certs_manager_test.go +++ b/internal/controller/apicerts/certs_manager_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package apicerts @@ -49,7 +49,7 @@ func TestManagerControllerOptions(t *testing.T) { observableWithInitialEventOption.WithInitialEvent, 0, "Pinniped CA", - "pinniped-api", + "ignored", ) secretsInformerFilter = observableWithInformerOption.GetFilterForInformer(secretsInformer) }) @@ -118,6 +118,7 @@ func TestManagerControllerSync(t *testing.T) { const installedInNamespace = "some-namespace" const certsSecretResourceName = "some-resource-name" const certDuration = 12345678 * time.Second + const defaultServiceName = "pinniped-api" var r *require.Assertions @@ -125,13 +126,13 @@ func TestManagerControllerSync(t *testing.T) { var kubeAPIClient *kubernetesfake.Clientset var kubeInformerClient *kubernetesfake.Clientset var kubeInformers kubeinformers.SharedInformerFactory - var timeoutContext context.Context - var timeoutContextCancel context.CancelFunc + var cancelContext context.Context + var cancelContextCancelFunc context.CancelFunc var syncContext *controllerlib.Context // Defer starting the informers until the last possible moment so that the // nested Before's can keep adding things to the informer caches. - var startInformersAndController = func() { + var startInformersAndController = func(serviceName string) { // Set this at the last second to allow for injection of server override. subject = NewCertsManagerController( installedInNamespace, @@ -146,12 +147,12 @@ func TestManagerControllerSync(t *testing.T) { controllerlib.WithInitialEvent, certDuration, "Pinniped CA", - "pinniped-api", + serviceName, ) // Set this at the last second to support calling subject.Name(). syncContext = &controllerlib.Context{ - Context: timeoutContext, + Context: cancelContext, Name: subject.Name(), Key: controllerlib.Key{ Namespace: installedInNamespace, @@ -160,14 +161,14 @@ func TestManagerControllerSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - kubeInformers.Start(timeoutContext.Done()) + kubeInformers.Start(cancelContext.Done()) controllerlib.TestRunSynchronously(t, subject) } it.Before(func() { r = require.New(t) - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) kubeInformerClient = kubernetesfake.NewSimpleClientset() kubeInformers = kubeinformers.NewSharedInformerFactory(kubeInformerClient, 0) @@ -175,7 +176,7 @@ func TestManagerControllerSync(t *testing.T) { }) it.After(func() { - timeoutContextCancel() + cancelContextCancelFunc() }) when("there is not yet a serving cert Secret in the installation namespace or it was deleted", func() { @@ -191,7 +192,7 @@ func TestManagerControllerSync(t *testing.T) { }) it("creates the serving cert Secret", func() { - startInformersAndController() + startInformersAndController(defaultServiceName) err := controllerlib.TestSync(t, subject, *syncContext) r.NoError(err) @@ -208,23 +209,54 @@ func TestManagerControllerSync(t *testing.T) { "myLabelKey2": "myLabelValue2", }, actualSecret.Labels) actualCACert := actualSecret.StringData["caCertificate"] + actualCAPrivateKey := actualSecret.StringData["caCertificatePrivateKey"] actualPrivateKey := actualSecret.StringData["tlsPrivateKey"] actualCertChain := actualSecret.StringData["tlsCertificateChain"] r.NotEmpty(actualCACert) + r.NotEmpty(actualCAPrivateKey) r.NotEmpty(actualPrivateKey) r.NotEmpty(actualCertChain) + r.Len(actualSecret.StringData, 4) - // Validate the created CA's lifetime. - validCACert := testutil.ValidateCertificate(t, actualCACert, actualCACert) + validCACert := testutil.ValidateServerCertificate(t, actualCACert, actualCACert) + validCACert.RequireMatchesPrivateKey(actualCAPrivateKey) validCACert.RequireLifetime(time.Now(), time.Now().Add(certDuration), 6*time.Minute) // Validate the created cert using the CA, and also validate the cert's hostname - validCert := testutil.ValidateCertificate(t, actualCACert, actualCertChain) + validCert := testutil.ValidateServerCertificate(t, actualCACert, actualCertChain) validCert.RequireDNSName("pinniped-api." + installedInNamespace + ".svc") validCert.RequireLifetime(time.Now(), time.Now().Add(certDuration), 6*time.Minute) validCert.RequireMatchesPrivateKey(actualPrivateKey) }) + it("creates the CA but not service when the service name is empty", func() { + startInformersAndController("") + err := controllerlib.TestSync(t, subject, *syncContext) + r.NoError(err) + + // Check all the relevant fields from the create Secret action + r.Len(kubeAPIClient.Actions(), 1) + actualAction := kubeAPIClient.Actions()[0].(coretesting.CreateActionImpl) + r.Equal(schema.GroupVersionResource{Group: "", Version: "v1", Resource: "secrets"}, actualAction.GetResource()) + r.Equal(installedInNamespace, actualAction.GetNamespace()) + actualSecret := actualAction.GetObject().(*corev1.Secret) + r.Equal(certsSecretResourceName, actualSecret.Name) + r.Equal(installedInNamespace, actualSecret.Namespace) + r.Equal(map[string]string{ + "myLabelKey1": "myLabelValue1", + "myLabelKey2": "myLabelValue2", + }, actualSecret.Labels) + actualCACert := actualSecret.StringData["caCertificate"] + actualCAPrivateKey := actualSecret.StringData["caCertificatePrivateKey"] + r.NotEmpty(actualCACert) + r.NotEmpty(actualCAPrivateKey) + r.Len(actualSecret.StringData, 2) + + validCACert := testutil.ValidateServerCertificate(t, actualCACert, actualCACert) + validCACert.RequireMatchesPrivateKey(actualCAPrivateKey) + validCACert.RequireLifetime(time.Now(), time.Now().Add(certDuration), 6*time.Minute) + }) + when("creating the Secret fails", func() { it.Before(func() { kubeAPIClient.PrependReactor( @@ -237,7 +269,7 @@ func TestManagerControllerSync(t *testing.T) { }) it("returns the create error", func() { - startInformersAndController() + startInformersAndController(defaultServiceName) err := controllerlib.TestSync(t, subject, *syncContext) r.EqualError(err, "could not create secret: create failed") }) @@ -257,7 +289,7 @@ func TestManagerControllerSync(t *testing.T) { }) it("does not need to make any API calls with its API client", func() { - startInformersAndController() + startInformersAndController(defaultServiceName) err := controllerlib.TestSync(t, subject, *syncContext) r.NoError(err) r.Empty(kubeAPIClient.Actions()) diff --git a/internal/controller/apicerts/certs_observer.go b/internal/controller/apicerts/certs_observer.go index 7f05d243f..f01312d59 100644 --- a/internal/controller/apicerts/certs_observer.go +++ b/internal/controller/apicerts/certs_observer.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package apicerts @@ -18,14 +18,14 @@ import ( type certsObserverController struct { namespace string certsSecretResourceName string - dynamicCertProvider dynamiccert.Provider + dynamicCertProvider dynamiccert.Private secretInformer corev1informers.SecretInformer } func NewCertsObserverController( namespace string, certsSecretResourceName string, - dynamicCertProvider dynamiccert.Provider, + dynamicCertProvider dynamiccert.Private, secretInformer corev1informers.SecretInformer, withInformer pinnipedcontroller.WithInformerOptionFunc, ) controllerlib.Controller { @@ -57,12 +57,15 @@ func (c *certsObserverController) Sync(_ controllerlib.Context) error { if notFound { klog.Info("certsObserverController Sync found that the secret does not exist yet or was deleted") // The secret does not exist yet or was deleted. - c.dynamicCertProvider.Set(nil, nil) + c.dynamicCertProvider.UnsetCertKeyContent() return nil } // Mutate the in-memory cert provider to update with the latest cert values. - c.dynamicCertProvider.Set(certSecret.Data[tlsCertificateChainSecretKey], certSecret.Data[tlsPrivateKeySecretKey]) + if err := c.dynamicCertProvider.SetCertKeyContent(certSecret.Data[TLSCertificateChainSecretKey], certSecret.Data[tlsPrivateKeySecretKey]); err != nil { + return fmt.Errorf("failed to set serving cert/key content from secret %s/%s: %w", c.namespace, c.certsSecretResourceName, err) + } + klog.Info("certsObserverController Sync updated certs in the dynamic cert provider") return nil } diff --git a/internal/controller/apicerts/certs_observer_test.go b/internal/controller/apicerts/certs_observer_test.go index 452c3544f..93de9801b 100644 --- a/internal/controller/apicerts/certs_observer_test.go +++ b/internal/controller/apicerts/certs_observer_test.go @@ -1,10 +1,11 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package apicerts import ( "context" + "strings" "testing" "time" @@ -16,6 +17,7 @@ import ( kubeinformers "k8s.io/client-go/informers" kubernetesfake "k8s.io/client-go/kubernetes/fake" + "go.pinniped.dev/internal/certauthority" "go.pinniped.dev/internal/controllerlib" "go.pinniped.dev/internal/dynamiccert" "go.pinniped.dev/internal/testutil" @@ -95,6 +97,7 @@ func TestObserverControllerInformerFilters(t *testing.T) { } func TestObserverControllerSync(t *testing.T) { + name := t.Name() spec.Run(t, "Sync", func(t *testing.T, when spec.G, it spec.S) { const installedInNamespace = "some-namespace" const certsSecretResourceName = "some-resource-name" @@ -104,10 +107,10 @@ func TestObserverControllerSync(t *testing.T) { var subject controllerlib.Controller var kubeInformerClient *kubernetesfake.Clientset var kubeInformers kubeinformers.SharedInformerFactory - var timeoutContext context.Context - var timeoutContextCancel context.CancelFunc + var cancelContext context.Context + var cancelContextCancelFunc context.CancelFunc var syncContext *controllerlib.Context - var dynamicCertProvider dynamiccert.Provider + var dynamicCertProvider dynamiccert.Private // Defer starting the informers until the last possible moment so that the // nested Before's can keep adding things to the informer caches. @@ -123,7 +126,7 @@ func TestObserverControllerSync(t *testing.T) { // Set this at the last second to support calling subject.Name(). syncContext = &controllerlib.Context{ - Context: timeoutContext, + Context: cancelContext, Name: subject.Name(), Key: controllerlib.Key{ Namespace: installedInNamespace, @@ -132,22 +135,22 @@ func TestObserverControllerSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - kubeInformers.Start(timeoutContext.Done()) + kubeInformers.Start(cancelContext.Done()) controllerlib.TestRunSynchronously(t, subject) } it.Before(func() { r = require.New(t) - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) kubeInformerClient = kubernetesfake.NewSimpleClientset() kubeInformers = kubeinformers.NewSharedInformerFactory(kubeInformerClient, 0) - dynamicCertProvider = dynamiccert.New() + dynamicCertProvider = dynamiccert.NewServingCert(name) }) it.After(func() { - timeoutContextCancel() + cancelContextCancelFunc() }) when("there is not yet a serving cert Secret in the installation namespace or it was deleted", func() { @@ -161,7 +164,20 @@ func TestObserverControllerSync(t *testing.T) { err := kubeInformerClient.Tracker().Add(unrelatedSecret) r.NoError(err) - dynamicCertProvider.Set([]byte("some cert"), []byte("some private key")) + caCrt, caKey, err := testutil.CreateCertificate( + time.Now().Add(-time.Hour), + time.Now().Add(time.Hour), + ) + require.NoError(t, err) + + ca, err := certauthority.Load(string(caCrt), string(caKey)) + require.NoError(t, err) + + crt, key, err := ca.IssueServerCertPEM(nil, nil, time.Hour) + require.NoError(t, err) + + err = dynamicCertProvider.SetCertKeyContent(crt, key) + r.NoError(err) }) it("sets the dynamicCertProvider's cert and key to nil", func() { @@ -177,6 +193,18 @@ func TestObserverControllerSync(t *testing.T) { when("there is a serving cert Secret with the expected keys already in the installation namespace", func() { it.Before(func() { + caCrt, caKey, err := testutil.CreateCertificate( + time.Now().Add(-time.Hour), + time.Now().Add(time.Hour), + ) + require.NoError(t, err) + + ca, err := certauthority.Load(string(caCrt), string(caKey)) + require.NoError(t, err) + + crt, key, err := ca.IssueServerCertPEM(nil, nil, time.Hour) + require.NoError(t, err) + apiServingCertSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: certsSecretResourceName, @@ -184,24 +212,29 @@ func TestObserverControllerSync(t *testing.T) { }, Data: map[string][]byte{ "caCertificate": []byte("fake cert"), - "tlsPrivateKey": []byte("fake private key"), - "tlsCertificateChain": []byte("fake cert chain"), + "tlsPrivateKey": key, + "tlsCertificateChain": crt, }, } - err := kubeInformerClient.Tracker().Add(apiServingCertSecret) + err = kubeInformerClient.Tracker().Add(apiServingCertSecret) r.NoError(err) - dynamicCertProvider.Set(nil, nil) + dynamicCertProvider.UnsetCertKeyContent() }) it("updates the dynamicCertProvider's cert and key", func() { startInformersAndController() + + actualCertChain, actualKey := dynamicCertProvider.CurrentCertKeyContent() + r.Nil(actualCertChain) + r.Nil(actualKey) + err := controllerlib.TestSync(t, subject, *syncContext) r.NoError(err) - actualCertChain, actualKey := dynamicCertProvider.CurrentCertKeyContent() - r.Equal("fake cert chain", string(actualCertChain)) - r.Equal("fake private key", string(actualKey)) + actualCertChain, actualKey = dynamicCertProvider.CurrentCertKeyContent() + r.True(strings.HasPrefix(string(actualCertChain), `-----BEGIN CERTIFICATE-----`), "not a cert:\n%s", string(actualCertChain)) + r.True(strings.HasPrefix(string(actualKey), `-----BEGIN PRIVATE KEY-----`), "not a key:\n%s", string(actualKey)) }) }) @@ -217,17 +250,22 @@ func TestObserverControllerSync(t *testing.T) { err := kubeInformerClient.Tracker().Add(apiServingCertSecret) r.NoError(err) - dynamicCertProvider.Set(nil, nil) + dynamicCertProvider.UnsetCertKeyContent() }) - it("set the missing values in the dynamicCertProvider as nil", func() { + it("returns an error and does not change the dynamicCertProvider", func() { startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - r.NoError(err) actualCertChain, actualKey := dynamicCertProvider.CurrentCertKeyContent() r.Nil(actualCertChain) r.Nil(actualKey) + + err := controllerlib.TestSync(t, subject, *syncContext) + r.EqualError(err, "failed to set serving cert/key content from secret some-namespace/some-resource-name: TestObserverControllerSync: attempt to set invalid key pair: tls: failed to find any PEM data in certificate input") + + actualCertChain, actualKey = dynamicCertProvider.CurrentCertKeyContent() + r.Nil(actualCertChain) + r.Nil(actualKey) }) }) }, spec.Parallel(), spec.Report(report.Terminal{})) diff --git a/internal/controller/authenticator/cachecleaner/cachecleaner_test.go b/internal/controller/authenticator/cachecleaner/cachecleaner_test.go index 4bac5be56..5683e26a7 100644 --- a/internal/controller/authenticator/cachecleaner/cachecleaner_test.go +++ b/internal/controller/authenticator/cachecleaner/cachecleaner_test.go @@ -6,7 +6,6 @@ package cachecleaner import ( "context" "testing" - "time" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" @@ -150,7 +149,7 @@ func TestController(t *testing.T) { jwtAuthenticators := informers.Authentication().V1alpha1().JWTAuthenticators() controller := New(cache, webhooks, jwtAuthenticators, testLog) - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() informers.Start(ctx.Done()) diff --git a/internal/controller/authenticator/jwtcachefiller/jwtcachefiller_test.go b/internal/controller/authenticator/jwtcachefiller/jwtcachefiller_test.go index 6e53876ba..5a43751e1 100644 --- a/internal/controller/authenticator/jwtcachefiller/jwtcachefiller_test.go +++ b/internal/controller/authenticator/jwtcachefiller/jwtcachefiller_test.go @@ -325,7 +325,7 @@ func TestController(t *testing.T) { controller := New(cache, informers.Authentication().V1alpha1().JWTAuthenticators(), testLog) - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() informers.Start(ctx.Done()) diff --git a/internal/controller/authenticator/webhookcachefiller/webhookcachefiller_test.go b/internal/controller/authenticator/webhookcachefiller/webhookcachefiller_test.go index b2dcbf843..4d6a0744d 100644 --- a/internal/controller/authenticator/webhookcachefiller/webhookcachefiller_test.go +++ b/internal/controller/authenticator/webhookcachefiller/webhookcachefiller_test.go @@ -11,7 +11,6 @@ import ( "net/http" "os" "testing" - "time" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -93,7 +92,7 @@ func TestController(t *testing.T) { controller := New(cache, informers.Authentication().V1alpha1().WebhookAuthenticators(), testLog) - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() informers.Start(ctx.Done()) diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go new file mode 100644 index 000000000..72cd4d68e --- /dev/null +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -0,0 +1,864 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package impersonatorconfig + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "fmt" + "net" + "strings" + "time" + + v1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/clock" + "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/intstr" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/sets" + corev1informers "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/kubernetes" + + "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" + pinnipedclientset "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned" + "go.pinniped.dev/internal/certauthority" + "go.pinniped.dev/internal/clusterhost" + "go.pinniped.dev/internal/concierge/impersonator" + "go.pinniped.dev/internal/constable" + pinnipedcontroller "go.pinniped.dev/internal/controller" + "go.pinniped.dev/internal/controller/apicerts" + "go.pinniped.dev/internal/controller/issuerconfig" + "go.pinniped.dev/internal/controllerlib" + "go.pinniped.dev/internal/dynamiccert" + "go.pinniped.dev/internal/plog" +) + +const ( + impersonationProxyPort = 8444 + defaultHTTPSPort = 443 + approximatelyOneHundredYears = 100 * 365 * 24 * time.Hour + caCommonName = "Pinniped Impersonation Proxy CA" + caCrtKey = "ca.crt" + caKeyKey = "ca.key" + appLabelKey = "app" +) + +type impersonatorConfigController struct { + namespace string + configMapResourceName string + credentialIssuerResourceName string + generatedLoadBalancerServiceName string + tlsSecretName string + caSecretName string + impersonationSignerSecretName string + + k8sClient kubernetes.Interface + pinnipedAPIClient pinnipedclientset.Interface + + configMapsInformer corev1informers.ConfigMapInformer + servicesInformer corev1informers.ServiceInformer + secretsInformer corev1informers.SecretInformer + + labels map[string]string + clock clock.Clock + impersonationSigningCertProvider dynamiccert.Provider + impersonatorFunc impersonator.FactoryFunc + + hasControlPlaneNodes *bool + serverStopCh chan struct{} + errorCh chan error + tlsServingCertDynamicCertProvider dynamiccert.Private +} + +func NewImpersonatorConfigController( + namespace string, + configMapResourceName string, + credentialIssuerResourceName string, + k8sClient kubernetes.Interface, + pinnipedAPIClient pinnipedclientset.Interface, + configMapsInformer corev1informers.ConfigMapInformer, + servicesInformer corev1informers.ServiceInformer, + secretsInformer corev1informers.SecretInformer, + withInformer pinnipedcontroller.WithInformerOptionFunc, + withInitialEvent pinnipedcontroller.WithInitialEventOptionFunc, + generatedLoadBalancerServiceName string, + tlsSecretName string, + caSecretName string, + labels map[string]string, + clock clock.Clock, + impersonatorFunc impersonator.FactoryFunc, + impersonationSignerSecretName string, + impersonationSigningCertProvider dynamiccert.Provider, +) controllerlib.Controller { + secretNames := sets.NewString(tlsSecretName, caSecretName, impersonationSignerSecretName) + return controllerlib.New( + controllerlib.Config{ + Name: "impersonator-config-controller", + Syncer: &impersonatorConfigController{ + namespace: namespace, + configMapResourceName: configMapResourceName, + credentialIssuerResourceName: credentialIssuerResourceName, + generatedLoadBalancerServiceName: generatedLoadBalancerServiceName, + tlsSecretName: tlsSecretName, + caSecretName: caSecretName, + impersonationSignerSecretName: impersonationSignerSecretName, + k8sClient: k8sClient, + pinnipedAPIClient: pinnipedAPIClient, + configMapsInformer: configMapsInformer, + servicesInformer: servicesInformer, + secretsInformer: secretsInformer, + labels: labels, + clock: clock, + impersonationSigningCertProvider: impersonationSigningCertProvider, + impersonatorFunc: impersonatorFunc, + tlsServingCertDynamicCertProvider: dynamiccert.NewServingCert("impersonation-proxy-serving-cert"), + }, + }, + withInformer( + configMapsInformer, + pinnipedcontroller.NameAndNamespaceExactMatchFilterFactory(configMapResourceName, namespace), + controllerlib.InformerOption{}, + ), + withInformer( + servicesInformer, + pinnipedcontroller.NameAndNamespaceExactMatchFilterFactory(generatedLoadBalancerServiceName, namespace), + controllerlib.InformerOption{}, + ), + withInformer( + secretsInformer, + pinnipedcontroller.SimpleFilter(func(obj metav1.Object) bool { + return obj.GetNamespace() == namespace && secretNames.Has(obj.GetName()) + }, nil), + controllerlib.InformerOption{}, + ), + // Be sure to run once even if the ConfigMap that the informer is watching doesn't exist so we can implement + // the default configuration behavior. + withInitialEvent(controllerlib.Key{ + Namespace: namespace, + Name: configMapResourceName, + }), + // TODO fix these controller options to make this a singleton queue + ) +} + +func (c *impersonatorConfigController) Sync(syncCtx controllerlib.Context) error { + plog.Debug("Starting impersonatorConfigController Sync") + + strategy, err := c.doSync(syncCtx) + + if err != nil { + strategy = &v1alpha1.CredentialIssuerStrategy{ + Type: v1alpha1.ImpersonationProxyStrategyType, + Status: v1alpha1.ErrorStrategyStatus, + Reason: v1alpha1.ErrorDuringSetupStrategyReason, + Message: err.Error(), + LastUpdateTime: metav1.NewTime(c.clock.Now()), + } + // The impersonator is not ready, so clear the signer CA from the dynamic provider. + c.clearSignerCA() + } + + updateStrategyErr := c.updateStrategy(syncCtx.Context, strategy) + if updateStrategyErr != nil { + plog.Error("error while updating the CredentialIssuer status", err) + if err == nil { + err = updateStrategyErr + } + } + + if err == nil { + plog.Debug("Successfully finished impersonatorConfigController Sync") + } + return err +} + +type certNameInfo struct { + // ready will be true when the certificate name information is known. + // ready will be false when it is pending because we are waiting for a load balancer to get assigned an ip/hostname. + // When false, the other fields in this struct should not be considered meaningful and may be zero values. + ready bool + + // The IP address or hostname which was selected to be used as the name in the cert. + // Either selectedIP or selectedHostname will be set, but not both. + selectedIP net.IP + selectedHostname string + + // The name of the endpoint to which a client should connect to talk to the impersonator. + // This may be a hostname or an IP, and may include a port number. + clientEndpoint string +} + +func (c *impersonatorConfigController) doSync(syncCtx controllerlib.Context) (*v1alpha1.CredentialIssuerStrategy, error) { + ctx := syncCtx.Context + + config, err := c.loadImpersonationProxyConfiguration() + if err != nil { + return nil, err + } + + // Make a live API call to avoid the cost of having an informer watch all node changes on the cluster, + // since there could be lots and we don't especially care about node changes. + // Once we have concluded that there is or is not a visible control plane, then cache that decision + // to avoid listing nodes very often. + if c.hasControlPlaneNodes == nil { + hasControlPlaneNodes, err := clusterhost.New(c.k8sClient).HasControlPlaneNodes(ctx) + if err != nil { + return nil, err + } + c.hasControlPlaneNodes = &hasControlPlaneNodes + plog.Debug("Queried for control plane nodes", "foundControlPlaneNodes", hasControlPlaneNodes) + } + + if c.shouldHaveImpersonator(config) { + if err = c.ensureImpersonatorIsStarted(syncCtx); err != nil { + return nil, err + } + } else { + if err = c.ensureImpersonatorIsStopped(true); err != nil { + return nil, err // TODO write unit test that errors during stopping the server are returned by sync + } + } + + if c.shouldHaveLoadBalancer(config) { + if err = c.ensureLoadBalancerIsStarted(ctx); err != nil { + return nil, err + } + } else { + if err = c.ensureLoadBalancerIsStopped(ctx); err != nil { + return nil, err + } + } + + nameInfo, err := c.findDesiredTLSCertificateName(config) + if err != nil { + // Unexpected error while determining the name that should go into the certs, so clear any existing certs. + c.tlsServingCertDynamicCertProvider.UnsetCertKeyContent() + return nil, err + } + + var impersonationCA *certauthority.CA + if c.shouldHaveTLSSecret(config) { + if impersonationCA, err = c.ensureCASecretIsCreated(ctx); err != nil { + return nil, err + } + if err = c.ensureTLSSecret(ctx, nameInfo, impersonationCA); err != nil { + return nil, err + } + } else if err = c.ensureTLSSecretIsRemoved(ctx); err != nil { + return nil, err + } + + credentialIssuerStrategyResult := c.doSyncResult(nameInfo, config, impersonationCA) + + if err = c.loadSignerCA(credentialIssuerStrategyResult.Status); err != nil { + return nil, err + } + + return credentialIssuerStrategyResult, nil +} + +func (c *impersonatorConfigController) loadImpersonationProxyConfiguration() (*impersonator.Config, error) { + configMap, err := c.configMapsInformer.Lister().ConfigMaps(c.namespace).Get(c.configMapResourceName) + notFound := k8serrors.IsNotFound(err) + if err != nil && !notFound { + return nil, fmt.Errorf("failed to get %s/%s configmap: %w", c.namespace, c.configMapResourceName, err) + } + + var config *impersonator.Config + if notFound { + plog.Info("Did not find impersonation proxy config: using default config values", + "configmap", c.configMapResourceName, + "namespace", c.namespace, + ) + config = impersonator.NewConfig() // use default configuration options + } else { + config, err = impersonator.ConfigFromConfigMap(configMap) + if err != nil { + return nil, fmt.Errorf("invalid impersonator configuration: %v", err) + } + plog.Info("Read impersonation proxy config", + "configmap", c.configMapResourceName, + "namespace", c.namespace, + ) + } + + return config, nil +} + +func (c *impersonatorConfigController) shouldHaveImpersonator(config *impersonator.Config) bool { + return c.enabledByAutoMode(config) || config.Mode == impersonator.ModeEnabled +} + +func (c *impersonatorConfigController) enabledByAutoMode(config *impersonator.Config) bool { + return config.Mode == impersonator.ModeAuto && !*c.hasControlPlaneNodes +} + +func (c *impersonatorConfigController) disabledByAutoMode(config *impersonator.Config) bool { + return config.Mode == impersonator.ModeAuto && *c.hasControlPlaneNodes +} + +func (c *impersonatorConfigController) disabledExplicitly(config *impersonator.Config) bool { + return config.Mode == impersonator.ModeDisabled +} + +func (c *impersonatorConfigController) shouldHaveLoadBalancer(config *impersonator.Config) bool { + return c.shouldHaveImpersonator(config) && !config.HasEndpoint() +} + +func (c *impersonatorConfigController) shouldHaveTLSSecret(config *impersonator.Config) bool { + return c.shouldHaveImpersonator(config) +} + +func (c *impersonatorConfigController) updateStrategy(ctx context.Context, strategy *v1alpha1.CredentialIssuerStrategy) error { + return issuerconfig.UpdateStrategy(ctx, c.credentialIssuerResourceName, c.labels, c.pinnipedAPIClient, *strategy) +} + +func (c *impersonatorConfigController) loadBalancerExists() (bool, error) { + _, err := c.servicesInformer.Lister().Services(c.namespace).Get(c.generatedLoadBalancerServiceName) + notFound := k8serrors.IsNotFound(err) + if notFound { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +func (c *impersonatorConfigController) tlsSecretExists() (bool, *v1.Secret, error) { + secret, err := c.secretsInformer.Lister().Secrets(c.namespace).Get(c.tlsSecretName) + notFound := k8serrors.IsNotFound(err) + if notFound { + return false, nil, nil + } + if err != nil { + return false, nil, err + } + return true, secret, nil +} + +func (c *impersonatorConfigController) ensureImpersonatorIsStarted(syncCtx controllerlib.Context) error { + if c.serverStopCh != nil { + // The server was already started, but it could have died in the background, so make a non-blocking + // check to see if it has sent any errors on the errorCh. + select { + case runningErr := <-c.errorCh: + if runningErr == nil { + // The server sent a nil error, meaning that it shutdown without reporting any particular + // error for some reason. We would still like to report this as an error for logging purposes. + runningErr = constable.Error("unexpected shutdown of proxy server") + } + // The server has stopped, so finish shutting it down. + // If that fails too, return both errors for logging purposes. + // By returning an error, the sync function will be called again + // and we'll have a chance to restart the server. + close(c.errorCh) // We don't want ensureImpersonatorIsStopped to block on reading this channel. + stoppingErr := c.ensureImpersonatorIsStopped(false) + return errors.NewAggregate([]error{runningErr, stoppingErr}) + default: + // Seems like it is still running, so nothing to do. + return nil + } + } + + plog.Info("Starting impersonation proxy", "port", impersonationProxyPort) + startImpersonatorFunc, err := c.impersonatorFunc( + impersonationProxyPort, + c.tlsServingCertDynamicCertProvider, + c.impersonationSigningCertProvider, + ) + if err != nil { + return err + } + + c.serverStopCh = make(chan struct{}) + // use a buffered channel so that startImpersonatorFunc can send + // on it without coordinating with the main controller go routine + c.errorCh = make(chan error, 1) + + // startImpersonatorFunc will block until the server shuts down (or fails to start), so run it in the background. + go func() { + defer utilruntime.HandleCrash() + + // The server has stopped, so enqueue ourselves for another sync, + // so we can try to start the server again as quickly as possible. + defer syncCtx.Queue.AddRateLimited(syncCtx.Key) + + // Forward any errors returned by startImpersonatorFunc on the errorCh. + c.errorCh <- startImpersonatorFunc(c.serverStopCh) + }() + + return nil +} + +func (c *impersonatorConfigController) ensureImpersonatorIsStopped(shouldCloseErrChan bool) error { + if c.serverStopCh == nil { + return nil + } + + plog.Info("Stopping impersonation proxy", "port", impersonationProxyPort) + close(c.serverStopCh) + stopErr := <-c.errorCh + + if shouldCloseErrChan { + close(c.errorCh) + } + + c.serverStopCh = nil + c.errorCh = nil + + return stopErr +} + +func (c *impersonatorConfigController) ensureLoadBalancerIsStarted(ctx context.Context) error { + running, err := c.loadBalancerExists() + if err != nil { + return err + } + if running { + return nil + } + appNameLabel := c.labels[appLabelKey] + loadBalancer := v1.Service{ + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeLoadBalancer, + Ports: []v1.ServicePort{ + { + TargetPort: intstr.FromInt(impersonationProxyPort), + Port: defaultHTTPSPort, + Protocol: v1.ProtocolTCP, + }, + }, + Selector: map[string]string{appLabelKey: appNameLabel}, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: c.generatedLoadBalancerServiceName, + Namespace: c.namespace, + Labels: c.labels, + }, + } + plog.Info("creating load balancer for impersonation proxy", + "service", c.generatedLoadBalancerServiceName, + "namespace", c.namespace) + _, err = c.k8sClient.CoreV1().Services(c.namespace).Create(ctx, &loadBalancer, metav1.CreateOptions{}) + return err +} + +func (c *impersonatorConfigController) ensureLoadBalancerIsStopped(ctx context.Context) error { + running, err := c.loadBalancerExists() + if err != nil { + return err + } + if !running { + return nil + } + + plog.Info("Deleting load balancer for impersonation proxy", + "service", c.generatedLoadBalancerServiceName, + "namespace", c.namespace) + return c.k8sClient.CoreV1().Services(c.namespace).Delete(ctx, c.generatedLoadBalancerServiceName, metav1.DeleteOptions{}) +} + +func (c *impersonatorConfigController) ensureTLSSecret(ctx context.Context, nameInfo *certNameInfo, ca *certauthority.CA) error { + secretFromInformer, err := c.secretsInformer.Lister().Secrets(c.namespace).Get(c.tlsSecretName) + notFound := k8serrors.IsNotFound(err) + if !notFound && err != nil { + return err + } + + if !notFound { + secretWasDeleted, err := c.deleteTLSSecretWhenCertificateDoesNotMatchDesiredState(ctx, nameInfo, ca, secretFromInformer) + if err != nil { + return err + } + // If it was deleted by the above call, then set it to nil. This allows us to avoid waiting + // for the informer cache to update before deciding to proceed to create the new Secret below. + if secretWasDeleted { + secretFromInformer = nil + } + } + + return c.ensureTLSSecretIsCreatedAndLoaded(ctx, nameInfo, secretFromInformer, ca) +} + +func (c *impersonatorConfigController) deleteTLSSecretWhenCertificateDoesNotMatchDesiredState(ctx context.Context, nameInfo *certNameInfo, ca *certauthority.CA, secret *v1.Secret) (bool, error) { + certPEM := secret.Data[v1.TLSCertKey] + block, _ := pem.Decode(certPEM) + if block == nil { + plog.Warning("Found missing or not PEM-encoded data in TLS Secret", + "invalidCertPEM", string(certPEM), + "secret", c.tlsSecretName, + "namespace", c.namespace) + deleteErr := c.ensureTLSSecretIsRemoved(ctx) + if deleteErr != nil { + return false, fmt.Errorf("found missing or not PEM-encoded data in TLS Secret, but got error while deleting it: %w", deleteErr) + } + return true, nil + } + + actualCertFromSecret, err := x509.ParseCertificate(block.Bytes) + if err != nil { + plog.Error("Found invalid PEM data in TLS Secret", err, + "invalidCertPEM", string(certPEM), + "secret", c.tlsSecretName, + "namespace", c.namespace) + if err = c.ensureTLSSecretIsRemoved(ctx); err != nil { + return false, fmt.Errorf("PEM data represented an invalid cert, but got error while deleting it: %w", err) + } + return true, nil + } + + keyPEM := secret.Data[v1.TLSPrivateKeyKey] + _, err = tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + plog.Error("Found invalid private key PEM data in TLS Secret", err, + "secret", c.tlsSecretName, + "namespace", c.namespace) + if err = c.ensureTLSSecretIsRemoved(ctx); err != nil { + return false, fmt.Errorf("cert had an invalid private key, but got error while deleting it: %w", err) + } + return true, nil + } + + opts := x509.VerifyOptions{Roots: ca.Pool()} + if _, err = actualCertFromSecret.Verify(opts); err != nil { + // The TLS cert was not signed by the current CA. Since they are mismatched, delete the TLS cert + // so we can recreate it using the current CA. + if err = c.ensureTLSSecretIsRemoved(ctx); err != nil { + return false, err + } + return true, nil + } + + if !nameInfo.ready { + // We currently have a secret but we are waiting for a load balancer to be assigned an ingress, so + // our current secret must be old/unwanted. + if err = c.ensureTLSSecretIsRemoved(ctx); err != nil { + return false, err + } + return true, nil + } + + actualIPs := actualCertFromSecret.IPAddresses + actualHostnames := actualCertFromSecret.DNSNames + plog.Info("Checking TLS certificate names", + "desiredIP", nameInfo.selectedIP, + "desiredHostname", nameInfo.selectedHostname, + "actualIPs", actualIPs, + "actualHostnames", actualHostnames, + "secret", c.tlsSecretName, + "namespace", c.namespace) + + if certHostnameAndIPMatchDesiredState(nameInfo.selectedIP, actualIPs, nameInfo.selectedHostname, actualHostnames) { + // The cert already matches the desired state, so there is no need to delete/recreate it. + return false, nil + } + + if err = c.ensureTLSSecretIsRemoved(ctx); err != nil { + return false, err + } + return true, nil +} + +func certHostnameAndIPMatchDesiredState(desiredIP net.IP, actualIPs []net.IP, desiredHostname string, actualHostnames []string) bool { + if desiredIP != nil && len(actualIPs) == 1 && desiredIP.Equal(actualIPs[0]) && len(actualHostnames) == 0 { + return true + } + if desiredHostname != "" && len(actualHostnames) == 1 && desiredHostname == actualHostnames[0] && len(actualIPs) == 0 { + return true + } + return false +} + +func (c *impersonatorConfigController) ensureTLSSecretIsCreatedAndLoaded(ctx context.Context, nameInfo *certNameInfo, secret *v1.Secret, ca *certauthority.CA) error { + if secret != nil { + err := c.loadTLSCertFromSecret(secret) + if err != nil { + return err + } + return nil + } + + if !nameInfo.ready { + return nil + } + + newTLSSecret, err := c.createNewTLSSecret(ctx, ca, nameInfo.selectedIP, nameInfo.selectedHostname) + if err != nil { + return err + } + + err = c.loadTLSCertFromSecret(newTLSSecret) + if err != nil { + return err + } + + return nil +} + +func (c *impersonatorConfigController) ensureCASecretIsCreated(ctx context.Context) (*certauthority.CA, error) { + caSecret, err := c.secretsInformer.Lister().Secrets(c.namespace).Get(c.caSecretName) + if err != nil && !k8serrors.IsNotFound(err) { + return nil, err + } + + var impersonationCA *certauthority.CA + if k8serrors.IsNotFound(err) { + impersonationCA, err = c.createCASecret(ctx) + } else { + crtBytes := caSecret.Data[caCrtKey] + keyBytes := caSecret.Data[caKeyKey] + impersonationCA, err = certauthority.Load(string(crtBytes), string(keyBytes)) + } + if err != nil { + return nil, err + } + + return impersonationCA, nil +} + +func (c *impersonatorConfigController) createCASecret(ctx context.Context) (*certauthority.CA, error) { + impersonationCA, err := certauthority.New(caCommonName, approximatelyOneHundredYears) + if err != nil { + return nil, fmt.Errorf("could not create impersonation CA: %w", err) + } + + caPrivateKeyPEM, err := impersonationCA.PrivateKeyToPEM() + if err != nil { + return nil, err + } + + secret := v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.caSecretName, + Namespace: c.namespace, + Labels: c.labels, + }, + Data: map[string][]byte{ + caCrtKey: impersonationCA.Bundle(), + caKeyKey: caPrivateKeyPEM, + }, + Type: v1.SecretTypeOpaque, + } + + plog.Info("Creating CA certificates for impersonation proxy", + "secret", c.caSecretName, + "namespace", c.namespace) + if _, err = c.k8sClient.CoreV1().Secrets(c.namespace).Create(ctx, &secret, metav1.CreateOptions{}); err != nil { + return nil, err + } + + return impersonationCA, nil +} + +func (c *impersonatorConfigController) findDesiredTLSCertificateName(config *impersonator.Config) (*certNameInfo, error) { + if config.HasEndpoint() { + return c.findTLSCertificateNameFromEndpointConfig(config), nil + } + return c.findTLSCertificateNameFromLoadBalancer() +} + +func (c *impersonatorConfigController) findTLSCertificateNameFromEndpointConfig(config *impersonator.Config) *certNameInfo { + endpointMaybeWithPort := config.Endpoint + endpointWithoutPort := strings.Split(endpointMaybeWithPort, ":")[0] + parsedAsIP := net.ParseIP(endpointWithoutPort) + if parsedAsIP != nil { + return &certNameInfo{ready: true, selectedIP: parsedAsIP, clientEndpoint: endpointMaybeWithPort} + } + return &certNameInfo{ready: true, selectedHostname: endpointWithoutPort, clientEndpoint: endpointMaybeWithPort} +} + +func (c *impersonatorConfigController) findTLSCertificateNameFromLoadBalancer() (*certNameInfo, error) { + lb, err := c.servicesInformer.Lister().Services(c.namespace).Get(c.generatedLoadBalancerServiceName) + notFound := k8serrors.IsNotFound(err) + if notFound { + // We aren't ready and will try again later in this case. + return &certNameInfo{ready: false}, nil + } + if err != nil { + return nil, err + } + ingresses := lb.Status.LoadBalancer.Ingress + if len(ingresses) == 0 || (ingresses[0].Hostname == "" && ingresses[0].IP == "") { + plog.Info("load balancer for impersonation proxy does not have an ingress yet, so skipping tls cert generation while we wait", + "service", c.generatedLoadBalancerServiceName, + "namespace", c.namespace) + return &certNameInfo{ready: false}, nil + } + for _, ingress := range ingresses { + hostname := ingress.Hostname + if hostname != "" { + return &certNameInfo{ready: true, selectedHostname: hostname, clientEndpoint: hostname}, nil + } + } + for _, ingress := range ingresses { + ip := ingress.IP + parsedIP := net.ParseIP(ip) + if parsedIP != nil { + return &certNameInfo{ready: true, selectedIP: parsedIP, clientEndpoint: ip}, nil + } + } + + return nil, fmt.Errorf("could not find valid IP addresses or hostnames from load balancer %s/%s", c.namespace, lb.Name) +} + +func (c *impersonatorConfigController) createNewTLSSecret(ctx context.Context, ca *certauthority.CA, ip net.IP, hostname string) (*v1.Secret, error) { + var hostnames []string + var ips []net.IP + if hostname != "" { + hostnames = []string{hostname} + } + if ip != nil { + ips = []net.IP{ip} + } + + impersonationCert, err := ca.IssueServerCert(hostnames, ips, approximatelyOneHundredYears) + if err != nil { + return nil, fmt.Errorf("could not create impersonation cert: %w", err) + } + + certPEM, keyPEM, err := certauthority.ToPEM(impersonationCert) + if err != nil { + return nil, err + } + + newTLSSecret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.tlsSecretName, + Namespace: c.namespace, + Labels: c.labels, + }, + Data: map[string][]byte{ + v1.TLSPrivateKeyKey: keyPEM, + v1.TLSCertKey: certPEM, + }, + Type: v1.SecretTypeTLS, + } + + plog.Info("Creating TLS certificates for impersonation proxy", + "ips", ips, + "hostnames", hostnames, + "secret", c.tlsSecretName, + "namespace", c.namespace) + return c.k8sClient.CoreV1().Secrets(c.namespace).Create(ctx, newTLSSecret, metav1.CreateOptions{}) +} + +func (c *impersonatorConfigController) loadTLSCertFromSecret(tlsSecret *v1.Secret) error { + certPEM := tlsSecret.Data[v1.TLSCertKey] + keyPEM := tlsSecret.Data[v1.TLSPrivateKeyKey] + + if err := c.tlsServingCertDynamicCertProvider.SetCertKeyContent(certPEM, keyPEM); err != nil { + c.tlsServingCertDynamicCertProvider.UnsetCertKeyContent() + return fmt.Errorf("could not parse TLS cert PEM data from Secret: %w", err) + } + + plog.Info("Loading TLS certificates for impersonation proxy", + "certPEM", string(certPEM), + "secret", c.tlsSecretName, + "namespace", c.namespace) + + return nil +} + +func (c *impersonatorConfigController) ensureTLSSecretIsRemoved(ctx context.Context) error { + tlsSecretExists, _, err := c.tlsSecretExists() + if err != nil { + return err + } + if !tlsSecretExists { + return nil + } + plog.Info("Deleting TLS certificates for impersonation proxy", + "secret", c.tlsSecretName, + "namespace", c.namespace) + err = c.k8sClient.CoreV1().Secrets(c.namespace).Delete(ctx, c.tlsSecretName, metav1.DeleteOptions{}) + if err != nil { + return err + } + + c.tlsServingCertDynamicCertProvider.UnsetCertKeyContent() + + return nil +} + +func (c *impersonatorConfigController) loadSignerCA(status v1alpha1.StrategyStatus) error { + // Clear it when the impersonator is not completely ready. + if status != v1alpha1.SuccessStrategyStatus { + c.clearSignerCA() + return nil + } + + signingCertSecret, err := c.secretsInformer.Lister().Secrets(c.namespace).Get(c.impersonationSignerSecretName) + if err != nil { + return fmt.Errorf("could not load the impersonator's credential signing secret: %w", err) + } + + certPEM := signingCertSecret.Data[apicerts.CACertificateSecretKey] + keyPEM := signingCertSecret.Data[apicerts.CACertificatePrivateKeySecretKey] + + if err := c.impersonationSigningCertProvider.SetCertKeyContent(certPEM, keyPEM); err != nil { + return fmt.Errorf("could not load the impersonator's credential signing secret: %w", err) + } + + plog.Info("Loading credential signing certificate for impersonation proxy", + "certPEM", string(certPEM), + "fromSecret", c.impersonationSignerSecretName, + "namespace", c.namespace) + + return nil +} + +func (c *impersonatorConfigController) clearSignerCA() { + plog.Info("Clearing credential signing certificate for impersonation proxy") + c.impersonationSigningCertProvider.UnsetCertKeyContent() +} + +func (c *impersonatorConfigController) doSyncResult(nameInfo *certNameInfo, config *impersonator.Config, ca *certauthority.CA) *v1alpha1.CredentialIssuerStrategy { + switch { + case c.disabledExplicitly(config): + return &v1alpha1.CredentialIssuerStrategy{ + Type: v1alpha1.ImpersonationProxyStrategyType, + Status: v1alpha1.ErrorStrategyStatus, + Reason: v1alpha1.DisabledStrategyReason, + Message: "impersonation proxy was explicitly disabled by configuration", + LastUpdateTime: metav1.NewTime(c.clock.Now()), + } + case c.disabledByAutoMode(config): + return &v1alpha1.CredentialIssuerStrategy{ + Type: v1alpha1.ImpersonationProxyStrategyType, + Status: v1alpha1.ErrorStrategyStatus, + Reason: v1alpha1.DisabledStrategyReason, + Message: "automatically determined that impersonation proxy should be disabled", + LastUpdateTime: metav1.NewTime(c.clock.Now()), + } + case !nameInfo.ready: + return &v1alpha1.CredentialIssuerStrategy{ + Type: v1alpha1.ImpersonationProxyStrategyType, + Status: v1alpha1.ErrorStrategyStatus, + Reason: v1alpha1.PendingStrategyReason, + Message: "waiting for load balancer Service to be assigned IP or hostname", + LastUpdateTime: metav1.NewTime(c.clock.Now()), + } + default: + return &v1alpha1.CredentialIssuerStrategy{ + Type: v1alpha1.ImpersonationProxyStrategyType, + Status: v1alpha1.SuccessStrategyStatus, + Reason: v1alpha1.ListeningStrategyReason, + Message: "impersonation proxy is ready to accept client connections", + LastUpdateTime: metav1.NewTime(c.clock.Now()), + Frontend: &v1alpha1.CredentialIssuerFrontend{ + Type: v1alpha1.ImpersonationProxyFrontendType, + ImpersonationProxyInfo: &v1alpha1.ImpersonationProxyInfo{ + Endpoint: "https://" + nameInfo.clientEndpoint, + CertificateAuthorityData: base64.StdEncoding.EncodeToString(ca.Bundle()), + }, + }, + } + } +} diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go new file mode 100644 index 000000000..d785d67fc --- /dev/null +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -0,0 +1,2494 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package impersonatorconfig + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "errors" + "fmt" + "io/ioutil" + "net" + "net/http" + "reflect" + "regexp" + "sync" + "testing" + "time" + + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/clock" + kubeinformers "k8s.io/client-go/informers" + kubernetesfake "k8s.io/client-go/kubernetes/fake" + coretesting "k8s.io/client-go/testing" + + "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" + pinnipedfake "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned/fake" + "go.pinniped.dev/internal/certauthority" + "go.pinniped.dev/internal/controller/apicerts" + "go.pinniped.dev/internal/controllerlib" + "go.pinniped.dev/internal/dynamiccert" + "go.pinniped.dev/internal/kubeclient" + "go.pinniped.dev/internal/testutil" +) + +func TestImpersonatorConfigControllerOptions(t *testing.T) { + spec.Run(t, "options", func(t *testing.T, when spec.G, it spec.S) { + const installedInNamespace = "some-namespace" + const configMapResourceName = "some-configmap-resource-name" + const generatedLoadBalancerServiceName = "some-service-resource-name" + const tlsSecretName = "some-tls-secret-name" //nolint:gosec // this is not a credential + const caSecretName = "some-ca-secret-name" + const caSignerName = "some-ca-signer-name" + + var r *require.Assertions + var observableWithInformerOption *testutil.ObservableWithInformerOption + var observableWithInitialEventOption *testutil.ObservableWithInitialEventOption + var configMapsInformerFilter controllerlib.Filter + var servicesInformerFilter controllerlib.Filter + var secretsInformerFilter controllerlib.Filter + + it.Before(func() { + r = require.New(t) + observableWithInformerOption = testutil.NewObservableWithInformerOption() + observableWithInitialEventOption = testutil.NewObservableWithInitialEventOption() + sharedInformerFactory := kubeinformers.NewSharedInformerFactory(nil, 0) + configMapsInformer := sharedInformerFactory.Core().V1().ConfigMaps() + servicesInformer := sharedInformerFactory.Core().V1().Services() + secretsInformer := sharedInformerFactory.Core().V1().Secrets() + + _ = NewImpersonatorConfigController( + installedInNamespace, + configMapResourceName, + "", + nil, + nil, + configMapsInformer, + servicesInformer, + secretsInformer, + observableWithInformerOption.WithInformer, + observableWithInitialEventOption.WithInitialEvent, + generatedLoadBalancerServiceName, + tlsSecretName, + caSecretName, + nil, + nil, + nil, + caSignerName, + nil, + ) + configMapsInformerFilter = observableWithInformerOption.GetFilterForInformer(configMapsInformer) + servicesInformerFilter = observableWithInformerOption.GetFilterForInformer(servicesInformer) + secretsInformerFilter = observableWithInformerOption.GetFilterForInformer(secretsInformer) + }) + + when("watching ConfigMap objects", func() { + var subject controllerlib.Filter + var target, wrongNamespace, wrongName, unrelated *corev1.ConfigMap + + it.Before(func() { + subject = configMapsInformerFilter + target = &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: configMapResourceName, Namespace: installedInNamespace}} + wrongNamespace = &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: configMapResourceName, Namespace: "wrong-namespace"}} + wrongName = &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "wrong-name", Namespace: installedInNamespace}} + unrelated = &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "wrong-name", Namespace: "wrong-namespace"}} + }) + + when("the target ConfigMap changes", func() { + it("returns true to trigger the sync method", func() { + r.True(subject.Add(target)) + r.True(subject.Update(target, unrelated)) + r.True(subject.Update(unrelated, target)) + r.True(subject.Delete(target)) + }) + }) + + when("a ConfigMap from another namespace changes", func() { + it("returns false to avoid triggering the sync method", func() { + r.False(subject.Add(wrongNamespace)) + r.False(subject.Update(wrongNamespace, unrelated)) + r.False(subject.Update(unrelated, wrongNamespace)) + r.False(subject.Delete(wrongNamespace)) + }) + }) + + when("a ConfigMap with a different name changes", func() { + it("returns false to avoid triggering the sync method", func() { + r.False(subject.Add(wrongName)) + r.False(subject.Update(wrongName, unrelated)) + r.False(subject.Update(unrelated, wrongName)) + r.False(subject.Delete(wrongName)) + }) + }) + + when("a ConfigMap with a different name and a different namespace changes", func() { + it("returns false to avoid triggering the sync method", func() { + r.False(subject.Add(unrelated)) + r.False(subject.Update(unrelated, unrelated)) + r.False(subject.Delete(unrelated)) + }) + }) + }) + + when("watching Service objects", func() { + var subject controllerlib.Filter + var target, wrongNamespace, wrongName, unrelated *corev1.Service + + it.Before(func() { + subject = servicesInformerFilter + target = &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: generatedLoadBalancerServiceName, Namespace: installedInNamespace}} + wrongNamespace = &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: generatedLoadBalancerServiceName, Namespace: "wrong-namespace"}} + wrongName = &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "wrong-name", Namespace: installedInNamespace}} + unrelated = &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "wrong-name", Namespace: "wrong-namespace"}} + }) + + when("the target Service changes", func() { + it("returns true to trigger the sync method", func() { + r.True(subject.Add(target)) + r.True(subject.Update(target, unrelated)) + r.True(subject.Update(unrelated, target)) + r.True(subject.Delete(target)) + }) + }) + + when("a Service from another namespace changes", func() { + it("returns false to avoid triggering the sync method", func() { + r.False(subject.Add(wrongNamespace)) + r.False(subject.Update(wrongNamespace, unrelated)) + r.False(subject.Update(unrelated, wrongNamespace)) + r.False(subject.Delete(wrongNamespace)) + }) + }) + + when("a Service with a different name changes", func() { + it("returns false to avoid triggering the sync method", func() { + r.False(subject.Add(wrongName)) + r.False(subject.Update(wrongName, unrelated)) + r.False(subject.Update(unrelated, wrongName)) + r.False(subject.Delete(wrongName)) + }) + }) + + when("a Service with a different name and a different namespace changes", func() { + it("returns false to avoid triggering the sync method", func() { + r.False(subject.Add(unrelated)) + r.False(subject.Update(unrelated, unrelated)) + r.False(subject.Delete(unrelated)) + }) + }) + }) + + when("watching Secret objects", func() { + var subject controllerlib.Filter + var target1, target2, target3, wrongNamespace1, wrongNamespace2, wrongName, unrelated *corev1.Secret + + it.Before(func() { + subject = secretsInformerFilter + target1 = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: tlsSecretName, Namespace: installedInNamespace}} + target2 = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: caSecretName, Namespace: installedInNamespace}} + target3 = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: caSignerName, Namespace: installedInNamespace}} + wrongNamespace1 = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: tlsSecretName, Namespace: "wrong-namespace"}} + wrongNamespace2 = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: caSecretName, Namespace: "wrong-namespace"}} + wrongName = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "wrong-name", Namespace: installedInNamespace}} + unrelated = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "wrong-name", Namespace: "wrong-namespace"}} + }) + + when("one of the target Secrets changes", func() { + it("returns true to trigger the sync method", func() { + r.True(subject.Add(target1)) + r.True(subject.Update(target1, unrelated)) + r.True(subject.Update(unrelated, target1)) + r.True(subject.Delete(target1)) + r.True(subject.Add(target2)) + r.True(subject.Update(target2, unrelated)) + r.True(subject.Update(unrelated, target2)) + r.True(subject.Delete(target2)) + r.True(subject.Add(target3)) + r.True(subject.Update(target3, unrelated)) + r.True(subject.Update(unrelated, target3)) + r.True(subject.Delete(target3)) + }) + }) + + when("a Secret from another namespace changes", func() { + it("returns false to avoid triggering the sync method", func() { + r.False(subject.Add(wrongNamespace1)) + r.False(subject.Update(wrongNamespace1, unrelated)) + r.False(subject.Update(unrelated, wrongNamespace1)) + r.False(subject.Delete(wrongNamespace1)) + r.False(subject.Add(wrongNamespace2)) + r.False(subject.Update(wrongNamespace2, unrelated)) + r.False(subject.Update(unrelated, wrongNamespace2)) + r.False(subject.Delete(wrongNamespace2)) + }) + }) + + when("a Secret with a different name changes", func() { + it("returns false to avoid triggering the sync method", func() { + r.False(subject.Add(wrongName)) + r.False(subject.Update(wrongName, unrelated)) + r.False(subject.Update(unrelated, wrongName)) + r.False(subject.Delete(wrongName)) + }) + }) + + when("a Secret with a different name and a different namespace changes", func() { + it("returns false to avoid triggering the sync method", func() { + r.False(subject.Add(unrelated)) + r.False(subject.Update(unrelated, unrelated)) + r.False(subject.Delete(unrelated)) + }) + }) + }) + + when("starting up", func() { + it("asks for an initial event because the ConfigMap may not exist yet and it needs to run anyway", func() { + r.Equal(&controllerlib.Key{ + Namespace: installedInNamespace, + Name: configMapResourceName, + }, observableWithInitialEventOption.GetInitialEventKey()) + }) + }) + }, spec.Parallel(), spec.Report(report.Terminal{})) +} + +func TestImpersonatorConfigControllerSync(t *testing.T) { + name := t.Name() + spec.Run(t, "Sync", func(t *testing.T, when spec.G, it spec.S) { + const installedInNamespace = "some-namespace" + const configMapResourceName = "some-configmap-resource-name" + const credentialIssuerResourceName = "some-credential-issuer-resource-name" + const loadBalancerServiceName = "some-service-resource-name" + const tlsSecretName = "some-tls-secret-name" //nolint:gosec // this is not a credential + const caSecretName = "some-ca-secret-name" + const caSignerName = "some-ca-signer-name" + const localhostIP = "127.0.0.1" + const httpsPort = ":443" + const fakeServerResponseBody = "hello, world!" + var labels = map[string]string{"app": "app-name", "other-key": "other-value"} + + var r *require.Assertions + + var subject controllerlib.Controller + var kubeAPIClient *kubernetesfake.Clientset + var pinnipedAPIClient *pinnipedfake.Clientset + var kubeInformerClient *kubernetesfake.Clientset + var kubeInformers kubeinformers.SharedInformerFactory + var cancelContext context.Context + var cancelContextCancelFunc context.CancelFunc + var syncContext *controllerlib.Context + var frozenNow time.Time + var signingCertProvider dynamiccert.Provider + var signingCACertPEM, signingCAKeyPEM []byte + var signingCASecret *corev1.Secret + var impersonatorFuncWasCalled int + var impersonatorFuncError error + var impersonatorFuncReturnedFuncError error + var startedTLSListener net.Listener + var startedTLSListenerMutex sync.RWMutex + var testHTTPServer *http.Server + var testHTTPServerMutex sync.RWMutex + var testHTTPServerInterruptCh chan struct{} + var queue *testQueue + var validClientCert *tls.Certificate + + var impersonatorFunc = func( + port int, + dynamicCertProvider dynamiccert.Private, + impersonationProxySignerCAProvider dynamiccert.Public, + ) (func(stopCh <-chan struct{}) error, error) { + impersonatorFuncWasCalled++ + r.Equal(8444, port) + r.NotNil(dynamicCertProvider) + r.NotNil(impersonationProxySignerCAProvider) + + if impersonatorFuncError != nil { + return nil, impersonatorFuncError + } + + startedTLSListenerMutex.Lock() // this is to satisfy the race detector + defer startedTLSListenerMutex.Unlock() + var err error + // Bind a listener to the port. Automatically choose the port for unit tests instead of using the real port. + startedTLSListener, err = tls.Listen("tcp", localhostIP+":0", &tls.Config{ + MinVersion: tls.VersionTLS12, + GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { + certPEM, keyPEM := dynamicCertProvider.CurrentCertKeyContent() + if certPEM != nil && keyPEM != nil { + tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) + r.NoError(err) + return &tlsCert, nil + } + return nil, nil // no cached TLS certs + }, + ClientAuth: tls.RequestClientCert, + VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + // Docs say that this will always be called in tls.RequestClientCert mode + // and that the second parameter will always be nil in that case. + // rawCerts will be raw ASN.1 certificates provided by the peer. + if len(rawCerts) != 1 { + return fmt.Errorf("expected to get one client cert on incoming request to test server") + } + clientCert := rawCerts[0] + currentClientCertCA := impersonationProxySignerCAProvider.CurrentCABundleContent() + if currentClientCertCA == nil { + return fmt.Errorf("impersonationProxySignerCAProvider does not have a current CA certificate") + } + // Assert that the client's cert was signed by the CA cert that the controller put into + // the CAContentProvider that was passed in. + parsed, err := x509.ParseCertificate(clientCert) + require.NoError(t, err) + roots := x509.NewCertPool() + require.True(t, roots.AppendCertsFromPEM(currentClientCertCA)) + opts := x509.VerifyOptions{ + Roots: roots, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + } + _, err = parsed.Verify(opts) + require.NoError(t, err) + return nil + }, + }) + r.NoError(err) + + // Return a func that starts a fake server when called, and shuts down the fake server when stopCh is closed. + // This fake server is enough like the real impersonation proxy server for this unit test because it + // uses the supplied providers to serve TLS. The goal of this unit test is to make sure that the server + // was started/stopped/configured correctly, not to test the actual impersonation behavior. + return func(stopCh <-chan struct{}) error { + if impersonatorFuncReturnedFuncError != nil { + return impersonatorFuncReturnedFuncError + } + + testHTTPServerMutex.Lock() // this is to satisfy the race detector + testHTTPServer = &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + _, err := fmt.Fprint(w, fakeServerResponseBody) + r.NoError(err) + })} + testHTTPServerMutex.Unlock() + + // Start serving requests in the background. + go func() { + startedTLSListenerMutex.RLock() // this is to satisfy the race detector + listener := startedTLSListener + startedTLSListenerMutex.RUnlock() + err := testHTTPServer.Serve(listener) + if !errors.Is(err, http.ErrServerClosed) { + t.Log("Got an unexpected error while starting the fake http server!") + r.NoError(err) // causes the test to crash, which is good enough because this should never happen + } + }() + + if testHTTPServerInterruptCh == nil { + // Wait in the foreground for the stopCh to be closed, and kill the server when that happens. + // This is similar to the behavior of the real impersonation server. + <-stopCh + } else { + // The test supplied an interrupt channel because it wants to test unexpected termination + // of the server, so wait for that channel to close instead of waiting for the one that + // was passed in from the production code. + <-testHTTPServerInterruptCh + } + + err := testHTTPServer.Close() + t.Log("Got an unexpected error while stopping the fake http server!") + r.NoError(err) // causes the test to crash, which is good enough because this should never happen + + return nil + }, nil + } + + var testServerAddr = func() string { + var listener net.Listener + require.Eventually(t, func() bool { + startedTLSListenerMutex.RLock() // this is to satisfy the race detector + listener = startedTLSListener + defer startedTLSListenerMutex.RUnlock() + return listener != nil + }, 20*time.Second, 50*time.Millisecond, "TLS listener never became not nil") + + return listener.Addr().String() + } + + var closeTestHTTPServer = func() { + // If a test left it running, then close it. + testHTTPServerMutex.RLock() // this is to satisfy the race detector + defer testHTTPServerMutex.RUnlock() + if testHTTPServer != nil { + err := testHTTPServer.Close() + r.NoError(err) + } + } + + var requireTLSServerIsRunning = func(caCrt []byte, addr string, dnsOverrides map[string]string) { + r.Greater(impersonatorFuncWasCalled, 0) + + realDialer := &net.Dialer{} + overrideDialContext := func(ctx context.Context, network, addr string) (net.Conn, error) { + replacementAddr, hasKey := dnsOverrides[addr] + if hasKey { + t.Logf("DialContext replacing addr %s with %s", addr, replacementAddr) + addr = replacementAddr + } else if dnsOverrides != nil { + t.Fatal("dnsOverrides was provided but not used, which was probably a mistake") + } + return realDialer.DialContext(ctx, network, addr) + } + + var tr *http.Transport + if caCrt == nil { + tr = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec + DialContext: overrideDialContext, + } + } else { + rootCAs := x509.NewCertPool() + rootCAs.AppendCertsFromPEM(caCrt) + tr = &http.Transport{ + TLSClientConfig: &tls.Config{ + // Server's TLS serving cert CA + RootCAs: rootCAs, + // Client cert which is supposed to work against the server's dynamic CAContentProvider + Certificates: []tls.Certificate{*validClientCert}, + }, + DialContext: overrideDialContext, + } + } + client := &http.Client{Transport: tr} + url := "https://" + addr + req, err := http.NewRequestWithContext(context.Background(), "GET", url, nil) + r.NoError(err) + + var resp *http.Response + assert.Eventually(t, func() bool { + resp, err = client.Do(req.Clone(context.Background())) //nolint:bodyclose + return err == nil + }, 20*time.Second, 50*time.Millisecond) + r.NoError(err) + + r.Equal(http.StatusOK, resp.StatusCode) + body, err := ioutil.ReadAll(resp.Body) + r.NoError(resp.Body.Close()) + r.NoError(err) + r.Equal(fakeServerResponseBody, string(body)) + } + + var requireTLSServerIsRunningWithoutCerts = func() { + r.Greater(impersonatorFuncWasCalled, 0) + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec + } + client := &http.Client{Transport: tr} + url := "https://" + testServerAddr() + req, err := http.NewRequestWithContext(context.Background(), "GET", url, nil) + r.NoError(err) + + expectedErrorRegex := "Get .*: remote error: tls: unrecognized name" + expectedErrorRegexCompiled, err := regexp.Compile(expectedErrorRegex) + r.NoError(err) + assert.Eventually(t, func() bool { + _, err = client.Do(req.Clone(context.Background())) //nolint:bodyclose + return err != nil && expectedErrorRegexCompiled.MatchString(err.Error()) + }, 20*time.Second, 50*time.Millisecond) + r.Error(err) + r.Regexp(expectedErrorRegex, err.Error()) + } + + var requireTLSServerIsNoLongerRunning = func() { + r.Greater(impersonatorFuncWasCalled, 0) + var err error + expectedErrorRegex := "dial tcp .*: connect: connection refused" + expectedErrorRegexCompiled, err := regexp.Compile(expectedErrorRegex) + r.NoError(err) + assert.Eventually(t, func() bool { + _, err = tls.Dial( + "tcp", + testServerAddr(), + &tls.Config{InsecureSkipVerify: true}, //nolint:gosec + ) + return err != nil && expectedErrorRegexCompiled.MatchString(err.Error()) + }, 20*time.Second, 50*time.Millisecond) + r.Error(err) + r.Regexp(expectedErrorRegex, err.Error()) + } + + var requireTLSServerWasNeverStarted = func() { + r.Equal(0, impersonatorFuncWasCalled) + } + + // Defer starting the informers until the last possible moment so that the + // nested Before's can keep adding things to the informer caches. + var startInformersAndController = func() { + // Set this at the last second to allow for injection of server override. + subject = NewImpersonatorConfigController( + installedInNamespace, + configMapResourceName, + credentialIssuerResourceName, + kubeAPIClient, + pinnipedAPIClient, + kubeInformers.Core().V1().ConfigMaps(), + kubeInformers.Core().V1().Services(), + kubeInformers.Core().V1().Secrets(), + controllerlib.WithInformer, + controllerlib.WithInitialEvent, + loadBalancerServiceName, + tlsSecretName, + caSecretName, + labels, + clock.NewFakeClock(frozenNow), + impersonatorFunc, + caSignerName, + signingCertProvider, + ) + + // Set this at the last second to support calling subject.Name(). + syncContext = &controllerlib.Context{ + Context: cancelContext, + Name: subject.Name(), + Key: controllerlib.Key{ + Namespace: installedInNamespace, + Name: configMapResourceName, + }, + Queue: queue, + } + + // Must start informers before calling TestRunSynchronously() + kubeInformers.Start(cancelContext.Done()) + controllerlib.TestRunSynchronously(t, subject) + } + + var addImpersonatorConfigMapToTracker = func(resourceName, configYAML string, client *kubernetesfake.Clientset) { + impersonatorConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: installedInNamespace, + }, + Data: map[string]string{ + "config.yaml": configYAML, + }, + } + r.NoError(client.Tracker().Add(impersonatorConfigMap)) + } + + var newSecretWithData = func(resourceName string, data map[string][]byte) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: installedInNamespace, + }, + Data: data, + } + } + + var newEmptySecret = func(resourceName string) *corev1.Secret { + return newSecretWithData(resourceName, map[string][]byte{}) + } + + var newCA = func() *certauthority.CA { + ca, err := certauthority.New("test CA", 24*time.Hour) + r.NoError(err) + return ca + } + + var newCACertSecretData = func(ca *certauthority.CA) map[string][]byte { + keyPEM, err := ca.PrivateKeyToPEM() + r.NoError(err) + return map[string][]byte{ + "ca.crt": ca.Bundle(), + "ca.key": keyPEM, + } + } + + var newTLSCertSecretData = func(ca *certauthority.CA, dnsNames []string, ip string) map[string][]byte { + impersonationCert, err := ca.IssueServerCert(dnsNames, []net.IP{net.ParseIP(ip)}, 24*time.Hour) + r.NoError(err) + certPEM, keyPEM, err := certauthority.ToPEM(impersonationCert) + r.NoError(err) + return map[string][]byte{ + corev1.TLSPrivateKeyKey: keyPEM, + corev1.TLSCertKey: certPEM, + } + } + + var newActualCASecret = func(ca *certauthority.CA, resourceName string) *corev1.Secret { + return newSecretWithData(resourceName, newCACertSecretData(ca)) + } + + var newActualTLSSecret = func(ca *certauthority.CA, resourceName string, ip string) *corev1.Secret { + return newSecretWithData(resourceName, newTLSCertSecretData(ca, nil, ip)) + } + + var newActualTLSSecretWithMultipleHostnames = func(ca *certauthority.CA, resourceName string, ip string) *corev1.Secret { + return newSecretWithData(resourceName, newTLSCertSecretData(ca, []string{"foo", "bar"}, ip)) + } + + var newSigningKeySecret = func(resourceName string, certPEM, keyPEM []byte) *corev1.Secret { + return newSecretWithData(resourceName, map[string][]byte{ + apicerts.CACertificateSecretKey: certPEM, + apicerts.CACertificatePrivateKeySecretKey: keyPEM, + }) + } + + var newLoadBalancerService = func(resourceName string, status corev1.ServiceStatus) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: installedInNamespace, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + }, + Status: status, + } + } + + // Anytime an object is added/updated/deleted in the informer's client *after* the informer is started, then we + // need to wait for the informer's cache to asynchronously pick up that change from its "watch". + // If an object is added to the informer's client *before* the informer is started, then waiting is + // not needed because the informer's initial "list" will pick up the object. + var waitForObjectToAppearInInformer = func(obj kubeclient.Object, informer controllerlib.InformerGetter) { + var objFromInformer interface{} + var exists bool + var err error + assert.Eventually(t, func() bool { + objFromInformer, exists, err = informer.Informer().GetIndexer().GetByKey(installedInNamespace + "/" + obj.GetName()) + return err == nil && exists && reflect.DeepEqual(objFromInformer.(kubeclient.Object), obj) + }, 30*time.Second, 10*time.Millisecond) + r.NoError(err) + r.True(exists, "this object should have existed in informer but didn't: %+v", obj) + r.Equal(obj, objFromInformer, "was waiting for expected to be found in informer, but found actual") + } + + // See comment for waitForObjectToAppearInInformer above. + var waitForObjectToBeDeletedFromInformer = func(resourceName string, informer controllerlib.InformerGetter) { + var objFromInformer interface{} + var exists bool + var err error + assert.Eventually(t, func() bool { + objFromInformer, exists, err = informer.Informer().GetIndexer().GetByKey(installedInNamespace + "/" + resourceName) + return err == nil && !exists + }, 30*time.Second, 10*time.Millisecond) + r.NoError(err) + r.False(exists, "this object should have been deleted from informer but wasn't: %s", objFromInformer) + } + + var addObjectToInformerAndWait = func(obj kubeclient.Object, informer controllerlib.InformerGetter) { + r.NoError(kubeInformerClient.Tracker().Add(obj)) + waitForObjectToAppearInInformer(obj, informer) + } + + var addObjectFromCreateActionToInformerAndWait = func(action coretesting.Action, informer controllerlib.InformerGetter) { + createdObject, ok := action.(coretesting.CreateAction).GetObject().(kubeclient.Object) + r.True(ok, "should have been able to cast this action's object to kubeclient.Object: %v", action) + addObjectToInformerAndWait(createdObject, informer) + } + + var updateImpersonatorConfigMapInInformerAndWait = func(resourceName, configYAML string, informer controllerlib.InformerGetter) { + configMapObj, err := kubeInformerClient.Tracker().Get( + schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, + installedInNamespace, + resourceName, + ) + r.NoError(err) + configMap := configMapObj.(*corev1.ConfigMap) + configMap = configMap.DeepCopy() // don't edit the original from the tracker + configMap.Data = map[string]string{ + "config.yaml": configYAML, + } + r.NoError(kubeInformerClient.Tracker().Update( + schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, + configMap, + installedInNamespace, + )) + waitForObjectToAppearInInformer(configMap, informer) + } + + var updateLoadBalancerServiceInInformerAndWait = func(resourceName string, ingresses []corev1.LoadBalancerIngress, informer controllerlib.InformerGetter) { + serviceObj, err := kubeInformerClient.Tracker().Get( + schema.GroupVersionResource{Version: "v1", Resource: "services"}, + installedInNamespace, + resourceName, + ) + r.NoError(err) + service := serviceObj.(*corev1.Service) + service = service.DeepCopy() // don't edit the original from the tracker + service.Status = corev1.ServiceStatus{LoadBalancer: corev1.LoadBalancerStatus{Ingress: ingresses}} + r.NoError(kubeInformerClient.Tracker().Update( + schema.GroupVersionResource{Version: "v1", Resource: "services"}, + service, + installedInNamespace, + )) + waitForObjectToAppearInInformer(service, informer) + } + + var addLoadBalancerServiceToTracker = func(resourceName string, client *kubernetesfake.Clientset) { + loadBalancerService := newLoadBalancerService(resourceName, corev1.ServiceStatus{}) + r.NoError(client.Tracker().Add(loadBalancerService)) + } + + var addLoadBalancerServiceWithIngressToTracker = func(resourceName string, ingress []corev1.LoadBalancerIngress, client *kubernetesfake.Clientset) { + loadBalancerService := newLoadBalancerService(resourceName, corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{Ingress: ingress}, + }) + r.NoError(client.Tracker().Add(loadBalancerService)) + } + + var addSecretToTrackers = func(secret *corev1.Secret, clients ...*kubernetesfake.Clientset) { + for _, client := range clients { + secretCopy := secret.DeepCopy() + r.NoError(client.Tracker().Add(secretCopy)) + } + } + + var deleteServiceFromTracker = func(resourceName string, client *kubernetesfake.Clientset) { + r.NoError(client.Tracker().Delete( + schema.GroupVersionResource{Version: "v1", Resource: "services"}, + installedInNamespace, + resourceName, + )) + } + + var deleteSecretFromTracker = func(resourceName string, client *kubernetesfake.Clientset) { + r.NoError(client.Tracker().Delete( + schema.GroupVersionResource{Version: "v1", Resource: "secrets"}, + installedInNamespace, + resourceName, + )) + } + + var addNodeWithRoleToTracker = func(role string, client *kubernetesfake.Clientset) { + r.NoError(client.Tracker().Add( + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node", + Labels: map[string]string{"kubernetes.io/node-role": role}, + }, + }, + )) + } + + var requireNodesListed = func(action coretesting.Action) { + r.Equal( + coretesting.NewListAction( + schema.GroupVersionResource{Version: "v1", Resource: "nodes"}, + schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Node"}, + "", + metav1.ListOptions{}), + action, + ) + } + + var newSuccessStrategy = func(endpoint string, ca []byte) v1alpha1.CredentialIssuerStrategy { + return v1alpha1.CredentialIssuerStrategy{ + Type: v1alpha1.ImpersonationProxyStrategyType, + Status: v1alpha1.SuccessStrategyStatus, + Reason: v1alpha1.ListeningStrategyReason, + Message: "impersonation proxy is ready to accept client connections", + LastUpdateTime: metav1.NewTime(frozenNow), + Frontend: &v1alpha1.CredentialIssuerFrontend{ + Type: v1alpha1.ImpersonationProxyFrontendType, + ImpersonationProxyInfo: &v1alpha1.ImpersonationProxyInfo{ + Endpoint: "https://" + endpoint, + CertificateAuthorityData: base64.StdEncoding.EncodeToString(ca), + }, + }, + } + } + + var newAutoDisabledStrategy = func() v1alpha1.CredentialIssuerStrategy { + return v1alpha1.CredentialIssuerStrategy{ + Type: v1alpha1.ImpersonationProxyStrategyType, + Status: v1alpha1.ErrorStrategyStatus, + Reason: v1alpha1.DisabledStrategyReason, + Message: "automatically determined that impersonation proxy should be disabled", + LastUpdateTime: metav1.NewTime(frozenNow), + Frontend: nil, + } + } + + var newManuallyDisabledStrategy = func() v1alpha1.CredentialIssuerStrategy { + s := newAutoDisabledStrategy() + s.Message = "impersonation proxy was explicitly disabled by configuration" + return s + } + + var newPendingStrategy = func() v1alpha1.CredentialIssuerStrategy { + return v1alpha1.CredentialIssuerStrategy{ + Type: v1alpha1.ImpersonationProxyStrategyType, + Status: v1alpha1.ErrorStrategyStatus, + Reason: v1alpha1.PendingStrategyReason, + Message: "waiting for load balancer Service to be assigned IP or hostname", + LastUpdateTime: metav1.NewTime(frozenNow), + Frontend: nil, + } + } + + var newErrorStrategy = func(msg string) v1alpha1.CredentialIssuerStrategy { + return v1alpha1.CredentialIssuerStrategy{ + Type: v1alpha1.ImpersonationProxyStrategyType, + Status: v1alpha1.ErrorStrategyStatus, + Reason: v1alpha1.ErrorDuringSetupStrategyReason, + Message: msg, + LastUpdateTime: metav1.NewTime(frozenNow), + Frontend: nil, + } + } + + var getCredentialIssuer = func() *v1alpha1.CredentialIssuer { + credentialIssuerObj, err := pinnipedAPIClient.Tracker().Get( + schema.GroupVersionResource{ + Group: v1alpha1.SchemeGroupVersion.Group, + Version: v1alpha1.SchemeGroupVersion.Version, + Resource: "credentialissuers", + }, "", credentialIssuerResourceName, + ) + r.NoError(err) + credentialIssuer, ok := credentialIssuerObj.(*v1alpha1.CredentialIssuer) + r.True(ok, "should have been able to cast this obj to CredentialIssuer: %v", credentialIssuerObj) + return credentialIssuer + } + + var requireCredentialIssuer = func(expectedStrategy v1alpha1.CredentialIssuerStrategy) { + // Rather than looking at the specific API actions on pinnipedAPIClient, we just look + // at the final result here. + // This is because the implementation is using a helper from another package to create + // and update the CredentialIssuer, and the specific API actions performed by that + // implementation are pretty complex and are already tested by its own unit tests. + // As long as we get the final result that we wanted then we are happy for the purposes + // of this test. + credentialIssuer := getCredentialIssuer() + r.Equal(labels, credentialIssuer.Labels) + r.Equal([]v1alpha1.CredentialIssuerStrategy{expectedStrategy}, credentialIssuer.Status.Strategies) + } + + var requireLoadBalancerWasCreated = func(action coretesting.Action) { + createAction, ok := action.(coretesting.CreateAction) + r.True(ok, "should have been able to cast this action to CreateAction: %v", action) + r.Equal("create", createAction.GetVerb()) + createdLoadBalancerService := createAction.GetObject().(*corev1.Service) + r.Equal(loadBalancerServiceName, createdLoadBalancerService.Name) + r.Equal(installedInNamespace, createdLoadBalancerService.Namespace) + r.Equal(corev1.ServiceTypeLoadBalancer, createdLoadBalancerService.Spec.Type) + r.Equal("app-name", createdLoadBalancerService.Spec.Selector["app"]) + r.Equal(labels, createdLoadBalancerService.Labels) + } + + var requireLoadBalancerWasDeleted = func(action coretesting.Action) { + deleteAction, ok := action.(coretesting.DeleteAction) + r.True(ok, "should have been able to cast this action to DeleteAction: %v", action) + r.Equal("delete", deleteAction.GetVerb()) + r.Equal(loadBalancerServiceName, deleteAction.GetName()) + r.Equal("services", deleteAction.GetResource().Resource) + } + + var requireTLSSecretWasDeleted = func(action coretesting.Action) { + deleteAction, ok := action.(coretesting.DeleteAction) + r.True(ok, "should have been able to cast this action to DeleteAction: %v", action) + r.Equal("delete", deleteAction.GetVerb()) + r.Equal(tlsSecretName, deleteAction.GetName()) + r.Equal("secrets", deleteAction.GetResource().Resource) + } + + var requireCASecretWasCreated = func(action coretesting.Action) []byte { + createAction, ok := action.(coretesting.CreateAction) + r.True(ok, "should have been able to cast this action to CreateAction: %v", action) + r.Equal("create", createAction.GetVerb()) + createdSecret := createAction.GetObject().(*corev1.Secret) + r.Equal(caSecretName, createdSecret.Name) + r.Equal(installedInNamespace, createdSecret.Namespace) + r.Equal(corev1.SecretTypeOpaque, createdSecret.Type) + r.Equal(labels, createdSecret.Labels) + r.Len(createdSecret.Data, 2) + createdCertPEM := createdSecret.Data["ca.crt"] + createdKeyPEM := createdSecret.Data["ca.key"] + r.NotNil(createdCertPEM) + r.NotNil(createdKeyPEM) + _, err := tls.X509KeyPair(createdCertPEM, createdKeyPEM) + r.NoError(err, "key does not match cert") + // Decode and parse the cert to check some of its fields. + block, _ := pem.Decode(createdCertPEM) + require.NotNil(t, block) + caCert, err := x509.ParseCertificate(block.Bytes) + require.NoError(t, err) + require.Equal(t, "Pinniped Impersonation Proxy CA", caCert.Subject.CommonName) + require.WithinDuration(t, time.Now().Add(-10*time.Second), caCert.NotBefore, 10*time.Second) + require.WithinDuration(t, time.Now().Add(100*time.Hour*24*365), caCert.NotAfter, 10*time.Second) + return createdCertPEM + } + + var requireTLSSecretWasCreated = func(action coretesting.Action, caCert []byte) { + createAction, ok := action.(coretesting.CreateAction) + r.True(ok, "should have been able to cast this action to CreateAction: %v", action) + r.Equal("create", createAction.GetVerb()) + createdSecret := createAction.GetObject().(*corev1.Secret) + r.Equal(tlsSecretName, createdSecret.Name) + r.Equal(installedInNamespace, createdSecret.Namespace) + r.Equal(corev1.SecretTypeTLS, createdSecret.Type) + r.Equal(labels, createdSecret.Labels) + r.Len(createdSecret.Data, 2) + createdCertPEM := createdSecret.Data[corev1.TLSCertKey] + createdKeyPEM := createdSecret.Data[corev1.TLSPrivateKeyKey] + r.NotNil(createdKeyPEM) + r.NotNil(createdCertPEM) + validCert := testutil.ValidateServerCertificate(t, string(caCert), string(createdCertPEM)) + validCert.RequireMatchesPrivateKey(string(createdKeyPEM)) + validCert.RequireLifetime(time.Now().Add(-10*time.Second), time.Now().Add(100*time.Hour*24*365), 10*time.Second) + } + + var requireSigningCertProviderHasLoadedCerts = func(certPEM, keyPEM []byte) { + actualCert, actualKey := signingCertProvider.CurrentCertKeyContent() + // Cast to string for better failure messages. + r.Equal(string(certPEM), string(actualCert)) + r.Equal(string(keyPEM), string(actualKey)) + } + + var requireSigningCertProviderIsEmpty = func() { + actualCert, actualKey := signingCertProvider.CurrentCertKeyContent() + r.Nil(actualCert) + r.Nil(actualKey) + } + + var runControllerSync = func() error { + return controllerlib.TestSync(t, subject, *syncContext) + } + + it.Before(func() { + r = require.New(t) + queue = &testQueue{} + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) + kubeInformerClient = kubernetesfake.NewSimpleClientset() + kubeInformers = kubeinformers.NewSharedInformerFactoryWithOptions(kubeInformerClient, 0, + kubeinformers.WithNamespace(installedInNamespace), + ) + kubeAPIClient = kubernetesfake.NewSimpleClientset() + pinnipedAPIClient = pinnipedfake.NewSimpleClientset() + frozenNow = time.Date(2021, time.March, 2, 7, 42, 0, 0, time.Local) + signingCertProvider = dynamiccert.NewCA(name) + + ca := newCA() + signingCACertPEM = ca.Bundle() + var err error + signingCAKeyPEM, err = ca.PrivateKeyToPEM() + r.NoError(err) + signingCASecret = newSigningKeySecret(caSignerName, signingCACertPEM, signingCAKeyPEM) + validClientCert, err = ca.IssueClientCert("username", nil, time.Hour) + r.NoError(err) + }) + + it.After(func() { + cancelContextCancelFunc() + closeTestHTTPServer() + }) + + when("the ConfigMap does not yet exist in the installation namespace or it was deleted (defaults to auto mode)", func() { + it.Before(func() { + addSecretToTrackers(signingCASecret, kubeInformerClient) + addImpersonatorConfigMapToTracker("some-other-unrelated-configmap", "foo: bar", kubeInformerClient) + }) + + when("there are visible control plane nodes", func() { + it.Before(func() { + addNodeWithRoleToTracker("control-plane", kubeAPIClient) + }) + + it("does not start the impersonator or load balancer", func() { + startInformersAndController() + r.NoError(runControllerSync()) + requireTLSServerWasNeverStarted() + r.Len(kubeAPIClient.Actions(), 1) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireCredentialIssuer(newAutoDisabledStrategy()) + requireSigningCertProviderIsEmpty() + }) + }) + + when("there are visible control plane nodes and a loadbalancer and a tls Secret", func() { + it.Before(func() { + addNodeWithRoleToTracker("control-plane", kubeAPIClient) + addLoadBalancerServiceToTracker(loadBalancerServiceName, kubeInformerClient) + addLoadBalancerServiceToTracker(loadBalancerServiceName, kubeAPIClient) + addSecretToTrackers(newEmptySecret(tlsSecretName), kubeAPIClient, kubeInformerClient) + }) + + it("does not start the impersonator, deletes the loadbalancer, deletes the Secret", func() { + startInformersAndController() + r.NoError(runControllerSync()) + requireTLSServerWasNeverStarted() + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerWasDeleted(kubeAPIClient.Actions()[1]) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[2]) + requireCredentialIssuer(newAutoDisabledStrategy()) + requireSigningCertProviderIsEmpty() + }) + }) + + when("there are not visible control plane nodes", func() { + it.Before(func() { + addNodeWithRoleToTracker("worker", kubeAPIClient) + startInformersAndController() + r.NoError(runControllerSync()) + }) + + it("starts the load balancer automatically", func() { + requireTLSServerIsRunningWithoutCerts() + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + requireCASecretWasCreated(kubeAPIClient.Actions()[2]) + requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() + }) + }) + + when("there are not visible control plane nodes and a load balancer already exists without an IP/hostname", func() { + it.Before(func() { + addNodeWithRoleToTracker("worker", kubeAPIClient) + addLoadBalancerServiceToTracker(loadBalancerServiceName, kubeInformerClient) + addLoadBalancerServiceToTracker(loadBalancerServiceName, kubeAPIClient) + startInformersAndController() + r.NoError(runControllerSync()) + }) + + it("does not start the load balancer automatically", func() { + requireTLSServerIsRunningWithoutCerts() + r.Len(kubeAPIClient.Actions(), 2) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() + }) + }) + + when("there are not visible control plane nodes and a load balancer already exists with empty ingress", func() { + it.Before(func() { + addNodeWithRoleToTracker("worker", kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "", Hostname: ""}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "", Hostname: ""}}, kubeAPIClient) + startInformersAndController() + r.NoError(runControllerSync()) + }) + + it("does not start the load balancer automatically", func() { + requireTLSServerIsRunningWithoutCerts() + r.Len(kubeAPIClient.Actions(), 2) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() + }) + }) + + when("there are not visible control plane nodes and a load balancer already exists with invalid ip", func() { + it.Before(func() { + addNodeWithRoleToTracker("worker", kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "not-an-ip"}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "not-an-ip"}}, kubeAPIClient) + startInformersAndController() + r.EqualError(runControllerSync(), "could not find valid IP addresses or hostnames from load balancer some-namespace/some-service-resource-name") + }) + + it("does not start the load balancer automatically", func() { + requireTLSServerIsRunningWithoutCerts() + r.Len(kubeAPIClient.Actions(), 1) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireCredentialIssuer(newErrorStrategy("could not find valid IP addresses or hostnames from load balancer some-namespace/some-service-resource-name")) + requireSigningCertProviderIsEmpty() + }) + }) + + when("there are not visible control plane nodes and a load balancer already exists with multiple ips", func() { + const fakeIP = "127.0.0.123" + it.Before(func() { + addNodeWithRoleToTracker("worker", kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: fakeIP}, {IP: "127.0.0.456"}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: fakeIP}, {IP: "127.0.0.456"}}, kubeAPIClient) + startInformersAndController() + r.NoError(runControllerSync()) + }) + + it("starts the impersonator with certs that match the first IP address", func() { + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) + requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + ":443": testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy(fakeIP, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + + // Simulate the informer cache's background update from its watch. + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Secrets()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) + + // keeps the secret around after resync + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) // nothing changed + requireCredentialIssuer(newSuccessStrategy(fakeIP, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + }) + }) + + when("there are not visible control plane nodes and a load balancer already exists with multiple hostnames", func() { + firstHostname := "fake-1.example.com" + it.Before(func() { + addNodeWithRoleToTracker("worker", kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{Hostname: firstHostname}, {Hostname: "fake-2.example.com"}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{Hostname: firstHostname}, {Hostname: "fake-2.example.com"}}, kubeAPIClient) + startInformersAndController() + r.NoError(runControllerSync()) + }) + + it("starts the impersonator with certs that match the first hostname", func() { + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) + requireTLSServerIsRunning(ca, firstHostname, map[string]string{firstHostname + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy(firstHostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + + // Simulate the informer cache's background update from its watch. + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Secrets()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) + + // keeps the secret around after resync + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) // nothing changed + requireCredentialIssuer(newSuccessStrategy(firstHostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + }) + }) + + when("there are not visible control plane nodes and a load balancer already exists with hostnames and ips", func() { + firstHostname := "fake-1.example.com" + it.Before(func() { + addNodeWithRoleToTracker("worker", kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.254"}, {Hostname: firstHostname}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.254"}, {Hostname: firstHostname}}, kubeAPIClient) + startInformersAndController() + r.NoError(runControllerSync()) + }) + + it("starts the impersonator with certs that match the first hostname", func() { + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) + requireTLSServerIsRunning(ca, firstHostname, map[string]string{firstHostname + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy(firstHostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + + // Simulate the informer cache's background update from its watch. + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Secrets()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) + + // keeps the secret around after resync + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) // nothing changed + requireCredentialIssuer(newSuccessStrategy(firstHostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + }) + }) + + when("there are not visible control plane nodes, a TLS secret exists with multiple hostnames and an IP", func() { + var caCrt []byte + it.Before(func() { + addNodeWithRoleToTracker("worker", kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) + ca := newCA() + caSecret := newActualCASecret(ca, caSecretName) + caCrt = caSecret.Data["ca.crt"] + addSecretToTrackers(caSecret, kubeAPIClient, kubeInformerClient) + addSecretToTrackers(newActualTLSSecretWithMultipleHostnames(ca, tlsSecretName, localhostIP), kubeAPIClient, kubeInformerClient) + startInformersAndController() + r.NoError(runControllerSync()) + }) + + it("deletes and recreates the secret to match the IP in the load balancer without the extra hostnames", func() { + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], caCrt) + requireTLSServerIsRunning(caCrt, testServerAddr(), nil) + requireCredentialIssuer(newSuccessStrategy(localhostIP, caCrt)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + }) + }) + + when("the cert's name needs to change but there is an error while deleting the tls Secret", func() { + it.Before(func() { + addNodeWithRoleToTracker("worker", kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.42"}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.42"}}, kubeAPIClient) + ca := newCA() + addSecretToTrackers(newActualCASecret(ca, caSecretName), kubeAPIClient, kubeInformerClient) + addSecretToTrackers(newActualTLSSecretWithMultipleHostnames(ca, tlsSecretName, localhostIP), kubeAPIClient, kubeInformerClient) + kubeAPIClient.PrependReactor("delete", "secrets", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, fmt.Errorf("error on delete") + }) + }) + + it("returns an error and runs the proxy without certs", func() { + startInformersAndController() + r.Error(runControllerSync(), "error on delete") + r.Len(kubeAPIClient.Actions(), 2) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[1]) + requireTLSServerIsRunningWithoutCerts() + requireCredentialIssuer(newErrorStrategy("error on delete")) + requireSigningCertProviderIsEmpty() + }) + }) + + when("the cert's name might need to change but there is an error while determining the new name", func() { + var caCrt []byte + it.Before(func() { + addNodeWithRoleToTracker("worker", kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) + ca := newCA() + caSecret := newActualCASecret(ca, caSecretName) + caCrt = caSecret.Data["ca.crt"] + addSecretToTrackers(caSecret, kubeAPIClient, kubeInformerClient) + tlsSecret := newActualTLSSecret(ca, tlsSecretName, localhostIP) + addSecretToTrackers(tlsSecret, kubeAPIClient, kubeInformerClient) + }) + + it("returns an error and keeps the proxy running but now without certs", func() { + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 1) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireTLSServerIsRunning(caCrt, testServerAddr(), nil) + + updateLoadBalancerServiceInInformerAndWait(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "not-an-ip"}}, kubeInformers.Core().V1().Services()) + + errString := "could not find valid IP addresses or hostnames from load balancer some-namespace/some-service-resource-name" + r.EqualError(runControllerSync(), errString) + r.Len(kubeAPIClient.Actions(), 1) // no new actions + requireTLSServerIsRunningWithoutCerts() + requireCredentialIssuer(newErrorStrategy(errString)) + requireSigningCertProviderIsEmpty() + }) + }) + }) + + when("the ConfigMap is already in the installation namespace", func() { + it.Before(func() { + addSecretToTrackers(signingCASecret, kubeInformerClient) + }) + + when("the configuration is auto mode with an endpoint", func() { + it.Before(func() { + configMapYAML := fmt.Sprintf("{mode: auto, endpoint: %s}", localhostIP) + addImpersonatorConfigMapToTracker(configMapResourceName, configMapYAML, kubeInformerClient) + }) + + when("there are visible control plane nodes", func() { + it.Before(func() { + addNodeWithRoleToTracker("control-plane", kubeAPIClient) + }) + + it("does not start the impersonator", func() { + startInformersAndController() + r.NoError(runControllerSync()) + requireTLSServerWasNeverStarted() + requireNodesListed(kubeAPIClient.Actions()[0]) + r.Len(kubeAPIClient.Actions(), 1) + requireCredentialIssuer(newAutoDisabledStrategy()) + requireSigningCertProviderIsEmpty() + }) + }) + + when("there are not visible control plane nodes", func() { + it.Before(func() { + addNodeWithRoleToTracker("worker", kubeAPIClient) + }) + + it("starts the impersonator according to the settings in the ConfigMap", func() { + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) + requireTLSServerIsRunning(ca, testServerAddr(), nil) + requireCredentialIssuer(newSuccessStrategy(localhostIP, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + }) + }) + }) + + when("the configuration is disabled mode", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: disabled", kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + }) + + it("does not start the impersonator", func() { + startInformersAndController() + r.NoError(runControllerSync()) + requireTLSServerWasNeverStarted() + requireNodesListed(kubeAPIClient.Actions()[0]) + r.Len(kubeAPIClient.Actions(), 1) + requireCredentialIssuer(newManuallyDisabledStrategy()) + requireSigningCertProviderIsEmpty() + }) + }) + + when("the configuration is enabled mode", func() { + when("no load balancer", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled", kubeInformerClient) + addNodeWithRoleToTracker("control-plane", kubeAPIClient) + }) + + it("starts the impersonator and creates a load balancer", func() { + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + requireCASecretWasCreated(kubeAPIClient.Actions()[2]) + requireTLSServerIsRunningWithoutCerts() + requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() + }) + + it("returns an error when the impersonation TLS server fails to start", func() { + impersonatorFuncError = errors.New("impersonation server start error") + startInformersAndController() + r.EqualError(runControllerSync(), "impersonation server start error") + requireCredentialIssuer(newErrorStrategy("impersonation server start error")) + requireSigningCertProviderIsEmpty() + }) + }) + + when("a loadbalancer already exists", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled", kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + addLoadBalancerServiceToTracker(loadBalancerServiceName, kubeInformerClient) + addLoadBalancerServiceToTracker(loadBalancerServiceName, kubeAPIClient) + }) + + it("starts the impersonator without creating a load balancer", func() { + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 2) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSServerIsRunningWithoutCerts() + requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() + }) + + it("returns an error when the impersonation TLS server fails to start", func() { + impersonatorFuncError = errors.New("impersonation server start error") + startInformersAndController() + r.EqualError(runControllerSync(), "impersonation server start error") + requireCredentialIssuer(newErrorStrategy("impersonation server start error")) + requireSigningCertProviderIsEmpty() + }) + }) + + when("a load balancer and a secret already exists", func() { + var caCrt []byte + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled", kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + ca := newCA() + caSecret := newActualCASecret(ca, caSecretName) + caCrt = caSecret.Data["ca.crt"] + addSecretToTrackers(caSecret, kubeAPIClient, kubeInformerClient) + tlsSecret := newActualTLSSecret(ca, tlsSecretName, localhostIP) + addSecretToTrackers(tlsSecret, kubeAPIClient, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) + }) + + it("starts the impersonator with the existing tls certs, does not start loadbalancer or make tls secret", func() { + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 1) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireTLSServerIsRunning(caCrt, testServerAddr(), nil) + requireCredentialIssuer(newSuccessStrategy(localhostIP, caCrt)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + }) + }) + + when("the configmap has a hostname specified for the endpoint", func() { + const fakeHostname = "fake.example.com" + it.Before(func() { + configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeHostname) + addImpersonatorConfigMapToTracker(configMapResourceName, configMapYAML, kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + }) + + it("starts the impersonator, generates a valid cert for the specified hostname", func() { + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) + // Check that the server is running and that TLS certs that are being served are are for fakeHostname. + requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + }) + }) + + when("the configmap has a endpoint which is an IP address with a port", func() { + const fakeIPWithPort = "127.0.0.1:3000" + it.Before(func() { + configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeIPWithPort) + addImpersonatorConfigMapToTracker(configMapResourceName, configMapYAML, kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + }) + + it("starts the impersonator, generates a valid cert for the specified IP address", func() { + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) + // Check that the server is running and that TLS certs that are being served are are for fakeIPWithPort. + requireTLSServerIsRunning(ca, fakeIPWithPort, map[string]string{fakeIPWithPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy(fakeIPWithPort, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + }) + }) + + when("the configmap has a endpoint which is a hostname with a port", func() { + const fakeHostnameWithPort = "fake.example.com:3000" + it.Before(func() { + configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeHostnameWithPort) + addImpersonatorConfigMapToTracker(configMapResourceName, configMapYAML, kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + }) + + it("starts the impersonator, generates a valid cert for the specified hostname", func() { + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) + // Check that the server is running and that TLS certs that are being served are are for fakeHostnameWithPort. + requireTLSServerIsRunning(ca, fakeHostnameWithPort, map[string]string{fakeHostnameWithPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy(fakeHostnameWithPort, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + }) + }) + + when("switching the configmap from ip address endpoint to hostname endpoint and back to ip address", func() { + const fakeHostname = "fake.example.com" + const fakeIP = "127.0.0.42" + var hostnameYAML = fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeHostname) + var ipAddressYAML = fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeIP) + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, ipAddressYAML, kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + }) + + it("regenerates the cert for the hostname, then regenerates it for the IP again", func() { + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) + // Check that the server is running and that TLS certs that are being served are are for fakeIP. + requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy(fakeIP, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + + // Simulate the informer cache's background update from its watch. + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Secrets()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) + + // Switch the endpoint config to a hostname. + updateImpersonatorConfigMapInInformerAndWait(configMapResourceName, hostnameYAML, kubeInformers.Core().V1().ConfigMaps()) + + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 5) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[3]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[4], ca) // reuses the old CA + // Check that the server is running and that TLS certs that are being served are are for fakeHostname. + requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + + // Simulate the informer cache's background update from its watch. + deleteSecretFromTracker(tlsSecretName, kubeInformerClient) + waitForObjectToBeDeletedFromInformer(tlsSecretName, kubeInformers.Core().V1().Secrets()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[4], kubeInformers.Core().V1().Secrets()) + + // Switch the endpoint config back to an IP. + updateImpersonatorConfigMapInInformerAndWait(configMapResourceName, ipAddressYAML, kubeInformers.Core().V1().ConfigMaps()) + + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 7) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[5]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[6], ca) // reuses the old CA again + // Check that the server is running and that TLS certs that are being served are are for fakeIP. + requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy(fakeIP, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + }) + }) + + when("the TLS cert goes missing and needs to be recreated, e.g. when a user manually deleted it", func() { + const fakeHostname = "fake.example.com" + it.Before(func() { + configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeHostname) + addImpersonatorConfigMapToTracker(configMapResourceName, configMapYAML, kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + startInformersAndController() + }) + + it("uses the existing CA cert the make a new TLS cert", func() { + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) + // Check that the server is running and that TLS certs that are being served are are for fakeHostname. + requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + + // Simulate the informer cache's background update from its watch. + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Secrets()) + + // Delete the TLS Secret that was just created from the Kube API server. Note that we never + // simulated it getting added to the informer cache, so we don't need to remove it from there. + deleteSecretFromTracker(tlsSecretName, kubeAPIClient) + + // Run again. It should create a new TLS cert using the old CA cert. + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 4) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[3], ca) + // Check that the server is running and that TLS certs that are being served are are for fakeHostname. + requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + }) + }) + + when("the CA cert goes missing and needs to be recreated, e.g. when a user manually deleted it", func() { + const fakeHostname = "fake.example.com" + it.Before(func() { + configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeHostname) + addImpersonatorConfigMapToTracker(configMapResourceName, configMapYAML, kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + startInformersAndController() + }) + + it("makes a new CA cert, deletes the old TLS cert, and makes a new TLS cert using the new CA", func() { + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) + // Check that the server is running and that TLS certs that are being served are are for fakeHostname. + requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + + // Simulate the informer cache's background update from its watch. + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) + + // Delete the CA Secret that was just created from the Kube API server. Note that we never + // simulated it getting added to the informer cache, so we don't need to remove it from there. + deleteSecretFromTracker(caSecretName, kubeAPIClient) + + // Run again. It should create both a new CA cert and a new TLS cert using the new CA cert. + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 6) + ca = requireCASecretWasCreated(kubeAPIClient.Actions()[3]) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[4]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[5], ca) // created using the new CA + // Check that the server is running and that TLS certs that are being served are are for fakeHostname. + requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + }) + }) + + when("the CA cert is overwritten by another valid CA cert", func() { + const fakeHostname = "fake.example.com" + var caCrt []byte + it.Before(func() { + configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeHostname) + addImpersonatorConfigMapToTracker(configMapResourceName, configMapYAML, kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) + // Check that the server is running and that TLS certs that are being served are are for fakeHostname. + requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + + // Simulate the informer cache's background update from its watch. + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) + + // Simulate someone updating the CA Secret out of band, e.g. when a human edits it with kubectl. + // Delete the CA Secret that was just created from the Kube API server. Note that we never + // simulated it getting added to the informer cache, so we don't need to remove it from there. + // Then add a new one. Delete + new = update, since only the final state is observed. + deleteSecretFromTracker(caSecretName, kubeAPIClient) + anotherCA := newCA() + newCASecret := newActualCASecret(anotherCA, caSecretName) + caCrt = newCASecret.Data["ca.crt"] + addSecretToTrackers(newCASecret, kubeAPIClient) + addObjectToInformerAndWait(newCASecret, kubeInformers.Core().V1().Secrets()) + }) + + it("deletes the old TLS cert and makes a new TLS cert using the new CA", func() { + // Run again. It should use the updated CA cert to create a new TLS cert. + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 5) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[3]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[4], caCrt) // created using the updated CA + // Check that the server is running and that TLS certs that are being served are are for fakeHostname. + requireTLSServerIsRunning(caCrt, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy(fakeHostname, caCrt)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + }) + + when("deleting the TLS cert due to mismatched CA results in an error", func() { + it.Before(func() { + kubeAPIClient.PrependReactor("delete", "secrets", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + if action.(coretesting.DeleteAction).GetName() == tlsSecretName { + return true, nil, fmt.Errorf("error on tls secret delete") + } + return false, nil, nil + }) + }) + + it("returns an error", func() { + r.Error(runControllerSync(), "error on tls secret delete") + r.Len(kubeAPIClient.Actions(), 4) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[3]) // tried to delete cert but failed + requireCredentialIssuer(newErrorStrategy("error on tls secret delete")) + requireSigningCertProviderIsEmpty() + }) + }) + }) + }) + + when("the configuration switches from enabled to disabled mode", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled", kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + }) + + it("starts the impersonator and loadbalancer, then shuts it down, then starts it again", func() { + startInformersAndController() + + r.NoError(runControllerSync()) + requireTLSServerIsRunningWithoutCerts() + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + requireCASecretWasCreated(kubeAPIClient.Actions()[2]) + requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() + + // Simulate the informer cache's background update from its watch. + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Services()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) + + // Update the configmap. + updateImpersonatorConfigMapInInformerAndWait(configMapResourceName, "mode: disabled", kubeInformers.Core().V1().ConfigMaps()) + + r.NoError(runControllerSync()) + requireTLSServerIsNoLongerRunning() + r.Len(kubeAPIClient.Actions(), 4) + requireLoadBalancerWasDeleted(kubeAPIClient.Actions()[3]) + requireCredentialIssuer(newManuallyDisabledStrategy()) + requireSigningCertProviderIsEmpty() + + deleteServiceFromTracker(loadBalancerServiceName, kubeInformerClient) + waitForObjectToBeDeletedFromInformer(loadBalancerServiceName, kubeInformers.Core().V1().Services()) + + // Update the configmap again. + updateImpersonatorConfigMapInInformerAndWait(configMapResourceName, "mode: enabled", kubeInformers.Core().V1().ConfigMaps()) + + r.NoError(runControllerSync()) + requireTLSServerIsRunningWithoutCerts() + r.Len(kubeAPIClient.Actions(), 5) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[4]) + requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() + }) + }) + + when("the endpoint switches from specified, to not specified, to specified again", func() { + it.Before(func() { + configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", localhostIP) + addImpersonatorConfigMapToTracker(configMapResourceName, configMapYAML, kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + }) + + it("doesn't create, then creates, then deletes the load balancer", func() { + startInformersAndController() + + // Should have started in "enabled" mode with an "endpoint", so no load balancer is needed. + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) // created immediately because "endpoint" was specified + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) + requireTLSServerIsRunning(ca, testServerAddr(), nil) + requireCredentialIssuer(newSuccessStrategy(localhostIP, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + + // Simulate the informer cache's background update from its watch. + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Secrets()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) + + // Switch to "enabled" mode without an "endpoint", so a load balancer is needed now. + updateImpersonatorConfigMapInInformerAndWait(configMapResourceName, "mode: enabled", kubeInformers.Core().V1().ConfigMaps()) + + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 5) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[3]) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[4]) // the Secret was deleted because it contained a cert with the wrong IP + requireTLSServerIsRunningWithoutCerts() + requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() + + // Simulate the informer cache's background update from its watch. + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[3], kubeInformers.Core().V1().Services()) + deleteSecretFromTracker(tlsSecretName, kubeInformerClient) + waitForObjectToBeDeletedFromInformer(tlsSecretName, kubeInformers.Core().V1().Secrets()) + + // The controller should be waiting for the load balancer's ingress to become available. + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 5) // no new actions while it is waiting for the load balancer's ingress + requireTLSServerIsRunningWithoutCerts() + requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() + + // Update the ingress of the LB in the informer's client and run Sync again. + fakeIP := "127.0.0.123" + updateLoadBalancerServiceInInformerAndWait(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: fakeIP}}, kubeInformers.Core().V1().Services()) + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 6) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[5], ca) // reuses the existing CA + // Check that the server is running and that TLS certs that are being served are are for fakeIP. + requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy(fakeIP, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + + // Simulate the informer cache's background update from its watch. + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[5], kubeInformers.Core().V1().Secrets()) + + // Now switch back to having the "endpoint" specified, so the load balancer is not needed anymore. + configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", localhostIP) + updateImpersonatorConfigMapInInformerAndWait(configMapResourceName, configMapYAML, kubeInformers.Core().V1().ConfigMaps()) + + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 9) + requireLoadBalancerWasDeleted(kubeAPIClient.Actions()[6]) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[7]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[8], ca) // recreated because the endpoint was updated, reused the old CA + requireTLSServerIsRunning(ca, testServerAddr(), nil) + requireCredentialIssuer(newSuccessStrategy(localhostIP, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + }) + }) + }) + + when("sync is called more than once", func() { + it.Before(func() { + addSecretToTrackers(signingCASecret, kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + }) + + it("only starts the impersonator once and only lists the cluster's nodes once", func() { + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + requireCASecretWasCreated(kubeAPIClient.Actions()[2]) + requireTLSServerIsRunningWithoutCerts() + requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() + + // Simulate the informer cache's background update from its watch. + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Services()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) + + r.NoError(runControllerSync()) + r.Equal(1, impersonatorFuncWasCalled) // wasn't started a second time + requireTLSServerIsRunningWithoutCerts() // still running + requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() + r.Len(kubeAPIClient.Actions(), 3) // no new API calls + }) + + it("creates certs from the ip address listed on the load balancer", func() { + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[2]) + requireTLSServerIsRunningWithoutCerts() + requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() + + // Simulate the informer cache's background update from its watch. + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Services()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) + + updateLoadBalancerServiceInInformerAndWait(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformers.Core().V1().Services()) + + r.NoError(runControllerSync()) + r.Equal(1, impersonatorFuncWasCalled) // wasn't started a second time + r.Len(kubeAPIClient.Actions(), 4) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[3], ca) // uses the ca from last time + requireTLSServerIsRunning(ca, testServerAddr(), nil) // running with certs now + requireCredentialIssuer(newSuccessStrategy(localhostIP, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + + // Simulate the informer cache's background update from its watch. + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[3], kubeInformers.Core().V1().Secrets()) + + r.NoError(runControllerSync()) + r.Equal(1, impersonatorFuncWasCalled) // wasn't started again + r.Len(kubeAPIClient.Actions(), 4) // no more actions + requireTLSServerIsRunning(ca, testServerAddr(), nil) // still running + requireCredentialIssuer(newSuccessStrategy(localhostIP, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + }) + + it("creates certs from the hostname listed on the load balancer", func() { + hostname := "fake.example.com" + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[2]) + requireTLSServerIsRunningWithoutCerts() + requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() + + // Simulate the informer cache's background update from its watch. + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Services()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) + + updateLoadBalancerServiceInInformerAndWait(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP, Hostname: hostname}}, kubeInformers.Core().V1().Services()) + + r.NoError(runControllerSync()) + r.Equal(1, impersonatorFuncWasCalled) // wasn't started a second time + r.Len(kubeAPIClient.Actions(), 4) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[3], ca) // uses the ca from last time + requireTLSServerIsRunning(ca, hostname, map[string]string{hostname + httpsPort: testServerAddr()}) // running with certs now + requireCredentialIssuer(newSuccessStrategy(hostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + + // Simulate the informer cache's background update from its watch. + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[3], kubeInformers.Core().V1().Secrets()) + + r.NoError(runControllerSync()) + r.Equal(1, impersonatorFuncWasCalled) // wasn't started a third time + r.Len(kubeAPIClient.Actions(), 4) // no more actions + requireTLSServerIsRunning(ca, hostname, map[string]string{hostname + httpsPort: testServerAddr()}) // still running + requireCredentialIssuer(newSuccessStrategy(hostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + }) + }) + + when("there is already a CredentialIssuer", func() { + preExistingStrategy := v1alpha1.CredentialIssuerStrategy{ + Type: v1alpha1.KubeClusterSigningCertificateStrategyType, + Status: v1alpha1.SuccessStrategyStatus, + Reason: v1alpha1.FetchedKeyStrategyReason, + Message: "happy other unrelated strategy", + LastUpdateTime: metav1.NewTime(frozenNow), + Frontend: &v1alpha1.CredentialIssuerFrontend{ + Type: v1alpha1.TokenCredentialRequestAPIFrontendType, + }, + } + + it.Before(func() { + addSecretToTrackers(signingCASecret, kubeInformerClient) + r.NoError(pinnipedAPIClient.Tracker().Add(&v1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: credentialIssuerResourceName}, + Status: v1alpha1.CredentialIssuerStatus{Strategies: []v1alpha1.CredentialIssuerStrategy{preExistingStrategy}}, + })) + addNodeWithRoleToTracker("worker", kubeAPIClient) + }) + + it("merges into the existing strategy array on the CredentialIssuer", func() { + startInformersAndController() + r.NoError(runControllerSync()) + requireTLSServerIsRunningWithoutCerts() + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + requireCASecretWasCreated(kubeAPIClient.Actions()[2]) + credentialIssuer := getCredentialIssuer() + r.Equal([]v1alpha1.CredentialIssuerStrategy{preExistingStrategy, newPendingStrategy()}, credentialIssuer.Status.Strategies) + }) + }) + + when("getting the control plane nodes returns an error, e.g. when there are no nodes", func() { + it("returns an error", func() { + startInformersAndController() + r.EqualError(runControllerSync(), "no nodes found") + requireCredentialIssuer(newErrorStrategy("no nodes found")) + requireSigningCertProviderIsEmpty() + requireTLSServerWasNeverStarted() + }) + }) + + when("the impersonator start function returned by the impersonatorFunc returns an error immediately", func() { + it.Before(func() { + addNodeWithRoleToTracker("worker", kubeAPIClient) + impersonatorFuncReturnedFuncError = errors.New("some immediate impersonator startup error") + }) + + it("causes an immediate resync, returns an error on that next sync, and then restarts the server in a following sync", func() { + startInformersAndController() + // The failure happens in a background goroutine, so the first sync succeeds. + r.NoError(runControllerSync()) + // The imperonatorFunc was called to construct an impersonator. + r.Equal(impersonatorFuncWasCalled, 1) + // Without waiting too long because we don't want the test to be slow, check if it seems like the + // server never started. + r.Never(func() bool { + testHTTPServerMutex.RLock() // this is to satisfy the race detector + defer testHTTPServerMutex.RUnlock() + return testHTTPServer != nil + }, 2*time.Second, 50*time.Millisecond) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + requireCASecretWasCreated(kubeAPIClient.Actions()[2]) + requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() + + // Simulate the informer cache's background update from its watch. + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Services()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) + + // The controller's first sync should have started a background routine which, when the server dies, + // requests to re-enqueue the original sync key to cause its sync method to get called again in the near future. + r.Eventually(func() bool { + queue.mutex.RLock() // this is to satisfy the race detector + defer queue.mutex.RUnlock() + return syncContext.Key == queue.key + }, 10*time.Second, 10*time.Millisecond) + + // The next sync should error because the server died in the background. This second + // sync should be able to detect the error and return it. + r.EqualError(runControllerSync(), "some immediate impersonator startup error") + requireCredentialIssuer(newErrorStrategy("some immediate impersonator startup error")) + requireSigningCertProviderIsEmpty() + + // Next time the controller starts the server, the server will start successfully. + impersonatorFuncReturnedFuncError = nil + + // One more sync and the controller should try to restart the server. + // Now everything should be working correctly. + r.NoError(runControllerSync()) + requireTLSServerIsRunningWithoutCerts() + requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() + }) + }) + + when("the impersonator server dies for no apparent reason after running for a while", func() { + it.Before(func() { + addNodeWithRoleToTracker("worker", kubeAPIClient) + }) + + it("causes an immediate resync, returns an error on that next sync, and then restarts the server in a following sync", func() { + // Prepare to be able to cause the server to die for no apparent reason. + testHTTPServerInterruptCh = make(chan struct{}) + + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + requireCASecretWasCreated(kubeAPIClient.Actions()[2]) + requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() + requireTLSServerIsRunningWithoutCerts() + + // Simulate the informer cache's background update from its watch. + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Services()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) + + // Simulate that impersonation server dies for no apparent reason. + close(testHTTPServerInterruptCh) + + // The controller's first sync should have started a background routine which, when the server dies, + // requests to re-enqueue the original sync key to cause its sync method to get called again in the near future. + r.Eventually(func() bool { + queue.mutex.RLock() // this is to satisfy the race detector + defer queue.mutex.RUnlock() + return syncContext.Key == queue.key + }, 10*time.Second, 10*time.Millisecond) + + // The next sync should error because the server died in the background. This second + // sync should be able to detect the error and return it. + r.EqualError(runControllerSync(), "unexpected shutdown of proxy server") + requireCredentialIssuer(newErrorStrategy("unexpected shutdown of proxy server")) + requireSigningCertProviderIsEmpty() + + // Next time the controller starts the server, the server should behave as normal. + testHTTPServerInterruptCh = nil + + // One more sync and the controller should try to restart the server. + // Now everything should be working correctly. + r.NoError(runControllerSync()) + requireTLSServerIsRunningWithoutCerts() + requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() + }) + }) + + when("the configmap is invalid", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "not yaml", kubeInformerClient) + }) + + it("returns an error", func() { + startInformersAndController() + errString := "invalid impersonator configuration: decode yaml: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type impersonator.Config" + r.EqualError(runControllerSync(), errString) + requireCredentialIssuer(newErrorStrategy(errString)) + requireSigningCertProviderIsEmpty() + requireTLSServerWasNeverStarted() + }) + }) + + when("there is an error creating the load balancer", func() { + it.Before(func() { + addNodeWithRoleToTracker("worker", kubeAPIClient) + kubeAPIClient.PrependReactor("create", "services", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, fmt.Errorf("error on create") + }) + }) + + it("returns an error", func() { + startInformersAndController() + r.EqualError(runControllerSync(), "error on create") + requireCredentialIssuer(newErrorStrategy("error on create")) + requireSigningCertProviderIsEmpty() + requireTLSServerIsRunningWithoutCerts() + }) + }) + + when("there is an error creating the tls secret", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "{mode: enabled, endpoint: example.com}", kubeInformerClient) + addNodeWithRoleToTracker("control-plane", kubeAPIClient) + kubeAPIClient.PrependReactor("create", "secrets", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + createdSecret := action.(coretesting.CreateAction).GetObject().(*corev1.Secret) + if createdSecret.Name == tlsSecretName { + return true, nil, fmt.Errorf("error on tls secret create") + } + return false, nil, nil + }) + }) + + it("starts the impersonator without certs and returns an error", func() { + startInformersAndController() + r.EqualError(runControllerSync(), "error on tls secret create") + requireCredentialIssuer(newErrorStrategy("error on tls secret create")) + requireSigningCertProviderIsEmpty() + requireTLSServerIsRunningWithoutCerts() + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) + }) + }) + + when("there is an error creating the CA secret", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "{mode: enabled, endpoint: example.com}", kubeInformerClient) + addNodeWithRoleToTracker("control-plane", kubeAPIClient) + kubeAPIClient.PrependReactor("create", "secrets", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + createdSecret := action.(coretesting.CreateAction).GetObject().(*corev1.Secret) + if createdSecret.Name == caSecretName { + return true, nil, fmt.Errorf("error on ca secret create") + } + return false, nil, nil + }) + }) + + it("starts the impersonator without certs and returns an error", func() { + startInformersAndController() + r.EqualError(runControllerSync(), "error on ca secret create") + requireCredentialIssuer(newErrorStrategy("error on ca secret create")) + requireSigningCertProviderIsEmpty() + requireTLSServerIsRunningWithoutCerts() + r.Len(kubeAPIClient.Actions(), 2) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + }) + }) + + when("the CA secret exists but is invalid while the TLS secret needs to be created", func() { + it.Before(func() { + addNodeWithRoleToTracker("control-plane", kubeAPIClient) + addImpersonatorConfigMapToTracker(configMapResourceName, "{mode: enabled, endpoint: example.com}", kubeInformerClient) + addSecretToTrackers(newEmptySecret(caSecretName), kubeAPIClient, kubeInformerClient) + }) + + it("starts the impersonator without certs and returns an error", func() { + startInformersAndController() + errString := "could not load CA: tls: failed to find any PEM data in certificate input" + r.EqualError(runControllerSync(), errString) + requireCredentialIssuer(newErrorStrategy(errString)) + requireSigningCertProviderIsEmpty() + requireTLSServerIsRunningWithoutCerts() + r.Len(kubeAPIClient.Actions(), 1) + requireNodesListed(kubeAPIClient.Actions()[0]) + }) + }) + + when("there is an error deleting the tls secret", func() { + it.Before(func() { + addNodeWithRoleToTracker("control-plane", kubeAPIClient) + addLoadBalancerServiceToTracker(loadBalancerServiceName, kubeInformerClient) + addLoadBalancerServiceToTracker(loadBalancerServiceName, kubeAPIClient) + addSecretToTrackers(newEmptySecret(tlsSecretName), kubeAPIClient, kubeInformerClient) + startInformersAndController() + kubeAPIClient.PrependReactor("delete", "secrets", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, fmt.Errorf("error on delete") + }) + }) + + it("does not start the impersonator, deletes the loadbalancer, returns an error", func() { + r.EqualError(runControllerSync(), "error on delete") + requireCredentialIssuer(newErrorStrategy("error on delete")) + requireSigningCertProviderIsEmpty() + requireTLSServerWasNeverStarted() + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerWasDeleted(kubeAPIClient.Actions()[1]) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[2]) + }) + }) + + when("the PEM formatted data in the TLS Secret is not a valid cert", func() { + it.Before(func() { + addSecretToTrackers(signingCASecret, kubeInformerClient) + configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", localhostIP) + addImpersonatorConfigMapToTracker(configMapResourceName, configMapYAML, kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + tlsSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: tlsSecretName, + Namespace: installedInNamespace, + }, + Data: map[string][]byte{ + // "aGVsbG8gd29ybGQK" is "hello world" base64 encoded which is not a valid cert + corev1.TLSCertKey: []byte("-----BEGIN CERTIFICATE-----\naGVsbG8gd29ybGQK\n-----END CERTIFICATE-----\n"), + }, + } + addSecretToTrackers(tlsSecret, kubeAPIClient, kubeInformerClient) + }) + + it("deletes the invalid certs, creates new certs, and starts the impersonator", func() { + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 4) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[2]) // deleted the bad cert + requireTLSSecretWasCreated(kubeAPIClient.Actions()[3], ca) + requireTLSServerIsRunning(ca, testServerAddr(), nil) + requireCredentialIssuer(newSuccessStrategy(localhostIP, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + }) + + when("there is an error while the invalid cert is being deleted", func() { + it.Before(func() { + kubeAPIClient.PrependReactor("delete", "secrets", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, fmt.Errorf("error on delete") + }) + }) + + it("tries to delete the invalid cert, starts the impersonator without certs, and returns an error", func() { + startInformersAndController() + errString := "PEM data represented an invalid cert, but got error while deleting it: error on delete" + r.EqualError(runControllerSync(), errString) + requireCredentialIssuer(newErrorStrategy(errString)) + requireSigningCertProviderIsEmpty() + requireTLSServerIsRunningWithoutCerts() + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[2]) // tried deleted the bad cert, which failed + requireTLSServerIsRunningWithoutCerts() + }) + }) + }) + + when("a tls secret already exists but it is not valid", func() { + var caCrt []byte + it.Before(func() { + addSecretToTrackers(signingCASecret, kubeInformerClient) + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled", kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + ca := newCA() + caSecret := newActualCASecret(ca, caSecretName) + caCrt = caSecret.Data["ca.crt"] + addSecretToTrackers(caSecret, kubeAPIClient, kubeInformerClient) + addSecretToTrackers(newEmptySecret(tlsSecretName), kubeAPIClient, kubeInformerClient) // secret exists but lacks certs + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) + }) + + it("deletes the invalid certs, creates new certs, and starts the impersonator", func() { + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[1]) // deleted the bad cert + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], caCrt) + requireTLSServerIsRunning(caCrt, testServerAddr(), nil) + requireCredentialIssuer(newSuccessStrategy(localhostIP, caCrt)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + }) + + when("there is an error while the invalid cert is being deleted", func() { + it.Before(func() { + kubeAPIClient.PrependReactor("delete", "secrets", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, fmt.Errorf("error on delete") + }) + }) + + it("tries to delete the invalid cert, starts the impersonator without certs, and returns an error", func() { + startInformersAndController() + errString := "found missing or not PEM-encoded data in TLS Secret, but got error while deleting it: error on delete" + r.EqualError(runControllerSync(), errString) + requireCredentialIssuer(newErrorStrategy(errString)) + requireSigningCertProviderIsEmpty() + requireTLSServerIsRunningWithoutCerts() + r.Len(kubeAPIClient.Actions(), 2) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[1]) // tried deleted the bad cert, which failed + requireTLSServerIsRunningWithoutCerts() + }) + }) + }) + + when("a tls secret already exists but the private key is not valid", func() { + var caCrt []byte + it.Before(func() { + addSecretToTrackers(signingCASecret, kubeInformerClient) + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled", kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + ca := newCA() + caSecret := newActualCASecret(ca, caSecretName) + caCrt = caSecret.Data["ca.crt"] + addSecretToTrackers(caSecret, kubeAPIClient, kubeInformerClient) + tlsSecret := newActualTLSSecret(ca, tlsSecretName, localhostIP) + tlsSecret.Data["tls.key"] = nil + addSecretToTrackers(tlsSecret, kubeAPIClient, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) + }) + + it("deletes the invalid certs, creates new certs, and starts the impersonator", func() { + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[1]) // deleted the bad cert + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], caCrt) + requireTLSServerIsRunning(caCrt, testServerAddr(), nil) + requireCredentialIssuer(newSuccessStrategy(localhostIP, caCrt)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + }) + + when("there is an error while the invalid cert is being deleted", func() { + it.Before(func() { + kubeAPIClient.PrependReactor("delete", "secrets", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, fmt.Errorf("error on delete") + }) + }) + + it("tries to delete the invalid cert, starts the impersonator without certs, and returns an error", func() { + startInformersAndController() + errString := "cert had an invalid private key, but got error while deleting it: error on delete" + r.EqualError(runControllerSync(), errString) + requireCredentialIssuer(newErrorStrategy(errString)) + requireSigningCertProviderIsEmpty() + requireTLSServerIsRunningWithoutCerts() + r.Len(kubeAPIClient.Actions(), 2) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[1]) // tried deleted the bad cert, which failed + requireTLSServerIsRunningWithoutCerts() + }) + }) + }) + + when("there is an error while creating or updating the CredentialIssuer status", func() { + it.Before(func() { + addSecretToTrackers(signingCASecret, kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + pinnipedAPIClient.PrependReactor("create", "credentialissuers", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, fmt.Errorf("error on create") + }) + }) + + it("returns the error", func() { + startInformersAndController() + r.EqualError(runControllerSync(), "could not create or update credentialissuer: create failed: error on create") + }) + + when("there is also a more fundamental error while starting the impersonator", func() { + it.Before(func() { + kubeAPIClient.PrependReactor("create", "services", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, fmt.Errorf("error on service creation") + }) + }) + + it("returns the more fundamental error instead of the CredentialIssuer error", func() { + startInformersAndController() + r.EqualError(runControllerSync(), "error on service creation") + }) + }) + }) + + when("the impersonator is ready but there is a problem with the signing secret, which should be created by another controller", func() { + const fakeHostname = "foo.example.com" + it.Before(func() { + configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeHostname) + addImpersonatorConfigMapToTracker(configMapResourceName, configMapYAML, kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + }) + + when("it does not exist in the informers", func() { + it("returns the error", func() { + startInformersAndController() + errString := `could not load the impersonator's credential signing secret: secret "some-ca-signer-name" not found` + r.EqualError(runControllerSync(), errString) + requireCredentialIssuer(newErrorStrategy(errString)) + requireSigningCertProviderIsEmpty() + }) + }) + + when("it does not have the expected fields", func() { + it.Before(func() { + addSecretToTrackers(newEmptySecret(caSignerName), kubeInformerClient) + }) + + it("returns the error", func() { + startInformersAndController() + errString := `could not load the impersonator's credential signing secret: TestImpersonatorConfigControllerSync: attempt to set invalid key pair: tls: failed to find any PEM data in certificate input` + r.EqualError(runControllerSync(), errString) + requireCredentialIssuer(newErrorStrategy(errString)) + requireSigningCertProviderIsEmpty() + }) + }) + + when("the cert is invalid", func() { + it.Before(func() { + signingCASecret.Data[apicerts.CACertificateSecretKey] = []byte("not a valid PEM formatted cert") + addSecretToTrackers(signingCASecret, kubeInformerClient) + }) + + it("returns the error", func() { + startInformersAndController() + errString := `could not load the impersonator's credential signing secret: TestImpersonatorConfigControllerSync: attempt to set invalid key pair: tls: failed to find any PEM data in certificate input` + r.EqualError(runControllerSync(), errString) + requireCredentialIssuer(newErrorStrategy(errString)) + requireSigningCertProviderIsEmpty() + }) + }) + + when("the cert goes from being valid to being invalid", func() { + const fakeHostname = "foo.example.com" + it.Before(func() { + addSecretToTrackers(signingCASecret, kubeInformerClient) + }) + + it("returns the error and clears the dynamic provider", func() { + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) + // Check that the server is running and that TLS certs that are being served are are for fakeHostname. + requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + + // Simulate the informer cache's background update from its watch. + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Secrets()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) + + // Now update the signer CA to something invalid. + deleteSecretFromTracker(caSignerName, kubeInformerClient) + waitForObjectToBeDeletedFromInformer(caSignerName, kubeInformers.Core().V1().Secrets()) + updatedSigner := newEmptySecret(caSignerName) + addSecretToTrackers(updatedSigner, kubeInformerClient) + waitForObjectToAppearInInformer(updatedSigner, kubeInformers.Core().V1().Secrets()) + + errString := `could not load the impersonator's credential signing secret: TestImpersonatorConfigControllerSync: attempt to set invalid key pair: tls: failed to find any PEM data in certificate input` + r.EqualError(runControllerSync(), errString) + requireCredentialIssuer(newErrorStrategy(errString)) + requireSigningCertProviderIsEmpty() + }) + }) + }) + }, spec.Parallel(), spec.Report(report.Terminal{})) +} + +type testQueue struct { + key controllerlib.Key + mutex sync.RWMutex + + controllerlib.Queue +} + +func (q *testQueue) AddRateLimited(key controllerlib.Key) { + q.mutex.Lock() // this is to satisfy the race detector + defer q.mutex.Unlock() + + if q.key != (controllerlib.Key{}) { + panic("called more than once") + } + + if key == (controllerlib.Key{}) { + panic("unexpected empty key") + } + + q.key = key +} diff --git a/internal/controller/issuerconfig/update_strategy.go b/internal/controller/issuerconfig/update_strategy.go index 669516ffc..f357aed91 100644 --- a/internal/controller/issuerconfig/update_strategy.go +++ b/internal/controller/issuerconfig/update_strategy.go @@ -52,9 +52,21 @@ func mergeStrategy(configToUpdate *v1alpha1.CredentialIssuerStatus, strategy v1a } } -// TODO: sort strategies by server preference rather than alphanumerically by type. +// weights are a set of priorities for each strategy type. +//nolint: gochecknoglobals +var weights = map[v1alpha1.StrategyType]int{ + v1alpha1.KubeClusterSigningCertificateStrategyType: 2, // most preferred strategy + v1alpha1.ImpersonationProxyStrategyType: 1, + // unknown strategy types will have weight 0 by default +} + type sortableStrategies []v1alpha1.CredentialIssuerStrategy -func (s sortableStrategies) Len() int { return len(s) } -func (s sortableStrategies) Less(i, j int) bool { return s[i].Type < s[j].Type } -func (s sortableStrategies) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s sortableStrategies) Len() int { return len(s) } +func (s sortableStrategies) Less(i, j int) bool { + if wi, wj := weights[s[i].Type], weights[s[j].Type]; wi != wj { + return wi > wj + } + return s[i].Type < s[j].Type +} +func (s sortableStrategies) Swap(i, j int) { s[i], s[j] = s[j], s[i] } diff --git a/internal/controller/issuerconfig/update_strategy_test.go b/internal/controller/issuerconfig/update_strategy_test.go index b1b904292..163024993 100644 --- a/internal/controller/issuerconfig/update_strategy_test.go +++ b/internal/controller/issuerconfig/update_strategy_test.go @@ -4,9 +4,13 @@ package issuerconfig import ( + "math/rand" + "sort" "testing" + "testing/quick" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -185,3 +189,30 @@ func TestMergeStrategy(t *testing.T) { }) } } + +func TestStrategySorting(t *testing.T) { + expected := []v1alpha1.CredentialIssuerStrategy{ + {Type: v1alpha1.KubeClusterSigningCertificateStrategyType}, + {Type: v1alpha1.ImpersonationProxyStrategyType}, + {Type: "Type1"}, + {Type: "Type2"}, + {Type: "Type3"}, + } + require.NoError(t, quick.Check(func(seed int64) bool { + // Create a randomly shuffled copy of the expected output. + //nolint:gosec // this is not meant to be a secure random, just a seeded RNG for shuffling deterministically + rng := rand.New(rand.NewSource(seed)) + output := make([]v1alpha1.CredentialIssuerStrategy, len(expected)) + copy(output, expected) + rng.Shuffle( + len(output), + func(i, j int) { output[i], output[j] = output[j], output[i] }, + ) + + // Sort it using the code under test. + sort.Stable(sortableStrategies(output)) + + // Assert that it's sorted back to the expected output order. + return assert.Equal(t, expected, output) + }, nil)) +} diff --git a/internal/controller/kubecertagent/annotater_test.go b/internal/controller/kubecertagent/annotater_test.go index bb767a474..379133fa5 100644 --- a/internal/controller/kubecertagent/annotater_test.go +++ b/internal/controller/kubecertagent/annotater_test.go @@ -79,8 +79,8 @@ func TestAnnotaterControllerSync(t *testing.T) { var agentInformerClient *kubernetesfake.Clientset var agentInformers kubeinformers.SharedInformerFactory var pinnipedAPIClient *pinnipedfake.Clientset - var timeoutContext context.Context - var timeoutContextCancel context.CancelFunc + var cancelContext context.Context + var cancelContextCancelFunc context.CancelFunc var syncContext *controllerlib.Context var controllerManagerPod, agentPod *corev1.Pod var podsGVR schema.GroupVersionResource @@ -116,7 +116,7 @@ func TestAnnotaterControllerSync(t *testing.T) { // Set this at the last second to support calling subject.Name(). syncContext = &controllerlib.Context{ - Context: timeoutContext, + Context: cancelContext, Name: subject.Name(), Key: controllerlib.Key{ Namespace: kubeSystemNamespace, @@ -125,8 +125,8 @@ func TestAnnotaterControllerSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - kubeSystemInformers.Start(timeoutContext.Done()) - agentInformers.Start(timeoutContext.Done()) + kubeSystemInformers.Start(cancelContext.Done()) + agentInformers.Start(cancelContext.Done()) controllerlib.TestRunSynchronously(t, subject) } @@ -143,7 +143,7 @@ func TestAnnotaterControllerSync(t *testing.T) { pinnipedAPIClient = pinnipedfake.NewSimpleClientset() - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) controllerManagerPod, agentPod = exampleControllerManagerAndAgentPods( kubeSystemNamespace, agentPodNamespace, certPath, keyPath, @@ -173,7 +173,7 @@ func TestAnnotaterControllerSync(t *testing.T) { }) it.After(func() { - timeoutContextCancel() + cancelContextCancelFunc() }) when("there is an agent pod without annotations set", func() { diff --git a/internal/controller/kubecertagent/creater_test.go b/internal/controller/kubecertagent/creater_test.go index eab57997a..13dd9943c 100644 --- a/internal/controller/kubecertagent/creater_test.go +++ b/internal/controller/kubecertagent/creater_test.go @@ -94,8 +94,8 @@ func TestCreaterControllerSync(t *testing.T) { var agentInformerClient *kubernetesfake.Clientset var agentInformers kubeinformers.SharedInformerFactory var pinnipedAPIClient *pinnipedfake.Clientset - var timeoutContext context.Context - var timeoutContextCancel context.CancelFunc + var cancelContext context.Context + var cancelContextCancelFunc context.CancelFunc var syncContext *controllerlib.Context var controllerManagerPod, agentPod *corev1.Pod var podsGVR schema.GroupVersionResource @@ -135,7 +135,7 @@ func TestCreaterControllerSync(t *testing.T) { // Set this at the last second to support calling subject.Name(). syncContext = &controllerlib.Context{ - Context: timeoutContext, + Context: cancelContext, Name: subject.Name(), Key: controllerlib.Key{ Namespace: kubeSystemNamespace, @@ -144,8 +144,8 @@ func TestCreaterControllerSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - kubeSystemInformers.Start(timeoutContext.Done()) - agentInformers.Start(timeoutContext.Done()) + kubeSystemInformers.Start(cancelContext.Done()) + agentInformers.Start(cancelContext.Done()) controllerlib.TestRunSynchronously(t, subject) } @@ -162,7 +162,7 @@ func TestCreaterControllerSync(t *testing.T) { pinnipedAPIClient = pinnipedfake.NewSimpleClientset() - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) controllerManagerPod, agentPod = exampleControllerManagerAndAgentPods( kubeSystemNamespace, agentPodNamespace, "ignored for this test", "ignored for this test", @@ -201,7 +201,7 @@ func TestCreaterControllerSync(t *testing.T) { }) it.After(func() { - timeoutContextCancel() + cancelContextCancelFunc() }) when("there is a controller manager pod", func() { diff --git a/internal/controller/kubecertagent/deleter_test.go b/internal/controller/kubecertagent/deleter_test.go index 2a8b5721d..ba5240af2 100644 --- a/internal/controller/kubecertagent/deleter_test.go +++ b/internal/controller/kubecertagent/deleter_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package kubecertagent @@ -6,7 +6,6 @@ package kubecertagent import ( "context" "testing" - "time" "github.com/sclevine/spec" "github.com/sclevine/spec/report" @@ -57,8 +56,8 @@ func TestDeleterControllerSync(t *testing.T) { var kubeSystemInformers kubeinformers.SharedInformerFactory var agentInformerClient *kubernetesfake.Clientset var agentInformers kubeinformers.SharedInformerFactory - var timeoutContext context.Context - var timeoutContextCancel context.CancelFunc + var cancelContext context.Context + var cancelContextCancelFunc context.CancelFunc var syncContext *controllerlib.Context var controllerManagerPod, agentPod *corev1.Pod var podsGVR schema.GroupVersionResource @@ -85,7 +84,7 @@ func TestDeleterControllerSync(t *testing.T) { // Set this at the last second to support calling subject.Name(). syncContext = &controllerlib.Context{ - Context: timeoutContext, + Context: cancelContext, Name: subject.Name(), Key: controllerlib.Key{ Namespace: kubeSystemNamespace, @@ -94,8 +93,8 @@ func TestDeleterControllerSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - kubeSystemInformers.Start(timeoutContext.Done()) - agentInformers.Start(timeoutContext.Done()) + kubeSystemInformers.Start(cancelContext.Done()) + agentInformers.Start(cancelContext.Done()) controllerlib.TestRunSynchronously(t, subject) } @@ -109,7 +108,7 @@ func TestDeleterControllerSync(t *testing.T) { it.Before(func() { r = require.New(t) - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) kubeAPIClient = kubernetesfake.NewSimpleClientset() @@ -139,7 +138,7 @@ func TestDeleterControllerSync(t *testing.T) { }) it.After(func() { - timeoutContextCancel() + cancelContextCancelFunc() }) when("there is an agent pod", func() { diff --git a/internal/controller/kubecertagent/execer.go b/internal/controller/kubecertagent/execer.go index b8ff1c83d..e021ff261 100644 --- a/internal/controller/kubecertagent/execer.go +++ b/internal/controller/kubecertagent/execer.go @@ -11,9 +11,9 @@ import ( k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/clock" + "k8s.io/apimachinery/pkg/util/errors" corev1informers "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/tools/clientcmd" - "k8s.io/klog/v2" configv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" pinnipedclientset "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned" @@ -33,7 +33,7 @@ type execerController struct { credentialIssuerLocationConfig *CredentialIssuerLocationConfig credentialIssuerLabels map[string]string discoveryURLOverride *string - dynamicCertProvider dynamiccert.Provider + dynamicCertProvider dynamiccert.Private podCommandExecutor PodCommandExecutor clock clock.Clock pinnipedAPIClient pinnipedclientset.Interface @@ -51,7 +51,7 @@ func NewExecerController( credentialIssuerLocationConfig *CredentialIssuerLocationConfig, credentialIssuerLabels map[string]string, discoveryURLOverride *string, - dynamicCertProvider dynamiccert.Provider, + dynamicCertProvider dynamiccert.Private, podCommandExecutor PodCommandExecutor, pinnipedAPIClient pinnipedclientset.Interface, clock clock.Clock, @@ -119,8 +119,7 @@ func (c *execerController) Sync(ctx controllerlib.Context) error { c.pinnipedAPIClient, strategyError(c.clock, err), ) - klog.ErrorS(strategyResultUpdateErr, "could not create or update CredentialIssuer with strategy success") - return err + return newAggregate(err, strategyResultUpdateErr) } keyPEM, err := c.podCommandExecutor.Exec(agentPod.Namespace, agentPod.Name, "cat", keyPath) @@ -132,11 +131,20 @@ func (c *execerController) Sync(ctx controllerlib.Context) error { c.pinnipedAPIClient, strategyError(c.clock, err), ) - klog.ErrorS(strategyResultUpdateErr, "could not create or update CredentialIssuer with strategy success") - return err + return newAggregate(err, strategyResultUpdateErr) } - c.dynamicCertProvider.Set([]byte(certPEM), []byte(keyPEM)) + if err := c.dynamicCertProvider.SetCertKeyContent([]byte(certPEM), []byte(keyPEM)); err != nil { + err = fmt.Errorf("failed to set signing cert/key content from agent pod %s/%s: %w", agentPod.Namespace, agentPod.Name, err) + strategyResultUpdateErr := issuerconfig.UpdateStrategy( + ctx.Context, + c.credentialIssuerLocationConfig.Name, + c.credentialIssuerLabels, + c.pinnipedAPIClient, + strategyError(c.clock, err), + ) + return newAggregate(err, strategyResultUpdateErr) + } apiInfo, err := c.getTokenCredentialRequestAPIInfo() if err != nil { @@ -153,8 +161,7 @@ func (c *execerController) Sync(ctx controllerlib.Context) error { LastUpdateTime: metav1.NewTime(c.clock.Now()), }, ) - klog.ErrorS(strategyResultUpdateErr, "could not create or update CredentialIssuer with strategy success") - return err + return newAggregate(err, strategyResultUpdateErr) } return issuerconfig.UpdateStrategy( @@ -219,3 +226,7 @@ func (c *execerController) getKeypairFilePaths(pod *v1.Pod) (string, string) { return certPath, keyPath } + +func newAggregate(errs ...error) error { + return errors.NewAggregate(errs) +} diff --git a/internal/controller/kubecertagent/execer_test.go b/internal/controller/kubecertagent/execer_test.go index b37891761..c9441010b 100644 --- a/internal/controller/kubecertagent/execer_test.go +++ b/internal/controller/kubecertagent/execer_test.go @@ -132,6 +132,7 @@ func (s *fakePodExecutor) Exec(podNamespace string, podName string, commandAndAr } func TestManagerControllerSync(t *testing.T) { + name := t.Name() spec.Run(t, "Sync", func(t *testing.T, when spec.G, it spec.S) { const agentPodNamespace = "some-namespace" const agentPodName = "some-agent-pod-name-123" @@ -139,15 +140,13 @@ func TestManagerControllerSync(t *testing.T) { const keyPathAnnotationName = "kube-cert-agent.pinniped.dev/key-path" const fakeCertPath = "/some/cert/path" const fakeKeyPath = "/some/key/path" - const defaultDynamicCertProviderCert = "initial-cert" - const defaultDynamicCertProviderKey = "initial-key" const credentialIssuerResourceName = "ci-resource-name" var r *require.Assertions var subject controllerlib.Controller - var timeoutContext context.Context - var timeoutContextCancel context.CancelFunc + var cancelContext context.Context + var cancelContextCancelFunc context.CancelFunc var syncContext *controllerlib.Context var pinnipedAPIClient *pinnipedfake.Clientset var kubeInformerFactory kubeinformers.SharedInformerFactory @@ -159,6 +158,8 @@ func TestManagerControllerSync(t *testing.T) { var fakeCertPEM, fakeKeyPEM string var credentialIssuerGVR schema.GroupVersionResource var frozenNow time.Time + var defaultDynamicCertProviderCert string + var defaultDynamicCertProviderKey string // Defer starting the informers until the last possible moment so that the // nested Before's can keep adding things to the informer caches. @@ -181,7 +182,7 @@ func TestManagerControllerSync(t *testing.T) { // Set this at the last second to support calling subject.Name(). syncContext = &controllerlib.Context{ - Context: timeoutContext, + Context: cancelContext, Name: subject.Name(), Key: controllerlib.Key{ Namespace: agentPodNamespace, @@ -190,7 +191,7 @@ func TestManagerControllerSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - kubeInformerFactory.Start(timeoutContext.Done()) + kubeInformerFactory.Start(cancelContext.Done()) controllerlib.TestRunSynchronously(t, subject) } @@ -228,14 +229,23 @@ func TestManagerControllerSync(t *testing.T) { it.Before(func() { r = require.New(t) - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + crt, key, err := testutil.CreateCertificate( + time.Now().Add(-time.Hour), + time.Now().Add(time.Hour), + ) + require.NoError(t, err) + defaultDynamicCertProviderCert = string(crt) + defaultDynamicCertProviderKey = string(key) + + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) pinnipedAPIClient = pinnipedfake.NewSimpleClientset() kubeClientset = kubernetesfake.NewSimpleClientset() kubeInformerFactory = kubeinformers.NewSharedInformerFactory(kubeClientset, 0) fakeExecutor = &fakePodExecutor{r: r} frozenNow = time.Date(2020, time.September, 23, 7, 42, 0, 0, time.Local) - dynamicCertProvider = dynamiccert.New() - dynamicCertProvider.Set([]byte(defaultDynamicCertProviderCert), []byte(defaultDynamicCertProviderKey)) + dynamicCertProvider = dynamiccert.NewCA(name) + err = dynamicCertProvider.SetCertKeyContent([]byte(defaultDynamicCertProviderCert), []byte(defaultDynamicCertProviderKey)) + r.NoError(err) loadFile := func(filename string) string { bytes, err := ioutil.ReadFile(filename) @@ -253,7 +263,7 @@ func TestManagerControllerSync(t *testing.T) { }) it.After(func() { - timeoutContextCancel() + cancelContextCancelFunc() }) when("there is not yet any agent pods or they were deleted", func() { @@ -669,6 +679,55 @@ func TestManagerControllerSync(t *testing.T) { r.Equal([]coretesting.Action{expectedGetAction, expectedCreateAction, expectedUpdateAction}, pinnipedAPIClient.Actions()) }) }) + + when("the third resulting pod exec has invalid key data", func() { + var keyParseErrorMessage string + + it.Before(func() { + keyParseErrorMessage = "failed to set signing cert/key content from agent pod some-namespace/some-agent-pod-name-123: TestManagerControllerSync: attempt to set invalid key pair: tls: failed to find any PEM data in key input" + fakeExecutor.errorsToReturn = []error{nil, nil} + fakeExecutor.resultsToReturn = []string{fakeCertPEM, ""} + startInformersAndController() + }) + + it("does not update the dynamic certificates provider", func() { + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), keyParseErrorMessage) + requireDynamicCertProviderHasDefaultValues() + }) + + it("creates or updates the the CredentialIssuer status field with an error", func() { + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), keyParseErrorMessage) + + expectedCreateCredentialIssuer := &configv1alpha1.CredentialIssuer{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: credentialIssuerResourceName, + }, + } + + expectedCredentialIssuer := &configv1alpha1.CredentialIssuer{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: credentialIssuerResourceName, + }, + Status: configv1alpha1.CredentialIssuerStatus{ + Strategies: []configv1alpha1.CredentialIssuerStrategy{ + { + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.ErrorStrategyStatus, + Reason: configv1alpha1.CouldNotFetchKeyStrategyReason, + Message: keyParseErrorMessage, + LastUpdateTime: metav1.NewTime(frozenNow), + }, + }, + }, + } + expectedGetAction := coretesting.NewRootGetAction(credentialIssuerGVR, credentialIssuerResourceName) + expectedCreateAction := coretesting.NewRootCreateAction(credentialIssuerGVR, expectedCreateCredentialIssuer) + expectedUpdateAction := coretesting.NewRootUpdateSubresourceAction(credentialIssuerGVR, "status", expectedCredentialIssuer) + r.Equal([]coretesting.Action{expectedGetAction, expectedCreateAction, expectedUpdateAction}, pinnipedAPIClient.Actions()) + }) + }) }) }, spec.Parallel(), spec.Report(report.Terminal{})) } diff --git a/internal/controller/supervisorconfig/federation_domain_watcher_test.go b/internal/controller/supervisorconfig/federation_domain_watcher_test.go index df4f20d92..8dda9704c 100644 --- a/internal/controller/supervisorconfig/federation_domain_watcher_test.go +++ b/internal/controller/supervisorconfig/federation_domain_watcher_test.go @@ -103,8 +103,8 @@ func TestSync(t *testing.T) { var federationDomainInformerClient *pinnipedfake.Clientset var federationDomainInformers pinnipedinformers.SharedInformerFactory var pinnipedAPIClient *pinnipedfake.Clientset - var timeoutContext context.Context - var timeoutContextCancel context.CancelFunc + var cancelContext context.Context + var cancelContextCancelFunc context.CancelFunc var syncContext *controllerlib.Context var frozenNow time.Time var providersSetter *fakeProvidersSetter @@ -124,7 +124,7 @@ func TestSync(t *testing.T) { // Set this at the last second to support calling subject.Name(). syncContext = &controllerlib.Context{ - Context: timeoutContext, + Context: cancelContext, Name: subject.Name(), Key: controllerlib.Key{ Namespace: namespace, @@ -133,7 +133,7 @@ func TestSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - federationDomainInformers.Start(timeoutContext.Done()) + federationDomainInformers.Start(cancelContext.Done()) controllerlib.TestRunSynchronously(t, subject) } @@ -143,7 +143,7 @@ func TestSync(t *testing.T) { providersSetter = &fakeProvidersSetter{} frozenNow = time.Date(2020, time.September, 23, 7, 42, 0, 0, time.Local) - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) federationDomainInformerClient = pinnipedfake.NewSimpleClientset() federationDomainInformers = pinnipedinformers.NewSharedInformerFactory(federationDomainInformerClient, 0) @@ -157,7 +157,7 @@ func TestSync(t *testing.T) { }) it.After(func() { - timeoutContextCancel() + cancelContextCancelFunc() }) when("there are some valid FederationDomains in the informer", func() { diff --git a/internal/controller/supervisorconfig/generator/federation_domain_secrets_test.go b/internal/controller/supervisorconfig/generator/federation_domain_secrets_test.go index 4fc92a003..a15a24774 100644 --- a/internal/controller/supervisorconfig/generator/federation_domain_secrets_test.go +++ b/internal/controller/supervisorconfig/generator/federation_domain_secrets_test.go @@ -10,7 +10,6 @@ import ( "fmt" "sync" "testing" - "time" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" @@ -630,7 +629,7 @@ func TestFederationDomainSecretsControllerSync(t *testing.T) { t.Run(test.name, func(t *testing.T) { t.Parallel() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() pinnipedAPIClient := pinnipedfake.NewSimpleClientset() diff --git a/internal/controller/supervisorconfig/generator/supervisor_secrets_test.go b/internal/controller/supervisorconfig/generator/supervisor_secrets_test.go index eeeb9930e..044d1cb00 100644 --- a/internal/controller/supervisorconfig/generator/supervisor_secrets_test.go +++ b/internal/controller/supervisorconfig/generator/supervisor_secrets_test.go @@ -8,7 +8,6 @@ import ( "errors" "sync" "testing" - "time" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" @@ -412,7 +411,7 @@ func TestSupervisorSecretsControllerSync(t *testing.T) { t.Run(test.name, func(t *testing.T) { // We cannot currently run this test in parallel since it uses the global generateKey function. - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() if test.generateKey != nil { diff --git a/internal/controller/supervisorconfig/jwks_observer_test.go b/internal/controller/supervisorconfig/jwks_observer_test.go index a1e7e3e30..07d6f62b3 100644 --- a/internal/controller/supervisorconfig/jwks_observer_test.go +++ b/internal/controller/supervisorconfig/jwks_observer_test.go @@ -7,7 +7,6 @@ import ( "context" "encoding/json" "testing" - "time" "github.com/sclevine/spec" "github.com/sclevine/spec/report" @@ -124,16 +123,16 @@ func TestJWKSObserverControllerSync(t *testing.T) { const installedInNamespace = "some-namespace" var ( - r *require.Assertions - subject controllerlib.Controller - pinnipedInformerClient *pinnipedfake.Clientset - kubeInformerClient *kubernetesfake.Clientset - pinnipedInformers pinnipedinformers.SharedInformerFactory - kubeInformers kubeinformers.SharedInformerFactory - timeoutContext context.Context - timeoutContextCancel context.CancelFunc - syncContext *controllerlib.Context - issuerToJWKSSetter *fakeIssuerToJWKSMapSetter + r *require.Assertions + subject controllerlib.Controller + pinnipedInformerClient *pinnipedfake.Clientset + kubeInformerClient *kubernetesfake.Clientset + pinnipedInformers pinnipedinformers.SharedInformerFactory + kubeInformers kubeinformers.SharedInformerFactory + cancelContext context.Context + cancelContextCancelFunc context.CancelFunc + syncContext *controllerlib.Context + issuerToJWKSSetter *fakeIssuerToJWKSMapSetter ) // Defer starting the informers until the last possible moment so that the @@ -149,7 +148,7 @@ func TestJWKSObserverControllerSync(t *testing.T) { // Set this at the last second to support calling subject.Name(). syncContext = &controllerlib.Context{ - Context: timeoutContext, + Context: cancelContext, Name: subject.Name(), Key: controllerlib.Key{ Namespace: installedInNamespace, @@ -158,15 +157,15 @@ func TestJWKSObserverControllerSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - kubeInformers.Start(timeoutContext.Done()) - pinnipedInformers.Start(timeoutContext.Done()) + kubeInformers.Start(cancelContext.Done()) + pinnipedInformers.Start(cancelContext.Done()) controllerlib.TestRunSynchronously(t, subject) } it.Before(func() { r = require.New(t) - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) kubeInformerClient = kubernetesfake.NewSimpleClientset() kubeInformers = kubeinformers.NewSharedInformerFactory(kubeInformerClient, 0) @@ -184,7 +183,7 @@ func TestJWKSObserverControllerSync(t *testing.T) { }) it.After(func() { - timeoutContextCancel() + cancelContextCancelFunc() }) when("there are no FederationDomains and no JWKS Secrets yet", func() { diff --git a/internal/controller/supervisorconfig/jwks_writer_test.go b/internal/controller/supervisorconfig/jwks_writer_test.go index c528be90d..f977c8f80 100644 --- a/internal/controller/supervisorconfig/jwks_writer_test.go +++ b/internal/controller/supervisorconfig/jwks_writer_test.go @@ -12,7 +12,6 @@ import ( "io" "io/ioutil" "testing" - "time" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" @@ -674,7 +673,7 @@ func TestJWKSWriterControllerSync(t *testing.T) { return goodKey, test.generateKeyErr } - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() kubeAPIClient := kubernetesfake.NewSimpleClientset() diff --git a/internal/controller/supervisorconfig/tls_cert_observer_test.go b/internal/controller/supervisorconfig/tls_cert_observer_test.go index f24888a20..3e3919424 100644 --- a/internal/controller/supervisorconfig/tls_cert_observer_test.go +++ b/internal/controller/supervisorconfig/tls_cert_observer_test.go @@ -9,7 +9,6 @@ import ( "io/ioutil" "net/url" "testing" - "time" "github.com/sclevine/spec" "github.com/sclevine/spec/report" @@ -130,16 +129,16 @@ func TestTLSCertObserverControllerSync(t *testing.T) { ) var ( - r *require.Assertions - subject controllerlib.Controller - pinnipedInformerClient *pinnipedfake.Clientset - kubeInformerClient *kubernetesfake.Clientset - pinnipedInformers pinnipedinformers.SharedInformerFactory - kubeInformers kubeinformers.SharedInformerFactory - timeoutContext context.Context - timeoutContextCancel context.CancelFunc - syncContext *controllerlib.Context - issuerTLSCertSetter *fakeIssuerTLSCertSetter + r *require.Assertions + subject controllerlib.Controller + pinnipedInformerClient *pinnipedfake.Clientset + kubeInformerClient *kubernetesfake.Clientset + pinnipedInformers pinnipedinformers.SharedInformerFactory + kubeInformers kubeinformers.SharedInformerFactory + cancelContext context.Context + cancelContextCancelFunc context.CancelFunc + syncContext *controllerlib.Context + issuerTLSCertSetter *fakeIssuerTLSCertSetter ) // Defer starting the informers until the last possible moment so that the @@ -156,7 +155,7 @@ func TestTLSCertObserverControllerSync(t *testing.T) { // Set this at the last second to support calling subject.Name(). syncContext = &controllerlib.Context{ - Context: timeoutContext, + Context: cancelContext, Name: subject.Name(), Key: controllerlib.Key{ Namespace: installedInNamespace, @@ -165,8 +164,8 @@ func TestTLSCertObserverControllerSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - kubeInformers.Start(timeoutContext.Done()) - pinnipedInformers.Start(timeoutContext.Done()) + kubeInformers.Start(cancelContext.Done()) + pinnipedInformers.Start(cancelContext.Done()) controllerlib.TestRunSynchronously(t, subject) } @@ -179,7 +178,7 @@ func TestTLSCertObserverControllerSync(t *testing.T) { it.Before(func() { r = require.New(t) - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) kubeInformerClient = kubernetesfake.NewSimpleClientset() kubeInformers = kubeinformers.NewSharedInformerFactory(kubeInformerClient, 0) @@ -197,7 +196,7 @@ func TestTLSCertObserverControllerSync(t *testing.T) { }) it.After(func() { - timeoutContextCancel() + cancelContextCancelFunc() }) when("there are no FederationDomains and no TLS Secrets yet", func() { diff --git a/internal/controller/supervisorconfig/upstreamwatcher/upstreamwatcher_test.go b/internal/controller/supervisorconfig/upstreamwatcher/upstreamwatcher_test.go index cba27c394..f73973522 100644 --- a/internal/controller/supervisorconfig/upstreamwatcher/upstreamwatcher_test.go +++ b/internal/controller/supervisorconfig/upstreamwatcher/upstreamwatcher_test.go @@ -624,7 +624,7 @@ func TestController(t *testing.T) { controllerlib.WithInformer, ) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() pinnipedInformers.Start(ctx.Done()) diff --git a/internal/controller/supervisorstorage/garbage_collector_test.go b/internal/controller/supervisorstorage/garbage_collector_test.go index a21067ee9..a5606cf88 100644 --- a/internal/controller/supervisorstorage/garbage_collector_test.go +++ b/internal/controller/supervisorstorage/garbage_collector_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package supervisorstorage @@ -108,16 +108,16 @@ func TestGarbageCollectorControllerSync(t *testing.T) { ) var ( - r *require.Assertions - subject controllerlib.Controller - kubeInformerClient *kubernetesfake.Clientset - kubeClient *kubernetesfake.Clientset - kubeInformers kubeinformers.SharedInformerFactory - timeoutContext context.Context - timeoutContextCancel context.CancelFunc - syncContext *controllerlib.Context - fakeClock *clock.FakeClock - frozenNow time.Time + r *require.Assertions + subject controllerlib.Controller + kubeInformerClient *kubernetesfake.Clientset + kubeClient *kubernetesfake.Clientset + kubeInformers kubeinformers.SharedInformerFactory + cancelContext context.Context + cancelContextCancelFunc context.CancelFunc + syncContext *controllerlib.Context + fakeClock *clock.FakeClock + frozenNow time.Time ) // Defer starting the informers until the last possible moment so that the @@ -133,7 +133,7 @@ func TestGarbageCollectorControllerSync(t *testing.T) { // Set this at the last second to support calling subject.Name(). syncContext = &controllerlib.Context{ - Context: timeoutContext, + Context: cancelContext, Name: subject.Name(), Key: controllerlib.Key{ Namespace: "", @@ -142,14 +142,14 @@ func TestGarbageCollectorControllerSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - kubeInformers.Start(timeoutContext.Done()) + kubeInformers.Start(cancelContext.Done()) controllerlib.TestRunSynchronously(t, subject) } it.Before(func() { r = require.New(t) - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) kubeInformerClient = kubernetesfake.NewSimpleClientset() kubeClient = kubernetesfake.NewSimpleClientset() @@ -168,7 +168,7 @@ func TestGarbageCollectorControllerSync(t *testing.T) { }) it.After(func() { - timeoutContextCancel() + cancelContextCancelFunc() }) when("there are secrets without the garbage-collect-after annotation", func() { diff --git a/internal/controllermanager/prepare_controllers.go b/internal/controllermanager/prepare_controllers.go index 61990a39f..6fe0b2127 100644 --- a/internal/controllermanager/prepare_controllers.go +++ b/internal/controllermanager/prepare_controllers.go @@ -18,12 +18,14 @@ import ( pinnipedclientset "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned" pinnipedinformers "go.pinniped.dev/generated/latest/client/concierge/informers/externalversions" "go.pinniped.dev/internal/apiserviceref" + "go.pinniped.dev/internal/concierge/impersonator" "go.pinniped.dev/internal/config/concierge" "go.pinniped.dev/internal/controller/apicerts" "go.pinniped.dev/internal/controller/authenticator/authncache" "go.pinniped.dev/internal/controller/authenticator/cachecleaner" "go.pinniped.dev/internal/controller/authenticator/jwtcachefiller" "go.pinniped.dev/internal/controller/authenticator/webhookcachefiller" + "go.pinniped.dev/internal/controller/impersonatorconfig" "go.pinniped.dev/internal/controller/kubecertagent" "go.pinniped.dev/internal/controllerlib" "go.pinniped.dev/internal/deploymentref" @@ -61,13 +63,23 @@ type Config struct { DiscoveryURLOverride *string // DynamicServingCertProvider provides a setter and a getter to the Pinniped API's serving cert. - DynamicServingCertProvider dynamiccert.Provider + DynamicServingCertProvider dynamiccert.Private + // DynamicSigningCertProvider provides a setter and a getter to the Pinniped API's // signing cert, i.e., the cert that it uses to sign certs for Pinniped clients wishing to login. - DynamicSigningCertProvider dynamiccert.Provider + // This is filled with the Kube API server's signing cert by a controller, if it can be found. + DynamicSigningCertProvider dynamiccert.Private + + // ImpersonationSigningCertProvider provides a setter and a getter to the CA cert that should be + // used to sign client certs for authentication to the impersonation proxy. This CA is used by + // the TokenCredentialRequest to sign certs and by the impersonation proxy to check certs. + // When the impersonation proxy is not running, the getter will return nil cert and nil key. + // (Note that the impersonation proxy also accepts client certs signed by the Kube API server's cert.) + ImpersonationSigningCertProvider dynamiccert.Provider // ServingCertDuration is the validity period, in seconds, of the API serving certificate. ServingCertDuration time.Duration + // ServingCertRenewBefore is the period of time, in seconds, that pinniped will wait before // rotating the serving certificate. This period of time starts upon issuance of the serving // certificate. @@ -179,6 +191,7 @@ func PrepareControllers(c *Config) (func(ctx context.Context), error) { informers.installationNamespaceK8s.Core().V1().Secrets(), controllerlib.WithInformer, c.ServingCertRenewBefore, + apicerts.TLSCertificateChainSecretKey, ), singletonWorker, ). @@ -266,6 +279,58 @@ func PrepareControllers(c *Config) (func(ctx context.Context), error) { klogr.New(), ), singletonWorker, + ). + + // The impersonator configuration controller dynamically configures the impersonation proxy feature. + WithController( + impersonatorconfig.NewImpersonatorConfigController( + c.ServerInstallationInfo.Namespace, + c.NamesConfig.ImpersonationConfigMap, + c.NamesConfig.CredentialIssuer, + client.Kubernetes, + client.PinnipedConcierge, + informers.installationNamespaceK8s.Core().V1().ConfigMaps(), + informers.installationNamespaceK8s.Core().V1().Services(), + informers.installationNamespaceK8s.Core().V1().Secrets(), + controllerlib.WithInformer, + controllerlib.WithInitialEvent, + c.NamesConfig.ImpersonationLoadBalancerService, + c.NamesConfig.ImpersonationTLSCertificateSecret, + c.NamesConfig.ImpersonationCACertificateSecret, + c.Labels, + clock.RealClock{}, + impersonator.New, + c.NamesConfig.ImpersonationSignerSecret, + c.ImpersonationSigningCertProvider, + ), + singletonWorker, + ). + WithController( + apicerts.NewCertsManagerController( + c.ServerInstallationInfo.Namespace, + c.NamesConfig.ImpersonationSignerSecret, + c.Labels, + client.Kubernetes, + informers.installationNamespaceK8s.Core().V1().Secrets(), + controllerlib.WithInformer, + controllerlib.WithInitialEvent, + 365*24*time.Hour, // 1 year hard coded value + "Pinniped Impersonation Proxy CA", + "", // optional, means do not give me a serving cert + ), + singletonWorker, + ). + WithController( + apicerts.NewCertsExpirerController( + c.ServerInstallationInfo.Namespace, + c.NamesConfig.ImpersonationSignerSecret, + client.Kubernetes, + informers.installationNamespaceK8s.Core().V1().Secrets(), + controllerlib.WithInformer, + c.ServingCertRenewBefore, + apicerts.CACertificateSecretKey, + ), + singletonWorker, ) // Return a function which starts the informers and controllers. diff --git a/internal/dynamiccert/provider.go b/internal/dynamiccert/provider.go index a4ca6ad5d..4e74d1183 100644 --- a/internal/dynamiccert/provider.go +++ b/internal/dynamiccert/provider.go @@ -1,45 +1,153 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package dynamiccert import ( + "crypto/tls" + "crypto/x509" + "fmt" "sync" "k8s.io/apiserver/pkg/server/dynamiccertificates" + + "go.pinniped.dev/internal/plog" ) -// Provider provides a getter, CurrentCertKeyContent(), and a setter, Set(), for a PEM-formatted -// certificate and matching key. type Provider interface { - dynamiccertificates.CertKeyContentProvider - Set(certPEM, keyPEM []byte) + Private + Public } +type Private interface { + dynamiccertificates.CertKeyContentProvider + SetCertKeyContent(certPEM, keyPEM []byte) error + UnsetCertKeyContent() + + notifier +} + +type Public interface { + dynamiccertificates.CAContentProvider + + notifier +} + +type notifier interface { + dynamiccertificates.Notifier + dynamiccertificates.ControllerRunner // we do not need this today, but it could grow and change in the future +} + +var _ Provider = &provider{} + type provider struct { - certPEM []byte - keyPEM []byte - mutex sync.RWMutex + // these fields are constant after struct initialization and thus do not need locking + name string + isCA bool + + // mutex guards all the fields below it + mutex sync.RWMutex + certPEM []byte + keyPEM []byte + listeners []dynamiccertificates.Listener } -// New returns an empty Provider. The returned Provider is thread-safe. -func New() Provider { - return &provider{} +// NewServingCert returns a Private that is go routine safe. +// It can only hold key pairs that have IsCA=false. +func NewServingCert(name string) Private { + return &provider{name: name} } -func (p *provider) Set(certPEM, keyPEM []byte) { - p.mutex.Lock() // acquire a write lock - defer p.mutex.Unlock() - p.certPEM = certPEM - p.keyPEM = keyPEM +// NewCA returns a Provider that is go routine safe. +// It can only hold key pairs that have IsCA=true. +func NewCA(name string) Provider { + return &provider{name: name, isCA: true} } func (p *provider) Name() string { - return "DynamicCertProvider" + return p.name } func (p *provider) CurrentCertKeyContent() (cert []byte, key []byte) { - p.mutex.RLock() // acquire a read lock + p.mutex.RLock() defer p.mutex.RUnlock() + return p.certPEM, p.keyPEM } + +func (p *provider) SetCertKeyContent(certPEM, keyPEM []byte) error { + // always make sure that we have valid PEM data, otherwise + // dynamiccertificates.NewUnionCAContentProvider.VerifyOptions will panic + cert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return fmt.Errorf("%s: attempt to set invalid key pair: %w", p.name, err) + } + + // these checks should always pass if tls.X509KeyPair did not error + if len(cert.Certificate) == 0 { + return fmt.Errorf("%s: key pair has empty cert slice", p.name) + } + x509Cert, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + return fmt.Errorf("%s: failed to parse key pair as x509 cert: %w", p.name, err) + } + + // confirm that we are not trying to use a CA as a serving cert and vice versa + if p.isCA != x509Cert.IsCA { + return fmt.Errorf("%s: attempt to set x509 cert with unexpected IsCA=%v", p.name, x509Cert.IsCA) + } + + p.setCertKeyContent(certPEM, keyPEM) + + return nil +} + +func (p *provider) UnsetCertKeyContent() { + p.setCertKeyContent(nil, nil) +} + +func (p *provider) setCertKeyContent(certPEM, keyPEM []byte) { + p.mutex.Lock() + defer p.mutex.Unlock() + + p.certPEM = certPEM + p.keyPEM = keyPEM + + // technically this only reads a read lock but we already have the write lock + for _, listener := range p.listeners { + listener.Enqueue() + } +} + +func (p *provider) CurrentCABundleContent() []byte { + if !p.isCA { + panic("*provider from NewServingCert was cast into wrong CA interface") + } + + ca, _ := p.CurrentCertKeyContent() + return ca +} + +func (p *provider) VerifyOptions() (x509.VerifyOptions, bool) { + if !p.isCA { + panic("*provider from NewServingCert was cast into wrong CA interface") + } + + plog.Warning("unexpected call to *provider.VerifyOptions; CA union logic is broken") + return x509.VerifyOptions{}, false // assume we are unioned via dynamiccertificates.NewUnionCAContentProvider +} + +func (p *provider) AddListener(listener dynamiccertificates.Listener) { + p.mutex.Lock() + defer p.mutex.Unlock() + + p.listeners = append(p.listeners, listener) +} + +func (p *provider) RunOnce() error { + return nil // no-op, but we want to make sure to stay in sync with dynamiccertificates.ControllerRunner +} + +func (p *provider) Run(workers int, stopCh <-chan struct{}) { + // no-op, but we want to make sure to stay in sync with dynamiccertificates.ControllerRunner +} diff --git a/internal/dynamiccert/provider_test.go b/internal/dynamiccert/provider_test.go new file mode 100644 index 000000000..140efe63c --- /dev/null +++ b/internal/dynamiccert/provider_test.go @@ -0,0 +1,226 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package dynamiccert + +import ( + "crypto/tls" + "crypto/x509" + "net" + "reflect" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/server/dynamiccertificates" + "k8s.io/apiserver/pkg/storage/names" + + "go.pinniped.dev/internal/certauthority" + "go.pinniped.dev/test/library" +) + +func TestProviderWithDynamicServingCertificateController(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + f func(t *testing.T, ca Provider, certKey Private) (wantClientCASubjects [][]byte, wantCerts []tls.Certificate) + }{ + { + name: "no-op leave everything alone", + f: func(t *testing.T, ca Provider, certKey Private) ([][]byte, []tls.Certificate) { + pool := x509.NewCertPool() + ok := pool.AppendCertsFromPEM(ca.CurrentCABundleContent()) + require.True(t, ok, "should have valid non-empty CA bundle") + + certPEM, keyPEM := certKey.CurrentCertKeyContent() + cert, err := tls.X509KeyPair(certPEM, keyPEM) + require.NoError(t, err) + + return pool.Subjects(), []tls.Certificate{cert} + }, + }, + { + name: "unset the CA", + f: func(t *testing.T, ca Provider, certKey Private) ([][]byte, []tls.Certificate) { + ca.UnsetCertKeyContent() + + certPEM, keyPEM := certKey.CurrentCertKeyContent() + cert, err := tls.X509KeyPair(certPEM, keyPEM) + require.NoError(t, err) + + return nil, []tls.Certificate{cert} + }, + }, + { + name: "unset the serving cert - still serves the old content", + f: func(t *testing.T, ca Provider, certKey Private) ([][]byte, []tls.Certificate) { + pool := x509.NewCertPool() + ok := pool.AppendCertsFromPEM(ca.CurrentCABundleContent()) + require.True(t, ok, "should have valid non-empty CA bundle") + + certPEM, keyPEM := certKey.CurrentCertKeyContent() + cert, err := tls.X509KeyPair(certPEM, keyPEM) + require.NoError(t, err) + + certKey.UnsetCertKeyContent() + + return pool.Subjects(), []tls.Certificate{cert} + }, + }, + { + name: "change to a new CA", + f: func(t *testing.T, ca Provider, certKey Private) ([][]byte, []tls.Certificate) { + // use unique names for all CAs to make sure the pool subjects are different + newCA, err := certauthority.New(names.SimpleNameGenerator.GenerateName("new-ca"), time.Hour) + require.NoError(t, err) + caKey, err := newCA.PrivateKeyToPEM() + require.NoError(t, err) + err = ca.SetCertKeyContent(newCA.Bundle(), caKey) + require.NoError(t, err) + + certPEM, keyPEM := certKey.CurrentCertKeyContent() + cert, err := tls.X509KeyPair(certPEM, keyPEM) + require.NoError(t, err) + + return newCA.Pool().Subjects(), []tls.Certificate{cert} + }, + }, + { + name: "change to new serving cert", + f: func(t *testing.T, ca Provider, certKey Private) ([][]byte, []tls.Certificate) { + // use unique names for all CAs to make sure the pool subjects are different + newCA, err := certauthority.New(names.SimpleNameGenerator.GenerateName("new-ca"), time.Hour) + require.NoError(t, err) + + certPEM, keyPEM, err := newCA.IssueServerCertPEM(nil, []net.IP{net.ParseIP("127.0.0.2")}, time.Hour) + require.NoError(t, err) + + err = certKey.SetCertKeyContent(certPEM, keyPEM) + require.NoError(t, err) + + cert, err := tls.X509KeyPair(certPEM, keyPEM) + require.NoError(t, err) + + pool := x509.NewCertPool() + ok := pool.AppendCertsFromPEM(ca.CurrentCABundleContent()) + require.True(t, ok, "should have valid non-empty CA bundle") + + return pool.Subjects(), []tls.Certificate{cert} + }, + }, + { + name: "change both CA and serving cert", + f: func(t *testing.T, ca Provider, certKey Private) ([][]byte, []tls.Certificate) { + // use unique names for all CAs to make sure the pool subjects are different + newCA, err := certauthority.New(names.SimpleNameGenerator.GenerateName("new-ca"), time.Hour) + require.NoError(t, err) + + certPEM, keyPEM, err := newCA.IssueServerCertPEM(nil, []net.IP{net.ParseIP("127.0.0.3")}, time.Hour) + require.NoError(t, err) + + err = certKey.SetCertKeyContent(certPEM, keyPEM) + require.NoError(t, err) + + cert, err := tls.X509KeyPair(certPEM, keyPEM) + require.NoError(t, err) + + // use unique names for all CAs to make sure the pool subjects are different + newOtherCA, err := certauthority.New(names.SimpleNameGenerator.GenerateName("new-other-ca"), time.Hour) + require.NoError(t, err) + caKey, err := newOtherCA.PrivateKeyToPEM() + require.NoError(t, err) + err = ca.SetCertKeyContent(newOtherCA.Bundle(), caKey) + require.NoError(t, err) + + return newOtherCA.Pool().Subjects(), []tls.Certificate{cert} + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + // use unique names for all CAs to make sure the pool subjects are different + ca, err := certauthority.New(names.SimpleNameGenerator.GenerateName("ca"), time.Hour) + require.NoError(t, err) + caKey, err := ca.PrivateKeyToPEM() + require.NoError(t, err) + caContent := NewCA("ca") + err = caContent.SetCertKeyContent(ca.Bundle(), caKey) + require.NoError(t, err) + + cert, key, err := ca.IssueServerCertPEM(nil, []net.IP{net.ParseIP("127.0.0.1")}, time.Hour) + require.NoError(t, err) + certKeyContent := NewServingCert("cert-key") + err = certKeyContent.SetCertKeyContent(cert, key) + require.NoError(t, err) + + tlsConfig := &tls.Config{ + MinVersion: tls.VersionTLS12, + NextProtos: []string{"h2", "http/1.1"}, + ClientAuth: tls.RequestClientCert, + } + + dynamicCertificateController := dynamiccertificates.NewDynamicServingCertificateController( + tlsConfig, + caContent, + certKeyContent, + nil, // we do not care about SNI + nil, // we do not care about events + ) + + caContent.AddListener(dynamicCertificateController) + certKeyContent.AddListener(dynamicCertificateController) + + err = dynamicCertificateController.RunOnce() + require.NoError(t, err) + + stopCh := make(chan struct{}) + defer close(stopCh) + go dynamicCertificateController.Run(1, stopCh) + + tlsConfig.GetConfigForClient = dynamicCertificateController.GetConfigForClient + + wantClientCASubjects, wantCerts := tt.f(t, caContent, certKeyContent) + + var lastTLSConfig *tls.Config + + // it will take some time for the controller to catch up + err = wait.PollImmediate(time.Second, 30*time.Second, func() (bool, error) { + actualTLSConfig, err := tlsConfig.GetConfigForClient(&tls.ClientHelloInfo{ServerName: "force-standard-sni"}) + if err != nil { + return false, err + } + + lastTLSConfig = actualTLSConfig + + return reflect.DeepEqual(wantClientCASubjects, poolSubjects(actualTLSConfig.ClientCAs)) && + reflect.DeepEqual(wantCerts, actualTLSConfig.Certificates), nil + }) + + if err != nil && lastTLSConfig != nil { + // for debugging failures + t.Log("diff between client CAs:\n", cmp.Diff( + library.Sdump(wantClientCASubjects), + library.Sdump(poolSubjects(lastTLSConfig.ClientCAs)), + )) + t.Log("diff between serving certs:\n", cmp.Diff( + library.Sdump(wantCerts), + library.Sdump(lastTLSConfig.Certificates), + )) + } + require.NoError(t, err) + }) + } +} + +func poolSubjects(pool *x509.CertPool) [][]byte { + if pool == nil { + return nil + } + return pool.Subjects() +} diff --git a/internal/httputil/roundtripper/roundtripper.go b/internal/httputil/roundtripper/roundtripper.go new file mode 100644 index 000000000..d0094b8a6 --- /dev/null +++ b/internal/httputil/roundtripper/roundtripper.go @@ -0,0 +1,14 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package roundtripper + +import "net/http" + +var _ http.RoundTripper = Func(nil) + +type Func func(*http.Request) (*http.Response, error) + +func (f Func) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req) +} diff --git a/internal/httputil/securityheader/securityheader_test.go b/internal/httputil/securityheader/securityheader_test.go index a0688c1a7..7ee7331f5 100644 --- a/internal/httputil/securityheader/securityheader_test.go +++ b/internal/httputil/securityheader/securityheader_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package securityheader @@ -22,7 +22,7 @@ func TestWrap(t *testing.T) { }))) t.Cleanup(testServer.Close) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodGet, testServer.URL, nil) diff --git a/internal/issuer/issuer.go b/internal/issuer/issuer.go new file mode 100644 index 000000000..ef563e996 --- /dev/null +++ b/internal/issuer/issuer.go @@ -0,0 +1,57 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package issuer + +import ( + "fmt" + "strings" + "time" + + "k8s.io/apimachinery/pkg/util/errors" + + "go.pinniped.dev/internal/constable" +) + +const defaultCertIssuerErr = constable.Error("failed to issue cert") + +type ClientCertIssuer interface { + Name() string + IssueClientCertPEM(username string, groups []string, ttl time.Duration) (certPEM, keyPEM []byte, err error) +} + +var _ ClientCertIssuer = ClientCertIssuers{} + +type ClientCertIssuers []ClientCertIssuer + +func (c ClientCertIssuers) Name() string { + if len(c) == 0 { + return "empty-client-cert-issuers" + } + + names := make([]string, 0, len(c)) + for _, issuer := range c { + names = append(names, issuer.Name()) + } + + return strings.Join(names, ",") +} + +func (c ClientCertIssuers) IssueClientCertPEM(username string, groups []string, ttl time.Duration) ([]byte, []byte, error) { + var errs []error + + for _, issuer := range c { + certPEM, keyPEM, err := issuer.IssueClientCertPEM(username, groups, ttl) + if err != nil { + errs = append(errs, fmt.Errorf("%s failed to issue client cert: %w", issuer.Name(), err)) + continue + } + return certPEM, keyPEM, nil + } + + if err := errors.NewAggregate(errs); err != nil { + return nil, nil, err + } + + return nil, nil, defaultCertIssuerErr +} diff --git a/internal/mocks/credentialrequestmocks/credentialrequestmocks.go b/internal/mocks/credentialrequestmocks/credentialrequestmocks.go index 58bd134db..68ff7f2ed 100644 --- a/internal/mocks/credentialrequestmocks/credentialrequestmocks.go +++ b/internal/mocks/credentialrequestmocks/credentialrequestmocks.go @@ -3,61 +3,20 @@ // // Code generated by MockGen. DO NOT EDIT. -// Source: go.pinniped.dev/internal/registry/credentialrequest (interfaces: CertIssuer,TokenCredentialRequestAuthenticator) +// Source: go.pinniped.dev/internal/registry/credentialrequest (interfaces: TokenCredentialRequestAuthenticator) // Package credentialrequestmocks is a generated GoMock package. package credentialrequestmocks import ( context "context" - pkix "crypto/x509/pkix" reflect "reflect" - time "time" gomock "github.com/golang/mock/gomock" login "go.pinniped.dev/generated/latest/apis/concierge/login" user "k8s.io/apiserver/pkg/authentication/user" ) -// MockCertIssuer is a mock of CertIssuer interface. -type MockCertIssuer struct { - ctrl *gomock.Controller - recorder *MockCertIssuerMockRecorder -} - -// MockCertIssuerMockRecorder is the mock recorder for MockCertIssuer. -type MockCertIssuerMockRecorder struct { - mock *MockCertIssuer -} - -// NewMockCertIssuer creates a new mock instance. -func NewMockCertIssuer(ctrl *gomock.Controller) *MockCertIssuer { - mock := &MockCertIssuer{ctrl: ctrl} - mock.recorder = &MockCertIssuerMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockCertIssuer) EXPECT() *MockCertIssuerMockRecorder { - return m.recorder -} - -// IssuePEM mocks base method. -func (m *MockCertIssuer) IssuePEM(arg0 pkix.Name, arg1 []string, arg2 time.Duration) ([]byte, []byte, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IssuePEM", arg0, arg1, arg2) - ret0, _ := ret[0].([]byte) - ret1, _ := ret[1].([]byte) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// IssuePEM indicates an expected call of IssuePEM. -func (mr *MockCertIssuerMockRecorder) IssuePEM(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IssuePEM", reflect.TypeOf((*MockCertIssuer)(nil).IssuePEM), arg0, arg1, arg2) -} - // MockTokenCredentialRequestAuthenticator is a mock of TokenCredentialRequestAuthenticator interface. type MockTokenCredentialRequestAuthenticator struct { ctrl *gomock.Controller diff --git a/internal/mocks/credentialrequestmocks/generate.go b/internal/mocks/credentialrequestmocks/generate.go index c22ec9787..ce19cf3f8 100644 --- a/internal/mocks/credentialrequestmocks/generate.go +++ b/internal/mocks/credentialrequestmocks/generate.go @@ -1,6 +1,6 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package credentialrequestmocks -//go:generate go run -v github.com/golang/mock/mockgen -destination=credentialrequestmocks.go -package=credentialrequestmocks -copyright_file=../../../hack/header.txt go.pinniped.dev/internal/registry/credentialrequest CertIssuer,TokenCredentialRequestAuthenticator +//go:generate go run -v github.com/golang/mock/mockgen -destination=credentialrequestmocks.go -package=credentialrequestmocks -copyright_file=../../../hack/header.txt go.pinniped.dev/internal/registry/credentialrequest TokenCredentialRequestAuthenticator diff --git a/internal/mocks/issuermocks/generate.go b/internal/mocks/issuermocks/generate.go new file mode 100644 index 000000000..7d0c9937e --- /dev/null +++ b/internal/mocks/issuermocks/generate.go @@ -0,0 +1,6 @@ +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package issuermocks + +//go:generate go run -v github.com/golang/mock/mockgen -destination=issuermocks.go -package=issuermocks -copyright_file=../../../hack/header.txt go.pinniped.dev/internal/issuer ClientCertIssuer diff --git a/internal/mocks/issuermocks/issuermocks.go b/internal/mocks/issuermocks/issuermocks.go new file mode 100644 index 000000000..045fbed34 --- /dev/null +++ b/internal/mocks/issuermocks/issuermocks.go @@ -0,0 +1,69 @@ +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// + +// Code generated by MockGen. DO NOT EDIT. +// Source: go.pinniped.dev/internal/issuer (interfaces: ClientCertIssuer) + +// Package issuermocks is a generated GoMock package. +package issuermocks + +import ( + reflect "reflect" + time "time" + + gomock "github.com/golang/mock/gomock" +) + +// MockClientCertIssuer is a mock of ClientCertIssuer interface. +type MockClientCertIssuer struct { + ctrl *gomock.Controller + recorder *MockClientCertIssuerMockRecorder +} + +// MockClientCertIssuerMockRecorder is the mock recorder for MockClientCertIssuer. +type MockClientCertIssuerMockRecorder struct { + mock *MockClientCertIssuer +} + +// NewMockClientCertIssuer creates a new mock instance. +func NewMockClientCertIssuer(ctrl *gomock.Controller) *MockClientCertIssuer { + mock := &MockClientCertIssuer{ctrl: ctrl} + mock.recorder = &MockClientCertIssuerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockClientCertIssuer) EXPECT() *MockClientCertIssuerMockRecorder { + return m.recorder +} + +// IssueClientCertPEM mocks base method. +func (m *MockClientCertIssuer) IssueClientCertPEM(arg0 string, arg1 []string, arg2 time.Duration) ([]byte, []byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IssueClientCertPEM", arg0, arg1, arg2) + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].([]byte) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// IssueClientCertPEM indicates an expected call of IssueClientCertPEM. +func (mr *MockClientCertIssuerMockRecorder) IssueClientCertPEM(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IssueClientCertPEM", reflect.TypeOf((*MockClientCertIssuer)(nil).IssueClientCertPEM), arg0, arg1, arg2) +} + +// Name mocks base method. +func (m *MockClientCertIssuer) Name() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Name") + ret0, _ := ret[0].(string) + return ret0 +} + +// Name indicates an expected call of Name. +func (mr *MockClientCertIssuerMockRecorder) Name() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockClientCertIssuer)(nil).Name)) +} diff --git a/internal/registry/credentialrequest/rest.go b/internal/registry/credentialrequest/rest.go index a89066761..9c9165d5f 100644 --- a/internal/registry/credentialrequest/rest.go +++ b/internal/registry/credentialrequest/rest.go @@ -6,7 +6,6 @@ package credentialrequest import ( "context" - "crypto/x509/pkix" "fmt" "time" @@ -22,20 +21,17 @@ import ( "k8s.io/utils/trace" loginapi "go.pinniped.dev/generated/latest/apis/concierge/login" + "go.pinniped.dev/internal/issuer" ) // clientCertificateTTL is the TTL for short-lived client certificates returned by this API. const clientCertificateTTL = 5 * time.Minute -type CertIssuer interface { - IssuePEM(subject pkix.Name, dnsNames []string, ttl time.Duration) ([]byte, []byte, error) -} - type TokenCredentialRequestAuthenticator interface { AuthenticateTokenCredentialRequest(ctx context.Context, req *loginapi.TokenCredentialRequest) (user.Info, error) } -func NewREST(authenticator TokenCredentialRequestAuthenticator, issuer CertIssuer, resource schema.GroupResource) *REST { +func NewREST(authenticator TokenCredentialRequestAuthenticator, issuer issuer.ClientCertIssuer, resource schema.GroupResource) *REST { return &REST{ authenticator: authenticator, issuer: issuer, @@ -45,7 +41,7 @@ func NewREST(authenticator TokenCredentialRequestAuthenticator, issuer CertIssue type REST struct { authenticator TokenCredentialRequestAuthenticator - issuer CertIssuer + issuer issuer.ClientCertIssuer tableConvertor rest.TableConvertor } @@ -100,30 +96,23 @@ func (r *REST) Create(ctx context.Context, obj runtime.Object, createValidation return nil, err } - user, err := r.authenticator.AuthenticateTokenCredentialRequest(ctx, credentialRequest) + userInfo, err := r.authenticator.AuthenticateTokenCredentialRequest(ctx, credentialRequest) if err != nil { traceFailureWithError(t, "token authentication", err) return failureResponse(), nil } - if user == nil || user.GetName() == "" { - traceSuccess(t, user, false) + if userInfo == nil || userInfo.GetName() == "" { + traceSuccess(t, userInfo, false) return failureResponse(), nil } - certPEM, keyPEM, err := r.issuer.IssuePEM( - pkix.Name{ - CommonName: user.GetName(), - Organization: user.GetGroups(), - }, - []string{}, - clientCertificateTTL, - ) + certPEM, keyPEM, err := r.issuer.IssueClientCertPEM(userInfo.GetName(), userInfo.GetGroups(), clientCertificateTTL) if err != nil { traceFailureWithError(t, "cert issuer", err) return failureResponse(), nil } - traceSuccess(t, user, true) + traceSuccess(t, userInfo, true) return &loginapi.TokenCredentialRequest{ Status: loginapi.TokenCredentialRequestStatus{ diff --git a/internal/registry/credentialrequest/rest_test.go b/internal/registry/credentialrequest/rest_test.go index 8542b99e8..9c05c2bb6 100644 --- a/internal/registry/credentialrequest/rest_test.go +++ b/internal/registry/credentialrequest/rest_test.go @@ -5,7 +5,6 @@ package credentialrequest import ( "context" - "crypto/x509/pkix" "errors" "fmt" "testing" @@ -24,7 +23,9 @@ import ( "k8s.io/klog/v2" loginapi "go.pinniped.dev/generated/latest/apis/concierge/login" + "go.pinniped.dev/internal/issuer" "go.pinniped.dev/internal/mocks/credentialrequestmocks" + "go.pinniped.dev/internal/mocks/issuermocks" "go.pinniped.dev/internal/testutil" ) @@ -88,16 +89,14 @@ func TestCreate(t *testing.T) { Groups: []string{"test-group-1", "test-group-2"}, }, nil) - issuer := credentialrequestmocks.NewMockCertIssuer(ctrl) - issuer.EXPECT().IssuePEM( - pkix.Name{ - CommonName: "test-user", - Organization: []string{"test-group-1", "test-group-2"}}, - []string{}, + clientCertIssuer := issuermocks.NewMockClientCertIssuer(ctrl) + clientCertIssuer.EXPECT().IssueClientCertPEM( + "test-user", + []string{"test-group-1", "test-group-2"}, 5*time.Minute, ).Return([]byte("test-cert"), []byte("test-key"), nil) - storage := NewREST(requestAuthenticator, issuer, schema.GroupResource{}) + storage := NewREST(requestAuthenticator, clientCertIssuer, schema.GroupResource{}) response, err := callCreate(context.Background(), storage, req) @@ -131,12 +130,12 @@ func TestCreate(t *testing.T) { Groups: []string{"test-group-1", "test-group-2"}, }, nil) - issuer := credentialrequestmocks.NewMockCertIssuer(ctrl) - issuer.EXPECT(). - IssuePEM(gomock.Any(), gomock.Any(), gomock.Any()). + clientCertIssuer := issuermocks.NewMockClientCertIssuer(ctrl) + clientCertIssuer.EXPECT(). + IssueClientCertPEM(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil, nil, fmt.Errorf("some certificate authority error")) - storage := NewREST(requestAuthenticator, issuer, schema.GroupResource{}) + storage := NewREST(requestAuthenticator, clientCertIssuer, schema.GroupResource{}) response, err := callCreate(context.Background(), storage, req) requireSuccessfulResponseWithAuthenticationFailureMessage(t, err, response) @@ -353,12 +352,12 @@ func requireSuccessfulResponseWithAuthenticationFailureMessage(t *testing.T, err }) } -func successfulIssuer(ctrl *gomock.Controller) CertIssuer { - issuer := credentialrequestmocks.NewMockCertIssuer(ctrl) - issuer.EXPECT(). - IssuePEM(gomock.Any(), gomock.Any(), gomock.Any()). +func successfulIssuer(ctrl *gomock.Controller) issuer.ClientCertIssuer { + clientCertIssuer := issuermocks.NewMockClientCertIssuer(ctrl) + clientCertIssuer.EXPECT(). + IssueClientCertPEM(gomock.Any(), gomock.Any(), gomock.Any()). Return([]byte("test-cert"), []byte("test-key"), nil) - return issuer + return clientCertIssuer } func stringPtr(s string) *string { diff --git a/internal/secret/cache_test.go b/internal/secret/cache_test.go index 0934bd589..0a6b500c3 100644 --- a/internal/secret/cache_test.go +++ b/internal/secret/cache_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package secret @@ -68,7 +68,7 @@ func TestCacheSynchronized(t *testing.T) { c.SetStateEncoderHashKey(issuer, stateEncoderHashKey) c.SetStateEncoderBlockKey(issuer, stateEncoderBlockKey) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() eg, _ := errgroup.WithContext(ctx) diff --git a/internal/testutil/certs.go b/internal/testutil/certs.go index 797921959..9d6b41d3c 100644 --- a/internal/testutil/certs.go +++ b/internal/testutil/certs.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package testutil @@ -12,6 +12,7 @@ import ( "crypto/x509/pkix" "encoding/pem" "math/big" + "net" "testing" "time" @@ -25,10 +26,18 @@ type ValidCert struct { parsed *x509.Certificate } -// ValidateCertificate validates a certificate and provides an object for asserting properties of the certificate. -func ValidateCertificate(t *testing.T, caPEM string, certPEM string) *ValidCert { +// ValidateServerCertificate validates a certificate and provides an object for asserting properties of the certificate. +func ValidateServerCertificate(t *testing.T, caPEM string, certPEM string) *ValidCert { t.Helper() + return validateCertificate(t, x509.ExtKeyUsageServerAuth, caPEM, certPEM) +} +func ValidateClientCertificate(t *testing.T, caPEM string, certPEM string) *ValidCert { + t.Helper() + return validateCertificate(t, x509.ExtKeyUsageClientAuth, caPEM, certPEM) +} + +func validateCertificate(t *testing.T, extKeyUsage x509.ExtKeyUsage, caPEM string, certPEM string) *ValidCert { block, _ := pem.Decode([]byte(certPEM)) require.NotNil(t, block) parsed, err := x509.ParseCertificate(block.Bytes) @@ -37,7 +46,10 @@ func ValidateCertificate(t *testing.T, caPEM string, certPEM string) *ValidCert // Validate the created cert using the CA. roots := x509.NewCertPool() require.True(t, roots.AppendCertsFromPEM([]byte(caPEM))) - opts := x509.VerifyOptions{Roots: roots} + opts := x509.VerifyOptions{ + Roots: roots, + KeyUsages: []x509.ExtKeyUsage{extKeyUsage}, + } _, err = parsed.Verify(opts) require.NoError(t, err) @@ -61,6 +73,35 @@ func (v *ValidCert) RequireDNSName(expectDNSName string) { require.Contains(v.t, v.parsed.DNSNames, expectDNSName, "expected an explicit DNS SAN, not just Common Name") } +func (v *ValidCert) RequireDNSNames(names []string) { + v.t.Helper() + require.Equal(v.t, names, v.parsed.DNSNames) +} + +func (v *ValidCert) RequireEmptyDNSNames() { + v.t.Helper() + require.Empty(v.t, v.parsed.DNSNames) +} + +func (v *ValidCert) RequireIPs(ips []net.IP) { + v.t.Helper() + actualIPs := v.parsed.IPAddresses + actualIPsStrings := make([]string, len(actualIPs)) + for i := range actualIPs { + actualIPsStrings[i] = actualIPs[i].String() + } + expectedIPsStrings := make([]string, len(ips)) + for i := range ips { + expectedIPsStrings[i] = ips[i].String() + } + require.Equal(v.t, expectedIPsStrings, actualIPsStrings) +} + +func (v *ValidCert) RequireEmptyIPs() { + v.t.Helper() + require.Empty(v.t, v.parsed.IPAddresses) +} + // RequireLifetime asserts that the lifetime of the certificate matches the expected timestamps. func (v *ValidCert) RequireLifetime(expectNotBefore time.Time, expectNotAfter time.Time, delta time.Duration) { v.t.Helper() @@ -81,6 +122,11 @@ func (v *ValidCert) RequireCommonName(commonName string) { require.Equal(v.t, commonName, v.parsed.Subject.CommonName) } +func (v *ValidCert) RequireOrganizations(orgs []string) { + v.t.Helper() + require.Equal(v.t, orgs, v.parsed.Subject.Organization) +} + // CreateCertificate creates a certificate with the provided time bounds, and returns the PEM // representation of the certificate and its private key. The returned certificate is capable of // signing child certificates. diff --git a/internal/testutil/testlogger/testlogger.go b/internal/testutil/testlogger/testlogger.go index f20cbc7ef..51b6ead65 100644 --- a/internal/testutil/testlogger/testlogger.go +++ b/internal/testutil/testlogger/testlogger.go @@ -6,6 +6,7 @@ package testlogger import ( "bytes" + "fmt" "log" "strings" "sync" @@ -13,6 +14,7 @@ import ( "github.com/go-logr/logr" "github.com/go-logr/stdr" + "github.com/stretchr/testify/require" ) // Logger implements logr.Logger in a way that captures logs for test assertions. @@ -46,6 +48,12 @@ func (l *Logger) Lines() []string { return result } +// Expect the emitted lines to match known-good output. +func (l *Logger) Expect(expected []string) { + actual := l.Lines() + require.Equalf(l.t, expected, actual, "did not see expected log output, actual output:\n%#v", backtickStrings(actual)) +} + // syncBuffer synchronizes access to a bytes.Buffer. type syncBuffer struct { mutex sync.Mutex @@ -57,3 +65,18 @@ func (s *syncBuffer) Write(p []byte) (n int, err error) { defer s.mutex.Unlock() return s.buffer.Write(p) } + +type backtickStrings []string + +func (b backtickStrings) GoString() string { + lines := make([]string, 0, len(b)) + for _, s := range b { + if strings.Contains(s, "`") { + s = fmt.Sprintf("%#v", s) + } else { + s = fmt.Sprintf("`%s`", s) + } + lines = append(lines, fmt.Sprintf("\t%s,", s)) + } + return fmt.Sprintf("[]string{\n%s\n}", strings.Join(lines, "\n")) +} diff --git a/pkg/conciergeclient/conciergeclient.go b/pkg/conciergeclient/conciergeclient.go index 0e5fa6f7d..895089bd8 100644 --- a/pkg/conciergeclient/conciergeclient.go +++ b/pkg/conciergeclient/conciergeclient.go @@ -109,7 +109,7 @@ func WithEndpoint(endpoint string) Option { func WithAPIGroupSuffix(apiGroupSuffix string) Option { return func(c *Client) error { if err := groupsuffix.Validate(apiGroupSuffix); err != nil { - return fmt.Errorf("invalid api group suffix: %w", err) + return fmt.Errorf("invalid API group suffix: %w", err) } c.apiGroupSuffix = apiGroupSuffix return nil diff --git a/pkg/conciergeclient/conciergeclient_test.go b/pkg/conciergeclient/conciergeclient_test.go index 028dda9fe..6787cc92a 100644 --- a/pkg/conciergeclient/conciergeclient_test.go +++ b/pkg/conciergeclient/conciergeclient_test.go @@ -5,7 +5,6 @@ package conciergeclient import ( "context" - "crypto/x509/pkix" "encoding/base64" "encoding/json" "fmt" @@ -26,7 +25,7 @@ import ( func TestNew(t *testing.T) { t.Parallel() - testCA, err := certauthority.New(pkix.Name{}, 1*time.Hour) + testCA, err := certauthority.New("Test CA", 1*time.Hour) require.NoError(t, err) tests := []struct { @@ -111,16 +110,16 @@ func TestNew(t *testing.T) { WithEndpoint("https://example.com"), WithAPIGroupSuffix(""), }, - wantErr: "invalid api group suffix: [must contain '.', a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')]", + wantErr: "invalid API group suffix: [must contain '.', a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')]", }, { - name: "invalid api group suffix", + name: "invalid API group suffix", opts: []Option{ WithAuthenticator("jwt", "test-authenticator"), WithEndpoint("https://example.com"), WithAPIGroupSuffix(".starts.with.dot"), }, - wantErr: "invalid api group suffix: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')", + wantErr: "invalid API group suffix: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')", }, { name: "valid", diff --git a/pkg/oidcclient/login_test.go b/pkg/oidcclient/login_test.go index eaa7a1b1a..8005bcafa 100644 --- a/pkg/oidcclient/login_test.go +++ b/pkg/oidcclient/login_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package oidcclient @@ -941,7 +941,7 @@ func TestHandleAuthCodeCallback(t *testing.T) { require.NoError(t, tt.opt(t)(h)) } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() resp := httptest.NewRecorder() diff --git a/proxy-kubeconfig.yaml b/proxy-kubeconfig.yaml new file mode 100644 index 000000000..37c60516a --- /dev/null +++ b/proxy-kubeconfig.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +clusters: + - cluster: + server: https://127.0.0.1:8444 + insecure-skip-tls-verify: true + name: kind-pinniped +contexts: + - context: + cluster: kind-pinniped + user: kind-pinniped + name: kind-pinniped +current-context: kind-pinniped +kind: Config +preferences: {} +users: + - name: kind-pinniped diff --git a/site/content/docs/background/architecture.md b/site/content/docs/background/architecture.md index 5e6469a2a..5eff94f1e 100644 --- a/site/content/docs/background/architecture.md +++ b/site/content/docs/background/architecture.md @@ -1,3 +1,4 @@ + --- title: Architecture description: Dive into the overall design and implementation details of Pinniped. @@ -21,7 +22,8 @@ to be passed on to clusters based on the user information from the IDP. 1. The Pinniped Concierge is a credential exchange API which takes as input a credential from an identity source (e.g., Pinniped Supervisor, proprietary IDP), authenticates the user via that credential, and returns another credential which is -understood by the host Kubernetes cluster. +understood by the host Kubernetes cluster or by an impersonation proxy which acts +on behalf of the user. ![Pinniped Architecture Sketch](/docs/img/pinniped_architecture_concierge_supervisor.svg) @@ -90,14 +92,13 @@ cleanly enable this integration. Pinniped supports the following cluster integration strategies. -* Pinniped hosts a credential exchange API endpoint via a Kubernetes aggregated API server. +* Token Credential Request API: Pinniped hosts a credential exchange API endpoint via a Kubernetes aggregated API server. This API returns a new cluster-specific credential using the cluster's signing keypair to issue short-lived cluster certificates. (In the future, when the Kubernetes CSR API provides a way to issue short-lived certificates, then the Pinniped credential exchange API will use that instead of using the cluster's signing keypair.) - -More cluster integration strategies are coming soon, which will allow Pinniped to -support more Kubernetes cluster types. +* Impersonation Proxy: Pinniped hosts an [impersonation](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation) +proxy that sends requests to the Kubernetes API server with user information and permissions based on a token. ## kubectl Integration @@ -116,8 +117,8 @@ both, or neither. ### Full Integration-- Concierge, Supervisor, and CLI Users can authenticate with the help of the Supervisor, which will issue tokens that -can be exchanged at the Concierge for a credential that is understood by the host Kubernetes -cluster. +can be exchanged at the Concierge for a credential that can be used to authenticate to +the host Kubernetes cluster. The Supervisor enables users to log in to their external identity provider once per day and access each cluster in a domain with a distinct scoped-down token. @@ -136,8 +137,8 @@ JWT authenticator via the Pinniped Concierge. ### Dynamic Cluster Authentication-- Concierge and CLI Users can authenticate directly with their OIDC compliant external identity provider to get credentials which -can be exchanged at the Concierge for a credential that is understood by the host Kubernetes -cluster. +can be exchanged at the Concierge for a credential that can be used to authenticate to +the host Kubernetes cluster. The diagram below shows the components involved in the login flow when the Concierge is configured. diff --git a/site/content/docs/reference/cli.md b/site/content/docs/reference/cli.md index de1c4e343..9cb92b33d 100644 --- a/site/content/docs/reference/cli.md +++ b/site/content/docs/reference/cli.md @@ -32,8 +32,7 @@ pinniped get kubeconfig [flags] - `-h`, `--help`: - help for kubeconfig - + help for kubeconfig - `--concierge-api-group-suffix string`: Concierge API group suffix (default "pinniped.dev") @@ -43,6 +42,21 @@ pinniped get kubeconfig [flags] - `--concierge-authenticator-type string`: Concierge authenticator type (e.g., 'webhook', 'jwt') (default: autodiscover) +- `--concierge-ca-bundle path`: + + Path to TLS certificate authority bundle (PEM format, optional, can be repeated) to use when connecting to the Concierge +- `--concierge-credential-issuer string`: + + Concierge CredentialIssuer object to use for autodiscovery (default: autodiscover) +- `--concierge-endpoint string`: + + API base for the Concierge endpoint +- `--concierge-mode mode`: + + Concierge mode of operation (default TokenCredentialRequestAPI) +- `--concierge-skip-wait`: + + Skip waiting for any pending Concierge strategies to become ready (default: false) - `--kubeconfig string`: Path to kubeconfig file @@ -51,8 +65,8 @@ pinniped get kubeconfig [flags] Kubeconfig context name (default: current active context) - `--no-concierge`: - Generate a configuration which does not use the concierge, but sends the credential to the cluster directly -- `--oidc-ca-bundle strings`: + Generate a configuration which does not use the Concierge, but sends the credential to the cluster directly +- `--oidc-ca-bundle path`: Path to TLS certificate authority bundle (PEM format, optional, can be repeated) - `--oidc-client-id string`: @@ -76,9 +90,18 @@ pinniped get kubeconfig [flags] - `--oidc-skip-browser`: During OpenID Connect login, skip opening the browser (just print the URL) +- `-o`, `--output string`: + + Output file path (default: stdout) +- `--skip-validation`: + + Skip final validation of the kubeconfig (default: false) - `--static-token string`: Instead of doing an OIDC-based login, specify a static token - `--static-token-env string`: Instead of doing an OIDC-based login, read a static token from the environment +- `--timeout duration`: + + Timeout for autodiscovery and validation (default 10m0s) diff --git a/site/content/docs/reference/supported-clusters.md b/site/content/docs/reference/supported-clusters.md index 62ff07fe8..28ba554b1 100644 --- a/site/content/docs/reference/supported-clusters.md +++ b/site/content/docs/reference/supported-clusters.md @@ -15,15 +15,23 @@ menu: | [VMware Tanzu Kubernetes Grid (TKG) clusters](https://tanzu.vmware.com/kubernetes-grid) | Yes | | [Kind clusters](https://kind.sigs.k8s.io/) | Yes | | [Kubeadm-based clusters](https://kubernetes.io/docs/reference/setup-tools/kubeadm/) | Yes | -| [Amazon Elastic Kubernetes Service (EKS)](https://aws.amazon.com/eks/) | No | -| [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine) | No | -| [Azure Kubernetes Service (AKS)](https://azure.microsoft.com/en-us/overview/kubernetes-on-azure) | No | +| [Amazon Elastic Kubernetes Service (EKS)](https://aws.amazon.com/eks/) | Yes | +| [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine) | Yes | +| [Azure Kubernetes Service (AKS)](https://azure.microsoft.com/en-us/overview/kubernetes-on-azure) | Yes | ## Background -The Pinniped Concierge currently supports clusters where a custom pod can be executed on the same node running `kube-controller-manager`. +The Pinniped Concierge has two strategies available to support clusters, under the following conditions: + +1. Token Credential Request API: Can be run on any Kubernetes cluster where a custom pod can be executed on the same node running `kube-controller-manager`. This type of cluster is typically called "self-hosted" because the cluster's control plane is running on nodes that are part of the cluster itself. +Most managed Kubernetes services do not support this. -In practice, this means that many Kubernetes distributions are supported, but not most managed Kubernetes services +2. Impersonation Proxy: Can be run on any Kubernetes cluster where a `LoadBalancer` service can be created. Most cloud-hosted Kubernetes environments have this +capability. The Impersonation Proxy automatically provisions a `LoadBalancer` for ingress to the impersonation endpoint. -Support for more cluster types, including managed Kubernetes environments, is planned. +If a cluster is capable of supporting both strategies, the Pinniped CLI will use the +token credential request API strategy by default. + +To choose the strategy to use with the concierge, use the `--concierge-mode` flag with `pinniped get kubeconfig`. +Possible values are `ImpersonationProxy` and `TokenCredentialRequestAPI`. diff --git a/site/content/docs/tutorials/concierge-and-supervisor-demo.md b/site/content/docs/tutorials/concierge-and-supervisor-demo.md index 382cdfc02..3d2ae8c25 100644 --- a/site/content/docs/tutorials/concierge-and-supervisor-demo.md +++ b/site/content/docs/tutorials/concierge-and-supervisor-demo.md @@ -11,7 +11,7 @@ menu: ## Prerequisites -1. A Kubernetes cluster of a type supported by Pinniped Concierge as described in [architecture](/docs/architecture). +1. A Kubernetes cluster of a type supported by Pinniped Concierge as described in [architecture](/docs/background/architecture). Don't have a cluster handy? Consider using [kind](https://kind.sigs.k8s.io/) on your local machine. See below for an example of using kind. diff --git a/site/content/docs/tutorials/concierge-only-demo.md b/site/content/docs/tutorials/concierge-only-demo.md index 22943c845..e625a2229 100644 --- a/site/content/docs/tutorials/concierge-only-demo.md +++ b/site/content/docs/tutorials/concierge-only-demo.md @@ -12,12 +12,12 @@ menu: ## Prerequisites -1. A Kubernetes cluster of a type supported by Pinniped as described in [architecture](/docs/architecture). +1. A Kubernetes cluster of a type supported by Pinniped as described in [architecture](/docs/background/architecture). Don't have a cluster handy? Consider using [kind](https://kind.sigs.k8s.io/) on your local machine. See below for an example of using kind. -1. An authenticator of a type supported by Pinniped as described in [architecture](/docs/architecture). +1. An authenticator of a type supported by Pinniped as described in [architecture](/docs/background/architecture). Don't have an authenticator of a type supported by Pinniped handy? No problem, there is a demo authenticator available. Start by installing local-user-authenticator on the same cluster where you would like to try Pinniped diff --git a/test/cluster_capabilities/gke.yaml b/test/cluster_capabilities/gke.yaml index e59470849..8bba8b8d5 100644 --- a/test/cluster_capabilities/gke.yaml +++ b/test/cluster_capabilities/gke.yaml @@ -7,6 +7,9 @@ capabilities: # Is it possible to borrow the cluster's signing key from the kube API server? clusterSigningKeyIsAvailable: false + # Will the cluster successfully provision a load balancer if requested? + hasExternalLoadBalancerProvider: true + # Does the cluster allow requests without authentication? # https://kubernetes.io/docs/reference/access-authn-authz/authentication/#anonymous-requests anonymousAuthenticationSupported: true diff --git a/test/cluster_capabilities/kind.yaml b/test/cluster_capabilities/kind.yaml index 6411cd954..0724edb96 100644 --- a/test/cluster_capabilities/kind.yaml +++ b/test/cluster_capabilities/kind.yaml @@ -7,6 +7,9 @@ capabilities: # Is it possible to borrow the cluster's signing key from the kube API server? clusterSigningKeyIsAvailable: true + # Will the cluster successfully provision a load balancer if requested? + hasExternalLoadBalancerProvider: false + # Does the cluster allow requests without authentication? # https://kubernetes.io/docs/reference/access-authn-authz/authentication/#anonymous-requests anonymousAuthenticationSupported: true diff --git a/test/cluster_capabilities/tkgs.yaml b/test/cluster_capabilities/tkgs.yaml index 6411cd954..2ea82b1ed 100644 --- a/test/cluster_capabilities/tkgs.yaml +++ b/test/cluster_capabilities/tkgs.yaml @@ -7,6 +7,9 @@ capabilities: # Is it possible to borrow the cluster's signing key from the kube API server? clusterSigningKeyIsAvailable: true + # Will the cluster successfully provision a load balancer if requested? + hasExternalLoadBalancerProvider: true + # Does the cluster allow requests without authentication? # https://kubernetes.io/docs/reference/access-authn-authz/authentication/#anonymous-requests anonymousAuthenticationSupported: true diff --git a/test/integration/cli_test.go b/test/integration/cli_test.go index bdc4a0448..1221a6046 100644 --- a/test/integration/cli_test.go +++ b/test/integration/cli_test.go @@ -20,22 +20,16 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" "gopkg.in/square/go-jose.v2" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientauthenticationv1beta1 "k8s.io/client-go/pkg/apis/clientauthentication/v1beta1" - identityapi "go.pinniped.dev/generated/latest/apis/concierge/identity" identityv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/identity/v1alpha1" - loginapi "go.pinniped.dev/generated/latest/apis/concierge/login" - loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1" - "go.pinniped.dev/internal/groupsuffix" - "go.pinniped.dev/internal/plog" + conciergescheme "go.pinniped.dev/internal/concierge/scheme" "go.pinniped.dev/internal/testutil" "go.pinniped.dev/pkg/oidcclient" "go.pinniped.dev/pkg/oidcclient/filesession" @@ -47,7 +41,7 @@ func TestCLIGetKubeconfigStaticToken(t *testing.T) { env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable) // Create a test webhook configuration to use with the CLI. - ctx, cancelFunc := context.WithTimeout(context.Background(), 4*time.Minute) + ctx, cancelFunc := context.WithTimeout(context.Background(), 5*time.Minute) defer cancelFunc() authenticator := library.CreateTestWebhookAuthenticator(ctx, t) @@ -55,79 +49,72 @@ func TestCLIGetKubeconfigStaticToken(t *testing.T) { // Build pinniped CLI. pinnipedExe := library.PinnipedCLIPath(t) - for _, tt := range []struct { - name string - args []string - expectStderr string - }{ - { - name: "newer command, but still using static parameters", - args: []string{ - "get", "kubeconfig", - "--static-token", env.TestUser.Token, - "--concierge-api-group-suffix", env.APIGroupSuffix, - "--concierge-authenticator-type", "webhook", - "--concierge-authenticator-name", authenticator.Name, - }, - }, - } { - tt := tt - t.Run(tt.name, func(t *testing.T) { - stdout, stderr := runPinnipedCLI(t, pinnipedExe, tt.args...) - require.Equal(t, tt.expectStderr, stderr) + stdout, stderr := runPinnipedCLI(t, nil, pinnipedExe, "get", "kubeconfig", + "--static-token", env.TestUser.Token, + "--concierge-api-group-suffix", env.APIGroupSuffix, + "--concierge-authenticator-type", "webhook", + "--concierge-authenticator-name", authenticator.Name, + ) + assert.Contains(t, stderr, "discovered CredentialIssuer") + assert.Contains(t, stderr, "discovered Concierge endpoint") + assert.Contains(t, stderr, "discovered Concierge certificate authority bundle") + assert.Contains(t, stderr, "validated connection to the cluster") - // Even the deprecated command should now generate a kubeconfig with the new "pinniped login static" command. - restConfig := library.NewRestConfigFromKubeconfig(t, stdout) - require.NotNil(t, restConfig.ExecProvider) - require.Equal(t, []string{"login", "static"}, restConfig.ExecProvider.Args[:2]) + // Even the deprecated command should now generate a kubeconfig with the new "pinniped login static" command. + restConfig := library.NewRestConfigFromKubeconfig(t, stdout) + require.NotNil(t, restConfig.ExecProvider) + require.Equal(t, []string{"login", "static"}, restConfig.ExecProvider.Args[:2]) - // In addition to the client-go based testing below, also try the kubeconfig - // with kubectl to validate that it works. - t.Run( - "access as user with kubectl", - library.AccessAsUserWithKubectlTest(stdout, env.TestUser.ExpectedUsername, env.ConciergeNamespace), - ) - for _, group := range env.TestUser.ExpectedGroups { - group := group - t.Run( - "access as group "+group+" with kubectl", - library.AccessAsGroupWithKubectlTest(stdout, group, env.ConciergeNamespace), - ) - } - - // Create Kubernetes client with kubeconfig from pinniped CLI. - kubeClient := library.NewClientsetForKubeConfig(t, stdout) - - // Validate that we can auth to the API via our user. - t.Run("access as user with client-go", library.AccessAsUserTest(ctx, env.TestUser.ExpectedUsername, kubeClient)) - for _, group := range env.TestUser.ExpectedGroups { - group := group - t.Run("access as group "+group+" with client-go", library.AccessAsGroupTest(ctx, group, kubeClient)) - } - - // Validate that `pinniped whoami` returns the correct identity. - kubeconfigPath := filepath.Join(testutil.TempDir(t), "whoami-kubeconfig") - require.NoError(t, ioutil.WriteFile(kubeconfigPath, []byte(stdout), 0600)) - assertWhoami( - ctx, - t, - false, - pinnipedExe, - kubeconfigPath, - env.TestUser.ExpectedUsername, - append(env.TestUser.ExpectedGroups, "system:authenticated"), - ) - }) + // In addition to the client-go based testing below, also try the kubeconfig + // with kubectl to validate that it works. + t.Run( + "access as user with kubectl", + library.AccessAsUserWithKubectlTest(stdout, env.TestUser.ExpectedUsername, env.ConciergeNamespace), + ) + for _, group := range env.TestUser.ExpectedGroups { + group := group + t.Run( + "access as group "+group+" with kubectl", + library.AccessAsGroupWithKubectlTest(stdout, group, env.ConciergeNamespace), + ) } + + // Create Kubernetes client with kubeconfig from pinniped CLI. + kubeClient := library.NewClientsetForKubeConfig(t, stdout) + + // Validate that we can auth to the API via our user. + t.Run("access as user with client-go", library.AccessAsUserTest(ctx, env.TestUser.ExpectedUsername, kubeClient)) + for _, group := range env.TestUser.ExpectedGroups { + group := group + t.Run("access as group "+group+" with client-go", library.AccessAsGroupTest(ctx, group, kubeClient)) + } + + t.Run("whoami", func(t *testing.T) { + // Validate that `pinniped whoami` returns the correct identity. + kubeconfigPath := filepath.Join(testutil.TempDir(t), "whoami-kubeconfig") + require.NoError(t, ioutil.WriteFile(kubeconfigPath, []byte(stdout), 0600)) + assertWhoami( + ctx, + t, + false, + pinnipedExe, + kubeconfigPath, + env.TestUser.ExpectedUsername, + append(env.TestUser.ExpectedGroups, "system:authenticated"), + ) + }) } -func runPinnipedCLI(t *testing.T, pinnipedExe string, args ...string) (string, string) { +func runPinnipedCLI(t *testing.T, envVars []string, pinnipedExe string, args ...string) (string, string) { t.Helper() + start := time.Now() var stdout, stderr bytes.Buffer cmd := exec.Command(pinnipedExe, args...) cmd.Stdout = &stdout cmd.Stderr = &stderr + cmd.Env = envVars require.NoErrorf(t, cmd.Run(), "stderr:\n%s\n\nstdout:\n%s\n\n", stderr.String(), stdout.String()) + t.Logf("ran %q in %s", library.MaskTokens("pinniped "+strings.Join(args, " ")), time.Since(start).Round(time.Millisecond)) return stdout.String(), stderr.String() } @@ -163,7 +150,7 @@ func assertWhoami(ctx context.Context, t *testing.T, useProxy bool, pinnipedExe, func deserializeWhoAmIRequest(t *testing.T, data string, apiGroupSuffix string) *identityv1alpha1.WhoAmIRequest { t.Helper() - scheme, _, _ := conciergeschemeNew(apiGroupSuffix) + scheme, _, _ := conciergescheme.New(apiGroupSuffix) codecs := serializer.NewCodecFactory(scheme) respInfo, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), runtime.ContentTypeYAML) require.True(t, ok) @@ -410,110 +397,3 @@ func oidcLoginCommand(ctx context.Context, t *testing.T, pinnipedExe string, ses cmd.Env = append(os.Environ(), env.ProxyEnv()...) return cmd } - -// conciergeschemeNew is a temporary private function to stand in place for -// "go.pinniped.dev/internal/concierge/scheme".New until the later function is merged to main. -func conciergeschemeNew(apiGroupSuffix string) (_ *runtime.Scheme, login, identity schema.GroupVersion) { - // standard set up of the server side scheme - scheme := runtime.NewScheme() - - // add the options to empty v1 - metav1.AddToGroupVersion(scheme, metav1.Unversioned) - - // nothing fancy is required if using the standard group suffix - if apiGroupSuffix == groupsuffix.PinnipedDefaultSuffix { - schemeBuilder := runtime.NewSchemeBuilder( - loginv1alpha1.AddToScheme, - loginapi.AddToScheme, - identityv1alpha1.AddToScheme, - identityapi.AddToScheme, - ) - utilruntime.Must(schemeBuilder.AddToScheme(scheme)) - return scheme, loginv1alpha1.SchemeGroupVersion, identityv1alpha1.SchemeGroupVersion - } - - loginConciergeGroupData, identityConciergeGroupData := groupsuffix.ConciergeAggregatedGroups(apiGroupSuffix) - - addToSchemeAtNewGroup(scheme, loginv1alpha1.GroupName, loginConciergeGroupData.Group, loginv1alpha1.AddToScheme, loginapi.AddToScheme) - addToSchemeAtNewGroup(scheme, identityv1alpha1.GroupName, identityConciergeGroupData.Group, identityv1alpha1.AddToScheme, identityapi.AddToScheme) - - // manually register conversions and defaulting into the correct scheme since we cannot directly call AddToScheme - schemeBuilder := runtime.NewSchemeBuilder( - loginv1alpha1.RegisterConversions, - loginv1alpha1.RegisterDefaults, - identityv1alpha1.RegisterConversions, - identityv1alpha1.RegisterDefaults, - ) - utilruntime.Must(schemeBuilder.AddToScheme(scheme)) - - // we do not want to return errors from the scheme and instead would prefer to defer - // to the REST storage layer for consistency. The simplest way to do this is to force - // a cache miss from the authenticator cache. Kube API groups are validated via the - // IsDNS1123Subdomain func thus we can easily create a group that is guaranteed never - // to be in the authenticator cache. Add a timestamp just to be extra sure. - const authenticatorCacheMissPrefix = "_INVALID_API_GROUP_" - authenticatorCacheMiss := authenticatorCacheMissPrefix + time.Now().UTC().String() - - // we do not have any defaulting functions for *loginv1alpha1.TokenCredentialRequest - // today, but we may have some in the future. Calling AddTypeDefaultingFunc overwrites - // any previously registered defaulting function. Thus to make sure that we catch - // a situation where we add a defaulting func, we attempt to call it here with a nil - // *loginv1alpha1.TokenCredentialRequest. This will do nothing when there is no - // defaulting func registered, but it will almost certainly panic if one is added. - scheme.Default((*loginv1alpha1.TokenCredentialRequest)(nil)) - - // on incoming requests, restore the authenticator API group to the standard group - // note that we are responsible for duplicating this logic for every external API version - scheme.AddTypeDefaultingFunc(&loginv1alpha1.TokenCredentialRequest{}, func(obj interface{}) { - credentialRequest := obj.(*loginv1alpha1.TokenCredentialRequest) - - if credentialRequest.Spec.Authenticator.APIGroup == nil { - // force a cache miss because this is an invalid request - plog.Debug("invalid token credential request, nil group", "authenticator", credentialRequest.Spec.Authenticator) - credentialRequest.Spec.Authenticator.APIGroup = &authenticatorCacheMiss - return - } - - restoredGroup, ok := groupsuffix.Unreplace(*credentialRequest.Spec.Authenticator.APIGroup, apiGroupSuffix) - if !ok { - // force a cache miss because this is an invalid request - plog.Debug("invalid token credential request, wrong group", "authenticator", credentialRequest.Spec.Authenticator) - credentialRequest.Spec.Authenticator.APIGroup = &authenticatorCacheMiss - return - } - - credentialRequest.Spec.Authenticator.APIGroup = &restoredGroup - }) - - return scheme, schema.GroupVersion(loginConciergeGroupData), schema.GroupVersion(identityConciergeGroupData) -} - -func addToSchemeAtNewGroup(scheme *runtime.Scheme, oldGroup, newGroup string, funcs ...func(*runtime.Scheme) error) { - // we need a temporary place to register our types to avoid double registering them - tmpScheme := runtime.NewScheme() - schemeBuilder := runtime.NewSchemeBuilder(funcs...) - utilruntime.Must(schemeBuilder.AddToScheme(tmpScheme)) - - for gvk := range tmpScheme.AllKnownTypes() { - if gvk.GroupVersion() == metav1.Unversioned { - continue // metav1.AddToGroupVersion registers types outside of our aggregated API group that we need to ignore - } - - if gvk.Group != oldGroup { - panic(fmt.Errorf("tmp scheme has type not in the old aggregated API group %s: %s", oldGroup, gvk)) // programmer error - } - - obj, err := tmpScheme.New(gvk) - if err != nil { - panic(err) // programmer error, scheme internal code is broken - } - newGVK := schema.GroupVersionKind{ - Group: newGroup, - Version: gvk.Version, - Kind: gvk.Kind, - } - - // register the existing type but with the new group in the correct scheme - scheme.AddKnownTypeWithName(newGVK, obj) - } -} diff --git a/test/integration/concierge_api_serving_certs_test.go b/test/integration/concierge_api_serving_certs_test.go index af6666d5d..c3e32b56c 100644 --- a/test/integration/concierge_api_serving_certs_test.go +++ b/test/integration/concierge_api_serving_certs_test.go @@ -77,7 +77,7 @@ func TestAPIServingCertificateAutoCreationAndRotation(t *testing.T) { kubeClient := library.NewKubernetesClientset(t) aggregatedClient := library.NewAggregatedClientset(t) conciergeClient := library.NewConciergeClientset(t) - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() apiServiceName := "v1alpha1.login.concierge." + env.APIGroupSuffix @@ -164,10 +164,10 @@ func TestAPIServingCertificateAutoCreationAndRotation(t *testing.T) { return err == nil } - // Unfortunately, although our code changes all the certs immediately, it seems to take ~1 minute for - // the API machinery to notice that we updated our serving cert, causing 1 minute of downtime for our endpoint. - assert.Eventually(t, aggregatedAPIWorking, 2*time.Minute, 250*time.Millisecond) - require.NoError(t, err) // prints out the error and stops the test in case of failure + // our code changes all the certs immediately thus this should be healthy fairly quickly + // if this starts flaking, check for bugs in our dynamiccertificates.Notifier implementation + assert.Eventually(t, aggregatedAPIWorking, 30*time.Second, 250*time.Millisecond) + require.NoError(t, err, "dynamiccertificates.Notifier broken?") // prints out the error and stops the test in case of failure }) } } diff --git a/test/integration/concierge_availability_test.go b/test/integration/concierge_availability_test.go index 575e4fc2a..6dc260bf7 100644 --- a/test/integration/concierge_availability_test.go +++ b/test/integration/concierge_availability_test.go @@ -20,7 +20,7 @@ func TestGetDeployment(t *testing.T) { env := library.IntegrationEnv(t) client := library.NewKubernetesClientset(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() appDeployment, err := client.AppsV1().Deployments(env.ConciergeNamespace).Get(ctx, env.ConciergeAppName, metav1.GetOptions{}) diff --git a/test/integration/concierge_client_test.go b/test/integration/concierge_client_test.go index 1bf1524e7..175daee59 100644 --- a/test/integration/concierge_client_test.go +++ b/test/integration/concierge_client_test.go @@ -57,7 +57,7 @@ var maskKey = func(s string) string { return strings.ReplaceAll(s, "TESTING KEY" func TestClient(t *testing.T) { env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() webhook := library.CreateTestWebhookAuthenticator(ctx, t) diff --git a/test/integration/concierge_credentialissuer_test.go b/test/integration/concierge_credentialissuer_test.go index 172022c63..c146dcc2b 100644 --- a/test/integration/concierge_credentialissuer_test.go +++ b/test/integration/concierge_credentialissuer_test.go @@ -23,7 +23,7 @@ func TestCredentialIssuer(t *testing.T) { client := library.NewConciergeClientset(t) aggregatedClientset := library.NewAggregatedClientset(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() t.Run("test successful CredentialIssuer", func(t *testing.T) { @@ -62,9 +62,20 @@ func TestCredentialIssuer(t *testing.T) { // Verify the cluster strategy status based on what's expected of the test cluster's ability to share signing keys. actualStatusStrategies := actualConfigList.Items[0].Status.Strategies - require.Len(t, actualStatusStrategies, 1) - actualStatusStrategy := actualStatusStrategies[0] - require.Equal(t, configv1alpha1.KubeClusterSigningCertificateStrategyType, actualStatusStrategy.Type) + + // There should be two. One of type KubeClusterSigningCertificate and one of type ImpersonationProxy. + require.Len(t, actualStatusStrategies, 2) + + // The details of the ImpersonationProxy type is tested by a different integration test for the impersonator. + // Grab the KubeClusterSigningCertificate result so we can check it in detail below. + var actualStatusStrategy configv1alpha1.CredentialIssuerStrategy + for _, s := range actualStatusStrategies { + if s.Type == configv1alpha1.KubeClusterSigningCertificateStrategyType { + actualStatusStrategy = s + break + } + } + require.NotNil(t, actualStatusStrategy) if env.HasCapability(library.ClusterSigningKeyIsAvailable) { require.Equal(t, configv1alpha1.SuccessStrategyStatus, actualStatusStrategy.Status) diff --git a/test/integration/concierge_credentialrequest_test.go b/test/integration/concierge_credentialrequest_test.go index 81984095f..20b86347a 100644 --- a/test/integration/concierge_credentialrequest_test.go +++ b/test/integration/concierge_credentialrequest_test.go @@ -23,16 +23,21 @@ import ( ) func TestUnsuccessfulCredentialRequest(t *testing.T) { - _ = library.IntegrationEnv(t).WithCapability(library.AnonymousAuthenticationSupported) + env := library.IntegrationEnv(t).WithCapability(library.AnonymousAuthenticationSupported) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - response, err := makeRequest(ctx, t, validCredentialRequestSpecWithRealToken(t, corev1.TypedLocalObjectReference{ - APIGroup: &auth1alpha1.SchemeGroupVersion.Group, - Kind: "WebhookAuthenticator", - Name: "some-webhook-that-does-not-exist", - })) + response, err := library.CreateTokenCredentialRequest(ctx, t, + loginv1alpha1.TokenCredentialRequestSpec{ + Token: env.TestUser.Token, + Authenticator: corev1.TypedLocalObjectReference{ + APIGroup: &auth1alpha1.SchemeGroupVersion.Group, + Kind: "WebhookAuthenticator", + Name: "some-webhook-that-does-not-exist", + }, + }, + ) require.NoError(t, err) require.Nil(t, response.Status.Credential) require.NotNil(t, response.Status.Message) @@ -84,10 +89,9 @@ func TestSuccessfulCredentialRequest(t *testing.T) { var response *loginv1alpha1.TokenCredentialRequest successfulResponse := func() bool { var err error - response, err = makeRequest(ctx, t, loginv1alpha1.TokenCredentialRequestSpec{ - Token: token, - Authenticator: authenticator, - }) + response, err = library.CreateTokenCredentialRequest(ctx, t, + loginv1alpha1.TokenCredentialRequestSpec{Token: token, Authenticator: authenticator}, + ) require.NoError(t, err, "the request should never fail at the HTTP level") return response.Status.Credential != nil } @@ -131,14 +135,13 @@ func TestFailedCredentialRequestWhenTheRequestIsValidButTheTokenDoesNotAuthentic // Create a testWebhook so we have a legitimate authenticator to pass to the // TokenCredentialRequest API. - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() testWebhook := library.CreateTestWebhookAuthenticator(ctx, t) - response, err := makeRequest(context.Background(), t, loginv1alpha1.TokenCredentialRequestSpec{ - Token: "not a good token", - Authenticator: testWebhook, - }) + response, err := library.CreateTokenCredentialRequest(context.Background(), t, + loginv1alpha1.TokenCredentialRequestSpec{Token: "not a good token", Authenticator: testWebhook}, + ) require.NoError(t, err) @@ -152,14 +155,13 @@ func TestCredentialRequest_ShouldFailWhenRequestDoesNotIncludeToken(t *testing.T // Create a testWebhook so we have a legitimate authenticator to pass to the // TokenCredentialRequest API. - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() testWebhook := library.CreateTestWebhookAuthenticator(ctx, t) - response, err := makeRequest(context.Background(), t, loginv1alpha1.TokenCredentialRequestSpec{ - Token: "", - Authenticator: testWebhook, - }) + response, err := library.CreateTokenCredentialRequest(context.Background(), t, + loginv1alpha1.TokenCredentialRequestSpec{Token: "", Authenticator: testWebhook}, + ) require.Error(t, err) statusError, isStatus := err.(*errors.StatusError) @@ -175,46 +177,6 @@ func TestCredentialRequest_ShouldFailWhenRequestDoesNotIncludeToken(t *testing.T require.Nil(t, response.Status.Credential) } -func TestCredentialRequest_OtherwiseValidRequestWithRealTokenShouldFailWhenTheClusterIsNotCapable(t *testing.T) { - _ = library.IntegrationEnv(t).WithoutCapability(library.ClusterSigningKeyIsAvailable).WithCapability(library.AnonymousAuthenticationSupported) - - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - - testWebhook := library.CreateTestWebhookAuthenticator(ctx, t) - - response, err := makeRequest(ctx, t, validCredentialRequestSpecWithRealToken(t, testWebhook)) - - require.NoError(t, err) - - require.Empty(t, response.Spec) - require.Nil(t, response.Status.Credential) - require.Equal(t, stringPtr("authentication failed"), response.Status.Message) -} - -func makeRequest(ctx context.Context, t *testing.T, spec loginv1alpha1.TokenCredentialRequestSpec) (*loginv1alpha1.TokenCredentialRequest, error) { - t.Helper() - env := library.IntegrationEnv(t) - - client := library.NewAnonymousConciergeClientset(t) - - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - - return client.LoginV1alpha1().TokenCredentialRequests().Create(ctx, &loginv1alpha1.TokenCredentialRequest{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{Namespace: env.ConciergeNamespace}, - Spec: spec, - }, metav1.CreateOptions{}) -} - -func validCredentialRequestSpecWithRealToken(t *testing.T, authenticator corev1.TypedLocalObjectReference) loginv1alpha1.TokenCredentialRequestSpec { - return loginv1alpha1.TokenCredentialRequestSpec{ - Token: library.IntegrationEnv(t).TestUser.Token, - Authenticator: authenticator, - } -} - func stringPtr(s string) *string { return &s } diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go new file mode 100644 index 000000000..7f196484c --- /dev/null +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -0,0 +1,1194 @@ +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package integration + +import ( + "bytes" + "context" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/gorilla/websocket" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/net/http2" + v1 "k8s.io/api/authorization/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" + k8sinformers "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "sigs.k8s.io/yaml" + + conciergev1alpha "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" + identityv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/identity/v1alpha1" + loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1" + "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned" + "go.pinniped.dev/internal/concierge/impersonator" + "go.pinniped.dev/internal/kubeclient" + "go.pinniped.dev/internal/testutil" + "go.pinniped.dev/test/library" +) + +// syncBuffer wraps bytes.Buffer with a mutex so we don't have races in our test code. +type syncBuffer struct { + buf bytes.Buffer + mu sync.Mutex +} + +func (sb *syncBuffer) String() string { + sb.mu.Lock() + defer sb.mu.Unlock() + return sb.buf.String() +} + +func (sb *syncBuffer) Read(b []byte) (int, error) { + sb.mu.Lock() + defer sb.mu.Unlock() + return sb.buf.Read(b) +} + +func (sb *syncBuffer) Write(b []byte) (int, error) { + sb.mu.Lock() + defer sb.mu.Unlock() + return sb.buf.Write(b) +} + +// Note that this test supports being run on all of our integration test cluster types: +// - load balancers not supported, has squid proxy (e.g. kind) +// - load balancers supported, has squid proxy (e.g. EKS) +// - load balancers supported, no squid proxy (e.g. GKE) +func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's complex. + env := library.IntegrationEnv(t) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute) + defer cancel() + + // Create a client using the admin kubeconfig. + adminClient := library.NewKubernetesClientset(t) + adminConciergeClient := library.NewConciergeClientset(t) + + // Create a WebhookAuthenticator and prepare a TokenCredentialRequestSpec using the authenticator for use later. + credentialRequestSpecWithWorkingCredentials := loginv1alpha1.TokenCredentialRequestSpec{ + Token: env.TestUser.Token, + Authenticator: library.CreateTestWebhookAuthenticator(ctx, t), + } + + // The address of the ClusterIP service that points at the impersonation proxy's port (used when there is no load balancer). + proxyServiceEndpoint := fmt.Sprintf("%s-proxy.%s.svc.cluster.local", env.ConciergeAppName, env.ConciergeNamespace) + + var mostRecentTokenCredentialRequestResponse *loginv1alpha1.TokenCredentialRequest + refreshCredential := func() *loginv1alpha1.ClusterCredential { + if mostRecentTokenCredentialRequestResponse == nil || credentialAlmostExpired(t, mostRecentTokenCredentialRequestResponse) { + var err error + // Make a TokenCredentialRequest. This can either return a cert signed by the Kube API server's CA (e.g. on kind) + // or a cert signed by the impersonator's signing CA (e.g. on GKE). Either should be accepted by the impersonation + // proxy server as a valid authentication. + // + // However, we issue short-lived certs, so this cert will only be valid for a few minutes. + // Cache it until it is almost expired and then refresh it whenever it is close to expired. + mostRecentTokenCredentialRequestResponse, err = library.CreateTokenCredentialRequest(ctx, t, credentialRequestSpecWithWorkingCredentials) + require.NoError(t, err) + + require.Nil(t, mostRecentTokenCredentialRequestResponse.Status.Message, + "expected no error message but got: %s", library.Sdump(mostRecentTokenCredentialRequestResponse.Status.Message)) + require.NotEmpty(t, mostRecentTokenCredentialRequestResponse.Status.Credential.ClientCertificateData) + require.NotEmpty(t, mostRecentTokenCredentialRequestResponse.Status.Credential.ClientKeyData) + + // At the moment the credential request should not have returned a token. In the future, if we make it return + // tokens, we should revisit this test's rest config below. + require.Empty(t, mostRecentTokenCredentialRequestResponse.Status.Credential.Token) + } + return mostRecentTokenCredentialRequestResponse.Status.Credential + } + + impersonationProxyViaSquidKubeClientWithoutCredential := func() kubernetes.Interface { + proxyURL := "https://" + proxyServiceEndpoint + kubeconfig := impersonationProxyRestConfig(&loginv1alpha1.ClusterCredential{}, proxyURL, nil, "") + kubeconfig.Proxy = kubeconfigProxyFunc(t, env.Proxy) + return library.NewKubeclient(t, kubeconfig).Kubernetes + } + + newImpersonationProxyClientWithCredentials := func(credentials *loginv1alpha1.ClusterCredential, impersonationProxyURL string, impersonationProxyCACertPEM []byte, doubleImpersonateUser string) *kubeclient.Client { + kubeconfig := impersonationProxyRestConfig(credentials, impersonationProxyURL, impersonationProxyCACertPEM, doubleImpersonateUser) + if !env.HasCapability(library.HasExternalLoadBalancerProvider) { + // Send traffic through the Squid proxy + kubeconfig.Proxy = kubeconfigProxyFunc(t, env.Proxy) + } + return library.NewKubeclient(t, kubeconfig) + } + + newImpersonationProxyClient := func(impersonationProxyURL string, impersonationProxyCACertPEM []byte, doubleImpersonateUser string) *kubeclient.Client { + refreshedCredentials := refreshCredential().DeepCopy() + refreshedCredentials.Token = "not a valid token" // demonstrates that client certs take precedence over tokens by setting both on the requests + return newImpersonationProxyClientWithCredentials(refreshedCredentials, impersonationProxyURL, impersonationProxyCACertPEM, doubleImpersonateUser) + } + + newAnonymousImpersonationProxyClient := func(impersonationProxyURL string, impersonationProxyCACertPEM []byte, doubleImpersonateUser string) *kubeclient.Client { + emptyCredentials := &loginv1alpha1.ClusterCredential{} + return newImpersonationProxyClientWithCredentials(emptyCredentials, impersonationProxyURL, impersonationProxyCACertPEM, doubleImpersonateUser) + } + + oldConfigMap, err := adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Get(ctx, impersonationProxyConfigMapName(env), metav1.GetOptions{}) + if !k8serrors.IsNotFound(err) { + require.NoError(t, err) // other errors aside from NotFound are unexpected + t.Logf("stashing a pre-existing configmap %s", oldConfigMap.Name) + require.NoError(t, adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Delete(ctx, impersonationProxyConfigMapName(env), metav1.DeleteOptions{})) + } + // At the end of the test, clean up the ConfigMap. + t.Cleanup(func() { + ctx, cancel = context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + // Delete any version that was created by this test. + t.Logf("cleaning up configmap at end of test %s", impersonationProxyConfigMapName(env)) + err := adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Delete(ctx, impersonationProxyConfigMapName(env), metav1.DeleteOptions{}) + if !k8serrors.IsNotFound(err) { + require.NoError(t, err) // only not found errors are acceptable + } + + // Only recreate it if it already existed at the start of this test. + if len(oldConfigMap.Data) != 0 { + t.Log(oldConfigMap) + oldConfigMap.UID = "" // cant have a UID yet + oldConfigMap.ResourceVersion = "" + t.Logf("restoring a pre-existing configmap %s", oldConfigMap.Name) + _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Create(ctx, oldConfigMap, metav1.CreateOptions{}) + require.NoError(t, err) + } + + // If we are running on an environment that has a load balancer, expect that the + // CredentialIssuer will be updated eventually with a successful impersonation proxy frontend. + // We do this to ensure that future tests that use the impersonation proxy (e.g., + // TestE2EFullIntegration) will start with a known-good state. + if env.HasCapability(library.HasExternalLoadBalancerProvider) { + performImpersonatorDiscovery(ctx, t, env, adminConciergeClient) + } + }) + + if env.HasCapability(library.HasExternalLoadBalancerProvider) { //nolint:nestif // come on... it's just a test + // Check that load balancer has been automatically created by the impersonator's "auto" mode. + library.RequireEventuallyWithoutError(t, func() (bool, error) { + return hasImpersonationProxyLoadBalancerService(ctx, env, adminClient) + }, 30*time.Second, 500*time.Millisecond) + } else { + require.NotEmpty(t, env.Proxy, + "test cluster does not support load balancers but also doesn't have a squid proxy... "+ + "this is not a supported configuration for test clusters") + + // Check that no load balancer has been created by the impersonator's "auto" mode. + library.RequireNeverWithoutError(t, func() (bool, error) { + return hasImpersonationProxyLoadBalancerService(ctx, env, adminClient) + }, 10*time.Second, 500*time.Millisecond) + + // Check that we can't use the impersonation proxy to execute kubectl commands yet. + _, err = impersonationProxyViaSquidKubeClientWithoutCredential().CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + isErr, message := isServiceUnavailableViaSquidError(err, proxyServiceEndpoint) + require.Truef(t, isErr, "wanted error %q to be service unavailable via squid error, but: %s", err, message) + + // Create configuration to make the impersonation proxy turn on with a hard coded endpoint (without a load balancer). + configMap := configMapForConfig(t, env, impersonator.Config{ + Mode: impersonator.ModeEnabled, + Endpoint: proxyServiceEndpoint, + }) + t.Logf("creating configmap %s", configMap.Name) + _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Create(ctx, &configMap, metav1.CreateOptions{}) + require.NoError(t, err) + } + + // At this point the impersonator should be starting/running. When it is ready, the CredentialIssuer's + // strategies array should be updated to include a successful impersonation strategy which can be used + // to discover the impersonator's URL and CA certificate. Until it has finished starting, it may not be included + // in the strategies array or it may be included in an error state. It can be in an error state for + // awhile when it is waiting for the load balancer to be assigned an ip/hostname. + impersonationProxyURL, impersonationProxyCACertPEM := performImpersonatorDiscovery(ctx, t, env, adminConciergeClient) + if !env.HasCapability(library.HasExternalLoadBalancerProvider) { + // In this case, we specified the endpoint in the configmap, so check that it was reported correctly in the CredentialIssuer. + require.Equal(t, "https://"+proxyServiceEndpoint, impersonationProxyURL) + } + + // Because our credentials expire so quickly, we'll always use a new client, to give us a chance to refresh our + // credentials before they expire. Create a closure to capture the arguments to newImpersonationProxyClient + // so we don't have to keep repeating them. + // This client performs TLS checks, so it also provides test coverage that the impersonation proxy server is generating TLS certs correctly. + impersonationProxyKubeClient := func() kubernetes.Interface { + return newImpersonationProxyClient(impersonationProxyURL, impersonationProxyCACertPEM, "").Kubernetes + } + + t.Run("positive tests", func(t *testing.T) { + // Create an RBAC rule to allow this user to read/write everything. + library.CreateTestClusterRoleBinding(t, + rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, + rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "edit"}, + ) + // Wait for the above RBAC rule to take effect. + library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ + Verb: "get", Group: "", Version: "v1", Resource: "namespaces", + }) + + // Get pods in concierge namespace and pick one. + // this is for tests that require performing actions against a running pod. We use the concierge pod because we already have it handy. + // We want to make sure it's a concierge pod (not cert agent), because we need to be able to "exec echo" and port-forward a running port. + pods, err := adminClient.CoreV1().Pods(env.ConciergeNamespace).List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + require.Greater(t, len(pods.Items), 0) + var conciergePod *corev1.Pod + for _, pod := range pods.Items { + pod := pod + if !strings.Contains(pod.Name, "kube-cert-agent") { + conciergePod = &pod + } + } + require.NotNil(t, conciergePod, "could not find a concierge pod") + + // Test that the user can perform basic actions through the client with their username and group membership + // influencing RBAC checks correctly. + t.Run( + "access as user", + library.AccessAsUserTest(ctx, env.TestUser.ExpectedUsername, impersonationProxyKubeClient()), + ) + for _, group := range env.TestUser.ExpectedGroups { + group := group + t.Run( + "access as group "+group, + library.AccessAsGroupTest(ctx, group, impersonationProxyKubeClient()), + ) + } + + t.Run("kubectl port-forward and keeping the connection open for over a minute", func(t *testing.T) { + kubeconfigPath, envVarsWithProxy, _ := getImpersonationKubeconfig(t, env, impersonationProxyURL, impersonationProxyCACertPEM) + + // Run the kubectl port-forward command. + timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) + defer cancelFunc() + portForwardCmd, _, portForwardStderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, "port-forward", "--namespace", env.ConciergeNamespace, conciergePod.Name, "8443:8443") + portForwardCmd.Env = envVarsWithProxy + + // Start, but don't wait for the command to finish. + err := portForwardCmd.Start() + require.NoError(t, err, `"kubectl port-forward" failed`) + go func() { + assert.EqualErrorf(t, portForwardCmd.Wait(), "signal: killed", `wanted "kubectl port-forward" to get signaled because context was cancelled (stderr: %q)`, portForwardStderr.String()) + }() + + // Wait to see if we time out. The default timeout is 60 seconds, but the server should recognize this this + // is going to be a long-running command and keep the connection open as long as the client stays connected. + time.Sleep(70 * time.Second) + + require.Eventually(t, func() bool { + timeout, cancelFunc = context.WithTimeout(ctx, 2*time.Minute) + defer cancelFunc() + curlCmd := exec.CommandContext(timeout, "curl", "-k", "-sS", "https://127.0.0.1:8443") // -sS turns off the progressbar but still prints errors + var curlStdOut, curlStdErr bytes.Buffer + curlCmd.Stdout = &curlStdOut + curlCmd.Stderr = &curlStdErr + err = curlCmd.Run() + if err != nil { + t.Log("curl error: " + err.Error()) + t.Log("curlStdErr: " + curlStdErr.String()) + t.Log("stdout: " + curlStdOut.String()) + } + // We expect this to 403, but all we care is that it gets through. + return err == nil && strings.Contains(curlStdOut.String(), "\"forbidden: User \\\"system:anonymous\\\" cannot get path \\\"/\\\"\"") + }, 1*time.Minute, 500*time.Millisecond) + }) + + t.Run("using and watching all the basic verbs", func(t *testing.T) { + // Create a namespace, because it will be easier to exercise "deletecollection" if we have a namespace. + namespaceName := createTestNamespace(t, adminClient) + + // Create and start informer to exercise the "watch" verb for us. + informerFactory := k8sinformers.NewSharedInformerFactoryWithOptions( + impersonationProxyKubeClient(), + 0, + k8sinformers.WithNamespace(namespaceName)) + informer := informerFactory.Core().V1().ConfigMaps() + informer.Informer() // makes sure that the informer will cache + stopChannel := make(chan struct{}) + informerFactory.Start(stopChannel) + t.Cleanup(func() { + // Shut down the informer. + close(stopChannel) + }) + informerFactory.WaitForCacheSync(ctx.Done()) + + // Use labels on our created ConfigMaps to avoid accidentally listing other ConfigMaps that might + // exist in the namespace. In Kube 1.20+ there is a default ConfigMap in every namespace. + configMapLabels := labels.Set{ + "pinniped.dev/testConfigMap": library.RandHex(t, 8), + } + + // Test "create" verb through the impersonation proxy. + _, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Create(ctx, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-1", Labels: configMapLabels}}, + metav1.CreateOptions{}, + ) + require.NoError(t, err) + _, err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Create(ctx, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-2", Labels: configMapLabels}}, + metav1.CreateOptions{}, + ) + require.NoError(t, err) + _, err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Create(ctx, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-3", Labels: configMapLabels}}, + metav1.CreateOptions{}, + ) + require.NoError(t, err) + + // Make sure that all of the created ConfigMaps show up in the informer's cache to + // demonstrate that the informer's "watch" verb is working through the impersonation proxy. + require.Eventually(t, func() bool { + _, err1 := informer.Lister().ConfigMaps(namespaceName).Get("configmap-1") + _, err2 := informer.Lister().ConfigMaps(namespaceName).Get("configmap-2") + _, err3 := informer.Lister().ConfigMaps(namespaceName).Get("configmap-3") + return err1 == nil && err2 == nil && err3 == nil + }, 10*time.Second, 50*time.Millisecond) + + // Test "get" verb through the impersonation proxy. + configMap3, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Get(ctx, "configmap-3", metav1.GetOptions{}) + require.NoError(t, err) + + // Test "list" verb through the impersonation proxy. + listResult, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).List(ctx, metav1.ListOptions{ + LabelSelector: configMapLabels.String(), + }) + require.NoError(t, err) + require.Len(t, listResult.Items, 3) + + // Test "update" verb through the impersonation proxy. + configMap3.Data = map[string]string{"foo": "bar"} + updateResult, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Update(ctx, configMap3, metav1.UpdateOptions{}) + require.NoError(t, err) + require.Equal(t, "bar", updateResult.Data["foo"]) + + // Make sure that the updated ConfigMap shows up in the informer's cache. + require.Eventually(t, func() bool { + configMap, err := informer.Lister().ConfigMaps(namespaceName).Get("configmap-3") + return err == nil && configMap.Data["foo"] == "bar" + }, 10*time.Second, 50*time.Millisecond) + + // Test "patch" verb through the impersonation proxy. + patchResult, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Patch(ctx, + "configmap-3", + types.MergePatchType, + []byte(`{"data":{"baz":"42"}}`), + metav1.PatchOptions{}, + ) + require.NoError(t, err) + require.Equal(t, "bar", patchResult.Data["foo"]) + require.Equal(t, "42", patchResult.Data["baz"]) + + // Make sure that the patched ConfigMap shows up in the informer's cache. + require.Eventually(t, func() bool { + configMap, err := informer.Lister().ConfigMaps(namespaceName).Get("configmap-3") + return err == nil && configMap.Data["foo"] == "bar" && configMap.Data["baz"] == "42" + }, 10*time.Second, 50*time.Millisecond) + + // Test "delete" verb through the impersonation proxy. + err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Delete(ctx, "configmap-3", metav1.DeleteOptions{}) + require.NoError(t, err) + + // Make sure that the deleted ConfigMap shows up in the informer's cache. + require.Eventually(t, func() bool { + _, getErr := informer.Lister().ConfigMaps(namespaceName).Get("configmap-3") + list, listErr := informer.Lister().ConfigMaps(namespaceName).List(configMapLabels.AsSelector()) + return k8serrors.IsNotFound(getErr) && listErr == nil && len(list) == 2 + }, 10*time.Second, 50*time.Millisecond) + + // Test "deletecollection" verb through the impersonation proxy. + err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{}) + require.NoError(t, err) + + // Make sure that the deleted ConfigMaps shows up in the informer's cache. + require.Eventually(t, func() bool { + list, listErr := informer.Lister().ConfigMaps(namespaceName).List(configMapLabels.AsSelector()) + return listErr == nil && len(list) == 0 + }, 10*time.Second, 50*time.Millisecond) + + // There should be no ConfigMaps left. + listResult, err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).List(ctx, metav1.ListOptions{ + LabelSelector: configMapLabels.String(), + }) + require.NoError(t, err) + require.Len(t, listResult.Items, 0) + }) + + t.Run("double impersonation as a regular user is blocked", func(t *testing.T) { + // Make a client which will send requests through the impersonation proxy and will also add + // impersonate headers to the request. + doubleImpersonationKubeClient := newImpersonationProxyClient(impersonationProxyURL, impersonationProxyCACertPEM, "other-user-to-impersonate").Kubernetes + + // Check that we can get some resource through the impersonation proxy without any impersonation headers on the request. + // We could use any resource for this, but we happen to know that this one should exist. + _, err := impersonationProxyKubeClient().CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) + require.NoError(t, err) + + // Now we'll see what happens when we add an impersonation header to the request. This should generate a + // request similar to the one above, except that it will also have an impersonation header. + _, err = doubleImpersonationKubeClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) + // Double impersonation is not supported yet, so we should get an error. + require.EqualError(t, err, fmt.Sprintf( + `users "other-user-to-impersonate" is forbidden: `+ + `User "%s" cannot impersonate resource "users" in API group "" at the cluster scope: `+ + `impersonation is not allowed or invalid verb`, + env.TestUser.ExpectedUsername)) + }) + + // This is a separate test from the above double impersonation test because the cluster admin user gets special + // authorization treatment from the Kube API server code that we are using, and we want to ensure that we are blocking + // double impersonation even for the cluster admin. + t.Run("double impersonation as a cluster admin user is blocked", func(t *testing.T) { + // Copy the admin credentials from the admin kubeconfig. + adminClientRestConfig := library.NewClientConfig(t) + + if adminClientRestConfig.BearerToken == "" && adminClientRestConfig.CertData == nil && adminClientRestConfig.KeyData == nil { + t.Skip("The admin kubeconfig does not include credentials, so skipping this test.") + } + + clusterAdminCredentials := &loginv1alpha1.ClusterCredential{ + Token: adminClientRestConfig.BearerToken, + ClientCertificateData: string(adminClientRestConfig.CertData), + ClientKeyData: string(adminClientRestConfig.KeyData), + } + + // Make a client using the admin credentials which will send requests through the impersonation proxy + // and will also add impersonate headers to the request. + doubleImpersonationKubeClient := newImpersonationProxyClientWithCredentials( + clusterAdminCredentials, impersonationProxyURL, impersonationProxyCACertPEM, "other-user-to-impersonate", + ).Kubernetes + + _, err := doubleImpersonationKubeClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) + // Double impersonation is not supported yet, so we should get an error. + require.Error(t, err) + require.Regexp(t, + `users "other-user-to-impersonate" is forbidden: `+ + `User ".*" cannot impersonate resource "users" in API group "" at the cluster scope: `+ + `impersonation is not allowed or invalid verb`, + err.Error(), + ) + }) + + t.Run("WhoAmIRequests and different kinds of authentication through the impersonation proxy", func(t *testing.T) { + // Test using the TokenCredentialRequest for authentication. + impersonationProxyPinnipedConciergeClient := newImpersonationProxyClient( + impersonationProxyURL, impersonationProxyCACertPEM, "", + ).PinnipedConcierge + whoAmI, err := impersonationProxyPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests(). + Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) + require.NoError(t, err) + require.Equal(t, + expectedWhoAmIRequestResponse( + env.TestUser.ExpectedUsername, + append(env.TestUser.ExpectedGroups, "system:authenticated"), + ), + whoAmI, + ) + + // Test an unauthenticated request which does not include any credentials. + impersonationProxyAnonymousPinnipedConciergeClient := newAnonymousImpersonationProxyClient( + impersonationProxyURL, impersonationProxyCACertPEM, "", + ).PinnipedConcierge + whoAmI, err = impersonationProxyAnonymousPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests(). + Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) + require.NoError(t, err) + require.Equal(t, + expectedWhoAmIRequestResponse( + "system:anonymous", + []string{"system:unauthenticated"}, + ), + whoAmI, + ) + + // Test using a service account token. Authenticating as Service Accounts through the impersonation + // proxy is not supported, so it should fail. + namespaceName := createTestNamespace(t, adminClient) + impersonationProxyServiceAccountPinnipedConciergeClient := newImpersonationProxyClientWithCredentials( + &loginv1alpha1.ClusterCredential{Token: createServiceAccountToken(ctx, t, adminClient, namespaceName)}, + impersonationProxyURL, impersonationProxyCACertPEM, "").PinnipedConcierge + _, err = impersonationProxyServiceAccountPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests(). + Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) + require.EqualError(t, err, "Internal error occurred: unimplemented functionality - unable to act as current user") + require.True(t, k8serrors.IsInternalError(err), err) + require.Equal(t, &k8serrors.StatusError{ + ErrStatus: metav1.Status{ + Status: metav1.StatusFailure, + Code: http.StatusInternalServerError, + Reason: metav1.StatusReasonInternalError, + Details: &metav1.StatusDetails{ + Causes: []metav1.StatusCause{ + { + Message: "unimplemented functionality - unable to act as current user", + }, + }, + }, + Message: "Internal error occurred: unimplemented functionality - unable to act as current user", + }, + }, err) + }) + + t.Run("kubectl as a client", func(t *testing.T) { + kubeconfigPath, envVarsWithProxy, tempDir := getImpersonationKubeconfig(t, env, impersonationProxyURL, impersonationProxyCACertPEM) + + // Try "kubectl exec" through the impersonation proxy. + echoString := "hello world" + remoteEchoFile := fmt.Sprintf("/tmp/test-impersonation-proxy-echo-file-%d.txt", time.Now().Unix()) + stdout, err := runKubectl(t, kubeconfigPath, envVarsWithProxy, "exec", "--namespace", env.ConciergeNamespace, conciergePod.Name, "--", "bash", "-c", fmt.Sprintf(`echo "%s" | tee %s`, echoString, remoteEchoFile)) + require.NoError(t, err, `"kubectl exec" failed`) + require.Equal(t, echoString+"\n", stdout) + + // run the kubectl cp command + localEchoFile := filepath.Join(tempDir, filepath.Base(remoteEchoFile)) + _, err = runKubectl(t, kubeconfigPath, envVarsWithProxy, "cp", fmt.Sprintf("%s/%s:%s", env.ConciergeNamespace, conciergePod.Name, remoteEchoFile), localEchoFile) + require.NoError(t, err, `"kubectl cp" failed`) + localEchoFileData, err := ioutil.ReadFile(localEchoFile) + require.NoError(t, err) + require.Equal(t, echoString+"\n", string(localEchoFileData)) + defer func() { + _, _ = runKubectl(t, kubeconfigPath, envVarsWithProxy, "exec", "--namespace", env.ConciergeNamespace, conciergePod.Name, "--", "rm", remoteEchoFile) // cleanup remote echo file + }() + + // run the kubectl logs command + logLinesCount := 10 + stdout, err = runKubectl(t, kubeconfigPath, envVarsWithProxy, "logs", "--namespace", env.ConciergeNamespace, conciergePod.Name, fmt.Sprintf("--tail=%d", logLinesCount)) + require.NoError(t, err, `"kubectl logs" failed`) + require.Equalf(t, logLinesCount, strings.Count(stdout, "\n"), "wanted %d newlines in kubectl logs output:\n%s", logLinesCount, stdout) + + // run the kubectl attach command + namespaceName := createTestNamespace(t, adminClient) + attachPod := library.CreatePod(ctx, t, "impersonation-proxy-attach", namespaceName, corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "impersonation-proxy-attach", + Image: conciergePod.Spec.Containers[0].Image, + Command: []string{"bash"}, + Args: []string{"-c", `while true; do read VAR; echo "VAR: $VAR"; done`}, + Stdin: true, + }, + }, + }) + timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) + defer cancelFunc() + attachCmd, attachStdout, attachStderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, "attach", "--stdin=true", "--namespace", namespaceName, attachPod.Name, "-v=10") + attachCmd.Env = envVarsWithProxy + attachStdin, err := attachCmd.StdinPipe() + require.NoError(t, err) + + // start but don't wait for the attach command + err = attachCmd.Start() + require.NoError(t, err) + attachExitCh := make(chan struct{}) + go func() { + assert.NoError(t, attachCmd.Wait()) + close(attachExitCh) + }() + + // write to stdin on the attach process + _, err = attachStdin.Write([]byte(echoString + "\n")) + require.NoError(t, err) + + // see that we can read stdout and it spits out stdin output back to us + wantAttachStdout := fmt.Sprintf("VAR: %s\n", echoString) + require.Eventuallyf(t, func() bool { return attachStdout.String() == wantAttachStdout }, time.Second*60, time.Millisecond*250, `got "kubectl attach" stdout: %q, wanted: %q (stderr: %q)`, attachStdout.String(), wantAttachStdout, attachStderr.String()) + + // close stdin and attach process should exit + err = attachStdin.Close() + require.NoError(t, err) + requireClose(t, attachExitCh, time.Second*20) + }) + + t.Run("websocket client", func(t *testing.T) { + namespaceName := createTestNamespace(t, adminClient) + + impersonationRestConfig := impersonationProxyRestConfig(refreshCredential(), impersonationProxyURL, impersonationProxyCACertPEM, "") + tlsConfig, err := rest.TLSConfigFor(impersonationRestConfig) + require.NoError(t, err) + + wantConfigMapLabelKey, wantConfigMapLabelValue := "some-label-key", "some-label-value" + dest, _ := url.Parse(impersonationProxyURL) + dest.Scheme = "wss" + dest.Path = "/api/v1/namespaces/" + namespaceName + "/configmaps" + dest.RawQuery = url.Values{ + "watch": {"1"}, + "labelSelector": {fmt.Sprintf("%s=%s", wantConfigMapLabelKey, wantConfigMapLabelValue)}, + "resourceVersion": {"0"}, + }.Encode() + dialer := websocket.Dialer{ + TLSClientConfig: tlsConfig, + } + if !env.HasCapability(library.HasExternalLoadBalancerProvider) { + dialer.Proxy = func(req *http.Request) (*url.URL, error) { + proxyURL, err := url.Parse(env.Proxy) + require.NoError(t, err) + t.Logf("passing request for %s through proxy %s", req.URL, proxyURL.String()) + return proxyURL, nil + } + } + c, r, err := dialer.Dial(dest.String(), http.Header{"Origin": {dest.String()}}) + if r != nil { + defer func() { + require.NoError(t, r.Body.Close()) + }() + } + if err != nil && r != nil { + body, _ := ioutil.ReadAll(r.Body) + t.Logf("websocket dial failed: %d:%s", r.StatusCode, body) + } + require.NoError(t, err) + + // perform a create through the admin client + wantConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "configmap-1", Labels: map[string]string{wantConfigMapLabelKey: wantConfigMapLabelValue}}, + } + wantConfigMap, err = adminClient.CoreV1().ConfigMaps(namespaceName).Create(ctx, + wantConfigMap, + metav1.CreateOptions{}, + ) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, adminClient.CoreV1().ConfigMaps(namespaceName). + DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{})) + }) + + // see if the websocket client received an event for the create + _, message, err := c.ReadMessage() + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + var got watchJSON + err = json.Unmarshal(message, &got) + require.NoError(t, err) + if got.Type != watch.Added { + t.Errorf("Unexpected type: %v", got.Type) + } + var actualConfigMap corev1.ConfigMap + require.NoError(t, json.Unmarshal(got.Object, &actualConfigMap)) + actualConfigMap.TypeMeta = metav1.TypeMeta{} // This isn't filled out in the wantConfigMap we got back from create. + require.Equal(t, *wantConfigMap, actualConfigMap) + }) + + t.Run("http2 client", func(t *testing.T) { + namespaceName := createTestNamespace(t, adminClient) + + wantConfigMapLabelKey, wantConfigMapLabelValue := "some-label-key", "some-label-value" + wantConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "configmap-1", Labels: map[string]string{wantConfigMapLabelKey: wantConfigMapLabelValue}}, + } + wantConfigMap, err = adminClient.CoreV1().ConfigMaps(namespaceName).Create(ctx, + wantConfigMap, + metav1.CreateOptions{}, + ) + require.NoError(t, err) + t.Cleanup(func() { + _ = adminClient.CoreV1().ConfigMaps(namespaceName).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{}) + }) + + // create rest client + restConfig := impersonationProxyRestConfig(refreshCredential(), impersonationProxyURL, impersonationProxyCACertPEM, "") + + tlsConfig, err := rest.TLSConfigFor(restConfig) + require.NoError(t, err) + httpTransport := http.Transport{ + TLSClientConfig: tlsConfig, + } + if !env.HasCapability(library.HasExternalLoadBalancerProvider) { + httpTransport.Proxy = func(req *http.Request) (*url.URL, error) { + proxyURL, err := url.Parse(env.Proxy) + require.NoError(t, err) + t.Logf("passing request for %s through proxy %s", req.URL, proxyURL.String()) + return proxyURL, nil + } + } + err = http2.ConfigureTransport(&httpTransport) + require.NoError(t, err) + + httpClient := http.Client{ + Transport: &httpTransport, + } + + dest, _ := url.Parse(impersonationProxyURL) + dest.Path = "/api/v1/namespaces/" + namespaceName + "/configmaps/configmap-1" + getConfigmapRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, dest.String(), nil) + require.NoError(t, err) + response, err := httpClient.Do(getConfigmapRequest) + require.NoError(t, err) + body, _ := ioutil.ReadAll(response.Body) + t.Logf("http2 status code: %d, proto: %s, message: %s", response.StatusCode, response.Proto, body) + require.Equal(t, "HTTP/2.0", response.Proto) + require.Equal(t, http.StatusOK, response.StatusCode) + defer func() { + require.NoError(t, response.Body.Close()) + }() + var actualConfigMap corev1.ConfigMap + require.NoError(t, json.Unmarshal(body, &actualConfigMap)) + actualConfigMap.TypeMeta = metav1.TypeMeta{} // This isn't filled out in the wantConfigMap we got back from create. + require.Equal(t, *wantConfigMap, actualConfigMap) + + // watch configmaps + dest.Path = "/api/v1/namespaces/" + namespaceName + "/configmaps" + dest.RawQuery = url.Values{ + "watch": {"1"}, + "labelSelector": {fmt.Sprintf("%s=%s", wantConfigMapLabelKey, wantConfigMapLabelValue)}, + "resourceVersion": {"0"}, + }.Encode() + watchConfigmapsRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, dest.String(), nil) + require.NoError(t, err) + response, err = httpClient.Do(watchConfigmapsRequest) + require.NoError(t, err) + require.Equal(t, "HTTP/2.0", response.Proto) + require.Equal(t, http.StatusOK, response.StatusCode) + defer func() { + require.NoError(t, response.Body.Close()) + }() + + // decode + decoder := json.NewDecoder(response.Body) + var got watchJSON + err = decoder.Decode(&got) + require.NoError(t, err) + if got.Type != watch.Added { + t.Errorf("Unexpected type: %v", got.Type) + } + err = json.Unmarshal(got.Object, &actualConfigMap) + require.NoError(t, err) + require.Equal(t, "configmap-1", actualConfigMap.Name) + actualConfigMap.TypeMeta = metav1.TypeMeta{} // This isn't filled out in the wantConfigMap we got back from create. + require.Equal(t, *wantConfigMap, actualConfigMap) + }) + }) + + t.Run("manually disabling the impersonation proxy feature", func(t *testing.T) { + // Update configuration to force the proxy to disabled mode + configMap := configMapForConfig(t, env, impersonator.Config{Mode: impersonator.ModeDisabled}) + if env.HasCapability(library.HasExternalLoadBalancerProvider) { + t.Logf("creating configmap %s", configMap.Name) + _, err := adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Create(ctx, &configMap, metav1.CreateOptions{}) + require.NoError(t, err) + } else { + t.Logf("updating configmap %s", configMap.Name) + _, err := adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Update(ctx, &configMap, metav1.UpdateOptions{}) + require.NoError(t, err) + } + + if env.HasCapability(library.HasExternalLoadBalancerProvider) { + // The load balancer should have been deleted when we disabled the impersonation proxy. + // Note that this can take kind of a long time on real cloud providers (e.g. ~22 seconds on EKS). + library.RequireEventuallyWithoutError(t, func() (bool, error) { + hasService, err := hasImpersonationProxyLoadBalancerService(ctx, env, adminClient) + return !hasService, err + }, 2*time.Minute, 500*time.Millisecond) + } + + // Check that the impersonation proxy port has shut down. + // Ideally we could always check that the impersonation proxy's port has shut down, but on clusters where we + // do not run the squid proxy we have no easy way to see beyond the load balancer to see inside the cluster, + // so we'll skip this check on clusters which have load balancers but don't run the squid proxy. + // The other cluster types that do run the squid proxy will give us sufficient coverage here. + if env.Proxy != "" { + require.Eventually(t, func() bool { + // It's okay if this returns RBAC errors because this user has no role bindings. + // What we want to see is that the proxy eventually shuts down entirely. + _, err := impersonationProxyViaSquidKubeClientWithoutCredential().CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + isErr, _ := isServiceUnavailableViaSquidError(err, proxyServiceEndpoint) + return isErr + }, 20*time.Second, 500*time.Millisecond) + } + + // Check that the generated TLS cert Secret was deleted by the controller because it's supposed to clean this up + // when we disable the impersonator. + require.Eventually(t, func() bool { + _, err := adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) + return k8serrors.IsNotFound(err) + }, 10*time.Second, 250*time.Millisecond) + + // Check that the generated CA cert Secret was not deleted by the controller because it's supposed to keep this + // around in case we decide to later re-enable the impersonator. We want to avoid generating new CA certs when + // possible because they make their way into kubeconfigs on client machines. + _, err := adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyCASecretName(env), metav1.GetOptions{}) + require.NoError(t, err) + + // At this point the impersonator should be stopped. The CredentialIssuer's strategies array should be updated to + // include an unsuccessful impersonation strategy saying that it was manually configured to be disabled. + requireDisabledByConfigurationStrategy(ctx, t, env, adminConciergeClient) + + if !env.HasCapability(library.ClusterSigningKeyIsAvailable) { + // This cluster does not support the cluster signing key strategy, so now that we've manually disabled the + // impersonation strategy, we should be left with no working strategies. + // Given that there are no working strategies, a TokenCredentialRequest which would otherwise work should now + // fail, because there is no point handing out credentials that are not going to work for any strategy. + tokenCredentialRequestResponse, err := library.CreateTokenCredentialRequest(ctx, t, credentialRequestSpecWithWorkingCredentials) + require.NoError(t, err) + + require.NotNil(t, tokenCredentialRequestResponse.Status.Message, "expected an error message but got nil") + require.Equal(t, "authentication failed", *tokenCredentialRequestResponse.Status.Message) + require.Nil(t, tokenCredentialRequestResponse.Status.Credential) + } + }) +} + +func createTestNamespace(t *testing.T, adminClient kubernetes.Interface) string { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + namespace, err := adminClient.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{GenerateName: "impersonation-integration-test-"}, + }, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + t.Logf("cleaning up test namespace %s", namespace.Name) + require.NoError(t, adminClient.CoreV1().Namespaces().Delete(ctx, namespace.Name, metav1.DeleteOptions{})) + }) + return namespace.Name +} + +func createServiceAccountToken(ctx context.Context, t *testing.T, adminClient kubernetes.Interface, namespaceName string) string { + t.Helper() + + serviceAccount, err := adminClient.CoreV1().ServiceAccounts(namespaceName).Create(ctx, + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{GenerateName: "int-test-service-account-"}}, metav1.CreateOptions{}) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, adminClient.CoreV1().ServiceAccounts(namespaceName). + Delete(context.Background(), serviceAccount.Name, metav1.DeleteOptions{})) + }) + + secret, err := adminClient.CoreV1().Secrets(namespaceName).Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "int-test-service-account-token-", + Annotations: map[string]string{ + corev1.ServiceAccountNameKey: serviceAccount.Name, + }, + }, + Type: corev1.SecretTypeServiceAccountToken, + }, metav1.CreateOptions{}) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, adminClient.CoreV1().Secrets(namespaceName). + Delete(context.Background(), secret.Name, metav1.DeleteOptions{})) + }) + + library.RequireEventuallyWithoutError(t, func() (bool, error) { + secret, err = adminClient.CoreV1().Secrets(namespaceName).Get(ctx, secret.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + return len(secret.Data[corev1.ServiceAccountTokenKey]) > 0, nil + }, time.Minute, time.Second) + + return string(secret.Data[corev1.ServiceAccountTokenKey]) +} + +func expectedWhoAmIRequestResponse(username string, groups []string) *identityv1alpha1.WhoAmIRequest { + return &identityv1alpha1.WhoAmIRequest{ + Status: identityv1alpha1.WhoAmIRequestStatus{ + KubernetesUserInfo: identityv1alpha1.KubernetesUserInfo{ + User: identityv1alpha1.UserInfo{ + Username: username, + UID: "", // no way to impersonate UID: https://github.com/kubernetes/kubernetes/issues/93699 + Groups: groups, + Extra: nil, + }, + }, + }, + } +} + +func performImpersonatorDiscovery(ctx context.Context, t *testing.T, env *library.TestEnv, adminConciergeClient versioned.Interface) (string, []byte) { + t.Helper() + var impersonationProxyURL string + var impersonationProxyCACertPEM []byte + + t.Log("Waiting for CredentialIssuer strategy to be successful") + + library.RequireEventuallyWithoutError(t, func() (bool, error) { + credentialIssuer, err := adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Get(ctx, credentialIssuerName(env), metav1.GetOptions{}) + if err != nil || credentialIssuer.Status.Strategies == nil { + t.Log("Did not find any CredentialIssuer with any strategies") + return false, nil // didn't find it, but keep trying + } + for _, strategy := range credentialIssuer.Status.Strategies { + // There will be other strategy types in the list, so ignore those. + if strategy.Type == conciergev1alpha.ImpersonationProxyStrategyType && strategy.Status == conciergev1alpha.SuccessStrategyStatus { //nolint:nestif + if strategy.Frontend == nil { + return false, fmt.Errorf("did not find a Frontend") // unexpected, fail the test + } + if strategy.Frontend.ImpersonationProxyInfo == nil { + return false, fmt.Errorf("did not find an ImpersonationProxyInfo") // unexpected, fail the test + } + impersonationProxyURL = strategy.Frontend.ImpersonationProxyInfo.Endpoint + impersonationProxyCACertPEM, err = base64.StdEncoding.DecodeString(strategy.Frontend.ImpersonationProxyInfo.CertificateAuthorityData) + if err != nil { + return false, err // unexpected, fail the test + } + return true, nil // found it, continue the test! + } else if strategy.Type == conciergev1alpha.ImpersonationProxyStrategyType { + t.Logf("Waiting for successful impersonation proxy strategy on %s: found status %s with reason %s and message: %s", + credentialIssuerName(env), strategy.Status, strategy.Reason, strategy.Message) + if strategy.Reason == conciergev1alpha.ErrorDuringSetupStrategyReason { + // The server encountered an unexpected error while starting the impersonator, so fail the test fast. + return false, fmt.Errorf("found impersonation strategy in %s state with message: %s", strategy.Reason, strategy.Message) + } + } + } + t.Log("Did not find any impersonation proxy strategy on CredentialIssuer") + return false, nil // didn't find it, but keep trying + }, 10*time.Minute, 10*time.Second) + + t.Log("Found successful CredentialIssuer strategy") + return impersonationProxyURL, impersonationProxyCACertPEM +} + +func requireDisabledByConfigurationStrategy(ctx context.Context, t *testing.T, env *library.TestEnv, adminConciergeClient versioned.Interface) { + t.Helper() + + library.RequireEventuallyWithoutError(t, func() (bool, error) { + credentialIssuer, err := adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Get(ctx, credentialIssuerName(env), metav1.GetOptions{}) + if err != nil || credentialIssuer.Status.Strategies == nil { + t.Log("Did not find any CredentialIssuer with any strategies") + return false, nil // didn't find it, but keep trying + } + for _, strategy := range credentialIssuer.Status.Strategies { + // There will be other strategy types in the list, so ignore those. + if strategy.Type == conciergev1alpha.ImpersonationProxyStrategyType && + strategy.Status == conciergev1alpha.ErrorStrategyStatus && + strategy.Reason == conciergev1alpha.DisabledStrategyReason { //nolint:nestif + return true, nil // found it, continue the test! + } else if strategy.Type == conciergev1alpha.ImpersonationProxyStrategyType { + t.Logf("Waiting for disabled impersonation proxy strategy on %s: found status %s with reason %s and message: %s", + credentialIssuerName(env), strategy.Status, strategy.Reason, strategy.Message) + if strategy.Reason == conciergev1alpha.ErrorDuringSetupStrategyReason { + // The server encountered an unexpected error while stopping the impersonator, so fail the test fast. + return false, fmt.Errorf("found impersonation strategy in %s state with message: %s", strategy.Reason, strategy.Message) + } + } + } + t.Log("Did not find any impersonation proxy strategy on CredentialIssuer") + return false, nil // didn't find it, but keep trying + }, 1*time.Minute, 500*time.Millisecond) +} + +func credentialAlmostExpired(t *testing.T, credential *loginv1alpha1.TokenCredentialRequest) bool { + t.Helper() + pemBlock, _ := pem.Decode([]byte(credential.Status.Credential.ClientCertificateData)) + parsedCredential, err := x509.ParseCertificate(pemBlock.Bytes) + require.NoError(t, err) + timeRemaining := time.Until(parsedCredential.NotAfter) + if timeRemaining < 2*time.Minute { + t.Logf("The TokenCredentialRequest cred is almost expired and needs to be refreshed. Expires in %s.", timeRemaining) + return true + } + t.Logf("The TokenCredentialRequest cred is good for some more time (%s) so using it.", timeRemaining) + return false +} + +func impersonationProxyRestConfig(credential *loginv1alpha1.ClusterCredential, host string, caData []byte, doubleImpersonateUser string) *rest.Config { + config := rest.Config{ + Host: host, + TLSClientConfig: rest.TLSClientConfig{ + Insecure: caData == nil, + CAData: caData, + CertData: []byte(credential.ClientCertificateData), + KeyData: []byte(credential.ClientKeyData), + }, + // kubectl would set both the client cert and the token, so we'll do that too. + // The Kube API server will ignore the token if the client cert successfully authenticates. + // Only if the client cert is not present or fails to authenticate will it use the token. + // Historically, it works that way because some web browsers will always send your + // corporate-assigned client cert even if it is not valid, and it doesn't want to treat + // that as a failure if you also sent a perfectly good token. + // We would like the impersonation proxy to imitate that behavior, so we test it here. + BearerToken: credential.Token, + } + if doubleImpersonateUser != "" { + config.Impersonate = rest.ImpersonationConfig{UserName: doubleImpersonateUser} + } + return &config +} + +func kubeconfigProxyFunc(t *testing.T, squidProxyURL string) func(req *http.Request) (*url.URL, error) { + return func(req *http.Request) (*url.URL, error) { + t.Helper() + parsedSquidProxyURL, err := url.Parse(squidProxyURL) + require.NoError(t, err) + t.Logf("passing request for %s through proxy %s", req.URL, parsedSquidProxyURL.String()) + return parsedSquidProxyURL, nil + } +} + +func configMapForConfig(t *testing.T, env *library.TestEnv, config impersonator.Config) corev1.ConfigMap { + t.Helper() + configString, err := yaml.Marshal(config) + require.NoError(t, err) + configMap := corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: impersonationProxyConfigMapName(env), + }, + Data: map[string]string{ + "config.yaml": string(configString), + }} + return configMap +} + +func hasImpersonationProxyLoadBalancerService(ctx context.Context, env *library.TestEnv, client kubernetes.Interface) (bool, error) { + service, err := client.CoreV1().Services(env.ConciergeNamespace).Get(ctx, impersonationProxyLoadBalancerName(env), metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, err + } + return service.Spec.Type == corev1.ServiceTypeLoadBalancer, nil +} + +func impersonationProxyConfigMapName(env *library.TestEnv) string { + return env.ConciergeAppName + "-impersonation-proxy-config" +} + +func impersonationProxyTLSSecretName(env *library.TestEnv) string { + return env.ConciergeAppName + "-impersonation-proxy-tls-serving-certificate" +} + +func impersonationProxyCASecretName(env *library.TestEnv) string { + return env.ConciergeAppName + "-impersonation-proxy-ca-certificate" +} + +func impersonationProxyLoadBalancerName(env *library.TestEnv) string { + return env.ConciergeAppName + "-impersonation-proxy-load-balancer" +} + +func credentialIssuerName(env *library.TestEnv) string { + return env.ConciergeAppName + "-config" +} + +func getImpersonationKubeconfig(t *testing.T, env *library.TestEnv, impersonationProxyURL string, impersonationProxyCACertPEM []byte) (string, []string, string) { + t.Helper() + + pinnipedExe := library.PinnipedCLIPath(t) + tempDir := testutil.TempDir(t) + + var envVarsWithProxy []string + if !env.HasCapability(library.HasExternalLoadBalancerProvider) { + // Only if you don't have a load balancer, use the squid proxy when it's available. + envVarsWithProxy = append(os.Environ(), env.ProxyEnv()...) + } + + // Get the kubeconfig. + getKubeConfigCmd := []string{"get", "kubeconfig", + "--concierge-api-group-suffix", env.APIGroupSuffix, + "--oidc-skip-browser", + "--static-token", env.TestUser.Token, + // Force the use of impersonation proxy strategy, but let it auto-discover the endpoint and CA. + "--concierge-mode", "ImpersonationProxy"} + t.Log("Running:", pinnipedExe, getKubeConfigCmd) + kubeconfigYAML, getKubeConfigStderr := runPinnipedCLI(t, envVarsWithProxy, pinnipedExe, getKubeConfigCmd...) + // "pinniped get kubectl" prints some status messages to stderr + t.Log(getKubeConfigStderr) + // Make sure that the "pinniped get kubeconfig" auto-discovered the impersonation proxy and we're going to + // make our kubectl requests through the impersonation proxy. Avoid using require.Contains because the error + // message would contain credentials. + require.True(t, + strings.Contains(kubeconfigYAML, "server: "+impersonationProxyURL+"\n"), + "the generated kubeconfig did not include the expected impersonation server address: %s", + impersonationProxyURL, + ) + require.True(t, + strings.Contains(kubeconfigYAML, "- --concierge-ca-bundle-data="+base64.StdEncoding.EncodeToString(impersonationProxyCACertPEM)+"\n"), + "the generated kubeconfig did not include the base64 encoded version of this expected impersonation CA cert: %s", + impersonationProxyCACertPEM, + ) + + // Write the kubeconfig to a temp file. + kubeconfigPath := filepath.Join(tempDir, "kubeconfig.yaml") + require.NoError(t, ioutil.WriteFile(kubeconfigPath, []byte(kubeconfigYAML), 0600)) + + return kubeconfigPath, envVarsWithProxy, tempDir +} + +// func to create kubectl commands with a kubeconfig. +func kubectlCommand(timeout context.Context, t *testing.T, kubeconfigPath string, envVarsWithProxy []string, args ...string) (*exec.Cmd, *syncBuffer, *syncBuffer) { + allArgs := append([]string{"--kubeconfig", kubeconfigPath}, args...) + //nolint:gosec // we are not performing malicious argument injection against ourselves + kubectlCmd := exec.CommandContext(timeout, "kubectl", allArgs...) + var stdout, stderr syncBuffer + kubectlCmd.Stdout = &stdout + kubectlCmd.Stderr = &stderr + kubectlCmd.Env = envVarsWithProxy + + t.Log("starting kubectl subprocess: kubectl", strings.Join(allArgs, " ")) + return kubectlCmd, &stdout, &stderr +} + +// Func to run kubeconfig commands. +func runKubectl(t *testing.T, kubeconfigPath string, envVarsWithProxy []string, args ...string) (string, error) { + timeout, cancelFunc := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancelFunc() + + kubectlCmd, stdout, stderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, args...) + + err := kubectlCmd.Run() + t.Logf("kubectl stdout output: %s", stdout.String()) + t.Logf("kubectl stderr output: %s", stderr.String()) + return stdout.String(), err +} + +// watchJSON defines the expected JSON wire equivalent of watch.Event. +type watchJSON struct { + Type watch.EventType `json:"type,omitempty"` + Object json.RawMessage `json:"object,omitempty"` +} + +// requireServiceUnavailableViaSquidError returns whether the provided err is the error that is +// returned by squid when the impersonation proxy port inside the cluster is not listening. +func isServiceUnavailableViaSquidError(err error, proxyServiceEndpoint string) (bool, string) { + if err == nil { + return false, "error is nil" + } + + for _, wantContains := range []string{ + fmt.Sprintf(`Get "https://%s/api/v1/namespaces"`, proxyServiceEndpoint), + ": Service Unavailable", + } { + if !strings.Contains(err.Error(), wantContains) { + return false, fmt.Sprintf("error does not contain %q", wantContains) + } + } + + return true, "" +} + +func requireClose(t *testing.T, c chan struct{}, timeout time.Duration) { + t.Helper() + timer := time.NewTimer(timeout) + select { + case <-c: + if !timer.Stop() { + <-timer.C + } + case <-timer.C: + require.FailNow(t, "failed to receive from channel within "+timeout.String()) + } +} diff --git a/test/integration/concierge_kubecertagent_test.go b/test/integration/concierge_kubecertagent_test.go index 5ce1cff44..0b0ae8ae2 100644 --- a/test/integration/concierge_kubecertagent_test.go +++ b/test/integration/concierge_kubecertagent_test.go @@ -18,6 +18,7 @@ import ( "k8s.io/apimachinery/pkg/util/diff" "k8s.io/apimachinery/pkg/util/wait" + conciergev1alpha "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" "go.pinniped.dev/test/library" ) @@ -28,7 +29,7 @@ const ( func TestKubeCertAgent(t *testing.T) { env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable) - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() kubeClient := library.NewKubernetesClientset(t) @@ -125,6 +126,30 @@ func TestKubeCertAgent(t *testing.T) { assert.Eventually(t, agentPodsReconciled, 10*time.Second, 250*time.Millisecond) require.NoError(t, err) }) + + // Because the above tests have purposefully put the kube cert issuer strategy into a broken + // state, wait for it to become healthy again before moving on to other integration tests, + // otherwise those tests would be polluted by this test and would have to wait for the + // strategy to become successful again. + library.RequireEventuallyWithoutError(t, func() (bool, error) { + adminConciergeClient := library.NewConciergeClientset(t) + credentialIssuer, err := adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Get(ctx, credentialIssuerName(env), metav1.GetOptions{}) + if err != nil || credentialIssuer.Status.Strategies == nil { + t.Log("Did not find any CredentialIssuer with any strategies") + return false, nil // didn't find it, but keep trying + } + for _, strategy := range credentialIssuer.Status.Strategies { + // There will be other strategy types in the list, so ignore those. + if strategy.Type == conciergev1alpha.KubeClusterSigningCertificateStrategyType && strategy.Status == conciergev1alpha.SuccessStrategyStatus { //nolint:nestif + if strategy.Frontend == nil { + return false, fmt.Errorf("did not find a Frontend") // unexpected, fail the test + } + return true, nil // found it, continue the test! + } + } + t.Log("Did not find any successful KubeClusterSigningCertificate strategy on CredentialIssuer") + return false, nil // didn't find it, but keep trying + }, 3*time.Minute, 3*time.Second) } func ensureKubeCertAgentSteadyState(t *testing.T, agentPodsReconciled func() bool) { diff --git a/test/integration/e2e_test.go b/test/integration/e2e_test.go index 572b1955c..c0137cab0 100644 --- a/test/integration/e2e_test.go +++ b/test/integration/e2e_test.go @@ -6,7 +6,6 @@ import ( "bufio" "bytes" "context" - "crypto/x509/pkix" "encoding/base64" "errors" "fmt" @@ -23,6 +22,7 @@ import ( coreosoidc "github.com/coreos/go-oidc/v3/oidc" "github.com/stretchr/testify/require" + authorizationv1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -40,7 +40,7 @@ import ( // TestE2EFullIntegration tests a full integration scenario that combines the supervisor, concierge, and CLI. func TestE2EFullIntegration(t *testing.T) { - env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable) + env := library.IntegrationEnv(t) // If anything in this test crashes, dump out the supervisor and proxy pod logs. defer library.DumpLogs(t, env.SupervisorNamespace, "") @@ -65,7 +65,7 @@ func TestE2EFullIntegration(t *testing.T) { // Generate a CA bundle with which to serve this provider. t.Logf("generating test CA") - ca, err := certauthority.New(pkix.Name{CommonName: "Downstream Test CA"}, 1*time.Hour) + ca, err := certauthority.New("Downstream Test CA", 1*time.Hour) require.NoError(t, err) // Save that bundle plus the one that signs the upstream issuer, for test purposes. @@ -76,12 +76,7 @@ func TestE2EFullIntegration(t *testing.T) { // Use the CA to issue a TLS server cert. t.Logf("issuing test certificate") - tlsCert, err := ca.Issue( - pkix.Name{CommonName: issuerURL.Hostname()}, - []string{issuerURL.Hostname()}, - nil, - 1*time.Hour, - ) + tlsCert, err := ca.IssueServerCert([]string{issuerURL.Hostname()}, nil, 1*time.Hour) require.NoError(t, err) certPEM, keyPEM, err := certauthority.ToPEM(tlsCert) require.NoError(t, err) @@ -132,12 +127,18 @@ func TestE2EFullIntegration(t *testing.T) { rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.SupervisorTestUpstream.Username}, rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "view"}, ) + library.WaitForUserToHaveAccess(t, env.SupervisorTestUpstream.Username, []string{}, &authorizationv1.ResourceAttributes{ + Verb: "get", + Group: "", + Version: "v1", + Resource: "namespaces", + }) // Use a specific session cache for this test. sessionCachePath := tempDir + "/sessions.yaml" // Run "pinniped get kubeconfig" to get a kubeconfig YAML. - kubeconfigYAML, stderr := runPinnipedCLI(t, pinnipedExe, "get", "kubeconfig", + kubeconfigYAML, stderr := runPinnipedCLI(t, nil, pinnipedExe, "get", "kubeconfig", "--concierge-api-group-suffix", env.APIGroupSuffix, "--concierge-authenticator-type", "jwt", "--concierge-authenticator-name", authenticator.Name, @@ -145,7 +146,8 @@ func TestE2EFullIntegration(t *testing.T) { "--oidc-ca-bundle", testCABundlePath, "--oidc-session-cache", sessionCachePath, ) - require.Equal(t, "", stderr) + t.Logf("stderr output from 'pinniped get kubeconfig':\n%s\n\n", stderr) + t.Logf("test kubeconfig:\n%s\n\n", kubeconfigYAML) restConfig := library.NewRestConfigFromKubeconfig(t, kubeconfigYAML) require.NotNil(t, restConfig.ExecProvider) diff --git a/test/integration/kubeclient_test.go b/test/integration/kubeclient_test.go index e5f759c5b..88edc8541 100644 --- a/test/integration/kubeclient_test.go +++ b/test/integration/kubeclient_test.go @@ -43,13 +43,12 @@ func TestKubeClientOwnerRef(t *testing.T) { ) require.NoError(t, err) - defer func() { - if t.Failed() { - return - } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() err := namespaces.Delete(ctx, namespace.Name, metav1.DeleteOptions{}) require.NoError(t, err) - }() + }) // create something that we can point to parentSecret, err := regularClient.CoreV1().Secrets(namespace.Name).Create( @@ -95,13 +94,15 @@ func TestKubeClientOwnerRef(t *testing.T) { metav1.CreateOptions{}, ) require.NoError(t, err) - defer func() { + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() err := regularAggregationClient.ApiregistrationV1().APIServices().Delete(ctx, parentAPIService.Name, metav1.DeleteOptions{}) if errors.IsNotFound(err) { return } require.NoError(t, err) - }() + }) // work around stupid behavior of WithoutVersionDecoder.Decode parentAPIService.APIVersion, parentAPIService.Kind = apiregistrationv1.SchemeGroupVersion.WithKind("APIService").ToAPIVersionAndKind() diff --git a/test/integration/supervisor_discovery_test.go b/test/integration/supervisor_discovery_test.go index c164cb7b5..4bbd41266 100644 --- a/test/integration/supervisor_discovery_test.go +++ b/test/integration/supervisor_discovery_test.go @@ -7,7 +7,6 @@ import ( "context" "crypto/tls" "crypto/x509" - "crypto/x509/pkix" "encoding/json" "fmt" "io/ioutil" @@ -282,11 +281,11 @@ func defaultTLSCertSecretName(env *library.TestEnv) string { func createTLSCertificateSecret(ctx context.Context, t *testing.T, ns string, hostname string, ips []net.IP, secretName string, kubeClient kubernetes.Interface) *certauthority.CA { // Create a CA. - ca, err := certauthority.New(pkix.Name{CommonName: "Acme Corp"}, 1000*time.Hour) + ca, err := certauthority.New("Acme Corp", 1000*time.Hour) require.NoError(t, err) // Using the CA, create a TLS server cert. - tlsCert, err := ca.Issue(pkix.Name{CommonName: hostname}, []string{hostname}, ips, 1000*time.Hour) + tlsCert, err := ca.IssueServerCert([]string{hostname}, ips, 1000*time.Hour) require.NoError(t, err) // Write the serving cert to the SNI secret. diff --git a/test/integration/supervisor_login_test.go b/test/integration/supervisor_login_test.go index b3e4760c5..efc0134a2 100644 --- a/test/integration/supervisor_login_test.go +++ b/test/integration/supervisor_login_test.go @@ -6,7 +6,6 @@ package integration import ( "context" "crypto/tls" - "crypto/x509/pkix" "encoding/base64" "encoding/json" "fmt" @@ -56,7 +55,7 @@ func TestSupervisorLogin(t *testing.T) { // Generate a CA bundle with which to serve this provider. t.Logf("generating test CA") - ca, err := certauthority.New(pkix.Name{CommonName: "Downstream Test CA"}, 1*time.Hour) + ca, err := certauthority.New("Downstream Test CA", 1*time.Hour) require.NoError(t, err) // Create an HTTP client that can reach the downstream discovery endpoint using the CA certs. @@ -83,12 +82,7 @@ func TestSupervisorLogin(t *testing.T) { // Use the CA to issue a TLS server cert. t.Logf("issuing test certificate") - tlsCert, err := ca.Issue( - pkix.Name{CommonName: issuerURL.Hostname()}, - []string{issuerURL.Hostname()}, - nil, - 1*time.Hour, - ) + tlsCert, err := ca.IssueServerCert([]string{issuerURL.Hostname()}, nil, 1*time.Hour) require.NoError(t, err) certPEM, keyPEM, err := certauthority.ToPEM(tlsCert) require.NoError(t, err) @@ -238,7 +232,7 @@ func verifyTokenResponse( nonceParam nonce.Nonce, expectedIDTokenClaims []string, ) { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() // Verify the ID Token. @@ -308,7 +302,7 @@ func (s *localCallbackServer) waitForCallback(timeout time.Duration) *http.Reque } func doTokenExchange(t *testing.T, config *oauth2.Config, tokenResponse *oauth2.Token, httpClient *http.Client, provider *coreosoidc.Provider) { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() // Form the HTTP POST request with the parameters specified by RFC8693. diff --git a/test/integration/supervisor_storage_garbage_collection_test.go b/test/integration/supervisor_storage_garbage_collection_test.go index c89a66e4d..84b82196b 100644 --- a/test/integration/supervisor_storage_garbage_collection_test.go +++ b/test/integration/supervisor_storage_garbage_collection_test.go @@ -99,7 +99,7 @@ func createSecret(ctx context.Context, t *testing.T, secrets corev1client.Secret // Make sure the Secret is deleted when the test ends. t.Cleanup(func() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() err := secrets.Delete(ctx, secret.Name, metav1.DeleteOptions{}) notFound := k8serrors.IsNotFound(err) diff --git a/test/integration/supervisor_storage_test.go b/test/integration/supervisor_storage_test.go index 51d0f70ea..5edc4e701 100644 --- a/test/integration/supervisor_storage_test.go +++ b/test/integration/supervisor_storage_test.go @@ -42,13 +42,13 @@ func TestAuthorizeCodeStorage(t *testing.T) { secrets := client.CoreV1().Secrets(env.SupervisorNamespace) t.Cleanup(func() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() err := secrets.Delete(ctx, name, metav1.DeleteOptions{}) require.NoError(t, err) }) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() // get a session with most of the data filled out diff --git a/test/integration/whoami_test.go b/test/integration/whoami_test.go index d39f828fe..15f87fe21 100644 --- a/test/integration/whoami_test.go +++ b/test/integration/whoami_test.go @@ -34,7 +34,7 @@ func TestWhoAmI_Kubeadm(t *testing.T) { // we should add more robust logic around skipping clusters based on vendor _ = library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() whoAmI, err := library.NewConciergeClientset(t).IdentityV1alpha1().WhoAmIRequests(). @@ -63,7 +63,7 @@ func TestWhoAmI_Kubeadm(t *testing.T) { func TestWhoAmI_ServiceAccount_Legacy(t *testing.T) { _ = library.IntegrationEnv(t) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() kubeClient := library.NewKubernetesClientset(t).CoreV1() @@ -75,13 +75,9 @@ func TestWhoAmI_ServiceAccount_Legacy(t *testing.T) { }, metav1.CreateOptions{}) require.NoError(t, err) - defer func() { - if t.Failed() { - return - } - err := kubeClient.Namespaces().Delete(ctx, ns.Name, metav1.DeleteOptions{}) - require.NoError(t, err) - }() + t.Cleanup(func() { + require.NoError(t, kubeClient.Namespaces().Delete(context.Background(), ns.Name, metav1.DeleteOptions{})) + }) sa, err := kubeClient.ServiceAccounts(ns.Name).Create(ctx, &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ @@ -107,7 +103,7 @@ func TestWhoAmI_ServiceAccount_Legacy(t *testing.T) { return false, err } return len(secret.Data[corev1.ServiceAccountTokenKey]) > 0, nil - }, 30*time.Second, time.Second) + }, time.Minute, time.Second) saConfig := library.NewAnonymousClientRestConfig(t) saConfig.BearerToken = string(secret.Data[corev1.ServiceAccountTokenKey]) @@ -140,7 +136,7 @@ func TestWhoAmI_ServiceAccount_Legacy(t *testing.T) { func TestWhoAmI_ServiceAccount_TokenRequest(t *testing.T) { _ = library.IntegrationEnv(t) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() kubeClient := library.NewKubernetesClientset(t).CoreV1() @@ -152,13 +148,9 @@ func TestWhoAmI_ServiceAccount_TokenRequest(t *testing.T) { }, metav1.CreateOptions{}) require.NoError(t, err) - defer func() { - if t.Failed() { - return - } - err := kubeClient.Namespaces().Delete(ctx, ns.Name, metav1.DeleteOptions{}) - require.NoError(t, err) - }() + t.Cleanup(func() { + require.NoError(t, kubeClient.Namespaces().Delete(context.Background(), ns.Name, metav1.DeleteOptions{})) + }) sa, err := kubeClient.ServiceAccounts(ns.Name).Create(ctx, &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ @@ -258,7 +250,7 @@ func TestWhoAmI_CSR(t *testing.T) { // we should add more robust logic around skipping clusters based on vendor _ = library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() kubeClient := library.NewKubernetesClientset(t) @@ -287,13 +279,10 @@ func TestWhoAmI_CSR(t *testing.T) { ) require.NoError(t, err) - defer func() { - if t.Failed() { - return - } - err := kubeClient.CertificatesV1beta1().CertificateSigningRequests().Delete(ctx, csrName, metav1.DeleteOptions{}) - require.NoError(t, err) - }() + t.Cleanup(func() { + require.NoError(t, kubeClient.CertificatesV1beta1().CertificateSigningRequests(). + Delete(context.Background(), csrName, metav1.DeleteOptions{})) + }) // this is a blind update with no resource version checks, which is only safe during tests // use the beta CSR API to support older clusters @@ -346,7 +335,7 @@ func TestWhoAmI_CSR(t *testing.T) { func TestWhoAmI_Anonymous(t *testing.T) { _ = library.IntegrationEnv(t).WithCapability(library.AnonymousAuthenticationSupported) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() anonymousConfig := library.NewAnonymousClientRestConfig(t) @@ -376,7 +365,7 @@ func TestWhoAmI_Anonymous(t *testing.T) { func TestWhoAmI_ImpersonateDirectly(t *testing.T) { _ = library.IntegrationEnv(t) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() impersonationConfig := library.NewClientConfig(t) @@ -442,11 +431,3 @@ func TestWhoAmI_ImpersonateDirectly(t *testing.T) { whoAmIAnonymous, ) } - -func TestWhoAmI_ImpersonateViaProxy(t *testing.T) { - _ = library.IntegrationEnv(t) - - // TODO: add this test after the impersonation proxy is done - // this should test all forms of auth understood by the proxy (certs, SA token, token cred req, anonymous, etc) - // remember that impersonation does not support UID: https://github.com/kubernetes/kubernetes/issues/93699 -} diff --git a/test/library/access.go b/test/library/access.go index 7aacf6b17..46c31bd3f 100644 --- a/test/library/access.go +++ b/test/library/access.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package library @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + authorizationv1 "k8s.io/api/authorization/v1" v1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -21,7 +22,7 @@ import ( const ( accessRetryInterval = 250 * time.Millisecond - accessRetryTimeout = 10 * time.Second + accessRetryTimeout = 60 * time.Second ) // AccessAsUserTest runs a generic test in which a clientUnderTest operating with username @@ -137,6 +138,12 @@ func addTestClusterUserCanViewEverythingRoleBinding(t *testing.T, testUsername s Name: "view", }, ) + WaitForUserToHaveAccess(t, testUsername, []string{}, &authorizationv1.ResourceAttributes{ + Verb: "get", + Group: "", + Version: "v1", + Resource: "namespaces", + }) } func addTestClusterGroupCanViewEverythingRoleBinding(t *testing.T, testGroup string) { @@ -154,6 +161,12 @@ func addTestClusterGroupCanViewEverythingRoleBinding(t *testing.T, testGroup str Name: "view", }, ) + WaitForUserToHaveAccess(t, "", []string{testGroup}, &authorizationv1.ResourceAttributes{ + Verb: "get", + Group: "", + Version: "v1", + Resource: "namespaces", + }) } func runKubectlGetNamespaces(t *testing.T, kubeConfigYAML string) (string, error) { diff --git a/test/library/assertions.go b/test/library/assertions.go index c74cdd959..35c2278b1 100644 --- a/test/library/assertions.go +++ b/test/library/assertions.go @@ -5,6 +5,7 @@ package library import ( "context" + "errors" "fmt" "testing" "time" @@ -16,7 +17,7 @@ import ( "k8s.io/client-go/kubernetes" ) -// RequireEventuallyWithoutError is a wrapper around require.Eventually() that allows the caller to +// RequireEventuallyWithoutError is similar to require.Eventually() except that it also allows the caller to // return an error from the condition function. If the condition function returns an error at any // point, the assertion will immediately fail. func RequireEventuallyWithoutError( @@ -30,6 +31,28 @@ func RequireEventuallyWithoutError( require.NoError(t, wait.PollImmediate(tick, waitFor, f), msgAndArgs...) } +// RequireNeverWithoutError is similar to require.Never() except that it also allows the caller to +// return an error from the condition function. If the condition function returns an error at any +// point, the assertion will immediately fail. +func RequireNeverWithoutError( + t *testing.T, + f func() (bool, error), + waitFor time.Duration, + tick time.Duration, + msgAndArgs ...interface{}, +) { + t.Helper() + err := wait.PollImmediate(tick, waitFor, f) + isWaitTimeout := errors.Is(err, wait.ErrWaitTimeout) + if err != nil && !isWaitTimeout { + require.NoError(t, err, msgAndArgs...) // this will fail and throw the right error message + } + if err == nil { + // This prints the same error message that require.Never would print in this case. + require.Fail(t, "Condition satisfied", msgAndArgs...) + } +} + // assertNoRestartsDuringTest allows a caller to assert that there were no restarts for a Pod in the // provided namespace with the provided labelSelector during the lifetime of a test. func assertNoRestartsDuringTest(t *testing.T, namespace, labelSelector string) { diff --git a/test/library/cli.go b/test/library/cli.go index b6618686c..b27fbed06 100644 --- a/test/library/cli.go +++ b/test/library/cli.go @@ -10,6 +10,7 @@ import ( "path/filepath" "sync" "testing" + "time" "github.com/stretchr/testify/require" @@ -42,8 +43,10 @@ func PinnipedCLIPath(t *testing.T) string { } t.Log("building pinniped CLI binary") + start := time.Now() output, err := exec.Command("go", "build", "-o", path, "go.pinniped.dev/cmd/pinniped").CombinedOutput() require.NoError(t, err, string(output)) + t.Logf("built CLI binary in %s", time.Since(start).Round(time.Millisecond)) // Fill our cache so we don't have to do this again. pinnipedCLIBinaryCache.buf, err = ioutil.ReadFile(path) diff --git a/test/library/client.go b/test/library/client.go index 8e90c4243..789ffb2d2 100644 --- a/test/library/client.go +++ b/test/library/client.go @@ -17,6 +17,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + authorizationv1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -27,6 +28,7 @@ import ( aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" auth1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/authentication/v1alpha1" + "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1" configv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/config/v1alpha1" idpv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/idp/v1alpha1" conciergeclientset "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned" @@ -53,7 +55,9 @@ func NewClientsetForKubeConfig(t *testing.T, kubeConfig string) kubernetes.Inter func NewRestConfigFromKubeconfig(t *testing.T, kubeConfig string) *rest.Config { kubeConfigFile, err := ioutil.TempFile("", "pinniped-cli-test-*") require.NoError(t, err) - defer os.Remove(kubeConfigFile.Name()) + defer func() { + require.NoError(t, os.Remove(kubeConfigFile.Name())) + }() _, err = kubeConfigFile.Write([]byte(kubeConfig)) require.NoError(t, err) @@ -155,7 +159,7 @@ func CreateTestWebhookAuthenticator(ctx context.Context, t *testing.T) corev1.Ty client := NewConciergeClientset(t) webhooks := client.AuthenticationV1alpha1().WebhookAuthenticators() - createContext, cancel := context.WithTimeout(ctx, 5*time.Second) + createContext, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() webhook, err := webhooks.Create(createContext, &auth1alpha1.WebhookAuthenticator{ @@ -163,18 +167,12 @@ func CreateTestWebhookAuthenticator(ctx context.Context, t *testing.T) corev1.Ty Spec: testEnv.TestWebhook, }, metav1.CreateOptions{}) require.NoError(t, err, "could not create test WebhookAuthenticator") - t.Logf("created test WebhookAuthenticator %s/%s", webhook.Namespace, webhook.Name) + t.Logf("created test WebhookAuthenticator %s", webhook.Name) t.Cleanup(func() { t.Helper() - - if t.Failed() { - t.Logf("skipping deletion of test WebhookAuthenticator %s/%s", webhook.Namespace, webhook.Name) - return - } - - t.Logf("cleaning up test WebhookAuthenticator %s/%s", webhook.Namespace, webhook.Name) - deleteCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + t.Logf("cleaning up test WebhookAuthenticator %s", webhook.Name) + deleteCtx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() err := webhooks.Delete(deleteCtx, webhook.Name, metav1.DeleteOptions{}) require.NoErrorf(t, err, "could not cleanup test WebhookAuthenticator %s/%s", webhook.Namespace, webhook.Name) @@ -224,7 +222,7 @@ func CreateTestJWTAuthenticator(ctx context.Context, t *testing.T, spec auth1alp client := NewConciergeClientset(t) jwtAuthenticators := client.AuthenticationV1alpha1().JWTAuthenticators() - createContext, cancel := context.WithTimeout(ctx, 5*time.Second) + createContext, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() jwtAuthenticator, err := jwtAuthenticators.Create(createContext, &auth1alpha1.JWTAuthenticator{ @@ -232,21 +230,15 @@ func CreateTestJWTAuthenticator(ctx context.Context, t *testing.T, spec auth1alp Spec: spec, }, metav1.CreateOptions{}) require.NoError(t, err, "could not create test JWTAuthenticator") - t.Logf("created test JWTAuthenticator %s/%s", jwtAuthenticator.Namespace, jwtAuthenticator.Name) + t.Logf("created test JWTAuthenticator %s", jwtAuthenticator.Name) t.Cleanup(func() { t.Helper() - - if t.Failed() { - t.Logf("skipping deletion of test JWTAuthenticator %s/%s", jwtAuthenticator.Namespace, jwtAuthenticator.Name) - return - } - t.Logf("cleaning up test JWTAuthenticator %s/%s", jwtAuthenticator.Namespace, jwtAuthenticator.Name) - deleteCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + deleteCtx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() err := jwtAuthenticators.Delete(deleteCtx, jwtAuthenticator.Name, metav1.DeleteOptions{}) - require.NoErrorf(t, err, "could not cleanup test JWTAuthenticator %s/%s", jwtAuthenticator.Namespace, jwtAuthenticator.Name) + require.NoErrorf(t, err, "could not cleanup test JWTAuthenticator %s", jwtAuthenticator.Name) }) return corev1.TypedLocalObjectReference{ @@ -266,7 +258,7 @@ func CreateTestFederationDomain(ctx context.Context, t *testing.T, issuer string t.Helper() testEnv := IntegrationEnv(t) - createContext, cancel := context.WithTimeout(ctx, 5*time.Second) + createContext, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() if issuer == "" { @@ -287,7 +279,7 @@ func CreateTestFederationDomain(ctx context.Context, t *testing.T, issuer string t.Cleanup(func() { t.Helper() t.Logf("cleaning up test FederationDomain %s/%s", federationDomain.Namespace, federationDomain.Name) - deleteCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + deleteCtx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() err := federationDomains.Delete(deleteCtx, federationDomain.Name, metav1.DeleteOptions{}) notFound := k8serrors.IsNotFound(err) @@ -341,7 +333,7 @@ func RandHex(t *testing.T, numBytes int) string { func CreateTestSecret(t *testing.T, namespace string, baseName string, secretType corev1.SecretType, stringData map[string]string) *corev1.Secret { t.Helper() client := NewKubernetesClientset(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() created, err := client.CoreV1().Secrets(namespace).Create(ctx, &corev1.Secret{ @@ -412,7 +404,7 @@ func CreateTestOIDCIdentityProvider(t *testing.T, spec idpv1alpha1.OIDCIdentityP func CreateTestClusterRoleBinding(t *testing.T, subject rbacv1.Subject, roleRef rbacv1.RoleRef) *rbacv1.ClusterRoleBinding { t.Helper() client := NewKubernetesClientset(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() // Create the ClusterRoleBinding using GenerateName to get a random name. @@ -434,6 +426,69 @@ func CreateTestClusterRoleBinding(t *testing.T, subject rbacv1.Subject, roleRef return created } +func CreateTokenCredentialRequest(ctx context.Context, t *testing.T, spec v1alpha1.TokenCredentialRequestSpec) (*v1alpha1.TokenCredentialRequest, error) { + t.Helper() + + client := NewAnonymousConciergeClientset(t) + + ctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + + return client.LoginV1alpha1().TokenCredentialRequests().Create(ctx, + &v1alpha1.TokenCredentialRequest{Spec: spec}, metav1.CreateOptions{}, + ) +} + +func CreatePod(ctx context.Context, t *testing.T, name, namespace string, spec corev1.PodSpec) *corev1.Pod { + t.Helper() + + client := NewKubernetesClientset(t) + pods := client.CoreV1().Pods(namespace) + + ctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + + created, err := pods.Create(ctx, &corev1.Pod{ObjectMeta: testObjectMeta(t, name), Spec: spec}, metav1.CreateOptions{}) + require.NoError(t, err) + t.Logf("created test Pod %s", created.Name) + + t.Cleanup(func() { + t.Logf("cleaning up test Pod %s", created.Name) + err := pods.Delete(context.Background(), created.Name, metav1.DeleteOptions{}) + require.NoError(t, err) + }) + + var result *corev1.Pod + require.Eventuallyf(t, func() bool { + var err error + result, err = pods.Get(ctx, created.Name, metav1.GetOptions{}) + require.NoError(t, err) + return result.Status.Phase == corev1.PodRunning + }, 15*time.Second, 1*time.Second, "expected the Pod to go into phase %s", corev1.PodRunning) + return result +} + +func WaitForUserToHaveAccess(t *testing.T, user string, groups []string, shouldHaveAccessTo *authorizationv1.ResourceAttributes) { + t.Helper() + client := NewKubernetesClientset(t) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + RequireEventuallyWithoutError(t, func() (bool, error) { + subjectAccessReview, err := client.AuthorizationV1().SubjectAccessReviews().Create(ctx, + &authorizationv1.SubjectAccessReview{ + Spec: authorizationv1.SubjectAccessReviewSpec{ + ResourceAttributes: shouldHaveAccessTo, + User: user, + Groups: groups, + }}, metav1.CreateOptions{}) + if err != nil { + return false, err + } + return subjectAccessReview.Status.Allowed, nil + }, time.Minute, 500*time.Millisecond) +} + func testObjectMeta(t *testing.T, baseName string) metav1.ObjectMeta { return metav1.ObjectMeta{ GenerateName: fmt.Sprintf("test-%s-", baseName), diff --git a/test/library/dumplogs.go b/test/library/dumplogs.go index 86dfd2525..dc0bf89cd 100644 --- a/test/library/dumplogs.go +++ b/test/library/dumplogs.go @@ -25,7 +25,7 @@ func DumpLogs(t *testing.T, namespace string, labelSelector string) { } kubeClient := NewKubernetesClientset(t) - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() pods, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector}) diff --git a/test/library/env.go b/test/library/env.go index 97b24c220..a19aaa2e1 100644 --- a/test/library/env.go +++ b/test/library/env.go @@ -21,6 +21,7 @@ type Capability string const ( ClusterSigningKeyIsAvailable Capability = "clusterSigningKeyIsAvailable" AnonymousAuthenticationSupported Capability = "anonymousAuthenticationSupported" + HasExternalLoadBalancerProvider Capability = "hasExternalLoadBalancerProvider" ) // TestEnv captures all the external parameters consumed by our integration tests. diff --git a/test/library/iotest.go b/test/library/iotest.go index 6e2f1e58b..7ac175b3a 100644 --- a/test/library/iotest.go +++ b/test/library/iotest.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package library @@ -43,6 +43,10 @@ func MaskTokens(in string) string { if strings.Count(t, ".") >= 4 { return t } + // Another heuristic, things that start with "--" are probably CLI flags. + if strings.HasPrefix(t, "--") { + return t + } return fmt.Sprintf("[...%d bytes...]", len(t)) }) }