Merge branch 'main' into upstream-oidc-refresh-groups

internal/config/supervisor/config.go

@@ -18,6 +18,12 @@ import (
 	"go.pinniped.dev/internal/plog"
 )
 
+const (
+	NetworkDisabled = "disabled"
+	NetworkUnix     = "unix"
+	NetworkTCP      = "tcp"
+)
+
 // FromPath loads an Config from a provided local file path, inserts any
 // defaults (from the Config documentation), and verifies that the config is
 // valid (Config documentation).
@@ -50,9 +56,40 @@ func FromPath(path string) (*Config, error) {
 		return nil, fmt.Errorf("validate log level: %w", err)
 	}
 
+	// support setting this to null or {} or empty in the YAML
+	if config.Endpoints == nil {
+		config.Endpoints = &Endpoints{}
+	}
+
+	maybeSetEndpointDefault(&config.Endpoints.HTTPS, Endpoint{
+		Network: NetworkTCP,
+		Address: ":8443",
+	})
+	maybeSetEndpointDefault(&config.Endpoints.HTTP, Endpoint{
+		Network: NetworkTCP,
+		Address: ":8080",
+	})
+
+	if err := validateEndpoint(*config.Endpoints.HTTPS); err != nil {
+		return nil, fmt.Errorf("validate https endpoint: %w", err)
+	}
+	if err := validateEndpoint(*config.Endpoints.HTTP); err != nil {
+		return nil, fmt.Errorf("validate http endpoint: %w", err)
+	}
+	if err := validateAtLeastOneEnabledEndpoint(*config.Endpoints.HTTPS, *config.Endpoints.HTTP); err != nil {
+		return nil, fmt.Errorf("validate endpoints: %w", err)
+	}
+
 	return &config, nil
 }
 
+func maybeSetEndpointDefault(endpoint **Endpoint, defaultEndpoint Endpoint) {
+	if *endpoint != nil {
+		return
+	}
+	*endpoint = &defaultEndpoint
+}
+
 func maybeSetAPIGroupSuffixDefault(apiGroupSuffix **string) {
 	if *apiGroupSuffix == nil {
 		*apiGroupSuffix = pointer.StringPtr(groupsuffix.PinnipedDefaultSuffix)
@@ -73,3 +110,29 @@ func validateNames(names *NamesConfigSpec) error {
 	}
 	return nil
 }
+
+func validateEndpoint(endpoint Endpoint) error {
+	switch n := endpoint.Network; n {
+	case NetworkTCP, NetworkUnix:
+		if len(endpoint.Address) == 0 {
+			return fmt.Errorf("address must be set with %q network", n)
+		}
+		return nil
+	case NetworkDisabled:
+		if len(endpoint.Address) != 0 {
+			return fmt.Errorf("address set to %q when disabled, should be empty", endpoint.Address)
+		}
+		return nil
+	default:
+		return fmt.Errorf("unknown network %q", n)
+	}
+}
+
+func validateAtLeastOneEnabledEndpoint(endpoints ...Endpoint) error {
+	for _, endpoint := range endpoints {
+		if endpoint.Network != NetworkDisabled {
+			return nil
+		}
+	}
+	return constable.Error("all endpoints are disabled")
+}

internal/config/supervisor/config_test.go

@@ -32,6 +32,12 @@ func TestFromPath(t *testing.T) {
 				  myLabelKey2: myLabelValue2
 				names:
 				  defaultTLSCertificateSecret: my-secret-name
+				endpoints:
+				  https:
+				    network: unix
+				    address: :1234
+				  http:
+				    network: disabled
 			`),
 			wantConfig: &Config{
 				APIGroupSuffix: pointer.StringPtr("some.suffix.com"),
@@ -42,6 +48,15 @@ func TestFromPath(t *testing.T) {
 				NamesConfig: NamesConfigSpec{
 					DefaultTLSCertificateSecret: "my-secret-name",
 				},
+				Endpoints: &Endpoints{
+					HTTPS: &Endpoint{
+						Network: "unix",
+						Address: ":1234",
+					},
+					HTTP: &Endpoint{
+						Network: "disabled",
+					},
+				},
 			},
 		},
 		{
@@ -57,8 +72,97 @@ func TestFromPath(t *testing.T) {
 				NamesConfig: NamesConfigSpec{
 					DefaultTLSCertificateSecret: "my-secret-name",
 				},
+				Endpoints: &Endpoints{
+					HTTPS: &Endpoint{
+						Network: "tcp",
+						Address: ":8443",
+					},
+					HTTP: &Endpoint{
+						Network: "tcp",
+						Address: ":8080",
+					},
+				},
 			},
 		},
+		{
+			name: "all endpoints disabled",
+			yaml: here.Doc(`
+				---
+				names:
+				  defaultTLSCertificateSecret: my-secret-name
+				endpoints:
+				  https:
+				    network: disabled
+				  http:
+				    network: disabled
+			`),
+			wantError: "validate endpoints: all endpoints are disabled",
+		},
+		{
+			name: "invalid https endpoint",
+			yaml: here.Doc(`
+				---
+				names:
+				  defaultTLSCertificateSecret: my-secret-name
+				endpoints:
+				  https:
+				    network: foo
+				  http:
+				    network: disabled
+			`),
+			wantError: `validate https endpoint: unknown network "foo"`,
+		},
+		{
+			name: "invalid http endpoint",
+			yaml: here.Doc(`
+				---
+				names:
+				  defaultTLSCertificateSecret: my-secret-name
+				endpoints:
+				  https:
+				    network: disabled
+				  http:
+				    network: bar
+			`),
+			wantError: `validate http endpoint: unknown network "bar"`,
+		},
+		{
+			name: "endpoint disabled with non-empty address",
+			yaml: here.Doc(`
+				---
+				names:
+				  defaultTLSCertificateSecret: my-secret-name
+				endpoints:
+				  https:
+				    network: disabled
+				    address: wee
+			`),
+			wantError: `validate https endpoint: address set to "wee" when disabled, should be empty`,
+		},
+		{
+			name: "endpoint tcp with empty address",
+			yaml: here.Doc(`
+				---
+				names:
+				  defaultTLSCertificateSecret: my-secret-name
+				endpoints:
+				  http:
+				    network: tcp
+			`),
+			wantError: `validate http endpoint: address must be set with "tcp" network`,
+		},
+		{
+			name: "endpoint unix with empty address",
+			yaml: here.Doc(`
+				---
+				names:
+				  defaultTLSCertificateSecret: my-secret-name
+				endpoints:
+				  https:
+				    network: unix
+			`),
+			wantError: `validate https endpoint: address must be set with "unix" network`,
+		},
 		{
 			name: "Missing defaultTLSCertificateSecret name",
 			yaml: here.Doc(`

internal/config/supervisor/types.go

@@ -11,9 +11,20 @@ type Config struct {
 	Labels         map[string]string `json:"labels"`
 	NamesConfig    NamesConfigSpec   `json:"names"`
 	LogLevel       plog.LogLevel     `json:"logLevel"`
+	Endpoints      *Endpoints        `json:"endpoints"`
 }
 
 // NamesConfigSpec configures the names of some Kubernetes resources for the Supervisor.
 type NamesConfigSpec struct {
 	DefaultTLSCertificateSecret string `json:"defaultTLSCertificateSecret"`
 }
+
+type Endpoints struct {
+	HTTPS *Endpoint `json:"https,omitempty"`
+	HTTP  *Endpoint `json:"http,omitempty"`
+}
+
+type Endpoint struct {
+	Network string `json:"network"`
+	Address string `json:"address"`
+}

internal/controllermanager/prepare_controllers.go

@@ -101,7 +101,7 @@ type Config struct {
 func PrepareControllers(c *Config) (controllerinit.RunnerBuilder, error) {
 	loginConciergeGroupData, identityConciergeGroupData := groupsuffix.ConciergeAggregatedGroups(c.APIGroupSuffix)
 
-	dref, deployment, err := deploymentref.New(c.ServerInstallationInfo)
+	dref, deployment, _, err := deploymentref.New(c.ServerInstallationInfo)
 	if err != nil {
 		return nil, fmt.Errorf("cannot create deployment ref: %w", err)
 	}

internal/deploymentref/deploymentref.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 
@@ -31,24 +32,24 @@ var getTempClient = func() (kubernetes.Interface, error) {
 	return client.Kubernetes, nil
 }
 
-func New(podInfo *downward.PodInfo) (kubeclient.Option, *appsv1.Deployment, error) {
+func New(podInfo *downward.PodInfo) (kubeclient.Option, *appsv1.Deployment, *corev1.Pod, error) {
 	tempClient, err := getTempClient()
 	if err != nil {
-		return nil, nil, fmt.Errorf("cannot create temp client: %w", err)
+		return nil, nil, nil, fmt.Errorf("cannot create temp client: %w", err)
 	}
 
-	deployment, err := getDeployment(tempClient, podInfo)
+	deployment, pod, err := getDeploymentAndPod(tempClient, podInfo)
 	if err != nil {
-		return nil, nil, fmt.Errorf("cannot get deployment: %w", err)
+		return nil, nil, nil, fmt.Errorf("cannot get deployment: %w", err)
 	}
 
 	// work around stupid behavior of WithoutVersionDecoder.Decode
 	deployment.APIVersion, deployment.Kind = appsv1.SchemeGroupVersion.WithKind("Deployment").ToAPIVersionAndKind()
 
-	return kubeclient.WithMiddleware(ownerref.New(deployment)), deployment, nil
+	return kubeclient.WithMiddleware(ownerref.New(deployment)), deployment, pod, nil
 }
 
-func getDeployment(kubeClient kubernetes.Interface, podInfo *downward.PodInfo) (*appsv1.Deployment, error) {
+func getDeploymentAndPod(kubeClient kubernetes.Interface, podInfo *downward.PodInfo) (*appsv1.Deployment, *corev1.Pod, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 
@@ -56,28 +57,28 @@ func getDeployment(kubeClient kubernetes.Interface, podInfo *downward.PodInfo) (
 
 	pod, err := kubeClient.CoreV1().Pods(ns).Get(ctx, podInfo.Name, metav1.GetOptions{})
 	if err != nil {
-		return nil, fmt.Errorf("could not get pod: %w", err)
+		return nil, nil, fmt.Errorf("could not get pod: %w", err)
 	}
 
 	podOwner := metav1.GetControllerOf(pod)
 	if podOwner == nil {
-		return nil, fmt.Errorf("pod %s/%s is missing owner", ns, podInfo.Name)
+		return nil, nil, fmt.Errorf("pod %s/%s is missing owner", ns, podInfo.Name)
 	}
 
 	rs, err := kubeClient.AppsV1().ReplicaSets(ns).Get(ctx, podOwner.Name, metav1.GetOptions{})
 	if err != nil {
-		return nil, fmt.Errorf("could not get replicaset: %w", err)
+		return nil, nil, fmt.Errorf("could not get replicaset: %w", err)
 	}
 
 	rsOwner := metav1.GetControllerOf(rs)
 	if rsOwner == nil {
-		return nil, fmt.Errorf("replicaset %s/%s is missing owner", ns, podInfo.Name)
+		return nil, nil, fmt.Errorf("replicaset %s/%s is missing owner", ns, podInfo.Name)
 	}
 
 	d, err := kubeClient.AppsV1().Deployments(ns).Get(ctx, rsOwner.Name, metav1.GetOptions{})
 	if err != nil {
-		return nil, fmt.Errorf("could not get deployment: %w", err)
+		return nil, nil, fmt.Errorf("could not get deployment: %w", err)
 	}
 
-	return d, nil
+	return d, pod, nil
 }

internal/deploymentref/deploymentref_test.go

@@ -31,6 +31,18 @@ func TestNew(t *testing.T) {
 			Name:      "some-name",
 		},
 	}
+	goodPod := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "some-namespace",
+			Name:      "some-name-rsname-podhash",
+			OwnerReferences: []metav1.OwnerReference{
+				{
+					Controller: &troo,
+					Name:       "some-name-rsname",
+				},
+			},
+		},
+	}
 	tests := []struct {
 		name            string
 		apiObjects      []runtime.Object
@@ -38,6 +50,7 @@ func TestNew(t *testing.T) {
 		createClientErr error
 		podInfo         *downward.PodInfo
 		wantDeployment  *appsv1.Deployment
+		wantPod         *corev1.Pod
 		wantError       string
 	}{
 		{
@@ -56,24 +69,14 @@ func TestNew(t *testing.T) {
 						},
 					},
 				},
-				&corev1.Pod{
-					ObjectMeta: metav1.ObjectMeta{
-						Namespace: "some-namespace",
-						Name:      "some-name-rsname-podhash",
-						OwnerReferences: []metav1.OwnerReference{
-							{
-								Controller: &troo,
-								Name:       "some-name-rsname",
-							},
-						},
-					},
-				},
+				goodPod,
 			},
 			podInfo: &downward.PodInfo{
 				Namespace: "some-namespace",
 				Name:      "some-name-rsname-podhash",
 			},
 			wantDeployment: goodDeployment,
+			wantPod:        goodPod,
 		},
 		{
 			name:            "failed to create client",
@@ -114,7 +117,7 @@ func TestNew(t *testing.T) {
 				return client, test.createClientErr
 			}
 
-			_, d, err := New(test.podInfo)
+			_, d, p, err := New(test.podInfo)
 			if test.wantError != "" {
 				require.EqualError(t, err, test.wantError)
 				return
@@ -122,6 +125,7 @@ func TestNew(t *testing.T) {
 
 			require.NoError(t, err)
 			require.Equal(t, test.wantDeployment, d)
+			require.Equal(t, test.wantPod, p)
 		})
 	}
 }

internal/supervisor/server/bootstrap.go

@@ -0,0 +1,66 @@
+// Copyright 2021 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package server
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+
+	"go.uber.org/atomic"
+	"k8s.io/apimachinery/pkg/util/sets"
+
+	"go.pinniped.dev/internal/certauthority"
+)
+
+// contextKey type is unexported to prevent collisions.
+type contextKey int
+
+const bootstrapKey contextKey = iota
+
+func withBootstrapConnCtx(ctx context.Context, _ net.Conn) context.Context {
+	isBootstrap := atomic.NewBool(false) // safe for concurrent access
+	return context.WithValue(ctx, bootstrapKey, isBootstrap)
+}
+
+func setIsBootstrapConn(ctx context.Context) {
+	isBootstrap, _ := ctx.Value(bootstrapKey).(*atomic.Bool)
+	if isBootstrap == nil {
+		return
+	}
+	isBootstrap.Store(true)
+}
+
+func withBootstrapPaths(handler http.Handler, paths ...string) http.Handler {
+	bootstrapPaths := sets.NewString(paths...)
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		isBootstrap, _ := req.Context().Value(bootstrapKey).(*atomic.Bool)
+
+		if isBootstrap != nil && isBootstrap.Load() && !bootstrapPaths.Has(req.URL.Path) {
+			http.Error(w, "pinniped supervisor has invalid TLS serving certificate configuration", http.StatusInternalServerError)
+			return
+		}
+
+		handler.ServeHTTP(w, req)
+	})
+}
+
+func getBootstrapCert() (*tls.Certificate, error) {
+	const forever = 10 * 365 * 24 * time.Hour
+
+	bootstrapCA, err := certauthority.New("pinniped-supervisor-bootstrap-ca", forever)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create bootstrap CA: %w", err)
+	}
+
+	bootstrapCert, err := bootstrapCA.IssueServerCert([]string{"pinniped-supervisor-bootstrap-cert"}, nil, forever)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create bootstrap cert: %w", err)
+	}
+
+	return bootstrapCert, nil // this is just enough to complete a TLS handshake, trust distribution does not matter
+}

internal/supervisor/server/server.go

@@ -13,11 +13,13 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"strconv"
 	"strings"
 	"sync"
 	"syscall"
 	"time"
 
+	"github.com/joshlf/go-acl"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
@@ -61,7 +63,13 @@ const (
 )
 
 func startServer(ctx context.Context, shutdown *sync.WaitGroup, l net.Listener, handler http.Handler) {
-	server := http.Server{Handler: genericapifilters.WithWarningRecorder(handler)}
+	handler = genericapifilters.WithWarningRecorder(handler)
+	handler = withBootstrapPaths(handler, "/healthz") // only health checks are allowed for bootstrap connections
+
+	server := http.Server{
+		Handler:     handler,
+		ConnContext: withBootstrapConnCtx,
+	}
 
 	shutdown.Add(1)
 	go func() {
@@ -306,10 +314,11 @@ func startControllers(ctx context.Context, shutdown *sync.WaitGroup, buildContro
 	return nil
 }
 
+//nolint:funlen
 func runSupervisor(podInfo *downward.PodInfo, cfg *supervisor.Config) error {
 	serverInstallationNamespace := podInfo.Namespace
 
-	dref, supervisorDeployment, err := deploymentref.New(podInfo)
+	dref, supervisorDeployment, supervisorPod, err := deploymentref.New(podInfo)
 	if err != nil {
 		return fmt.Errorf("cannot create deployment ref: %w", err)
 	}
@@ -387,40 +396,76 @@ func runSupervisor(podInfo *downward.PodInfo, cfg *supervisor.Config) error {
 		return err
 	}
 
-	//nolint: gosec // Intentionally binding to all network interfaces.
-	httpListener, err := net.Listen("tcp", ":8080")
-	if err != nil {
-		return fmt.Errorf("cannot create listener: %w", err)
-	}
-	defer func() { _ = httpListener.Close() }()
-	startServer(ctx, shutdown, httpListener, oidProvidersManager)
+	if e := cfg.Endpoints.HTTP; e.Network != supervisor.NetworkDisabled {
+		finishSetupPerms := maybeSetupUnixPerms(e, supervisorPod)
 
-	c := ptls.Default(nil)
-	c.GetCertificate = func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
-		cert := dynamicTLSCertProvider.GetTLSCert(strings.ToLower(info.ServerName))
-		defaultCert := dynamicTLSCertProvider.GetDefaultTLSCert()
-		plog.Debug("GetCertificate called for port 8443",
-			"info.ServerName", info.ServerName,
-			"foundSNICert", cert != nil,
-			"foundDefaultCert", defaultCert != nil,
-		)
-		if cert == nil {
-			cert = defaultCert
+		httpListener, err := net.Listen(e.Network, e.Address)
+		if err != nil {
+			return fmt.Errorf("cannot create http listener with network %q and address %q: %w", e.Network, e.Address, err)
 		}
-		return cert, nil
-	}
-	//nolint: gosec // Intentionally binding to all network interfaces.
-	httpsListener, err := tls.Listen("tcp", ":8443", c)
-	if err != nil {
-		return fmt.Errorf("cannot create listener: %w", err)
-	}
-	defer func() { _ = httpsListener.Close() }()
-	startServer(ctx, shutdown, httpsListener, oidProvidersManager)
 
-	plog.Debug("supervisor is ready",
-		"httpAddress", httpListener.Addr().String(),
-		"httpsAddress", httpsListener.Addr().String(),
-	)
+		if err := finishSetupPerms(); err != nil {
+			return fmt.Errorf("cannot setup http listener permissions for network %q and address %q: %w", e.Network, e.Address, err)
+		}
+
+		defer func() { _ = httpListener.Close() }()
+		startServer(ctx, shutdown, httpListener, oidProvidersManager)
+		plog.Debug("supervisor http listener started", "address", httpListener.Addr().String())
+	}
+
+	if e := cfg.Endpoints.HTTPS; e.Network != supervisor.NetworkDisabled { //nolint:nestif
+		finishSetupPerms := maybeSetupUnixPerms(e, supervisorPod)
+
+		bootstrapCert, err := getBootstrapCert() // generate this in-memory once per process startup
+		if err != nil {
+			return fmt.Errorf("https listener bootstrap error: %w", err)
+		}
+
+		c := ptls.Default(nil)
+		c.GetCertificate = func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
+			cert := dynamicTLSCertProvider.GetTLSCert(strings.ToLower(info.ServerName))
+
+			defaultCert := dynamicTLSCertProvider.GetDefaultTLSCert()
+
+			if plog.Enabled(plog.LevelTrace) { // minor CPU optimization as this is generally just noise
+				host, port, _ := net.SplitHostPort(info.Conn.LocalAddr().String()) // error is safe to ignore here
+
+				plog.Trace("GetCertificate called",
+					"info.ServerName", info.ServerName,
+					"foundSNICert", cert != nil,
+					"foundDefaultCert", defaultCert != nil,
+					"host", host,
+					"port", port,
+				)
+			}
+
+			if cert == nil {
+				cert = defaultCert
+			}
+
+			if cert == nil {
+				setIsBootstrapConn(info.Context()) // make this connection only work for bootstrap requests
+				cert = bootstrapCert
+			}
+
+			return cert, nil
+		}
+
+		httpsListener, err := tls.Listen(e.Network, e.Address, c)
+		if err != nil {
+			return fmt.Errorf("cannot create https listener with network %q and address %q: %w", e.Network, e.Address, err)
+		}
+
+		if err := finishSetupPerms(); err != nil {
+			return fmt.Errorf("cannot setup https listener permissions for network %q and address %q: %w", e.Network, e.Address, err)
+		}
+
+		defer func() { _ = httpsListener.Close() }()
+		startServer(ctx, shutdown, httpsListener, oidProvidersManager)
+		plog.Debug("supervisor https listener started", "address", httpsListener.Addr().String())
+	}
+
+	plog.Debug("supervisor started")
 	defer plog.Debug("supervisor exiting")
 
 	shutdown.Wait()
@@ -428,6 +473,37 @@ func runSupervisor(podInfo *downward.PodInfo, cfg *supervisor.Config) error {
 	return nil
 }
 
+func maybeSetupUnixPerms(endpoint *supervisor.Endpoint, pod *corev1.Pod) func() error {
+	if endpoint.Network != supervisor.NetworkUnix {
+		return func() error { return nil }
+	}
+
+	_ = os.Remove(endpoint.Address) // empty dir volumes persist across container crashes
+
+	return func() error {
+		selfUser := int64(os.Getuid())
+		var entries []acl.Entry
+		for _, container := range pod.Spec.Containers {
+			if container.SecurityContext == nil ||
+				container.SecurityContext.RunAsUser == nil ||
+				*container.SecurityContext.RunAsUser == selfUser {
+				continue
+			}
+
+			plog.Debug("adding write permission",
+				"address", endpoint.Address,
+				"uid", *container.SecurityContext.RunAsUser,
+			)
+			entries = append(entries, acl.Entry{
+				Tag:       acl.TagUser,
+				Qualifier: strconv.FormatInt(*container.SecurityContext.RunAsUser, 10),
+				Perms:     2, // write permission
+			})
+		}
+		return acl.Add(endpoint.Address, entries...) // allow all containers in the pod to write to the socket
+	}
+}
+
 func main() error { // return an error instead of klog.Fatal to allow defer statements to run
 	logs.InitLogs()
 	defer logs.FlushLogs()