Allow configuration of supervisor endpoints

This change allows configuration of the http and https listeners
used by the supervisor.

TCP (IPv4 and IPv6 with any interface and port) and Unix domain
socket based listeners are supported.  Listeners may also be
disabled.

Binding the http listener to TCP addresses other than 127.0.0.1 or
::1 is deprecated.

The deployment now uses https health checks.  The supervisor is
always able to complete a TLS connection with the use of a bootstrap
certificate that is signed by an in-memory certificate authority.

To support sidecar containers used by service meshes, Unix domain
socket based listeners include ACLs that allow writes to the socket
file from any runAsUser specified in the pod's containers.

Signed-off-by: Monis Khan <mok@vmware.com>

internal/config/supervisor/config.go

@@ -18,6 +18,12 @@ import (
 	"go.pinniped.dev/internal/plog"
 )
 
+const (
+	NetworkDisabled = "disabled"
+	NetworkUnix     = "unix"
+	NetworkTCP      = "tcp"
+)
+
 // FromPath loads an Config from a provided local file path, inserts any
 // defaults (from the Config documentation), and verifies that the config is
 // valid (Config documentation).
@@ -50,9 +56,40 @@ func FromPath(path string) (*Config, error) {
 		return nil, fmt.Errorf("validate log level: %w", err)
 	}
 
+	// support setting this to null or {} or empty in the YAML
+	if config.Endpoints == nil {
+		config.Endpoints = &Endpoints{}
+	}
+
+	maybeSetEndpointDefault(&config.Endpoints.HTTPS, Endpoint{
+		Network: NetworkTCP,
+		Address: ":8443",
+	})
+	maybeSetEndpointDefault(&config.Endpoints.HTTP, Endpoint{
+		Network: NetworkTCP,
+		Address: ":8080",
+	})
+
+	if err := validateEndpoint(*config.Endpoints.HTTPS); err != nil {
+		return nil, fmt.Errorf("validate https endpoint: %w", err)
+	}
+	if err := validateEndpoint(*config.Endpoints.HTTP); err != nil {
+		return nil, fmt.Errorf("validate http endpoint: %w", err)
+	}
+	if err := validateAtLeastOneEnabledEndpoint(*config.Endpoints.HTTPS, *config.Endpoints.HTTP); err != nil {
+		return nil, fmt.Errorf("validate endpoints: %w", err)
+	}
+
 	return &config, nil
 }
 
+func maybeSetEndpointDefault(endpoint **Endpoint, defaultEndpoint Endpoint) {
+	if *endpoint != nil {
+		return
+	}
+	*endpoint = &defaultEndpoint
+}
+
 func maybeSetAPIGroupSuffixDefault(apiGroupSuffix **string) {
 	if *apiGroupSuffix == nil {
 		*apiGroupSuffix = pointer.StringPtr(groupsuffix.PinnipedDefaultSuffix)
@@ -73,3 +110,29 @@ func validateNames(names *NamesConfigSpec) error {
 	}
 	return nil
 }
+
+func validateEndpoint(endpoint Endpoint) error {
+	switch n := endpoint.Network; n {
+	case NetworkTCP, NetworkUnix:
+		if len(endpoint.Address) == 0 {
+			return fmt.Errorf("address must be set with %q network", n)
+		}
+		return nil
+	case NetworkDisabled:
+		if len(endpoint.Address) != 0 {
+			return fmt.Errorf("address set to %q when disabled, should be empty", endpoint.Address)
+		}
+		return nil
+	default:
+		return fmt.Errorf("unknown network %q", n)
+	}
+}
+
+func validateAtLeastOneEnabledEndpoint(endpoints ...Endpoint) error {
+	for _, endpoint := range endpoints {
+		if endpoint.Network != NetworkDisabled {
+			return nil
+		}
+	}
+	return constable.Error("all endpoints are disabled")
+}

internal/config/supervisor/config_test.go

@@ -32,6 +32,12 @@ func TestFromPath(t *testing.T) {
 				  myLabelKey2: myLabelValue2
 				names:
 				  defaultTLSCertificateSecret: my-secret-name
+				endpoints:
+				  https:
+				    network: unix
+				    address: :1234
+				  http:
+				    network: disabled
 			`),
 			wantConfig: &Config{
 				APIGroupSuffix: pointer.StringPtr("some.suffix.com"),
@@ -42,6 +48,15 @@ func TestFromPath(t *testing.T) {
 				NamesConfig: NamesConfigSpec{
 					DefaultTLSCertificateSecret: "my-secret-name",
 				},
+				Endpoints: &Endpoints{
+					HTTPS: &Endpoint{
+						Network: "unix",
+						Address: ":1234",
+					},
+					HTTP: &Endpoint{
+						Network: "disabled",
+					},
+				},
 			},
 		},
 		{
@@ -57,8 +72,97 @@ func TestFromPath(t *testing.T) {
 				NamesConfig: NamesConfigSpec{
 					DefaultTLSCertificateSecret: "my-secret-name",
 				},
+				Endpoints: &Endpoints{
+					HTTPS: &Endpoint{
+						Network: "tcp",
+						Address: ":8443",
+					},
+					HTTP: &Endpoint{
+						Network: "tcp",
+						Address: ":8080",
+					},
+				},
 			},
 		},
+		{
+			name: "all endpoints disabled",
+			yaml: here.Doc(`
+				---
+				names:
+				  defaultTLSCertificateSecret: my-secret-name
+				endpoints:
+				  https:
+				    network: disabled
+				  http:
+				    network: disabled
+			`),
+			wantError: "validate endpoints: all endpoints are disabled",
+		},
+		{
+			name: "invalid https endpoint",
+			yaml: here.Doc(`
+				---
+				names:
+				  defaultTLSCertificateSecret: my-secret-name
+				endpoints:
+				  https:
+				    network: foo
+				  http:
+				    network: disabled
+			`),
+			wantError: `validate https endpoint: unknown network "foo"`,
+		},
+		{
+			name: "invalid http endpoint",
+			yaml: here.Doc(`
+				---
+				names:
+				  defaultTLSCertificateSecret: my-secret-name
+				endpoints:
+				  https:
+				    network: disabled
+				  http:
+				    network: bar
+			`),
+			wantError: `validate http endpoint: unknown network "bar"`,
+		},
+		{
+			name: "endpoint disabled with non-empty address",
+			yaml: here.Doc(`
+				---
+				names:
+				  defaultTLSCertificateSecret: my-secret-name
+				endpoints:
+				  https:
+				    network: disabled
+				    address: wee
+			`),
+			wantError: `validate https endpoint: address set to "wee" when disabled, should be empty`,
+		},
+		{
+			name: "endpoint tcp with empty address",
+			yaml: here.Doc(`
+				---
+				names:
+				  defaultTLSCertificateSecret: my-secret-name
+				endpoints:
+				  http:
+				    network: tcp
+			`),
+			wantError: `validate http endpoint: address must be set with "tcp" network`,
+		},
+		{
+			name: "endpoint unix with empty address",
+			yaml: here.Doc(`
+				---
+				names:
+				  defaultTLSCertificateSecret: my-secret-name
+				endpoints:
+				  https:
+				    network: unix
+			`),
+			wantError: `validate https endpoint: address must be set with "unix" network`,
+		},
 		{
 			name: "Missing defaultTLSCertificateSecret name",
 			yaml: here.Doc(`

internal/config/supervisor/types.go

@@ -11,9 +11,20 @@ type Config struct {
 	Labels         map[string]string `json:"labels"`
 	NamesConfig    NamesConfigSpec   `json:"names"`
 	LogLevel       plog.LogLevel     `json:"logLevel"`
+	Endpoints      *Endpoints        `json:"endpoints"`
 }
 
 // NamesConfigSpec configures the names of some Kubernetes resources for the Supervisor.
 type NamesConfigSpec struct {
 	DefaultTLSCertificateSecret string `json:"defaultTLSCertificateSecret"`
 }
+
+type Endpoints struct {
+	HTTPS *Endpoint `json:"https,omitempty"`
+	HTTP  *Endpoint `json:"http,omitempty"`
+}
+
+type Endpoint struct {
+	Network string `json:"network"`
+	Address string `json:"address"`
+}

internal/controllermanager/prepare_controllers.go

@@ -101,7 +101,7 @@ type Config struct {
 func PrepareControllers(c *Config) (controllerinit.RunnerBuilder, error) {
 	loginConciergeGroupData, identityConciergeGroupData := groupsuffix.ConciergeAggregatedGroups(c.APIGroupSuffix)
 
-	dref, deployment, err := deploymentref.New(c.ServerInstallationInfo)
+	dref, deployment, _, err := deploymentref.New(c.ServerInstallationInfo)
 	if err != nil {
 		return nil, fmt.Errorf("cannot create deployment ref: %w", err)
 	}

internal/deploymentref/deploymentref.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 
@@ -31,24 +32,24 @@ var getTempClient = func() (kubernetes.Interface, error) {
 	return client.Kubernetes, nil
 }
 
-func New(podInfo *downward.PodInfo) (kubeclient.Option, *appsv1.Deployment, error) {
+func New(podInfo *downward.PodInfo) (kubeclient.Option, *appsv1.Deployment, *corev1.Pod, error) {
 	tempClient, err := getTempClient()
 	if err != nil {
-		return nil, nil, fmt.Errorf("cannot create temp client: %w", err)
+		return nil, nil, nil, fmt.Errorf("cannot create temp client: %w", err)
 	}
 
-	deployment, err := getDeployment(tempClient, podInfo)
+	deployment, pod, err := getDeploymentAndPod(tempClient, podInfo)
 	if err != nil {
-		return nil, nil, fmt.Errorf("cannot get deployment: %w", err)
+		return nil, nil, nil, fmt.Errorf("cannot get deployment: %w", err)
 	}
 
 	// work around stupid behavior of WithoutVersionDecoder.Decode
 	deployment.APIVersion, deployment.Kind = appsv1.SchemeGroupVersion.WithKind("Deployment").ToAPIVersionAndKind()
 
-	return kubeclient.WithMiddleware(ownerref.New(deployment)), deployment, nil
+	return kubeclient.WithMiddleware(ownerref.New(deployment)), deployment, pod, nil
 }
 
-func getDeployment(kubeClient kubernetes.Interface, podInfo *downward.PodInfo) (*appsv1.Deployment, error) {
+func getDeploymentAndPod(kubeClient kubernetes.Interface, podInfo *downward.PodInfo) (*appsv1.Deployment, *corev1.Pod, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 
@@ -56,28 +57,28 @@ func getDeployment(kubeClient kubernetes.Interface, podInfo *downward.PodInfo) (
 
 	pod, err := kubeClient.CoreV1().Pods(ns).Get(ctx, podInfo.Name, metav1.GetOptions{})
 	if err != nil {
-		return nil, fmt.Errorf("could not get pod: %w", err)
+		return nil, nil, fmt.Errorf("could not get pod: %w", err)
 	}
 
 	podOwner := metav1.GetControllerOf(pod)
 	if podOwner == nil {
-		return nil, fmt.Errorf("pod %s/%s is missing owner", ns, podInfo.Name)
+		return nil, nil, fmt.Errorf("pod %s/%s is missing owner", ns, podInfo.Name)
 	}
 
 	rs, err := kubeClient.AppsV1().ReplicaSets(ns).Get(ctx, podOwner.Name, metav1.GetOptions{})
 	if err != nil {
-		return nil, fmt.Errorf("could not get replicaset: %w", err)
+		return nil, nil, fmt.Errorf("could not get replicaset: %w", err)
 	}
 
 	rsOwner := metav1.GetControllerOf(rs)
 	if rsOwner == nil {
-		return nil, fmt.Errorf("replicaset %s/%s is missing owner", ns, podInfo.Name)
+		return nil, nil, fmt.Errorf("replicaset %s/%s is missing owner", ns, podInfo.Name)
 	}
 
 	d, err := kubeClient.AppsV1().Deployments(ns).Get(ctx, rsOwner.Name, metav1.GetOptions{})
 	if err != nil {
-		return nil, fmt.Errorf("could not get deployment: %w", err)
+		return nil, nil, fmt.Errorf("could not get deployment: %w", err)
 	}
 
-	return d, nil
+	return d, pod, nil
 }

internal/deploymentref/deploymentref_test.go

@@ -31,6 +31,18 @@ func TestNew(t *testing.T) {
 			Name:      "some-name",
 		},
 	}
+	goodPod := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "some-namespace",
+			Name:      "some-name-rsname-podhash",
+			OwnerReferences: []metav1.OwnerReference{
+				{
+					Controller: &troo,
+					Name:       "some-name-rsname",
+				},
+			},
+		},
+	}
 	tests := []struct {
 		name            string
 		apiObjects      []runtime.Object
@@ -38,6 +50,7 @@ func TestNew(t *testing.T) {
 		createClientErr error
 		podInfo         *downward.PodInfo
 		wantDeployment  *appsv1.Deployment
+		wantPod         *corev1.Pod
 		wantError       string
 	}{
 		{
@@ -56,24 +69,14 @@ func TestNew(t *testing.T) {
 						},
 					},
 				},
-				&corev1.Pod{
-					ObjectMeta: metav1.ObjectMeta{
-						Namespace: "some-namespace",
-						Name:      "some-name-rsname-podhash",
-						OwnerReferences: []metav1.OwnerReference{
-							{
-								Controller: &troo,
-								Name:       "some-name-rsname",
-							},
-						},
-					},
-				},
+				goodPod,
 			},
 			podInfo: &downward.PodInfo{
 				Namespace: "some-namespace",
 				Name:      "some-name-rsname-podhash",
 			},
 			wantDeployment: goodDeployment,
+			wantPod:        goodPod,
 		},
 		{
 			name:            "failed to create client",
@@ -114,7 +117,7 @@ func TestNew(t *testing.T) {
 				return client, test.createClientErr
 			}
 
-			_, d, err := New(test.podInfo)
+			_, d, p, err := New(test.podInfo)
 			if test.wantError != "" {
 				require.EqualError(t, err, test.wantError)
 				return
@@ -122,6 +125,7 @@ func TestNew(t *testing.T) {
 
 			require.NoError(t, err)
 			require.Equal(t, test.wantDeployment, d)
+			require.Equal(t, test.wantPod, p)
 		})
 	}
 }

internal/supervisor/server/bootstrap.go

@@ -0,0 +1,66 @@
+// Copyright 2021 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package server
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+
+	"go.uber.org/atomic"
+	"k8s.io/apimachinery/pkg/util/sets"
+
+	"go.pinniped.dev/internal/certauthority"
+)
+
+// contextKey type is unexported to prevent collisions.
+type contextKey int
+
+const bootstrapKey contextKey = iota
+
+func withBootstrapConnCtx(ctx context.Context, _ net.Conn) context.Context {
+	isBootstrap := atomic.NewBool(false) // safe for concurrent access
+	return context.WithValue(ctx, bootstrapKey, isBootstrap)
+}
+
+func setIsBootstrapConn(ctx context.Context) {
+	isBootstrap, _ := ctx.Value(bootstrapKey).(*atomic.Bool)
+	if isBootstrap == nil {
+		return
+	}
+	isBootstrap.Store(true)
+}
+
+func withBootstrapPaths(handler http.Handler, paths ...string) http.Handler {
+	bootstrapPaths := sets.NewString(paths...)
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		isBootstrap, _ := req.Context().Value(bootstrapKey).(*atomic.Bool)
+
+		if isBootstrap != nil && isBootstrap.Load() && !bootstrapPaths.Has(req.URL.Path) {
+			http.Error(w, "pinniped supervisor has invalid TLS serving certificate configuration", http.StatusInternalServerError)
+			return
+		}
+
+		handler.ServeHTTP(w, req)
+	})
+}
+
+func getBootstrapCert() (*tls.Certificate, error) {
+	const forever = 10 * 365 * 24 * time.Hour
+
+	bootstrapCA, err := certauthority.New("pinniped-supervisor-bootstrap-ca", forever)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create bootstrap CA: %w", err)
+	}
+
+	bootstrapCert, err := bootstrapCA.IssueServerCert([]string{"pinniped-supervisor-bootstrap-cert"}, nil, forever)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create bootstrap cert: %w", err)
+	}
+
+	return bootstrapCert, nil // this is just enough to complete a TLS handshake, trust distribution does not matter
+}

internal/supervisor/server/server.go

@@ -13,11 +13,13 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"strconv"
 	"strings"
 	"sync"
 	"syscall"
 	"time"
 
+	"github.com/joshlf/go-acl"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
@@ -61,7 +63,13 @@ const (
 )
 
 func startServer(ctx context.Context, shutdown *sync.WaitGroup, l net.Listener, handler http.Handler) {
-	server := http.Server{Handler: genericapifilters.WithWarningRecorder(handler)}
+	handler = genericapifilters.WithWarningRecorder(handler)
+	handler = withBootstrapPaths(handler, "/healthz") // only health checks are allowed for bootstrap connections
+
+	server := http.Server{
+		Handler:     handler,
+		ConnContext: withBootstrapConnCtx,
+	}
 
 	shutdown.Add(1)
 	go func() {
@@ -306,10 +314,11 @@ func startControllers(ctx context.Context, shutdown *sync.WaitGroup, buildContro
 	return nil
 }
 
+//nolint:funlen
 func runSupervisor(podInfo *downward.PodInfo, cfg *supervisor.Config) error {
 	serverInstallationNamespace := podInfo.Namespace
 
-	dref, supervisorDeployment, err := deploymentref.New(podInfo)
+	dref, supervisorDeployment, supervisorPod, err := deploymentref.New(podInfo)
 	if err != nil {
 		return fmt.Errorf("cannot create deployment ref: %w", err)
 	}
@@ -387,40 +396,76 @@ func runSupervisor(podInfo *downward.PodInfo, cfg *supervisor.Config) error {
 		return err
 	}
 
-	//nolint: gosec // Intentionally binding to all network interfaces.
-	httpListener, err := net.Listen("tcp", ":8080")
-	if err != nil {
-		return fmt.Errorf("cannot create listener: %w", err)
-	}
-	defer func() { _ = httpListener.Close() }()
-	startServer(ctx, shutdown, httpListener, oidProvidersManager)
+	if e := cfg.Endpoints.HTTP; e.Network != supervisor.NetworkDisabled {
+		finishSetupPerms := maybeSetupUnixPerms(e, supervisorPod)
 
-	c := ptls.Default(nil)
-	c.GetCertificate = func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
-		cert := dynamicTLSCertProvider.GetTLSCert(strings.ToLower(info.ServerName))
-		defaultCert := dynamicTLSCertProvider.GetDefaultTLSCert()
-		plog.Debug("GetCertificate called for port 8443",
-			"info.ServerName", info.ServerName,
-			"foundSNICert", cert != nil,
-			"foundDefaultCert", defaultCert != nil,
-		)
-		if cert == nil {
-			cert = defaultCert
+		httpListener, err := net.Listen(e.Network, e.Address)
+		if err != nil {
+			return fmt.Errorf("cannot create http listener with network %q and address %q: %w", e.Network, e.Address, err)
 		}
-		return cert, nil
-	}
-	//nolint: gosec // Intentionally binding to all network interfaces.
-	httpsListener, err := tls.Listen("tcp", ":8443", c)
-	if err != nil {
-		return fmt.Errorf("cannot create listener: %w", err)
-	}
-	defer func() { _ = httpsListener.Close() }()
-	startServer(ctx, shutdown, httpsListener, oidProvidersManager)
 
-	plog.Debug("supervisor is ready",
-		"httpAddress", httpListener.Addr().String(),
-		"httpsAddress", httpsListener.Addr().String(),
-	)
+		if err := finishSetupPerms(); err != nil {
+			return fmt.Errorf("cannot setup http listener permissions for network %q and address %q: %w", e.Network, e.Address, err)
+		}
+
+		defer func() { _ = httpListener.Close() }()
+		startServer(ctx, shutdown, httpListener, oidProvidersManager)
+		plog.Debug("supervisor http listener started", "address", httpListener.Addr().String())
+	}
+
+	if e := cfg.Endpoints.HTTPS; e.Network != supervisor.NetworkDisabled { //nolint:nestif
+		finishSetupPerms := maybeSetupUnixPerms(e, supervisorPod)
+
+		bootstrapCert, err := getBootstrapCert() // generate this in-memory once per process startup
+		if err != nil {
+			return fmt.Errorf("https listener bootstrap error: %w", err)
+		}
+
+		c := ptls.Default(nil)
+		c.GetCertificate = func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
+			cert := dynamicTLSCertProvider.GetTLSCert(strings.ToLower(info.ServerName))
+
+			defaultCert := dynamicTLSCertProvider.GetDefaultTLSCert()
+
+			if plog.Enabled(plog.LevelTrace) { // minor CPU optimization as this is generally just noise
+				host, port, _ := net.SplitHostPort(info.Conn.LocalAddr().String()) // error is safe to ignore here
+
+				plog.Trace("GetCertificate called",
+					"info.ServerName", info.ServerName,
+					"foundSNICert", cert != nil,
+					"foundDefaultCert", defaultCert != nil,
+					"host", host,
+					"port", port,
+				)
+			}
+
+			if cert == nil {
+				cert = defaultCert
+			}
+
+			if cert == nil {
+				setIsBootstrapConn(info.Context()) // make this connection only work for bootstrap requests
+				cert = bootstrapCert
+			}
+
+			return cert, nil
+		}
+
+		httpsListener, err := tls.Listen(e.Network, e.Address, c)
+		if err != nil {
+			return fmt.Errorf("cannot create https listener with network %q and address %q: %w", e.Network, e.Address, err)
+		}
+
+		if err := finishSetupPerms(); err != nil {
+			return fmt.Errorf("cannot setup https listener permissions for network %q and address %q: %w", e.Network, e.Address, err)
+		}
+
+		defer func() { _ = httpsListener.Close() }()
+		startServer(ctx, shutdown, httpsListener, oidProvidersManager)
+		plog.Debug("supervisor https listener started", "address", httpsListener.Addr().String())
+	}
+
+	plog.Debug("supervisor started")
 	defer plog.Debug("supervisor exiting")
 
 	shutdown.Wait()
@@ -428,6 +473,37 @@ func runSupervisor(podInfo *downward.PodInfo, cfg *supervisor.Config) error {
 	return nil
 }
 
+func maybeSetupUnixPerms(endpoint *supervisor.Endpoint, pod *corev1.Pod) func() error {
+	if endpoint.Network != supervisor.NetworkUnix {
+		return func() error { return nil }
+	}
+
+	_ = os.Remove(endpoint.Address) // empty dir volumes persist across container crashes
+
+	return func() error {
+		selfUser := int64(os.Getuid())
+		var entries []acl.Entry
+		for _, container := range pod.Spec.Containers {
+			if container.SecurityContext == nil ||
+				container.SecurityContext.RunAsUser == nil ||
+				*container.SecurityContext.RunAsUser == selfUser {
+				continue
+			}
+
+			plog.Debug("adding write permission",
+				"address", endpoint.Address,
+				"uid", *container.SecurityContext.RunAsUser,
+			)
+			entries = append(entries, acl.Entry{
+				Tag:       acl.TagUser,
+				Qualifier: strconv.FormatInt(*container.SecurityContext.RunAsUser, 10),
+				Perms:     2, // write permission
+			})
+		}
+		return acl.Add(endpoint.Address, entries...) // allow all containers in the pod to write to the socket
+	}
+}
+
 func main() error { // return an error instead of klog.Fatal to allow defer statements to run
 	logs.InitLogs()
 	defer logs.FlushLogs()