Merge pull request #1190 from vmware-tanzu/client-secret-api-noop

aggregated api for oidcclientsecretrequest

internal/config/supervisor/config.go

@@ -24,6 +24,12 @@ const (
 	NetworkDisabled = "disabled"
 	NetworkUnix     = "unix"
 	NetworkTCP      = "tcp"
+
+	// Use 10250 because it happens to be the same port on which the Kubelet listens, so some cluster types
+	// are more permissive with servers that run on this port. For example, GKE private clusters do not
+	// allow traffic from the control plane to most ports, but do allow traffic to port 10250. This allows
+	// the Concierge to work without additional configuration on these types of clusters.
+	aggregatedAPIServerPortDefault = 10250
 )
 
 // FromPath loads an Config from a provided local file path, inserts any
@@ -50,6 +56,12 @@ func FromPath(ctx context.Context, path string) (*Config, error) {
 		return nil, fmt.Errorf("validate apiGroupSuffix: %w", err)
 	}
 
+	maybeSetAggregatedAPIServerPortDefaults(&config.AggregatedAPIServerPort)
+
+	if err := validateServerPort(config.AggregatedAPIServerPort); err != nil {
+		return nil, fmt.Errorf("validate aggregatedAPIServerPort: %w", err)
+	}
+
 	if err := validateNames(&config.NamesConfig); err != nil {
 		return nil, fmt.Errorf("validate names: %w", err)
 	}
@@ -105,6 +117,12 @@ func validateAPIGroupSuffix(apiGroupSuffix string) error {
 	return groupsuffix.Validate(apiGroupSuffix)
 }
 
+func maybeSetAggregatedAPIServerPortDefaults(port **int64) {
+	if *port == nil {
+		*port = pointer.Int64Ptr(aggregatedAPIServerPortDefault)
+	}
+}
+
 func validateNames(names *NamesConfigSpec) error {
 	missingNames := []string{}
 	if names.DefaultTLSCertificateSecret == "" {
@@ -193,3 +211,11 @@ func addrIsOnlyOnLoopback(addr string) bool {
 	}
 	return ip.IsLoopback()
 }
+
+func validateServerPort(port *int64) error {
+	// It cannot be below 1024 because the container is not running as root.
+	if *port < 1024 || *port > 65535 {
+		return constable.Error("must be within range 1024 to 65535")
+	}
+	return nil
+}

internal/config/supervisor/config_test.go

@@ -43,6 +43,7 @@ func TestFromPath(t *testing.T) {
 					address: 127.0.0.1:1234
 				insecureAcceptExternalUnencryptedHttpRequests: false
 				logLevel: trace
+				aggregatedAPIServerPort: 12345
 			`),
 			wantConfig: &Config{
 				APIGroupSuffix: pointer.StringPtr("some.suffix.com"),
@@ -68,6 +69,7 @@ func TestFromPath(t *testing.T) {
 				Log: plog.LogSpec{
 					Level: plog.LevelTrace,
 				},
+				AggregatedAPIServerPort: pointer.Int64Ptr(12345),
 			},
 		},
 		{
@@ -91,6 +93,7 @@ func TestFromPath(t *testing.T) {
 				log:
 				  level: info
 				  format: text
+				aggregatedAPIServerPort: 12345
 			`),
 			wantConfig: &Config{
 				APIGroupSuffix: pointer.StringPtr("some.suffix.com"),
@@ -116,6 +119,7 @@ func TestFromPath(t *testing.T) {
 					Level:  plog.LevelInfo,
 					Format: plog.FormatText,
 				},
+				AggregatedAPIServerPort: pointer.Int64Ptr(12345),
 			},
 		},
 		{
@@ -166,6 +170,7 @@ func TestFromPath(t *testing.T) {
 					Level:  plog.LevelTrace,
 					Format: plog.FormatText,
 				},
+				AggregatedAPIServerPort: pointer.Int64Ptr(10250),
 			},
 		},
 		{
@@ -202,7 +207,8 @@ func TestFromPath(t *testing.T) {
 						Network: "disabled",
 					},
 				},
-				AllowExternalHTTP: false,
+				AllowExternalHTTP:       false,
+				AggregatedAPIServerPort: pointer.Int64Ptr(10250),
 			},
 		},
 		{
@@ -332,7 +338,8 @@ func TestFromPath(t *testing.T) {
 						Address: ":1234",
 					},
 				},
-				AllowExternalHTTP: true,
+				AllowExternalHTTP:       true,
+				AggregatedAPIServerPort: pointer.Int64Ptr(10250),
 			},
 		},
 		{
@@ -363,7 +370,8 @@ func TestFromPath(t *testing.T) {
 						Address: ":1234",
 					},
 				},
-				AllowExternalHTTP: true,
+				AllowExternalHTTP:       true,
+				AggregatedAPIServerPort: pointer.Int64Ptr(10250),
 			},
 		},
 		{
@@ -420,6 +428,22 @@ func TestFromPath(t *testing.T) {
 			`),
 			wantError: "validate apiGroupSuffix: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')",
 		},
+		{
+			name: "AggregatedAPIServerPortDefault too small",
+			yaml: here.Doc(`
+				---
+				aggregatedAPIServerPort: 1023
+			`),
+			wantError: "validate aggregatedAPIServerPort: must be within range 1024 to 65535",
+		},
+		{
+			name: "AggregatedAPIServerPortDefault too large",
+			yaml: here.Doc(`
+				---
+				aggregatedAPIServerPort: 65536
+			`),
+			wantError: "validate aggregatedAPIServerPort: must be within range 1024 to 65535",
+		},
 	}
 	for _, test := range tests {
 		test := test

internal/config/supervisor/types.go

@@ -15,15 +15,17 @@ type Config struct {
 	Labels         map[string]string `json:"labels"`
 	NamesConfig    NamesConfigSpec   `json:"names"`
 	// Deprecated: use log.level instead
-	LogLevel          *plog.LogLevel     `json:"logLevel"`
-	Log               plog.LogSpec       `json:"log"`
-	Endpoints         *Endpoints         `json:"endpoints"`
-	AllowExternalHTTP stringOrBoolAsBool `json:"insecureAcceptExternalUnencryptedHttpRequests"`
+	LogLevel                *plog.LogLevel     `json:"logLevel"`
+	Log                     plog.LogSpec       `json:"log"`
+	Endpoints               *Endpoints         `json:"endpoints"`
+	AllowExternalHTTP       stringOrBoolAsBool `json:"insecureAcceptExternalUnencryptedHttpRequests"`
+	AggregatedAPIServerPort *int64             `json:"aggregatedAPIServerPort"`
 }
 
 // NamesConfigSpec configures the names of some Kubernetes resources for the Supervisor.
 type NamesConfigSpec struct {
 	DefaultTLSCertificateSecret string `json:"defaultTLSCertificateSecret"`
+	APIService                  string `json:"apiService"`
 }
 
 type Endpoints struct {

internal/groupsuffix/groupdata.go

@@ -1,4 +1,4 @@
-// Copyright 2021 the Pinniped contributors. All Rights Reserved.
+// Copyright 2021-2022 the Pinniped contributors. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
 package groupsuffix
@@ -8,6 +8,7 @@ import (
 
 	identityv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/identity/v1alpha1"
 	loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1"
+	clientsecretv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/clientsecret/v1alpha1"
 )
 
 type GroupData schema.GroupVersion
@@ -32,3 +33,16 @@ func ConciergeAggregatedGroups(apiGroupSuffix string) (login, identity GroupData
 			Version: identityv1alpha1.SchemeGroupVersion.Version,
 		}
 }
+
+func SupervisorAggregatedGroups(apiGroupSuffix string) (clientSecret GroupData) {
+	clientSecretVirtualSupervisorAPIGroup, ok1 := Replace(clientsecretv1alpha1.GroupName, apiGroupSuffix)
+
+	if !ok1 {
+		panic("static group input is invalid")
+	}
+
+	return GroupData{
+		Group:   clientSecretVirtualSupervisorAPIGroup,
+		Version: clientsecretv1alpha1.SchemeGroupVersion.Version,
+	}
+}

internal/registry/clientsecretrequest/rest.go

@@ -0,0 +1,107 @@
+// Copyright 2022 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package clientsecretrequest provides REST functionality for the CredentialRequest resource.
+package clientsecretrequest
+
+import (
+	"context"
+	"fmt"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/registry/rest"
+	"k8s.io/utils/trace"
+
+	clientsecretapi "go.pinniped.dev/generated/latest/apis/supervisor/clientsecret"
+)
+
+func NewREST(resource schema.GroupResource) *REST {
+	return &REST{
+		tableConvertor: rest.NewDefaultTableConvertor(resource),
+	}
+}
+
+type REST struct {
+	tableConvertor rest.TableConvertor
+}
+
+// Assert that our *REST implements all the optional interfaces that we expect it to implement.
+var _ interface {
+	rest.Creater
+	rest.NamespaceScopedStrategy
+	rest.Scoper
+	rest.Storage
+	rest.CategoriesProvider
+	rest.Lister
+	rest.TableConvertor
+} = (*REST)(nil)
+
+func (*REST) New() runtime.Object {
+	return &clientsecretapi.OIDCClientSecretRequest{}
+}
+
+func (*REST) NewList() runtime.Object {
+	return &clientsecretapi.OIDCClientSecretRequestList{}
+}
+
+func (*REST) List(_ context.Context, _ *metainternalversion.ListOptions) (runtime.Object, error) {
+	return &clientsecretapi.OIDCClientSecretRequestList{
+		ListMeta: metav1.ListMeta{
+			ResourceVersion: "0", // this resource version means "from the API server cache"
+		},
+		Items: []clientsecretapi.OIDCClientSecretRequest{}, // avoid sending nil items list
+	}, nil
+}
+
+func (r *REST) ConvertToTable(ctx context.Context, obj runtime.Object, tableOptions runtime.Object) (*metav1.Table, error) {
+	return r.tableConvertor.ConvertToTable(ctx, obj, tableOptions)
+}
+
+func (*REST) NamespaceScoped() bool {
+	return true
+}
+
+func (*REST) Categories() []string {
+	return []string{"pinniped"}
+}
+
+func (r *REST) Create(ctx context.Context, obj runtime.Object, createValidation rest.ValidateObjectFunc, options *metav1.CreateOptions) (runtime.Object, error) {
+	t := trace.FromContext(ctx).Nest("create", trace.Field{
+		Key:   "kind",
+		Value: "OIDCClientSecretRequest",
+	})
+	defer t.Log()
+
+	_, err := validateRequest(obj, t)
+	if err != nil {
+		return nil, err
+	}
+
+	return &clientsecretapi.OIDCClientSecretRequest{
+		Status: clientsecretapi.OIDCClientSecretRequestStatus{
+			GeneratedSecret:    "not-a-real-secret",
+			TotalClientSecrets: 20,
+		},
+	}, nil
+}
+
+func validateRequest(obj runtime.Object, t *trace.Trace) (*clientsecretapi.OIDCClientSecretRequest, error) {
+	clientSecretRequest, ok := obj.(*clientsecretapi.OIDCClientSecretRequest)
+	if !ok {
+		traceValidationFailure(t, "not an OIDCClientSecretRequest")
+		return nil, apierrors.NewBadRequest(fmt.Sprintf("not an OIDCClientSecretRequest: %#v", obj))
+	}
+
+	return clientSecretRequest, nil
+}
+
+func traceValidationFailure(t *trace.Trace, msg string) {
+	t.Step("failure",
+		trace.Field{Key: "failureType", Value: "request validation"},
+		trace.Field{Key: "msg", Value: msg},
+	)
+}

internal/supervisor/apiserver/apiserver.go

@@ -0,0 +1,139 @@
+// Copyright 2022 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package apiserver
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apiserver/pkg/registry/rest"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	"k8s.io/client-go/pkg/version"
+
+	"go.pinniped.dev/internal/controllerinit"
+	"go.pinniped.dev/internal/plog"
+	"go.pinniped.dev/internal/registry/clientsecretrequest"
+)
+
+type Config struct {
+	GenericConfig *genericapiserver.RecommendedConfig
+	ExtraConfig   ExtraConfig
+}
+
+type ExtraConfig struct {
+	BuildControllersPostStartHook      controllerinit.RunnerBuilder
+	Scheme                             *runtime.Scheme
+	NegotiatedSerializer               runtime.NegotiatedSerializer
+	ClientSecretSupervisorGroupVersion schema.GroupVersion
+}
+
+type PinnipedServer struct {
+	GenericAPIServer *genericapiserver.GenericAPIServer
+}
+
+type completedConfig struct {
+	GenericConfig genericapiserver.CompletedConfig
+	ExtraConfig   *ExtraConfig
+}
+
+type CompletedConfig struct {
+	// Embed a private pointer that cannot be instantiated outside of this package.
+	*completedConfig
+}
+
+// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
+func (c *Config) Complete() CompletedConfig {
+	completedCfg := completedConfig{
+		c.GenericConfig.Complete(),
+		&c.ExtraConfig,
+	}
+
+	versionInfo := version.Get()
+	completedCfg.GenericConfig.Version = &versionInfo
+
+	return CompletedConfig{completedConfig: &completedCfg}
+}
+
+// New returns a new instance of AdmissionServer from the given config.
+func (c completedConfig) New() (*PinnipedServer, error) {
+	genericServer, err := c.GenericConfig.New("pinniped-supervisor", genericapiserver.NewEmptyDelegate()) // completion is done in Complete, no need for a second time
+	if err != nil {
+		return nil, fmt.Errorf("completion error: %w", err)
+	}
+
+	s := &PinnipedServer{
+		GenericAPIServer: genericServer,
+	}
+
+	var errs []error //nolint: prealloc
+	for _, f := range []func() (schema.GroupVersionResource, rest.Storage){
+		func() (schema.GroupVersionResource, rest.Storage) {
+			clientSecretReqGVR := c.ExtraConfig.ClientSecretSupervisorGroupVersion.WithResource("oidcclientsecretrequests")
+			clientSecretReqStorage := clientsecretrequest.NewREST(clientSecretReqGVR.GroupResource())
+			return clientSecretReqGVR, clientSecretReqStorage
+		},
+	} {
+		gvr, storage := f()
+		errs = append(errs,
+			s.GenericAPIServer.InstallAPIGroup(
+				&genericapiserver.APIGroupInfo{
+					PrioritizedVersions:          []schema.GroupVersion{gvr.GroupVersion()},
+					VersionedResourcesStorageMap: map[string]map[string]rest.Storage{gvr.Version: {gvr.Resource: storage}},
+					OptionsExternalVersion:       &schema.GroupVersion{Version: "v1"},
+					Scheme:                       c.ExtraConfig.Scheme,
+					ParameterCodec:               metav1.ParameterCodec,
+					NegotiatedSerializer:         c.ExtraConfig.NegotiatedSerializer,
+				},
+			),
+		)
+	}
+	if err := errors.NewAggregate(errs); err != nil {
+		return nil, fmt.Errorf("could not install API groups: %w", err)
+	}
+
+	shutdown := &sync.WaitGroup{}
+	s.GenericAPIServer.AddPostStartHookOrDie("start-controllers",
+		func(postStartContext genericapiserver.PostStartHookContext) error {
+			plog.Debug("start-controllers post start hook starting")
+
+			ctx, cancel := context.WithCancel(context.Background())
+			go func() {
+				defer cancel()
+
+				<-postStartContext.StopCh
+			}()
+
+			runControllers, err := c.ExtraConfig.BuildControllersPostStartHook(ctx)
+			if err != nil {
+				return fmt.Errorf("cannot create run controller func: %w", err)
+			}
+
+			shutdown.Add(1)
+			go func() {
+				defer shutdown.Done()
+
+				runControllers(ctx)
+			}()
+
+			return nil
+		},
+	)
+	s.GenericAPIServer.AddPreShutdownHookOrDie("stop-controllers",
+		func() error {
+			plog.Debug("stop-controllers pre shutdown hook starting")
+			defer plog.Debug("stop-controllers pre shutdown hook completed")
+
+			shutdown.Wait()
+
+			return nil
+		},
+	)
+
+	return s, nil
+}

internal/supervisor/scheme/scheme.go

@@ -0,0 +1,91 @@
+// Copyright 2022 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package scheme contains code to construct a proper runtime.Scheme for the Concierge aggregated
+// API.
+package scheme
+
+import (
+	"fmt"
+
+	clientsecretapi "go.pinniped.dev/generated/latest/apis/supervisor/clientsecret"
+	clientsecretv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/clientsecret/v1alpha1"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+
+	"go.pinniped.dev/internal/groupsuffix"
+)
+
+// New returns a runtime.Scheme for use by the Supervisor aggregated API running with the provided
+// apiGroupSuffix.
+func New(apiGroupSuffix string) (_ *runtime.Scheme, oauth schema.GroupVersion) {
+	// standard set up of the server side scheme
+	scheme := runtime.NewScheme()
+
+	// add the options to empty v1
+	metav1.AddToGroupVersion(scheme, metav1.Unversioned)
+
+	// nothing fancy is required if using the standard group suffix
+	if apiGroupSuffix == groupsuffix.PinnipedDefaultSuffix {
+		schemeBuilder := runtime.NewSchemeBuilder(
+			clientsecretv1alpha1.AddToScheme,
+			clientsecretapi.AddToScheme,
+		)
+		utilruntime.Must(schemeBuilder.AddToScheme(scheme))
+		return scheme, clientsecretv1alpha1.SchemeGroupVersion
+	}
+
+	clientSecretSupervisorGroupData := groupsuffix.SupervisorAggregatedGroups(apiGroupSuffix)
+
+	addToSchemeAtNewGroup(scheme, clientsecretv1alpha1.GroupName, clientSecretSupervisorGroupData.Group, clientsecretv1alpha1.AddToScheme, clientsecretapi.AddToScheme)
+
+	// manually register conversions and defaulting into the correct scheme since we cannot directly call AddToScheme
+	schemeBuilder := runtime.NewSchemeBuilder(
+		clientsecretv1alpha1.RegisterConversions,
+		clientsecretv1alpha1.RegisterDefaults,
+	)
+	utilruntime.Must(schemeBuilder.AddToScheme(scheme))
+
+	// we do not have any defaulting functions for *loginv1alpha1.OIDCClientSecretRequest
+	// today, but we may have some in the future.  Calling AddTypeDefaultingFunc overwrites
+	// any previously registered defaulting function.  Thus to make sure that we catch
+	// a situation where we add a defaulting func, we attempt to call it here with a nil
+	// *clientsecretv1alpha1.OIDCClientSecretRequest.  This will do nothing when there is no
+	// defaulting func registered, but it will almost certainly panic if one is added.
+	scheme.Default((*clientsecretv1alpha1.OIDCClientSecretRequest)(nil))
+
+	return scheme, schema.GroupVersion(clientSecretSupervisorGroupData)
+}
+
+func addToSchemeAtNewGroup(scheme *runtime.Scheme, oldGroup, newGroup string, funcs ...func(*runtime.Scheme) error) {
+	// we need a temporary place to register our types to avoid double registering them
+	tmpScheme := runtime.NewScheme()
+	schemeBuilder := runtime.NewSchemeBuilder(funcs...)
+	utilruntime.Must(schemeBuilder.AddToScheme(tmpScheme))
+
+	for gvk := range tmpScheme.AllKnownTypes() {
+		if gvk.GroupVersion() == metav1.Unversioned {
+			continue // metav1.AddToGroupVersion registers types outside of our aggregated API group that we need to ignore
+		}
+
+		if gvk.Group != oldGroup {
+			panic(fmt.Errorf("tmp scheme has type not in the old aggregated API group %s: %s", oldGroup, gvk)) // programmer error
+		}
+
+		obj, err := tmpScheme.New(gvk)
+		if err != nil {
+			panic(err) // programmer error, scheme internal code is broken
+		}
+		newGVK := schema.GroupVersionKind{
+			Group:   newGroup,
+			Version: gvk.Version,
+			Kind:    gvk.Kind,
+		}
+
+		// register the existing type but with the new group in the correct scheme
+		scheme.AddKnownTypeWithName(newGVK, obj)
+	}
+}

internal/supervisor/scheme/scheme_test.go

@@ -0,0 +1,143 @@
+// Copyright 2022 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package scheme
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
+	clientsecretapi "go.pinniped.dev/generated/latest/apis/supervisor/clientsecret"
+	clientsecretv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/clientsecret/v1alpha1"
+)
+
+func TestNew(t *testing.T) {
+	// the standard group
+	regularClientSecretGV := schema.GroupVersion{
+		Group:   "clientsecret.supervisor.pinniped.dev",
+		Version: "v1alpha1",
+	}
+	regularClientSecretGVInternal := schema.GroupVersion{
+		Group:   "clientsecret.supervisor.pinniped.dev",
+		Version: runtime.APIVersionInternal,
+	}
+
+	// the canonical other group
+	otherClientSecretGV := schema.GroupVersion{
+		Group:   "clientsecret.supervisor.walrus.tld",
+		Version: "v1alpha1",
+	}
+	otherClientSecretGVInternal := schema.GroupVersion{
+		Group:   "clientsecret.supervisor.walrus.tld",
+		Version: runtime.APIVersionInternal,
+	}
+
+	// kube's core internal
+	internalGV := schema.GroupVersion{
+		Group:   "",
+		Version: runtime.APIVersionInternal,
+	}
+
+	tests := []struct {
+		name                         string
+		apiGroupSuffix               string
+		want                         map[schema.GroupVersionKind]reflect.Type
+		wantClientSecretGroupVersion schema.GroupVersion
+	}{
+		{
+			name:           "regular api group",
+			apiGroupSuffix: "pinniped.dev",
+			want: map[schema.GroupVersionKind]reflect.Type{
+				// all the types that are in the aggregated API group
+
+				regularClientSecretGV.WithKind("OIDCClientSecretRequest"):     reflect.TypeOf(&clientsecretv1alpha1.OIDCClientSecretRequest{}).Elem(),
+				regularClientSecretGV.WithKind("OIDCClientSecretRequestList"): reflect.TypeOf(&clientsecretv1alpha1.OIDCClientSecretRequestList{}).Elem(),
+
+				regularClientSecretGVInternal.WithKind("OIDCClientSecretRequest"):     reflect.TypeOf(&clientsecretapi.OIDCClientSecretRequest{}).Elem(),
+				regularClientSecretGVInternal.WithKind("OIDCClientSecretRequestList"): reflect.TypeOf(&clientsecretapi.OIDCClientSecretRequestList{}).Elem(),
+
+				regularClientSecretGV.WithKind("CreateOptions"): reflect.TypeOf(&metav1.CreateOptions{}).Elem(),
+				regularClientSecretGV.WithKind("DeleteOptions"): reflect.TypeOf(&metav1.DeleteOptions{}).Elem(),
+				regularClientSecretGV.WithKind("GetOptions"):    reflect.TypeOf(&metav1.GetOptions{}).Elem(),
+				regularClientSecretGV.WithKind("ListOptions"):   reflect.TypeOf(&metav1.ListOptions{}).Elem(),
+				regularClientSecretGV.WithKind("PatchOptions"):  reflect.TypeOf(&metav1.PatchOptions{}).Elem(),
+				regularClientSecretGV.WithKind("UpdateOptions"): reflect.TypeOf(&metav1.UpdateOptions{}).Elem(),
+				regularClientSecretGV.WithKind("WatchEvent"):    reflect.TypeOf(&metav1.WatchEvent{}).Elem(),
+
+				regularClientSecretGVInternal.WithKind("WatchEvent"): reflect.TypeOf(&metav1.InternalEvent{}).Elem(),
+
+				// the types below this line do not really matter to us because they are in the core group
+
+				internalGV.WithKind("WatchEvent"): reflect.TypeOf(&metav1.InternalEvent{}).Elem(),
+
+				metav1.Unversioned.WithKind("APIGroup"):        reflect.TypeOf(&metav1.APIGroup{}).Elem(),
+				metav1.Unversioned.WithKind("APIGroupList"):    reflect.TypeOf(&metav1.APIGroupList{}).Elem(),
+				metav1.Unversioned.WithKind("APIResourceList"): reflect.TypeOf(&metav1.APIResourceList{}).Elem(),
+				metav1.Unversioned.WithKind("APIVersions"):     reflect.TypeOf(&metav1.APIVersions{}).Elem(),
+				metav1.Unversioned.WithKind("CreateOptions"):   reflect.TypeOf(&metav1.CreateOptions{}).Elem(),
+				metav1.Unversioned.WithKind("DeleteOptions"):   reflect.TypeOf(&metav1.DeleteOptions{}).Elem(),
+				metav1.Unversioned.WithKind("GetOptions"):      reflect.TypeOf(&metav1.GetOptions{}).Elem(),
+				metav1.Unversioned.WithKind("ListOptions"):     reflect.TypeOf(&metav1.ListOptions{}).Elem(),
+				metav1.Unversioned.WithKind("PatchOptions"):    reflect.TypeOf(&metav1.PatchOptions{}).Elem(),
+				metav1.Unversioned.WithKind("Status"):          reflect.TypeOf(&metav1.Status{}).Elem(),
+				metav1.Unversioned.WithKind("UpdateOptions"):   reflect.TypeOf(&metav1.UpdateOptions{}).Elem(),
+				metav1.Unversioned.WithKind("WatchEvent"):      reflect.TypeOf(&metav1.WatchEvent{}).Elem(),
+			},
+			wantClientSecretGroupVersion: regularClientSecretGV,
+		},
+		{
+			name:           "other api group",
+			apiGroupSuffix: "walrus.tld",
+			want: map[schema.GroupVersionKind]reflect.Type{
+				// all the types that are in the aggregated API group
+
+				otherClientSecretGV.WithKind("OIDCClientSecretRequest"):     reflect.TypeOf(&clientsecretv1alpha1.OIDCClientSecretRequest{}).Elem(),
+				otherClientSecretGV.WithKind("OIDCClientSecretRequestList"): reflect.TypeOf(&clientsecretv1alpha1.OIDCClientSecretRequestList{}).Elem(),
+
+				otherClientSecretGVInternal.WithKind("OIDCClientSecretRequest"):     reflect.TypeOf(&clientsecretapi.OIDCClientSecretRequest{}).Elem(),
+				otherClientSecretGVInternal.WithKind("OIDCClientSecretRequestList"): reflect.TypeOf(&clientsecretapi.OIDCClientSecretRequestList{}).Elem(),
+
+				otherClientSecretGV.WithKind("CreateOptions"): reflect.TypeOf(&metav1.CreateOptions{}).Elem(),
+				otherClientSecretGV.WithKind("DeleteOptions"): reflect.TypeOf(&metav1.DeleteOptions{}).Elem(),
+				otherClientSecretGV.WithKind("GetOptions"):    reflect.TypeOf(&metav1.GetOptions{}).Elem(),
+				otherClientSecretGV.WithKind("ListOptions"):   reflect.TypeOf(&metav1.ListOptions{}).Elem(),
+				otherClientSecretGV.WithKind("PatchOptions"):  reflect.TypeOf(&metav1.PatchOptions{}).Elem(),
+				otherClientSecretGV.WithKind("UpdateOptions"): reflect.TypeOf(&metav1.UpdateOptions{}).Elem(),
+				otherClientSecretGV.WithKind("WatchEvent"):    reflect.TypeOf(&metav1.WatchEvent{}).Elem(),
+
+				otherClientSecretGVInternal.WithKind("WatchEvent"): reflect.TypeOf(&metav1.InternalEvent{}).Elem(),
+
+				// the types below this line do not really matter to us because they are in the core group
+
+				internalGV.WithKind("WatchEvent"): reflect.TypeOf(&metav1.InternalEvent{}).Elem(),
+
+				metav1.Unversioned.WithKind("APIGroup"):        reflect.TypeOf(&metav1.APIGroup{}).Elem(),
+				metav1.Unversioned.WithKind("APIGroupList"):    reflect.TypeOf(&metav1.APIGroupList{}).Elem(),
+				metav1.Unversioned.WithKind("APIResourceList"): reflect.TypeOf(&metav1.APIResourceList{}).Elem(),
+				metav1.Unversioned.WithKind("APIVersions"):     reflect.TypeOf(&metav1.APIVersions{}).Elem(),
+				metav1.Unversioned.WithKind("CreateOptions"):   reflect.TypeOf(&metav1.CreateOptions{}).Elem(),
+				metav1.Unversioned.WithKind("DeleteOptions"):   reflect.TypeOf(&metav1.DeleteOptions{}).Elem(),
+				metav1.Unversioned.WithKind("GetOptions"):      reflect.TypeOf(&metav1.GetOptions{}).Elem(),
+				metav1.Unversioned.WithKind("ListOptions"):     reflect.TypeOf(&metav1.ListOptions{}).Elem(),
+				metav1.Unversioned.WithKind("PatchOptions"):    reflect.TypeOf(&metav1.PatchOptions{}).Elem(),
+				metav1.Unversioned.WithKind("Status"):          reflect.TypeOf(&metav1.Status{}).Elem(),
+				metav1.Unversioned.WithKind("UpdateOptions"):   reflect.TypeOf(&metav1.UpdateOptions{}).Elem(),
+				metav1.Unversioned.WithKind("WatchEvent"):      reflect.TypeOf(&metav1.WatchEvent{}).Elem(),
+			},
+			wantClientSecretGroupVersion: otherClientSecretGV,
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			scheme, clientSecretGV := New(tt.apiGroupSuffix)
+			require.Equal(t, tt.want, scheme.AllKnownTypes())
+			require.Equal(t, tt.wantClientSecretGroupVersion, clientSecretGV)
+		})
+	}
+}

internal/supervisor/server/server.go

@@ -22,18 +22,26 @@ import (
 	"github.com/joshlf/go-acl"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
 	apimachineryversion "k8s.io/apimachinery/pkg/version"
 	genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	genericoptions "k8s.io/apiserver/pkg/server/options"
 	kubeinformers "k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/pkg/version"
 	"k8s.io/client-go/rest"
+	aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
 	"k8s.io/utils/clock"
 
 	configv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/config/v1alpha1"
 	pinnipedclientset "go.pinniped.dev/generated/latest/client/supervisor/clientset/versioned"
 	pinnipedinformers "go.pinniped.dev/generated/latest/client/supervisor/informers/externalversions"
+	"go.pinniped.dev/internal/apiserviceref"
 	"go.pinniped.dev/internal/config/supervisor"
+	"go.pinniped.dev/internal/controller/apicerts"
 	"go.pinniped.dev/internal/controller/supervisorconfig"
 	"go.pinniped.dev/internal/controller/supervisorconfig/activedirectoryupstreamwatcher"
 	"go.pinniped.dev/internal/controller/supervisorconfig/generator"
@@ -45,6 +53,7 @@ import (
 	"go.pinniped.dev/internal/crypto/ptls"
 	"go.pinniped.dev/internal/deploymentref"
 	"go.pinniped.dev/internal/downward"
+	"go.pinniped.dev/internal/dynamiccert"
 	"go.pinniped.dev/internal/groupsuffix"
 	"go.pinniped.dev/internal/kubeclient"
 	"go.pinniped.dev/internal/leaderelection"
@@ -53,6 +62,8 @@ import (
 	"go.pinniped.dev/internal/oidc/provider/manager"
 	"go.pinniped.dev/internal/plog"
 	"go.pinniped.dev/internal/secret"
+	"go.pinniped.dev/internal/supervisor/apiserver"
+	supervisorscheme "go.pinniped.dev/internal/supervisor/scheme"
 )
 
 const (
@@ -116,14 +127,19 @@ func prepareControllers(
 	dynamicJWKSProvider jwks.DynamicJWKSProvider,
 	dynamicTLSCertProvider provider.DynamicTLSCertProvider,
 	dynamicUpstreamIDPProvider provider.DynamicUpstreamIDPProvider,
+	dynamicServingCertProvider dynamiccert.Private,
 	secretCache *secret.Cache,
 	supervisorDeployment *appsv1.Deployment,
 	kubeClient kubernetes.Interface,
 	pinnipedClient pinnipedclientset.Interface,
+	aggregatorClient aggregatorclient.Interface,
 	kubeInformers kubeinformers.SharedInformerFactory,
 	pinnipedInformers pinnipedinformers.SharedInformerFactory,
 	leaderElector controllerinit.RunnerWrapper,
+	podInfo *downward.PodInfo,
 ) controllerinit.RunnerBuilder {
+	const certificateName string = "pinniped-supervisor-api-tls-serving-certificate"
+	clientSecretSupervisorGroupData := groupsuffix.SupervisorAggregatedGroups(*cfg.APIGroupSuffix)
 	federationDomainInformer := pinnipedInformers.Config().V1alpha1().FederationDomains()
 	secretInformer := kubeInformers.Core().V1().Secrets()
 
@@ -291,30 +307,69 @@ func prepareControllers(
 				secretInformer,
 				controllerlib.WithInformer,
 			),
-			singletonWorker)
+			singletonWorker).
+		WithController(
+			apicerts.NewCertsManagerController(
+				podInfo.Namespace,
+				certificateName,
+				cfg.Labels,
+				kubeClient,
+				secretInformer,
+				controllerlib.WithInformer,
+				controllerlib.WithInitialEvent,
+				365*24*time.Hour, // about one year
+				"Pinniped Supervisor Aggregation CA",
+				cfg.NamesConfig.APIService,
+			),
+			singletonWorker,
+		).
+		WithController(
+			apicerts.NewAPIServiceUpdaterController(
+				podInfo.Namespace,
+				certificateName,
+				clientSecretSupervisorGroupData.APIServiceName(),
+				aggregatorClient,
+				secretInformer,
+				controllerlib.WithInformer,
+			),
+			singletonWorker,
+		).
+		WithController(
+			apicerts.NewCertsObserverController(
+				podInfo.Namespace,
+				certificateName,
+				dynamicServingCertProvider,
+				secretInformer,
+				controllerlib.WithInformer,
+			),
+			singletonWorker,
+		).
+		WithController(
+			apicerts.NewCertsExpirerController(
+				podInfo.Namespace,
+				certificateName,
+				kubeClient,
+				secretInformer,
+				controllerlib.WithInformer,
+				9*30*24*time.Hour, // about 9 months
+				apicerts.TLSCertificateChainSecretKey,
+				plog.New(),
+			),
+			singletonWorker,
+		)
 
 	return controllerinit.Prepare(controllerManager.Start, leaderElector, kubeInformers, pinnipedInformers)
 }
 
-func startControllers(ctx context.Context, shutdown *sync.WaitGroup, buildControllers controllerinit.RunnerBuilder) error {
-	runControllers, err := buildControllers(ctx)
-	if err != nil {
-		return fmt.Errorf("cannot create run controller func: %w", err)
-	}
-
-	shutdown.Add(1)
-	go func() {
-		defer shutdown.Done()
-
-		runControllers(ctx)
-	}()
-
-	return nil
-}
-
 //nolint:funlen
 func runSupervisor(ctx context.Context, podInfo *downward.PodInfo, cfg *supervisor.Config) error {
 	serverInstallationNamespace := podInfo.Namespace
+	clientSecretSupervisorGroupData := groupsuffix.SupervisorAggregatedGroups(*cfg.APIGroupSuffix)
+
+	apiServiceRef, err := apiserviceref.New(clientSecretSupervisorGroupData.APIServiceName())
+	if err != nil {
+		return fmt.Errorf("cannot create API service ref: %w", err)
+	}
 
 	dref, supervisorDeployment, supervisorPod, err := deploymentref.New(podInfo)
 	if err != nil {
@@ -323,6 +378,7 @@ func runSupervisor(ctx context.Context, podInfo *downward.PodInfo, cfg *supervis
 
 	opts := []kubeclient.Option{
 		dref,
+		apiServiceRef,
 		kubeclient.WithMiddleware(groupsuffix.New(*cfg.APIGroupSuffix)),
 	}
 
@@ -358,6 +414,8 @@ func runSupervisor(ctx context.Context, podInfo *downward.PodInfo, cfg *supervis
 		_, _ = writer.Write([]byte("ok"))
 	}))
 
+	dynamicServingCertProvider := dynamiccert.NewServingCert("supervisor-serving-cert")
+
 	dynamicJWKSProvider := jwks.NewDynamicJWKSProvider()
 	dynamicTLSCertProvider := provider.NewDynamicTLSCertProvider()
 	dynamicUpstreamIDPProvider := provider.NewDynamicUpstreamIDPProvider()
@@ -372,25 +430,47 @@ func runSupervisor(ctx context.Context, podInfo *downward.PodInfo, cfg *supervis
 		clientWithoutLeaderElection.Kubernetes.CoreV1().Secrets(serverInstallationNamespace), // writes to kube storage are allowed for non-leaders
 	)
 
+	// Get the "real" name of the client secret supervisor API group (i.e., the API group name with the
+	// injected suffix).
+	scheme, clientSecretGV := supervisorscheme.New(*cfg.APIGroupSuffix)
+
 	buildControllersFunc := prepareControllers(
 		cfg,
 		oidProvidersManager,
 		dynamicJWKSProvider,
 		dynamicTLSCertProvider,
 		dynamicUpstreamIDPProvider,
+		dynamicServingCertProvider,
 		&secretCache,
 		supervisorDeployment,
 		client.Kubernetes,
 		client.PinnipedSupervisor,
+		client.Aggregation,
 		kubeInformers,
 		pinnipedInformers,
 		leaderElector,
+		podInfo,
 	)
 
 	shutdown := &sync.WaitGroup{}
 
-	if err := startControllers(ctx, shutdown, buildControllersFunc); err != nil {
-		return err
+	// Get the aggregated API server config.
+	aggregatedAPIServerConfig, err := getAggregatedAPIServerConfig(
+		dynamicServingCertProvider,
+		buildControllersFunc,
+		*cfg.APIGroupSuffix,
+		*cfg.AggregatedAPIServerPort,
+		scheme,
+		clientSecretGV,
+	)
+	if err != nil {
+		return fmt.Errorf("could not configure aggregated API server: %w", err)
+	}
+
+	// Complete the aggregated API server config and make a server instance.
+	server, err := aggregatedAPIServerConfig.Complete().New()
+	if err != nil {
+		return fmt.Errorf("could not create aggregated API server: %w", err)
 	}
 
 	if e := cfg.Endpoints.HTTP; e.Network != supervisor.NetworkDisabled {
@@ -465,11 +545,73 @@ func runSupervisor(ctx context.Context, podInfo *downward.PodInfo, cfg *supervis
 	plog.Debug("supervisor started")
 	defer plog.Debug("supervisor exiting")
 
+	// Run the server. Its post-start hook will start the controllers.
+	err = server.GenericAPIServer.PrepareRun().Run(ctx.Done())
+	if err != nil {
+		return err
+	}
 	shutdown.Wait()
 
 	return nil
 }
 
+// Create a configuration for the aggregated API server.
+func getAggregatedAPIServerConfig(
+	dynamicCertProvider dynamiccert.Private,
+	buildControllers controllerinit.RunnerBuilder,
+	apiGroupSuffix string,
+	aggregatedAPIServerPort int64,
+	scheme *runtime.Scheme,
+	clientSecretSupervisorGroupVersion schema.GroupVersion,
+) (*apiserver.Config, error) {
+	codecs := serializer.NewCodecFactory(scheme)
+
+	// this is unused for now but it is a safe value that we could use in the future
+	defaultEtcdPathPrefix := fmt.Sprintf("/pinniped-supervisor-registry/%s", apiGroupSuffix)
+
+	recommendedOptions := genericoptions.NewRecommendedOptions(
+		defaultEtcdPathPrefix,
+		codecs.LegacyCodec(clientSecretSupervisorGroupVersion),
+	)
+	recommendedOptions.Etcd = nil // turn off etcd storage because we don't need it yet
+	recommendedOptions.SecureServing.ServerCert.GeneratedCert = dynamicCertProvider
+
+	// This port is configurable. It should be safe to cast because the config reader already validated it.
+	recommendedOptions.SecureServing.BindPort = int(aggregatedAPIServerPort)
+
+	// secure TLS for connections coming from and going to the Kube API server
+	// this is best effort because not all options provide the right hooks to override TLS config
+	// since our only client is the Kube API server, this uses the most secure TLS config
+	if err := ptls.SecureRecommendedOptions(recommendedOptions, kubeclient.Secure); err != nil {
+		return nil, fmt.Errorf("failed to secure recommended options: %w", err)
+	}
+
+	serverConfig := genericapiserver.NewRecommendedConfig(codecs)
+	// Note that among other things, this ApplyTo() function copies
+	// `recommendedOptions.SecureServing.ServerCert.GeneratedCert` into
+	// `serverConfig.SecureServing.Cert` thus making `dynamicCertProvider`
+	// the cert provider for the running server. The provider will be called
+	// by the API machinery periodically. When the provider returns nil certs,
+	// the API server will return "the server is currently unable to
+	// handle the request" error responses for all incoming requests.
+	// If the provider later starts returning certs, then the API server
+	// will use them to handle the incoming requests successfully.
+	if err := recommendedOptions.ApplyTo(serverConfig); err != nil {
+		return nil, fmt.Errorf("failed to apply recommended options: %w", err)
+	}
+
+	apiServerConfig := &apiserver.Config{
+		GenericConfig: serverConfig,
+		ExtraConfig: apiserver.ExtraConfig{
+			BuildControllersPostStartHook:      buildControllers,
+			Scheme:                             scheme,
+			NegotiatedSerializer:               codecs,
+			ClientSecretSupervisorGroupVersion: clientSecretSupervisorGroupVersion,
+		},
+	}
+	return apiServerConfig, nil
+}
+
 func maybeSetupUnixPerms(endpoint *supervisor.Endpoint, pod *corev1.Pod) func() error {
 	if endpoint.Network != supervisor.NetworkUnix {
 		return func() error { return nil }