First draft of moving API server TLS cert generation to controllers

- Refactors the existing cert generation code into controllers
  which read and write a Secret containing the certs
- Does not add any new functionality yet, e.g. no new handling
  for cert expiration, and no leader election to allow for
  multiple servers running simultaneously
- This commit also doesn't add new tests for the cert generation
  code, but it should be more unit testable now as controllers

internal/controller/apicerts/certs_manager.go

@@ -0,0 +1,164 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package apicerts
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"fmt"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	corev1informers "k8s.io/client-go/informers/core/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
+	aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
+
+	"github.com/suzerain-io/controller-go"
+	"github.com/suzerain-io/placeholder-name/internal/autoregistration"
+	"github.com/suzerain-io/placeholder-name/internal/certauthority"
+	placeholdernamecontroller "github.com/suzerain-io/placeholder-name/internal/controller"
+	placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1"
+)
+
+const (
+	//nolint: gosec
+	certsSecretName              = "api-serving-cert"
+	caCertificateSecretKey       = "caCertificate"
+	tlsPrivateKeySecretKey       = "tlsPrivateKey"
+	tlsCertificateChainSecretKey = "tlsCertificateChain"
+)
+
+type certsManagerController struct {
+	namespace        string
+	apiServiceName   string
+	k8sClient        kubernetes.Interface
+	aggregatorClient *aggregatorclient.Clientset
+	secretInformer   corev1informers.SecretInformer
+}
+
+func NewCertsManagerController(
+	namespace string,
+	k8sClient kubernetes.Interface,
+	aggregationClient *aggregatorclient.Clientset,
+	secretInformer corev1informers.SecretInformer,
+	withInformer placeholdernamecontroller.WithInformerOptionFunc,
+) controller.Controller {
+	apiServiceName := placeholderv1alpha1.SchemeGroupVersion.Version + "." + placeholderv1alpha1.GroupName
+	return controller.New(
+		controller.Config{
+			Name: "certs-manager-controller",
+			Syncer: &certsManagerController{
+				apiServiceName:   apiServiceName,
+				namespace:        namespace,
+				k8sClient:        k8sClient,
+				aggregatorClient: aggregationClient,
+				secretInformer:   secretInformer,
+			},
+		},
+		withInformer(
+			secretInformer,
+			placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(certsSecretName, namespace),
+			controller.InformerOption{},
+		),
+		// Be sure to run once even if the Secret that the informer is watching doesn't exist.
+		controller.WithInitialEvent(controller.Key{
+			Namespace: namespace,
+			Name:      certsSecretName,
+		}),
+	)
+}
+
+func (c *certsManagerController) Sync(ctx controller.Context) error {
+	// Try to get the secret from the informer cache.
+	_, err := c.secretInformer.Lister().Secrets(c.namespace).Get(certsSecretName)
+	notFound := k8serrors.IsNotFound(err)
+	if err != nil && !notFound {
+		return fmt.Errorf("failed to get %s/%s secret: %w", c.namespace, certsSecretName, err)
+	}
+	if !notFound {
+		// The secret already exists, so nothing to do.
+		return nil
+	}
+
+	// Create a CA.
+	aggregatedAPIServerCA, err := certauthority.New(pkix.Name{CommonName: "Placeholder CA"})
+	if err != nil {
+		return fmt.Errorf("could not initialize CA: %w", err)
+	}
+
+	// This string must match the name of the Service declared in the deployment yaml.
+	const serviceName = "placeholder-name-api"
+
+	// Using the CA from above, create a TLS server cert for the aggregated API server to use.
+	aggregatedAPIServerTLSCert, err := aggregatedAPIServerCA.Issue(
+		pkix.Name{CommonName: serviceName + "." + c.namespace + ".svc"},
+		[]string{},
+		24*365*time.Hour,
+	)
+	if err != nil {
+		return fmt.Errorf("could not issue serving certificate: %w", err)
+	}
+
+	// Write the CA's public key bundle and the serving certs to a secret.
+	tlsPrivateKeyPEM, tlsCertChainPEM, err := pemEncode(aggregatedAPIServerTLSCert)
+	if err != nil {
+		return fmt.Errorf("could not PEM encode serving certificate: %w", err)
+	}
+	secret := corev1.Secret{
+		TypeMeta: metav1.TypeMeta{},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      certsSecretName,
+			Namespace: c.namespace,
+		},
+		StringData: map[string]string{
+			caCertificateSecretKey:       string(aggregatedAPIServerCA.Bundle()),
+			tlsPrivateKeySecretKey:       string(tlsPrivateKeyPEM),
+			tlsCertificateChainSecretKey: string(tlsCertChainPEM),
+		},
+	}
+	_, err = c.k8sClient.CoreV1().Secrets(c.namespace).Create(ctx.Context, &secret, metav1.CreateOptions{})
+	if err != nil {
+		return fmt.Errorf("could not create secret: %w", err)
+	}
+
+	// Update the APIService to give it the new CA bundle.
+	if err := autoregistration.UpdateAPIService(ctx.Context, c.aggregatorClient, aggregatedAPIServerCA.Bundle()); err != nil {
+		return fmt.Errorf("could not update the API service: %w", err)
+	}
+
+	klog.Info("certsManagerController Sync successfully created secret and updated API service")
+	return nil
+}
+
+// Encode a tls.Certificate into a private key PEM and a cert chain PEM.
+func pemEncode(cert *tls.Certificate) ([]byte, []byte, error) {
+	privateKeyDER, err := x509.MarshalPKCS8PrivateKey(cert.PrivateKey)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error marshalling private key: %w", err)
+	}
+	privateKeyPEM := pem.EncodeToMemory(&pem.Block{
+		Type:    "PRIVATE KEY",
+		Headers: nil,
+		Bytes:   privateKeyDER,
+	})
+
+	certChainPEM := make([]byte, 0)
+	for _, certFromChain := range cert.Certificate {
+		certPEMBytes := pem.EncodeToMemory(&pem.Block{
+			Type:    "CERTIFICATE",
+			Headers: nil,
+			Bytes:   certFromChain,
+		})
+		certChainPEM = append(certChainPEM, certPEMBytes...)
+	}
+
+	return privateKeyPEM, certChainPEM, nil
+}

internal/controller/apicerts/certs_observer.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package apicerts
+
+import (
+	"fmt"
+
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	corev1informers "k8s.io/client-go/informers/core/v1"
+	"k8s.io/klog/v2"
+
+	"github.com/suzerain-io/controller-go"
+	placeholdernamecontroller "github.com/suzerain-io/placeholder-name/internal/controller"
+	"github.com/suzerain-io/placeholder-name/internal/provider"
+)
+
+type certsObserverController struct {
+	namespace           string
+	dynamicCertProvider *provider.DynamicTLSServingCertProvider
+	secretInformer      corev1informers.SecretInformer
+}
+
+func NewCertsObserverController(
+	namespace string,
+	dynamicCertProvider *provider.DynamicTLSServingCertProvider,
+	secretInformer corev1informers.SecretInformer,
+	withInformer placeholdernamecontroller.WithInformerOptionFunc,
+) controller.Controller {
+	return controller.New(
+		controller.Config{
+			Name: "certs-observer-controller",
+			Syncer: &certsObserverController{
+				namespace:           namespace,
+				dynamicCertProvider: dynamicCertProvider,
+				secretInformer:      secretInformer,
+			},
+		},
+		withInformer(
+			secretInformer,
+			placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(certsSecretName, namespace),
+			controller.InformerOption{},
+		),
+	)
+}
+
+func (c *certsObserverController) Sync(_ controller.Context) error {
+	// Try to get the secret from the informer cache.
+	certSecret, err := c.secretInformer.Lister().Secrets(c.namespace).Get(certsSecretName)
+	notFound := k8serrors.IsNotFound(err)
+	if err != nil && !notFound {
+		return fmt.Errorf("failed to get %s/%s secret: %w", c.namespace, certsSecretName, err)
+	}
+	if notFound {
+		klog.Info("certsObserverController Sync() found that the secret does not exist yet or was deleted")
+		// The secret does not exist yet or was deleted.
+		c.dynamicCertProvider.CertPEM = nil
+		c.dynamicCertProvider.KeyPEM = nil
+		return nil
+	}
+
+	// Mutate the in-memory cert provider to update with the latest cert values.
+	c.dynamicCertProvider.CertPEM = certSecret.Data[tlsCertificateChainSecretKey]
+	c.dynamicCertProvider.KeyPEM = certSecret.Data[tlsPrivateKeySecretKey]
+	klog.Info("certsObserverController Sync updated certs in the dynamic cert provider")
+	return nil
+}

internal/controller/logindiscovery/publisher.go

@@ -17,6 +17,7 @@ import (
 	"k8s.io/klog/v2"
 
 	"github.com/suzerain-io/controller-go"
+	placeholdernamecontroller "github.com/suzerain-io/placeholder-name/internal/controller"
 	crdsplaceholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/crdsplaceholder/v1alpha1"
 	placeholderclientset "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/clientset/versioned"
 	crdsplaceholderv1alpha1informers "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/informers/externalversions/crdsplaceholder/v1alpha1"
@@ -31,25 +32,6 @@ const (
 	configName = "placeholder-name-config"
 )
 
-func nameAndNamespaceExactMatchFilterFactory(name, namespace string) controller.FilterFuncs {
-	objMatchesFunc := func(obj metav1.Object) bool {
-		return obj.GetName() == name && obj.GetNamespace() == namespace
-	}
-	return controller.FilterFuncs{
-		AddFunc: objMatchesFunc,
-		UpdateFunc: func(oldObj, newObj metav1.Object) bool {
-			return objMatchesFunc(oldObj) || objMatchesFunc(newObj)
-		},
-		DeleteFunc: objMatchesFunc,
-	}
-}
-
-// Same signature as controller.WithInformer().
-type withInformerOptionFunc func(
-	getter controller.InformerGetter,
-	filter controller.Filter,
-	opt controller.InformerOption) controller.Option
-
 type publisherController struct {
 	namespace                    string
 	serverOverride               *string
@@ -64,7 +46,7 @@ func NewPublisherController(
 	placeholderClient placeholderclientset.Interface,
 	configMapInformer corev1informers.ConfigMapInformer,
 	loginDiscoveryConfigInformer crdsplaceholderv1alpha1informers.LoginDiscoveryConfigInformer,
-	withInformer withInformerOptionFunc,
+	withInformer placeholdernamecontroller.WithInformerOptionFunc,
 ) controller.Controller {
 	return controller.New(
 		controller.Config{
@@ -79,12 +61,12 @@ func NewPublisherController(
 		},
 		withInformer(
 			configMapInformer,
-			nameAndNamespaceExactMatchFilterFactory(clusterInfoName, ClusterInfoNamespace),
+			placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(clusterInfoName, ClusterInfoNamespace),
 			controller.InformerOption{},
 		),
 		withInformer(
 			loginDiscoveryConfigInformer,
-			nameAndNamespaceExactMatchFilterFactory(configName, namespace),
+			placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(configName, namespace),
 			controller.InformerOption{},
 		),
 	)

internal/controller/prepare_controllers.go

@@ -1,139 +0,0 @@
-/*
-Copyright 2020 VMware, Inc.
-SPDX-License-Identifier: Apache-2.0
-*/
-
-package controller
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	"k8s.io/apimachinery/pkg/runtime"
-	k8sinformers "k8s.io/client-go/informers"
-	"k8s.io/client-go/kubernetes"
-	restclient "k8s.io/client-go/rest"
-	aggregationv1client "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
-
-	"github.com/suzerain-io/controller-go"
-	"github.com/suzerain-io/placeholder-name/internal/autoregistration"
-	"github.com/suzerain-io/placeholder-name/internal/controller/logindiscovery"
-	placeholderclientset "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/clientset/versioned"
-	placeholderinformers "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/informers/externalversions"
-)
-
-const (
-	singletonWorker       = 1
-	defaultResyncInterval = 3 * time.Minute
-)
-
-// Prepare the controllers and their informers and return a function that will start them when called.
-func PrepareControllers(
-	ctx context.Context,
-	caBundle []byte,
-	serverInstallationNamespace string,
-	discoveryURLOverride *string,
-) (func(ctx context.Context), error) {
-	// Create k8s clients.
-	k8sClient, aggregationClient, placeholderClient, err := createClients()
-	if err != nil {
-		return nil, fmt.Errorf("could not create clients for the controllers: %w", err)
-	}
-
-	// TODO Putting this here temporarily on the way toward moving it elsewhere.
-	//  When it moves elsewhere then PrepareControllers() will not need to take ctx and caBundle parameters.
-	if err := autoregistration.UpdateAPIService(ctx, aggregationClient, caBundle); err != nil {
-		return nil, fmt.Errorf("could not update the API service: %w", err)
-	}
-
-	// Create informers.
-	k8sInformers, placeholderInformers := createInformers(serverInstallationNamespace, k8sClient, placeholderClient)
-
-	// Create controller manager.
-	controllerManager := controller.
-		NewManager().
-		WithController(
-			logindiscovery.NewPublisherController(
-				serverInstallationNamespace,
-				discoveryURLOverride,
-				placeholderClient,
-				k8sInformers.Core().V1().ConfigMaps(),
-				placeholderInformers.Crds().V1alpha1().LoginDiscoveryConfigs(),
-				controller.WithInformer,
-			),
-			singletonWorker,
-		)
-
-	// Return a function which starts the informers and controllers.
-	return func(ctx context.Context) {
-		k8sInformers.Start(ctx.Done())
-		placeholderInformers.Start(ctx.Done())
-		go controllerManager.Start(ctx)
-	}, nil
-}
-
-// Create the k8s clients that will be used by the controllers.
-func createClients() (*kubernetes.Clientset, *aggregationv1client.Clientset, *placeholderclientset.Clientset, error) {
-	// Load the Kubernetes client configuration (kubeconfig),
-	kubeConfig, err := restclient.InClusterConfig()
-	if err != nil {
-		return nil, nil, nil, fmt.Errorf("could not load in-cluster configuration: %w", err)
-	}
-
-	// explicitly use protobuf when talking to built-in kube APIs
-	protoKubeConfig := createProtoKubeConfig(kubeConfig)
-
-	// Connect to the core Kubernetes API.
-	k8sClient, err := kubernetes.NewForConfig(protoKubeConfig)
-	if err != nil {
-		return nil, nil, nil, fmt.Errorf("could not initialize Kubernetes client: %w", err)
-	}
-
-	// Connect to the Kubernetes aggregation API.
-	aggregationClient, err := aggregationv1client.NewForConfig(protoKubeConfig)
-	if err != nil {
-		return nil, nil, nil, fmt.Errorf("could not initialize Kubernetes client: %w", err)
-	}
-
-	// Connect to the placeholder API.
-	// I think we can't use protobuf encoding here because we are using CRDs
-	// (for which protobuf encoding is not supported).
-	placeholderClient, err := placeholderclientset.NewForConfig(kubeConfig)
-	if err != nil {
-		return nil, nil, nil, fmt.Errorf("could not initialize placeholder client: %w", err)
-	}
-
-	return k8sClient, aggregationClient, placeholderClient, nil
-}
-
-// Create the informers that will be used by the controllers.
-func createInformers(
-	serverInstallationNamespace string,
-	k8sClient *kubernetes.Clientset,
-	placeholderClient *placeholderclientset.Clientset,
-) (k8sinformers.SharedInformerFactory, placeholderinformers.SharedInformerFactory) {
-	k8sInformers := k8sinformers.NewSharedInformerFactoryWithOptions(
-		k8sClient,
-		defaultResyncInterval,
-		k8sinformers.WithNamespace(
-			logindiscovery.ClusterInfoNamespace,
-		),
-	)
-	placeholderInformers := placeholderinformers.NewSharedInformerFactoryWithOptions(
-		placeholderClient,
-		defaultResyncInterval,
-		placeholderinformers.WithNamespace(serverInstallationNamespace),
-	)
-	return k8sInformers, placeholderInformers
-}
-
-// Returns a copy of the input config with the ContentConfig set to use protobuf.
-// Do not use this config to communicate with any CRD based APIs.
-func createProtoKubeConfig(kubeConfig *restclient.Config) *restclient.Config {
-	protoKubeConfig := restclient.CopyConfig(kubeConfig)
-	const protoThenJSON = runtime.ContentTypeProtobuf + "," + runtime.ContentTypeJSON
-	protoKubeConfig.AcceptContentTypes = protoThenJSON
-	protoKubeConfig.ContentType = runtime.ContentTypeProtobuf
-	return protoKubeConfig
-}

internal/controller/utils.go

@@ -0,0 +1,31 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package controller
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/suzerain-io/controller-go"
+)
+
+func NameAndNamespaceExactMatchFilterFactory(name, namespace string) controller.FilterFuncs {
+	objMatchesFunc := func(obj metav1.Object) bool {
+		return obj.GetName() == name && obj.GetNamespace() == namespace
+	}
+	return controller.FilterFuncs{
+		AddFunc: objMatchesFunc,
+		UpdateFunc: func(oldObj, newObj metav1.Object) bool {
+			return objMatchesFunc(oldObj) || objMatchesFunc(newObj)
+		},
+		DeleteFunc: objMatchesFunc,
+	}
+}
+
+// Same signature as controller.WithInformer().
+type WithInformerOptionFunc func(
+	getter controller.InformerGetter,
+	filter controller.Filter,
+	opt controller.InformerOption) controller.Option