From 52546fad901f272127e55912de5fe77f200c5a56 Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Fri, 31 Jul 2020 12:08:07 -0400 Subject: [PATCH] WIP: start on publisher controller integration --- deploy/crd.yaml | 4 +- deploy/rbac.yaml | 27 ++++ internal/apiserver/apiserver.go | 20 ++- .../controller/logindiscovery/publisher.go | 9 +- internal/server/server.go | 85 +++++++++++-- test/integration/logindiscoveryconfig_test.go | 116 ++++++++++++++++++ 6 files changed, 242 insertions(+), 19 deletions(-) create mode 100644 test/integration/logindiscoveryconfig_test.go diff --git a/deploy/crd.yaml b/deploy/crd.yaml index de8147751..a859de09b 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -15,9 +15,9 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: logindiscoveryconfigs.suzerain-io.github.io + name: logindiscoveryconfigs.placeholder.suzerain-io.github.io spec: - group: suzerain-io.github.io + group: placeholder.suzerain-io.github.io versions: #! Any changes to these schemas should also be reflected in the types.go file(s) #! in https://github.com/suzerain-io/placeholder-name-api/tree/main/pkg/apis/placeholder diff --git a/deploy/rbac.yaml b/deploy/rbac.yaml index e3d0858a4..ba6cec000 100644 --- a/deploy/rbac.yaml +++ b/deploy/rbac.yaml @@ -38,6 +38,9 @@ rules: - apiGroups: [""] resources: [services] verbs: [create, get, list, patch, update, watch] + - apiGroups: [placeholder.suzerain-io.github.io] + resources: [logindiscoveryconfigs] + verbs: [create, get, list, update, watch] --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -108,3 +111,27 @@ roleRef: #! give permissions for a special configmap of CA bundles that is needed by aggregated api servers name: extension-apiserver-authentication-reader apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: #@ data.values.app_name + "-cluster-info-lister-watcher-role" + namespace: kube-public +rules: + - apiGroups: [""] + resources: [configmaps] + verbs: [list, watch] #! TODO: do we neeed a get here for the controller? +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: #@ data.values.app_name + "-cluster-info-lister-watcher-role-binding" + namespace: kube-public +subjects: + - kind: ServiceAccount + name: #@ data.values.app_name + "-service-account" + namespace: #@ data.values.namespace +roleRef: + kind: Role + name: #@ data.values.app_name + "-cluster-info-lister-watcher-role" + apiGroup: rbac.authorization.k8s.io diff --git a/internal/apiserver/apiserver.go b/internal/apiserver/apiserver.go index 2fb42b0f1..338e740bf 100644 --- a/internal/apiserver/apiserver.go +++ b/internal/apiserver/apiserver.go @@ -6,6 +6,7 @@ SPDX-License-Identifier: Apache-2.0 package apiserver import ( + "context" "fmt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -56,8 +57,9 @@ type Config struct { } type ExtraConfig struct { - Webhook authenticator.Token - Issuer loginrequest.CertIssuer + Webhook authenticator.Token + Issuer loginrequest.CertIssuer + StartControllersPostStartHook func(ctx context.Context) } type PlaceHolderServer struct { @@ -122,9 +124,17 @@ func (c completedConfig) New() (*PlaceHolderServer, error) { return nil, fmt.Errorf("install API group error: %w", err) } - s.GenericAPIServer.AddPostStartHookOrDie("place-holder-post-start-hook", - func(context genericapiserver.PostStartHookContext) error { - klog.InfoS("post start hook", "foo", "bar") + s.GenericAPIServer.AddPostStartHookOrDie("start-controllers", + func(postStartContext genericapiserver.PostStartHookContext) error { + klog.InfoS("post start hook") + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + <-postStartContext.StopCh + cancel() + }() + c.ExtraConfig.StartControllersPostStartHook(ctx) + return nil }, ) diff --git a/internal/controller/logindiscovery/publisher.go b/internal/controller/logindiscovery/publisher.go index 0d6d04b8e..0725437b2 100644 --- a/internal/controller/logindiscovery/publisher.go +++ b/internal/controller/logindiscovery/publisher.go @@ -23,8 +23,9 @@ import ( ) const ( + ClusterInfoNamespace = "kube-public" + clusterInfoName = "cluster-info" - clusterInfoNamespace = "kube-public" clusterInfoConfigMapKey = "kubeconfig" configName = "placeholder-name-config" @@ -75,7 +76,7 @@ func NewPublisherController( }, withInformer( configMapInformer, - nameAndNamespaceExactMatchFilterFactory(clusterInfoName, clusterInfoNamespace), + nameAndNamespaceExactMatchFilterFactory(clusterInfoName, ClusterInfoNamespace), controller.InformerOption{}, ), withInformer( @@ -89,7 +90,7 @@ func NewPublisherController( func (c *publisherController) Sync(ctx controller.Context) error { configMap, err := c.configMapInformer. Lister(). - ConfigMaps(clusterInfoNamespace). + ConfigMaps(ClusterInfoNamespace). Get(clusterInfoName) notFound := k8serrors.IsNotFound(err) if err != nil && !notFound { @@ -99,7 +100,7 @@ func (c *publisherController) Sync(ctx controller.Context) error { klog.InfoS( "could not find config map", "configmap", - klog.KRef(clusterInfoNamespace, clusterInfoName), + klog.KRef(ClusterInfoNamespace, clusterInfoName), ) return nil } diff --git a/internal/server/server.go b/internal/server/server.go index 442e4fd5d..550ba745c 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -25,20 +25,30 @@ import ( "k8s.io/apiserver/pkg/server/dynamiccertificates" genericoptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/plugin/pkg/authenticator/token/webhook" + k8sinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" - corev1client "k8s.io/client-go/kubernetes/typed/core/v1" restclient "k8s.io/client-go/rest" apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" aggregationv1client "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" + "github.com/suzerain-io/controller-go" placeholderv1alpha1 "github.com/suzerain-io/placeholder-name-api/pkg/apis/placeholder/v1alpha1" + placeholderclientset "github.com/suzerain-io/placeholder-name-client-go/pkg/generated/clientset/versioned" + placeholderinformers "github.com/suzerain-io/placeholder-name-client-go/pkg/generated/informers/externalversions" "github.com/suzerain-io/placeholder-name/internal/apiserver" "github.com/suzerain-io/placeholder-name/internal/autoregistration" "github.com/suzerain-io/placeholder-name/internal/certauthority" + "github.com/suzerain-io/placeholder-name/internal/controller/logindiscovery" "github.com/suzerain-io/placeholder-name/internal/downward" "github.com/suzerain-io/placeholder-name/pkg/config" ) +// TODO(akeesler): what should these controller settings be? +const ( + defaultWorkers = 3 + defaultResync = 20 * time.Minute +) + // App is an object that represents the placeholder-name-server application. type App struct { cmd *cobra.Command @@ -94,7 +104,15 @@ authenticating to the Kubernetes API.`, return fmt.Errorf("could not initialize Kubernetes client: %w", err) } - return a.run(ctx, k8s.CoreV1(), aggregation) + // Connect to the placeholder API. + // I think we can't use protobuf encoding here because we are using CRDs + // (for which protobuf encoding is not supported). + placeholder, err := placeholderclientset.NewForConfig(kubeConfig) + if err != nil { + return fmt.Errorf("could not initialize placeholder client: %w", err) + } + + return a.run(ctx, k8s, aggregation, placeholder) }, Args: cobra.NoArgs, } @@ -143,8 +161,9 @@ func (a *App) Run() error { func (a *App) run( ctx context.Context, - k8s corev1client.CoreV1Interface, + k8s kubernetes.Interface, aggregation aggregationv1client.Interface, + placeholder placeholderclientset.Interface, ) error { cfg, err := config.FromPath(a.configPath) if err != nil { @@ -213,7 +232,7 @@ func (a *App) run( }, } if err := autoregistration.Setup(ctx, autoregistration.SetupOptions{ - CoreV1: k8s, + CoreV1: k8s.CoreV1(), AggregationV1: aggregation, Namespace: podinfo.Namespace, ServiceTemplate: service, @@ -222,7 +241,13 @@ func (a *App) run( return fmt.Errorf("could not register API service: %w", err) } - apiServerConfig, err := a.ConfigServer(cert, webhookTokenAuthenticator, clientCA) + cmrf := wireControllerManagerRunFunc(podinfo, k8s, placeholder) + apiServerConfig, err := a.configServer( + cert, + webhookTokenAuthenticator, + clientCA, + cmrf, + ) if err != nil { return err } @@ -235,7 +260,12 @@ func (a *App) run( return server.GenericAPIServer.PrepareRun().Run(ctx.Done()) } -func (a *App) ConfigServer(cert *tls.Certificate, webhookTokenAuthenticator *webhook.WebhookTokenAuthenticator, ca *certauthority.CA) (*apiserver.Config, error) { +func (a *App) configServer( + cert *tls.Certificate, + webhookTokenAuthenticator *webhook.WebhookTokenAuthenticator, + ca *certauthority.CA, + startControllersPostStartHook func(context.Context), +) (*apiserver.Config, error) { provider, err := createStaticCertKeyProvider(cert) if err != nil { return nil, fmt.Errorf("could not create static cert key provider: %w", err) @@ -250,8 +280,9 @@ func (a *App) ConfigServer(cert *tls.Certificate, webhookTokenAuthenticator *web apiServerConfig := &apiserver.Config{ GenericConfig: serverConfig, ExtraConfig: apiserver.ExtraConfig{ - Webhook: webhookTokenAuthenticator, - Issuer: ca, + Webhook: webhookTokenAuthenticator, + Issuer: ca, + StartControllersPostStartHook: startControllersPostStartHook, }, } return apiServerConfig, nil @@ -290,3 +321,41 @@ func createStaticCertKeyProvider(cert *tls.Certificate) (dynamiccertificates.Cer return dynamiccertificates.NewStaticCertKeyContent("some-name???", certChainPEM, privateKeyPEM) } + +func wireControllerManagerRunFunc( + podinfo *downward.PodInfo, + k8s kubernetes.Interface, + placeholder placeholderclientset.Interface, +) func(ctx context.Context) { + k8sInformers := k8sinformers.NewSharedInformerFactoryWithOptions( + k8s, + defaultResync, + k8sinformers.WithNamespace( + logindiscovery.ClusterInfoNamespace, + ), + ) + placeholderInformers := placeholderinformers.NewSharedInformerFactoryWithOptions( + placeholder, + defaultResync, + placeholderinformers.WithNamespace( + "integration", // TODO(akeesler): unhardcode this. + ), + ) + cm := controller. + NewManager(). + WithController( + logindiscovery.NewPublisherController( + podinfo.Namespace, + placeholder, + k8sInformers.Core().V1().ConfigMaps(), + placeholderInformers.Placeholder().V1alpha1().LoginDiscoveryConfigs(), + controller.WithInformer, + ), + defaultWorkers, + ) + return func(ctx context.Context) { + k8sInformers.Start(ctx.Done()) + placeholderInformers.Start(ctx.Done()) + go cm.Start(ctx) + } +} diff --git a/test/integration/logindiscoveryconfig_test.go b/test/integration/logindiscoveryconfig_test.go new file mode 100644 index 000000000..043c82981 --- /dev/null +++ b/test/integration/logindiscoveryconfig_test.go @@ -0,0 +1,116 @@ +/* +Copyright 2020 VMware, Inc. +SPDX-License-Identifier: Apache-2.0 +*/ + +package integration + +import ( + "context" + "encoding/base64" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/rest" + + placeholderv1alpha1 "github.com/suzerain-io/placeholder-name-api/pkg/apis/placeholder/v1alpha1" + "github.com/suzerain-io/placeholder-name/test/library" +) + +func TestSuccessfulLoginDiscoveryConfig(t *testing.T) { + namespaceName := os.Getenv("PLACEHOLDER_NAME_NAMESPACE") + require.NotEmptyf(t, namespaceName, "must specify PLACEHOLDER_NAME_NAMESPACE env var for integration tests") + + client := library.NewPlaceholderNameClientset(t) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // TODO(akeesler): is there a race here between this test running and the + // placeholder-name-server creating the CR? + + config := library.NewClientConfig(t) + expectedLDC := getExpectedLDC(namespaceName, config) + configList, err := client. + PlaceholderV1alpha1(). + LoginDiscoveryConfigs(namespaceName). + List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + require.Len(t, configList.Items, 1) + require.Equal(t, expectedLDC, configList.Items[0]) +} + +func TestReconcilingLoginDiscoveryConfig(t *testing.T) { + t.Skip() + + namespaceName := os.Getenv("PLACEHOLDER_NAME_NAMESPACE") + require.NotEmptyf(t, namespaceName, "must specify PLACEHOLDER_NAME_NAMESPACE env var for integration tests") + + client := library.NewPlaceholderNameClientset(t) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // TODO(akeesler): is there a race here between this test running and the + // placeholder-name-server creating the CR? + + w, err := client. + PlaceholderV1alpha1(). + LoginDiscoveryConfigs(namespaceName). + Watch(ctx, metav1.ListOptions{}) + require.NoError(t, err) + + err = client. + PlaceholderV1alpha1(). + LoginDiscoveryConfigs(namespaceName). + Delete(ctx, "placeholder-name-config", metav1.DeleteOptions{}) + require.NoError(t, err) + + config := library.NewClientConfig(t) + expectedLDC := getExpectedLDC(namespaceName, config) + received := func(et watch.EventType, o runtime.Object) func() bool { + return func() bool { + select { + case e := <-w.ResultChan(): + require.Equal(t, et, e.Type) + require.Equal(t, o, e.Object) + return true + default: + return false + } + } + } + require.Eventually( + t, + received(watch.Deleted, expectedLDC), + time.Second, + 3*time.Second, + ) + require.Eventually( + t, + received(watch.Added, expectedLDC), + time.Second, + 3*time.Second, + ) +} + +func getExpectedLDC( + namespaceName string, + config *rest.Config, +) *placeholderv1alpha1.LoginDiscoveryConfig { + return &placeholderv1alpha1.LoginDiscoveryConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: "placeholder-name-config", + Namespace: namespaceName, + }, + Spec: placeholderv1alpha1.LoginDiscoveryConfigSpec{ + Server: config.Host, + CertificateAuthorityData: base64.StdEncoding.EncodeToString(config.TLSClientConfig.CAData), + }, + } +}