Allow multiple Pinnipeds to work on same cluster

Yes, this is a huge commit. The middleware allows you to customize the API groups of all of the *.pinniped.dev API groups. Some notes about other small things in this commit: - We removed the internal/client package in favor of pkg/conciergeclient. The two packages do basically the same thing. I don't think we use the former anymore. - We re-enabled cluster-scoped owner assertions in the integration tests. This code was added in internal/ownerref. See a0546942 for when this assertion was removed. - Note: the middlware code is in charge of restoring the GV of a request object, so we should never need to write mutations that do that. - We updated the supervisor secret generation to no longer manually set an owner reference to the deployment since the middleware code now does this. I think we still need some way to make an initial event for the secret generator controller, which involves knowing the namespace and the name of the generated secret, so I still wired the deployment through. We could use a namespace/name tuple here, but I was lazy. Signed-off-by: Andrew Keesler <akeesler@vmware.com> Co-authored-by: Ryan Richard <richardry@vmware.com>
2026-09-05 23:57:12 +00:00 · 2021-02-02 15:18:41 -08:00
parent 93d25a349f
commit efe1fa89fe
67 changed files with 4285 additions and 820 deletions
@@ -0,0 +1,7 @@
+// Copyright 2021 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package testutil contains shared test utilities for the Pinniped project.
+//
+// As of right now, it is more or less a dumping ground for our test utilities.
+package testutil
@@ -0,0 +1,221 @@
+// Copyright 2021 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+// Package fakekubeapi contains a *very* simple httptest.Server that can be used to stand in for
+// a real Kube API server in tests.
+//
+// Usage:
+//   func TestSomething(t *testing.T) {
+//     resources := map[string]kubeclient.Object{
+//       // store preexisting resources here
+//       "/api/v1/namespaces/default/pods/some-pod-name": &corev1.Pod{...},
+//     }
+//     server, restConfig := fakekubeapi.Start(t, resources)
+//     defer server.Close()
+//     client := kubeclient.New(kubeclient.WithConfig(restConfig))
+//     // do stuff with client...
+//   }
+package fakekubeapi
+
+import (
+	"encoding/pem"
+	"io/ioutil"
+	"mime"
+	"net/http"
+	"net/http/httptest"
+	"path"
+	"strings"
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	kubescheme "k8s.io/client-go/kubernetes/scheme"
+	restclient "k8s.io/client-go/rest"
+	aggregatorclientscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme"
+
+	pinnipedconciergeclientsetscheme "go.pinniped.dev/generated/1.20/client/concierge/clientset/versioned/scheme"
+	pinnipedsupervisorclientsetscheme "go.pinniped.dev/generated/1.20/client/supervisor/clientset/versioned/scheme"
+	"go.pinniped.dev/internal/httputil/httperr"
+	"go.pinniped.dev/internal/multierror"
+)
+
+// Unlike the standard httperr.New(), this one does not prepend error messages with any prefix.
+type plainHTTPErr struct {
+	code int
+	msg  string
+}
+
+func (e plainHTTPErr) Error() string {
+	return e.msg
+}
+
+func (e plainHTTPErr) Respond(w http.ResponseWriter) {
+	http.Error(w, e.msg, e.code)
+}
+
+// Start starts an httptest.Server (with TLS) that pretends to be a Kube API server.
+//
+// The server uses the provided resources map to store API Object's. The map should be from API path
+// to Object (e.g., /api/v1/namespaces/default/pods/some-pod-name => &corev1.Pod{}).
+//
+// Start returns an already started httptest.Server and a restclient.Config that can be used to talk
+// to the server.
+//
+// Note! Only these following verbs are (partially) supported: create, get, update, delete.
+func Start(t *testing.T, resources map[string]metav1.Object) (*httptest.Server, *restclient.Config) {
+	if resources == nil {
+		resources = make(map[string]metav1.Object)
+	}
+
+	server := httptest.NewTLSServer(httperr.HandlerFunc(func(w http.ResponseWriter, r *http.Request) (err error) {
+		obj, err := decodeObj(r)
+		if err != nil {
+			return err
+		}
+
+		obj, err = handleObj(r, obj, resources)
+		if err != nil {
+			return err
+		}
+
+		if r.Method != http.MethodDelete && obj == nil {
+			return &plainHTTPErr{
+				code: http.StatusNotFound,
+				// This is representative of a real Kube 404 message body.
+				msg: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"not found","reason":"NotFound","details":{"name":"not-found","kind":"pods"},"code":404}`,
+			}
+		}
+
+		if err := encodeObj(w, r, obj); err != nil {
+			return err
+		}
+
+		return nil
+	}))
+	restConfig := &restclient.Config{
+		Host: server.URL,
+		TLSClientConfig: restclient.TLSClientConfig{
+			CAData: pem.EncodeToMemory(&pem.Block{Bytes: server.Certificate().Raw, Type: "CERTIFICATE"}),
+		},
+	}
+	return server, restConfig
+}
+
+func decodeObj(r *http.Request) (metav1.Object, error) {
+	switch r.Method {
+	case http.MethodPut, http.MethodPost:
+	default:
+		return nil, nil
+	}
+
+	contentType := r.Header.Get("Content-Type")
+	if len(contentType) == 0 {
+		return nil, httperr.New(http.StatusUnsupportedMediaType, "empty content-type header is not allowed")
+	}
+
+	mediaType, _, err := mime.ParseMediaType(contentType)
+	if err != nil {
+		return nil, httperr.Wrap(http.StatusUnsupportedMediaType, "could not parse mime type from content-type header", err)
+	}
+
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		return nil, httperr.Wrap(http.StatusInternalServerError, "read body", err)
+	}
+
+	var obj metav1.Object
+	multiErr := multierror.New()
+	codecsThatWeUseInOurCode := []runtime.NegotiatedSerializer{
+		kubescheme.Codecs,
+		aggregatorclientscheme.Codecs,
+		pinnipedconciergeclientsetscheme.Codecs,
+		pinnipedsupervisorclientsetscheme.Codecs,
+	}
+	for _, codec := range codecsThatWeUseInOurCode {
+		obj, err = tryDecodeObj(mediaType, body, codec)
+		if err == nil {
+			return obj, nil
+		}
+		multiErr.Add(err)
+	}
+	return nil, multiErr.ErrOrNil()
+}
+
+func tryDecodeObj(
+	mediaType string,
+	body []byte,
+	negotiatedSerializer runtime.NegotiatedSerializer,
+) (metav1.Object, error) {
+	serializerInfo, ok := runtime.SerializerInfoForMediaType(negotiatedSerializer.SupportedMediaTypes(), mediaType)
+	if !ok {
+		return nil, httperr.Newf(http.StatusInternalServerError, "unable to find serialier with content-type %s", mediaType)
+	}
+
+	obj, err := runtime.Decode(serializerInfo.Serializer, body)
+	if err != nil {
+		return nil, httperr.Wrap(http.StatusInternalServerError, "decode obj", err)
+	}
+
+	return obj.(metav1.Object), nil
+}
+
+func handleObj(r *http.Request, obj metav1.Object, resources map[string]metav1.Object) (metav1.Object, error) {
+	switch r.Method {
+	case http.MethodGet:
+		obj = resources[r.URL.Path]
+	case http.MethodPost, http.MethodPut:
+		resources[path.Join(r.URL.Path, obj.GetName())] = obj
+	case http.MethodDelete:
+		if _, ok := resources[r.URL.Path]; !ok {
+			return nil, httperr.Newf(http.StatusNotFound, "no resource with path %q", r.URL.Path)
+		}
+		delete(resources, r.URL.Path)
+	default:
+		return nil, httperr.New(http.StatusMethodNotAllowed, "check source code for methods supported")
+	}
+
+	return obj, nil
+}
+
+func encodeObj(w http.ResponseWriter, r *http.Request, obj metav1.Object) error {
+	if r.Method == http.MethodDelete {
+		return nil
+	}
+
+	accepts := strings.Split(r.Header.Get("Accept"), ",")
+	contentType := findGoodContentType(accepts)
+	if len(contentType) == 0 {
+		return httperr.Newf(http.StatusUnsupportedMediaType, "can't find good content type in %s", accepts)
+	}
+
+	mediaType, _, err := mime.ParseMediaType(contentType)
+	if err != nil {
+		return httperr.Wrap(http.StatusUnsupportedMediaType, "could not parse mime type from accept header", err)
+	}
+
+	serializerInfo, ok := runtime.SerializerInfoForMediaType(kubescheme.Codecs.SupportedMediaTypes(), mediaType)
+	if !ok {
+		return httperr.Newf(http.StatusInternalServerError, "unable to find serialier with content-type %s", mediaType)
+	}
+
+	data, err := runtime.Encode(serializerInfo.Serializer, obj.(runtime.Object))
+	if err != nil {
+		return httperr.Wrap(http.StatusInternalServerError, "decode obj", err)
+	}
+
+	w.Header().Set("Content-Type", contentType)
+	if _, err := w.Write(data); err != nil {
+		return httperr.Wrap(http.StatusInternalServerError, "write response", err)
+	}
+
+	return nil
+}
+
+func findGoodContentType(contentTypes []string) string {
+	for _, contentType := range contentTypes {
+		if strings.Contains(contentType, "json") || strings.Contains(contentType, "yaml") || strings.Contains(contentType, "protobuf") {
+			return contentType
+		}
+	}
+	return ""
+}
@@ -0,0 +1,70 @@
+// Copyright 2021 the Pinniped contributors. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package testutil
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+
+	"go.pinniped.dev/internal/kubeclient"
+)
+
+// RoundTrip is an implementation of kubeclient.RoundTrip that is easy to use in tests.
+type RoundTrip struct {
+	verb            kubeclient.Verb
+	namespace       string
+	namespaceScoped bool
+	resource        schema.GroupVersionResource
+	subresource     string
+
+	MutateRequests, MutateResponses []func(kubeclient.Object)
+}
+
+func (rt *RoundTrip) WithVerb(verb kubeclient.Verb) *RoundTrip {
+	rt.verb = verb
+	return rt
+}
+
+func (rt *RoundTrip) Verb() kubeclient.Verb {
+	return rt.verb
+}
+
+func (rt *RoundTrip) WithNamespace(namespace string) *RoundTrip {
+	rt.namespace = namespace
+	rt.namespaceScoped = len(namespace) != 0
+	return rt
+}
+
+func (rt *RoundTrip) Namespace() string {
+	return rt.namespace
+}
+
+func (rt *RoundTrip) NamespaceScoped() bool {
+	return rt.namespaceScoped
+}
+
+func (rt *RoundTrip) WithResource(resource schema.GroupVersionResource) *RoundTrip {
+	rt.resource = resource
+	return rt
+}
+
+func (rt *RoundTrip) Resource() schema.GroupVersionResource {
+	return rt.resource
+}
+
+func (rt *RoundTrip) WithSubresource(subresource string) *RoundTrip {
+	rt.subresource = subresource
+	return rt
+}
+
+func (rt *RoundTrip) Subresource() string {
+	return rt.subresource
+}
+
+func (rt *RoundTrip) MutateRequest(fn func(kubeclient.Object)) {
+	rt.MutateRequests = append(rt.MutateRequests, fn)
+}
+
+func (rt *RoundTrip) MutateResponse(fn func(kubeclient.Object)) {
+	rt.MutateResponses = append(rt.MutateResponses, fn)
+}