mirrors/pinniped
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/pinniped.git synced 2026-01-07 14:05:50 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

impersonatorconfig: only unload dynamiccert when proxy is disabled

Browse Source 
In the upstream dynamiccertificates package, we rely on two pieces
of code:

1. DynamicServingCertificateController.newTLSContent which calls
   - clientCA.CurrentCABundleContent
   - servingCert.CurrentCertKeyContent
2. unionCAContent.VerifyOptions which calls
   - unionCAContent.CurrentCABundleContent

This results in calls to our tlsServingCertDynamicCertProvider and
impersonationSigningCertProvider.  If we Unset these providers, we
subtly break these consumers.  At best this results in test slowness
and flakes while we wait for reconcile loops to converge.  At worst,
it results in actual errors during runtime.  For example, we
previously would Unset the impersonationSigningCertProvider on any
sync loop error (even a transient one caused by a network blip or
a conflict between writes from different replicas of the concierge).
This would cause us to transiently fail to issue new certificates
from the token credential require API.  It would also cause us to
transiently fail to authenticate previously issued client certs
(which results in occasional Unauthorized errors in CI).

Signed-off-by: Monis Khan <mok@vmware.com>
This commit is contained in:
Monis Khan
2021-08-10 12:05:04 -04:00
parent bb30569e41
commit 7a812ac5ed
6 changed files with 344 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
internal/controller/apicerts/certs_expirer_test.go
View File

@@ -20,9 +20,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
kubeinformers "k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
kubernetesfake "k8s.io/client-go/kubernetes/fake"
corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
kubetesting "k8s.io/client-go/testing"
"go.pinniped.dev/internal/controllerlib"
@@ -253,7 +251,8 @@ func TestExpirerControllerSync(t *testing.T) {
0,
)
trackDeleteClient := &clientWrapper{Interface: kubeAPIClient, opts: &[]metav1.DeleteOptions{}}
opts := &[]metav1.DeleteOptions{}
trackDeleteClient := testutil.NewDeleteOptionsRecorder(kubeAPIClient, opts)
c := NewCertsExpirerController(
namespace,
@@ -297,44 +296,16 @@ func TestExpirerControllerSync(t *testing.T) {
require.Equal(t, exActions, acActions)
if test.wantDelete {
require.Len(t, *trackDeleteClient.opts, 1)
require.Len(t, *opts, 1)
require.Equal(t, metav1.DeleteOptions{
Preconditions: &metav1.Preconditions{
UID: &testUID,
ResourceVersion: &testRV,
},
}, (*trackDeleteClient.opts)[0])
}, (*opts)[0])
} else {
require.Len(t, *trackDeleteClient.opts, 0)
require.Len(t, *opts, 0)
}
})
}
}
type clientWrapper struct {
kubernetes.Interface
opts *[]metav1.DeleteOptions
}
func (c *clientWrapper) CoreV1() corev1client.CoreV1Interface {
return &coreWrapper{CoreV1Interface: c.Interface.CoreV1(), opts: c.opts}
}
type coreWrapper struct {
corev1client.CoreV1Interface
opts *[]metav1.DeleteOptions
}
func (c *coreWrapper) Secrets(namespace string) corev1client.SecretInterface {
return &secretsWrapper{SecretInterface: c.CoreV1Interface.Secrets(namespace), opts: c.opts}
}
type secretsWrapper struct {
corev1client.SecretInterface
opts *[]metav1.DeleteOptions
}
func (s *secretsWrapper) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error {
*s.opts = append(*s.opts, opts)
return s.SecretInterface.Delete(ctx, name, opts)
}

94
internal/controller/impersonatorconfig/impersonator_config.go
View File

@@ -183,8 +183,6 @@ func (c *impersonatorConfigController) Sync(syncCtx controllerlib.Context) error
Message: err.Error(),
LastUpdateTime: metav1.NewTime(c.clock.Now()),
}
// The impersonator is not ready, so clear the signer CA from the dynamic provider.
c.clearSignerCA()
}
err = utilerrors.NewAggregate([]error{err, issuerconfig.Update(
@@ -281,27 +279,32 @@ func (c *impersonatorConfigController) doSync(syncCtx controllerlib.Context, cre
nameInfo, err := c.findDesiredTLSCertificateName(impersonationSpec)
if err != nil {
// Unexpected error while determining the name that should go into the certs, so clear any existing certs.
c.tlsServingCertDynamicCertProvider.UnsetCertKeyContent()
return nil, err
}
var impersonationCA *certauthority.CA
if c.shouldHaveTLSSecret(impersonationSpec) {
if c.shouldHaveImpersonator(impersonationSpec) {
if impersonationCA, err = c.ensureCASecretIsCreated(ctx); err != nil {
return nil, err
}
if err = c.ensureTLSSecret(ctx, nameInfo, impersonationCA); err != nil {
return nil, err
}
} else if err = c.ensureTLSSecretIsRemoved(ctx); err != nil {
return nil, err
} else {
if err = c.ensureTLSSecretIsRemoved(ctx); err != nil {
return nil, err
}
c.clearTLSSecret()
}
credentialIssuerStrategyResult := c.doSyncResult(nameInfo, impersonationSpec, impersonationCA)
if err = c.loadSignerCA(credentialIssuerStrategyResult.Status); err != nil {
return nil, err
if c.shouldHaveImpersonator(impersonationSpec) {
if err = c.loadSignerCA(); err != nil {
return nil, err
}
} else {
c.clearSignerCA()
}
return credentialIssuerStrategyResult, nil
@@ -350,20 +353,16 @@ func (c *impersonatorConfigController) shouldHaveClusterIPService(config *v1alph
return c.shouldHaveImpersonator(config) && config.Service.Type == v1alpha1.ImpersonationProxyServiceTypeClusterIP
}
func (c *impersonatorConfigController) shouldHaveTLSSecret(config *v1alpha1.ImpersonationProxySpec) bool {
return c.shouldHaveImpersonator(config)
}
func (c *impersonatorConfigController) serviceExists(serviceName string) (bool, error) {
_, err := c.servicesInformer.Lister().Services(c.namespace).Get(serviceName)
func (c *impersonatorConfigController) serviceExists(serviceName string) (bool, *v1.Service, error) {
service, err := c.servicesInformer.Lister().Services(c.namespace).Get(serviceName)
notFound := k8serrors.IsNotFound(err)
if notFound {
return false, nil
return false, nil, nil
}
if err != nil {
return false, err
return false, nil, err
}
return true, nil
return true, service, nil
}
func (c *impersonatorConfigController) tlsSecretExists() (bool, *v1.Secret, error) {
@@ -477,7 +476,7 @@ func (c *impersonatorConfigController) ensureLoadBalancerIsStarted(ctx context.C
}
func (c *impersonatorConfigController) ensureLoadBalancerIsStopped(ctx context.Context) error {
running, err := c.serviceExists(c.generatedLoadBalancerServiceName)
running, service, err := c.serviceExists(c.generatedLoadBalancerServiceName)
if err != nil {
return err
}
@@ -488,7 +487,12 @@ func (c *impersonatorConfigController) ensureLoadBalancerIsStopped(ctx context.C
c.infoLog.Info("deleting load balancer for impersonation proxy",
"service", klog.KRef(c.namespace, c.generatedLoadBalancerServiceName),
)
err = c.k8sClient.CoreV1().Services(c.namespace).Delete(ctx, c.generatedLoadBalancerServiceName, metav1.DeleteOptions{})
err = c.k8sClient.CoreV1().Services(c.namespace).Delete(ctx, c.generatedLoadBalancerServiceName, metav1.DeleteOptions{
Preconditions: &metav1.Preconditions{
UID: &service.UID,
ResourceVersion: &service.ResourceVersion,
},
})
return utilerrors.FilterOut(err, k8serrors.IsNotFound)
}
@@ -517,7 +521,7 @@ func (c *impersonatorConfigController) ensureClusterIPServiceIsStarted(ctx conte
}
func (c *impersonatorConfigController) ensureClusterIPServiceIsStopped(ctx context.Context) error {
running, err := c.serviceExists(c.generatedClusterIPServiceName)
running, service, err := c.serviceExists(c.generatedClusterIPServiceName)
if err != nil {
return err
}
@@ -528,7 +532,12 @@ func (c *impersonatorConfigController) ensureClusterIPServiceIsStopped(ctx conte
c.infoLog.Info("deleting cluster ip for impersonation proxy",
"service", klog.KRef(c.namespace, c.generatedClusterIPServiceName),
)
err = c.k8sClient.CoreV1().Services(c.namespace).Delete(ctx, c.generatedClusterIPServiceName, metav1.DeleteOptions{})
err = c.k8sClient.CoreV1().Services(c.namespace).Delete(ctx, c.generatedClusterIPServiceName, metav1.DeleteOptions{
Preconditions: &metav1.Preconditions{
UID: &service.UID,
ResourceVersion: &service.ResourceVersion,
},
})
return utilerrors.FilterOut(err, k8serrors.IsNotFound)
}
@@ -942,7 +951,6 @@ func (c *impersonatorConfigController) loadTLSCertFromSecret(tlsSecret *v1.Secre
keyPEM := tlsSecret.Data[v1.TLSPrivateKeyKey]
if err := c.tlsServingCertDynamicCertProvider.SetCertKeyContent(certPEM, keyPEM); err != nil {
c.tlsServingCertDynamicCertProvider.UnsetCertKeyContent()
return fmt.Errorf("could not parse TLS cert PEM data from Secret: %w", err)
}
@@ -955,39 +963,33 @@ func (c *impersonatorConfigController) loadTLSCertFromSecret(tlsSecret *v1.Secre
}
func (c *impersonatorConfigController) ensureTLSSecretIsRemoved(ctx context.Context) error {
tlsSecretExists, _, err := c.tlsSecretExists()
tlsSecretExists, secret, err := c.tlsSecretExists()
if err != nil {
return err
}
if !tlsSecretExists {
return nil
}
c.infoLog.Info("deleting TLS certificates for impersonation proxy",
c.infoLog.Info("deleting TLS serving certificate for impersonation proxy",
"secret", klog.KRef(c.namespace, c.tlsSecretName),
)
err = c.k8sClient.CoreV1().Secrets(c.namespace).Delete(ctx, c.tlsSecretName, metav1.DeleteOptions{})
notFound := k8serrors.IsNotFound(err)
if notFound {
// its okay if we tried to delete and we got a not found error. This probably means
// another instance of the concierge got here first so there's nothing to delete.
return nil
}
if err != nil {
return err
}
c.tlsServingCertDynamicCertProvider.UnsetCertKeyContent()
return nil
err = c.k8sClient.CoreV1().Secrets(c.namespace).Delete(ctx, c.tlsSecretName, metav1.DeleteOptions{
Preconditions: &metav1.Preconditions{
UID: &secret.UID,
ResourceVersion: &secret.ResourceVersion,
},
})
// it is okay if we tried to delete and we got a not found error. This probably means
// another instance of the concierge got here first so there's nothing to delete.
return utilerrors.FilterOut(err, k8serrors.IsNotFound)
}
func (c *impersonatorConfigController) loadSignerCA(status v1alpha1.StrategyStatus) error {
// Clear it when the impersonator is not completely ready.
if status != v1alpha1.SuccessStrategyStatus {
c.clearSignerCA()
return nil
}
func (c *impersonatorConfigController) clearTLSSecret() {
c.debugLog.Info("clearing TLS serving certificate for impersonation proxy")
c.tlsServingCertDynamicCertProvider.UnsetCertKeyContent()
}
func (c *impersonatorConfigController) loadSignerCA() error {
signingCertSecret, err := c.secretsInformer.Lister().Secrets(c.namespace).Get(c.impersonationSignerSecretName)
if err != nil {
return fmt.Errorf("could not load the impersonator's credential signing secret: %w", err)
@@ -997,7 +999,7 @@ func (c *impersonatorConfigController) loadSignerCA(status v1alpha1.StrategyStat
keyPEM := signingCertSecret.Data[apicerts.CACertificatePrivateKeySecretKey]
if err := c.impersonationSigningCertProvider.SetCertKeyContent(certPEM, keyPEM); err != nil {
return fmt.Errorf("could not load the impersonator's credential signing secret: %w", err)
return fmt.Errorf("could not set the impersonator's credential signing secret: %w", err)
}
c.infoLog.Info("loading credential signing certificate for impersonation proxy",

229
internal/controller/impersonatorconfig/impersonator_config_test.go
View File

@@ -29,11 +29,14 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/clock"
"k8s.io/apimachinery/pkg/util/intstr"
kubeinformers "k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
kubernetesfake "k8s.io/client-go/kubernetes/fake"
coretesting "k8s.io/client-go/testing"
"k8s.io/utils/pointer"
"go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1"
pinnipedfake "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned/fake"
@@ -266,6 +269,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
var subject controllerlib.Controller
var kubeAPIClient *kubernetesfake.Clientset
var deleteOptions *[]metav1.DeleteOptions
var deleteOptionsRecorder kubernetes.Interface
var pinnipedAPIClient *pinnipedfake.Clientset
var pinnipedInformerClient *pinnipedfake.Clientset
var pinnipedInformers pinnipedinformers.SharedInformerFactory
@@ -275,6 +280,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
var cancelContextCancelFunc context.CancelFunc
var syncContext *controllerlib.Context
var frozenNow time.Time
var tlsServingCertDynamicCertProvider dynamiccert.Private
var signingCertProvider dynamiccert.Provider
var signingCACertPEM, signingCAKeyPEM []byte
var signingCASecret *corev1.Secret
@@ -418,6 +424,20 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
}
}
var requireTLSSecretProviderHasLoadedCerts = func() {
actualCert, actualKey := tlsServingCertDynamicCertProvider.CurrentCertKeyContent()
r.NotEmpty(actualCert)
r.NotEmpty(actualKey)
_, err := tls.X509KeyPair(actualCert, actualKey)
r.NoError(err)
}
var requireTLSSecretProviderIsEmpty = func() {
actualCert, actualKey := tlsServingCertDynamicCertProvider.CurrentCertKeyContent()
r.Nil(actualCert)
r.Nil(actualKey)
}
var requireTLSServerIsRunning = func(caCrt []byte, addr string, dnsOverrides map[string]string) {
r.Greater(impersonatorFuncWasCalled, 0)
@@ -469,6 +489,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
r.NoError(resp.Body.Close())
r.NoError(err)
r.Equal(fakeServerResponseBody, string(body))
requireTLSSecretProviderHasLoadedCerts()
}
var requireTLSServerIsRunningWithoutCerts = func() {
@@ -490,6 +512,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
}, 20*time.Second, 50*time.Millisecond)
r.Error(err)
r.Regexp(expectedErrorRegex, err.Error())
requireTLSSecretProviderIsEmpty()
}
var requireTLSServerIsNoLongerRunning = func() {
@@ -508,10 +532,14 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
}, 20*time.Second, 50*time.Millisecond)
r.Error(err)
r.Regexp(expectedErrorRegex, err.Error())
requireTLSSecretProviderIsEmpty()
}
var requireTLSServerWasNeverStarted = func() {
r.Equal(0, impersonatorFuncWasCalled)
requireTLSSecretProviderIsEmpty()
}
// Defer starting the informers until the last possible moment so that the
@@ -521,7 +549,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
subject = NewImpersonatorConfigController(
installedInNamespace,
credentialIssuerResourceName,
kubeAPIClient,
deleteOptionsRecorder,
pinnipedAPIClient,
pinnipedInformers.Config().V1alpha1().CredentialIssuers(),
kubeInformers.Core().V1().Services(),
@@ -538,6 +566,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
signingCertProvider,
testLog,
)
controllerlib.TestWrap(t, subject, func(syncer controllerlib.Syncer) controllerlib.Syncer {
tlsServingCertDynamicCertProvider = syncer.(*impersonatorConfigController).tlsServingCertDynamicCertProvider
return syncer
})
// Set this at the last second to support calling subject.Name().
syncContext = &controllerlib.Context{
@@ -564,8 +596,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
var newSecretWithData = func(resourceName string, data map[string][]byte) *corev1.Secret {
return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: resourceName,
Namespace: installedInNamespace,
Name: resourceName,
Namespace: installedInNamespace,
UID: "uid-1234", // simulate KAS filling out UID and RV
ResourceVersion: "rv-5678",
},
Data: data,
}
@@ -705,6 +739,14 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
var addObjectFromCreateActionToInformerAndWait = func(action coretesting.Action, informer controllerlib.InformerGetter) {
createdObject, ok := action.(coretesting.CreateAction).GetObject().(kubeclient.Object)
r.True(ok, "should have been able to cast this action's object to kubeclient.Object: %v", action)
if secret, ok := createdObject.(*corev1.Secret); ok && len(secret.ResourceVersion) == 0 {
secret = secret.DeepCopy()
secret.UID = "uid-1234" // simulate KAS filling out UID and RV
secret.ResourceVersion = "rv-5678"
createdObject = secret
}
addObjectToKubeInformerAndWait(createdObject, informer)
}
@@ -986,6 +1028,18 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
r.Equal("delete", deleteAction.GetVerb())
r.Equal(tlsSecretName, deleteAction.GetName())
r.Equal("secrets", deleteAction.GetResource().Resource)
// validate that we set delete preconditions correctly
r.NotEmpty(*deleteOptions)
for _, opt := range *deleteOptions {
uid := types.UID("uid-1234")
r.Equal(metav1.DeleteOptions{
Preconditions: &metav1.Preconditions{
UID: &uid,
ResourceVersion: pointer.String("rv-5678"),
},
}, opt)
}
}
var requireCASecretWasCreated = func(action coretesting.Action) []byte {
@@ -1064,6 +1118,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
kubeinformers.WithNamespace(installedInNamespace),
)
kubeAPIClient = kubernetesfake.NewSimpleClientset()
deleteOptions = &[]metav1.DeleteOptions{}
deleteOptionsRecorder = testutil.NewDeleteOptionsRecorder(kubeAPIClient, deleteOptions)
pinnipedAPIClient = pinnipedfake.NewSimpleClientset()
frozenNow = time.Date(2021, time.March, 2, 7, 42, 0, 0, time.Local)
signingCertProvider = dynamiccert.NewCA(name)
@@ -1222,7 +1278,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1])
requireCASecretWasCreated(kubeAPIClient.Actions()[2])
requireCredentialIssuer(newPendingStrategyWaitingForLB())
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM)
})
})
@@ -1241,7 +1297,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
requireNodesListed(kubeAPIClient.Actions()[0])
requireCASecretWasCreated(kubeAPIClient.Actions()[1])
requireCredentialIssuer(newPendingStrategyWaitingForLB())
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM)
})
})
@@ -1260,7 +1316,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
requireNodesListed(kubeAPIClient.Actions()[0])
requireCASecretWasCreated(kubeAPIClient.Actions()[1])
requireCredentialIssuer(newPendingStrategyWaitingForLB())
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM)
})
})
@@ -1451,10 +1507,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
errString := "could not find valid IP addresses or hostnames from load balancer some-namespace/some-service-resource-name"
r.EqualError(runControllerSync(), errString)
r.Len(kubeAPIClient.Actions(), 1) // no new actions
requireTLSServerIsRunningWithoutCerts()
r.Len(kubeAPIClient.Actions(), 1) // no new actions
requireTLSServerIsRunning(caCrt, testServerAddr(), nil) // serving certificate is not unloaded in this case
requireCredentialIssuer(newErrorStrategy(errString))
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM)
})
})
})
@@ -1510,7 +1566,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
requireCASecretWasCreated(kubeAPIClient.Actions()[2])
requireTLSServerIsRunningWithoutCerts()
requireCredentialIssuer(newPendingStrategyWaitingForLB())
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM)
})
it("returns an error when the impersonation TLS server fails to start", func() {
@@ -1545,7 +1601,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
requireCASecretWasCreated(kubeAPIClient.Actions()[1])
requireTLSServerIsRunningWithoutCerts()
requireCredentialIssuer(newPendingStrategyWaitingForLB())
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM)
})
it("returns an error when the impersonation TLS server fails to start", func() {
@@ -1685,7 +1741,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
requireCASecretWasCreated(kubeAPIClient.Actions()[2])
requireTLSServerIsRunningWithoutCerts()
requireCredentialIssuer(newPendingStrategyWaitingForLB())
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM)
})
})
@@ -1780,7 +1836,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
// Check that the server is running without certs.
requireTLSServerIsRunningWithoutCerts()
requireCredentialIssuer(newPendingStrategyWaitingForLB())
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM)
})
})
@@ -2129,7 +2185,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
r.Len(kubeAPIClient.Actions(), 4)
requireTLSSecretWasDeleted(kubeAPIClient.Actions()[3]) // tried to delete cert but failed
requireCredentialIssuer(newErrorStrategy("error on tls secret delete"))
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM)
})
})
})
@@ -2160,7 +2216,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1])
requireCASecretWasCreated(kubeAPIClient.Actions()[2])
requireCredentialIssuer(newPendingStrategyWaitingForLB())
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) // load when enabled
// Simulate the informer cache's background update from its watch.
addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Services())
@@ -2178,7 +2234,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
r.Len(kubeAPIClient.Actions(), 4)
requireServiceWasDeleted(kubeAPIClient.Actions()[3], loadBalancerServiceName)
requireCredentialIssuer(newManuallyDisabledStrategy())
requireSigningCertProviderIsEmpty()
requireSigningCertProviderIsEmpty() // only unload when disabled
deleteServiceFromTracker(loadBalancerServiceName, kubeInformerClient)
waitForObjectToBeDeletedFromInformer(loadBalancerServiceName, kubeInformers.Core().V1().Services())
@@ -2195,7 +2251,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
r.Len(kubeAPIClient.Actions(), 5)
requireLoadBalancerWasCreated(kubeAPIClient.Actions()[4])
requireCredentialIssuer(newPendingStrategyWaitingForLB())
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) // load again when enabled
})
})
@@ -2226,7 +2282,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
requireClusterIPWasCreated(kubeAPIClient.Actions()[1])
requireCASecretWasCreated(kubeAPIClient.Actions()[2])
requireCredentialIssuer(newPendingStrategyWaitingForLB())
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) // load when enabled
// Simulate the informer cache's background update from its watch.
addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Services())
@@ -2244,7 +2300,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
r.Len(kubeAPIClient.Actions(), 4)
requireServiceWasDeleted(kubeAPIClient.Actions()[3], clusterIPServiceName)
requireCredentialIssuer(newManuallyDisabledStrategy())
requireSigningCertProviderIsEmpty()
requireSigningCertProviderIsEmpty() // only unload when disabled
deleteServiceFromTracker(clusterIPServiceName, kubeInformerClient)
waitForObjectToBeDeletedFromInformer(clusterIPServiceName, kubeInformers.Core().V1().Services())
@@ -2264,7 +2320,88 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
r.Len(kubeAPIClient.Actions(), 5)
requireClusterIPWasCreated(kubeAPIClient.Actions()[4])
requireCredentialIssuer(newPendingStrategyWaitingForLB())
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) // load again when enabled
})
})
when("service type none with a hostname", func() {
const fakeHostname = "hello.com"
it.Before(func() {
addSecretToTrackers(signingCASecret, kubeInformerClient)
addCredentialIssuerToTrackers(v1alpha1.CredentialIssuer{
ObjectMeta: metav1.ObjectMeta{Name: credentialIssuerResourceName},
Spec: v1alpha1.CredentialIssuerSpec{
ImpersonationProxy: &v1alpha1.ImpersonationProxySpec{
Mode: v1alpha1.ImpersonationProxyModeEnabled,
ExternalEndpoint: fakeHostname,
Service: v1alpha1.ImpersonationProxyServiceSpec{
Type: v1alpha1.ImpersonationProxyServiceTypeNone,
},
},
},
}, pinnipedInformerClient, pinnipedAPIClient)
addNodeWithRoleToTracker("worker", kubeAPIClient)
})
it("starts the impersonator, then shuts it down, then starts it again", func() {
startInformersAndController()
r.NoError(runControllerSync())
r.Len(kubeAPIClient.Actions(), 3)
requireNodesListed(kubeAPIClient.Actions()[0])
ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1])
requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca)
requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()})
requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca))
// load when enabled
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM)
requireTLSSecretProviderHasLoadedCerts()
// Simulate the informer cache's background update from its watch.
addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Secrets())
addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets())
// Update the CredentialIssuer.
updateCredentialIssuerInInformerAndWait(credentialIssuerResourceName, v1alpha1.CredentialIssuerSpec{
ImpersonationProxy: &v1alpha1.ImpersonationProxySpec{
Mode: v1alpha1.ImpersonationProxyModeDisabled,
},
}, pinnipedInformers.Config().V1alpha1().CredentialIssuers())
r.NoError(runControllerSync())
requireTLSServerIsNoLongerRunning()
r.Len(kubeAPIClient.Actions(), 4)
requireTLSSecretWasDeleted(kubeAPIClient.Actions()[3])
requireCredentialIssuer(newManuallyDisabledStrategy())
// only unload when disabled
requireSigningCertProviderIsEmpty()
requireTLSSecretProviderIsEmpty()
deleteSecretFromTracker(tlsSecretName, kubeInformerClient)
waitForObjectToBeDeletedFromInformer(tlsSecretName, kubeInformers.Core().V1().Secrets())
// Update the CredentialIssuer again.
updateCredentialIssuerInInformerAndWait(credentialIssuerResourceName, v1alpha1.CredentialIssuerSpec{
ImpersonationProxy: &v1alpha1.ImpersonationProxySpec{
Mode: v1alpha1.ImpersonationProxyModeEnabled,
ExternalEndpoint: fakeHostname,
Service: v1alpha1.ImpersonationProxyServiceSpec{
Type: v1alpha1.ImpersonationProxyServiceTypeNone,
},
},
}, pinnipedInformers.Config().V1alpha1().CredentialIssuers())
r.NoError(runControllerSync())
requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()})
r.Len(kubeAPIClient.Actions(), 5)
requireTLSSecretWasCreated(kubeAPIClient.Actions()[4], ca)
requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca))
// load again when enabled
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM)
requireTLSSecretProviderHasLoadedCerts()
})
})
})
@@ -2315,9 +2452,9 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
r.Len(kubeAPIClient.Actions(), 5)
requireLoadBalancerWasCreated(kubeAPIClient.Actions()[3])
requireTLSSecretWasDeleted(kubeAPIClient.Actions()[4]) // the Secret was deleted because it contained a cert with the wrong IP
requireTLSServerIsRunningWithoutCerts()
requireTLSServerIsRunning(ca, testServerAddr(), nil) // serving certificate is not unloaded in this case
requireCredentialIssuer(newPendingStrategyWaitingForLB())
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM)
// Simulate the informer cache's background update from its watch.
addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[3], kubeInformers.Core().V1().Services())
@@ -2326,10 +2463,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
// The controller should be waiting for the load balancer's ingress to become available.
r.NoError(runControllerSync())
r.Len(kubeAPIClient.Actions(), 5) // no new actions while it is waiting for the load balancer's ingress
requireTLSServerIsRunningWithoutCerts()
r.Len(kubeAPIClient.Actions(), 5) // no new actions while it is waiting for the load balancer's ingress
requireTLSServerIsRunning(ca, testServerAddr(), nil) // serving certificate is not unloaded in this case
requireCredentialIssuer(newPendingStrategyWaitingForLB())
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM)
// Update the ingress of the LB in the informer's client and run Sync again.
fakeIP := "127.0.0.123"
@@ -2767,7 +2904,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
requireCASecretWasCreated(kubeAPIClient.Actions()[2])
requireTLSServerIsRunningWithoutCerts()
requireCredentialIssuer(newPendingStrategyWaitingForLB())
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM)
// Simulate the informer cache's background update from its watch.
addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Services())
@@ -2777,7 +2914,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
r.Equal(1, impersonatorFuncWasCalled) // wasn't started a second time
requireTLSServerIsRunningWithoutCerts() // still running
requireCredentialIssuer(newPendingStrategyWaitingForLB())
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM)
r.Len(kubeAPIClient.Actions(), 3) // no new API calls
})
@@ -2790,7 +2927,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
ca := requireCASecretWasCreated(kubeAPIClient.Actions()[2])
requireTLSServerIsRunningWithoutCerts()
requireCredentialIssuer(newPendingStrategyWaitingForLB())
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM)
// Simulate the informer cache's background update from its watch.
addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Services())
@@ -2827,7 +2964,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
ca := requireCASecretWasCreated(kubeAPIClient.Actions()[2])
requireTLSServerIsRunningWithoutCerts()
requireCredentialIssuer(newPendingStrategyWaitingForLB())
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM)
// Simulate the informer cache's background update from its watch.
addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Services())
@@ -2918,6 +3055,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
when("the impersonator start function returned by the impersonatorFunc returns an error immediately", func() {
it.Before(func() {
addSecretToTrackers(signingCASecret, kubeInformerClient)
addNodeWithRoleToTracker("worker", kubeAPIClient)
impersonatorFuncReturnedFuncError = errors.New("some immediate impersonator startup error")
addCredentialIssuerToTrackers(v1alpha1.CredentialIssuer{
@@ -2948,7 +3086,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1])
requireCASecretWasCreated(kubeAPIClient.Actions()[2])
requireCredentialIssuer(newPendingStrategyWaitingForLB())
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM)
// Simulate the informer cache's background update from its watch.
addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Services())
@@ -2966,7 +3104,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
// sync should be able to detect the error and return it.
r.EqualError(runControllerSync(), "some immediate impersonator startup error")
requireCredentialIssuer(newErrorStrategy("some immediate impersonator startup error"))
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM)
// Next time the controller starts the server, the server will start successfully.
impersonatorFuncReturnedFuncError = nil
@@ -2976,12 +3114,13 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
r.NoError(runControllerSync())
requireTLSServerIsRunningWithoutCerts()
requireCredentialIssuer(newPendingStrategyWaitingForLB())
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM)
})
})
when("the impersonator server dies for no apparent reason after running for a while", func() {
it.Before(func() {
addSecretToTrackers(signingCASecret, kubeInformerClient)
addNodeWithRoleToTracker("worker", kubeAPIClient)
addCredentialIssuerToTrackers(v1alpha1.CredentialIssuer{
ObjectMeta: metav1.ObjectMeta{Name: credentialIssuerResourceName},
@@ -3004,7 +3143,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1])
requireCASecretWasCreated(kubeAPIClient.Actions()[2])
requireCredentialIssuer(newPendingStrategyWaitingForLB())
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM)
requireTLSServerIsRunningWithoutCerts()
// Simulate the informer cache's background update from its watch.
@@ -3026,7 +3165,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
// sync should be able to detect the error and return it.
r.EqualError(runControllerSync(), "unexpected shutdown of proxy server")
requireCredentialIssuer(newErrorStrategy("unexpected shutdown of proxy server"))
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM)
// Next time the controller starts the server, the server should behave as normal.
testHTTPServerInterruptCh = nil
@@ -3036,7 +3175,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
r.NoError(runControllerSync())
requireTLSServerIsRunningWithoutCerts()
requireCredentialIssuer(newPendingStrategyWaitingForLB())
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM)
})
})
@@ -3471,16 +3610,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
},
}, pinnipedInformerClient, pinnipedAPIClient)
addNodeWithRoleToTracker("worker", kubeAPIClient)
tlsSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: tlsSecretName,
Namespace: installedInNamespace,
},
Data: map[string][]byte{
// "aGVsbG8gd29ybGQK" is "hello world" base64 encoded which is not a valid cert
corev1.TLSCertKey: []byte("-----BEGIN CERTIFICATE-----\naGVsbG8gd29ybGQK\n-----END CERTIFICATE-----\n"),
},
}
tlsSecret := newSecretWithData(tlsSecretName, map[string][]byte{
// "aGVsbG8gd29ybGQK" is "hello world" base64 encoded which is not a valid cert
corev1.TLSCertKey: []byte("-----BEGIN CERTIFICATE-----\naGVsbG8gd29ybGQK\n-----END CERTIFICATE-----\n"),
})
addSecretToTrackers(tlsSecret, kubeAPIClient, kubeInformerClient)
})
@@ -3705,7 +3838,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
it("returns the error", func() {
startInformersAndController()
errString := `could not load the impersonator's credential signing secret: TestImpersonatorConfigControllerSync: attempt to set invalid key pair: tls: failed to find any PEM data in certificate input`
errString := `could not set the impersonator's credential signing secret: TestImpersonatorConfigControllerSync: attempt to set invalid key pair: tls: failed to find any PEM data in certificate input`
r.EqualError(runControllerSync(), errString)
requireCredentialIssuer(newErrorStrategy(errString))
requireSigningCertProviderIsEmpty()
@@ -3720,7 +3853,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
it("returns the error", func() {
startInformersAndController()
errString := `could not load the impersonator's credential signing secret: TestImpersonatorConfigControllerSync: attempt to set invalid key pair: tls: failed to find any PEM data in certificate input`
errString := `could not set the impersonator's credential signing secret: TestImpersonatorConfigControllerSync: attempt to set invalid key pair: tls: failed to find any PEM data in certificate input`
r.EqualError(runControllerSync(), errString)
requireCredentialIssuer(newErrorStrategy(errString))
requireSigningCertProviderIsEmpty()
@@ -3756,10 +3889,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
addSecretToTrackers(updatedSigner, kubeInformerClient)
waitForObjectToAppearInInformer(updatedSigner, kubeInformers.Core().V1().Secrets())
errString := `could not load the impersonator's credential signing secret: TestImpersonatorConfigControllerSync: attempt to set invalid key pair: tls: failed to find any PEM data in certificate input`
errString := `could not set the impersonator's credential signing secret: TestImpersonatorConfigControllerSync: attempt to set invalid key pair: tls: failed to find any PEM data in certificate input`
r.EqualError(runControllerSync(), errString)
requireCredentialIssuer(newErrorStrategy(errString))
requireSigningCertProviderIsEmpty()
requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM)
})
})
})

1
internal/controller/kubecertagent/kubecertagent_test.go
View File

@@ -961,7 +961,6 @@ func runControllerUntilQuiet(ctx context.Context, t *testing.T, controller contr
errorStream := make(chan error)
controllerlib.TestWrap(t, controller, func(syncer controllerlib.Syncer) controllerlib.Syncer {
controller.Name()
return controllerlib.SyncFunc(func(ctx controllerlib.Context) error {
err := syncer.Sync(ctx)
errorStream <- err