Add initial controller boilerplate and example controller

Signed-off-by: Monis Khan <mok@vmware.com>

test/integration/examplecontroller/api/api.go

@@ -0,0 +1,20 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package api
+
+// Annotation on service.
+const SecretNameAnnotation = "service.placeholder.io/secret-name"
+
+// Annotations on secret.
+const (
+	// ServiceUIDAnnotation is an annotation on a secret that indicates which service created it, by UID
+	ServiceUIDAnnotation = "service.placeholder.io/service-uid"
+	// ServiceNameAnnotation is an annotation on a secret that indicates which service created it, by Name
+	// to allow reverse lookups on services for comparison against UIDs
+	ServiceNameAnnotation = "service.placeholder.io/service-name"
+)
+
+const SecretDataKey = "secret-data"

test/integration/examplecontroller/controller/creating.go

@@ -0,0 +1,182 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package controller
+
+import (
+	"context"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	corev1informers "k8s.io/client-go/informers/core/v1"
+	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/tools/events"
+	"k8s.io/klog/v2"
+
+	"github.com/suzerain-io/placeholder-name/internal/controller"
+	"github.com/suzerain-io/placeholder-name/test/integration/examplecontroller/api"
+)
+
+//nolint:funlen
+func NewExampleCreatingController(
+	services corev1informers.ServiceInformer,
+	secrets corev1informers.SecretInformer,
+	secretClient corev1client.SecretsGetter,
+	recorder events.EventRecorder,
+	secretData string,
+) controller.Controller {
+	serviceLister := services.Lister()
+	secretLister := secrets.Lister()
+
+	// note that these functions do not need to be inlined
+	// this just demonstrates that for simple Syncer implementations, everything can be in one place
+
+	requiresSecretGeneration := func(service *corev1.Service) (bool, error) {
+		// check the secret since it could not have been created yet
+		secretName := service.Annotations[api.SecretNameAnnotation]
+		if len(secretName) == 0 {
+			return false, nil
+		}
+
+		secret, err := secretLister.Secrets(service.Namespace).Get(secretName)
+		if apierrors.IsNotFound(err) {
+			// we have not created the secret yet
+			return true, nil
+		}
+		if err != nil {
+			return false, fmt.Errorf("unable to get the secret %s/%s: %w", service.Namespace, secretName, err)
+		}
+
+		if string(secret.Data[api.SecretDataKey]) == secretData {
+			return false, nil
+		}
+
+		// the secret exists but the data does not match what we expect (i.e. we have new secretData now)
+		return true, nil
+	}
+
+	generateSecret := func(service *corev1.Service) error {
+		klog.V(4).InfoS("generating new secret for service", "namespace", service.Namespace, "name", service.Name)
+
+		secret := &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      service.Annotations[api.SecretNameAnnotation],
+				Namespace: service.Namespace,
+				Annotations: map[string]string{
+					api.ServiceUIDAnnotation:  string(service.UID),
+					api.ServiceNameAnnotation: service.Name,
+				},
+				OwnerReferences: []metav1.OwnerReference{
+					{
+						APIVersion: "v1",
+						Kind:       "Service",
+						Name:       service.Name,
+						UID:        service.UID,
+					},
+				},
+				Finalizers: nil, // TODO maybe add finalizer to guarantee we never miss a delete event?
+			},
+			Type: corev1.SecretTypeOpaque,
+			Data: map[string][]byte{
+				api.SecretDataKey: []byte(secretData),
+			},
+		}
+
+		_, err := secretClient.Secrets(service.Namespace).Create(context.TODO(), secret, metav1.CreateOptions{})
+		if apierrors.IsAlreadyExists(err) {
+			actualSecret, getErr := secretClient.Secrets(service.Namespace).Get(context.TODO(), secret.Name, metav1.GetOptions{})
+			if getErr != nil {
+				return getErr
+			}
+
+			if actualSecret.Annotations[api.ServiceUIDAnnotation] != string(service.UID) {
+				//nolint: goerr113
+				utilruntime.HandleError(fmt.Errorf("secret %s/%s does not have corresponding service UID %v", actualSecret.Namespace, actualSecret.Name, service.UID))
+				return nil // drop from queue because we cannot safely update this secret
+			}
+
+			klog.V(4).InfoS("updating data in existing secret", "namespace", secret.Namespace, "name", secret.Name)
+			// Actually update the secret in the regeneration case (the secret already exists but we want to update to new secretData).
+			_, updateErr := secretClient.Secrets(secret.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{})
+			return updateErr
+		}
+		if err != nil {
+			return fmt.Errorf("unable to create secret %s/%s: %w", secret.Namespace, secret.Name, err)
+		}
+
+		return nil
+	}
+
+	syncer := controller.SyncFunc(func(ctx controller.Context) error {
+		service, err := serviceLister.Services(ctx.Key.Namespace).Get(ctx.Key.Name)
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+		if err != nil {
+			return fmt.Errorf("unable to get the service %s/%s: %w", service.Namespace, service.Name, err)
+		}
+
+		ok, err := requiresSecretGeneration(service)
+		if err != nil || !ok {
+			return err
+		}
+
+		return generateSecret(service)
+	})
+
+	config := controller.Config{
+		Name:   "example-controller-creating",
+		Syncer: syncer,
+	}
+
+	toServiceName := func(secret *corev1.Secret) (string, bool) {
+		serviceName := secret.Annotations[api.ServiceNameAnnotation]
+		return serviceName, len(serviceName) != 0
+	}
+
+	hasSecretNameAnnotation := func(obj metav1.Object) bool {
+		return len(obj.GetAnnotations()[api.SecretNameAnnotation]) != 0
+	}
+	hasSecretNameAnnotationUpdate := func(oldObj, newObj metav1.Object) bool {
+		return hasSecretNameAnnotation(newObj) || hasSecretNameAnnotation(oldObj)
+	}
+
+	return controller.New(config,
+		controller.WithInformer(services, controller.FilterFuncs{
+			AddFunc:    hasSecretNameAnnotation,
+			UpdateFunc: hasSecretNameAnnotationUpdate,
+		}, controller.InformerOption{}),
+
+		controller.WithInformer(secrets, controller.FilterFuncs{
+			ParentFunc: func(obj metav1.Object) controller.Key {
+				secret := obj.(*corev1.Secret)
+				serviceName, _ := toServiceName(secret)
+				return controller.Key{Namespace: secret.Namespace, Name: serviceName}
+			},
+			DeleteFunc: func(obj metav1.Object) bool {
+				secret := obj.(*corev1.Secret)
+				serviceName, ok := toServiceName(secret)
+				if !ok {
+					return false
+				}
+				service, err := serviceLister.Services(secret.Namespace).Get(serviceName)
+				if apierrors.IsNotFound(err) {
+					return false
+				}
+				if err != nil {
+					utilruntime.HandleError(fmt.Errorf("unable to get service %s/%s: %w", secret.Namespace, serviceName, err))
+					return false
+				}
+				klog.V(4).InfoS("recreating secret", "namespace", service.Namespace, "name", service.Name)
+				return true
+			},
+		}, controller.InformerOption{}),
+
+		controller.WithRecorder(recorder), // TODO actually use the recorder
+	)
+}

test/integration/examplecontroller/controller/creating_test.go

@@ -0,0 +1,170 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package controller
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes/fake"
+	coretesting "k8s.io/client-go/testing"
+	"k8s.io/client-go/tools/events"
+
+	"github.com/suzerain-io/placeholder-name/internal/controller"
+	"github.com/suzerain-io/placeholder-name/test/integration/examplecontroller/api"
+)
+
+func TestNewExampleCreatingController(t *testing.T) {
+	secretsGVR := schema.GroupVersionResource{Version: "v1", Resource: "secrets"}
+
+	type args struct {
+		services   []*corev1.Service
+		secrets    []*corev1.Secret
+		secretData string
+	}
+	type keyErr struct {
+		key controller.Key
+		err error
+	}
+	tests := []struct {
+		name        string
+		args        args
+		wantActions []coretesting.Action
+		wantKeyErrs []keyErr
+	}{
+		{
+			name: "service has annotation but secret does not exist",
+			args: args{
+				services: []*corev1.Service{
+					{
+						ObjectMeta: metav1.ObjectMeta{
+							Namespace: "ns-1",
+							Name:      "service-1",
+							Annotations: map[string]string{
+								api.SecretNameAnnotation: "secret-1",
+							},
+							UID: "0001",
+						},
+					},
+				},
+				secretData: "foo-secret-1",
+			},
+			wantKeyErrs: []keyErr{
+				{
+					key: controller.Key{
+						Namespace: "ns-1",
+						Name:      "service-1",
+					},
+					err: nil, // we expect no error with this key
+				},
+			},
+			wantActions: []coretesting.Action{
+				coretesting.NewCreateAction(secretsGVR, "ns-1", &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "secret-1",
+						Namespace: "ns-1",
+						Annotations: map[string]string{
+							api.ServiceUIDAnnotation:  "0001",
+							api.ServiceNameAnnotation: "service-1",
+						},
+						OwnerReferences: []metav1.OwnerReference{
+							{
+								APIVersion: "v1",
+								Kind:       "Service",
+								Name:       "service-1",
+								UID:        "0001",
+							},
+						},
+					},
+					Type: corev1.SecretTypeOpaque,
+					Data: map[string][]byte{
+						api.SecretDataKey: []byte("foo-secret-1"),
+					},
+				}),
+			},
+		},
+	}
+	for _, tt := range tests {
+		tt := tt
+
+		t.Run(tt.name, func(t *testing.T) {
+			kubeClient := fake.NewSimpleClientset()
+			for i := range tt.args.services {
+				service := tt.args.services[i]
+				err := kubeClient.Tracker().Add(service)
+				require.NoError(t, err)
+			}
+			for i := range tt.args.secrets {
+				secret := tt.args.secrets[i]
+				err := kubeClient.Tracker().Add(secret)
+				require.NoError(t, err)
+			}
+
+			recorder := events.NewEventBroadcasterAdapter(kubeClient).NewRecorder("example-controller")
+			kubeInformers := informers.NewSharedInformerFactory(kubeClient, 0)
+
+			creatingController := NewExampleCreatingController(
+				kubeInformers.Core().V1().Services(),
+				kubeInformers.Core().V1().Secrets(),
+				kubeClient.CoreV1(),
+				recorder,
+				tt.args.secretData,
+			)
+
+			keyErrs := make(chan keyErr)
+			controller.TestWrap(t, creatingController, func(syncer controller.Syncer) controller.Syncer {
+				return controller.SyncFunc(func(ctx controller.Context) error {
+					err := syncer.Sync(ctx)
+
+					keyErrs <- keyErr{
+						key: ctx.Key,
+						err: err,
+					}
+
+					return err
+				})
+			})
+
+			// a different approach would be to use TestSync and run each iteration manually:
+			//
+			// err := controller.TestSync(t, c, ...)
+
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+
+			kubeInformers.Start(ctx.Done())
+			go creatingController.Run(ctx, 5) // TODO maybe only use one worker?
+
+			var actualKeyErrs []keyErr
+		done:
+			for {
+				select {
+				case key := <-keyErrs:
+					actualKeyErrs = append(actualKeyErrs, key)
+
+				case <-time.After(3 * time.Second):
+					// this assumes that calls to Sync are never more than three seconds apart
+					// we have five workers so there is little chance they all hang around doing nothing for that long
+					break done
+				}
+			}
+
+			// TODO: Figure out how to capture actions from informers
+			// TODO: I think we need some more fancy order independent equal comparison here
+
+			require.Equal(t, tt.wantKeyErrs, actualKeyErrs)
+
+			// ignore the discovery call from the event recorder and the list/watch from both informers (first five events)
+			require.Equal(t, tt.wantActions, kubeClient.Actions()[5:])
+		})
+	}
+}

test/integration/examplecontroller/controller/updating.go

@@ -0,0 +1,149 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package controller
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	corev1informers "k8s.io/client-go/informers/core/v1"
+	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/tools/events"
+
+	"github.com/suzerain-io/placeholder-name/internal/controller"
+	"github.com/suzerain-io/placeholder-name/test/integration/examplecontroller/api"
+)
+
+func NewExampleUpdatingController(
+	services corev1informers.ServiceInformer,
+	secrets corev1informers.SecretInformer,
+	secretClient corev1client.SecretsGetter,
+	recorder events.EventRecorder,
+	secretData string,
+) controller.Controller {
+	serviceLister := services.Lister()
+	secretLister := secrets.Lister()
+
+	// note that these functions do not need to be inlined
+	// this just demonstrates that for simple Syncer implementations, everything can be in one place
+
+	toServiceName := func(secret *corev1.Secret) (string, bool) {
+		serviceName := secret.Annotations[api.ServiceNameAnnotation]
+		return serviceName, len(serviceName) != 0
+	}
+
+	ensureSecretData := func(service *corev1.Service, secretCopy *corev1.Secret) bool {
+		var needsUpdate bool
+
+		expectedData := map[string][]byte{
+			api.SecretDataKey: []byte(secretData),
+		}
+		if !reflect.DeepEqual(secretCopy.Data, expectedData) {
+			secretCopy.Data = expectedData
+			needsUpdate = true
+		}
+
+		expectedOwnerReferences := []metav1.OwnerReference{
+			{
+				APIVersion: "v1",
+				Kind:       "Service",
+				Name:       service.Name,
+				UID:        service.UID,
+			},
+		}
+		if !reflect.DeepEqual(secretCopy.OwnerReferences, expectedOwnerReferences) {
+			secretCopy.OwnerReferences = expectedOwnerReferences
+			needsUpdate = true
+		}
+
+		return needsUpdate
+	}
+
+	isSecretValidForService := func(service *corev1.Service, secret *corev1.Secret) bool {
+		if service.Annotations[api.SecretNameAnnotation] != secret.Name {
+			return false
+		}
+		if secret.Annotations[api.ServiceUIDAnnotation] != string(service.UID) {
+			return false
+		}
+		return true
+	}
+
+	getServiceForSecret := func(secret *corev1.Secret) (*corev1.Service, error) {
+		serviceName, ok := toServiceName(secret)
+		if !ok {
+			return nil, nil
+		}
+		service, err := serviceLister.Services(secret.Namespace).Get(serviceName)
+		if apierrors.IsNotFound(err) {
+			return nil, nil
+		}
+		if err != nil {
+			return nil, fmt.Errorf("unable to get service %s/%s: %w", secret.Namespace, serviceName, err)
+		}
+		return service, nil
+	}
+
+	syncer := controller.SyncFunc(func(ctx controller.Context) error {
+		secret, err := secretLister.Secrets(ctx.Key.Namespace).Get(ctx.Key.Name)
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+		if err != nil {
+			return fmt.Errorf("unable to get the secret %s/%s: %w", secret.Namespace, secret.Name, err)
+		}
+
+		service, err := getServiceForSecret(secret)
+		if err != nil || service == nil {
+			return err
+		}
+
+		if !isSecretValidForService(service, secret) {
+			//nolint: goerr113
+			utilruntime.HandleError(fmt.Errorf("secret %s/%s does not have corresponding service UID %v", secret.Namespace, secret.Name, service.UID))
+			return nil
+		}
+
+		// make a copy to avoid mutating cache state
+		secretCopy := secret.DeepCopy()
+
+		if needsUpdate := ensureSecretData(service, secretCopy); needsUpdate {
+			_, updateErr := secretClient.Secrets(secretCopy.Namespace).Update(context.TODO(), secretCopy, metav1.UpdateOptions{})
+			return updateErr
+		}
+
+		return nil
+	})
+
+	config := controller.Config{
+		Name:   "example-controller-updating",
+		Syncer: syncer,
+	}
+
+	addSecret := func(obj metav1.Object) bool {
+		secret := obj.(*corev1.Secret)
+		_, ok := toServiceName(secret)
+		return ok
+	}
+
+	return controller.New(config,
+		controller.WithInformer(services, controller.FilterFuncs{}, controller.InformerOption{SkipEvents: true}),
+
+		controller.WithInformer(secrets, controller.FilterFuncs{
+			AddFunc: addSecret,
+			UpdateFunc: func(oldObj, newObj metav1.Object) bool {
+				return addSecret(newObj) || addSecret(oldObj)
+			},
+		}, controller.InformerOption{}),
+
+		controller.WithRecorder(recorder), // TODO actually use the recorder
+	)
+}

test/integration/examplecontroller/starter/starter.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package starter
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/events"
+
+	"github.com/suzerain-io/placeholder-name/internal/controller"
+	examplecontroller "github.com/suzerain-io/placeholder-name/test/integration/examplecontroller/controller"
+)
+
+func StartExampleController(ctx context.Context, config *rest.Config, secretData string) error {
+	kubeClient, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return fmt.Errorf("failed to build client: %w", err)
+	}
+
+	kubeInformers := informers.NewSharedInformerFactory(kubeClient, 20*time.Minute)
+
+	recorder := events.NewEventBroadcasterAdapter(kubeClient).NewRecorder("example-controller")
+
+	manager := controller.NewManager().
+		WithController(
+			examplecontroller.NewExampleCreatingController(
+				kubeInformers.Core().V1().Services(),
+				kubeInformers.Core().V1().Secrets(),
+				kubeClient.CoreV1(),
+				recorder,
+				secretData,
+			), 5,
+		).
+		WithController(
+			examplecontroller.NewExampleUpdatingController(
+				kubeInformers.Core().V1().Services(),
+				kubeInformers.Core().V1().Secrets(),
+				kubeClient.CoreV1(),
+				recorder,
+				secretData,
+			), 5,
+		)
+
+	kubeInformers.Start(ctx.Done())
+	go manager.Start(ctx)
+
+	return nil
+}

test/integration/examplecontroller_test.go

@@ -0,0 +1,161 @@
+/*
+Copyright 2020 VMware, Inc.
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package integration
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/watch"
+
+	"github.com/suzerain-io/placeholder-name/test/integration/examplecontroller/api"
+	examplestart "github.com/suzerain-io/placeholder-name/test/integration/examplecontroller/starter"
+	"github.com/suzerain-io/placeholder-name/test/library"
+)
+
+func TestExampleController(t *testing.T) {
+	config := library.NewClientConfig(t)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	secretData := "super-secret-data-1"
+
+	err := examplestart.StartExampleController(ctx, config, secretData)
+	require.NoError(t, err)
+
+	client := library.NewClientset(t)
+
+	namespaces := client.CoreV1().Namespaces()
+
+	namespace, err := namespaces.Create(ctx, &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "example-controller-test-",
+		},
+	}, metav1.CreateOptions{})
+	require.NoError(t, err)
+
+	defer func() {
+		deleteErr := namespaces.Delete(context.Background(), namespace.Name, metav1.DeleteOptions{})
+		require.NoError(t, deleteErr)
+	}()
+
+	services := client.CoreV1().Services(namespace.Name)
+	secrets := client.CoreV1().Secrets(namespace.Name)
+
+	secretsWatch, err := secrets.Watch(context.Background(), metav1.ListOptions{})
+	require.NoError(t, err)
+	defer secretsWatch.Stop()
+
+	service := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "example-service-test",
+			Annotations: map[string]string{
+				api.SecretNameAnnotation: "example-secret-name",
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			Ports: []corev1.ServicePort{
+				{
+					Port: 443,
+				},
+			},
+		},
+	}
+	_, err = services.Create(ctx, service, metav1.CreateOptions{})
+	require.NoError(t, err)
+
+	timeout := time.After(10 * time.Second)
+done:
+	for {
+		select {
+		case event := <-secretsWatch.ResultChan():
+			if event.Type != watch.Added {
+				continue
+			}
+			secret, ok := event.Object.(*corev1.Secret)
+			if !ok {
+				continue
+			}
+			if secret.Name != service.Annotations[api.SecretNameAnnotation] {
+				continue
+			}
+
+			expectedData := map[string][]byte{
+				api.SecretDataKey: []byte(secretData),
+			}
+			require.Equal(t, expectedData, secret.Data, "expected to see new secret data: %s", library.Sdump(secret))
+			break done // immediately stop consuming events because we want to check for updated events below
+
+		case <-timeout:
+			t.Fatal("timed out waiting to see new secret")
+		}
+	}
+
+	// shutdown the controllers so we can change the secret data
+	cancel()
+	time.Sleep(5 * time.Second) // wait a bit for the controllers to shut down
+
+	ctx, cancel = context.WithCancel(context.Background())
+	defer cancel()
+
+	secretData2 := "super-secret-data-2"
+
+	err = examplestart.StartExampleController(ctx, config, secretData2)
+	require.NoError(t, err)
+
+	timeout = time.After(10 * time.Second)
+done2:
+	for {
+		select {
+		case event := <-secretsWatch.ResultChan():
+			if event.Type != watch.Modified {
+				continue
+			}
+			secret, ok := event.Object.(*corev1.Secret)
+			if !ok {
+				continue
+			}
+			if secret.Name != service.Annotations[api.SecretNameAnnotation] {
+				continue
+			}
+
+			expectedData := map[string][]byte{
+				api.SecretDataKey: []byte(secretData2),
+			}
+			require.Equal(t, expectedData, secret.Data, "expected to see updated secret data: %s", library.Sdump(secret))
+			break done2 // immediately stop consuming events because we want to check for hot loops below
+
+		case <-timeout:
+			t.Fatal("timed out waiting to see updated secret")
+		}
+	}
+
+	timeout = time.After(5 * time.Second)
+done3:
+	for {
+		select {
+		case event := <-secretsWatch.ResultChan():
+			secret, ok := event.Object.(*corev1.Secret)
+			if !ok {
+				continue
+			}
+			if secret.Name != service.Annotations[api.SecretNameAnnotation] {
+				continue
+			}
+
+			// this assumes that no other actor in the system is trying to mutate this secret
+			t.Errorf("unexpected event seen for secret: %s", library.Sdump(event))
+
+		case <-timeout:
+			break done3 // we saw no events matching our secret meaning that we are not hot looping
+		}
+	}
+}