webhookcachefiller and jwtcachefiller always update status when needed

Even when the authenticator is found in the cache, try to update its
status. Failing to do so would mean that the actual status will not
be overwritten by the controller's newly computed desired status.

Co-authored-by: Ashish Amarnath <ashish.amarnath@broadcom.com>

internal/controller/authenticator/jwtcachefiller/jwtcachefiller.go

@@ -225,6 +225,7 @@ func (c *jwtCacheFillerController) syncIndividualJWTAuthenticator(ctx context.Co
 
 	var errs []error
 	conditions := make([]*metav1.Condition, 0)
+	var newJWTAuthenticatorForCache *cachedJWTAuthenticator
 
 	caBundle, conditions, tlsBundleOk := c.validateTLSBundle(jwtAuthenticator.Spec.TLS, conditions)
 
@@ -232,22 +233,75 @@ func (c *jwtCacheFillerController) syncIndividualJWTAuthenticator(ctx context.Co
 	okSoFar := tlsBundleOk && issuerOk
 
 	// Only revalidate and update the cache if the cached authenticator is different from the desired authenticator.
-	// There is no need to repeat validations for a spec that was already successfully validated. We are making a
-	// design decision to avoid repeating the validation which dials the server, even though the server's TLS
-	// configuration could have changed, because it is also possible that the network could be flaky. We are choosing
-	// to prefer to keep the authenticator cached (available for end-user auth attempts) during times of network flakes
-	// rather than trying to show the most up-to-date status possible. These validations are for administrator
-	// convenience at the time of a configuration change, to catch typos and blatant misconfigurations, rather
-	// than to constantly monitor for external issues.
-	foundAuthenticatorInCache, alreadyValidatedTheseSettings := c.havePreviouslyValidated(
+	// There is no need to repeat connection probe validations for a URL and CA bundle combination that was already
+	// successfully validated. We are making a design decision to avoid repeating the validation which dials the server,
+	// even though the server's TLS configuration could have changed, because it is also possible that the network
+	// could be flaky. We are choosing to prefer to keep the authenticator cached (available for end-user auth attempts)
+	// during times of network flakes rather than trying to show the most up-to-date status possible. These validations
+	// are for administrator convenience at the time of a configuration change, to catch typos and blatant
+	// misconfigurations, rather than to constantly monitor for external issues.
+	foundAuthenticatorInCache, previouslyValidatedWithSameEndpointAndBundle := c.havePreviouslyValidated(
 		cacheKey, jwtAuthenticator.Spec.Issuer, tlsBundleOk, caBundle.Hash(), logger)
-	if alreadyValidatedTheseSettings {
-		// Stop, no more work to be done. This authenticator is already validated and cached.
-		// TODO: Append hardcoded list of remaining success conditions and skip redoing the remaining validations below.
-		//       Make sure that the conditions array is still checked for any errors, and the status and cache are still updated, as below.
-		return nil
+	if previouslyValidatedWithSameEndpointAndBundle {
+		// Because the authenticator was previously cached, that implies that the following conditions were
+		// previously validated. These are the expensive validations to repeat, so skip them this time.
+		// However, the status may be lagging behind due to the informer cache being slow to catch up
+		// after previous status updates, so always calculate the new status conditions again and check
+		// if they need to be updated.
+		conditions = append(conditions,
+			successfulDiscoveryValidCondition(),
+			successfulJWKSURLValidCondition(),
+			successfulJWKSFetchValidCondition(),
+			successfulAuthenticatorValidCondition(),
+		)
+	} else {
+		// Run all remaining validations.
+		a, moreConditions, moreErrs := c.doExpensiveValidations(jwtAuthenticator, caBundle, okSoFar)
+		newJWTAuthenticatorForCache = a
+		conditions = append(conditions, moreConditions...)
+		errs = append(errs, moreErrs...)
 	}
 
+	authenticatorValid := !conditionsutil.HadErrorCondition(conditions)
+
+	// If we calculated a failed status condition, then remove it from the cache even before we try to write
+	// the status, because writing the status can fail for various reasons.
+	if !authenticatorValid {
+		// The authenticator was determined to be invalid. Remove it from the cache, in case it was previously
+		// validated and cached. Do not allow an old, previously validated spec of the authenticator to continue
+		// being used for authentication.
+		c.cache.Delete(cacheKey)
+		logger.Info("invalid jwt authenticator",
+			"removedFromCache", foundAuthenticatorInCache)
+	}
+
+	// Always try to update the status, even when we found it in the authenticator cache.
+	updateErr := c.updateStatus(ctx, jwtAuthenticator, conditions, logger)
+	errs = append(errs, updateErr)
+
+	// Only add/update this authenticator to the cache when we have a new one and the status update succeeded.
+	if newJWTAuthenticatorForCache != nil && authenticatorValid && updateErr == nil {
+		c.cache.Store(cacheKey, newJWTAuthenticatorForCache)
+		logger.Info("added or updated jwt authenticator in cache",
+			"isOverwrite", foundAuthenticatorInCache)
+	}
+
+	// Sync loop errors:
+	// - Should not be configuration errors. Config errors a user must correct belong on the .Status
+	//   object. The controller simply must wait for a user to correct before running again.
+	// - Other errors, such as networking errors, etc. are the types of errors that should return here
+	//   and signal the controller to retry the sync loop. These may be corrected by machines.
+	return utilerrors.NewAggregate(errs)
+}
+
+func (c *jwtCacheFillerController) doExpensiveValidations(
+	jwtAuthenticator *authenticationv1alpha1.JWTAuthenticator,
+	caBundle *tlsconfigutil.CABundle,
+	okSoFar bool,
+) (*cachedJWTAuthenticator, []*metav1.Condition, []error) {
+	var conditions []*metav1.Condition
+	var errs []error
+
 	client := phttp.Default(caBundle.CertPool())
 	client.Timeout = 30 * time.Second // copied from Kube OIDC code
 	coreOSCtx := coreosoidc.ClientContext(context.Background(), client)
@@ -273,37 +327,7 @@ func (c *jwtCacheFillerController) syncIndividualJWTAuthenticator(ctx context.Co
 		okSoFar)
 	errs = append(errs, err)
 
-	authenticatorValid := !conditionsutil.HadErrorCondition(conditions)
-
-	// If we calculated a failed status condition, then remove it from the cache even before we try to write
-	// the status, because writing the status can fail for various reasons.
-	if !authenticatorValid {
-		// The authenticator was determined to be invalid. Remove it from the cache, in case it was previously
-		// validated and cached. Do not allow an old, previously validated spec of the authenticator to continue
-		// being used for authentication.
-		c.cache.Delete(cacheKey)
-		logger.Info("invalid jwt authenticator",
-			"removedFromCache", foundAuthenticatorInCache)
-	}
-
-	updateErr := c.updateStatus(ctx, jwtAuthenticator, conditions, logger)
-	errs = append(errs, updateErr)
-
-	// Only add this JWTAuthenticator to the cache if the status update succeeds.
-	// If it were in the cache after failing to update the status, then the next Sync loop would see it in the cache
-	// and skip trying to update its status again, which would leave its old status permanently intact.
-	if authenticatorValid && updateErr == nil {
-		c.cache.Store(cacheKey, newJWTAuthenticatorForCache)
-		logger.Info("added or updated jwt authenticator in cache",
-			"isOverwrite", foundAuthenticatorInCache)
-	}
-
-	// Sync loop errors:
-	// - Should not be configuration errors. Config errors a user must correct belong on the .Status
-	//   object. The controller simply must wait for a user to correct before running again.
-	// - Other errors, such as networking errors, etc. are the types of errors that should return here
-	//   and signal the controller to retry the sync loop. These may be corrected by machines.
-	return utilerrors.NewAggregate(errs)
+	return newJWTAuthenticatorForCache, conditions, errs
 }
 
 func (c *jwtCacheFillerController) havePreviouslyValidated(
@@ -423,6 +447,15 @@ func (c *jwtCacheFillerController) validateIssuer(issuer string, conditions []*m
 	return conditions, true
 }
 
+func successfulDiscoveryValidCondition() *metav1.Condition {
+	return &metav1.Condition{
+		Type:    typeDiscoveryValid,
+		Status:  metav1.ConditionTrue,
+		Reason:  conditionsutil.ReasonSuccess,
+		Message: "discovery performed successfully",
+	}
+}
+
 func (c *jwtCacheFillerController) validateProviderDiscovery(ctx context.Context, issuer string, conditions []*metav1.Condition, prereqOk bool) (*providerJSON, *coreosoidc.Provider, []*metav1.Condition, error) {
 	if !prereqOk {
 		conditions = append(conditions, &metav1.Condition{
@@ -448,14 +481,17 @@ func (c *jwtCacheFillerController) validateProviderDiscovery(ctx context.Context
 		// resync err, may be machine or other types of non-config error
 		return nil, nil, conditions, fmt.Errorf("%s: %s", errText, err)
 	}
-	msg := "discovery performed successfully"
-	conditions = append(conditions, &metav1.Condition{
-		Type:    typeDiscoveryValid,
+	conditions = append(conditions, successfulDiscoveryValidCondition())
+	return pJSON, provider, conditions, nil
+}
+
+func successfulJWKSURLValidCondition() *metav1.Condition {
+	return &metav1.Condition{
+		Type:    typeJWKSURLValid,
 		Status:  metav1.ConditionTrue,
 		Reason:  conditionsutil.ReasonSuccess,
-		Message: msg,
-	})
-	return pJSON, provider, conditions, nil
+		Message: "jwks_uri is a valid URL",
+	}
 }
 
 func (c *jwtCacheFillerController) validateProviderJWKSURL(provider *coreosoidc.Provider, pJSON *providerJSON, conditions []*metav1.Condition, prereqOk bool) (string, []*metav1.Condition, error) {
@@ -508,13 +544,17 @@ func (c *jwtCacheFillerController) validateProviderJWKSURL(provider *coreosoidc.
 		return pJSON.JWKSURL, conditions, fmt.Errorf("%s", msg)
 	}
 
-	conditions = append(conditions, &metav1.Condition{
-		Type:    typeJWKSURLValid,
+	conditions = append(conditions, successfulJWKSURLValidCondition())
+	return pJSON.JWKSURL, conditions, nil
+}
+
+func successfulJWKSFetchValidCondition() *metav1.Condition {
+	return &metav1.Condition{
+		Type:    typeJWKSFetchValid,
 		Status:  metav1.ConditionTrue,
 		Reason:  conditionsutil.ReasonSuccess,
-		Message: "jwks_uri is a valid URL",
-	})
-	return pJSON.JWKSURL, conditions, nil
+		Message: "successfully fetched jwks",
+	}
 }
 
 // validateJWKSFetch deliberately takes an unsigned JWT to trigger coreosoidc.NewRemoteKeySet to
@@ -567,12 +607,7 @@ func (c *jwtCacheFillerController) validateJWKSFetch(ctx context.Context, jwksUR
 	// This error indicates success of this check. We only wanted to test if we could fetch, we aren't actually
 	// testing for valid signature verification.
 	if strings.Contains(verifyErrString, "failed to verify id token signature") {
-		conditions = append(conditions, &metav1.Condition{
-			Type:    typeJWKSFetchValid,
-			Status:  metav1.ConditionTrue,
-			Reason:  conditionsutil.ReasonSuccess,
-			Message: "successfully fetched jwks",
-		})
+		conditions = append(conditions, successfulJWKSFetchValidCondition())
 		return keySet, conditions, nil
 	}
 
@@ -588,6 +623,15 @@ func (c *jwtCacheFillerController) validateJWKSFetch(ctx context.Context, jwksUR
 	return nil, conditions, fmt.Errorf("%s: %w", errText, verifyWithKeySetErr)
 }
 
+func successfulAuthenticatorValidCondition() *metav1.Condition {
+	return &metav1.Condition{
+		Type:    typeAuthenticatorValid,
+		Status:  metav1.ConditionTrue,
+		Reason:  conditionsutil.ReasonSuccess,
+		Message: "authenticator initialized",
+	}
+}
+
 // newCachedJWTAuthenticator creates a jwt authenticator from the provided spec.
 func (c *jwtCacheFillerController) newCachedJWTAuthenticator(
 	client *http.Client,
@@ -656,13 +700,7 @@ func (c *jwtCacheFillerController) newCachedJWTAuthenticator(
 		// resync err, lots of possible issues that may or may not be machine related
 		return nil, conditions, fmt.Errorf("%s: %w", errText, err)
 	}
-	msg := "authenticator initialized"
-	conditions = append(conditions, &metav1.Condition{
-		Type:    typeAuthenticatorValid,
-		Status:  metav1.ConditionTrue,
-		Reason:  conditionsutil.ReasonSuccess,
-		Message: msg,
-	})
+	conditions = append(conditions, successfulAuthenticatorValidCondition())
 	return &cachedJWTAuthenticator{
 		Token:        oidcAuthenticator,
 		issuer:       spec.Issuer,

internal/controller/authenticator/jwtcachefiller/jwtcachefiller_test.go

@@ -1158,7 +1158,7 @@ func TestController(t *testing.T) {
 			wantClose:                           true,
 		},
 		{
-			name: "Sync: previously cached JWTAuthenticator gets new spec fields, but status update fails: loop will leave it in the cache and avoid calling close",
+			name: "Sync: previously cached authenticator gets new spec fields, but status update fails: loop will leave it in the cache and avoid calling close",
 			cache: func(t *testing.T, cache *authncache.Cache, wantClose bool) {
 				oldCA, err := base64.StdEncoding.DecodeString(otherJWTAuthenticatorSpec.TLS.CertificateAuthorityData)
 				require.NoError(t, err)
@@ -1207,11 +1207,72 @@ func TestController(t *testing.T) {
 					updateStatusAction,
 				}
 			},
+			// skip the tests because the authenticator preloaded into the cache is the mock version that was added above
 			skipTestingCachedAuthenticator:      true,
 			wantSyncErr:                         testutil.WantExactErrorString("error for JWTAuthenticator test-name: some update error"),
 			wantNamesOfJWTAuthenticatorsInCache: []string{"test-name"}, // keeps the old entry in the cache
 			wantClose:                           false,
 		},
+		{
+			name: "Sync: previously cached valid authenticator with unchanged issuer URL and CA bundle hash has invalid status conditions in informer cache, as can happen on subsequent sync soon after multiple quick status updates (when the informer cache finally catches up): should update status in current sync",
+			cache: func(t *testing.T, cache *authncache.Cache, wantClose bool) {
+				oldCA, err := base64.StdEncoding.DecodeString(someJWTAuthenticatorSpec.TLS.CertificateAuthorityData)
+				require.NoError(t, err)
+				cache.Store(
+					authncache.Key{
+						Name:     "test-name",
+						Kind:     "JWTAuthenticator",
+						APIGroup: authenticationv1alpha1.SchemeGroupVersion.Group,
+					},
+					newCacheValue(t, *someJWTAuthenticatorSpec, string(oldCA), wantClose),
+				)
+			},
+			jwtAuthenticators: []runtime.Object{
+				&authenticationv1alpha1.JWTAuthenticator{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "test-name",
+					},
+					Spec: *someJWTAuthenticatorSpec,
+					Status: authenticationv1alpha1.JWTAuthenticatorStatus{
+						Conditions: conditionstestutil.Replace(
+							allHappyConditionsSuccess(goodIssuer, frozenMetav1Now, 0),
+							[]metav1.Condition{
+								sadReadyCondition(frozenMetav1Now, 0),
+								unknownAuthenticatorValid(frozenMetav1Now, 0),
+								sadJWKSFetch("some remote jwks error", frozenMetav1Now, 0),
+							},
+						),
+						Phase: "Error",
+					},
+				},
+			},
+			wantLogLines: []string{
+				fmt.Sprintf(`{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","logger":"jwtcachefiller-controller","caller":"jwtcachefiller/jwtcachefiller.go:<line>$jwtcachefiller.(*jwtCacheFillerController).havePreviouslyValidated","message":"cached jwt authenticator and desired jwt authenticator are the same: already cached, so skipping validations","jwtAuthenticator":"test-name","issuer":"%s"}`, someJWTAuthenticatorSpec.Issuer),
+				fmt.Sprintf(`{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","logger":"jwtcachefiller-controller","caller":"jwtcachefiller/jwtcachefiller.go:<line>$jwtcachefiller.(*jwtCacheFillerController).updateStatus","message":"jwtauthenticator status successfully updated","jwtAuthenticator":"test-name","issuer":"%s","phase":"Ready"}`, someJWTAuthenticatorSpec.Issuer),
+			},
+			wantActions: func() []coretesting.Action {
+				updateStatusAction := coretesting.NewUpdateAction(jwtAuthenticatorsGVR, "", &authenticationv1alpha1.JWTAuthenticator{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "test-name",
+					},
+					Spec: *someJWTAuthenticatorSpec,
+					Status: authenticationv1alpha1.JWTAuthenticatorStatus{ // updates the status to ready
+						Conditions: allHappyConditionsSuccess(goodIssuer, frozenMetav1Now, 0),
+						Phase:      "Ready",
+					},
+				})
+				updateStatusAction.Subresource = "status"
+				return []coretesting.Action{
+					coretesting.NewListAction(jwtAuthenticatorsGVR, jwtAUthenticatorGVK, "", metav1.ListOptions{}),
+					coretesting.NewWatchAction(jwtAuthenticatorsGVR, "", metav1.ListOptions{}),
+					updateStatusAction,
+				}
+			},
+			// skip the tests because the authenticator preloaded into the cache is the mock version that was added above
+			skipTestingCachedAuthenticator:      true,
+			wantNamesOfJWTAuthenticatorsInCache: []string{"test-name"}, // keeps the old entry in the cache
+			wantClose:                           false,
+		},
 		{
 			name: "Sync: JWTAuthenticator with external and changed CA bundle: loop will close previous instance of JWTAuthenticator and complete successfully and update status conditions",
 			cache: func(t *testing.T, cache *authncache.Cache, wantClose bool) {
@@ -1261,7 +1322,7 @@ func TestController(t *testing.T) {
 			wantNamesOfJWTAuthenticatorsInCache: []string{"test-name"},
 		},
 		{
-			name: "Sync: JWTAuthenticator with no change: loop will abort early and not update status conditions",
+			name: "Sync: previously cached JWTAuthenticator with no change: will not update status conditions",
 			cache: func(t *testing.T, cache *authncache.Cache, wantClose bool) {
 				oldCA, err := base64.StdEncoding.DecodeString(someJWTAuthenticatorSpec.TLS.CertificateAuthorityData)
 				require.NoError(t, err)
@@ -1280,10 +1341,15 @@ func TestController(t *testing.T) {
 						Name: "test-name",
 					},
 					Spec: *someJWTAuthenticatorSpec,
+					Status: authenticationv1alpha1.JWTAuthenticatorStatus{
+						Conditions: allHappyConditionsSuccess(goodIssuer, frozenMetav1Now, 0),
+						Phase:      "Ready",
+					},
 				},
 			},
 			wantLogLines: []string{
 				fmt.Sprintf(`{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","logger":"jwtcachefiller-controller","caller":"jwtcachefiller/jwtcachefiller.go:<line>$jwtcachefiller.(*jwtCacheFillerController).havePreviouslyValidated","message":"cached jwt authenticator and desired jwt authenticator are the same: already cached, so skipping validations","jwtAuthenticator":"test-name","issuer":"%s"}`, goodIssuer),
+				fmt.Sprintf(`{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","logger":"jwtcachefiller-controller","caller":"jwtcachefiller/jwtcachefiller.go:<line>$jwtcachefiller.(*jwtCacheFillerController).updateStatus","message":"choosing to not update the jwtauthenticator status since there is no update to make","jwtAuthenticator":"test-name","issuer":"%s","phase":"Ready"}`, goodIssuer),
 			},
 			wantActions: func() []coretesting.Action {
 				return []coretesting.Action{
@@ -1291,10 +1357,10 @@ func TestController(t *testing.T) {
 					coretesting.NewWatchAction(jwtAuthenticatorsGVR, "", metav1.ListOptions{}),
 				}
 			},
+			// skip the tests because the authenticator preloaded into the cache is the mock version that was added above
+			skipTestingCachedAuthenticator:      true,
 			wantClose:                           false,
 			wantNamesOfJWTAuthenticatorsInCache: []string{"test-name"},
-			// skip the tests because the authenticator pre-loaded into the cache is the mock version that was added above
-			skipTestingCachedAuthenticator: true,
 		},
 		{
 			name: "Sync: authenticator update when cached authenticator is the wrong data type, which should never really happen: loop will complete successfully and update status conditions",

internal/controller/authenticator/webhookcachefiller/webhookcachefiller.go

@@ -169,6 +169,7 @@ func (c *webhookCacheFillerController) syncIndividualWebhookAuthenticator(ctx co
 
 	var errs []error
 	conditions := make([]*metav1.Condition, 0)
+	var newWebhookAuthenticatorForCache *cachedWebhookAuthenticator
 
 	caBundle, conditions, tlsBundleOk := c.validateTLSBundle(webhookAuthenticator.Spec.TLS, conditions)
 
@@ -176,36 +177,33 @@ func (c *webhookCacheFillerController) syncIndividualWebhookAuthenticator(ctx co
 	okSoFar := tlsBundleOk && endpointOk
 
 	// Only revalidate and update the cache if the cached authenticator is different from the desired authenticator.
-	// There is no need to repeat validations for a spec that was already successfully validated. We are making a
-	// design decision to avoid repeating the validation which dials the server, even though the server's TLS
-	// configuration could have changed, because it is also possible that the network could be flaky. We are choosing
-	// to prefer to keep the authenticator cached (available for end-user auth attempts) during times of network flakes
-	// rather than trying to show the most up-to-date status possible. These validations are for administrator
-	// convenience at the time of a configuration change, to catch typos and blatant misconfigurations, rather
-	// than to constantly monitor for external issues.
-	foundAuthenticatorInCache, alreadyValidatedTheseSettings := c.havePreviouslyValidated(
+	// There is no need to repeat connection probe validations for a URL and CA bundle combination that was already
+	// successfully validated. We are making a design decision to avoid repeating the validation which dials the server,
+	// even though the server's TLS configuration could have changed, because it is also possible that the network
+	// could be flaky. We are choosing to prefer to keep the authenticator cached (available for end-user auth attempts)
+	// during times of network flakes rather than trying to show the most up-to-date status possible. These validations
+	// are for administrator convenience at the time of a configuration change, to catch typos and blatant
+	// misconfigurations, rather than to constantly monitor for external issues.
+	foundAuthenticatorInCache, previouslyValidatedWithSameEndpointAndBundle := c.havePreviouslyValidated(
 		cacheKey, webhookAuthenticator.Spec.Endpoint, tlsBundleOk, caBundle.Hash(), logger)
-	if alreadyValidatedTheseSettings {
-		// Stop, no more work to be done. This authenticator is already validated and cached.
-		// TODO: Append hardcoded list of remaining success conditions and skip redoing the remaining validations below.
-		//       Make sure that the conditions array is still checked for any errors, and the status and cache are still updated, as below.
-		return nil
+	if previouslyValidatedWithSameEndpointAndBundle {
+		// Because the authenticator was previously cached, that implies that the following conditions were
+		// previously validated. These are the expensive validations to repeat, so skip them this time.
+		// However, the status may be lagging behind due to the informer cache being slow to catch up
+		// after previous status updates, so always calculate the new status conditions again and check
+		// if they need to be updated.
+		conditions = append(conditions,
+			successfulWebhookConnectionValidCondition(),
+			successfulAuthenticatorValidCondition(),
+		)
+	} else {
+		// Run all remaining validations.
+		a, moreConditions, moreErrs := c.doExpensiveValidations(webhookAuthenticator, endpointHostPort, caBundle, okSoFar, logger)
+		newWebhookAuthenticatorForCache = a
+		conditions = append(conditions, moreConditions...)
+		errs = append(errs, moreErrs...)
 	}
 
-	conditions, tlsNegotiateErr := c.validateConnection(caBundle.CertPool(), endpointHostPort, conditions, okSoFar, logger)
-	errs = append(errs, tlsNegotiateErr)
-	okSoFar = okSoFar && tlsNegotiateErr == nil
-
-	newWebhookAuthenticatorForCache, conditions, err := newWebhookAuthenticator(
-		// Note that we use the whole URL when constructing the webhook client,
-		// not just the host and port that we validated above. We need the path, etc.
-		webhookAuthenticator.Spec.Endpoint,
-		caBundle.PEMBytes(),
-		conditions,
-		okSoFar,
-	)
-	errs = append(errs, err)
-
 	authenticatorValid := !conditionsutil.HadErrorCondition(conditions)
 
 	// If we calculated a failed status condition, then remove it from the cache even before we try to write
@@ -219,18 +217,13 @@ func (c *webhookCacheFillerController) syncIndividualWebhookAuthenticator(ctx co
 			"removedFromCache", foundAuthenticatorInCache)
 	}
 
+	// Always try to update the status, even when we found it in the authenticator cache.
 	updateErr := c.updateStatus(ctx, webhookAuthenticator, conditions, logger)
 	errs = append(errs, updateErr)
 
-	// Only add this WebhookAuthenticator to the cache if the status update succeeds.
-	// If it were in the cache after failing to update the status, then the next Sync loop would see it in the cache
-	// and skip trying to update its status again, which would leave its old status permanently intact.
-	if authenticatorValid && updateErr == nil {
-		c.cache.Store(cacheKey, &cachedWebhookAuthenticator{
-			Token:        newWebhookAuthenticatorForCache,
-			endpoint:     webhookAuthenticator.Spec.Endpoint,
-			caBundleHash: caBundle.Hash(),
-		})
+	// Only add/update this authenticator to the cache when we have a new one and the status update succeeded.
+	if newWebhookAuthenticatorForCache != nil && authenticatorValid && updateErr == nil {
+		c.cache.Store(cacheKey, newWebhookAuthenticatorForCache)
 		logger.Info("added or updated webhook authenticator in cache",
 			"isOverwrite", foundAuthenticatorInCache)
 	}
@@ -243,6 +236,41 @@ func (c *webhookCacheFillerController) syncIndividualWebhookAuthenticator(ctx co
 	return utilerrors.NewAggregate(errs)
 }
 
+func (c *webhookCacheFillerController) doExpensiveValidations(
+	webhookAuthenticator *authenticationv1alpha1.WebhookAuthenticator,
+	endpointHostPort *endpointaddr.HostPort,
+	caBundle *tlsconfigutil.CABundle,
+	okSoFar bool,
+	logger plog.Logger,
+) (*cachedWebhookAuthenticator, []*metav1.Condition, []error) {
+	var newWebhookAuthenticatorForCache *cachedWebhookAuthenticator
+	var conditions []*metav1.Condition
+	var errs []error
+
+	conditions, tlsNegotiateErr := c.validateConnection(caBundle.CertPool(), endpointHostPort, conditions, okSoFar, logger)
+	errs = append(errs, tlsNegotiateErr)
+	okSoFar = okSoFar && tlsNegotiateErr == nil
+
+	newAuthenticator, conditions, err := newWebhookAuthenticator(
+		// Note that we use the whole URL when constructing the webhook client,
+		// not just the host and port that we validated above. We need the path, etc.
+		webhookAuthenticator.Spec.Endpoint,
+		caBundle.PEMBytes(),
+		conditions,
+		okSoFar,
+	)
+	errs = append(errs, err)
+
+	if newAuthenticator != nil {
+		newWebhookAuthenticatorForCache = &cachedWebhookAuthenticator{
+			Token:        newAuthenticator,
+			endpoint:     webhookAuthenticator.Spec.Endpoint,
+			caBundleHash: caBundle.Hash(),
+		}
+	}
+	return newWebhookAuthenticatorForCache, conditions, errs
+}
+
 func (c *webhookCacheFillerController) havePreviouslyValidated(
 	cacheKey authncache.Key,
 	endpoint string,
@@ -294,6 +322,15 @@ func (c *webhookCacheFillerController) validateTLSBundle(tlsSpec *authentication
 	return caBundle, conditions, condition.Status == metav1.ConditionTrue
 }
 
+func successfulAuthenticatorValidCondition() *metav1.Condition {
+	return &metav1.Condition{
+		Type:    typeAuthenticatorValid,
+		Status:  metav1.ConditionTrue,
+		Reason:  conditionsutil.ReasonSuccess,
+		Message: "authenticator initialized",
+	}
+}
+
 // newWebhookAuthenticator creates a webhook from the provided API server url and caBundle
 // used to validate TLS connections.
 func newWebhookAuthenticator(
@@ -362,17 +399,20 @@ func newWebhookAuthenticator(
 		return nil, conditions, fmt.Errorf("%s: %w", errText, err)
 	}
 
-	msg := "authenticator initialized"
-	conditions = append(conditions, &metav1.Condition{
-		Type:    typeAuthenticatorValid,
-		Status:  metav1.ConditionTrue,
-		Reason:  conditionsutil.ReasonSuccess,
-		Message: msg,
-	})
+	conditions = append(conditions, successfulAuthenticatorValidCondition())
 
 	return webhookAuthenticator, conditions, nil
 }
 
+func successfulWebhookConnectionValidCondition() *metav1.Condition {
+	return &metav1.Condition{
+		Type:    typeWebhookConnectionValid,
+		Status:  metav1.ConditionTrue,
+		Reason:  conditionsutil.ReasonSuccess,
+		Message: "successfully dialed webhook server",
+	}
+}
+
 func (c *webhookCacheFillerController) validateConnection(
 	certPool *x509.CertPool,
 	endpointHostPort *endpointaddr.HostPort,
@@ -411,12 +451,7 @@ func (c *webhookCacheFillerController) validateConnection(
 		logger.Error("error closing dialer", err)
 	}
 
-	conditions = append(conditions, &metav1.Condition{
-		Type:    typeWebhookConnectionValid,
-		Status:  metav1.ConditionTrue,
-		Reason:  conditionsutil.ReasonSuccess,
-		Message: "successfully dialed webhook server",
-	})
+	conditions = append(conditions, successfulWebhookConnectionValidCondition())
 	return conditions, nil
 }
 

internal/controller/authenticator/webhookcachefiller/webhookcachefiller_test.go

@@ -769,6 +769,7 @@ func TestController(t *testing.T) {
 			},
 			wantLogLines: []string{
 				fmt.Sprintf(`{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","logger":"webhookcachefiller-controller","caller":"webhookcachefiller/webhookcachefiller.go:<line>$webhookcachefiller.(*webhookCacheFillerController).havePreviouslyValidated","message":"cached webhook authenticator and desired webhook authenticator are the same: already cached, so skipping validations","webhookAuthenticator":"test-name","endpoint":"%s"}`, goodWebhookDefaultServingCertEndpoint),
+				fmt.Sprintf(`{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","logger":"webhookcachefiller-controller","caller":"webhookcachefiller/webhookcachefiller.go:<line>$webhookcachefiller.(*webhookCacheFillerController).updateStatus","message":"choosing to not update the webhookauthenticator status since there is no update to make","webhookAuthenticator":"test-name","endpoint":"%s","phase":"Ready"}`, goodWebhookDefaultServingCertEndpoint),
 			},
 			wantActions: func() []coretesting.Action {
 				return []coretesting.Action{
@@ -950,6 +951,66 @@ func TestController(t *testing.T) {
 			wantSyncErr:                             testutil.WantExactErrorString("error for WebhookAuthenticator test-name: some update error"),
 			wantNamesOfWebhookAuthenticatorsInCache: []string{"test-name"}, // keeps the old entry in the cache
 		},
+		{
+			name: "Sync: previously cached valid authenticator with unchanged endpoint URL and CA bundle hash has invalid status conditions in informer cache, as can happen on subsequent sync soon after multiple quick status updates (when the informer cache finally catches up): should update status in current sync",
+			cache: func(t *testing.T, cache *authncache.Cache) {
+				oldCA, err := base64.StdEncoding.DecodeString(goodWebhookAuthenticatorSpecWithCA.TLS.CertificateAuthorityData)
+				require.NoError(t, err)
+				cache.Store(
+					authncache.Key{
+						Name:     "test-name",
+						Kind:     "WebhookAuthenticator",
+						APIGroup: authenticationv1alpha1.SchemeGroupVersion.Group,
+					},
+					newCacheValue(t, goodWebhookAuthenticatorSpecWithCA, string(oldCA)),
+				)
+			},
+			webhookAuthenticators: []runtime.Object{
+				&authenticationv1alpha1.WebhookAuthenticator{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:       "test-name",
+						Generation: 1234,
+					},
+					Spec: goodWebhookAuthenticatorSpecWithCA,
+					Status: authenticationv1alpha1.WebhookAuthenticatorStatus{
+						Conditions: conditionstestutil.Replace(
+							allHappyConditionsSuccess(goodWebhookDefaultServingCertEndpoint, frozenMetav1Now, 0),
+							[]metav1.Condition{
+								sadTLSConfigurationValid(frozenMetav1Now, 0),
+								unknownWebhookConnectionValid(frozenMetav1Now, 0),
+								unknownAuthenticatorValid(frozenMetav1Now, 0),
+								sadReadyCondition(frozenMetav1Now, 0),
+							},
+						),
+						Phase: "Error",
+					},
+				},
+			},
+			wantLogLines: []string{
+				fmt.Sprintf(`{"level":"info","timestamp":"2099-08-08T13:57:36.123456Z","logger":"webhookcachefiller-controller","caller":"webhookcachefiller/webhookcachefiller.go:<line>$webhookcachefiller.(*webhookCacheFillerController).havePreviouslyValidated","message":"cached webhook authenticator and desired webhook authenticator are the same: already cached, so skipping validations","webhookAuthenticator":"test-name","endpoint":"%s"}`, goodWebhookAuthenticatorSpecWithCA.Endpoint),
+				fmt.Sprintf(`{"level":"debug","timestamp":"2099-08-08T13:57:36.123456Z","logger":"webhookcachefiller-controller","caller":"webhookcachefiller/webhookcachefiller.go:<line>$webhookcachefiller.(*webhookCacheFillerController).updateStatus","message":"webhookauthenticator status successfully updated","webhookAuthenticator":"test-name","endpoint":"%s","phase":"Ready"}`, goodWebhookAuthenticatorSpecWithCA.Endpoint),
+			},
+			wantActions: func() []coretesting.Action {
+				updateStatusAction := coretesting.NewUpdateAction(webhookAuthenticatorGVR, "", &authenticationv1alpha1.WebhookAuthenticator{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:       "test-name",
+						Generation: 1234,
+					},
+					Spec: goodWebhookAuthenticatorSpecWithCA,
+					Status: authenticationv1alpha1.WebhookAuthenticatorStatus{ // updates the status to ready
+						Conditions: allHappyConditionsSuccess(goodWebhookDefaultServingCertEndpoint, frozenMetav1Now, 1234),
+						Phase:      "Ready",
+					},
+				})
+				updateStatusAction.Subresource = "status"
+				return []coretesting.Action{
+					coretesting.NewListAction(webhookAuthenticatorGVR, webhookAuthenticatorGVK, "", metav1.ListOptions{}),
+					coretesting.NewWatchAction(webhookAuthenticatorGVR, "", metav1.ListOptions{}),
+					updateStatusAction,
+				}
+			},
+			wantNamesOfWebhookAuthenticatorsInCache: []string{"test-name"}, // keeps the old entry in the cache
+		},
 		{
 			name: "Sync: valid WebhookAuthenticator with CA: will complete sync loop successfully with success conditions and ready phase",
 			webhookAuthenticators: []runtime.Object{

test/integration/concierge_jwtauthenticator_status_test.go

@@ -115,6 +115,48 @@ func TestConciergeJWTAuthenticatorWithExternalCABundleStatusIsUpdatedWhenExterna
 	}
 }
 
+func TestConciergeJWTAuthenticatorStatusShouldBeOverwrittenByControllerAfterAnyManualEdits_Parallel(t *testing.T) {
+	env := testlib.IntegrationEnv(t)
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+	t.Cleanup(cancel)
+
+	conciergeClient := testlib.NewConciergeClientset(t)
+
+	// Run several times because there is always a chance that the test could pass because the controller
+	// will resync every 3 minutes even if it does not pay attention to changes in status.
+	for i := range 3 {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			t.Parallel()
+
+			authenticator := testlib.CreateTestJWTAuthenticator(ctx, t, authenticationv1alpha1.JWTAuthenticatorSpec{
+				Issuer:   env.SupervisorUpstreamOIDC.Issuer,
+				Audience: "does-not-matter",
+				TLS: &authenticationv1alpha1.TLSSpec{
+					CertificateAuthorityData: base64.StdEncoding.EncodeToString([]byte(env.SupervisorUpstreamOIDC.CABundle)),
+				},
+			}, authenticationv1alpha1.JWTAuthenticatorPhaseReady)
+
+			updatedAuthenticator, err := conciergeClient.AuthenticationV1alpha1().JWTAuthenticators().Get(ctx, authenticator.Name, metav1.GetOptions{})
+			require.NoError(t, err)
+
+			updatedAuthenticator.Status.Phase = "Pending"
+			originalFirstConditionMessage := updatedAuthenticator.Status.Conditions[0].Message
+			updatedAuthenticator.Status.Conditions[0].Message = "this is a manually edited message that should go away"
+			_, err = conciergeClient.AuthenticationV1alpha1().JWTAuthenticators().UpdateStatus(ctx, updatedAuthenticator, metav1.UpdateOptions{})
+			require.NoError(t, err)
+
+			testlib.RequireEventually(t, func(requireEventually *require.Assertions) {
+				gotAuthenticator, err := conciergeClient.AuthenticationV1alpha1().JWTAuthenticators().Get(ctx, authenticator.Name, metav1.GetOptions{})
+				requireEventually.NoError(err)
+				requireEventually.Equal(authenticationv1alpha1.JWTAuthenticatorPhaseReady, gotAuthenticator.Status.Phase,
+					"the controller should have changed the phase back to Ready")
+				requireEventually.Equal(originalFirstConditionMessage, gotAuthenticator.Status.Conditions[0].Message,
+					"the controller should have changed the message back to the correct value but it didn't")
+			}, 30*time.Second, 250*time.Millisecond)
+		})
+	}
+}
+
 func TestConciergeJWTAuthenticatorStatus_Parallel(t *testing.T) {
 	env := testlib.IntegrationEnv(t)
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)

test/integration/concierge_webhookauthenticator_status_test.go

@@ -112,6 +112,42 @@ func TestConciergeWebhookAuthenticatorWithExternalCABundleStatusIsUpdatedWhenExt
 	}
 }
 
+func TestConciergeWebhookAuthenticatorStatusShouldBeOverwrittenByControllerAfterAnyManualEdits_Parallel(t *testing.T) {
+	env := testlib.IntegrationEnv(t)
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+	t.Cleanup(cancel)
+
+	conciergeClient := testlib.NewConciergeClientset(t)
+
+	// Run several times because there is always a chance that the test could pass because the controller
+	// will resync every 3 minutes even if it does not pay attention to changes in status.
+	for i := range 3 {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			t.Parallel()
+
+			authenticator := testlib.CreateTestWebhookAuthenticator(ctx, t, &env.TestWebhook, authenticationv1alpha1.WebhookAuthenticatorPhaseReady)
+
+			updatedAuthenticator, err := conciergeClient.AuthenticationV1alpha1().WebhookAuthenticators().Get(ctx, authenticator.Name, metav1.GetOptions{})
+			require.NoError(t, err)
+
+			updatedAuthenticator.Status.Phase = "Pending"
+			originalFirstConditionMessage := updatedAuthenticator.Status.Conditions[0].Message
+			updatedAuthenticator.Status.Conditions[0].Message = "this is a manually edited message that should go away"
+			_, err = conciergeClient.AuthenticationV1alpha1().WebhookAuthenticators().UpdateStatus(ctx, updatedAuthenticator, metav1.UpdateOptions{})
+			require.NoError(t, err)
+
+			testlib.RequireEventually(t, func(requireEventually *require.Assertions) {
+				gotAuthenticator, err := conciergeClient.AuthenticationV1alpha1().WebhookAuthenticators().Get(ctx, authenticator.Name, metav1.GetOptions{})
+				requireEventually.NoError(err)
+				requireEventually.Equal(authenticationv1alpha1.WebhookAuthenticatorPhaseReady, gotAuthenticator.Status.Phase,
+					"the controller should have changed the phase back to Ready")
+				requireEventually.Equal(originalFirstConditionMessage, gotAuthenticator.Status.Conditions[0].Message,
+					"the controller should have changed the message back to the correct value but it didn't")
+			}, 30*time.Second, 250*time.Millisecond)
+		})
+	}
+}
+
 func TestConciergeWebhookAuthenticatorStatus_Parallel(t *testing.T) {
 	env := testlib.IntegrationEnv(t)
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)