appview: stop serving service tokens past their expiry, and challenge the client when the hold rejects one

Seen in production on 2026-09-11: three cold pulls of a 22-layer image failed
with BLOB_UNKNOWN for layers that exist. The hold had answered 403 "service
token authentication failed: token has expired", and the same blobs served
fine a minute later.

Three things lined up. The registry middleware's validation cache kept a
fetched service token for a flat 45 seconds regardless of its real remaining
life, so a token fetched with 12 seconds left was still handed to the hold
half a minute after it died. The registry JWT is stamped from the auth cache's
expiry, which trailed the real exp by only 10 seconds, while distribution
accepts a JWT for 60 seconds past its exp, so a client could hold an accepted
JWT for most of a minute after the credential behind it was gone. And the
hold's 403 was flattened to BLOB_UNKNOWN, so the client failed instead of
re-authenticating.

Now the validation cache bounds an entry by the token's exp minus a shared
ServiceTokenSafetyMargin of 60 seconds, the same margin the auth cache and the
JWT stamp use, chosen to equal distribution's leeway so the last instant a JWT
is accepted is the service token's real exp. A PDS that grants less than the
margin gets half its remaining life instead of an already-past deadline. When
the hold rejects the service token as expired or missing, the appview drops
both cached copies and returns a 401 challenge so Docker and crane re-run the
token dance and retry; a genuine permission denial stays a 403, and a hold
that is down still maps to blob unknown.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EvFJr4Dwz8p2NDAeXmgmBt

pkg/auth/cache.go

@@ -1,7 +1,9 @@
 // Package auth provides service token caching and management for AppView.
 // Service tokens are JWTs issued by a user's PDS to authorize AppView to
 // act on their behalf when communicating with hold services. Tokens are
-// cached with automatic expiry parsing and 10-second safety margins.
+// cached with automatic expiry parsing and a safety margin
+// (ServiceTokenSafetyMargin) so the cache stops serving a token before the
+// hold would reject it.
 package auth
 
 import (
@@ -14,6 +16,28 @@ import (
 	"time"
 )
 
+// ServiceTokenSafetyMargin is how far ahead of a service token's real exp the
+// AppView stops treating it as usable. Every cache that holds a service token
+// (this one, and the registry middleware's per-process validation cache)
+// subtracts it, and the registry JWT's exp is stamped from the value this
+// package returns, so the JWT never outlives the credential behind it.
+//
+// It is 60s because that is exactly distribution's token.Leeway: the registry
+// auth package accepts a registry JWT for 60s past its exp. With a 60s margin
+// the JWT is stamped at (service token exp - 60s), so the last moment
+// distribution will accept it is the service token's real exp. Shrinking this
+// below distribution's leeway reopens the window this constant closes: a client
+// would hold an accepted JWT while the service token behind it is already dead,
+// the hold would answer 403 "token has expired", and the pull would fail
+// instead of re-authenticating.
+const ServiceTokenSafetyMargin = 60 * time.Second
+
+// unparsableTokenTTL is how long a token whose exp claim could not be read is
+// cached. PDS-granted service tokens are requested with a 5 minute expiry (see
+// servicetoken.go), so a fixed 50s is comfortably inside any plausible real
+// lifetime and the next request re-mints.
+const unparsableTokenTTL = 50 * time.Second
+
 // serviceTokenEntry represents a cached service token.
 type serviceTokenEntry struct {
 	token     string
@@ -63,17 +87,34 @@ func (c *Cache) Get(did, holdDID string) (string, time.Time) {
 }
 
 // Set stores token for (did, holdDID), parsing its JWT exp claim and
-// applying a 10s safety margin so the cache expires before the real
-// token does. Falls back to a 50s TTL if the JWT can't be parsed.
+// applying ServiceTokenSafetyMargin so the cache expires before the real
+// token does. Falls back to unparsableTokenTTL if the JWT can't be parsed.
+//
+// A PDS is free to grant less than the margin (ATCR asks for 5 minutes;
+// reference PDSes grant up to an hour, others may grant less). Subtracting a
+// 60s margin from a 30s token would store an entry that is already expired,
+// which Get would evict on sight, so every single request would re-mint: a
+// refetch storm against the user's PDS. In that case the entry is kept for half
+// of whatever life the token actually has instead, which is always positive
+// while the token is alive and still leaves headroom proportional to it. A
+// token that arrives already expired gets a past expiry, which is correct: it
+// is unusable and the next call must mint a new one.
 func (c *Cache) Set(did, holdDID, token string) error {
 	cacheKey := did + ":" + holdDID
 
 	expiry, err := parseJWTExpiry(token)
 	if err != nil {
-		slog.Warn("Failed to parse JWT expiry, using default 50s", "error", err, "cacheKey", cacheKey)
-		expiry = time.Now().Add(50 * time.Second)
+		slog.Warn("Failed to parse JWT expiry, using fallback TTL",
+			"error", err, "cacheKey", cacheKey, "ttl", unparsableTokenTTL)
+		expiry = time.Now().Add(unparsableTokenTTL)
+	} else if remaining := time.Until(expiry); remaining <= ServiceTokenSafetyMargin {
+		slog.Warn("PDS granted a service token shorter than the safety margin",
+			"cacheKey", cacheKey,
+			"grantedLife", remaining.Round(time.Second),
+			"margin", ServiceTokenSafetyMargin)
+		expiry = time.Now().Add(remaining / 2)
 	} else {
-		expiry = expiry.Add(-10 * time.Second)
+		expiry = expiry.Add(-ServiceTokenSafetyMargin)
 	}
 
 	c.mu.Lock()
@@ -163,7 +204,7 @@ func GetServiceToken(did, holdDID string) (token string, expiresAt time.Time) {
 }
 
 // SetServiceToken stores token under (did, holdDID) in the default cache,
-// applying the standard 10s safety margin against the JWT's exp claim.
+// applying ServiceTokenSafetyMargin against the JWT's exp claim.
 func SetServiceToken(did, holdDID, token string) error {
 	return defaultCache.Set(did, holdDID, token)
 }
@@ -191,6 +232,18 @@ func DefaultCache() *Cache {
 	return defaultCache
 }
 
+// ServiceTokenExpiry reports the exp claim of a PDS-issued service token,
+// without verifying its signature (we trust tokens minted by the user's PDS,
+// and the hold verifies them anyway).
+//
+// Exported for callers that hold a service token outside this cache and must
+// not keep it past its real life. The registry middleware's validation cache is
+// the one that matters: it used to pin every token for a flat 45s, so a token
+// with 12s left was handed to the hold for another 33s after it died.
+func ServiceTokenExpiry(token string) (time.Time, error) {
+	return parseJWTExpiry(token)
+}
+
 // parseJWTExpiry extracts the exp claim from a JWT without verifying its
 // signature. We trust tokens from the user's PDS, so signature
 // verification isn't needed here.

pkg/auth/cache_test.go

@@ -1,6 +1,8 @@
 package auth
 
 import (
+	"encoding/base64"
+	"fmt"
 	"testing"
 	"time"
 )
@@ -232,3 +234,97 @@ func TestCache_PackageFunctionsDelegateToDefault(t *testing.T) {
 		t.Errorf("after InvalidateServiceToken, DefaultCache().Get() = %q, want empty", tok)
 	}
 }
+
+// testServiceToken builds an unsigned JWT whose exp claim is expiresAt. Only
+// the payload is meaningful: the cache reads exp without verifying anything.
+func testServiceToken(expiresAt time.Time) string {
+	payload := fmt.Sprintf(`{"exp":%d}`, expiresAt.Unix())
+	return "header." + base64.RawURLEncoding.EncodeToString([]byte(payload)) + ".signature"
+}
+
+func TestSetServiceToken_AppliesSafetyMargin(t *testing.T) {
+	defaultCache.Clear()
+
+	did := "did:plc:margin"
+	holdDID := "did:web:hold.example.com"
+
+	realExp := time.Now().Add(5 * time.Minute)
+	if err := SetServiceToken(did, holdDID, testServiceToken(realExp)); err != nil {
+		t.Fatalf("SetServiceToken() error = %v", err)
+	}
+
+	_, expiresAt := GetServiceToken(did, holdDID)
+	if expiresAt.IsZero() {
+		t.Fatal("expected the token to be cached")
+	}
+
+	want := realExp.Add(-ServiceTokenSafetyMargin)
+	if diff := expiresAt.Sub(want); diff < -2*time.Second || diff > 2*time.Second {
+		t.Errorf("cached expiry off by %v (want exp minus %v)", diff, ServiceTokenSafetyMargin)
+	}
+
+	// The point of the margin: the cache must stop serving the token at least
+	// distribution's 60s JWT leeway before the hold would reject it.
+	if got := realExp.Sub(expiresAt); got < 60*time.Second {
+		t.Errorf("cache serves the token until %v before its real exp, want >= 60s", got)
+	}
+}
+
+func TestSetServiceToken_ShortGrantKeepsPositiveTTL(t *testing.T) {
+	defaultCache.Clear()
+
+	did := "did:plc:shortgrant"
+	holdDID := "did:web:hold.example.com"
+
+	// A PDS that grants far less than the safety margin. Subtracting the margin
+	// outright would cache an already-expired entry and make every request
+	// re-mint, so the cache keeps half the remaining life instead.
+	realExp := time.Now().Add(20 * time.Second)
+	if err := SetServiceToken(did, holdDID, testServiceToken(realExp)); err != nil {
+		t.Fatalf("SetServiceToken() error = %v", err)
+	}
+
+	token, expiresAt := GetServiceToken(did, holdDID)
+	if token == "" {
+		t.Fatal("short-lived token should still be cached, not dropped on sight")
+	}
+	if !expiresAt.After(time.Now()) {
+		t.Fatalf("cached expiry %v is not in the future", expiresAt)
+	}
+	if !expiresAt.Before(realExp) {
+		t.Errorf("cached expiry %v should be before the token's real exp %v", expiresAt, realExp)
+	}
+}
+
+func TestSetServiceToken_UnparsableExpUsesFallbackTTL(t *testing.T) {
+	defaultCache.Clear()
+
+	did := "did:plc:unparsable"
+	holdDID := "did:web:hold.example.com"
+
+	if err := SetServiceToken(did, holdDID, "not-a-jwt"); err != nil {
+		t.Fatalf("SetServiceToken() error = %v", err)
+	}
+
+	_, expiresAt := GetServiceToken(did, holdDID)
+	want := time.Now().Add(unparsableTokenTTL)
+	if diff := expiresAt.Sub(want); diff < -5*time.Second || diff > 5*time.Second {
+		t.Errorf("expiry off by %v (want ~%v from now)", diff, unparsableTokenTTL)
+	}
+}
+
+func TestServiceTokenExpiry(t *testing.T) {
+	want := time.Now().Add(3 * time.Minute).Truncate(time.Second)
+
+	got, err := ServiceTokenExpiry(testServiceToken(want))
+	if err != nil {
+		t.Fatalf("ServiceTokenExpiry() error = %v", err)
+	}
+	if !got.Equal(want) {
+		t.Errorf("ServiceTokenExpiry() = %v, want %v", got, want)
+	}
+
+	if _, err := ServiceTokenExpiry("not-a-jwt"); err == nil {
+		t.Error("expected an error for a token that is not a JWT")
+	}
+}

pkg/auth/token/handler.go

@@ -710,11 +710,17 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 		if !res.expiresAt.IsZero() {
-			// Cap JWT lifetime at the service-auth's expiry. The cache's
-			// expiresAt already includes a 10s safety margin
-			// (pkg/auth/cache.go:71), so this guarantees the service-auth
-			// is still cache-valid for any /v2/* request the JWT can
-			// authorize. We never extend beyond the configured default.
+			// Cap JWT lifetime at the service-auth's expiry, stamping exactly
+			// the value the fetcher returned. That value already has
+			// auth.ServiceTokenSafetyMargin subtracted from the PDS-granted
+			// exp, and the margin is deliberately >= distribution's
+			// token.Leeway (60s): the registry auth package accepts this JWT
+			// for Leeway past its exp, so the last instant a client can use it
+			// is still inside the service token's real life. Do not shrink the
+			// margin below that leeway or the gap reopens, and Docker never
+			// gets the 401 that would make it re-authenticate.
+			//
+			// We never extend beyond the configured default.
 			until := time.Until(res.expiresAt)
 			if until < issueExp {
 				issueExp = until

pkg/auth/token/handler_margin_test.go

@@ -0,0 +1,81 @@
+package token
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	disttoken "github.com/distribution/distribution/v3/registry/auth/token"
+
+	"atcr.io/pkg/auth"
+)
+
+// The margin exists to cover distribution's JWT leeway. distribution's registry
+// auth accepts a token for Leeway past its exp, and the JWT's exp is stamped
+// from (service token exp - margin). If the margin ever drops below the leeway,
+// a client can hold an accepted JWT after the service token behind it is dead,
+// the hold answers 403 "token has expired", and the pull fails with no 401 to
+// make the client re-authenticate. That is the production bug this closes.
+func TestServiceTokenSafetyMarginCoversDistributionLeeway(t *testing.T) {
+	if auth.ServiceTokenSafetyMargin < disttoken.Leeway {
+		t.Fatalf("auth.ServiceTokenSafetyMargin is %v, which is under distribution's token.Leeway of %v: "+
+			"a registry JWT would stay acceptable after its service token expired",
+			auth.ServiceTokenSafetyMargin, disttoken.Leeway)
+	}
+}
+
+// The handler must stamp exactly the expiry the fetcher hands it. That value
+// already has the margin subtracted (pkg/auth/cache.go), so stamping anything
+// later would reopen the window, and the margin has to survive the round trip
+// through the token response.
+func TestHandler_ServiceAuthFetcher_MarginSurvivesStamping(t *testing.T) {
+	keyPath := getSharedTestKey(t)
+	issuer, err := NewIssuer(keyPath, "atcr.io", "registry", 15*time.Minute)
+	if err != nil {
+		t.Fatalf("NewIssuer() error = %v", err)
+	}
+
+	deviceStore, database := setupTestDeviceStore(t)
+	deviceSecret := createTestDevice(t, deviceStore, database, "did:plc:alice123", "alice.bsky.social")
+
+	handler := NewHandler(issuer, deviceStore)
+
+	// What a 5 minute PDS grant looks like coming out of the cache.
+	serviceTokenExp := time.Now().Add(5 * time.Minute)
+	handler.SetServiceAuthFetcher(&stubServiceAuthFetcher{
+		expiresAt: serviceTokenExp.Add(-auth.ServiceTokenSafetyMargin),
+	})
+
+	req := httptest.NewRequest(http.MethodGet, "/auth/token?service=registry&scope=repository:alice.bsky.social/myapp:pull,push", nil)
+	req.SetBasicAuth("alice", deviceSecret)
+	w := httptest.NewRecorder()
+
+	handler.ServeHTTP(w, req)
+
+	if w.Code != http.StatusOK {
+		t.Fatalf("expected 200, got %d. Body: %s", w.Code, w.Body.String())
+	}
+
+	var resp TokenResponse
+	if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {
+		t.Fatalf("decode response: %v", err)
+	}
+
+	jwtExpiresAt := time.Now().Add(time.Duration(resp.ExpiresIn) * time.Second)
+
+	// The last instant distribution will accept this JWT must still be inside
+	// the service token's real life.
+	lastAccepted := jwtExpiresAt.Add(disttoken.Leeway)
+	if lastAccepted.After(serviceTokenExp.Add(2 * time.Second)) {
+		t.Errorf("JWT is accepted until %v but the service token dies at %v",
+			lastAccepted, serviceTokenExp)
+	}
+
+	// And it must still be a usable token, not one squeezed to nothing.
+	if resp.ExpiresIn < 200 {
+		t.Errorf("expires_in = %d, want ~240s (5 min grant minus the %v margin)",
+			resp.ExpiresIn, auth.ServiceTokenSafetyMargin)
+	}
+}