appview: stop serving service tokens past their expiry, and challenge the client when the hold rejects one

Seen in production on 2026-09-11: three cold pulls of a 22-layer image failed
with BLOB_UNKNOWN for layers that exist. The hold had answered 403 "service
token authentication failed: token has expired", and the same blobs served
fine a minute later.

Three things lined up. The registry middleware's validation cache kept a
fetched service token for a flat 45 seconds regardless of its real remaining
life, so a token fetched with 12 seconds left was still handed to the hold
half a minute after it died. The registry JWT is stamped from the auth cache's
expiry, which trailed the real exp by only 10 seconds, while distribution
accepts a JWT for 60 seconds past its exp, so a client could hold an accepted
JWT for most of a minute after the credential behind it was gone. And the
hold's 403 was flattened to BLOB_UNKNOWN, so the client failed instead of
re-authenticating.

Now the validation cache bounds an entry by the token's exp minus a shared
ServiceTokenSafetyMargin of 60 seconds, the same margin the auth cache and the
JWT stamp use, chosen to equal distribution's leeway so the last instant a JWT
is accepted is the service token's real exp. A PDS that grants less than the
margin gets half its remaining life instead of an already-past deadline. When
the hold rejects the service token as expired or missing, the appview drops
both cached copies and returns a 401 challenge so Docker and crane re-run the
token dance and retry; a genuine permission denial stays a 403, and a hold
that is down still maps to blob unknown.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EvFJr4Dwz8p2NDAeXmgmBt

pkg/auth/cache_test.go

@@ -1,6 +1,8 @@
 package auth
 
 import (
+	"encoding/base64"
+	"fmt"
 	"testing"
 	"time"
 )
@@ -232,3 +234,97 @@ func TestCache_PackageFunctionsDelegateToDefault(t *testing.T) {
 		t.Errorf("after InvalidateServiceToken, DefaultCache().Get() = %q, want empty", tok)
 	}
 }
+
+// testServiceToken builds an unsigned JWT whose exp claim is expiresAt. Only
+// the payload is meaningful: the cache reads exp without verifying anything.
+func testServiceToken(expiresAt time.Time) string {
+	payload := fmt.Sprintf(`{"exp":%d}`, expiresAt.Unix())
+	return "header." + base64.RawURLEncoding.EncodeToString([]byte(payload)) + ".signature"
+}
+
+func TestSetServiceToken_AppliesSafetyMargin(t *testing.T) {
+	defaultCache.Clear()
+
+	did := "did:plc:margin"
+	holdDID := "did:web:hold.example.com"
+
+	realExp := time.Now().Add(5 * time.Minute)
+	if err := SetServiceToken(did, holdDID, testServiceToken(realExp)); err != nil {
+		t.Fatalf("SetServiceToken() error = %v", err)
+	}
+
+	_, expiresAt := GetServiceToken(did, holdDID)
+	if expiresAt.IsZero() {
+		t.Fatal("expected the token to be cached")
+	}
+
+	want := realExp.Add(-ServiceTokenSafetyMargin)
+	if diff := expiresAt.Sub(want); diff < -2*time.Second || diff > 2*time.Second {
+		t.Errorf("cached expiry off by %v (want exp minus %v)", diff, ServiceTokenSafetyMargin)
+	}
+
+	// The point of the margin: the cache must stop serving the token at least
+	// distribution's 60s JWT leeway before the hold would reject it.
+	if got := realExp.Sub(expiresAt); got < 60*time.Second {
+		t.Errorf("cache serves the token until %v before its real exp, want >= 60s", got)
+	}
+}
+
+func TestSetServiceToken_ShortGrantKeepsPositiveTTL(t *testing.T) {
+	defaultCache.Clear()
+
+	did := "did:plc:shortgrant"
+	holdDID := "did:web:hold.example.com"
+
+	// A PDS that grants far less than the safety margin. Subtracting the margin
+	// outright would cache an already-expired entry and make every request
+	// re-mint, so the cache keeps half the remaining life instead.
+	realExp := time.Now().Add(20 * time.Second)
+	if err := SetServiceToken(did, holdDID, testServiceToken(realExp)); err != nil {
+		t.Fatalf("SetServiceToken() error = %v", err)
+	}
+
+	token, expiresAt := GetServiceToken(did, holdDID)
+	if token == "" {
+		t.Fatal("short-lived token should still be cached, not dropped on sight")
+	}
+	if !expiresAt.After(time.Now()) {
+		t.Fatalf("cached expiry %v is not in the future", expiresAt)
+	}
+	if !expiresAt.Before(realExp) {
+		t.Errorf("cached expiry %v should be before the token's real exp %v", expiresAt, realExp)
+	}
+}
+
+func TestSetServiceToken_UnparsableExpUsesFallbackTTL(t *testing.T) {
+	defaultCache.Clear()
+
+	did := "did:plc:unparsable"
+	holdDID := "did:web:hold.example.com"
+
+	if err := SetServiceToken(did, holdDID, "not-a-jwt"); err != nil {
+		t.Fatalf("SetServiceToken() error = %v", err)
+	}
+
+	_, expiresAt := GetServiceToken(did, holdDID)
+	want := time.Now().Add(unparsableTokenTTL)
+	if diff := expiresAt.Sub(want); diff < -5*time.Second || diff > 5*time.Second {
+		t.Errorf("expiry off by %v (want ~%v from now)", diff, unparsableTokenTTL)
+	}
+}
+
+func TestServiceTokenExpiry(t *testing.T) {
+	want := time.Now().Add(3 * time.Minute).Truncate(time.Second)
+
+	got, err := ServiceTokenExpiry(testServiceToken(want))
+	if err != nil {
+		t.Fatalf("ServiceTokenExpiry() error = %v", err)
+	}
+	if !got.Equal(want) {
+		t.Errorf("ServiceTokenExpiry() = %v, want %v", got, want)
+	}
+
+	if _, err := ServiceTokenExpiry("not-a-jwt"); err == nil {
+		t.Error("expected an error for a token that is not a JWT")
+	}
+}