appview: stop serving service tokens past their expiry, and challenge the client when the hold rejects one

Seen in production on 2026-09-11: three cold pulls of a 22-layer image failed
with BLOB_UNKNOWN for layers that exist. The hold had answered 403 "service
token authentication failed: token has expired", and the same blobs served
fine a minute later.

Three things lined up. The registry middleware's validation cache kept a
fetched service token for a flat 45 seconds regardless of its real remaining
life, so a token fetched with 12 seconds left was still handed to the hold
half a minute after it died. The registry JWT is stamped from the auth cache's
expiry, which trailed the real exp by only 10 seconds, while distribution
accepts a JWT for 60 seconds past its exp, so a client could hold an accepted
JWT for most of a minute after the credential behind it was gone. And the
hold's 403 was flattened to BLOB_UNKNOWN, so the client failed instead of
re-authenticating.

Now the validation cache bounds an entry by the token's exp minus a shared
ServiceTokenSafetyMargin of 60 seconds, the same margin the auth cache and the
JWT stamp use, chosen to equal distribution's leeway so the last instant a JWT
is accepted is the service token's real exp. A PDS that grants less than the
margin gets half its remaining life instead of an already-past deadline. When
the hold rejects the service token as expired or missing, the appview drops
both cached copies and returns a 401 challenge so Docker and crane re-run the
token dance and retry; a genuine permission denial stays a 403, and a hold
that is down still maps to blob unknown.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EvFJr4Dwz8p2NDAeXmgmBt

pkg/appview/authgate/service_auth_fetcher.go

@@ -89,8 +89,10 @@ func (f *ServiceAuthFetcher) Fetch(ctx context.Context, did, authMethod string)
 	}
 
 	// GetOrFetchServiceToken* caches as a side effect — read the granted
-	// expiry back. The cached value already has the 10s safety margin from
-	// pkg/auth/cache.go applied.
+	// expiry back. The cached value already has auth.ServiceTokenSafetyMargin
+	// subtracted from the PDS-granted exp (pkg/auth/cache.go), and the token
+	// handler stamps the JWT's exp from exactly this value, so the JWT cannot
+	// outlive the service token behind it.
 	_, expiresAt := f.cache.Get(did, holdDID)
 	if expiresAt.IsZero() {
 		return time.Time{}, errors.New("service-auth fetched but cache miss")

pkg/appview/middleware/registry.go

@@ -35,6 +35,21 @@ const pullerDIDKey contextKey = "puller.did"
 // hasPushScopeKey is the context key for storing whether the JWT has push scope
 const hasPushScopeKey contextKey = "token.has_push_scope"
 
+// validationCacheTTL is the longest a fetched service token is reused from the
+// validation cache. It covers a typical Docker push, whose many blob requests
+// would otherwise each race on OAuth/DPoP.
+//
+// It is a ceiling, not the actual lifetime: getOrFetch also clamps the entry to
+// the token's own exp minus auth.ServiceTokenSafetyMargin. Without that clamp a
+// token fetched with 12s of life left was still served for the full 45s, so the
+// hold saw an expired service token and answered 403 "token has expired" on
+// blobs that exist.
+const validationCacheTTL = 45 * time.Second
+
+// validationCacheErrorTTL is how long a failed fetch is remembered so
+// concurrent requests fast-fail instead of stampeding the PDS.
+const validationCacheErrorTTL = 5 * time.Second
+
 // validationCacheEntry stores a validated service token with expiration
 type validationCacheEntry struct {
 	serviceToken string
@@ -154,15 +169,14 @@ func (vc *validationCache) getOrFetch(ctx context.Context, cacheKey string, fetc
 	entry.inFlight = false
 
 	if err != nil {
-		// Cache errors for 5 seconds (fast-fail for subsequent requests)
+		// Cache errors briefly (fast-fail for subsequent requests)
 		entry.err = err
-		entry.validUntil = time.Now().Add(5 * time.Second)
+		entry.validUntil = time.Now().Add(validationCacheErrorTTL)
 		entry.serviceToken = ""
 	} else {
-		// Cache token for 45 seconds (covers typical Docker push operation)
 		entry.err = nil
 		entry.serviceToken = serviceToken
-		entry.validUntil = time.Now().Add(45 * time.Second)
+		entry.validUntil = tokenValidUntil(serviceToken)
 	}
 
 	// Signal completion to waiting goroutines
@@ -172,6 +186,65 @@ func (vc *validationCache) getOrFetch(ctx context.Context, cacheKey string, fetc
 	return serviceToken, err
 }
 
+// tokenValidUntil bounds a cached service token by its own exp claim, not just
+// by the flat validation-cache TTL.
+//
+// The cache used to pin any successful fetch for validationCacheTTL regardless
+// of how much life the token actually had. The token is minted with a fixed
+// absolute expiry, so a fetch that landed near the end of one (the auth cache
+// hands back a token until it is close to expiry, and the PDS may grant less
+// than asked) left the appview presenting a dead credential to the hold for the
+// rest of the 45s. The hold answered 403 "token has expired" and cold pulls
+// failed on layers that exist.
+//
+// A token whose exp cannot be parsed keeps the flat TTL: the appview cannot do
+// better than its previous behaviour for a token shape it does not understand,
+// and pkg/auth's cache applies the same fallback.
+func tokenValidUntil(serviceToken string) time.Time {
+	validUntil := time.Now().Add(validationCacheTTL)
+
+	exp, err := auth.ServiceTokenExpiry(serviceToken)
+	if err != nil {
+		slog.Warn("Service token exp unreadable, using flat validation cache TTL",
+			"component", "registry/middleware",
+			"error", err,
+			"ttl", validationCacheTTL)
+		return validUntil
+	}
+
+	// Same margin the auth cache and the registry JWT's exp use, so all three
+	// stop trusting the token at the same moment.
+	if safe := exp.Add(-auth.ServiceTokenSafetyMargin); safe.Before(validUntil) {
+		return safe
+	}
+	return validUntil
+}
+
+// invalidate expires the entry for cacheKey so the next getOrFetch re-mints.
+// Called when the hold rejects the token we handed it: the entry is stale by
+// definition and replaying it would fail the client's retry the same way.
+//
+// The entry is expired in place rather than deleted from the map because
+// concurrent goroutines already hold the pointer; an in-flight fetch is left
+// alone because it is about to store a fresh token anyway.
+func (vc *validationCache) invalidate(cacheKey string) {
+	vc.mu.RLock()
+	entry, exists := vc.entries[cacheKey]
+	vc.mu.RUnlock()
+
+	if !exists {
+		return
+	}
+
+	entry.mu.Lock()
+	if !entry.inFlight {
+		entry.serviceToken = ""
+		entry.err = nil
+		entry.validUntil = time.Time{}
+	}
+	entry.mu.Unlock()
+}
+
 // LabelChecker checks whether content has been taken down via ATProto labels.
 type LabelChecker interface {
 	IsTakenDown(did, repository string) (bool, error)
@@ -451,6 +524,9 @@ func (nr *NamespaceResolver) Repository(ctx context.Context, name reference.Name
 	// IMPORTANT: Use PULLER's DID/PDS for service token, not owner's!
 	// The puller (authenticated user) needs to authenticate to the hold service.
 	var serviceToken string
+	// invalidateServiceToken is handed to the blob store so it can drop this
+	// token when the hold rejects it; nil unless we actually fetched one.
+	var invalidateServiceToken func()
 	authMethod, _ := ctx.Value(authMethodKey).(string)
 	pullerDID, _ := ctx.Value(pullerDIDKey).(string)
 	hasPushScope, _ := ctx.Value(hasPushScopeKey).(bool)
@@ -537,6 +613,16 @@ func (nr *NamespaceResolver) Repository(ctx context.Context, name reference.Name
 				// Generic service token error
 				return nil, nr.authErrorMessage(fmt.Sprintf("Failed to obtain storage credentials: %v", fetchErr))
 			}
+
+			// Both caches have to go: the validation cache would otherwise
+			// replay the rejected token for the rest of its window, and
+			// pkg/auth's cache would hand the same one straight back to the
+			// refetch.
+			vc := nr.validationCache
+			invalidateServiceToken = func() {
+				vc.invalidate(cacheKey)
+				auth.InvalidateServiceToken(pullerDID, holdDID)
+			}
 		}
 	} else {
 		slog.Debug("Skipping service token fetch for unauthenticated request",
@@ -631,26 +717,27 @@ func (nr *NamespaceResolver) Repository(ctx context.Context, name reference.Name
 	// 3. The refresher already caches sessions efficiently (in-memory + DB)
 	// 4. Caching the repository with a stale ATProtoClient causes refresh token errors
 	registryCtx := &storage.RegistryContext{
-		DID:                did,
-		Handle:             handle,
-		HoldDID:            holdDID,
-		HoldURL:            holdURL,
-		PDSEndpoint:        pdsEndpoint,
-		Repository:         repositoryName,
-		ServiceToken:       serviceToken, // Cached service token from puller's PDS
-		ATProtoClient:      atprotoClient,
-		AuthMethod:         authMethod,        // Auth method from JWT token
-		PullerDID:          pullerDID,         // Authenticated user making the request
-		PullerPDSEndpoint:  pullerPDSEndpoint, // Puller's PDS for service token refresh
-		HasPushScope:       hasPushScope,      // Whether JWT has push scope (for pull stats filtering)
-		Anonymous:          pullerDID == "",   // No puller identity: hold decides via captain.Public
-		AutoRemoveUntagged: prefs.AutoRemoveUntagged,
-		Database:           nr.database,
-		Authorizer:         nr.authorizer,
-		Refresher:          nr.refresher,
-		ReadmeFetcher:      nr.readmeFetcher,
-		WebhookDispatcher:  nr.webhookDispatcher,
-		ManifestRefChecker: nr.manifestRefChecker,
+		DID:                    did,
+		Handle:                 handle,
+		HoldDID:                holdDID,
+		HoldURL:                holdURL,
+		PDSEndpoint:            pdsEndpoint,
+		Repository:             repositoryName,
+		ServiceToken:           serviceToken, // Cached service token from puller's PDS
+		InvalidateServiceToken: invalidateServiceToken,
+		ATProtoClient:          atprotoClient,
+		AuthMethod:             authMethod,        // Auth method from JWT token
+		PullerDID:              pullerDID,         // Authenticated user making the request
+		PullerPDSEndpoint:      pullerPDSEndpoint, // Puller's PDS for service token refresh
+		HasPushScope:           hasPushScope,      // Whether JWT has push scope (for pull stats filtering)
+		Anonymous:              pullerDID == "",   // No puller identity: hold decides via captain.Public
+		AutoRemoveUntagged:     prefs.AutoRemoveUntagged,
+		Database:               nr.database,
+		Authorizer:             nr.authorizer,
+		Refresher:              nr.refresher,
+		ReadmeFetcher:          nr.readmeFetcher,
+		WebhookDispatcher:      nr.webhookDispatcher,
+		ManifestRefChecker:     nr.manifestRefChecker,
 	}
 
 	return storage.NewRoutingRepository(repo, registryCtx), nil

pkg/appview/middleware/validation_cache_test.go

@@ -0,0 +1,151 @@
+package middleware
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"testing"
+	"time"
+
+	"atcr.io/pkg/auth"
+)
+
+// serviceTokenExpiring builds an unsigned JWT whose exp claim is the given
+// time. The validation cache only reads exp; the hold is what verifies
+// signatures.
+func serviceTokenExpiring(at time.Time) string {
+	payload := fmt.Sprintf(`{"exp":%d}`, at.Unix())
+	return "header." + base64.RawURLEncoding.EncodeToString([]byte(payload)) + ".signature"
+}
+
+// entryValidUntil reads back the cached entry's deadline.
+func entryValidUntil(t *testing.T, vc *validationCache, cacheKey string) time.Time {
+	t.Helper()
+
+	vc.mu.RLock()
+	entry, ok := vc.entries[cacheKey]
+	vc.mu.RUnlock()
+	if !ok {
+		t.Fatalf("no validation cache entry for %q", cacheKey)
+	}
+
+	entry.mu.Lock()
+	defer entry.mu.Unlock()
+	return entry.validUntil
+}
+
+// A token with only 20s of life must not be pinned for the flat 45s TTL: that
+// is the production bug, where the appview kept presenting a dead service token
+// and the hold answered 403 "token has expired" on blobs that exist.
+func TestValidationCache_BoundsEntryByTokenExpiry(t *testing.T) {
+	vc := newValidationCache()
+	cacheKey := "did:plc:puller:did:web:hold.example.com"
+
+	realExp := time.Now().Add(20 * time.Second)
+	token := serviceTokenExpiring(realExp)
+
+	got, err := vc.getOrFetch(context.Background(), cacheKey, func() (string, error) {
+		return token, nil
+	})
+	if err != nil {
+		t.Fatalf("getOrFetch() error = %v", err)
+	}
+	if got != token {
+		t.Fatalf("getOrFetch() returned %q, want the fetched token", got)
+	}
+
+	validUntil := entryValidUntil(t, vc, cacheKey)
+
+	want := realExp.Add(-auth.ServiceTokenSafetyMargin)
+	if validUntil.After(want) {
+		t.Errorf("entry valid until %v, want no later than %v (exp minus %v)",
+			validUntil, want, auth.ServiceTokenSafetyMargin)
+	}
+
+	if flat := time.Now().Add(validationCacheTTL); !validUntil.Before(flat) {
+		t.Errorf("entry valid until %v, which is the flat %v TTL: the token's own exp was ignored",
+			validUntil, validationCacheTTL)
+	}
+}
+
+// A long-lived token is still capped by the flat TTL, so the cache keeps
+// rechecking the underlying session rather than holding one token for minutes.
+func TestValidationCache_LongLivedTokenKeepsFlatTTL(t *testing.T) {
+	vc := newValidationCache()
+	cacheKey := "did:plc:puller:did:web:hold.example.com"
+
+	token := serviceTokenExpiring(time.Now().Add(1 * time.Hour))
+
+	if _, err := vc.getOrFetch(context.Background(), cacheKey, func() (string, error) {
+		return token, nil
+	}); err != nil {
+		t.Fatalf("getOrFetch() error = %v", err)
+	}
+
+	validUntil := entryValidUntil(t, vc, cacheKey)
+
+	want := time.Now().Add(validationCacheTTL)
+	if diff := validUntil.Sub(want); diff < -2*time.Second || diff > 2*time.Second {
+		t.Errorf("entry valid until %v, want ~%v (the flat TTL)", validUntil, want)
+	}
+}
+
+// A token whose exp cannot be read falls back to the behaviour that shipped
+// before: the flat TTL. The appview can't do better for a shape it doesn't
+// understand, and refusing to cache would stampede the PDS.
+func TestValidationCache_UnparsableTokenKeepsFlatTTL(t *testing.T) {
+	vc := newValidationCache()
+	cacheKey := "did:plc:puller:did:web:hold.example.com"
+
+	if _, err := vc.getOrFetch(context.Background(), cacheKey, func() (string, error) {
+		return "not-a-jwt", nil
+	}); err != nil {
+		t.Fatalf("getOrFetch() error = %v", err)
+	}
+
+	validUntil := entryValidUntil(t, vc, cacheKey)
+
+	want := time.Now().Add(validationCacheTTL)
+	if diff := validUntil.Sub(want); diff < -2*time.Second || diff > 2*time.Second {
+		t.Errorf("entry valid until %v, want ~%v (the flat TTL)", validUntil, want)
+	}
+}
+
+// invalidate must force the next call to re-mint, which is what makes the 401
+// we send to Docker worth retrying.
+func TestValidationCache_InvalidateForcesRefetch(t *testing.T) {
+	vc := newValidationCache()
+	cacheKey := "did:plc:puller:did:web:hold.example.com"
+
+	fetches := 0
+	fetch := func() (string, error) {
+		fetches++
+		return serviceTokenExpiring(time.Now().Add(5 * time.Minute)), nil
+	}
+
+	if _, err := vc.getOrFetch(context.Background(), cacheKey, fetch); err != nil {
+		t.Fatalf("getOrFetch() error = %v", err)
+	}
+	if _, err := vc.getOrFetch(context.Background(), cacheKey, fetch); err != nil {
+		t.Fatalf("getOrFetch() error = %v", err)
+	}
+	if fetches != 1 {
+		t.Fatalf("second call fetched again (%d fetches), expected a cache hit", fetches)
+	}
+
+	vc.invalidate(cacheKey)
+
+	if _, err := vc.getOrFetch(context.Background(), cacheKey, fetch); err != nil {
+		t.Fatalf("getOrFetch() after invalidate error = %v", err)
+	}
+	if fetches != 2 {
+		t.Errorf("got %d fetches after invalidate, want 2", fetches)
+	}
+}
+
+// invalidate on a key that was never cached is a no-op, not a panic: the blob
+// store calls it on whatever request hit the rejection.
+func TestValidationCache_InvalidateUnknownKey(t *testing.T) {
+	vc := newValidationCache()
+	vc.invalidate("did:plc:nobody:did:web:hold.example.com")
+}

pkg/appview/storage/context.go

@@ -75,6 +75,15 @@ type RegistryContext struct {
 	HasPushScope      bool            // Whether the JWT token has push scope (used to filter pull stats)
 	Anonymous         bool            // Request carries no puller identity (anonymous pull); hold decides via captain.Public
 
+	// InvalidateServiceToken drops every process-level cache holding the
+	// puller's service token for this hold. Called when the hold rejects the
+	// token as expired or invalid: without it the client's retry (prompted by
+	// the 401 we return) would be handed the very same dead token and fail
+	// again. Set by the registry middleware, which owns the per-process
+	// validation cache; nil on requests that never fetched a token (anonymous
+	// pulls) and in tests.
+	InvalidateServiceToken func()
+
 	// Per-request user preferences
 	AutoRemoveUntagged bool // Whether to auto-delete untagged manifests on tag overwrite
 

pkg/appview/storage/proxy_blob_auth_test.go

@@ -0,0 +1,226 @@
+package storage
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/distribution/distribution/v3"
+	"github.com/distribution/distribution/v3/registry/api/errcode"
+	"github.com/opencontainers/go-digest"
+)
+
+// The two bodies below are copied from what the hold actually writes. Its OCI
+// blob handler answers every read authorization failure with
+// http.Error(w, "authorization failed: "+err.Error(), http.StatusForbidden),
+// and the error underneath is either the service-token wrapper from
+// ValidateBlobReadAccess or an *AuthError. Same status, opposite meaning.
+const (
+	holdExpiredTokenBody = "authorization failed: service token authentication failed: token has expired"
+	holdPermissionBody   = "authorization failed: access denied for blob:read: crew member lacks permission (required: blob:read or blob:write)"
+)
+
+const authTestDigest = digest.Digest("sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd")
+
+// statWithHoldFailure points a blob store at a hold that rejects the presign
+// with the given status and body, and returns Stat's error plus whether the
+// service token was invalidated.
+func statWithHoldFailure(t *testing.T, status int, body string) (error, bool) {
+	t.Helper()
+
+	holdServer := newMockHoldServer(t, "http://s3.invalid")
+	defer holdServer.Close()
+
+	holdServer.mu.Lock()
+	holdServer.PresignAuthFailure = &mockAuthFailure{Status: status, Body: body}
+	holdServer.mu.Unlock()
+
+	invalidated := false
+	store := createTestProxyBlobStore(t, holdServer.URL)
+	store.ctx.PullerDID = "did:plc:puller"
+	store.ctx.InvalidateServiceToken = func() { invalidated = true }
+
+	_, err := store.Stat(context.Background(), authTestDigest)
+	return err, invalidated
+}
+
+// The production failure: the hold rejects our service token as expired, which
+// used to surface as BLOB_UNKNOWN (or a bare 403) and fail the pull outright.
+// It has to become a 401 so BearerChallenge attaches WWW-Authenticate and the
+// client re-runs the token dance.
+func TestStat_HoldRejectsExpiredServiceToken_ReturnsUnauthorized(t *testing.T) {
+	err, invalidated := statWithHoldFailure(t, http.StatusForbidden, holdExpiredTokenBody)
+
+	var ecErr errcode.Error
+	if !errors.As(err, &ecErr) {
+		t.Fatalf("Stat() error = %v (%T), want an errcode.Error", err, err)
+	}
+	if ecErr.Code != errcode.ErrorCodeUnauthorized {
+		t.Errorf("Stat() code = %v, want %v", ecErr.Code, errcode.ErrorCodeUnauthorized)
+	}
+	if !invalidated {
+		t.Error("the stale service token was not invalidated: the client's retry would be handed the same dead token")
+	}
+}
+
+// A crew member without read access authenticated fine. Sending them back to
+// re-authenticate would loop forever, so this stays a 403.
+func TestStat_HoldDeniesPermission_StaysDenied(t *testing.T) {
+	err, invalidated := statWithHoldFailure(t, http.StatusForbidden, holdPermissionBody)
+
+	var ecErr errcode.Error
+	if !errors.As(err, &ecErr) {
+		t.Fatalf("Stat() error = %v (%T), want an errcode.Error", err, err)
+	}
+	if ecErr.Code != errcode.ErrorCodeDenied {
+		t.Errorf("Stat() code = %v, want %v", ecErr.Code, errcode.ErrorCodeDenied)
+	}
+	if invalidated {
+		t.Error("a permission denial must not throw away a perfectly good service token")
+	}
+}
+
+// A bare 401 from the hold is a challenge by definition, whatever its body.
+func TestStat_HoldAnswers401_ReturnsUnauthorized(t *testing.T) {
+	err, invalidated := statWithHoldFailure(t, http.StatusUnauthorized, "unauthorized")
+
+	var ecErr errcode.Error
+	if !errors.As(err, &ecErr) {
+		t.Fatalf("Stat() error = %v (%T), want an errcode.Error", err, err)
+	}
+	if ecErr.Code != errcode.ErrorCodeUnauthorized {
+		t.Errorf("Stat() code = %v, want %v", ecErr.Code, errcode.ErrorCodeUnauthorized)
+	}
+	if !invalidated {
+		t.Error("a 401 from the hold should drop the credential it rejected")
+	}
+}
+
+// An anonymous pull that the hold refuses keeps its existing 401 "authenticate
+// first" path, and must not touch a service token it never had.
+func TestStat_AnonymousDenial_StillAsksForCredentials(t *testing.T) {
+	holdServer := newMockHoldServer(t, "http://s3.invalid")
+	defer holdServer.Close()
+
+	holdServer.mu.Lock()
+	holdServer.PresignAuthFailure = &mockAuthFailure{
+		Status: http.StatusForbidden,
+		Body:   holdPermissionBody,
+	}
+	holdServer.mu.Unlock()
+
+	store := createTestProxyBlobStore(t, holdServer.URL)
+	store.ctx.Anonymous = true
+	store.ctx.ServiceToken = ""
+
+	_, err := store.Stat(context.Background(), authTestDigest)
+
+	var ecErr errcode.Error
+	if !errors.As(err, &ecErr) {
+		t.Fatalf("Stat() error = %v (%T), want an errcode.Error", err, err)
+	}
+	if ecErr.Code != errcode.ErrorCodeUnauthorized {
+		t.Errorf("Stat() code = %v, want %v", ecErr.Code, errcode.ErrorCodeUnauthorized)
+	}
+	if ecErr.Message != "authentication required" {
+		t.Errorf("Stat() message = %q, want the anonymous challenge to be unchanged", ecErr.Message)
+	}
+}
+
+// A hold that is broken, rather than one that refused us, must keep answering
+// the way it did before: blob-unknown, not a re-auth challenge. A client that
+// re-authenticated over a 5xx would just burn tokens against a sick hold.
+func TestStat_HoldServerError_StillBlobUnknown(t *testing.T) {
+	holdServer := newMockHoldServer(t, "http://s3.invalid")
+	defer holdServer.Close()
+
+	holdServer.mu.Lock()
+	holdServer.PresignError = errors.New("internal error")
+	holdServer.mu.Unlock()
+
+	store := createTestProxyBlobStore(t, holdServer.URL)
+	store.ctx.InvalidateServiceToken = func() {
+		t.Error("a 5xx must not invalidate the service token")
+	}
+
+	_, err := store.Stat(context.Background(), authTestDigest)
+	if !errors.Is(err, distribution.ErrBlobUnknown) {
+		t.Errorf("Stat() error = %v, want %v", err, distribution.ErrBlobUnknown)
+	}
+}
+
+// Same for a hold that is simply unreachable.
+func TestStat_HoldUnreachable_StillBlobUnknown(t *testing.T) {
+	holdServer := newMockHoldServer(t, "http://s3.invalid")
+	holdURL := holdServer.URL
+	holdServer.Close()
+
+	store := createTestProxyBlobStore(t, holdURL)
+	store.ctx.InvalidateServiceToken = func() {
+		t.Error("a connection failure must not invalidate the service token")
+	}
+
+	_, err := store.Stat(context.Background(), authTestDigest)
+	if !errors.Is(err, distribution.ErrBlobUnknown) {
+		t.Errorf("Stat() error = %v, want %v", err, distribution.ErrBlobUnknown)
+	}
+}
+
+// ServeBlob runs the same presign path, so the challenge has to reach the
+// client from there too: on a GET, distribution calls Stat and then ServeBlob.
+func TestServeBlob_HoldRejectsExpiredServiceToken_ReturnsUnauthorized(t *testing.T) {
+	holdServer := newMockHoldServer(t, "http://s3.invalid")
+	defer holdServer.Close()
+
+	holdServer.mu.Lock()
+	holdServer.PresignAuthFailure = &mockAuthFailure{
+		Status: http.StatusForbidden,
+		Body:   holdExpiredTokenBody,
+	}
+	holdServer.mu.Unlock()
+
+	invalidated := false
+	store := createTestProxyBlobStore(t, holdServer.URL)
+	store.ctx.InvalidateServiceToken = func() { invalidated = true }
+
+	req := httptest.NewRequest(http.MethodGet, "/v2/alice/img/blobs/"+authTestDigest.String(), nil)
+	err := store.ServeBlob(context.Background(), httptest.NewRecorder(), req, authTestDigest)
+
+	var ecErr errcode.Error
+	if !errors.As(err, &ecErr) {
+		t.Fatalf("ServeBlob() error = %v (%T), want an errcode.Error", err, err)
+	}
+	if ecErr.Code != errcode.ErrorCodeUnauthorized {
+		t.Errorf("ServeBlob() code = %v, want %v", ecErr.Code, errcode.ErrorCodeUnauthorized)
+	}
+	if !invalidated {
+		t.Error("the stale service token was not invalidated")
+	}
+}
+
+func TestIsHoldAuthenticationFailure(t *testing.T) {
+	tests := []struct {
+		name   string
+		status int
+		body   string
+		want   bool
+	}{
+		{"expired service token", http.StatusForbidden, holdExpiredTokenBody, true},
+		{"dpop failure", http.StatusForbidden, "authorization failed: DPoP authentication failed: bad proof", true},
+		{"no auth scheme", http.StatusForbidden, "authorization failed: invalid authorization scheme: expected 'Bearer' or 'DPoP'", true},
+		{"any 401", http.StatusUnauthorized, "", true},
+		{"crew member lacks permission", http.StatusForbidden, holdPermissionBody, false},
+		{"not a crew member", http.StatusForbidden, "authorization failed: access denied for blob:read: user is not a crew member (required: blob:read or blob:write)", false},
+		{"unrecognised 403", http.StatusForbidden, "authorization failed: failed to get captain record: boom", false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := isHoldAuthenticationFailure(tt.status, []byte(tt.body)); got != tt.want {
+				t.Errorf("isHoldAuthenticationFailure(%d, %q) = %v, want %v", tt.status, tt.body, got, tt.want)
+			}
+		})
+	}
+}

pkg/appview/storage/proxy_blob_store.go

@@ -10,6 +10,7 @@ import (
 	"log/slog"
 	"net/http"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -567,6 +568,58 @@ func (p *ProxyBlobStore) presign(ctx context.Context, method string, dgst digest
 	return blob, nil
 }
 
+// maxHoldErrorBody bounds how much of a hold error response is read for
+// classification. The hold's is a single http.Error line.
+const maxHoldErrorBody = 4 << 10
+
+// holdAuthFailureMarkers are the substrings that mark a hold 401/403 as "we
+// could not authenticate you" rather than "you may not read this".
+//
+// The hold answers every blob authorization failure the same way
+// (pkg/hold/pds/xrpc.go): http.Error with 403 and a body of
+// "authorization failed: " + err.Error(). Two shapes come out of that, and they
+// need opposite handling:
+//
+//	authorization failed: service token authentication failed: token has expired
+//	authorization failed: access denied for blob:read: crew member lacks permission (required: blob:read or blob:write)
+//
+// The first says our own credential is stale, which a retry with a fresh one
+// fixes, so it has to reach the client as a 401 challenge. The second is the
+// hold's verdict on the user and must stay a 403, or the client would be sent
+// to re-authenticate in a loop it can never win.
+//
+// Anything unrecognised is treated as a permission denial. That is the
+// conservative direction: a misread denial costs one failed request, whereas a
+// misread challenge costs a re-auth loop.
+var holdAuthFailureMarkers = []string{
+	// "service token authentication failed", "DPoP authentication failed"
+	"authentication failed",
+	// pds.ErrTokenExpired, in case the hold ever reports it unwrapped
+	"token has expired",
+	"missing authorization header",
+	"invalid authorization header format",
+	"invalid authorization scheme",
+	"missing token",
+}
+
+// isHoldAuthenticationFailure reports whether a hold rejection is about the
+// credential we presented rather than about what the caller is allowed to do.
+func isHoldAuthenticationFailure(status int, body []byte) bool {
+	// A 401 is by definition a challenge: the hold is asking for credentials,
+	// not refusing a request it understood.
+	if status == http.StatusUnauthorized {
+		return true
+	}
+
+	lower := strings.ToLower(string(body))
+	for _, marker := range holdAuthFailureMarkers {
+		if strings.Contains(lower, marker) {
+			return true
+		}
+	}
+	return false
+}
+
 // getPresignedURL asks the hold for a presigned URL for a blob operation, and
 // for reads gets the blob's size back with it.
 func (p *ProxyBlobStore) getPresignedURL(ctx context.Context, operation string, dgst digest.Digest) (presignedBlob, error) {
@@ -591,10 +644,38 @@ func (p *ProxyBlobStore) getPresignedURL(ctx context.Context, operation string,
 	}
 	defer resp.Body.Close()
 
-	if resp.StatusCode == http.StatusForbidden && p.ctx.Anonymous {
-		// Stale local captain cache let an anonymous request through, but the
-		// hold says private. Surface a 401 so the client re-authenticates.
-		return presignedBlob{}, errcode.ErrorCodeUnauthorized.WithMessage("authentication required")
+	if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
+		if p.ctx.Anonymous {
+			// Stale local captain cache let an anonymous request through, but the
+			// hold says private. Surface a 401 so the client re-authenticates.
+			return presignedBlob{}, errcode.ErrorCodeUnauthorized.WithMessage("authentication required")
+		}
+
+		// The body is a short http.Error line; cap the read anyway so a
+		// misbehaving hold cannot make us buffer a response of its choosing.
+		bodyBytes, _ := io.ReadAll(io.LimitReader(resp.Body, maxHoldErrorBody))
+
+		if isHoldAuthenticationFailure(resp.StatusCode, bodyBytes) {
+			// Our credential is stale, not the user's permissions. Drop the
+			// cached token so the retry mints a new one, and answer with a 401
+			// so BearerChallenge attaches WWW-Authenticate and the client runs
+			// the token dance again instead of failing the pull.
+			slog.Warn("Hold rejected the service token, challenging client to re-authenticate",
+				"component", "proxy_blob_store",
+				"hold_did", p.ctx.HoldDID,
+				"puller_did", p.ctx.PullerDID,
+				"status", resp.StatusCode,
+				"body", strings.TrimSpace(string(bodyBytes)))
+			if p.ctx.InvalidateServiceToken != nil {
+				p.ctx.InvalidateServiceToken()
+			}
+			return presignedBlob{}, errcode.ErrorCodeUnauthorized.WithMessage("service token expired, re-authenticate")
+		}
+
+		// A real permission decision: the puller authenticated fine and may
+		// not read this hold. Sending them back to re-authenticate would just
+		// loop, so keep it a 403.
+		return presignedBlob{}, errcode.ErrorCodeDenied.WithMessage("read access denied")
 	}
 
 	if resp.StatusCode == http.StatusNotFound {

pkg/appview/storage/proxy_blob_store_test.go

@@ -262,8 +262,10 @@ func TestDoAuthenticatedRequest_AnonymousWhenNoToken(t *testing.T) {
 }
 
 // TestGetPresignedURL_HoldForbiddenMapsByAnonymity verifies a hold 403 becomes a
-// client 401 for anonymous requests (stale captain cache) but stays a generic
-// error for authenticated ones.
+// client 401 for anonymous requests (stale captain cache) and, for an
+// authenticated request whose rejection names no authentication failure, a 403
+// DENIED: the caller got through the door and may not read. The authentication
+// cases live in proxy_blob_auth_test.go.
 func TestGetPresignedURL_HoldForbiddenMapsByAnonymity(t *testing.T) {
 	holdServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 		w.WriteHeader(http.StatusForbidden)
@@ -289,7 +291,7 @@ func TestGetPresignedURL_HoldForbiddenMapsByAnonymity(t *testing.T) {
 		}
 	})
 
-	t.Run("authenticated -> generic error", func(t *testing.T) {
+	t.Run("authenticated -> 403 denied", func(t *testing.T) {
 		store := NewProxyBlobStore(&RegistryContext{
 			DID:          "did:plc:owner",
 			HoldDID:      "did:web:hold.example.com",
@@ -298,11 +300,12 @@ func TestGetPresignedURL_HoldForbiddenMapsByAnonymity(t *testing.T) {
 			Anonymous:    false,
 		})
 		_, err := store.getPresignedURL(context.Background(), "GET", dgst)
-		if err == nil {
-			t.Fatal("expected an error for authenticated 403")
+		ec, ok := err.(errcode.Error)
+		if !ok {
+			t.Fatalf("expected errcode.Error, got %T: %v", err, err)
 		}
-		if _, ok := err.(errcode.Error); ok {
-			t.Errorf("expected a generic (non-errcode) error for authenticated 403, got errcode: %v", err)
+		if ec.Code != errcode.ErrorCodeDenied {
+			t.Errorf("expected DENIED, got %v", ec.Code)
 		}
 	})
 }
@@ -676,6 +679,13 @@ type mockHoldServer struct {
 	AbortError    error
 	PresignError  error
 
+	// PresignAuthFailure, when set, makes the presign endpoint answer the way
+	// the real hold answers a blob authorization failure: an http.Error line
+	// with the hold's status and body shape (pkg/hold/pds/xrpc.go writes
+	// "authorization failed: " + err.Error() with status 403 for every one).
+	// It is checked before PresignError so a test can inject either.
+	PresignAuthFailure *mockAuthFailure
+
 	// AbortHook, when set, runs as an abort request is handled. It exists so a
 	// test can order the abort against other events (an in-flight part
 	// finishing, say) rather than only counting aborts after the fact.
@@ -685,6 +695,13 @@ type mockHoldServer struct {
 	UploadID string
 }
 
+// mockAuthFailure is one hold authorization rejection: the status it answers
+// with and the body it writes.
+type mockAuthFailure struct {
+	Status int
+	Body   string
+}
+
 type mockInitiateCall struct {
 	Digest string
 }
@@ -793,6 +810,15 @@ func newMockHoldServer(t *testing.T, s3URL string) *mockHoldServer {
 
 		switch {
 		case strings.Contains(r.URL.Path, atproto.SyncGetBlob):
+			if m.PresignAuthFailure != nil {
+				// http.Error, exactly as the hold sends it: text/plain with a
+				// trailing newline, not the JSON the success path uses.
+				w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+				w.WriteHeader(m.PresignAuthFailure.Status)
+				fmt.Fprintln(w, m.PresignAuthFailure.Body)
+				return
+			}
+
 			if m.PresignError != nil {
 				w.WriteHeader(http.StatusInternalServerError)
 				fmt.Fprintf(w, `{"error":"%s"}`, m.PresignError.Error())