Files
at-container-registry/pkg/appview/middleware/validation_cache_test.go
T
Evan JarrettandClaude Fable 5.1 9dbc53b670 appview: stop serving service tokens past their expiry, and challenge the client when the hold rejects one
Seen in production on 2026-09-11: three cold pulls of a 22-layer image failed
with BLOB_UNKNOWN for layers that exist. The hold had answered 403 "service
token authentication failed: token has expired", and the same blobs served
fine a minute later.

Three things lined up. The registry middleware's validation cache kept a
fetched service token for a flat 45 seconds regardless of its real remaining
life, so a token fetched with 12 seconds left was still handed to the hold
half a minute after it died. The registry JWT is stamped from the auth cache's
expiry, which trailed the real exp by only 10 seconds, while distribution
accepts a JWT for 60 seconds past its exp, so a client could hold an accepted
JWT for most of a minute after the credential behind it was gone. And the
hold's 403 was flattened to BLOB_UNKNOWN, so the client failed instead of
re-authenticating.

Now the validation cache bounds an entry by the token's exp minus a shared
ServiceTokenSafetyMargin of 60 seconds, the same margin the auth cache and the
JWT stamp use, chosen to equal distribution's leeway so the last instant a JWT
is accepted is the service token's real exp. A PDS that grants less than the
margin gets half its remaining life instead of an already-past deadline. When
the hold rejects the service token as expired or missing, the appview drops
both cached copies and returns a 401 challenge so Docker and crane re-run the
token dance and retry; a genuine permission denial stays a 403, and a hold
that is down still maps to blob unknown.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EvFJr4Dwz8p2NDAeXmgmBt
2026-09-11 19:26:51 -05:00

152 lines
4.8 KiB
Go

package middleware
import (
"context"
"encoding/base64"
"fmt"
"testing"
"time"
"atcr.io/pkg/auth"
)
// serviceTokenExpiring builds an unsigned JWT whose exp claim is the given
// time. The validation cache only reads exp; the hold is what verifies
// signatures.
func serviceTokenExpiring(at time.Time) string {
payload := fmt.Sprintf(`{"exp":%d}`, at.Unix())
return "header." + base64.RawURLEncoding.EncodeToString([]byte(payload)) + ".signature"
}
// entryValidUntil reads back the cached entry's deadline.
func entryValidUntil(t *testing.T, vc *validationCache, cacheKey string) time.Time {
t.Helper()
vc.mu.RLock()
entry, ok := vc.entries[cacheKey]
vc.mu.RUnlock()
if !ok {
t.Fatalf("no validation cache entry for %q", cacheKey)
}
entry.mu.Lock()
defer entry.mu.Unlock()
return entry.validUntil
}
// A token with only 20s of life must not be pinned for the flat 45s TTL: that
// is the production bug, where the appview kept presenting a dead service token
// and the hold answered 403 "token has expired" on blobs that exist.
func TestValidationCache_BoundsEntryByTokenExpiry(t *testing.T) {
vc := newValidationCache()
cacheKey := "did:plc:puller:did:web:hold.example.com"
realExp := time.Now().Add(20 * time.Second)
token := serviceTokenExpiring(realExp)
got, err := vc.getOrFetch(context.Background(), cacheKey, func() (string, error) {
return token, nil
})
if err != nil {
t.Fatalf("getOrFetch() error = %v", err)
}
if got != token {
t.Fatalf("getOrFetch() returned %q, want the fetched token", got)
}
validUntil := entryValidUntil(t, vc, cacheKey)
want := realExp.Add(-auth.ServiceTokenSafetyMargin)
if validUntil.After(want) {
t.Errorf("entry valid until %v, want no later than %v (exp minus %v)",
validUntil, want, auth.ServiceTokenSafetyMargin)
}
if flat := time.Now().Add(validationCacheTTL); !validUntil.Before(flat) {
t.Errorf("entry valid until %v, which is the flat %v TTL: the token's own exp was ignored",
validUntil, validationCacheTTL)
}
}
// A long-lived token is still capped by the flat TTL, so the cache keeps
// rechecking the underlying session rather than holding one token for minutes.
func TestValidationCache_LongLivedTokenKeepsFlatTTL(t *testing.T) {
vc := newValidationCache()
cacheKey := "did:plc:puller:did:web:hold.example.com"
token := serviceTokenExpiring(time.Now().Add(1 * time.Hour))
if _, err := vc.getOrFetch(context.Background(), cacheKey, func() (string, error) {
return token, nil
}); err != nil {
t.Fatalf("getOrFetch() error = %v", err)
}
validUntil := entryValidUntil(t, vc, cacheKey)
want := time.Now().Add(validationCacheTTL)
if diff := validUntil.Sub(want); diff < -2*time.Second || diff > 2*time.Second {
t.Errorf("entry valid until %v, want ~%v (the flat TTL)", validUntil, want)
}
}
// A token whose exp cannot be read falls back to the behaviour that shipped
// before: the flat TTL. The appview can't do better for a shape it doesn't
// understand, and refusing to cache would stampede the PDS.
func TestValidationCache_UnparsableTokenKeepsFlatTTL(t *testing.T) {
vc := newValidationCache()
cacheKey := "did:plc:puller:did:web:hold.example.com"
if _, err := vc.getOrFetch(context.Background(), cacheKey, func() (string, error) {
return "not-a-jwt", nil
}); err != nil {
t.Fatalf("getOrFetch() error = %v", err)
}
validUntil := entryValidUntil(t, vc, cacheKey)
want := time.Now().Add(validationCacheTTL)
if diff := validUntil.Sub(want); diff < -2*time.Second || diff > 2*time.Second {
t.Errorf("entry valid until %v, want ~%v (the flat TTL)", validUntil, want)
}
}
// invalidate must force the next call to re-mint, which is what makes the 401
// we send to Docker worth retrying.
func TestValidationCache_InvalidateForcesRefetch(t *testing.T) {
vc := newValidationCache()
cacheKey := "did:plc:puller:did:web:hold.example.com"
fetches := 0
fetch := func() (string, error) {
fetches++
return serviceTokenExpiring(time.Now().Add(5 * time.Minute)), nil
}
if _, err := vc.getOrFetch(context.Background(), cacheKey, fetch); err != nil {
t.Fatalf("getOrFetch() error = %v", err)
}
if _, err := vc.getOrFetch(context.Background(), cacheKey, fetch); err != nil {
t.Fatalf("getOrFetch() error = %v", err)
}
if fetches != 1 {
t.Fatalf("second call fetched again (%d fetches), expected a cache hit", fetches)
}
vc.invalidate(cacheKey)
if _, err := vc.getOrFetch(context.Background(), cacheKey, fetch); err != nil {
t.Fatalf("getOrFetch() after invalidate error = %v", err)
}
if fetches != 2 {
t.Errorf("got %d fetches after invalidate, want 2", fetches)
}
}
// invalidate on a key that was never cached is a no-op, not a panic: the blob
// store calls it on whatever request hit the rejection.
func TestValidationCache_InvalidateUnknownKey(t *testing.T) {
vc := newValidationCache()
vc.invalidate("did:plc:nobody:did:web:hold.example.com")
}