mirror of
https://tangled.org/evan.jarrett.net/at-container-registry
synced 2026-09-23 02:34:17 +00:00
Seen in production on 2026-09-11: three cold pulls of a 22-layer image failed with BLOB_UNKNOWN for layers that exist. The hold had answered 403 "service token authentication failed: token has expired", and the same blobs served fine a minute later. Three things lined up. The registry middleware's validation cache kept a fetched service token for a flat 45 seconds regardless of its real remaining life, so a token fetched with 12 seconds left was still handed to the hold half a minute after it died. The registry JWT is stamped from the auth cache's expiry, which trailed the real exp by only 10 seconds, while distribution accepts a JWT for 60 seconds past its exp, so a client could hold an accepted JWT for most of a minute after the credential behind it was gone. And the hold's 403 was flattened to BLOB_UNKNOWN, so the client failed instead of re-authenticating. Now the validation cache bounds an entry by the token's exp minus a shared ServiceTokenSafetyMargin of 60 seconds, the same margin the auth cache and the JWT stamp use, chosen to equal distribution's leeway so the last instant a JWT is accepted is the service token's real exp. A PDS that grants less than the margin gets half its remaining life instead of an already-past deadline. When the hold rejects the service token as expired or missing, the appview drops both cached copies and returns a 401 challenge so Docker and crane re-run the token dance and retry; a genuine permission denial stays a 403, and a hold that is down still maps to blob unknown. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EvFJr4Dwz8p2NDAeXmgmBt
152 lines
4.8 KiB
Go
152 lines
4.8 KiB
Go
package middleware
|
|
|
|
import (
|
|
"context"
|
|
"encoding/base64"
|
|
"fmt"
|
|
"testing"
|
|
"time"
|
|
|
|
"atcr.io/pkg/auth"
|
|
)
|
|
|
|
// serviceTokenExpiring builds an unsigned JWT whose exp claim is the given
|
|
// time. The validation cache only reads exp; the hold is what verifies
|
|
// signatures.
|
|
func serviceTokenExpiring(at time.Time) string {
|
|
payload := fmt.Sprintf(`{"exp":%d}`, at.Unix())
|
|
return "header." + base64.RawURLEncoding.EncodeToString([]byte(payload)) + ".signature"
|
|
}
|
|
|
|
// entryValidUntil reads back the cached entry's deadline.
|
|
func entryValidUntil(t *testing.T, vc *validationCache, cacheKey string) time.Time {
|
|
t.Helper()
|
|
|
|
vc.mu.RLock()
|
|
entry, ok := vc.entries[cacheKey]
|
|
vc.mu.RUnlock()
|
|
if !ok {
|
|
t.Fatalf("no validation cache entry for %q", cacheKey)
|
|
}
|
|
|
|
entry.mu.Lock()
|
|
defer entry.mu.Unlock()
|
|
return entry.validUntil
|
|
}
|
|
|
|
// A token with only 20s of life must not be pinned for the flat 45s TTL: that
|
|
// is the production bug, where the appview kept presenting a dead service token
|
|
// and the hold answered 403 "token has expired" on blobs that exist.
|
|
func TestValidationCache_BoundsEntryByTokenExpiry(t *testing.T) {
|
|
vc := newValidationCache()
|
|
cacheKey := "did:plc:puller:did:web:hold.example.com"
|
|
|
|
realExp := time.Now().Add(20 * time.Second)
|
|
token := serviceTokenExpiring(realExp)
|
|
|
|
got, err := vc.getOrFetch(context.Background(), cacheKey, func() (string, error) {
|
|
return token, nil
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("getOrFetch() error = %v", err)
|
|
}
|
|
if got != token {
|
|
t.Fatalf("getOrFetch() returned %q, want the fetched token", got)
|
|
}
|
|
|
|
validUntil := entryValidUntil(t, vc, cacheKey)
|
|
|
|
want := realExp.Add(-auth.ServiceTokenSafetyMargin)
|
|
if validUntil.After(want) {
|
|
t.Errorf("entry valid until %v, want no later than %v (exp minus %v)",
|
|
validUntil, want, auth.ServiceTokenSafetyMargin)
|
|
}
|
|
|
|
if flat := time.Now().Add(validationCacheTTL); !validUntil.Before(flat) {
|
|
t.Errorf("entry valid until %v, which is the flat %v TTL: the token's own exp was ignored",
|
|
validUntil, validationCacheTTL)
|
|
}
|
|
}
|
|
|
|
// A long-lived token is still capped by the flat TTL, so the cache keeps
|
|
// rechecking the underlying session rather than holding one token for minutes.
|
|
func TestValidationCache_LongLivedTokenKeepsFlatTTL(t *testing.T) {
|
|
vc := newValidationCache()
|
|
cacheKey := "did:plc:puller:did:web:hold.example.com"
|
|
|
|
token := serviceTokenExpiring(time.Now().Add(1 * time.Hour))
|
|
|
|
if _, err := vc.getOrFetch(context.Background(), cacheKey, func() (string, error) {
|
|
return token, nil
|
|
}); err != nil {
|
|
t.Fatalf("getOrFetch() error = %v", err)
|
|
}
|
|
|
|
validUntil := entryValidUntil(t, vc, cacheKey)
|
|
|
|
want := time.Now().Add(validationCacheTTL)
|
|
if diff := validUntil.Sub(want); diff < -2*time.Second || diff > 2*time.Second {
|
|
t.Errorf("entry valid until %v, want ~%v (the flat TTL)", validUntil, want)
|
|
}
|
|
}
|
|
|
|
// A token whose exp cannot be read falls back to the behaviour that shipped
|
|
// before: the flat TTL. The appview can't do better for a shape it doesn't
|
|
// understand, and refusing to cache would stampede the PDS.
|
|
func TestValidationCache_UnparsableTokenKeepsFlatTTL(t *testing.T) {
|
|
vc := newValidationCache()
|
|
cacheKey := "did:plc:puller:did:web:hold.example.com"
|
|
|
|
if _, err := vc.getOrFetch(context.Background(), cacheKey, func() (string, error) {
|
|
return "not-a-jwt", nil
|
|
}); err != nil {
|
|
t.Fatalf("getOrFetch() error = %v", err)
|
|
}
|
|
|
|
validUntil := entryValidUntil(t, vc, cacheKey)
|
|
|
|
want := time.Now().Add(validationCacheTTL)
|
|
if diff := validUntil.Sub(want); diff < -2*time.Second || diff > 2*time.Second {
|
|
t.Errorf("entry valid until %v, want ~%v (the flat TTL)", validUntil, want)
|
|
}
|
|
}
|
|
|
|
// invalidate must force the next call to re-mint, which is what makes the 401
|
|
// we send to Docker worth retrying.
|
|
func TestValidationCache_InvalidateForcesRefetch(t *testing.T) {
|
|
vc := newValidationCache()
|
|
cacheKey := "did:plc:puller:did:web:hold.example.com"
|
|
|
|
fetches := 0
|
|
fetch := func() (string, error) {
|
|
fetches++
|
|
return serviceTokenExpiring(time.Now().Add(5 * time.Minute)), nil
|
|
}
|
|
|
|
if _, err := vc.getOrFetch(context.Background(), cacheKey, fetch); err != nil {
|
|
t.Fatalf("getOrFetch() error = %v", err)
|
|
}
|
|
if _, err := vc.getOrFetch(context.Background(), cacheKey, fetch); err != nil {
|
|
t.Fatalf("getOrFetch() error = %v", err)
|
|
}
|
|
if fetches != 1 {
|
|
t.Fatalf("second call fetched again (%d fetches), expected a cache hit", fetches)
|
|
}
|
|
|
|
vc.invalidate(cacheKey)
|
|
|
|
if _, err := vc.getOrFetch(context.Background(), cacheKey, fetch); err != nil {
|
|
t.Fatalf("getOrFetch() after invalidate error = %v", err)
|
|
}
|
|
if fetches != 2 {
|
|
t.Errorf("got %d fetches after invalidate, want 2", fetches)
|
|
}
|
|
}
|
|
|
|
// invalidate on a key that was never cached is a no-op, not a panic: the blob
|
|
// store calls it on whatever request hit the rejection.
|
|
func TestValidationCache_InvalidateUnknownKey(t *testing.T) {
|
|
vc := newValidationCache()
|
|
vc.invalidate("did:plc:nobody:did:web:hold.example.com")
|
|
}
|