mirror of
https://tangled.org/evan.jarrett.net/at-container-registry
synced 2026-09-23 10:44:16 +00:00
Evan JarrettandClaude Fable 5.1
9dbc53b670
Seen in production on 2026-09-11: three cold pulls of a 22-layer image failed with BLOB_UNKNOWN for layers that exist. The hold had answered 403 "service token authentication failed: token has expired", and the same blobs served fine a minute later. Three things lined up. The registry middleware's validation cache kept a fetched service token for a flat 45 seconds regardless of its real remaining life, so a token fetched with 12 seconds left was still handed to the hold half a minute after it died. The registry JWT is stamped from the auth cache's expiry, which trailed the real exp by only 10 seconds, while distribution accepts a JWT for 60 seconds past its exp, so a client could hold an accepted JWT for most of a minute after the credential behind it was gone. And the hold's 403 was flattened to BLOB_UNKNOWN, so the client failed instead of re-authenticating. Now the validation cache bounds an entry by the token's exp minus a shared ServiceTokenSafetyMargin of 60 seconds, the same margin the auth cache and the JWT stamp use, chosen to equal distribution's leeway so the last instant a JWT is accepted is the service token's real exp. A PDS that grants less than the margin gets half its remaining life instead of an already-past deadline. When the hold rejects the service token as expired or missing, the appview drops both cached copies and returns a 401 challenge so Docker and crane re-run the token dance and retry; a genuine permission denial stays a 403, and a hold that is down still maps to blob unknown. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EvFJr4Dwz8p2NDAeXmgmBt
82 lines
3.0 KiB
Go
82 lines
3.0 KiB
Go
package token
|
|
|
|
import (
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
"time"
|
|
|
|
disttoken "github.com/distribution/distribution/v3/registry/auth/token"
|
|
|
|
"atcr.io/pkg/auth"
|
|
)
|
|
|
|
// The margin exists to cover distribution's JWT leeway. distribution's registry
|
|
// auth accepts a token for Leeway past its exp, and the JWT's exp is stamped
|
|
// from (service token exp - margin). If the margin ever drops below the leeway,
|
|
// a client can hold an accepted JWT after the service token behind it is dead,
|
|
// the hold answers 403 "token has expired", and the pull fails with no 401 to
|
|
// make the client re-authenticate. That is the production bug this closes.
|
|
func TestServiceTokenSafetyMarginCoversDistributionLeeway(t *testing.T) {
|
|
if auth.ServiceTokenSafetyMargin < disttoken.Leeway {
|
|
t.Fatalf("auth.ServiceTokenSafetyMargin is %v, which is under distribution's token.Leeway of %v: "+
|
|
"a registry JWT would stay acceptable after its service token expired",
|
|
auth.ServiceTokenSafetyMargin, disttoken.Leeway)
|
|
}
|
|
}
|
|
|
|
// The handler must stamp exactly the expiry the fetcher hands it. That value
|
|
// already has the margin subtracted (pkg/auth/cache.go), so stamping anything
|
|
// later would reopen the window, and the margin has to survive the round trip
|
|
// through the token response.
|
|
func TestHandler_ServiceAuthFetcher_MarginSurvivesStamping(t *testing.T) {
|
|
keyPath := getSharedTestKey(t)
|
|
issuer, err := NewIssuer(keyPath, "atcr.io", "registry", 15*time.Minute)
|
|
if err != nil {
|
|
t.Fatalf("NewIssuer() error = %v", err)
|
|
}
|
|
|
|
deviceStore, database := setupTestDeviceStore(t)
|
|
deviceSecret := createTestDevice(t, deviceStore, database, "did:plc:alice123", "alice.bsky.social")
|
|
|
|
handler := NewHandler(issuer, deviceStore)
|
|
|
|
// What a 5 minute PDS grant looks like coming out of the cache.
|
|
serviceTokenExp := time.Now().Add(5 * time.Minute)
|
|
handler.SetServiceAuthFetcher(&stubServiceAuthFetcher{
|
|
expiresAt: serviceTokenExp.Add(-auth.ServiceTokenSafetyMargin),
|
|
})
|
|
|
|
req := httptest.NewRequest(http.MethodGet, "/auth/token?service=registry&scope=repository:alice.bsky.social/myapp:pull,push", nil)
|
|
req.SetBasicAuth("alice", deviceSecret)
|
|
w := httptest.NewRecorder()
|
|
|
|
handler.ServeHTTP(w, req)
|
|
|
|
if w.Code != http.StatusOK {
|
|
t.Fatalf("expected 200, got %d. Body: %s", w.Code, w.Body.String())
|
|
}
|
|
|
|
var resp TokenResponse
|
|
if err := json.NewDecoder(w.Body).Decode(&resp); err != nil {
|
|
t.Fatalf("decode response: %v", err)
|
|
}
|
|
|
|
jwtExpiresAt := time.Now().Add(time.Duration(resp.ExpiresIn) * time.Second)
|
|
|
|
// The last instant distribution will accept this JWT must still be inside
|
|
// the service token's real life.
|
|
lastAccepted := jwtExpiresAt.Add(disttoken.Leeway)
|
|
if lastAccepted.After(serviceTokenExp.Add(2 * time.Second)) {
|
|
t.Errorf("JWT is accepted until %v but the service token dies at %v",
|
|
lastAccepted, serviceTokenExp)
|
|
}
|
|
|
|
// And it must still be a usable token, not one squeezed to nothing.
|
|
if resp.ExpiresIn < 200 {
|
|
t.Errorf("expires_in = %d, want ~240s (5 min grant minus the %v margin)",
|
|
resp.ExpiresIn, auth.ServiceTokenSafetyMargin)
|
|
}
|
|
}
|