mirror of
https://tangled.org/evan.jarrett.net/at-container-registry
synced 2026-09-23 10:44:16 +00:00
Seen in production on 2026-09-11: three cold pulls of a 22-layer image failed with BLOB_UNKNOWN for layers that exist. The hold had answered 403 "service token authentication failed: token has expired", and the same blobs served fine a minute later. Three things lined up. The registry middleware's validation cache kept a fetched service token for a flat 45 seconds regardless of its real remaining life, so a token fetched with 12 seconds left was still handed to the hold half a minute after it died. The registry JWT is stamped from the auth cache's expiry, which trailed the real exp by only 10 seconds, while distribution accepts a JWT for 60 seconds past its exp, so a client could hold an accepted JWT for most of a minute after the credential behind it was gone. And the hold's 403 was flattened to BLOB_UNKNOWN, so the client failed instead of re-authenticating. Now the validation cache bounds an entry by the token's exp minus a shared ServiceTokenSafetyMargin of 60 seconds, the same margin the auth cache and the JWT stamp use, chosen to equal distribution's leeway so the last instant a JWT is accepted is the service token's real exp. A PDS that grants less than the margin gets half its remaining life instead of an already-past deadline. When the hold rejects the service token as expired or missing, the appview drops both cached copies and returns a 401 challenge so Docker and crane re-run the token dance and retry; a genuine permission denial stays a 403, and a hold that is down still maps to blob unknown. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EvFJr4Dwz8p2NDAeXmgmBt
227 lines
8.4 KiB
Go
227 lines
8.4 KiB
Go
package storage
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
|
|
"github.com/distribution/distribution/v3"
|
|
"github.com/distribution/distribution/v3/registry/api/errcode"
|
|
"github.com/opencontainers/go-digest"
|
|
)
|
|
|
|
// The two bodies below are copied from what the hold actually writes. Its OCI
|
|
// blob handler answers every read authorization failure with
|
|
// http.Error(w, "authorization failed: "+err.Error(), http.StatusForbidden),
|
|
// and the error underneath is either the service-token wrapper from
|
|
// ValidateBlobReadAccess or an *AuthError. Same status, opposite meaning.
|
|
const (
|
|
holdExpiredTokenBody = "authorization failed: service token authentication failed: token has expired"
|
|
holdPermissionBody = "authorization failed: access denied for blob:read: crew member lacks permission (required: blob:read or blob:write)"
|
|
)
|
|
|
|
const authTestDigest = digest.Digest("sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd")
|
|
|
|
// statWithHoldFailure points a blob store at a hold that rejects the presign
|
|
// with the given status and body, and returns Stat's error plus whether the
|
|
// service token was invalidated.
|
|
func statWithHoldFailure(t *testing.T, status int, body string) (error, bool) {
|
|
t.Helper()
|
|
|
|
holdServer := newMockHoldServer(t, "http://s3.invalid")
|
|
defer holdServer.Close()
|
|
|
|
holdServer.mu.Lock()
|
|
holdServer.PresignAuthFailure = &mockAuthFailure{Status: status, Body: body}
|
|
holdServer.mu.Unlock()
|
|
|
|
invalidated := false
|
|
store := createTestProxyBlobStore(t, holdServer.URL)
|
|
store.ctx.PullerDID = "did:plc:puller"
|
|
store.ctx.InvalidateServiceToken = func() { invalidated = true }
|
|
|
|
_, err := store.Stat(context.Background(), authTestDigest)
|
|
return err, invalidated
|
|
}
|
|
|
|
// The production failure: the hold rejects our service token as expired, which
|
|
// used to surface as BLOB_UNKNOWN (or a bare 403) and fail the pull outright.
|
|
// It has to become a 401 so BearerChallenge attaches WWW-Authenticate and the
|
|
// client re-runs the token dance.
|
|
func TestStat_HoldRejectsExpiredServiceToken_ReturnsUnauthorized(t *testing.T) {
|
|
err, invalidated := statWithHoldFailure(t, http.StatusForbidden, holdExpiredTokenBody)
|
|
|
|
var ecErr errcode.Error
|
|
if !errors.As(err, &ecErr) {
|
|
t.Fatalf("Stat() error = %v (%T), want an errcode.Error", err, err)
|
|
}
|
|
if ecErr.Code != errcode.ErrorCodeUnauthorized {
|
|
t.Errorf("Stat() code = %v, want %v", ecErr.Code, errcode.ErrorCodeUnauthorized)
|
|
}
|
|
if !invalidated {
|
|
t.Error("the stale service token was not invalidated: the client's retry would be handed the same dead token")
|
|
}
|
|
}
|
|
|
|
// A crew member without read access authenticated fine. Sending them back to
|
|
// re-authenticate would loop forever, so this stays a 403.
|
|
func TestStat_HoldDeniesPermission_StaysDenied(t *testing.T) {
|
|
err, invalidated := statWithHoldFailure(t, http.StatusForbidden, holdPermissionBody)
|
|
|
|
var ecErr errcode.Error
|
|
if !errors.As(err, &ecErr) {
|
|
t.Fatalf("Stat() error = %v (%T), want an errcode.Error", err, err)
|
|
}
|
|
if ecErr.Code != errcode.ErrorCodeDenied {
|
|
t.Errorf("Stat() code = %v, want %v", ecErr.Code, errcode.ErrorCodeDenied)
|
|
}
|
|
if invalidated {
|
|
t.Error("a permission denial must not throw away a perfectly good service token")
|
|
}
|
|
}
|
|
|
|
// A bare 401 from the hold is a challenge by definition, whatever its body.
|
|
func TestStat_HoldAnswers401_ReturnsUnauthorized(t *testing.T) {
|
|
err, invalidated := statWithHoldFailure(t, http.StatusUnauthorized, "unauthorized")
|
|
|
|
var ecErr errcode.Error
|
|
if !errors.As(err, &ecErr) {
|
|
t.Fatalf("Stat() error = %v (%T), want an errcode.Error", err, err)
|
|
}
|
|
if ecErr.Code != errcode.ErrorCodeUnauthorized {
|
|
t.Errorf("Stat() code = %v, want %v", ecErr.Code, errcode.ErrorCodeUnauthorized)
|
|
}
|
|
if !invalidated {
|
|
t.Error("a 401 from the hold should drop the credential it rejected")
|
|
}
|
|
}
|
|
|
|
// An anonymous pull that the hold refuses keeps its existing 401 "authenticate
|
|
// first" path, and must not touch a service token it never had.
|
|
func TestStat_AnonymousDenial_StillAsksForCredentials(t *testing.T) {
|
|
holdServer := newMockHoldServer(t, "http://s3.invalid")
|
|
defer holdServer.Close()
|
|
|
|
holdServer.mu.Lock()
|
|
holdServer.PresignAuthFailure = &mockAuthFailure{
|
|
Status: http.StatusForbidden,
|
|
Body: holdPermissionBody,
|
|
}
|
|
holdServer.mu.Unlock()
|
|
|
|
store := createTestProxyBlobStore(t, holdServer.URL)
|
|
store.ctx.Anonymous = true
|
|
store.ctx.ServiceToken = ""
|
|
|
|
_, err := store.Stat(context.Background(), authTestDigest)
|
|
|
|
var ecErr errcode.Error
|
|
if !errors.As(err, &ecErr) {
|
|
t.Fatalf("Stat() error = %v (%T), want an errcode.Error", err, err)
|
|
}
|
|
if ecErr.Code != errcode.ErrorCodeUnauthorized {
|
|
t.Errorf("Stat() code = %v, want %v", ecErr.Code, errcode.ErrorCodeUnauthorized)
|
|
}
|
|
if ecErr.Message != "authentication required" {
|
|
t.Errorf("Stat() message = %q, want the anonymous challenge to be unchanged", ecErr.Message)
|
|
}
|
|
}
|
|
|
|
// A hold that is broken, rather than one that refused us, must keep answering
|
|
// the way it did before: blob-unknown, not a re-auth challenge. A client that
|
|
// re-authenticated over a 5xx would just burn tokens against a sick hold.
|
|
func TestStat_HoldServerError_StillBlobUnknown(t *testing.T) {
|
|
holdServer := newMockHoldServer(t, "http://s3.invalid")
|
|
defer holdServer.Close()
|
|
|
|
holdServer.mu.Lock()
|
|
holdServer.PresignError = errors.New("internal error")
|
|
holdServer.mu.Unlock()
|
|
|
|
store := createTestProxyBlobStore(t, holdServer.URL)
|
|
store.ctx.InvalidateServiceToken = func() {
|
|
t.Error("a 5xx must not invalidate the service token")
|
|
}
|
|
|
|
_, err := store.Stat(context.Background(), authTestDigest)
|
|
if !errors.Is(err, distribution.ErrBlobUnknown) {
|
|
t.Errorf("Stat() error = %v, want %v", err, distribution.ErrBlobUnknown)
|
|
}
|
|
}
|
|
|
|
// Same for a hold that is simply unreachable.
|
|
func TestStat_HoldUnreachable_StillBlobUnknown(t *testing.T) {
|
|
holdServer := newMockHoldServer(t, "http://s3.invalid")
|
|
holdURL := holdServer.URL
|
|
holdServer.Close()
|
|
|
|
store := createTestProxyBlobStore(t, holdURL)
|
|
store.ctx.InvalidateServiceToken = func() {
|
|
t.Error("a connection failure must not invalidate the service token")
|
|
}
|
|
|
|
_, err := store.Stat(context.Background(), authTestDigest)
|
|
if !errors.Is(err, distribution.ErrBlobUnknown) {
|
|
t.Errorf("Stat() error = %v, want %v", err, distribution.ErrBlobUnknown)
|
|
}
|
|
}
|
|
|
|
// ServeBlob runs the same presign path, so the challenge has to reach the
|
|
// client from there too: on a GET, distribution calls Stat and then ServeBlob.
|
|
func TestServeBlob_HoldRejectsExpiredServiceToken_ReturnsUnauthorized(t *testing.T) {
|
|
holdServer := newMockHoldServer(t, "http://s3.invalid")
|
|
defer holdServer.Close()
|
|
|
|
holdServer.mu.Lock()
|
|
holdServer.PresignAuthFailure = &mockAuthFailure{
|
|
Status: http.StatusForbidden,
|
|
Body: holdExpiredTokenBody,
|
|
}
|
|
holdServer.mu.Unlock()
|
|
|
|
invalidated := false
|
|
store := createTestProxyBlobStore(t, holdServer.URL)
|
|
store.ctx.InvalidateServiceToken = func() { invalidated = true }
|
|
|
|
req := httptest.NewRequest(http.MethodGet, "/v2/alice/img/blobs/"+authTestDigest.String(), nil)
|
|
err := store.ServeBlob(context.Background(), httptest.NewRecorder(), req, authTestDigest)
|
|
|
|
var ecErr errcode.Error
|
|
if !errors.As(err, &ecErr) {
|
|
t.Fatalf("ServeBlob() error = %v (%T), want an errcode.Error", err, err)
|
|
}
|
|
if ecErr.Code != errcode.ErrorCodeUnauthorized {
|
|
t.Errorf("ServeBlob() code = %v, want %v", ecErr.Code, errcode.ErrorCodeUnauthorized)
|
|
}
|
|
if !invalidated {
|
|
t.Error("the stale service token was not invalidated")
|
|
}
|
|
}
|
|
|
|
func TestIsHoldAuthenticationFailure(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
status int
|
|
body string
|
|
want bool
|
|
}{
|
|
{"expired service token", http.StatusForbidden, holdExpiredTokenBody, true},
|
|
{"dpop failure", http.StatusForbidden, "authorization failed: DPoP authentication failed: bad proof", true},
|
|
{"no auth scheme", http.StatusForbidden, "authorization failed: invalid authorization scheme: expected 'Bearer' or 'DPoP'", true},
|
|
{"any 401", http.StatusUnauthorized, "", true},
|
|
{"crew member lacks permission", http.StatusForbidden, holdPermissionBody, false},
|
|
{"not a crew member", http.StatusForbidden, "authorization failed: access denied for blob:read: user is not a crew member (required: blob:read or blob:write)", false},
|
|
{"unrecognised 403", http.StatusForbidden, "authorization failed: failed to get captain record: boom", false},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
if got := isHoldAuthenticationFailure(tt.status, []byte(tt.body)); got != tt.want {
|
|
t.Errorf("isHoldAuthenticationFailure(%d, %q) = %v, want %v", tt.status, tt.body, got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|