Files
at-container-registry/pkg/appview/storage/proxy_blob_auth_test.go
T
Evan JarrettandClaude Fable 5.1 9dbc53b670 appview: stop serving service tokens past their expiry, and challenge the client when the hold rejects one
Seen in production on 2026-09-11: three cold pulls of a 22-layer image failed
with BLOB_UNKNOWN for layers that exist. The hold had answered 403 "service
token authentication failed: token has expired", and the same blobs served
fine a minute later.

Three things lined up. The registry middleware's validation cache kept a
fetched service token for a flat 45 seconds regardless of its real remaining
life, so a token fetched with 12 seconds left was still handed to the hold
half a minute after it died. The registry JWT is stamped from the auth cache's
expiry, which trailed the real exp by only 10 seconds, while distribution
accepts a JWT for 60 seconds past its exp, so a client could hold an accepted
JWT for most of a minute after the credential behind it was gone. And the
hold's 403 was flattened to BLOB_UNKNOWN, so the client failed instead of
re-authenticating.

Now the validation cache bounds an entry by the token's exp minus a shared
ServiceTokenSafetyMargin of 60 seconds, the same margin the auth cache and the
JWT stamp use, chosen to equal distribution's leeway so the last instant a JWT
is accepted is the service token's real exp. A PDS that grants less than the
margin gets half its remaining life instead of an already-past deadline. When
the hold rejects the service token as expired or missing, the appview drops
both cached copies and returns a 401 challenge so Docker and crane re-run the
token dance and retry; a genuine permission denial stays a 403, and a hold
that is down still maps to blob unknown.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EvFJr4Dwz8p2NDAeXmgmBt
2026-09-11 19:26:51 -05:00

227 lines
8.4 KiB
Go

package storage
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"testing"
"github.com/distribution/distribution/v3"
"github.com/distribution/distribution/v3/registry/api/errcode"
"github.com/opencontainers/go-digest"
)
// The two bodies below are copied from what the hold actually writes. Its OCI
// blob handler answers every read authorization failure with
// http.Error(w, "authorization failed: "+err.Error(), http.StatusForbidden),
// and the error underneath is either the service-token wrapper from
// ValidateBlobReadAccess or an *AuthError. Same status, opposite meaning.
const (
holdExpiredTokenBody = "authorization failed: service token authentication failed: token has expired"
holdPermissionBody = "authorization failed: access denied for blob:read: crew member lacks permission (required: blob:read or blob:write)"
)
const authTestDigest = digest.Digest("sha256:abc123def456abc123def456abc123def456abc123def456abc123def456abcd")
// statWithHoldFailure points a blob store at a hold that rejects the presign
// with the given status and body, and returns Stat's error plus whether the
// service token was invalidated.
func statWithHoldFailure(t *testing.T, status int, body string) (error, bool) {
t.Helper()
holdServer := newMockHoldServer(t, "http://s3.invalid")
defer holdServer.Close()
holdServer.mu.Lock()
holdServer.PresignAuthFailure = &mockAuthFailure{Status: status, Body: body}
holdServer.mu.Unlock()
invalidated := false
store := createTestProxyBlobStore(t, holdServer.URL)
store.ctx.PullerDID = "did:plc:puller"
store.ctx.InvalidateServiceToken = func() { invalidated = true }
_, err := store.Stat(context.Background(), authTestDigest)
return err, invalidated
}
// The production failure: the hold rejects our service token as expired, which
// used to surface as BLOB_UNKNOWN (or a bare 403) and fail the pull outright.
// It has to become a 401 so BearerChallenge attaches WWW-Authenticate and the
// client re-runs the token dance.
func TestStat_HoldRejectsExpiredServiceToken_ReturnsUnauthorized(t *testing.T) {
err, invalidated := statWithHoldFailure(t, http.StatusForbidden, holdExpiredTokenBody)
var ecErr errcode.Error
if !errors.As(err, &ecErr) {
t.Fatalf("Stat() error = %v (%T), want an errcode.Error", err, err)
}
if ecErr.Code != errcode.ErrorCodeUnauthorized {
t.Errorf("Stat() code = %v, want %v", ecErr.Code, errcode.ErrorCodeUnauthorized)
}
if !invalidated {
t.Error("the stale service token was not invalidated: the client's retry would be handed the same dead token")
}
}
// A crew member without read access authenticated fine. Sending them back to
// re-authenticate would loop forever, so this stays a 403.
func TestStat_HoldDeniesPermission_StaysDenied(t *testing.T) {
err, invalidated := statWithHoldFailure(t, http.StatusForbidden, holdPermissionBody)
var ecErr errcode.Error
if !errors.As(err, &ecErr) {
t.Fatalf("Stat() error = %v (%T), want an errcode.Error", err, err)
}
if ecErr.Code != errcode.ErrorCodeDenied {
t.Errorf("Stat() code = %v, want %v", ecErr.Code, errcode.ErrorCodeDenied)
}
if invalidated {
t.Error("a permission denial must not throw away a perfectly good service token")
}
}
// A bare 401 from the hold is a challenge by definition, whatever its body.
func TestStat_HoldAnswers401_ReturnsUnauthorized(t *testing.T) {
err, invalidated := statWithHoldFailure(t, http.StatusUnauthorized, "unauthorized")
var ecErr errcode.Error
if !errors.As(err, &ecErr) {
t.Fatalf("Stat() error = %v (%T), want an errcode.Error", err, err)
}
if ecErr.Code != errcode.ErrorCodeUnauthorized {
t.Errorf("Stat() code = %v, want %v", ecErr.Code, errcode.ErrorCodeUnauthorized)
}
if !invalidated {
t.Error("a 401 from the hold should drop the credential it rejected")
}
}
// An anonymous pull that the hold refuses keeps its existing 401 "authenticate
// first" path, and must not touch a service token it never had.
func TestStat_AnonymousDenial_StillAsksForCredentials(t *testing.T) {
holdServer := newMockHoldServer(t, "http://s3.invalid")
defer holdServer.Close()
holdServer.mu.Lock()
holdServer.PresignAuthFailure = &mockAuthFailure{
Status: http.StatusForbidden,
Body: holdPermissionBody,
}
holdServer.mu.Unlock()
store := createTestProxyBlobStore(t, holdServer.URL)
store.ctx.Anonymous = true
store.ctx.ServiceToken = ""
_, err := store.Stat(context.Background(), authTestDigest)
var ecErr errcode.Error
if !errors.As(err, &ecErr) {
t.Fatalf("Stat() error = %v (%T), want an errcode.Error", err, err)
}
if ecErr.Code != errcode.ErrorCodeUnauthorized {
t.Errorf("Stat() code = %v, want %v", ecErr.Code, errcode.ErrorCodeUnauthorized)
}
if ecErr.Message != "authentication required" {
t.Errorf("Stat() message = %q, want the anonymous challenge to be unchanged", ecErr.Message)
}
}
// A hold that is broken, rather than one that refused us, must keep answering
// the way it did before: blob-unknown, not a re-auth challenge. A client that
// re-authenticated over a 5xx would just burn tokens against a sick hold.
func TestStat_HoldServerError_StillBlobUnknown(t *testing.T) {
holdServer := newMockHoldServer(t, "http://s3.invalid")
defer holdServer.Close()
holdServer.mu.Lock()
holdServer.PresignError = errors.New("internal error")
holdServer.mu.Unlock()
store := createTestProxyBlobStore(t, holdServer.URL)
store.ctx.InvalidateServiceToken = func() {
t.Error("a 5xx must not invalidate the service token")
}
_, err := store.Stat(context.Background(), authTestDigest)
if !errors.Is(err, distribution.ErrBlobUnknown) {
t.Errorf("Stat() error = %v, want %v", err, distribution.ErrBlobUnknown)
}
}
// Same for a hold that is simply unreachable.
func TestStat_HoldUnreachable_StillBlobUnknown(t *testing.T) {
holdServer := newMockHoldServer(t, "http://s3.invalid")
holdURL := holdServer.URL
holdServer.Close()
store := createTestProxyBlobStore(t, holdURL)
store.ctx.InvalidateServiceToken = func() {
t.Error("a connection failure must not invalidate the service token")
}
_, err := store.Stat(context.Background(), authTestDigest)
if !errors.Is(err, distribution.ErrBlobUnknown) {
t.Errorf("Stat() error = %v, want %v", err, distribution.ErrBlobUnknown)
}
}
// ServeBlob runs the same presign path, so the challenge has to reach the
// client from there too: on a GET, distribution calls Stat and then ServeBlob.
func TestServeBlob_HoldRejectsExpiredServiceToken_ReturnsUnauthorized(t *testing.T) {
holdServer := newMockHoldServer(t, "http://s3.invalid")
defer holdServer.Close()
holdServer.mu.Lock()
holdServer.PresignAuthFailure = &mockAuthFailure{
Status: http.StatusForbidden,
Body: holdExpiredTokenBody,
}
holdServer.mu.Unlock()
invalidated := false
store := createTestProxyBlobStore(t, holdServer.URL)
store.ctx.InvalidateServiceToken = func() { invalidated = true }
req := httptest.NewRequest(http.MethodGet, "/v2/alice/img/blobs/"+authTestDigest.String(), nil)
err := store.ServeBlob(context.Background(), httptest.NewRecorder(), req, authTestDigest)
var ecErr errcode.Error
if !errors.As(err, &ecErr) {
t.Fatalf("ServeBlob() error = %v (%T), want an errcode.Error", err, err)
}
if ecErr.Code != errcode.ErrorCodeUnauthorized {
t.Errorf("ServeBlob() code = %v, want %v", ecErr.Code, errcode.ErrorCodeUnauthorized)
}
if !invalidated {
t.Error("the stale service token was not invalidated")
}
}
func TestIsHoldAuthenticationFailure(t *testing.T) {
tests := []struct {
name string
status int
body string
want bool
}{
{"expired service token", http.StatusForbidden, holdExpiredTokenBody, true},
{"dpop failure", http.StatusForbidden, "authorization failed: DPoP authentication failed: bad proof", true},
{"no auth scheme", http.StatusForbidden, "authorization failed: invalid authorization scheme: expected 'Bearer' or 'DPoP'", true},
{"any 401", http.StatusUnauthorized, "", true},
{"crew member lacks permission", http.StatusForbidden, holdPermissionBody, false},
{"not a crew member", http.StatusForbidden, "authorization failed: access denied for blob:read: user is not a crew member (required: blob:read or blob:write)", false},
{"unrecognised 403", http.StatusForbidden, "authorization failed: failed to get captain record: boom", false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := isHoldAuthenticationFailure(tt.status, []byte(tt.body)); got != tt.want {
t.Errorf("isHoldAuthenticationFailure(%d, %q) = %v, want %v", tt.status, tt.body, got, tt.want)
}
})
}
}