Files
at-container-registry/scanner/internal/client/hold_test.go
T
Evan JarrettandClaude Opus 5 1853c0c1d3 hold/scanner: detect dead connections and get storage off the reader
Three defects that all turn on whether the hold knows what a scanner is doing.

Neither end had a keepalive, a read deadline, or a read limit, so a half-open
connection was invisible until some other timeout fired. That got worse with
capacity-aware dispatch: a dead-but-connected scanner holds its advertised
worker count out of the budget and keeps winning jobs. Both ends now ping every
30s against a 90s read deadline, so three unanswered pings condemn a
connection. Detection takes about 90 seconds, after which the existing
reconnect grace reclaims the rows, against the 60-minute queueing timeout that
was previously the only escape. Liveness decides when a scanner is gone; the
grace window decides when its work is reassignable.

Read limits are asymmetric and deliberately generous, because exceeding one
closes the connection rather than truncating, which would turn a large but
legitimate result into a permanent retry loop. Write deadlines were absent
everywhere; the scanner in particular held a mutex across an unbounded write,
so a wedged write silenced it without disconnecting it.

handleResult did two S3 uploads and a CAR commit inline on the reader
goroutine, so a slow S3 looked exactly like a dead scanner. Terminal messages
now go to a per-subscriber storage goroutine while acks and starts stay on the
reader. One goroutine, not a pool: every path ends in CreateScanRecord, which
serialises on the repo lock anyway, and ordering is worth more than parallelism
that cannot be used. Uploads and the record write get separate budgets, so a
stalled upload cannot spend the time the record needs and the record write
stays unconditional.

checkPredecessor cached an inconclusive answer as a definitive negative in a
map that is never reset, so one unreachable hold meant its manifests were never
scanned again for the life of the process. gc.go already carried the corrected
logic for the same problem; this follows it rather than inventing a second
approach, and also stops treating an unparseable captain record as definitive.

The test PDS had to move off ":memory:", which go-libsql scopes per connection:
writing a scan record from any goroutine but the caller's got a connection with
no tables. That was invisible while every record write happened inline.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01U1Km3N3uUmeGaj7VbaM8PF
2026-09-05 15:41:27 -05:00

157 lines
4.8 KiB
Go

package client
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"atcr.io/scanner/internal/queue"
"github.com/gorilla/websocket"
)
// upgradeOnly accepts a WebSocket and hands the server side to onConn, which
// decides how lifelike the peer is. The returned URL is what a HoldClient dials.
func upgradeOnly(t *testing.T, onConn func(*websocket.Conn)) string {
t.Helper()
upgrader := websocket.Upgrader{}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
return
}
onConn(conn)
}))
t.Cleanup(srv.Close)
return "ws" + strings.TrimPrefix(srv.URL, "http")
}
func newTestHoldClient(t *testing.T, url string) *HoldClient {
t.Helper()
c := NewHoldClient(url, "secret", queue.NewJobQueue(8))
c.pingInterval = 20 * time.Millisecond
c.pongWait = 200 * time.Millisecond
c.writeWait = time.Second
t.Cleanup(func() {
defer func() { _ = recover() }() // Close panics if already closed
c.Close()
})
return c
}
// TestHoldClient_DetectsHalfOpenConnection is the scanner half of F7.
//
// connectOnce sat in ReadMessage with no deadline and sent no pings, so a hold
// that went away without closing the TCP connection left this scanner
// permanently "connected": it never reconnects, never re-drains the hold's
// pending rows, and every result it computes is written into a socket that goes
// nowhere. The peer here upgrades and then does nothing at all — it never reads,
// so it never pongs.
func TestHoldClient_DetectsHalfOpenConnection(t *testing.T) {
held := make(chan struct{})
url := upgradeOnly(t, func(conn *websocket.Conn) {
<-held // hold the connection open, answering nothing
_ = conn.Close()
})
t.Cleanup(func() { close(held) })
c := newTestHoldClient(t, url)
errCh := make(chan error, 1)
go func() { errCh <- c.connectOnce(-1) }()
select {
case err := <-errCh:
if err == nil {
t.Fatal("connectOnce returned nil for a peer that answered nothing")
}
case <-time.After(5 * time.Second):
t.Fatal("a hold that answers no pings never disconnected the scanner: " +
"the client waits in ReadMessage with no deadline and sends no pings")
}
}
// The other side of the rule. An idle hold is the normal state — the dfd604b
// commit message records a hold and scanner that shared one WebSocket for a
// week — so silence alone must not drop the connection. gorilla answers a ping
// from inside ReadMessage, which is what the real hold's reader does.
func TestHoldClient_StaysConnectedWhileHoldAnswers(t *testing.T) {
url := upgradeOnly(t, func(conn *websocket.Conn) {
for {
if _, _, err := conn.ReadMessage(); err != nil {
_ = conn.Close()
return
}
}
})
c := newTestHoldClient(t, url)
errCh := make(chan error, 1)
go func() { errCh <- c.connectOnce(-1) }()
// Many pongWait periods with no traffic but pings.
select {
case err := <-errCh:
t.Fatalf("an idle but responsive hold was dropped: %v", err)
case <-time.After(time.Second):
}
c.Close()
select {
case <-errCh:
case <-time.After(5 * time.Second):
t.Fatal("connectOnce did not return after Close")
}
}
// A job frame is small — a config descriptor and a layer list — so the read
// limit that bounds a hostile or broken hold must still admit anything the real
// one sends with room to spare.
func TestHoldClient_AcceptsARealisticJobFrame(t *testing.T) {
delivered := make(chan struct{})
url := upgradeOnly(t, func(conn *websocket.Conn) {
// A 128-layer image, which is well past the OCI-practical limit.
var layers []string
for i := 0; i < 128; i++ {
layers = append(layers,
`{"digest":"sha256:`+strings.Repeat("a", 64)+`","size":123456,"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip"}`)
}
frame := `{"type":"job","seq":1,"manifestDigest":"sha256:` + strings.Repeat("b", 64) + `",` +
`"repository":"repo","tag":"latest","userDid":"did:plc:user","userHandle":"user.example.com",` +
`"holdDid":"did:web:hold.example.com","holdEndpoint":"https://hold.example.com","tier":"deckhand",` +
`"config":{"digest":"sha256:` + strings.Repeat("c", 64) + `","size":1024,"mediaType":"application/vnd.oci.image.config.v1+json"},` +
`"layers":[` + strings.Join(layers, ",") + `]}`
if err := conn.WriteMessage(websocket.TextMessage, []byte(frame)); err != nil {
return
}
for {
if _, _, err := conn.ReadMessage(); err != nil {
_ = conn.Close()
return
}
select {
case delivered <- struct{}{}:
default:
}
}
})
c := newTestHoldClient(t, url)
c.pongWait = 5 * time.Second
go func() { _ = c.connectOnce(-1) }()
select {
case <-delivered:
case <-time.After(5 * time.Second):
t.Fatal("a realistic job frame was never acknowledged: the read limit rejected it")
}
if c.queue.Len() != 1 {
t.Errorf("queue depth = %d, want 1: the job frame did not survive the read", c.queue.Len())
}
}