mirror of
https://tangled.org/evan.jarrett.net/at-container-registry
synced 2026-09-25 03:34:14 +00:00
Three defects that all turn on whether the hold knows what a scanner is doing. Neither end had a keepalive, a read deadline, or a read limit, so a half-open connection was invisible until some other timeout fired. That got worse with capacity-aware dispatch: a dead-but-connected scanner holds its advertised worker count out of the budget and keeps winning jobs. Both ends now ping every 30s against a 90s read deadline, so three unanswered pings condemn a connection. Detection takes about 90 seconds, after which the existing reconnect grace reclaims the rows, against the 60-minute queueing timeout that was previously the only escape. Liveness decides when a scanner is gone; the grace window decides when its work is reassignable. Read limits are asymmetric and deliberately generous, because exceeding one closes the connection rather than truncating, which would turn a large but legitimate result into a permanent retry loop. Write deadlines were absent everywhere; the scanner in particular held a mutex across an unbounded write, so a wedged write silenced it without disconnecting it. handleResult did two S3 uploads and a CAR commit inline on the reader goroutine, so a slow S3 looked exactly like a dead scanner. Terminal messages now go to a per-subscriber storage goroutine while acks and starts stay on the reader. One goroutine, not a pool: every path ends in CreateScanRecord, which serialises on the repo lock anyway, and ordering is worth more than parallelism that cannot be used. Uploads and the record write get separate budgets, so a stalled upload cannot spend the time the record needs and the record write stays unconditional. checkPredecessor cached an inconclusive answer as a definitive negative in a map that is never reset, so one unreachable hold meant its manifests were never scanned again for the life of the process. gc.go already carried the corrected logic for the same problem; this follows it rather than inventing a second approach, and also stops treating an unparseable captain record as definitive. The test PDS had to move off ":memory:", which go-libsql scopes per connection: writing a scan record from any goroutine but the caller's got a connection with no tables. That was invisible while every record write happened inline. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01U1Km3N3uUmeGaj7VbaM8PF
157 lines
4.8 KiB
Go
157 lines
4.8 KiB
Go
package client
|
|
|
|
import (
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"atcr.io/scanner/internal/queue"
|
|
"github.com/gorilla/websocket"
|
|
)
|
|
|
|
// upgradeOnly accepts a WebSocket and hands the server side to onConn, which
|
|
// decides how lifelike the peer is. The returned URL is what a HoldClient dials.
|
|
func upgradeOnly(t *testing.T, onConn func(*websocket.Conn)) string {
|
|
t.Helper()
|
|
|
|
upgrader := websocket.Upgrader{}
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
conn, err := upgrader.Upgrade(w, r, nil)
|
|
if err != nil {
|
|
return
|
|
}
|
|
onConn(conn)
|
|
}))
|
|
t.Cleanup(srv.Close)
|
|
return "ws" + strings.TrimPrefix(srv.URL, "http")
|
|
}
|
|
|
|
func newTestHoldClient(t *testing.T, url string) *HoldClient {
|
|
t.Helper()
|
|
|
|
c := NewHoldClient(url, "secret", queue.NewJobQueue(8))
|
|
c.pingInterval = 20 * time.Millisecond
|
|
c.pongWait = 200 * time.Millisecond
|
|
c.writeWait = time.Second
|
|
t.Cleanup(func() {
|
|
defer func() { _ = recover() }() // Close panics if already closed
|
|
c.Close()
|
|
})
|
|
return c
|
|
}
|
|
|
|
// TestHoldClient_DetectsHalfOpenConnection is the scanner half of F7.
|
|
//
|
|
// connectOnce sat in ReadMessage with no deadline and sent no pings, so a hold
|
|
// that went away without closing the TCP connection left this scanner
|
|
// permanently "connected": it never reconnects, never re-drains the hold's
|
|
// pending rows, and every result it computes is written into a socket that goes
|
|
// nowhere. The peer here upgrades and then does nothing at all — it never reads,
|
|
// so it never pongs.
|
|
func TestHoldClient_DetectsHalfOpenConnection(t *testing.T) {
|
|
held := make(chan struct{})
|
|
url := upgradeOnly(t, func(conn *websocket.Conn) {
|
|
<-held // hold the connection open, answering nothing
|
|
_ = conn.Close()
|
|
})
|
|
t.Cleanup(func() { close(held) })
|
|
|
|
c := newTestHoldClient(t, url)
|
|
|
|
errCh := make(chan error, 1)
|
|
go func() { errCh <- c.connectOnce(-1) }()
|
|
|
|
select {
|
|
case err := <-errCh:
|
|
if err == nil {
|
|
t.Fatal("connectOnce returned nil for a peer that answered nothing")
|
|
}
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("a hold that answers no pings never disconnected the scanner: " +
|
|
"the client waits in ReadMessage with no deadline and sends no pings")
|
|
}
|
|
}
|
|
|
|
// The other side of the rule. An idle hold is the normal state — the dfd604b
|
|
// commit message records a hold and scanner that shared one WebSocket for a
|
|
// week — so silence alone must not drop the connection. gorilla answers a ping
|
|
// from inside ReadMessage, which is what the real hold's reader does.
|
|
func TestHoldClient_StaysConnectedWhileHoldAnswers(t *testing.T) {
|
|
url := upgradeOnly(t, func(conn *websocket.Conn) {
|
|
for {
|
|
if _, _, err := conn.ReadMessage(); err != nil {
|
|
_ = conn.Close()
|
|
return
|
|
}
|
|
}
|
|
})
|
|
|
|
c := newTestHoldClient(t, url)
|
|
|
|
errCh := make(chan error, 1)
|
|
go func() { errCh <- c.connectOnce(-1) }()
|
|
|
|
// Many pongWait periods with no traffic but pings.
|
|
select {
|
|
case err := <-errCh:
|
|
t.Fatalf("an idle but responsive hold was dropped: %v", err)
|
|
case <-time.After(time.Second):
|
|
}
|
|
|
|
c.Close()
|
|
select {
|
|
case <-errCh:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("connectOnce did not return after Close")
|
|
}
|
|
}
|
|
|
|
// A job frame is small — a config descriptor and a layer list — so the read
|
|
// limit that bounds a hostile or broken hold must still admit anything the real
|
|
// one sends with room to spare.
|
|
func TestHoldClient_AcceptsARealisticJobFrame(t *testing.T) {
|
|
delivered := make(chan struct{})
|
|
url := upgradeOnly(t, func(conn *websocket.Conn) {
|
|
// A 128-layer image, which is well past the OCI-practical limit.
|
|
var layers []string
|
|
for i := 0; i < 128; i++ {
|
|
layers = append(layers,
|
|
`{"digest":"sha256:`+strings.Repeat("a", 64)+`","size":123456,"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip"}`)
|
|
}
|
|
frame := `{"type":"job","seq":1,"manifestDigest":"sha256:` + strings.Repeat("b", 64) + `",` +
|
|
`"repository":"repo","tag":"latest","userDid":"did:plc:user","userHandle":"user.example.com",` +
|
|
`"holdDid":"did:web:hold.example.com","holdEndpoint":"https://hold.example.com","tier":"deckhand",` +
|
|
`"config":{"digest":"sha256:` + strings.Repeat("c", 64) + `","size":1024,"mediaType":"application/vnd.oci.image.config.v1+json"},` +
|
|
`"layers":[` + strings.Join(layers, ",") + `]}`
|
|
if err := conn.WriteMessage(websocket.TextMessage, []byte(frame)); err != nil {
|
|
return
|
|
}
|
|
for {
|
|
if _, _, err := conn.ReadMessage(); err != nil {
|
|
_ = conn.Close()
|
|
return
|
|
}
|
|
select {
|
|
case delivered <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
})
|
|
|
|
c := newTestHoldClient(t, url)
|
|
c.pongWait = 5 * time.Second
|
|
go func() { _ = c.connectOnce(-1) }()
|
|
|
|
select {
|
|
case <-delivered:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("a realistic job frame was never acknowledged: the read limit rejected it")
|
|
}
|
|
|
|
if c.queue.Len() != 1 {
|
|
t.Errorf("queue depth = %d, want 1: the job frame did not survive the read", c.queue.Len())
|
|
}
|
|
}
|