appview: read the sailor profile from the local users row, not the PDS, per request

Hold discovery in the registry middleware called getRecord on the repository owner's PDS for every request under /v2/: every HEAD, POST, PATCH, PUT and GET. A 10-layer push was 40 or more PDS round trips, and it was the last per-request network call on the push path that had nothing to do with moving bytes. Only two profile fields are used there: the default hold and the auto-remove-untagged flag. The users row already caches the default hold, written by the Jetstream processor on every profile event and prefilled by the backfill, and the auth gate already reads it from there. This makes the row a faithful copy of what the registry needs and switches the middleware to it. The auto-remove flag gets a nullable users column. NULL means the value has never been learned; the processor writes 0 or 1 on every profile event and never NULL. On a request whose row is missing or still NULL, the middleware does one live fetch, uses it, and writes both fields back, including a 0 for a user with no profile at all, so the fallback runs at most once per user. A failed fetch writes nothing and uses the appview default for that request, so a network error is never cached. That single mechanism covers the minutes after a deploy while the startup backfill fills the column, a brand-new user, and a user the backfill has not reached. The processor also stops returning early on an empty default hold, which left a user who removed their custom hold pushing to it forever. Empty is now written through and means the appview default, matching what the auth gate already reads. Tests count PDS requests with a test server: a populated row makes none, a NULL row makes exactly one and then none, a missing profile is cached as known, and a failed fetch degrades without writing. The migration was applied to a fresh database and to one built from the previous schema. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_018Yf1ZVA7sXYhQNb9tCo1m5
2026-09-29 05:25:35 +00:00 · 2026-09-09 20:42:58 -05:00
parent 47a107058d
commit 7045e84c00
12 changed files with 726 additions and 45 deletions
@@ -573,9 +573,12 @@ func (p *Processor) ProcessSailorProfile(ctx context.Context, did string, record
 		slog.Warn("Failed to cache registry domain preference", "component", "processor", "did", did, "registryDomain", profileRecord.RegistryDomain, "error", err)
 	}

-	// Skip hold processing if no default hold set
-	if profileRecord.DefaultHold == "" {
-		return nil
+	// Cache the untagged-cleanup preference. A processed profile is always a
+	// known value, so this writes 0 or 1 and never leaves the column NULL. NULL
+	// means "never learned", which is what makes the registry middleware fetch
+	// the profile live once; writing here is what stops it doing so again.
+	if err := db.UpdateUserAutoRemoveUntagged(p.db, did, profileRecord.AutoRemoveUntagged); err != nil {
+		slog.Warn("Failed to cache autoRemoveUntagged preference", "component", "processor", "did", did, "autoRemoveUntagged", profileRecord.AutoRemoveUntagged, "error", err)
 	}

 	// Convert hold URL/DID to canonical DID. On failure, cache the raw reference
@@ -584,11 +587,22 @@ func (p *Processor) ProcessSailorProfile(ctx context.Context, did string, record
 	// unresolvable must not fall through to managed. A managed hold's DID never
 	// fails resolution (DIDs return as-is), so only unreachable URL-form refs land
 	// here, and a raw non-DID value correctly reads as non-managed (fail closed).
-	holdDID, err := atproto.ResolveHoldDID(ctx, profileRecord.DefaultHold)
-	if err != nil {
-		slog.Warn("Invalid hold reference in profile; caching raw value (fails closed for billing)",
-			"component", "processor", "did", did, "default_hold", profileRecord.DefaultHold, "error", err)
-		holdDID = profileRecord.DefaultHold
+	//
+	// An empty defaultHold is not resolved, it is written straight through. The
+	// user cleared their custom hold, and "" is exactly how both the billing gate
+	// and the auth gate's resolveHoldDID spell "fall back to the operator
+	// default". Returning early here instead, as this used to, left the old value
+	// in the row forever, so a user who moved back to the managed hold kept being
+	// routed to a hold they had abandoned.
+	holdDID := ""
+	if profileRecord.DefaultHold != "" {
+		resolved, err := atproto.ResolveHoldDID(ctx, profileRecord.DefaultHold)
+		if err != nil {
+			slog.Warn("Invalid hold reference in profile; caching raw value (fails closed for billing)",
+				"component", "processor", "did", did, "default_hold", profileRecord.DefaultHold, "error", err)
+			resolved = profileRecord.DefaultHold
+		}
+		holdDID = resolved
 	}

 	// Cache default hold DID on the user record
@@ -598,7 +612,8 @@ func (p *Processor) ProcessSailorProfile(ctx context.Context, did string, record

 	// Query and cache the captain record using provided function
 	// This allows backfill-specific logic (retries, test mode handling) without duplicating it here
-	if queryCaptainFn != nil {
+	// There is nothing to query when the user has no hold of their own.
+	if queryCaptainFn != nil && holdDID != "" {
 		return queryCaptainFn(ctx, holdDID)
 	}

@@ -0,0 +1,184 @@
+package jetstream
+
+import (
+	"context"
+	"database/sql"
+	"testing"
+	"time"
+
+	"atcr.io/pkg/appview/db"
+	"atcr.io/pkg/atproto"
+	"github.com/bluesky-social/indigo/atproto/identity"
+	"github.com/bluesky-social/indigo/atproto/syntax"
+)
+
+// readPrefs returns the cached row state, with auto_remove_untagged as a
+// tri-state: NULL means the processor has never seen a profile for this user,
+// which is what makes the registry middleware fetch one live.
+func readPrefs(t *testing.T, database *sql.DB, did string) (found bool, holdDID sql.NullString, autoRemove sql.NullBool) {
+	t.Helper()
+	err := database.QueryRow(
+		`SELECT default_hold_did, auto_remove_untagged FROM users WHERE did = ?`, did,
+	).Scan(&holdDID, &autoRemove)
+	if err == sql.ErrNoRows {
+		return false, holdDID, autoRemove
+	}
+	if err != nil {
+		t.Fatalf("read cached prefs: %v", err)
+	}
+	return true, holdDID, autoRemove
+}
+
+// TestProcessSailorProfile_ClearingDefaultHold covers a bug that had no
+// expiry: the processor returned early when defaultHold was empty, so a user
+// who removed their custom hold kept the old DID in users.default_hold_did
+// forever and went on being routed to a hold they had abandoned. Empty is a
+// value, and it means "use the operator's default" to both the auth gate's
+// resolveHoldDID and the billing gate.
+func TestProcessSailorProfile_ClearingDefaultHold(t *testing.T) {
+	database := setupTestDB(t)
+	defer database.Close()
+
+	const did = "did:plc:clearhold"
+	if err := db.UpsertUser(database, &db.User{
+		DID: did, Handle: "clear.test", PDSEndpoint: "https://pds", LastSeen: time.Now(),
+	}); err != nil {
+		t.Fatalf("upsert user: %v", err)
+	}
+
+	p := NewProcessor(database, false, nil)
+	ctx := context.Background()
+
+	withHold := []byte(`{"$type":"io.atcr.sailor.profile","defaultHold":"did:web:user.hold.io","createdAt":"2025-01-01T00:00:00Z"}`)
+	if err := p.ProcessSailorProfile(ctx, did, withHold, nil); err != nil {
+		t.Fatalf("ProcessSailorProfile (with hold): %v", err)
+	}
+	if got := db.GetUserDefaultHoldDID(database, did); got != "did:web:user.hold.io" {
+		t.Fatalf("default_hold_did = %q, want the profile's hold", got)
+	}
+
+	cleared := []byte(`{"$type":"io.atcr.sailor.profile","createdAt":"2025-01-01T00:00:00Z"}`)
+	if err := p.ProcessSailorProfile(ctx, did, cleared, nil); err != nil {
+		t.Fatalf("ProcessSailorProfile (cleared): %v", err)
+	}
+	if got := db.GetUserDefaultHoldDID(database, did); got != "" {
+		t.Errorf("default_hold_did = %q after the user cleared it, want \"\" (operator default)", got)
+	}
+}
+
+// TestProcessSailorProfile_ClearedHoldSkipsCaptainQuery: with no hold of their
+// own there is no captain record to go and fetch.
+func TestProcessSailorProfile_ClearedHoldSkipsCaptainQuery(t *testing.T) {
+	database := setupTestDB(t)
+	defer database.Close()
+
+	const did = "did:plc:clearcaptain"
+	if err := db.UpsertUser(database, &db.User{
+		DID: did, Handle: "clearcap.test", PDSEndpoint: "https://pds", LastSeen: time.Now(),
+	}); err != nil {
+		t.Fatalf("upsert user: %v", err)
+	}
+
+	queried := 0
+	queryCaptain := func(context.Context, string) error {
+		queried++
+		return nil
+	}
+
+	p := NewProcessor(database, false, nil)
+	cleared := []byte(`{"$type":"io.atcr.sailor.profile","createdAt":"2025-01-01T00:00:00Z"}`)
+	if err := p.ProcessSailorProfile(context.Background(), did, cleared, queryCaptain); err != nil {
+		t.Fatalf("ProcessSailorProfile: %v", err)
+	}
+	if queried != 0 {
+		t.Errorf("captain query ran %d times for a user with no hold, want 0", queried)
+	}
+}
+
+// TestProcessSailorProfile_CachesAutoRemoveUntagged: a processed profile is
+// always a known value, so the column must come out 0 or 1 and never NULL.
+// NULL is reserved for "never learned", and only the registry's one-shot
+// fallback is allowed to resolve it.
+func TestProcessSailorProfile_CachesAutoRemoveUntagged(t *testing.T) {
+	database := setupTestDB(t)
+	defer database.Close()
+
+	const did = "did:plc:autoremove"
+	if err := db.UpsertUser(database, &db.User{
+		DID: did, Handle: "auto.test", PDSEndpoint: "https://pds", LastSeen: time.Now(),
+	}); err != nil {
+		t.Fatalf("upsert user: %v", err)
+	}
+
+	if _, _, autoRemove := readPrefs(t, database, did); autoRemove.Valid {
+		t.Fatalf("a fresh user row should start with auto_remove_untagged NULL, got %v", autoRemove.Bool)
+	}
+
+	p := NewProcessor(database, false, nil)
+	ctx := context.Background()
+
+	on := []byte(`{"$type":"io.atcr.sailor.profile","autoRemoveUntagged":true,"createdAt":"2025-01-01T00:00:00Z"}`)
+	if err := p.ProcessSailorProfile(ctx, did, on, nil); err != nil {
+		t.Fatalf("ProcessSailorProfile (on): %v", err)
+	}
+	_, _, autoRemove := readPrefs(t, database, did)
+	if !autoRemove.Valid || !autoRemove.Bool {
+		t.Errorf("auto_remove_untagged = %+v, want a valid true", autoRemove)
+	}
+
+	// autoRemoveUntagged is omitempty, so "off" arrives as an absent field.
+	// That still has to be written as a known 0, not left at the previous true.
+	off := []byte(`{"$type":"io.atcr.sailor.profile","createdAt":"2025-01-01T00:00:00Z"}`)
+	if err := p.ProcessSailorProfile(ctx, did, off, nil); err != nil {
+		t.Fatalf("ProcessSailorProfile (off): %v", err)
+	}
+	_, _, autoRemove = readPrefs(t, database, did)
+	if !autoRemove.Valid {
+		t.Fatalf("auto_remove_untagged went back to NULL; the registry would refetch forever")
+	}
+	if autoRemove.Bool {
+		t.Errorf("auto_remove_untagged = true after the user turned it off")
+	}
+}
+
+// TestProcessRecord_SailorProfileCreatesMissingUserRow pins the ensure step.
+// ProcessSailorProfile writes with UPDATE, which is a silent no-op against a
+// missing row, so a profile event for a DID that has never opened the web UI
+// would land nowhere at all without it. ProcessRecord is the only entry point
+// on both the live worker and the backfill, and both go through EnsureUser.
+func TestProcessRecord_SailorProfileCreatesMissingUserRow(t *testing.T) {
+	database := setupTestDB(t)
+	defer database.Close()
+
+	const did = "did:plc:neverseen"
+	atproto.SetDirectory(&fakeDirectory{byDID: map[string]*identity.Identity{
+		did: {
+			DID:    syntax.DID(did),
+			Handle: syntax.Handle("neverseen.example.com"),
+			Services: map[string]identity.ServiceEndpoint{
+				"atproto_pds": {Type: "AtprotoPersonalDataServer", URL: "https://pds.example.com"},
+			},
+		},
+	}})
+
+	if found, _, _ := readPrefs(t, database, did); found {
+		t.Fatal("user row should not exist yet")
+	}
+
+	p := NewProcessor(database, true, nil)
+	record := []byte(`{"$type":"io.atcr.sailor.profile","defaultHold":"did:web:user.hold.io","autoRemoveUntagged":true,"createdAt":"2025-01-01T00:00:00Z"}`)
+	if err := p.ProcessRecord(context.Background(), did, atproto.SailorProfileCollection, "self", record, false, nil); err != nil {
+		t.Fatalf("ProcessRecord: %v", err)
+	}
+
+	found, holdDID, autoRemove := readPrefs(t, database, did)
+	if !found {
+		t.Fatal("a profile event for an unknown DID left no row behind")
+	}
+	if holdDID.String != "did:web:user.hold.io" {
+		t.Errorf("default_hold_did = %q, want the profile's hold", holdDID.String)
+	}
+	if !autoRemove.Valid || !autoRemove.Bool {
+		t.Errorf("auto_remove_untagged = %+v, want a valid true", autoRemove)
+	}
+}