improve backfill and jetstream db connections

pkg/appview/jetstream/worker.go

@@ -49,6 +49,12 @@ type Worker struct {
 	// In-memory cursor tracking for reconnects
 	lastCursor  int64
 	cursorMutex sync.RWMutex
+
+	// Cursor persistence: a single-slot channel carries the most recent
+	// cursor to a background saver goroutine. The saver writes to
+	// jetstream_cursor every tick, dropping any older value that has not
+	// yet been flushed so the WS read loop is never blocked on DB I/O.
+	cursorSave chan int64
 }
 
 // NewWorker creates a new Jetstream worker
@@ -74,6 +80,7 @@ func NewWorker(database *sql.DB, urls []string, startCursor int64) *Worker {
 		},
 		statsCache: statsCache,
 		processor:  NewProcessor(database, true, statsCache), // Use cache for live streaming
+		cursorSave: make(chan int64, 1),
 	}
 }
 
@@ -258,7 +265,7 @@ func (w *Worker) Start(ctx context.Context) error {
 			// TODO: Re-enable compression once debugging is complete
 			_ = decoder // Keep decoder to avoid unused variable error
 
-			if err := w.processMessage(message); err != nil {
+			if err := w.processMessageResilient(ctx, message); err != nil {
 				slog.Error("ERROR processing message", "error", err)
 				// Continue processing other messages
 			} else {
@@ -274,6 +281,37 @@ func (w *Worker) Start(ctx context.Context) error {
 // 30 seconds to avoid missing events (events are idempotent DB upserts).
 // Cycles through all endpoints indefinitely and never gives up.
 func (w *Worker) StartWithFailover(ctx context.Context) {
+	// Bootstrap from the persisted cursor the first time we run. If the DB
+	// has a saved cursor we resume from it (minus a small safety rewind so
+	// any gap from the previous shutdown is covered). Events are idempotent
+	// UPSERTs, so re-processing a handful is harmless.
+	if w.startCursor == 0 {
+		if cursor, err := db.GetJetstreamCursor(w.db); err != nil {
+			slog.Warn("Jetstream failed to load persisted cursor", "error", err)
+		} else if cursor > 0 {
+			const rewind = int64(30 * 1_000_000) // 30s safety rewind, same units as cursor
+			resume := cursor - rewind
+			if resume < 0 {
+				resume = 0
+			}
+			w.cursorMutex.Lock()
+			w.startCursor = resume
+			w.lastCursor = resume
+			w.cursorMutex.Unlock()
+			slog.Info("Jetstream resuming from persisted cursor",
+				"persisted_cursor", cursor,
+				"resume_cursor", resume)
+		}
+	}
+
+	// Launch the background cursor saver. It runs for the lifetime of this
+	// Start call and exits on ctx.Done with a final flush.
+	saverDone := make(chan struct{})
+	go w.runCursorSaver(ctx, saverDone)
+	defer func() {
+		<-saverDone
+	}()
+
 	retryDelays := []time.Duration{1 * time.Second, 5 * time.Second, 10 * time.Second}
 
 	for {
@@ -358,6 +396,49 @@ func (w *Worker) Processor() *Processor {
 	return w.processor
 }
 
+// runCursorSaver is a long-running goroutine that persists the most recent
+// Jetstream cursor to SQLite. It writes at most once every cursorSaveInterval
+// so we never hit the DB faster than it can keep up, and always flushes a
+// final value on shutdown so the next Start resumes from the right place.
+//
+// The goroutine intentionally uses b.db directly (not ExecResilient) because
+// the INSERT ... ON CONFLICT statement is a single round-trip that cannot
+// trigger the poisoned-tx cascade.
+func (w *Worker) runCursorSaver(ctx context.Context, done chan<- struct{}) {
+	defer close(done)
+
+	const cursorSaveInterval = 5 * time.Second
+	ticker := time.NewTicker(cursorSaveInterval)
+	defer ticker.Stop()
+
+	var pending int64
+	flush := func() {
+		if pending == 0 {
+			return
+		}
+		if err := db.SaveJetstreamCursor(w.db, pending); err != nil {
+			slog.Warn("Jetstream failed to persist cursor", "cursor", pending, "error", err)
+			return
+		}
+		pending = 0
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+			flush()
+			return
+		case c := <-w.cursorSave:
+			// Keep only the newest value; the ticker decides when to flush.
+			if c > pending {
+				pending = c
+			}
+		case <-ticker.C:
+			flush()
+		}
+	}
+}
+
 // GetLastCursor returns the last processed cursor (time_us) for reconnects
 func (w *Worker) GetLastCursor() int64 {
 	w.cursorMutex.RLock()
@@ -365,6 +446,49 @@ func (w *Worker) GetLastCursor() int64 {
 	return w.lastCursor
 }
 
+// processMessageResilient runs processMessage and, if the underlying DB
+// connection was poisoned by a remote tx timeout (common with Bunny Database
+// after a backfill chunk exceeded the server-side transaction limit), drains
+// the poisoned connections from the pool and retries once. A second failure
+// returns the error so the caller's Error log line fires — replacing silent
+// data loss with a loud, attributable one.
+func (w *Worker) processMessageResilient(ctx context.Context, message []byte) error {
+	err := w.processMessage(message)
+	if err == nil || !db.IsPoisonedTxErr(err) {
+		return err
+	}
+
+	slog.Warn("Jetstream poisoned connection detected, draining pool and retrying",
+		"error", err)
+	drainPool(ctx, w.db)
+	time.Sleep(100 * time.Millisecond)
+	return w.processMessage(message)
+}
+
+// drainPool borrows each idle connection from the pool in turn and runs a
+// trivial probe. A poisoned connection fails the probe, and db.ExecResilient
+// evicts it via driver.ErrBadConn. Loops up to the pool's open-connection
+// limit so a single call can clear every bad conn.
+func drainPool(ctx context.Context, database *sql.DB) {
+	// MaxOpenConns is 8 (see pkg/appview/db/schema.go). We probe one more time
+	// than that to ensure we cycle through every conn if any were mid-use.
+	const maxProbes = 10
+	for i := 0; i < maxProbes; i++ {
+		err := db.ExecResilient(ctx, database, func(conn *sql.Conn) error {
+			_, err := conn.ExecContext(ctx, "SELECT 1")
+			return err
+		})
+		if err == nil {
+			// Got a healthy conn; no need to probe further — any remaining
+			// poisoned conns will be evicted on their next use.
+			return
+		}
+		if ctx.Err() != nil {
+			return
+		}
+	}
+}
+
 // processMessage processes a single Jetstream event
 func (w *Worker) processMessage(message []byte) error {
 	var event JetstreamEvent
@@ -377,6 +501,26 @@ func (w *Worker) processMessage(message []byte) error {
 	w.lastCursor = event.TimeUS
 	w.cursorMutex.Unlock()
 
+	// Offer the cursor to the async saver. Non-blocking: if the saver is
+	// still writing the previous value, we drop-and-replace so the DB always
+	// converges on the freshest cursor without ever stalling the read loop.
+	if w.cursorSave != nil {
+		select {
+		case w.cursorSave <- event.TimeUS:
+		default:
+			// Drain any stale value and try once more — if that still fails
+			// we just skip this tick; the saver's timer will catch up.
+			select {
+			case <-w.cursorSave:
+			default:
+			}
+			select {
+			case w.cursorSave <- event.TimeUS:
+			default:
+			}
+		}
+	}
+
 	// Call callback if set
 	if w.eventCallback != nil {
 		w.eventCallback(event.TimeUS)