backfill cleanup

pkg/appview/db/schema.go

@@ -80,20 +80,6 @@ CREATE TABLE IF NOT EXISTS tags (
 );
 CREATE INDEX IF NOT EXISTS idx_tags_did_repo ON tags(did, repository);
 
-CREATE TABLE IF NOT EXISTS firehose_cursor (
-    id INTEGER PRIMARY KEY CHECK (id = 1),
-    cursor INTEGER NOT NULL,
-    updated_at TIMESTAMP NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS backfill_state (
-    id INTEGER PRIMARY KEY CHECK (id = 1),
-    start_cursor INTEGER NOT NULL,
-    current_cursor INTEGER NOT NULL,
-    completed BOOLEAN NOT NULL DEFAULT 0,
-    updated_at TIMESTAMP NOT NULL
-);
-
 CREATE TABLE IF NOT EXISTS oauth_sessions (
     session_key TEXT PRIMARY KEY,
     account_did TEXT NOT NULL,

pkg/appview/jetstream/worker.go

@@ -44,6 +44,10 @@ type Worker struct {
 	pongsReceived int64
 	lastPongTime  time.Time
 	pongMutex     sync.Mutex
+
+	// In-memory cursor tracking for reconnects
+	lastCursor int64
+	cursorMutex sync.RWMutex
 }
 
 // NewWorker creates a new Jetstream worker
@@ -83,10 +87,14 @@ func (w *Worker) Start(ctx context.Context) error {
 		q.Add("wantedCollections", collection)
 	}
 
-	// Add cursor if specified (for backfilling historical data)
+	// Add cursor if specified (for backfilling historical data or reconnects)
 	if w.startCursor > 0 {
 		q.Set("cursor", fmt.Sprintf("%d", w.startCursor))
-		fmt.Printf("Starting from cursor: %d (replaying historical events)\n", w.startCursor)
+
+		// Calculate lag (cursor is in microseconds)
+		now := time.Now().UnixMicro()
+		lagSeconds := float64(now-w.startCursor) / 1_000_000.0
+		fmt.Printf("Jetstream: Starting from cursor %d (%.1f seconds behind live)\n", w.startCursor, lagSeconds)
 	}
 
 	// Disable compression for now to debug
@@ -263,6 +271,13 @@ func (w *Worker) SetEventCallback(cb EventCallback) {
 	w.eventCallback = cb
 }
 
+// GetLastCursor returns the last processed cursor (time_us) for reconnects
+func (w *Worker) GetLastCursor() int64 {
+	w.cursorMutex.RLock()
+	defer w.cursorMutex.RUnlock()
+	return w.lastCursor
+}
+
 // processMessage processes a single Jetstream event
 func (w *Worker) processMessage(message []byte) error {
 	var event JetstreamEvent
@@ -270,6 +285,11 @@ func (w *Worker) processMessage(message []byte) error {
 		return fmt.Errorf("failed to unmarshal event: %w", err)
 	}
 
+	// Update cursor for reconnects (do this first, even if processing fails)
+	w.cursorMutex.Lock()
+	w.lastCursor = event.TimeUS
+	w.cursorMutex.Unlock()
+
 	// Call callback if set
 	if w.eventCallback != nil {
 		w.eventCallback(event.TimeUS)