improve backfill and jetstream db connections

This commit is contained in:
Evan Jarrett
2026-04-19 18:01:17 -05:00
parent 38c693acc9
commit 7c6b8945ed
4 changed files with 205 additions and 29 deletions
+13 -2
View File
@@ -92,6 +92,16 @@ func InitDB(path string, cfg LibsqlConfig) (*sql.DB, error) {
return nil, err
}
// Bound the connection pool. With a remote target (Bunny Database), each
// idle conn is a stable libsql stream — keeping a handful warm avoids
// reconnect cost, capping the total prevents runaway contention. Short
// lifetimes ensure we recycle past any idle-side disconnects and drop any
// poisoned conn that survived IsPoisonedTxErr eviction.
db.SetMaxOpenConns(8)
db.SetMaxIdleConns(4)
db.SetConnMaxLifetime(5 * time.Minute)
db.SetConnMaxIdleTime(2 * time.Minute)
// Check if this is an existing database with migrations applied
isExisting, err := hasAppliedMigrations(db)
if err != nil {
@@ -202,20 +212,21 @@ func runMigrations(db *sql.DB, freshDB bool) error {
if err != nil {
return fmt.Errorf("failed to begin transaction for migration %d: %w", m.Version, err)
}
// Deferred rollback is a no-op once Commit succeeds; it guards against
// panics and any early return that forgets an explicit rollback.
defer func() { _ = tx.Rollback() }()
// Split query into individual statements and execute each
// go-sqlite3's Exec() doesn't reliably execute all statements in multi-statement queries
statements := splitSQLStatements(m.Query)
for i, stmt := range statements {
if _, err := tx.Exec(stmt); err != nil {
tx.Rollback()
return fmt.Errorf("failed to apply migration %d (%s) statement %d: %w", m.Version, m.Name, i+1, err)
}
}
// Record migration
if _, err := tx.Exec("INSERT INTO schema_migrations (version) VALUES (?)", m.Version); err != nil {
tx.Rollback()
return fmt.Errorf("failed to record migration %d: %w", m.Version, err)
}
+6
View File
@@ -180,6 +180,12 @@ CREATE TABLE IF NOT EXISTS repository_stats_daily (
);
CREATE INDEX IF NOT EXISTS idx_repo_stats_daily_date ON repository_stats_daily(date DESC);
CREATE TABLE IF NOT EXISTS jetstream_cursor (
id INTEGER PRIMARY KEY CHECK (id = 1),
cursor INTEGER NOT NULL,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS stars (
starrer_did TEXT NOT NULL,
owner_did TEXT NOT NULL,