// Package db provides the database layer for the AppView web UI, including // SQLite schema initialization, migrations, and query functions for OAuth // sessions, device flows, repository metadata, stars, pull counts, and // user profiles. package db import ( "context" "database/sql" "database/sql/driver" "embed" "fmt" "io/fs" "log/slog" "path/filepath" "sort" "strconv" "strings" "time" "github.com/tursodatabase/go-libsql" "go.yaml.in/yaml/v4" ) //go:embed migrations/*.yaml var migrationsFS embed.FS //go:embed schema.sql var schemaSQL string // LibsqlConfig holds optional libSQL sync settings for embedded replicas. // When SyncURL is empty, the database operates in local-only mode. type LibsqlConfig struct { SyncURL string AuthToken string SyncInterval time.Duration } // InitDB initializes the database with the schema. // Uses libSQL driver: local-only when cfg.SyncURL is empty, // embedded replica when cfg.SyncURL is set. func InitDB(path string, cfg LibsqlConfig) (*sql.DB, error) { var db *sql.DB if cfg.SyncURL != "" { // Embedded replica mode: local file + sync to remote opts := []libsql.Option{ libsql.WithAuthToken(cfg.AuthToken), } if cfg.SyncInterval > 0 { opts = append(opts, libsql.WithSyncInterval(cfg.SyncInterval)) } connector, err := libsql.NewEmbeddedReplicaConnector(path, cfg.SyncURL, opts...) if err != nil { return nil, fmt.Errorf("failed to create libsql embedded replica connector: %w", err) } db = sql.OpenDB(connector) slog.Info("Database opened in embedded replica mode", "path", path, "sync_url", cfg.SyncURL) } else { // Local-only mode: plain file via libsql driver, wrapped so every new // connection gets PRAGMA busy_timeout. SQLite's busy_timeout is // per-connection, so a one-shot db.Exec only configures whichever // pooled conn served the call — leaving the rest to fail SQLITE_BUSY // instantly on any write contention with the jetstream/backfill workers. // Paths starting with "file:" or ":memory:" are already valid libsql URIs. dsn := path if !strings.HasPrefix(path, "file:") && !strings.HasPrefix(path, ":memory:") { dsn = "file:" + path } baseConnector, err := openLibsqlLocalConnector(dsn) if err != nil { return nil, err } db = sql.OpenDB(&busyTimeoutConnector{base: baseConnector, timeoutMs: 5000}) slog.Info("Database opened in local-only mode", "path", path) } // In local-only mode, set WAL mode (database-wide setting, persists // across connections — single call is sufficient unlike busy_timeout). // In embedded replica mode, the remote server manages this and the // PRAGMA is rejected as "unsupported statement" (observed with Bunny; // Turso may behave similarly). if cfg.SyncURL == "" { var journalMode string if err := db.QueryRow("PRAGMA journal_mode = WAL").Scan(&journalMode); err != nil { return nil, err } } // Enable foreign keys if _, err := db.Exec("PRAGMA foreign_keys = ON"); err != nil { return nil, err } // Bound the connection pool. With a remote target (Bunny Database), each // idle conn is a stable libsql stream — keeping a handful warm avoids // reconnect cost, capping the total prevents runaway contention. Short // lifetimes ensure we recycle past any idle-side disconnects and drop any // poisoned conn that survived IsPoisonedTxErr eviction. db.SetMaxOpenConns(8) db.SetMaxIdleConns(4) db.SetConnMaxLifetime(5 * time.Minute) db.SetConnMaxIdleTime(2 * time.Minute) // Check if this is an existing database with migrations applied isExisting, err := hasAppliedMigrations(db) if err != nil { return nil, fmt.Errorf("failed to check database state: %w", err) } if isExisting { // Existing database: skip schema.sql, only run pending migrations slog.Debug("Existing database detected, skipping schema.sql") } else { // Fresh database: apply schema.sql slog.Info("Fresh database detected, applying schema") if err := applySchema(db); err != nil { return nil, err } } // Run migrations // For fresh databases, migrations are recorded but not executed (schema.sql is already complete) if err := runMigrations(db, !isExisting); err != nil { return nil, err } // Report any structural drift from schema.sql. Only meaningful for existing // databases: a fresh one was just built from schema.sql, so it agrees by // construction. Warn and continue — see SchemaDrift for why this never // fails the boot. if isExisting { reportSchemaDrift(db) } return db, nil } // reportSchemaDrift logs any difference between the live database and // schema.sql. Migrations can be recorded without being executed (0004 was, on // production, which is the whole reason migration 0009 exists), and until now // nothing surfaced that. Failure to run the check is itself only a warning: an // introspection problem must not stop the server from starting. func reportSchemaDrift(db *sql.DB) { findings, err := SchemaDrift(db) if err != nil { slog.Warn("Could not check database schema for drift", "error", err) return } if len(findings) == 0 { slog.Debug("Database schema matches schema.sql") return } slog.Warn("Database schema differs from schema.sql", "differences", len(findings), "hint", "a migration may have been recorded without being executed; a corrective migration is probably needed") for _, f := range findings { slog.Warn("Schema drift", "detail", f) } } // hasAppliedMigrations checks if this is an existing database with migrations applied func hasAppliedMigrations(db *sql.DB) (bool, error) { // Check if schema_migrations table exists var count int err := db.QueryRow(` SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='schema_migrations' `).Scan(&count) if err != nil { return false, err } if count == 0 { return false, nil // No migrations table = fresh DB } // Table exists, check if it has entries err = db.QueryRow("SELECT COUNT(*) FROM schema_migrations").Scan(&count) if err != nil { return false, err } return count > 0, nil } // applySchema executes schema.sql for fresh databases func applySchema(db *sql.DB) error { for _, stmt := range splitSQLStatements(schemaSQL) { if _, err := db.Exec(stmt); err != nil { return fmt.Errorf("failed to apply schema: %w", err) } } return nil } // Migration represents a database migration type Migration struct { Version int Name string Description string `yaml:"description"` Query string `yaml:"query"` } // runMigrations applies any pending database migrations // If freshDB is true, migrations are recorded but not executed (schema.sql already includes their changes) func runMigrations(db *sql.DB, freshDB bool) error { // Load migrations from files migrations, err := loadMigrations() if err != nil { return fmt.Errorf("failed to load migrations: %w", err) } // Sort migrations by version sort.Slice(migrations, func(i, j int) bool { return migrations[i].Version < migrations[j].Version }) for _, m := range migrations { // Check if migration already applied var count int err := db.QueryRow("SELECT COUNT(*) FROM schema_migrations WHERE version = ?", m.Version).Scan(&count) if err != nil { return fmt.Errorf("failed to check migration status: %w", err) } if count > 0 { // Migration already applied continue } if freshDB { // Fresh database: schema.sql already has everything, just record the migration slog.Debug("Recording migration as applied (fresh DB)", "version", m.Version, "name", m.Name) if _, err := db.Exec("INSERT INTO schema_migrations (version) VALUES (?)", m.Version); err != nil { return fmt.Errorf("failed to record migration %d: %w", m.Version, err) } continue } // Existing database: apply migration in a transaction slog.Info("Applying migration", "version", m.Version, "name", m.Name, "description", m.Description) tx, err := db.Begin() if err != nil { return fmt.Errorf("failed to begin transaction for migration %d: %w", m.Version, err) } // Deferred rollback is a no-op once Commit succeeds; it guards against // panics and any early return that forgets an explicit rollback. defer func() { _ = tx.Rollback() }() // Split query into individual statements and execute each // go-sqlite3's Exec() doesn't reliably execute all statements in multi-statement queries statements := splitSQLStatements(m.Query) for i, stmt := range statements { if _, err := tx.Exec(stmt); err != nil { return fmt.Errorf("failed to apply migration %d (%s) statement %d: %w", m.Version, m.Name, i+1, err) } } // Some migrations need a step SQL cannot express. It runs in this same // transaction, after the DDL it depends on and before the version is // recorded, so the version is never recorded without its Go half. if hook, ok := goMigrations[m.Version]; ok { slog.Info("Running Go migration step", "version", m.Version, "name", m.Name) if err := hook(tx); err != nil { return fmt.Errorf("failed to apply migration %d (%s) Go step: %w", m.Version, m.Name, err) } } // Record migration if _, err := tx.Exec("INSERT INTO schema_migrations (version) VALUES (?)", m.Version); err != nil { return fmt.Errorf("failed to record migration %d: %w", m.Version, err) } if err := tx.Commit(); err != nil { return fmt.Errorf("failed to commit migration %d: %w", m.Version, err) } slog.Info("Migration applied successfully", "version", m.Version) } return nil } // loadMigrations loads all migration files from embedded filesystem func loadMigrations() ([]Migration, error) { // Read all migration files from embedded FS entries, err := fs.Glob(migrationsFS, "migrations/[0-9][0-9][0-9][0-9]_*.yaml") if err != nil { return nil, fmt.Errorf("failed to list migration files: %w", err) } var migrations []Migration for _, file := range entries { // Parse version and name from filename basename := filepath.Base(file) version, name, err := parseMigrationFilename(basename) if err != nil { return nil, fmt.Errorf("invalid migration filename %s: %w", basename, err) } // Read YAML content from embedded FS data, err := migrationsFS.ReadFile(file) if err != nil { return nil, fmt.Errorf("failed to read migration file %s: %w", file, err) } var m Migration if err := yaml.Unmarshal(data, &m); err != nil { return nil, fmt.Errorf("failed to parse migration file %s: %w", file, err) } // Set version and name from filename m.Version = version m.Name = name // Validate migration. A migration may be pure SQL, SQL plus a Go step, // or Go only; what it may not be is empty. if m.Query == "" { if _, hasHook := goMigrations[version]; !hasHook { return nil, fmt.Errorf("missing migration 'query' in %s", file) } } migrations = append(migrations, m) } return migrations, nil } // splitSQLStatements splits a SQL query into individual statements. // It splits on semicolons that are not inside -- line comments or 'string literals'. func splitSQLStatements(query string) []string { var statements []string var current strings.Builder inLineComment := false inString := false for i := 0; i < len(query); i++ { ch := query[i] if inLineComment { current.WriteByte(ch) if ch == '\n' { inLineComment = false } continue } if inString { current.WriteByte(ch) if ch == '\'' { // Check for escaped quote ('') if i+1 < len(query) && query[i+1] == '\'' { current.WriteByte(query[i+1]) i++ } else { inString = false } } continue } switch { case ch == '-' && i+1 < len(query) && query[i+1] == '-': inLineComment = true current.WriteByte(ch) case ch == '\'': inString = true current.WriteByte(ch) case ch == ';': // Statement boundary — flush if non-empty stmt := strings.TrimSpace(current.String()) if stmt != "" { statements = append(statements, stmt) } current.Reset() default: current.WriteByte(ch) } } // Flush trailing statement if stmt := strings.TrimSpace(current.String()); stmt != "" { statements = append(statements, stmt) } // Filter out comment-only statements filtered := statements[:0] for _, stmt := range statements { hasCode := false for line := range strings.SplitSeq(stmt, "\n") { trimmed := strings.TrimSpace(line) if trimmed != "" && !strings.HasPrefix(trimmed, "--") { hasCode = true break } } if hasCode { filtered = append(filtered, stmt) } } return filtered } // parseMigrationFilename extracts version and name from migration filename // Expected format: 0001_migration_name.yaml // Returns: version (int), name (string), error // Note: Glob pattern ensures format is valid, so minimal validation needed func parseMigrationFilename(filename string) (int, string, error) { // Remove extension (.yaml or .yml) ext := filepath.Ext(filename) fileNameWithoutExt := filename[:len(filename)-len(ext)] // First 4 characters are the version (glob guarantees they're digits) version, _ := strconv.Atoi(fileNameWithoutExt[:4]) // Remainder after position 5 is the name (glob guarantees it exists) name := strings.ReplaceAll(fileNameWithoutExt[5:], "_", " ") name = strings.TrimSpace(name) return version, name, nil } // openLibsqlLocalConnector returns a driver.Connector for a local libsql DSN. // go-libsql exports NewEmbeddedReplicaConnector for replica mode but no public // constructor for local files, so we obtain the driver via a probe sql.Open // (which is lazy and opens no connection) and ask it for a Connector. func openLibsqlLocalConnector(dsn string) (driver.Connector, error) { probe, err := sql.Open("libsql", dsn) if err != nil { return nil, fmt.Errorf("probe libsql driver: %w", err) } drv := probe.Driver() _ = probe.Close() dctx, ok := drv.(driver.DriverContext) if !ok { return nil, fmt.Errorf("libsql driver does not implement driver.DriverContext") } return dctx.OpenConnector(dsn) } // busyTimeoutConnector wraps a driver.Connector and runs PRAGMA busy_timeout // on every newly opened connection. SQLite's busy_timeout is per-connection, // so this is the only way to ensure every conn in the pool waits on lock // contention instead of returning SQLITE_BUSY immediately. type busyTimeoutConnector struct { base driver.Connector timeoutMs int } func (c *busyTimeoutConnector) Connect(ctx context.Context) (driver.Conn, error) { conn, err := c.base.Connect(ctx) if err != nil { return nil, err } // libsql treats PRAGMA assignments as queries that return a row, so we // must use QueryerContext rather than ExecerContext. queryer, ok := conn.(driver.QueryerContext) if !ok { _ = conn.Close() return nil, fmt.Errorf("libsql conn does not support QueryerContext") } rows, err := queryer.QueryContext(ctx, fmt.Sprintf("PRAGMA busy_timeout = %d", c.timeoutMs), nil) if err != nil { _ = conn.Close() return nil, fmt.Errorf("set busy_timeout on new conn: %w", err) } _ = rows.Close() return conn, nil } func (c *busyTimeoutConnector) Driver() driver.Driver { return c.base.Driver() }