package db import ( "database/sql" "fmt" "regexp" "slices" "sort" "strings" ) // schemaSnapshot is a structural description of a database: the column set of // every user table, plus the explicitly-created indexes. // // Columns are held as sorted descriptor strings rather than as ordered lists, // because ordinal position legitimately differs between a fresh install and an // upgraded one. Migrations append with ALTER TABLE ADD COLUMN; schema.sql places // the same column mid-table. Comparing as a set is what makes the two // comparable at all. type schemaSnapshot struct { Tables map[string][]string Indexes []string } var ( driftWhitespaceRun = regexp.MustCompile(`\s+`) // Space that sits immediately before "(", ")" or "," carries no meaning in // index DDL, and space immediately after "(" or "," carries none either. // Space *after* ")" is deliberately left alone, since removing it would run // a trailing clause into the column list. driftSpaceBeforePunct = regexp.MustCompile(` ([(),])`) driftSpaceAfterPunct = regexp.MustCompile(`([(,]) `) ) // normalizeIndexDDL reduces an index definition to a form that can be compared // by string equality regardless of who emitted the text. // // This matters because the two sides of the comparison do not come from the same // engine. referenceSnapshot applies schema.sql to a local libsql database, which // stores the CREATE statement verbatim; a production database is an embedded // replica whose writes go to the remote first, and the remote re-emits the DDL // through its own parser. The two spellings differ only in spacing: // // schema.sql: CREATE INDEX idx_webhooks_user ON webhooks(user_did) // replica: CREATE INDEX idx_webhooks_user ON webhooks (user_did) // // Collapsing whitespace *runs* cannot fix that, because the difference is a // space that exists on one side and not the other. Left unhandled, every index // was reported twice on every boot (missing, and unexpected), burying any real // drift under the noise. // // Only the spacing is touched; identifiers, ordering (DESC) and uniqueness all // still have to match exactly. func normalizeIndexDDL(ddl string) string { norm := driftWhitespaceRun.ReplaceAllString(strings.TrimSpace(ddl), " ") norm = strings.ReplaceAll(norm, "IF NOT EXISTS ", "") norm = driftSpaceBeforePunct.ReplaceAllString(norm, "$1") norm = driftSpaceAfterPunct.ReplaceAllString(norm, "$1") return norm } // introspectSchema reads the structure of db. // // SQLite internal tables (sqlite_sequence and friends, created implicitly by // AUTOINCREMENT) are skipped: they are an artifact of the engine rather than // something either schema.sql or a migration declares. func introspectSchema(db DBTX) (schemaSnapshot, error) { snap := schemaSnapshot{Tables: make(map[string][]string)} tables, err := listUserTables(db) if err != nil { return snap, err } for _, table := range tables { cols, err := describeColumns(db, table) if err != nil { return snap, err } snap.Tables[table] = cols } snap.Indexes, err = describeIndexes(db) if err != nil { return snap, err } return snap, nil } func listUserTables(db DBTX) ([]string, error) { rows, err := db.Query(` SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name `) if err != nil { return nil, fmt.Errorf("list tables: %w", err) } defer rows.Close() var names []string for rows.Next() { var n string if err := rows.Scan(&n); err != nil { return nil, fmt.Errorf("scan table name: %w", err) } names = append(names, n) } return names, rows.Err() } // describeColumns returns one sorted descriptor per column: name, declared // type, not-null, default and primary-key position. Ordinal position is // deliberately excluded; see schemaSnapshot. func describeColumns(db DBTX, table string) ([]string, error) { rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%q)", table)) if err != nil { return nil, fmt.Errorf("table_info(%s): %w", table, err) } defer rows.Close() var cols []string for rows.Next() { var ( cid int name string colType string notNull int dfltValue sql.NullString pk int ) if err := rows.Scan(&cid, &name, &colType, ¬Null, &dfltValue, &pk); err != nil { return nil, fmt.Errorf("scan table_info(%s): %w", table, err) } dflt := "" if dfltValue.Valid { dflt = dfltValue.String } cols = append(cols, fmt.Sprintf("%s type=%s notnull=%d default=%s pk=%d", name, strings.ToUpper(colType), notNull, dflt, pk)) } if err := rows.Err(); err != nil { return nil, err } sort.Strings(cols) return cols, nil } // describeIndexes returns normalized DDL for the explicitly-created indexes. // Indexes SQLite creates implicitly for UNIQUE and PRIMARY KEY constraints have // a NULL sql column and are skipped, since the column set already implies them. func describeIndexes(db DBTX) ([]string, error) { rows, err := db.Query(` SELECT sql FROM sqlite_master WHERE type = 'index' AND sql IS NOT NULL `) if err != nil { return nil, fmt.Errorf("list indexes: %w", err) } defer rows.Close() var out []string for rows.Next() { var ddl string if err := rows.Scan(&ddl); err != nil { return nil, fmt.Errorf("scan index sql: %w", err) } out = append(out, normalizeIndexDDL(ddl)) } if err := rows.Err(); err != nil { return nil, err } sort.Strings(out) return out, nil } // referenceSnapshot builds the structure schema.sql describes, by applying it to // a throwaway in-memory database and introspecting that. // // Introspecting a real database beats parsing the DDL: SQLite's own resolution // of types, defaults and implicit indexes is the thing we want to compare // against, and reimplementing it in a parser would drift from the engine. func referenceSnapshot() (schemaSnapshot, error) { connector, err := openLibsqlLocalConnector(":memory:") if err != nil { return schemaSnapshot{}, fmt.Errorf("open reference connector: %w", err) } ref := sql.OpenDB(connector) // One connection, so every statement lands in the same in-memory database. ref.SetMaxOpenConns(1) defer ref.Close() for i, stmt := range splitSQLStatements(schemaSQL) { if _, err := ref.Exec(stmt); err != nil { return schemaSnapshot{}, fmt.Errorf("apply schema.sql statement %d: %w", i+1, err) } } return introspectSchema(ref) } // SchemaDrift compares a live database against what schema.sql describes and // returns one human-readable line per difference. An empty result means they // agree. // // This exists because migrations can be recorded without being executed. That is // not hypothetical: migration 0009 had to clean up after 0004, which production // recorded but never applied, leaving eleven columns behind that fresh installs // did not have. Nothing reported that at the time; it surfaced later as // confusing behavior. A check at boot turns that class of problem into a log // line at deploy time. // // Callers should warn on the result and continue. Drift is a signal to write a // corrective migration, not a reason to refuse to serve traffic: a database that // is merely ahead of or behind schema.sql is usually still perfectly able to run // the application, and failing the boot would turn a cosmetic diff into an // outage. func SchemaDrift(live DBTX) ([]string, error) { want, err := referenceSnapshot() if err != nil { return nil, err } got, err := introspectSchema(live) if err != nil { return nil, fmt.Errorf("introspect live database: %w", err) } var findings []string wantTables := sortedKeys(want.Tables) gotTables := sortedKeys(got.Tables) for _, table := range wantTables { if !slices.Contains(gotTables, table) { findings = append(findings, fmt.Sprintf( "table %q is in schema.sql but missing from the database (a migration to create it is probably missing)", table)) continue } for _, col := range want.Tables[table] { if !slices.Contains(got.Tables[table], col) { findings = append(findings, fmt.Sprintf( "table %q: schema.sql expects column [%s] but the database does not have it", table, col)) } } for _, col := range got.Tables[table] { if !slices.Contains(want.Tables[table], col) { findings = append(findings, fmt.Sprintf( "table %q: database has column [%s] which schema.sql does not declare", table, col)) } } } for _, table := range gotTables { if !slices.Contains(wantTables, table) { findings = append(findings, fmt.Sprintf( "table %q exists in the database but is not declared in schema.sql", table)) } } for _, idx := range want.Indexes { if !slices.Contains(got.Indexes, idx) { findings = append(findings, fmt.Sprintf("index missing from the database: %s", idx)) } } for _, idx := range got.Indexes { if !slices.Contains(want.Indexes, idx) { findings = append(findings, fmt.Sprintf("index in the database but not in schema.sql: %s", idx)) } } return findings, nil } func sortedKeys(m map[string][]string) []string { out := make([]string, 0, len(m)) for k := range m { out = append(out, k) } sort.Strings(out) return out }