package db import ( "database/sql" "fmt" "log/slog" "os" "path/filepath" "slices" "strings" "testing" ) // TestSchemaMatchesMigrations enforces the invariant that schema.sql and the // migrations describe the same database. // // The two are supposed to be updated in lockstep, and nothing checked that they // were. The cost of them drifting is already on the record: migration 0009 // exists only because "Migration 0004 was supposed to drop these columns but // either failed or was only recorded (not executed) on production", which left // production carrying eleven columns that fresh installs did not have. // // The check builds the schema both ways and compares them: // // A: apply schema.sql (what a fresh install gets) // B: apply testdata/base_schema.sql, then // run every migration (what an upgrade gets) // // Columns are compared as a SET, ignoring ordinal position. That is deliberate. // Migrations append with ALTER TABLE ADD COLUMN while schema.sql places the same // column mid-table, so the two orders legitimately differ on manifests, users, // devices and repo_pages. Reordering those tables would mean four rebuild // migrations for no functional gain, and nothing in pkg/appview depends on // ordinal position (no SELECT *, no column-less INSERT ... VALUES). If you add // code that does depend on column order, this test will not save you. func TestSchemaMatchesMigrations(t *testing.T) { fresh := freshSchemaDB(t) defer fresh.Close() migrated := migratedSchemaDB(t) defer migrated.Close() freshSnap, err := introspectSchema(fresh) if err != nil { t.Fatalf("introspect fresh database: %v", err) } migratedSnap, err := introspectSchema(migrated) if err != nil { t.Fatalf("introspect migrated database: %v", err) } freshTables := sortedKeys(freshSnap.Tables) migratedTables := sortedKeys(migratedSnap.Tables) if diff := diffStringSets(freshTables, migratedTables); diff != "" { t.Errorf("table sets differ between schema.sql and the migrations:\n%s\n\n"+ "A table only in schema.sql will never exist on an upgraded database, because InitDB\n"+ "skips schema.sql entirely once schema_migrations has rows. New tables need BOTH a\n"+ "schema.sql entry and a migration.", diff) } for _, table := range freshTables { if !slices.Contains(migratedTables, table) { continue // already reported above } if diff := diffStringSets(freshSnap.Tables[table], migratedSnap.Tables[table]); diff != "" { t.Errorf("table %q differs between schema.sql and the migrations:\n%s", table, diff) } } if diff := diffStringSets(freshSnap.Indexes, migratedSnap.Indexes); diff != "" { t.Errorf("index sets differ between schema.sql and the migrations:\n%s", diff) } } // TestSchemaDriftCleanOnFreshDatabase pins the other end of the same invariant: // SchemaDrift, which runs at boot against real databases, must report nothing // for a database that was just built from schema.sql. If this ever fails, the // startup check has become noisy and operators will learn to ignore it, which // costs more than not having it at all. func TestSchemaDriftCleanOnFreshDatabase(t *testing.T) { db := freshSchemaDB(t) defer db.Close() findings, err := SchemaDrift(db) if err != nil { t.Fatalf("SchemaDrift: %v", err) } if len(findings) != 0 { t.Errorf("expected no drift against a database built from schema.sql, got %d:\n %s", len(findings), strings.Join(findings, "\n ")) } } // TestSchemaDriftDetectsMissingColumn proves SchemaDrift actually compares // rather than always returning clean. A database missing a column schema.sql // declares is exactly the shape of the 0004 incident. func TestSchemaDriftDetectsMissingColumn(t *testing.T) { db := freshSchemaDB(t) defer db.Close() // Rebuild users without registry_domain, imitating a migration that was // recorded but never executed. stmts := []string{ `CREATE TABLE users_drifted ( did TEXT PRIMARY KEY, handle TEXT NOT NULL, pds_endpoint TEXT NOT NULL, avatar TEXT, default_hold_did TEXT, oci_client TEXT DEFAULT '', last_seen TIMESTAMP NOT NULL, UNIQUE(handle) )`, `DROP TABLE users`, `ALTER TABLE users_drifted RENAME TO users`, } for _, s := range stmts { if _, err := db.Exec(s); err != nil { t.Fatalf("set up drifted schema: %v", err) } } findings, err := SchemaDrift(db) if err != nil { t.Fatalf("SchemaDrift: %v", err) } if !anyContains(findings, "registry_domain") { t.Errorf("expected a finding naming the missing registry_domain column, got: %v", findings) } } // TestSchemaDriftIgnoresIndexSpacing covers the case that made the boot-time // check useless in production. // // referenceSnapshot applies schema.sql to a LOCAL libsql database, which stores // the CREATE statement byte for byte. Production runs an embedded replica: the // write goes to the remote first and the remote re-emits the DDL through its own // parser, which puts a space before the column list. Every index then failed the // equality check from both directions at once, so all 38 were reported twice and // the warning read "differences=76" on every single boot. Real drift would have // been one line in seventy-seven. // // Nothing caught it because every test runs local-only, where both sides are // produced by the same writer and therefore agree by accident. This test injects // the remote's spelling by hand so the discrimination is actually exercised. func TestSchemaDriftIgnoresIndexSpacing(t *testing.T) { db := freshSchemaDB(t) defer db.Close() // Each pair is (index to drop, spaced respelling of the identical index). // The spellings on the right are what a remote-parsed database stores. respell := []struct{ drop, create string }{ {"idx_webhooks_user", `CREATE INDEX idx_webhooks_user ON webhooks (user_did)`}, {"idx_manifests_did_repo", `CREATE INDEX idx_manifests_did_repo ON manifests (did, repository)`}, {"idx_manifests_created_at", `CREATE INDEX idx_manifests_created_at ON manifests ( created_at DESC )`}, {"idx_instance_leases_expires", `CREATE INDEX idx_instance_leases_expires ON instance_leases (expires_at)`}, } for _, r := range respell { if _, err := db.Exec("DROP INDEX " + r.drop); err != nil { t.Fatalf("drop %s: %v", r.drop, err) } if _, err := db.Exec(r.create); err != nil { t.Fatalf("recreate %s: %v", r.drop, err) } } findings, err := SchemaDrift(db) if err != nil { t.Fatalf("SchemaDrift: %v", err) } if len(findings) != 0 { t.Errorf("index spacing is not a schema difference, but SchemaDrift reported %d:\n %s", len(findings), strings.Join(findings, "\n ")) } } // TestSchemaDriftDetectsChangedIndex is the other half of the pair above. // Normalizing spacing must not blunt the check: an index that genuinely differs // in its columns, its table or its uniqueness still has to be reported, in the // spaced spelling as much as the unspaced one. func TestSchemaDriftDetectsChangedIndex(t *testing.T) { cases := []struct { name string drop string create string wantHit string }{ { name: "different column", drop: "idx_webhooks_user", create: `CREATE INDEX idx_webhooks_user ON webhooks (url)`, wantHit: "idx_webhooks_user", }, { name: "extra column", drop: "idx_manifests_did_repo", create: `CREATE INDEX idx_manifests_did_repo ON manifests (did, repository, digest)`, wantHit: "idx_manifests_did_repo", }, { name: "uniqueness changed", drop: "idx_stars_starrer", create: `CREATE UNIQUE INDEX idx_stars_starrer ON stars (starrer_did)`, wantHit: "idx_stars_starrer", }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { db := freshSchemaDB(t) defer db.Close() if _, err := db.Exec("DROP INDEX " + tc.drop); err != nil { t.Fatalf("drop %s: %v", tc.drop, err) } if _, err := db.Exec(tc.create); err != nil { t.Fatalf("create replacement index: %v", err) } findings, err := SchemaDrift(db) if err != nil { t.Fatalf("SchemaDrift: %v", err) } if !anyContains(findings, tc.wantHit) { t.Errorf("expected a finding naming %s, got: %v", tc.wantHit, findings) } }) } } // TestNormalizeIndexDDL pins the normalization itself, without a database in the // way: spacing around punctuation is noise, everything else is signal. func TestNormalizeIndexDDL(t *testing.T) { same := [][]string{ { `CREATE INDEX idx_webhooks_user ON webhooks(user_did)`, `CREATE INDEX idx_webhooks_user ON webhooks (user_did)`, `CREATE INDEX IF NOT EXISTS idx_webhooks_user ON webhooks ( user_did )`, "CREATE INDEX idx_webhooks_user\n ON webhooks\n (user_did)", }, { `CREATE INDEX i ON t(a, b)`, `CREATE INDEX i ON t(a,b)`, `CREATE INDEX i ON t ( a , b )`, }, { `CREATE INDEX i ON t(created_at DESC)`, `CREATE INDEX i ON t (created_at DESC)`, }, } for _, group := range same { want := normalizeIndexDDL(group[0]) for _, variant := range group[1:] { if got := normalizeIndexDDL(variant); got != want { t.Errorf("spelling difference survived normalization:\n %q -> %q\n %q -> %q", group[0], want, variant, got) } } } different := [][2]string{ {`CREATE INDEX i ON t(a)`, `CREATE INDEX i ON t(b)`}, {`CREATE INDEX i ON t(a)`, `CREATE INDEX i ON u(a)`}, {`CREATE INDEX i ON t(a)`, `CREATE UNIQUE INDEX i ON t(a)`}, {`CREATE INDEX i ON t(a, b)`, `CREATE INDEX i ON t(b, a)`}, {`CREATE INDEX i ON t(a)`, `CREATE INDEX i ON t(a DESC)`}, } for _, pair := range different { if normalizeIndexDDL(pair[0]) == normalizeIndexDDL(pair[1]) { t.Errorf("normalization collapsed two genuinely different indexes: %q and %q", pair[0], pair[1]) } } } // TestInitDBWarnsOnDriftAndStillBoots covers the integration rather than the // comparison: an existing database whose schema has drifted must still open. // // The check is warn-only on purpose. A database that is merely ahead of or // behind schema.sql is almost always still able to serve traffic, so refusing to // boot would convert a diff that wants a corrective migration into an outage, // and it would do so during a deploy, which is the worst possible moment. func TestInitDBWarnsOnDriftAndStillBoots(t *testing.T) { path := filepath.Join(t.TempDir(), "app.db") first, err := InitDB(path, LibsqlConfig{}) if err != nil { t.Fatalf("first InitDB: %v", err) } // Imitate a migration that was recorded but never executed. if _, err := first.Exec(`ALTER TABLE users DROP COLUMN registry_domain`); err != nil { t.Fatalf("drop column: %v", err) } first.Close() var logged strings.Builder restore := slog.Default() slog.SetDefault(slog.New(slog.NewTextHandler(&logged, &slog.HandlerOptions{Level: slog.LevelWarn}))) defer slog.SetDefault(restore) // The second open takes the existing-database path, which is the only one // that runs the drift check. second, err := InitDB(path, LibsqlConfig{}) if err != nil { t.Fatalf("second InitDB must succeed despite drift, got: %v", err) } defer second.Close() out := logged.String() if !strings.Contains(out, "Schema drift") { t.Errorf("expected a schema drift warning at startup, got log output:\n%s", out) } if !strings.Contains(out, "registry_domain") { t.Errorf("expected the warning to name the missing column, got log output:\n%s", out) } } // freshSchemaDB returns a database built the way a fresh install builds one: // schema.sql applied directly, migrations recorded but not executed. func freshSchemaDB(t *testing.T) *sql.DB { t.Helper() db, err := InitDB(":memory:", LibsqlConfig{}) if err != nil { t.Fatalf("InitDB (fresh install path): %v", err) } return db } // migratedSchemaDB returns a database built the way an upgrade builds one: the // pre-0009 base snapshot, then every migration executed in order. func migratedSchemaDB(t *testing.T) *sql.DB { t.Helper() base, err := os.ReadFile("testdata/base_schema.sql") if err != nil { t.Fatalf("read testdata/base_schema.sql: %v", err) } connector, err := openLibsqlLocalConnector(":memory:") if err != nil { t.Fatalf("open libsql connector: %v", err) } db := sql.OpenDB(connector) // One connection so every statement lands in the same in-memory database. db.SetMaxOpenConns(1) if _, err := db.Exec("PRAGMA foreign_keys = ON"); err != nil { db.Close() t.Fatalf("enable foreign keys: %v", err) } for i, stmt := range splitSQLStatements(string(base)) { if _, err := db.Exec(stmt); err != nil { db.Close() t.Fatalf("apply base_schema.sql statement %d: %v\n%s", i+1, err, stmt) } } // freshDB=false so the DDL actually executes rather than merely being // recorded. base_schema.sql records versions 1-8 as already applied, so // everything from 0009 onward is pending. if err := runMigrations(db, false); err != nil { db.Close() t.Fatalf("runMigrations against the base snapshot: %v", err) } return db } // diffStringSets returns a human-readable diff, or "" when the sets match. // "schema.sql" is the a side, "migrations" the b side. func diffStringSets(a, b []string) string { var sb strings.Builder for _, s := range a { if !slices.Contains(b, s) { fmt.Fprintf(&sb, " only in schema.sql: %s\n", s) } } for _, s := range b { if !slices.Contains(a, s) { fmt.Fprintf(&sb, " only in migrations: %s\n", s) } } return strings.TrimRight(sb.String(), "\n") } func anyContains(haystack []string, needle string) bool { for _, s := range haystack { if strings.Contains(s, needle) { return true } } return false }