at-container-registry/pkg/appview/db/drift.go

package db

import (
	"database/sql"
	"fmt"
	"regexp"
	"slices"
	"sort"
	"strings"
)

// schemaSnapshot is a structural description of a database: the column set of
// every user table, plus the explicitly-created indexes.
//
// Columns are held as sorted descriptor strings rather than as ordered lists,
// because ordinal position legitimately differs between a fresh install and an
// upgraded one. Migrations append with ALTER TABLE ADD COLUMN; schema.sql places
// the same column mid-table. Comparing as a set is what makes the two
// comparable at all.
type schemaSnapshot struct {
	Tables  map[string][]string
	Indexes []string
}

var (
	driftWhitespaceRun = regexp.MustCompile(`\s+`)
	// Space that sits immediately before "(", ")" or "," carries no meaning in
	// index DDL, and space immediately after "(" or "," carries none either.
	// Space *after* ")" is deliberately left alone, since removing it would run
	// a trailing clause into the column list.
	driftSpaceBeforePunct = regexp.MustCompile(` ([(),])`)
	driftSpaceAfterPunct  = regexp.MustCompile(`([(,]) `)
)

// normalizeIndexDDL reduces an index definition to a form that can be compared
// by string equality regardless of who emitted the text.
//
// This matters because the two sides of the comparison do not come from the same
// engine. referenceSnapshot applies schema.sql to a local libsql database, which
// stores the CREATE statement verbatim; a production database is an embedded
// replica whose writes go to the remote first, and the remote re-emits the DDL
// through its own parser. The two spellings differ only in spacing:
//
//	schema.sql:  CREATE INDEX idx_webhooks_user ON webhooks(user_did)
//	replica:     CREATE INDEX idx_webhooks_user ON webhooks (user_did)
//
// Collapsing whitespace *runs* cannot fix that, because the difference is a
// space that exists on one side and not the other. Left unhandled, every index
// was reported twice on every boot (missing, and unexpected), burying any real
// drift under the noise.
//
// Only the spacing is touched; identifiers, ordering (DESC) and uniqueness all
// still have to match exactly.
func normalizeIndexDDL(ddl string) string {
	norm := driftWhitespaceRun.ReplaceAllString(strings.TrimSpace(ddl), " ")
	norm = strings.ReplaceAll(norm, "IF NOT EXISTS ", "")
	norm = driftSpaceBeforePunct.ReplaceAllString(norm, "$1")
	norm = driftSpaceAfterPunct.ReplaceAllString(norm, "$1")
	return norm
}

// introspectSchema reads the structure of db.
//
// SQLite internal tables (sqlite_sequence and friends, created implicitly by
// AUTOINCREMENT) are skipped: they are an artifact of the engine rather than
// something either schema.sql or a migration declares.
func introspectSchema(db DBTX) (schemaSnapshot, error) {
	snap := schemaSnapshot{Tables: make(map[string][]string)}

	tables, err := listUserTables(db)
	if err != nil {
		return snap, err
	}
	for _, table := range tables {
		cols, err := describeColumns(db, table)
		if err != nil {
			return snap, err
		}
		snap.Tables[table] = cols
	}

	snap.Indexes, err = describeIndexes(db)
	if err != nil {
		return snap, err
	}
	return snap, nil
}

func listUserTables(db DBTX) ([]string, error) {
	rows, err := db.Query(`
		SELECT name FROM sqlite_master
		WHERE type = 'table' AND name NOT LIKE 'sqlite_%'
		ORDER BY name
	`)
	if err != nil {
		return nil, fmt.Errorf("list tables: %w", err)
	}
	defer rows.Close()

	var names []string
	for rows.Next() {
		var n string
		if err := rows.Scan(&n); err != nil {
			return nil, fmt.Errorf("scan table name: %w", err)
		}
		names = append(names, n)
	}
	return names, rows.Err()
}

// describeColumns returns one sorted descriptor per column: name, declared
// type, not-null, default and primary-key position. Ordinal position is
// deliberately excluded; see schemaSnapshot.
func describeColumns(db DBTX, table string) ([]string, error) {
	rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%q)", table))
	if err != nil {
		return nil, fmt.Errorf("table_info(%s): %w", table, err)
	}
	defer rows.Close()

	var cols []string
	for rows.Next() {
		var (
			cid       int
			name      string
			colType   string
			notNull   int
			dfltValue sql.NullString
			pk        int
		)
		if err := rows.Scan(&cid, &name, &colType, &notNull, &dfltValue, &pk); err != nil {
			return nil, fmt.Errorf("scan table_info(%s): %w", table, err)
		}
		dflt := "<none>"
		if dfltValue.Valid {
			dflt = dfltValue.String
		}
		cols = append(cols, fmt.Sprintf("%s type=%s notnull=%d default=%s pk=%d",
			name, strings.ToUpper(colType), notNull, dflt, pk))
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}
	sort.Strings(cols)
	return cols, nil
}

// describeIndexes returns normalized DDL for the explicitly-created indexes.
// Indexes SQLite creates implicitly for UNIQUE and PRIMARY KEY constraints have
// a NULL sql column and are skipped, since the column set already implies them.
func describeIndexes(db DBTX) ([]string, error) {
	rows, err := db.Query(`
		SELECT sql FROM sqlite_master
		WHERE type = 'index' AND sql IS NOT NULL
	`)
	if err != nil {
		return nil, fmt.Errorf("list indexes: %w", err)
	}
	defer rows.Close()

	var out []string
	for rows.Next() {
		var ddl string
		if err := rows.Scan(&ddl); err != nil {
			return nil, fmt.Errorf("scan index sql: %w", err)
		}
		out = append(out, normalizeIndexDDL(ddl))
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}
	sort.Strings(out)
	return out, nil
}

// referenceSnapshot builds the structure schema.sql describes, by applying it to
// a throwaway in-memory database and introspecting that.
//
// Introspecting a real database beats parsing the DDL: SQLite's own resolution
// of types, defaults and implicit indexes is the thing we want to compare
// against, and reimplementing it in a parser would drift from the engine.
func referenceSnapshot() (schemaSnapshot, error) {
	connector, err := openLibsqlLocalConnector(":memory:")
	if err != nil {
		return schemaSnapshot{}, fmt.Errorf("open reference connector: %w", err)
	}
	ref := sql.OpenDB(connector)
	// One connection, so every statement lands in the same in-memory database.
	ref.SetMaxOpenConns(1)
	defer ref.Close()

	for i, stmt := range splitSQLStatements(schemaSQL) {
		if _, err := ref.Exec(stmt); err != nil {
			return schemaSnapshot{}, fmt.Errorf("apply schema.sql statement %d: %w", i+1, err)
		}
	}
	return introspectSchema(ref)
}

// SchemaDrift compares a live database against what schema.sql describes and
// returns one human-readable line per difference. An empty result means they
// agree.
//
// This exists because migrations can be recorded without being executed. That is
// not hypothetical: migration 0009 had to clean up after 0004, which production
// recorded but never applied, leaving eleven columns behind that fresh installs
// did not have. Nothing reported that at the time; it surfaced later as
// confusing behavior. A check at boot turns that class of problem into a log
// line at deploy time.
//
// Callers should warn on the result and continue. Drift is a signal to write a
// corrective migration, not a reason to refuse to serve traffic: a database that
// is merely ahead of or behind schema.sql is usually still perfectly able to run
// the application, and failing the boot would turn a cosmetic diff into an
// outage.
func SchemaDrift(live DBTX) ([]string, error) {
	want, err := referenceSnapshot()
	if err != nil {
		return nil, err
	}
	got, err := introspectSchema(live)
	if err != nil {
		return nil, fmt.Errorf("introspect live database: %w", err)
	}

	var findings []string

	wantTables := sortedKeys(want.Tables)
	gotTables := sortedKeys(got.Tables)

	for _, table := range wantTables {
		if !slices.Contains(gotTables, table) {
			findings = append(findings, fmt.Sprintf(
				"table %q is in schema.sql but missing from the database (a migration to create it is probably missing)", table))
			continue
		}
		for _, col := range want.Tables[table] {
			if !slices.Contains(got.Tables[table], col) {
				findings = append(findings, fmt.Sprintf(
					"table %q: schema.sql expects column [%s] but the database does not have it", table, col))
			}
		}
		for _, col := range got.Tables[table] {
			if !slices.Contains(want.Tables[table], col) {
				findings = append(findings, fmt.Sprintf(
					"table %q: database has column [%s] which schema.sql does not declare", table, col))
			}
		}
	}

	for _, table := range gotTables {
		if !slices.Contains(wantTables, table) {
			findings = append(findings, fmt.Sprintf(
				"table %q exists in the database but is not declared in schema.sql", table))
		}
	}

	for _, idx := range want.Indexes {
		if !slices.Contains(got.Indexes, idx) {
			findings = append(findings, fmt.Sprintf("index missing from the database: %s", idx))
		}
	}
	for _, idx := range got.Indexes {
		if !slices.Contains(want.Indexes, idx) {
			findings = append(findings, fmt.Sprintf("index in the database but not in schema.sql: %s", idx))
		}
	}

	return findings, nil
}

func sortedKeys(m map[string][]string) []string {
	out := make([]string, 0, len(m))
	for k := range m {
		out = append(out, k)
	}
	sort.Strings(out)
	return out
}