// db-migrate copies all tables and data from a local SQLite database to a // remote libsql database (e.g. Bunny Database, Turso). It reads the schema // from sqlite_master, creates tables on the remote, and inserts all rows // in batches. Generic — works with any SQLite DB (appview, hold, etc.). // // Usage: // // go run ./cmd/db-migrate --local /path/to/local.db --remote "libsql://..." --token "..." // go run ./cmd/db-migrate --local /path/to/local.db --remote "libsql://..." --token "..." --skip-existing package main import ( "database/sql" "flag" "fmt" "log" "os" "strings" "time" _ "github.com/tursodatabase/go-libsql" ) func main() { localPath := flag.String("local", "", "Path to local SQLite database file") remoteURL := flag.String("remote", "", "Remote libsql URL (libsql://...)") authToken := flag.String("token", "", "Auth token for remote database") skipExisting := flag.Bool("skip-existing", false, "Skip tables that already have data on remote") batchSize := flag.Int("batch-size", 100, "Number of rows per INSERT batch") dryRun := flag.Bool("dry-run", false, "Show what would be migrated without writing") flag.Parse() if *localPath == "" || *remoteURL == "" || *authToken == "" { flag.Usage() os.Exit(1) } // Open local database read-only localDSN := *localPath if !strings.HasPrefix(localDSN, "file:") { localDSN = "file:" + localDSN } localDSN += "?mode=ro" localDB, err := sql.Open("libsql", localDSN) if err != nil { log.Fatalf("Failed to open local database: %v", err) } defer localDB.Close() if err := localDB.Ping(); err != nil { log.Fatalf("Failed to ping local database: %v", err) } // Open remote database remoteDSN := fmt.Sprintf("%s?authToken=%s", *remoteURL, *authToken) remoteDB, err := sql.Open("libsql", remoteDSN) if err != nil { log.Fatalf("Failed to open remote database: %v", err) } defer remoteDB.Close() if err := remoteDB.Ping(); err != nil { log.Fatalf("Failed to ping remote database: %v", err) } // Get all user tables from local tables, err := getTables(localDB) if err != nil { log.Fatalf("Failed to list tables: %v", err) } if len(tables) == 0 { log.Println("No tables found in local database") return } fmt.Printf("Found %d tables to migrate\n\n", len(tables)) start := time.Now() if !*dryRun { // Phase 1: Create all tables first so FK references resolve fmt.Println("Creating tables...") for _, t := range tables { if err := createTable(remoteDB, t); err != nil { log.Fatalf("Failed to create table %s: %v", t.name, err) } } fmt.Println() } // Phase 2: Copy data fmt.Println("Migrating data...") totalRows := 0 for _, t := range tables { count, err := migrateTable(localDB, remoteDB, t, *batchSize, *skipExisting, *dryRun) if err != nil { log.Fatalf("Failed to migrate table %s: %v", t.name, err) } totalRows += count } if !*dryRun { // Phase 3: Create indexes after data is loaded (faster than indexing during insert) fmt.Println("\nCreating indexes...") for _, t := range tables { if err := createIndexes(localDB, remoteDB, t.name); err != nil { log.Fatalf("Failed to create indexes for %s: %v", t.name, err) } } } fmt.Printf("\nDone. %d total rows across %d tables in %s\n", totalRows, len(tables), time.Since(start).Round(time.Millisecond)) if *dryRun { fmt.Println("(dry run — nothing was written)") } } type tableInfo struct { name string ddl string } func getTables(db *sql.DB) ([]tableInfo, error) { rows, err := db.Query(` SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' AND name NOT LIKE '_litestream_%' AND name NOT LIKE 'libsql_%' ORDER BY name `) if err != nil { return nil, err } defer rows.Close() var tables []tableInfo for rows.Next() { var t tableInfo var ddl sql.NullString if err := rows.Scan(&t.name, &ddl); err != nil { return nil, err } if ddl.Valid { t.ddl = ddl.String } tables = append(tables, t) } if err := rows.Err(); err != nil { return nil, err } // Sort tables so those referenced by foreign keys come first. // Tables with FK references depend on other tables existing and // having data, so we insert referenced tables first. return topoSortTables(db, tables) } // topoSortTables orders tables so that referenced (parent) tables come before // tables that reference them via foreign keys. func topoSortTables(db *sql.DB, tables []tableInfo) ([]tableInfo, error) { byName := make(map[string]tableInfo, len(tables)) for _, t := range tables { byName[t.name] = t } // Build dependency graph: table -> tables it references deps := make(map[string][]string) for _, t := range tables { fkRows, err := db.Query(fmt.Sprintf("PRAGMA foreign_key_list([%s])", t.name)) if err != nil { // PRAGMA might not return rows for tables without FKs continue } seen := make(map[string]bool) for fkRows.Next() { var id, seq int var table, from, to, onUpdate, onDelete, match string if err := fkRows.Scan(&id, &seq, &table, &from, &to, &onUpdate, &onDelete, &match); err != nil { fkRows.Close() return nil, err } if !seen[table] { deps[t.name] = append(deps[t.name], table) seen[table] = true } } fkRows.Close() } // Topological sort (Kahn's algorithm) visited := make(map[string]bool) var sorted []tableInfo var visit func(name string) visit = func(name string) { if visited[name] { return } visited[name] = true for _, dep := range deps[name] { visit(dep) } if t, ok := byName[name]; ok { sorted = append(sorted, t) } } for _, t := range tables { visit(t.name) } return sorted, nil } func getIndexes(db *sql.DB, tableName string) ([]string, error) { rows, err := db.Query(` SELECT sql FROM sqlite_master WHERE type = 'index' AND tbl_name = ? AND sql IS NOT NULL `, tableName) if err != nil { return nil, err } defer rows.Close() var indexes []string for rows.Next() { var ddl string if err := rows.Scan(&ddl); err != nil { return nil, err } indexes = append(indexes, ddl) } return indexes, rows.Err() } func createTable(remoteDB *sql.DB, t tableInfo) error { if t.ddl == "" { return nil } ddl := t.ddl if !strings.Contains(strings.ToUpper(ddl), "IF NOT EXISTS") { ddl = strings.Replace(ddl, "CREATE TABLE", "CREATE TABLE IF NOT EXISTS", 1) } if _, err := remoteDB.Exec(ddl); err != nil { return fmt.Errorf("create table %s: %w", t.name, err) } fmt.Printf(" %s\n", t.name) return nil } func createIndexes(localDB, remoteDB *sql.DB, tableName string) error { indexes, err := getIndexes(localDB, tableName) if err != nil { return err } for _, idx := range indexes { ddl := idx if !strings.Contains(strings.ToUpper(ddl), "IF NOT EXISTS") { ddl = strings.Replace(ddl, "CREATE INDEX", "CREATE INDEX IF NOT EXISTS", 1) ddl = strings.Replace(ddl, "CREATE UNIQUE INDEX", "CREATE UNIQUE INDEX IF NOT EXISTS", 1) } if _, err := remoteDB.Exec(ddl); err != nil { return fmt.Errorf("create index on %s: %w", tableName, err) } } if len(indexes) > 0 { fmt.Printf(" %s: %d indexes\n", tableName, len(indexes)) } return nil } func migrateTable(localDB, remoteDB *sql.DB, t tableInfo, batchSize int, skipExisting, dryRun bool) (int, error) { var localCount int if err := localDB.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM [%s]", t.name)).Scan(&localCount); err != nil { return 0, fmt.Errorf("count local rows: %w", err) } if localCount == 0 { fmt.Printf(" %-30s %6d rows (empty)\n", t.name, 0) return 0, nil } if dryRun { fmt.Printf(" %-30s %6d rows (would migrate)\n", t.name, localCount) return localCount, nil } if skipExisting { var remoteCount int if err := remoteDB.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM [%s]", t.name)).Scan(&remoteCount); err != nil { return 0, fmt.Errorf("count remote rows: %w", err) } if remoteCount > 0 { fmt.Printf(" %-30s %6d rows (skipped, %d on remote)\n", t.name, localCount, remoteCount) return 0, nil } } rows, err := localDB.Query(fmt.Sprintf("SELECT * FROM [%s]", t.name)) if err != nil { return 0, fmt.Errorf("select: %w", err) } defer rows.Close() cols, err := rows.Columns() if err != nil { return 0, fmt.Errorf("columns: %w", err) } placeholders := make([]string, len(cols)) quotedCols := make([]string, len(cols)) for i, c := range cols { placeholders[i] = "?" quotedCols[i] = fmt.Sprintf("[%s]", c) } insertPrefix := fmt.Sprintf("INSERT INTO [%s] (%s) VALUES ", t.name, strings.Join(quotedCols, ", ")) rowPlaceholder := "(" + strings.Join(placeholders, ", ") + ")" inserted := 0 batch := make([][]any, 0, batchSize) for rows.Next() { vals := make([]any, len(cols)) ptrs := make([]any, len(cols)) for i := range vals { ptrs[i] = &vals[i] } if err := rows.Scan(ptrs...); err != nil { return 0, fmt.Errorf("scan: %w", err) } batch = append(batch, vals) if len(batch) >= batchSize { if err := insertBatch(remoteDB, insertPrefix, rowPlaceholder, batch); err != nil { return 0, fmt.Errorf("insert batch at row %d: %w", inserted, err) } inserted += len(batch) batch = batch[:0] } } if len(batch) > 0 { if err := insertBatch(remoteDB, insertPrefix, rowPlaceholder, batch); err != nil { return 0, fmt.Errorf("insert final batch: %w", err) } inserted += len(batch) } if err := rows.Err(); err != nil { return 0, fmt.Errorf("rows iteration: %w", err) } fmt.Printf(" %-30s %6d rows migrated\n", t.name, inserted) return inserted, nil } func insertBatch(db *sql.DB, prefix, rowPlaceholder string, batch [][]any) error { if len(batch) == 0 { return nil } placeholders := make([]string, len(batch)) var args []any for i, row := range batch { placeholders[i] = rowPlaceholder args = append(args, row...) } query := prefix + strings.Join(placeholders, ", ") _, err := db.Exec(query, args...) return err }