at-container-registry/pkg/hold/db/sqlite_store_contention_test.go

package db

import (
	"context"
	"database/sql"
	"fmt"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/bluesky-social/indigo/models"
	blockformat "github.com/ipfs/go-block-format"
	"github.com/ipfs/go-cid"
)

// These tests pin the one property that keeps a hold's writers from failing
// each other: every connection in every pool on a hold database waits for the
// write lock (busy_timeout) instead of failing the statement the moment
// another writer holds it. Before c44a874 only the first connection of a pool
// had the timeout, and production writes failed with "database is locked".
//
// They test the mechanism, not the weather. An earlier version hammered the
// file with concurrent writers for a few seconds and asserted that no lock
// error surfaced; that verdict depended on how fast the CI disk was that day
// and failed on a loaded runner (2026-09-13) with the code correct. Here one
// connection deliberately holds the write lock, a second writer is shown to
// block rather than fail, and it is shown to succeed once the lock is
// released. A regression fails every time, on any machine.

// isLockErr reports whether err is a SQLite busy/locked error rather than a
// logic error, so a typo cannot masquerade as a reproduction.
func isLockErr(err error) bool {
	if err == nil {
		return false
	}
	s := strings.ToLower(err.Error())
	return strings.Contains(s, "database is locked") ||
		strings.Contains(s, "database table is locked") ||
		strings.Contains(s, "sqlite_busy") ||
		strings.Contains(s, "busy")
}

func makeBlocks(t *testing.T, n int, salt string) (cid.Cid, map[cid.Cid]blockformat.Block) {
	t.Helper()
	blks := make(map[cid.Cid]blockformat.Block, n)
	var root cid.Cid
	for i := range n {
		b := blockformat.NewBlock(fmt.Appendf(nil, "%s-block-%d-%s", salt, i, strings.Repeat("x", 2048)))
		blks[b.Cid()] = b
		if i == 0 {
			root = b.Cid()
		}
	}
	return root, blks
}

// holdWriteLock pins one connection from db, opens a write transaction on it
// and leaves it open, so every other writer on the file must wait. The
// returned release commits and hands the connection back.
func holdWriteLock(t *testing.T, db *sql.DB) (release func()) {
	t.Helper()
	ctx := context.Background()
	conn, err := db.Conn(ctx)
	if err != nil {
		t.Fatalf("pin connection: %v", err)
	}
	if _, err := conn.ExecContext(ctx, "CREATE TABLE IF NOT EXISTS lock_probe (k INTEGER PRIMARY KEY, v TEXT)"); err != nil {
		t.Fatalf("create lock_probe: %v", err)
	}
	// BEGIN IMMEDIATE takes the write lock now rather than at the first write,
	// so the lock is held from the moment this returns.
	if _, err := conn.ExecContext(ctx, "BEGIN IMMEDIATE"); err != nil {
		t.Fatalf("begin immediate: %v", err)
	}
	if _, err := conn.ExecContext(ctx, "INSERT INTO lock_probe (v) VALUES ('held')"); err != nil {
		t.Fatalf("insert under lock: %v", err)
	}
	return func() {
		if _, err := conn.ExecContext(ctx, "COMMIT"); err != nil {
			t.Errorf("commit lock holder: %v", err)
		}
		_ = conn.Close()
	}
}

// holdDuration is how long the lock is held before release. The blocked
// writer must take at least most of this, which proves it waited rather than
// racing past an already-released lock, and it must finish well inside
// DefaultBusyTimeoutMs, which proves busy_timeout is what let it through.
const holdDuration = 300 * time.Millisecond

// awaitBlockedWrite runs write while the lock is held, releases the lock after
// holdDuration, and checks that write blocked and then succeeded.
func awaitBlockedWrite(t *testing.T, what string, release func(), write func() error) {
	t.Helper()
	done := make(chan error, 1)
	start := time.Now()
	go func() { done <- write() }()

	select {
	case err := <-done:
		// Finished while the lock was still held: either it failed (the
		// regression) or it did not actually contend for the lock (a broken
		// test), and both are reported.
		if err != nil {
			if isLockErr(err) {
				t.Fatalf("%s failed instead of waiting for the write lock: %v", what, err)
			}
			t.Fatalf("%s failed: %v", what, err)
		}
		t.Fatalf("%s completed in %s while another connection held the write lock; the test is not contending", what, time.Since(start))
	case <-time.After(holdDuration):
	}

	release()
	select {
	case err := <-done:
		if err != nil {
			t.Fatalf("%s failed after the lock was released: %v", what, err)
		}
	case <-time.After(time.Duration(DefaultBusyTimeoutMs) * time.Millisecond):
		t.Fatalf("%s still blocked %dms after the lock was released", what, DefaultBusyTimeoutMs)
	}
	if waited := time.Since(start); waited < holdDuration {
		t.Fatalf("%s took %s, less than the %s the lock was held", what, waited, holdDuration)
	}
}

func carstoreWrite(sqs *SQLiteStore, root cid.Cid, blks map[cid.Cid]blockformat.Block) func() error {
	return func() error {
		_, err := sqs.writeNewShard(context.Background(), root, "rev-001", models.Uid(1), 1, blks, nil)
		return err
	}
}

// TestCarstoreWriteWaitsForLockSharedPool is the production topology: one
// *sql.DB pool (OpenHoldDB) shared by the carstore, records index, events and
// scan broadcaster. database/sql hands a different connection to each
// concurrent caller, so the carstore's write runs on a connection that is not
// the one holding the lock and must have its own busy_timeout.
func TestCarstoreWriteWaitsForLockSharedPool(t *testing.T) {
	path := filepath.Join(t.TempDir(), "db.sqlite3")
	h, err := OpenHoldDB(path, LibsqlConfig{})
	if err != nil {
		t.Fatalf("OpenHoldDB: %v", err)
	}
	defer h.Close()

	sqs, err := NewSQLiteStoreWithDB(path, h.DB)
	if err != nil {
		t.Fatalf("NewSQLiteStoreWithDB: %v", err)
	}
	root, blks := makeBlocks(t, 12, "shared")

	release := holdWriteLock(t, h.DB)
	awaitBlockedWrite(t, "shared-pool carstore write", release, carstoreWrite(sqs, root, blks))
}

// TestCarstoreWriteWaitsForLockTwoOpeners covers a second subsystem opening
// <dir>/db.sqlite3 again through its own pool, in both directions: the
// carstore waiting on the other pool's lock, and the other pool waiting on the
// carstore's.
func TestCarstoreWriteWaitsForLockTwoOpeners(t *testing.T) {
	dir := t.TempDir()
	sqs, err := NewSqliteStore(dir)
	if err != nil {
		t.Fatalf("NewSqliteStore: %v", err)
	}
	defer sqs.Close()

	other, err := OpenLocalDB(filepath.Join(dir, "db.sqlite3"))
	if err != nil {
		t.Fatalf("open second pool: %v", err)
	}
	defer other.Close()
	if _, err := other.Exec("CREATE TABLE IF NOT EXISTS records (collection TEXT, rkey TEXT, cid TEXT, PRIMARY KEY (collection, rkey))"); err != nil {
		t.Fatalf("create records table: %v", err)
	}

	root, blks := makeBlocks(t, 12, "two-openers")
	release := holdWriteLock(t, other)
	awaitBlockedWrite(t, "carstore write behind the second pool's lock", release, carstoreWrite(sqs, root, blks))

	release = holdWriteLock(t, sqs.db)
	awaitBlockedWrite(t, "second pool write behind the carstore's lock", release, func() error {
		_, err := other.Exec(
			"INSERT INTO records (collection, rkey, cid) VALUES (?, ?, ?) ON CONFLICT (collection, rkey) DO UPDATE SET cid=excluded.cid",
			"io.atcr.hold.scan", "rkey-1", "cid-1")
		return err
	})
}

// TestRawPoolFailsWithoutBusyTimeout is the control: a pool opened around the
// bare driver, with no busy_timeout, fails immediately under the same held
// lock. It proves the two tests above are observing busy_timeout and not some
// other reason the writes happened to get through.
func TestRawPoolFailsWithoutBusyTimeout(t *testing.T) {
	path := filepath.Join(t.TempDir(), "db.sqlite3")
	guarded, err := OpenLocalDB(path)
	if err != nil {
		t.Fatalf("OpenLocalDB: %v", err)
	}
	defer guarded.Close()

	raw, err := sql.Open("libsql", localDSN(path))
	if err != nil {
		t.Fatalf("raw open: %v", err)
	}
	defer raw.Close()
	if _, err := raw.Exec("CREATE TABLE IF NOT EXISTS records (k INTEGER PRIMARY KEY, v TEXT)"); err != nil {
		t.Fatalf("create table: %v", err)
	}

	release := holdWriteLock(t, guarded)
	defer release()

	start := time.Now()
	_, err = raw.Exec("INSERT INTO records (v) VALUES ('raw')")
	took := time.Since(start)
	if err == nil {
		t.Fatalf("raw pool wrote through a held write lock in %s; the control cannot distinguish a regression", took)
	}
	if !isLockErr(err) {
		t.Fatalf("raw pool failed for a reason other than the lock: %v", err)
	}
	if took > holdDuration {
		t.Fatalf("raw pool waited %s before failing; expected an immediate SQLITE_BUSY with no busy_timeout", took)
	}
}