at-container-registry/pkg/hold/db/sqlite_store_contention_test.go

package db

import (
	"context"
	"fmt"
	"path/filepath"
	"strings"
	"sync"
	"testing"

	"github.com/bluesky-social/indigo/models"
	blockformat "github.com/ipfs/go-block-format"
	"github.com/ipfs/go-cid"
)

// isLockErr reports whether err is a SQLite busy/locked error rather than a
// logic error. Contention tests must fail on lock errors specifically, so a
// typo cannot masquerade as a reproduction.
func isLockErr(err error) bool {
	if err == nil {
		return false
	}
	s := strings.ToLower(err.Error())
	return strings.Contains(s, "database is locked") ||
		strings.Contains(s, "database table is locked") ||
		strings.Contains(s, "sqlite_busy") ||
		strings.Contains(s, "busy")
}

func makeBlocks(t *testing.T, n int, salt string) (cid.Cid, map[cid.Cid]blockformat.Block) {
	t.Helper()
	blks := make(map[cid.Cid]blockformat.Block, n)
	var root cid.Cid
	for i := 0; i < n; i++ {
		b := blockformat.NewBlock([]byte(fmt.Sprintf("%s-block-%d-%s", salt, i, strings.Repeat("x", 2048))))
		blks[b.Cid()] = b
		if i == 0 {
			root = b.Cid()
		}
	}
	return root, blks
}

// writeShards hammers the carstore with concurrent shard writes and reports any
// lock error it hits. This is the exact path every hold record write takes.
func writeShards(t *testing.T, sqs *SQLiteStore, writers, iters int) error {
	t.Helper()
	ctx := context.Background()
	var wg sync.WaitGroup
	var mu sync.Mutex
	var firstErr error

	for w := 0; w < writers; w++ {
		wg.Add(1)
		go func(w int) {
			defer wg.Done()
			for i := 0; i < iters; i++ {
				salt := fmt.Sprintf("w%d-i%d", w, i)
				root, blks := makeBlocks(t, 12, salt)
				rev := fmt.Sprintf("rev-%03d-%03d", w, i)
				_, err := sqs.writeNewShard(ctx, root, rev, models.Uid(1), i, blks, nil)
				if err != nil {
					mu.Lock()
					if firstErr == nil {
						firstErr = err
					}
					mu.Unlock()
					return
				}
			}
		}(w)
	}
	wg.Wait()
	return firstErr
}

// TestCarstoreConcurrentWritesSharedPool reproduces the production topology:
// one *sql.DB pool (OpenHoldDB) shared by the carstore, records index, events
// and scan broadcaster. database/sql opens a fresh connection per concurrent
// caller, and busy_timeout is per-connection, so any connection beyond the
// first commits with busy_timeout=0 and fails immediately under contention.
func TestCarstoreConcurrentWritesSharedPool(t *testing.T) {
	path := filepath.Join(t.TempDir(), "db.sqlite3")
	h, err := OpenHoldDB(path, LibsqlConfig{})
	if err != nil {
		t.Fatalf("OpenHoldDB: %v", err)
	}
	defer h.Close()

	sqs, err := NewSQLiteStoreWithDB(path, h.DB)
	if err != nil {
		t.Fatalf("NewSQLiteStoreWithDB: %v", err)
	}

	if err := writeShards(t, sqs, 8, 25); err != nil {
		if isLockErr(err) {
			t.Fatalf("shared-pool carstore write hit a lock error: %v", err)
		}
		t.Fatalf("shared-pool carstore write failed: %v", err)
	}
}

// TestCarstoreConcurrentWritesTwoOpeners covers the topology where a second
// subsystem (the scan broadcaster or the event broadcaster, when they are not
// handed the shared pool) opens <dir>/db.sqlite3 again through its own pool
// while the carstore is writing.
func TestCarstoreConcurrentWritesTwoOpeners(t *testing.T) {
	dir := t.TempDir()
	sqs, err := NewSqliteStore(dir)
	if err != nil {
		t.Fatalf("NewSqliteStore: %v", err)
	}
	defer sqs.Close()

	// Second, independent pool on the same file, opened the way every hold
	// subsystem now opens one.
	ri, err := OpenLocalDB(filepath.Join(dir, "db.sqlite3"))
	if err != nil {
		t.Fatalf("open records index: %v", err)
	}
	defer ri.Close()
	if _, err := ri.Exec("CREATE TABLE IF NOT EXISTS records (collection TEXT, rkey TEXT, cid TEXT, PRIMARY KEY (collection, rkey))"); err != nil {
		t.Fatalf("create records table: %v", err)
	}

	ctx := context.Background()
	stop := make(chan struct{})
	idxErr := make(chan error, 1)
	go func() {
		var err error
		for i := 0; ; i++ {
			select {
			case <-stop:
				idxErr <- err
				return
			default:
			}
			_, e := ri.ExecContext(ctx,
				"INSERT INTO records (collection, rkey, cid) VALUES (?, ?, ?) ON CONFLICT (collection, rkey) DO UPDATE SET cid=excluded.cid",
				"io.atcr.hold.scan", fmt.Sprintf("rkey-%d", i), fmt.Sprintf("cid-%d", i))
			if e != nil && err == nil {
				err = e
			}
		}
	}()

	csErr := writeShards(t, sqs, 4, 30)
	close(stop)
	riResult := <-idxErr

	for _, e := range []error{csErr, riResult} {
		if e == nil {
			continue
		}
		if isLockErr(e) {
			t.Fatalf("two-opener write hit a lock error: %v", e)
		}
		t.Fatalf("two-opener write failed: %v", e)
	}
}