mirrors/at-container-registry
1
0
Fork 0
mirror of https://tangled.org/evan.jarrett.net/at-container-registry synced 2026-09-21 01:34:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

improve backfill and jetstream db connections

Browse Source
This commit is contained in:
Evan Jarrett
2026-04-19 18:01:17 -05:00
parent 38c693acc9
commit 7c6b8945ed
4 changed files with 205 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/appview/jetstream/backfill.go
+41 -26
View File
@@ -267,36 +267,51 @@ func (b *BackfillWorker) backfillRepo(ctx context.Context, did, collection strin
// ProcessManifest calls ResolveHoldDID for legacy manifests.
b.prewarmHoldCaches(ctx, collection, allRecords)
// Phase 3: Process records in chunked transactions.
// All network I/O should be cached by now, so transactions stay fast.
const chunkSize = 20
recordCount := 0
for i := 0; i < len(allRecords); i += chunkSize {
end := i + chunkSize
if end > len(allRecords) {
end = len(allRecords)
}
tx, err := b.db.Begin()
if err != nil {
return recordCount, fmt.Errorf("failed to begin transaction: %w", err)
}
txProcessor := NewProcessor(tx, false, b.processor.statsCache)
for j := i; j < end; j++ {
if err := b.processRecordWith(ctx, txProcessor, did, collection, &allRecords[j]); err != nil {
slog.Warn("Backfill failed to process record", "uri", allRecords[j].URI, "error", err)
// Phase 3: Write records to the DB.
//
// For collections whose writes are straightforward idempotent upserts, we
// batch every record in the repo into one multi-row INSERT per table. This
// replaces the previous 20-record chunked transaction loop, which exceeded
// Bunny Database's remote transaction timeout (~5s) once chunks grew large
// and poisoned the connection pool on timeout.
//
// Collections that do network I/O per record (SailorProfile) or have
// conditional read-then-write logic (Scan) stay on the single-record path
// where each write is its own statement and cannot hold a long transaction.
var recordCount int
var procErr error
switch collection {
case atproto.ManifestCollection:
recordCount, procErr = b.batchManifests(ctx, did, allRecords)
case atproto.TagCollection:
recordCount, procErr = b.batchTags(did, allRecords)
case atproto.StarCollection:
recordCount, procErr = b.batchStars(ctx, did, allRecords)
case atproto.RepoPageCollection:
recordCount, procErr = b.batchRepoPages(did, allRecords)
case atproto.DailyStatsCollection:
recordCount, procErr = b.batchDailyStats(ctx, did, allRecords)
case atproto.StatsCollection:
recordCount, procErr = b.batchStats(ctx, did, allRecords)
case atproto.CaptainCollection:
recordCount, procErr = b.batchCaptains(did, allRecords)
case atproto.CrewCollection:
recordCount, procErr = b.batchCrew(did, allRecords)
default:
// SailorProfileCollection and ScanCollection keep per-record processing
// because they do network I/O or conditional reads that would be awkward
// to batch. Each call writes a single row, so there is no long-lived
// transaction at risk.
for i := range allRecords {
if err := b.processRecordWith(ctx, b.processor, did, collection, &allRecords[i]); err != nil {
slog.Warn("Backfill failed to process record", "uri", allRecords[i].URI, "error", err)
continue
}
recordCount++
}
if err := tx.Commit(); err != nil {
tx.Rollback()
return recordCount, fmt.Errorf("failed to commit transaction: %w", err)
}
}
if procErr != nil {
return recordCount, procErr
}
// Reconciliation runs outside the transaction (involves network I/O and fewer writes)