fix(vacuum): batch fsync in makeupDiff to prevent test timeout (#11289)

makeupDiff called dstDatBackend.Sync() (fsync) per needle in the loop
over incrementedHasUpdatedIndexEntry. With 20000 entries in
TestLDBIndexCompaction this resulted in up to 20000 fsync calls, which
on slow CI disks exceeded the 10-minute test timeout.

Batch the sync: write all needles/tombstones first, then fsync the dat
file once in the defer alongside the existing idx fsync. The durability
guarantee is unchanged — both files are still synced before CommitCompact
writes the .cpc commit marker and swaps the files.
This commit is contained in:
Chris Lu
2026-09-13 00:05:30 -07:00
committed by GitHub
parent 8db41d0217
commit 99d2479528
+6 -4
View File
@@ -486,7 +486,12 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI
defer func() {
// makeupDiff appends new needles/tombstones to the .cpx; its fsync is the
// durability gate that must succeed before CommitCompact writes the .cpc
// marker and swaps the files.
// marker and swaps the files. Sync the dat file once after all appends
// rather than per-needle — a fsync per entry scales poorly on slow disks
// and can exceed test timeouts with large entry counts.
if syncErr := dstDatBackend.Sync(); syncErr != nil && err == nil {
err = fmt.Errorf("sync dat %s: %v", newDatFileName, syncErr)
}
if syncErr := idx.Sync(); syncErr != nil && err == nil {
err = fmt.Errorf("sync idx %s: %v", newIdxFileName, syncErr)
}
@@ -536,9 +541,6 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI
return fmt.Errorf("ReadNeedleBlob %s key %d offset %d size %d failed: %w", oldDatFile.Name(), key, increIdxEntry.offset.ToActualOffset(), increIdxEntry.size, err)
}
dstDatBackend.Write(needleBytes)
if err := dstDatBackend.Sync(); err != nil {
return fmt.Errorf("cannot sync needle %s: %v", dstDatBackend.File.Name(), err)
}
util.Uint32toBytes(idxEntryBytes[8:12], uint32(offset/NeedlePaddingSize))
} else { //deleted needle
//fakeDelNeedle's default Data field is nil