mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-20 15:04:37 +00:00
fix(vacuum): batch fsync in makeupDiff to prevent test timeout (#11289)
makeupDiff called dstDatBackend.Sync() (fsync) per needle in the loop over incrementedHasUpdatedIndexEntry. With 20000 entries in TestLDBIndexCompaction this resulted in up to 20000 fsync calls, which on slow CI disks exceeded the 10-minute test timeout. Batch the sync: write all needles/tombstones first, then fsync the dat file once in the defer alongside the existing idx fsync. The durability guarantee is unchanged — both files are still synced before CommitCompact writes the .cpc commit marker and swaps the files.
This commit is contained in:
@@ -486,7 +486,12 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI
|
||||
defer func() {
|
||||
// makeupDiff appends new needles/tombstones to the .cpx; its fsync is the
|
||||
// durability gate that must succeed before CommitCompact writes the .cpc
|
||||
// marker and swaps the files.
|
||||
// marker and swaps the files. Sync the dat file once after all appends
|
||||
// rather than per-needle — a fsync per entry scales poorly on slow disks
|
||||
// and can exceed test timeouts with large entry counts.
|
||||
if syncErr := dstDatBackend.Sync(); syncErr != nil && err == nil {
|
||||
err = fmt.Errorf("sync dat %s: %v", newDatFileName, syncErr)
|
||||
}
|
||||
if syncErr := idx.Sync(); syncErr != nil && err == nil {
|
||||
err = fmt.Errorf("sync idx %s: %v", newIdxFileName, syncErr)
|
||||
}
|
||||
@@ -536,9 +541,6 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI
|
||||
return fmt.Errorf("ReadNeedleBlob %s key %d offset %d size %d failed: %w", oldDatFile.Name(), key, increIdxEntry.offset.ToActualOffset(), increIdxEntry.size, err)
|
||||
}
|
||||
dstDatBackend.Write(needleBytes)
|
||||
if err := dstDatBackend.Sync(); err != nil {
|
||||
return fmt.Errorf("cannot sync needle %s: %v", dstDatBackend.File.Name(), err)
|
||||
}
|
||||
util.Uint32toBytes(idxEntryBytes[8:12], uint32(offset/NeedlePaddingSize))
|
||||
} else { //deleted needle
|
||||
//fakeDelNeedle's default Data field is nil
|
||||
|
||||
Reference in New Issue
Block a user