From d848b8ed0045b0167146cca05238787cf31a12db Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 23 Sep 2026 18:08:20 +0800 Subject: [PATCH] rust volume: regression test for makeup_diff replay across a 32 GiB offset boundary (#11410) (#11427) * rust volume: test makeup_diff replay across a 32 GiB offset boundary Issue #11410 corrupted a replayed write's index offset in Go's makeupDiff by patching only four of the five offset bytes. The Rust makeup_diff already encodes the whole offset through idx_entry_to_bytes and Offset::from_actual_offset; this adds the mirror of TestConcurrentWriteCrossesOffsetBoundary so a regression would fail here the same way it does under -tags=5BytesOffset on the Go side. Sparse-truncate the .dat to 64 GiB, compact, write, commit: the index offset must equal the .cpd size and the needle must stay readable through a second vacuum. Gated on the 5bytes feature since a 64 GiB .dat exceeds the 32 GiB range of 4-byte offsets. * rust volume: skip the offset-boundary replay test on Windows Windows set_len allocates the full 64 GiB extension instead of a sparse range, so the test fails with StorageFull on CI runners. Gate it to unix, where set_len leaves the extension unallocated. --- seaweed-volume/src/storage/volume.rs | 56 ++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/seaweed-volume/src/storage/volume.rs b/seaweed-volume/src/storage/volume.rs index ae3127f22..0b7f4fd62 100644 --- a/seaweed-volume/src/storage/volume.rs +++ b/seaweed-volume/src/storage/volume.rs @@ -7001,6 +7001,62 @@ mod tests { assert_eq!(got.data, b"late-write"); } + /// Mirrors Go's TestConcurrentWriteCrossesOffsetBoundary (issue #11410): + /// a replayed write's index entry must encode all offset bytes even when + /// old and new offsets sit in different 32 GiB ranges. Unix-only: Windows + /// set_len eagerly allocates the 64 GiB extension. + #[test] + #[cfg(all(unix, feature = "5bytes"))] + fn test_makeup_diff_replay_crosses_offset_boundary() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let mut v = make_test_volume(dir); + + write_test_needle(&mut v, 1, &[b'x'; 32]); + v.dat_file.as_ref().unwrap().set_len(64u64 << 30).unwrap(); + v.compact_by_index(0, 0, |_| true).unwrap(); + + let late_data = vec![b'x'; 113]; + write_test_needle(&mut v, 342511246, &late_data); + + let mut before = Needle { + id: NeedleId(342511246), + ..Needle::default() + }; + v.read_needle(&mut before).unwrap(); + assert_eq!(before.data, late_data); + + let expected_offset = fs::metadata(v.file_name(".cpd")).unwrap().len() as i64; + v.commit_compact().unwrap(); + + let mut idx_file = File::open(v.file_name(".idx")).unwrap(); + let mut actual_offset = 0i64; + crate::storage::idx::walk_index_file(&mut idx_file, 0, |key, offset, _size| { + if key == NeedleId(342511246) { + actual_offset = offset.to_actual_offset(); + } + Ok(()) + }) + .unwrap(); + assert_eq!(actual_offset, expected_offset); + + let mut after = Needle { + id: NeedleId(342511246), + ..Needle::default() + }; + v.read_needle(&mut after).unwrap(); + assert_eq!(after.data, late_data); + + v.compact_by_index(0, 0, |_| true).unwrap(); + v.commit_compact().unwrap(); + let mut third = Needle { + id: NeedleId(342511246), + ..Needle::default() + }; + v.read_needle(&mut third).unwrap(); + assert_eq!(third.data, late_data); + } + /// Vacuum compaction must tolerate an .idx entry whose offset points past /// the end of the .dat file (the failure mode in issue #8928). The bad /// entry is silently dropped from the resulting .cpx; healthy needles