diff --git a/seaweed-volume/src/storage/erasure_coding/ec_bitrot.rs b/seaweed-volume/src/storage/erasure_coding/ec_bitrot.rs new file mode 100644 index 000000000..06a31167f --- /dev/null +++ b/seaweed-volume/src/storage/erasure_coding/ec_bitrot.rs @@ -0,0 +1,827 @@ +//! EC bitrot detection — checksum sidecar. +//! +//! A per-volume sidecar file stores a CRC32C (Castagnoli) checksum for every +//! fixed-size block of every EC shard, so a scrub (and the reconstruction path) +//! can detect silent disk corruption in any shard — including cold parity shards +//! that are never read during normal serving. +//! +//! The sidecar is OPTIONAL: an absent or generation-mismatched sidecar simply +//! means "feature off" for that generation, so old binaries, JSON-only nodes, +//! and rollback deployments ignore it and degrade gracefully. +//! +//! On-disk layout of `.ecsum` (legacy/generation 0) and `.ecsum.v`: +//! +//! ```text +//! [ magic(4) | format_version(2) | payload_len(4) | payload_crc32c(4) ] [ proto payload ] +//! ``` +//! +//! All header fields are BIG-ENDIAN. The header's `payload_crc32c` lets a loader +//! detect corruption of the sidecar itself BEFORE trusting any contents, so a +//! rotted sidecar can never be mistaken for shard corruption. This format is +//! byte-identical to the Go implementation in +//! `weed/storage/erasure_coding/ec_bitrot.go`. + +use std::fs::File; +use std::io::{self, Read, Write}; + +use prost::Message; + +use crate::pb::volume_server_pb::{ + ChecksumAlgorithm, EcBitrotProtection, EcShardChecksums, EcShardConfig, +}; +use crate::storage::erasure_coding::ec_shard::MAX_SHARD_COUNT; +use crate::storage::needle::crc::CRC; + +/// Canonical extension for the checksum sidecar. Generation 0 (legacy/fresh +/// encode) uses `.ecsum`; vacuum generation N uses `.ecsum.v`, +/// mirroring the `.vif`/`.ecx` versioned convention. +pub const BITROT_SIDECAR_EXT: &str = ".ecsum"; + +/// Default checksum granularity (16 MiB). It is a power-of-two multiple of +/// `ERASURE_CODING_SMALL_BLOCK_SIZE` (1 MiB) and keeps the sidecar tiny +/// (~11 KB for a 30 GB volume) while localizing corruption to a 16 MiB region. +pub const DEFAULT_BITROT_BLOCK_SIZE: usize = 16 * 1024 * 1024; + +/// Caps the block granularity so a loaded sidecar cannot force a huge +/// scrub/verify scratch buffer. Power-of-two multiple of 1 MiB. +pub const MAX_BITROT_BLOCK_SIZE: u32 = 64 * 1024 * 1024; + +/// Magic "ECSU". +pub const BITROT_MAGIC: u32 = 0x4543_5355; +/// On-disk format version. +pub const BITROT_FORMAT_VERSION: u16 = 1; +/// Header size: magic(4) + version(2) + payload_len(4) + payload_crc32c(4). +pub const BITROT_HEADER_SIZE: usize = 14; + +/// Resolved protection state of an EC volume's active generation after loading +/// and validating its sidecar. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BitrotStatus { + /// No sidecar, or a sidecar that does not describe the active generation. + /// The generation is unprotected; this is NOT corruption. + Off, + /// A complete, well-formed, generation-matching sidecar is loaded. + On, + /// A generation-matching sidecar that is malformed, incomplete, or + /// self-integrity-failed. The generation is unprotected pending repair of + /// the sidecar, and an integrity alarm should fire. The rebuild path treats + /// this as fail-closed. + Invalid, +} + +impl std::fmt::Display for BitrotStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + BitrotStatus::On => "on", + BitrotStatus::Invalid => "invalid", + BitrotStatus::Off => "off", + }; + f.write_str(s) + } +} + +/// Error returned by [`load_bitrot_sidecar`]. A self-integrity failure is a +/// sidecar-integrity problem (the caller maps it to [`BitrotStatus::Invalid`]), +/// never a shard-corruption signal. +#[derive(Debug)] +pub enum BitrotLoadError { + /// File missing or other underlying I/O error. + Io(io::Error), + /// File shorter than the fixed header. + TooShort(usize), + /// Header magic did not match. + BadMagic(u32), + /// Header format version is unsupported. + UnsupportedVersion(u16), + /// Header payload_len disagrees with the actual payload length. + LengthMismatch { header: u32, actual: usize }, + /// Header payload_crc32c disagrees with the computed CRC32C of the payload. + CrcMismatch { header: u32, computed: u32 }, + /// Protobuf payload failed to decode. + Decode(prost::DecodeError), +} + +impl std::fmt::Display for BitrotLoadError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BitrotLoadError::Io(e) => write!(f, "bitrot sidecar io error: {}", e), + BitrotLoadError::TooShort(n) => { + write!(f, "bitrot sidecar too short ({} bytes)", n) + } + BitrotLoadError::BadMagic(m) => write!(f, "bitrot sidecar bad magic {:#x}", m), + BitrotLoadError::UnsupportedVersion(v) => { + write!(f, "bitrot sidecar unsupported format version {}", v) + } + BitrotLoadError::LengthMismatch { header, actual } => write!( + f, + "bitrot sidecar length mismatch: header {}, actual {}", + header, actual + ), + BitrotLoadError::CrcMismatch { header, computed } => write!( + f, + "bitrot sidecar self-integrity CRC mismatch: header {:#x}, computed {:#x}", + header, computed + ), + BitrotLoadError::Decode(e) => write!(f, "unmarshal bitrot sidecar: {}", e), + } + } +} + +impl std::error::Error for BitrotLoadError {} + +impl From for BitrotLoadError { + fn from(e: io::Error) -> Self { + BitrotLoadError::Io(e) + } +} + +/// Returns the sidecar path for a base file name and EC generation. Generation +/// 0 is the un-suffixed legacy path; generation N>0 is the versioned path, +/// consistent with how `.vif`/`.ecx` are versioned by the 2PC switch. +pub fn bitrot_sidecar_path(base: &str, generation: u32) -> String { + if generation == 0 { + format!("{}{}", base, BITROT_SIDECAR_EXT) + } else { + format!("{}{}.v{}", base, BITROT_SIDECAR_EXT, generation) + } +} + +/// Returns a fresh random per-encode identity used to detect a stale sidecar +/// left behind by an in-place re-encode. +pub fn new_encode_uuid() -> Vec { + use rand::RngCore; + let mut b = vec![0u8; 16]; + rand::thread_rng().fill_bytes(&mut b); + b +} + +/// Reports whether `block_size` is a power of two in [1 MiB, MAX_BITROT_BLOCK_SIZE]. +pub fn is_pow2_multiple_of_1mib(block_size: u32) -> bool { + block_size >= (1 << 20) && block_size <= MAX_BITROT_BLOCK_SIZE && block_size.count_ones() == 1 +} + +/// Returns ceil(covered_size / block_size). +fn expected_block_count(covered_size: i64, block_size: i64) -> usize { + if block_size <= 0 { + return 0; + } + ((covered_size + block_size - 1) / block_size) as usize +} + +/// Packs a slice of u32 into little-endian bytes. +fn pack_u32_le(vals: &[u32]) -> Vec { + let mut out = Vec::with_capacity(vals.len() * 4); + for v in vals { + out.extend_from_slice(&v.to_le_bytes()); + } + out +} + +/// Unpacks little-endian bytes into a Vec. Trailing bytes (len % 4) are +/// ignored, matching the Go implementation's `len(b)/4` truncation. +fn unpack_u32_le(b: &[u8]) -> Vec { + let n = b.len() / 4; + let mut out = Vec::with_capacity(n); + for i in 0..n { + out.push(u32::from_le_bytes([ + b[i * 4], + b[i * 4 + 1], + b[i * 4 + 2], + b[i * 4 + 3], + ])); + } + out +} + +/// Accumulates the per-block CRC32C of a single shard's byte stream as it is +/// written. Tolerates arbitrary chunk sizes that cross block boundaries, so it +/// works for both small encode buffers and the larger rebuild buffers. +pub struct ShardChecksumBuilder { + block_size: i64, + cur: CRC, + cur_len: i64, + total: i64, + blocks: Vec, +} + +impl ShardChecksumBuilder { + /// Creates a builder over a given block size in bytes. + pub fn new(block_size: i64) -> Self { + ShardChecksumBuilder { + block_size, + cur: CRC(0), + cur_len: 0, + total: 0, + blocks: Vec::new(), + } + } + + /// Feeds a chunk of arbitrary size, splitting it across block boundaries. + pub fn write(&mut self, mut p: &[u8]) { + while !p.is_empty() { + let room = self.block_size - self.cur_len; + let n = (p.len() as i64).min(room) as usize; + self.cur = self.cur.update(&p[..n]); + self.cur_len += n as i64; + self.total += n as i64; + p = &p[n..]; + if self.cur_len == self.block_size { + self.blocks.push(self.cur.0); + self.cur = CRC(0); + self.cur_len = 0; + } + } + } + + /// Flushes any partial last block and returns the covered size and the + /// packed little-endian u32 CRC array. + pub fn finalize(mut self) -> (i64, Vec) { + if self.cur_len > 0 { + self.blocks.push(self.cur.0); + self.cur = CRC(0); + self.cur_len = 0; + } + (self.total, pack_u32_le(&self.blocks)) + } +} + +/// Atomically writes `prot` to `path`, wrapped in the on-disk header with a +/// CRC32C over the serialized payload (temp file + rename). +pub fn save_bitrot_sidecar(path: &str, prot: &EcBitrotProtection) -> io::Result<()> { + let payload = prot.encode_to_vec(); + let mut buf = Vec::with_capacity(BITROT_HEADER_SIZE + payload.len()); + buf.extend_from_slice(&BITROT_MAGIC.to_be_bytes()); + buf.extend_from_slice(&BITROT_FORMAT_VERSION.to_be_bytes()); + buf.extend_from_slice(&(payload.len() as u32).to_be_bytes()); + buf.extend_from_slice(&CRC::new(&payload).0.to_be_bytes()); + buf.extend_from_slice(&payload); + + let tmp = format!("{}.tmp", path); + { + let mut f = File::create(&tmp)?; + f.write_all(&buf)?; + f.sync_all()?; + } + if let Err(e) = std::fs::rename(&tmp, path) { + let _ = std::fs::remove_file(&tmp); + return Err(e); + } + Ok(()) +} + +/// Reads and self-integrity-checks a sidecar file. Returns the parsed message, +/// or an error if the file is missing, truncated, has a bad magic/version, or +/// fails the payload CRC. A self-integrity failure is a sidecar-integrity +/// problem (caller maps it to [`BitrotStatus::Invalid`]), never a shard +/// corruption signal. +pub fn load_bitrot_sidecar(path: &str) -> Result { + let mut data = Vec::new(); + File::open(path)?.read_to_end(&mut data)?; + if data.len() < BITROT_HEADER_SIZE { + return Err(BitrotLoadError::TooShort(data.len())); + } + let magic = u32::from_be_bytes([data[0], data[1], data[2], data[3]]); + if magic != BITROT_MAGIC { + return Err(BitrotLoadError::BadMagic(magic)); + } + let ver = u16::from_be_bytes([data[4], data[5]]); + if ver != BITROT_FORMAT_VERSION { + return Err(BitrotLoadError::UnsupportedVersion(ver)); + } + let payload_len = u32::from_be_bytes([data[6], data[7], data[8], data[9]]); + let want_crc = u32::from_be_bytes([data[10], data[11], data[12], data[13]]); + let payload = &data[BITROT_HEADER_SIZE..]; + if payload_len as usize != payload.len() { + return Err(BitrotLoadError::LengthMismatch { + header: payload_len, + actual: payload.len(), + }); + } + let got = CRC::new(payload).0; + if got != want_crc { + return Err(BitrotLoadError::CrcMismatch { + header: want_crc, + computed: got, + }); + } + EcBitrotProtection::decode(payload).map_err(BitrotLoadError::Decode) +} + +/// Performs the disk-free manifest/syntax checks that every loader runs: +/// supported algorithm, valid block size, exactly one entry per shard id in the +/// active layout (no duplicates, no out-of-range ids), positive covered_size, +/// and a packed-CRC count consistent with covered_size. +/// +/// It does NOT compare covered_size against on-disk shard lengths — that is a +/// per-node physical check done only for locally-held shards. +pub fn validate_manifest( + prot: &EcBitrotProtection, + data_shards: usize, + parity_shards: usize, +) -> Result<(), String> { + if prot.algorithm != ChecksumAlgorithm::ChecksumCrc32c as i32 { + return Err(format!("unsupported checksum algorithm {}", prot.algorithm)); + } + if !is_pow2_multiple_of_1mib(prot.block_size) { + return Err(format!( + "invalid block_size {} (must be a power-of-two multiple of 1 MiB, at most {})", + prot.block_size, MAX_BITROT_BLOCK_SIZE + )); + } + let bs = prot.block_size as i64; + let total = data_shards + parity_shards; + if total == 0 || total > MAX_SHARD_COUNT { + return Err(format!( + "invalid active layout: data={} parity={}", + data_shards, parity_shards + )); + } + if prot.shards.len() != total { + return Err(format!( + "incomplete manifest: {} shard entries, expected {}", + prot.shards.len(), + total + )); + } + let mut seen = vec![false; MAX_SHARD_COUNT]; + for s in &prot.shards { + if s.shard_id >= total as u32 { + return Err(format!( + "shard id {} out of range [0,{})", + s.shard_id, total + )); + } + if seen[s.shard_id as usize] { + return Err(format!("duplicate shard id {}", s.shard_id)); + } + seen[s.shard_id as usize] = true; + if s.covered_size <= 0 { + return Err(format!( + "shard {} has non-positive covered_size {}", + s.shard_id, s.covered_size + )); + } + let want_count = expected_block_count(s.covered_size, bs); + if s.block_crc32c.len() != want_count * 4 { + return Err(format!( + "shard {} crc count mismatch: {} bytes, expected {} (covered_size={} block_size={})", + s.shard_id, + s.block_crc32c.len(), + want_count * 4, + s.covered_size, + prot.block_size + )); + } + } + Ok(()) +} + +/// Resolves the protection status of a loaded-or-missing sidecar against the +/// active generation and layout. `loaded` is the result of attempting to load +/// the sidecar at the active generation's path. +/// +/// - missing sidecar (NotFound) => [`BitrotStatus::Off`] +/// - load/self-integrity failure => [`BitrotStatus::Invalid`] +/// - generation mismatch => [`BitrotStatus::Off`] +/// - manifest validation failure => [`BitrotStatus::Invalid`] +/// - otherwise => [`BitrotStatus::On`] +pub fn resolve_status( + loaded: &Result, + active_generation: u32, + data_shards: usize, + parity_shards: usize, +) -> BitrotStatus { + match loaded { + Err(BitrotLoadError::Io(e)) if e.kind() == io::ErrorKind::NotFound => BitrotStatus::Off, + Err(_) => BitrotStatus::Invalid, + Ok(prot) => { + if prot.generation != active_generation { + return BitrotStatus::Off; + } + if validate_manifest(prot, data_shards, parity_shards).is_err() { + return BitrotStatus::Invalid; + } + BitrotStatus::On + } + } +} + +/// Returns the [`EcShardChecksums`] entry for a shard id, or `None`. +pub fn shard_checksums(prot: &EcBitrotProtection, shard_id: u32) -> Option<&EcShardChecksums> { + prot.shards.iter().find(|s| s.shard_id == shard_id) +} + +/// Reads a shard file at `path` in `block_size` chunks and compares each block's +/// CRC32C against the manifest entry. Returns the list of mismatching block +/// indices (empty == clean), or a fatal `io::Error` for genuine I/O problems. +/// +/// A length mismatch (truncation or unexpected trailing bytes) is itself shard +/// corruption: every block index is reported as mismatched so the caller treats +/// the shard as bad. It does not interpret the result — the caller (scrub / +/// rebuild) arbitrates shard-vs-sidecar via Reed-Solomon before acting. +pub fn verify_shard_file_blocks( + path: &str, + entry: &EcShardChecksums, + block_size: i64, +) -> io::Result> { + let f = File::open(path)?; + let file_size = f.metadata()?.len() as i64; + let want = unpack_u32_le(&entry.block_crc32c); + + if file_size != entry.covered_size { + // Length drift is shard corruption: report every block as mismatched. + return Ok((0..want.len()).collect()); + } + + let mut mismatched = Vec::new(); + let mut buf = vec![0u8; block_size.max(1) as usize]; + let mut offset: i64 = 0; + for (i, want_crc) in want.iter().enumerate() { + let to_read = (entry.covered_size - offset).min(block_size); + if to_read <= 0 { + break; + } + let to_read = to_read as usize; + read_full_at(&f, &mut buf[..to_read], offset as u64)?; + if CRC::new(&buf[..to_read]).0 != *want_crc { + mismatched.push(i); + } + offset += to_read as i64; + } + Ok(mismatched) +} + +/// Reads exactly `buf.len()` bytes from `f` at `offset`, erroring on early EOF. +fn read_full_at(f: &File, buf: &mut [u8], offset: u64) -> io::Result<()> { + let mut total = 0usize; + while total < buf.len() { + #[cfg(unix)] + let n = { + use std::os::unix::fs::FileExt; + f.read_at(&mut buf[total..], offset + total as u64)? + }; + #[cfg(not(unix))] + let n = { + use std::io::{Read, Seek, SeekFrom}; + let mut fc = f.try_clone()?; + fc.seek(SeekFrom::Start(offset + total as u64))?; + fc.read(&mut buf[total..])? + }; + if n == 0 { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "short read on shard block", + )); + } + total += n; + } + Ok(()) +} + +/// Builds the `EcShardConfig` proto for the given layout. The bitrot sidecar +/// carries its own top-level encode_uuid, so the nested config leaves it empty. +pub fn ec_shard_config(data_shards: u32, parity_shards: u32) -> EcShardConfig { + EcShardConfig { + data_shards, + parity_shards, + encode_ts_ns: 0, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Cross-binary byte-exact proof: CANONICAL_HEX equals `canonicalInteropHex` + /// in weed/storage/erasure_coding/ec_bitrot_interop_test.go (the Go reference + /// asserts the same constant). Both binaries port the same format, so a + /// sidecar written by either must be byte-identical; this pins the Rust side + /// of that guarantee. If you change the format, regenerate and update BOTH. + #[test] + fn test_byte_exact_go_interop() { + const CANONICAL_HEX: &str = "45435355000100000039cc1b826a080110808080082204080a10042a0a108080401a04040302012a0c0801108080401a04080706053210000102030405060708090a0b0c0d0e0f"; + let prot = EcBitrotProtection { + algorithm: ChecksumAlgorithm::ChecksumCrc32c as i32, + block_size: DEFAULT_BITROT_BLOCK_SIZE as u32, + generation: 0, + ec_shard_config: Some(EcShardConfig { + data_shards: 10, + parity_shards: 4, + encode_ts_ns: 0, + }), + shards: vec![ + EcShardChecksums { + shard_id: 0, + covered_size: 1024 * 1024, + block_crc32c: pack_u32_le(&[0x0102_0304]), + }, + EcShardChecksums { + shard_id: 1, + covered_size: 1024 * 1024, + block_crc32c: pack_u32_le(&[0x0506_0708]), + }, + ], + encode_uuid: (0..16u8).collect(), + }; + let dir = std::env::temp_dir().join(format!("ecsum_interop_{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("v1.ecsum"); + let path = path.to_str().unwrap(); + save_bitrot_sidecar(path, &prot).unwrap(); + let bytes = std::fs::read(path).unwrap(); + let hex: String = bytes.iter().map(|b| format!("{:02x}", b)).collect(); + assert_eq!(hex, CANONICAL_HEX, "Rust .ecsum bytes drifted from the Go canonical form"); + let _ = std::fs::remove_file(path); + } + + #[test] + fn test_sidecar_path_generations() { + assert_eq!(bitrot_sidecar_path("/d/1", 0), "/d/1.ecsum"); + assert_eq!(bitrot_sidecar_path("/d/1", 3), "/d/1.ecsum.v3"); + } + + #[test] + fn test_is_pow2_multiple_of_1mib() { + assert!(is_pow2_multiple_of_1mib(1 << 20)); // 1 MiB + assert!(is_pow2_multiple_of_1mib(16 * 1024 * 1024)); // 16 MiB default + assert!(is_pow2_multiple_of_1mib(1 << 25)); + assert!(is_pow2_multiple_of_1mib(MAX_BITROT_BLOCK_SIZE)); // 64 MiB boundary + assert!(!is_pow2_multiple_of_1mib(0)); + assert!(!is_pow2_multiple_of_1mib(1 << 19)); // 512 KiB, too small + assert!(!is_pow2_multiple_of_1mib(3 << 20)); // 3 MiB, not pow2 + assert!(!is_pow2_multiple_of_1mib(128 * 1024 * 1024)); // pow2 but > MAX_BITROT_BLOCK_SIZE + assert!(!is_pow2_multiple_of_1mib(DEFAULT_BITROT_BLOCK_SIZE as u32 + 1)); + } + + #[test] + fn test_expected_block_count() { + assert_eq!(expected_block_count(0, 16), 0); + assert_eq!(expected_block_count(1, 16), 1); + assert_eq!(expected_block_count(16, 16), 1); + assert_eq!(expected_block_count(17, 16), 2); + assert_eq!(expected_block_count(32, 16), 2); + assert_eq!(expected_block_count(100, 0), 0); + } + + #[test] + fn test_pack_unpack_roundtrip() { + let vals = vec![0x0102_0304u32, 0xdead_beef, 0, u32::MAX]; + let packed = pack_u32_le(&vals); + assert_eq!(packed.len(), 16); + // Verify little-endian byte order of the first entry. + assert_eq!(&packed[0..4], &[0x04, 0x03, 0x02, 0x01]); + assert_eq!(unpack_u32_le(&packed), vals); + } + + #[test] + fn test_builder_block_boundaries() { + // block_size = 4; feed 10 bytes in chunks that cross boundaries. + let mut b = ShardChecksumBuilder::new(4); + let data = b"0123456789"; + b.write(&data[0..3]); // partial block 0 + b.write(&data[3..7]); // completes block 0 (idx 3), fills block 1 + b.write(&data[7..10]); // partial block 2 + let (covered, packed) = b.finalize(); + assert_eq!(covered, 10); + let crcs = unpack_u32_le(&packed); + // ceil(10/4) = 3 blocks + assert_eq!(crcs.len(), 3); + // Compare against direct per-block CRCs. + assert_eq!(crcs[0], CRC::new(&data[0..4]).0); + assert_eq!(crcs[1], CRC::new(&data[4..8]).0); + assert_eq!(crcs[2], CRC::new(&data[8..10]).0); + } + + #[test] + fn test_builder_exact_block_multiple() { + let mut b = ShardChecksumBuilder::new(4); + b.write(b"01234567"); // exactly 2 blocks, no partial + let (covered, packed) = b.finalize(); + assert_eq!(covered, 8); + assert_eq!(unpack_u32_le(&packed).len(), 2); + } + + #[test] + fn test_save_load_roundtrip() { + let tmp = tempfile::TempDir::new().unwrap(); + let path = tmp + .path() + .join("vol.ecsum") + .to_str() + .unwrap() + .to_string(); + + let mut builder = ShardChecksumBuilder::new(DEFAULT_BITROT_BLOCK_SIZE as i64); + builder.write(b"hello world"); + let (covered, packed) = builder.finalize(); + + let prot = EcBitrotProtection { + algorithm: ChecksumAlgorithm::ChecksumCrc32c as i32, + block_size: DEFAULT_BITROT_BLOCK_SIZE as u32, + generation: 0, + ec_shard_config: Some(ec_shard_config(10, 4)), + shards: vec![EcShardChecksums { + shard_id: 0, + covered_size: covered, + block_crc32c: packed, + }], + encode_uuid: new_encode_uuid(), + }; + + save_bitrot_sidecar(&path, &prot).unwrap(); + let loaded = load_bitrot_sidecar(&path).unwrap(); + assert_eq!(loaded, prot); + } + + #[test] + fn test_load_rejects_bad_magic() { + let tmp = tempfile::TempDir::new().unwrap(); + let path = tmp.path().join("bad.ecsum").to_str().unwrap().to_string(); + std::fs::write(&path, vec![0u8; BITROT_HEADER_SIZE + 4]).unwrap(); + match load_bitrot_sidecar(&path) { + Err(BitrotLoadError::BadMagic(_)) => {} + other => panic!("expected BadMagic, got {:?}", other), + } + } + + #[test] + fn test_load_rejects_corrupted_payload() { + let tmp = tempfile::TempDir::new().unwrap(); + let path = tmp + .path() + .join("corrupt.ecsum") + .to_str() + .unwrap() + .to_string(); + let prot = EcBitrotProtection { + algorithm: ChecksumAlgorithm::ChecksumCrc32c as i32, + block_size: DEFAULT_BITROT_BLOCK_SIZE as u32, + generation: 0, + ec_shard_config: Some(ec_shard_config(10, 4)), + shards: vec![EcShardChecksums { + shard_id: 0, + covered_size: 5, + block_crc32c: pack_u32_le(&[CRC::new(b"hello").0]), + }], + encode_uuid: vec![0u8; 16], + }; + save_bitrot_sidecar(&path, &prot).unwrap(); + + // Flip a byte in the payload (after the 14-byte header). + let mut data = std::fs::read(&path).unwrap(); + let last = data.len() - 1; + data[last] ^= 0xff; + std::fs::write(&path, &data).unwrap(); + + match load_bitrot_sidecar(&path) { + Err(BitrotLoadError::CrcMismatch { .. }) => {} + other => panic!("expected CrcMismatch, got {:?}", other), + } + } + + #[test] + fn test_load_missing_is_notfound() { + let res = load_bitrot_sidecar("/nonexistent/path/x.ecsum"); + match res { + Err(BitrotLoadError::Io(e)) => assert_eq!(e.kind(), io::ErrorKind::NotFound), + other => panic!("expected Io(NotFound), got {:?}", other), + } + } + + fn good_manifest() -> EcBitrotProtection { + let mut shards = Vec::new(); + for id in 0..14u32 { + shards.push(EcShardChecksums { + shard_id: id, + covered_size: 5, + block_crc32c: pack_u32_le(&[CRC::new(b"hello").0]), + }); + } + EcBitrotProtection { + algorithm: ChecksumAlgorithm::ChecksumCrc32c as i32, + block_size: DEFAULT_BITROT_BLOCK_SIZE as u32, + generation: 0, + ec_shard_config: Some(ec_shard_config(10, 4)), + shards, + encode_uuid: vec![0u8; 16], + } + } + + #[test] + fn test_validate_manifest_ok() { + assert!(validate_manifest(&good_manifest(), 10, 4).is_ok()); + } + + #[test] + fn test_validate_rejects_wrong_algorithm() { + let mut m = good_manifest(); + m.algorithm = ChecksumAlgorithm::ChecksumNone as i32; + assert!(validate_manifest(&m, 10, 4).is_err()); + } + + #[test] + fn test_validate_rejects_bad_block_size() { + let mut m = good_manifest(); + m.block_size = 3 << 20; + assert!(validate_manifest(&m, 10, 4).is_err()); + } + + #[test] + fn test_validate_rejects_incomplete() { + let mut m = good_manifest(); + m.shards.pop(); + assert!(validate_manifest(&m, 10, 4).is_err()); + } + + #[test] + fn test_validate_rejects_duplicate_shard_id() { + let mut m = good_manifest(); + m.shards[1].shard_id = 0; + assert!(validate_manifest(&m, 10, 4).is_err()); + } + + #[test] + fn test_validate_rejects_out_of_range_id() { + let mut m = good_manifest(); + m.shards[13].shard_id = 14; + assert!(validate_manifest(&m, 10, 4).is_err()); + } + + #[test] + fn test_validate_rejects_nonpositive_covered_size() { + let mut m = good_manifest(); + m.shards[0].covered_size = 0; + assert!(validate_manifest(&m, 10, 4).is_err()); + } + + #[test] + fn test_validate_rejects_crc_count_mismatch() { + let mut m = good_manifest(); + m.shards[0].block_crc32c = vec![0u8; 8]; // 2 entries but covered_size=5 => want 1 + assert!(validate_manifest(&m, 10, 4).is_err()); + } + + #[test] + fn test_resolve_status() { + // Missing => Off. + let notfound: Result = Err(BitrotLoadError::Io( + io::Error::new(io::ErrorKind::NotFound, "x"), + )); + assert_eq!(resolve_status(¬found, 0, 10, 4), BitrotStatus::Off); + + // Integrity failure => Invalid. + let bad: Result = + Err(BitrotLoadError::BadMagic(0)); + assert_eq!(resolve_status(&bad, 0, 10, 4), BitrotStatus::Invalid); + + // Generation mismatch => Off. + let mut m = good_manifest(); + m.generation = 7; + let ok: Result = Ok(m); + assert_eq!(resolve_status(&ok, 0, 10, 4), BitrotStatus::Off); + + // Matching + valid => On. + let ok2: Result = Ok(good_manifest()); + assert_eq!(resolve_status(&ok2, 0, 10, 4), BitrotStatus::On); + + // Matching generation but invalid manifest => Invalid. + let mut bad_m = good_manifest(); + bad_m.shards.pop(); + let ok3: Result = Ok(bad_m); + assert_eq!(resolve_status(&ok3, 0, 10, 4), BitrotStatus::Invalid); + } + + #[test] + fn test_verify_shard_file_blocks() { + let tmp = tempfile::TempDir::new().unwrap(); + let path = tmp.path().join("s.ec00").to_str().unwrap().to_string(); + let block_size: i64 = 4; + let data = b"0123456789"; // 10 bytes, 3 blocks + std::fs::write(&path, data).unwrap(); + + let mut b = ShardChecksumBuilder::new(block_size); + b.write(data); + let (covered, packed) = b.finalize(); + let entry = EcShardChecksums { + shard_id: 0, + covered_size: covered, + block_crc32c: packed, + }; + + // Clean file => no mismatches. + let mm = verify_shard_file_blocks(&path, &entry, block_size).unwrap(); + assert!(mm.is_empty()); + + // Corrupt block index 1 (bytes 4..8). + let mut corrupt = data.to_vec(); + corrupt[5] ^= 0xff; + std::fs::write(&path, &corrupt).unwrap(); + let mm = verify_shard_file_blocks(&path, &entry, block_size).unwrap(); + assert_eq!(mm, vec![1]); + + // Truncation => all blocks mismatched. + std::fs::write(&path, b"012").unwrap(); + let mm = verify_shard_file_blocks(&path, &entry, block_size).unwrap(); + assert_eq!(mm, vec![0, 1, 2]); + } +} diff --git a/seaweed-volume/src/storage/erasure_coding/mod.rs b/seaweed-volume/src/storage/erasure_coding/mod.rs index b6c07b450..fd1dab928 100644 --- a/seaweed-volume/src/storage/erasure_coding/mod.rs +++ b/seaweed-volume/src/storage/erasure_coding/mod.rs @@ -3,6 +3,7 @@ //! Encodes a volume's .dat file into 10 data + 4 parity shards using //! Reed-Solomon erasure coding. Can reconstruct from any 10 of 14 shards. +pub mod ec_bitrot; pub mod ec_decoder; pub mod ec_encoder; pub mod ec_locate;