diff --git a/seaweed-volume/src/server/grpc_server.rs b/seaweed-volume/src/server/grpc_server.rs index 89bf995a1..bd708ce8c 100644 --- a/seaweed-volume/src/server/grpc_server.rs +++ b/seaweed-volume/src/server/grpc_server.rs @@ -2464,7 +2464,10 @@ impl VolumeServer for VolumeGrpcService { )); } - // Rebuild missing shards, searching all locations for input shards + // Rebuild missing shards, searching all locations for input shards. + // Pass other_dirs so shards on sibling disks are found even when the + // primary rebuild dir doesn't hold them. + let other_dir_refs: Vec<&str> = other_dirs.iter().map(|s| s.as_str()).collect(); crate::storage::erasure_coding::ec_encoder::rebuild_ec_files( &rebuild_dir, collection, @@ -2472,6 +2475,7 @@ impl VolumeServer for VolumeGrpcService { &missing, data_shards as usize, parity_shards as usize, + &other_dir_refs, ) .map_err(|e| Status::internal(format!("RebuildEcFiles: {}", e)))?; @@ -2485,11 +2489,23 @@ impl VolumeServer for VolumeGrpcService { rebuild_idx_dir }; + // .ecx may live in a different directory than the data shards (split + // data/idx layout). Ensure rebuild_dir is searchable for data shards + // even when it isn't the chosen .ecx rebuild directory and isn't + // already covered by other_dir_refs. + let mut ecx_dir_refs: Vec<&str> = + Vec::with_capacity(other_dir_refs.len() + usize::from(ecx_rebuild_dir != rebuild_dir)); + if ecx_rebuild_dir != rebuild_dir { + ecx_dir_refs.push(rebuild_dir.as_str()); + } + ecx_dir_refs.extend(other_dir_refs.iter().copied()); + crate::storage::erasure_coding::ec_encoder::rebuild_ecx_file( &ecx_rebuild_dir, collection, vid, data_shards as usize, + &ecx_dir_refs, ) .map_err(|e| Status::internal(format!("RebuildEcxFile: {}", e)))?; diff --git a/seaweed-volume/src/storage/erasure_coding/ec_encoder.rs b/seaweed-volume/src/storage/erasure_coding/ec_encoder.rs index 58fcce5c1..4bec3c228 100644 --- a/seaweed-volume/src/storage/erasure_coding/ec_encoder.rs +++ b/seaweed-volume/src/storage/erasure_coding/ec_encoder.rs @@ -73,6 +73,10 @@ pub fn write_ec_files( /// Rebuild missing EC shard files from existing shards using Reed-Solomon reconstruct. /// /// This does not require the `.dat` file, only the existing `.ecXX` shard files. +/// +/// `additional_dirs` lists sibling disk locations to search for existing shards when +/// they are not found in `dir`. This is required on multi-disk volume servers where +/// shards for the same volume may be spread across disks. pub fn rebuild_ec_files( dir: &str, collection: &str, @@ -80,6 +84,7 @@ pub fn rebuild_ec_files( missing_shard_ids: &[u32], data_shards: usize, parity_shards: usize, + additional_dirs: &[&str], ) -> io::Result<()> { if missing_shard_ids.is_empty() { return Ok(()); @@ -93,7 +98,9 @@ pub fn rebuild_ec_files( .map(|i| EcVolumeShard::new(dir, collection, volume_id, i)) .collect(); - // Determine the exact shard size from the first available existing shard + // Determine the exact shard size from the first available existing shard. + // When a shard is not found in `dir`, search `additional_dirs` (sibling disks + // on the same volume server) before failing — mirrors Go's findShardFile logic. let mut shard_size = 0; for (i, shard) in shards.iter_mut().enumerate() { if !missing_shard_ids.contains(&(i as u32)) { @@ -103,10 +110,26 @@ pub fn rebuild_ec_files( shard_size = size; } } else { - return Err(io::Error::new( - io::ErrorKind::NotFound, - format!("missing non-rebuild shard {}", i), - )); + // Try sibling disk locations before giving up. + let mut found = false; + for &other_dir in additional_dirs { + let mut alt = EcVolumeShard::new(other_dir, collection, volume_id, i as u8); + if let Ok(_) = alt.open() { + let size = alt.file_size(); + if size > shard_size { + shard_size = size; + } + *shard = alt; + found = true; + break; + } + } + if !found { + return Err(io::Error::new( + io::ErrorKind::NotFound, + format!("missing non-rebuild shard {}", i), + )); + } } } } @@ -331,11 +354,16 @@ fn write_sorted_ecx_from_idx(idx_path: &str, ecx_path: &str) -> io::Result<()> { /// content from the EC data shards, walks through needle headers to extract /// (needle_id, offset, size) entries, deduplicates them, and writes a sorted /// .ecx index file. +/// +/// `additional_dirs` lists sibling disk locations to search for data shards when +/// they are not found in `dir` — mirrors the same fallback used by rebuild_ec_files, +/// required on multi-disk volume servers where shards may be spread across disks. pub fn rebuild_ecx_file( dir: &str, collection: &str, volume_id: VolumeId, data_shards: usize, + additional_dirs: &[&str], ) -> io::Result<()> { use crate::storage::needle::needle::get_actual_size; use crate::storage::super_block::SUPER_BLOCK_SIZE; @@ -343,21 +371,34 @@ pub fn rebuild_ecx_file( let base = volume_file_name(dir, collection, volume_id); let ecx_path = format!("{}.ecx", base); - // Open data shards to read logical .dat content + // Open data shards to read logical .dat content. When a shard isn't found + // in `dir`, search `additional_dirs` (sibling disks on the same volume + // server) before giving up. let mut shards: Vec = (0..data_shards as u8) .map(|i| EcVolumeShard::new(dir, collection, volume_id, i)) .collect(); - for shard in &mut shards { + for (i, shard) in shards.iter_mut().enumerate() { if let Err(_) = shard.open() { - // If a data shard is missing, we can't rebuild ecx - for s in &mut shards { - s.close(); + let mut found = false; + for &other_dir in additional_dirs { + let mut alt = EcVolumeShard::new(other_dir, collection, volume_id, i as u8); + if alt.open().is_ok() { + *shard = alt; + found = true; + break; + } + } + if !found { + // If a data shard is missing, we can't rebuild ecx + for s in &mut shards { + s.close(); + } + return Err(io::Error::new( + io::ErrorKind::NotFound, + format!("cannot open data shard for ecx rebuild"), + )); } - return Err(io::Error::new( - io::ErrorKind::NotFound, - format!("cannot open data shard for ecx rebuild"), - )); } } @@ -717,7 +758,7 @@ mod tests { // Rebuild a different (genuinely missing) shard. std::fs::remove_file(format!("{}/1.ec07", dir)).unwrap(); - let res = rebuild_ec_files(&dir, "", VolumeId(1), &[7], 10, 4); + let res = rebuild_ec_files(&dir, "", VolumeId(1), &[7], 10, 4, &[]); assert!(res.is_err(), "truncated input shard must abort the rebuild"); } @@ -730,7 +771,7 @@ mod tests { let dropped = format!("{}/1.ec07", dir); std::fs::remove_file(&dropped).unwrap(); - rebuild_ec_files(&dir, "", VolumeId(1), &[7], 10, 4).unwrap(); + rebuild_ec_files(&dir, "", VolumeId(1), &[7], 10, 4, &[]).unwrap(); assert!( std::path::Path::new(&dropped).exists(), "rebuilt shard .ec07 should exist" @@ -741,6 +782,124 @@ mod tests { ); } + // Multi-disk rebuild: shards for the same volume are split across two + // directories (simulating a volume server with two disk locations). The + // shard to rebuild is in the primary dir; some of the input shards needed + // for RS reconstruction exist only in the secondary dir. Passing the + // secondary dir via `additional_dirs` must allow the rebuild to succeed + // where it would previously return "missing non-rebuild shard N". + #[test] + fn test_rebuild_ec_files_multi_disk() { + let tmp_primary = TempDir::new().unwrap(); + let tmp_secondary = TempDir::new().unwrap(); + let primary = tmp_primary.path().to_str().unwrap().to_string(); + let secondary = tmp_secondary.path().to_str().unwrap().to_string(); + + // Encode into primary dir first (all 14 shards land there). + let dir = encode_sample_volume(&tmp_primary); + assert_eq!(dir, primary); + + // Simulate a multi-disk layout: move shards 0, 4, 8 to the secondary + // dir, as if the master had placed them on a different disk. + let moved_shards: &[u8] = &[0, 4, 8]; + for &shard_id in moved_shards { + let src = format!("{}/1.ec{:02}", primary, shard_id); + let dst = format!("{}/1.ec{:02}", secondary, shard_id); + std::fs::rename(&src, &dst).unwrap(); + } + + // Remove shard 7 from primary — this is the shard we want to rebuild. + let missing_path = format!("{}/1.ec07", primary); + std::fs::remove_file(&missing_path).unwrap(); + + // Without additional_dirs the rebuild must fail: shards 0, 4, 8 are + // not in primary and shard 7 cannot be reconstructed without them. + let res = rebuild_ec_files(&primary, "", VolumeId(1), &[7], 10, 4, &[]); + assert!( + res.is_err(), + "rebuild without additional_dirs must fail when input shards are on another disk" + ); + assert!( + !std::path::Path::new(&missing_path).exists(), + "failed rebuild must not leave a partial shard file behind" + ); + + // With additional_dirs pointing at the secondary, the rebuild must succeed. + rebuild_ec_files( + &primary, + "", + VolumeId(1), + &[7], + 10, + 4, + &[secondary.as_str()], + ) + .unwrap(); + + assert!( + std::path::Path::new(&missing_path).exists(), + "rebuilt shard .ec07 should exist in primary dir" + ); + assert!( + std::fs::metadata(&missing_path).unwrap().len() > 0, + "rebuilt shard must be non-empty" + ); + } + + // Multi-disk .ecx rebuild: data shards (0-9) needed to reconstruct the + // logical .dat content are split across two directories, as on a + // multi-disk volume server. Passing the secondary dir via + // `additional_dirs` must allow rebuild_ecx_file to find them and + // succeed where it would previously fail with + // "cannot open data shard for ecx rebuild". + #[test] + fn test_rebuild_ecx_file_multi_disk() { + let tmp_primary = TempDir::new().unwrap(); + let tmp_secondary = TempDir::new().unwrap(); + let primary = tmp_primary.path().to_str().unwrap().to_string(); + let secondary = tmp_secondary.path().to_str().unwrap().to_string(); + + let dir = encode_sample_volume(&tmp_primary); + assert_eq!(dir, primary); + + // Move some data shards (0-9) to the secondary dir, simulating a + // multi-disk layout where not all data shards landed on the same disk. + let moved_shards: &[u8] = &[1, 3, 6]; + for &shard_id in moved_shards { + let src = format!("{}/1.ec{:02}", primary, shard_id); + let dst = format!("{}/1.ec{:02}", secondary, shard_id); + std::fs::rename(&src, &dst).unwrap(); + } + + // Delete the existing .ecx to force a rebuild. + let ecx_path = format!("{}/1.ecx", primary); + std::fs::remove_file(&ecx_path).unwrap(); + + // Without additional_dirs, rebuild must fail: shards 1, 3, 6 are not + // in primary and the full logical .dat content can't be reconstructed. + let res = rebuild_ecx_file(&primary, "", VolumeId(1), 10, &[]); + assert!( + res.is_err(), + "ecx rebuild without additional_dirs must fail when data shards are on another disk" + ); + assert!( + !std::path::Path::new(&ecx_path).exists(), + "failed ecx rebuild must not leave a partial .ecx file behind" + ); + + // With additional_dirs pointing at the secondary, the rebuild must succeed. + rebuild_ecx_file(&primary, "", VolumeId(1), 10, &[secondary.as_str()]).unwrap(); + + assert!( + std::path::Path::new(&ecx_path).exists(), + "rebuilt .ecx should exist in primary dir" + ); + assert!( + std::fs::metadata(&ecx_path).unwrap().len() > 0, + "rebuilt .ecx must be non-empty" + ); + } + #[test] fn test_reed_solomon_basic() { let data_shards = 10;