mirrors/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-08-21 14:46:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(volume [rust] + ec): search sibling disk locations when rebuilding missing EC shards + .ecx files (#10145)

Browse Source 
* fix(volume [rust] + ec): search sibling disk locations when rebuilding missing EC shards

* fix(volume [rust] + ec): apply sibling-disk shard lookup to .ecx rebuild as well

* fix(volume [rust] + ec): include rebuild_dir in .ecx rebuild's shard search dirs

---------

Co-authored-by: adri <adri@digitalunited.net>
This commit is contained in:
adri
2026-06-30 00:07:44 -07:00
committed by GitHub
co-authored by adri
parent 72009c607b
commit 1df7a0e653
2 changed files with 192 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
seaweed-volume/src/server/grpc_server.rs
+17 -1
View File
@@ -2464,7 +2464,10 @@ impl VolumeServer for VolumeGrpcService {
));
}
// Rebuild missing shards, searching all locations for input shards
// Rebuild missing shards, searching all locations for input shards.
// Pass other_dirs so shards on sibling disks are found even when the
// primary rebuild dir doesn't hold them.
let other_dir_refs: Vec<&str> = other_dirs.iter().map(|s| s.as_str()).collect();
crate::storage::erasure_coding::ec_encoder::rebuild_ec_files(
&rebuild_dir,
collection,
@@ -2472,6 +2475,7 @@ impl VolumeServer for VolumeGrpcService {
&missing,
data_shards as usize,
parity_shards as usize,
&other_dir_refs,
)
.map_err(|e| Status::internal(format!("RebuildEcFiles: {}", e)))?;
@@ -2485,11 +2489,23 @@ impl VolumeServer for VolumeGrpcService {
rebuild_idx_dir
};
// .ecx may live in a different directory than the data shards (split
// data/idx layout). Ensure rebuild_dir is searchable for data shards
// even when it isn't the chosen .ecx rebuild directory and isn't
// already covered by other_dir_refs.
let mut ecx_dir_refs: Vec<&str> =
Vec::with_capacity(other_dir_refs.len() + usize::from(ecx_rebuild_dir != rebuild_dir));
if ecx_rebuild_dir != rebuild_dir {
ecx_dir_refs.push(rebuild_dir.as_str());
}
ecx_dir_refs.extend(other_dir_refs.iter().copied());
crate::storage::erasure_coding::ec_encoder::rebuild_ecx_file(
&ecx_rebuild_dir,
collection,
vid,
data_shards as usize,
&ecx_dir_refs,
)
.map_err(|e| Status::internal(format!("RebuildEcxFile: {}", e)))?;
seaweed-volume/src/storage/erasure_coding/ec_encoder.rs
+175 -16
View File
@@ -73,6 +73,10 @@ pub fn write_ec_files(
/// Rebuild missing EC shard files from existing shards using Reed-Solomon reconstruct.
///
/// This does not require the `.dat` file, only the existing `.ecXX` shard files.
///
/// `additional_dirs` lists sibling disk locations to search for existing shards when
/// they are not found in `dir`. This is required on multi-disk volume servers where
/// shards for the same volume may be spread across disks.
pub fn rebuild_ec_files(
dir: &str,
collection: &str,
@@ -80,6 +84,7 @@ pub fn rebuild_ec_files(
missing_shard_ids: &[u32],
data_shards: usize,
parity_shards: usize,
additional_dirs: &[&str],
) -> io::Result<()> {
if missing_shard_ids.is_empty() {
return Ok(());
@@ -93,7 +98,9 @@ pub fn rebuild_ec_files(
.map(|i| EcVolumeShard::new(dir, collection, volume_id, i))
.collect();
// Determine the exact shard size from the first available existing shard
// Determine the exact shard size from the first available existing shard.
// When a shard is not found in `dir`, search `additional_dirs` (sibling disks
// on the same volume server) before failing — mirrors Go's findShardFile logic.
let mut shard_size = 0;
for (i, shard) in shards.iter_mut().enumerate() {
if !missing_shard_ids.contains(&(i as u32)) {
@@ -103,10 +110,26 @@ pub fn rebuild_ec_files(
shard_size = size;
}
} else {
return Err(io::Error::new(
io::ErrorKind::NotFound,
format!("missing non-rebuild shard {}", i),
));
// Try sibling disk locations before giving up.
let mut found = false;
for &other_dir in additional_dirs {
let mut alt = EcVolumeShard::new(other_dir, collection, volume_id, i as u8);
if let Ok(_) = alt.open() {
let size = alt.file_size();
if size > shard_size {
shard_size = size;
}
*shard = alt;
found = true;
break;
}
}
if !found {
return Err(io::Error::new(
io::ErrorKind::NotFound,
format!("missing non-rebuild shard {}", i),
));
}
}
}
}
@@ -331,11 +354,16 @@ fn write_sorted_ecx_from_idx(idx_path: &str, ecx_path: &str) -> io::Result<()> {
/// content from the EC data shards, walks through needle headers to extract
/// (needle_id, offset, size) entries, deduplicates them, and writes a sorted
/// .ecx index file.
///
/// `additional_dirs` lists sibling disk locations to search for data shards when
/// they are not found in `dir` — mirrors the same fallback used by rebuild_ec_files,
/// required on multi-disk volume servers where shards may be spread across disks.
pub fn rebuild_ecx_file(
dir: &str,
collection: &str,
volume_id: VolumeId,
data_shards: usize,
additional_dirs: &[&str],
) -> io::Result<()> {
use crate::storage::needle::needle::get_actual_size;
use crate::storage::super_block::SUPER_BLOCK_SIZE;
@@ -343,21 +371,34 @@ pub fn rebuild_ecx_file(
let base = volume_file_name(dir, collection, volume_id);
let ecx_path = format!("{}.ecx", base);
// Open data shards to read logical .dat content
// Open data shards to read logical .dat content. When a shard isn't found
// in `dir`, search `additional_dirs` (sibling disks on the same volume
// server) before giving up.
let mut shards: Vec<EcVolumeShard> = (0..data_shards as u8)
.map(|i| EcVolumeShard::new(dir, collection, volume_id, i))
.collect();
for shard in &mut shards {
for (i, shard) in shards.iter_mut().enumerate() {
if let Err(_) = shard.open() {
// If a data shard is missing, we can't rebuild ecx
for s in &mut shards {
s.close();
let mut found = false;
for &other_dir in additional_dirs {
let mut alt = EcVolumeShard::new(other_dir, collection, volume_id, i as u8);
if alt.open().is_ok() {
*shard = alt;
found = true;
break;
}
}
if !found {
// If a data shard is missing, we can't rebuild ecx
for s in &mut shards {
s.close();
}
return Err(io::Error::new(
io::ErrorKind::NotFound,
format!("cannot open data shard for ecx rebuild"),
));
}
return Err(io::Error::new(
io::ErrorKind::NotFound,
format!("cannot open data shard for ecx rebuild"),
));
}
}
@@ -717,7 +758,7 @@ mod tests {
// Rebuild a different (genuinely missing) shard.
std::fs::remove_file(format!("{}/1.ec07", dir)).unwrap();
let res = rebuild_ec_files(&dir, "", VolumeId(1), &[7], 10, 4);
let res = rebuild_ec_files(&dir, "", VolumeId(1), &[7], 10, 4, &[]);
assert!(res.is_err(), "truncated input shard must abort the rebuild");
}
@@ -730,7 +771,7 @@ mod tests {
let dropped = format!("{}/1.ec07", dir);
std::fs::remove_file(&dropped).unwrap();
rebuild_ec_files(&dir, "", VolumeId(1), &[7], 10, 4).unwrap();
rebuild_ec_files(&dir, "", VolumeId(1), &[7], 10, 4, &[]).unwrap();
assert!(
std::path::Path::new(&dropped).exists(),
"rebuilt shard .ec07 should exist"
@@ -741,6 +782,124 @@ mod tests {
);
}
// Multi-disk rebuild: shards for the same volume are split across two
// directories (simulating a volume server with two disk locations). The
// shard to rebuild is in the primary dir; some of the input shards needed
// for RS reconstruction exist only in the secondary dir. Passing the
// secondary dir via `additional_dirs` must allow the rebuild to succeed
// where it would previously return "missing non-rebuild shard N".
#[test]
fn test_rebuild_ec_files_multi_disk() {
let tmp_primary = TempDir::new().unwrap();
let tmp_secondary = TempDir::new().unwrap();
let primary = tmp_primary.path().to_str().unwrap().to_string();
let secondary = tmp_secondary.path().to_str().unwrap().to_string();
// Encode into primary dir first (all 14 shards land there).
let dir = encode_sample_volume(&tmp_primary);
assert_eq!(dir, primary);
// Simulate a multi-disk layout: move shards 0, 4, 8 to the secondary
// dir, as if the master had placed them on a different disk.
let moved_shards: &[u8] = &[0, 4, 8];
for &shard_id in moved_shards {
let src = format!("{}/1.ec{:02}", primary, shard_id);
let dst = format!("{}/1.ec{:02}", secondary, shard_id);
std::fs::rename(&src, &dst).unwrap();
}
// Remove shard 7 from primary — this is the shard we want to rebuild.
let missing_path = format!("{}/1.ec07", primary);
std::fs::remove_file(&missing_path).unwrap();
// Without additional_dirs the rebuild must fail: shards 0, 4, 8 are
// not in primary and shard 7 cannot be reconstructed without them.
let res = rebuild_ec_files(&primary, "", VolumeId(1), &[7], 10, 4, &[]);
assert!(
res.is_err(),
"rebuild without additional_dirs must fail when input shards are on another disk"
);
assert!(
!std::path::Path::new(&missing_path).exists(),
"failed rebuild must not leave a partial shard file behind"
);
// With additional_dirs pointing at the secondary, the rebuild must succeed.
rebuild_ec_files(
&primary,
"",
VolumeId(1),
&[7],
10,
4,
&[secondary.as_str()],
)
.unwrap();
assert!(
std::path::Path::new(&missing_path).exists(),
"rebuilt shard .ec07 should exist in primary dir"
);
assert!(
std::fs::metadata(&missing_path).unwrap().len() > 0,
"rebuilt shard must be non-empty"
);
}
// Multi-disk .ecx rebuild: data shards (0-9) needed to reconstruct the
// logical .dat content are split across two directories, as on a
// multi-disk volume server. Passing the secondary dir via
// `additional_dirs` must allow rebuild_ecx_file to find them and
// succeed where it would previously fail with
// "cannot open data shard for ecx rebuild".
#[test]
fn test_rebuild_ecx_file_multi_disk() {
let tmp_primary = TempDir::new().unwrap();
let tmp_secondary = TempDir::new().unwrap();
let primary = tmp_primary.path().to_str().unwrap().to_string();
let secondary = tmp_secondary.path().to_str().unwrap().to_string();
let dir = encode_sample_volume(&tmp_primary);
assert_eq!(dir, primary);
// Move some data shards (0-9) to the secondary dir, simulating a
// multi-disk layout where not all data shards landed on the same disk.
let moved_shards: &[u8] = &[1, 3, 6];
for &shard_id in moved_shards {
let src = format!("{}/1.ec{:02}", primary, shard_id);
let dst = format!("{}/1.ec{:02}", secondary, shard_id);
std::fs::rename(&src, &dst).unwrap();
}
// Delete the existing .ecx to force a rebuild.
let ecx_path = format!("{}/1.ecx", primary);
std::fs::remove_file(&ecx_path).unwrap();
// Without additional_dirs, rebuild must fail: shards 1, 3, 6 are not
// in primary and the full logical .dat content can't be reconstructed.
let res = rebuild_ecx_file(&primary, "", VolumeId(1), 10, &[]);
assert!(
res.is_err(),
"ecx rebuild without additional_dirs must fail when data shards are on another disk"
);
assert!(
!std::path::Path::new(&ecx_path).exists(),
"failed ecx rebuild must not leave a partial .ecx file behind"
);
// With additional_dirs pointing at the secondary, the rebuild must succeed.
rebuild_ecx_file(&primary, "", VolumeId(1), 10, &[secondary.as_str()]).unwrap();
assert!(
std::path::Path::new(&ecx_path).exists(),
"rebuilt .ecx should exist in primary dir"
);
assert!(
std::fs::metadata(&ecx_path).unwrap().len() > 0,
"rebuilt .ecx must be non-empty"
);
}
#[test]
fn test_reed_solomon_basic() {
let data_shards = 10;