diff --git a/seaweed-volume/proto/volume_server.proto b/seaweed-volume/proto/volume_server.proto index ab5d2e5af..8db9f15e6 100644 --- a/seaweed-volume/proto/volume_server.proto +++ b/seaweed-volume/proto/volume_server.proto @@ -462,6 +462,7 @@ message VolumeEcShardsMountRequest { string collection = 2; repeated uint32 shard_ids = 3; string source_disk_type = 4; // disk type of the source volume, applied to the in-memory EC volume so heartbeats report under it (#9423) + bool recover_missing_index = 5; // first fetch a missing .ecx index from a peer so on-disk shards without a local index become mountable (#10104) } message VolumeEcShardsMountResponse { } diff --git a/seaweed-volume/src/server/grpc_server.rs b/seaweed-volume/src/server/grpc_server.rs index 1b704abbe..a46971bec 100644 --- a/seaweed-volume/src/server/grpc_server.rs +++ b/seaweed-volume/src/server/grpc_server.rs @@ -2840,6 +2840,14 @@ impl VolumeServer for VolumeGrpcService { let req = request.into_inner(); let vid = VolumeId(req.volume_id); + // Fetch a missing .ecx from a peer first so on-disk shards that never had + // a local index can be mounted (issue #10104). Driven on demand by + // ec.rebuild. volume_id 0 recovers every orphan on this server, including + // volumes the master never learned about. + if req.recover_missing_index { + crate::server::store_ec::recover_missing_ec_indexes(&self.state, req.volume_id).await; + } + // Mount one shard at a time, returning error on first failure. // Matches Go: for _, shardId := range req.ShardIds { err = vs.store.MountEcShards(...) } let mut store = self.state.store.write().unwrap(); diff --git a/seaweed-volume/src/server/store_ec.rs b/seaweed-volume/src/server/store_ec.rs index a71eb14d9..718325f6a 100644 --- a/seaweed-volume/src/server/store_ec.rs +++ b/seaweed-volume/src/server/store_ec.rs @@ -27,6 +27,7 @@ //! `RwLock` so we do not contend with the Store-level lock at all. use std::collections::HashMap; +use std::fs; use std::io; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -37,14 +38,16 @@ use tonic::Request; use crate::pb::master_pb::{self, seaweed_client::SeaweedClient, LookupEcVolumeRequest}; use crate::pb::volume_server_pb::{ - volume_server_client::VolumeServerClient, VolumeEcShardReadRequest, + volume_server_client::VolumeServerClient, CopyFileRequest, VolumeEcShardReadRequest, }; use crate::server::grpc_client::{build_grpc_endpoint, parse_grpc_address, GRPC_MAX_MESSAGE_SIZE}; use crate::server::request_id::outgoing_request_id_interceptor; -use crate::server::volume_server::VolumeServerState; +use crate::server::volume_server::{to_http_address, VolumeServerState}; use crate::storage::erasure_coding::ec_shard::ShardId; use crate::storage::needle::needle::{get_actual_size, Needle}; +use crate::storage::store_ec_reconcile::EcVolumeMissingIndex; use crate::storage::types::*; +use crate::storage::volume::volume_file_name; /// One interval's data after Phase A. enum IntervalResult { @@ -703,3 +706,227 @@ async fn recover_one_remote_ec_shard_interval( // parse_grpc_address lives in `grpc_client.rs` and is re-exported // here via the use above so this module shares a single // HTTP↔gRPC port-translation routine with grpc_server.rs. + +// --------------------------------------------------------------------------- +// Missing-index recovery (issue #10104). +// +// Mirrors weed/server/volume_grpc_erasure_coding_recover.go. EC shards whose +// .ecx index lives only on a peer server cannot be mounted locally, so the +// master never learns about them. recover_missing_ec_indexes fetches the index +// from a peer and mounts the on-disk shards. Driven on demand by +// VolumeEcShardsMount(recover_missing_index), so an operator triggers it through +// ec.rebuild rather than a background loop. +// --------------------------------------------------------------------------- + +/// Recover EC volumes whose shards sit on this server while the index lives only +/// on a peer. `filter_vid` 0 recovers every orphan on this server (including +/// volumes the master never registered); otherwise just that volume. Returns the +/// number of volumes whose index was recovered. +pub(crate) async fn recover_missing_ec_indexes( + state: &Arc, + filter_vid: u32, +) -> usize { + let missing: Vec = { + let store = state.store.read().unwrap(); + store + .collect_ec_volumes_missing_index() + .into_iter() + .filter(|m| filter_vid == 0 || m.vid.0 == filter_vid) + .collect() + }; + if missing.is_empty() { + return 0; + } + + let self_http = to_http_address(&state.self_url).into_owned(); + let mut recovered = 0usize; + for m in &missing { + let peers = match cached_lookup_ec_shard_locations(state, m.vid).await { + Ok(map) => { + let mut peers: Vec = Vec::new(); + for addrs in map.values() { + for a in addrs { + if to_http_address(a).as_ref() == self_http.as_str() { + continue; + } + if !peers.contains(a) { + peers.push(a.clone()); + } + } + } + peers + } + Err(e) => { + tracing::warn!( + volume_id = m.vid.0, + "cannot look up peers to recover missing .ecx: {}", + e + ); + continue; + } + }; + if peers.is_empty() { + tracing::warn!( + volume_id = m.vid.0, + "shards present locally but .ecx missing and no peer holds it; leaving shards unloaded" + ); + continue; + } + if fetch_ec_index_from_peers(state, m, &peers).await { + recovered += 1; + } + } + + if recovered > 0 { + state.store.write().unwrap().mount_recovered_ec_shards(); + tracing::info!( + "recovered missing EC index for {} volume(s) from peers and mounted their shards", + recovered + ); + } + recovered +} + +/// Try each peer in turn, copying the `.ecx` (required) and `.ecj` / `.vif` +/// (best-effort) into m's local dirs. The `.ecx` is an immutable encode-time +/// index, identical on every holder, so any peer's copy serves. The `.ecj` is a +/// per-holder deletion journal that differs across holders; the recovered node +/// adopts the source peer's deletion view, like a balanced or rebuilt shard. The +/// first peer with a non-empty `.ecx` wins. +async fn fetch_ec_index_from_peers( + state: &Arc, + m: &EcVolumeMissingIndex, + peers: &[String], +) -> bool { + let idx_base = volume_file_name(&m.idx_dir, &m.collection, m.vid); + let data_base = volume_file_name(&m.data_dir, &m.collection, m.vid); + let ecx_path = format!("{}.ecx", idx_base); + let ecj_path = format!("{}.ecj", idx_base); + let vif_path = format!("{}.vif", data_base); + + for peer in peers { + match fetch_ec_index_from_one_peer(state, m, peer, &ecx_path, &ecj_path, &vif_path).await { + Ok(()) => { + tracing::info!( + volume_id = m.vid.0, + peer = %peer, + "fetched missing .ecx into {}", + m.idx_dir + ); + return true; + } + Err(e) => { + // Remove any partial .ecx so a later attempt is not blocked by a stub. + let _ = fs::remove_file(&ecx_path); + tracing::debug!( + volume_id = m.vid.0, + peer = %peer, + "fetch missing .ecx failed: {}", + e + ); + } + } + } + false +} + +async fn fetch_ec_index_from_one_peer( + state: &Arc, + m: &EcVolumeMissingIndex, + peer: &str, + ecx_path: &str, + ecj_path: &str, + vif_path: &str, +) -> io::Result<()> { + let grpc_addr = + parse_grpc_address(peer).map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?; + let channel = build_grpc_endpoint(&grpc_addr, state.outgoing_grpc_tls.as_ref()) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))? + .connect_timeout(Duration::from_secs(5)) + .timeout(Duration::from_secs(30)) + .connect() + .await + .map_err(|e| io::Error::new(io::ErrorKind::Other, format!("connect {}: {}", peer, e)))?; + let mut client = VolumeServerClient::with_interceptor(channel, outgoing_request_id_interceptor) + .max_decoding_message_size(GRPC_MAX_MESSAGE_SIZE) + .max_encoding_message_size(GRPC_MAX_MESSAGE_SIZE); + + let copy_req = |ext: &str, ignore_not_found: bool| CopyFileRequest { + volume_id: m.vid.0, + collection: m.collection.clone(), + is_ec_volume: true, + ext: ext.to_string(), + compaction_revision: u32::MAX, + stop_offset: i64::MAX as u64, + ignore_source_file_not_found: ignore_not_found, + ..Default::default() + }; + + // .ecx is mandatory and written in place (create/truncate); a peer without it + // errors and the caller moves on. + let stream = client + .copy_file(copy_req(".ecx", false)) + .await + .map_err(|e| io::Error::new(io::ErrorKind::Other, format!("copy .ecx: {}", e)))? + .into_inner(); + drain_copy_stream(stream, ecx_path, false).await?; + + let meta = fs::metadata(ecx_path) + .map_err(|e| io::Error::new(io::ErrorKind::Other, format!("stat copied .ecx: {}", e)))?; + if meta.is_dir() || meta.len() == 0 { + let _ = fs::remove_file(ecx_path); + return Err(io::Error::new( + io::ErrorKind::Other, + format!("peer {} served an unusable .ecx (size {})", peer, meta.len()), + )); + } + + // .ecj is the source peer's deletion journal (appended); .vif carries EC + // params. Both are best-effort: a missing .ecj is recreated at mount and a + // missing .vif falls back to default EC parameters. A failed .ecj append + // leaves a partial file, so drop it. + match client.copy_file(copy_req(".ecj", true)).await { + Ok(resp) => { + if let Err(e) = drain_copy_stream(resp.into_inner(), ecj_path, true).await { + tracing::warn!(volume_id = m.vid.0, peer = %peer, "copy .ecj: {}", e); + let _ = fs::remove_file(ecj_path); + } + } + Err(e) => tracing::warn!(volume_id = m.vid.0, peer = %peer, "copy .ecj: {}", e), + } + + match client.copy_file(copy_req(".vif", true)).await { + Ok(resp) => { + if let Err(e) = drain_copy_stream(resp.into_inner(), vif_path, false).await { + tracing::warn!(volume_id = m.vid.0, peer = %peer, "copy .vif: {}", e); + } + } + Err(e) => tracing::warn!(volume_id = m.vid.0, peer = %peer, "copy .vif: {}", e), + } + + Ok(()) +} + +/// Drain a CopyFile stream into a local file, appending or truncating. +async fn drain_copy_stream( + mut stream: tonic::Streaming, + dest_path: &str, + append: bool, +) -> io::Result<()> { + use std::io::Write; + let mut file = if append { + fs::OpenOptions::new().create(true).append(true).open(dest_path) + } else { + fs::File::create(dest_path) + } + .map_err(|e| io::Error::new(io::ErrorKind::Other, format!("create {}: {}", dest_path, e)))?; + while let Some(chunk) = stream + .message() + .await + .map_err(|e| io::Error::new(io::ErrorKind::Other, format!("recv {}: {}", dest_path, e)))? + { + file.write_all(&chunk.file_content) + .map_err(|e| io::Error::new(io::ErrorKind::Other, format!("write {}: {}", dest_path, e)))?; + } + Ok(()) +} diff --git a/seaweed-volume/src/storage/store_ec_reconcile.rs b/seaweed-volume/src/storage/store_ec_reconcile.rs index 9c494f29a..1f9494f2f 100644 --- a/seaweed-volume/src/storage/store_ec_reconcile.rs +++ b/seaweed-volume/src/storage/store_ec_reconcile.rs @@ -15,16 +15,28 @@ //! sibling disk's index files so it can serve reads and route deletes //! through a real `.ecx` / `.ecj`. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::fs; -use tracing::{info, warn}; +use tracing::{error, info, warn}; use crate::storage::disk_location::{is_ec_shard_extension, parse_collection_volume_id_pub}; use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT; use crate::storage::store::Store; use crate::storage::types::VolumeId; +/// An EC volume with shard files on a local disk but no usable `.ecx` on any +/// local disk, so the shards cannot mount. The same-server reconcile/mirror +/// handle a `.ecx` on a sibling disk; this is the cross-server case whose index +/// must be fetched from a peer (issue #10104). Mirrors Go's `EcVolumeMissingIndex`. +#[derive(Clone, Debug)] +pub(crate) struct EcVolumeMissingIndex { + pub collection: String, + pub vid: VolumeId, + pub idx_dir: String, + pub data_dir: String, +} + pub(crate) fn ec_local_ecx_path(dir: &str, collection: &str, vid: VolumeId) -> String { if collection.is_empty() { format!("{}/{}.ecx", dir, vid.0) @@ -420,6 +432,71 @@ impl Store { } owners } + + /// Cross-server orphans: EC volumes that have shard files on a local disk but + /// no usable `.ecx` on any local disk. A `.ecx` merely on a sibling disk is + /// excluded (the same-server reconcile/mirror handles it). Scans on-disk shard + /// files, so it surfaces volumes the master never learned about — including + /// those whose every holder is missing its index. + /// + /// Mirrors `Store.CollectEcVolumesMissingIndex` in Go. + pub(crate) fn collect_ec_volumes_missing_index(&self) -> Vec { + let owners = self.index_ecx_owners(); + let mut seen: HashSet = HashSet::new(); + let mut missing = Vec::new(); + for (loc_idx, loc) in self.locations.iter().enumerate() { + for (key, _shards) in collect_orphan_ec_shards(loc, loc_idx) { + if owners.contains_key(&key) || !seen.insert(key.clone()) { + continue; + } + missing.push(EcVolumeMissingIndex { + collection: key.collection, + vid: key.vid, + idx_dir: loc.idx_directory.clone(), + data_dir: loc.directory.clone(), + }); + } + } + missing + } + + /// Mount EC shards that became loadable after a missing `.ecx` was fetched + /// onto a local disk: mirror the index onto every shard-bearing disk, mount + /// the disks that now have a local index, then fall back to the cross-disk + /// virtual mount. Mirrors `Store.MountRecoveredEcShards` in Go. + pub(crate) fn mount_recovered_ec_shards(&mut self) { + self.mirror_ec_metadata_to_shard_disks(); + self.load_orphan_ec_shards_with_local_index(); + self.reconcile_ec_shards_across_disks(); + } + + /// Mount on-disk EC shards whose `.ecx` index is now present on the same disk. + /// Unlike `reconcile_ec_shards_across_disks` it needs no sibling disk, so a + /// single-disk store recovers once its index has been fetched from a peer. + fn load_orphan_ec_shards_with_local_index(&mut self) { + let mut work: Vec<(usize, EcKey, Vec)> = Vec::new(); + for (loc_idx, loc) in self.locations.iter().enumerate() { + for (key, shards) in collect_orphan_ec_shards(loc, loc_idx) { + if !loc.has_ecx_file_on_disk(&key.collection, key.vid) { + continue; + } + let ids: Vec = shards.iter().map(|(_, sid)| *sid).collect(); + work.push((loc_idx, key, ids)); + } + } + for (loc_idx, key, ids) in work { + let loc_dir = self.locations[loc_idx].directory.clone(); + let loc = &mut self.locations[loc_idx]; + if let Err(e) = loc.mount_ec_shards(key.vid, &key.collection, &ids, "") { + error!( + volume_id = key.vid.0, + directory = %loc_dir, + "load after index recovery failed: {}", + e, + ); + } + } + } } #[cfg(test)] @@ -944,6 +1021,103 @@ mod tests { } } + #[test] + fn test_collect_missing_index_recovers_cross_server_orphan() { + // Reproduces issue #10104: shards spread across this server's disks with + // no .ecx anywhere local. collect_ec_volumes_missing_index must surface + // the volume; after the index lands on the orphan disk (as a peer fetch + // would deliver it), mount_recovered_ec_shards mounts all shards. + let tmp = TempDir::new().unwrap(); + let dir0 = tmp.path().join("data0"); + let dir1 = tmp.path().join("data1"); + std::fs::create_dir_all(&dir0).unwrap(); + std::fs::create_dir_all(&dir1).unwrap(); + + let collection = "video-recordings"; + let vid = 6190u32; + write_shard(dir0.to_str().unwrap(), collection, vid, 0); + write_shard(dir0.to_str().unwrap(), collection, vid, 5); + write_shard(dir1.to_str().unwrap(), collection, vid, 6); + + let mut store = Store::new(NeedleMapKind::InMemory); + for d in [&dir0, &dir1] { + store + .add_location( + d.to_str().unwrap(), + d.to_str().unwrap(), + 100, + DiskType::HardDrive, + MinFreeSpace::Percent(0.0), + Vec::new(), + ) + .unwrap(); + } + + // Bug state: nothing mounted. + assert!(store.locations[0].find_ec_volume(VolumeId(vid)).is_none()); + + let missing = store.collect_ec_volumes_missing_index(); + let m = missing + .iter() + .find(|m| m.vid == VolumeId(vid) && m.collection == collection) + .expect("cross-server orphan not reported"); + + // Simulate the peer fetch dropping the index onto the orphan disk. + write_index_files(&m.idx_dir, collection, vid, 10, 4); + + store.mount_recovered_ec_shards(); + + let ev0 = store.locations[0] + .find_ec_volume(VolumeId(vid)) + .expect("dir0 not mounted after recovery"); + assert!(ev0.has_shard(0) && ev0.has_shard(5), "dir0 shards missing"); + let ev1 = store.locations[1] + .find_ec_volume(VolumeId(vid)) + .expect("dir1 not mounted after recovery"); + assert!(ev1.has_shard(6), "dir1 shard missing"); + + // Nothing left to recover. + assert!(store + .collect_ec_volumes_missing_index() + .iter() + .all(|m| m.vid != VolumeId(vid))); + } + + #[test] + fn test_collect_missing_index_excludes_cross_disk_orphan() { + // A .ecx merely on a sibling disk is the same-server case the reconcile + // already handles; it must not be reported as a cross-server orphan. + let tmp = TempDir::new().unwrap(); + let dir0 = tmp.path().join("data0"); + let dir1 = tmp.path().join("data1"); + std::fs::create_dir_all(&dir0).unwrap(); + std::fs::create_dir_all(&dir1).unwrap(); + + let collection = "mybucket"; + let vid = 42u32; + write_shard(dir0.to_str().unwrap(), collection, vid, 3); + write_index_files(dir1.to_str().unwrap(), collection, vid, 10, 4); + + let mut store = Store::new(NeedleMapKind::InMemory); + for d in [&dir0, &dir1] { + store + .add_location( + d.to_str().unwrap(), + d.to_str().unwrap(), + 100, + DiskType::HardDrive, + MinFreeSpace::Percent(0.0), + Vec::new(), + ) + .unwrap(); + } + + assert!(store + .collect_ec_volumes_missing_index() + .iter() + .all(|m| m.vid != VolumeId(vid))); + } + /// Helper: build a 2-disk store where reconcile produces the /// cross-disk split layout (shards 0/12 on dir0, shard 1 + .ecx /// on dir1). Mirrors the report-from-the-issue layout that diff --git a/weed/pb/volume_server.proto b/weed/pb/volume_server.proto index 5825445d3..889b4dc73 100644 --- a/weed/pb/volume_server.proto +++ b/weed/pb/volume_server.proto @@ -464,6 +464,7 @@ message VolumeEcShardsMountRequest { string collection = 2; repeated uint32 shard_ids = 3; string source_disk_type = 4; // disk type of the source volume, applied to the in-memory EC volume so heartbeats report under it (#9423) + bool recover_missing_index = 5; // first fetch a missing .ecx index from a peer so on-disk shards without a local index become mountable (#10104) } message VolumeEcShardsMountResponse { } diff --git a/weed/pb/volume_server_pb/volume_server.pb.go b/weed/pb/volume_server_pb/volume_server.pb.go index ff0f4786d..d02b35bb1 100644 --- a/weed/pb/volume_server_pb/volume_server.pb.go +++ b/weed/pb/volume_server_pb/volume_server.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.6 -// protoc v6.33.4 +// protoc v7.35.0 // source: volume_server.proto package volume_server_pb @@ -3721,13 +3721,14 @@ func (x *VolumeEcShardsDeleteResponse) GetFullTeardownDone() bool { } type VolumeEcShardsMountRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` - Collection string `protobuf:"bytes,2,opt,name=collection,proto3" json:"collection,omitempty"` - ShardIds []uint32 `protobuf:"varint,3,rep,packed,name=shard_ids,json=shardIds,proto3" json:"shard_ids,omitempty"` - SourceDiskType string `protobuf:"bytes,4,opt,name=source_disk_type,json=sourceDiskType,proto3" json:"source_disk_type,omitempty"` // disk type of the source volume, applied to the in-memory EC volume so heartbeats report under it (#9423) - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + Collection string `protobuf:"bytes,2,opt,name=collection,proto3" json:"collection,omitempty"` + ShardIds []uint32 `protobuf:"varint,3,rep,packed,name=shard_ids,json=shardIds,proto3" json:"shard_ids,omitempty"` + SourceDiskType string `protobuf:"bytes,4,opt,name=source_disk_type,json=sourceDiskType,proto3" json:"source_disk_type,omitempty"` // disk type of the source volume, applied to the in-memory EC volume so heartbeats report under it (#9423) + RecoverMissingIndex bool `protobuf:"varint,5,opt,name=recover_missing_index,json=recoverMissingIndex,proto3" json:"recover_missing_index,omitempty"` // first fetch a missing .ecx index from a peer so on-disk shards without a local index become mountable (#10104) + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *VolumeEcShardsMountRequest) Reset() { @@ -3788,6 +3789,13 @@ func (x *VolumeEcShardsMountRequest) GetSourceDiskType() string { return "" } +func (x *VolumeEcShardsMountRequest) GetRecoverMissingIndex() bool { + if x != nil { + return x.RecoverMissingIndex + } + return false +} + type VolumeEcShardsMountResponse struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -7295,14 +7303,15 @@ const file_volume_server_proto_rawDesc = "" + "\fencode_ts_ns\x18\x05 \x01(\x03R\n" + "encodeTsNs\"L\n" + "\x1cVolumeEcShardsDeleteResponse\x12,\n" + - "\x12full_teardown_done\x18\x01 \x01(\bR\x10fullTeardownDone\"\xa0\x01\n" + + "\x12full_teardown_done\x18\x01 \x01(\bR\x10fullTeardownDone\"\xd4\x01\n" + "\x1aVolumeEcShardsMountRequest\x12\x1b\n" + "\tvolume_id\x18\x01 \x01(\rR\bvolumeId\x12\x1e\n" + "\n" + "collection\x18\x02 \x01(\tR\n" + "collection\x12\x1b\n" + "\tshard_ids\x18\x03 \x03(\rR\bshardIds\x12(\n" + - "\x10source_disk_type\x18\x04 \x01(\tR\x0esourceDiskType\"\x1d\n" + + "\x10source_disk_type\x18\x04 \x01(\tR\x0esourceDiskType\x122\n" + + "\x15recover_missing_index\x18\x05 \x01(\bR\x13recoverMissingIndex\"\x1d\n" + "\x1bVolumeEcShardsMountResponse\"z\n" + "\x1cVolumeEcShardsUnmountRequest\x12\x1b\n" + "\tvolume_id\x18\x01 \x01(\rR\bvolumeId\x12\x1b\n" + diff --git a/weed/server/volume_grpc_erasure_coding.go b/weed/server/volume_grpc_erasure_coding.go index 1d6725651..5362e0c7e 100644 --- a/weed/server/volume_grpc_erasure_coding.go +++ b/weed/server/volume_grpc_erasure_coding.go @@ -717,6 +717,14 @@ func (vs *VolumeServer) VolumeEcShardsMount(ctx context.Context, req *volume_ser glog.V(0).Infof("VolumeEcShardsMount: %v", req) + // Fetch a missing .ecx from a peer first so on-disk shards that never had a + // local index can be mounted (issue #10104). Driven on demand by ec.rebuild. + // volume_id 0 recovers every orphan on this server, including volumes the + // master never learned about. + if req.RecoverMissingIndex { + vs.recoverMissingEcIndexes(req.VolumeId) + } + for _, shardId := range req.ShardIds { err := vs.store.MountEcShards(req.Collection, needle.VolumeId(req.VolumeId), erasure_coding.ShardId(shardId), req.SourceDiskType) diff --git a/weed/server/volume_grpc_erasure_coding_recover.go b/weed/server/volume_grpc_erasure_coding_recover.go new file mode 100644 index 000000000..c05def22f --- /dev/null +++ b/weed/server/volume_grpc_erasure_coding_recover.go @@ -0,0 +1,168 @@ +package weed_server + +import ( + "context" + "fmt" + "math" + "os" + "time" + + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/operation" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage" +) + +// ecRecoveryLookupTimeout bounds the master LookupEcVolume call so a slow or +// unresponsive master cannot hang the synchronous recovery RPC. +const ecRecoveryLookupTimeout = time.Minute + +// recoverMissingEcIndexes fetches the .ecx / .ecj / .vif index files for EC +// volumes whose shards sit on this server while the index lives only on a peer, +// then mounts the now-recoverable shards. This self-heals the "shards present, +// .ecx missing everywhere local" layout (issue #10104) that the per-disk loader +// and the same-server cross-disk reconcile both leave unmounted, including for +// volumes already broken before the upgrade. +// +// filterVid limits recovery to a single volume id; 0 recovers every orphan on +// this server (used by ec.rebuild to heal volumes the master never learned +// about). It is driven on demand by VolumeEcShardsMount (recover_missing_index), +// so an operator triggers it through ec.rebuild rather than a background loop. +// Returns the number of volumes whose index was recovered. +func (vs *VolumeServer) recoverMissingEcIndexes(filterVid uint32) int { + orphans := vs.store.CollectEcVolumesMissingIndex() + if filterVid != 0 { + filtered := orphans[:0] + for _, m := range orphans { + if uint32(m.VolumeId) == filterVid { + filtered = append(filtered, m) + } + } + orphans = filtered + } + if len(orphans) == 0 { + return 0 + } + + master := vs.getCurrentMaster() + if master == "" { + glog.Warningf("cannot recover missing EC index without a master connection") + return 0 + } + self := pb.NewServerAddress(vs.store.Ip, vs.store.Port, vs.store.GrpcPort) + + recovered := 0 + for _, m := range orphans { + peers, err := vs.lookupEcVolumePeers(master, uint32(m.VolumeId), self) + if err != nil { + glog.Warningf("ec volume %d: cannot look up peers to recover missing .ecx: %v", m.VolumeId, err) + continue + } + if len(peers) == 0 { + glog.Warningf("ec volume %d: shards present locally but .ecx missing and no peer holds it; leaving shards unloaded", m.VolumeId) + continue + } + if vs.fetchEcIndexFromPeers(peers, m) { + recovered++ + } + } + + if recovered > 0 { + vs.store.MountRecoveredEcShards() + glog.V(0).Infof("recovered missing EC index for %d volume(s) from peers and mounted their shards", recovered) + } + return recovered +} + +// lookupEcVolumePeers asks the master which servers hold shards for vid and +// returns the unique peer addresses, excluding this server itself. +func (vs *VolumeServer) lookupEcVolumePeers(master pb.ServerAddress, vid uint32, self pb.ServerAddress) ([]pb.ServerAddress, error) { + ctx, cancel := context.WithTimeout(context.Background(), ecRecoveryLookupTimeout) + defer cancel() + + var peers []pb.ServerAddress + seen := make(map[pb.ServerAddress]bool) + err := operation.WithMasterServerClient(ctx, false, master, vs.grpcDialOption, func(client master_pb.SeaweedClient) error { + resp, err := client.LookupEcVolume(ctx, &master_pb.LookupEcVolumeRequest{VolumeId: vid}) + if err != nil { + return err + } + for _, shardIdLocations := range resp.ShardIdLocations { + for _, loc := range shardIdLocations.Locations { + addr := pb.NewServerAddressFromLocation(loc) + if addr.Equals(self) || seen[addr] { + continue + } + seen[addr] = true + peers = append(peers, addr) + } + } + return nil + }) + return peers, err +} + +// fetchEcIndexFromPeers tries each peer in turn, copying the .ecx (required) and +// the .ecj / .vif (best-effort) for the volume onto the local disk recorded in +// m. The .ecx is an immutable encode-time index, identical on every holder, so +// any peer's copy serves. The .ecj is a per-holder deletion journal that differs +// across holders (a delete is journaled on only one node); the recovered node +// adopts the source peer's deletion view, exactly as a balanced or rebuilt shard +// does — the EC delete model already tolerates that divergence. The first peer +// that yields a non-empty .ecx wins; a peer with no index or a 0-byte stub is +// skipped (the orphan shards are non-empty, so a 0-byte index cannot be theirs). +// A failed copy removes its partial file so a later attempt is not blocked by a +// stub. +func (vs *VolumeServer) fetchEcIndexFromPeers(peers []pb.ServerAddress, m storage.EcVolumeMissingIndex) bool { + idxBaseFileName := storage.VolumeFileName(m.IdxDir, m.Collection, int(m.VolumeId)) + dataBaseFileName := storage.VolumeFileName(m.DataDir, m.Collection, int(m.VolumeId)) + ecxPath := idxBaseFileName + ".ecx" + ecjPath := idxBaseFileName + ".ecj" + + removePartial := func(path string) { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + glog.Warningf("ec volume %d: remove partial %s: %v", m.VolumeId, path, err) + } + } + + for _, peer := range peers { + err := operation.WithVolumeServerClient(true, peer, vs.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + // .ecx is mandatory; a peer without it errors and we move on. + if _, err := vs.doCopyFile(client, true, m.Collection, uint32(m.VolumeId), math.MaxUint32, math.MaxInt64, idxBaseFileName, ".ecx", false, false, nil); err != nil { + removePartial(ecxPath) + return err + } + info, statErr := os.Stat(ecxPath) + if statErr != nil { + removePartial(ecxPath) + return fmt.Errorf("stat copied .ecx %s: %w", ecxPath, statErr) + } + if info.IsDir() || info.Size() == 0 { + removePartial(ecxPath) + return fmt.Errorf("peer %s served an unusable .ecx (size %d)", peer, info.Size()) + } + // .ecj is the source peer's deletion journal; .vif carries EC params + // and EncodeTsNs. Both are best-effort: a missing .ecj is recreated at + // mount and a missing .vif falls back to default EC parameters. A failed + // .ecj copy is an in-place append, so drop the partial file; the .vif + // copy stages and renames, leaving nothing to clean up. + if _, err := vs.doCopyFile(client, true, m.Collection, uint32(m.VolumeId), math.MaxUint32, math.MaxInt64, idxBaseFileName, ".ecj", true, true, nil); err != nil { + glog.Warningf("ec volume %d: copy .ecj from %s: %v", m.VolumeId, peer, err) + removePartial(ecjPath) + } + if _, err := vs.doCopyFile(client, true, m.Collection, uint32(m.VolumeId), math.MaxUint32, math.MaxInt64, dataBaseFileName, ".vif", false, true, nil); err != nil { + glog.Warningf("ec volume %d: copy .vif from %s: %v", m.VolumeId, peer, err) + } + return nil + }) + if err != nil { + glog.V(1).Infof("ec volume %d: fetch missing .ecx from %s failed: %v", m.VolumeId, peer, err) + continue + } + glog.V(0).Infof("ec volume %d: fetched missing .ecx from %s into %s", m.VolumeId, peer, m.IdxDir) + return true + } + return false +} diff --git a/weed/server/volume_grpc_erasure_coding_recover_test.go b/weed/server/volume_grpc_erasure_coding_recover_test.go new file mode 100644 index 000000000..9f21edc94 --- /dev/null +++ b/weed/server/volume_grpc_erasure_coding_recover_test.go @@ -0,0 +1,285 @@ +package weed_server + +import ( + "context" + "net" + "os" + "path/filepath" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/stats" + "github.com/seaweedfs/seaweedfs/weed/storage" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +// fakeMaster serves only LookupEcVolume, returning the configured holders for +// every shard of one volume. Used to drive the receiver's peer discovery. +type fakeMaster struct { + master_pb.UnimplementedSeaweedServer + volumeId uint32 + locations []*master_pb.Location +} + +func (m *fakeMaster) LookupEcVolume(_ context.Context, req *master_pb.LookupEcVolumeRequest) (*master_pb.LookupEcVolumeResponse, error) { + resp := &master_pb.LookupEcVolumeResponse{VolumeId: req.VolumeId} + if req.VolumeId != m.volumeId { + return resp, nil + } + // Recovery only needs one peer holding the index; report a couple of shards + // pointing at the holder, independent of the EC ratio. + for shardId := 0; shardId < 2; shardId++ { + resp.ShardIdLocations = append(resp.ShardIdLocations, &master_pb.LookupEcVolumeResponse_EcShardIdLocation{ + ShardId: uint32(shardId), + Locations: m.locations, + }) + } + return resp, nil +} + +// serveGrpc registers register(s) on a fresh localhost listener and returns its +// grpc port, stopping the server on test cleanup. +func serveGrpc(t *testing.T, register func(*grpc.Server)) int { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + srv := grpc.NewServer() + register(srv) + go srv.Serve(lis) + t.Cleanup(srv.Stop) + return lis.Addr().(*net.TCPAddr).Port +} + +func newRecoverTestStore(t *testing.T, dir string) *storage.Store { + t.Helper() + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", dir, err) + } + store := storage.NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "store-id", + []string{dir}, []int32{100}, []util.MinFreeSpace{{}}, "", + storage.NeedleMapInMemory, []types.DiskType{types.HardDriveType}, nil, 3, stats.DefaultDiskIOProbeConfig()) + done := make(chan struct{}) + go func() { + for { + select { + case <-store.NewEcShardsChan: + case <-store.NewVolumesChan: + case <-store.DeletedVolumesChan: + case <-store.DeletedEcShardsChan: + case <-store.StateUpdateChan: + case <-done: + return + } + } + }() + t.Cleanup(func() { + store.Close() + close(done) + }) + return store +} + +// TestFetchEcIndexFromPeers_CopiesIndexOverGrpc stands up a source volume server +// that holds the .ecx/.ecj/.vif for a volume and verifies the receiver pulls +// them over a real CopyFile gRPC stream into its own disk, the way #10104 +// recovery does when the index lives only on a peer. +func TestFetchEcIndexFromPeers_CopiesIndexOverGrpc(t *testing.T) { + const collection = "video-recordings" + vid := needle.VolumeId(6190) + + // Source server: has the index files on disk. + srcDir := filepath.Join(t.TempDir(), "src") + srcStore := newRecoverTestStore(t, srcDir) + srcBase := erasure_coding.EcShardFileName(collection, srcDir, int(vid)) + ecxBytes := make([]byte, types.NeedleMapEntrySize*3) + for i := range ecxBytes { + ecxBytes[i] = byte(i) + } + if err := os.WriteFile(srcBase+".ecx", ecxBytes, 0o644); err != nil { + t.Fatalf("write source .ecx: %v", err) + } + if err := os.WriteFile(srcBase+".ecj", []byte("journal"), 0o644); err != nil { + t.Fatalf("write source .ecj: %v", err) + } + if err := os.WriteFile(srcBase+".vif", []byte("volinfo"), 0o644); err != nil { + t.Fatalf("write source .vif: %v", err) + } + + // Serve the source over a real TCP gRPC listener. + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + grpcServer := grpc.NewServer() + volume_server_pb.RegisterVolumeServerServer(grpcServer, &VolumeServer{store: srcStore}) + go grpcServer.Serve(lis) + t.Cleanup(grpcServer.Stop) + + grpcPort := lis.Addr().(*net.TCPAddr).Port + peer := pb.NewServerAddress("127.0.0.1", grpcPort-10000, grpcPort) + + // Receiver server: empty disk, ready to receive the index. + dstDir := filepath.Join(t.TempDir(), "dst") + dstStore := newRecoverTestStore(t, dstDir) + receiver := &VolumeServer{ + store: dstStore, + grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()), + } + + m := storage.EcVolumeMissingIndex{ + Collection: collection, + VolumeId: vid, + IdxDir: dstDir, + DataDir: dstDir, + } + if !receiver.fetchEcIndexFromPeers([]pb.ServerAddress{peer}, m) { + t.Fatalf("fetchEcIndexFromPeers returned false; expected a successful copy") + } + + dstBase := erasure_coding.EcShardFileName(collection, dstDir, int(vid)) + got, err := os.ReadFile(dstBase + ".ecx") + if err != nil { + t.Fatalf("read copied .ecx: %v", err) + } + if len(got) != len(ecxBytes) { + t.Errorf("copied .ecx size = %d, want %d", len(got), len(ecxBytes)) + } + if _, err := os.Stat(dstBase + ".ecj"); err != nil { + t.Errorf("copied .ecj missing: %v", err) + } + if _, err := os.Stat(dstBase + ".vif"); err != nil { + t.Errorf("copied .vif missing: %v", err) + } +} + +// TestFetchEcIndexFromPeers_SkipsPeerWithoutIndex verifies the receiver moves on +// to the next peer when the first has no .ecx, and reports failure when no peer +// can serve a usable index. +func TestFetchEcIndexFromPeers_SkipsPeerWithoutIndex(t *testing.T) { + const collection = "video-recordings" + vid := needle.VolumeId(6191) + + // Source server with NO index files for this volume. + srcDir := filepath.Join(t.TempDir(), "src") + srcStore := newRecoverTestStore(t, srcDir) + + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + grpcServer := grpc.NewServer() + volume_server_pb.RegisterVolumeServerServer(grpcServer, &VolumeServer{store: srcStore}) + go grpcServer.Serve(lis) + t.Cleanup(grpcServer.Stop) + + grpcPort := lis.Addr().(*net.TCPAddr).Port + peer := pb.NewServerAddress("127.0.0.1", grpcPort-10000, grpcPort) + + dstDir := filepath.Join(t.TempDir(), "dst") + dstStore := newRecoverTestStore(t, dstDir) + receiver := &VolumeServer{ + store: dstStore, + grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()), + } + + m := storage.EcVolumeMissingIndex{Collection: collection, VolumeId: vid, IdxDir: dstDir, DataDir: dstDir} + if receiver.fetchEcIndexFromPeers([]pb.ServerAddress{peer}, m) { + t.Fatalf("fetchEcIndexFromPeers should fail when no peer has the index") + } + + // No stub .ecx must be left behind. + dstBase := erasure_coding.EcShardFileName(collection, dstDir, int(vid)) + if _, err := os.Stat(dstBase + ".ecx"); !os.IsNotExist(err) { + t.Errorf("a .ecx stub was left behind; stat err = %v", err) + } +} + +// TestVolumeEcShardsMount_RecoverMissingIndex drives the full on-demand path: +// VolumeEcShardsMount with recover_missing_index (volume_id 0 = recover every +// orphan on this server, as ec.rebuild broadcasts it) looks up holders via a +// (fake) master, fetches the missing .ecx/.ecj/.vif from the holding peer over +// gRPC, and mounts the previously-orphaned on-disk shards (issue #10104). +func TestVolumeEcShardsMount_RecoverMissingIndex(t *testing.T) { + const collection = "video-recordings" + vid := needle.VolumeId(6190) + + // Holder peer: serves the index files for the volume. + srcDir := filepath.Join(t.TempDir(), "src") + srcStore := newRecoverTestStore(t, srcDir) + srcBase := erasure_coding.EcShardFileName(collection, srcDir, int(vid)) + if err := os.WriteFile(srcBase+".ecx", make([]byte, types.NeedleMapEntrySize*4), 0o644); err != nil { + t.Fatalf("write source .ecx: %v", err) + } + if err := os.WriteFile(srcBase+".ecj", nil, 0o644); err != nil { + t.Fatalf("write source .ecj: %v", err) + } + if err := os.WriteFile(srcBase+".vif", []byte("volinfo"), 0o644); err != nil { + t.Fatalf("write source .vif: %v", err) + } + srcGrpcPort := serveGrpc(t, func(s *grpc.Server) { + volume_server_pb.RegisterVolumeServerServer(s, &VolumeServer{store: srcStore}) + }) + + // Fake master points every shard at the holder peer. + masterGrpcPort := serveGrpc(t, func(s *grpc.Server) { + master_pb.RegisterSeaweedServer(s, &fakeMaster{ + volumeId: uint32(vid), + locations: []*master_pb.Location{{Url: "127.0.0.1:1", GrpcPort: uint32(srcGrpcPort)}}, + }) + }) + + // Receiver: holds orphan shard files on disk, no .ecx anywhere. + dstDir := filepath.Join(t.TempDir(), "dst") + dstStore := newRecoverTestStore(t, dstDir) + const shardSize = 1 << 20 + for _, sid := range []erasure_coding.ShardId{0, 5} { + base := erasure_coding.EcShardFileName(collection, dstDir, int(vid)) + f, err := os.Create(base + erasure_coding.ToExt(int(sid))) + if err != nil { + t.Fatalf("create shard %d: %v", sid, err) + } + if err := f.Truncate(shardSize); err != nil { + f.Close() + t.Fatalf("truncate shard %d: %v", sid, err) + } + f.Close() + } + + receiver := &VolumeServer{ + store: dstStore, + grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()), + } + receiver.setCurrentMaster(pb.NewServerAddress("127.0.0.1", masterGrpcPort-10000, masterGrpcPort)) + + if _, found := dstStore.FindEcVolume(vid); found { + t.Fatalf("EC volume %d unexpectedly mounted before recovery", vid) + } + + // volume_id 0: the receiver discovers the orphan (6190) on disk itself, even + // though the request names no volume and the master never registered it here. + if _, err := receiver.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{ + RecoverMissingIndex: true, + }); err != nil { + t.Fatalf("VolumeEcShardsMount with recover_missing_index: %v", err) + } + + ev, found := dstStore.FindEcVolume(vid) + if !found { + t.Fatalf("EC volume %d not mounted after recovery", vid) + } + for _, sid := range []erasure_coding.ShardId{0, 5} { + if _, ok := ev.FindEcVolumeShard(sid); !ok { + t.Errorf("shard %d.%d not registered after recovery", vid, sid) + } + } +} diff --git a/weed/shell/command_ec_rebuild.go b/weed/shell/command_ec_rebuild.go index 2ada4c244..a18bcddc6 100644 --- a/weed/shell/command_ec_rebuild.go +++ b/weed/shell/command_ec_rebuild.go @@ -47,6 +47,11 @@ func (c *commandEcRebuild) Help() string { ec.rebuild [-c EACH_COLLECTION|] [-apply] [-maxParallelization N] [-diskType=] + Before rebuilding, asks volume servers to recover any shards left unmounted by + a missing .ecx index (the index resides only on a peer server). Such shards are + invisible to the master, so recovering them first avoids regenerating data that + is actually present (issue #10104). + Options: -collection: specify a collection name, or "EACH_COLLECTION" to process all collections -apply: actually perform the rebuild operations (default is dry-run mode) @@ -149,6 +154,11 @@ func (c *commandEcRebuild) Do(args []string, commandEnv *CommandEnv, writer io.W ewg: NewErrorWaitGroup(*maxParallelization), } + // Recover shards left unmounted by a missing .ecx index before planning: such + // shards never register with the master, so the rebuild below would treat the + // volume as short or unrepairable even though its data is intact (issue #10104). + erb.recoverMissingIndexes() + fmt.Printf("rebuildEcVolumes for %d collection(s)\n", len(collections)) for _, c := range collections { erb.rebuildEcVolumes(c) @@ -288,6 +298,61 @@ func (erb *ecRebuilder) rebuildEcVolumes(collection string) { } } +// recoverMissingIndexes asks every ec node to fetch a missing .ecx index from a +// peer and mount the on-disk shards it could not load on its own. Shards +// orphaned this way (index only on another server) are absent from the master +// topology, so without this pass ec.rebuild would regenerate or give up on +// shards whose data is actually present — and a volume whose every holder lacks +// the index would not appear in the topology at all. Each node therefore +// recovers all of its on-disk orphans (volume_id 0); an explicit -volumeIds +// list narrows that to the requested volumes. On apply it refreshes the topology +// so the rebuild planning sees the recovered shards (issue #10104). +func (erb *ecRebuilder) recoverMissingIndexes() { + erb.ecNodesMu.Lock() + nodes := append([]*EcNode(nil), erb.ecNodes...) + erb.ecNodesMu.Unlock() + if len(nodes) == 0 { + return + } + + // volume_id 0 means "recover every orphan on the node"; a -volumeIds list + // narrows recovery to those ids (each scanned across collections server-side). + vids := erb.volumeIds + if len(vids) == 0 { + vids = []needle.VolumeId{0} + } + + if !erb.applyChanges { + erb.write("would ask %d ec node(s) to recover EC shards left unmounted by a missing .ecx index\n", len(nodes)) + return + } + + for _, node := range nodes { + for _, vid := range vids { + err := operation.WithVolumeServerClient(false, pb.NewServerAddressFromDataNode(node.info), erb.commandEnv.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + _, mountErr := client.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{ + VolumeId: uint32(vid), + RecoverMissingIndex: true, + }) + return mountErr + }) + if err != nil { + erb.write("%s recover missing index (volume %d): %v\n", node.info.Id, vid, err) + } + } + } + + // Refresh topology so the rebuild planning sees shards the recovery registered. + refreshed, _, err := collectEcNodes(erb.commandEnv, erb.diskType) + if err != nil { + erb.write("failed to refresh ec nodes after index recovery: %v\n", err) + return + } + erb.ecNodesMu.Lock() + erb.ecNodes = refreshed + erb.ecNodesMu.Unlock() +} + func (erb *ecRebuilder) rebuildOneEcVolume(collection string, volumeId needle.VolumeId, locations EcShardLocations, rebuilder *EcNode) error { if !erb.isLocked() { return fmt.Errorf("lock is lost") diff --git a/weed/storage/store_ec_recover.go b/weed/storage/store_ec_recover.go new file mode 100644 index 000000000..74de9d78b --- /dev/null +++ b/weed/storage/store_ec_recover.go @@ -0,0 +1,83 @@ +package storage + +import ( + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" +) + +// EcVolumeMissingIndex names an EC volume whose .ec?? shard files are present on +// this store but whose .ecx index is missing on every local disk, so the shards +// cannot be mounted. The per-disk loader and the same-server cross-disk reconcile +// both leave such shards unmounted because the index lives only on a different +// server (issue #10104). The recovery path fetches the index from a peer that +// still holds it and drops it into IdxDir / DataDir, then re-runs the mount. +type EcVolumeMissingIndex struct { + Collection string + VolumeId needle.VolumeId + IdxDir string // destination for the fetched .ecx / .ecj + DataDir string // destination for the fetched .vif +} + +// CollectEcVolumesMissingIndex returns every EC volume that has shard files on a +// local disk but no usable .ecx on any local disk — the cross-server orphans a +// peer index fetch must recover. Volumes whose index merely sits on a sibling +// disk are excluded (the same-server reconcile/mirror handles those). The +// destination dirs are taken from the first disk found holding orphan shards; +// MountRecoveredEcShards mirrors the index onto the remaining shard-bearing disks. +// +// It scans on-disk shard files directly, so it surfaces volumes the master never +// learned about — including those whose every holder is missing its index. +func (s *Store) CollectEcVolumesMissingIndex() []EcVolumeMissingIndex { + ecxOwners := s.indexEcxOwners() + + seen := make(map[ecKeyForReconcile]bool) + var missing []EcVolumeMissingIndex + for _, loc := range s.Locations { + for key := range loc.collectOrphanEcShards() { + if _, hasLocalIndex := ecxOwners[key]; hasLocalIndex { + continue + } + if seen[key] { + continue + } + seen[key] = true + missing = append(missing, EcVolumeMissingIndex{ + Collection: key.collection, + VolumeId: key.vid, + IdxDir: loc.IdxDirectory, + DataDir: loc.Directory, + }) + } + } + return missing +} + +// MountRecoveredEcShards mounts EC shards that became loadable after a missing +// .ecx was fetched onto a local disk. It mirrors the index onto every +// shard-bearing disk, mounts the disks that now have a local index, and falls +// back to the cross-disk virtual mount for any disk the mirror could not reach. +// Each shard load announces itself through ecShardNotifyHandler so the master +// learns about the now-registered shards. +func (s *Store) MountRecoveredEcShards() { + s.mirrorEcMetadataToShardDisks() + s.loadOrphanEcShardsWithLocalIndex() + s.reconcileEcShardsAcrossDisks() +} + +// loadOrphanEcShardsWithLocalIndex mounts on-disk EC shards whose .ecx index is +// now present on the same disk. Unlike reconcileEcShardsAcrossDisks it does not +// require a sibling disk, so a single-disk store recovers too once its index has +// been fetched from a peer. +func (s *Store) loadOrphanEcShardsWithLocalIndex() { + for _, loc := range s.Locations { + orphans := loc.collectOrphanEcShards() + for key, shards := range orphans { + if !loc.HasEcxFileOnDisk(key.collection, key.vid) { + continue + } + if err := loc.loadEcShards(shards, key.collection, key.vid, loc.ecShardNotifyHandler); err != nil { + glog.Errorf("ec volume %d on %s: load after index recovery failed: %v", key.vid, loc.Directory, err) + } + } + } +} diff --git a/weed/storage/store_ec_recover_test.go b/weed/storage/store_ec_recover_test.go new file mode 100644 index 000000000..ecc0c7905 --- /dev/null +++ b/weed/storage/store_ec_recover_test.go @@ -0,0 +1,201 @@ +package storage + +import ( + "os" + "path/filepath" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/stats" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/storage/volume_info" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +// newDrainingStore builds a Store over dirs and drains its announcement +// channels in the background so loads never block. Files planted before this +// call are picked up by the startup loaders; files planted after are not. +func newDrainingStore(t *testing.T, dirs []string) *Store { + t.Helper() + for _, d := range dirs { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", d, err) + } + } + maxCounts := make([]int32, len(dirs)) + minFree := make([]util.MinFreeSpace, len(dirs)) + diskTypes := make([]types.DiskType, len(dirs)) + for i := range dirs { + maxCounts[i] = 100 + diskTypes[i] = types.HardDriveType + } + store := NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "store-id", + dirs, maxCounts, minFree, "", + NeedleMapInMemory, diskTypes, nil, 3, stats.DefaultDiskIOProbeConfig()) + done := make(chan struct{}) + go func() { + for { + select { + case <-store.NewEcShardsChan: + case <-store.NewVolumesChan: + case <-store.DeletedVolumesChan: + case <-store.DeletedEcShardsChan: + case <-store.StateUpdateChan: + case <-done: + return + } + } + }() + t.Cleanup(func() { + store.Close() + close(done) + }) + return store +} + +func plantEcShard(t *testing.T, dir, collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, size int64) { + t.Helper() + base := erasure_coding.EcShardFileName(collection, dir, int(vid)) + f, err := os.Create(base + erasure_coding.ToExt(int(shardId))) + if err != nil { + t.Fatalf("create shard %d.%d: %v", vid, shardId, err) + } + if err := f.Truncate(size); err != nil { + f.Close() + t.Fatalf("truncate shard %d.%d: %v", vid, shardId, err) + } + f.Close() +} + +func plantEcIndex(t *testing.T, idxDir, dataDir, collection string, vid needle.VolumeId, datSize int64) { + t.Helper() + idxBase := erasure_coding.EcShardFileName(collection, idxDir, int(vid)) + if err := os.WriteFile(idxBase+".ecx", make([]byte, types.NeedleMapEntrySize), 0o644); err != nil { + t.Fatalf("write .ecx: %v", err) + } + if err := os.WriteFile(idxBase+".ecj", nil, 0o644); err != nil { + t.Fatalf("write .ecj: %v", err) + } + dataBase := erasure_coding.EcShardFileName(collection, dataDir, int(vid)) + if err := volume_info.SaveVolumeInfo(dataBase+".vif", &volume_server_pb.VolumeInfo{ + Version: uint32(needle.Version3), + DatFileSize: datSize, + EcShardConfig: &volume_server_pb.EcShardConfig{ + DataShards: 10, + ParityShards: 4, + }, + }); err != nil { + t.Fatalf("save .vif: %v", err) + } +} + +// TestEcIndexRecovery_CrossServerOrphan reproduces issue #10104: a volume +// server reboots with EC shards on disk but no .ecx index anywhere local (the +// index lives only on a peer server). The startup loaders leave the shards +// unmounted, so the master never learns about them. After the recovery path +// drops the index fetched from a peer onto the local disk, MountRecoveredEcShards +// must mount and announce the shards. +func TestEcIndexRecovery_CrossServerOrphan(t *testing.T) { + tempDir := t.TempDir() + dir0 := filepath.Join(tempDir, "disk0") + dir1 := filepath.Join(tempDir, "disk1") + + const collection = "video-recordings" + vid := needle.VolumeId(6190) + const datSize int64 = 10 * 1024 * 1024 + shardSize := calculateExpectedShardSize(datSize, 10) + + // Pre-seed the on-disk layout the issue describes: shards spread across + // this server's disks, with no .ecx / .ecj / .vif on any of them. + for _, d := range []string{dir0, dir1} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", d, err) + } + } + plantEcShard(t, dir0, collection, vid, 0, shardSize) + plantEcShard(t, dir0, collection, vid, 5, shardSize) + plantEcShard(t, dir1, collection, vid, 6, shardSize) + + store := newDrainingStore(t, []string{dir0, dir1}) + + // Bug reproduction: the startup loaders left every shard unmounted. + if _, found := store.FindEcVolume(vid); found { + t.Fatalf("EC volume %d unexpectedly mounted without any local .ecx", vid) + } + + // The recovery scan must surface this volume as missing its index. + m, ok := findMissingIndex(store.CollectEcVolumesMissingIndex(), collection, vid) + if !ok { + t.Fatalf("CollectEcVolumesMissingIndex did not report volume %d as a cross-server orphan", vid) + } + + // Simulate the peer fetch: the server-side path copies .ecx/.ecj/.vif from a + // peer that still has them into m.IdxDir / m.DataDir. + plantEcIndex(t, m.IdxDir, m.DataDir, collection, vid, datSize) + + store.MountRecoveredEcShards() + + // All shards must now be mounted on the disks that physically hold them. + loc0 := store.Locations[0] + ev, found := loc0.FindEcVolume(vid) + if !found { + t.Fatalf("EC volume %d not mounted on disk0 after index recovery", vid) + } + for _, sid := range []erasure_coding.ShardId{0, 5} { + if _, ok := ev.FindEcVolumeShard(sid); !ok { + t.Errorf("shard %d.%d not registered on disk0 after recovery", vid, sid) + } + } + loc1 := store.Locations[1] + ev1, found := loc1.FindEcVolume(vid) + if !found { + t.Fatalf("EC volume %d not mounted on disk1 after index recovery", vid) + } + if _, ok := ev1.FindEcVolumeShard(6); !ok { + t.Errorf("shard %d.6 not registered on disk1 after recovery", vid) + } + + // A second scan must report nothing left to recover. + if _, again := findMissingIndex(store.CollectEcVolumesMissingIndex(), collection, vid); again { + t.Errorf("CollectEcVolumesMissingIndex still reports volume %d after recovery", vid) + } +} + +// findMissingIndex returns the entry for (collection, vid) among the reported +// missing-index volumes, if present. +func findMissingIndex(missing []EcVolumeMissingIndex, collection string, vid needle.VolumeId) (EcVolumeMissingIndex, bool) { + for _, m := range missing { + if m.Collection == collection && m.VolumeId == vid { + return m, true + } + } + return EcVolumeMissingIndex{}, false +} + +// TestCollectEcVolumesMissingIndex_ExcludesCrossDiskOrphan guards the boundary +// with the existing same-server reconcile: a volume whose .ecx merely sits on a +// sibling disk is NOT a cross-server orphan and must not be reported here — +// mirrorEcMetadataToShardDisks / reconcileEcShardsAcrossDisks already mount it. +func TestCollectEcVolumesMissingIndex_ExcludesCrossDiskOrphan(t *testing.T) { + tempDir := t.TempDir() + dir0 := filepath.Join(tempDir, "disk0") + dir1 := filepath.Join(tempDir, "disk1") + store := newDrainingStore(t, []string{dir0, dir1}) + + const collection = "mybucket" + vid := needle.VolumeId(42) + const datSize int64 = 10 * 1024 * 1024 + shardSize := calculateExpectedShardSize(datSize, 10) + + // Plant after NewStore so the startup reconcile does not consume the layout; + // CollectEcVolumesMissingIndex is what's under test. Shard on disk0, index on + // disk1 — the cross-disk layout handled by the same-server reconcile. + plantEcShard(t, dir0, collection, vid, 3, shardSize) + plantEcIndex(t, dir1, dir1, collection, vid, datSize) + + if _, missing := findMissingIndex(store.CollectEcVolumesMissingIndex(), collection, vid); missing { + t.Errorf("cross-disk .ecx must not be reported as a cross-server orphan") + } +}