ec: recover EC shards whose .ecx index lives only on a peer server (#10108)

* ec: recover EC shards whose .ecx index lives only on a peer server

A volume server that boots with EC shard files on disk but no .ecx index
on any local disk cannot mount the shards, so the master never learns
about them. ec.rebuild works off master-registered shards, so it sees the
volume as short and gives up even though the shard data is intact.

Add an operator-triggered recovery: VolumeEcShardsMount gains a
recover_missing_index flag that makes the volume server fetch the missing
.ecx (plus .ecj/.vif) from a peer holding it and mount the on-disk shards.
ec.rebuild runs this across the cluster before planning, so orphaned
shards register and the rebuild sees the true shard set.

.ecx is an immutable encode-time index, identical on every holder. .ecj
is a per-holder deletion journal that differs across holders, so the
recovered node adopts the source peer's deletion view, like a balanced or
rebuilt shard does.

* ec: mirror missing-index recovery into the Rust volume server

Port the #10104 recovery to seaweed-volume so the Rust volume server
self-heals the same layout: EC shards on disk with the .ecx index only on
a peer. Adds collect_ec_volumes_missing_index / mount_recovered_ec_shards
to the store, recover_missing_ec_indexes (master LookupEcVolume + peer
CopyFile fetch + mount) to the server, and the recover_missing_index flag
on VolumeEcShardsMount.

.ecx is the immutable encode-time index, identical on every holder. .ecj
is a per-holder deletion journal, so the recovered node adopts the source
peer's deletion view, matching the Go path.

seaweed-volume/proto/volume_server.proto

@@ -462,6 +462,7 @@ message VolumeEcShardsMountRequest {
     string collection = 2;
     repeated uint32 shard_ids = 3;
     string source_disk_type = 4; // disk type of the source volume, applied to the in-memory EC volume so heartbeats report under it (#9423)
+    bool recover_missing_index = 5; // first fetch a missing .ecx index from a peer so on-disk shards without a local index become mountable (#10104)
 }
 message VolumeEcShardsMountResponse {
 }

seaweed-volume/src/server/grpc_server.rs

@@ -2840,6 +2840,14 @@ impl VolumeServer for VolumeGrpcService {
         let req = request.into_inner();
         let vid = VolumeId(req.volume_id);
 
+        // Fetch a missing .ecx from a peer first so on-disk shards that never had
+        // a local index can be mounted (issue #10104). Driven on demand by
+        // ec.rebuild. volume_id 0 recovers every orphan on this server, including
+        // volumes the master never learned about.
+        if req.recover_missing_index {
+            crate::server::store_ec::recover_missing_ec_indexes(&self.state, req.volume_id).await;
+        }
+
         // Mount one shard at a time, returning error on first failure.
         // Matches Go: for _, shardId := range req.ShardIds { err = vs.store.MountEcShards(...) }
         let mut store = self.state.store.write().unwrap();

seaweed-volume/src/server/store_ec.rs

@@ -27,6 +27,7 @@
 //! `RwLock` so we do not contend with the Store-level lock at all.
 
 use std::collections::HashMap;
+use std::fs;
 use std::io;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
@@ -37,14 +38,16 @@ use tonic::Request;
 
 use crate::pb::master_pb::{self, seaweed_client::SeaweedClient, LookupEcVolumeRequest};
 use crate::pb::volume_server_pb::{
-    volume_server_client::VolumeServerClient, VolumeEcShardReadRequest,
+    volume_server_client::VolumeServerClient, CopyFileRequest, VolumeEcShardReadRequest,
 };
 use crate::server::grpc_client::{build_grpc_endpoint, parse_grpc_address, GRPC_MAX_MESSAGE_SIZE};
 use crate::server::request_id::outgoing_request_id_interceptor;
-use crate::server::volume_server::VolumeServerState;
+use crate::server::volume_server::{to_http_address, VolumeServerState};
 use crate::storage::erasure_coding::ec_shard::ShardId;
 use crate::storage::needle::needle::{get_actual_size, Needle};
+use crate::storage::store_ec_reconcile::EcVolumeMissingIndex;
 use crate::storage::types::*;
+use crate::storage::volume::volume_file_name;
 
 /// One interval's data after Phase A.
 enum IntervalResult {
@@ -703,3 +706,227 @@ async fn recover_one_remote_ec_shard_interval(
 // parse_grpc_address lives in `grpc_client.rs` and is re-exported
 // here via the use above so this module shares a single
 // HTTP↔gRPC port-translation routine with grpc_server.rs.
+
+// ---------------------------------------------------------------------------
+// Missing-index recovery (issue #10104).
+//
+// Mirrors weed/server/volume_grpc_erasure_coding_recover.go. EC shards whose
+// .ecx index lives only on a peer server cannot be mounted locally, so the
+// master never learns about them. recover_missing_ec_indexes fetches the index
+// from a peer and mounts the on-disk shards. Driven on demand by
+// VolumeEcShardsMount(recover_missing_index), so an operator triggers it through
+// ec.rebuild rather than a background loop.
+// ---------------------------------------------------------------------------
+
+/// Recover EC volumes whose shards sit on this server while the index lives only
+/// on a peer. `filter_vid` 0 recovers every orphan on this server (including
+/// volumes the master never registered); otherwise just that volume. Returns the
+/// number of volumes whose index was recovered.
+pub(crate) async fn recover_missing_ec_indexes(
+    state: &Arc<VolumeServerState>,
+    filter_vid: u32,
+) -> usize {
+    let missing: Vec<EcVolumeMissingIndex> = {
+        let store = state.store.read().unwrap();
+        store
+            .collect_ec_volumes_missing_index()
+            .into_iter()
+            .filter(|m| filter_vid == 0 || m.vid.0 == filter_vid)
+            .collect()
+    };
+    if missing.is_empty() {
+        return 0;
+    }
+
+    let self_http = to_http_address(&state.self_url).into_owned();
+    let mut recovered = 0usize;
+    for m in &missing {
+        let peers = match cached_lookup_ec_shard_locations(state, m.vid).await {
+            Ok(map) => {
+                let mut peers: Vec<String> = Vec::new();
+                for addrs in map.values() {
+                    for a in addrs {
+                        if to_http_address(a).as_ref() == self_http.as_str() {
+                            continue;
+                        }
+                        if !peers.contains(a) {
+                            peers.push(a.clone());
+                        }
+                    }
+                }
+                peers
+            }
+            Err(e) => {
+                tracing::warn!(
+                    volume_id = m.vid.0,
+                    "cannot look up peers to recover missing .ecx: {}",
+                    e
+                );
+                continue;
+            }
+        };
+        if peers.is_empty() {
+            tracing::warn!(
+                volume_id = m.vid.0,
+                "shards present locally but .ecx missing and no peer holds it; leaving shards unloaded"
+            );
+            continue;
+        }
+        if fetch_ec_index_from_peers(state, m, &peers).await {
+            recovered += 1;
+        }
+    }
+
+    if recovered > 0 {
+        state.store.write().unwrap().mount_recovered_ec_shards();
+        tracing::info!(
+            "recovered missing EC index for {} volume(s) from peers and mounted their shards",
+            recovered
+        );
+    }
+    recovered
+}
+
+/// Try each peer in turn, copying the `.ecx` (required) and `.ecj` / `.vif`
+/// (best-effort) into m's local dirs. The `.ecx` is an immutable encode-time
+/// index, identical on every holder, so any peer's copy serves. The `.ecj` is a
+/// per-holder deletion journal that differs across holders; the recovered node
+/// adopts the source peer's deletion view, like a balanced or rebuilt shard. The
+/// first peer with a non-empty `.ecx` wins.
+async fn fetch_ec_index_from_peers(
+    state: &Arc<VolumeServerState>,
+    m: &EcVolumeMissingIndex,
+    peers: &[String],
+) -> bool {
+    let idx_base = volume_file_name(&m.idx_dir, &m.collection, m.vid);
+    let data_base = volume_file_name(&m.data_dir, &m.collection, m.vid);
+    let ecx_path = format!("{}.ecx", idx_base);
+    let ecj_path = format!("{}.ecj", idx_base);
+    let vif_path = format!("{}.vif", data_base);
+
+    for peer in peers {
+        match fetch_ec_index_from_one_peer(state, m, peer, &ecx_path, &ecj_path, &vif_path).await {
+            Ok(()) => {
+                tracing::info!(
+                    volume_id = m.vid.0,
+                    peer = %peer,
+                    "fetched missing .ecx into {}",
+                    m.idx_dir
+                );
+                return true;
+            }
+            Err(e) => {
+                // Remove any partial .ecx so a later attempt is not blocked by a stub.
+                let _ = fs::remove_file(&ecx_path);
+                tracing::debug!(
+                    volume_id = m.vid.0,
+                    peer = %peer,
+                    "fetch missing .ecx failed: {}",
+                    e
+                );
+            }
+        }
+    }
+    false
+}
+
+async fn fetch_ec_index_from_one_peer(
+    state: &Arc<VolumeServerState>,
+    m: &EcVolumeMissingIndex,
+    peer: &str,
+    ecx_path: &str,
+    ecj_path: &str,
+    vif_path: &str,
+) -> io::Result<()> {
+    let grpc_addr =
+        parse_grpc_address(peer).map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
+    let channel = build_grpc_endpoint(&grpc_addr, state.outgoing_grpc_tls.as_ref())
+        .map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))?
+        .connect_timeout(Duration::from_secs(5))
+        .timeout(Duration::from_secs(30))
+        .connect()
+        .await
+        .map_err(|e| io::Error::new(io::ErrorKind::Other, format!("connect {}: {}", peer, e)))?;
+    let mut client = VolumeServerClient::with_interceptor(channel, outgoing_request_id_interceptor)
+        .max_decoding_message_size(GRPC_MAX_MESSAGE_SIZE)
+        .max_encoding_message_size(GRPC_MAX_MESSAGE_SIZE);
+
+    let copy_req = |ext: &str, ignore_not_found: bool| CopyFileRequest {
+        volume_id: m.vid.0,
+        collection: m.collection.clone(),
+        is_ec_volume: true,
+        ext: ext.to_string(),
+        compaction_revision: u32::MAX,
+        stop_offset: i64::MAX as u64,
+        ignore_source_file_not_found: ignore_not_found,
+        ..Default::default()
+    };
+
+    // .ecx is mandatory and written in place (create/truncate); a peer without it
+    // errors and the caller moves on.
+    let stream = client
+        .copy_file(copy_req(".ecx", false))
+        .await
+        .map_err(|e| io::Error::new(io::ErrorKind::Other, format!("copy .ecx: {}", e)))?
+        .into_inner();
+    drain_copy_stream(stream, ecx_path, false).await?;
+
+    let meta = fs::metadata(ecx_path)
+        .map_err(|e| io::Error::new(io::ErrorKind::Other, format!("stat copied .ecx: {}", e)))?;
+    if meta.is_dir() || meta.len() == 0 {
+        let _ = fs::remove_file(ecx_path);
+        return Err(io::Error::new(
+            io::ErrorKind::Other,
+            format!("peer {} served an unusable .ecx (size {})", peer, meta.len()),
+        ));
+    }
+
+    // .ecj is the source peer's deletion journal (appended); .vif carries EC
+    // params. Both are best-effort: a missing .ecj is recreated at mount and a
+    // missing .vif falls back to default EC parameters. A failed .ecj append
+    // leaves a partial file, so drop it.
+    match client.copy_file(copy_req(".ecj", true)).await {
+        Ok(resp) => {
+            if let Err(e) = drain_copy_stream(resp.into_inner(), ecj_path, true).await {
+                tracing::warn!(volume_id = m.vid.0, peer = %peer, "copy .ecj: {}", e);
+                let _ = fs::remove_file(ecj_path);
+            }
+        }
+        Err(e) => tracing::warn!(volume_id = m.vid.0, peer = %peer, "copy .ecj: {}", e),
+    }
+
+    match client.copy_file(copy_req(".vif", true)).await {
+        Ok(resp) => {
+            if let Err(e) = drain_copy_stream(resp.into_inner(), vif_path, false).await {
+                tracing::warn!(volume_id = m.vid.0, peer = %peer, "copy .vif: {}", e);
+            }
+        }
+        Err(e) => tracing::warn!(volume_id = m.vid.0, peer = %peer, "copy .vif: {}", e),
+    }
+
+    Ok(())
+}
+
+/// Drain a CopyFile stream into a local file, appending or truncating.
+async fn drain_copy_stream(
+    mut stream: tonic::Streaming<crate::pb::volume_server_pb::CopyFileResponse>,
+    dest_path: &str,
+    append: bool,
+) -> io::Result<()> {
+    use std::io::Write;
+    let mut file = if append {
+        fs::OpenOptions::new().create(true).append(true).open(dest_path)
+    } else {
+        fs::File::create(dest_path)
+    }
+    .map_err(|e| io::Error::new(io::ErrorKind::Other, format!("create {}: {}", dest_path, e)))?;
+    while let Some(chunk) = stream
+        .message()
+        .await
+        .map_err(|e| io::Error::new(io::ErrorKind::Other, format!("recv {}: {}", dest_path, e)))?
+    {
+        file.write_all(&chunk.file_content)
+            .map_err(|e| io::Error::new(io::ErrorKind::Other, format!("write {}: {}", dest_path, e)))?;
+    }
+    Ok(())
+}

seaweed-volume/src/storage/store_ec_reconcile.rs

@@ -15,16 +15,28 @@
 //! sibling disk's index files so it can serve reads and route deletes
 //! through a real `.ecx` / `.ecj`.
 
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::fs;
 
-use tracing::{info, warn};
+use tracing::{error, info, warn};
 
 use crate::storage::disk_location::{is_ec_shard_extension, parse_collection_volume_id_pub};
 use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT;
 use crate::storage::store::Store;
 use crate::storage::types::VolumeId;
 
+/// An EC volume with shard files on a local disk but no usable `.ecx` on any
+/// local disk, so the shards cannot mount. The same-server reconcile/mirror
+/// handle a `.ecx` on a sibling disk; this is the cross-server case whose index
+/// must be fetched from a peer (issue #10104). Mirrors Go's `EcVolumeMissingIndex`.
+#[derive(Clone, Debug)]
+pub(crate) struct EcVolumeMissingIndex {
+    pub collection: String,
+    pub vid: VolumeId,
+    pub idx_dir: String,
+    pub data_dir: String,
+}
+
 pub(crate) fn ec_local_ecx_path(dir: &str, collection: &str, vid: VolumeId) -> String {
     if collection.is_empty() {
         format!("{}/{}.ecx", dir, vid.0)
@@ -420,6 +432,71 @@ impl Store {
         }
         owners
     }
+
+    /// Cross-server orphans: EC volumes that have shard files on a local disk but
+    /// no usable `.ecx` on any local disk. A `.ecx` merely on a sibling disk is
+    /// excluded (the same-server reconcile/mirror handles it). Scans on-disk shard
+    /// files, so it surfaces volumes the master never learned about — including
+    /// those whose every holder is missing its index.
+    ///
+    /// Mirrors `Store.CollectEcVolumesMissingIndex` in Go.
+    pub(crate) fn collect_ec_volumes_missing_index(&self) -> Vec<EcVolumeMissingIndex> {
+        let owners = self.index_ecx_owners();
+        let mut seen: HashSet<EcKey> = HashSet::new();
+        let mut missing = Vec::new();
+        for (loc_idx, loc) in self.locations.iter().enumerate() {
+            for (key, _shards) in collect_orphan_ec_shards(loc, loc_idx) {
+                if owners.contains_key(&key) || !seen.insert(key.clone()) {
+                    continue;
+                }
+                missing.push(EcVolumeMissingIndex {
+                    collection: key.collection,
+                    vid: key.vid,
+                    idx_dir: loc.idx_directory.clone(),
+                    data_dir: loc.directory.clone(),
+                });
+            }
+        }
+        missing
+    }
+
+    /// Mount EC shards that became loadable after a missing `.ecx` was fetched
+    /// onto a local disk: mirror the index onto every shard-bearing disk, mount
+    /// the disks that now have a local index, then fall back to the cross-disk
+    /// virtual mount. Mirrors `Store.MountRecoveredEcShards` in Go.
+    pub(crate) fn mount_recovered_ec_shards(&mut self) {
+        self.mirror_ec_metadata_to_shard_disks();
+        self.load_orphan_ec_shards_with_local_index();
+        self.reconcile_ec_shards_across_disks();
+    }
+
+    /// Mount on-disk EC shards whose `.ecx` index is now present on the same disk.
+    /// Unlike `reconcile_ec_shards_across_disks` it needs no sibling disk, so a
+    /// single-disk store recovers once its index has been fetched from a peer.
+    fn load_orphan_ec_shards_with_local_index(&mut self) {
+        let mut work: Vec<(usize, EcKey, Vec<u32>)> = Vec::new();
+        for (loc_idx, loc) in self.locations.iter().enumerate() {
+            for (key, shards) in collect_orphan_ec_shards(loc, loc_idx) {
+                if !loc.has_ecx_file_on_disk(&key.collection, key.vid) {
+                    continue;
+                }
+                let ids: Vec<u32> = shards.iter().map(|(_, sid)| *sid).collect();
+                work.push((loc_idx, key, ids));
+            }
+        }
+        for (loc_idx, key, ids) in work {
+            let loc_dir = self.locations[loc_idx].directory.clone();
+            let loc = &mut self.locations[loc_idx];
+            if let Err(e) = loc.mount_ec_shards(key.vid, &key.collection, &ids, "") {
+                error!(
+                    volume_id = key.vid.0,
+                    directory = %loc_dir,
+                    "load after index recovery failed: {}",
+                    e,
+                );
+            }
+        }
+    }
 }
 
 #[cfg(test)]
@@ -944,6 +1021,103 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_collect_missing_index_recovers_cross_server_orphan() {
+        // Reproduces issue #10104: shards spread across this server's disks with
+        // no .ecx anywhere local. collect_ec_volumes_missing_index must surface
+        // the volume; after the index lands on the orphan disk (as a peer fetch
+        // would deliver it), mount_recovered_ec_shards mounts all shards.
+        let tmp = TempDir::new().unwrap();
+        let dir0 = tmp.path().join("data0");
+        let dir1 = tmp.path().join("data1");
+        std::fs::create_dir_all(&dir0).unwrap();
+        std::fs::create_dir_all(&dir1).unwrap();
+
+        let collection = "video-recordings";
+        let vid = 6190u32;
+        write_shard(dir0.to_str().unwrap(), collection, vid, 0);
+        write_shard(dir0.to_str().unwrap(), collection, vid, 5);
+        write_shard(dir1.to_str().unwrap(), collection, vid, 6);
+
+        let mut store = Store::new(NeedleMapKind::InMemory);
+        for d in [&dir0, &dir1] {
+            store
+                .add_location(
+                    d.to_str().unwrap(),
+                    d.to_str().unwrap(),
+                    100,
+                    DiskType::HardDrive,
+                    MinFreeSpace::Percent(0.0),
+                    Vec::new(),
+                )
+                .unwrap();
+        }
+
+        // Bug state: nothing mounted.
+        assert!(store.locations[0].find_ec_volume(VolumeId(vid)).is_none());
+
+        let missing = store.collect_ec_volumes_missing_index();
+        let m = missing
+            .iter()
+            .find(|m| m.vid == VolumeId(vid) && m.collection == collection)
+            .expect("cross-server orphan not reported");
+
+        // Simulate the peer fetch dropping the index onto the orphan disk.
+        write_index_files(&m.idx_dir, collection, vid, 10, 4);
+
+        store.mount_recovered_ec_shards();
+
+        let ev0 = store.locations[0]
+            .find_ec_volume(VolumeId(vid))
+            .expect("dir0 not mounted after recovery");
+        assert!(ev0.has_shard(0) && ev0.has_shard(5), "dir0 shards missing");
+        let ev1 = store.locations[1]
+            .find_ec_volume(VolumeId(vid))
+            .expect("dir1 not mounted after recovery");
+        assert!(ev1.has_shard(6), "dir1 shard missing");
+
+        // Nothing left to recover.
+        assert!(store
+            .collect_ec_volumes_missing_index()
+            .iter()
+            .all(|m| m.vid != VolumeId(vid)));
+    }
+
+    #[test]
+    fn test_collect_missing_index_excludes_cross_disk_orphan() {
+        // A .ecx merely on a sibling disk is the same-server case the reconcile
+        // already handles; it must not be reported as a cross-server orphan.
+        let tmp = TempDir::new().unwrap();
+        let dir0 = tmp.path().join("data0");
+        let dir1 = tmp.path().join("data1");
+        std::fs::create_dir_all(&dir0).unwrap();
+        std::fs::create_dir_all(&dir1).unwrap();
+
+        let collection = "mybucket";
+        let vid = 42u32;
+        write_shard(dir0.to_str().unwrap(), collection, vid, 3);
+        write_index_files(dir1.to_str().unwrap(), collection, vid, 10, 4);
+
+        let mut store = Store::new(NeedleMapKind::InMemory);
+        for d in [&dir0, &dir1] {
+            store
+                .add_location(
+                    d.to_str().unwrap(),
+                    d.to_str().unwrap(),
+                    100,
+                    DiskType::HardDrive,
+                    MinFreeSpace::Percent(0.0),
+                    Vec::new(),
+                )
+                .unwrap();
+        }
+
+        assert!(store
+            .collect_ec_volumes_missing_index()
+            .iter()
+            .all(|m| m.vid != VolumeId(vid)));
+    }
+
     /// Helper: build a 2-disk store where reconcile produces the
     /// cross-disk split layout (shards 0/12 on dir0, shard 1 + .ecx
     /// on dir1). Mirrors the report-from-the-issue layout that

weed/pb/volume_server.proto

@@ -464,6 +464,7 @@ message VolumeEcShardsMountRequest {
     string collection = 2;
     repeated uint32 shard_ids = 3;
     string source_disk_type = 4; // disk type of the source volume, applied to the in-memory EC volume so heartbeats report under it (#9423)
+    bool recover_missing_index = 5; // first fetch a missing .ecx index from a peer so on-disk shards without a local index become mountable (#10104)
 }
 message VolumeEcShardsMountResponse {
 }

weed/pb/volume_server_pb/volume_server.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.36.6
-// 	protoc        v6.33.4
+// 	protoc        v7.35.0
 // source: volume_server.proto
 
 package volume_server_pb
@@ -3721,13 +3721,14 @@ func (x *VolumeEcShardsDeleteResponse) GetFullTeardownDone() bool {
 }
 
 type VolumeEcShardsMountRequest struct {
-	state          protoimpl.MessageState `protogen:"open.v1"`
-	VolumeId       uint32                 `protobuf:"varint,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"`
-	Collection     string                 `protobuf:"bytes,2,opt,name=collection,proto3" json:"collection,omitempty"`
-	ShardIds       []uint32               `protobuf:"varint,3,rep,packed,name=shard_ids,json=shardIds,proto3" json:"shard_ids,omitempty"`
-	SourceDiskType string                 `protobuf:"bytes,4,opt,name=source_disk_type,json=sourceDiskType,proto3" json:"source_disk_type,omitempty"` // disk type of the source volume, applied to the in-memory EC volume so heartbeats report under it (#9423)
-	unknownFields  protoimpl.UnknownFields
-	sizeCache      protoimpl.SizeCache
+	state               protoimpl.MessageState `protogen:"open.v1"`
+	VolumeId            uint32                 `protobuf:"varint,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"`
+	Collection          string                 `protobuf:"bytes,2,opt,name=collection,proto3" json:"collection,omitempty"`
+	ShardIds            []uint32               `protobuf:"varint,3,rep,packed,name=shard_ids,json=shardIds,proto3" json:"shard_ids,omitempty"`
+	SourceDiskType      string                 `protobuf:"bytes,4,opt,name=source_disk_type,json=sourceDiskType,proto3" json:"source_disk_type,omitempty"`                 // disk type of the source volume, applied to the in-memory EC volume so heartbeats report under it (#9423)
+	RecoverMissingIndex bool                   `protobuf:"varint,5,opt,name=recover_missing_index,json=recoverMissingIndex,proto3" json:"recover_missing_index,omitempty"` // first fetch a missing .ecx index from a peer so on-disk shards without a local index become mountable (#10104)
+	unknownFields       protoimpl.UnknownFields
+	sizeCache           protoimpl.SizeCache
 }
 
 func (x *VolumeEcShardsMountRequest) Reset() {
@@ -3788,6 +3789,13 @@ func (x *VolumeEcShardsMountRequest) GetSourceDiskType() string {
 	return ""
 }
 
+func (x *VolumeEcShardsMountRequest) GetRecoverMissingIndex() bool {
+	if x != nil {
+		return x.RecoverMissingIndex
+	}
+	return false
+}
+
 type VolumeEcShardsMountResponse struct {
 	state         protoimpl.MessageState `protogen:"open.v1"`
 	unknownFields protoimpl.UnknownFields
@@ -7295,14 +7303,15 @@ const file_volume_server_proto_rawDesc = "" +
 	"\fencode_ts_ns\x18\x05 \x01(\x03R\n" +
 	"encodeTsNs\"L\n" +
 	"\x1cVolumeEcShardsDeleteResponse\x12,\n" +
-	"\x12full_teardown_done\x18\x01 \x01(\bR\x10fullTeardownDone\"\xa0\x01\n" +
+	"\x12full_teardown_done\x18\x01 \x01(\bR\x10fullTeardownDone\"\xd4\x01\n" +
 	"\x1aVolumeEcShardsMountRequest\x12\x1b\n" +
 	"\tvolume_id\x18\x01 \x01(\rR\bvolumeId\x12\x1e\n" +
 	"\n" +
 	"collection\x18\x02 \x01(\tR\n" +
 	"collection\x12\x1b\n" +
 	"\tshard_ids\x18\x03 \x03(\rR\bshardIds\x12(\n" +
-	"\x10source_disk_type\x18\x04 \x01(\tR\x0esourceDiskType\"\x1d\n" +
+	"\x10source_disk_type\x18\x04 \x01(\tR\x0esourceDiskType\x122\n" +
+	"\x15recover_missing_index\x18\x05 \x01(\bR\x13recoverMissingIndex\"\x1d\n" +
 	"\x1bVolumeEcShardsMountResponse\"z\n" +
 	"\x1cVolumeEcShardsUnmountRequest\x12\x1b\n" +
 	"\tvolume_id\x18\x01 \x01(\rR\bvolumeId\x12\x1b\n" +

weed/server/volume_grpc_erasure_coding.go

@@ -717,6 +717,14 @@ func (vs *VolumeServer) VolumeEcShardsMount(ctx context.Context, req *volume_ser
 
 	glog.V(0).Infof("VolumeEcShardsMount: %v", req)
 
+	// Fetch a missing .ecx from a peer first so on-disk shards that never had a
+	// local index can be mounted (issue #10104). Driven on demand by ec.rebuild.
+	// volume_id 0 recovers every orphan on this server, including volumes the
+	// master never learned about.
+	if req.RecoverMissingIndex {
+		vs.recoverMissingEcIndexes(req.VolumeId)
+	}
+
 	for _, shardId := range req.ShardIds {
 		err := vs.store.MountEcShards(req.Collection, needle.VolumeId(req.VolumeId), erasure_coding.ShardId(shardId), req.SourceDiskType)
 

weed/server/volume_grpc_erasure_coding_recover.go

@@ -0,0 +1,168 @@
+package weed_server
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"os"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+)
+
+// ecRecoveryLookupTimeout bounds the master LookupEcVolume call so a slow or
+// unresponsive master cannot hang the synchronous recovery RPC.
+const ecRecoveryLookupTimeout = time.Minute
+
+// recoverMissingEcIndexes fetches the .ecx / .ecj / .vif index files for EC
+// volumes whose shards sit on this server while the index lives only on a peer,
+// then mounts the now-recoverable shards. This self-heals the "shards present,
+// .ecx missing everywhere local" layout (issue #10104) that the per-disk loader
+// and the same-server cross-disk reconcile both leave unmounted, including for
+// volumes already broken before the upgrade.
+//
+// filterVid limits recovery to a single volume id; 0 recovers every orphan on
+// this server (used by ec.rebuild to heal volumes the master never learned
+// about). It is driven on demand by VolumeEcShardsMount (recover_missing_index),
+// so an operator triggers it through ec.rebuild rather than a background loop.
+// Returns the number of volumes whose index was recovered.
+func (vs *VolumeServer) recoverMissingEcIndexes(filterVid uint32) int {
+	orphans := vs.store.CollectEcVolumesMissingIndex()
+	if filterVid != 0 {
+		filtered := orphans[:0]
+		for _, m := range orphans {
+			if uint32(m.VolumeId) == filterVid {
+				filtered = append(filtered, m)
+			}
+		}
+		orphans = filtered
+	}
+	if len(orphans) == 0 {
+		return 0
+	}
+
+	master := vs.getCurrentMaster()
+	if master == "" {
+		glog.Warningf("cannot recover missing EC index without a master connection")
+		return 0
+	}
+	self := pb.NewServerAddress(vs.store.Ip, vs.store.Port, vs.store.GrpcPort)
+
+	recovered := 0
+	for _, m := range orphans {
+		peers, err := vs.lookupEcVolumePeers(master, uint32(m.VolumeId), self)
+		if err != nil {
+			glog.Warningf("ec volume %d: cannot look up peers to recover missing .ecx: %v", m.VolumeId, err)
+			continue
+		}
+		if len(peers) == 0 {
+			glog.Warningf("ec volume %d: shards present locally but .ecx missing and no peer holds it; leaving shards unloaded", m.VolumeId)
+			continue
+		}
+		if vs.fetchEcIndexFromPeers(peers, m) {
+			recovered++
+		}
+	}
+
+	if recovered > 0 {
+		vs.store.MountRecoveredEcShards()
+		glog.V(0).Infof("recovered missing EC index for %d volume(s) from peers and mounted their shards", recovered)
+	}
+	return recovered
+}
+
+// lookupEcVolumePeers asks the master which servers hold shards for vid and
+// returns the unique peer addresses, excluding this server itself.
+func (vs *VolumeServer) lookupEcVolumePeers(master pb.ServerAddress, vid uint32, self pb.ServerAddress) ([]pb.ServerAddress, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), ecRecoveryLookupTimeout)
+	defer cancel()
+
+	var peers []pb.ServerAddress
+	seen := make(map[pb.ServerAddress]bool)
+	err := operation.WithMasterServerClient(ctx, false, master, vs.grpcDialOption, func(client master_pb.SeaweedClient) error {
+		resp, err := client.LookupEcVolume(ctx, &master_pb.LookupEcVolumeRequest{VolumeId: vid})
+		if err != nil {
+			return err
+		}
+		for _, shardIdLocations := range resp.ShardIdLocations {
+			for _, loc := range shardIdLocations.Locations {
+				addr := pb.NewServerAddressFromLocation(loc)
+				if addr.Equals(self) || seen[addr] {
+					continue
+				}
+				seen[addr] = true
+				peers = append(peers, addr)
+			}
+		}
+		return nil
+	})
+	return peers, err
+}
+
+// fetchEcIndexFromPeers tries each peer in turn, copying the .ecx (required) and
+// the .ecj / .vif (best-effort) for the volume onto the local disk recorded in
+// m. The .ecx is an immutable encode-time index, identical on every holder, so
+// any peer's copy serves. The .ecj is a per-holder deletion journal that differs
+// across holders (a delete is journaled on only one node); the recovered node
+// adopts the source peer's deletion view, exactly as a balanced or rebuilt shard
+// does — the EC delete model already tolerates that divergence. The first peer
+// that yields a non-empty .ecx wins; a peer with no index or a 0-byte stub is
+// skipped (the orphan shards are non-empty, so a 0-byte index cannot be theirs).
+// A failed copy removes its partial file so a later attempt is not blocked by a
+// stub.
+func (vs *VolumeServer) fetchEcIndexFromPeers(peers []pb.ServerAddress, m storage.EcVolumeMissingIndex) bool {
+	idxBaseFileName := storage.VolumeFileName(m.IdxDir, m.Collection, int(m.VolumeId))
+	dataBaseFileName := storage.VolumeFileName(m.DataDir, m.Collection, int(m.VolumeId))
+	ecxPath := idxBaseFileName + ".ecx"
+	ecjPath := idxBaseFileName + ".ecj"
+
+	removePartial := func(path string) {
+		if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+			glog.Warningf("ec volume %d: remove partial %s: %v", m.VolumeId, path, err)
+		}
+	}
+
+	for _, peer := range peers {
+		err := operation.WithVolumeServerClient(true, peer, vs.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+			// .ecx is mandatory; a peer without it errors and we move on.
+			if _, err := vs.doCopyFile(client, true, m.Collection, uint32(m.VolumeId), math.MaxUint32, math.MaxInt64, idxBaseFileName, ".ecx", false, false, nil); err != nil {
+				removePartial(ecxPath)
+				return err
+			}
+			info, statErr := os.Stat(ecxPath)
+			if statErr != nil {
+				removePartial(ecxPath)
+				return fmt.Errorf("stat copied .ecx %s: %w", ecxPath, statErr)
+			}
+			if info.IsDir() || info.Size() == 0 {
+				removePartial(ecxPath)
+				return fmt.Errorf("peer %s served an unusable .ecx (size %d)", peer, info.Size())
+			}
+			// .ecj is the source peer's deletion journal; .vif carries EC params
+			// and EncodeTsNs. Both are best-effort: a missing .ecj is recreated at
+			// mount and a missing .vif falls back to default EC parameters. A failed
+			// .ecj copy is an in-place append, so drop the partial file; the .vif
+			// copy stages and renames, leaving nothing to clean up.
+			if _, err := vs.doCopyFile(client, true, m.Collection, uint32(m.VolumeId), math.MaxUint32, math.MaxInt64, idxBaseFileName, ".ecj", true, true, nil); err != nil {
+				glog.Warningf("ec volume %d: copy .ecj from %s: %v", m.VolumeId, peer, err)
+				removePartial(ecjPath)
+			}
+			if _, err := vs.doCopyFile(client, true, m.Collection, uint32(m.VolumeId), math.MaxUint32, math.MaxInt64, dataBaseFileName, ".vif", false, true, nil); err != nil {
+				glog.Warningf("ec volume %d: copy .vif from %s: %v", m.VolumeId, peer, err)
+			}
+			return nil
+		})
+		if err != nil {
+			glog.V(1).Infof("ec volume %d: fetch missing .ecx from %s failed: %v", m.VolumeId, peer, err)
+			continue
+		}
+		glog.V(0).Infof("ec volume %d: fetched missing .ecx from %s into %s", m.VolumeId, peer, m.IdxDir)
+		return true
+	}
+	return false
+}

weed/server/volume_grpc_erasure_coding_recover_test.go

@@ -0,0 +1,285 @@
+package weed_server
+
+import (
+	"context"
+	"net"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// fakeMaster serves only LookupEcVolume, returning the configured holders for
+// every shard of one volume. Used to drive the receiver's peer discovery.
+type fakeMaster struct {
+	master_pb.UnimplementedSeaweedServer
+	volumeId  uint32
+	locations []*master_pb.Location
+}
+
+func (m *fakeMaster) LookupEcVolume(_ context.Context, req *master_pb.LookupEcVolumeRequest) (*master_pb.LookupEcVolumeResponse, error) {
+	resp := &master_pb.LookupEcVolumeResponse{VolumeId: req.VolumeId}
+	if req.VolumeId != m.volumeId {
+		return resp, nil
+	}
+	// Recovery only needs one peer holding the index; report a couple of shards
+	// pointing at the holder, independent of the EC ratio.
+	for shardId := 0; shardId < 2; shardId++ {
+		resp.ShardIdLocations = append(resp.ShardIdLocations, &master_pb.LookupEcVolumeResponse_EcShardIdLocation{
+			ShardId:   uint32(shardId),
+			Locations: m.locations,
+		})
+	}
+	return resp, nil
+}
+
+// serveGrpc registers register(s) on a fresh localhost listener and returns its
+// grpc port, stopping the server on test cleanup.
+func serveGrpc(t *testing.T, register func(*grpc.Server)) int {
+	t.Helper()
+	lis, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	srv := grpc.NewServer()
+	register(srv)
+	go srv.Serve(lis)
+	t.Cleanup(srv.Stop)
+	return lis.Addr().(*net.TCPAddr).Port
+}
+
+func newRecoverTestStore(t *testing.T, dir string) *storage.Store {
+	t.Helper()
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		t.Fatalf("mkdir %s: %v", dir, err)
+	}
+	store := storage.NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "store-id",
+		[]string{dir}, []int32{100}, []util.MinFreeSpace{{}}, "",
+		storage.NeedleMapInMemory, []types.DiskType{types.HardDriveType}, nil, 3, stats.DefaultDiskIOProbeConfig())
+	done := make(chan struct{})
+	go func() {
+		for {
+			select {
+			case <-store.NewEcShardsChan:
+			case <-store.NewVolumesChan:
+			case <-store.DeletedVolumesChan:
+			case <-store.DeletedEcShardsChan:
+			case <-store.StateUpdateChan:
+			case <-done:
+				return
+			}
+		}
+	}()
+	t.Cleanup(func() {
+		store.Close()
+		close(done)
+	})
+	return store
+}
+
+// TestFetchEcIndexFromPeers_CopiesIndexOverGrpc stands up a source volume server
+// that holds the .ecx/.ecj/.vif for a volume and verifies the receiver pulls
+// them over a real CopyFile gRPC stream into its own disk, the way #10104
+// recovery does when the index lives only on a peer.
+func TestFetchEcIndexFromPeers_CopiesIndexOverGrpc(t *testing.T) {
+	const collection = "video-recordings"
+	vid := needle.VolumeId(6190)
+
+	// Source server: has the index files on disk.
+	srcDir := filepath.Join(t.TempDir(), "src")
+	srcStore := newRecoverTestStore(t, srcDir)
+	srcBase := erasure_coding.EcShardFileName(collection, srcDir, int(vid))
+	ecxBytes := make([]byte, types.NeedleMapEntrySize*3)
+	for i := range ecxBytes {
+		ecxBytes[i] = byte(i)
+	}
+	if err := os.WriteFile(srcBase+".ecx", ecxBytes, 0o644); err != nil {
+		t.Fatalf("write source .ecx: %v", err)
+	}
+	if err := os.WriteFile(srcBase+".ecj", []byte("journal"), 0o644); err != nil {
+		t.Fatalf("write source .ecj: %v", err)
+	}
+	if err := os.WriteFile(srcBase+".vif", []byte("volinfo"), 0o644); err != nil {
+		t.Fatalf("write source .vif: %v", err)
+	}
+
+	// Serve the source over a real TCP gRPC listener.
+	lis, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	grpcServer := grpc.NewServer()
+	volume_server_pb.RegisterVolumeServerServer(grpcServer, &VolumeServer{store: srcStore})
+	go grpcServer.Serve(lis)
+	t.Cleanup(grpcServer.Stop)
+
+	grpcPort := lis.Addr().(*net.TCPAddr).Port
+	peer := pb.NewServerAddress("127.0.0.1", grpcPort-10000, grpcPort)
+
+	// Receiver server: empty disk, ready to receive the index.
+	dstDir := filepath.Join(t.TempDir(), "dst")
+	dstStore := newRecoverTestStore(t, dstDir)
+	receiver := &VolumeServer{
+		store:          dstStore,
+		grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
+	}
+
+	m := storage.EcVolumeMissingIndex{
+		Collection: collection,
+		VolumeId:   vid,
+		IdxDir:     dstDir,
+		DataDir:    dstDir,
+	}
+	if !receiver.fetchEcIndexFromPeers([]pb.ServerAddress{peer}, m) {
+		t.Fatalf("fetchEcIndexFromPeers returned false; expected a successful copy")
+	}
+
+	dstBase := erasure_coding.EcShardFileName(collection, dstDir, int(vid))
+	got, err := os.ReadFile(dstBase + ".ecx")
+	if err != nil {
+		t.Fatalf("read copied .ecx: %v", err)
+	}
+	if len(got) != len(ecxBytes) {
+		t.Errorf("copied .ecx size = %d, want %d", len(got), len(ecxBytes))
+	}
+	if _, err := os.Stat(dstBase + ".ecj"); err != nil {
+		t.Errorf("copied .ecj missing: %v", err)
+	}
+	if _, err := os.Stat(dstBase + ".vif"); err != nil {
+		t.Errorf("copied .vif missing: %v", err)
+	}
+}
+
+// TestFetchEcIndexFromPeers_SkipsPeerWithoutIndex verifies the receiver moves on
+// to the next peer when the first has no .ecx, and reports failure when no peer
+// can serve a usable index.
+func TestFetchEcIndexFromPeers_SkipsPeerWithoutIndex(t *testing.T) {
+	const collection = "video-recordings"
+	vid := needle.VolumeId(6191)
+
+	// Source server with NO index files for this volume.
+	srcDir := filepath.Join(t.TempDir(), "src")
+	srcStore := newRecoverTestStore(t, srcDir)
+
+	lis, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	grpcServer := grpc.NewServer()
+	volume_server_pb.RegisterVolumeServerServer(grpcServer, &VolumeServer{store: srcStore})
+	go grpcServer.Serve(lis)
+	t.Cleanup(grpcServer.Stop)
+
+	grpcPort := lis.Addr().(*net.TCPAddr).Port
+	peer := pb.NewServerAddress("127.0.0.1", grpcPort-10000, grpcPort)
+
+	dstDir := filepath.Join(t.TempDir(), "dst")
+	dstStore := newRecoverTestStore(t, dstDir)
+	receiver := &VolumeServer{
+		store:          dstStore,
+		grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
+	}
+
+	m := storage.EcVolumeMissingIndex{Collection: collection, VolumeId: vid, IdxDir: dstDir, DataDir: dstDir}
+	if receiver.fetchEcIndexFromPeers([]pb.ServerAddress{peer}, m) {
+		t.Fatalf("fetchEcIndexFromPeers should fail when no peer has the index")
+	}
+
+	// No stub .ecx must be left behind.
+	dstBase := erasure_coding.EcShardFileName(collection, dstDir, int(vid))
+	if _, err := os.Stat(dstBase + ".ecx"); !os.IsNotExist(err) {
+		t.Errorf("a .ecx stub was left behind; stat err = %v", err)
+	}
+}
+
+// TestVolumeEcShardsMount_RecoverMissingIndex drives the full on-demand path:
+// VolumeEcShardsMount with recover_missing_index (volume_id 0 = recover every
+// orphan on this server, as ec.rebuild broadcasts it) looks up holders via a
+// (fake) master, fetches the missing .ecx/.ecj/.vif from the holding peer over
+// gRPC, and mounts the previously-orphaned on-disk shards (issue #10104).
+func TestVolumeEcShardsMount_RecoverMissingIndex(t *testing.T) {
+	const collection = "video-recordings"
+	vid := needle.VolumeId(6190)
+
+	// Holder peer: serves the index files for the volume.
+	srcDir := filepath.Join(t.TempDir(), "src")
+	srcStore := newRecoverTestStore(t, srcDir)
+	srcBase := erasure_coding.EcShardFileName(collection, srcDir, int(vid))
+	if err := os.WriteFile(srcBase+".ecx", make([]byte, types.NeedleMapEntrySize*4), 0o644); err != nil {
+		t.Fatalf("write source .ecx: %v", err)
+	}
+	if err := os.WriteFile(srcBase+".ecj", nil, 0o644); err != nil {
+		t.Fatalf("write source .ecj: %v", err)
+	}
+	if err := os.WriteFile(srcBase+".vif", []byte("volinfo"), 0o644); err != nil {
+		t.Fatalf("write source .vif: %v", err)
+	}
+	srcGrpcPort := serveGrpc(t, func(s *grpc.Server) {
+		volume_server_pb.RegisterVolumeServerServer(s, &VolumeServer{store: srcStore})
+	})
+
+	// Fake master points every shard at the holder peer.
+	masterGrpcPort := serveGrpc(t, func(s *grpc.Server) {
+		master_pb.RegisterSeaweedServer(s, &fakeMaster{
+			volumeId:  uint32(vid),
+			locations: []*master_pb.Location{{Url: "127.0.0.1:1", GrpcPort: uint32(srcGrpcPort)}},
+		})
+	})
+
+	// Receiver: holds orphan shard files on disk, no .ecx anywhere.
+	dstDir := filepath.Join(t.TempDir(), "dst")
+	dstStore := newRecoverTestStore(t, dstDir)
+	const shardSize = 1 << 20
+	for _, sid := range []erasure_coding.ShardId{0, 5} {
+		base := erasure_coding.EcShardFileName(collection, dstDir, int(vid))
+		f, err := os.Create(base + erasure_coding.ToExt(int(sid)))
+		if err != nil {
+			t.Fatalf("create shard %d: %v", sid, err)
+		}
+		if err := f.Truncate(shardSize); err != nil {
+			f.Close()
+			t.Fatalf("truncate shard %d: %v", sid, err)
+		}
+		f.Close()
+	}
+
+	receiver := &VolumeServer{
+		store:          dstStore,
+		grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
+	}
+	receiver.setCurrentMaster(pb.NewServerAddress("127.0.0.1", masterGrpcPort-10000, masterGrpcPort))
+
+	if _, found := dstStore.FindEcVolume(vid); found {
+		t.Fatalf("EC volume %d unexpectedly mounted before recovery", vid)
+	}
+
+	// volume_id 0: the receiver discovers the orphan (6190) on disk itself, even
+	// though the request names no volume and the master never registered it here.
+	if _, err := receiver.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{
+		RecoverMissingIndex: true,
+	}); err != nil {
+		t.Fatalf("VolumeEcShardsMount with recover_missing_index: %v", err)
+	}
+
+	ev, found := dstStore.FindEcVolume(vid)
+	if !found {
+		t.Fatalf("EC volume %d not mounted after recovery", vid)
+	}
+	for _, sid := range []erasure_coding.ShardId{0, 5} {
+		if _, ok := ev.FindEcVolumeShard(sid); !ok {
+			t.Errorf("shard %d.%d not registered after recovery", vid, sid)
+		}
+	}
+}

weed/shell/command_ec_rebuild.go

@@ -47,6 +47,11 @@ func (c *commandEcRebuild) Help() string {
 
 	ec.rebuild [-c EACH_COLLECTION|<collection_name>] [-apply] [-maxParallelization N] [-diskType=<disk_type>]
 
+	Before rebuilding, asks volume servers to recover any shards left unmounted by
+	a missing .ecx index (the index resides only on a peer server). Such shards are
+	invisible to the master, so recovering them first avoids regenerating data that
+	is actually present (issue #10104).
+
 	Options:
 	  -collection: specify a collection name, or "EACH_COLLECTION" to process all collections
 	  -apply: actually perform the rebuild operations (default is dry-run mode)
@@ -149,6 +154,11 @@ func (c *commandEcRebuild) Do(args []string, commandEnv *CommandEnv, writer io.W
 		ewg: NewErrorWaitGroup(*maxParallelization),
 	}
 
+	// Recover shards left unmounted by a missing .ecx index before planning: such
+	// shards never register with the master, so the rebuild below would treat the
+	// volume as short or unrepairable even though its data is intact (issue #10104).
+	erb.recoverMissingIndexes()
+
 	fmt.Printf("rebuildEcVolumes for %d collection(s)\n", len(collections))
 	for _, c := range collections {
 		erb.rebuildEcVolumes(c)
@@ -288,6 +298,61 @@ func (erb *ecRebuilder) rebuildEcVolumes(collection string) {
 	}
 }
 
+// recoverMissingIndexes asks every ec node to fetch a missing .ecx index from a
+// peer and mount the on-disk shards it could not load on its own. Shards
+// orphaned this way (index only on another server) are absent from the master
+// topology, so without this pass ec.rebuild would regenerate or give up on
+// shards whose data is actually present — and a volume whose every holder lacks
+// the index would not appear in the topology at all. Each node therefore
+// recovers all of its on-disk orphans (volume_id 0); an explicit -volumeIds
+// list narrows that to the requested volumes. On apply it refreshes the topology
+// so the rebuild planning sees the recovered shards (issue #10104).
+func (erb *ecRebuilder) recoverMissingIndexes() {
+	erb.ecNodesMu.Lock()
+	nodes := append([]*EcNode(nil), erb.ecNodes...)
+	erb.ecNodesMu.Unlock()
+	if len(nodes) == 0 {
+		return
+	}
+
+	// volume_id 0 means "recover every orphan on the node"; a -volumeIds list
+	// narrows recovery to those ids (each scanned across collections server-side).
+	vids := erb.volumeIds
+	if len(vids) == 0 {
+		vids = []needle.VolumeId{0}
+	}
+
+	if !erb.applyChanges {
+		erb.write("would ask %d ec node(s) to recover EC shards left unmounted by a missing .ecx index\n", len(nodes))
+		return
+	}
+
+	for _, node := range nodes {
+		for _, vid := range vids {
+			err := operation.WithVolumeServerClient(false, pb.NewServerAddressFromDataNode(node.info), erb.commandEnv.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+				_, mountErr := client.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{
+					VolumeId:            uint32(vid),
+					RecoverMissingIndex: true,
+				})
+				return mountErr
+			})
+			if err != nil {
+				erb.write("%s recover missing index (volume %d): %v\n", node.info.Id, vid, err)
+			}
+		}
+	}
+
+	// Refresh topology so the rebuild planning sees shards the recovery registered.
+	refreshed, _, err := collectEcNodes(erb.commandEnv, erb.diskType)
+	if err != nil {
+		erb.write("failed to refresh ec nodes after index recovery: %v\n", err)
+		return
+	}
+	erb.ecNodesMu.Lock()
+	erb.ecNodes = refreshed
+	erb.ecNodesMu.Unlock()
+}
+
 func (erb *ecRebuilder) rebuildOneEcVolume(collection string, volumeId needle.VolumeId, locations EcShardLocations, rebuilder *EcNode) error {
 	if !erb.isLocked() {
 		return fmt.Errorf("lock is lost")

weed/storage/store_ec_recover.go

@@ -0,0 +1,83 @@
+package storage
+
+import (
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+// EcVolumeMissingIndex names an EC volume whose .ec?? shard files are present on
+// this store but whose .ecx index is missing on every local disk, so the shards
+// cannot be mounted. The per-disk loader and the same-server cross-disk reconcile
+// both leave such shards unmounted because the index lives only on a different
+// server (issue #10104). The recovery path fetches the index from a peer that
+// still holds it and drops it into IdxDir / DataDir, then re-runs the mount.
+type EcVolumeMissingIndex struct {
+	Collection string
+	VolumeId   needle.VolumeId
+	IdxDir     string // destination for the fetched .ecx / .ecj
+	DataDir    string // destination for the fetched .vif
+}
+
+// CollectEcVolumesMissingIndex returns every EC volume that has shard files on a
+// local disk but no usable .ecx on any local disk — the cross-server orphans a
+// peer index fetch must recover. Volumes whose index merely sits on a sibling
+// disk are excluded (the same-server reconcile/mirror handles those). The
+// destination dirs are taken from the first disk found holding orphan shards;
+// MountRecoveredEcShards mirrors the index onto the remaining shard-bearing disks.
+//
+// It scans on-disk shard files directly, so it surfaces volumes the master never
+// learned about — including those whose every holder is missing its index.
+func (s *Store) CollectEcVolumesMissingIndex() []EcVolumeMissingIndex {
+	ecxOwners := s.indexEcxOwners()
+
+	seen := make(map[ecKeyForReconcile]bool)
+	var missing []EcVolumeMissingIndex
+	for _, loc := range s.Locations {
+		for key := range loc.collectOrphanEcShards() {
+			if _, hasLocalIndex := ecxOwners[key]; hasLocalIndex {
+				continue
+			}
+			if seen[key] {
+				continue
+			}
+			seen[key] = true
+			missing = append(missing, EcVolumeMissingIndex{
+				Collection: key.collection,
+				VolumeId:   key.vid,
+				IdxDir:     loc.IdxDirectory,
+				DataDir:    loc.Directory,
+			})
+		}
+	}
+	return missing
+}
+
+// MountRecoveredEcShards mounts EC shards that became loadable after a missing
+// .ecx was fetched onto a local disk. It mirrors the index onto every
+// shard-bearing disk, mounts the disks that now have a local index, and falls
+// back to the cross-disk virtual mount for any disk the mirror could not reach.
+// Each shard load announces itself through ecShardNotifyHandler so the master
+// learns about the now-registered shards.
+func (s *Store) MountRecoveredEcShards() {
+	s.mirrorEcMetadataToShardDisks()
+	s.loadOrphanEcShardsWithLocalIndex()
+	s.reconcileEcShardsAcrossDisks()
+}
+
+// loadOrphanEcShardsWithLocalIndex mounts on-disk EC shards whose .ecx index is
+// now present on the same disk. Unlike reconcileEcShardsAcrossDisks it does not
+// require a sibling disk, so a single-disk store recovers too once its index has
+// been fetched from a peer.
+func (s *Store) loadOrphanEcShardsWithLocalIndex() {
+	for _, loc := range s.Locations {
+		orphans := loc.collectOrphanEcShards()
+		for key, shards := range orphans {
+			if !loc.HasEcxFileOnDisk(key.collection, key.vid) {
+				continue
+			}
+			if err := loc.loadEcShards(shards, key.collection, key.vid, loc.ecShardNotifyHandler); err != nil {
+				glog.Errorf("ec volume %d on %s: load after index recovery failed: %v", key.vid, loc.Directory, err)
+			}
+		}
+	}
+}

weed/storage/store_ec_recover_test.go

@@ -0,0 +1,201 @@
+package storage
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// newDrainingStore builds a Store over dirs and drains its announcement
+// channels in the background so loads never block. Files planted before this
+// call are picked up by the startup loaders; files planted after are not.
+func newDrainingStore(t *testing.T, dirs []string) *Store {
+	t.Helper()
+	for _, d := range dirs {
+		if err := os.MkdirAll(d, 0o755); err != nil {
+			t.Fatalf("mkdir %s: %v", d, err)
+		}
+	}
+	maxCounts := make([]int32, len(dirs))
+	minFree := make([]util.MinFreeSpace, len(dirs))
+	diskTypes := make([]types.DiskType, len(dirs))
+	for i := range dirs {
+		maxCounts[i] = 100
+		diskTypes[i] = types.HardDriveType
+	}
+	store := NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "store-id",
+		dirs, maxCounts, minFree, "",
+		NeedleMapInMemory, diskTypes, nil, 3, stats.DefaultDiskIOProbeConfig())
+	done := make(chan struct{})
+	go func() {
+		for {
+			select {
+			case <-store.NewEcShardsChan:
+			case <-store.NewVolumesChan:
+			case <-store.DeletedVolumesChan:
+			case <-store.DeletedEcShardsChan:
+			case <-store.StateUpdateChan:
+			case <-done:
+				return
+			}
+		}
+	}()
+	t.Cleanup(func() {
+		store.Close()
+		close(done)
+	})
+	return store
+}
+
+func plantEcShard(t *testing.T, dir, collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, size int64) {
+	t.Helper()
+	base := erasure_coding.EcShardFileName(collection, dir, int(vid))
+	f, err := os.Create(base + erasure_coding.ToExt(int(shardId)))
+	if err != nil {
+		t.Fatalf("create shard %d.%d: %v", vid, shardId, err)
+	}
+	if err := f.Truncate(size); err != nil {
+		f.Close()
+		t.Fatalf("truncate shard %d.%d: %v", vid, shardId, err)
+	}
+	f.Close()
+}
+
+func plantEcIndex(t *testing.T, idxDir, dataDir, collection string, vid needle.VolumeId, datSize int64) {
+	t.Helper()
+	idxBase := erasure_coding.EcShardFileName(collection, idxDir, int(vid))
+	if err := os.WriteFile(idxBase+".ecx", make([]byte, types.NeedleMapEntrySize), 0o644); err != nil {
+		t.Fatalf("write .ecx: %v", err)
+	}
+	if err := os.WriteFile(idxBase+".ecj", nil, 0o644); err != nil {
+		t.Fatalf("write .ecj: %v", err)
+	}
+	dataBase := erasure_coding.EcShardFileName(collection, dataDir, int(vid))
+	if err := volume_info.SaveVolumeInfo(dataBase+".vif", &volume_server_pb.VolumeInfo{
+		Version:     uint32(needle.Version3),
+		DatFileSize: datSize,
+		EcShardConfig: &volume_server_pb.EcShardConfig{
+			DataShards:   10,
+			ParityShards: 4,
+		},
+	}); err != nil {
+		t.Fatalf("save .vif: %v", err)
+	}
+}
+
+// TestEcIndexRecovery_CrossServerOrphan reproduces issue #10104: a volume
+// server reboots with EC shards on disk but no .ecx index anywhere local (the
+// index lives only on a peer server). The startup loaders leave the shards
+// unmounted, so the master never learns about them. After the recovery path
+// drops the index fetched from a peer onto the local disk, MountRecoveredEcShards
+// must mount and announce the shards.
+func TestEcIndexRecovery_CrossServerOrphan(t *testing.T) {
+	tempDir := t.TempDir()
+	dir0 := filepath.Join(tempDir, "disk0")
+	dir1 := filepath.Join(tempDir, "disk1")
+
+	const collection = "video-recordings"
+	vid := needle.VolumeId(6190)
+	const datSize int64 = 10 * 1024 * 1024
+	shardSize := calculateExpectedShardSize(datSize, 10)
+
+	// Pre-seed the on-disk layout the issue describes: shards spread across
+	// this server's disks, with no .ecx / .ecj / .vif on any of them.
+	for _, d := range []string{dir0, dir1} {
+		if err := os.MkdirAll(d, 0o755); err != nil {
+			t.Fatalf("mkdir %s: %v", d, err)
+		}
+	}
+	plantEcShard(t, dir0, collection, vid, 0, shardSize)
+	plantEcShard(t, dir0, collection, vid, 5, shardSize)
+	plantEcShard(t, dir1, collection, vid, 6, shardSize)
+
+	store := newDrainingStore(t, []string{dir0, dir1})
+
+	// Bug reproduction: the startup loaders left every shard unmounted.
+	if _, found := store.FindEcVolume(vid); found {
+		t.Fatalf("EC volume %d unexpectedly mounted without any local .ecx", vid)
+	}
+
+	// The recovery scan must surface this volume as missing its index.
+	m, ok := findMissingIndex(store.CollectEcVolumesMissingIndex(), collection, vid)
+	if !ok {
+		t.Fatalf("CollectEcVolumesMissingIndex did not report volume %d as a cross-server orphan", vid)
+	}
+
+	// Simulate the peer fetch: the server-side path copies .ecx/.ecj/.vif from a
+	// peer that still has them into m.IdxDir / m.DataDir.
+	plantEcIndex(t, m.IdxDir, m.DataDir, collection, vid, datSize)
+
+	store.MountRecoveredEcShards()
+
+	// All shards must now be mounted on the disks that physically hold them.
+	loc0 := store.Locations[0]
+	ev, found := loc0.FindEcVolume(vid)
+	if !found {
+		t.Fatalf("EC volume %d not mounted on disk0 after index recovery", vid)
+	}
+	for _, sid := range []erasure_coding.ShardId{0, 5} {
+		if _, ok := ev.FindEcVolumeShard(sid); !ok {
+			t.Errorf("shard %d.%d not registered on disk0 after recovery", vid, sid)
+		}
+	}
+	loc1 := store.Locations[1]
+	ev1, found := loc1.FindEcVolume(vid)
+	if !found {
+		t.Fatalf("EC volume %d not mounted on disk1 after index recovery", vid)
+	}
+	if _, ok := ev1.FindEcVolumeShard(6); !ok {
+		t.Errorf("shard %d.6 not registered on disk1 after recovery", vid)
+	}
+
+	// A second scan must report nothing left to recover.
+	if _, again := findMissingIndex(store.CollectEcVolumesMissingIndex(), collection, vid); again {
+		t.Errorf("CollectEcVolumesMissingIndex still reports volume %d after recovery", vid)
+	}
+}
+
+// findMissingIndex returns the entry for (collection, vid) among the reported
+// missing-index volumes, if present.
+func findMissingIndex(missing []EcVolumeMissingIndex, collection string, vid needle.VolumeId) (EcVolumeMissingIndex, bool) {
+	for _, m := range missing {
+		if m.Collection == collection && m.VolumeId == vid {
+			return m, true
+		}
+	}
+	return EcVolumeMissingIndex{}, false
+}
+
+// TestCollectEcVolumesMissingIndex_ExcludesCrossDiskOrphan guards the boundary
+// with the existing same-server reconcile: a volume whose .ecx merely sits on a
+// sibling disk is NOT a cross-server orphan and must not be reported here —
+// mirrorEcMetadataToShardDisks / reconcileEcShardsAcrossDisks already mount it.
+func TestCollectEcVolumesMissingIndex_ExcludesCrossDiskOrphan(t *testing.T) {
+	tempDir := t.TempDir()
+	dir0 := filepath.Join(tempDir, "disk0")
+	dir1 := filepath.Join(tempDir, "disk1")
+	store := newDrainingStore(t, []string{dir0, dir1})
+
+	const collection = "mybucket"
+	vid := needle.VolumeId(42)
+	const datSize int64 = 10 * 1024 * 1024
+	shardSize := calculateExpectedShardSize(datSize, 10)
+
+	// Plant after NewStore so the startup reconcile does not consume the layout;
+	// CollectEcVolumesMissingIndex is what's under test. Shard on disk0, index on
+	// disk1 — the cross-disk layout handled by the same-server reconcile.
+	plantEcShard(t, dir0, collection, vid, 3, shardSize)
+	plantEcIndex(t, dir1, dir1, collection, vid, datSize)
+
+	if _, missing := findMissingIndex(store.CollectEcVolumesMissingIndex(), collection, vid); missing {
+		t.Errorf("cross-disk .ecx must not be reported as a cross-server orphan")
+	}
+}