volume: fix EC decode/reconstruct index locality under -dir.idx (#10442)

* volume: fix EC decode/reconstruct index locality under -dir.idx

EC->replicated decode failed under -dir.idx and on multi-disk with "volume not
found on disk". The reconstruct rebuilds the .dat on the data disk but the
on-demand VolumeMount scans only the data directory, matching on .idx/.vif;
with the rebuilt .idx off in the index directory it matched the volume's
leftover EC .vif and skipped the volume as EC metadata.

- Resolve the EC .ecx local-first: prefer the copy co-located with the shards
  over the shared -dir.idx copy, with a non-empty preference so a 0-byte local
  stub still yields to a valid sibling (the cross-disk fallback).
- Co-locate the rebuilt .idx with the .dat at the end of the reconstruct so the
  mount finds it; sweep .ecx/.ecj from both the data and index directories on
  Destroy so a stale copy cannot re-mount as a phantom EC volume.
- Add VolumeConsolidateIndex: once the EC shards are deleted, unmount, move the
  .idx/.sdx from the data disk back to the -dir.idx directory (copy fallback
  across filesystems), and remount. A no-op without -dir.idx.

* volume: tests for EC index locality (local-first .ecx, sweep, consolidate)

- NewEcVolume prefers a non-empty local .ecx over the shared index dir, and a
  0-byte local stub yields to a non-empty shared copy (the #9212 fallback).
- Destroy sweeps .ecx/.ecj from both the data and index directories.
- ConsolidateVolumeIndex moves a co-located index back to the -dir.idx dir and
  keeps the volume mounted; no-op without a separate index dir.
- RenameOrCopyFile moves a file and drops the source.

* volume: relocate the decoded index in place, without a read gap

ConsolidateVolumeIndex previously unmounted the volume, moved the index, and
remounted it. Between the EC-shard delete and the remount the volume had neither
a normal nor an EC form mounted, so a read landing in that window got a
not-found (or was proxied away).

Move the index in place instead: RelocateIndexTo takes the data-file write lock,
closes the needle map and data backend, moves the .idx (and derived .sdx), then
retargets dirIdx and reloads — the same close-swap-load CommitCompact uses. The
volume never leaves the mounted set, so a concurrent read blocks briefly on the
lock rather than failing. The test now writes a needle before consolidating and
reads it back after, proving the in-place reload keeps the volume serving.

* volume: address review — maintenance guard, no orphan on copy failure

- VolumeConsolidateIndex now rejects the request under maintenance mode, like
  VolumeConfigure and the other mutating volume RPCs.
- RenameOrCopyFile rolls the cross-device copy back when the source cannot be
  removed, so a failed move never leaves two divergent copies (the loader would
  keep the data-dir one while the idx-dir orphan goes stale).
- RelocateIndexTo logs a failed reopen-after-failed-move instead of swallowing
  it, since that leaves the volume unusable until the next load.

weed/pb/volume_server.proto

@@ -45,6 +45,8 @@ service VolumeServer {
     }
     rpc VolumeUnmount (VolumeUnmountRequest) returns (VolumeUnmountResponse) {
     }
+    rpc VolumeConsolidateIndex (VolumeConsolidateIndexRequest) returns (VolumeConsolidateIndexResponse) {
+    }
     rpc VolumeDelete (VolumeDeleteRequest) returns (VolumeDeleteResponse) {
     }
     rpc VolumeMarkReadonly (VolumeMarkReadonlyRequest) returns (VolumeMarkReadonlyResponse) {
@@ -244,6 +246,12 @@ message VolumeUnmountRequest {
 message VolumeUnmountResponse {
 }
 
+message VolumeConsolidateIndexRequest {
+    uint32 volume_id = 1;
+}
+message VolumeConsolidateIndexResponse {
+}
+
 message VolumeDeleteRequest {
     uint32 volume_id = 1;
     bool only_empty = 2;

weed/pb/volume_server_pb/volume_server_grpc.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
 // versions:
 // - protoc-gen-go-grpc v1.5.1
-// - protoc             v6.33.4
+// - protoc             v7.35.0
 // source: volume_server.proto
 
 package volume_server_pb
@@ -30,6 +30,7 @@ const (
 	VolumeServer_VolumeIncrementalCopy_FullMethodName       = "/volume_server_pb.VolumeServer/VolumeIncrementalCopy"
 	VolumeServer_VolumeMount_FullMethodName                 = "/volume_server_pb.VolumeServer/VolumeMount"
 	VolumeServer_VolumeUnmount_FullMethodName               = "/volume_server_pb.VolumeServer/VolumeUnmount"
+	VolumeServer_VolumeConsolidateIndex_FullMethodName      = "/volume_server_pb.VolumeServer/VolumeConsolidateIndex"
 	VolumeServer_VolumeDelete_FullMethodName                = "/volume_server_pb.VolumeServer/VolumeDelete"
 	VolumeServer_VolumeMarkReadonly_FullMethodName          = "/volume_server_pb.VolumeServer/VolumeMarkReadonly"
 	VolumeServer_VolumeMarkWritable_FullMethodName          = "/volume_server_pb.VolumeServer/VolumeMarkWritable"
@@ -85,6 +86,7 @@ type VolumeServerClient interface {
 	VolumeIncrementalCopy(ctx context.Context, in *VolumeIncrementalCopyRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[VolumeIncrementalCopyResponse], error)
 	VolumeMount(ctx context.Context, in *VolumeMountRequest, opts ...grpc.CallOption) (*VolumeMountResponse, error)
 	VolumeUnmount(ctx context.Context, in *VolumeUnmountRequest, opts ...grpc.CallOption) (*VolumeUnmountResponse, error)
+	VolumeConsolidateIndex(ctx context.Context, in *VolumeConsolidateIndexRequest, opts ...grpc.CallOption) (*VolumeConsolidateIndexResponse, error)
 	VolumeDelete(ctx context.Context, in *VolumeDeleteRequest, opts ...grpc.CallOption) (*VolumeDeleteResponse, error)
 	VolumeMarkReadonly(ctx context.Context, in *VolumeMarkReadonlyRequest, opts ...grpc.CallOption) (*VolumeMarkReadonlyResponse, error)
 	VolumeMarkWritable(ctx context.Context, in *VolumeMarkWritableRequest, opts ...grpc.CallOption) (*VolumeMarkWritableResponse, error)
@@ -266,6 +268,16 @@ func (c *volumeServerClient) VolumeUnmount(ctx context.Context, in *VolumeUnmoun
 	return out, nil
 }
 
+func (c *volumeServerClient) VolumeConsolidateIndex(ctx context.Context, in *VolumeConsolidateIndexRequest, opts ...grpc.CallOption) (*VolumeConsolidateIndexResponse, error) {
+	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
+	out := new(VolumeConsolidateIndexResponse)
+	err := c.cc.Invoke(ctx, VolumeServer_VolumeConsolidateIndex_FullMethodName, in, out, cOpts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func (c *volumeServerClient) VolumeDelete(ctx context.Context, in *VolumeDeleteRequest, opts ...grpc.CallOption) (*VolumeDeleteResponse, error) {
 	cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
 	out := new(VolumeDeleteResponse)
@@ -727,6 +739,7 @@ type VolumeServerServer interface {
 	VolumeIncrementalCopy(*VolumeIncrementalCopyRequest, grpc.ServerStreamingServer[VolumeIncrementalCopyResponse]) error
 	VolumeMount(context.Context, *VolumeMountRequest) (*VolumeMountResponse, error)
 	VolumeUnmount(context.Context, *VolumeUnmountRequest) (*VolumeUnmountResponse, error)
+	VolumeConsolidateIndex(context.Context, *VolumeConsolidateIndexRequest) (*VolumeConsolidateIndexResponse, error)
 	VolumeDelete(context.Context, *VolumeDeleteRequest) (*VolumeDeleteResponse, error)
 	VolumeMarkReadonly(context.Context, *VolumeMarkReadonlyRequest) (*VolumeMarkReadonlyResponse, error)
 	VolumeMarkWritable(context.Context, *VolumeMarkWritableRequest) (*VolumeMarkWritableResponse, error)
@@ -813,6 +826,9 @@ func (UnimplementedVolumeServerServer) VolumeMount(context.Context, *VolumeMount
 func (UnimplementedVolumeServerServer) VolumeUnmount(context.Context, *VolumeUnmountRequest) (*VolumeUnmountResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method VolumeUnmount not implemented")
 }
+func (UnimplementedVolumeServerServer) VolumeConsolidateIndex(context.Context, *VolumeConsolidateIndexRequest) (*VolumeConsolidateIndexResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method VolumeConsolidateIndex not implemented")
+}
 func (UnimplementedVolumeServerServer) VolumeDelete(context.Context, *VolumeDeleteRequest) (*VolumeDeleteResponse, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method VolumeDelete not implemented")
 }
@@ -1129,6 +1145,24 @@ func _VolumeServer_VolumeUnmount_Handler(srv interface{}, ctx context.Context, d
 	return interceptor(ctx, in, info, handler)
 }
 
+func _VolumeServer_VolumeConsolidateIndex_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(VolumeConsolidateIndexRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(VolumeServerServer).VolumeConsolidateIndex(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: VolumeServer_VolumeConsolidateIndex_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(VolumeServerServer).VolumeConsolidateIndex(ctx, req.(*VolumeConsolidateIndexRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 func _VolumeServer_VolumeDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(VolumeDeleteRequest)
 	if err := dec(in); err != nil {
@@ -1771,6 +1805,10 @@ var VolumeServer_ServiceDesc = grpc.ServiceDesc{
 			MethodName: "VolumeUnmount",
 			Handler:    _VolumeServer_VolumeUnmount_Handler,
 		},
+		{
+			MethodName: "VolumeConsolidateIndex",
+			Handler:    _VolumeServer_VolumeConsolidateIndex_Handler,
+		},
 		{
 			MethodName: "VolumeDelete",
 			Handler:    _VolumeServer_VolumeDelete_Handler,

weed/server/volume_grpc_admin.go

@@ -161,6 +161,30 @@ func (vs *VolumeServer) VolumeUnmount(ctx context.Context, req *volume_server_pb
 
 }
 
+func (vs *VolumeServer) VolumeConsolidateIndex(ctx context.Context, req *volume_server_pb.VolumeConsolidateIndexRequest) (*volume_server_pb.VolumeConsolidateIndexResponse, error) {
+
+	resp := &volume_server_pb.VolumeConsolidateIndexResponse{}
+
+	if err := vs.checkGrpcAdminAuth(ctx); err != nil {
+		return resp, err
+	}
+
+	if err := vs.CheckMaintenanceMode(); err != nil {
+		return resp, err
+	}
+
+	err := vs.store.ConsolidateVolumeIndex(needle.VolumeId(req.VolumeId))
+
+	if err != nil {
+		glog.Errorf("volume consolidate index %v: %v", req, err)
+	} else {
+		glog.V(2).Infof("volume consolidate index %v", req)
+	}
+
+	return resp, err
+
+}
+
 func (vs *VolumeServer) VolumeDelete(ctx context.Context, req *volume_server_pb.VolumeDeleteRequest) (*volume_server_pb.VolumeDeleteResponse, error) {
 	resp := &volume_server_pb.VolumeDeleteResponse{}
 

weed/server/volume_grpc_erasure_coding.go

@@ -1009,6 +1009,23 @@ func (vs *VolumeServer) VolumeEcShardsToVolume(ctx context.Context, req *volume_
 		glog.Errorf("CompactVolumeFiles %s: %v", dataBaseFileName, err)
 	}
 
+	// Co-locate the rebuilt index with the data. The rebuild wrote the .idx to
+	// the shared -dir.idx directory, but the on-demand VolumeMount scans only
+	// the data directory and matches on .idx/.vif: with the index off in the
+	// index directory it would find the volume's leftover EC .vif instead and
+	// skip it as EC metadata. Moving the .idx next to the .dat lets the mount
+	// find the volume; VolumeConsolidateIndex returns it to the index directory
+	// once the EC shards are deleted.
+	if volumeLocation.IdxDirectory != volumeLocation.Directory {
+		idxSrc := storage.VolumeFileName(volumeLocation.IdxDirectory, v.Collection, int(req.VolumeId)) + ".idx"
+		idxDst := storage.VolumeFileName(volumeLocation.Directory, v.Collection, int(req.VolumeId)) + ".idx"
+		if util.FileExists(idxSrc) {
+			if moveErr := storage.RenameOrCopyFile(idxSrc, idxDst); moveErr != nil {
+				glog.Warningf("co-locate rebuilt index %s -> %s: %v", idxSrc, idxDst, moveErr)
+			}
+		}
+	}
+
 	return &volume_server_pb.VolumeEcShardsToVolumeResponse{}, nil
 }
 

weed/storage/disk_location.go

@@ -160,14 +160,14 @@ func getValidVolumeName(basename string) string {
 }
 
 // hasEcxFile reports whether an .ecx for volumeName exists on this disk.
-// Checks IdxDirectory first, then falls back to Directory (the .ecx may
-// have been created before -dir.idx was configured).
+// Checks the local Directory first (where the index sits co-located with the
+// shards during a move or reconstruct), then the shared IdxDirectory.
 func (l *DiskLocation) hasEcxFile(volumeName string) bool {
-	if util.FileExists(filepath.Join(l.IdxDirectory, volumeName+".ecx")) {
+	if util.FileExists(filepath.Join(l.Directory, volumeName+".ecx")) {
 		return true
 	}
 	if l.IdxDirectory != l.Directory {
-		return util.FileExists(filepath.Join(l.Directory, volumeName+".ecx"))
+		return util.FileExists(filepath.Join(l.IdxDirectory, volumeName+".ecx"))
 	}
 	return false
 }

weed/storage/disk_location_ec.go

@@ -110,17 +110,19 @@ func (l *DiskLocation) FindEcShard(vid needle.VolumeId, shardId erasure_coding.S
 // from the .ecx that travels with the first shard, which is the source of
 // the orphan-shard layout reported in #9212.
 func (l *DiskLocation) HasEcxFileOnDisk(collection string, vid needle.VolumeId) bool {
-	idxBase := erasure_coding.EcShardFileName(collection, l.IdxDirectory, int(vid))
+	// Prefer the local data directory, where the index sits co-located with the
+	// shards during a move or reconstruct, then the shared IdxDirectory.
 	// A 0-byte .ecx is a corrupt stub left by a failed EC distribute copy;
 	// it cannot drive mount and must not steer placement decisions toward
 	// this disk. Treat it as absent so the caller falls through to a
 	// sibling disk that may hold a valid index.
-	if info, err := os.Stat(idxBase + ".ecx"); err == nil && !info.IsDir() && info.Size() > 0 {
+	dataBase := erasure_coding.EcShardFileName(collection, l.Directory, int(vid))
+	if info, err := os.Stat(dataBase + ".ecx"); err == nil && !info.IsDir() && info.Size() > 0 {
 		return true
 	}
 	if l.IdxDirectory != l.Directory {
-		dataBase := erasure_coding.EcShardFileName(collection, l.Directory, int(vid))
-		if info, err := os.Stat(dataBase + ".ecx"); err == nil && !info.IsDir() && info.Size() > 0 {
+		idxBase := erasure_coding.EcShardFileName(collection, l.IdxDirectory, int(vid))
+		if info, err := os.Stat(idxBase + ".ecx"); err == nil && !info.IsDir() && info.Size() > 0 {
 			return true
 		}
 	}

weed/storage/erasure_coding/ec_volume.go

@@ -72,6 +72,19 @@ type EcVolume struct {
 	bitrotStatus BitrotStatus
 }
 
+// statEcxSize returns the size of an .ecx file, os.ErrNotExist when it is absent
+// (or a directory), so the resolver can prefer a non-empty copy.
+func statEcxSize(path string) (int64, error) {
+	info, statErr := os.Stat(path)
+	if statErr != nil {
+		return 0, statErr
+	}
+	if info.IsDir() {
+		return 0, os.ErrNotExist
+	}
+	return info.Size(), nil
+}
+
 func NewEcVolume(diskType types.DiskType, dir string, dirIdx string, collection string, vid needle.VolumeId) (ev *EcVolume, err error) {
 	ev = &EcVolume{dir: dir, dirIdx: dirIdx, Collection: collection, VolumeId: vid, diskType: diskType}
 
@@ -87,25 +100,36 @@ func NewEcVolume(diskType types.DiskType, dir string, dirIdx string, collection
 	// stream is indistinguishable from that empty case by file size alone;
 	// preventing such stubs is the receiver-side cleanup in writeToFile's
 	// job, not this open path.
-	ev.ecxActualDir = dirIdx
+	// Resolve the .ecx, preferring the copy co-located with the shard data on
+	// this disk — where a move or reconstruct leaves it — then the caller's
+	// index directory. That directory is either the shared -dir.idx dir or a
+	// sibling disk that owns the .ecx when this disk holds only a 0-byte stub
+	// left by an interrupted copy (#9212). A 0-byte .ecx is a legitimate empty
+	// index, so the local copy yields only to a *non-empty* copy elsewhere,
+	// never to a mere absence: prefer a non-empty .ecx local-first, then fall
+	// back to whichever exists at all.
+	localBaseFileName := dataBaseFileName
+	sharedBaseFileName := indexBaseFileName
+	localSize, localErr := statEcxSize(localBaseFileName + ".ecx")
+	sharedSize, sharedErr := int64(0), os.ErrNotExist
+	if dirIdx != dir {
+		sharedSize, sharedErr = statEcxSize(sharedBaseFileName + ".ecx")
+	}
+	switch {
+	case localErr == nil && localSize > 0:
+		indexBaseFileName, ev.ecxActualDir = localBaseFileName, dir
+	case sharedErr == nil && sharedSize > 0:
+		indexBaseFileName, ev.ecxActualDir = sharedBaseFileName, dirIdx
+		glog.V(1).Infof("ecx not local at %s.ecx, using %s.ecx", localBaseFileName, sharedBaseFileName)
+	case localErr == nil: // local exists but is a 0-byte empty index
+		indexBaseFileName, ev.ecxActualDir = localBaseFileName, dir
+	case sharedErr == nil: // only a 0-byte copy in the index dir
+		indexBaseFileName, ev.ecxActualDir = sharedBaseFileName, dirIdx
+	default:
+		return nil, fmt.Errorf("cannot open ec volume index %s.ecx (or %s.ecx): %w", localBaseFileName, sharedBaseFileName, os.ErrNotExist)
+	}
 	if ev.ecxFile, err = os.OpenFile(indexBaseFileName+".ecx", os.O_RDWR, 0644); err != nil {
-		if dirIdx != dir && os.IsNotExist(err) {
-			// fall back to data directory if idx directory does not have the .ecx file
-			firstErr := err
-			glog.V(1).Infof("ecx file not found at %s.ecx, falling back to %s.ecx", indexBaseFileName, dataBaseFileName)
-			if ev.ecxFile, err = os.OpenFile(dataBaseFileName+".ecx", os.O_RDWR, 0644); err != nil {
-				if os.IsNotExist(err) {
-					return nil, fmt.Errorf("open ecx index %s.ecx (fallback %s.ecx): %w", indexBaseFileName, dataBaseFileName, os.ErrNotExist)
-				}
-				return nil, fmt.Errorf("open ecx index %s.ecx: %v; fallback %s.ecx: %w", indexBaseFileName, firstErr, dataBaseFileName, err)
-			}
-			indexBaseFileName = dataBaseFileName
-			ev.ecxActualDir = dir
-		} else if os.IsNotExist(err) {
-			return nil, fmt.Errorf("cannot open ec volume index %s.ecx: %w", indexBaseFileName, os.ErrNotExist)
-		} else {
-			return nil, fmt.Errorf("cannot open ec volume index %s.ecx: %w", indexBaseFileName, err)
-		}
+		return nil, fmt.Errorf("cannot open ec volume index %s.ecx: %w", indexBaseFileName, err)
 	}
 	ecxFi, statErr := ev.ecxFile.Stat()
 	if statErr != nil {
@@ -277,8 +301,17 @@ func (ev *EcVolume) Destroy() {
 	for _, s := range ev.Shards {
 		s.Destroy()
 	}
-	os.Remove(ev.FileName(".ecx"))
-	os.Remove(ev.FileName(".ecj"))
+	// Sweep the EC-only index files from BOTH the data directory and the shared
+	// index directory. A move or reconstruct can leave a copy in whichever
+	// directory is not ecxActualDir; removing only the active one leaves a stale
+	// index that a later reload could pick up and re-mount as a phantom EC
+	// volume. .ecx/.ecj are EC-specific, so removing both copies is safe.
+	for _, base := range ev.ecIndexBaseNames() {
+		os.Remove(base + ".ecx")
+		os.Remove(base + ".ecj")
+	}
+	// The .vif is shared with a coexisting normal volume (e.g. mid-decode), so
+	// only remove the active copy, not both.
 	os.Remove(ev.FileName(".vif"))
 	// Remove the bitrot checksum sidecar(s) so a later volume reuse cannot load
 	// stale protection. Search both the data and index bases.
@@ -288,6 +321,16 @@ func (ev *EcVolume) Destroy() {
 	}
 }
 
+// ecIndexBaseNames returns the base paths for the volume's EC index files in
+// both the data and index directories, deduplicated when they coincide.
+func (ev *EcVolume) ecIndexBaseNames() []string {
+	bases := []string{ev.DataBaseFileName()}
+	if ev.IndexBaseFileName() != ev.DataBaseFileName() {
+		bases = append(bases, ev.IndexBaseFileName())
+	}
+	return bases
+}
+
 // DiskType returns the disk type the EC volume currently reports under.
 // Defaults to the physical location's disk type; orchestrators can override
 // it via SetDiskType so the volume keeps reporting under the source

weed/storage/erasure_coding/ec_volume_test.go

@@ -3,6 +3,7 @@ package erasure_coding
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -155,3 +156,103 @@ func TestNewEcVolumeLoadsCustomRatio(t *testing.T) {
 		t.Fatalf("ECContext = %d+%d, want 9+3", ev.ECContext.DataShards, ev.ECContext.ParityShards)
 	}
 }
+
+// ecIndexDirsFixture lays out an EC volume across a data dir and a separate
+// index dir (the -dir.idx split), planting a .vif next to the data and .ecx
+// files of the given sizes (-1 = absent). It returns the two bases.
+func ecIndexDirsFixture(t *testing.T, vid needle.VolumeId, localEcx, sharedEcx int) (dataDir, idxDir, dataBase, idxBase string) {
+	t.Helper()
+	root := t.TempDir()
+	dataDir = filepath.Join(root, "data")
+	idxDir = filepath.Join(root, "idx")
+	for _, d := range []string{dataDir, idxDir} {
+		if err := os.MkdirAll(d, 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+	dataBase = EcShardFileName("", dataDir, int(vid))
+	idxBase = EcShardFileName("", idxDir, int(vid))
+	writeEcx := func(base string, size int) {
+		if size < 0 {
+			return
+		}
+		if err := os.WriteFile(base+".ecx", make([]byte, size), 0o644); err != nil {
+			t.Fatalf("write .ecx: %v", err)
+		}
+	}
+	writeEcx(dataBase, localEcx)
+	writeEcx(idxBase, sharedEcx)
+	if err := volume_info.SaveVolumeInfo(dataBase+".vif", &volume_server_pb.VolumeInfo{
+		Version:       uint32(needle.Version3),
+		EcShardConfig: &volume_server_pb.EcShardConfig{DataShards: 10, ParityShards: 4},
+	}); err != nil {
+		t.Fatalf("save .vif: %v", err)
+	}
+	return
+}
+
+// TestNewEcVolumePrefersLocalEcx pins the local-first resolution: with a
+// non-empty .ecx co-located with the shard data AND one in the shared index
+// dir, NewEcVolume opens the local copy — where a move or reconstruct leaves
+// the index.
+func TestNewEcVolumePrefersLocalEcx(t *testing.T) {
+	const vid = needle.VolumeId(200)
+	dataDir, idxDir, _, _ := ecIndexDirsFixture(t, vid, 16, 16)
+
+	ev, err := NewEcVolume(types.HardDriveType, dataDir, idxDir, "", vid)
+	if err != nil {
+		t.Fatalf("NewEcVolume: %v", err)
+	}
+	defer ev.Close()
+
+	if got := filepath.Dir(ev.FileName(".ecx")); got != dataDir {
+		t.Errorf(".ecx resolved in %q, want the data dir %q (local-first)", got, dataDir)
+	}
+}
+
+// TestNewEcVolumeZeroByteLocalYieldsToSharedEcx pins that a 0-byte local .ecx
+// (an interrupted-copy stub is indistinguishable from a legitimately empty
+// index by size) yields to a *non-empty* copy in the shared index dir, keeping
+// the cross-disk fallback (#9212).
+func TestNewEcVolumeZeroByteLocalYieldsToSharedEcx(t *testing.T) {
+	const vid = needle.VolumeId(201)
+	dataDir, idxDir, _, _ := ecIndexDirsFixture(t, vid, 0, 16)
+
+	ev, err := NewEcVolume(types.HardDriveType, dataDir, idxDir, "", vid)
+	if err != nil {
+		t.Fatalf("NewEcVolume: %v", err)
+	}
+	defer ev.Close()
+
+	if got := filepath.Dir(ev.FileName(".ecx")); got != idxDir {
+		t.Errorf(".ecx resolved in %q, want the shared idx dir %q (non-empty wins over 0-byte local)", got, idxDir)
+	}
+}
+
+// TestEcVolumeDestroySweepsBothDirs pins that Destroy removes the EC index
+// files from both the data and index directories, so a stale copy left by a
+// move or reconstruct cannot re-mount as a phantom EC volume.
+func TestEcVolumeDestroySweepsBothDirs(t *testing.T) {
+	const vid = needle.VolumeId(202)
+	dataDir, idxDir, dataBase, idxBase := ecIndexDirsFixture(t, vid, 16, 16)
+	// A stale .ecj in each dir too.
+	for _, base := range []string{dataBase, idxBase} {
+		if err := os.WriteFile(base+".ecj", nil, 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	ev, err := NewEcVolume(types.HardDriveType, dataDir, idxDir, "", vid)
+	if err != nil {
+		t.Fatalf("NewEcVolume: %v", err)
+	}
+	ev.Destroy()
+
+	for _, base := range []string{dataBase, idxBase} {
+		for _, ext := range []string{".ecx", ".ecj"} {
+			if _, err := os.Stat(base + ext); !os.IsNotExist(err) {
+				t.Errorf("%s%s survived Destroy (err=%v), want removed from both dirs", base, ext, err)
+			}
+		}
+	}
+}

weed/storage/store.go

@@ -4,10 +4,12 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"os"
 	"path/filepath"
 	"strings"
 	"sync"
 	"sync/atomic"
+	"syscall"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
@@ -810,6 +812,70 @@ func (s *Store) UnmountVolume(i needle.VolumeId) error {
 	return errors.Join(errs...)
 }
 
+// ConsolidateVolumeIndex returns a volume's index to the configured -dir.idx
+// directory when it is currently co-located with the data. A decode/reconstruct
+// leaves the rebuilt .idx next to the .dat so the on-demand mount can find the
+// volume while the old EC .ecx still coexists in the index directory; once the
+// shards are gone this puts the index back on its own tier. It is a no-op when
+// no separate index directory is configured or the index is already there.
+//
+// The relocation happens in place under the volume lock (see RelocateIndexTo),
+// so the volume never leaves the mounted set and a concurrent read blocks
+// briefly rather than failing.
+func (s *Store) ConsolidateVolumeIndex(i needle.VolumeId) error {
+	for _, location := range s.Locations {
+		if v, found := location.FindVolume(i); found {
+			if location.IdxDirectory == location.Directory {
+				return nil
+			}
+			return v.RelocateIndexTo(location.IdxDirectory)
+		}
+	}
+	return fmt.Errorf("volume %d not found on disk", i)
+}
+
+// RenameOrCopyFile moves src to dst, falling back to a copy when the two sit on
+// different filesystems (os.Rename returns EXDEV across the data and -dir.idx
+// disks, the separate media the flag exists to use).
+func RenameOrCopyFile(src, dst string) error {
+	if err := os.Rename(src, dst); err == nil {
+		return nil
+	} else if !errors.Is(err, syscall.EXDEV) {
+		return err
+	}
+	in, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer in.Close()
+	out, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(out, in); err != nil {
+		out.Close()
+		os.Remove(dst)
+		return err
+	}
+	if err := out.Sync(); err != nil {
+		out.Close()
+		os.Remove(dst)
+		return err
+	}
+	if err := out.Close(); err != nil {
+		os.Remove(dst)
+		return err
+	}
+	// Roll the copy back if the source cannot be removed, so a failure never
+	// leaves two divergent copies (the loader would keep using the data-dir one
+	// while the idx-dir orphan goes stale).
+	if err := os.Remove(src); err != nil {
+		os.Remove(dst)
+		return err
+	}
+	return nil
+}
+
 func (s *Store) DeleteVolume(i needle.VolumeId, onlyEmpty bool, keepRemoteData bool) error {
 	// Delete every copy of the volume id across disks, not just the first match, so
 	// a stale twin (e.g. a re-attached disk; NewStore has no cross-disk duplicate

weed/storage/store_consolidate_index_test.go

@@ -0,0 +1,130 @@
+package storage
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRenameOrCopyFile(t *testing.T) {
+	dir := t.TempDir()
+	src := filepath.Join(dir, "a.idx")
+	dst := filepath.Join(dir, "sub", "a.idx")
+	require.NoError(t, os.MkdirAll(filepath.Dir(dst), 0o755))
+	content := []byte("index-bytes")
+	require.NoError(t, os.WriteFile(src, content, 0o644))
+
+	require.NoError(t, RenameOrCopyFile(src, dst))
+
+	_, err := os.Stat(src)
+	require.True(t, os.IsNotExist(err), "source should be gone after the move")
+	got, err := os.ReadFile(dst)
+	require.NoError(t, err)
+	require.Equal(t, content, got, "content must survive the move")
+}
+
+// newIdxSplitStore builds a single-disk store whose index directory differs
+// from its data directory (-dir.idx), draining every notify channel so mount
+// and unmount never block.
+func newIdxSplitStore(t *testing.T, dataDir, idxDir string) *Store {
+	t.Helper()
+	require.NoError(t, os.MkdirAll(dataDir, 0o755))
+	require.NoError(t, os.MkdirAll(idxDir, 0o755))
+	store := NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "store-id",
+		[]string{dataDir}, []int32{100}, []util.MinFreeSpace{{}}, idxDir,
+		NeedleMapInMemory, []types.DiskType{types.HardDriveType}, nil, 3,
+		stats.DefaultDiskIOProbeConfig())
+	done := make(chan struct{})
+	go func() {
+		for {
+			select {
+			case <-store.NewVolumesChan:
+			case <-store.DeletedVolumesChan:
+			case <-store.NewEcShardsChan:
+			case <-store.DeletedEcShardsChan:
+			case <-store.StateUpdateChan:
+			case <-done:
+				return
+			}
+		}
+	}()
+	t.Cleanup(func() {
+		store.Close()
+		close(done)
+	})
+	return store
+}
+
+// TestConsolidateVolumeIndexMovesIdxToIdxDir pins the relocate a decode runs
+// once the EC shards are gone: an index co-located with the data (where the
+// reconstruct left it) is moved back to the -dir.idx directory, and the volume
+// stays mounted.
+func TestConsolidateVolumeIndexMovesIdxToIdxDir(t *testing.T) {
+	root := t.TempDir()
+	dataDir := filepath.Join(root, "data")
+	idxDir := filepath.Join(root, "idx")
+	require.NoError(t, os.MkdirAll(dataDir, 0o755))
+	require.NoError(t, os.MkdirAll(idxDir, 0o755))
+	const vid = needle.VolumeId(7)
+
+	// Create the volume with its index co-located in the data dir (the state a
+	// reconstruct leaves), then close it so the store mounts it fresh.
+	v, err := NewVolume(dataDir, dataDir, "", vid, NeedleMapInMemory,
+		&super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
+	require.NoError(t, err)
+	v.Close()
+
+	dataIdx := filepath.Join(dataDir, "7.idx")
+	idxDirIdx := filepath.Join(idxDir, "7.idx")
+	require.FileExists(t, dataIdx, "precondition: index co-located with the data")
+
+	store := newIdxSplitStore(t, dataDir, idxDir)
+	require.NoError(t, store.MountVolume(vid))
+
+	// Write a needle so the relocate has real index state to preserve.
+	mounted := store.findVolume(vid)
+	require.NotNil(t, mounted)
+	n := &needle.Needle{Id: types.Uint64ToNeedleId(42), Data: []byte("payload-across-relocate")}
+	n.Checksum = needle.NewCRC(n.Data)
+	_, _, _, err = mounted.writeNeedle2(n, true, false)
+	require.NoError(t, err)
+
+	require.NoError(t, store.ConsolidateVolumeIndex(vid))
+
+	require.FileExists(t, idxDirIdx, "index should have moved to the idx dir")
+	_, err = os.Stat(dataIdx)
+	require.True(t, os.IsNotExist(err), "index should be gone from the data dir")
+	require.NotNil(t, store.findVolume(vid), "volume should stay mounted after consolidation")
+
+	// The volume stayed mounted and reloaded in place, so the needle still reads
+	// back — a concurrent read would have blocked on the lock, never failed.
+	got := &needle.Needle{Id: n.Id}
+	_, err = store.ReadVolumeNeedle(vid, got, &ReadOption{}, func(types.Size) {})
+	require.NoError(t, err, "volume must serve reads after consolidation")
+	require.Equal(t, n.Data, got.Data, "needle content survives the relocate")
+}
+
+// TestConsolidateVolumeIndexNoopWithoutIdxDir pins that the relocate is a no-op
+// when no separate index directory is configured.
+func TestConsolidateVolumeIndexNoopWithoutIdxDir(t *testing.T) {
+	dataDir := t.TempDir()
+	const vid = needle.VolumeId(8)
+	v, err := NewVolume(dataDir, dataDir, "", vid, NeedleMapInMemory,
+		&super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
+	require.NoError(t, err)
+	v.Close()
+
+	store := newIdxSplitStore(t, dataDir, dataDir) // idx dir == data dir
+	require.NoError(t, store.MountVolume(vid))
+
+	require.NoError(t, store.ConsolidateVolumeIndex(vid))
+	require.FileExists(t, filepath.Join(dataDir, "8.idx"), "index stays put without -dir.idx")
+	require.NotNil(t, store.findVolume(vid), "volume stays mounted")
+}

weed/storage/volume.go

@@ -173,6 +173,59 @@ func (v *Volume) FileName(ext string) (fileName string) {
 	return VolumeFileName(v.dir, v.Collection, int(v.Id)) + ext
 }
 
+// RelocateIndexTo moves the volume's index to newIdxDir and reopens the volume
+// against it in place, without unmounting. It takes the data-file write lock —
+// so a concurrent read blocks briefly instead of failing — closes the needle
+// map and data backend, moves the .idx (and the derived .sdx best-effort), then
+// retargets dirIdx and reloads, mirroring CommitCompact's close-swap-load. A
+// decode co-locates the rebuilt index with the data so the on-demand mount can
+// find the volume; this returns it to the -dir.idx tier once the EC shards are
+// gone. A no-op when the index already lives in newIdxDir. A derived .ldb is
+// not moved: the reload rebuilds it in newIdxDir from the .idx.
+func (v *Volume) RelocateIndexTo(newIdxDir string) error {
+	v.dataFileAccessLock.Lock()
+	defer v.dataFileAccessLock.Unlock()
+
+	if v.dirIdx == newIdxDir {
+		return nil
+	}
+	oldBase := VolumeFileName(v.dirIdx, v.Collection, int(v.Id))
+	if _, err := os.Stat(oldBase + ".idx"); err != nil {
+		return nil // nothing co-located to move
+	}
+	newBase := VolumeFileName(newIdxDir, v.Collection, int(v.Id))
+
+	if v.nm != nil {
+		_ = v.nm.Sync()
+		v.nm.Close()
+		v.nm = nil
+	}
+	if v.DataBackend != nil {
+		_ = v.DataBackend.Sync()
+		_ = v.DataBackend.Close()
+		v.DataBackend = nil
+	}
+
+	if err := RenameOrCopyFile(oldBase+".idx", newBase+".idx"); err != nil {
+		// Reopen against the old dir so the volume is not left down; surface a
+		// failed reopen since it leaves the volume unusable until the next load.
+		if reopenErr := v.load(true, false, v.needleMapKind, 0, v.Version()); reopenErr != nil {
+			glog.Errorf("relocate volume %d: reopen after failed .idx move: %v", v.Id, reopenErr)
+		}
+		return fmt.Errorf("relocate index for volume %d: move .idx: %w", v.Id, err)
+	}
+	// The .sdx is a derived sorted index; move it when present, but a failure is
+	// not fatal — drop the stale copy so the reload rebuilds it in the new dir.
+	if _, err := os.Stat(oldBase + ".sdx"); err == nil {
+		if err := RenameOrCopyFile(oldBase+".sdx", newBase+".sdx"); err != nil {
+			glog.Warningf("relocate volume %d: move .sdx: %v (will rebuild)", v.Id, err)
+			_ = os.Remove(oldBase + ".sdx")
+		}
+	}
+	v.dirIdx = newIdxDir
+	return v.load(true, false, v.needleMapKind, 0, v.Version())
+}
+
 func (v *Volume) Version() needle.Version {
 	v.superBlockAccessLock.Lock()
 	defer v.superBlockAccessLock.Unlock()