refactor: extract EC orchestration into a shared weed/ec package (#10760)

* shell: move ErrorWaitGroup to weed/util

* shell: remove unused CandidateEcNode and EcRack types

* ec: extract EC orchestration logic from weed/shell into weed/ec

Move the EC node/topology model, balance engine, encode pipeline, decode
pipeline, and rebuild engine into a new weed/ec package so shell commands
and maintenance workers can share the logic. Shell commands keep flag
parsing and delegate through a small ec.Env (dial option, topology fetch,
volume locations, lock check). Tests move along with the code.

* shell: remove unused proportional-rebalance type stubs

* ec: move scrub, replication check, and shard unmount engines into weed/ec

* worker: share the EC generation-aware shard counter from weed/ec

* ec: gofmt

* shell: drop EC aliases with no remaining callers

* ec: guard a missing topology hook and nil disk entries in topology helpers

* ec: drop trailing newlines from decode error strings

* ec: re-check the shell lock before applying shard unmounts

* shell: trim -node entries in ec.scrub

weed/ec/ec_check_replication.go

@@ -0,0 +1,211 @@
+package ec
+
+import (
+	"fmt"
+	"io"
+	"slices"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+)
+
+// CheckEcVolumeReplication reports EC volumes with under- or over-replicated
+// shards among the given data nodes. volumeIDs, when non-empty, narrows the
+// check to those volume ids.
+func CheckEcVolumeReplication(writer io.Writer, dataNodes []*master_pb.DataNodeInfo, volumeIDs map[uint32]bool, showDetails bool) error {
+	runner := &ecCheckReplicationRunner{
+		writer:      writer,
+		dataNodes:   dataNodes,
+		volumeIDMap: volumeIDs,
+	}
+	return runner.checkEcVolumes(showDetails)
+}
+
+type ecCheckReplicationRunner struct {
+	writer      io.Writer
+	dataNodes   []*master_pb.DataNodeInfo
+	volumeIDMap map[uint32]bool
+}
+
+func (r *ecCheckReplicationRunner) write(format string, a ...any) {
+	fmt.Fprintf(r.writer, format, a...)
+}
+
+func (r *ecCheckReplicationRunner) isVolumeIDValid(vid uint32) bool {
+	if len(r.volumeIDMap) == 0 {
+		return true
+	}
+	return r.volumeIDMap[vid]
+}
+
+// ecVolumeShardReplication aggregates the observed shards for a single EC volume,
+// together with the data+parity ratio the volume was encoded with. The ratio is
+// taken per volume (via erasure_coding.EcShardsVolume*Shards) so custom EC
+// ratios are checked against their own expected shard count.
+type ecVolumeShardReplication struct {
+	dataShards   int
+	parityShards int
+	// shardAddresses maps a shard id to the sorted node addresses hosting it.
+	shardAddresses map[erasure_coding.ShardId][]string
+}
+
+func (h *ecVolumeShardReplication) totalShards() int {
+	return h.dataShards + h.parityShards
+}
+
+// replicaCount is the total number of shard copies observed, counting each
+// over-replicated shard once per hosting node.
+func (h *ecVolumeShardReplication) replicaCount() int {
+	n := 0
+	for _, addrs := range h.shardAddresses {
+		n += len(addrs)
+	}
+	return n
+}
+
+// unexpectedShardIds returns, in ascending order, any observed shard ids that
+// fall outside the volume's expected data+parity range. These are anomalies
+// (e.g. a stray shard) and are treated as over-replication.
+func (h *ecVolumeShardReplication) unexpectedShardIds() []erasure_coding.ShardId {
+	var ids []erasure_coding.ShardId
+	for sid := range h.shardAddresses {
+		if int(sid) >= h.totalShards() {
+			ids = append(ids, sid)
+		}
+	}
+	slices.Sort(ids)
+	return ids
+}
+
+// TODO: check shard sizes?
+func (r *ecCheckReplicationRunner) checkEcVolumes(showDetails bool) error {
+	// collect EC shard placement, keyed by volume id
+	volumes := map[uint32]*ecVolumeShardReplication{}
+	for _, dni := range r.dataNodes {
+		nodeAddress := dni.GetAddress()
+		for _, di := range dni.GetDiskInfos() {
+			for _, eci := range di.GetEcShardInfos() {
+				vid := eci.GetId()
+				if !r.isVolumeIDValid(vid) {
+					continue
+				}
+
+				h, ok := volumes[vid]
+				if !ok {
+					// all shards of a volume share one ratio; take it from the
+					// first shard message seen for the volume.
+					h = &ecVolumeShardReplication{
+						dataShards:     erasure_coding.EcShardsVolumeDataShards(eci),
+						parityShards:   erasure_coding.EcShardsVolumeParityShards(eci),
+						shardAddresses: map[erasure_coding.ShardId][]string{},
+					}
+					volumes[vid] = h
+				}
+
+				sinfo := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(eci)
+				for _, sid := range sinfo.Ids() {
+					h.shardAddresses[sid] = append(h.shardAddresses[sid], nodeAddress)
+				}
+			}
+		}
+	}
+
+	if len(volumes) == 0 {
+		// Asking about specific volume IDs that turn out not to be EC volumes is
+		// an error; an unfiltered run over a cluster with no EC volumes is a
+		// legitimate, healthy state rather than a failure.
+		if len(r.volumeIDMap) > 0 {
+			return fmt.Errorf("no EC volumes found")
+		}
+		r.write("No EC volumes found.\n")
+		return nil
+	}
+
+	// keep the shard address lists nicely sorted, for the sake of readability.
+	for _, h := range volumes {
+		for _, addrs := range h.shardAddresses {
+			slices.Sort(addrs)
+		}
+	}
+
+	// classify each volume against its own expected shard count
+	underreplicatedVolumeIDs := []uint32{}
+	overreplicatedVolumeIDs := []uint32{}
+	for vid, h := range volumes {
+		under := false
+		over := false
+		for sid := 0; sid < h.totalShards(); sid++ {
+			switch len(h.shardAddresses[erasure_coding.ShardId(sid)]) {
+			case 0:
+				under = true
+			case 1:
+			default:
+				over = true
+			}
+		}
+		// shard ids beyond the expected data+parity range are unexpected extras,
+		// i.e. redundant data the ratio doesn't call for.
+		if len(h.unexpectedShardIds()) > 0 {
+			over = true
+		}
+
+		// under- and over-replication are independent problems (missing shards risk
+		// data loss, redundant shards waste space), so a volume exhibiting both is
+		// reported in both lists.
+		if under {
+			underreplicatedVolumeIDs = append(underreplicatedVolumeIDs, vid)
+		}
+		if over {
+			overreplicatedVolumeIDs = append(overreplicatedVolumeIDs, vid)
+		}
+	}
+	slices.Sort(underreplicatedVolumeIDs)
+	slices.Sort(overreplicatedVolumeIDs)
+
+	// ...and display results
+	if len(underreplicatedVolumeIDs) == 0 && len(overreplicatedVolumeIDs) == 0 {
+		r.write("EC volumes are healthy.\n")
+		return nil
+	}
+
+	if len(underreplicatedVolumeIDs) != 0 {
+		r.write("Found %d/%d under-replicated EC volumes: %v\n", len(underreplicatedVolumeIDs), len(volumes), underreplicatedVolumeIDs)
+	}
+	if len(overreplicatedVolumeIDs) != 0 {
+		r.write("Found %d/%d over-replicated EC volumes: %v\n", len(overreplicatedVolumeIDs), len(volumes), overreplicatedVolumeIDs)
+	}
+
+	if showDetails {
+		if len(underreplicatedVolumeIDs) != 0 {
+			r.write("\n")
+			r.writeShardMaps("under-replicated", underreplicatedVolumeIDs, volumes)
+		}
+		if len(overreplicatedVolumeIDs) != 0 {
+			r.write("\n")
+			r.writeShardMaps("over-replicated", overreplicatedVolumeIDs, volumes)
+		}
+	}
+
+	return nil
+}
+
+func (r *ecCheckReplicationRunner) writeShardMaps(kind string, volumeIDs []uint32, volumes map[uint32]*ecVolumeShardReplication) {
+	for _, vid := range volumeIDs {
+		h := volumes[vid]
+		r.write("Shards map for %s EC volume %v (%d/%d shards):\n", kind, vid, h.replicaCount(), h.totalShards())
+		for sid := 0; sid < h.totalShards(); sid++ {
+			shardTypeDesc := ""
+			if sid >= h.dataShards {
+				shardTypeDesc = " (parity)"
+			}
+			if addrs, ok := h.shardAddresses[erasure_coding.ShardId(sid)]; ok {
+				r.write("\t%02d%s => %v\n", sid, shardTypeDesc, addrs)
+			} else {
+				r.write("\t%02d%s is missing\n", sid, shardTypeDesc)
+			}
+		}
+		for _, sid := range h.unexpectedShardIds() {
+			r.write("\t%02d (unexpected) => %v\n", int(sid), h.shardAddresses[sid])
+		}
+	}
+}

weed/ec/ec_check_replication_test.go

@@ -1,4 +1,4 @@
-package shell
+package ec
 
 import (
 	"bytes"

weed/ec/ec_common_test.go

@@ -1,4 +1,4 @@
-package shell
+package ec
 
 import (
 	"strings"
@@ -15,15 +15,15 @@ import (
 // best-scoring disk 0 (0 is also the zero value) and returned the fallback.
 func TestPickBestDiskOnNodeSelectsPhysicalDiskZero(t *testing.T) {
 	ecNode := &EcNode{
-		disks: map[uint32]*EcDisk{
-			0: {diskId: 0, diskType: string(types.SsdType), freeEcSlots: 10, ecShards: map[needle.VolumeId]*erasure_coding.ShardsInfo{}},
-			5: {diskId: 5, diskType: string(types.HardDriveType), freeEcSlots: 10, ecShards: map[needle.VolumeId]*erasure_coding.ShardsInfo{}},
+		Disks: map[uint32]*EcDisk{
+			0: {DiskId: 0, DiskType: string(types.SsdType), FreeEcSlots: 10, EcShards: map[needle.VolumeId]*erasure_coding.ShardsInfo{}},
+			5: {DiskId: 5, DiskType: string(types.HardDriveType), FreeEcSlots: 10, EcShards: map[needle.VolumeId]*erasure_coding.ShardsInfo{}},
 		},
 	}
 
 	// Disk 0 matches the requested type; it must win over the non-matching
 	// fallback on disk 5 instead of being treated as "no match".
-	if got := pickBestDiskOnNode(ecNode, needle.VolumeId(42), types.SsdType, false, 0, 0); got != 0 {
+	if got := PickBestDiskOnNode(ecNode, needle.VolumeId(42), types.SsdType, false, 0, 0); got != 0 {
 		t.Fatalf("want physical disk 0 (matching type), got %d", got)
 	}
 }
@@ -142,18 +142,18 @@ func TestCommandEcBalanceVolumeEvenButRackUneven(t *testing.T) {
 
 func newEcNode(dc string, rack string, dataNodeId string, freeEcSlot int) *EcNode {
 	return &EcNode{
-		info: &master_pb.DataNodeInfo{
+		Info: &master_pb.DataNodeInfo{
 			Id:        dataNodeId,
 			DiskInfos: make(map[string]*master_pb.DiskInfo),
 		},
-		dc:         DataCenterId(dc),
-		rack:       RackId(rack),
-		freeEcSlot: freeEcSlot,
+		DC:         DataCenterId(dc),
+		Rack:       RackId(rack),
+		FreeEcSlot: freeEcSlot,
 	}
 }
 
 func (ecNode *EcNode) addEcVolumeAndShardsForTest(vid uint32, collection string, shardIds []erasure_coding.ShardId) *EcNode {
-	return ecNode.addEcVolumeShards(needle.VolumeId(vid), collection, shardIds, types.HardDriveType)
+	return ecNode.AddEcVolumeShards(needle.VolumeId(vid), collection, shardIds, types.HardDriveType)
 }
 
 // TestCommandEcBalanceEvenDataAndParityDistribution verifies that after balancing:
@@ -190,8 +190,8 @@ func TestCommandEcBalanceEvenDataAndParityDistribution(t *testing.T) {
 	// With 6 racks:
 	// - Data shards (10): max 2 per rack (ceil(10/6) = 2)
 	// - Parity shards (4): max 1 per rack (ceil(4/6) = 1)
-	maxDataPerRack := ceilDivide(dataShardCount, 6)     // 2
-	maxParityPerRack := ceilDivide(parityShardCount, 6) // 1
+	maxDataPerRack := CeilDivide(dataShardCount, 6)     // 2
+	maxParityPerRack := CeilDivide(parityShardCount, 6) // 1
 
 	// Verify no rack has more than max data shards
 	for rackId, count := range dataPerRack {
@@ -247,9 +247,9 @@ func countDataAndParityShardsPerRack(ecNodes []*EcNode, vid needle.VolumeId, dat
 	parityPerRack = make(map[string]int)
 
 	for _, ecNode := range ecNodes {
-		si := findEcVolumeShardsInfo(ecNode, vid, types.HardDriveType)
+		si := FindEcVolumeShardsInfo(ecNode, vid, types.HardDriveType)
 		for _, shardId := range si.Ids() {
-			rackId := string(ecNode.rack)
+			rackId := string(ecNode.Rack)
 			if int(shardId) < dataShardCount {
 				dataPerRack[rackId]++
 			} else {
@@ -286,8 +286,8 @@ func TestCommandEcBalanceMultipleVolumesEvenDistribution(t *testing.T) {
 	for _, vid := range []needle.VolumeId{1, 2} {
 		dataPerRack, parityPerRack := countDataAndParityShardsPerRack(ecb.ecNodes, vid, erasure_coding.DataShardsCount)
 
-		maxDataPerRack := ceilDivide(erasure_coding.DataShardsCount, 6)
-		maxParityPerRack := ceilDivide(erasure_coding.ParityShardsCount, 6)
+		maxDataPerRack := CeilDivide(erasure_coding.DataShardsCount, 6)
+		maxParityPerRack := CeilDivide(erasure_coding.ParityShardsCount, 6)
 
 		for rackId, count := range dataPerRack {
 			if count > maxDataPerRack {
@@ -339,16 +339,16 @@ func TestCommandEcBalanceAllNodesShareAllVolumes(t *testing.T) {
 	// Count total shards per node after balancing
 	for _, node := range ecb.ecNodes {
 		count := 0
-		if diskInfo, found := node.info.DiskInfos[string(types.HardDriveType)]; found {
+		if diskInfo, found := node.Info.DiskInfos[string(types.HardDriveType)]; found {
 			for _, ecsi := range diskInfo.EcShardInfos {
 				count += erasure_coding.GetShardCount(ecsi)
 			}
 		}
 		// Average is 7, so all nodes should be at 7 (ceil(28/4) = 7)
 		if count > 7 {
-			t.Errorf("node %s has %d shards after balancing, expected at most 7", node.info.Id, count)
+			t.Errorf("node %s has %d shards after balancing, expected at most 7", node.Info.Id, count)
 		}
-		t.Logf("node %s: %d shards", node.info.Id, count)
+		t.Logf("node %s: %d shards", node.Info.Id, count)
 	}
 }
 
@@ -409,12 +409,12 @@ func TestCommandEcBalanceIssue8793Topology(t *testing.T) {
 	// Log initial state
 	for _, node := range ecb.ecNodes {
 		count := 0
-		if diskInfo, found := node.info.DiskInfos[string(types.HardDriveType)]; found {
+		if diskInfo, found := node.Info.DiskInfos[string(types.HardDriveType)]; found {
 			for _, ecsi := range diskInfo.EcShardInfos {
 				count += erasure_coding.GetShardCount(ecsi)
 			}
 		}
-		t.Logf("BEFORE node %s (max %d): %d shards", node.info.Id, node.freeEcSlot+count, count)
+		t.Logf("BEFORE node %s (max %d): %d shards", node.Info.Id, node.FreeEcSlot+count, count)
 	}
 
 	ecb.balance([]string{"cldata"})
@@ -433,14 +433,14 @@ func TestCommandEcBalanceIssue8793Topology(t *testing.T) {
 	shardCounts := make(map[string]int)
 	for _, node := range ecb.ecNodes {
 		count := 0
-		if diskInfo, found := node.info.DiskInfos[string(types.HardDriveType)]; found {
+		if diskInfo, found := node.Info.DiskInfos[string(types.HardDriveType)]; found {
 			for _, ecsi := range diskInfo.EcShardInfos {
 				count += erasure_coding.GetShardCount(ecsi)
 			}
 		}
-		shardCounts[node.info.Id] = count
+		shardCounts[node.Info.Id] = count
 		totalShards += count
-		totalCapacity += capacityByID[node.info.Id]
+		totalCapacity += capacityByID[node.Info.Id]
 	}
 	overallFullness := float64(totalShards) / float64(totalCapacity)
 
@@ -448,14 +448,14 @@ func TestCommandEcBalanceIssue8793Topology(t *testing.T) {
 	// would sit ~38 points above overall), but above integer-rounding skew.
 	const tolerance = 0.05
 	for _, node := range ecb.ecNodes {
-		count := shardCounts[node.info.Id]
-		capacity := capacityByID[node.info.Id]
+		count := shardCounts[node.Info.Id]
+		capacity := capacityByID[node.Info.Id]
 		fullness := float64(count) / float64(capacity)
 		t.Logf("AFTER  node %s: %d/%d shards (%.0f%% full, overall %.0f%%)",
-			node.info.Id, count, capacity, fullness*100, overallFullness*100)
+			node.Info.Id, count, capacity, fullness*100, overallFullness*100)
 		if diff := fullness - overallFullness; diff > tolerance || diff < -tolerance {
 			t.Errorf("node %s fullness %.1f%% deviates from overall %.1f%% by more than %.0f points",
-				node.info.Id, fullness*100, overallFullness*100, tolerance*100)
+				node.Info.Id, fullness*100, overallFullness*100, tolerance*100)
 		}
 	}
 }
@@ -484,15 +484,15 @@ func TestCommandEcBalanceVolumeIdsFilter(t *testing.T) {
 	}
 
 	// Volume 1 spreads out.
-	if count := ecb.ecNodes[0].localShardIdCount(1); count == 14 {
+	if count := ecb.ecNodes[0].LocalShardIdCount(1); count == 14 {
 		t.Errorf("volume 1 was not balanced: dn1 still holds all %d shards", count)
 	}
 
 	// Volume 2 keeps every shard where it was, duplicate included.
-	if count := ecb.ecNodes[1].localShardIdCount(2); count != 14 {
+	if count := ecb.ecNodes[1].LocalShardIdCount(2); count != 14 {
 		t.Errorf("dn2 holds %d shards of the unselected volume 2, want 14", count)
 	}
-	if count := ecb.ecNodes[2].localShardIdCount(2); count != 1 {
+	if count := ecb.ecNodes[2].LocalShardIdCount(2); count != 1 {
 		t.Errorf("dn3 holds %d shards of the unselected volume 2, want the duplicate to survive", count)
 	}
 }
@@ -513,7 +513,7 @@ func TestCommandEcBalanceVolumeIdsNotFound(t *testing.T) {
 		t.Fatalf("want an error naming volume 99, got %v", err)
 	}
 	// The valid id must not be balanced either: the plan is all-or-nothing.
-	if count := ecb.ecNodes[0].localShardIdCount(1); count != 14 {
+	if count := ecb.ecNodes[0].LocalShardIdCount(1); count != 14 {
 		t.Errorf("dn1 holds %d shards, want the rejected plan to leave volume 1 untouched", count)
 	}
 }

weed/ec/ec_decode.go

@@ -0,0 +1,376 @@
+package ec
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+	"strings"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+func DoEcDecode(env *Env, topoInfo *master_pb.TopologyInfo, collection string, vid needle.VolumeId, diskType types.DiskType, checkMinFreeSpace bool, diskUsageState *DecodeDiskUsageState) (err error) {
+
+	if !env.isLocked() {
+		return fmt.Errorf("lock is lost")
+	}
+
+	// find volume location
+	nodeToEcShardsInfo, dataShards := collectEcNodeShardsInfo(topoInfo, vid, diskType)
+
+	fmt.Printf("ec volume %d shard locations: %+v\n", vid, nodeToEcShardsInfo)
+
+	if len(nodeToEcShardsInfo) == 0 {
+		return fmt.Errorf("no EC shards found for volume %d (diskType %s)", vid, diskType.ReadableString())
+	}
+
+	var originalShardCounts map[pb.ServerAddress]int
+	if diskUsageState != nil {
+		originalShardCounts = make(map[pb.ServerAddress]int, len(nodeToEcShardsInfo))
+		for location, si := range nodeToEcShardsInfo {
+			originalShardCounts[location] = si.Count()
+		}
+	}
+
+	var eligibleTargets map[pb.ServerAddress]struct{}
+	if checkMinFreeSpace {
+		if diskUsageState == nil {
+			return fmt.Errorf("min free space checking requires disk usage state")
+		}
+		eligibleTargets = make(map[pb.ServerAddress]struct{})
+		for location := range nodeToEcShardsInfo {
+			if freeCount, found := diskUsageState.freeVolumeCount(location); found && freeCount > 0 {
+				eligibleTargets[location] = struct{}{}
+			}
+		}
+		if len(eligibleTargets) == 0 {
+			return fmt.Errorf("no eligible target datanodes with free volume slots for volume %d (diskType %s); use -checkMinFreeSpace=false to override", vid, diskType.ReadableString())
+		}
+	}
+
+	// collect ec shards to the server with most space
+	targetNodeLocation, err := collectEcShards(env, nodeToEcShardsInfo, collection, vid, eligibleTargets, dataShards)
+	if err != nil {
+		return fmt.Errorf("collectEcShards for volume %d: %v", vid, err)
+	}
+
+	// generate a normal volume
+	err = generateNormalVolume(env.GrpcDialOption, vid, collection, targetNodeLocation)
+	if err != nil {
+		// Special case: if the EC index has no live entries, decoding is a no-op.
+		// Just purge EC shards and return success without generating/mounting an empty volume.
+		if isEcDecodeEmptyVolumeErr(err) {
+			if err := unmountAndDeleteEcShards(env.GrpcDialOption, collection, nodeToEcShardsInfo, vid); err != nil {
+				return err
+			}
+			if diskUsageState != nil {
+				diskUsageState.applyDecode(targetNodeLocation, originalShardCounts, false)
+			}
+			return nil
+		}
+		return fmt.Errorf("generate normal volume %d on %s: %v", vid, targetNodeLocation, err)
+	}
+
+	// mount the decoded volume after server-side offline compaction succeeded
+	err = mountDecodedVolume(env.GrpcDialOption, targetNodeLocation, vid)
+	if err != nil {
+		return fmt.Errorf("mount decoded volume %d on %s: %v", vid, targetNodeLocation, err)
+	}
+
+	// Confirm the regenerated .dat is present and non-empty before destroying
+	// the shards. Without this gate, a silent failure in generate/mount could
+	// leave the cluster with neither shards nor volume.
+	if err := verifyDecodedVolumeBeforeDelete(env.GrpcDialOption, targetNodeLocation, vid); err != nil {
+		return fmt.Errorf("verify decoded volume %d on %s before deleting shards: %w", vid, targetNodeLocation, err)
+	}
+
+	// delete the previous ec shards
+	err = unmountAndDeleteEcShardsWithPrefix("deleteDecodedEcShards", env.GrpcDialOption, collection, nodeToEcShardsInfo, vid)
+	if err != nil {
+		return fmt.Errorf("delete ec shards for volume %d: %v", vid, err)
+	}
+	if diskUsageState != nil {
+		diskUsageState.applyDecode(targetNodeLocation, originalShardCounts, true)
+	}
+
+	return nil
+}
+
+func isEcDecodeEmptyVolumeErr(err error) bool {
+	st, ok := status.FromError(err)
+	if !ok {
+		return false
+	}
+	if st.Code() != codes.FailedPrecondition {
+		return false
+	}
+	// Keep this robust against wording tweaks while still being specific.
+	return strings.Contains(st.Message(), erasure_coding.EcNoLiveEntriesSubstring)
+}
+
+func unmountAndDeleteEcShards(grpcDialOption grpc.DialOption, collection string, nodeToShardsInfo map[pb.ServerAddress]*erasure_coding.ShardsInfo, vid needle.VolumeId) error {
+	return unmountAndDeleteEcShardsWithPrefix("unmountAndDeleteEcShards", grpcDialOption, collection, nodeToShardsInfo, vid)
+}
+
+func unmountAndDeleteEcShardsWithPrefix(prefix string, grpcDialOption grpc.DialOption, collection string, nodeToShardsInfo map[pb.ServerAddress]*erasure_coding.ShardsInfo, vid needle.VolumeId) error {
+	ewg := util.NewErrorWaitGroup(len(nodeToShardsInfo))
+
+	// unmount and delete ec shards in parallel (one goroutine per location)
+	for location, si := range nodeToShardsInfo {
+		location, si := location, si // capture loop variables for goroutine
+		ewg.Add(func() error {
+			fmt.Printf("unmount ec volume %d on %s has shards: %+v\n", vid, location, si.Ids())
+			if err := UnmountEcShards(grpcDialOption, vid, location, si.Ids()); err != nil {
+				return fmt.Errorf("%s unmount ec volume %d on %s: %w", prefix, vid, location, err)
+			}
+
+			fmt.Printf("delete ec volume %d on %s has shards: %+v\n", vid, location, si.Ids())
+			if err := SourceServerDeleteEcShards(grpcDialOption, collection, vid, location, si.Ids()); err != nil {
+				return fmt.Errorf("%s delete ec volume %d on %s: %w", prefix, vid, location, err)
+			}
+			return nil
+		})
+	}
+	return ewg.Wait()
+}
+
+func verifyDecodedVolumeBeforeDelete(grpcDialOption grpc.DialOption, target pb.ServerAddress, vid needle.VolumeId) error {
+	var resp *volume_server_pb.ReadVolumeFileStatusResponse
+	if err := operation.WithVolumeServerClient(false, target, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+		r, e := client.ReadVolumeFileStatus(context.Background(), &volume_server_pb.ReadVolumeFileStatusRequest{
+			VolumeId: uint32(vid),
+		})
+		if e != nil {
+			return e
+		}
+		resp = r
+		return nil
+	}); err != nil {
+		return fmt.Errorf("read volume file status: %w", err)
+	}
+	if resp.DatFileSize == 0 {
+		return fmt.Errorf("decoded .dat is 0 bytes")
+	}
+	if resp.IdxFileSize == 0 {
+		return fmt.Errorf("decoded .idx is 0 bytes")
+	}
+	glog.V(0).Infof("ec decode verification ok for volume %d on %s: dat=%d idx=%d", vid, target, resp.DatFileSize, resp.IdxFileSize)
+	return nil
+}
+
+func mountDecodedVolume(grpcDialOption grpc.DialOption, targetNodeLocation pb.ServerAddress, vid needle.VolumeId) error {
+	return operation.WithVolumeServerClient(false, targetNodeLocation, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+		_, mountErr := volumeServerClient.VolumeMount(context.Background(), &volume_server_pb.VolumeMountRequest{
+			VolumeId: uint32(vid),
+		})
+		return mountErr
+	})
+}
+
+func generateNormalVolume(grpcDialOption grpc.DialOption, vid needle.VolumeId, collection string, sourceVolumeServer pb.ServerAddress) error {
+	fmt.Printf("generateNormalVolume from ec volume %d on %s\n", vid, sourceVolumeServer)
+
+	err := operation.WithVolumeServerClient(false, sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+		_, genErr := volumeServerClient.VolumeEcShardsToVolume(context.Background(), &volume_server_pb.VolumeEcShardsToVolumeRequest{
+			VolumeId:   uint32(vid),
+			Collection: collection,
+		})
+		return genErr
+	})
+
+	return err
+
+}
+
+func collectEcShards(env *Env, nodeToShardsInfo map[pb.ServerAddress]*erasure_coding.ShardsInfo, collection string, vid needle.VolumeId, eligibleTargets map[pb.ServerAddress]struct{}, dataShards int) (targetNodeLocation pb.ServerAddress, err error) {
+
+	maxShardCount := -1
+	existingShardsInfo := erasure_coding.NewShardsInfo()
+	for loc, si := range nodeToShardsInfo {
+		if eligibleTargets != nil {
+			if _, ok := eligibleTargets[loc]; !ok {
+				continue
+			}
+		}
+		toBeCopiedShardCount := si.MinusParityShards(dataShards).Count()
+		if toBeCopiedShardCount > maxShardCount {
+			maxShardCount = toBeCopiedShardCount
+			targetNodeLocation = loc
+			existingShardsInfo = si
+		}
+	}
+	if targetNodeLocation == "" {
+		return "", fmt.Errorf("no eligible target datanodes available to decode volume %d", vid)
+	}
+
+	fmt.Printf("collectEcShards: ec volume %d collect shards to %s from: %+v\n", vid, targetNodeLocation, nodeToShardsInfo)
+
+	copiedShardsInfo := erasure_coding.NewShardsInfo()
+	for loc, si := range nodeToShardsInfo {
+		if loc == targetNodeLocation {
+			continue
+		}
+
+		needToCopyShardsInfo := si.Minus(existingShardsInfo).MinusParityShards(dataShards)
+
+		err = operation.WithVolumeServerClient(false, targetNodeLocation, env.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+
+			// Always collect .ecj from every shard location. Each server's .ecj
+			// only contains deletions for needles whose data resides in shards
+			// held by that server. Without merging all .ecj files, deletions
+			// recorded on other servers would be lost during decode.
+			if needToCopyShardsInfo.Count() > 0 {
+				fmt.Printf("copy %d.%v %s => %s\n", vid, needToCopyShardsInfo.Ids(), loc, targetNodeLocation)
+			} else {
+				fmt.Printf("collect ecj %d %s => %s\n", vid, loc, targetNodeLocation)
+			}
+
+			_, copyErr := volumeServerClient.VolumeEcShardsCopy(context.Background(), &volume_server_pb.VolumeEcShardsCopyRequest{
+				VolumeId:       uint32(vid),
+				Collection:     collection,
+				ShardIds:       needToCopyShardsInfo.IdsUint32(),
+				CopyEcxFile:    false,
+				CopyEcjFile:    true,
+				CopyVifFile:    needToCopyShardsInfo.Count() > 0,
+				SourceDataNode: string(loc),
+			})
+			if copyErr != nil {
+				return fmt.Errorf("copy %d.%v %s => %s: %v", vid, needToCopyShardsInfo.Ids(), loc, targetNodeLocation, copyErr)
+			}
+
+			if needToCopyShardsInfo.Count() > 0 {
+				fmt.Printf("mount %d.%v on %s\n", vid, needToCopyShardsInfo.Ids(), targetNodeLocation)
+				_, mountErr := volumeServerClient.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{
+					VolumeId:   uint32(vid),
+					Collection: collection,
+					ShardIds:   needToCopyShardsInfo.IdsUint32(),
+				})
+				if mountErr != nil {
+					return fmt.Errorf("mount %d.%v on %s: %v", vid, needToCopyShardsInfo.Ids(), targetNodeLocation, mountErr)
+				}
+			}
+
+			return nil
+		})
+
+		if err != nil {
+			break
+		}
+
+		copiedShardsInfo.Add(needToCopyShardsInfo)
+	}
+
+	nodeToShardsInfo[targetNodeLocation] = existingShardsInfo.Plus(copiedShardsInfo)
+
+	return targetNodeLocation, err
+}
+func CollectEcShardIds(topoInfo *master_pb.TopologyInfo, collectionRegex *regexp.Regexp, diskType types.DiskType) (vids []needle.VolumeId) {
+	vidMap := make(map[uint32]bool)
+	EachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
+		if diskInfo, found := dn.DiskInfos[string(diskType)]; found {
+			for _, v := range diskInfo.EcShardInfos {
+				if collectionRegex.MatchString(v.Collection) {
+					vidMap[v.Id] = true
+				}
+			}
+		}
+	})
+
+	for vid := range vidMap {
+		vids = append(vids, needle.VolumeId(vid))
+	}
+
+	return
+}
+
+func collectEcNodeShardsInfo(topoInfo *master_pb.TopologyInfo, vid needle.VolumeId, diskType types.DiskType) (map[pb.ServerAddress]*erasure_coding.ShardsInfo, int) {
+	res := make(map[pb.ServerAddress]*erasure_coding.ShardsInfo)
+	EachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
+		if diskInfo, found := dn.DiskInfos[string(diskType)]; found {
+			// A node may report several EcShardInfos for one volume — one per
+			// physical disk holding shards of it (multi-disk nodes). Union them
+			// rather than overwriting, or only the last disk's shards survive and
+			// the node looks like it is missing shards it actually has.
+			for _, v := range diskInfo.EcShardInfos {
+				if v.Id == uint32(vid) {
+					addr := pb.NewServerAddressFromDataNode(dn)
+					si := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(v)
+					if existing, ok := res[addr]; ok {
+						existing.Add(si)
+					} else {
+						res[addr] = si
+					}
+				}
+			}
+		}
+	})
+
+	// OSS is always 10+4; the per-volume ratio override lives in the enterprise build.
+	return res, erasure_coding.DataShardsCount
+}
+
+type DecodeDiskUsageState struct {
+	byNode map[pb.ServerAddress]*decodeDiskUsageCounts
+}
+
+type decodeDiskUsageCounts struct {
+	maxVolumeCount    int64
+	volumeCount       int64
+	remoteVolumeCount int64
+	ecShardCount      int64
+}
+
+func NewDecodeDiskUsageState(topoInfo *master_pb.TopologyInfo, diskType types.DiskType) *DecodeDiskUsageState {
+	state := &DecodeDiskUsageState{byNode: make(map[pb.ServerAddress]*decodeDiskUsageCounts)}
+	EachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
+		if diskInfo, found := dn.DiskInfos[string(diskType)]; found {
+			state.byNode[pb.NewServerAddressFromDataNode(dn)] = &decodeDiskUsageCounts{
+				maxVolumeCount:    diskInfo.MaxVolumeCount,
+				volumeCount:       diskInfo.VolumeCount,
+				remoteVolumeCount: diskInfo.RemoteVolumeCount,
+				ecShardCount:      int64(CountShards(diskInfo.EcShardInfos)),
+			}
+		}
+	})
+	return state
+}
+
+func (state *DecodeDiskUsageState) freeVolumeCount(location pb.ServerAddress) (int64, bool) {
+	if state == nil {
+		return 0, false
+	}
+	usage, found := state.byNode[location]
+	if !found {
+		return 0, false
+	}
+	free := usage.maxVolumeCount - (usage.volumeCount - usage.remoteVolumeCount)
+	free -= (usage.ecShardCount + int64(erasure_coding.DataShardsCount) - 1) / int64(erasure_coding.DataShardsCount)
+	return free, true
+}
+
+func (state *DecodeDiskUsageState) applyDecode(targetNodeLocation pb.ServerAddress, shardCounts map[pb.ServerAddress]int, createdVolume bool) {
+	if state == nil {
+		return
+	}
+	for location, shardCount := range shardCounts {
+		if usage, found := state.byNode[location]; found {
+			usage.ecShardCount -= int64(shardCount)
+		}
+	}
+	if createdVolume {
+		if usage, found := state.byNode[targetNodeLocation]; found {
+			usage.volumeCount++
+		}
+	}
+}

weed/ec/ec_destructive_guards_test.go

@@ -1,4 +1,4 @@
-package shell
+package ec
 
 import (
 	"bytes"
@@ -52,7 +52,7 @@ func TestAssertEncodableRegularVolumes(t *testing.T) {
 	}
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
-			err := assertEncodableRegularVolumes(topoWith(tc.regular, tc.ec), tc.vids)
+			err := AssertEncodableRegularVolumes(topoWith(tc.regular, tc.ec), tc.vids)
 			if tc.wantErrSubstr == "" {
 				if err != nil {
 					t.Fatalf("want nil error, got %v", err)
@@ -68,7 +68,7 @@ func TestAssertEncodableRegularVolumes(t *testing.T) {
 
 func newDryRunRebuilder(log *bytes.Buffer, nodes ...*EcNode) *ecRebuilder {
 	return &ecRebuilder{
-		commandEnv:   &CommandEnv{env: make(map[string]string), noLock: true},
+		env:          &Env{},
 		ecNodes:      nodes,
 		writer:       log,
 		applyChanges: false,
@@ -114,7 +114,7 @@ func TestPrepareDataToRecover_UnionsLocalShardsAcrossDisks(t *testing.T) {
 	var log bytes.Buffer
 	rebuilder := newEcNode("dc1", "rack1", "rebuilder", 100).
 		addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1}).
-		addEcVolumeShards(needle.VolumeId(1), "c1", []erasure_coding.ShardId{5}, types.SsdType)
+		AddEcVolumeShards(needle.VolumeId(1), "c1", []erasure_coding.ShardId{5}, types.SsdType)
 	remote := newEcNode("dc1", "rack1", "remote", 100)
 	erb := newDryRunRebuilder(&log, rebuilder)
 
@@ -151,7 +151,7 @@ func TestCountLocalShards_UnionsAcrossDisks(t *testing.T) {
 	var log bytes.Buffer
 	rebuilder := newEcNode("dc1", "rack1", "rebuilder", 100).
 		addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1}).
-		addEcVolumeShards(needle.VolumeId(1), "c1", []erasure_coding.ShardId{5}, types.SsdType)
+		AddEcVolumeShards(needle.VolumeId(1), "c1", []erasure_coding.ShardId{5}, types.SsdType)
 	erb := newDryRunRebuilder(&log, rebuilder)
 
 	if got := erb.countLocalShards(rebuilder, "c1", needle.VolumeId(1)); got != 3 {
@@ -201,12 +201,8 @@ func TestPrepareDataToRecover_ApplyCopyFailureDoesNotCountAsRecoverable(t *testi
 	remote := newEcNode("dc1", "rack1", "127.0.0.1:2", 100).
 		addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{9})
 	erb := &ecRebuilder{
-		commandEnv: &CommandEnv{
-			env:    make(map[string]string),
-			noLock: true,
-			option: &ShellOptions{
-				GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
-			},
+		env: &Env{
+			GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
 		},
 		ecNodes:      []*EcNode{rebuilder, remote},
 		writer:       &log,

weed/ec/ec_encode.go

@@ -0,0 +1,931 @@
+package ec
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"regexp"
+	"slices"
+	"sort"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/operation/volume_move"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/storage/volume_replica"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+	"google.golang.org/grpc"
+)
+
+// markVolumeReplicaWritable marks one replica writable/readonly with a progress
+// line, delegating to the canonical volume_move helper.
+func markVolumeReplicaWritable(ctx context.Context, grpcDialOption grpc.DialOption, volumeId needle.VolumeId, location wdclient.Location, writable, persist bool) error {
+	if writable {
+		fmt.Printf("markVolumeWritable %d on %s ...\n", volumeId, location.Url)
+	} else {
+		fmt.Printf("markVolumeReadonly %d on %s persist=%v ...\n", volumeId, location.Url, persist)
+	}
+	return volume_move.NewMover(grpcDialOption).MarkVolumeWritable(ctx, volumeId, location.ServerAddress(), writable, persist)
+}
+
+// deleteVolume removes the volume from sourceVolumeServer via the canonical
+// volume_move helper.
+func deleteVolume(ctx context.Context, grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sourceVolumeServer pb.ServerAddress, onlyEmpty bool, keepRemoteData bool) (err error) {
+	return volume_move.NewMover(grpcDialOption).DeleteVolume(ctx, volumeId, sourceVolumeServer, onlyEmpty, keepRemoteData)
+}
+
+func ChunkVolumeIds(volumeIds []needle.VolumeId, batchSize int) [][]needle.VolumeId {
+	if batchSize <= 0 || len(volumeIds) == 0 {
+		return [][]needle.VolumeId{volumeIds}
+	}
+	var batches [][]needle.VolumeId
+	for start := 0; start < len(volumeIds); start += batchSize {
+		end := start + batchSize
+		if end > len(volumeIds) {
+			end = len(volumeIds)
+		}
+		batches = append(batches, volumeIds[start:end])
+	}
+	return batches
+}
+
+func ProcessEcEncodeBatch(env *Env, writer io.Writer, volumeIds []needle.VolumeId, rp *super_block.ReplicaPlacement, diskType types.DiskType, maxParallelization int, applyBalancing bool, collectionForMessage string) (err error) {
+	topologyInfo, _, err := env.FetchTopology(0)
+	if err != nil {
+		return err
+	}
+
+	// Refuse to encode a volume that is already EC (present only as shards):
+	// an EC volume has no .dat, so re-encoding it would tear down its only
+	// copy before failing. A regular volume (with a .dat) passes. This closes
+	// the operator-rerun / script-retry path; a worker racing the snapshot is
+	// handled by encode fencing, not here.
+	if err := AssertEncodableRegularVolumes(topologyInfo, volumeIds); err != nil {
+		return err
+	}
+
+	volumeIdToCollection := CollectVolumeIdToCollection(topologyInfo, volumeIds)
+	balanceCollections := CollectCollectionsForVolumeIds(topologyInfo, volumeIds)
+
+	fmt.Printf("Collecting volume locations for %d volumes before EC encoding...\n", len(volumeIds))
+	volumeLocationsMap, err := volumeLocations(env, volumeIds)
+	if err != nil {
+		return fmt.Errorf("failed to collect volume locations before EC encoding: %w", err)
+	}
+
+	if err := checkEcEncodeCapacity(topologyInfo, len(volumeIds), diskType, collectionForMessage); err != nil {
+		return err
+	}
+
+	// From here doEcEncode marks the volumes readonly and generates EC shards.
+	// If any step before the originals are deleted fails, roll the encode back:
+	// tear down the shards produced this run and restore the sources to writable,
+	// so a failed (and possibly abandoned) ec.encode does not strand volumes
+	// readonly or leave orphan EC shards behind. Once the shards are verified
+	// recoverable we are committed to the EC copy and must not roll back.
+	committed := false
+	defer func() {
+		if err != nil && !committed {
+			rollbackFailedEcEncode(env, writer, volumeIds, volumeIdToCollection, volumeLocationsMap, maxParallelization)
+		}
+	}()
+
+	skippedNodes, err := doEcEncode(env, writer, volumeIdToCollection, volumeIds, maxParallelization, topologyInfo)
+	if err != nil {
+		return fmt.Errorf("ec encode for volumes %v: %w", volumeIds, err)
+	}
+	// Mounting the new shards notifies the master asynchronously, and EcBalance
+	// plans from a fresh topology snapshot: one taken before the mounts land
+	// shows no shards for these volumes, so the balance plans no moves and
+	// silently leaves every shard on the generation host.
+	if err := waitForEcShardsToRegister(env, volumeIds); err != nil {
+		return fmt.Errorf("wait for ec shards to register with the master: %w", err)
+	}
+	// EcBalance works at collection scope. In batch mode this intentionally
+	// rebalances each collection after every batch so source volumes can be
+	// safely verified and deleted without waiting for all batches to finish.
+	// skippedNodes are excluded so a recovered node's stale orphan is never
+	// paired with a new-generation shard.
+	if err := EcBalance(env, balanceCollections, "", rp, diskType, maxParallelization, 0, applyBalancing, skippedNodes, nil); err != nil {
+		return fmt.Errorf("re-balance ec shards for collection(s) %v: %w", balanceCollections, err)
+	}
+	if err := verifyEcShardsBeforeDelete(env, volumeIds, diskType, applyBalancing); err != nil {
+		return fmt.Errorf("verify EC shards before deleting originals: %w", err)
+	}
+	// Past verify the EC copy is recoverable; a delete failure below must not
+	// tear the shards down.
+	committed = true
+	fmt.Printf("Deleting original volumes after EC encoding...\n")
+	if err := doDeleteVolumesWithLocations(env, volumeIds, volumeLocationsMap, maxParallelization); err != nil {
+		return fmt.Errorf("delete original volumes after EC encoding: %w", err)
+	}
+	fmt.Printf("Successfully completed EC encoding for %d volumes\n", len(volumeIds))
+	return nil
+}
+
+// rollbackFailedEcEncode is a best-effort cleanup for an ec.encode that failed
+// after marking volumes readonly / generating shards but before the originals
+// were deleted. It tears down the EC shards this run produced (so they do not
+// survive as orphans until the next encode) and restores the source volumes to
+// writable (so a failed and possibly abandoned encode does not strand them
+// readonly). Errors are logged, not returned — we are already on the failure
+// path. Both operations are idempotent: clearPreexistingEcShards is a no-op when
+// no shards were generated, and marking an already-writable volume is a no-op,
+// so it is safe even for a failure before the volumes were marked readonly.
+func rollbackFailedEcEncode(env *Env, writer io.Writer, volumeIds []needle.VolumeId, volumeIdToCollection map[needle.VolumeId]string, volumeLocationsMap map[needle.VolumeId][]wdclient.Location, maxParallelization int) {
+	fmt.Fprintf(writer, "rolling back failed EC encode for volumes %v...\n", volumeIds)
+
+	// Tear down any EC shards this run produced. A fresh topology snapshot finds
+	// them wherever generate/balance left them; the teardown is blanket.
+	if topologyInfo, _, err := env.FetchTopology(0); err != nil {
+		fmt.Fprintf(writer, "rollback: collect topology to clear ec shards: %v\n", err)
+	} else if _, err := clearPreexistingEcShards(env, topologyInfo, volumeIds, volumeIdToCollection, maxParallelization); err != nil {
+		fmt.Fprintf(writer, "rollback: clear ec shards: %v\n", err)
+	}
+
+	// Restore the source volumes to writable. doEcEncode re-reads the locations
+	// and marks every replica of that later snapshot readonly, so re-read here
+	// too: a replica added or moved between the batch's initial snapshot
+	// (volumeLocationsMap) and doEcEncode's readonly-marking would otherwise be
+	// left readonly. Fall back to the initial snapshot if the re-read fails.
+	locations := volumeLocationsMap
+	if fresh, err := volumeLocations(env, volumeIds); err != nil {
+		fmt.Fprintf(writer, "rollback: re-read volume locations (using pre-encode snapshot): %v\n", err)
+	} else {
+		locations = fresh
+	}
+	ewg := util.NewErrorWaitGroup(maxParallelization)
+	for _, vid := range volumeIds {
+		for _, l := range locations[vid] {
+			ewg.Add(func() error {
+				if err := markVolumeReplicaWritable(context.Background(), env.GrpcDialOption, vid, l, true, false); err != nil {
+					return fmt.Errorf("restore volume %d writable on %s: %w", vid, l.Url, err)
+				}
+				return nil
+			})
+		}
+	}
+	if err := ewg.Wait(); err != nil {
+		fmt.Fprintf(writer, "rollback: %v\n", err)
+	}
+}
+
+func checkEcEncodeCapacity(topologyInfo *master_pb.TopologyInfo, volumeCount int, diskType types.DiskType, collectionForMessage string) error {
+	// Pre-flight check: verify the target disk type has capacity for EC shards.
+	// This prevents encoding shards only to fail during rebalance. Reuse the
+	// caller's topology snapshot instead of issuing another VolumeList to the
+	// master per batch.
+	_, totalFreeEcSlots := CollectEcVolumeServersByDc(topologyInfo, "", diskType)
+
+	// Each volume needs TotalShardsCount (14) shards distributed.
+	requiredSlots := volumeCount * erasure_coding.TotalShardsCount
+	if totalFreeEcSlots < 1 {
+		if diskType != types.HardDriveType {
+			tryDiskTypeMessage := "Try passing -diskType=hdd, or omit -diskType to use the default (hdd)"
+			if collectionForMessage != "" {
+				tryDiskTypeMessage = fmt.Sprintf("Try:\n  ec.encode -collection=%s -diskType=hdd\nOr omit -diskType to use the default (hdd)", collectionForMessage)
+			}
+			return fmt.Errorf("no free ec shard slots on disk type '%s'. The target disk type has no capacity.\n"+
+				"Your volumes are likely on a different disk type. %s", diskType, tryDiskTypeMessage)
+		}
+		return fmt.Errorf("no free ec shard slots. only %d left on disk type '%s'", totalFreeEcSlots, diskType)
+	}
+
+	if totalFreeEcSlots < requiredSlots {
+		fmt.Printf("Warning: limited EC shard capacity. Need %d slots for %d volumes, but only %d slots available on disk type '%s'.\n",
+			requiredSlots, volumeCount, totalFreeEcSlots, diskType)
+		fmt.Printf("Rebalancing may not achieve optimal distribution.\n")
+	}
+	return nil
+}
+
+func volumeLocations(env *Env, volumeIds []needle.VolumeId) (map[needle.VolumeId][]wdclient.Location, error) {
+	res := map[needle.VolumeId][]wdclient.Location{}
+	for _, vid := range volumeIds {
+		ls, ok := env.GetVolumeLocations(uint32(vid))
+		if !ok {
+			return nil, fmt.Errorf("volume %d not found", vid)
+		}
+		res[vid] = ls
+	}
+
+	return res, nil
+}
+
+func doEcEncode(env *Env, writer io.Writer, volumeIdToCollection map[needle.VolumeId]string, volumeIds []needle.VolumeId, maxParallelization int, topologyInfo *master_pb.TopologyInfo) (skippedNodes map[pb.ServerAddress]struct{}, err error) {
+	if !env.isLocked() {
+		return nil, fmt.Errorf("lock is lost")
+	}
+	locations, err := volumeLocations(env, volumeIds)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get volume locations for EC encoding: %w", err)
+	}
+
+	// Clear EC shards left by a previous failed/partial encode so a retry
+	// starts clean and never mixes two encode runs. A node skipped here as
+	// unreachable is excluded from the later balance: it may still hold a stale
+	// orphan that, paired with a new-generation shard from a balance copy, would
+	// mix generations on that node.
+	skippedNodes, err = clearPreexistingEcShards(env, topologyInfo, volumeIds, volumeIdToCollection, maxParallelization)
+	if err != nil {
+		return nil, fmt.Errorf("clear pre-existing ec shards before encoding: %w", err)
+	}
+
+	// Build a map of (volumeId, serverAddress) -> freeVolumeCount.
+	// Key by dn.Address so it matches wdclient.Location.Url. In deployments
+	// where dn.Id is a short name (e.g. Kubernetes StatefulSet pod name)
+	// while dn.Address is a FQDN:port, keying by dn.Id would never match the
+	// location Url during the health-check lookup below.
+	freeVolumeCountMap := make(map[string]int) // key: volumeId-serverAddress
+	EachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
+		addr := dn.Address
+		if addr == "" {
+			addr = dn.Id // older nodes use ip:port as id
+		}
+		for _, diskInfo := range dn.DiskInfos {
+			for _, v := range diskInfo.VolumeInfos {
+				key := fmt.Sprintf("%d-%s", v.Id, addr)
+				freeVolumeCountMap[key] = int(diskInfo.FreeVolumeCount)
+			}
+		}
+	})
+
+	// Filter replicas by free capacity BEFORE marking volumes readonly so that
+	// a failed health check does not strand volumes in readonly state.
+	filteredLocations := make(map[needle.VolumeId][]wdclient.Location)
+	for _, vid := range volumeIds {
+		var filteredLocs []wdclient.Location
+		for _, l := range locations[vid] {
+			key := fmt.Sprintf("%d-%s", vid, l.Url)
+			if freeCount, found := freeVolumeCountMap[key]; found && freeCount >= 2 {
+				filteredLocs = append(filteredLocs, l)
+			}
+		}
+		if len(filteredLocs) == 0 {
+			return nil, fmt.Errorf("no healthy replicas (FreeVolumeCount >= 2) found for volume %d to use as source for EC encoding", vid)
+		}
+		filteredLocations[vid] = filteredLocs
+	}
+
+	// mark volumes as readonly
+	ewg := util.NewErrorWaitGroup(maxParallelization)
+	for _, vid := range volumeIds {
+		for _, l := range locations[vid] {
+			ewg.Add(func() error {
+				if err := markVolumeReplicaWritable(context.Background(), env.GrpcDialOption, vid, l, false, false); err != nil {
+					return fmt.Errorf("mark volume %d as readonly on %s: %v", vid, l.Url, err)
+				}
+				return nil
+			})
+		}
+	}
+	if err := ewg.Wait(); err != nil {
+		return nil, err
+	}
+
+	// Sync replicas and select the best one for each volume (with highest file count)
+	// This addresses data inconsistency risk in multi-replica volumes (issue #7797)
+	// by syncing missing entries between replicas before encoding
+	bestReplicas := make(map[needle.VolumeId]wdclient.Location)
+	for _, vid := range volumeIds {
+		collection := volumeIdToCollection[vid]
+
+		// Sync missing entries between replicas, then select the best one
+		bestLoc, selectErr := volume_replica.SyncAndSelectBestReplica(env.GrpcDialOption, vid, collection, filteredLocations[vid], "", writer)
+		if selectErr != nil {
+			return nil, fmt.Errorf("failed to sync and select replica for volume %d: %v", vid, selectErr)
+		}
+		bestReplicas[vid] = bestLoc
+	}
+
+	// Re-attempt the orphan sweep on the nodes skipped as unreachable, now that
+	// any node that recovered during readonly-marking and replica sync answers
+	// again. A node whose teardown now succeeds is clean (and the generation host
+	// re-wipes its own disks regardless), so it leaves the skipped set and can be
+	// a balance source/target — otherwise its shards would never distribute off
+	// it. A node that is still down stays skipped and excluded, preserving the
+	// leniency for a genuinely-down node; such a node also cannot be the
+	// generation host below, since VolumeEcShardsGenerate would fail to read .dat.
+	if err := resweepSkippedNodes(env, skippedNodes, volumeIds, volumeIdToCollection, maxParallelization); err != nil {
+		return nil, err
+	}
+
+	// A selected generation host still in skippedNodes after the re-sweep was
+	// transport-down when we tried to clean it, so its stale orphans were never
+	// removed and EcBalance excludes it as both source and target. If it recovers
+	// just in time for generation, all shards land on a node we can neither clean
+	// nor balance off — a single point of failure that union-only verification
+	// still accepts, after which the originals are deleted. Abort instead.
+	for _, vid := range volumeIds {
+		genHost := bestReplicas[vid].ServerAddress()
+		if _, stillSkipped := skippedNodes[genHost]; stillSkipped {
+			return nil, fmt.Errorf("generate ec shards for volume %d aborted: selected source %s is still skipped after the orphan re-sweep", vid, genHost)
+		}
+	}
+
+	// generate ec shards using the best replica for each volume
+	ewg.Reset()
+	for _, vid := range volumeIds {
+		target := bestReplicas[vid]
+		collection := volumeIdToCollection[vid]
+		ewg.Add(func() error {
+			if err := generateEcShards(env.GrpcDialOption, vid, collection, target.ServerAddress()); err != nil {
+				return fmt.Errorf("generate ec shards for volume %d on %s: %v", vid, target.Url, err)
+			}
+			return nil
+		})
+	}
+	if err := ewg.Wait(); err != nil {
+		return nil, err
+	}
+
+	// mount all ec shards for the converted volume
+	shardIds := erasure_coding.AllShardIds()
+
+	ewg.Reset()
+	for _, vid := range volumeIds {
+		target := bestReplicas[vid]
+		collection := volumeIdToCollection[vid]
+		ewg.Add(func() error {
+			if err := MountEcShards(env.GrpcDialOption, collection, vid, target.ServerAddress(), shardIds); err != nil {
+				return fmt.Errorf("mount ec shards for volume %d on %s: %v", vid, target.Url, err)
+			}
+			return nil
+		})
+	}
+	if err := ewg.Wait(); err != nil {
+		return nil, err
+	}
+
+	return skippedNodes, nil
+}
+
+// clearPreexistingEcShards removes EC shards and index files left over from a
+// previous (failed or partial) encode of the given volume ids, on every node
+// that still reports them, so a fresh encode regenerates from a clean slate.
+// Scans all disk types. The normal .dat/.idx — the source of truth for this
+// encode — is untouched; only orphaned EC artifacts are deleted.
+//
+// Returns the set of nodes skipped as unreachable. A skipped node may still hold
+// an un-deleted orphan from a prior run; if it recovers it must be kept out of
+// this encode's shard distribution, or the balance could install the new
+// generation alongside the stale orphan and mix generations on one node.
+func clearPreexistingEcShards(env *Env, topologyInfo *master_pb.TopologyInfo, volumeIds []needle.VolumeId, volumeIdToCollection map[needle.VolumeId]string, maxParallelization int) (skipped map[pb.ServerAddress]struct{}, err error) {
+	wanted := make(map[uint32]bool, len(volumeIds))
+	for _, vid := range volumeIds {
+		wanted[uint32(vid)] = true
+	}
+
+	// Note which (node, vid) pairs the topology already reports EC shards for:
+	// those are mounted leftovers and cleaning them is required (fatal on
+	// error). Every other (node, vid) is swept best-effort to catch UNMOUNTED
+	// orphans left by a failed copy — invisible to the heartbeat, so absent
+	// here. A node that is down or holds nothing is a harmless no-op; a node
+	// unreachable now also cannot receive this encode's new generation, so a
+	// surviving orphan there keeps its old identity and the read guard rejects
+	// it. Always delete the full shard-id range so a wider custom ratio's
+	// leftovers are covered too.
+	reportedKey := func(addr pb.ServerAddress, vid uint32) string {
+		return string(addr) + "\x00" + strconv.FormatUint(uint64(vid), 10)
+	}
+	reported := make(map[string]struct{})
+	var nodes []pb.ServerAddress
+	EachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
+		addr := pb.NewServerAddressFromDataNode(dn)
+		nodes = append(nodes, addr)
+		for _, diskInfo := range dn.DiskInfos {
+			for _, ecInfo := range diskInfo.EcShardInfos {
+				if wanted[ecInfo.Id] {
+					reported[reportedKey(addr, ecInfo.Id)] = struct{}{}
+				}
+			}
+		}
+	})
+
+	allShardIds := make([]erasure_coding.ShardId, erasure_coding.MaxShardCount)
+	for i := range allShardIds {
+		allShardIds[i] = erasure_coding.ShardId(i)
+	}
+
+	if len(reported) > 0 {
+		fmt.Printf("clearing stale EC shards reported for %d (node,volume) pair(s) before regenerating...\n", len(reported))
+	}
+	// Nodes skipped as unreachable, accumulated across the concurrent sweep tasks.
+	skipped = make(map[pb.ServerAddress]struct{})
+	var skippedMu sync.Mutex
+	ewg := util.NewErrorWaitGroup(maxParallelization)
+	for _, addr := range nodes {
+		for _, vid := range volumeIds {
+			fatal := false
+			if _, ok := reported[reportedKey(addr, uint32(vid))]; ok {
+				fatal = true
+			}
+			collection := volumeIdToCollection[vid]
+			ewg.Add(func() error {
+				if err := UnmountAndDeleteEcShardsQuiet(env.GrpcDialOption, collection, vid, addr, allShardIds); err != nil {
+					// Surface a reachable node whose delete genuinely failed (its orphan would
+					// be re-stamped by a later copy installing the new .vif). A missing
+					// full_teardown ack from a reachable pre-upgrade node is fatal too: it may
+					// still hold an orphan a later copy would re-stamp into the new generation.
+					// Stay best-effort only for a node that is truly unreachable: codes.Unavailable
+					// alone is ambiguous — a genuinely-down node and a reachable Rust volume
+					// server in maintenance mode both return it (a Go server returns Unknown for
+					// maintenance, already fatal above). Confirm with a non-maintenance-gated Ping
+					// before skipping; skip only when the Ping itself transport-failed (NodeDown).
+					// A reachable maintenance node (nodeUp) CAN receive this generation, and an
+					// inconclusive Ping (nodeLivenessUnknown, e.g. a pre-Ping server returning
+					// Unimplemented — which means the node is up) does not prove the node is down,
+					// so both stay fatal rather than silently leaving a stale EC generation.
+					if fatal || errors.Is(err, ErrFullTeardownNotAcked) || !IsNodeUnreachable(err) ||
+						ClassifyNodeLiveness(PingVolumeServer(env.GrpcDialOption, addr)) != NodeDown {
+						return fmt.Errorf("clear stale ec shards for volume %d on %s: %w", vid, addr, err)
+					}
+					glog.V(1).Infof("orphan sweep: volume %d on %s skipped (unreachable): %v", vid, addr, err)
+					skippedMu.Lock()
+					skipped[addr] = struct{}{}
+					skippedMu.Unlock()
+				}
+				return nil
+			})
+		}
+	}
+	if err := ewg.Wait(); err != nil {
+		return nil, err
+	}
+	return skipped, nil
+}
+
+// resweepSkippedNodes re-attempts the orphan teardown on the nodes that the
+// initial sweep skipped as unreachable, just before shard generation. A node
+// that recovered in the meantime — and is therefore eligible to host this
+// encode's generation — has its teardown retried; if it now fully succeeds it is
+// removed from skipped so the rebalance can use it as a source and move its
+// shards off, instead of stranding all shards on the single generation host and
+// collapsing fault tolerance. A node still transport-down stays skipped (the
+// same leniency the initial sweep grants), and a node that came back reachable
+// but whose delete genuinely failed is fatal, exactly as in the initial sweep,
+// so a stale generation is never silently left behind. Mutates skipped in place.
+func resweepSkippedNodes(env *Env, skipped map[pb.ServerAddress]struct{}, volumeIds []needle.VolumeId, volumeIdToCollection map[needle.VolumeId]string, maxParallelization int) error {
+	if len(skipped) == 0 {
+		return nil
+	}
+
+	allShardIds := make([]erasure_coding.ShardId, erasure_coding.MaxShardCount)
+	for i := range allShardIds {
+		allShardIds[i] = erasure_coding.ShardId(i)
+	}
+
+	addrs := make([]pb.ServerAddress, 0, len(skipped))
+	for addr := range skipped {
+		addrs = append(addrs, addr)
+	}
+
+	fmt.Printf("re-checking %d node(s) skipped by the orphan sweep before generating shards...\n", len(addrs))
+
+	// A node still down on every retried vid stays skipped; one that fully
+	// succeeds is un-skipped. Track per-node whether any retry still failed
+	// (down) so a node whose state is mixed across vids never gets un-skipped.
+	stillDown := make(map[pb.ServerAddress]struct{})
+	var mu sync.Mutex
+	ewg := util.NewErrorWaitGroup(maxParallelization)
+	for _, addr := range addrs {
+		for _, vid := range volumeIds {
+			collection := volumeIdToCollection[vid]
+			ewg.Add(func() error {
+				if err := UnmountAndDeleteEcShardsQuiet(env.GrpcDialOption, collection, vid, addr, allShardIds); err != nil {
+					// Same decision as the initial sweep: a reachable node whose delete
+					// genuinely failed (or did not ack a full teardown, or whose liveness is
+					// inconclusive) is fatal, since it could hold an orphan a later copy
+					// re-stamps into this generation. Only a node still transport-down stays
+					// skipped.
+					if errors.Is(err, ErrFullTeardownNotAcked) || !IsNodeUnreachable(err) ||
+						ClassifyNodeLiveness(PingVolumeServer(env.GrpcDialOption, addr)) != NodeDown {
+						return fmt.Errorf("re-clear stale ec shards for volume %d on %s: %w", vid, addr, err)
+					}
+					glog.V(1).Infof("orphan re-sweep: volume %d on %s still skipped (unreachable): %v", vid, addr, err)
+					mu.Lock()
+					stillDown[addr] = struct{}{}
+					mu.Unlock()
+				}
+				return nil
+			})
+		}
+	}
+	if err := ewg.Wait(); err != nil {
+		return err
+	}
+
+	for _, addr := range addrs {
+		if _, down := stillDown[addr]; !down {
+			delete(skipped, addr)
+			glog.V(0).Infof("orphan re-sweep: node %s recovered and was cleaned; it will participate in the EC rebalance", addr)
+		}
+	}
+	return nil
+}
+
+// collectEcShardBitsByNode returns, for one volume, the EC shard bits each node
+// reports, unioned across all its disk types. Freshly generated shards sit on
+// the disk that held the source .dat, which may differ from the balance target
+// disk type, so visibility questions ("has the master heard about these shards
+// at all?") must not filter by disk type. Only the newest encode generation
+// (largest EncodeTsNs) counts: an orphaned older generation — a failed earlier
+// encode on a node the pre-encode sweep could not reach but the master still
+// hears from — must neither satisfy the registration wait nor pose as a second
+// holder in the clump check. Entries without a timestamp form the legacy
+// generation zero, so they only count when no stamped generation exists;
+// dropping them otherwise errs toward keeping the source volume.
+func CollectEcShardBitsByNode(topoInfo *master_pb.TopologyInfo, vid needle.VolumeId) map[pb.ServerAddress]erasure_coding.ShardBits {
+	type shardEntry struct {
+		addr pb.ServerAddress
+		ts   int64
+		bits erasure_coding.ShardBits
+	}
+	var entries []shardEntry
+	var newestTs int64
+	EachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
+		for _, diskInfo := range dn.DiskInfos {
+			for _, ecInfo := range diskInfo.EcShardInfos {
+				if ecInfo.Id != uint32(vid) {
+					continue
+				}
+				entries = append(entries, shardEntry{pb.NewServerAddressFromDataNode(dn), ecInfo.EncodeTsNs, erasure_coding.ShardBits(ecInfo.EcIndexBits)})
+				if ecInfo.EncodeTsNs > newestTs {
+					newestTs = ecInfo.EncodeTsNs
+				}
+			}
+		}
+	})
+	res := make(map[pb.ServerAddress]erasure_coding.ShardBits)
+	for _, e := range entries {
+		if e.ts == newestTs {
+			res[e.addr] |= e.bits
+		}
+	}
+	return res
+}
+
+// waitForEcShardsToRegister polls the master topology until every given volume
+// reports a full EC shard set. Mounting shards notifies the master
+// asynchronously (mount -> NewEcShardsChan -> delta heartbeat), so a topology
+// snapshot taken right after doEcEncode can predate the mounts. Failing after
+// the retries keeps the source volumes, and a re-run of ec.encode starts clean.
+func waitForEcShardsToRegister(env *Env, volumeIds []needle.VolumeId) error {
+	const maxAttempts = 10
+	const retryInterval = 2 * time.Second
+
+	var lastMissing []string
+	for attempt := 0; attempt < maxAttempts; attempt++ {
+		if attempt > 0 {
+			time.Sleep(retryInterval)
+		}
+		topoInfo, _, err := env.FetchTopology(0)
+		if err != nil {
+			return fmt.Errorf("fetch topology while waiting for ec shards to register: %w", err)
+		}
+		lastMissing = lastMissing[:0]
+		for _, vid := range volumeIds {
+			var union erasure_coding.ShardBits
+			for _, bits := range CollectEcShardBitsByNode(topoInfo, vid) {
+				union |= bits
+			}
+			if union.Count() < erasure_coding.TotalShardsCount {
+				lastMissing = append(lastMissing, fmt.Sprintf("volume %d: %d/%d shards", vid, union.Count(), erasure_coding.TotalShardsCount))
+			}
+		}
+		if len(lastMissing) == 0 {
+			return nil
+		}
+		glog.V(0).Infof("waiting for newly generated ec shards to register with the master (attempt %d/%d): %v",
+			attempt+1, maxAttempts, lastMissing)
+	}
+	return fmt.Errorf("newly generated ec shards did not register with the master after %d attempts: %v", maxAttempts, lastMissing)
+}
+
+// ecShardsClumpedOnOneNode reports whether every EC shard the master sees for
+// vid sits on a single node while at least one other node has free EC shard
+// slots on the target disk type — i.e. the preceding rebalance could have
+// spread the shards but did not. Zero visible shards is not a clump; the
+// recoverability check owns that case.
+func ecShardsClumpedOnOneNode(topoInfo *master_pb.TopologyInfo, vid needle.VolumeId, diskType types.DiskType) (holder pb.ServerAddress, clumped bool) {
+	byNode := CollectEcShardBitsByNode(topoInfo, vid)
+	if len(byNode) != 1 {
+		return "", false
+	}
+	for addr := range byNode {
+		holder = addr
+	}
+	ecNodes, _ := CollectEcVolumeServersByDc(topoInfo, "", diskType)
+	for _, en := range ecNodes {
+		if pb.NewServerAddressFromDataNode(en.Info) == holder {
+			continue
+		}
+		if en.FreeEcSlot > 0 {
+			return holder, true
+		}
+	}
+	return "", false
+}
+
+// ecShardSummaryByNode says where a volume's shards are, one entry per node,
+// sorted so the message is stable. It names the ids and not just the count: a
+// set holding shards 0-9 and one holding 4-13 are both "10 shards", and which
+// ones survived is what says whether the set is recoverable and from where.
+func ecShardSummaryByNode(byNode map[pb.ServerAddress]erasure_coding.ShardBits) []string {
+	summary := make([]string, 0, len(byNode))
+	for node, bits := range byNode {
+		summary = append(summary, fmt.Sprintf("%s=%d shards %v", node, bits.Count(), slices.Collect(bits.All())))
+	}
+	sort.Strings(summary)
+	return summary
+}
+
+func verifyEcShardsBeforeDelete(env *Env, volumeIds []needle.VolumeId, diskType types.DiskType, expectSpread bool) error {
+	// Shard relocations from the preceding EC balance reach the master via
+	// volume-server heartbeats, so freshly distributed shards may not all be
+	// visible in the master topology immediately. Poll a few times before
+	// concluding the shard set is incomplete, so a heartbeat-propagation lag is
+	// not mistaken for missing data. After the retries: a volume below the
+	// recoverable threshold (dataShards) aborts the deletion; a recoverable
+	// but degraded set proceeds with a warning, since the missing shards can
+	// be rebuilt from the survivors while keeping the source next to live
+	// shards is the more dangerous mixed state. When expectSpread is set (the
+	// rebalance ran in apply mode), a volume whose shards all still sit on one
+	// node while another node has free slots also aborts the deletion: losing
+	// that node would lose the volume, so the original is the safer copy.
+	const maxAttempts = 10
+	const retryInterval = 2 * time.Second
+
+	var lastErr error
+	var lastDegraded []string
+	var lastClumped []string
+	for attempt := 0; attempt < maxAttempts; attempt++ {
+		topoInfo, _, err := env.FetchTopology(0)
+		if err != nil {
+			return fmt.Errorf("fetch topology for shard verification: %w", err)
+		}
+
+		lastErr = nil
+		lastDegraded = lastDegraded[:0]
+		lastClumped = lastClumped[:0]
+		for _, vid := range volumeIds {
+			// Count the shards wherever they landed, as waitForEcShardsToRegister
+			// above already does. generateEcShards writes them beside the source
+			// volume, so encoding a volume that lives on a non-default medium
+			// puts them on that medium while -diskType still says hdd. Counting
+			// only the -diskType bucket then reports a complete set as entirely
+			// missing and aborts an encode that in fact succeeded, leaving the
+			// volume as both a .dat and a full set of shards.
+			byNode := CollectEcShardBitsByNode(topoInfo, vid)
+
+			var union erasure_coding.ShardBits
+			for _, bits := range byNode {
+				union |= bits
+			}
+
+			totalShards := erasure_coding.TotalShardsCount
+			degraded, err := erasure_coding.RequireRecoverableShardSet(uint32(vid), union, erasure_coding.DataShardsCount, totalShards)
+			if err != nil {
+				lastErr = fmt.Errorf("volume %d: %w (observed: %v)", vid, err, ecShardSummaryByNode(byNode))
+				break
+			}
+			if expectSpread {
+				if holder, clumped := ecShardsClumpedOnOneNode(topoInfo, vid, diskType); clumped {
+					lastClumped = append(lastClumped, fmt.Sprintf("volume %d: all shards on %s", vid, holder))
+					continue
+				}
+			}
+			if degraded {
+				lastDegraded = append(lastDegraded, fmt.Sprintf("volume %d: %d/%d shards", vid, union.Count(), totalShards))
+				continue
+			}
+
+			glog.V(0).Infof("EC shard verification ok for volume %d: %d/%d shards present across %d nodes",
+				vid, union.Count(), totalShards, len(byNode))
+		}
+
+		if lastErr == nil && len(lastDegraded) == 0 && len(lastClumped) == 0 {
+			return nil
+		}
+		if attempt < maxAttempts-1 {
+			glog.V(0).Infof("EC shard verification incomplete (attempt %d/%d), waiting for shard locations to propagate: %v %v %v",
+				attempt+1, maxAttempts, lastErr, lastDegraded, lastClumped)
+			time.Sleep(retryInterval)
+		}
+	}
+
+	if lastErr != nil {
+		glog.Errorf("EC shard verification failed after %d attempts: %v", maxAttempts, lastErr)
+		return lastErr
+	}
+	if len(lastClumped) > 0 {
+		return fmt.Errorf("EC shards still sit on a single node after rebalance even though other nodes have free slots (%v); keeping the original volumes. Run ec.balance -apply, verify the spread, then delete the originals or re-run ec.encode", lastClumped)
+	}
+	glog.Warningf("EC shard set incomplete but recoverable after %d attempts, proceeding with source deletion (rebuild missing shards with ec.rebuild): %v",
+		maxAttempts, lastDegraded)
+	return nil
+}
+
+// doDeleteVolumesWithLocations deletes volumes using pre-collected location information
+// This avoids race conditions where master metadata is updated after EC encoding
+func doDeleteVolumesWithLocations(env *Env, volumeIds []needle.VolumeId, volumeLocationsMap map[needle.VolumeId][]wdclient.Location, maxParallelization int) error {
+	if !env.isLocked() {
+		return fmt.Errorf("lock is lost")
+	}
+
+	ewg := util.NewErrorWaitGroup(maxParallelization)
+	for _, vid := range volumeIds {
+		locations, found := volumeLocationsMap[vid]
+		if !found {
+			fmt.Printf("warning: no locations found for volume %d, skipping deletion\n", vid)
+			continue
+		}
+
+		for _, l := range locations {
+			ewg.Add(func() error {
+				if err := deleteVolume(context.Background(), env.GrpcDialOption, vid, l.ServerAddress(), false, false); err != nil {
+					return fmt.Errorf("deleteVolume %s volume %d: %v", l.Url, vid, err)
+				}
+				fmt.Printf("deleted volume %d from %s\n", vid, l.Url)
+				return nil
+			})
+		}
+	}
+	if err := ewg.Wait(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func generateEcShards(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, collection string, sourceVolumeServer pb.ServerAddress) error {
+
+	fmt.Printf("generateEcShards %d (collection %q) on %s ...\n", volumeId, collection, sourceVolumeServer)
+
+	err := operation.WithVolumeServerClient(false, sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+		_, genErr := volumeServerClient.VolumeEcShardsGenerate(context.Background(), &volume_server_pb.VolumeEcShardsGenerateRequest{
+			VolumeId:   uint32(volumeId),
+			Collection: collection,
+		})
+		return genErr
+	})
+
+	return err
+
+}
+
+func SelectVolumeIdsFromTopology(topologyInfo *master_pb.TopologyInfo, volumeSizeLimitMb uint64, collectionRegex *regexp.Regexp, sourceDiskType *types.DiskType, quietSeconds int64, nowUnixSeconds int64, fullPercentage float64, verbose bool) (vids []needle.VolumeId, matchedCollections []string) {
+	// Statistics for verbose mode
+	var (
+		totalVolumes    int
+		remoteVolumes   int
+		wrongCollection int
+		wrongDiskType   int
+		tooRecent       int
+		tooSmall        int
+		noFreeDisk      int
+	)
+
+	vidMap := make(map[uint32]bool)
+	collectionSet := make(map[string]bool)
+	EachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
+		for _, diskInfo := range dn.DiskInfos {
+			for _, v := range diskInfo.VolumeInfos {
+				totalVolumes++
+
+				// ignore remote volumes
+				if v.RemoteStorageName != "" {
+					remoteVolumes++
+					if verbose {
+						fmt.Printf("skip volume %d on %s: remote volume (storage: %s)\n",
+							v.Id, dn.Id, v.RemoteStorageName)
+					}
+					continue
+				}
+
+				// check collection against regex pattern
+				if !collectionRegex.MatchString(v.Collection) {
+					wrongCollection++
+					if verbose {
+						fmt.Printf("skip volume %d on %s: collection doesn't match pattern (pattern: %s, actual: %s)\n",
+							v.Id, dn.Id, collectionRegex.String(), v.Collection)
+					}
+					continue
+				}
+
+				// track matched collection
+				collectionSet[v.Collection] = true
+
+				// check disk type
+				if sourceDiskType != nil && types.ToDiskType(v.DiskType) != *sourceDiskType {
+					wrongDiskType++
+					if verbose {
+						fmt.Printf("skip volume %d on %s: wrong disk type (expected: %s, actual: %s)\n",
+							v.Id, dn.Id, sourceDiskType.ReadableString(), types.ToDiskType(v.DiskType).ReadableString())
+					}
+					continue
+				}
+
+				// check quiet period
+				if v.ModifiedAtSecond+quietSeconds >= nowUnixSeconds {
+					tooRecent++
+					if verbose {
+						fmt.Printf("skip volume %d on %s: too recently modified (last modified: %d seconds ago, required: %d seconds)\n",
+							v.Id, dn.Id, nowUnixSeconds-v.ModifiedAtSecond, quietSeconds)
+					}
+					continue
+				}
+
+				// check size
+				sizeThreshold := fullPercentage / 100 * float64(volumeSizeLimitMb) * 1024 * 1024
+				if float64(v.Size) <= sizeThreshold {
+					tooSmall++
+					if verbose {
+						fmt.Printf("skip volume %d on %s: too small (size: %.1f MB, threshold: %.1f MB, %.1f%% full)\n",
+							v.Id, dn.Id, float64(v.Size)/(1024*1024), sizeThreshold/(1024*1024),
+							float64(v.Size)*100/(float64(volumeSizeLimitMb)*1024*1024))
+					}
+					continue
+				}
+
+				// check free disk space
+				if diskInfo.FreeVolumeCount < 2 {
+					glog.V(0).Infof("replica %s %d on %s has no free disk", v.Collection, v.Id, dn.Id)
+					if verbose {
+						fmt.Printf("skip replica of volume %d on %s: insufficient free disk space (free volumes: %d, required: 2)\n",
+							v.Id, dn.Id, diskInfo.FreeVolumeCount)
+					}
+					if _, found := vidMap[v.Id]; !found {
+						vidMap[v.Id] = false
+					}
+				} else {
+					if verbose {
+						fmt.Printf("selected volume %d on %s: size %.1f MB (%.1f%% full), last modified %d seconds ago, free volumes: %d\n",
+							v.Id, dn.Id, float64(v.Size)/(1024*1024),
+							float64(v.Size)*100/(float64(volumeSizeLimitMb)*1024*1024),
+							nowUnixSeconds-v.ModifiedAtSecond, diskInfo.FreeVolumeCount)
+					}
+					vidMap[v.Id] = true
+				}
+			}
+		}
+	})
+
+	for vid, good := range vidMap {
+		if good {
+			vids = append(vids, needle.VolumeId(vid))
+		} else {
+			noFreeDisk++
+		}
+	}
+
+	// Convert collection set to slice
+	for collection := range collectionSet {
+		matchedCollections = append(matchedCollections, collection)
+	}
+	sort.Strings(matchedCollections)
+
+	// Print summary statistics in verbose mode or when no volumes selected
+	if verbose || len(vids) == 0 {
+		fmt.Printf("\nVolume selection summary:\n")
+		fmt.Printf("  Total volumes examined: %d\n", totalVolumes)
+		fmt.Printf("  Selected for encoding: %d\n", len(vids))
+		fmt.Printf("  Collections matched: %v\n", matchedCollections)
+
+		if totalVolumes > 0 {
+			fmt.Printf("\nReasons for exclusion:\n")
+			if remoteVolumes > 0 {
+				fmt.Printf("  Remote volumes: %d\n", remoteVolumes)
+			}
+			if wrongCollection > 0 {
+				fmt.Printf("  Collection doesn't match pattern: %d\n", wrongCollection)
+			}
+			if wrongDiskType > 0 {
+				fmt.Printf("  Wrong disk type: %d\n", wrongDiskType)
+			}
+			if tooRecent > 0 {
+				fmt.Printf("  Too recently modified: %d\n", tooRecent)
+			}
+			if tooSmall > 0 {
+				fmt.Printf("  Too small (< %.1f%% full): %d\n", fullPercentage, tooSmall)
+			}
+			if noFreeDisk > 0 {
+				fmt.Printf("  Insufficient free disk space: %d\n", noFreeDisk)
+			}
+		}
+		fmt.Println()
+	}
+
+	return
+}

weed/ec/ec_encode_test.go

@@ -1,4 +1,4 @@
-package shell
+package ec
 
 import (
 	"regexp"
@@ -177,7 +177,7 @@ func TestSelectVolumeIdsFromTopology(t *testing.T) {
 	// In the provided topology, FreeVolumeCount is 1 ("free:1"), so it is < 2.
 	// So ALL volumes should be skipped due to insufficient free disk space.
 
-	vids, _ := selectVolumeIdsFromTopology(topologyInfo, volumeSizeLimitMb, collectionRegex, nil, quietSeconds, nowUnixSeconds, fullPercentage, verbose)
+	vids, _ := SelectVolumeIdsFromTopology(topologyInfo, volumeSizeLimitMb, collectionRegex, nil, quietSeconds, nowUnixSeconds, fullPercentage, verbose)
 
 	assert.Equal(t, 0, len(vids), "Should select 0 volumes because FreeVolumeCount is 1 (less than 2)")
 
@@ -197,7 +197,7 @@ func TestSelectVolumeIdsFromTopology(t *testing.T) {
 
 	// So expected volumes: 10, 11, 12.
 
-	vids, _ = selectVolumeIdsFromTopology(topologyInfo, volumeSizeLimitMb, collectionRegex, nil, quietSeconds, nowUnixSeconds, fullPercentage, verbose)
+	vids, _ = SelectVolumeIdsFromTopology(topologyInfo, volumeSizeLimitMb, collectionRegex, nil, quietSeconds, nowUnixSeconds, fullPercentage, verbose)
 
 	expectedVids := []needle.VolumeId{10, 11, 12}
 	assert.Equal(t, len(expectedVids), len(vids), "Should select 3 volumes")
@@ -232,7 +232,7 @@ func TestEcEncodeNodeCountCheck(t *testing.T) {
 	}
 
 	nodeCount := 0
-	eachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
+	EachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
 		nodeCount++
 	})
 
@@ -257,9 +257,9 @@ func TestChunkVolumeIds(t *testing.T) {
 		{101, 102},
 		{103, 104},
 		{105},
-	}, chunkVolumeIds(vids, 2))
+	}, ChunkVolumeIds(vids, 2))
 
-	assert.Equal(t, [][]needle.VolumeId{vids}, chunkVolumeIds(vids, 0))
+	assert.Equal(t, [][]needle.VolumeId{vids}, ChunkVolumeIds(vids, 0))
 }
 
 func ecShardVisibilityTestTopology(nodes ...*master_pb.DataNodeInfo) *master_pb.TopologyInfo {
@@ -304,11 +304,11 @@ func TestCollectEcShardBitsByNode(t *testing.T) {
 	}
 	topo := ecShardVisibilityTestTopology(node1, node2)
 
-	byNode := collectEcShardBitsByNode(topo, needle.VolumeId(1))
+	byNode := CollectEcShardBitsByNode(topo, needle.VolumeId(1))
 	assert.Len(t, byNode, 1)
 	assert.Equal(t, erasure_coding.ShardBits(allBits), byNode[pb.NewServerAddressFromDataNode(node1)])
 
-	assert.Empty(t, collectEcShardBitsByNode(topo, needle.VolumeId(3)))
+	assert.Empty(t, CollectEcShardBitsByNode(topo, needle.VolumeId(3)))
 }
 
 func TestCollectEcShardBitsByNode_MixedGenerations(t *testing.T) {
@@ -332,18 +332,18 @@ func TestCollectEcShardBitsByNode_MixedGenerations(t *testing.T) {
 	// the registration union nor as a second holder.
 	fresh := nodeWithGeneration("node1:8080", 200)
 	orphan := nodeWithGeneration("node2:8080", 100)
-	byNode := collectEcShardBitsByNode(ecShardVisibilityTestTopology(fresh, orphan), needle.VolumeId(1))
+	byNode := CollectEcShardBitsByNode(ecShardVisibilityTestTopology(fresh, orphan), needle.VolumeId(1))
 	assert.Len(t, byNode, 1)
 	assert.Equal(t, erasure_coding.ShardBits(allBits), byNode[pb.NewServerAddressFromDataNode(fresh)])
 
 	// Un-stamped entries are the legacy generation zero: dropped when a stamped
 	// generation exists, all counted when nothing is stamped.
 	legacy := nodeWithGeneration("node2:8080", 0)
-	byNode = collectEcShardBitsByNode(ecShardVisibilityTestTopology(fresh, legacy), needle.VolumeId(1))
+	byNode = CollectEcShardBitsByNode(ecShardVisibilityTestTopology(fresh, legacy), needle.VolumeId(1))
 	assert.Len(t, byNode, 1)
 	assert.Equal(t, erasure_coding.ShardBits(allBits), byNode[pb.NewServerAddressFromDataNode(fresh)])
 
-	byNode = collectEcShardBitsByNode(
+	byNode = CollectEcShardBitsByNode(
 		ecShardVisibilityTestTopology(nodeWithGeneration("node1:8080", 0), nodeWithGeneration("node2:8080", 0)),
 		needle.VolumeId(1))
 	assert.Len(t, byNode, 2)
@@ -438,7 +438,7 @@ func TestEcShardCountIgnoresDiskTypeOfTheShards(t *testing.T) {
 	topo := ecShardVisibilityTestTopology(node)
 
 	var union erasure_coding.ShardBits
-	for _, bits := range collectEcShardBitsByNode(topo, needle.VolumeId(1)) {
+	for _, bits := range CollectEcShardBitsByNode(topo, needle.VolumeId(1)) {
 		union |= bits
 	}
 	assert.Equal(t, erasure_coding.TotalShardsCount, union.Count(),

weed/ec/ec_flags_test.go

@@ -1,4 +1,4 @@
-package shell
+package ec
 
 import (
 	"testing"
@@ -25,27 +25,27 @@ func TestCountFreeShardSlotsPhysicalDiskGate(t *testing.T) {
 	}
 
 	// Physically empty: slot math applies, positive free slots.
-	if got := countFreeShardSlots(mk(1000*gb, 900*gb), types.HardDriveType); got <= 0 {
+	if got := CountFreeShardSlots(mk(1000*gb, 900*gb), types.HardDriveType); got <= 0 {
 		t.Errorf("physically empty disk free shard slots = %d, want > 0", got)
 	}
 	// Physically 96% full: gated to zero regardless of slot room.
-	if got := countFreeShardSlots(mk(1000*gb, 40*gb), types.HardDriveType); got != 0 {
+	if got := CountFreeShardSlots(mk(1000*gb, 40*gb), types.HardDriveType); got != 0 {
 		t.Errorf("physically full disk free shard slots = %d, want 0", got)
 	}
 	// No byte report (older server): fall back to slot math, positive.
-	if got := countFreeShardSlots(mk(0, 0), types.HardDriveType); got <= 0 {
+	if got := CountFreeShardSlots(mk(0, 0), types.HardDriveType); got <= 0 {
 		t.Errorf("unreported-bytes disk free shard slots = %d, want > 0 (slot fallback)", got)
 	}
 }
 
 func TestParseVolumeIdsFlag(t *testing.T) {
-	vids, err := parseVolumeIdsFlag("101, 102,101, 103")
+	vids, err := ParseVolumeIdsFlag("101, 102,101, 103")
 	assert.NoError(t, err)
 	assert.Equal(t, []needle.VolumeId{101, 102, 103}, vids)
 
-	_, err = parseVolumeIdsFlag("101,abc")
+	_, err = ParseVolumeIdsFlag("101,abc")
 	assert.Error(t, err)
 
-	_, err = parseVolumeIdsFlag(" , ")
+	_, err = ParseVolumeIdsFlag(" , ")
 	assert.Error(t, err)
 }

weed/ec/ec_rebalance_slots_test.go

@@ -1,4 +1,4 @@
-package shell
+package ec
 
 import (
 	"testing"
@@ -33,7 +33,7 @@ func TestECRebalanceWithLimitedSlots(t *testing.T) {
 	topology := buildLimitedSlotsTopology()
 
 	// Collect EC nodes from the topology
-	ecNodes, totalFreeEcSlots := collectEcVolumeServersByDc(topology, "", types.HardDriveType)
+	ecNodes, totalFreeEcSlots := CollectEcVolumeServersByDc(topology, "", types.HardDriveType)
 
 	t.Logf("Topology summary:")
 	t.Logf("  Number of EC nodes: %d", len(ecNodes))
@@ -42,19 +42,19 @@ func TestECRebalanceWithLimitedSlots(t *testing.T) {
 	// Log per-node details
 	for _, node := range ecNodes {
 		shardCount := 0
-		for _, diskInfo := range node.info.DiskInfos {
+		for _, diskInfo := range node.Info.DiskInfos {
 			for _, ecShard := range diskInfo.EcShardInfos {
 				shardCount += erasure_coding.GetShardCount(ecShard)
 			}
 		}
 		t.Logf("  Node %s (rack %s): %d shards, %d free slots",
-			node.info.Id, node.rack, shardCount, node.freeEcSlot)
+			node.Info.Id, node.Rack, shardCount, node.FreeEcSlot)
 	}
 
 	// Calculate total EC shards
 	totalEcShards := 0
 	for _, node := range ecNodes {
-		for _, diskInfo := range node.info.DiskInfos {
+		for _, diskInfo := range node.Info.DiskInfos {
 			for _, ecShard := range diskInfo.EcShardInfos {
 				totalEcShards += erasure_coding.GetShardCount(ecShard)
 			}
@@ -115,20 +115,20 @@ func TestECRebalanceZeroFreeSlots(t *testing.T) {
 	// (VolumeCount still reflects the original volumes)
 	topology := buildZeroFreeSlotTopology()
 
-	ecNodes, totalFreeEcSlots := collectEcVolumeServersByDc(topology, "", types.HardDriveType)
+	ecNodes, totalFreeEcSlots := CollectEcVolumeServersByDc(topology, "", types.HardDriveType)
 
 	t.Logf("Zero free slots scenario:")
 	for _, node := range ecNodes {
 		shardCount := 0
-		for _, diskInfo := range node.info.DiskInfos {
+		for _, diskInfo := range node.Info.DiskInfos {
 			for _, ecShard := range diskInfo.EcShardInfos {
 				shardCount += erasure_coding.GetShardCount(ecShard)
 			}
 		}
 		t.Logf("  Node %s: %d shards, %d free slots, volumeCount=%d, max=%d",
-			node.info.Id, shardCount, node.freeEcSlot,
-			node.info.DiskInfos[string(types.HardDriveType)].VolumeCount,
-			node.info.DiskInfos[string(types.HardDriveType)].MaxVolumeCount)
+			node.Info.Id, shardCount, node.FreeEcSlot,
+			node.Info.DiskInfos[string(types.HardDriveType)].VolumeCount,
+			node.Info.DiskInfos[string(types.HardDriveType)].MaxVolumeCount)
 	}
 	t.Logf("  Total free slots: %d", totalFreeEcSlots)
 

weed/ec/ec_rebuild.go

@@ -0,0 +1,435 @@
+package ec
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"slices"
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+type ecRebuilder struct {
+	env          *Env
+	ecNodes      []*EcNode
+	writer       io.Writer
+	applyChanges bool
+	collections  []string
+	volumeIds    []needle.VolumeId
+	diskType     types.DiskType
+
+	ewg       *util.ErrorWaitGroup
+	ecNodesMu sync.Mutex
+}
+
+// RebuildEcVolumes finds and rebuilds missing EC shards for the given
+// collections. Before rebuilding it asks volume servers to recover any shards
+// left unmounted by a missing .ecx index; such shards are invisible to the
+// master, so recovering them first avoids regenerating data that is actually
+// present. volumeIds, when non-empty, restricts the rebuild to those volumes.
+func RebuildEcVolumes(env *Env, ecNodes []*EcNode, writer io.Writer, collections []string, volumeIds []needle.VolumeId, diskType types.DiskType, maxParallelization int, applyChanges bool) error {
+	erb := &ecRebuilder{
+		env:          env,
+		ecNodes:      ecNodes,
+		writer:       writer,
+		applyChanges: applyChanges,
+		collections:  collections,
+		volumeIds:    volumeIds,
+		diskType:     diskType,
+
+		ewg: util.NewErrorWaitGroup(maxParallelization),
+	}
+
+	// Recover shards left unmounted by a missing .ecx index before planning: such
+	// shards never register with the master, so the rebuild below would treat the
+	// volume as short or unrepairable even though its data is intact (issue #10104).
+	erb.recoverMissingIndexes()
+
+	fmt.Printf("rebuildEcVolumes for %d collection(s)\n", len(collections))
+	for _, c := range collections {
+		erb.rebuildEcVolumes(c)
+	}
+
+	return erb.ewg.Wait()
+}
+
+func (erb *ecRebuilder) write(format string, a ...any) {
+	fmt.Fprintf(erb.writer, format, a...)
+}
+
+func (erb *ecRebuilder) isLocked() bool {
+	return erb.env.isLocked()
+}
+
+// matchesVolumeId verifies whether the rebuilder is targeted at a given volume ID.
+func (erb *ecRebuilder) matchesVolumeId(vid needle.VolumeId) bool {
+	if len(erb.volumeIds) == 0 {
+		return true
+	}
+
+	return slices.Contains(erb.volumeIds, vid)
+}
+
+// countLocalShards returns the number of shards already present locally on the node for the given volume.
+// Unions across all of the node's disks, like prepareDataToRecover, so slot
+// accounting matches what the rebuild will actually treat as local.
+func (erb *ecRebuilder) countLocalShards(node *EcNode, collection string, volumeId needle.VolumeId) int {
+	localShardsInfo := erasure_coding.NewShardsInfo()
+	for _, diskInfo := range node.Info.DiskInfos {
+		for _, ecShardInfo := range diskInfo.EcShardInfos {
+			if ecShardInfo.Collection == collection && needle.VolumeId(ecShardInfo.Id) == volumeId {
+				localShardsInfo.Add(erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(ecShardInfo))
+			}
+		}
+	}
+	return localShardsInfo.Count()
+}
+
+// selectAndReserveRebuilder atomically selects a rebuilder node with sufficient free slots
+// and reserves slots only for the non-local shards that need to be copied/generated.
+func (erb *ecRebuilder) selectAndReserveRebuilder(collection string, volumeId needle.VolumeId) (*EcNode, int, error) {
+	erb.ecNodesMu.Lock()
+	defer erb.ecNodesMu.Unlock()
+
+	if len(erb.ecNodes) == 0 {
+		return nil, 0, fmt.Errorf("no ec nodes available")
+	}
+
+	// Find the node with the most free slots, considering local shards
+	var bestNode *EcNode
+	var bestSlotsNeeded int
+	var maxAvailableSlots int
+	var minSlotsNeeded int = erasure_coding.TotalShardsCount // Start with maximum possible
+	for _, node := range erb.ecNodes {
+		localShards := erb.countLocalShards(node, collection, volumeId)
+		slotsNeeded := erasure_coding.TotalShardsCount - localShards
+		if slotsNeeded < 0 {
+			slotsNeeded = 0
+		}
+
+		if node.FreeEcSlot > maxAvailableSlots {
+			maxAvailableSlots = node.FreeEcSlot
+		}
+
+		if slotsNeeded < minSlotsNeeded {
+			minSlotsNeeded = slotsNeeded
+		}
+
+		if node.FreeEcSlot >= slotsNeeded {
+			if bestNode == nil || node.FreeEcSlot > bestNode.FreeEcSlot {
+				bestNode = node
+				bestSlotsNeeded = slotsNeeded
+			}
+		}
+	}
+
+	if bestNode == nil {
+		return nil, 0, fmt.Errorf("no node has sufficient free slots for volume %d (need at least %d slots, max available: %d)",
+			volumeId, minSlotsNeeded, maxAvailableSlots)
+	}
+
+	// Reserve slots only for non-local shards
+	bestNode.FreeEcSlot -= bestSlotsNeeded
+
+	return bestNode, bestSlotsNeeded, nil
+}
+
+// releaseRebuilder releases the reserved slots back to the rebuilder node.
+func (erb *ecRebuilder) releaseRebuilder(node *EcNode, slotsToRelease int) {
+	erb.ecNodesMu.Lock()
+	defer erb.ecNodesMu.Unlock()
+
+	// Release slots by incrementing the free slot count
+	node.FreeEcSlot += slotsToRelease
+}
+
+func (erb *ecRebuilder) rebuildEcVolumes(collection string) {
+	fmt.Printf("rebuildEcVolumes for %q\n", collection)
+
+	// collect vid => each shard locations, similar to ecShardMap in topology.go
+	ecShardMap := make(EcShardMap)
+	erb.ecNodesMu.Lock()
+	for _, ecNode := range erb.ecNodes {
+		ecShardMap.registerEcNode(ecNode, collection)
+	}
+	erb.ecNodesMu.Unlock()
+
+	for vid, locations := range ecShardMap {
+		if !erb.matchesVolumeId(vid) {
+			continue
+		}
+		shardCount := locations.shardCount()
+		if shardCount == erasure_coding.TotalShardsCount {
+			continue
+		}
+		if shardCount < erasure_coding.DataShardsCount {
+			erb.write("ec volume %d is unrepairable with %d shards (need %d), skipping\n", vid, shardCount, erasure_coding.DataShardsCount)
+			continue
+		}
+
+		// Capture variables for closure
+		vid := vid
+		locations := locations
+
+		erb.ewg.Add(func() error {
+			// Select rebuilder and reserve slots atomically per volume
+			rebuilder, slotsToReserve, err := erb.selectAndReserveRebuilder(collection, vid)
+			if err != nil {
+				return fmt.Errorf("failed to select rebuilder for volume %d: %v", vid, err)
+			}
+			defer erb.releaseRebuilder(rebuilder, slotsToReserve)
+
+			return erb.rebuildOneEcVolume(collection, vid, locations, rebuilder)
+		})
+	}
+}
+
+// recoverMissingIndexes asks every ec node to fetch a missing .ecx index from a
+// peer and mount the on-disk shards it could not load on its own. Shards
+// orphaned this way (index only on another server) are absent from the master
+// topology, so without this pass ec.rebuild would regenerate or give up on
+// shards whose data is actually present — and a volume whose every holder lacks
+// the index would not appear in the topology at all. Each node therefore
+// recovers all of its on-disk orphans (volume_id 0); an explicit -volumeIds
+// list narrows that to the requested volumes. On apply it refreshes the topology
+// so the rebuild planning sees the recovered shards (issue #10104).
+func (erb *ecRebuilder) recoverMissingIndexes() {
+	erb.ecNodesMu.Lock()
+	nodes := append([]*EcNode(nil), erb.ecNodes...)
+	erb.ecNodesMu.Unlock()
+	if len(nodes) == 0 {
+		return
+	}
+
+	// volume_id 0 means "recover every orphan on the node"; a -volumeIds list
+	// narrows recovery to those ids (each scanned across collections server-side).
+	vids := erb.volumeIds
+	if len(vids) == 0 {
+		vids = []needle.VolumeId{0}
+	}
+
+	if !erb.applyChanges {
+		erb.write("would ask %d ec node(s) to recover EC shards left unmounted by a missing .ecx index\n", len(nodes))
+		return
+	}
+
+	for _, node := range nodes {
+		for _, vid := range vids {
+			err := operation.WithVolumeServerClient(false, pb.NewServerAddressFromDataNode(node.Info), erb.env.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+				_, mountErr := client.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{
+					VolumeId:            uint32(vid),
+					RecoverMissingIndex: true,
+				})
+				return mountErr
+			})
+			if err != nil {
+				erb.write("%s recover missing index (volume %d): %v\n", node.Info.Id, vid, err)
+			}
+		}
+	}
+
+	// Refresh topology so the rebuild planning sees shards the recovery registered.
+	refreshed, _, err := CollectEcNodes(erb.env, erb.diskType)
+	if err != nil {
+		erb.write("failed to refresh ec nodes after index recovery: %v\n", err)
+		return
+	}
+	erb.ecNodesMu.Lock()
+	erb.ecNodes = refreshed
+	erb.ecNodesMu.Unlock()
+}
+
+func (erb *ecRebuilder) rebuildOneEcVolume(collection string, volumeId needle.VolumeId, locations EcShardLocations, rebuilder *EcNode) error {
+	if !erb.isLocked() {
+		return fmt.Errorf("lock is lost")
+	}
+
+	fmt.Printf("rebuildOneEcVolume %s %d\n", collection, volumeId)
+
+	// collect shard files to rebuilder local disk
+	var generatedShardIds []erasure_coding.ShardId
+	copiedShardIds, _, err := erb.prepareDataToRecover(rebuilder, collection, volumeId, locations)
+	defer func() {
+		// Clean up the working copies this run actually made, even when the
+		// recoverability gate failed after some copies already succeeded:
+		// they are temp files on the rebuilder nothing else reclaims. Dry-run
+		// copies nothing (copiedShardIds is empty), so this issues no delete
+		// RPC. Use a local error so a cleanup failure cannot mask the return.
+		if !erb.applyChanges || len(copiedShardIds) == 0 {
+			return
+		}
+		if derr := SourceServerDeleteEcShards(erb.env.GrpcDialOption, collection, volumeId, pb.NewServerAddressFromDataNode(rebuilder.Info), copiedShardIds); derr != nil {
+			erb.write("%s delete copied ec shards %s %d.%v: %v\n", rebuilder.Info.Id, collection, volumeId, copiedShardIds, derr)
+		}
+	}()
+	if err != nil {
+		return err
+	}
+
+	if !erb.applyChanges {
+		return nil
+	}
+
+	// generate ec shards, and maybe ecx file
+	generatedShardIds, err = erb.generateMissingShards(collection, volumeId, pb.NewServerAddressFromDataNode(rebuilder.Info))
+	if err != nil {
+		return err
+	}
+
+	// mount the generated shards
+	err = MountEcShards(erb.env.GrpcDialOption, collection, volumeId, pb.NewServerAddressFromDataNode(rebuilder.Info), generatedShardIds)
+	if err != nil {
+		return err
+	}
+
+	// ensure ECNode updates are atomic
+	erb.ecNodesMu.Lock()
+	defer erb.ecNodesMu.Unlock()
+	rebuilder.AddEcVolumeShards(volumeId, collection, generatedShardIds, erb.diskType)
+
+	return nil
+}
+
+func (erb *ecRebuilder) generateMissingShards(collection string, volumeId needle.VolumeId, sourceLocation pb.ServerAddress) (rebuiltShardIds []erasure_coding.ShardId, err error) {
+
+	err = operation.WithVolumeServerClient(false, sourceLocation, erb.env.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+		resp, rebuildErr := volumeServerClient.VolumeEcShardsRebuild(context.Background(), &volume_server_pb.VolumeEcShardsRebuildRequest{
+			VolumeId:   uint32(volumeId),
+			Collection: collection,
+		})
+		if rebuildErr == nil {
+			rebuiltShardIds = erasure_coding.Uint32ToShardIds(resp.RebuiltShardIds)
+		}
+		return rebuildErr
+	})
+	return
+}
+
+func (erb *ecRebuilder) prepareDataToRecover(rebuilder *EcNode, collection string, volumeId needle.VolumeId, locations EcShardLocations) (copiedShardIds []erasure_coding.ShardId, localShardIds []erasure_coding.ShardId, err error) {
+
+	needEcxFile := true
+	localShardsInfo := erasure_coding.NewShardsInfo()
+	for _, diskInfo := range rebuilder.Info.DiskInfos {
+		for _, ecShardInfo := range diskInfo.EcShardInfos {
+			if ecShardInfo.Collection == collection && needle.VolumeId(ecShardInfo.Id) == volumeId {
+				needEcxFile = false
+				// Union across disks: the rebuilder may hold this volume's
+				// shards on more than one disk. Overwriting per-disk would
+				// make a shard on a non-last disk look remote and get copied
+				// onto itself (O_TRUNC) and then node-wide deleted.
+				localShardsInfo.Add(erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(ecShardInfo))
+			}
+		}
+	}
+
+	targetShardCount := erasure_coding.TotalShardsCount
+	for i := erasure_coding.TotalShardsCount; i < len(locations); i++ {
+		if len(locations[i]) > 0 {
+			targetShardCount = i + 1
+		}
+	}
+
+	// recoverableRemoteShards counts remote shards that can contribute to the
+	// rebuild. Dry-run counts the plan; apply mode counts only successful copies.
+	recoverableRemoteShards := 0
+	for i := 0; i < targetShardCount; i++ {
+		ecNodes := locations[i]
+		shardId := erasure_coding.ShardId(i)
+		if len(ecNodes) == 0 {
+			erb.write("missing shard %d.%d\n", volumeId, shardId)
+			continue
+		}
+
+		if localShardsInfo.Has(shardId) {
+			localShardIds = append(localShardIds, shardId)
+			erb.write("use existing shard %d.%d\n", volumeId, shardId)
+			continue
+		}
+
+		// The rebuilder is itself the only listed holder: never copy a shard
+		// onto itself (the in-place O_TRUNC would destroy it) nor schedule it
+		// for the post-rebuild delete. Treat it as already local.
+		if ecNodes[0].Info.Id == rebuilder.Info.Id {
+			localShardIds = append(localShardIds, shardId)
+			erb.write("use existing shard %d.%d (already on rebuilder)\n", volumeId, shardId)
+			continue
+		}
+
+		if !erb.applyChanges {
+			recoverableRemoteShards++
+			erb.write("would copy %d.%d from %s\n", volumeId, shardId, ecNodes[0].Info.Id)
+			continue
+		}
+
+		copyErr := operation.WithVolumeServerClient(false, pb.NewServerAddressFromDataNode(rebuilder.Info), erb.env.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+			_, copyErr := volumeServerClient.VolumeEcShardsCopy(context.Background(), &volume_server_pb.VolumeEcShardsCopyRequest{
+				VolumeId:       uint32(volumeId),
+				Collection:     collection,
+				ShardIds:       []uint32{uint32(shardId)},
+				CopyEcxFile:    needEcxFile,
+				CopyEcjFile:    true,
+				CopyVifFile:    needEcxFile,
+				SourceDataNode: string(pb.NewServerAddressFromDataNode(ecNodes[0].Info)),
+			})
+			return copyErr
+		})
+		if copyErr != nil {
+			erb.write("%s failed to copy %d.%d from %s: %v\n", rebuilder.Info.Id, volumeId, shardId, ecNodes[0].Info.Id, copyErr)
+			continue
+		}
+		recoverableRemoteShards++
+		if needEcxFile {
+			needEcxFile = false
+		}
+		erb.write("%s copied %d.%d from %s\n", rebuilder.Info.Id, volumeId, shardId, ecNodes[0].Info.Id)
+		// Only shards this run actually copied are temp working files to be
+		// deleted afterward; never a pre-existing local or remote shard.
+		copiedShardIds = append(copiedShardIds, shardId)
+	}
+
+	if len(localShardIds)+recoverableRemoteShards >= erasure_coding.DataShardsCount {
+		return copiedShardIds, localShardIds, nil
+	}
+
+	// Hand back what was copied so the caller deletes these orphaned working
+	// shards: recovery failed, but the temp files are already on the rebuilder.
+	return copiedShardIds, localShardIds, fmt.Errorf("%d shards are not enough to recover volume %d", len(localShardIds)+recoverableRemoteShards, volumeId)
+
+}
+
+type EcShardMap map[needle.VolumeId]EcShardLocations
+type EcShardLocations [][]*EcNode
+
+func (ecShardMap EcShardMap) registerEcNode(ecNode *EcNode, collection string) {
+	for _, diskInfo := range ecNode.Info.DiskInfos {
+		for _, shardInfo := range diskInfo.EcShardInfos {
+			if shardInfo.Collection == collection {
+				existing, found := ecShardMap[needle.VolumeId(shardInfo.Id)]
+				if !found {
+					// Use MaxShardCount (32) to support custom EC ratios
+					existing = make([][]*EcNode, erasure_coding.MaxShardCount)
+					ecShardMap[needle.VolumeId(shardInfo.Id)] = existing
+				}
+				for _, shardId := range erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(shardInfo).Ids() {
+					existing[shardId] = append(existing[shardId], ecNode)
+				}
+			}
+		}
+	}
+}
+
+func (ecShardLocations EcShardLocations) shardCount() (count int) {
+	for _, locations := range ecShardLocations {
+		if len(locations) > 0 {
+			count++
+		}
+	}
+	return
+}

weed/ec/ec_rebuild_test.go

@@ -1,4 +1,4 @@
-package shell
+package ec
 
 import (
 	"bytes"
@@ -8,6 +8,7 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
 // TestEcShardMapRegister tests that EC shards are properly registered
@@ -37,12 +38,12 @@ func TestEcShardMapRegister(t *testing.T) {
 
 	// Verify shard distribution
 	for i := 0; i < 7; i++ {
-		if len(locations[i]) != 1 || locations[i][0].info.Id != "node1" {
+		if len(locations[i]) != 1 || locations[i][0].Info.Id != "node1" {
 			t.Errorf("Shard %d should be on node1", i)
 		}
 	}
 	for i := 7; i < erasure_coding.TotalShardsCount; i++ {
-		if len(locations[i]) != 1 || locations[i][0].info.Id != "node2" {
+		if len(locations[i]) != 1 || locations[i][0].Info.Id != "node2" {
 			t.Errorf("Shard %d should be on node2", i)
 		}
 	}
@@ -89,11 +90,8 @@ func TestRebuildEcVolumesInsufficientShards(t *testing.T) {
 		addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4}) // Only 5 shards
 
 	erb := &ecRebuilder{
-		commandEnv: &CommandEnv{
-			env:    make(map[string]string),
-			noLock: true, // Bypass lock check for unit test
-		},
-		ewg:     NewErrorWaitGroup(DefaultMaxParallelization),
+		env:     &Env{},
+		ewg:     util.NewErrorWaitGroup(10),
 		ecNodes: []*EcNode{node1},
 		writer:  &logBuffer,
 	}
@@ -120,11 +118,8 @@ func TestRebuildEcVolumesCompleteVolume(t *testing.T) {
 		addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13})
 
 	erb := &ecRebuilder{
-		commandEnv: &CommandEnv{
-			env:    make(map[string]string),
-			noLock: true, // Bypass lock check for unit test
-		},
-		ewg:          NewErrorWaitGroup(DefaultMaxParallelization),
+		env:          &Env{},
+		ewg:          util.NewErrorWaitGroup(10),
 		ecNodes:      []*EcNode{node1},
 		writer:       &logBuffer,
 		applyChanges: false,
@@ -152,11 +147,8 @@ func TestRebuildEcVolumesInsufficientSpace(t *testing.T) {
 							addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
 
 	erb := &ecRebuilder{
-		commandEnv: &CommandEnv{
-			env:    make(map[string]string),
-			noLock: true, // Bypass lock check for unit test
-		},
-		ewg:          NewErrorWaitGroup(DefaultMaxParallelization),
+		env:          &Env{},
+		ewg:          util.NewErrorWaitGroup(10),
 		ecNodes:      []*EcNode{node1},
 		writer:       &logBuffer,
 		applyChanges: false,
@@ -203,17 +195,17 @@ func TestMultipleNodesWithShards(t *testing.T) {
 
 	// Verify each shard is on the correct node
 	for i := 0; i < 4; i++ {
-		if len(locations[i]) != 1 || locations[i][0].info.Id != "node1" {
+		if len(locations[i]) != 1 || locations[i][0].Info.Id != "node1" {
 			t.Errorf("Shard %d should be on node1", i)
 		}
 	}
 	for i := 4; i < 8; i++ {
-		if len(locations[i]) != 1 || locations[i][0].info.Id != "node2" {
+		if len(locations[i]) != 1 || locations[i][0].Info.Id != "node2" {
 			t.Errorf("Shard %d should be on node2", i)
 		}
 	}
 	for i := 8; i < 10; i++ {
-		if len(locations[i]) != 1 || locations[i][0].info.Id != "node3" {
+		if len(locations[i]) != 1 || locations[i][0].Info.Id != "node3" {
 			t.Errorf("Shard %d should be on node3", i)
 		}
 	}
@@ -243,10 +235,10 @@ func TestDuplicateShards(t *testing.T) {
 	foundNode1 := false
 	foundNode2 := false
 	for _, node := range locations[0] {
-		if node.info.Id == "node1" {
+		if node.Info.Id == "node1" {
 			foundNode1 = true
 		}
-		if node.info.Id == "node2" {
+		if node.Info.Id == "node2" {
 			foundNode2 = true
 		}
 	}
@@ -270,10 +262,7 @@ func TestPrepareDataToRecoverTargetShardCount(t *testing.T) {
 		addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
 
 	erb := &ecRebuilder{
-		commandEnv: &CommandEnv{
-			env:    make(map[string]string),
-			noLock: true,
-		},
+		env:     &Env{},
 		ecNodes: []*EcNode{node1},
 		writer:  &logBuffer,
 	}

weed/ec/ec_scrub.go

@@ -0,0 +1,81 @@
+package ec
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+	"sync"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// ScrubEcVolumes asks each volume server to scrub its EC volumes (optionally
+// narrowed to volumeIds) in the given mode, aggregating and reporting results.
+func ScrubEcVolumes(env *Env, writer io.Writer, volumeServerAddrs []pb.ServerAddress, volumeIds []uint32, mode volume_server_pb.VolumeScrubMode, forceDeletedNeedlesCheck bool, maxParallelization int, showDetails bool) error {
+	var brokenVolumesStr, brokenShardsStr []string
+	var details []string
+	var totalVolumes, brokenVolumes, brokenShards, totalFiles uint64
+	var mu sync.Mutex
+
+	ewg := util.NewErrorWaitGroup(maxParallelization)
+	count := 0
+	for _, addr := range volumeServerAddrs {
+		ewg.Add(func() error {
+			mu.Lock()
+			count++
+			fmt.Fprintf(writer, "Scrubbing %s (%d/%d)...\n", addr.String(), count, len(volumeServerAddrs))
+			mu.Unlock()
+
+			err := operation.WithVolumeServerClient(false, addr, env.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
+				res, err := volumeServerClient.ScrubEcVolume(context.Background(), &volume_server_pb.ScrubEcVolumeRequest{
+					Mode:                     mode,
+					VolumeIds:                volumeIds,
+					ForceDeletedNeedlesCheck: forceDeletedNeedlesCheck,
+				})
+				if err != nil {
+					return err
+				}
+
+				mu.Lock()
+				defer mu.Unlock()
+
+				totalVolumes += res.GetTotalVolumes()
+				totalFiles += res.GetTotalFiles()
+				brokenVolumes += uint64(len(res.GetBrokenVolumeIds()))
+				brokenShards += uint64(len(res.GetBrokenShardInfos()))
+				for _, d := range res.GetDetails() {
+					details = append(details, fmt.Sprintf("[%s] %s", addr, d))
+				}
+				for _, vid := range res.GetBrokenVolumeIds() {
+					brokenVolumesStr = append(brokenVolumesStr, fmt.Sprintf("%s:%v", addr, vid))
+				}
+				for _, si := range res.GetBrokenShardInfos() {
+					brokenShardsStr = append(brokenShardsStr, fmt.Sprintf("%s:%v:%v", addr, si.VolumeId, si.ShardId))
+				}
+
+				return nil
+			})
+			return err
+		})
+	}
+	if err := ewg.Wait(); err != nil {
+		return err
+	}
+
+	fmt.Fprintf(writer, "Scrubbed %d EC files and %d volumes on %d nodes\n", totalFiles, totalVolumes, len(volumeServerAddrs))
+	if brokenVolumes != 0 {
+		fmt.Fprintf(writer, "\nGot scrub failures on %d EC volumes and %d EC shards :(\n", brokenVolumes, brokenShards)
+		fmt.Fprintf(writer, "Affected volumes: %s\n", strings.Join(brokenVolumesStr, ", "))
+		if len(brokenShardsStr) != 0 {
+			fmt.Fprintf(writer, "Affected shards:  %s\n", strings.Join(brokenShardsStr, ", "))
+		}
+		if showDetails && len(details) != 0 {
+			fmt.Fprintf(writer, "Details:\n\t%s\n", strings.Join(details, "\n\t"))
+		}
+	}
+	return nil
+}

weed/ec/ec_shard_unmount.go

@@ -0,0 +1,210 @@
+package ec
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+)
+
+const (
+	ecShardActionTimeout = 1 * time.Minute
+)
+
+// ShardRef names one EC shard copy, optionally pinned to a node address.
+type ShardRef struct {
+	ShardID     uint32
+	Collection  string
+	NodeAddress string
+}
+
+func (s *ShardRef) String() string {
+	if s.NodeAddress == "" {
+		return fmt.Sprintf("%d", s.ShardID)
+	}
+	return fmt.Sprintf("%d@%s", s.ShardID, s.NodeAddress)
+}
+
+// ShardRefsFromString parses a comma-separated list of shard IDs, each either a
+// bare numeric ID or <shard_id>@<node_address> to pick one copy.
+func ShardRefsFromString(shards string) ([]*ShardRef, error) {
+	res := []*ShardRef{}
+
+	for _, s := range strings.Split(shards, ",") {
+		s = strings.TrimSpace(s)
+		if s == "" {
+			return nil, fmt.Errorf("empty shard ID in %q", shards)
+		}
+
+		// optional <shard ID>@<node address> to pick one copy
+		idStr, addr, _ := strings.Cut(s, "@")
+
+		id, err := strconv.Atoi(idStr)
+		if err != nil || id < 0 || id >= erasure_coding.MaxShardCount {
+			return nil, fmt.Errorf("invalid shard ID %q", s)
+		}
+
+		res = append(res, &ShardRef{ShardID: uint32(id), NodeAddress: addr})
+	}
+
+	return res, nil
+}
+
+// ShardUnmountRequest describes one ec.shard.unmount run.
+type ShardUnmountRequest struct {
+	Topology      *master_pb.TopologyInfo
+	VolumeID      uint32
+	Shards        []*ShardRef
+	Delete        bool
+	IgnoreInvalid bool
+	Apply         bool
+}
+
+type ecShardUnmounter struct {
+	env    *Env
+	writer io.Writer
+	req    ShardUnmountRequest
+}
+
+// UnmountShards unmounts, and optionally deletes, the requested EC shard
+// copies, resolving them against the live topology first. Dry-run unless
+// req.Apply is set.
+func UnmountShards(env *Env, writer io.Writer, req ShardUnmountRequest) error {
+	c := &ecShardUnmounter{env: env, writer: writer, req: req}
+	return c.doShardsUnmount()
+}
+
+func (c *ecShardUnmounter) write(format string, a ...any) {
+	fmt.Fprintf(c.writer, format, a...)
+}
+
+func (c *ecShardUnmounter) liveShardsForVolume() []*ShardRef {
+	shards := []*ShardRef{}
+
+	for _, dci := range c.req.Topology.GetDataCenterInfos() {
+		for _, ri := range dci.GetRackInfos() {
+			for _, dni := range ri.GetDataNodeInfos() {
+				nodeAddress := dni.GetAddress()
+				for _, di := range dni.GetDiskInfos() {
+					for _, eci := range di.GetEcShardInfos() {
+						if eci.GetId() == c.req.VolumeID {
+							sinfo := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(eci)
+							for _, sid := range sinfo.Ids() {
+								shards = append(shards, &ShardRef{
+									ShardID:     uint32(sid),
+									Collection:  eci.GetCollection(),
+									NodeAddress: nodeAddress,
+								})
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	sort.SliceStable(shards, func(i, j int) bool { return shards[i].ShardID < shards[j].ShardID })
+	return shards
+}
+
+func (c *ecShardUnmounter) printShards(ss []*ShardRef) {
+	for _, s := range ss {
+		c.write("\t%v\n", s)
+	}
+	c.write("\n")
+}
+
+func (c *ecShardUnmounter) doShardsUnmount() error {
+	liveShards := c.liveShardsForVolume()
+	c.write("Live shard topology for volume ID %d (%d shards):\n", c.req.VolumeID, len(liveShards))
+	c.printShards(liveShards)
+
+	// resolve target shards against the live topology
+	targetShards := []*ShardRef{}
+	for _, ps := range c.req.Shards {
+		var result *ShardRef
+		for _, ts := range liveShards {
+			if ts.ShardID == ps.ShardID {
+				if ps.NodeAddress == "" || ps.NodeAddress == ts.NodeAddress {
+					if result != nil {
+						return fmt.Errorf("shard %v is ambiguous", ps)
+					}
+					result = ts
+				}
+			}
+		}
+		if result == nil {
+			if !c.req.IgnoreInvalid {
+				return fmt.Errorf("shard %v is invalid", ps)
+			}
+			c.write("!!! ignoring invalid shard %v\n", ps)
+		} else {
+			targetShards = append(targetShards, result)
+		}
+	}
+	if len(targetShards) == 0 {
+		return fmt.Errorf("got no shards to process")
+	}
+
+	mode := "unmount"
+	if c.req.Delete {
+		mode = "unmount + delete"
+	}
+	c.write("Will %s %d shard(s):\n", mode, len(targetShards))
+	c.printShards(targetShards)
+
+	if !c.req.Apply {
+		c.write("Not proceeding in dry-run mode\n")
+		return nil
+	}
+
+	if !c.env.isLocked() {
+		return fmt.Errorf("lock is lost")
+	}
+
+	for _, s := range targetShards {
+		if err := c.unmountShard(s); err != nil {
+			return err
+		}
+	}
+
+	c.write("\nAll done!\n")
+	return nil
+}
+
+func (c *ecShardUnmounter) unmountShard(s *ShardRef) error {
+	ctx, cancel := context.WithTimeout(context.Background(), ecShardActionTimeout)
+	defer cancel()
+
+	return operation.WithVolumeServerClient(false, pb.ServerAddress(s.NodeAddress), c.env.GrpcDialOption, func(vsc volume_server_pb.VolumeServerClient) error {
+		c.write("Unmounting shard %v for volume ID %d...\n", s, c.req.VolumeID)
+		if _, err := vsc.VolumeEcShardsUnmount(ctx, &volume_server_pb.VolumeEcShardsUnmountRequest{
+			VolumeId: c.req.VolumeID,
+			ShardIds: []uint32{s.ShardID},
+		}); err != nil {
+			return err
+		}
+
+		if c.req.Delete {
+			c.write("Deleting shard %v for volume ID %d...\n", s, c.req.VolumeID)
+			if _, err := vsc.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
+				VolumeId:   c.req.VolumeID,
+				Collection: s.Collection,
+				ShardIds:   []uint32{s.ShardID},
+			}); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}

weed/ec/ec_shard_unmount_test.go

@@ -1,4 +1,4 @@
-package shell
+package ec
 
 import (
 	"testing"
@@ -11,28 +11,28 @@ func TestEcShardsFromString(t *testing.T) {
 	tests := []struct {
 		name    string
 		in      string
-		want    []*ecShard
+		want    []*ShardRef
 		wantErr bool
 	}{
 		{
 			name: "single id",
 			in:   "2",
-			want: []*ecShard{{ShardID: 2}},
+			want: []*ShardRef{{ShardID: 2}},
 		},
 		{
 			name: "list of ids",
 			in:   "0, 3 ,5",
-			want: []*ecShard{{ShardID: 0}, {ShardID: 3}, {ShardID: 5}},
+			want: []*ShardRef{{ShardID: 0}, {ShardID: 3}, {ShardID: 5}},
 		},
 		{
 			name: "qualified keeps host:port",
 			in:   "3@10.200.18.88:9007",
-			want: []*ecShard{{ShardID: 3, NodeAddress: "10.200.18.88:9007"}},
+			want: []*ShardRef{{ShardID: 3, NodeAddress: "10.200.18.88:9007"}},
 		},
 		{
 			name: "mixed bare and qualified",
 			in:   "2,3@10.200.18.88:9007",
-			want: []*ecShard{{ShardID: 2}, {ShardID: 3, NodeAddress: "10.200.18.88:9007"}},
+			want: []*ShardRef{{ShardID: 2}, {ShardID: 3, NodeAddress: "10.200.18.88:9007"}},
 		},
 		{
 			name:    "colon without node marker is not a shard ID",
@@ -43,7 +43,7 @@ func TestEcShardsFromString(t *testing.T) {
 			// shard IDs beyond the default 10+4 total are valid on custom-ratio volumes.
 			name: "shard id within MaxShardCount",
 			in:   "20",
-			want: []*ecShard{{ShardID: 20}},
+			want: []*ShardRef{{ShardID: 20}},
 		},
 		{
 			name:    "shard id at MaxShardCount is rejected",
@@ -78,7 +78,7 @@ func TestEcShardsFromString(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := ecShardsFromString(tt.in)
+			got, err := ShardRefsFromString(tt.in)
 			if tt.wantErr {
 				assert.Error(t, err)
 				return
@@ -91,12 +91,12 @@ func TestEcShardsFromString(t *testing.T) {
 
 // the printed topology form must parse back, so entries can be copied verbatim.
 func TestEcShardStringRoundTrips(t *testing.T) {
-	for _, s := range []*ecShard{
+	for _, s := range []*ShardRef{
 		{ShardID: 2},
 		{ShardID: 3, NodeAddress: "10.200.18.88:9007"},
 	} {
-		got, err := ecShardsFromString(s.String())
+		got, err := ShardRefsFromString(s.String())
 		assert.NoError(t, err)
-		assert.Equal(t, []*ecShard{s}, got)
+		assert.Equal(t, []*ShardRef{s}, got)
 	}
 }

weed/ec/env.go

@@ -0,0 +1,39 @@
+// Package ec holds the EC (erasure coding) cluster orchestration logic shared
+// by the weed shell commands and the maintenance workers: topology analysis of
+// EC shards, the encode/balance pipelines, and the volume-server RPC wrappers
+// they drive. The placement policy itself lives in
+// weed/storage/erasure_coding/ecbalancer; low-level shard mechanics live in
+// weed/storage/erasure_coding.
+package ec
+
+import (
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/wdclient"
+	"google.golang.org/grpc"
+)
+
+// Env carries the cluster access hooks EC operations need, decoupled from any
+// particular caller (shell CommandEnv, worker task, admin server).
+type Env struct {
+	GrpcDialOption grpc.DialOption
+	// FetchTopology returns a fresh master topology snapshot (and the master's
+	// volume size limit in MB) after an optional delay.
+	FetchTopology func(delay time.Duration) (*master_pb.TopologyInfo, uint64, error)
+	// GetVolumeLocations returns the current replica locations for a volume id,
+	// or false if the volume is unknown.
+	GetVolumeLocations func(vid uint32) ([]wdclient.Location, bool)
+	// IsLocked reports whether the caller still holds the cluster admin lock.
+	// Callers without a lock concept return true.
+	IsLocked func() bool
+}
+
+// isLocked treats a nil Env or nil hook as locked, matching the shell's
+// nil-receiver behavior so dry-run paths work without a cluster connection.
+func (env *Env) isLocked() bool {
+	if env == nil || env.IsLocked == nil {
+		return true
+	}
+	return env.IsLocked()
+}

weed/shell/command_ec_check_replication.go

@@ -4,12 +4,10 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"slices"
 	"strconv"
 	"strings"
 
-	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
-	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/ec"
 )
 
 func init() {
@@ -63,13 +61,7 @@ func (c *commandEcCheckReplication) Do(args []string, commandEnv *CommandEnv, wr
 		return
 	}
 
-	// Keep execution state in a per-run runner rather than on the long-lived
-	// command singleton, so concurrent or repeated invocations don't share state.
-	runner := &ecCheckReplicationRunner{
-		writer:      writer,
-		volumeIDMap: map[uint32]bool{},
-	}
-
+	volumeIDMap := map[uint32]bool{}
 	if *volumeIDsStr != "" {
 		for _, vids := range strings.Split(*volumeIDsStr, ",") {
 			vids = strings.TrimSpace(vids)
@@ -77,207 +69,19 @@ func (c *commandEcCheckReplication) Do(args []string, commandEnv *CommandEnv, wr
 				continue
 			}
 			if vid, err := strconv.ParseUint(vids, 10, 32); err == nil {
-				runner.volumeIDMap[uint32(vid)] = true
+				volumeIDMap[uint32(vid)] = true
 			} else {
 				return fmt.Errorf("invalid volume ID %q", vids)
 			}
 		}
 	}
 
-	runner.dataNodes, err = collectDataNodes(commandEnv, 0)
+	dataNodes, err := collectDataNodes(commandEnv, 0)
 	if err != nil {
 		return err
 	}
 
-	return runner.checkEcVolumes(*showDetails)
+	return ec.CheckEcVolumeReplication(writer, dataNodes, volumeIDMap, *showDetails)
 }
 
 // ecCheckReplicationRunner holds the state for a single ec.check.replication invocation.
-type ecCheckReplicationRunner struct {
-	writer      io.Writer
-	dataNodes   []*master_pb.DataNodeInfo
-	volumeIDMap map[uint32]bool
-}
-
-func (r *ecCheckReplicationRunner) write(format string, a ...any) {
-	fmt.Fprintf(r.writer, format, a...)
-}
-
-func (r *ecCheckReplicationRunner) isVolumeIDValid(vid uint32) bool {
-	if len(r.volumeIDMap) == 0 {
-		return true
-	}
-	return r.volumeIDMap[vid]
-}
-
-// ecVolumeShardReplication aggregates the observed shards for a single EC volume,
-// together with the data+parity ratio the volume was encoded with. The ratio is
-// taken per volume (via erasure_coding.EcShardsVolume*Shards) so custom EC
-// ratios are checked against their own expected shard count.
-type ecVolumeShardReplication struct {
-	dataShards   int
-	parityShards int
-	// shardAddresses maps a shard id to the sorted node addresses hosting it.
-	shardAddresses map[erasure_coding.ShardId][]string
-}
-
-func (h *ecVolumeShardReplication) totalShards() int {
-	return h.dataShards + h.parityShards
-}
-
-// replicaCount is the total number of shard copies observed, counting each
-// over-replicated shard once per hosting node.
-func (h *ecVolumeShardReplication) replicaCount() int {
-	n := 0
-	for _, addrs := range h.shardAddresses {
-		n += len(addrs)
-	}
-	return n
-}
-
-// unexpectedShardIds returns, in ascending order, any observed shard ids that
-// fall outside the volume's expected data+parity range. These are anomalies
-// (e.g. a stray shard) and are treated as over-replication.
-func (h *ecVolumeShardReplication) unexpectedShardIds() []erasure_coding.ShardId {
-	var ids []erasure_coding.ShardId
-	for sid := range h.shardAddresses {
-		if int(sid) >= h.totalShards() {
-			ids = append(ids, sid)
-		}
-	}
-	slices.Sort(ids)
-	return ids
-}
-
-// TODO: check shard sizes?
-func (r *ecCheckReplicationRunner) checkEcVolumes(showDetails bool) error {
-	// collect EC shard placement, keyed by volume id
-	volumes := map[uint32]*ecVolumeShardReplication{}
-	for _, dni := range r.dataNodes {
-		nodeAddress := dni.GetAddress()
-		for _, di := range dni.GetDiskInfos() {
-			for _, eci := range di.GetEcShardInfos() {
-				vid := eci.GetId()
-				if !r.isVolumeIDValid(vid) {
-					continue
-				}
-
-				h, ok := volumes[vid]
-				if !ok {
-					// all shards of a volume share one ratio; take it from the
-					// first shard message seen for the volume.
-					h = &ecVolumeShardReplication{
-						dataShards:     erasure_coding.EcShardsVolumeDataShards(eci),
-						parityShards:   erasure_coding.EcShardsVolumeParityShards(eci),
-						shardAddresses: map[erasure_coding.ShardId][]string{},
-					}
-					volumes[vid] = h
-				}
-
-				sinfo := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(eci)
-				for _, sid := range sinfo.Ids() {
-					h.shardAddresses[sid] = append(h.shardAddresses[sid], nodeAddress)
-				}
-			}
-		}
-	}
-
-	if len(volumes) == 0 {
-		// Asking about specific volume IDs that turn out not to be EC volumes is
-		// an error; an unfiltered run over a cluster with no EC volumes is a
-		// legitimate, healthy state rather than a failure.
-		if len(r.volumeIDMap) > 0 {
-			return fmt.Errorf("no EC volumes found")
-		}
-		r.write("No EC volumes found.\n")
-		return nil
-	}
-
-	// keep the shard address lists nicely sorted, for the sake of readability.
-	for _, h := range volumes {
-		for _, addrs := range h.shardAddresses {
-			slices.Sort(addrs)
-		}
-	}
-
-	// classify each volume against its own expected shard count
-	underreplicatedVolumeIDs := []uint32{}
-	overreplicatedVolumeIDs := []uint32{}
-	for vid, h := range volumes {
-		under := false
-		over := false
-		for sid := 0; sid < h.totalShards(); sid++ {
-			switch len(h.shardAddresses[erasure_coding.ShardId(sid)]) {
-			case 0:
-				under = true
-			case 1:
-			default:
-				over = true
-			}
-		}
-		// shard ids beyond the expected data+parity range are unexpected extras,
-		// i.e. redundant data the ratio doesn't call for.
-		if len(h.unexpectedShardIds()) > 0 {
-			over = true
-		}
-
-		// under- and over-replication are independent problems (missing shards risk
-		// data loss, redundant shards waste space), so a volume exhibiting both is
-		// reported in both lists.
-		if under {
-			underreplicatedVolumeIDs = append(underreplicatedVolumeIDs, vid)
-		}
-		if over {
-			overreplicatedVolumeIDs = append(overreplicatedVolumeIDs, vid)
-		}
-	}
-	slices.Sort(underreplicatedVolumeIDs)
-	slices.Sort(overreplicatedVolumeIDs)
-
-	// ...and display results
-	if len(underreplicatedVolumeIDs) == 0 && len(overreplicatedVolumeIDs) == 0 {
-		r.write("EC volumes are healthy.\n")
-		return nil
-	}
-
-	if len(underreplicatedVolumeIDs) != 0 {
-		r.write("Found %d/%d under-replicated EC volumes: %v\n", len(underreplicatedVolumeIDs), len(volumes), underreplicatedVolumeIDs)
-	}
-	if len(overreplicatedVolumeIDs) != 0 {
-		r.write("Found %d/%d over-replicated EC volumes: %v\n", len(overreplicatedVolumeIDs), len(volumes), overreplicatedVolumeIDs)
-	}
-
-	if showDetails {
-		if len(underreplicatedVolumeIDs) != 0 {
-			r.write("\n")
-			r.writeShardMaps("under-replicated", underreplicatedVolumeIDs, volumes)
-		}
-		if len(overreplicatedVolumeIDs) != 0 {
-			r.write("\n")
-			r.writeShardMaps("over-replicated", overreplicatedVolumeIDs, volumes)
-		}
-	}
-
-	return nil
-}
-
-func (r *ecCheckReplicationRunner) writeShardMaps(kind string, volumeIDs []uint32, volumes map[uint32]*ecVolumeShardReplication) {
-	for _, vid := range volumeIDs {
-		h := volumes[vid]
-		r.write("Shards map for %s EC volume %v (%d/%d shards):\n", kind, vid, h.replicaCount(), h.totalShards())
-		for sid := 0; sid < h.totalShards(); sid++ {
-			shardTypeDesc := ""
-			if sid >= h.dataShards {
-				shardTypeDesc = " (parity)"
-			}
-			if addrs, ok := h.shardAddresses[erasure_coding.ShardId(sid)]; ok {
-				r.write("\t%02d%s => %v\n", sid, shardTypeDesc, addrs)
-			} else {
-				r.write("\t%02d%s is missing\n", sid, shardTypeDesc)
-			}
-		}
-		for _, sid := range h.unexpectedShardIds() {
-			r.write("\t%02d (unexpected) => %v\n", int(sid), h.shardAddresses[sid])
-		}
-	}
-}

weed/shell/command_ec_decode.go

@@ -5,21 +5,11 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"strings"
 
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"github.com/seaweedfs/seaweedfs/weed/storage/types"
-
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
-
-	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/ec"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 )
 
 func init() {
@@ -82,27 +72,29 @@ func (c *commandEcDecode) Do(args []string, commandEnv *CommandEnv, writer io.Wr
 
 	vid := needle.VolumeId(*volumeId)
 	diskType := types.ToDiskType(*diskTypeStr)
+	env := commandEnv.ecEnv()
 
 	// collect topology information
 	topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
 	if err != nil {
 		return err
 	}
-	var diskUsageState *decodeDiskUsageState
+	var diskUsageState *ec.DecodeDiskUsageState
 	if *checkMinFreeSpace {
-		diskUsageState = newDecodeDiskUsageState(topologyInfo, diskType)
+		diskUsageState = ec.NewDecodeDiskUsageState(topologyInfo, diskType)
 	}
 
 	// volumeId is provided
 	if vid != 0 {
-		return doEcDecode(commandEnv, topologyInfo, *collection, vid, diskType, *checkMinFreeSpace, diskUsageState)
+		return ec.DoEcDecode(env, topologyInfo, *collection, vid, diskType, *checkMinFreeSpace, diskUsageState)
 	}
 
 	// apply to all volumes in the collection
-	volumeIds, err := collectEcShardIds(topologyInfo, *collection, diskType)
+	collectionRegex, err := compileCollectionPattern(*collection)
 	if err != nil {
-		return err
+		return fmt.Errorf("invalid collection pattern '%s': %v", *collection, err)
 	}
+	volumeIds := ec.CollectEcShardIds(topologyInfo, collectionRegex, diskType)
 	fmt.Printf("ec decode volumes: %v\n", volumeIds)
 	batches := chunkVolumeIds(volumeIds, *batchSize)
 	for i, batch := range batches {
@@ -114,14 +106,14 @@ func (c *commandEcDecode) Do(args []string, commandEnv *CommandEnv, writer io.Wr
 				return err
 			}
 			if *checkMinFreeSpace {
-				diskUsageState = newDecodeDiskUsageState(topologyInfo, diskType)
+				diskUsageState = ec.NewDecodeDiskUsageState(topologyInfo, diskType)
 			}
 		}
 		if len(batches) > 1 {
 			fmt.Printf("ec decode batch %d/%d: %v\n", i+1, len(batches), batch)
 		}
 		for _, vid := range batch {
-			if err = doEcDecode(commandEnv, topologyInfo, *collection, vid, diskType, *checkMinFreeSpace, diskUsageState); err != nil {
+			if err = ec.DoEcDecode(env, topologyInfo, *collection, vid, diskType, *checkMinFreeSpace, diskUsageState); err != nil {
 				return err
 			}
 		}
@@ -130,262 +122,6 @@ func (c *commandEcDecode) Do(args []string, commandEnv *CommandEnv, writer io.Wr
 	return nil
 }
 
-func doEcDecode(commandEnv *CommandEnv, topoInfo *master_pb.TopologyInfo, collection string, vid needle.VolumeId, diskType types.DiskType, checkMinFreeSpace bool, diskUsageState *decodeDiskUsageState) (err error) {
-
-	if !commandEnv.isLocked() {
-		return fmt.Errorf("lock is lost")
-	}
-
-	// find volume location
-	nodeToEcShardsInfo, dataShards := collectEcNodeShardsInfo(topoInfo, vid, diskType)
-
-	fmt.Printf("ec volume %d shard locations: %+v\n", vid, nodeToEcShardsInfo)
-
-	if len(nodeToEcShardsInfo) == 0 {
-		return fmt.Errorf("no EC shards found for volume %d (diskType %s)", vid, diskType.ReadableString())
-	}
-
-	var originalShardCounts map[pb.ServerAddress]int
-	if diskUsageState != nil {
-		originalShardCounts = make(map[pb.ServerAddress]int, len(nodeToEcShardsInfo))
-		for location, si := range nodeToEcShardsInfo {
-			originalShardCounts[location] = si.Count()
-		}
-	}
-
-	var eligibleTargets map[pb.ServerAddress]struct{}
-	if checkMinFreeSpace {
-		if diskUsageState == nil {
-			return fmt.Errorf("min free space checking requires disk usage state")
-		}
-		eligibleTargets = make(map[pb.ServerAddress]struct{})
-		for location := range nodeToEcShardsInfo {
-			if freeCount, found := diskUsageState.freeVolumeCount(location); found && freeCount > 0 {
-				eligibleTargets[location] = struct{}{}
-			}
-		}
-		if len(eligibleTargets) == 0 {
-			return fmt.Errorf("no eligible target datanodes with free volume slots for volume %d (diskType %s); use -checkMinFreeSpace=false to override", vid, diskType.ReadableString())
-		}
-	}
-
-	// collect ec shards to the server with most space
-	targetNodeLocation, err := collectEcShards(commandEnv, nodeToEcShardsInfo, collection, vid, eligibleTargets, dataShards)
-	if err != nil {
-		return fmt.Errorf("collectEcShards for volume %d: %v", vid, err)
-	}
-
-	// generate a normal volume
-	err = generateNormalVolume(commandEnv.option.GrpcDialOption, vid, collection, targetNodeLocation)
-	if err != nil {
-		// Special case: if the EC index has no live entries, decoding is a no-op.
-		// Just purge EC shards and return success without generating/mounting an empty volume.
-		if isEcDecodeEmptyVolumeErr(err) {
-			if err := unmountAndDeleteEcShards(commandEnv.option.GrpcDialOption, collection, nodeToEcShardsInfo, vid); err != nil {
-				return err
-			}
-			if diskUsageState != nil {
-				diskUsageState.applyDecode(targetNodeLocation, originalShardCounts, false)
-			}
-			return nil
-		}
-		return fmt.Errorf("generate normal volume %d on %s: %v", vid, targetNodeLocation, err)
-	}
-
-	// mount the decoded volume after server-side offline compaction succeeded
-	err = mountDecodedVolume(commandEnv.option.GrpcDialOption, targetNodeLocation, vid)
-	if err != nil {
-		return fmt.Errorf("mount decoded volume %d on %s: %v", vid, targetNodeLocation, err)
-	}
-
-	// Confirm the regenerated .dat is present and non-empty before destroying
-	// the shards. Without this gate, a silent failure in generate/mount could
-	// leave the cluster with neither shards nor volume.
-	if err := verifyDecodedVolumeBeforeDelete(commandEnv.option.GrpcDialOption, targetNodeLocation, vid); err != nil {
-		return fmt.Errorf("verify decoded volume %d on %s before deleting shards: %w", vid, targetNodeLocation, err)
-	}
-
-	// delete the previous ec shards
-	err = unmountAndDeleteEcShardsWithPrefix("deleteDecodedEcShards", commandEnv.option.GrpcDialOption, collection, nodeToEcShardsInfo, vid)
-	if err != nil {
-		return fmt.Errorf("delete ec shards for volume %d: %v", vid, err)
-	}
-	if diskUsageState != nil {
-		diskUsageState.applyDecode(targetNodeLocation, originalShardCounts, true)
-	}
-
-	return nil
-}
-
-func isEcDecodeEmptyVolumeErr(err error) bool {
-	st, ok := status.FromError(err)
-	if !ok {
-		return false
-	}
-	if st.Code() != codes.FailedPrecondition {
-		return false
-	}
-	// Keep this robust against wording tweaks while still being specific.
-	return strings.Contains(st.Message(), erasure_coding.EcNoLiveEntriesSubstring)
-}
-
-func unmountAndDeleteEcShards(grpcDialOption grpc.DialOption, collection string, nodeToShardsInfo map[pb.ServerAddress]*erasure_coding.ShardsInfo, vid needle.VolumeId) error {
-	return unmountAndDeleteEcShardsWithPrefix("unmountAndDeleteEcShards", grpcDialOption, collection, nodeToShardsInfo, vid)
-}
-
-func unmountAndDeleteEcShardsWithPrefix(prefix string, grpcDialOption grpc.DialOption, collection string, nodeToShardsInfo map[pb.ServerAddress]*erasure_coding.ShardsInfo, vid needle.VolumeId) error {
-	ewg := NewErrorWaitGroup(len(nodeToShardsInfo))
-
-	// unmount and delete ec shards in parallel (one goroutine per location)
-	for location, si := range nodeToShardsInfo {
-		location, si := location, si // capture loop variables for goroutine
-		ewg.Add(func() error {
-			fmt.Printf("unmount ec volume %d on %s has shards: %+v\n", vid, location, si.Ids())
-			if err := unmountEcShards(grpcDialOption, vid, location, si.Ids()); err != nil {
-				return fmt.Errorf("%s unmount ec volume %d on %s: %w", prefix, vid, location, err)
-			}
-
-			fmt.Printf("delete ec volume %d on %s has shards: %+v\n", vid, location, si.Ids())
-			if err := sourceServerDeleteEcShards(grpcDialOption, collection, vid, location, si.Ids()); err != nil {
-				return fmt.Errorf("%s delete ec volume %d on %s: %w", prefix, vid, location, err)
-			}
-			return nil
-		})
-	}
-	return ewg.Wait()
-}
-
-func verifyDecodedVolumeBeforeDelete(grpcDialOption grpc.DialOption, target pb.ServerAddress, vid needle.VolumeId) error {
-	var resp *volume_server_pb.ReadVolumeFileStatusResponse
-	if err := operation.WithVolumeServerClient(false, target, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
-		r, e := client.ReadVolumeFileStatus(context.Background(), &volume_server_pb.ReadVolumeFileStatusRequest{
-			VolumeId: uint32(vid),
-		})
-		if e != nil {
-			return e
-		}
-		resp = r
-		return nil
-	}); err != nil {
-		return fmt.Errorf("read volume file status: %w", err)
-	}
-	if resp.DatFileSize == 0 {
-		return fmt.Errorf("decoded .dat is 0 bytes")
-	}
-	if resp.IdxFileSize == 0 {
-		return fmt.Errorf("decoded .idx is 0 bytes")
-	}
-	glog.V(0).Infof("ec decode verification ok for volume %d on %s: dat=%d idx=%d", vid, target, resp.DatFileSize, resp.IdxFileSize)
-	return nil
-}
-
-func mountDecodedVolume(grpcDialOption grpc.DialOption, targetNodeLocation pb.ServerAddress, vid needle.VolumeId) error {
-	return operation.WithVolumeServerClient(false, targetNodeLocation, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
-		_, mountErr := volumeServerClient.VolumeMount(context.Background(), &volume_server_pb.VolumeMountRequest{
-			VolumeId: uint32(vid),
-		})
-		return mountErr
-	})
-}
-
-func generateNormalVolume(grpcDialOption grpc.DialOption, vid needle.VolumeId, collection string, sourceVolumeServer pb.ServerAddress) error {
-	fmt.Printf("generateNormalVolume from ec volume %d on %s\n", vid, sourceVolumeServer)
-
-	err := operation.WithVolumeServerClient(false, sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
-		_, genErr := volumeServerClient.VolumeEcShardsToVolume(context.Background(), &volume_server_pb.VolumeEcShardsToVolumeRequest{
-			VolumeId:   uint32(vid),
-			Collection: collection,
-		})
-		return genErr
-	})
-
-	return err
-
-}
-
-func collectEcShards(commandEnv *CommandEnv, nodeToShardsInfo map[pb.ServerAddress]*erasure_coding.ShardsInfo, collection string, vid needle.VolumeId, eligibleTargets map[pb.ServerAddress]struct{}, dataShards int) (targetNodeLocation pb.ServerAddress, err error) {
-
-	maxShardCount := -1
-	existingShardsInfo := erasure_coding.NewShardsInfo()
-	for loc, si := range nodeToShardsInfo {
-		if eligibleTargets != nil {
-			if _, ok := eligibleTargets[loc]; !ok {
-				continue
-			}
-		}
-		toBeCopiedShardCount := si.MinusParityShards(dataShards).Count()
-		if toBeCopiedShardCount > maxShardCount {
-			maxShardCount = toBeCopiedShardCount
-			targetNodeLocation = loc
-			existingShardsInfo = si
-		}
-	}
-	if targetNodeLocation == "" {
-		return "", fmt.Errorf("no eligible target datanodes available to decode volume %d", vid)
-	}
-
-	fmt.Printf("collectEcShards: ec volume %d collect shards to %s from: %+v\n", vid, targetNodeLocation, nodeToShardsInfo)
-
-	copiedShardsInfo := erasure_coding.NewShardsInfo()
-	for loc, si := range nodeToShardsInfo {
-		if loc == targetNodeLocation {
-			continue
-		}
-
-		needToCopyShardsInfo := si.Minus(existingShardsInfo).MinusParityShards(dataShards)
-
-		err = operation.WithVolumeServerClient(false, targetNodeLocation, commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
-
-			// Always collect .ecj from every shard location. Each server's .ecj
-			// only contains deletions for needles whose data resides in shards
-			// held by that server. Without merging all .ecj files, deletions
-			// recorded on other servers would be lost during decode.
-			if needToCopyShardsInfo.Count() > 0 {
-				fmt.Printf("copy %d.%v %s => %s\n", vid, needToCopyShardsInfo.Ids(), loc, targetNodeLocation)
-			} else {
-				fmt.Printf("collect ecj %d %s => %s\n", vid, loc, targetNodeLocation)
-			}
-
-			_, copyErr := volumeServerClient.VolumeEcShardsCopy(context.Background(), &volume_server_pb.VolumeEcShardsCopyRequest{
-				VolumeId:       uint32(vid),
-				Collection:     collection,
-				ShardIds:       needToCopyShardsInfo.IdsUint32(),
-				CopyEcxFile:    false,
-				CopyEcjFile:    true,
-				CopyVifFile:    needToCopyShardsInfo.Count() > 0,
-				SourceDataNode: string(loc),
-			})
-			if copyErr != nil {
-				return fmt.Errorf("copy %d.%v %s => %s : %v\n", vid, needToCopyShardsInfo.Ids(), loc, targetNodeLocation, copyErr)
-			}
-
-			if needToCopyShardsInfo.Count() > 0 {
-				fmt.Printf("mount %d.%v on %s\n", vid, needToCopyShardsInfo.Ids(), targetNodeLocation)
-				_, mountErr := volumeServerClient.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{
-					VolumeId:   uint32(vid),
-					Collection: collection,
-					ShardIds:   needToCopyShardsInfo.IdsUint32(),
-				})
-				if mountErr != nil {
-					return fmt.Errorf("mount %d.%v on %s : %v\n", vid, needToCopyShardsInfo.Ids(), targetNodeLocation, mountErr)
-				}
-			}
-
-			return nil
-		})
-
-		if err != nil {
-			break
-		}
-
-		copiedShardsInfo.Add(needToCopyShardsInfo)
-	}
-
-	nodeToShardsInfo[targetNodeLocation] = existingShardsInfo.Plus(copiedShardsInfo)
-
-	return targetNodeLocation, err
-}
-
 func lookupVolumeIds(commandEnv *CommandEnv, volumeIds []string) (volumeIdLocations []*master_pb.LookupVolumeResponse_VolumeIdLocation, err error) {
 	var resp *master_pb.LookupVolumeResponse
 	err = commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error {
@@ -397,109 +133,3 @@ func lookupVolumeIds(commandEnv *CommandEnv, volumeIds []string) (volumeIdLocati
 	}
 	return resp.VolumeIdLocations, nil
 }
-
-func collectEcShardIds(topoInfo *master_pb.TopologyInfo, collectionPattern string, diskType types.DiskType) (vids []needle.VolumeId, err error) {
-	// compile regex pattern for collection matching
-	collectionRegex, err := compileCollectionPattern(collectionPattern)
-	if err != nil {
-		return nil, fmt.Errorf("invalid collection pattern '%s': %v", collectionPattern, err)
-	}
-
-	vidMap := make(map[uint32]bool)
-	eachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
-		if diskInfo, found := dn.DiskInfos[string(diskType)]; found {
-			for _, v := range diskInfo.EcShardInfos {
-				if collectionRegex.MatchString(v.Collection) {
-					vidMap[v.Id] = true
-				}
-			}
-		}
-	})
-
-	for vid := range vidMap {
-		vids = append(vids, needle.VolumeId(vid))
-	}
-
-	return
-}
-
-func collectEcNodeShardsInfo(topoInfo *master_pb.TopologyInfo, vid needle.VolumeId, diskType types.DiskType) (map[pb.ServerAddress]*erasure_coding.ShardsInfo, int) {
-	res := make(map[pb.ServerAddress]*erasure_coding.ShardsInfo)
-	eachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
-		if diskInfo, found := dn.DiskInfos[string(diskType)]; found {
-			// A node may report several EcShardInfos for one volume — one per
-			// physical disk holding shards of it (multi-disk nodes). Union them
-			// rather than overwriting, or only the last disk's shards survive and
-			// the node looks like it is missing shards it actually has.
-			for _, v := range diskInfo.EcShardInfos {
-				if v.Id == uint32(vid) {
-					addr := pb.NewServerAddressFromDataNode(dn)
-					si := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(v)
-					if existing, ok := res[addr]; ok {
-						existing.Add(si)
-					} else {
-						res[addr] = si
-					}
-				}
-			}
-		}
-	})
-
-	// OSS is always 10+4; the per-volume ratio override lives in the enterprise build.
-	return res, erasure_coding.DataShardsCount
-}
-
-type decodeDiskUsageState struct {
-	byNode map[pb.ServerAddress]*decodeDiskUsageCounts
-}
-
-type decodeDiskUsageCounts struct {
-	maxVolumeCount    int64
-	volumeCount       int64
-	remoteVolumeCount int64
-	ecShardCount      int64
-}
-
-func newDecodeDiskUsageState(topoInfo *master_pb.TopologyInfo, diskType types.DiskType) *decodeDiskUsageState {
-	state := &decodeDiskUsageState{byNode: make(map[pb.ServerAddress]*decodeDiskUsageCounts)}
-	eachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
-		if diskInfo, found := dn.DiskInfos[string(diskType)]; found {
-			state.byNode[pb.NewServerAddressFromDataNode(dn)] = &decodeDiskUsageCounts{
-				maxVolumeCount:    diskInfo.MaxVolumeCount,
-				volumeCount:       diskInfo.VolumeCount,
-				remoteVolumeCount: diskInfo.RemoteVolumeCount,
-				ecShardCount:      int64(countShards(diskInfo.EcShardInfos)),
-			}
-		}
-	})
-	return state
-}
-
-func (state *decodeDiskUsageState) freeVolumeCount(location pb.ServerAddress) (int64, bool) {
-	if state == nil {
-		return 0, false
-	}
-	usage, found := state.byNode[location]
-	if !found {
-		return 0, false
-	}
-	free := usage.maxVolumeCount - (usage.volumeCount - usage.remoteVolumeCount)
-	free -= (usage.ecShardCount + int64(erasure_coding.DataShardsCount) - 1) / int64(erasure_coding.DataShardsCount)
-	return free, true
-}
-
-func (state *decodeDiskUsageState) applyDecode(targetNodeLocation pb.ServerAddress, shardCounts map[pb.ServerAddress]int, createdVolume bool) {
-	if state == nil {
-		return
-	}
-	for location, shardCount := range shardCounts {
-		if usage, found := state.byNode[location]; found {
-			usage.ecShardCount -= int64(shardCount)
-		}
-	}
-	if createdVolume {
-		if usage, found := state.byNode[targetNodeLocation]; found {
-			usage.volumeCount++
-		}
-	}
-}

weed/shell/command_ec_encode.go

@@ -1,36 +1,19 @@
 package shell
 
 import (
-	"context"
-	"errors"
 	"flag"
 	"fmt"
 	"io"
-	"regexp"
-	"slices"
-	"sort"
-	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 
+	"github.com/seaweedfs/seaweedfs/weed/ec"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"github.com/seaweedfs/seaweedfs/weed/wdclient"
-
-	"google.golang.org/grpc"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
-
-	"github.com/seaweedfs/seaweedfs/weed/operation"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
-	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
-	"github.com/seaweedfs/seaweedfs/weed/storage/volume_replica"
 )
 
 func init() {
@@ -183,7 +166,7 @@ func (c *commandEcEncode) Do(args []string, commandEnv *CommandEnv, writer io.Wr
 		return fmt.Errorf("-batchSize must be >= 0")
 	}
 
-	batches := chunkVolumeIds(volumeIds, *batchSize)
+	batches := ec.ChunkVolumeIds(volumeIds, *batchSize)
 	if len(batches) > 1 {
 		fmt.Printf("Processing %d volumes in %d batch(es), batchSize=%d\n", len(volumeIds), len(batches), *batchSize)
 	}
@@ -191,7 +174,7 @@ func (c *commandEcEncode) Do(args []string, commandEnv *CommandEnv, writer io.Wr
 		if len(batches) > 1 {
 			fmt.Printf("Starting EC encoding batch %d/%d with %d volumes: %v\n", i+1, len(batches), len(batchVolumeIds), batchVolumeIds)
 		}
-		if err := processEcEncodeBatch(commandEnv, writer, batchVolumeIds, rp, diskType, *maxParallelization, *applyBalancing, *collection); err != nil {
+		if err := ec.ProcessEcEncodeBatch(commandEnv.ecEnv(), writer, batchVolumeIds, rp, diskType, *maxParallelization, *applyBalancing, *collection); err != nil {
 			return fmt.Errorf("ec encode batch %d/%d for volumes %v: %w", i+1, len(batches), batchVolumeIds, err)
 		}
 	}
@@ -201,790 +184,6 @@ func (c *commandEcEncode) Do(args []string, commandEnv *CommandEnv, writer io.Wr
 
 	return nil
 }
-
-func chunkVolumeIds(volumeIds []needle.VolumeId, batchSize int) [][]needle.VolumeId {
-	if batchSize <= 0 || len(volumeIds) == 0 {
-		return [][]needle.VolumeId{volumeIds}
-	}
-	var batches [][]needle.VolumeId
-	for start := 0; start < len(volumeIds); start += batchSize {
-		end := start + batchSize
-		if end > len(volumeIds) {
-			end = len(volumeIds)
-		}
-		batches = append(batches, volumeIds[start:end])
-	}
-	return batches
-}
-
-func processEcEncodeBatch(commandEnv *CommandEnv, writer io.Writer, volumeIds []needle.VolumeId, rp *super_block.ReplicaPlacement, diskType types.DiskType, maxParallelization int, applyBalancing bool, collectionForMessage string) (err error) {
-	topologyInfo, _, err := collectTopologyInfo(commandEnv, 0)
-	if err != nil {
-		return err
-	}
-
-	// Refuse to encode a volume that is already EC (present only as shards):
-	// an EC volume has no .dat, so re-encoding it would tear down its only
-	// copy before failing. A regular volume (with a .dat) passes. This closes
-	// the operator-rerun / script-retry path; a worker racing the snapshot is
-	// handled by encode fencing, not here.
-	if err := assertEncodableRegularVolumes(topologyInfo, volumeIds); err != nil {
-		return err
-	}
-
-	volumeIdToCollection := collectVolumeIdToCollection(topologyInfo, volumeIds)
-	balanceCollections := collectCollectionsForVolumeIds(topologyInfo, volumeIds)
-
-	fmt.Printf("Collecting volume locations for %d volumes before EC encoding...\n", len(volumeIds))
-	volumeLocationsMap, err := volumeLocations(commandEnv, volumeIds)
-	if err != nil {
-		return fmt.Errorf("failed to collect volume locations before EC encoding: %w", err)
-	}
-
-	if err := checkEcEncodeCapacity(topologyInfo, len(volumeIds), diskType, collectionForMessage); err != nil {
-		return err
-	}
-
-	// From here doEcEncode marks the volumes readonly and generates EC shards.
-	// If any step before the originals are deleted fails, roll the encode back:
-	// tear down the shards produced this run and restore the sources to writable,
-	// so a failed (and possibly abandoned) ec.encode does not strand volumes
-	// readonly or leave orphan EC shards behind. Once the shards are verified
-	// recoverable we are committed to the EC copy and must not roll back.
-	committed := false
-	defer func() {
-		if err != nil && !committed {
-			rollbackFailedEcEncode(commandEnv, writer, volumeIds, volumeIdToCollection, volumeLocationsMap, maxParallelization)
-		}
-	}()
-
-	skippedNodes, err := doEcEncode(commandEnv, writer, volumeIdToCollection, volumeIds, maxParallelization, topologyInfo)
-	if err != nil {
-		return fmt.Errorf("ec encode for volumes %v: %w", volumeIds, err)
-	}
-	// Mounting the new shards notifies the master asynchronously, and EcBalance
-	// plans from a fresh topology snapshot: one taken before the mounts land
-	// shows no shards for these volumes, so the balance plans no moves and
-	// silently leaves every shard on the generation host.
-	if err := waitForEcShardsToRegister(commandEnv, volumeIds); err != nil {
-		return fmt.Errorf("wait for ec shards to register with the master: %w", err)
-	}
-	// EcBalance works at collection scope. In batch mode this intentionally
-	// rebalances each collection after every batch so source volumes can be
-	// safely verified and deleted without waiting for all batches to finish.
-	// skippedNodes are excluded so a recovered node's stale orphan is never
-	// paired with a new-generation shard.
-	if err := EcBalance(commandEnv, balanceCollections, "", rp, diskType, maxParallelization, 0, applyBalancing, skippedNodes, nil); err != nil {
-		return fmt.Errorf("re-balance ec shards for collection(s) %v: %w", balanceCollections, err)
-	}
-	if err := verifyEcShardsBeforeDelete(commandEnv, volumeIds, diskType, applyBalancing); err != nil {
-		return fmt.Errorf("verify EC shards before deleting originals: %w", err)
-	}
-	// Past verify the EC copy is recoverable; a delete failure below must not
-	// tear the shards down.
-	committed = true
-	fmt.Printf("Deleting original volumes after EC encoding...\n")
-	if err := doDeleteVolumesWithLocations(commandEnv, volumeIds, volumeLocationsMap, maxParallelization); err != nil {
-		return fmt.Errorf("delete original volumes after EC encoding: %w", err)
-	}
-	fmt.Printf("Successfully completed EC encoding for %d volumes\n", len(volumeIds))
-	return nil
-}
-
-// rollbackFailedEcEncode is a best-effort cleanup for an ec.encode that failed
-// after marking volumes readonly / generating shards but before the originals
-// were deleted. It tears down the EC shards this run produced (so they do not
-// survive as orphans until the next encode) and restores the source volumes to
-// writable (so a failed and possibly abandoned encode does not strand them
-// readonly). Errors are logged, not returned — we are already on the failure
-// path. Both operations are idempotent: clearPreexistingEcShards is a no-op when
-// no shards were generated, and marking an already-writable volume is a no-op,
-// so it is safe even for a failure before the volumes were marked readonly.
-func rollbackFailedEcEncode(commandEnv *CommandEnv, writer io.Writer, volumeIds []needle.VolumeId, volumeIdToCollection map[needle.VolumeId]string, volumeLocationsMap map[needle.VolumeId][]wdclient.Location, maxParallelization int) {
-	fmt.Fprintf(writer, "rolling back failed EC encode for volumes %v...\n", volumeIds)
-
-	// Tear down any EC shards this run produced. A fresh topology snapshot finds
-	// them wherever generate/balance left them; the teardown is blanket.
-	if topologyInfo, _, err := collectTopologyInfo(commandEnv, 0); err != nil {
-		fmt.Fprintf(writer, "rollback: collect topology to clear ec shards: %v\n", err)
-	} else if _, err := clearPreexistingEcShards(commandEnv, topologyInfo, volumeIds, volumeIdToCollection, maxParallelization); err != nil {
-		fmt.Fprintf(writer, "rollback: clear ec shards: %v\n", err)
-	}
-
-	// Restore the source volumes to writable. doEcEncode re-reads the locations
-	// and marks every replica of that later snapshot readonly, so re-read here
-	// too: a replica added or moved between the batch's initial snapshot
-	// (volumeLocationsMap) and doEcEncode's readonly-marking would otherwise be
-	// left readonly. Fall back to the initial snapshot if the re-read fails.
-	locations := volumeLocationsMap
-	if fresh, err := volumeLocations(commandEnv, volumeIds); err != nil {
-		fmt.Fprintf(writer, "rollback: re-read volume locations (using pre-encode snapshot): %v\n", err)
-	} else {
-		locations = fresh
-	}
-	ewg := NewErrorWaitGroup(maxParallelization)
-	for _, vid := range volumeIds {
-		for _, l := range locations[vid] {
-			ewg.Add(func() error {
-				if err := markVolumeReplicaWritable(context.Background(), commandEnv.option.GrpcDialOption, vid, l, true, false); err != nil {
-					return fmt.Errorf("restore volume %d writable on %s: %w", vid, l.Url, err)
-				}
-				return nil
-			})
-		}
-	}
-	if err := ewg.Wait(); err != nil {
-		fmt.Fprintf(writer, "rollback: %v\n", err)
-	}
-}
-
-func checkEcEncodeCapacity(topologyInfo *master_pb.TopologyInfo, volumeCount int, diskType types.DiskType, collectionForMessage string) error {
-	// Pre-flight check: verify the target disk type has capacity for EC shards.
-	// This prevents encoding shards only to fail during rebalance. Reuse the
-	// caller's topology snapshot instead of issuing another VolumeList to the
-	// master per batch.
-	_, totalFreeEcSlots := collectEcVolumeServersByDc(topologyInfo, "", diskType)
-
-	// Each volume needs TotalShardsCount (14) shards distributed.
-	requiredSlots := volumeCount * erasure_coding.TotalShardsCount
-	if totalFreeEcSlots < 1 {
-		if diskType != types.HardDriveType {
-			tryDiskTypeMessage := "Try passing -diskType=hdd, or omit -diskType to use the default (hdd)"
-			if collectionForMessage != "" {
-				tryDiskTypeMessage = fmt.Sprintf("Try:\n  ec.encode -collection=%s -diskType=hdd\nOr omit -diskType to use the default (hdd)", collectionForMessage)
-			}
-			return fmt.Errorf("no free ec shard slots on disk type '%s'. The target disk type has no capacity.\n"+
-				"Your volumes are likely on a different disk type. %s", diskType, tryDiskTypeMessage)
-		}
-		return fmt.Errorf("no free ec shard slots. only %d left on disk type '%s'", totalFreeEcSlots, diskType)
-	}
-
-	if totalFreeEcSlots < requiredSlots {
-		fmt.Printf("Warning: limited EC shard capacity. Need %d slots for %d volumes, but only %d slots available on disk type '%s'.\n",
-			requiredSlots, volumeCount, totalFreeEcSlots, diskType)
-		fmt.Printf("Rebalancing may not achieve optimal distribution.\n")
-	}
-	return nil
-}
-
-func volumeLocations(commandEnv *CommandEnv, volumeIds []needle.VolumeId) (map[needle.VolumeId][]wdclient.Location, error) {
-	res := map[needle.VolumeId][]wdclient.Location{}
-	for _, vid := range volumeIds {
-		ls, ok := commandEnv.MasterClient.GetLocationsClone(uint32(vid))
-		if !ok {
-			return nil, fmt.Errorf("volume %d not found", vid)
-		}
-		res[vid] = ls
-	}
-
-	return res, nil
-}
-
-func doEcEncode(commandEnv *CommandEnv, writer io.Writer, volumeIdToCollection map[needle.VolumeId]string, volumeIds []needle.VolumeId, maxParallelization int, topologyInfo *master_pb.TopologyInfo) (skippedNodes map[pb.ServerAddress]struct{}, err error) {
-	if !commandEnv.isLocked() {
-		return nil, fmt.Errorf("lock is lost")
-	}
-	locations, err := volumeLocations(commandEnv, volumeIds)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get volume locations for EC encoding: %w", err)
-	}
-
-	// Clear EC shards left by a previous failed/partial encode so a retry
-	// starts clean and never mixes two encode runs. A node skipped here as
-	// unreachable is excluded from the later balance: it may still hold a stale
-	// orphan that, paired with a new-generation shard from a balance copy, would
-	// mix generations on that node.
-	skippedNodes, err = clearPreexistingEcShards(commandEnv, topologyInfo, volumeIds, volumeIdToCollection, maxParallelization)
-	if err != nil {
-		return nil, fmt.Errorf("clear pre-existing ec shards before encoding: %w", err)
-	}
-
-	// Build a map of (volumeId, serverAddress) -> freeVolumeCount.
-	// Key by dn.Address so it matches wdclient.Location.Url. In deployments
-	// where dn.Id is a short name (e.g. Kubernetes StatefulSet pod name)
-	// while dn.Address is a FQDN:port, keying by dn.Id would never match the
-	// location Url during the health-check lookup below.
-	freeVolumeCountMap := make(map[string]int) // key: volumeId-serverAddress
-	eachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
-		addr := dn.Address
-		if addr == "" {
-			addr = dn.Id // older nodes use ip:port as id
-		}
-		for _, diskInfo := range dn.DiskInfos {
-			for _, v := range diskInfo.VolumeInfos {
-				key := fmt.Sprintf("%d-%s", v.Id, addr)
-				freeVolumeCountMap[key] = int(diskInfo.FreeVolumeCount)
-			}
-		}
-	})
-
-	// Filter replicas by free capacity BEFORE marking volumes readonly so that
-	// a failed health check does not strand volumes in readonly state.
-	filteredLocations := make(map[needle.VolumeId][]wdclient.Location)
-	for _, vid := range volumeIds {
-		var filteredLocs []wdclient.Location
-		for _, l := range locations[vid] {
-			key := fmt.Sprintf("%d-%s", vid, l.Url)
-			if freeCount, found := freeVolumeCountMap[key]; found && freeCount >= 2 {
-				filteredLocs = append(filteredLocs, l)
-			}
-		}
-		if len(filteredLocs) == 0 {
-			return nil, fmt.Errorf("no healthy replicas (FreeVolumeCount >= 2) found for volume %d to use as source for EC encoding", vid)
-		}
-		filteredLocations[vid] = filteredLocs
-	}
-
-	// mark volumes as readonly
-	ewg := NewErrorWaitGroup(maxParallelization)
-	for _, vid := range volumeIds {
-		for _, l := range locations[vid] {
-			ewg.Add(func() error {
-				if err := markVolumeReplicaWritable(context.Background(), commandEnv.option.GrpcDialOption, vid, l, false, false); err != nil {
-					return fmt.Errorf("mark volume %d as readonly on %s: %v", vid, l.Url, err)
-				}
-				return nil
-			})
-		}
-	}
-	if err := ewg.Wait(); err != nil {
-		return nil, err
-	}
-
-	// Sync replicas and select the best one for each volume (with highest file count)
-	// This addresses data inconsistency risk in multi-replica volumes (issue #7797)
-	// by syncing missing entries between replicas before encoding
-	bestReplicas := make(map[needle.VolumeId]wdclient.Location)
-	for _, vid := range volumeIds {
-		collection := volumeIdToCollection[vid]
-
-		// Sync missing entries between replicas, then select the best one
-		bestLoc, selectErr := volume_replica.SyncAndSelectBestReplica(commandEnv.option.GrpcDialOption, vid, collection, filteredLocations[vid], "", writer)
-		if selectErr != nil {
-			return nil, fmt.Errorf("failed to sync and select replica for volume %d: %v", vid, selectErr)
-		}
-		bestReplicas[vid] = bestLoc
-	}
-
-	// Re-attempt the orphan sweep on the nodes skipped as unreachable, now that
-	// any node that recovered during readonly-marking and replica sync answers
-	// again. A node whose teardown now succeeds is clean (and the generation host
-	// re-wipes its own disks regardless), so it leaves the skipped set and can be
-	// a balance source/target — otherwise its shards would never distribute off
-	// it. A node that is still down stays skipped and excluded, preserving the
-	// leniency for a genuinely-down node; such a node also cannot be the
-	// generation host below, since VolumeEcShardsGenerate would fail to read .dat.
-	if err := resweepSkippedNodes(commandEnv, skippedNodes, volumeIds, volumeIdToCollection, maxParallelization); err != nil {
-		return nil, err
-	}
-
-	// A selected generation host still in skippedNodes after the re-sweep was
-	// transport-down when we tried to clean it, so its stale orphans were never
-	// removed and EcBalance excludes it as both source and target. If it recovers
-	// just in time for generation, all shards land on a node we can neither clean
-	// nor balance off — a single point of failure that union-only verification
-	// still accepts, after which the originals are deleted. Abort instead.
-	for _, vid := range volumeIds {
-		genHost := bestReplicas[vid].ServerAddress()
-		if _, stillSkipped := skippedNodes[genHost]; stillSkipped {
-			return nil, fmt.Errorf("generate ec shards for volume %d aborted: selected source %s is still skipped after the orphan re-sweep", vid, genHost)
-		}
-	}
-
-	// generate ec shards using the best replica for each volume
-	ewg.Reset()
-	for _, vid := range volumeIds {
-		target := bestReplicas[vid]
-		collection := volumeIdToCollection[vid]
-		ewg.Add(func() error {
-			if err := generateEcShards(commandEnv.option.GrpcDialOption, vid, collection, target.ServerAddress()); err != nil {
-				return fmt.Errorf("generate ec shards for volume %d on %s: %v", vid, target.Url, err)
-			}
-			return nil
-		})
-	}
-	if err := ewg.Wait(); err != nil {
-		return nil, err
-	}
-
-	// mount all ec shards for the converted volume
-	shardIds := erasure_coding.AllShardIds()
-
-	ewg.Reset()
-	for _, vid := range volumeIds {
-		target := bestReplicas[vid]
-		collection := volumeIdToCollection[vid]
-		ewg.Add(func() error {
-			if err := mountEcShards(commandEnv.option.GrpcDialOption, collection, vid, target.ServerAddress(), shardIds); err != nil {
-				return fmt.Errorf("mount ec shards for volume %d on %s: %v", vid, target.Url, err)
-			}
-			return nil
-		})
-	}
-	if err := ewg.Wait(); err != nil {
-		return nil, err
-	}
-
-	return skippedNodes, nil
-}
-
-// clearPreexistingEcShards removes EC shards and index files left over from a
-// previous (failed or partial) encode of the given volume ids, on every node
-// that still reports them, so a fresh encode regenerates from a clean slate.
-// Scans all disk types. The normal .dat/.idx — the source of truth for this
-// encode — is untouched; only orphaned EC artifacts are deleted.
-//
-// Returns the set of nodes skipped as unreachable. A skipped node may still hold
-// an un-deleted orphan from a prior run; if it recovers it must be kept out of
-// this encode's shard distribution, or the balance could install the new
-// generation alongside the stale orphan and mix generations on one node.
-func clearPreexistingEcShards(commandEnv *CommandEnv, topologyInfo *master_pb.TopologyInfo, volumeIds []needle.VolumeId, volumeIdToCollection map[needle.VolumeId]string, maxParallelization int) (skipped map[pb.ServerAddress]struct{}, err error) {
-	wanted := make(map[uint32]bool, len(volumeIds))
-	for _, vid := range volumeIds {
-		wanted[uint32(vid)] = true
-	}
-
-	// Note which (node, vid) pairs the topology already reports EC shards for:
-	// those are mounted leftovers and cleaning them is required (fatal on
-	// error). Every other (node, vid) is swept best-effort to catch UNMOUNTED
-	// orphans left by a failed copy — invisible to the heartbeat, so absent
-	// here. A node that is down or holds nothing is a harmless no-op; a node
-	// unreachable now also cannot receive this encode's new generation, so a
-	// surviving orphan there keeps its old identity and the read guard rejects
-	// it. Always delete the full shard-id range so a wider custom ratio's
-	// leftovers are covered too.
-	reportedKey := func(addr pb.ServerAddress, vid uint32) string {
-		return string(addr) + "\x00" + strconv.FormatUint(uint64(vid), 10)
-	}
-	reported := make(map[string]struct{})
-	var nodes []pb.ServerAddress
-	eachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
-		addr := pb.NewServerAddressFromDataNode(dn)
-		nodes = append(nodes, addr)
-		for _, diskInfo := range dn.DiskInfos {
-			for _, ecInfo := range diskInfo.EcShardInfos {
-				if wanted[ecInfo.Id] {
-					reported[reportedKey(addr, ecInfo.Id)] = struct{}{}
-				}
-			}
-		}
-	})
-
-	allShardIds := make([]erasure_coding.ShardId, erasure_coding.MaxShardCount)
-	for i := range allShardIds {
-		allShardIds[i] = erasure_coding.ShardId(i)
-	}
-
-	if len(reported) > 0 {
-		fmt.Printf("clearing stale EC shards reported for %d (node,volume) pair(s) before regenerating...\n", len(reported))
-	}
-	// Nodes skipped as unreachable, accumulated across the concurrent sweep tasks.
-	skipped = make(map[pb.ServerAddress]struct{})
-	var skippedMu sync.Mutex
-	ewg := NewErrorWaitGroup(maxParallelization)
-	for _, addr := range nodes {
-		for _, vid := range volumeIds {
-			fatal := false
-			if _, ok := reported[reportedKey(addr, uint32(vid))]; ok {
-				fatal = true
-			}
-			collection := volumeIdToCollection[vid]
-			ewg.Add(func() error {
-				if err := unmountAndDeleteEcShardsQuiet(commandEnv.option.GrpcDialOption, collection, vid, addr, allShardIds); err != nil {
-					// Surface a reachable node whose delete genuinely failed (its orphan would
-					// be re-stamped by a later copy installing the new .vif). A missing
-					// full_teardown ack from a reachable pre-upgrade node is fatal too: it may
-					// still hold an orphan a later copy would re-stamp into the new generation.
-					// Stay best-effort only for a node that is truly unreachable: codes.Unavailable
-					// alone is ambiguous — a genuinely-down node and a reachable Rust volume
-					// server in maintenance mode both return it (a Go server returns Unknown for
-					// maintenance, already fatal above). Confirm with a non-maintenance-gated Ping
-					// before skipping; skip only when the Ping itself transport-failed (nodeDown).
-					// A reachable maintenance node (nodeUp) CAN receive this generation, and an
-					// inconclusive Ping (nodeLivenessUnknown, e.g. a pre-Ping server returning
-					// Unimplemented — which means the node is up) does not prove the node is down,
-					// so both stay fatal rather than silently leaving a stale EC generation.
-					if fatal || errors.Is(err, errFullTeardownNotAcked) || !isNodeUnreachable(err) ||
-						classifyNodeLiveness(pingVolumeServer(commandEnv.option.GrpcDialOption, addr)) != nodeDown {
-						return fmt.Errorf("clear stale ec shards for volume %d on %s: %w", vid, addr, err)
-					}
-					glog.V(1).Infof("orphan sweep: volume %d on %s skipped (unreachable): %v", vid, addr, err)
-					skippedMu.Lock()
-					skipped[addr] = struct{}{}
-					skippedMu.Unlock()
-				}
-				return nil
-			})
-		}
-	}
-	if err := ewg.Wait(); err != nil {
-		return nil, err
-	}
-	return skipped, nil
-}
-
-// resweepSkippedNodes re-attempts the orphan teardown on the nodes that the
-// initial sweep skipped as unreachable, just before shard generation. A node
-// that recovered in the meantime — and is therefore eligible to host this
-// encode's generation — has its teardown retried; if it now fully succeeds it is
-// removed from skipped so the rebalance can use it as a source and move its
-// shards off, instead of stranding all shards on the single generation host and
-// collapsing fault tolerance. A node still transport-down stays skipped (the
-// same leniency the initial sweep grants), and a node that came back reachable
-// but whose delete genuinely failed is fatal, exactly as in the initial sweep,
-// so a stale generation is never silently left behind. Mutates skipped in place.
-func resweepSkippedNodes(commandEnv *CommandEnv, skipped map[pb.ServerAddress]struct{}, volumeIds []needle.VolumeId, volumeIdToCollection map[needle.VolumeId]string, maxParallelization int) error {
-	if len(skipped) == 0 {
-		return nil
-	}
-
-	allShardIds := make([]erasure_coding.ShardId, erasure_coding.MaxShardCount)
-	for i := range allShardIds {
-		allShardIds[i] = erasure_coding.ShardId(i)
-	}
-
-	addrs := make([]pb.ServerAddress, 0, len(skipped))
-	for addr := range skipped {
-		addrs = append(addrs, addr)
-	}
-
-	fmt.Printf("re-checking %d node(s) skipped by the orphan sweep before generating shards...\n", len(addrs))
-
-	// A node still down on every retried vid stays skipped; one that fully
-	// succeeds is un-skipped. Track per-node whether any retry still failed
-	// (down) so a node whose state is mixed across vids never gets un-skipped.
-	stillDown := make(map[pb.ServerAddress]struct{})
-	var mu sync.Mutex
-	ewg := NewErrorWaitGroup(maxParallelization)
-	for _, addr := range addrs {
-		for _, vid := range volumeIds {
-			collection := volumeIdToCollection[vid]
-			ewg.Add(func() error {
-				if err := unmountAndDeleteEcShardsQuiet(commandEnv.option.GrpcDialOption, collection, vid, addr, allShardIds); err != nil {
-					// Same decision as the initial sweep: a reachable node whose delete
-					// genuinely failed (or did not ack a full teardown, or whose liveness is
-					// inconclusive) is fatal, since it could hold an orphan a later copy
-					// re-stamps into this generation. Only a node still transport-down stays
-					// skipped.
-					if errors.Is(err, errFullTeardownNotAcked) || !isNodeUnreachable(err) ||
-						classifyNodeLiveness(pingVolumeServer(commandEnv.option.GrpcDialOption, addr)) != nodeDown {
-						return fmt.Errorf("re-clear stale ec shards for volume %d on %s: %w", vid, addr, err)
-					}
-					glog.V(1).Infof("orphan re-sweep: volume %d on %s still skipped (unreachable): %v", vid, addr, err)
-					mu.Lock()
-					stillDown[addr] = struct{}{}
-					mu.Unlock()
-				}
-				return nil
-			})
-		}
-	}
-	if err := ewg.Wait(); err != nil {
-		return err
-	}
-
-	for _, addr := range addrs {
-		if _, down := stillDown[addr]; !down {
-			delete(skipped, addr)
-			glog.V(0).Infof("orphan re-sweep: node %s recovered and was cleaned; it will participate in the EC rebalance", addr)
-		}
-	}
-	return nil
-}
-
-// isNodeUnreachable reports whether err means the volume server could not be
-// reached at all, as opposed to an RPC that reached the node and failed. Only an
-// unreachable node is safe to skip in the orphan sweep. A dead peer surfaces as
-// a gRPC codes.Unavailable from the RPC (the dial is lazy, so it never fails at
-// connect time); any non-status error reached node logic and is treated as
-// reachable, so the sweep stays fatal rather than silently leaving stale state.
-func isNodeUnreachable(err error) bool {
-	if err == nil {
-		return false
-	}
-	st, ok := status.FromError(err)
-	return ok && st.Code() == codes.Unavailable
-}
-
-// nodeLiveness is the tri-state result of a pingVolumeServer probe.
-type nodeLiveness int
-
-const (
-	// nodeUp: Ping succeeded — the node is reachable (e.g. a Rust volume server
-	// in maintenance mode that fails the delete but answers Ping).
-	nodeUp nodeLiveness = iota
-	// nodeDown: Ping itself transport-failed with codes.Unavailable — the node is
-	// confirmed unreachable. The only state the orphan sweep may skip.
-	nodeDown
-	// nodeLivenessUnknown: Ping reached failing logic with any non-Unavailable
-	// code (Internal, ResourceExhausted, Unimplemented from a pre-Ping server, …)
-	// or a non-status error. This does NOT prove the node is down, so it is fatal.
-	nodeLivenessUnknown
-)
-
-// classifyNodeLiveness maps a pingVolumeServer error into the tri-state. A nil
-// error is nodeUp, a transport codes.Unavailable is nodeDown (reusing the same
-// rule as isNodeUnreachable), and every other Ping failure is nodeLivenessUnknown.
-func classifyNodeLiveness(pingErr error) nodeLiveness {
-	if pingErr == nil {
-		return nodeUp
-	}
-	if isNodeUnreachable(pingErr) {
-		return nodeDown
-	}
-	return nodeLivenessUnknown
-}
-
-// collectEcShardBitsByNode returns, for one volume, the EC shard bits each node
-// reports, unioned across all its disk types. Freshly generated shards sit on
-// the disk that held the source .dat, which may differ from the balance target
-// disk type, so visibility questions ("has the master heard about these shards
-// at all?") must not filter by disk type. Only the newest encode generation
-// (largest EncodeTsNs) counts: an orphaned older generation — a failed earlier
-// encode on a node the pre-encode sweep could not reach but the master still
-// hears from — must neither satisfy the registration wait nor pose as a second
-// holder in the clump check. Entries without a timestamp form the legacy
-// generation zero, so they only count when no stamped generation exists;
-// dropping them otherwise errs toward keeping the source volume.
-func collectEcShardBitsByNode(topoInfo *master_pb.TopologyInfo, vid needle.VolumeId) map[pb.ServerAddress]erasure_coding.ShardBits {
-	type shardEntry struct {
-		addr pb.ServerAddress
-		ts   int64
-		bits erasure_coding.ShardBits
-	}
-	var entries []shardEntry
-	var newestTs int64
-	eachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
-		for _, diskInfo := range dn.DiskInfos {
-			for _, ecInfo := range diskInfo.EcShardInfos {
-				if ecInfo.Id != uint32(vid) {
-					continue
-				}
-				entries = append(entries, shardEntry{pb.NewServerAddressFromDataNode(dn), ecInfo.EncodeTsNs, erasure_coding.ShardBits(ecInfo.EcIndexBits)})
-				if ecInfo.EncodeTsNs > newestTs {
-					newestTs = ecInfo.EncodeTsNs
-				}
-			}
-		}
-	})
-	res := make(map[pb.ServerAddress]erasure_coding.ShardBits)
-	for _, e := range entries {
-		if e.ts == newestTs {
-			res[e.addr] |= e.bits
-		}
-	}
-	return res
-}
-
-// waitForEcShardsToRegister polls the master topology until every given volume
-// reports a full EC shard set. Mounting shards notifies the master
-// asynchronously (mount -> NewEcShardsChan -> delta heartbeat), so a topology
-// snapshot taken right after doEcEncode can predate the mounts. Failing after
-// the retries keeps the source volumes, and a re-run of ec.encode starts clean.
-func waitForEcShardsToRegister(commandEnv *CommandEnv, volumeIds []needle.VolumeId) error {
-	const maxAttempts = 10
-	const retryInterval = 2 * time.Second
-
-	var lastMissing []string
-	for attempt := 0; attempt < maxAttempts; attempt++ {
-		if attempt > 0 {
-			time.Sleep(retryInterval)
-		}
-		topoInfo, _, err := collectTopologyInfo(commandEnv, 0)
-		if err != nil {
-			return fmt.Errorf("fetch topology while waiting for ec shards to register: %w", err)
-		}
-		lastMissing = lastMissing[:0]
-		for _, vid := range volumeIds {
-			var union erasure_coding.ShardBits
-			for _, bits := range collectEcShardBitsByNode(topoInfo, vid) {
-				union |= bits
-			}
-			if union.Count() < erasure_coding.TotalShardsCount {
-				lastMissing = append(lastMissing, fmt.Sprintf("volume %d: %d/%d shards", vid, union.Count(), erasure_coding.TotalShardsCount))
-			}
-		}
-		if len(lastMissing) == 0 {
-			return nil
-		}
-		glog.V(0).Infof("waiting for newly generated ec shards to register with the master (attempt %d/%d): %v",
-			attempt+1, maxAttempts, lastMissing)
-	}
-	return fmt.Errorf("newly generated ec shards did not register with the master after %d attempts: %v", maxAttempts, lastMissing)
-}
-
-// ecShardsClumpedOnOneNode reports whether every EC shard the master sees for
-// vid sits on a single node while at least one other node has free EC shard
-// slots on the target disk type — i.e. the preceding rebalance could have
-// spread the shards but did not. Zero visible shards is not a clump; the
-// recoverability check owns that case.
-func ecShardsClumpedOnOneNode(topoInfo *master_pb.TopologyInfo, vid needle.VolumeId, diskType types.DiskType) (holder pb.ServerAddress, clumped bool) {
-	byNode := collectEcShardBitsByNode(topoInfo, vid)
-	if len(byNode) != 1 {
-		return "", false
-	}
-	for addr := range byNode {
-		holder = addr
-	}
-	ecNodes, _ := collectEcVolumeServersByDc(topoInfo, "", diskType)
-	for _, en := range ecNodes {
-		if pb.NewServerAddressFromDataNode(en.info) == holder {
-			continue
-		}
-		if en.freeEcSlot > 0 {
-			return holder, true
-		}
-	}
-	return "", false
-}
-
-// ecShardSummaryByNode says where a volume's shards are, one entry per node,
-// sorted so the message is stable. It names the ids and not just the count: a
-// set holding shards 0-9 and one holding 4-13 are both "10 shards", and which
-// ones survived is what says whether the set is recoverable and from where.
-func ecShardSummaryByNode(byNode map[pb.ServerAddress]erasure_coding.ShardBits) []string {
-	summary := make([]string, 0, len(byNode))
-	for node, bits := range byNode {
-		summary = append(summary, fmt.Sprintf("%s=%d shards %v", node, bits.Count(), slices.Collect(bits.All())))
-	}
-	sort.Strings(summary)
-	return summary
-}
-
-func verifyEcShardsBeforeDelete(commandEnv *CommandEnv, volumeIds []needle.VolumeId, diskType types.DiskType, expectSpread bool) error {
-	// Shard relocations from the preceding EC balance reach the master via
-	// volume-server heartbeats, so freshly distributed shards may not all be
-	// visible in the master topology immediately. Poll a few times before
-	// concluding the shard set is incomplete, so a heartbeat-propagation lag is
-	// not mistaken for missing data. After the retries: a volume below the
-	// recoverable threshold (dataShards) aborts the deletion; a recoverable
-	// but degraded set proceeds with a warning, since the missing shards can
-	// be rebuilt from the survivors while keeping the source next to live
-	// shards is the more dangerous mixed state. When expectSpread is set (the
-	// rebalance ran in apply mode), a volume whose shards all still sit on one
-	// node while another node has free slots also aborts the deletion: losing
-	// that node would lose the volume, so the original is the safer copy.
-	const maxAttempts = 10
-	const retryInterval = 2 * time.Second
-
-	var lastErr error
-	var lastDegraded []string
-	var lastClumped []string
-	for attempt := 0; attempt < maxAttempts; attempt++ {
-		topoInfo, _, err := collectTopologyInfo(commandEnv, 0)
-		if err != nil {
-			return fmt.Errorf("fetch topology for shard verification: %w", err)
-		}
-
-		lastErr = nil
-		lastDegraded = lastDegraded[:0]
-		lastClumped = lastClumped[:0]
-		for _, vid := range volumeIds {
-			// Count the shards wherever they landed, as waitForEcShardsToRegister
-			// above already does. generateEcShards writes them beside the source
-			// volume, so encoding a volume that lives on a non-default medium
-			// puts them on that medium while -diskType still says hdd. Counting
-			// only the -diskType bucket then reports a complete set as entirely
-			// missing and aborts an encode that in fact succeeded, leaving the
-			// volume as both a .dat and a full set of shards.
-			byNode := collectEcShardBitsByNode(topoInfo, vid)
-
-			var union erasure_coding.ShardBits
-			for _, bits := range byNode {
-				union |= bits
-			}
-
-			totalShards := erasure_coding.TotalShardsCount
-			degraded, err := erasure_coding.RequireRecoverableShardSet(uint32(vid), union, erasure_coding.DataShardsCount, totalShards)
-			if err != nil {
-				lastErr = fmt.Errorf("volume %d: %w (observed: %v)", vid, err, ecShardSummaryByNode(byNode))
-				break
-			}
-			if expectSpread {
-				if holder, clumped := ecShardsClumpedOnOneNode(topoInfo, vid, diskType); clumped {
-					lastClumped = append(lastClumped, fmt.Sprintf("volume %d: all shards on %s", vid, holder))
-					continue
-				}
-			}
-			if degraded {
-				lastDegraded = append(lastDegraded, fmt.Sprintf("volume %d: %d/%d shards", vid, union.Count(), totalShards))
-				continue
-			}
-
-			glog.V(0).Infof("EC shard verification ok for volume %d: %d/%d shards present across %d nodes",
-				vid, union.Count(), totalShards, len(byNode))
-		}
-
-		if lastErr == nil && len(lastDegraded) == 0 && len(lastClumped) == 0 {
-			return nil
-		}
-		if attempt < maxAttempts-1 {
-			glog.V(0).Infof("EC shard verification incomplete (attempt %d/%d), waiting for shard locations to propagate: %v %v %v",
-				attempt+1, maxAttempts, lastErr, lastDegraded, lastClumped)
-			time.Sleep(retryInterval)
-		}
-	}
-
-	if lastErr != nil {
-		glog.Errorf("EC shard verification failed after %d attempts: %v", maxAttempts, lastErr)
-		return lastErr
-	}
-	if len(lastClumped) > 0 {
-		return fmt.Errorf("EC shards still sit on a single node after rebalance even though other nodes have free slots (%v); keeping the original volumes. Run ec.balance -apply, verify the spread, then delete the originals or re-run ec.encode", lastClumped)
-	}
-	glog.Warningf("EC shard set incomplete but recoverable after %d attempts, proceeding with source deletion (rebuild missing shards with ec.rebuild): %v",
-		maxAttempts, lastDegraded)
-	return nil
-}
-
-// doDeleteVolumesWithLocations deletes volumes using pre-collected location information
-// This avoids race conditions where master metadata is updated after EC encoding
-func doDeleteVolumesWithLocations(commandEnv *CommandEnv, volumeIds []needle.VolumeId, volumeLocationsMap map[needle.VolumeId][]wdclient.Location, maxParallelization int) error {
-	if !commandEnv.isLocked() {
-		return fmt.Errorf("lock is lost")
-	}
-
-	ewg := NewErrorWaitGroup(maxParallelization)
-	for _, vid := range volumeIds {
-		locations, found := volumeLocationsMap[vid]
-		if !found {
-			fmt.Printf("warning: no locations found for volume %d, skipping deletion\n", vid)
-			continue
-		}
-
-		for _, l := range locations {
-			ewg.Add(func() error {
-				if err := deleteVolume(context.Background(), commandEnv.option.GrpcDialOption, vid, l.ServerAddress(), false, false); err != nil {
-					return fmt.Errorf("deleteVolume %s volume %d: %v", l.Url, vid, err)
-				}
-				fmt.Printf("deleted volume %d from %s\n", vid, l.Url)
-				return nil
-			})
-		}
-	}
-	if err := ewg.Wait(); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func generateEcShards(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, collection string, sourceVolumeServer pb.ServerAddress) error {
-
-	fmt.Printf("generateEcShards %d (collection %q) on %s ...\n", volumeId, collection, sourceVolumeServer)
-
-	err := operation.WithVolumeServerClient(false, sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
-		_, genErr := volumeServerClient.VolumeEcShardsGenerate(context.Background(), &volume_server_pb.VolumeEcShardsGenerateRequest{
-			VolumeId:   uint32(volumeId),
-			Collection: collection,
-		})
-		return genErr
-	})
-
-	return err
-
-}
-
 func collectVolumeIdsForEcEncode(commandEnv *CommandEnv, collectionPattern string, sourceDiskType *types.DiskType, fullPercentage float64, quietPeriod time.Duration, verbose bool) (vids []needle.VolumeId, matchedCollections []string, err error) {
 	// compile regex pattern for collection matching
 	collectionRegex, err := compileCollectionPattern(collectionPattern)
@@ -1003,151 +202,6 @@ func collectVolumeIdsForEcEncode(commandEnv *CommandEnv, collectionPattern strin
 
 	fmt.Printf("collect volumes with collection pattern '%s', quiet for: %d seconds and %.1f%% full\n", collectionPattern, quietSeconds, fullPercentage)
 
-	vids, matchedCollections = selectVolumeIdsFromTopology(topologyInfo, volumeSizeLimitMb, collectionRegex, sourceDiskType, quietSeconds, nowUnixSeconds, fullPercentage, verbose)
-	return
-}
-
-func selectVolumeIdsFromTopology(topologyInfo *master_pb.TopologyInfo, volumeSizeLimitMb uint64, collectionRegex *regexp.Regexp, sourceDiskType *types.DiskType, quietSeconds int64, nowUnixSeconds int64, fullPercentage float64, verbose bool) (vids []needle.VolumeId, matchedCollections []string) {
-	// Statistics for verbose mode
-	var (
-		totalVolumes    int
-		remoteVolumes   int
-		wrongCollection int
-		wrongDiskType   int
-		tooRecent       int
-		tooSmall        int
-		noFreeDisk      int
-	)
-
-	vidMap := make(map[uint32]bool)
-	collectionSet := make(map[string]bool)
-	eachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
-		for _, diskInfo := range dn.DiskInfos {
-			for _, v := range diskInfo.VolumeInfos {
-				totalVolumes++
-
-				// ignore remote volumes
-				if v.RemoteStorageName != "" {
-					remoteVolumes++
-					if verbose {
-						fmt.Printf("skip volume %d on %s: remote volume (storage: %s)\n",
-							v.Id, dn.Id, v.RemoteStorageName)
-					}
-					continue
-				}
-
-				// check collection against regex pattern
-				if !collectionRegex.MatchString(v.Collection) {
-					wrongCollection++
-					if verbose {
-						fmt.Printf("skip volume %d on %s: collection doesn't match pattern (pattern: %s, actual: %s)\n",
-							v.Id, dn.Id, collectionRegex.String(), v.Collection)
-					}
-					continue
-				}
-
-				// track matched collection
-				collectionSet[v.Collection] = true
-
-				// check disk type
-				if sourceDiskType != nil && types.ToDiskType(v.DiskType) != *sourceDiskType {
-					wrongDiskType++
-					if verbose {
-						fmt.Printf("skip volume %d on %s: wrong disk type (expected: %s, actual: %s)\n",
-							v.Id, dn.Id, sourceDiskType.ReadableString(), types.ToDiskType(v.DiskType).ReadableString())
-					}
-					continue
-				}
-
-				// check quiet period
-				if v.ModifiedAtSecond+quietSeconds >= nowUnixSeconds {
-					tooRecent++
-					if verbose {
-						fmt.Printf("skip volume %d on %s: too recently modified (last modified: %d seconds ago, required: %d seconds)\n",
-							v.Id, dn.Id, nowUnixSeconds-v.ModifiedAtSecond, quietSeconds)
-					}
-					continue
-				}
-
-				// check size
-				sizeThreshold := fullPercentage / 100 * float64(volumeSizeLimitMb) * 1024 * 1024
-				if float64(v.Size) <= sizeThreshold {
-					tooSmall++
-					if verbose {
-						fmt.Printf("skip volume %d on %s: too small (size: %.1f MB, threshold: %.1f MB, %.1f%% full)\n",
-							v.Id, dn.Id, float64(v.Size)/(1024*1024), sizeThreshold/(1024*1024),
-							float64(v.Size)*100/(float64(volumeSizeLimitMb)*1024*1024))
-					}
-					continue
-				}
-
-				// check free disk space
-				if diskInfo.FreeVolumeCount < 2 {
-					glog.V(0).Infof("replica %s %d on %s has no free disk", v.Collection, v.Id, dn.Id)
-					if verbose {
-						fmt.Printf("skip replica of volume %d on %s: insufficient free disk space (free volumes: %d, required: 2)\n",
-							v.Id, dn.Id, diskInfo.FreeVolumeCount)
-					}
-					if _, found := vidMap[v.Id]; !found {
-						vidMap[v.Id] = false
-					}
-				} else {
-					if verbose {
-						fmt.Printf("selected volume %d on %s: size %.1f MB (%.1f%% full), last modified %d seconds ago, free volumes: %d\n",
-							v.Id, dn.Id, float64(v.Size)/(1024*1024),
-							float64(v.Size)*100/(float64(volumeSizeLimitMb)*1024*1024),
-							nowUnixSeconds-v.ModifiedAtSecond, diskInfo.FreeVolumeCount)
-					}
-					vidMap[v.Id] = true
-				}
-			}
-		}
-	})
-
-	for vid, good := range vidMap {
-		if good {
-			vids = append(vids, needle.VolumeId(vid))
-		} else {
-			noFreeDisk++
-		}
-	}
-
-	// Convert collection set to slice
-	for collection := range collectionSet {
-		matchedCollections = append(matchedCollections, collection)
-	}
-	sort.Strings(matchedCollections)
-
-	// Print summary statistics in verbose mode or when no volumes selected
-	if verbose || len(vids) == 0 {
-		fmt.Printf("\nVolume selection summary:\n")
-		fmt.Printf("  Total volumes examined: %d\n", totalVolumes)
-		fmt.Printf("  Selected for encoding: %d\n", len(vids))
-		fmt.Printf("  Collections matched: %v\n", matchedCollections)
-
-		if totalVolumes > 0 {
-			fmt.Printf("\nReasons for exclusion:\n")
-			if remoteVolumes > 0 {
-				fmt.Printf("  Remote volumes: %d\n", remoteVolumes)
-			}
-			if wrongCollection > 0 {
-				fmt.Printf("  Collection doesn't match pattern: %d\n", wrongCollection)
-			}
-			if wrongDiskType > 0 {
-				fmt.Printf("  Wrong disk type: %d\n", wrongDiskType)
-			}
-			if tooRecent > 0 {
-				fmt.Printf("  Too recently modified: %d\n", tooRecent)
-			}
-			if tooSmall > 0 {
-				fmt.Printf("  Too small (< %.1f%% full): %d\n", fullPercentage, tooSmall)
-			}
-			if noFreeDisk > 0 {
-				fmt.Printf("  Insufficient free disk space: %d\n", noFreeDisk)
-			}
-		}
-		fmt.Println()
-	}
-
+	vids, matchedCollections = ec.SelectVolumeIdsFromTopology(topologyInfo, volumeSizeLimitMb, collectionRegex, sourceDiskType, quietSeconds, nowUnixSeconds, fullPercentage, verbose)
 	return
 }

weed/shell/command_ec_rebuild.go

@@ -1,19 +1,13 @@
 package shell
 
 import (
-	"context"
 	"flag"
 	"fmt"
 	"io"
-	"slices"
 	"strconv"
 	"strings"
-	"sync"
 
-	"github.com/seaweedfs/seaweedfs/weed/operation"
-	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/ec"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
 	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 )
@@ -22,19 +16,6 @@ func init() {
 	Commands = append(Commands, &commandEcRebuild{})
 }
 
-type ecRebuilder struct {
-	commandEnv   *CommandEnv
-	ecNodes      []*EcNode
-	writer       io.Writer
-	applyChanges bool
-	collections  []string
-	volumeIds    []needle.VolumeId
-	diskType     types.DiskType
-
-	ewg       *ErrorWaitGroup
-	ecNodesMu sync.Mutex
-}
-
 type commandEcRebuild struct {
 }
 
@@ -142,401 +123,5 @@ func (c *commandEcRebuild) Do(args []string, commandEnv *CommandEnv, writer io.W
 		}
 	}
 
-	erb := &ecRebuilder{
-		commandEnv:   commandEnv,
-		ecNodes:      allEcNodes,
-		writer:       writer,
-		applyChanges: *applyChanges,
-		collections:  collections,
-		volumeIds:    volumeIds,
-		diskType:     diskType,
-
-		ewg: NewErrorWaitGroup(*maxParallelization),
-	}
-
-	// Recover shards left unmounted by a missing .ecx index before planning: such
-	// shards never register with the master, so the rebuild below would treat the
-	// volume as short or unrepairable even though its data is intact (issue #10104).
-	erb.recoverMissingIndexes()
-
-	fmt.Printf("rebuildEcVolumes for %d collection(s)\n", len(collections))
-	for _, c := range collections {
-		erb.rebuildEcVolumes(c)
-	}
-
-	return erb.ewg.Wait()
-}
-
-func (erb *ecRebuilder) write(format string, a ...any) {
-	fmt.Fprintf(erb.writer, format, a...)
-}
-
-func (erb *ecRebuilder) isLocked() bool {
-	return erb.commandEnv.isLocked()
-}
-
-// matchesVolumeId verifies whether the rebuilder is targeted at a given volume ID.
-func (erb *ecRebuilder) matchesVolumeId(vid needle.VolumeId) bool {
-	if len(erb.volumeIds) == 0 {
-		return true
-	}
-
-	return slices.Contains(erb.volumeIds, vid)
-}
-
-// countLocalShards returns the number of shards already present locally on the node for the given volume.
-// Unions across all of the node's disks, like prepareDataToRecover, so slot
-// accounting matches what the rebuild will actually treat as local.
-func (erb *ecRebuilder) countLocalShards(node *EcNode, collection string, volumeId needle.VolumeId) int {
-	localShardsInfo := erasure_coding.NewShardsInfo()
-	for _, diskInfo := range node.info.DiskInfos {
-		for _, ecShardInfo := range diskInfo.EcShardInfos {
-			if ecShardInfo.Collection == collection && needle.VolumeId(ecShardInfo.Id) == volumeId {
-				localShardsInfo.Add(erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(ecShardInfo))
-			}
-		}
-	}
-	return localShardsInfo.Count()
-}
-
-// selectAndReserveRebuilder atomically selects a rebuilder node with sufficient free slots
-// and reserves slots only for the non-local shards that need to be copied/generated.
-func (erb *ecRebuilder) selectAndReserveRebuilder(collection string, volumeId needle.VolumeId) (*EcNode, int, error) {
-	erb.ecNodesMu.Lock()
-	defer erb.ecNodesMu.Unlock()
-
-	if len(erb.ecNodes) == 0 {
-		return nil, 0, fmt.Errorf("no ec nodes available")
-	}
-
-	// Find the node with the most free slots, considering local shards
-	var bestNode *EcNode
-	var bestSlotsNeeded int
-	var maxAvailableSlots int
-	var minSlotsNeeded int = erasure_coding.TotalShardsCount // Start with maximum possible
-	for _, node := range erb.ecNodes {
-		localShards := erb.countLocalShards(node, collection, volumeId)
-		slotsNeeded := erasure_coding.TotalShardsCount - localShards
-		if slotsNeeded < 0 {
-			slotsNeeded = 0
-		}
-
-		if node.freeEcSlot > maxAvailableSlots {
-			maxAvailableSlots = node.freeEcSlot
-		}
-
-		if slotsNeeded < minSlotsNeeded {
-			minSlotsNeeded = slotsNeeded
-		}
-
-		if node.freeEcSlot >= slotsNeeded {
-			if bestNode == nil || node.freeEcSlot > bestNode.freeEcSlot {
-				bestNode = node
-				bestSlotsNeeded = slotsNeeded
-			}
-		}
-	}
-
-	if bestNode == nil {
-		return nil, 0, fmt.Errorf("no node has sufficient free slots for volume %d (need at least %d slots, max available: %d)",
-			volumeId, minSlotsNeeded, maxAvailableSlots)
-	}
-
-	// Reserve slots only for non-local shards
-	bestNode.freeEcSlot -= bestSlotsNeeded
-
-	return bestNode, bestSlotsNeeded, nil
-}
-
-// releaseRebuilder releases the reserved slots back to the rebuilder node.
-func (erb *ecRebuilder) releaseRebuilder(node *EcNode, slotsToRelease int) {
-	erb.ecNodesMu.Lock()
-	defer erb.ecNodesMu.Unlock()
-
-	// Release slots by incrementing the free slot count
-	node.freeEcSlot += slotsToRelease
-}
-
-func (erb *ecRebuilder) rebuildEcVolumes(collection string) {
-	fmt.Printf("rebuildEcVolumes for %q\n", collection)
-
-	// collect vid => each shard locations, similar to ecShardMap in topology.go
-	ecShardMap := make(EcShardMap)
-	erb.ecNodesMu.Lock()
-	for _, ecNode := range erb.ecNodes {
-		ecShardMap.registerEcNode(ecNode, collection)
-	}
-	erb.ecNodesMu.Unlock()
-
-	for vid, locations := range ecShardMap {
-		if !erb.matchesVolumeId(vid) {
-			continue
-		}
-		shardCount := locations.shardCount()
-		if shardCount == erasure_coding.TotalShardsCount {
-			continue
-		}
-		if shardCount < erasure_coding.DataShardsCount {
-			erb.write("ec volume %d is unrepairable with %d shards (need %d), skipping\n", vid, shardCount, erasure_coding.DataShardsCount)
-			continue
-		}
-
-		// Capture variables for closure
-		vid := vid
-		locations := locations
-
-		erb.ewg.Add(func() error {
-			// Select rebuilder and reserve slots atomically per volume
-			rebuilder, slotsToReserve, err := erb.selectAndReserveRebuilder(collection, vid)
-			if err != nil {
-				return fmt.Errorf("failed to select rebuilder for volume %d: %v", vid, err)
-			}
-			defer erb.releaseRebuilder(rebuilder, slotsToReserve)
-
-			return erb.rebuildOneEcVolume(collection, vid, locations, rebuilder)
-		})
-	}
-}
-
-// recoverMissingIndexes asks every ec node to fetch a missing .ecx index from a
-// peer and mount the on-disk shards it could not load on its own. Shards
-// orphaned this way (index only on another server) are absent from the master
-// topology, so without this pass ec.rebuild would regenerate or give up on
-// shards whose data is actually present — and a volume whose every holder lacks
-// the index would not appear in the topology at all. Each node therefore
-// recovers all of its on-disk orphans (volume_id 0); an explicit -volumeIds
-// list narrows that to the requested volumes. On apply it refreshes the topology
-// so the rebuild planning sees the recovered shards (issue #10104).
-func (erb *ecRebuilder) recoverMissingIndexes() {
-	erb.ecNodesMu.Lock()
-	nodes := append([]*EcNode(nil), erb.ecNodes...)
-	erb.ecNodesMu.Unlock()
-	if len(nodes) == 0 {
-		return
-	}
-
-	// volume_id 0 means "recover every orphan on the node"; a -volumeIds list
-	// narrows recovery to those ids (each scanned across collections server-side).
-	vids := erb.volumeIds
-	if len(vids) == 0 {
-		vids = []needle.VolumeId{0}
-	}
-
-	if !erb.applyChanges {
-		erb.write("would ask %d ec node(s) to recover EC shards left unmounted by a missing .ecx index\n", len(nodes))
-		return
-	}
-
-	for _, node := range nodes {
-		for _, vid := range vids {
-			err := operation.WithVolumeServerClient(false, pb.NewServerAddressFromDataNode(node.info), erb.commandEnv.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
-				_, mountErr := client.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{
-					VolumeId:            uint32(vid),
-					RecoverMissingIndex: true,
-				})
-				return mountErr
-			})
-			if err != nil {
-				erb.write("%s recover missing index (volume %d): %v\n", node.info.Id, vid, err)
-			}
-		}
-	}
-
-	// Refresh topology so the rebuild planning sees shards the recovery registered.
-	refreshed, _, err := collectEcNodes(erb.commandEnv, erb.diskType)
-	if err != nil {
-		erb.write("failed to refresh ec nodes after index recovery: %v\n", err)
-		return
-	}
-	erb.ecNodesMu.Lock()
-	erb.ecNodes = refreshed
-	erb.ecNodesMu.Unlock()
-}
-
-func (erb *ecRebuilder) rebuildOneEcVolume(collection string, volumeId needle.VolumeId, locations EcShardLocations, rebuilder *EcNode) error {
-	if !erb.isLocked() {
-		return fmt.Errorf("lock is lost")
-	}
-
-	fmt.Printf("rebuildOneEcVolume %s %d\n", collection, volumeId)
-
-	// collect shard files to rebuilder local disk
-	var generatedShardIds []erasure_coding.ShardId
-	copiedShardIds, _, err := erb.prepareDataToRecover(rebuilder, collection, volumeId, locations)
-	defer func() {
-		// Clean up the working copies this run actually made, even when the
-		// recoverability gate failed after some copies already succeeded:
-		// they are temp files on the rebuilder nothing else reclaims. Dry-run
-		// copies nothing (copiedShardIds is empty), so this issues no delete
-		// RPC. Use a local error so a cleanup failure cannot mask the return.
-		if !erb.applyChanges || len(copiedShardIds) == 0 {
-			return
-		}
-		if derr := sourceServerDeleteEcShards(erb.commandEnv.option.GrpcDialOption, collection, volumeId, pb.NewServerAddressFromDataNode(rebuilder.info), copiedShardIds); derr != nil {
-			erb.write("%s delete copied ec shards %s %d.%v: %v\n", rebuilder.info.Id, collection, volumeId, copiedShardIds, derr)
-		}
-	}()
-	if err != nil {
-		return err
-	}
-
-	if !erb.applyChanges {
-		return nil
-	}
-
-	// generate ec shards, and maybe ecx file
-	generatedShardIds, err = erb.generateMissingShards(collection, volumeId, pb.NewServerAddressFromDataNode(rebuilder.info))
-	if err != nil {
-		return err
-	}
-
-	// mount the generated shards
-	err = mountEcShards(erb.commandEnv.option.GrpcDialOption, collection, volumeId, pb.NewServerAddressFromDataNode(rebuilder.info), generatedShardIds)
-	if err != nil {
-		return err
-	}
-
-	// ensure ECNode updates are atomic
-	erb.ecNodesMu.Lock()
-	defer erb.ecNodesMu.Unlock()
-	rebuilder.addEcVolumeShards(volumeId, collection, generatedShardIds, erb.diskType)
-
-	return nil
-}
-
-func (erb *ecRebuilder) generateMissingShards(collection string, volumeId needle.VolumeId, sourceLocation pb.ServerAddress) (rebuiltShardIds []erasure_coding.ShardId, err error) {
-
-	err = operation.WithVolumeServerClient(false, sourceLocation, erb.commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
-		resp, rebuildErr := volumeServerClient.VolumeEcShardsRebuild(context.Background(), &volume_server_pb.VolumeEcShardsRebuildRequest{
-			VolumeId:   uint32(volumeId),
-			Collection: collection,
-		})
-		if rebuildErr == nil {
-			rebuiltShardIds = erasure_coding.Uint32ToShardIds(resp.RebuiltShardIds)
-		}
-		return rebuildErr
-	})
-	return
-}
-
-func (erb *ecRebuilder) prepareDataToRecover(rebuilder *EcNode, collection string, volumeId needle.VolumeId, locations EcShardLocations) (copiedShardIds []erasure_coding.ShardId, localShardIds []erasure_coding.ShardId, err error) {
-
-	needEcxFile := true
-	localShardsInfo := erasure_coding.NewShardsInfo()
-	for _, diskInfo := range rebuilder.info.DiskInfos {
-		for _, ecShardInfo := range diskInfo.EcShardInfos {
-			if ecShardInfo.Collection == collection && needle.VolumeId(ecShardInfo.Id) == volumeId {
-				needEcxFile = false
-				// Union across disks: the rebuilder may hold this volume's
-				// shards on more than one disk. Overwriting per-disk would
-				// make a shard on a non-last disk look remote and get copied
-				// onto itself (O_TRUNC) and then node-wide deleted.
-				localShardsInfo.Add(erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(ecShardInfo))
-			}
-		}
-	}
-
-	targetShardCount := erasure_coding.TotalShardsCount
-	for i := erasure_coding.TotalShardsCount; i < len(locations); i++ {
-		if len(locations[i]) > 0 {
-			targetShardCount = i + 1
-		}
-	}
-
-	// recoverableRemoteShards counts remote shards that can contribute to the
-	// rebuild. Dry-run counts the plan; apply mode counts only successful copies.
-	recoverableRemoteShards := 0
-	for i := 0; i < targetShardCount; i++ {
-		ecNodes := locations[i]
-		shardId := erasure_coding.ShardId(i)
-		if len(ecNodes) == 0 {
-			erb.write("missing shard %d.%d\n", volumeId, shardId)
-			continue
-		}
-
-		if localShardsInfo.Has(shardId) {
-			localShardIds = append(localShardIds, shardId)
-			erb.write("use existing shard %d.%d\n", volumeId, shardId)
-			continue
-		}
-
-		// The rebuilder is itself the only listed holder: never copy a shard
-		// onto itself (the in-place O_TRUNC would destroy it) nor schedule it
-		// for the post-rebuild delete. Treat it as already local.
-		if ecNodes[0].info.Id == rebuilder.info.Id {
-			localShardIds = append(localShardIds, shardId)
-			erb.write("use existing shard %d.%d (already on rebuilder)\n", volumeId, shardId)
-			continue
-		}
-
-		if !erb.applyChanges {
-			recoverableRemoteShards++
-			erb.write("would copy %d.%d from %s\n", volumeId, shardId, ecNodes[0].info.Id)
-			continue
-		}
-
-		copyErr := operation.WithVolumeServerClient(false, pb.NewServerAddressFromDataNode(rebuilder.info), erb.commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
-			_, copyErr := volumeServerClient.VolumeEcShardsCopy(context.Background(), &volume_server_pb.VolumeEcShardsCopyRequest{
-				VolumeId:       uint32(volumeId),
-				Collection:     collection,
-				ShardIds:       []uint32{uint32(shardId)},
-				CopyEcxFile:    needEcxFile,
-				CopyEcjFile:    true,
-				CopyVifFile:    needEcxFile,
-				SourceDataNode: string(pb.NewServerAddressFromDataNode(ecNodes[0].info)),
-			})
-			return copyErr
-		})
-		if copyErr != nil {
-			erb.write("%s failed to copy %d.%d from %s: %v\n", rebuilder.info.Id, volumeId, shardId, ecNodes[0].info.Id, copyErr)
-			continue
-		}
-		recoverableRemoteShards++
-		if needEcxFile {
-			needEcxFile = false
-		}
-		erb.write("%s copied %d.%d from %s\n", rebuilder.info.Id, volumeId, shardId, ecNodes[0].info.Id)
-		// Only shards this run actually copied are temp working files to be
-		// deleted afterward; never a pre-existing local or remote shard.
-		copiedShardIds = append(copiedShardIds, shardId)
-	}
-
-	if len(localShardIds)+recoverableRemoteShards >= erasure_coding.DataShardsCount {
-		return copiedShardIds, localShardIds, nil
-	}
-
-	// Hand back what was copied so the caller deletes these orphaned working
-	// shards: recovery failed, but the temp files are already on the rebuilder.
-	return copiedShardIds, localShardIds, fmt.Errorf("%d shards are not enough to recover volume %d", len(localShardIds)+recoverableRemoteShards, volumeId)
-
-}
-
-type EcShardMap map[needle.VolumeId]EcShardLocations
-type EcShardLocations [][]*EcNode
-
-func (ecShardMap EcShardMap) registerEcNode(ecNode *EcNode, collection string) {
-	for _, diskInfo := range ecNode.info.DiskInfos {
-		for _, shardInfo := range diskInfo.EcShardInfos {
-			if shardInfo.Collection == collection {
-				existing, found := ecShardMap[needle.VolumeId(shardInfo.Id)]
-				if !found {
-					// Use MaxShardCount (32) to support custom EC ratios
-					existing = make([][]*EcNode, erasure_coding.MaxShardCount)
-					ecShardMap[needle.VolumeId(shardInfo.Id)] = existing
-				}
-				for _, shardId := range erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(shardInfo).Ids() {
-					existing[shardId] = append(existing[shardId], ecNode)
-				}
-			}
-		}
-	}
-}
-
-func (ecShardLocations EcShardLocations) shardCount() (count int) {
-	for _, locations := range ecShardLocations {
-		if len(locations) > 0 {
-			count++
-		}
-	}
-	return
+	return ec.RebuildEcVolumes(commandEnv.ecEnv(), allEcNodes, writer, collections, volumeIds, diskType, *maxParallelization, *applyChanges)
 }

weed/shell/command_ec_scrub.go

@@ -1,18 +1,15 @@
 package shell
 
 import (
-	"context"
 	"flag"
 	"fmt"
 	"io"
 	"strconv"
 	"strings"
-	"sync"
 
-	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/ec"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-	"google.golang.org/grpc"
 )
 
 func init() {
@@ -20,12 +17,6 @@ func init() {
 }
 
 type commandEcVolumeScrub struct {
-	env                      *CommandEnv
-	volumeServerAddrs        []pb.ServerAddress
-	volumeIDs                []uint32
-	mode                     volume_server_pb.VolumeScrubMode
-	forceDeletedNeedlesCheck bool
-	grpcDialOption           grpc.DialOption
 }
 
 func (c *commandEcVolumeScrub) Name() string {
@@ -62,10 +53,14 @@ func (c *commandEcVolumeScrub) Do(args []string, commandEnv *CommandEnv, writer
 		return
 	}
 
-	c.volumeServerAddrs = []pb.ServerAddress{}
+	volumeServerAddrs := []pb.ServerAddress{}
 	if *nodesStr != "" {
 		for _, addr := range strings.Split(*nodesStr, ",") {
-			c.volumeServerAddrs = append(c.volumeServerAddrs, pb.ServerAddress(addr))
+			addr = strings.TrimSpace(addr)
+			if addr == "" {
+				continue
+			}
+			volumeServerAddrs = append(volumeServerAddrs, pb.ServerAddress(addr))
 		}
 	} else {
 		dns, err := collectDataNodes(commandEnv, 0)
@@ -73,11 +68,11 @@ func (c *commandEcVolumeScrub) Do(args []string, commandEnv *CommandEnv, writer
 			return err
 		}
 		for _, dn := range dns {
-			c.volumeServerAddrs = append(c.volumeServerAddrs, pb.ServerAddress(dn.Address))
+			volumeServerAddrs = append(volumeServerAddrs, pb.ServerAddress(dn.Address))
 		}
 	}
 
-	c.volumeIDs = []uint32{}
+	volumeIDs := []uint32{}
 	if *volumeIDsStr != "" {
 		for _, vids := range strings.Split(*volumeIDsStr, ",") {
 			vids = strings.TrimSpace(vids)
@@ -85,96 +80,30 @@ func (c *commandEcVolumeScrub) Do(args []string, commandEnv *CommandEnv, writer
 				continue
 			}
 			if vid, err := strconv.ParseUint(vids, 10, 32); err == nil {
-				c.volumeIDs = append(c.volumeIDs, uint32(vid))
+				volumeIDs = append(volumeIDs, uint32(vid))
 			} else {
 				return fmt.Errorf("invalid volume ID %q", vids)
 			}
 		}
 	}
 
+	var scrubMode volume_server_pb.VolumeScrubMode
 	switch strings.ToUpper(*mode) {
 	case "INDEX":
-		c.mode = volume_server_pb.VolumeScrubMode_INDEX
+		scrubMode = volume_server_pb.VolumeScrubMode_INDEX
 	case "LOCAL":
-		c.mode = volume_server_pb.VolumeScrubMode_LOCAL
+		scrubMode = volume_server_pb.VolumeScrubMode_LOCAL
 	case "FULL":
-		c.mode = volume_server_pb.VolumeScrubMode_FULL
+		scrubMode = volume_server_pb.VolumeScrubMode_FULL
 	case "CHECKSUM":
-		c.mode = volume_server_pb.VolumeScrubMode_CHECKSUM
+		scrubMode = volume_server_pb.VolumeScrubMode_CHECKSUM
 	default:
 		return fmt.Errorf("unsupported scrubbing mode %q", *mode)
 	}
-	fmt.Fprintf(writer, "using %s mode\n", c.mode.String())
-	c.env = commandEnv
-	c.forceDeletedNeedlesCheck = *forceDeletedNeedlesCheck
-	if c.forceDeletedNeedlesCheck && c.mode != volume_server_pb.VolumeScrubMode_FULL {
+	fmt.Fprintf(writer, "using %s mode\n", scrubMode.String())
+	if *forceDeletedNeedlesCheck && scrubMode != volume_server_pb.VolumeScrubMode_FULL {
 		return fmt.Errorf("deleted needle checks are only supported for FULL scrubs")
 	}
 
-	return c.scrubEcVolumes(writer, *maxParallelization, *showDetails)
-}
-
-func (c *commandEcVolumeScrub) scrubEcVolumes(writer io.Writer, maxParallelization int, showDetails bool) error {
-	var brokenVolumesStr, brokenShardsStr []string
-	var details []string
-	var totalVolumes, brokenVolumes, brokenShards, totalFiles uint64
-	var mu sync.Mutex
-
-	ewg := NewErrorWaitGroup(maxParallelization)
-	count := 0
-	for _, addr := range c.volumeServerAddrs {
-		ewg.Add(func() error {
-			mu.Lock()
-			count++
-			fmt.Fprintf(writer, "Scrubbing %s (%d/%d)...\n", addr.String(), count, len(c.volumeServerAddrs))
-			mu.Unlock()
-
-			err := operation.WithVolumeServerClient(false, addr, c.env.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
-				res, err := volumeServerClient.ScrubEcVolume(context.Background(), &volume_server_pb.ScrubEcVolumeRequest{
-					Mode:                     c.mode,
-					VolumeIds:                c.volumeIDs,
-					ForceDeletedNeedlesCheck: c.forceDeletedNeedlesCheck,
-				})
-				if err != nil {
-					return err
-				}
-
-				mu.Lock()
-				defer mu.Unlock()
-
-				totalVolumes += res.GetTotalVolumes()
-				totalFiles += res.GetTotalFiles()
-				brokenVolumes += uint64(len(res.GetBrokenVolumeIds()))
-				brokenShards += uint64(len(res.GetBrokenShardInfos()))
-				for _, d := range res.GetDetails() {
-					details = append(details, fmt.Sprintf("[%s] %s", addr, d))
-				}
-				for _, vid := range res.GetBrokenVolumeIds() {
-					brokenVolumesStr = append(brokenVolumesStr, fmt.Sprintf("%s:%v", addr, vid))
-				}
-				for _, si := range res.GetBrokenShardInfos() {
-					brokenShardsStr = append(brokenShardsStr, fmt.Sprintf("%s:%v:%v", addr, si.VolumeId, si.ShardId))
-				}
-
-				return nil
-			})
-			return err
-		})
-	}
-	if err := ewg.Wait(); err != nil {
-		return err
-	}
-
-	fmt.Fprintf(writer, "Scrubbed %d EC files and %d volumes on %d nodes\n", totalFiles, totalVolumes, len(c.volumeServerAddrs))
-	if brokenVolumes != 0 {
-		fmt.Fprintf(writer, "\nGot scrub failures on %d EC volumes and %d EC shards :(\n", brokenVolumes, brokenShards)
-		fmt.Fprintf(writer, "Affected volumes: %s\n", strings.Join(brokenVolumesStr, ", "))
-		if len(brokenShardsStr) != 0 {
-			fmt.Fprintf(writer, "Affected shards:  %s\n", strings.Join(brokenShardsStr, ", "))
-		}
-		if showDetails && len(details) != 0 {
-			fmt.Fprintf(writer, "Details:\n\t%s\n", strings.Join(details, "\n\t"))
-		}
-	}
-	return nil
+	return ec.ScrubEcVolumes(commandEnv.ecEnv(), writer, volumeServerAddrs, volumeIDs, scrubMode, *forceDeletedNeedlesCheck, *maxParallelization, *showDetails)
 }

weed/shell/command_ec_shard_unmount.go

@@ -1,76 +1,19 @@
 package shell
 
 import (
-	"context"
 	"flag"
 	"fmt"
 	"io"
 	"math"
-	"sort"
-	"strconv"
-	"strings"
-	"time"
 
-	"github.com/seaweedfs/seaweedfs/weed/operation"
-	"github.com/seaweedfs/seaweedfs/weed/pb"
-	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
-	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
-	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+	"github.com/seaweedfs/seaweedfs/weed/ec"
 )
 
-const (
-	ecShardActionTimeout = 1 * time.Minute
-)
-
-type ecShard struct {
-	ShardID     uint32
-	Collection  string
-	NodeAddress string
-}
-
-func (s *ecShard) String() string {
-	if s.NodeAddress == "" {
-		return fmt.Sprintf("%d", s.ShardID)
-	}
-	return fmt.Sprintf("%d@%s", s.ShardID, s.NodeAddress)
-}
-
-func ecShardsFromString(shards string) ([]*ecShard, error) {
-	res := []*ecShard{}
-
-	for _, s := range strings.Split(shards, ",") {
-		s = strings.TrimSpace(s)
-		if s == "" {
-			return nil, fmt.Errorf("empty shard ID in %q", shards)
-		}
-
-		// optional <shard ID>@<node address> to pick one copy
-		idStr, addr, _ := strings.Cut(s, "@")
-
-		id, err := strconv.Atoi(idStr)
-		if err != nil || id < 0 || id >= erasure_coding.MaxShardCount {
-			return nil, fmt.Errorf("invalid shard ID %q", s)
-		}
-
-		res = append(res, &ecShard{ShardID: uint32(id), NodeAddress: addr})
-	}
-
-	return res, nil
-}
-
 func init() {
 	Commands = append(Commands, &commandEcShardUnmount{})
 }
 
 type commandEcShardUnmount struct {
-	env           *CommandEnv
-	writer        io.Writer
-	topology      *master_pb.TopologyInfo
-	volumeID      uint32
-	delete        bool
-	ignoreInvalid bool
-	apply         bool
-	shards        []*ecShard
 }
 
 func (c *commandEcShardUnmount) Name() string {
@@ -107,7 +50,6 @@ func (c *commandEcShardUnmount) Do(args []string, commandEnv *CommandEnv, writer
 	if handleHelpRequest(c, args, writer) {
 		return nil
 	}
-
 	ecShardUnmountCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
 	volumeID := ecShardUnmountCommand.Uint("volumeId", 0, "volume ID for the shards to process")
 	shardIDsStr := ecShardUnmountCommand.String("shardId", "", "comma-separated EC shard IDs for the volume")
@@ -133,7 +75,7 @@ func (c *commandEcShardUnmount) Do(args []string, commandEnv *CommandEnv, writer
 	if *shardIDsStr == "" {
 		return fmt.Errorf("missing shardId")
 	}
-	shards, err := ecShardsFromString(*shardIDsStr)
+	shards, err := ec.ShardRefsFromString(*shardIDsStr)
 	if err != nil {
 		return err
 	}
@@ -144,136 +86,12 @@ func (c *commandEcShardUnmount) Do(args []string, commandEnv *CommandEnv, writer
 		return err
 	}
 
-	c.env = commandEnv
-	c.writer = writer
-	c.topology = topology
-	c.volumeID = uint32(*volumeID)
-	c.delete = *delete
-	c.ignoreInvalid = *ignoreInvalid
-	c.apply = *apply || *applyAlias
-	c.shards = shards
-
-	return c.doShardsUnmount()
-}
-
-func (c *commandEcShardUnmount) write(format string, a ...any) {
-	fmt.Fprintf(c.writer, format, a...)
-}
-
-func (c *commandEcShardUnmount) liveShardsForVolume() []*ecShard {
-	shards := []*ecShard{}
-
-	for _, dci := range c.topology.GetDataCenterInfos() {
-		for _, ri := range dci.GetRackInfos() {
-			for _, dni := range ri.GetDataNodeInfos() {
-				nodeAddress := dni.GetAddress()
-				for _, di := range dni.GetDiskInfos() {
-					for _, eci := range di.GetEcShardInfos() {
-						if eci.GetId() == c.volumeID {
-							sinfo := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(eci)
-							for _, sid := range sinfo.Ids() {
-								shards = append(shards, &ecShard{
-									ShardID:     uint32(sid),
-									Collection:  eci.GetCollection(),
-									NodeAddress: nodeAddress,
-								})
-							}
-						}
-					}
-				}
-			}
-		}
-	}
-
-	sort.SliceStable(shards, func(i, j int) bool { return shards[i].ShardID < shards[j].ShardID })
-	return shards
-}
-
-func (c *commandEcShardUnmount) printShards(ss []*ecShard) {
-	for _, s := range ss {
-		c.write("\t%v\n", s)
-	}
-	c.write("\n")
-}
-
-func (c *commandEcShardUnmount) doShardsUnmount() error {
-	liveShards := c.liveShardsForVolume()
-	c.write("Live shard topology for volume ID %d (%d shards):\n", c.volumeID, len(liveShards))
-	c.printShards(liveShards)
-
-	// resolve target shards against the live topology
-	targetShards := []*ecShard{}
-	for _, ps := range c.shards {
-		var result *ecShard
-		for _, ts := range liveShards {
-			if ts.ShardID == ps.ShardID {
-				if ps.NodeAddress == "" || ps.NodeAddress == ts.NodeAddress {
-					if result != nil {
-						return fmt.Errorf("shard %v is ambiguous", ps)
-					}
-					result = ts
-				}
-			}
-		}
-		if result == nil {
-			if !c.ignoreInvalid {
-				return fmt.Errorf("shard %v is invalid", ps)
-			}
-			c.write("!!! ignoring invalid shard %v\n", ps)
-		} else {
-			targetShards = append(targetShards, result)
-		}
-	}
-	if len(targetShards) == 0 {
-		return fmt.Errorf("got no shards to process")
-	}
-
-	mode := "unmount"
-	if c.delete {
-		mode = "unmount + delete"
-	}
-	c.write("Will %s %d shard(s):\n", mode, len(targetShards))
-	c.printShards(targetShards)
-
-	if !c.apply {
-		c.write("Not proceeding in dry-run mode\n")
-		return nil
-	}
-
-	for _, s := range targetShards {
-		if err := c.unmountShard(s); err != nil {
-			return err
-		}
-	}
-
-	c.write("\nAll done!\n")
-	return nil
-}
-
-func (c *commandEcShardUnmount) unmountShard(s *ecShard) error {
-	ctx, cancel := context.WithTimeout(context.Background(), ecShardActionTimeout)
-	defer cancel()
-
-	return operation.WithVolumeServerClient(false, pb.ServerAddress(s.NodeAddress), c.env.option.GrpcDialOption, func(vsc volume_server_pb.VolumeServerClient) error {
-		c.write("Unmounting shard %v for volume ID %d...\n", s, c.volumeID)
-		if _, err := vsc.VolumeEcShardsUnmount(ctx, &volume_server_pb.VolumeEcShardsUnmountRequest{
-			VolumeId: c.volumeID,
-			ShardIds: []uint32{s.ShardID},
-		}); err != nil {
-			return err
-		}
-
-		if c.delete {
-			c.write("Deleting shard %v for volume ID %d...\n", s, c.volumeID)
-			if _, err := vsc.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{
-				VolumeId:   c.volumeID,
-				Collection: s.Collection,
-				ShardIds:   []uint32{s.ShardID},
-			}); err != nil {
-				return err
-			}
-		}
-
-		return nil
+	return ec.UnmountShards(commandEnv.ecEnv(), writer, ec.ShardUnmountRequest{
+		Topology:      topology,
+		VolumeID:      uint32(*volumeID),
+		Shards:        shards,
+		Delete:        *delete,
+		IgnoreInvalid: *ignoreInvalid,
+		Apply:         *apply || *applyAlias,
 	})
 }

weed/shell/command_volume_server_evacuate.go

@@ -177,7 +177,7 @@ func (c *commandVolumeServerEvacuate) evacuateEcVolumes(commandEnv *CommandEnv,
 
 		// move away ec volumes for this disk type
 		for _, thisNode := range thisNodes {
-			diskInfo, found := thisNode.info.DiskInfos[string(diskType)]
+			diskInfo, found := thisNode.Info.DiskInfos[string(diskType)]
 			if !found {
 				continue
 			}
@@ -205,12 +205,12 @@ func (c *commandVolumeServerEvacuate) moveAwayOneEcVolume(commandEnv *CommandEnv
 		// Sort by: 1) fewest shards of this volume, 2) most free EC slots
 		// This ensures we prefer nodes with capacity and balanced shard distribution
 		slices.SortFunc(otherNodes, func(a, b *EcNode) int {
-			aShards := a.localShardIdCount(ecShardInfo.Id)
-			bShards := b.localShardIdCount(ecShardInfo.Id)
+			aShards := a.LocalShardIdCount(ecShardInfo.Id)
+			bShards := b.LocalShardIdCount(ecShardInfo.Id)
 			if aShards != bShards {
 				return aShards - bShards // Prefer fewer shards
 			}
-			return b.freeEcSlot - a.freeEcSlot // Then prefer more free slots
+			return b.FreeEcSlot - a.FreeEcSlot // Then prefer more free slots
 		})
 
 		shardMoved := false
@@ -219,7 +219,7 @@ func (c *commandVolumeServerEvacuate) moveAwayOneEcVolume(commandEnv *CommandEnv
 			emptyNode := otherNodes[i]
 
 			// Skip nodes with no free EC slots
-			if emptyNode.freeEcSlot <= 0 {
+			if emptyNode.FreeEcSlot <= 0 {
 				skippedNodes++
 				continue
 			}
@@ -233,9 +233,9 @@ func (c *commandVolumeServerEvacuate) moveAwayOneEcVolume(commandEnv *CommandEnv
 			// No anti-affinity needed for evacuation (dataShardCount=0)
 			destDiskId := pickBestDiskOnNode(emptyNode, vid, diskType, false, shardId, 0)
 			if destDiskId > 0 {
-				fmt.Fprintf(writer, "moving ec volume %s%d.%d %s => %s (disk %d)\n", collectionPrefix, ecShardInfo.Id, shardId, thisNode.info.Id, emptyNode.info.Id, destDiskId)
+				fmt.Fprintf(writer, "moving ec volume %s%d.%d %s => %s (disk %d)\n", collectionPrefix, ecShardInfo.Id, shardId, thisNode.Info.Id, emptyNode.Info.Id, destDiskId)
 			} else {
-				fmt.Fprintf(writer, "moving ec volume %s%d.%d %s => %s\n", collectionPrefix, ecShardInfo.Id, shardId, thisNode.info.Id, emptyNode.info.Id)
+				fmt.Fprintf(writer, "moving ec volume %s%d.%d %s => %s\n", collectionPrefix, ecShardInfo.Id, shardId, thisNode.Info.Id, emptyNode.Info.Id)
 			}
 			err = moveMountedShardToEcNode(commandEnv, thisNode, ecShardInfo.Collection, vid, shardId, emptyNode, destDiskId, applyChange, diskType)
 			if err != nil {
@@ -245,7 +245,7 @@ func (c *commandVolumeServerEvacuate) moveAwayOneEcVolume(commandEnv *CommandEnv
 				hasMoved = true
 				shardMoved = true
 				// Update the node's free slot count after successful move
-				emptyNode.freeEcSlot--
+				emptyNode.FreeEcSlot--
 				break
 			}
 		}
@@ -310,14 +310,14 @@ func (c *commandVolumeServerEvacuate) nodesOtherThan(volumeServers []*Node, this
 
 func (c *commandVolumeServerEvacuate) ecNodesOtherThan(volumeServers []*EcNode, thisServer string) (thisNodes []*EcNode, otherNodes []*EcNode) {
 	for _, node := range volumeServers {
-		if node.info.Id == thisServer || (*c.volumeRack != "" && string(node.rack) == *c.volumeRack) {
+		if node.Info.Id == thisServer || (*c.volumeRack != "" && string(node.Rack) == *c.volumeRack) {
 			thisNodes = append(thisNodes, node)
 			continue
 		}
-		if *c.volumeRack != "" && *c.volumeRack == string(node.rack) {
+		if *c.volumeRack != "" && *c.volumeRack == string(node.Rack) {
 			continue
 		}
-		if *c.targetServer != "" && *c.targetServer != node.info.Id {
+		if *c.targetServer != "" && *c.targetServer != node.Info.Id {
 			continue
 		}
 		otherNodes = append(otherNodes, node)

weed/shell/common.go

@@ -1,9 +1,7 @@
 package shell
 
 import (
-	"errors"
-	"fmt"
-	"sync"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
 var (
@@ -16,89 +14,9 @@ var (
 	CollectionDefault = "_default"
 )
 
-// ErrorWaitGroup implements a goroutine wait group which aggregates errors, if any.
-type ErrorWaitGroup struct {
-	maxConcurrency int
-	wg             *sync.WaitGroup
-	wgSem          chan bool
-	errors         []error
-	errorsMu       sync.Mutex
-}
+// ErrorWaitGroup lives in weed/util so packages outside the shell can share it.
+type ErrorWaitGroup = util.ErrorWaitGroup
+type ErrorWaitGroupTask = util.ErrorWaitGroupTask
 
-type ErrorWaitGroupTask func() error
-
-// executeParallelTaskGroups runs tasks from each group sequentially and runs
-// independent groups with up to maxParallelization concurrency. EC balancing
-// uses one group per volume because its shard sidecar files are shared; volume
-// balancing uses one group per already-reserved volume move.
-func executeParallelTaskGroups(maxParallelization int, taskGroups [][]ErrorWaitGroupTask) error {
-	ewg := NewErrorWaitGroup(maxParallelization)
-	for _, taskGroup := range taskGroups {
-		taskGroup := taskGroup
-		ewg.Add(func() error {
-			for _, task := range taskGroup {
-				if err := task(); err != nil {
-					return err
-				}
-			}
-			return nil
-		})
-	}
-	return ewg.Wait()
-}
-
-func NewErrorWaitGroup(maxConcurrency int) *ErrorWaitGroup {
-	if maxConcurrency <= 0 {
-		// no concurrency = one task at the time
-		maxConcurrency = 1
-	}
-	return &ErrorWaitGroup{
-		maxConcurrency: maxConcurrency,
-		wg:             &sync.WaitGroup{},
-		wgSem:          make(chan bool, maxConcurrency),
-	}
-}
-
-// Reset restarts an ErrorWaitGroup, keeping original settings. Errors and pending goroutines, if any, are flushed.
-func (ewg *ErrorWaitGroup) Reset() {
-	close(ewg.wgSem)
-
-	ewg.wg = &sync.WaitGroup{}
-	ewg.wgSem = make(chan bool, ewg.maxConcurrency)
-	ewg.errors = nil
-}
-
-// Add queues an ErrorWaitGroupTask to be executed as a goroutine.
-func (ewg *ErrorWaitGroup) Add(f ErrorWaitGroupTask) {
-	if ewg.maxConcurrency <= 1 {
-		// keep run order deterministic when parallelization is off
-		ewg.errors = append(ewg.errors, f())
-		return
-	}
-
-	ewg.wg.Add(1)
-	go func() {
-		ewg.wgSem <- true
-
-		err := f()
-		ewg.errorsMu.Lock()
-		ewg.errors = append(ewg.errors, err)
-		ewg.errorsMu.Unlock()
-
-		<-ewg.wgSem
-		ewg.wg.Done()
-	}()
-}
-
-// AddErrorf adds an error to an ErrorWaitGroupTask result, without queueing any goroutines.
-func (ewg *ErrorWaitGroup) AddErrorf(format string, a ...interface{}) {
-	ewg.errorsMu.Lock()
-	ewg.errors = append(ewg.errors, fmt.Errorf(format, a...))
-	ewg.errorsMu.Unlock()
-}
-
-// Wait sleeps until all ErrorWaitGroupTasks are completed, then returns errors for them.
-func (ewg *ErrorWaitGroup) Wait() error {
-	ewg.wg.Wait()
-	return errors.Join(ewg.errors...)
-}
+var NewErrorWaitGroup = util.NewErrorWaitGroup
+var executeParallelTaskGroups = util.ExecuteParallelTaskGroups

weed/shell/common_test.go

@@ -2,9 +2,6 @@ package shell
 
 import (
 	_ "embed"
-	"errors"
-	"testing"
-	"time"
 )
 
 //go:embed volume.list.txt
@@ -21,90 +18,3 @@ var (
 	testTopology2  = parseOutput(topoData2)
 	testTopologyEc = parseOutput(topoDataEc)
 )
-
-func TestExecuteParallelTaskGroups(t *testing.T) {
-	firstTaskStarted := make(chan struct{})
-	firstTaskRelease := make(chan struct{})
-	secondTaskStarted := make(chan struct{})
-	secondTaskInFirstGroupStarted := make(chan struct{})
-	done := make(chan error, 1)
-
-	go func() {
-		done <- executeParallelTaskGroups(2, [][]ErrorWaitGroupTask{
-			{
-				func() error {
-					close(firstTaskStarted)
-					<-firstTaskRelease
-					return nil
-				},
-				func() error {
-					close(secondTaskInFirstGroupStarted)
-					return nil
-				},
-			},
-			{
-				func() error {
-					close(secondTaskStarted)
-					return nil
-				},
-			},
-		})
-	}()
-
-	select {
-	case <-firstTaskStarted:
-	case <-time.After(time.Second):
-		t.Fatal("first task did not start")
-	}
-	select {
-	case <-secondTaskStarted:
-	case <-time.After(time.Second):
-		t.Fatal("independent task group did not start in parallel")
-	}
-	select {
-	case <-secondTaskInFirstGroupStarted:
-		t.Fatal("tasks in the same group ran in parallel")
-	default:
-	}
-
-	close(firstTaskRelease)
-	select {
-	case err := <-done:
-		if err != nil {
-			t.Fatalf("execute parallel task groups: %v", err)
-		}
-	case <-time.After(time.Second):
-		t.Fatal("parallel task groups did not finish")
-	}
-}
-
-func TestExecuteParallelTaskGroupsStopsOnlyFailedGroup(t *testing.T) {
-	expectedErr := errors.New("move failed")
-	failedGroupContinued := false
-	otherGroupRan := false
-
-	err := executeParallelTaskGroups(1, [][]ErrorWaitGroupTask{
-		{
-			func() error { return expectedErr },
-			func() error {
-				failedGroupContinued = true
-				return nil
-			},
-		},
-		{
-			func() error {
-				otherGroupRan = true
-				return nil
-			},
-		},
-	})
-	if !errors.Is(err, expectedErr) {
-		t.Fatalf("expected task error, got %v", err)
-	}
-	if failedGroupContinued {
-		t.Fatal("task after a failed task in the same group ran")
-	}
-	if !otherGroupRan {
-		t.Fatal("independent task group did not run")
-	}
-}

weed/shell/ec_proportional_rebalance.go

@@ -1,38 +0,0 @@
-package shell
-
-import (
-	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
-	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/distribution"
-	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
-	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
-	"github.com/seaweedfs/seaweedfs/weed/storage/types"
-)
-
-// ECDistribution is an alias to the distribution package type for backward compatibility
-type ECDistribution = distribution.ECDistribution
-
-// TopologyDistributionAnalysis holds the current shard distribution analysis
-// This wraps the distribution package's TopologyAnalysis with shell-specific EcNode handling
-type TopologyDistributionAnalysis struct {
-	inner *distribution.TopologyAnalysis
-
-	// Shell-specific mappings
-	nodeMap map[string]*EcNode // nodeID -> EcNode
-}
-
-// ECShardMove represents a planned shard move (shell-specific with EcNode references)
-type ECShardMove struct {
-	VolumeId   needle.VolumeId
-	ShardId    erasure_coding.ShardId
-	SourceNode *EcNode
-	DestNode   *EcNode
-	Reason     string
-}
-
-// ProportionalECRebalancer implements proportional shard distribution for shell commands
-type ProportionalECRebalancer struct {
-	ecNodes          []*EcNode
-	replicaPlacement *super_block.ReplicaPlacement
-	diskType         types.DiskType
-	ecConfig         distribution.ECConfig
-}

weed/util/error_wait_group.go

@@ -0,0 +1,94 @@
+package util
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+)
+
+// ErrorWaitGroup implements a goroutine wait group which aggregates errors, if any.
+type ErrorWaitGroup struct {
+	maxConcurrency int
+	wg             *sync.WaitGroup
+	wgSem          chan bool
+	errors         []error
+	errorsMu       sync.Mutex
+}
+
+type ErrorWaitGroupTask func() error
+
+// ExecuteParallelTaskGroups runs tasks from each group sequentially and runs
+// independent groups with up to maxParallelization concurrency. EC balancing
+// uses one group per volume because its shard sidecar files are shared; volume
+// balancing uses one group per already-reserved volume move.
+func ExecuteParallelTaskGroups(maxParallelization int, taskGroups [][]ErrorWaitGroupTask) error {
+	ewg := NewErrorWaitGroup(maxParallelization)
+	for _, taskGroup := range taskGroups {
+		taskGroup := taskGroup
+		ewg.Add(func() error {
+			for _, task := range taskGroup {
+				if err := task(); err != nil {
+					return err
+				}
+			}
+			return nil
+		})
+	}
+	return ewg.Wait()
+}
+
+func NewErrorWaitGroup(maxConcurrency int) *ErrorWaitGroup {
+	if maxConcurrency <= 0 {
+		// no concurrency = one task at the time
+		maxConcurrency = 1
+	}
+	return &ErrorWaitGroup{
+		maxConcurrency: maxConcurrency,
+		wg:             &sync.WaitGroup{},
+		wgSem:          make(chan bool, maxConcurrency),
+	}
+}
+
+// Reset restarts an ErrorWaitGroup, keeping original settings. Errors and pending goroutines, if any, are flushed.
+func (ewg *ErrorWaitGroup) Reset() {
+	close(ewg.wgSem)
+
+	ewg.wg = &sync.WaitGroup{}
+	ewg.wgSem = make(chan bool, ewg.maxConcurrency)
+	ewg.errors = nil
+}
+
+// Add queues an ErrorWaitGroupTask to be executed as a goroutine.
+func (ewg *ErrorWaitGroup) Add(f ErrorWaitGroupTask) {
+	if ewg.maxConcurrency <= 1 {
+		// keep run order deterministic when parallelization is off
+		ewg.errors = append(ewg.errors, f())
+		return
+	}
+
+	ewg.wg.Add(1)
+	go func() {
+		ewg.wgSem <- true
+
+		err := f()
+		ewg.errorsMu.Lock()
+		ewg.errors = append(ewg.errors, err)
+		ewg.errorsMu.Unlock()
+
+		<-ewg.wgSem
+		ewg.wg.Done()
+	}()
+}
+
+// AddErrorf adds an error to an ErrorWaitGroupTask result, without queueing any goroutines.
+func (ewg *ErrorWaitGroup) AddErrorf(format string, a ...interface{}) {
+	ewg.errorsMu.Lock()
+	ewg.errors = append(ewg.errors, fmt.Errorf(format, a...))
+	ewg.errorsMu.Unlock()
+}
+
+// Wait sleeps until all ErrorWaitGroupTasks are completed, then returns errors for them.
+func (ewg *ErrorWaitGroup) Wait() error {
+	ewg.wg.Wait()
+	return errors.Join(ewg.errors...)
+}

weed/util/error_wait_group_test.go

@@ -0,0 +1,94 @@
+package util
+
+import (
+	"errors"
+	"testing"
+	"time"
+)
+
+func TestExecuteParallelTaskGroups(t *testing.T) {
+	firstTaskStarted := make(chan struct{})
+	firstTaskRelease := make(chan struct{})
+	secondTaskStarted := make(chan struct{})
+	secondTaskInFirstGroupStarted := make(chan struct{})
+	done := make(chan error, 1)
+
+	go func() {
+		done <- ExecuteParallelTaskGroups(2, [][]ErrorWaitGroupTask{
+			{
+				func() error {
+					close(firstTaskStarted)
+					<-firstTaskRelease
+					return nil
+				},
+				func() error {
+					close(secondTaskInFirstGroupStarted)
+					return nil
+				},
+			},
+			{
+				func() error {
+					close(secondTaskStarted)
+					return nil
+				},
+			},
+		})
+	}()
+
+	select {
+	case <-firstTaskStarted:
+	case <-time.After(time.Second):
+		t.Fatal("first task did not start")
+	}
+	select {
+	case <-secondTaskStarted:
+	case <-time.After(time.Second):
+		t.Fatal("independent task group did not start in parallel")
+	}
+	select {
+	case <-secondTaskInFirstGroupStarted:
+		t.Fatal("tasks in the same group ran in parallel")
+	default:
+	}
+
+	close(firstTaskRelease)
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Fatalf("execute parallel task groups: %v", err)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("parallel task groups did not finish")
+	}
+}
+
+func TestExecuteParallelTaskGroupsStopsOnlyFailedGroup(t *testing.T) {
+	expectedErr := errors.New("move failed")
+	failedGroupContinued := false
+	otherGroupRan := false
+
+	err := ExecuteParallelTaskGroups(1, [][]ErrorWaitGroupTask{
+		{
+			func() error { return expectedErr },
+			func() error {
+				failedGroupContinued = true
+				return nil
+			},
+		},
+		{
+			func() error {
+				otherGroupRan = true
+				return nil
+			},
+		},
+	})
+	if !errors.Is(err, expectedErr) {
+		t.Fatalf("expected task error, got %v", err)
+	}
+	if failedGroupContinued {
+		t.Fatal("task after a failed task in the same group ran")
+	}
+	if !otherGroupRan {
+		t.Fatal("independent task group did not run")
+	}
+}

weed/worker/tasks/erasure_coding/detection.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/admin/topology"
+	"github.com/seaweedfs/seaweedfs/weed/ec"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/operation"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -757,50 +758,12 @@ func cleanupOrphanSourceReplicas(ctx context.Context, clusterInfo *types.Cluster
 	return deleted, nil
 }
 
-// countExistingEcShardsForVolume returns the number of distinct EC shard IDs
-// for (volumeID, collection) present in the topology, counting only the single
-// largest encode generation. Shards are grouped by encode_ts_ns (the per-encode
-// identity from .vif), so two interrupted encode runs whose shard sets overlap
-// are never unioned into a false-complete set that would wrongly trigger the
-// orphaned-source delete. Walks every disk's EcIndexBits bitmap rather than
-// trusting len(EcShardInfos), because a single info entry can carry multiple
-// shards. Shards reporting encode_ts_ns==0 (pre-upgrade servers) form their own
-// generation bucket.
-//
-// Limitation: the heartbeat carries one encode_ts_ns per (volume, disk), so this
-// separates generations living on different disks; same-disk mixing is prevented
-// upstream by the pre-encode artifact wipe and the cross-run read guard.
+// countExistingEcShardsForVolume counts (volumeID, collection) EC shards in the
+// topology, counting only the single largest encode generation; see
+// ec.CountExistingEcShardsForVolume for the generation semantics.
 func countExistingEcShardsForVolume(activeTopology *topology.ActiveTopology, volumeID uint32, collection string) int {
 	if activeTopology == nil {
 		return 0
 	}
-	topologyInfo := activeTopology.GetTopologyInfo()
-	if topologyInfo == nil {
-		return 0
-	}
-	perGeneration := make(map[int64]erasure_coding.ShardBits)
-	for _, dc := range topologyInfo.DataCenterInfos {
-		for _, rack := range dc.RackInfos {
-			for _, node := range rack.DataNodeInfos {
-				for _, diskInfo := range node.DiskInfos {
-					for _, ecShardInfo := range diskInfo.EcShardInfos {
-						if ecShardInfo == nil {
-							continue
-						}
-						if ecShardInfo.Id != volumeID || ecShardInfo.Collection != collection {
-							continue
-						}
-						perGeneration[ecShardInfo.EncodeTsNs] |= erasure_coding.ShardBits(ecShardInfo.EcIndexBits)
-					}
-				}
-			}
-		}
-	}
-	best := 0
-	for _, bits := range perGeneration {
-		if c := bits.Count(); c > best {
-			best = c
-		}
-	}
-	return best
+	return ec.CountExistingEcShardsForVolume(activeTopology.GetTopologyInfo(), volumeID, collection)
 }