diff --git a/weed/ec/ec_check_replication.go b/weed/ec/ec_check_replication.go new file mode 100644 index 000000000..6f445b482 --- /dev/null +++ b/weed/ec/ec_check_replication.go @@ -0,0 +1,211 @@ +package ec + +import ( + "fmt" + "io" + "slices" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" +) + +// CheckEcVolumeReplication reports EC volumes with under- or over-replicated +// shards among the given data nodes. volumeIDs, when non-empty, narrows the +// check to those volume ids. +func CheckEcVolumeReplication(writer io.Writer, dataNodes []*master_pb.DataNodeInfo, volumeIDs map[uint32]bool, showDetails bool) error { + runner := &ecCheckReplicationRunner{ + writer: writer, + dataNodes: dataNodes, + volumeIDMap: volumeIDs, + } + return runner.checkEcVolumes(showDetails) +} + +type ecCheckReplicationRunner struct { + writer io.Writer + dataNodes []*master_pb.DataNodeInfo + volumeIDMap map[uint32]bool +} + +func (r *ecCheckReplicationRunner) write(format string, a ...any) { + fmt.Fprintf(r.writer, format, a...) +} + +func (r *ecCheckReplicationRunner) isVolumeIDValid(vid uint32) bool { + if len(r.volumeIDMap) == 0 { + return true + } + return r.volumeIDMap[vid] +} + +// ecVolumeShardReplication aggregates the observed shards for a single EC volume, +// together with the data+parity ratio the volume was encoded with. The ratio is +// taken per volume (via erasure_coding.EcShardsVolume*Shards) so custom EC +// ratios are checked against their own expected shard count. +type ecVolumeShardReplication struct { + dataShards int + parityShards int + // shardAddresses maps a shard id to the sorted node addresses hosting it. + shardAddresses map[erasure_coding.ShardId][]string +} + +func (h *ecVolumeShardReplication) totalShards() int { + return h.dataShards + h.parityShards +} + +// replicaCount is the total number of shard copies observed, counting each +// over-replicated shard once per hosting node. +func (h *ecVolumeShardReplication) replicaCount() int { + n := 0 + for _, addrs := range h.shardAddresses { + n += len(addrs) + } + return n +} + +// unexpectedShardIds returns, in ascending order, any observed shard ids that +// fall outside the volume's expected data+parity range. These are anomalies +// (e.g. a stray shard) and are treated as over-replication. +func (h *ecVolumeShardReplication) unexpectedShardIds() []erasure_coding.ShardId { + var ids []erasure_coding.ShardId + for sid := range h.shardAddresses { + if int(sid) >= h.totalShards() { + ids = append(ids, sid) + } + } + slices.Sort(ids) + return ids +} + +// TODO: check shard sizes? +func (r *ecCheckReplicationRunner) checkEcVolumes(showDetails bool) error { + // collect EC shard placement, keyed by volume id + volumes := map[uint32]*ecVolumeShardReplication{} + for _, dni := range r.dataNodes { + nodeAddress := dni.GetAddress() + for _, di := range dni.GetDiskInfos() { + for _, eci := range di.GetEcShardInfos() { + vid := eci.GetId() + if !r.isVolumeIDValid(vid) { + continue + } + + h, ok := volumes[vid] + if !ok { + // all shards of a volume share one ratio; take it from the + // first shard message seen for the volume. + h = &ecVolumeShardReplication{ + dataShards: erasure_coding.EcShardsVolumeDataShards(eci), + parityShards: erasure_coding.EcShardsVolumeParityShards(eci), + shardAddresses: map[erasure_coding.ShardId][]string{}, + } + volumes[vid] = h + } + + sinfo := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(eci) + for _, sid := range sinfo.Ids() { + h.shardAddresses[sid] = append(h.shardAddresses[sid], nodeAddress) + } + } + } + } + + if len(volumes) == 0 { + // Asking about specific volume IDs that turn out not to be EC volumes is + // an error; an unfiltered run over a cluster with no EC volumes is a + // legitimate, healthy state rather than a failure. + if len(r.volumeIDMap) > 0 { + return fmt.Errorf("no EC volumes found") + } + r.write("No EC volumes found.\n") + return nil + } + + // keep the shard address lists nicely sorted, for the sake of readability. + for _, h := range volumes { + for _, addrs := range h.shardAddresses { + slices.Sort(addrs) + } + } + + // classify each volume against its own expected shard count + underreplicatedVolumeIDs := []uint32{} + overreplicatedVolumeIDs := []uint32{} + for vid, h := range volumes { + under := false + over := false + for sid := 0; sid < h.totalShards(); sid++ { + switch len(h.shardAddresses[erasure_coding.ShardId(sid)]) { + case 0: + under = true + case 1: + default: + over = true + } + } + // shard ids beyond the expected data+parity range are unexpected extras, + // i.e. redundant data the ratio doesn't call for. + if len(h.unexpectedShardIds()) > 0 { + over = true + } + + // under- and over-replication are independent problems (missing shards risk + // data loss, redundant shards waste space), so a volume exhibiting both is + // reported in both lists. + if under { + underreplicatedVolumeIDs = append(underreplicatedVolumeIDs, vid) + } + if over { + overreplicatedVolumeIDs = append(overreplicatedVolumeIDs, vid) + } + } + slices.Sort(underreplicatedVolumeIDs) + slices.Sort(overreplicatedVolumeIDs) + + // ...and display results + if len(underreplicatedVolumeIDs) == 0 && len(overreplicatedVolumeIDs) == 0 { + r.write("EC volumes are healthy.\n") + return nil + } + + if len(underreplicatedVolumeIDs) != 0 { + r.write("Found %d/%d under-replicated EC volumes: %v\n", len(underreplicatedVolumeIDs), len(volumes), underreplicatedVolumeIDs) + } + if len(overreplicatedVolumeIDs) != 0 { + r.write("Found %d/%d over-replicated EC volumes: %v\n", len(overreplicatedVolumeIDs), len(volumes), overreplicatedVolumeIDs) + } + + if showDetails { + if len(underreplicatedVolumeIDs) != 0 { + r.write("\n") + r.writeShardMaps("under-replicated", underreplicatedVolumeIDs, volumes) + } + if len(overreplicatedVolumeIDs) != 0 { + r.write("\n") + r.writeShardMaps("over-replicated", overreplicatedVolumeIDs, volumes) + } + } + + return nil +} + +func (r *ecCheckReplicationRunner) writeShardMaps(kind string, volumeIDs []uint32, volumes map[uint32]*ecVolumeShardReplication) { + for _, vid := range volumeIDs { + h := volumes[vid] + r.write("Shards map for %s EC volume %v (%d/%d shards):\n", kind, vid, h.replicaCount(), h.totalShards()) + for sid := 0; sid < h.totalShards(); sid++ { + shardTypeDesc := "" + if sid >= h.dataShards { + shardTypeDesc = " (parity)" + } + if addrs, ok := h.shardAddresses[erasure_coding.ShardId(sid)]; ok { + r.write("\t%02d%s => %v\n", sid, shardTypeDesc, addrs) + } else { + r.write("\t%02d%s is missing\n", sid, shardTypeDesc) + } + } + for _, sid := range h.unexpectedShardIds() { + r.write("\t%02d (unexpected) => %v\n", int(sid), h.shardAddresses[sid]) + } + } +} diff --git a/weed/shell/command_ec_check_replication_test.go b/weed/ec/ec_check_replication_test.go similarity index 99% rename from weed/shell/command_ec_check_replication_test.go rename to weed/ec/ec_check_replication_test.go index f7621f76d..3a6b0460f 100644 --- a/weed/shell/command_ec_check_replication_test.go +++ b/weed/ec/ec_check_replication_test.go @@ -1,4 +1,4 @@ -package shell +package ec import ( "bytes" diff --git a/weed/ec/ec_common.go b/weed/ec/ec_common.go new file mode 100644 index 000000000..1fda10a09 --- /dev/null +++ b/weed/ec/ec_common.go @@ -0,0 +1,1154 @@ +package ec + +import ( + "context" + "fmt" + "os" + "slices" + "sort" + "strconv" + "strings" + "time" + + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/operation" + "github.com/seaweedfs/seaweedfs/weed/operation/volume_move" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/ecbalancer" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/super_block" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/topology/balancer" + "github.com/seaweedfs/seaweedfs/weed/util" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type DataCenterId string +type EcNodeId string +type RackId string + +// EcDisk represents a single disk on a volume server +type EcDisk struct { + DiskId uint32 + DiskType string + FreeEcSlots int + EcShardCount int // Total EC shards on this disk + // Map of volumeId -> ShardsInfo for shards on this disk + EcShards map[needle.VolumeId]*erasure_coding.ShardsInfo +} + +type EcNode struct { + Info *master_pb.DataNodeInfo + DC DataCenterId + Rack RackId + FreeEcSlot int + // Disks maps diskId -> EcDisk for disk-level balancing + Disks map[uint32]*EcDisk +} + +var BalanceAlgorithmDescription = ` + func EcBalance() { + for each collection: + balanceEcVolumes(collectionName) + for each rack: + balanceEcRack(rack) + } + + func balanceEcVolumes(collectionName){ + for each volume: + doDeduplicateEcShards(volumeId) + + tracks rack~shardCount mapping + for each volume: + doBalanceEcShardsAcrossRacks(volumeId) + + for each volume: + doBalanceEcShardsWithinRacks(volumeId) + } + + // spread ec shards into more racks + func doBalanceEcShardsAcrossRacks(volumeId){ + tracks rack~volumeIdShardCount mapping + averageShardsPerEcRack = totalShardNumber / numRacks // totalShardNumber is 14 for now, later could varies for each dc + ecShardsToMove = select overflown ec shards from racks with ec shard counts > averageShardsPerEcRack + for each ecShardsToMove { + destRack = pickOneRack(rack~shardCount, rack~volumeIdShardCount, ecShardReplicaPlacement) + destVolumeServers = volume servers on the destRack + pickOneEcNodeAndMoveOneShard(destVolumeServers) + } + } + + func doBalanceEcShardsWithinRacks(volumeId){ + racks = collect all racks that the volume id is on + for rack, shards := range racks + doBalanceEcShardsWithinOneRack(volumeId, shards, rack) + } + + // move ec shards + func doBalanceEcShardsWithinOneRack(volumeId, shards, rackId){ + tracks volumeServer~volumeIdShardCount mapping + averageShardCount = len(shards) / numVolumeServers + volumeServersOverAverage = volume servers with volumeId's ec shard counts > averageShardsPerEcRack + ecShardsToMove = select overflown ec shards from volumeServersOverAverage + for each ecShardsToMove { + destVolumeServer = pickOneVolumeServer(volumeServer~shardCount, volumeServer~volumeIdShardCount, ecShardReplicaPlacement) + pickOneEcNodeAndMoveOneShard(destVolumeServers) + } + } + + // move ec shards while keeping shard distribution for the same volume unchanged or more even + func balanceEcRack(rack){ + averageShardCount = total shards / numVolumeServers + for hasMovedOneEcShard { + sort all volume servers ordered by the number of local ec shards + pick the volume server A with the lowest number of ec shards x + pick the volume server B with the highest number of ec shards y + if y > averageShardCount and x +1 <= averageShardCount { + if B has a ec shard with volume id v that A does not have { + move one ec shard v from B to A + hasMovedOneEcShard = true + } + } + } + } + ` + +func CollectEcNodesForDC(env *Env, selectedDataCenter string, diskType types.DiskType) (ecNodes []*EcNode, totalFreeEcSlots int, err error) { + if env == nil || env.FetchTopology == nil { + return nil, 0, fmt.Errorf("no topology source configured") + } + // list all possible locations + // collect topology information + topologyInfo, _, err := env.FetchTopology(0) + if err != nil { + return + } + + // find out all volume servers with one slot left. + ecNodes, totalFreeEcSlots = CollectEcVolumeServersByDc(topologyInfo, selectedDataCenter, diskType) + + SortEcNodesByFreeslotsDescending(ecNodes) + + return +} + +func CollectEcNodes(env *Env, diskType types.DiskType) (ecNodes []*EcNode, totalFreeEcSlots int, err error) { + return CollectEcNodesForDC(env, "", diskType) +} + +// AssertEncodableRegularVolumes rejects volume ids that are not encodable +// regular volumes in the topology snapshot: an already-EC volume (present only +// as EC shards, with no .dat) or an id absent from the cluster. Encoding an +// already-EC volume would clear its shards before failing, destroying the only +// copy. A volume present as BOTH a regular .dat and stale orphan shards (a +// failed-encode retry) passes, so the retry + orphan sweep still works. +func AssertEncodableRegularVolumes(t *master_pb.TopologyInfo, vids []needle.VolumeId) error { + want := make(map[needle.VolumeId]bool, len(vids)) + for _, vid := range vids { + want[vid] = true + } + regular := make(map[needle.VolumeId]bool) + hasEcShards := make(map[needle.VolumeId]bool) + for _, dc := range t.DataCenterInfos { + for _, r := range dc.RackInfos { + for _, dn := range r.DataNodeInfos { + for _, diskInfo := range dn.DiskInfos { + if diskInfo == nil { + continue + } + for _, vi := range diskInfo.VolumeInfos { + if want[needle.VolumeId(vi.Id)] { + regular[needle.VolumeId(vi.Id)] = true + } + } + for _, ecShardInfo := range diskInfo.EcShardInfos { + if want[needle.VolumeId(ecShardInfo.Id)] { + hasEcShards[needle.VolumeId(ecShardInfo.Id)] = true + } + } + } + } + } + } + for _, vid := range vids { + if regular[vid] { + continue + } + if hasEcShards[vid] { + return fmt.Errorf("volume %d is already EC-encoded (no .dat replica); refusing to re-encode, which would destroy its shards", vid) + } + return fmt.Errorf("volume %d not found as a regular volume in the cluster; refusing to encode", vid) + } + return nil +} + +// CollectVolumeIdToCollection returns a map from volume ID to its collection name +func CollectVolumeIdToCollection(t *master_pb.TopologyInfo, vids []needle.VolumeId) map[needle.VolumeId]string { + result := make(map[needle.VolumeId]string) + if len(vids) == 0 { + return result + } + + vidSet := make(map[needle.VolumeId]bool) + for _, vid := range vids { + vidSet[vid] = true + } + + for _, dc := range t.DataCenterInfos { + for _, r := range dc.RackInfos { + for _, dn := range r.DataNodeInfos { + for _, diskInfo := range dn.DiskInfos { + if diskInfo == nil { + continue + } + for _, vi := range diskInfo.VolumeInfos { + vid := needle.VolumeId(vi.Id) + if vidSet[vid] { + result[vid] = vi.Collection + } + } + } + } + } + } + return result +} + +func CollectCollectionsForVolumeIds(t *master_pb.TopologyInfo, vids []needle.VolumeId) []string { + if len(vids) == 0 { + return nil + } + + found := map[string]bool{} + for _, dc := range t.DataCenterInfos { + for _, r := range dc.RackInfos { + for _, dn := range r.DataNodeInfos { + for _, diskInfo := range dn.DiskInfos { + for _, vi := range diskInfo.VolumeInfos { + for _, vid := range vids { + if needle.VolumeId(vi.Id) == vid { + found[vi.Collection] = true + } + } + } + for _, ecs := range diskInfo.EcShardInfos { + for _, vid := range vids { + if needle.VolumeId(ecs.Id) == vid { + found[ecs.Collection] = true + } + } + } + } + } + } + } + if len(found) == 0 { + return nil + } + + collections := []string{} + for k := range found { + collections = append(collections, k) + } + sort.Strings(collections) + return collections +} + +func MoveMountedShardToEcNode(env *Env, existingLocation *EcNode, collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, destinationEcNode *EcNode, destDiskId uint32, applyBalancing bool, diskType types.DiskType) (err error) { + + if !env.isLocked() { + return fmt.Errorf("lock is lost") + } + + copiedShardIds := []erasure_coding.ShardId{shardId} + + if applyBalancing { + + existingServerAddress := pb.NewServerAddressFromDataNode(existingLocation.Info) + + // ask destination node to copy shard and the ecx file from source node, and mount it + copiedShardIds, err = OneServerCopyAndMountEcShardsFromSource(env.GrpcDialOption, destinationEcNode, []erasure_coding.ShardId{shardId}, vid, collection, existingServerAddress, destDiskId) + if err != nil { + return err + } + + // unmount the to be deleted shards + err = UnmountEcShards(env.GrpcDialOption, vid, existingServerAddress, copiedShardIds) + if err != nil { + return err + } + + // ask source node to delete the shard, and maybe the ecx file + err = SourceServerDeleteEcShards(env.GrpcDialOption, collection, vid, existingServerAddress, copiedShardIds) + if err != nil { + return err + } + + if destDiskId > 0 { + fmt.Printf("moved ec shard %d.%d %s => %s (disk %d)\n", vid, shardId, existingLocation.Info.Id, destinationEcNode.Info.Id, destDiskId) + } else { + fmt.Printf("moved ec shard %d.%d %s => %s\n", vid, shardId, existingLocation.Info.Id, destinationEcNode.Info.Id) + } + + } + + destinationEcNode.AddEcVolumeShards(vid, collection, copiedShardIds, diskType) + existingLocation.DeleteEcVolumeShards(vid, copiedShardIds, diskType) + + return nil + +} + +func OneServerCopyAndMountEcShardsFromSource(grpcDialOption grpc.DialOption, + targetServer *EcNode, shardIdsToCopy []erasure_coding.ShardId, + volumeId needle.VolumeId, collection string, existingLocation pb.ServerAddress, destDiskId uint32) (copiedShardIds []erasure_coding.ShardId, err error) { + + fmt.Printf("allocate %d.%v %s => %s\n", volumeId, shardIdsToCopy, existingLocation, targetServer.Info.Id) + + targetAddress := pb.NewServerAddressFromDataNode(targetServer.Info) + err = volume_move.NewMover(grpcDialOption).CopyAndMountEcShards(context.Background(), volumeId, collection, shardIdsToCopy, existingLocation, targetAddress, destDiskId, 0, os.Stdout) + if err != nil { + return + } + + // SameServer, not ==: a representation mismatch here would report a + // same-server mount-in-place as a copy and have the caller delete the + // shards it kept. + if !volume_move.SameServer(targetAddress, existingLocation) { + copiedShardIds = shardIdsToCopy + glog.V(0).Infof("%s ec volume %d deletes shards %+v", existingLocation, volumeId, copiedShardIds) + } + + return +} + +func EachDataNode(topo *master_pb.TopologyInfo, fn func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo)) { + for _, dc := range topo.DataCenterInfos { + for _, rack := range dc.RackInfos { + for _, dn := range rack.DataNodeInfos { + fn(DataCenterId(dc.Id), RackId(rack.Id), dn) + } + } + } +} + +func SortEcNodesByFreeslotsDescending(ecNodes []*EcNode) { + slices.SortFunc(ecNodes, func(a, b *EcNode) int { + return b.FreeEcSlot - a.FreeEcSlot + }) +} + +func SortEcNodesByFreeslotsAscending(ecNodes []*EcNode) { + slices.SortFunc(ecNodes, func(a, b *EcNode) int { + return a.FreeEcSlot - b.FreeEcSlot + }) +} + +func CountShards(ecShardInfos []*master_pb.VolumeEcShardInformationMessage) (count int) { + for _, eci := range ecShardInfos { + count += erasure_coding.GetShardCount(eci) + } + return +} + +func CountFreeShardSlots(dn *master_pb.DataNodeInfo, diskType types.DiskType) (count int) { + if dn.DiskInfos == nil { + return 0 + } + diskInfo := dn.DiskInfos[string(diskType)] + if diskInfo == nil { + return 0 + } + + // A physically near-full disk has no room for more EC shards regardless of + // slot math (an over-set maxVolumeCount hides real fullness; statfs free bytes + // already include EC shard files). No-opinion when the server reports no bytes. + if balancer.DiskTooFullAfter(diskInfo.DiskTotalBytes, diskInfo.DiskFreeBytes, 0, balancer.DefaultMaxDiskUsagePercent) { + return 0 + } + + slots := int(diskInfo.MaxVolumeCount-diskInfo.VolumeCount)*erasure_coding.DataShardsCount - CountShards(diskInfo.EcShardInfos) + if slots < 0 { + return 0 + } + + return slots +} + +func (ecNode *EcNode) LocalShardIdCount(vid uint32) int { + for _, diskInfo := range ecNode.Info.DiskInfos { + for _, eci := range diskInfo.EcShardInfos { + if vid == eci.Id { + return erasure_coding.GetShardCount(eci) + } + } + } + return 0 +} + +func CollectEcVolumeServersByDc(topo *master_pb.TopologyInfo, selectedDataCenter string, diskType types.DiskType) (ecNodes []*EcNode, totalFreeEcSlots int) { + EachDataNode(topo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) { + if selectedDataCenter != "" && selectedDataCenter != string(dc) { + return + } + + freeEcSlots := CountFreeShardSlots(dn, diskType) + ecNode := &EcNode{ + Info: dn, + DC: dc, + Rack: rack, + FreeEcSlot: int(freeEcSlots), + Disks: make(map[uint32]*EcDisk), + } + + // Build disk-level information from volumes and EC shards + // First, discover all unique disk IDs from VolumeInfos (includes empty disks) + allDiskIds := make(map[uint32]string) // diskId -> diskType + for diskTypeKey, diskInfo := range dn.DiskInfos { + if diskInfo == nil { + continue + } + // Get all disk IDs from volumes + for _, vi := range diskInfo.VolumeInfos { + allDiskIds[vi.DiskId] = diskTypeKey + } + // Also get disk IDs from EC shards + for _, ecShardInfo := range diskInfo.EcShardInfos { + allDiskIds[ecShardInfo.DiskId] = diskTypeKey + } + } + + // Group EC shards by disk_id + diskShards := make(map[uint32]map[needle.VolumeId]*erasure_coding.ShardsInfo) + for _, diskInfo := range dn.DiskInfos { + if diskInfo == nil { + continue + } + for _, eci := range diskInfo.EcShardInfos { + diskId := eci.DiskId + if diskShards[diskId] == nil { + diskShards[diskId] = make(map[needle.VolumeId]*erasure_coding.ShardsInfo) + } + vid := needle.VolumeId(eci.Id) + diskShards[diskId][vid] = erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(eci) + } + } + + // Create EcDisk for each discovered disk + diskCount := len(allDiskIds) + if diskCount == 0 { + diskCount = 1 + } + freePerDisk := int(freeEcSlots) / diskCount + + for diskId, diskTypeStr := range allDiskIds { + shards := diskShards[diskId] + if shards == nil { + shards = make(map[needle.VolumeId]*erasure_coding.ShardsInfo) + } + totalShardCount := 0 + for _, shardsInfo := range shards { + totalShardCount += shardsInfo.Count() + } + + ecNode.Disks[diskId] = &EcDisk{ + DiskId: diskId, + DiskType: diskTypeStr, + FreeEcSlots: freePerDisk, + EcShardCount: totalShardCount, + EcShards: shards, + } + } + + ecNodes = append(ecNodes, ecNode) + totalFreeEcSlots += freeEcSlots + }) + return +} + +func SourceServerDeleteEcShards(grpcDialOption grpc.DialOption, collection string, volumeId needle.VolumeId, sourceLocation pb.ServerAddress, toBeDeletedShardIds []erasure_coding.ShardId) error { + + fmt.Printf("delete %d.%v from %s\n", volumeId, toBeDeletedShardIds, sourceLocation) + + return volume_move.NewMover(grpcDialOption).DeleteEcShards(context.Background(), volumeId, collection, sourceLocation, toBeDeletedShardIds) +} + +// ErrFullTeardownNotAcked marks a reachable server that completed the delete RPC +// but did not report full_teardown_done (a pre-upgrade volume server). The orphan +// sweep must treat this as fatal: the node may still hold an orphan that a later +// copy would re-stamp into the new generation. Aliased to the shared sentinel so +// the shell and the plugin-worker EC task agree on the teardown-not-acked signal. +var ErrFullTeardownNotAcked = erasure_coding.ErrFullTeardownNotAcked + +// PingVolumeServer probes node liveness with an empty-target Ping, which is never +// maintenance-gated, and returns the raw Ping error (nil on success). It lets the +// orphan sweep disambiguate a delete codes.Unavailable: a Rust volume server in +// maintenance mode fails the maintenance-gated delete with Unavailable yet answers +// Ping, whereas a genuinely-down node fails Ping with a transport Unavailable too. +// A Go server returns Unknown for maintenance, which IsNodeUnreachable already +// treats as fatal. The caller classifies the result with ClassifyNodeLiveness: +// only a Ping that itself transport-failed (codes.Unavailable) confirms the node +// is down; a nil error (reachable) or any other Ping error (inconclusive — e.g. a +// pre-Ping server returning Unimplemented, which means the node is up) is fatal. +func PingVolumeServer(grpcDialOption grpc.DialOption, location pb.ServerAddress) error { + return operation.WithVolumeServerClient(false, location, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + _, pingErr := client.Ping(context.Background(), &volume_server_pb.PingRequest{}) + return pingErr + }) +} + +// IsNodeUnreachable reports whether err means the volume server could not be +// reached at all, as opposed to an RPC that reached the node and failed. Only an +// unreachable node is safe to skip in the orphan sweep. A dead peer surfaces as +// a gRPC codes.Unavailable from the RPC (the dial is lazy, so it never fails at +// connect time); any non-status error reached node logic and is treated as +// reachable, so the sweep stays fatal rather than silently leaving stale state. +func IsNodeUnreachable(err error) bool { + if err == nil { + return false + } + st, ok := status.FromError(err) + return ok && st.Code() == codes.Unavailable +} + +// NodeLiveness is the tri-state result of a PingVolumeServer probe. +type NodeLiveness int + +const ( + // NodeUp: Ping succeeded — the node is reachable (e.g. a Rust volume server + // in maintenance mode that fails the delete but answers Ping). + NodeUp NodeLiveness = iota + // NodeDown: Ping itself transport-failed with codes.Unavailable — the node is + // confirmed unreachable. The only state the orphan sweep may skip. + NodeDown + // NodeLivenessUnknown: Ping reached failing logic with any non-Unavailable + // code (Internal, ResourceExhausted, Unimplemented from a pre-Ping server, …) + // or a non-status error. This does NOT prove the node is down, so it is fatal. + NodeLivenessUnknown +) + +// ClassifyNodeLiveness maps a PingVolumeServer error into the tri-state. A nil +// error is NodeUp, a transport codes.Unavailable is NodeDown (reusing the same +// rule as IsNodeUnreachable), and every other Ping failure is NodeLivenessUnknown. +func ClassifyNodeLiveness(pingErr error) NodeLiveness { + if pingErr == nil { + return NodeUp + } + if IsNodeUnreachable(pingErr) { + return NodeDown + } + return NodeLivenessUnknown +} + +// UnmountAndDeleteEcShardsQuiet unmounts then deletes shards on one server in a +// single connection, without the per-call logging the interactive helpers emit. +// Used by the orphan sweep, which fans out to every node x volume and would +// otherwise flood the shell with no-op lines. +func UnmountAndDeleteEcShardsQuiet(grpcDialOption grpc.DialOption, collection string, volumeId needle.VolumeId, location pb.ServerAddress, shardIds []erasure_coding.ShardId) error { + return erasure_coding.UnmountAndDeleteEcShards(context.Background(), grpcDialOption, location, collection, + uint32(volumeId), erasure_coding.ShardIdsToUint32(shardIds), 0) +} + +func UnmountEcShards(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sourceLocation pb.ServerAddress, toBeUnmountedShardIds []erasure_coding.ShardId) error { + + fmt.Printf("unmount %d.%v from %s\n", volumeId, toBeUnmountedShardIds, sourceLocation) + + return volume_move.NewMover(grpcDialOption).UnmountEcShards(context.Background(), volumeId, sourceLocation, toBeUnmountedShardIds) +} + +func MountEcShards(grpcDialOption grpc.DialOption, collection string, volumeId needle.VolumeId, sourceLocation pb.ServerAddress, toBeMountedShardIds []erasure_coding.ShardId) error { + + fmt.Printf("mount %d.%v on %s\n", volumeId, toBeMountedShardIds, sourceLocation) + + return volume_move.NewMover(grpcDialOption).MountEcShards(context.Background(), volumeId, collection, sourceLocation, toBeMountedShardIds) +} + +func CeilDivide(a, b int) int { + var r int + if (a % b) != 0 { + r = 1 + } + return (a / b) + r +} + +func FindEcVolumeShardsInfo(ecNode *EcNode, vid needle.VolumeId, diskType types.DiskType) *erasure_coding.ShardsInfo { + if diskInfo, found := ecNode.Info.DiskInfos[string(diskType)]; found { + for _, shardInfo := range diskInfo.EcShardInfos { + if needle.VolumeId(shardInfo.Id) == vid { + return erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(shardInfo) + } + } + } + + // Returns an empty ShardsInfo struct on failure, to avoid potential nil dereferences. + return erasure_coding.NewShardsInfo() +} + +// TODO: simplify me +func (ecNode *EcNode) AddEcVolumeShards(vid needle.VolumeId, collection string, shardIds []erasure_coding.ShardId, diskType types.DiskType) *EcNode { + + foundVolume := false + diskInfo, found := ecNode.Info.DiskInfos[string(diskType)] + if found { + for _, ecsi := range diskInfo.EcShardInfos { + if needle.VolumeId(ecsi.Id) == vid { + si := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(ecsi) + oldShardCount := si.Count() + for _, shardId := range shardIds { + si.Set(erasure_coding.NewShardInfo(shardId, 0)) + } + ecsi.EcIndexBits = si.Bitmap() + ecsi.ShardSizes = si.SizesInt64() + ecNode.FreeEcSlot -= si.Count() - oldShardCount + foundVolume = true + break + } + } + } else { + diskInfo = &master_pb.DiskInfo{ + Type: string(diskType), + } + ecNode.Info.DiskInfos[string(diskType)] = diskInfo + } + + if !foundVolume { + si := erasure_coding.NewShardsInfo() + for _, id := range shardIds { + si.Set(erasure_coding.NewShardInfo(id, 0)) + } + diskInfo.EcShardInfos = append(diskInfo.EcShardInfos, &master_pb.VolumeEcShardInformationMessage{ + Id: uint32(vid), + Collection: collection, + EcIndexBits: si.Bitmap(), + ShardSizes: si.SizesInt64(), + DiskType: string(diskType), + }) + ecNode.FreeEcSlot -= si.Count() + } + + return ecNode +} + +func (ecNode *EcNode) DeleteEcVolumeShards(vid needle.VolumeId, shardIds []erasure_coding.ShardId, diskType types.DiskType) *EcNode { + + if diskInfo, found := ecNode.Info.DiskInfos[string(diskType)]; found { + for _, eci := range diskInfo.EcShardInfos { + if needle.VolumeId(eci.Id) == vid { + si := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(eci) + oldCount := si.Count() + for _, shardId := range shardIds { + si.Delete(shardId) + } + eci.EcIndexBits = si.Bitmap() + eci.ShardSizes = si.SizesInt64() + ecNode.FreeEcSlot -= si.Count() - oldCount + } + } + } + + return ecNode +} + +// PickBestDiskOnNode selects the best disk on a node for placing a new EC shard +// It prefers disks of the specified type with fewer shards and more free slots +// When shardId is provided and dataShardCount > 0, it applies anti-affinity: +// - For data shards (shardId < dataShardCount): prefer disks without parity shards +// - For parity shards (shardId >= dataShardCount): prefer disks without data shards +// If strictDiskType is false, it will fall back to other disk types if no matching disk is found +func PickBestDiskOnNode(ecNode *EcNode, vid needle.VolumeId, diskType types.DiskType, strictDiskType bool, shardId erasure_coding.ShardId, dataShardCount int) uint32 { + if len(ecNode.Disks) == 0 { + return 0 // No disk info available, let the server decide + } + + var bestDiskId uint32 + bestScore := -1 + var fallbackDiskId uint32 + fallbackScore := -1 + + // Determine if we're placing a data or parity shard + isDataShard := dataShardCount > 0 && int(shardId) < dataShardCount + + for diskId, disk := range ecNode.Disks { + if disk.FreeEcSlots <= 0 { + continue + } + + // Check existing shards on this disk for this volume + existingShards := 0 + hasDataShards := false + hasParityShards := false + if si, ok := disk.EcShards[vid]; ok { + existingShards = si.Count() + // Check what type of shards are on this disk + if dataShardCount > 0 { + for _, existingShardId := range si.Ids() { + if int(existingShardId) < dataShardCount { + hasDataShards = true + } else { + hasParityShards = true + } + } + } + } + + // Score: prefer disks with fewer total shards and fewer shards of this volume + // Lower score is better + score := disk.EcShardCount*10 + existingShards*100 + + // Apply anti-affinity penalty if applicable + if dataShardCount > 0 { + if isDataShard && hasParityShards { + // Penalize placing data shard on disk with parity shards + score += 1000 + } else if !isDataShard && hasDataShards { + // Penalize placing parity shard on disk with data shards + score += 1000 + } + } + + if disk.DiskType == string(diskType) { + // Matching disk type - this is preferred + if bestScore == -1 || score < bestScore { + bestScore = score + bestDiskId = diskId + } + } else if !strictDiskType { + // Non-matching disk type - use as fallback if allowed + if fallbackScore == -1 || score < fallbackScore { + fallbackScore = score + fallbackDiskId = diskId + } + } + } + + // Return matching disk type if found, otherwise fallback. Gate on bestScore, + // not bestDiskId: physical disk 0 is a valid target and 0 is also the "no + // match" zero value, so testing bestDiskId would never select disk 0. + if bestScore != -1 { + return bestDiskId + } + return fallbackDiskId +} + +// ecBalancer drives an EC balance run: it collects the cluster's EC nodes, hands +// them to the shared ecbalancer planner, and executes the planned shard moves. +// The balancing policy lives in weed/storage/erasure_coding/ecbalancer, shared +// with the EC balance worker so the two cannot drift. +type ecBalancer struct { + env *Env + ecNodes []*EcNode + replicaPlacement *super_block.ReplicaPlacement + applyBalancing bool + maxParallelization int + ioBytePerSecond int64 + diskType types.DiskType + // volumeIds narrows the plan to these ec volume ids; nil balances every volume + // of the selected collections. + volumeIds map[uint32]bool +} + +// excludeNodes is a set of server addresses kept out of the balance as copy/move +// targets and sources. ec.encode passes the nodes its orphan sweep could not +// reach: such a node may still hold a stale-generation shard orphan, and pairing +// it with a new-generation shard from a balance copy would mix generations on one +// node. The standalone ec.balance command passes nil. +// +// volumeIds, when non-empty, restricts the plan to those ec volume ids; empty +// balances every volume of the given collections. +func EcBalance(env *Env, collections []string, dc string, ecReplicaPlacement *super_block.ReplicaPlacement, diskType types.DiskType, maxParallelization int, ioBytePerSecond int64, applyBalancing bool, excludeNodes map[pb.ServerAddress]struct{}, volumeIds []needle.VolumeId) (err error) { + // collect all ec nodes + allEcNodes, totalFreeEcSlots, err := CollectEcNodesForDC(env, dc, diskType) + if err != nil { + return err + } + + // Drop excluded nodes (and the slots they contribute) before planning so they + // can be neither a target nor a source for any move this balance plans. + if len(excludeNodes) > 0 { + kept := allEcNodes[:0] + var excludedFreeSlots int + for _, en := range allEcNodes { + if _, skip := excludeNodes[pb.NewServerAddressFromDataNode(en.Info)]; skip { + excludedFreeSlots += en.FreeEcSlot + glog.V(0).Infof("EC balance excluding node %s: skipped as unreachable by the encode orphan sweep", en.Info.Id) + continue + } + kept = append(kept, en) + } + allEcNodes = kept + totalFreeEcSlots -= excludedFreeSlots + } + + if totalFreeEcSlots < 1 { + return fmt.Errorf("no free ec shard slots. only %d left", totalFreeEcSlots) + } + + var volumeIdFilter map[uint32]bool + if len(volumeIds) > 0 { + volumeIdFilter = make(map[uint32]bool, len(volumeIds)) + for _, vid := range volumeIds { + volumeIdFilter[uint32(vid)] = true + } + } + + ecb := &ecBalancer{ + env: env, + ecNodes: allEcNodes, + replicaPlacement: ecReplicaPlacement, + applyBalancing: applyBalancing, + maxParallelization: maxParallelization, + ioBytePerSecond: ioBytePerSecond, + diskType: diskType, + volumeIds: volumeIdFilter, + } + + if len(collections) == 0 { + glog.V(1).Infof("WARNING: No collections to balance EC volumes across.\n") + } + return ecb.balance(collections) +} + +// defaultECRatio resolves a collection's EC data/parity counts, defaulting to +// the standard scheme. This is the admin-side plug-in point for custom ratios. +func defaultECRatio(_ string) (int, int) { + // Custom EC ratios are an enterprise feature; OSS uses the standard scheme. + return erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount +} + +// balance plans EC shard moves with the shared planner and executes them. When +// collections is empty all collections present are balanced. +func (ecb *ecBalancer) balance(collections []string) error { + topo, volumeRatio, selected := toBalancerTopology(ecb.ecNodes, collections, ecb.diskType, ecb.volumeIds) + if len(ecb.volumeIds) > 0 { + requested := make([]uint32, 0, len(ecb.volumeIds)) + for vid := range ecb.volumeIds { + requested = append(requested, vid) + } + slices.Sort(requested) + var missing []uint32 + for _, vid := range requested { + if !selected[vid] { + missing = append(missing, vid) + } + } + if len(missing) > 0 { + return fmt.Errorf("no ec shards found for volume(s) %v: not an ec volume, or outside the selected collection, dataCenter or diskType", missing) + } + fmt.Printf("balancing ec volume(s) %v\n", requested) + } + moves := ecbalancer.Plan(topo, ecbalancer.Options{ + DiskType: string(ecb.diskType), + ImbalanceThreshold: 0, // the shell balances to an even distribution + ReplicaPlacement: ecb.replicaPlacement, + Ratio: defaultECRatio, + // Prefer each volume's own heartbeat-reported ratio over the collection + // default so a mixed-ratio collection is spread per volume; 0 defers to + // defaultECRatio (and is the always-0 OSS case). + VolumeRatio: volumeRatio, + // Balance the global phase by fractional fullness so heterogeneous-capacity + // nodes fill proportionally (matching the worker). This is identical to raw + // shard count when capacities are uniform. + GlobalUtilizationBased: true, + }) + if len(ecb.volumeIds) > 0 { + var deletions int + for _, m := range moves { + if m.Phase == "dedup" { + deletions++ + } + } + fmt.Printf("planned %d ec shard move(s) and %d ec shard deletion(s)\n", len(moves)-deletions, deletions) + } + return ecb.executeMoves(moves) +} + +// toBalancerTopology builds an ecbalancer.Topology from the EcNode model, +// including the shards of the requested collections (all collections when empty) +// and, when volumeIds is non-nil, only those volume ids. Volumes left out here are +// invisible to the planner, so no phase - dedup included - can plan against them. +// It also returns a per-volume ratio lookup built from each shard's heartbeat +// (0,0 when unreported, e.g. always in OSS), which Plan prefers over the +// collection ratio for mixed-ratio clusters, and the set of volume ids that made +// it into the topology. +func toBalancerTopology(ecNodes []*EcNode, collections []string, diskType types.DiskType, volumeIds map[uint32]bool) (*ecbalancer.Topology, func(collection string, vid uint32) (int, int), map[uint32]bool) { + allowed := make(map[string]bool, len(collections)) + for _, c := range collections { + allowed[c] = true + } + + type volRatioKey struct { + collection string + vid uint32 + } + volRatios := make(map[volRatioKey][2]int) + selected := make(map[uint32]bool) + + topo := ecbalancer.NewTopology() + for _, en := range ecNodes { + rackKey := string(en.DC) + ":" + string(en.Rack) + node := topo.AddNode(en.Info.Id, string(en.DC), rackKey, en.FreeEcSlot) + // Group by physical machine (host) so shards spread across machines, not just + // nodes; the id stays the node identity used for moves. + node.SetHost(pb.NewServerAddressFromDataNode(en.Info).ToHost()) + for diskId, d := range en.Disks { + node.AddDisk(diskId, d.DiskType, d.FreeEcSlots, d.EcShardCount) + } + diskInfo, found := en.Info.DiskInfos[string(diskType)] + if !found { + continue + } + for _, eci := range diskInfo.EcShardInfos { + if len(allowed) > 0 && !allowed[eci.Collection] { + continue + } + if volumeIds != nil && !volumeIds[eci.Id] { + continue + } + selected[eci.Id] = true + node.AddShards(eci.Id, eci.Collection, eci.DiskId, erasure_coding.ShardBits(eci.EcIndexBits)) + if d, p := ecbalancer.VolumeShardRatio(eci); d > 0 || p > 0 { + volRatios[volRatioKey{eci.Collection, eci.Id}] = [2]int{d, p} + } + } + } + + volumeRatio := func(collection string, vid uint32) (int, int) { + r := volRatios[volRatioKey{collection, vid}] + return r[0], r[1] + } + return topo, volumeRatio, selected +} + +// executeMoves carries out the planned moves. Phases run in order (a within-rack +// move can depend on a cross-rack move's result), and the independent moves +// within a phase run with up to maxParallelization concurrency. Apply mode does +// only the RPCs; dry-run mode runs sequentially and mutates the in-memory EcNode +// model so callers/tests can inspect the planned end state. +func (ecb *ecBalancer) executeMoves(moves []ecbalancer.Move) error { + byID := make(map[string]*EcNode, len(ecb.ecNodes)) + for _, en := range ecb.ecNodes { + byID[en.Info.Id] = en + } + + // Plan emits moves grouped by phase; run each contiguous same-phase group + // together, waiting before the next so cross-phase dependencies hold. + for i := 0; i < len(moves); { + j := i + for j < len(moves) && moves[j].Phase == moves[i].Phase { + j++ + } + if err := ecb.executePhase(byID, moves[i:j]); err != nil { + return err + } + i = j + } + return nil +} + +func (ecb *ecBalancer) executePhase(byID map[string]*EcNode, moves []ecbalancer.Move) error { + if !ecb.applyBalancing { + // Dry-run: sequential so the in-memory model updates are race-free and + // reflect the full plan for inspection. + for _, m := range moves { + if err := ecb.executeMove(byID, m); err != nil { + return err + } + } + return nil + } + // Apply mode: parallelize across volumes, but run one volume's moves within a + // phase sequentially. Concurrent moves of the same volume to a node can race + // on its shared .ecx/.ecj/.vif sidecar files. + var order []uint32 + byVol := make(map[uint32][]ecbalancer.Move) + for _, m := range moves { + if _, ok := byVol[m.VolumeID]; !ok { + order = append(order, m.VolumeID) + } + byVol[m.VolumeID] = append(byVol[m.VolumeID], m) + } + taskGroups := make([][]util.ErrorWaitGroupTask, 0, len(order)) + for _, vid := range order { + movesForVolume := byVol[vid] + taskGroup := make([]util.ErrorWaitGroupTask, 0, len(movesForVolume)) + for _, move := range movesForVolume { + move := move + taskGroup = append(taskGroup, func() error { + return ecb.executeMove(byID, move) + }) + } + taskGroups = append(taskGroups, taskGroup) + } + return util.ExecuteParallelTaskGroups(ecb.maxParallelization, taskGroups) +} + +// verifyEcShardOnKeepNode confirms the node a dedup move chose to keep actually +// holds the shard, so a duplicate is only removed when a real copy remains. An +// unreachable keep node is unknown, not confirmed, and blocks the delete — as +// does one that answers too slowly to be waited on, which is why this is +// bounded rather than left to hang the whole balance run. +// ecShardVerifyTimeout bounds the keep-node inventory query. A node that accepts +// the connection but never answers must not stall the whole balance run. +const ecShardVerifyTimeout = 30 * time.Second + +func verifyEcShardOnKeepNode(grpcDialOption grpc.DialOption, collection string, vid needle.VolumeId, keepNode string, shardId erasure_coding.ShardId) error { + if keepNode == "" { + return fmt.Errorf("refusing dedup delete of %d.%d: no keep node recorded", vid, shardId) + } + ctx, cancel := context.WithTimeout(context.Background(), ecShardVerifyTimeout) + defer cancel() + if err := erasure_coding.VerifyShardsOnServer(ctx, collection, uint32(vid), keepNode, + []uint32{uint32(shardId)}, grpcDialOption); err != nil { + return fmt.Errorf("refusing dedup delete: %w", err) + } + return nil +} + +func (ecb *ecBalancer) executeMove(byID map[string]*EcNode, m ecbalancer.Move) error { + src := byID[m.SourceNode] + if src == nil { + return nil + } + vid := needle.VolumeId(m.VolumeID) + shardId := erasure_coding.ShardId(m.ShardID) + shardIds := []erasure_coding.ShardId{shardId} + + if m.Phase == "dedup" { + fmt.Printf("dedup: delete ec shard %d.%d on %s\n", vid, shardId, m.SourceNode) + if !ecb.applyBalancing { + src.DeleteEcVolumeShards(vid, shardIds, ecb.diskType) + return nil + } + grpcDialOption := ecb.env.GrpcDialOption + // Nothing is copied first, so the shard surviving elsewhere is the only + // thing making this safe -- and the plan saying so is not evidence. A + // topology entry can name a location holding nothing, and deleting on + // that basis removes the last copy. Confirm the keep node has it. + if err := verifyEcShardOnKeepNode(grpcDialOption, m.Collection, vid, m.KeepNode, shardId); err != nil { + return err + } + addr := pb.NewServerAddressFromDataNode(src.Info) + if err := UnmountEcShards(grpcDialOption, vid, addr, shardIds); err != nil { + return err + } + return SourceServerDeleteEcShards(grpcDialOption, m.Collection, vid, addr, shardIds) + } + + dst := byID[m.TargetNode] + if dst == nil { + return nil + } + if m.TargetDisk > 0 { + fmt.Printf("%s moves ec shard %d.%d to %s (disk %d)\n", m.SourceNode, vid, shardId, m.TargetNode, m.TargetDisk) + } else { + fmt.Printf("%s moves ec shard %d.%d to %s\n", m.SourceNode, vid, shardId, m.TargetNode) + } + if !ecb.applyBalancing { + // Dry-run: update the in-memory model only. + return MoveMountedShardToEcNode(ecb.env, src, m.Collection, vid, shardId, dst, m.TargetDisk, false, ecb.diskType) + } + return ecb.applyShardMoveRPC(src, dst, m.Collection, vid, shardId, m.TargetDisk) +} + +// applyShardMoveRPC copies a shard to the destination disk, verifies the +// destination registered it, then unmounts and deletes it on the source. It +// does not touch the in-memory model, so it is safe to run concurrently across +// the moves of a phase. +func (ecb *ecBalancer) applyShardMoveRPC(src, dst *EcNode, collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, destDiskId uint32) error { + srcAddr := pb.NewServerAddressFromDataNode(src.Info) + dstAddr := pb.NewServerAddressFromDataNode(dst.Info) + if volume_move.SameServer(srcAddr, dstAddr) { + // A same-server (cross-disk) move cannot be expressed with these RPCs; + // leave the shard where it is. + return nil + } + return volume_move.NewMover(ecb.env.GrpcDialOption).MoveEcShards(context.Background(), volume_move.EcShardMove{ + VolumeId: vid, + Collection: collection, + ShardIds: []erasure_coding.ShardId{shardId}, + Source: srcAddr, + Target: dstAddr, + TargetDisk: destDiskId, + }, volume_move.EcMoveOptions{IoBytePerSecond: ecb.ioBytePerSecond, Writer: os.Stdout}) +} + +// CountExistingEcShardsForVolume returns the number of distinct EC shard IDs +// for (volumeID, collection) present in the topology, counting only the single +// largest encode generation. Shards are grouped by encode_ts_ns (the per-encode +// identity from .vif), so two interrupted encode runs whose shard sets overlap +// are never unioned into a false-complete set that would wrongly trigger the +// orphaned-source delete. Walks every disk's EcIndexBits bitmap rather than +// trusting len(EcShardInfos), because a single info entry can carry multiple +// shards. Shards reporting encode_ts_ns==0 (pre-upgrade servers) form their own +// generation bucket. +// +// Limitation: the heartbeat carries one encode_ts_ns per (volume, disk), so this +// separates generations living on different disks; same-disk mixing is prevented +// upstream by the pre-encode artifact wipe and the cross-run read guard. +func CountExistingEcShardsForVolume(topologyInfo *master_pb.TopologyInfo, volumeID uint32, collection string) int { + if topologyInfo == nil { + return 0 + } + perGeneration := make(map[int64]erasure_coding.ShardBits) + for _, dc := range topologyInfo.DataCenterInfos { + for _, rack := range dc.RackInfos { + for _, node := range rack.DataNodeInfos { + for _, diskInfo := range node.DiskInfos { + if diskInfo == nil { + continue + } + for _, ecShardInfo := range diskInfo.EcShardInfos { + if ecShardInfo == nil { + continue + } + if ecShardInfo.Id != volumeID || ecShardInfo.Collection != collection { + continue + } + perGeneration[ecShardInfo.EncodeTsNs] |= erasure_coding.ShardBits(ecShardInfo.EcIndexBits) + } + } + } + } + } + best := 0 + for _, bits := range perGeneration { + if c := bits.Count(); c > best { + best = c + } + } + return best +} + +// ParseVolumeIdsFlag parses a comma-separated -volumeIds flag value, dropping +// duplicates and keeping the given order. +func ParseVolumeIdsFlag(volumeIdsStr string) ([]needle.VolumeId, error) { + var volumeIds []needle.VolumeId + seen := make(map[needle.VolumeId]bool) + for _, part := range strings.Split(volumeIdsStr, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + vidValue, err := strconv.ParseUint(part, 10, 32) + if err != nil || vidValue == 0 { + return nil, fmt.Errorf("invalid volume id %q in -volumeIds", part) + } + // ParseUint with bitSize 32 bounds the value; convert through uint32 + // (matching the rest of the codebase) so the narrowing is provably safe. + vid := needle.VolumeId(uint32(vidValue)) + if seen[vid] { + continue + } + seen[vid] = true + volumeIds = append(volumeIds, vid) + } + if len(volumeIds) == 0 { + return nil, fmt.Errorf("-volumeIds does not contain any valid volume id") + } + return volumeIds, nil +} diff --git a/weed/shell/command_ec_test.go b/weed/ec/ec_common_test.go similarity index 92% rename from weed/shell/command_ec_test.go rename to weed/ec/ec_common_test.go index 24f6f04c0..4d6387fd2 100644 --- a/weed/shell/command_ec_test.go +++ b/weed/ec/ec_common_test.go @@ -1,4 +1,4 @@ -package shell +package ec import ( "strings" @@ -15,15 +15,15 @@ import ( // best-scoring disk 0 (0 is also the zero value) and returned the fallback. func TestPickBestDiskOnNodeSelectsPhysicalDiskZero(t *testing.T) { ecNode := &EcNode{ - disks: map[uint32]*EcDisk{ - 0: {diskId: 0, diskType: string(types.SsdType), freeEcSlots: 10, ecShards: map[needle.VolumeId]*erasure_coding.ShardsInfo{}}, - 5: {diskId: 5, diskType: string(types.HardDriveType), freeEcSlots: 10, ecShards: map[needle.VolumeId]*erasure_coding.ShardsInfo{}}, + Disks: map[uint32]*EcDisk{ + 0: {DiskId: 0, DiskType: string(types.SsdType), FreeEcSlots: 10, EcShards: map[needle.VolumeId]*erasure_coding.ShardsInfo{}}, + 5: {DiskId: 5, DiskType: string(types.HardDriveType), FreeEcSlots: 10, EcShards: map[needle.VolumeId]*erasure_coding.ShardsInfo{}}, }, } // Disk 0 matches the requested type; it must win over the non-matching // fallback on disk 5 instead of being treated as "no match". - if got := pickBestDiskOnNode(ecNode, needle.VolumeId(42), types.SsdType, false, 0, 0); got != 0 { + if got := PickBestDiskOnNode(ecNode, needle.VolumeId(42), types.SsdType, false, 0, 0); got != 0 { t.Fatalf("want physical disk 0 (matching type), got %d", got) } } @@ -142,18 +142,18 @@ func TestCommandEcBalanceVolumeEvenButRackUneven(t *testing.T) { func newEcNode(dc string, rack string, dataNodeId string, freeEcSlot int) *EcNode { return &EcNode{ - info: &master_pb.DataNodeInfo{ + Info: &master_pb.DataNodeInfo{ Id: dataNodeId, DiskInfos: make(map[string]*master_pb.DiskInfo), }, - dc: DataCenterId(dc), - rack: RackId(rack), - freeEcSlot: freeEcSlot, + DC: DataCenterId(dc), + Rack: RackId(rack), + FreeEcSlot: freeEcSlot, } } func (ecNode *EcNode) addEcVolumeAndShardsForTest(vid uint32, collection string, shardIds []erasure_coding.ShardId) *EcNode { - return ecNode.addEcVolumeShards(needle.VolumeId(vid), collection, shardIds, types.HardDriveType) + return ecNode.AddEcVolumeShards(needle.VolumeId(vid), collection, shardIds, types.HardDriveType) } // TestCommandEcBalanceEvenDataAndParityDistribution verifies that after balancing: @@ -190,8 +190,8 @@ func TestCommandEcBalanceEvenDataAndParityDistribution(t *testing.T) { // With 6 racks: // - Data shards (10): max 2 per rack (ceil(10/6) = 2) // - Parity shards (4): max 1 per rack (ceil(4/6) = 1) - maxDataPerRack := ceilDivide(dataShardCount, 6) // 2 - maxParityPerRack := ceilDivide(parityShardCount, 6) // 1 + maxDataPerRack := CeilDivide(dataShardCount, 6) // 2 + maxParityPerRack := CeilDivide(parityShardCount, 6) // 1 // Verify no rack has more than max data shards for rackId, count := range dataPerRack { @@ -247,9 +247,9 @@ func countDataAndParityShardsPerRack(ecNodes []*EcNode, vid needle.VolumeId, dat parityPerRack = make(map[string]int) for _, ecNode := range ecNodes { - si := findEcVolumeShardsInfo(ecNode, vid, types.HardDriveType) + si := FindEcVolumeShardsInfo(ecNode, vid, types.HardDriveType) for _, shardId := range si.Ids() { - rackId := string(ecNode.rack) + rackId := string(ecNode.Rack) if int(shardId) < dataShardCount { dataPerRack[rackId]++ } else { @@ -286,8 +286,8 @@ func TestCommandEcBalanceMultipleVolumesEvenDistribution(t *testing.T) { for _, vid := range []needle.VolumeId{1, 2} { dataPerRack, parityPerRack := countDataAndParityShardsPerRack(ecb.ecNodes, vid, erasure_coding.DataShardsCount) - maxDataPerRack := ceilDivide(erasure_coding.DataShardsCount, 6) - maxParityPerRack := ceilDivide(erasure_coding.ParityShardsCount, 6) + maxDataPerRack := CeilDivide(erasure_coding.DataShardsCount, 6) + maxParityPerRack := CeilDivide(erasure_coding.ParityShardsCount, 6) for rackId, count := range dataPerRack { if count > maxDataPerRack { @@ -339,16 +339,16 @@ func TestCommandEcBalanceAllNodesShareAllVolumes(t *testing.T) { // Count total shards per node after balancing for _, node := range ecb.ecNodes { count := 0 - if diskInfo, found := node.info.DiskInfos[string(types.HardDriveType)]; found { + if diskInfo, found := node.Info.DiskInfos[string(types.HardDriveType)]; found { for _, ecsi := range diskInfo.EcShardInfos { count += erasure_coding.GetShardCount(ecsi) } } // Average is 7, so all nodes should be at 7 (ceil(28/4) = 7) if count > 7 { - t.Errorf("node %s has %d shards after balancing, expected at most 7", node.info.Id, count) + t.Errorf("node %s has %d shards after balancing, expected at most 7", node.Info.Id, count) } - t.Logf("node %s: %d shards", node.info.Id, count) + t.Logf("node %s: %d shards", node.Info.Id, count) } } @@ -409,12 +409,12 @@ func TestCommandEcBalanceIssue8793Topology(t *testing.T) { // Log initial state for _, node := range ecb.ecNodes { count := 0 - if diskInfo, found := node.info.DiskInfos[string(types.HardDriveType)]; found { + if diskInfo, found := node.Info.DiskInfos[string(types.HardDriveType)]; found { for _, ecsi := range diskInfo.EcShardInfos { count += erasure_coding.GetShardCount(ecsi) } } - t.Logf("BEFORE node %s (max %d): %d shards", node.info.Id, node.freeEcSlot+count, count) + t.Logf("BEFORE node %s (max %d): %d shards", node.Info.Id, node.FreeEcSlot+count, count) } ecb.balance([]string{"cldata"}) @@ -433,14 +433,14 @@ func TestCommandEcBalanceIssue8793Topology(t *testing.T) { shardCounts := make(map[string]int) for _, node := range ecb.ecNodes { count := 0 - if diskInfo, found := node.info.DiskInfos[string(types.HardDriveType)]; found { + if diskInfo, found := node.Info.DiskInfos[string(types.HardDriveType)]; found { for _, ecsi := range diskInfo.EcShardInfos { count += erasure_coding.GetShardCount(ecsi) } } - shardCounts[node.info.Id] = count + shardCounts[node.Info.Id] = count totalShards += count - totalCapacity += capacityByID[node.info.Id] + totalCapacity += capacityByID[node.Info.Id] } overallFullness := float64(totalShards) / float64(totalCapacity) @@ -448,14 +448,14 @@ func TestCommandEcBalanceIssue8793Topology(t *testing.T) { // would sit ~38 points above overall), but above integer-rounding skew. const tolerance = 0.05 for _, node := range ecb.ecNodes { - count := shardCounts[node.info.Id] - capacity := capacityByID[node.info.Id] + count := shardCounts[node.Info.Id] + capacity := capacityByID[node.Info.Id] fullness := float64(count) / float64(capacity) t.Logf("AFTER node %s: %d/%d shards (%.0f%% full, overall %.0f%%)", - node.info.Id, count, capacity, fullness*100, overallFullness*100) + node.Info.Id, count, capacity, fullness*100, overallFullness*100) if diff := fullness - overallFullness; diff > tolerance || diff < -tolerance { t.Errorf("node %s fullness %.1f%% deviates from overall %.1f%% by more than %.0f points", - node.info.Id, fullness*100, overallFullness*100, tolerance*100) + node.Info.Id, fullness*100, overallFullness*100, tolerance*100) } } } @@ -484,15 +484,15 @@ func TestCommandEcBalanceVolumeIdsFilter(t *testing.T) { } // Volume 1 spreads out. - if count := ecb.ecNodes[0].localShardIdCount(1); count == 14 { + if count := ecb.ecNodes[0].LocalShardIdCount(1); count == 14 { t.Errorf("volume 1 was not balanced: dn1 still holds all %d shards", count) } // Volume 2 keeps every shard where it was, duplicate included. - if count := ecb.ecNodes[1].localShardIdCount(2); count != 14 { + if count := ecb.ecNodes[1].LocalShardIdCount(2); count != 14 { t.Errorf("dn2 holds %d shards of the unselected volume 2, want 14", count) } - if count := ecb.ecNodes[2].localShardIdCount(2); count != 1 { + if count := ecb.ecNodes[2].LocalShardIdCount(2); count != 1 { t.Errorf("dn3 holds %d shards of the unselected volume 2, want the duplicate to survive", count) } } @@ -513,7 +513,7 @@ func TestCommandEcBalanceVolumeIdsNotFound(t *testing.T) { t.Fatalf("want an error naming volume 99, got %v", err) } // The valid id must not be balanced either: the plan is all-or-nothing. - if count := ecb.ecNodes[0].localShardIdCount(1); count != 14 { + if count := ecb.ecNodes[0].LocalShardIdCount(1); count != 14 { t.Errorf("dn1 holds %d shards, want the rejected plan to leave volume 1 untouched", count) } } diff --git a/weed/ec/ec_decode.go b/weed/ec/ec_decode.go new file mode 100644 index 000000000..57665614e --- /dev/null +++ b/weed/ec/ec_decode.go @@ -0,0 +1,376 @@ +package ec + +import ( + "context" + "fmt" + "regexp" + "strings" + + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/operation" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/util" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func DoEcDecode(env *Env, topoInfo *master_pb.TopologyInfo, collection string, vid needle.VolumeId, diskType types.DiskType, checkMinFreeSpace bool, diskUsageState *DecodeDiskUsageState) (err error) { + + if !env.isLocked() { + return fmt.Errorf("lock is lost") + } + + // find volume location + nodeToEcShardsInfo, dataShards := collectEcNodeShardsInfo(topoInfo, vid, diskType) + + fmt.Printf("ec volume %d shard locations: %+v\n", vid, nodeToEcShardsInfo) + + if len(nodeToEcShardsInfo) == 0 { + return fmt.Errorf("no EC shards found for volume %d (diskType %s)", vid, diskType.ReadableString()) + } + + var originalShardCounts map[pb.ServerAddress]int + if diskUsageState != nil { + originalShardCounts = make(map[pb.ServerAddress]int, len(nodeToEcShardsInfo)) + for location, si := range nodeToEcShardsInfo { + originalShardCounts[location] = si.Count() + } + } + + var eligibleTargets map[pb.ServerAddress]struct{} + if checkMinFreeSpace { + if diskUsageState == nil { + return fmt.Errorf("min free space checking requires disk usage state") + } + eligibleTargets = make(map[pb.ServerAddress]struct{}) + for location := range nodeToEcShardsInfo { + if freeCount, found := diskUsageState.freeVolumeCount(location); found && freeCount > 0 { + eligibleTargets[location] = struct{}{} + } + } + if len(eligibleTargets) == 0 { + return fmt.Errorf("no eligible target datanodes with free volume slots for volume %d (diskType %s); use -checkMinFreeSpace=false to override", vid, diskType.ReadableString()) + } + } + + // collect ec shards to the server with most space + targetNodeLocation, err := collectEcShards(env, nodeToEcShardsInfo, collection, vid, eligibleTargets, dataShards) + if err != nil { + return fmt.Errorf("collectEcShards for volume %d: %v", vid, err) + } + + // generate a normal volume + err = generateNormalVolume(env.GrpcDialOption, vid, collection, targetNodeLocation) + if err != nil { + // Special case: if the EC index has no live entries, decoding is a no-op. + // Just purge EC shards and return success without generating/mounting an empty volume. + if isEcDecodeEmptyVolumeErr(err) { + if err := unmountAndDeleteEcShards(env.GrpcDialOption, collection, nodeToEcShardsInfo, vid); err != nil { + return err + } + if diskUsageState != nil { + diskUsageState.applyDecode(targetNodeLocation, originalShardCounts, false) + } + return nil + } + return fmt.Errorf("generate normal volume %d on %s: %v", vid, targetNodeLocation, err) + } + + // mount the decoded volume after server-side offline compaction succeeded + err = mountDecodedVolume(env.GrpcDialOption, targetNodeLocation, vid) + if err != nil { + return fmt.Errorf("mount decoded volume %d on %s: %v", vid, targetNodeLocation, err) + } + + // Confirm the regenerated .dat is present and non-empty before destroying + // the shards. Without this gate, a silent failure in generate/mount could + // leave the cluster with neither shards nor volume. + if err := verifyDecodedVolumeBeforeDelete(env.GrpcDialOption, targetNodeLocation, vid); err != nil { + return fmt.Errorf("verify decoded volume %d on %s before deleting shards: %w", vid, targetNodeLocation, err) + } + + // delete the previous ec shards + err = unmountAndDeleteEcShardsWithPrefix("deleteDecodedEcShards", env.GrpcDialOption, collection, nodeToEcShardsInfo, vid) + if err != nil { + return fmt.Errorf("delete ec shards for volume %d: %v", vid, err) + } + if diskUsageState != nil { + diskUsageState.applyDecode(targetNodeLocation, originalShardCounts, true) + } + + return nil +} + +func isEcDecodeEmptyVolumeErr(err error) bool { + st, ok := status.FromError(err) + if !ok { + return false + } + if st.Code() != codes.FailedPrecondition { + return false + } + // Keep this robust against wording tweaks while still being specific. + return strings.Contains(st.Message(), erasure_coding.EcNoLiveEntriesSubstring) +} + +func unmountAndDeleteEcShards(grpcDialOption grpc.DialOption, collection string, nodeToShardsInfo map[pb.ServerAddress]*erasure_coding.ShardsInfo, vid needle.VolumeId) error { + return unmountAndDeleteEcShardsWithPrefix("unmountAndDeleteEcShards", grpcDialOption, collection, nodeToShardsInfo, vid) +} + +func unmountAndDeleteEcShardsWithPrefix(prefix string, grpcDialOption grpc.DialOption, collection string, nodeToShardsInfo map[pb.ServerAddress]*erasure_coding.ShardsInfo, vid needle.VolumeId) error { + ewg := util.NewErrorWaitGroup(len(nodeToShardsInfo)) + + // unmount and delete ec shards in parallel (one goroutine per location) + for location, si := range nodeToShardsInfo { + location, si := location, si // capture loop variables for goroutine + ewg.Add(func() error { + fmt.Printf("unmount ec volume %d on %s has shards: %+v\n", vid, location, si.Ids()) + if err := UnmountEcShards(grpcDialOption, vid, location, si.Ids()); err != nil { + return fmt.Errorf("%s unmount ec volume %d on %s: %w", prefix, vid, location, err) + } + + fmt.Printf("delete ec volume %d on %s has shards: %+v\n", vid, location, si.Ids()) + if err := SourceServerDeleteEcShards(grpcDialOption, collection, vid, location, si.Ids()); err != nil { + return fmt.Errorf("%s delete ec volume %d on %s: %w", prefix, vid, location, err) + } + return nil + }) + } + return ewg.Wait() +} + +func verifyDecodedVolumeBeforeDelete(grpcDialOption grpc.DialOption, target pb.ServerAddress, vid needle.VolumeId) error { + var resp *volume_server_pb.ReadVolumeFileStatusResponse + if err := operation.WithVolumeServerClient(false, target, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + r, e := client.ReadVolumeFileStatus(context.Background(), &volume_server_pb.ReadVolumeFileStatusRequest{ + VolumeId: uint32(vid), + }) + if e != nil { + return e + } + resp = r + return nil + }); err != nil { + return fmt.Errorf("read volume file status: %w", err) + } + if resp.DatFileSize == 0 { + return fmt.Errorf("decoded .dat is 0 bytes") + } + if resp.IdxFileSize == 0 { + return fmt.Errorf("decoded .idx is 0 bytes") + } + glog.V(0).Infof("ec decode verification ok for volume %d on %s: dat=%d idx=%d", vid, target, resp.DatFileSize, resp.IdxFileSize) + return nil +} + +func mountDecodedVolume(grpcDialOption grpc.DialOption, targetNodeLocation pb.ServerAddress, vid needle.VolumeId) error { + return operation.WithVolumeServerClient(false, targetNodeLocation, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + _, mountErr := volumeServerClient.VolumeMount(context.Background(), &volume_server_pb.VolumeMountRequest{ + VolumeId: uint32(vid), + }) + return mountErr + }) +} + +func generateNormalVolume(grpcDialOption grpc.DialOption, vid needle.VolumeId, collection string, sourceVolumeServer pb.ServerAddress) error { + fmt.Printf("generateNormalVolume from ec volume %d on %s\n", vid, sourceVolumeServer) + + err := operation.WithVolumeServerClient(false, sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + _, genErr := volumeServerClient.VolumeEcShardsToVolume(context.Background(), &volume_server_pb.VolumeEcShardsToVolumeRequest{ + VolumeId: uint32(vid), + Collection: collection, + }) + return genErr + }) + + return err + +} + +func collectEcShards(env *Env, nodeToShardsInfo map[pb.ServerAddress]*erasure_coding.ShardsInfo, collection string, vid needle.VolumeId, eligibleTargets map[pb.ServerAddress]struct{}, dataShards int) (targetNodeLocation pb.ServerAddress, err error) { + + maxShardCount := -1 + existingShardsInfo := erasure_coding.NewShardsInfo() + for loc, si := range nodeToShardsInfo { + if eligibleTargets != nil { + if _, ok := eligibleTargets[loc]; !ok { + continue + } + } + toBeCopiedShardCount := si.MinusParityShards(dataShards).Count() + if toBeCopiedShardCount > maxShardCount { + maxShardCount = toBeCopiedShardCount + targetNodeLocation = loc + existingShardsInfo = si + } + } + if targetNodeLocation == "" { + return "", fmt.Errorf("no eligible target datanodes available to decode volume %d", vid) + } + + fmt.Printf("collectEcShards: ec volume %d collect shards to %s from: %+v\n", vid, targetNodeLocation, nodeToShardsInfo) + + copiedShardsInfo := erasure_coding.NewShardsInfo() + for loc, si := range nodeToShardsInfo { + if loc == targetNodeLocation { + continue + } + + needToCopyShardsInfo := si.Minus(existingShardsInfo).MinusParityShards(dataShards) + + err = operation.WithVolumeServerClient(false, targetNodeLocation, env.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + + // Always collect .ecj from every shard location. Each server's .ecj + // only contains deletions for needles whose data resides in shards + // held by that server. Without merging all .ecj files, deletions + // recorded on other servers would be lost during decode. + if needToCopyShardsInfo.Count() > 0 { + fmt.Printf("copy %d.%v %s => %s\n", vid, needToCopyShardsInfo.Ids(), loc, targetNodeLocation) + } else { + fmt.Printf("collect ecj %d %s => %s\n", vid, loc, targetNodeLocation) + } + + _, copyErr := volumeServerClient.VolumeEcShardsCopy(context.Background(), &volume_server_pb.VolumeEcShardsCopyRequest{ + VolumeId: uint32(vid), + Collection: collection, + ShardIds: needToCopyShardsInfo.IdsUint32(), + CopyEcxFile: false, + CopyEcjFile: true, + CopyVifFile: needToCopyShardsInfo.Count() > 0, + SourceDataNode: string(loc), + }) + if copyErr != nil { + return fmt.Errorf("copy %d.%v %s => %s: %v", vid, needToCopyShardsInfo.Ids(), loc, targetNodeLocation, copyErr) + } + + if needToCopyShardsInfo.Count() > 0 { + fmt.Printf("mount %d.%v on %s\n", vid, needToCopyShardsInfo.Ids(), targetNodeLocation) + _, mountErr := volumeServerClient.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{ + VolumeId: uint32(vid), + Collection: collection, + ShardIds: needToCopyShardsInfo.IdsUint32(), + }) + if mountErr != nil { + return fmt.Errorf("mount %d.%v on %s: %v", vid, needToCopyShardsInfo.Ids(), targetNodeLocation, mountErr) + } + } + + return nil + }) + + if err != nil { + break + } + + copiedShardsInfo.Add(needToCopyShardsInfo) + } + + nodeToShardsInfo[targetNodeLocation] = existingShardsInfo.Plus(copiedShardsInfo) + + return targetNodeLocation, err +} +func CollectEcShardIds(topoInfo *master_pb.TopologyInfo, collectionRegex *regexp.Regexp, diskType types.DiskType) (vids []needle.VolumeId) { + vidMap := make(map[uint32]bool) + EachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) { + if diskInfo, found := dn.DiskInfos[string(diskType)]; found { + for _, v := range diskInfo.EcShardInfos { + if collectionRegex.MatchString(v.Collection) { + vidMap[v.Id] = true + } + } + } + }) + + for vid := range vidMap { + vids = append(vids, needle.VolumeId(vid)) + } + + return +} + +func collectEcNodeShardsInfo(topoInfo *master_pb.TopologyInfo, vid needle.VolumeId, diskType types.DiskType) (map[pb.ServerAddress]*erasure_coding.ShardsInfo, int) { + res := make(map[pb.ServerAddress]*erasure_coding.ShardsInfo) + EachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) { + if diskInfo, found := dn.DiskInfos[string(diskType)]; found { + // A node may report several EcShardInfos for one volume — one per + // physical disk holding shards of it (multi-disk nodes). Union them + // rather than overwriting, or only the last disk's shards survive and + // the node looks like it is missing shards it actually has. + for _, v := range diskInfo.EcShardInfos { + if v.Id == uint32(vid) { + addr := pb.NewServerAddressFromDataNode(dn) + si := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(v) + if existing, ok := res[addr]; ok { + existing.Add(si) + } else { + res[addr] = si + } + } + } + } + }) + + // OSS is always 10+4; the per-volume ratio override lives in the enterprise build. + return res, erasure_coding.DataShardsCount +} + +type DecodeDiskUsageState struct { + byNode map[pb.ServerAddress]*decodeDiskUsageCounts +} + +type decodeDiskUsageCounts struct { + maxVolumeCount int64 + volumeCount int64 + remoteVolumeCount int64 + ecShardCount int64 +} + +func NewDecodeDiskUsageState(topoInfo *master_pb.TopologyInfo, diskType types.DiskType) *DecodeDiskUsageState { + state := &DecodeDiskUsageState{byNode: make(map[pb.ServerAddress]*decodeDiskUsageCounts)} + EachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) { + if diskInfo, found := dn.DiskInfos[string(diskType)]; found { + state.byNode[pb.NewServerAddressFromDataNode(dn)] = &decodeDiskUsageCounts{ + maxVolumeCount: diskInfo.MaxVolumeCount, + volumeCount: diskInfo.VolumeCount, + remoteVolumeCount: diskInfo.RemoteVolumeCount, + ecShardCount: int64(CountShards(diskInfo.EcShardInfos)), + } + } + }) + return state +} + +func (state *DecodeDiskUsageState) freeVolumeCount(location pb.ServerAddress) (int64, bool) { + if state == nil { + return 0, false + } + usage, found := state.byNode[location] + if !found { + return 0, false + } + free := usage.maxVolumeCount - (usage.volumeCount - usage.remoteVolumeCount) + free -= (usage.ecShardCount + int64(erasure_coding.DataShardsCount) - 1) / int64(erasure_coding.DataShardsCount) + return free, true +} + +func (state *DecodeDiskUsageState) applyDecode(targetNodeLocation pb.ServerAddress, shardCounts map[pb.ServerAddress]int, createdVolume bool) { + if state == nil { + return + } + for location, shardCount := range shardCounts { + if usage, found := state.byNode[location]; found { + usage.ecShardCount -= int64(shardCount) + } + } + if createdVolume { + if usage, found := state.byNode[targetNodeLocation]; found { + usage.volumeCount++ + } + } +} diff --git a/weed/shell/command_ec_destructive_guards_test.go b/weed/ec/ec_destructive_guards_test.go similarity index 94% rename from weed/shell/command_ec_destructive_guards_test.go rename to weed/ec/ec_destructive_guards_test.go index c3a124c0d..b084bd955 100644 --- a/weed/shell/command_ec_destructive_guards_test.go +++ b/weed/ec/ec_destructive_guards_test.go @@ -1,4 +1,4 @@ -package shell +package ec import ( "bytes" @@ -52,7 +52,7 @@ func TestAssertEncodableRegularVolumes(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - err := assertEncodableRegularVolumes(topoWith(tc.regular, tc.ec), tc.vids) + err := AssertEncodableRegularVolumes(topoWith(tc.regular, tc.ec), tc.vids) if tc.wantErrSubstr == "" { if err != nil { t.Fatalf("want nil error, got %v", err) @@ -68,7 +68,7 @@ func TestAssertEncodableRegularVolumes(t *testing.T) { func newDryRunRebuilder(log *bytes.Buffer, nodes ...*EcNode) *ecRebuilder { return &ecRebuilder{ - commandEnv: &CommandEnv{env: make(map[string]string), noLock: true}, + env: &Env{}, ecNodes: nodes, writer: log, applyChanges: false, @@ -114,7 +114,7 @@ func TestPrepareDataToRecover_UnionsLocalShardsAcrossDisks(t *testing.T) { var log bytes.Buffer rebuilder := newEcNode("dc1", "rack1", "rebuilder", 100). addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1}). - addEcVolumeShards(needle.VolumeId(1), "c1", []erasure_coding.ShardId{5}, types.SsdType) + AddEcVolumeShards(needle.VolumeId(1), "c1", []erasure_coding.ShardId{5}, types.SsdType) remote := newEcNode("dc1", "rack1", "remote", 100) erb := newDryRunRebuilder(&log, rebuilder) @@ -151,7 +151,7 @@ func TestCountLocalShards_UnionsAcrossDisks(t *testing.T) { var log bytes.Buffer rebuilder := newEcNode("dc1", "rack1", "rebuilder", 100). addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1}). - addEcVolumeShards(needle.VolumeId(1), "c1", []erasure_coding.ShardId{5}, types.SsdType) + AddEcVolumeShards(needle.VolumeId(1), "c1", []erasure_coding.ShardId{5}, types.SsdType) erb := newDryRunRebuilder(&log, rebuilder) if got := erb.countLocalShards(rebuilder, "c1", needle.VolumeId(1)); got != 3 { @@ -201,12 +201,8 @@ func TestPrepareDataToRecover_ApplyCopyFailureDoesNotCountAsRecoverable(t *testi remote := newEcNode("dc1", "rack1", "127.0.0.1:2", 100). addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{9}) erb := &ecRebuilder{ - commandEnv: &CommandEnv{ - env: make(map[string]string), - noLock: true, - option: &ShellOptions{ - GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()), - }, + env: &Env{ + GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()), }, ecNodes: []*EcNode{rebuilder, remote}, writer: &log, diff --git a/weed/ec/ec_encode.go b/weed/ec/ec_encode.go new file mode 100644 index 000000000..3994a6ad9 --- /dev/null +++ b/weed/ec/ec_encode.go @@ -0,0 +1,931 @@ +package ec + +import ( + "context" + "errors" + "fmt" + "io" + "regexp" + "slices" + "sort" + "strconv" + "sync" + "time" + + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/operation" + "github.com/seaweedfs/seaweedfs/weed/operation/volume_move" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/super_block" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/storage/volume_replica" + "github.com/seaweedfs/seaweedfs/weed/util" + "github.com/seaweedfs/seaweedfs/weed/wdclient" + "google.golang.org/grpc" +) + +// markVolumeReplicaWritable marks one replica writable/readonly with a progress +// line, delegating to the canonical volume_move helper. +func markVolumeReplicaWritable(ctx context.Context, grpcDialOption grpc.DialOption, volumeId needle.VolumeId, location wdclient.Location, writable, persist bool) error { + if writable { + fmt.Printf("markVolumeWritable %d on %s ...\n", volumeId, location.Url) + } else { + fmt.Printf("markVolumeReadonly %d on %s persist=%v ...\n", volumeId, location.Url, persist) + } + return volume_move.NewMover(grpcDialOption).MarkVolumeWritable(ctx, volumeId, location.ServerAddress(), writable, persist) +} + +// deleteVolume removes the volume from sourceVolumeServer via the canonical +// volume_move helper. +func deleteVolume(ctx context.Context, grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sourceVolumeServer pb.ServerAddress, onlyEmpty bool, keepRemoteData bool) (err error) { + return volume_move.NewMover(grpcDialOption).DeleteVolume(ctx, volumeId, sourceVolumeServer, onlyEmpty, keepRemoteData) +} + +func ChunkVolumeIds(volumeIds []needle.VolumeId, batchSize int) [][]needle.VolumeId { + if batchSize <= 0 || len(volumeIds) == 0 { + return [][]needle.VolumeId{volumeIds} + } + var batches [][]needle.VolumeId + for start := 0; start < len(volumeIds); start += batchSize { + end := start + batchSize + if end > len(volumeIds) { + end = len(volumeIds) + } + batches = append(batches, volumeIds[start:end]) + } + return batches +} + +func ProcessEcEncodeBatch(env *Env, writer io.Writer, volumeIds []needle.VolumeId, rp *super_block.ReplicaPlacement, diskType types.DiskType, maxParallelization int, applyBalancing bool, collectionForMessage string) (err error) { + topologyInfo, _, err := env.FetchTopology(0) + if err != nil { + return err + } + + // Refuse to encode a volume that is already EC (present only as shards): + // an EC volume has no .dat, so re-encoding it would tear down its only + // copy before failing. A regular volume (with a .dat) passes. This closes + // the operator-rerun / script-retry path; a worker racing the snapshot is + // handled by encode fencing, not here. + if err := AssertEncodableRegularVolumes(topologyInfo, volumeIds); err != nil { + return err + } + + volumeIdToCollection := CollectVolumeIdToCollection(topologyInfo, volumeIds) + balanceCollections := CollectCollectionsForVolumeIds(topologyInfo, volumeIds) + + fmt.Printf("Collecting volume locations for %d volumes before EC encoding...\n", len(volumeIds)) + volumeLocationsMap, err := volumeLocations(env, volumeIds) + if err != nil { + return fmt.Errorf("failed to collect volume locations before EC encoding: %w", err) + } + + if err := checkEcEncodeCapacity(topologyInfo, len(volumeIds), diskType, collectionForMessage); err != nil { + return err + } + + // From here doEcEncode marks the volumes readonly and generates EC shards. + // If any step before the originals are deleted fails, roll the encode back: + // tear down the shards produced this run and restore the sources to writable, + // so a failed (and possibly abandoned) ec.encode does not strand volumes + // readonly or leave orphan EC shards behind. Once the shards are verified + // recoverable we are committed to the EC copy and must not roll back. + committed := false + defer func() { + if err != nil && !committed { + rollbackFailedEcEncode(env, writer, volumeIds, volumeIdToCollection, volumeLocationsMap, maxParallelization) + } + }() + + skippedNodes, err := doEcEncode(env, writer, volumeIdToCollection, volumeIds, maxParallelization, topologyInfo) + if err != nil { + return fmt.Errorf("ec encode for volumes %v: %w", volumeIds, err) + } + // Mounting the new shards notifies the master asynchronously, and EcBalance + // plans from a fresh topology snapshot: one taken before the mounts land + // shows no shards for these volumes, so the balance plans no moves and + // silently leaves every shard on the generation host. + if err := waitForEcShardsToRegister(env, volumeIds); err != nil { + return fmt.Errorf("wait for ec shards to register with the master: %w", err) + } + // EcBalance works at collection scope. In batch mode this intentionally + // rebalances each collection after every batch so source volumes can be + // safely verified and deleted without waiting for all batches to finish. + // skippedNodes are excluded so a recovered node's stale orphan is never + // paired with a new-generation shard. + if err := EcBalance(env, balanceCollections, "", rp, diskType, maxParallelization, 0, applyBalancing, skippedNodes, nil); err != nil { + return fmt.Errorf("re-balance ec shards for collection(s) %v: %w", balanceCollections, err) + } + if err := verifyEcShardsBeforeDelete(env, volumeIds, diskType, applyBalancing); err != nil { + return fmt.Errorf("verify EC shards before deleting originals: %w", err) + } + // Past verify the EC copy is recoverable; a delete failure below must not + // tear the shards down. + committed = true + fmt.Printf("Deleting original volumes after EC encoding...\n") + if err := doDeleteVolumesWithLocations(env, volumeIds, volumeLocationsMap, maxParallelization); err != nil { + return fmt.Errorf("delete original volumes after EC encoding: %w", err) + } + fmt.Printf("Successfully completed EC encoding for %d volumes\n", len(volumeIds)) + return nil +} + +// rollbackFailedEcEncode is a best-effort cleanup for an ec.encode that failed +// after marking volumes readonly / generating shards but before the originals +// were deleted. It tears down the EC shards this run produced (so they do not +// survive as orphans until the next encode) and restores the source volumes to +// writable (so a failed and possibly abandoned encode does not strand them +// readonly). Errors are logged, not returned — we are already on the failure +// path. Both operations are idempotent: clearPreexistingEcShards is a no-op when +// no shards were generated, and marking an already-writable volume is a no-op, +// so it is safe even for a failure before the volumes were marked readonly. +func rollbackFailedEcEncode(env *Env, writer io.Writer, volumeIds []needle.VolumeId, volumeIdToCollection map[needle.VolumeId]string, volumeLocationsMap map[needle.VolumeId][]wdclient.Location, maxParallelization int) { + fmt.Fprintf(writer, "rolling back failed EC encode for volumes %v...\n", volumeIds) + + // Tear down any EC shards this run produced. A fresh topology snapshot finds + // them wherever generate/balance left them; the teardown is blanket. + if topologyInfo, _, err := env.FetchTopology(0); err != nil { + fmt.Fprintf(writer, "rollback: collect topology to clear ec shards: %v\n", err) + } else if _, err := clearPreexistingEcShards(env, topologyInfo, volumeIds, volumeIdToCollection, maxParallelization); err != nil { + fmt.Fprintf(writer, "rollback: clear ec shards: %v\n", err) + } + + // Restore the source volumes to writable. doEcEncode re-reads the locations + // and marks every replica of that later snapshot readonly, so re-read here + // too: a replica added or moved between the batch's initial snapshot + // (volumeLocationsMap) and doEcEncode's readonly-marking would otherwise be + // left readonly. Fall back to the initial snapshot if the re-read fails. + locations := volumeLocationsMap + if fresh, err := volumeLocations(env, volumeIds); err != nil { + fmt.Fprintf(writer, "rollback: re-read volume locations (using pre-encode snapshot): %v\n", err) + } else { + locations = fresh + } + ewg := util.NewErrorWaitGroup(maxParallelization) + for _, vid := range volumeIds { + for _, l := range locations[vid] { + ewg.Add(func() error { + if err := markVolumeReplicaWritable(context.Background(), env.GrpcDialOption, vid, l, true, false); err != nil { + return fmt.Errorf("restore volume %d writable on %s: %w", vid, l.Url, err) + } + return nil + }) + } + } + if err := ewg.Wait(); err != nil { + fmt.Fprintf(writer, "rollback: %v\n", err) + } +} + +func checkEcEncodeCapacity(topologyInfo *master_pb.TopologyInfo, volumeCount int, diskType types.DiskType, collectionForMessage string) error { + // Pre-flight check: verify the target disk type has capacity for EC shards. + // This prevents encoding shards only to fail during rebalance. Reuse the + // caller's topology snapshot instead of issuing another VolumeList to the + // master per batch. + _, totalFreeEcSlots := CollectEcVolumeServersByDc(topologyInfo, "", diskType) + + // Each volume needs TotalShardsCount (14) shards distributed. + requiredSlots := volumeCount * erasure_coding.TotalShardsCount + if totalFreeEcSlots < 1 { + if diskType != types.HardDriveType { + tryDiskTypeMessage := "Try passing -diskType=hdd, or omit -diskType to use the default (hdd)" + if collectionForMessage != "" { + tryDiskTypeMessage = fmt.Sprintf("Try:\n ec.encode -collection=%s -diskType=hdd\nOr omit -diskType to use the default (hdd)", collectionForMessage) + } + return fmt.Errorf("no free ec shard slots on disk type '%s'. The target disk type has no capacity.\n"+ + "Your volumes are likely on a different disk type. %s", diskType, tryDiskTypeMessage) + } + return fmt.Errorf("no free ec shard slots. only %d left on disk type '%s'", totalFreeEcSlots, diskType) + } + + if totalFreeEcSlots < requiredSlots { + fmt.Printf("Warning: limited EC shard capacity. Need %d slots for %d volumes, but only %d slots available on disk type '%s'.\n", + requiredSlots, volumeCount, totalFreeEcSlots, diskType) + fmt.Printf("Rebalancing may not achieve optimal distribution.\n") + } + return nil +} + +func volumeLocations(env *Env, volumeIds []needle.VolumeId) (map[needle.VolumeId][]wdclient.Location, error) { + res := map[needle.VolumeId][]wdclient.Location{} + for _, vid := range volumeIds { + ls, ok := env.GetVolumeLocations(uint32(vid)) + if !ok { + return nil, fmt.Errorf("volume %d not found", vid) + } + res[vid] = ls + } + + return res, nil +} + +func doEcEncode(env *Env, writer io.Writer, volumeIdToCollection map[needle.VolumeId]string, volumeIds []needle.VolumeId, maxParallelization int, topologyInfo *master_pb.TopologyInfo) (skippedNodes map[pb.ServerAddress]struct{}, err error) { + if !env.isLocked() { + return nil, fmt.Errorf("lock is lost") + } + locations, err := volumeLocations(env, volumeIds) + if err != nil { + return nil, fmt.Errorf("failed to get volume locations for EC encoding: %w", err) + } + + // Clear EC shards left by a previous failed/partial encode so a retry + // starts clean and never mixes two encode runs. A node skipped here as + // unreachable is excluded from the later balance: it may still hold a stale + // orphan that, paired with a new-generation shard from a balance copy, would + // mix generations on that node. + skippedNodes, err = clearPreexistingEcShards(env, topologyInfo, volumeIds, volumeIdToCollection, maxParallelization) + if err != nil { + return nil, fmt.Errorf("clear pre-existing ec shards before encoding: %w", err) + } + + // Build a map of (volumeId, serverAddress) -> freeVolumeCount. + // Key by dn.Address so it matches wdclient.Location.Url. In deployments + // where dn.Id is a short name (e.g. Kubernetes StatefulSet pod name) + // while dn.Address is a FQDN:port, keying by dn.Id would never match the + // location Url during the health-check lookup below. + freeVolumeCountMap := make(map[string]int) // key: volumeId-serverAddress + EachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) { + addr := dn.Address + if addr == "" { + addr = dn.Id // older nodes use ip:port as id + } + for _, diskInfo := range dn.DiskInfos { + for _, v := range diskInfo.VolumeInfos { + key := fmt.Sprintf("%d-%s", v.Id, addr) + freeVolumeCountMap[key] = int(diskInfo.FreeVolumeCount) + } + } + }) + + // Filter replicas by free capacity BEFORE marking volumes readonly so that + // a failed health check does not strand volumes in readonly state. + filteredLocations := make(map[needle.VolumeId][]wdclient.Location) + for _, vid := range volumeIds { + var filteredLocs []wdclient.Location + for _, l := range locations[vid] { + key := fmt.Sprintf("%d-%s", vid, l.Url) + if freeCount, found := freeVolumeCountMap[key]; found && freeCount >= 2 { + filteredLocs = append(filteredLocs, l) + } + } + if len(filteredLocs) == 0 { + return nil, fmt.Errorf("no healthy replicas (FreeVolumeCount >= 2) found for volume %d to use as source for EC encoding", vid) + } + filteredLocations[vid] = filteredLocs + } + + // mark volumes as readonly + ewg := util.NewErrorWaitGroup(maxParallelization) + for _, vid := range volumeIds { + for _, l := range locations[vid] { + ewg.Add(func() error { + if err := markVolumeReplicaWritable(context.Background(), env.GrpcDialOption, vid, l, false, false); err != nil { + return fmt.Errorf("mark volume %d as readonly on %s: %v", vid, l.Url, err) + } + return nil + }) + } + } + if err := ewg.Wait(); err != nil { + return nil, err + } + + // Sync replicas and select the best one for each volume (with highest file count) + // This addresses data inconsistency risk in multi-replica volumes (issue #7797) + // by syncing missing entries between replicas before encoding + bestReplicas := make(map[needle.VolumeId]wdclient.Location) + for _, vid := range volumeIds { + collection := volumeIdToCollection[vid] + + // Sync missing entries between replicas, then select the best one + bestLoc, selectErr := volume_replica.SyncAndSelectBestReplica(env.GrpcDialOption, vid, collection, filteredLocations[vid], "", writer) + if selectErr != nil { + return nil, fmt.Errorf("failed to sync and select replica for volume %d: %v", vid, selectErr) + } + bestReplicas[vid] = bestLoc + } + + // Re-attempt the orphan sweep on the nodes skipped as unreachable, now that + // any node that recovered during readonly-marking and replica sync answers + // again. A node whose teardown now succeeds is clean (and the generation host + // re-wipes its own disks regardless), so it leaves the skipped set and can be + // a balance source/target — otherwise its shards would never distribute off + // it. A node that is still down stays skipped and excluded, preserving the + // leniency for a genuinely-down node; such a node also cannot be the + // generation host below, since VolumeEcShardsGenerate would fail to read .dat. + if err := resweepSkippedNodes(env, skippedNodes, volumeIds, volumeIdToCollection, maxParallelization); err != nil { + return nil, err + } + + // A selected generation host still in skippedNodes after the re-sweep was + // transport-down when we tried to clean it, so its stale orphans were never + // removed and EcBalance excludes it as both source and target. If it recovers + // just in time for generation, all shards land on a node we can neither clean + // nor balance off — a single point of failure that union-only verification + // still accepts, after which the originals are deleted. Abort instead. + for _, vid := range volumeIds { + genHost := bestReplicas[vid].ServerAddress() + if _, stillSkipped := skippedNodes[genHost]; stillSkipped { + return nil, fmt.Errorf("generate ec shards for volume %d aborted: selected source %s is still skipped after the orphan re-sweep", vid, genHost) + } + } + + // generate ec shards using the best replica for each volume + ewg.Reset() + for _, vid := range volumeIds { + target := bestReplicas[vid] + collection := volumeIdToCollection[vid] + ewg.Add(func() error { + if err := generateEcShards(env.GrpcDialOption, vid, collection, target.ServerAddress()); err != nil { + return fmt.Errorf("generate ec shards for volume %d on %s: %v", vid, target.Url, err) + } + return nil + }) + } + if err := ewg.Wait(); err != nil { + return nil, err + } + + // mount all ec shards for the converted volume + shardIds := erasure_coding.AllShardIds() + + ewg.Reset() + for _, vid := range volumeIds { + target := bestReplicas[vid] + collection := volumeIdToCollection[vid] + ewg.Add(func() error { + if err := MountEcShards(env.GrpcDialOption, collection, vid, target.ServerAddress(), shardIds); err != nil { + return fmt.Errorf("mount ec shards for volume %d on %s: %v", vid, target.Url, err) + } + return nil + }) + } + if err := ewg.Wait(); err != nil { + return nil, err + } + + return skippedNodes, nil +} + +// clearPreexistingEcShards removes EC shards and index files left over from a +// previous (failed or partial) encode of the given volume ids, on every node +// that still reports them, so a fresh encode regenerates from a clean slate. +// Scans all disk types. The normal .dat/.idx — the source of truth for this +// encode — is untouched; only orphaned EC artifacts are deleted. +// +// Returns the set of nodes skipped as unreachable. A skipped node may still hold +// an un-deleted orphan from a prior run; if it recovers it must be kept out of +// this encode's shard distribution, or the balance could install the new +// generation alongside the stale orphan and mix generations on one node. +func clearPreexistingEcShards(env *Env, topologyInfo *master_pb.TopologyInfo, volumeIds []needle.VolumeId, volumeIdToCollection map[needle.VolumeId]string, maxParallelization int) (skipped map[pb.ServerAddress]struct{}, err error) { + wanted := make(map[uint32]bool, len(volumeIds)) + for _, vid := range volumeIds { + wanted[uint32(vid)] = true + } + + // Note which (node, vid) pairs the topology already reports EC shards for: + // those are mounted leftovers and cleaning them is required (fatal on + // error). Every other (node, vid) is swept best-effort to catch UNMOUNTED + // orphans left by a failed copy — invisible to the heartbeat, so absent + // here. A node that is down or holds nothing is a harmless no-op; a node + // unreachable now also cannot receive this encode's new generation, so a + // surviving orphan there keeps its old identity and the read guard rejects + // it. Always delete the full shard-id range so a wider custom ratio's + // leftovers are covered too. + reportedKey := func(addr pb.ServerAddress, vid uint32) string { + return string(addr) + "\x00" + strconv.FormatUint(uint64(vid), 10) + } + reported := make(map[string]struct{}) + var nodes []pb.ServerAddress + EachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) { + addr := pb.NewServerAddressFromDataNode(dn) + nodes = append(nodes, addr) + for _, diskInfo := range dn.DiskInfos { + for _, ecInfo := range diskInfo.EcShardInfos { + if wanted[ecInfo.Id] { + reported[reportedKey(addr, ecInfo.Id)] = struct{}{} + } + } + } + }) + + allShardIds := make([]erasure_coding.ShardId, erasure_coding.MaxShardCount) + for i := range allShardIds { + allShardIds[i] = erasure_coding.ShardId(i) + } + + if len(reported) > 0 { + fmt.Printf("clearing stale EC shards reported for %d (node,volume) pair(s) before regenerating...\n", len(reported)) + } + // Nodes skipped as unreachable, accumulated across the concurrent sweep tasks. + skipped = make(map[pb.ServerAddress]struct{}) + var skippedMu sync.Mutex + ewg := util.NewErrorWaitGroup(maxParallelization) + for _, addr := range nodes { + for _, vid := range volumeIds { + fatal := false + if _, ok := reported[reportedKey(addr, uint32(vid))]; ok { + fatal = true + } + collection := volumeIdToCollection[vid] + ewg.Add(func() error { + if err := UnmountAndDeleteEcShardsQuiet(env.GrpcDialOption, collection, vid, addr, allShardIds); err != nil { + // Surface a reachable node whose delete genuinely failed (its orphan would + // be re-stamped by a later copy installing the new .vif). A missing + // full_teardown ack from a reachable pre-upgrade node is fatal too: it may + // still hold an orphan a later copy would re-stamp into the new generation. + // Stay best-effort only for a node that is truly unreachable: codes.Unavailable + // alone is ambiguous — a genuinely-down node and a reachable Rust volume + // server in maintenance mode both return it (a Go server returns Unknown for + // maintenance, already fatal above). Confirm with a non-maintenance-gated Ping + // before skipping; skip only when the Ping itself transport-failed (NodeDown). + // A reachable maintenance node (nodeUp) CAN receive this generation, and an + // inconclusive Ping (nodeLivenessUnknown, e.g. a pre-Ping server returning + // Unimplemented — which means the node is up) does not prove the node is down, + // so both stay fatal rather than silently leaving a stale EC generation. + if fatal || errors.Is(err, ErrFullTeardownNotAcked) || !IsNodeUnreachable(err) || + ClassifyNodeLiveness(PingVolumeServer(env.GrpcDialOption, addr)) != NodeDown { + return fmt.Errorf("clear stale ec shards for volume %d on %s: %w", vid, addr, err) + } + glog.V(1).Infof("orphan sweep: volume %d on %s skipped (unreachable): %v", vid, addr, err) + skippedMu.Lock() + skipped[addr] = struct{}{} + skippedMu.Unlock() + } + return nil + }) + } + } + if err := ewg.Wait(); err != nil { + return nil, err + } + return skipped, nil +} + +// resweepSkippedNodes re-attempts the orphan teardown on the nodes that the +// initial sweep skipped as unreachable, just before shard generation. A node +// that recovered in the meantime — and is therefore eligible to host this +// encode's generation — has its teardown retried; if it now fully succeeds it is +// removed from skipped so the rebalance can use it as a source and move its +// shards off, instead of stranding all shards on the single generation host and +// collapsing fault tolerance. A node still transport-down stays skipped (the +// same leniency the initial sweep grants), and a node that came back reachable +// but whose delete genuinely failed is fatal, exactly as in the initial sweep, +// so a stale generation is never silently left behind. Mutates skipped in place. +func resweepSkippedNodes(env *Env, skipped map[pb.ServerAddress]struct{}, volumeIds []needle.VolumeId, volumeIdToCollection map[needle.VolumeId]string, maxParallelization int) error { + if len(skipped) == 0 { + return nil + } + + allShardIds := make([]erasure_coding.ShardId, erasure_coding.MaxShardCount) + for i := range allShardIds { + allShardIds[i] = erasure_coding.ShardId(i) + } + + addrs := make([]pb.ServerAddress, 0, len(skipped)) + for addr := range skipped { + addrs = append(addrs, addr) + } + + fmt.Printf("re-checking %d node(s) skipped by the orphan sweep before generating shards...\n", len(addrs)) + + // A node still down on every retried vid stays skipped; one that fully + // succeeds is un-skipped. Track per-node whether any retry still failed + // (down) so a node whose state is mixed across vids never gets un-skipped. + stillDown := make(map[pb.ServerAddress]struct{}) + var mu sync.Mutex + ewg := util.NewErrorWaitGroup(maxParallelization) + for _, addr := range addrs { + for _, vid := range volumeIds { + collection := volumeIdToCollection[vid] + ewg.Add(func() error { + if err := UnmountAndDeleteEcShardsQuiet(env.GrpcDialOption, collection, vid, addr, allShardIds); err != nil { + // Same decision as the initial sweep: a reachable node whose delete + // genuinely failed (or did not ack a full teardown, or whose liveness is + // inconclusive) is fatal, since it could hold an orphan a later copy + // re-stamps into this generation. Only a node still transport-down stays + // skipped. + if errors.Is(err, ErrFullTeardownNotAcked) || !IsNodeUnreachable(err) || + ClassifyNodeLiveness(PingVolumeServer(env.GrpcDialOption, addr)) != NodeDown { + return fmt.Errorf("re-clear stale ec shards for volume %d on %s: %w", vid, addr, err) + } + glog.V(1).Infof("orphan re-sweep: volume %d on %s still skipped (unreachable): %v", vid, addr, err) + mu.Lock() + stillDown[addr] = struct{}{} + mu.Unlock() + } + return nil + }) + } + } + if err := ewg.Wait(); err != nil { + return err + } + + for _, addr := range addrs { + if _, down := stillDown[addr]; !down { + delete(skipped, addr) + glog.V(0).Infof("orphan re-sweep: node %s recovered and was cleaned; it will participate in the EC rebalance", addr) + } + } + return nil +} + +// collectEcShardBitsByNode returns, for one volume, the EC shard bits each node +// reports, unioned across all its disk types. Freshly generated shards sit on +// the disk that held the source .dat, which may differ from the balance target +// disk type, so visibility questions ("has the master heard about these shards +// at all?") must not filter by disk type. Only the newest encode generation +// (largest EncodeTsNs) counts: an orphaned older generation — a failed earlier +// encode on a node the pre-encode sweep could not reach but the master still +// hears from — must neither satisfy the registration wait nor pose as a second +// holder in the clump check. Entries without a timestamp form the legacy +// generation zero, so they only count when no stamped generation exists; +// dropping them otherwise errs toward keeping the source volume. +func CollectEcShardBitsByNode(topoInfo *master_pb.TopologyInfo, vid needle.VolumeId) map[pb.ServerAddress]erasure_coding.ShardBits { + type shardEntry struct { + addr pb.ServerAddress + ts int64 + bits erasure_coding.ShardBits + } + var entries []shardEntry + var newestTs int64 + EachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) { + for _, diskInfo := range dn.DiskInfos { + for _, ecInfo := range diskInfo.EcShardInfos { + if ecInfo.Id != uint32(vid) { + continue + } + entries = append(entries, shardEntry{pb.NewServerAddressFromDataNode(dn), ecInfo.EncodeTsNs, erasure_coding.ShardBits(ecInfo.EcIndexBits)}) + if ecInfo.EncodeTsNs > newestTs { + newestTs = ecInfo.EncodeTsNs + } + } + } + }) + res := make(map[pb.ServerAddress]erasure_coding.ShardBits) + for _, e := range entries { + if e.ts == newestTs { + res[e.addr] |= e.bits + } + } + return res +} + +// waitForEcShardsToRegister polls the master topology until every given volume +// reports a full EC shard set. Mounting shards notifies the master +// asynchronously (mount -> NewEcShardsChan -> delta heartbeat), so a topology +// snapshot taken right after doEcEncode can predate the mounts. Failing after +// the retries keeps the source volumes, and a re-run of ec.encode starts clean. +func waitForEcShardsToRegister(env *Env, volumeIds []needle.VolumeId) error { + const maxAttempts = 10 + const retryInterval = 2 * time.Second + + var lastMissing []string + for attempt := 0; attempt < maxAttempts; attempt++ { + if attempt > 0 { + time.Sleep(retryInterval) + } + topoInfo, _, err := env.FetchTopology(0) + if err != nil { + return fmt.Errorf("fetch topology while waiting for ec shards to register: %w", err) + } + lastMissing = lastMissing[:0] + for _, vid := range volumeIds { + var union erasure_coding.ShardBits + for _, bits := range CollectEcShardBitsByNode(topoInfo, vid) { + union |= bits + } + if union.Count() < erasure_coding.TotalShardsCount { + lastMissing = append(lastMissing, fmt.Sprintf("volume %d: %d/%d shards", vid, union.Count(), erasure_coding.TotalShardsCount)) + } + } + if len(lastMissing) == 0 { + return nil + } + glog.V(0).Infof("waiting for newly generated ec shards to register with the master (attempt %d/%d): %v", + attempt+1, maxAttempts, lastMissing) + } + return fmt.Errorf("newly generated ec shards did not register with the master after %d attempts: %v", maxAttempts, lastMissing) +} + +// ecShardsClumpedOnOneNode reports whether every EC shard the master sees for +// vid sits on a single node while at least one other node has free EC shard +// slots on the target disk type — i.e. the preceding rebalance could have +// spread the shards but did not. Zero visible shards is not a clump; the +// recoverability check owns that case. +func ecShardsClumpedOnOneNode(topoInfo *master_pb.TopologyInfo, vid needle.VolumeId, diskType types.DiskType) (holder pb.ServerAddress, clumped bool) { + byNode := CollectEcShardBitsByNode(topoInfo, vid) + if len(byNode) != 1 { + return "", false + } + for addr := range byNode { + holder = addr + } + ecNodes, _ := CollectEcVolumeServersByDc(topoInfo, "", diskType) + for _, en := range ecNodes { + if pb.NewServerAddressFromDataNode(en.Info) == holder { + continue + } + if en.FreeEcSlot > 0 { + return holder, true + } + } + return "", false +} + +// ecShardSummaryByNode says where a volume's shards are, one entry per node, +// sorted so the message is stable. It names the ids and not just the count: a +// set holding shards 0-9 and one holding 4-13 are both "10 shards", and which +// ones survived is what says whether the set is recoverable and from where. +func ecShardSummaryByNode(byNode map[pb.ServerAddress]erasure_coding.ShardBits) []string { + summary := make([]string, 0, len(byNode)) + for node, bits := range byNode { + summary = append(summary, fmt.Sprintf("%s=%d shards %v", node, bits.Count(), slices.Collect(bits.All()))) + } + sort.Strings(summary) + return summary +} + +func verifyEcShardsBeforeDelete(env *Env, volumeIds []needle.VolumeId, diskType types.DiskType, expectSpread bool) error { + // Shard relocations from the preceding EC balance reach the master via + // volume-server heartbeats, so freshly distributed shards may not all be + // visible in the master topology immediately. Poll a few times before + // concluding the shard set is incomplete, so a heartbeat-propagation lag is + // not mistaken for missing data. After the retries: a volume below the + // recoverable threshold (dataShards) aborts the deletion; a recoverable + // but degraded set proceeds with a warning, since the missing shards can + // be rebuilt from the survivors while keeping the source next to live + // shards is the more dangerous mixed state. When expectSpread is set (the + // rebalance ran in apply mode), a volume whose shards all still sit on one + // node while another node has free slots also aborts the deletion: losing + // that node would lose the volume, so the original is the safer copy. + const maxAttempts = 10 + const retryInterval = 2 * time.Second + + var lastErr error + var lastDegraded []string + var lastClumped []string + for attempt := 0; attempt < maxAttempts; attempt++ { + topoInfo, _, err := env.FetchTopology(0) + if err != nil { + return fmt.Errorf("fetch topology for shard verification: %w", err) + } + + lastErr = nil + lastDegraded = lastDegraded[:0] + lastClumped = lastClumped[:0] + for _, vid := range volumeIds { + // Count the shards wherever they landed, as waitForEcShardsToRegister + // above already does. generateEcShards writes them beside the source + // volume, so encoding a volume that lives on a non-default medium + // puts them on that medium while -diskType still says hdd. Counting + // only the -diskType bucket then reports a complete set as entirely + // missing and aborts an encode that in fact succeeded, leaving the + // volume as both a .dat and a full set of shards. + byNode := CollectEcShardBitsByNode(topoInfo, vid) + + var union erasure_coding.ShardBits + for _, bits := range byNode { + union |= bits + } + + totalShards := erasure_coding.TotalShardsCount + degraded, err := erasure_coding.RequireRecoverableShardSet(uint32(vid), union, erasure_coding.DataShardsCount, totalShards) + if err != nil { + lastErr = fmt.Errorf("volume %d: %w (observed: %v)", vid, err, ecShardSummaryByNode(byNode)) + break + } + if expectSpread { + if holder, clumped := ecShardsClumpedOnOneNode(topoInfo, vid, diskType); clumped { + lastClumped = append(lastClumped, fmt.Sprintf("volume %d: all shards on %s", vid, holder)) + continue + } + } + if degraded { + lastDegraded = append(lastDegraded, fmt.Sprintf("volume %d: %d/%d shards", vid, union.Count(), totalShards)) + continue + } + + glog.V(0).Infof("EC shard verification ok for volume %d: %d/%d shards present across %d nodes", + vid, union.Count(), totalShards, len(byNode)) + } + + if lastErr == nil && len(lastDegraded) == 0 && len(lastClumped) == 0 { + return nil + } + if attempt < maxAttempts-1 { + glog.V(0).Infof("EC shard verification incomplete (attempt %d/%d), waiting for shard locations to propagate: %v %v %v", + attempt+1, maxAttempts, lastErr, lastDegraded, lastClumped) + time.Sleep(retryInterval) + } + } + + if lastErr != nil { + glog.Errorf("EC shard verification failed after %d attempts: %v", maxAttempts, lastErr) + return lastErr + } + if len(lastClumped) > 0 { + return fmt.Errorf("EC shards still sit on a single node after rebalance even though other nodes have free slots (%v); keeping the original volumes. Run ec.balance -apply, verify the spread, then delete the originals or re-run ec.encode", lastClumped) + } + glog.Warningf("EC shard set incomplete but recoverable after %d attempts, proceeding with source deletion (rebuild missing shards with ec.rebuild): %v", + maxAttempts, lastDegraded) + return nil +} + +// doDeleteVolumesWithLocations deletes volumes using pre-collected location information +// This avoids race conditions where master metadata is updated after EC encoding +func doDeleteVolumesWithLocations(env *Env, volumeIds []needle.VolumeId, volumeLocationsMap map[needle.VolumeId][]wdclient.Location, maxParallelization int) error { + if !env.isLocked() { + return fmt.Errorf("lock is lost") + } + + ewg := util.NewErrorWaitGroup(maxParallelization) + for _, vid := range volumeIds { + locations, found := volumeLocationsMap[vid] + if !found { + fmt.Printf("warning: no locations found for volume %d, skipping deletion\n", vid) + continue + } + + for _, l := range locations { + ewg.Add(func() error { + if err := deleteVolume(context.Background(), env.GrpcDialOption, vid, l.ServerAddress(), false, false); err != nil { + return fmt.Errorf("deleteVolume %s volume %d: %v", l.Url, vid, err) + } + fmt.Printf("deleted volume %d from %s\n", vid, l.Url) + return nil + }) + } + } + if err := ewg.Wait(); err != nil { + return err + } + + return nil +} + +func generateEcShards(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, collection string, sourceVolumeServer pb.ServerAddress) error { + + fmt.Printf("generateEcShards %d (collection %q) on %s ...\n", volumeId, collection, sourceVolumeServer) + + err := operation.WithVolumeServerClient(false, sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + _, genErr := volumeServerClient.VolumeEcShardsGenerate(context.Background(), &volume_server_pb.VolumeEcShardsGenerateRequest{ + VolumeId: uint32(volumeId), + Collection: collection, + }) + return genErr + }) + + return err + +} + +func SelectVolumeIdsFromTopology(topologyInfo *master_pb.TopologyInfo, volumeSizeLimitMb uint64, collectionRegex *regexp.Regexp, sourceDiskType *types.DiskType, quietSeconds int64, nowUnixSeconds int64, fullPercentage float64, verbose bool) (vids []needle.VolumeId, matchedCollections []string) { + // Statistics for verbose mode + var ( + totalVolumes int + remoteVolumes int + wrongCollection int + wrongDiskType int + tooRecent int + tooSmall int + noFreeDisk int + ) + + vidMap := make(map[uint32]bool) + collectionSet := make(map[string]bool) + EachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) { + for _, diskInfo := range dn.DiskInfos { + for _, v := range diskInfo.VolumeInfos { + totalVolumes++ + + // ignore remote volumes + if v.RemoteStorageName != "" { + remoteVolumes++ + if verbose { + fmt.Printf("skip volume %d on %s: remote volume (storage: %s)\n", + v.Id, dn.Id, v.RemoteStorageName) + } + continue + } + + // check collection against regex pattern + if !collectionRegex.MatchString(v.Collection) { + wrongCollection++ + if verbose { + fmt.Printf("skip volume %d on %s: collection doesn't match pattern (pattern: %s, actual: %s)\n", + v.Id, dn.Id, collectionRegex.String(), v.Collection) + } + continue + } + + // track matched collection + collectionSet[v.Collection] = true + + // check disk type + if sourceDiskType != nil && types.ToDiskType(v.DiskType) != *sourceDiskType { + wrongDiskType++ + if verbose { + fmt.Printf("skip volume %d on %s: wrong disk type (expected: %s, actual: %s)\n", + v.Id, dn.Id, sourceDiskType.ReadableString(), types.ToDiskType(v.DiskType).ReadableString()) + } + continue + } + + // check quiet period + if v.ModifiedAtSecond+quietSeconds >= nowUnixSeconds { + tooRecent++ + if verbose { + fmt.Printf("skip volume %d on %s: too recently modified (last modified: %d seconds ago, required: %d seconds)\n", + v.Id, dn.Id, nowUnixSeconds-v.ModifiedAtSecond, quietSeconds) + } + continue + } + + // check size + sizeThreshold := fullPercentage / 100 * float64(volumeSizeLimitMb) * 1024 * 1024 + if float64(v.Size) <= sizeThreshold { + tooSmall++ + if verbose { + fmt.Printf("skip volume %d on %s: too small (size: %.1f MB, threshold: %.1f MB, %.1f%% full)\n", + v.Id, dn.Id, float64(v.Size)/(1024*1024), sizeThreshold/(1024*1024), + float64(v.Size)*100/(float64(volumeSizeLimitMb)*1024*1024)) + } + continue + } + + // check free disk space + if diskInfo.FreeVolumeCount < 2 { + glog.V(0).Infof("replica %s %d on %s has no free disk", v.Collection, v.Id, dn.Id) + if verbose { + fmt.Printf("skip replica of volume %d on %s: insufficient free disk space (free volumes: %d, required: 2)\n", + v.Id, dn.Id, diskInfo.FreeVolumeCount) + } + if _, found := vidMap[v.Id]; !found { + vidMap[v.Id] = false + } + } else { + if verbose { + fmt.Printf("selected volume %d on %s: size %.1f MB (%.1f%% full), last modified %d seconds ago, free volumes: %d\n", + v.Id, dn.Id, float64(v.Size)/(1024*1024), + float64(v.Size)*100/(float64(volumeSizeLimitMb)*1024*1024), + nowUnixSeconds-v.ModifiedAtSecond, diskInfo.FreeVolumeCount) + } + vidMap[v.Id] = true + } + } + } + }) + + for vid, good := range vidMap { + if good { + vids = append(vids, needle.VolumeId(vid)) + } else { + noFreeDisk++ + } + } + + // Convert collection set to slice + for collection := range collectionSet { + matchedCollections = append(matchedCollections, collection) + } + sort.Strings(matchedCollections) + + // Print summary statistics in verbose mode or when no volumes selected + if verbose || len(vids) == 0 { + fmt.Printf("\nVolume selection summary:\n") + fmt.Printf(" Total volumes examined: %d\n", totalVolumes) + fmt.Printf(" Selected for encoding: %d\n", len(vids)) + fmt.Printf(" Collections matched: %v\n", matchedCollections) + + if totalVolumes > 0 { + fmt.Printf("\nReasons for exclusion:\n") + if remoteVolumes > 0 { + fmt.Printf(" Remote volumes: %d\n", remoteVolumes) + } + if wrongCollection > 0 { + fmt.Printf(" Collection doesn't match pattern: %d\n", wrongCollection) + } + if wrongDiskType > 0 { + fmt.Printf(" Wrong disk type: %d\n", wrongDiskType) + } + if tooRecent > 0 { + fmt.Printf(" Too recently modified: %d\n", tooRecent) + } + if tooSmall > 0 { + fmt.Printf(" Too small (< %.1f%% full): %d\n", fullPercentage, tooSmall) + } + if noFreeDisk > 0 { + fmt.Printf(" Insufficient free disk space: %d\n", noFreeDisk) + } + } + fmt.Println() + } + + return +} diff --git a/weed/shell/command_ec_encode_test.go b/weed/ec/ec_encode_test.go similarity index 97% rename from weed/shell/command_ec_encode_test.go rename to weed/ec/ec_encode_test.go index 5ac95a1b1..ce9b70e52 100644 --- a/weed/shell/command_ec_encode_test.go +++ b/weed/ec/ec_encode_test.go @@ -1,4 +1,4 @@ -package shell +package ec import ( "regexp" @@ -177,7 +177,7 @@ func TestSelectVolumeIdsFromTopology(t *testing.T) { // In the provided topology, FreeVolumeCount is 1 ("free:1"), so it is < 2. // So ALL volumes should be skipped due to insufficient free disk space. - vids, _ := selectVolumeIdsFromTopology(topologyInfo, volumeSizeLimitMb, collectionRegex, nil, quietSeconds, nowUnixSeconds, fullPercentage, verbose) + vids, _ := SelectVolumeIdsFromTopology(topologyInfo, volumeSizeLimitMb, collectionRegex, nil, quietSeconds, nowUnixSeconds, fullPercentage, verbose) assert.Equal(t, 0, len(vids), "Should select 0 volumes because FreeVolumeCount is 1 (less than 2)") @@ -197,7 +197,7 @@ func TestSelectVolumeIdsFromTopology(t *testing.T) { // So expected volumes: 10, 11, 12. - vids, _ = selectVolumeIdsFromTopology(topologyInfo, volumeSizeLimitMb, collectionRegex, nil, quietSeconds, nowUnixSeconds, fullPercentage, verbose) + vids, _ = SelectVolumeIdsFromTopology(topologyInfo, volumeSizeLimitMb, collectionRegex, nil, quietSeconds, nowUnixSeconds, fullPercentage, verbose) expectedVids := []needle.VolumeId{10, 11, 12} assert.Equal(t, len(expectedVids), len(vids), "Should select 3 volumes") @@ -232,7 +232,7 @@ func TestEcEncodeNodeCountCheck(t *testing.T) { } nodeCount := 0 - eachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) { + EachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) { nodeCount++ }) @@ -257,9 +257,9 @@ func TestChunkVolumeIds(t *testing.T) { {101, 102}, {103, 104}, {105}, - }, chunkVolumeIds(vids, 2)) + }, ChunkVolumeIds(vids, 2)) - assert.Equal(t, [][]needle.VolumeId{vids}, chunkVolumeIds(vids, 0)) + assert.Equal(t, [][]needle.VolumeId{vids}, ChunkVolumeIds(vids, 0)) } func ecShardVisibilityTestTopology(nodes ...*master_pb.DataNodeInfo) *master_pb.TopologyInfo { @@ -304,11 +304,11 @@ func TestCollectEcShardBitsByNode(t *testing.T) { } topo := ecShardVisibilityTestTopology(node1, node2) - byNode := collectEcShardBitsByNode(topo, needle.VolumeId(1)) + byNode := CollectEcShardBitsByNode(topo, needle.VolumeId(1)) assert.Len(t, byNode, 1) assert.Equal(t, erasure_coding.ShardBits(allBits), byNode[pb.NewServerAddressFromDataNode(node1)]) - assert.Empty(t, collectEcShardBitsByNode(topo, needle.VolumeId(3))) + assert.Empty(t, CollectEcShardBitsByNode(topo, needle.VolumeId(3))) } func TestCollectEcShardBitsByNode_MixedGenerations(t *testing.T) { @@ -332,18 +332,18 @@ func TestCollectEcShardBitsByNode_MixedGenerations(t *testing.T) { // the registration union nor as a second holder. fresh := nodeWithGeneration("node1:8080", 200) orphan := nodeWithGeneration("node2:8080", 100) - byNode := collectEcShardBitsByNode(ecShardVisibilityTestTopology(fresh, orphan), needle.VolumeId(1)) + byNode := CollectEcShardBitsByNode(ecShardVisibilityTestTopology(fresh, orphan), needle.VolumeId(1)) assert.Len(t, byNode, 1) assert.Equal(t, erasure_coding.ShardBits(allBits), byNode[pb.NewServerAddressFromDataNode(fresh)]) // Un-stamped entries are the legacy generation zero: dropped when a stamped // generation exists, all counted when nothing is stamped. legacy := nodeWithGeneration("node2:8080", 0) - byNode = collectEcShardBitsByNode(ecShardVisibilityTestTopology(fresh, legacy), needle.VolumeId(1)) + byNode = CollectEcShardBitsByNode(ecShardVisibilityTestTopology(fresh, legacy), needle.VolumeId(1)) assert.Len(t, byNode, 1) assert.Equal(t, erasure_coding.ShardBits(allBits), byNode[pb.NewServerAddressFromDataNode(fresh)]) - byNode = collectEcShardBitsByNode( + byNode = CollectEcShardBitsByNode( ecShardVisibilityTestTopology(nodeWithGeneration("node1:8080", 0), nodeWithGeneration("node2:8080", 0)), needle.VolumeId(1)) assert.Len(t, byNode, 2) @@ -438,7 +438,7 @@ func TestEcShardCountIgnoresDiskTypeOfTheShards(t *testing.T) { topo := ecShardVisibilityTestTopology(node) var union erasure_coding.ShardBits - for _, bits := range collectEcShardBitsByNode(topo, needle.VolumeId(1)) { + for _, bits := range CollectEcShardBitsByNode(topo, needle.VolumeId(1)) { union |= bits } assert.Equal(t, erasure_coding.TotalShardsCount, union.Count(), diff --git a/weed/shell/command_ec_common_test.go b/weed/ec/ec_flags_test.go similarity index 81% rename from weed/shell/command_ec_common_test.go rename to weed/ec/ec_flags_test.go index c7a81ec7f..28943ea3d 100644 --- a/weed/shell/command_ec_common_test.go +++ b/weed/ec/ec_flags_test.go @@ -1,4 +1,4 @@ -package shell +package ec import ( "testing" @@ -25,27 +25,27 @@ func TestCountFreeShardSlotsPhysicalDiskGate(t *testing.T) { } // Physically empty: slot math applies, positive free slots. - if got := countFreeShardSlots(mk(1000*gb, 900*gb), types.HardDriveType); got <= 0 { + if got := CountFreeShardSlots(mk(1000*gb, 900*gb), types.HardDriveType); got <= 0 { t.Errorf("physically empty disk free shard slots = %d, want > 0", got) } // Physically 96% full: gated to zero regardless of slot room. - if got := countFreeShardSlots(mk(1000*gb, 40*gb), types.HardDriveType); got != 0 { + if got := CountFreeShardSlots(mk(1000*gb, 40*gb), types.HardDriveType); got != 0 { t.Errorf("physically full disk free shard slots = %d, want 0", got) } // No byte report (older server): fall back to slot math, positive. - if got := countFreeShardSlots(mk(0, 0), types.HardDriveType); got <= 0 { + if got := CountFreeShardSlots(mk(0, 0), types.HardDriveType); got <= 0 { t.Errorf("unreported-bytes disk free shard slots = %d, want > 0 (slot fallback)", got) } } func TestParseVolumeIdsFlag(t *testing.T) { - vids, err := parseVolumeIdsFlag("101, 102,101, 103") + vids, err := ParseVolumeIdsFlag("101, 102,101, 103") assert.NoError(t, err) assert.Equal(t, []needle.VolumeId{101, 102, 103}, vids) - _, err = parseVolumeIdsFlag("101,abc") + _, err = ParseVolumeIdsFlag("101,abc") assert.Error(t, err) - _, err = parseVolumeIdsFlag(" , ") + _, err = ParseVolumeIdsFlag(" , ") assert.Error(t, err) } diff --git a/weed/shell/ec_rebalance_slots_test.go b/weed/ec/ec_rebalance_slots_test.go similarity index 94% rename from weed/shell/ec_rebalance_slots_test.go rename to weed/ec/ec_rebalance_slots_test.go index f47ddbb13..a6f15364f 100644 --- a/weed/shell/ec_rebalance_slots_test.go +++ b/weed/ec/ec_rebalance_slots_test.go @@ -1,4 +1,4 @@ -package shell +package ec import ( "testing" @@ -33,7 +33,7 @@ func TestECRebalanceWithLimitedSlots(t *testing.T) { topology := buildLimitedSlotsTopology() // Collect EC nodes from the topology - ecNodes, totalFreeEcSlots := collectEcVolumeServersByDc(topology, "", types.HardDriveType) + ecNodes, totalFreeEcSlots := CollectEcVolumeServersByDc(topology, "", types.HardDriveType) t.Logf("Topology summary:") t.Logf(" Number of EC nodes: %d", len(ecNodes)) @@ -42,19 +42,19 @@ func TestECRebalanceWithLimitedSlots(t *testing.T) { // Log per-node details for _, node := range ecNodes { shardCount := 0 - for _, diskInfo := range node.info.DiskInfos { + for _, diskInfo := range node.Info.DiskInfos { for _, ecShard := range diskInfo.EcShardInfos { shardCount += erasure_coding.GetShardCount(ecShard) } } t.Logf(" Node %s (rack %s): %d shards, %d free slots", - node.info.Id, node.rack, shardCount, node.freeEcSlot) + node.Info.Id, node.Rack, shardCount, node.FreeEcSlot) } // Calculate total EC shards totalEcShards := 0 for _, node := range ecNodes { - for _, diskInfo := range node.info.DiskInfos { + for _, diskInfo := range node.Info.DiskInfos { for _, ecShard := range diskInfo.EcShardInfos { totalEcShards += erasure_coding.GetShardCount(ecShard) } @@ -115,20 +115,20 @@ func TestECRebalanceZeroFreeSlots(t *testing.T) { // (VolumeCount still reflects the original volumes) topology := buildZeroFreeSlotTopology() - ecNodes, totalFreeEcSlots := collectEcVolumeServersByDc(topology, "", types.HardDriveType) + ecNodes, totalFreeEcSlots := CollectEcVolumeServersByDc(topology, "", types.HardDriveType) t.Logf("Zero free slots scenario:") for _, node := range ecNodes { shardCount := 0 - for _, diskInfo := range node.info.DiskInfos { + for _, diskInfo := range node.Info.DiskInfos { for _, ecShard := range diskInfo.EcShardInfos { shardCount += erasure_coding.GetShardCount(ecShard) } } t.Logf(" Node %s: %d shards, %d free slots, volumeCount=%d, max=%d", - node.info.Id, shardCount, node.freeEcSlot, - node.info.DiskInfos[string(types.HardDriveType)].VolumeCount, - node.info.DiskInfos[string(types.HardDriveType)].MaxVolumeCount) + node.Info.Id, shardCount, node.FreeEcSlot, + node.Info.DiskInfos[string(types.HardDriveType)].VolumeCount, + node.Info.DiskInfos[string(types.HardDriveType)].MaxVolumeCount) } t.Logf(" Total free slots: %d", totalFreeEcSlots) diff --git a/weed/ec/ec_rebuild.go b/weed/ec/ec_rebuild.go new file mode 100644 index 000000000..4afdabace --- /dev/null +++ b/weed/ec/ec_rebuild.go @@ -0,0 +1,435 @@ +package ec + +import ( + "context" + "fmt" + "io" + "slices" + "sync" + + "github.com/seaweedfs/seaweedfs/weed/operation" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +type ecRebuilder struct { + env *Env + ecNodes []*EcNode + writer io.Writer + applyChanges bool + collections []string + volumeIds []needle.VolumeId + diskType types.DiskType + + ewg *util.ErrorWaitGroup + ecNodesMu sync.Mutex +} + +// RebuildEcVolumes finds and rebuilds missing EC shards for the given +// collections. Before rebuilding it asks volume servers to recover any shards +// left unmounted by a missing .ecx index; such shards are invisible to the +// master, so recovering them first avoids regenerating data that is actually +// present. volumeIds, when non-empty, restricts the rebuild to those volumes. +func RebuildEcVolumes(env *Env, ecNodes []*EcNode, writer io.Writer, collections []string, volumeIds []needle.VolumeId, diskType types.DiskType, maxParallelization int, applyChanges bool) error { + erb := &ecRebuilder{ + env: env, + ecNodes: ecNodes, + writer: writer, + applyChanges: applyChanges, + collections: collections, + volumeIds: volumeIds, + diskType: diskType, + + ewg: util.NewErrorWaitGroup(maxParallelization), + } + + // Recover shards left unmounted by a missing .ecx index before planning: such + // shards never register with the master, so the rebuild below would treat the + // volume as short or unrepairable even though its data is intact (issue #10104). + erb.recoverMissingIndexes() + + fmt.Printf("rebuildEcVolumes for %d collection(s)\n", len(collections)) + for _, c := range collections { + erb.rebuildEcVolumes(c) + } + + return erb.ewg.Wait() +} + +func (erb *ecRebuilder) write(format string, a ...any) { + fmt.Fprintf(erb.writer, format, a...) +} + +func (erb *ecRebuilder) isLocked() bool { + return erb.env.isLocked() +} + +// matchesVolumeId verifies whether the rebuilder is targeted at a given volume ID. +func (erb *ecRebuilder) matchesVolumeId(vid needle.VolumeId) bool { + if len(erb.volumeIds) == 0 { + return true + } + + return slices.Contains(erb.volumeIds, vid) +} + +// countLocalShards returns the number of shards already present locally on the node for the given volume. +// Unions across all of the node's disks, like prepareDataToRecover, so slot +// accounting matches what the rebuild will actually treat as local. +func (erb *ecRebuilder) countLocalShards(node *EcNode, collection string, volumeId needle.VolumeId) int { + localShardsInfo := erasure_coding.NewShardsInfo() + for _, diskInfo := range node.Info.DiskInfos { + for _, ecShardInfo := range diskInfo.EcShardInfos { + if ecShardInfo.Collection == collection && needle.VolumeId(ecShardInfo.Id) == volumeId { + localShardsInfo.Add(erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(ecShardInfo)) + } + } + } + return localShardsInfo.Count() +} + +// selectAndReserveRebuilder atomically selects a rebuilder node with sufficient free slots +// and reserves slots only for the non-local shards that need to be copied/generated. +func (erb *ecRebuilder) selectAndReserveRebuilder(collection string, volumeId needle.VolumeId) (*EcNode, int, error) { + erb.ecNodesMu.Lock() + defer erb.ecNodesMu.Unlock() + + if len(erb.ecNodes) == 0 { + return nil, 0, fmt.Errorf("no ec nodes available") + } + + // Find the node with the most free slots, considering local shards + var bestNode *EcNode + var bestSlotsNeeded int + var maxAvailableSlots int + var minSlotsNeeded int = erasure_coding.TotalShardsCount // Start with maximum possible + for _, node := range erb.ecNodes { + localShards := erb.countLocalShards(node, collection, volumeId) + slotsNeeded := erasure_coding.TotalShardsCount - localShards + if slotsNeeded < 0 { + slotsNeeded = 0 + } + + if node.FreeEcSlot > maxAvailableSlots { + maxAvailableSlots = node.FreeEcSlot + } + + if slotsNeeded < minSlotsNeeded { + minSlotsNeeded = slotsNeeded + } + + if node.FreeEcSlot >= slotsNeeded { + if bestNode == nil || node.FreeEcSlot > bestNode.FreeEcSlot { + bestNode = node + bestSlotsNeeded = slotsNeeded + } + } + } + + if bestNode == nil { + return nil, 0, fmt.Errorf("no node has sufficient free slots for volume %d (need at least %d slots, max available: %d)", + volumeId, minSlotsNeeded, maxAvailableSlots) + } + + // Reserve slots only for non-local shards + bestNode.FreeEcSlot -= bestSlotsNeeded + + return bestNode, bestSlotsNeeded, nil +} + +// releaseRebuilder releases the reserved slots back to the rebuilder node. +func (erb *ecRebuilder) releaseRebuilder(node *EcNode, slotsToRelease int) { + erb.ecNodesMu.Lock() + defer erb.ecNodesMu.Unlock() + + // Release slots by incrementing the free slot count + node.FreeEcSlot += slotsToRelease +} + +func (erb *ecRebuilder) rebuildEcVolumes(collection string) { + fmt.Printf("rebuildEcVolumes for %q\n", collection) + + // collect vid => each shard locations, similar to ecShardMap in topology.go + ecShardMap := make(EcShardMap) + erb.ecNodesMu.Lock() + for _, ecNode := range erb.ecNodes { + ecShardMap.registerEcNode(ecNode, collection) + } + erb.ecNodesMu.Unlock() + + for vid, locations := range ecShardMap { + if !erb.matchesVolumeId(vid) { + continue + } + shardCount := locations.shardCount() + if shardCount == erasure_coding.TotalShardsCount { + continue + } + if shardCount < erasure_coding.DataShardsCount { + erb.write("ec volume %d is unrepairable with %d shards (need %d), skipping\n", vid, shardCount, erasure_coding.DataShardsCount) + continue + } + + // Capture variables for closure + vid := vid + locations := locations + + erb.ewg.Add(func() error { + // Select rebuilder and reserve slots atomically per volume + rebuilder, slotsToReserve, err := erb.selectAndReserveRebuilder(collection, vid) + if err != nil { + return fmt.Errorf("failed to select rebuilder for volume %d: %v", vid, err) + } + defer erb.releaseRebuilder(rebuilder, slotsToReserve) + + return erb.rebuildOneEcVolume(collection, vid, locations, rebuilder) + }) + } +} + +// recoverMissingIndexes asks every ec node to fetch a missing .ecx index from a +// peer and mount the on-disk shards it could not load on its own. Shards +// orphaned this way (index only on another server) are absent from the master +// topology, so without this pass ec.rebuild would regenerate or give up on +// shards whose data is actually present — and a volume whose every holder lacks +// the index would not appear in the topology at all. Each node therefore +// recovers all of its on-disk orphans (volume_id 0); an explicit -volumeIds +// list narrows that to the requested volumes. On apply it refreshes the topology +// so the rebuild planning sees the recovered shards (issue #10104). +func (erb *ecRebuilder) recoverMissingIndexes() { + erb.ecNodesMu.Lock() + nodes := append([]*EcNode(nil), erb.ecNodes...) + erb.ecNodesMu.Unlock() + if len(nodes) == 0 { + return + } + + // volume_id 0 means "recover every orphan on the node"; a -volumeIds list + // narrows recovery to those ids (each scanned across collections server-side). + vids := erb.volumeIds + if len(vids) == 0 { + vids = []needle.VolumeId{0} + } + + if !erb.applyChanges { + erb.write("would ask %d ec node(s) to recover EC shards left unmounted by a missing .ecx index\n", len(nodes)) + return + } + + for _, node := range nodes { + for _, vid := range vids { + err := operation.WithVolumeServerClient(false, pb.NewServerAddressFromDataNode(node.Info), erb.env.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + _, mountErr := client.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{ + VolumeId: uint32(vid), + RecoverMissingIndex: true, + }) + return mountErr + }) + if err != nil { + erb.write("%s recover missing index (volume %d): %v\n", node.Info.Id, vid, err) + } + } + } + + // Refresh topology so the rebuild planning sees shards the recovery registered. + refreshed, _, err := CollectEcNodes(erb.env, erb.diskType) + if err != nil { + erb.write("failed to refresh ec nodes after index recovery: %v\n", err) + return + } + erb.ecNodesMu.Lock() + erb.ecNodes = refreshed + erb.ecNodesMu.Unlock() +} + +func (erb *ecRebuilder) rebuildOneEcVolume(collection string, volumeId needle.VolumeId, locations EcShardLocations, rebuilder *EcNode) error { + if !erb.isLocked() { + return fmt.Errorf("lock is lost") + } + + fmt.Printf("rebuildOneEcVolume %s %d\n", collection, volumeId) + + // collect shard files to rebuilder local disk + var generatedShardIds []erasure_coding.ShardId + copiedShardIds, _, err := erb.prepareDataToRecover(rebuilder, collection, volumeId, locations) + defer func() { + // Clean up the working copies this run actually made, even when the + // recoverability gate failed after some copies already succeeded: + // they are temp files on the rebuilder nothing else reclaims. Dry-run + // copies nothing (copiedShardIds is empty), so this issues no delete + // RPC. Use a local error so a cleanup failure cannot mask the return. + if !erb.applyChanges || len(copiedShardIds) == 0 { + return + } + if derr := SourceServerDeleteEcShards(erb.env.GrpcDialOption, collection, volumeId, pb.NewServerAddressFromDataNode(rebuilder.Info), copiedShardIds); derr != nil { + erb.write("%s delete copied ec shards %s %d.%v: %v\n", rebuilder.Info.Id, collection, volumeId, copiedShardIds, derr) + } + }() + if err != nil { + return err + } + + if !erb.applyChanges { + return nil + } + + // generate ec shards, and maybe ecx file + generatedShardIds, err = erb.generateMissingShards(collection, volumeId, pb.NewServerAddressFromDataNode(rebuilder.Info)) + if err != nil { + return err + } + + // mount the generated shards + err = MountEcShards(erb.env.GrpcDialOption, collection, volumeId, pb.NewServerAddressFromDataNode(rebuilder.Info), generatedShardIds) + if err != nil { + return err + } + + // ensure ECNode updates are atomic + erb.ecNodesMu.Lock() + defer erb.ecNodesMu.Unlock() + rebuilder.AddEcVolumeShards(volumeId, collection, generatedShardIds, erb.diskType) + + return nil +} + +func (erb *ecRebuilder) generateMissingShards(collection string, volumeId needle.VolumeId, sourceLocation pb.ServerAddress) (rebuiltShardIds []erasure_coding.ShardId, err error) { + + err = operation.WithVolumeServerClient(false, sourceLocation, erb.env.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + resp, rebuildErr := volumeServerClient.VolumeEcShardsRebuild(context.Background(), &volume_server_pb.VolumeEcShardsRebuildRequest{ + VolumeId: uint32(volumeId), + Collection: collection, + }) + if rebuildErr == nil { + rebuiltShardIds = erasure_coding.Uint32ToShardIds(resp.RebuiltShardIds) + } + return rebuildErr + }) + return +} + +func (erb *ecRebuilder) prepareDataToRecover(rebuilder *EcNode, collection string, volumeId needle.VolumeId, locations EcShardLocations) (copiedShardIds []erasure_coding.ShardId, localShardIds []erasure_coding.ShardId, err error) { + + needEcxFile := true + localShardsInfo := erasure_coding.NewShardsInfo() + for _, diskInfo := range rebuilder.Info.DiskInfos { + for _, ecShardInfo := range diskInfo.EcShardInfos { + if ecShardInfo.Collection == collection && needle.VolumeId(ecShardInfo.Id) == volumeId { + needEcxFile = false + // Union across disks: the rebuilder may hold this volume's + // shards on more than one disk. Overwriting per-disk would + // make a shard on a non-last disk look remote and get copied + // onto itself (O_TRUNC) and then node-wide deleted. + localShardsInfo.Add(erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(ecShardInfo)) + } + } + } + + targetShardCount := erasure_coding.TotalShardsCount + for i := erasure_coding.TotalShardsCount; i < len(locations); i++ { + if len(locations[i]) > 0 { + targetShardCount = i + 1 + } + } + + // recoverableRemoteShards counts remote shards that can contribute to the + // rebuild. Dry-run counts the plan; apply mode counts only successful copies. + recoverableRemoteShards := 0 + for i := 0; i < targetShardCount; i++ { + ecNodes := locations[i] + shardId := erasure_coding.ShardId(i) + if len(ecNodes) == 0 { + erb.write("missing shard %d.%d\n", volumeId, shardId) + continue + } + + if localShardsInfo.Has(shardId) { + localShardIds = append(localShardIds, shardId) + erb.write("use existing shard %d.%d\n", volumeId, shardId) + continue + } + + // The rebuilder is itself the only listed holder: never copy a shard + // onto itself (the in-place O_TRUNC would destroy it) nor schedule it + // for the post-rebuild delete. Treat it as already local. + if ecNodes[0].Info.Id == rebuilder.Info.Id { + localShardIds = append(localShardIds, shardId) + erb.write("use existing shard %d.%d (already on rebuilder)\n", volumeId, shardId) + continue + } + + if !erb.applyChanges { + recoverableRemoteShards++ + erb.write("would copy %d.%d from %s\n", volumeId, shardId, ecNodes[0].Info.Id) + continue + } + + copyErr := operation.WithVolumeServerClient(false, pb.NewServerAddressFromDataNode(rebuilder.Info), erb.env.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + _, copyErr := volumeServerClient.VolumeEcShardsCopy(context.Background(), &volume_server_pb.VolumeEcShardsCopyRequest{ + VolumeId: uint32(volumeId), + Collection: collection, + ShardIds: []uint32{uint32(shardId)}, + CopyEcxFile: needEcxFile, + CopyEcjFile: true, + CopyVifFile: needEcxFile, + SourceDataNode: string(pb.NewServerAddressFromDataNode(ecNodes[0].Info)), + }) + return copyErr + }) + if copyErr != nil { + erb.write("%s failed to copy %d.%d from %s: %v\n", rebuilder.Info.Id, volumeId, shardId, ecNodes[0].Info.Id, copyErr) + continue + } + recoverableRemoteShards++ + if needEcxFile { + needEcxFile = false + } + erb.write("%s copied %d.%d from %s\n", rebuilder.Info.Id, volumeId, shardId, ecNodes[0].Info.Id) + // Only shards this run actually copied are temp working files to be + // deleted afterward; never a pre-existing local or remote shard. + copiedShardIds = append(copiedShardIds, shardId) + } + + if len(localShardIds)+recoverableRemoteShards >= erasure_coding.DataShardsCount { + return copiedShardIds, localShardIds, nil + } + + // Hand back what was copied so the caller deletes these orphaned working + // shards: recovery failed, but the temp files are already on the rebuilder. + return copiedShardIds, localShardIds, fmt.Errorf("%d shards are not enough to recover volume %d", len(localShardIds)+recoverableRemoteShards, volumeId) + +} + +type EcShardMap map[needle.VolumeId]EcShardLocations +type EcShardLocations [][]*EcNode + +func (ecShardMap EcShardMap) registerEcNode(ecNode *EcNode, collection string) { + for _, diskInfo := range ecNode.Info.DiskInfos { + for _, shardInfo := range diskInfo.EcShardInfos { + if shardInfo.Collection == collection { + existing, found := ecShardMap[needle.VolumeId(shardInfo.Id)] + if !found { + // Use MaxShardCount (32) to support custom EC ratios + existing = make([][]*EcNode, erasure_coding.MaxShardCount) + ecShardMap[needle.VolumeId(shardInfo.Id)] = existing + } + for _, shardId := range erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(shardInfo).Ids() { + existing[shardId] = append(existing[shardId], ecNode) + } + } + } + } +} + +func (ecShardLocations EcShardLocations) shardCount() (count int) { + for _, locations := range ecShardLocations { + if len(locations) > 0 { + count++ + } + } + return +} diff --git a/weed/shell/command_ec_rebuild_test.go b/weed/ec/ec_rebuild_test.go similarity index 90% rename from weed/shell/command_ec_rebuild_test.go rename to weed/ec/ec_rebuild_test.go index ecc0da31f..18db79ec1 100644 --- a/weed/shell/command_ec_rebuild_test.go +++ b/weed/ec/ec_rebuild_test.go @@ -1,4 +1,4 @@ -package shell +package ec import ( "bytes" @@ -8,6 +8,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/util" ) // TestEcShardMapRegister tests that EC shards are properly registered @@ -37,12 +38,12 @@ func TestEcShardMapRegister(t *testing.T) { // Verify shard distribution for i := 0; i < 7; i++ { - if len(locations[i]) != 1 || locations[i][0].info.Id != "node1" { + if len(locations[i]) != 1 || locations[i][0].Info.Id != "node1" { t.Errorf("Shard %d should be on node1", i) } } for i := 7; i < erasure_coding.TotalShardsCount; i++ { - if len(locations[i]) != 1 || locations[i][0].info.Id != "node2" { + if len(locations[i]) != 1 || locations[i][0].Info.Id != "node2" { t.Errorf("Shard %d should be on node2", i) } } @@ -89,11 +90,8 @@ func TestRebuildEcVolumesInsufficientShards(t *testing.T) { addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4}) // Only 5 shards erb := &ecRebuilder{ - commandEnv: &CommandEnv{ - env: make(map[string]string), - noLock: true, // Bypass lock check for unit test - }, - ewg: NewErrorWaitGroup(DefaultMaxParallelization), + env: &Env{}, + ewg: util.NewErrorWaitGroup(10), ecNodes: []*EcNode{node1}, writer: &logBuffer, } @@ -120,11 +118,8 @@ func TestRebuildEcVolumesCompleteVolume(t *testing.T) { addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}) erb := &ecRebuilder{ - commandEnv: &CommandEnv{ - env: make(map[string]string), - noLock: true, // Bypass lock check for unit test - }, - ewg: NewErrorWaitGroup(DefaultMaxParallelization), + env: &Env{}, + ewg: util.NewErrorWaitGroup(10), ecNodes: []*EcNode{node1}, writer: &logBuffer, applyChanges: false, @@ -152,11 +147,8 @@ func TestRebuildEcVolumesInsufficientSpace(t *testing.T) { addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) erb := &ecRebuilder{ - commandEnv: &CommandEnv{ - env: make(map[string]string), - noLock: true, // Bypass lock check for unit test - }, - ewg: NewErrorWaitGroup(DefaultMaxParallelization), + env: &Env{}, + ewg: util.NewErrorWaitGroup(10), ecNodes: []*EcNode{node1}, writer: &logBuffer, applyChanges: false, @@ -203,17 +195,17 @@ func TestMultipleNodesWithShards(t *testing.T) { // Verify each shard is on the correct node for i := 0; i < 4; i++ { - if len(locations[i]) != 1 || locations[i][0].info.Id != "node1" { + if len(locations[i]) != 1 || locations[i][0].Info.Id != "node1" { t.Errorf("Shard %d should be on node1", i) } } for i := 4; i < 8; i++ { - if len(locations[i]) != 1 || locations[i][0].info.Id != "node2" { + if len(locations[i]) != 1 || locations[i][0].Info.Id != "node2" { t.Errorf("Shard %d should be on node2", i) } } for i := 8; i < 10; i++ { - if len(locations[i]) != 1 || locations[i][0].info.Id != "node3" { + if len(locations[i]) != 1 || locations[i][0].Info.Id != "node3" { t.Errorf("Shard %d should be on node3", i) } } @@ -243,10 +235,10 @@ func TestDuplicateShards(t *testing.T) { foundNode1 := false foundNode2 := false for _, node := range locations[0] { - if node.info.Id == "node1" { + if node.Info.Id == "node1" { foundNode1 = true } - if node.info.Id == "node2" { + if node.Info.Id == "node2" { foundNode2 = true } } @@ -270,10 +262,7 @@ func TestPrepareDataToRecoverTargetShardCount(t *testing.T) { addEcVolumeAndShardsForTest(1, "c1", []erasure_coding.ShardId{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}) erb := &ecRebuilder{ - commandEnv: &CommandEnv{ - env: make(map[string]string), - noLock: true, - }, + env: &Env{}, ecNodes: []*EcNode{node1}, writer: &logBuffer, } diff --git a/weed/ec/ec_scrub.go b/weed/ec/ec_scrub.go new file mode 100644 index 000000000..e108db16b --- /dev/null +++ b/weed/ec/ec_scrub.go @@ -0,0 +1,81 @@ +package ec + +import ( + "context" + "fmt" + "io" + "strings" + "sync" + + "github.com/seaweedfs/seaweedfs/weed/operation" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/util" +) + +// ScrubEcVolumes asks each volume server to scrub its EC volumes (optionally +// narrowed to volumeIds) in the given mode, aggregating and reporting results. +func ScrubEcVolumes(env *Env, writer io.Writer, volumeServerAddrs []pb.ServerAddress, volumeIds []uint32, mode volume_server_pb.VolumeScrubMode, forceDeletedNeedlesCheck bool, maxParallelization int, showDetails bool) error { + var brokenVolumesStr, brokenShardsStr []string + var details []string + var totalVolumes, brokenVolumes, brokenShards, totalFiles uint64 + var mu sync.Mutex + + ewg := util.NewErrorWaitGroup(maxParallelization) + count := 0 + for _, addr := range volumeServerAddrs { + ewg.Add(func() error { + mu.Lock() + count++ + fmt.Fprintf(writer, "Scrubbing %s (%d/%d)...\n", addr.String(), count, len(volumeServerAddrs)) + mu.Unlock() + + err := operation.WithVolumeServerClient(false, addr, env.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { + res, err := volumeServerClient.ScrubEcVolume(context.Background(), &volume_server_pb.ScrubEcVolumeRequest{ + Mode: mode, + VolumeIds: volumeIds, + ForceDeletedNeedlesCheck: forceDeletedNeedlesCheck, + }) + if err != nil { + return err + } + + mu.Lock() + defer mu.Unlock() + + totalVolumes += res.GetTotalVolumes() + totalFiles += res.GetTotalFiles() + brokenVolumes += uint64(len(res.GetBrokenVolumeIds())) + brokenShards += uint64(len(res.GetBrokenShardInfos())) + for _, d := range res.GetDetails() { + details = append(details, fmt.Sprintf("[%s] %s", addr, d)) + } + for _, vid := range res.GetBrokenVolumeIds() { + brokenVolumesStr = append(brokenVolumesStr, fmt.Sprintf("%s:%v", addr, vid)) + } + for _, si := range res.GetBrokenShardInfos() { + brokenShardsStr = append(brokenShardsStr, fmt.Sprintf("%s:%v:%v", addr, si.VolumeId, si.ShardId)) + } + + return nil + }) + return err + }) + } + if err := ewg.Wait(); err != nil { + return err + } + + fmt.Fprintf(writer, "Scrubbed %d EC files and %d volumes on %d nodes\n", totalFiles, totalVolumes, len(volumeServerAddrs)) + if brokenVolumes != 0 { + fmt.Fprintf(writer, "\nGot scrub failures on %d EC volumes and %d EC shards :(\n", brokenVolumes, brokenShards) + fmt.Fprintf(writer, "Affected volumes: %s\n", strings.Join(brokenVolumesStr, ", ")) + if len(brokenShardsStr) != 0 { + fmt.Fprintf(writer, "Affected shards: %s\n", strings.Join(brokenShardsStr, ", ")) + } + if showDetails && len(details) != 0 { + fmt.Fprintf(writer, "Details:\n\t%s\n", strings.Join(details, "\n\t")) + } + } + return nil +} diff --git a/weed/ec/ec_shard_unmount.go b/weed/ec/ec_shard_unmount.go new file mode 100644 index 000000000..e4cfee94b --- /dev/null +++ b/weed/ec/ec_shard_unmount.go @@ -0,0 +1,210 @@ +package ec + +import ( + "context" + "fmt" + "io" + "sort" + "strconv" + "strings" + "time" + + "github.com/seaweedfs/seaweedfs/weed/operation" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" +) + +const ( + ecShardActionTimeout = 1 * time.Minute +) + +// ShardRef names one EC shard copy, optionally pinned to a node address. +type ShardRef struct { + ShardID uint32 + Collection string + NodeAddress string +} + +func (s *ShardRef) String() string { + if s.NodeAddress == "" { + return fmt.Sprintf("%d", s.ShardID) + } + return fmt.Sprintf("%d@%s", s.ShardID, s.NodeAddress) +} + +// ShardRefsFromString parses a comma-separated list of shard IDs, each either a +// bare numeric ID or @ to pick one copy. +func ShardRefsFromString(shards string) ([]*ShardRef, error) { + res := []*ShardRef{} + + for _, s := range strings.Split(shards, ",") { + s = strings.TrimSpace(s) + if s == "" { + return nil, fmt.Errorf("empty shard ID in %q", shards) + } + + // optional @ to pick one copy + idStr, addr, _ := strings.Cut(s, "@") + + id, err := strconv.Atoi(idStr) + if err != nil || id < 0 || id >= erasure_coding.MaxShardCount { + return nil, fmt.Errorf("invalid shard ID %q", s) + } + + res = append(res, &ShardRef{ShardID: uint32(id), NodeAddress: addr}) + } + + return res, nil +} + +// ShardUnmountRequest describes one ec.shard.unmount run. +type ShardUnmountRequest struct { + Topology *master_pb.TopologyInfo + VolumeID uint32 + Shards []*ShardRef + Delete bool + IgnoreInvalid bool + Apply bool +} + +type ecShardUnmounter struct { + env *Env + writer io.Writer + req ShardUnmountRequest +} + +// UnmountShards unmounts, and optionally deletes, the requested EC shard +// copies, resolving them against the live topology first. Dry-run unless +// req.Apply is set. +func UnmountShards(env *Env, writer io.Writer, req ShardUnmountRequest) error { + c := &ecShardUnmounter{env: env, writer: writer, req: req} + return c.doShardsUnmount() +} + +func (c *ecShardUnmounter) write(format string, a ...any) { + fmt.Fprintf(c.writer, format, a...) +} + +func (c *ecShardUnmounter) liveShardsForVolume() []*ShardRef { + shards := []*ShardRef{} + + for _, dci := range c.req.Topology.GetDataCenterInfos() { + for _, ri := range dci.GetRackInfos() { + for _, dni := range ri.GetDataNodeInfos() { + nodeAddress := dni.GetAddress() + for _, di := range dni.GetDiskInfos() { + for _, eci := range di.GetEcShardInfos() { + if eci.GetId() == c.req.VolumeID { + sinfo := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(eci) + for _, sid := range sinfo.Ids() { + shards = append(shards, &ShardRef{ + ShardID: uint32(sid), + Collection: eci.GetCollection(), + NodeAddress: nodeAddress, + }) + } + } + } + } + } + } + } + + sort.SliceStable(shards, func(i, j int) bool { return shards[i].ShardID < shards[j].ShardID }) + return shards +} + +func (c *ecShardUnmounter) printShards(ss []*ShardRef) { + for _, s := range ss { + c.write("\t%v\n", s) + } + c.write("\n") +} + +func (c *ecShardUnmounter) doShardsUnmount() error { + liveShards := c.liveShardsForVolume() + c.write("Live shard topology for volume ID %d (%d shards):\n", c.req.VolumeID, len(liveShards)) + c.printShards(liveShards) + + // resolve target shards against the live topology + targetShards := []*ShardRef{} + for _, ps := range c.req.Shards { + var result *ShardRef + for _, ts := range liveShards { + if ts.ShardID == ps.ShardID { + if ps.NodeAddress == "" || ps.NodeAddress == ts.NodeAddress { + if result != nil { + return fmt.Errorf("shard %v is ambiguous", ps) + } + result = ts + } + } + } + if result == nil { + if !c.req.IgnoreInvalid { + return fmt.Errorf("shard %v is invalid", ps) + } + c.write("!!! ignoring invalid shard %v\n", ps) + } else { + targetShards = append(targetShards, result) + } + } + if len(targetShards) == 0 { + return fmt.Errorf("got no shards to process") + } + + mode := "unmount" + if c.req.Delete { + mode = "unmount + delete" + } + c.write("Will %s %d shard(s):\n", mode, len(targetShards)) + c.printShards(targetShards) + + if !c.req.Apply { + c.write("Not proceeding in dry-run mode\n") + return nil + } + + if !c.env.isLocked() { + return fmt.Errorf("lock is lost") + } + + for _, s := range targetShards { + if err := c.unmountShard(s); err != nil { + return err + } + } + + c.write("\nAll done!\n") + return nil +} + +func (c *ecShardUnmounter) unmountShard(s *ShardRef) error { + ctx, cancel := context.WithTimeout(context.Background(), ecShardActionTimeout) + defer cancel() + + return operation.WithVolumeServerClient(false, pb.ServerAddress(s.NodeAddress), c.env.GrpcDialOption, func(vsc volume_server_pb.VolumeServerClient) error { + c.write("Unmounting shard %v for volume ID %d...\n", s, c.req.VolumeID) + if _, err := vsc.VolumeEcShardsUnmount(ctx, &volume_server_pb.VolumeEcShardsUnmountRequest{ + VolumeId: c.req.VolumeID, + ShardIds: []uint32{s.ShardID}, + }); err != nil { + return err + } + + if c.req.Delete { + c.write("Deleting shard %v for volume ID %d...\n", s, c.req.VolumeID) + if _, err := vsc.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{ + VolumeId: c.req.VolumeID, + Collection: s.Collection, + ShardIds: []uint32{s.ShardID}, + }); err != nil { + return err + } + } + + return nil + }) +} diff --git a/weed/shell/command_ec_shard_unmount_test.go b/weed/ec/ec_shard_unmount_test.go similarity index 79% rename from weed/shell/command_ec_shard_unmount_test.go rename to weed/ec/ec_shard_unmount_test.go index 3d522cea0..82b11fe58 100644 --- a/weed/shell/command_ec_shard_unmount_test.go +++ b/weed/ec/ec_shard_unmount_test.go @@ -1,4 +1,4 @@ -package shell +package ec import ( "testing" @@ -11,28 +11,28 @@ func TestEcShardsFromString(t *testing.T) { tests := []struct { name string in string - want []*ecShard + want []*ShardRef wantErr bool }{ { name: "single id", in: "2", - want: []*ecShard{{ShardID: 2}}, + want: []*ShardRef{{ShardID: 2}}, }, { name: "list of ids", in: "0, 3 ,5", - want: []*ecShard{{ShardID: 0}, {ShardID: 3}, {ShardID: 5}}, + want: []*ShardRef{{ShardID: 0}, {ShardID: 3}, {ShardID: 5}}, }, { name: "qualified keeps host:port", in: "3@10.200.18.88:9007", - want: []*ecShard{{ShardID: 3, NodeAddress: "10.200.18.88:9007"}}, + want: []*ShardRef{{ShardID: 3, NodeAddress: "10.200.18.88:9007"}}, }, { name: "mixed bare and qualified", in: "2,3@10.200.18.88:9007", - want: []*ecShard{{ShardID: 2}, {ShardID: 3, NodeAddress: "10.200.18.88:9007"}}, + want: []*ShardRef{{ShardID: 2}, {ShardID: 3, NodeAddress: "10.200.18.88:9007"}}, }, { name: "colon without node marker is not a shard ID", @@ -43,7 +43,7 @@ func TestEcShardsFromString(t *testing.T) { // shard IDs beyond the default 10+4 total are valid on custom-ratio volumes. name: "shard id within MaxShardCount", in: "20", - want: []*ecShard{{ShardID: 20}}, + want: []*ShardRef{{ShardID: 20}}, }, { name: "shard id at MaxShardCount is rejected", @@ -78,7 +78,7 @@ func TestEcShardsFromString(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := ecShardsFromString(tt.in) + got, err := ShardRefsFromString(tt.in) if tt.wantErr { assert.Error(t, err) return @@ -91,12 +91,12 @@ func TestEcShardsFromString(t *testing.T) { // the printed topology form must parse back, so entries can be copied verbatim. func TestEcShardStringRoundTrips(t *testing.T) { - for _, s := range []*ecShard{ + for _, s := range []*ShardRef{ {ShardID: 2}, {ShardID: 3, NodeAddress: "10.200.18.88:9007"}, } { - got, err := ecShardsFromString(s.String()) + got, err := ShardRefsFromString(s.String()) assert.NoError(t, err) - assert.Equal(t, []*ecShard{s}, got) + assert.Equal(t, []*ShardRef{s}, got) } } diff --git a/weed/ec/env.go b/weed/ec/env.go new file mode 100644 index 000000000..748e5de9d --- /dev/null +++ b/weed/ec/env.go @@ -0,0 +1,39 @@ +// Package ec holds the EC (erasure coding) cluster orchestration logic shared +// by the weed shell commands and the maintenance workers: topology analysis of +// EC shards, the encode/balance pipelines, and the volume-server RPC wrappers +// they drive. The placement policy itself lives in +// weed/storage/erasure_coding/ecbalancer; low-level shard mechanics live in +// weed/storage/erasure_coding. +package ec + +import ( + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/wdclient" + "google.golang.org/grpc" +) + +// Env carries the cluster access hooks EC operations need, decoupled from any +// particular caller (shell CommandEnv, worker task, admin server). +type Env struct { + GrpcDialOption grpc.DialOption + // FetchTopology returns a fresh master topology snapshot (and the master's + // volume size limit in MB) after an optional delay. + FetchTopology func(delay time.Duration) (*master_pb.TopologyInfo, uint64, error) + // GetVolumeLocations returns the current replica locations for a volume id, + // or false if the volume is unknown. + GetVolumeLocations func(vid uint32) ([]wdclient.Location, bool) + // IsLocked reports whether the caller still holds the cluster admin lock. + // Callers without a lock concept return true. + IsLocked func() bool +} + +// isLocked treats a nil Env or nil hook as locked, matching the shell's +// nil-receiver behavior so dry-run paths work without a cluster connection. +func (env *Env) isLocked() bool { + if env == nil || env.IsLocked == nil { + return true + } + return env.IsLocked() +} diff --git a/weed/shell/command_ec_check_replication.go b/weed/shell/command_ec_check_replication.go index 953650dea..a5a8b8ff8 100644 --- a/weed/shell/command_ec_check_replication.go +++ b/weed/shell/command_ec_check_replication.go @@ -4,12 +4,10 @@ import ( "flag" "fmt" "io" - "slices" "strconv" "strings" - "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" - "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" + "github.com/seaweedfs/seaweedfs/weed/ec" ) func init() { @@ -63,13 +61,7 @@ func (c *commandEcCheckReplication) Do(args []string, commandEnv *CommandEnv, wr return } - // Keep execution state in a per-run runner rather than on the long-lived - // command singleton, so concurrent or repeated invocations don't share state. - runner := &ecCheckReplicationRunner{ - writer: writer, - volumeIDMap: map[uint32]bool{}, - } - + volumeIDMap := map[uint32]bool{} if *volumeIDsStr != "" { for _, vids := range strings.Split(*volumeIDsStr, ",") { vids = strings.TrimSpace(vids) @@ -77,207 +69,19 @@ func (c *commandEcCheckReplication) Do(args []string, commandEnv *CommandEnv, wr continue } if vid, err := strconv.ParseUint(vids, 10, 32); err == nil { - runner.volumeIDMap[uint32(vid)] = true + volumeIDMap[uint32(vid)] = true } else { return fmt.Errorf("invalid volume ID %q", vids) } } } - runner.dataNodes, err = collectDataNodes(commandEnv, 0) + dataNodes, err := collectDataNodes(commandEnv, 0) if err != nil { return err } - return runner.checkEcVolumes(*showDetails) + return ec.CheckEcVolumeReplication(writer, dataNodes, volumeIDMap, *showDetails) } // ecCheckReplicationRunner holds the state for a single ec.check.replication invocation. -type ecCheckReplicationRunner struct { - writer io.Writer - dataNodes []*master_pb.DataNodeInfo - volumeIDMap map[uint32]bool -} - -func (r *ecCheckReplicationRunner) write(format string, a ...any) { - fmt.Fprintf(r.writer, format, a...) -} - -func (r *ecCheckReplicationRunner) isVolumeIDValid(vid uint32) bool { - if len(r.volumeIDMap) == 0 { - return true - } - return r.volumeIDMap[vid] -} - -// ecVolumeShardReplication aggregates the observed shards for a single EC volume, -// together with the data+parity ratio the volume was encoded with. The ratio is -// taken per volume (via erasure_coding.EcShardsVolume*Shards) so custom EC -// ratios are checked against their own expected shard count. -type ecVolumeShardReplication struct { - dataShards int - parityShards int - // shardAddresses maps a shard id to the sorted node addresses hosting it. - shardAddresses map[erasure_coding.ShardId][]string -} - -func (h *ecVolumeShardReplication) totalShards() int { - return h.dataShards + h.parityShards -} - -// replicaCount is the total number of shard copies observed, counting each -// over-replicated shard once per hosting node. -func (h *ecVolumeShardReplication) replicaCount() int { - n := 0 - for _, addrs := range h.shardAddresses { - n += len(addrs) - } - return n -} - -// unexpectedShardIds returns, in ascending order, any observed shard ids that -// fall outside the volume's expected data+parity range. These are anomalies -// (e.g. a stray shard) and are treated as over-replication. -func (h *ecVolumeShardReplication) unexpectedShardIds() []erasure_coding.ShardId { - var ids []erasure_coding.ShardId - for sid := range h.shardAddresses { - if int(sid) >= h.totalShards() { - ids = append(ids, sid) - } - } - slices.Sort(ids) - return ids -} - -// TODO: check shard sizes? -func (r *ecCheckReplicationRunner) checkEcVolumes(showDetails bool) error { - // collect EC shard placement, keyed by volume id - volumes := map[uint32]*ecVolumeShardReplication{} - for _, dni := range r.dataNodes { - nodeAddress := dni.GetAddress() - for _, di := range dni.GetDiskInfos() { - for _, eci := range di.GetEcShardInfos() { - vid := eci.GetId() - if !r.isVolumeIDValid(vid) { - continue - } - - h, ok := volumes[vid] - if !ok { - // all shards of a volume share one ratio; take it from the - // first shard message seen for the volume. - h = &ecVolumeShardReplication{ - dataShards: erasure_coding.EcShardsVolumeDataShards(eci), - parityShards: erasure_coding.EcShardsVolumeParityShards(eci), - shardAddresses: map[erasure_coding.ShardId][]string{}, - } - volumes[vid] = h - } - - sinfo := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(eci) - for _, sid := range sinfo.Ids() { - h.shardAddresses[sid] = append(h.shardAddresses[sid], nodeAddress) - } - } - } - } - - if len(volumes) == 0 { - // Asking about specific volume IDs that turn out not to be EC volumes is - // an error; an unfiltered run over a cluster with no EC volumes is a - // legitimate, healthy state rather than a failure. - if len(r.volumeIDMap) > 0 { - return fmt.Errorf("no EC volumes found") - } - r.write("No EC volumes found.\n") - return nil - } - - // keep the shard address lists nicely sorted, for the sake of readability. - for _, h := range volumes { - for _, addrs := range h.shardAddresses { - slices.Sort(addrs) - } - } - - // classify each volume against its own expected shard count - underreplicatedVolumeIDs := []uint32{} - overreplicatedVolumeIDs := []uint32{} - for vid, h := range volumes { - under := false - over := false - for sid := 0; sid < h.totalShards(); sid++ { - switch len(h.shardAddresses[erasure_coding.ShardId(sid)]) { - case 0: - under = true - case 1: - default: - over = true - } - } - // shard ids beyond the expected data+parity range are unexpected extras, - // i.e. redundant data the ratio doesn't call for. - if len(h.unexpectedShardIds()) > 0 { - over = true - } - - // under- and over-replication are independent problems (missing shards risk - // data loss, redundant shards waste space), so a volume exhibiting both is - // reported in both lists. - if under { - underreplicatedVolumeIDs = append(underreplicatedVolumeIDs, vid) - } - if over { - overreplicatedVolumeIDs = append(overreplicatedVolumeIDs, vid) - } - } - slices.Sort(underreplicatedVolumeIDs) - slices.Sort(overreplicatedVolumeIDs) - - // ...and display results - if len(underreplicatedVolumeIDs) == 0 && len(overreplicatedVolumeIDs) == 0 { - r.write("EC volumes are healthy.\n") - return nil - } - - if len(underreplicatedVolumeIDs) != 0 { - r.write("Found %d/%d under-replicated EC volumes: %v\n", len(underreplicatedVolumeIDs), len(volumes), underreplicatedVolumeIDs) - } - if len(overreplicatedVolumeIDs) != 0 { - r.write("Found %d/%d over-replicated EC volumes: %v\n", len(overreplicatedVolumeIDs), len(volumes), overreplicatedVolumeIDs) - } - - if showDetails { - if len(underreplicatedVolumeIDs) != 0 { - r.write("\n") - r.writeShardMaps("under-replicated", underreplicatedVolumeIDs, volumes) - } - if len(overreplicatedVolumeIDs) != 0 { - r.write("\n") - r.writeShardMaps("over-replicated", overreplicatedVolumeIDs, volumes) - } - } - - return nil -} - -func (r *ecCheckReplicationRunner) writeShardMaps(kind string, volumeIDs []uint32, volumes map[uint32]*ecVolumeShardReplication) { - for _, vid := range volumeIDs { - h := volumes[vid] - r.write("Shards map for %s EC volume %v (%d/%d shards):\n", kind, vid, h.replicaCount(), h.totalShards()) - for sid := 0; sid < h.totalShards(); sid++ { - shardTypeDesc := "" - if sid >= h.dataShards { - shardTypeDesc = " (parity)" - } - if addrs, ok := h.shardAddresses[erasure_coding.ShardId(sid)]; ok { - r.write("\t%02d%s => %v\n", sid, shardTypeDesc, addrs) - } else { - r.write("\t%02d%s is missing\n", sid, shardTypeDesc) - } - } - for _, sid := range h.unexpectedShardIds() { - r.write("\t%02d (unexpected) => %v\n", int(sid), h.shardAddresses[sid]) - } - } -} diff --git a/weed/shell/command_ec_common.go b/weed/shell/command_ec_common.go index 6b824127c..5165baa2b 100644 --- a/weed/shell/command_ec_common.go +++ b/weed/shell/command_ec_common.go @@ -2,134 +2,54 @@ package shell import ( "context" - "fmt" - "os" "regexp" - "slices" - "sort" - "strconv" - "strings" "time" + "github.com/seaweedfs/seaweedfs/weed/ec" "github.com/seaweedfs/seaweedfs/weed/glog" - "github.com/seaweedfs/seaweedfs/weed/operation" - "github.com/seaweedfs/seaweedfs/weed/operation/volume_move" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" - "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" - "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/ecbalancer" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/super_block" "github.com/seaweedfs/seaweedfs/weed/storage/types" - "github.com/seaweedfs/seaweedfs/weed/topology/balancer" - "google.golang.org/grpc" + "github.com/seaweedfs/seaweedfs/weed/wdclient" ) -type DataCenterId string -type EcNodeId string -type RackId string - -// EcDisk represents a single disk on a volume server -type EcDisk struct { - diskId uint32 - diskType string - freeEcSlots int - ecShardCount int // Total EC shards on this disk - // Map of volumeId -> ShardsInfo for shards on this disk - ecShards map[needle.VolumeId]*erasure_coding.ShardsInfo -} - -type EcNode struct { - info *master_pb.DataNodeInfo - dc DataCenterId - rack RackId - freeEcSlot int - // disks maps diskId -> EcDisk for disk-level balancing - disks map[uint32]*EcDisk -} - -type CandidateEcNode struct { - ecNode *EcNode - shardCount int -} - -type EcRack struct { - ecNodes map[EcNodeId]*EcNode - freeEcSlot int -} +// The EC orchestration logic lives in weed/ec so the shell commands and the +// maintenance workers share it. The aliases below keep the shell-internal +// names working for the command files that still use them. +type DataCenterId = ec.DataCenterId +type RackId = ec.RackId +type EcNode = ec.EcNode var ( - ecBalanceAlgorithmDescription = ` - func EcBalance() { - for each collection: - balanceEcVolumes(collectionName) - for each rack: - balanceEcRack(rack) - } - - func balanceEcVolumes(collectionName){ - for each volume: - doDeduplicateEcShards(volumeId) - - tracks rack~shardCount mapping - for each volume: - doBalanceEcShardsAcrossRacks(volumeId) - - for each volume: - doBalanceEcShardsWithinRacks(volumeId) - } - - // spread ec shards into more racks - func doBalanceEcShardsAcrossRacks(volumeId){ - tracks rack~volumeIdShardCount mapping - averageShardsPerEcRack = totalShardNumber / numRacks // totalShardNumber is 14 for now, later could varies for each dc - ecShardsToMove = select overflown ec shards from racks with ec shard counts > averageShardsPerEcRack - for each ecShardsToMove { - destRack = pickOneRack(rack~shardCount, rack~volumeIdShardCount, ecShardReplicaPlacement) - destVolumeServers = volume servers on the destRack - pickOneEcNodeAndMoveOneShard(destVolumeServers) - } - } - - func doBalanceEcShardsWithinRacks(volumeId){ - racks = collect all racks that the volume id is on - for rack, shards := range racks - doBalanceEcShardsWithinOneRack(volumeId, shards, rack) - } - - // move ec shards - func doBalanceEcShardsWithinOneRack(volumeId, shards, rackId){ - tracks volumeServer~volumeIdShardCount mapping - averageShardCount = len(shards) / numVolumeServers - volumeServersOverAverage = volume servers with volumeId's ec shard counts > averageShardsPerEcRack - ecShardsToMove = select overflown ec shards from volumeServersOverAverage - for each ecShardsToMove { - destVolumeServer = pickOneVolumeServer(volumeServer~shardCount, volumeServer~volumeIdShardCount, ecShardReplicaPlacement) - pickOneEcNodeAndMoveOneShard(destVolumeServers) - } - } - - // move ec shards while keeping shard distribution for the same volume unchanged or more even - func balanceEcRack(rack){ - averageShardCount = total shards / numVolumeServers - for hasMovedOneEcShard { - sort all volume servers ordered by the number of local ec shards - pick the volume server A with the lowest number of ec shards x - pick the volume server B with the highest number of ec shards y - if y > averageShardCount and x +1 <= averageShardCount { - if B has a ec shard with volume id v that A does not have { - move one ec shard v from B to A - hasMovedOneEcShard = true - } - } - } - } - ` - // Overridable functions for testing. - getDefaultReplicaPlacement = _getDefaultReplicaPlacement + ecBalanceAlgorithmDescription = ec.BalanceAlgorithmDescription + eachDataNode = ec.EachDataNode + collectEcVolumeServersByDc = ec.CollectEcVolumeServersByDc + pickBestDiskOnNode = ec.PickBestDiskOnNode + collectVolumeIdToCollection = ec.CollectVolumeIdToCollection + parseVolumeIdsFlag = ec.ParseVolumeIdsFlag + chunkVolumeIds = ec.ChunkVolumeIds ) +// ecEnv adapts a CommandEnv to the cluster access hooks the ec package needs. +func (ce *CommandEnv) ecEnv() *ec.Env { + return &ec.Env{ + GrpcDialOption: ce.option.GrpcDialOption, + FetchTopology: func(delay time.Duration) (*master_pb.TopologyInfo, uint64, error) { + return collectTopologyInfo(ce, delay) + }, + GetVolumeLocations: func(vid uint32) ([]wdclient.Location, bool) { + return ce.MasterClient.GetLocationsClone(vid) + }, + IsLocked: ce.isLocked, + } +} + +// Overridable functions for testing. +var getDefaultReplicaPlacement = _getDefaultReplicaPlacement + func _getDefaultReplicaPlacement(commandEnv *CommandEnv) (*super_block.ReplicaPlacement, error) { var resp *master_pb.GetMasterConfigurationResponse var err error @@ -204,942 +124,21 @@ func collectDataNodes(commandEnv *CommandEnv, delayBeforeCollecting time.Duratio } func collectEcNodesForDC(commandEnv *CommandEnv, selectedDataCenter string, diskType types.DiskType) (ecNodes []*EcNode, totalFreeEcSlots int, err error) { - // list all possible locations - // collect topology information - topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) - if err != nil { - return - } - - // find out all volume servers with one slot left. - ecNodes, totalFreeEcSlots = collectEcVolumeServersByDc(topologyInfo, selectedDataCenter, diskType) - - sortEcNodesByFreeslotsDescending(ecNodes) - - return + return ec.CollectEcNodesForDC(commandEnv.ecEnv(), selectedDataCenter, diskType) } func collectEcNodes(commandEnv *CommandEnv, diskType types.DiskType) (ecNodes []*EcNode, totalFreeEcSlots int, err error) { - return collectEcNodesForDC(commandEnv, "", diskType) -} - -// assertEncodableRegularVolumes rejects volume ids that are not encodable -// regular volumes in the topology snapshot: an already-EC volume (present only -// as EC shards, with no .dat) or an id absent from the cluster. Encoding an -// already-EC volume would clear its shards before failing, destroying the only -// copy. A volume present as BOTH a regular .dat and stale orphan shards (a -// failed-encode retry) passes, so the retry + orphan sweep still works. -func assertEncodableRegularVolumes(t *master_pb.TopologyInfo, vids []needle.VolumeId) error { - want := make(map[needle.VolumeId]bool, len(vids)) - for _, vid := range vids { - want[vid] = true - } - regular := make(map[needle.VolumeId]bool) - hasEcShards := make(map[needle.VolumeId]bool) - for _, dc := range t.DataCenterInfos { - for _, r := range dc.RackInfos { - for _, dn := range r.DataNodeInfos { - for _, diskInfo := range dn.DiskInfos { - if diskInfo == nil { - continue - } - for _, vi := range diskInfo.VolumeInfos { - if want[needle.VolumeId(vi.Id)] { - regular[needle.VolumeId(vi.Id)] = true - } - } - for _, ec := range diskInfo.EcShardInfos { - if want[needle.VolumeId(ec.Id)] { - hasEcShards[needle.VolumeId(ec.Id)] = true - } - } - } - } - } - } - for _, vid := range vids { - if regular[vid] { - continue - } - if hasEcShards[vid] { - return fmt.Errorf("volume %d is already EC-encoded (no .dat replica); refusing to re-encode, which would destroy its shards", vid) - } - return fmt.Errorf("volume %d not found as a regular volume in the cluster; refusing to encode", vid) - } - return nil -} - -// collectVolumeIdToCollection returns a map from volume ID to its collection name -func collectVolumeIdToCollection(t *master_pb.TopologyInfo, vids []needle.VolumeId) map[needle.VolumeId]string { - result := make(map[needle.VolumeId]string) - if len(vids) == 0 { - return result - } - - vidSet := make(map[needle.VolumeId]bool) - for _, vid := range vids { - vidSet[vid] = true - } - - for _, dc := range t.DataCenterInfos { - for _, r := range dc.RackInfos { - for _, dn := range r.DataNodeInfos { - for _, diskInfo := range dn.DiskInfos { - for _, vi := range diskInfo.VolumeInfos { - vid := needle.VolumeId(vi.Id) - if vidSet[vid] { - result[vid] = vi.Collection - } - } - } - } - } - } - return result -} - -func collectCollectionsForVolumeIds(t *master_pb.TopologyInfo, vids []needle.VolumeId) []string { - if len(vids) == 0 { - return nil - } - - found := map[string]bool{} - for _, dc := range t.DataCenterInfos { - for _, r := range dc.RackInfos { - for _, dn := range r.DataNodeInfos { - for _, diskInfo := range dn.DiskInfos { - for _, vi := range diskInfo.VolumeInfos { - for _, vid := range vids { - if needle.VolumeId(vi.Id) == vid { - found[vi.Collection] = true - } - } - } - for _, ecs := range diskInfo.EcShardInfos { - for _, vid := range vids { - if needle.VolumeId(ecs.Id) == vid { - found[ecs.Collection] = true - } - } - } - } - } - } - } - if len(found) == 0 { - return nil - } - - collections := []string{} - for k, _ := range found { - collections = append(collections, k) - } - sort.Strings(collections) - return collections + return ec.CollectEcNodes(commandEnv.ecEnv(), diskType) } func moveMountedShardToEcNode(commandEnv *CommandEnv, existingLocation *EcNode, collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, destinationEcNode *EcNode, destDiskId uint32, applyBalancing bool, diskType types.DiskType) (err error) { - - if !commandEnv.isLocked() { - return fmt.Errorf("lock is lost") - } - - copiedShardIds := []erasure_coding.ShardId{shardId} - - if applyBalancing { - - existingServerAddress := pb.NewServerAddressFromDataNode(existingLocation.info) - - // ask destination node to copy shard and the ecx file from source node, and mount it - copiedShardIds, err = oneServerCopyAndMountEcShardsFromSource(commandEnv.option.GrpcDialOption, destinationEcNode, []erasure_coding.ShardId{shardId}, vid, collection, existingServerAddress, destDiskId) - if err != nil { - return err - } - - // unmount the to be deleted shards - err = unmountEcShards(commandEnv.option.GrpcDialOption, vid, existingServerAddress, copiedShardIds) - if err != nil { - return err - } - - // ask source node to delete the shard, and maybe the ecx file - err = sourceServerDeleteEcShards(commandEnv.option.GrpcDialOption, collection, vid, existingServerAddress, copiedShardIds) - if err != nil { - return err - } - - if destDiskId > 0 { - fmt.Printf("moved ec shard %d.%d %s => %s (disk %d)\n", vid, shardId, existingLocation.info.Id, destinationEcNode.info.Id, destDiskId) - } else { - fmt.Printf("moved ec shard %d.%d %s => %s\n", vid, shardId, existingLocation.info.Id, destinationEcNode.info.Id) - } - - } - - destinationEcNode.addEcVolumeShards(vid, collection, copiedShardIds, diskType) - existingLocation.deleteEcVolumeShards(vid, copiedShardIds, diskType) - - return nil - + return ec.MoveMountedShardToEcNode(commandEnv.ecEnv(), existingLocation, collection, vid, shardId, destinationEcNode, destDiskId, applyBalancing, diskType) } -func oneServerCopyAndMountEcShardsFromSource(grpcDialOption grpc.DialOption, - targetServer *EcNode, shardIdsToCopy []erasure_coding.ShardId, - volumeId needle.VolumeId, collection string, existingLocation pb.ServerAddress, destDiskId uint32) (copiedShardIds []erasure_coding.ShardId, err error) { - - fmt.Printf("allocate %d.%v %s => %s\n", volumeId, shardIdsToCopy, existingLocation, targetServer.info.Id) - - targetAddress := pb.NewServerAddressFromDataNode(targetServer.info) - err = volume_move.NewMover(grpcDialOption).CopyAndMountEcShards(context.Background(), volumeId, collection, shardIdsToCopy, existingLocation, targetAddress, destDiskId, 0, os.Stdout) - if err != nil { - return - } - - // SameServer, not ==: a representation mismatch here would report a - // same-server mount-in-place as a copy and have the caller delete the - // shards it kept. - if !volume_move.SameServer(targetAddress, existingLocation) { - copiedShardIds = shardIdsToCopy - glog.V(0).Infof("%s ec volume %d deletes shards %+v", existingLocation, volumeId, copiedShardIds) - } - - return -} - -func eachDataNode(topo *master_pb.TopologyInfo, fn func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo)) { - for _, dc := range topo.DataCenterInfos { - for _, rack := range dc.RackInfos { - for _, dn := range rack.DataNodeInfos { - fn(DataCenterId(dc.Id), RackId(rack.Id), dn) - } - } - } -} - -func sortEcNodesByFreeslotsDescending(ecNodes []*EcNode) { - slices.SortFunc(ecNodes, func(a, b *EcNode) int { - return b.freeEcSlot - a.freeEcSlot - }) -} - -func sortEcNodesByFreeslotsAscending(ecNodes []*EcNode) { - slices.SortFunc(ecNodes, func(a, b *EcNode) int { - return a.freeEcSlot - b.freeEcSlot - }) -} - -func countShards(ecShardInfos []*master_pb.VolumeEcShardInformationMessage) (count int) { - for _, eci := range ecShardInfos { - count += erasure_coding.GetShardCount(eci) - } - return -} - -func countFreeShardSlots(dn *master_pb.DataNodeInfo, diskType types.DiskType) (count int) { - if dn.DiskInfos == nil { - return 0 - } - diskInfo := dn.DiskInfos[string(diskType)] - if diskInfo == nil { - return 0 - } - - // A physically near-full disk has no room for more EC shards regardless of - // slot math (an over-set maxVolumeCount hides real fullness; statfs free bytes - // already include EC shard files). No-opinion when the server reports no bytes. - if balancer.DiskTooFullAfter(diskInfo.DiskTotalBytes, diskInfo.DiskFreeBytes, 0, balancer.DefaultMaxDiskUsagePercent) { - return 0 - } - - slots := int(diskInfo.MaxVolumeCount-diskInfo.VolumeCount)*erasure_coding.DataShardsCount - countShards(diskInfo.EcShardInfos) - if slots < 0 { - return 0 - } - - return slots -} - -func (ecNode *EcNode) localShardIdCount(vid uint32) int { - for _, diskInfo := range ecNode.info.DiskInfos { - for _, eci := range diskInfo.EcShardInfos { - if vid == eci.Id { - return erasure_coding.GetShardCount(eci) - } - } - } - return 0 -} - -func collectEcVolumeServersByDc(topo *master_pb.TopologyInfo, selectedDataCenter string, diskType types.DiskType) (ecNodes []*EcNode, totalFreeEcSlots int) { - eachDataNode(topo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) { - if selectedDataCenter != "" && selectedDataCenter != string(dc) { - return - } - - freeEcSlots := countFreeShardSlots(dn, diskType) - ecNode := &EcNode{ - info: dn, - dc: dc, - rack: rack, - freeEcSlot: int(freeEcSlots), - disks: make(map[uint32]*EcDisk), - } - - // Build disk-level information from volumes and EC shards - // First, discover all unique disk IDs from VolumeInfos (includes empty disks) - allDiskIds := make(map[uint32]string) // diskId -> diskType - for diskTypeKey, diskInfo := range dn.DiskInfos { - if diskInfo == nil { - continue - } - // Get all disk IDs from volumes - for _, vi := range diskInfo.VolumeInfos { - allDiskIds[vi.DiskId] = diskTypeKey - } - // Also get disk IDs from EC shards - for _, ecShardInfo := range diskInfo.EcShardInfos { - allDiskIds[ecShardInfo.DiskId] = diskTypeKey - } - } - - // Group EC shards by disk_id - diskShards := make(map[uint32]map[needle.VolumeId]*erasure_coding.ShardsInfo) - for _, diskInfo := range dn.DiskInfos { - if diskInfo == nil { - continue - } - for _, eci := range diskInfo.EcShardInfos { - diskId := eci.DiskId - if diskShards[diskId] == nil { - diskShards[diskId] = make(map[needle.VolumeId]*erasure_coding.ShardsInfo) - } - vid := needle.VolumeId(eci.Id) - diskShards[diskId][vid] = erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(eci) - } - } - - // Create EcDisk for each discovered disk - diskCount := len(allDiskIds) - if diskCount == 0 { - diskCount = 1 - } - freePerDisk := int(freeEcSlots) / diskCount - - for diskId, diskTypeStr := range allDiskIds { - shards := diskShards[diskId] - if shards == nil { - shards = make(map[needle.VolumeId]*erasure_coding.ShardsInfo) - } - totalShardCount := 0 - for _, shardsInfo := range shards { - totalShardCount += shardsInfo.Count() - } - - ecNode.disks[diskId] = &EcDisk{ - diskId: diskId, - diskType: diskTypeStr, - freeEcSlots: freePerDisk, - ecShardCount: totalShardCount, - ecShards: shards, - } - } - - ecNodes = append(ecNodes, ecNode) - totalFreeEcSlots += freeEcSlots - }) - return -} - -func sourceServerDeleteEcShards(grpcDialOption grpc.DialOption, collection string, volumeId needle.VolumeId, sourceLocation pb.ServerAddress, toBeDeletedShardIds []erasure_coding.ShardId) error { - - fmt.Printf("delete %d.%v from %s\n", volumeId, toBeDeletedShardIds, sourceLocation) - - return volume_move.NewMover(grpcDialOption).DeleteEcShards(context.Background(), volumeId, collection, sourceLocation, toBeDeletedShardIds) -} - -// errFullTeardownNotAcked marks a reachable server that completed the delete RPC -// but did not report full_teardown_done (a pre-upgrade volume server). The orphan -// sweep must treat this as fatal: the node may still hold an orphan that a later -// copy would re-stamp into the new generation. Aliased to the shared sentinel so -// the shell and the plugin-worker EC task agree on the teardown-not-acked signal. -var errFullTeardownNotAcked = erasure_coding.ErrFullTeardownNotAcked - -// pingVolumeServer probes node liveness with an empty-target Ping, which is never -// maintenance-gated, and returns the raw Ping error (nil on success). It lets the -// orphan sweep disambiguate a delete codes.Unavailable: a Rust volume server in -// maintenance mode fails the maintenance-gated delete with Unavailable yet answers -// Ping, whereas a genuinely-down node fails Ping with a transport Unavailable too. -// A Go server returns Unknown for maintenance, which isNodeUnreachable already -// treats as fatal. The caller classifies the result with classifyNodeLiveness: -// only a Ping that itself transport-failed (codes.Unavailable) confirms the node -// is down; a nil error (reachable) or any other Ping error (inconclusive — e.g. a -// pre-Ping server returning Unimplemented, which means the node is up) is fatal. -func pingVolumeServer(grpcDialOption grpc.DialOption, location pb.ServerAddress) error { - return operation.WithVolumeServerClient(false, location, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { - _, pingErr := client.Ping(context.Background(), &volume_server_pb.PingRequest{}) - return pingErr - }) -} - -// unmountAndDeleteEcShardsQuiet unmounts then deletes shards on one server in a -// single connection, without the per-call logging the interactive helpers emit. -// Used by the orphan sweep, which fans out to every node x volume and would -// otherwise flood the shell with no-op lines. -func unmountAndDeleteEcShardsQuiet(grpcDialOption grpc.DialOption, collection string, volumeId needle.VolumeId, location pb.ServerAddress, shardIds []erasure_coding.ShardId) error { - return erasure_coding.UnmountAndDeleteEcShards(context.Background(), grpcDialOption, location, collection, - uint32(volumeId), erasure_coding.ShardIdsToUint32(shardIds), 0) -} - -func unmountEcShards(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, sourceLocation pb.ServerAddress, toBeUnmountedShardIds []erasure_coding.ShardId) error { - - fmt.Printf("unmount %d.%v from %s\n", volumeId, toBeUnmountedShardIds, sourceLocation) - - return volume_move.NewMover(grpcDialOption).UnmountEcShards(context.Background(), volumeId, sourceLocation, toBeUnmountedShardIds) -} - -func mountEcShards(grpcDialOption grpc.DialOption, collection string, volumeId needle.VolumeId, sourceLocation pb.ServerAddress, toBeMountedShardIds []erasure_coding.ShardId) error { - - fmt.Printf("mount %d.%v on %s\n", volumeId, toBeMountedShardIds, sourceLocation) - - return volume_move.NewMover(grpcDialOption).MountEcShards(context.Background(), volumeId, collection, sourceLocation, toBeMountedShardIds) -} - -func ceilDivide(a, b int) int { - var r int - if (a % b) != 0 { - r = 1 - } - return (a / b) + r -} - -func findEcVolumeShardsInfo(ecNode *EcNode, vid needle.VolumeId, diskType types.DiskType) *erasure_coding.ShardsInfo { - if diskInfo, found := ecNode.info.DiskInfos[string(diskType)]; found { - for _, shardInfo := range diskInfo.EcShardInfos { - if needle.VolumeId(shardInfo.Id) == vid { - return erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(shardInfo) - } - } - } - - // Returns an empty ShardsInfo struct on failure, to avoid potential nil dereferences. - return erasure_coding.NewShardsInfo() -} - -// TODO: simplify me -func (ecNode *EcNode) addEcVolumeShards(vid needle.VolumeId, collection string, shardIds []erasure_coding.ShardId, diskType types.DiskType) *EcNode { - - foundVolume := false - diskInfo, found := ecNode.info.DiskInfos[string(diskType)] - if found { - for _, ecsi := range diskInfo.EcShardInfos { - if needle.VolumeId(ecsi.Id) == vid { - si := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(ecsi) - oldShardCount := si.Count() - for _, shardId := range shardIds { - si.Set(erasure_coding.NewShardInfo(shardId, 0)) - } - ecsi.EcIndexBits = si.Bitmap() - ecsi.ShardSizes = si.SizesInt64() - ecNode.freeEcSlot -= si.Count() - oldShardCount - foundVolume = true - break - } - } - } else { - diskInfo = &master_pb.DiskInfo{ - Type: string(diskType), - } - ecNode.info.DiskInfos[string(diskType)] = diskInfo - } - - if !foundVolume { - si := erasure_coding.NewShardsInfo() - for _, id := range shardIds { - si.Set(erasure_coding.NewShardInfo(id, 0)) - } - diskInfo.EcShardInfos = append(diskInfo.EcShardInfos, &master_pb.VolumeEcShardInformationMessage{ - Id: uint32(vid), - Collection: collection, - EcIndexBits: si.Bitmap(), - ShardSizes: si.SizesInt64(), - DiskType: string(diskType), - }) - ecNode.freeEcSlot -= si.Count() - } - - return ecNode -} - -func (ecNode *EcNode) deleteEcVolumeShards(vid needle.VolumeId, shardIds []erasure_coding.ShardId, diskType types.DiskType) *EcNode { - - if diskInfo, found := ecNode.info.DiskInfos[string(diskType)]; found { - for _, eci := range diskInfo.EcShardInfos { - if needle.VolumeId(eci.Id) == vid { - si := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(eci) - oldCount := si.Count() - for _, shardId := range shardIds { - si.Delete(shardId) - } - eci.EcIndexBits = si.Bitmap() - eci.ShardSizes = si.SizesInt64() - ecNode.freeEcSlot -= si.Count() - oldCount - } - } - } - - return ecNode -} - -// pickBestDiskOnNode selects the best disk on a node for placing a new EC shard -// It prefers disks of the specified type with fewer shards and more free slots -// When shardId is provided and dataShardCount > 0, it applies anti-affinity: -// - For data shards (shardId < dataShardCount): prefer disks without parity shards -// - For parity shards (shardId >= dataShardCount): prefer disks without data shards -// If strictDiskType is false, it will fall back to other disk types if no matching disk is found -func pickBestDiskOnNode(ecNode *EcNode, vid needle.VolumeId, diskType types.DiskType, strictDiskType bool, shardId erasure_coding.ShardId, dataShardCount int) uint32 { - if len(ecNode.disks) == 0 { - return 0 // No disk info available, let the server decide - } - - var bestDiskId uint32 - bestScore := -1 - var fallbackDiskId uint32 - fallbackScore := -1 - - // Determine if we're placing a data or parity shard - isDataShard := dataShardCount > 0 && int(shardId) < dataShardCount - - for diskId, disk := range ecNode.disks { - if disk.freeEcSlots <= 0 { - continue - } - - // Check existing shards on this disk for this volume - existingShards := 0 - hasDataShards := false - hasParityShards := false - if si, ok := disk.ecShards[vid]; ok { - existingShards = si.Count() - // Check what type of shards are on this disk - if dataShardCount > 0 { - for _, existingShardId := range si.Ids() { - if int(existingShardId) < dataShardCount { - hasDataShards = true - } else { - hasParityShards = true - } - } - } - } - - // Score: prefer disks with fewer total shards and fewer shards of this volume - // Lower score is better - score := disk.ecShardCount*10 + existingShards*100 - - // Apply anti-affinity penalty if applicable - if dataShardCount > 0 { - if isDataShard && hasParityShards { - // Penalize placing data shard on disk with parity shards - score += 1000 - } else if !isDataShard && hasDataShards { - // Penalize placing parity shard on disk with data shards - score += 1000 - } - } - - if disk.diskType == string(diskType) { - // Matching disk type - this is preferred - if bestScore == -1 || score < bestScore { - bestScore = score - bestDiskId = diskId - } - } else if !strictDiskType { - // Non-matching disk type - use as fallback if allowed - if fallbackScore == -1 || score < fallbackScore { - fallbackScore = score - fallbackDiskId = diskId - } - } - } - - // Return matching disk type if found, otherwise fallback. Gate on bestScore, - // not bestDiskId: physical disk 0 is a valid target and 0 is also the "no - // match" zero value, so testing bestDiskId would never select disk 0. - if bestScore != -1 { - return bestDiskId - } - return fallbackDiskId -} - -// ecBalancer drives an EC balance run: it collects the cluster's EC nodes, hands -// them to the shared ecbalancer planner, and executes the planned shard moves. -// The balancing policy lives in weed/storage/erasure_coding/ecbalancer, shared -// with the EC balance worker so the two cannot drift. -type ecBalancer struct { - commandEnv *CommandEnv - ecNodes []*EcNode - replicaPlacement *super_block.ReplicaPlacement - applyBalancing bool - maxParallelization int - ioBytePerSecond int64 - diskType types.DiskType - // volumeIds narrows the plan to these ec volume ids; nil balances every volume - // of the selected collections. - volumeIds map[uint32]bool -} - -// excludeNodes is a set of server addresses kept out of the balance as copy/move -// targets and sources. ec.encode passes the nodes its orphan sweep could not -// reach: such a node may still hold a stale-generation shard orphan, and pairing -// it with a new-generation shard from a balance copy would mix generations on one -// node. The standalone ec.balance command passes nil. -// -// volumeIds, when non-empty, restricts the plan to those ec volume ids; empty -// balances every volume of the given collections. +// EcBalance balances EC shards across the cluster; see ec.EcBalance for the +// excludeNodes and volumeIds semantics. func EcBalance(commandEnv *CommandEnv, collections []string, dc string, ecReplicaPlacement *super_block.ReplicaPlacement, diskType types.DiskType, maxParallelization int, ioBytePerSecond int64, applyBalancing bool, excludeNodes map[pb.ServerAddress]struct{}, volumeIds []needle.VolumeId) (err error) { - // collect all ec nodes - allEcNodes, totalFreeEcSlots, err := collectEcNodesForDC(commandEnv, dc, diskType) - if err != nil { - return err - } - - // Drop excluded nodes (and the slots they contribute) before planning so they - // can be neither a target nor a source for any move this balance plans. - if len(excludeNodes) > 0 { - kept := allEcNodes[:0] - var excludedFreeSlots int - for _, en := range allEcNodes { - if _, skip := excludeNodes[pb.NewServerAddressFromDataNode(en.info)]; skip { - excludedFreeSlots += en.freeEcSlot - glog.V(0).Infof("EC balance excluding node %s: skipped as unreachable by the encode orphan sweep", en.info.Id) - continue - } - kept = append(kept, en) - } - allEcNodes = kept - totalFreeEcSlots -= excludedFreeSlots - } - - if totalFreeEcSlots < 1 { - return fmt.Errorf("no free ec shard slots. only %d left", totalFreeEcSlots) - } - - var volumeIdFilter map[uint32]bool - if len(volumeIds) > 0 { - volumeIdFilter = make(map[uint32]bool, len(volumeIds)) - for _, vid := range volumeIds { - volumeIdFilter[uint32(vid)] = true - } - } - - ecb := &ecBalancer{ - commandEnv: commandEnv, - ecNodes: allEcNodes, - replicaPlacement: ecReplicaPlacement, - applyBalancing: applyBalancing, - maxParallelization: maxParallelization, - ioBytePerSecond: ioBytePerSecond, - diskType: diskType, - volumeIds: volumeIdFilter, - } - - if len(collections) == 0 { - glog.V(1).Infof("WARNING: No collections to balance EC volumes across.\n") - } - return ecb.balance(collections) -} - -// shellECRatio resolves a collection's EC data/parity counts, defaulting to the -// standard scheme. This is the shell's plug-in point for custom ratios. -func shellECRatio(_ string) (int, int) { - // Custom EC ratios are an enterprise feature; OSS uses the standard scheme. - return erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount -} - -// balance plans EC shard moves with the shared planner and executes them. When -// collections is empty all collections present are balanced. -func (ecb *ecBalancer) balance(collections []string) error { - topo, volumeRatio, selected := toBalancerTopology(ecb.ecNodes, collections, ecb.diskType, ecb.volumeIds) - if len(ecb.volumeIds) > 0 { - requested := make([]uint32, 0, len(ecb.volumeIds)) - for vid := range ecb.volumeIds { - requested = append(requested, vid) - } - slices.Sort(requested) - var missing []uint32 - for _, vid := range requested { - if !selected[vid] { - missing = append(missing, vid) - } - } - if len(missing) > 0 { - return fmt.Errorf("no ec shards found for volume(s) %v: not an ec volume, or outside the selected collection, dataCenter or diskType", missing) - } - fmt.Printf("balancing ec volume(s) %v\n", requested) - } - moves := ecbalancer.Plan(topo, ecbalancer.Options{ - DiskType: string(ecb.diskType), - ImbalanceThreshold: 0, // the shell balances to an even distribution - ReplicaPlacement: ecb.replicaPlacement, - Ratio: shellECRatio, - // Prefer each volume's own heartbeat-reported ratio over the collection - // default so a mixed-ratio collection is spread per volume; 0 defers to - // shellECRatio (and is the always-0 OSS case). - VolumeRatio: volumeRatio, - // Balance the global phase by fractional fullness so heterogeneous-capacity - // nodes fill proportionally (matching the worker). This is identical to raw - // shard count when capacities are uniform. - GlobalUtilizationBased: true, - }) - if len(ecb.volumeIds) > 0 { - var deletions int - for _, m := range moves { - if m.Phase == "dedup" { - deletions++ - } - } - fmt.Printf("planned %d ec shard move(s) and %d ec shard deletion(s)\n", len(moves)-deletions, deletions) - } - return ecb.executeMoves(moves) -} - -// toBalancerTopology builds an ecbalancer.Topology from the shell's EcNode model, -// including the shards of the requested collections (all collections when empty) -// and, when volumeIds is non-nil, only those volume ids. Volumes left out here are -// invisible to the planner, so no phase - dedup included - can plan against them. -// It also returns a per-volume ratio lookup built from each shard's heartbeat -// (0,0 when unreported, e.g. always in OSS), which Plan prefers over the -// collection ratio for mixed-ratio clusters, and the set of volume ids that made -// it into the topology. -func toBalancerTopology(ecNodes []*EcNode, collections []string, diskType types.DiskType, volumeIds map[uint32]bool) (*ecbalancer.Topology, func(collection string, vid uint32) (int, int), map[uint32]bool) { - allowed := make(map[string]bool, len(collections)) - for _, c := range collections { - allowed[c] = true - } - - type volRatioKey struct { - collection string - vid uint32 - } - volRatios := make(map[volRatioKey][2]int) - selected := make(map[uint32]bool) - - topo := ecbalancer.NewTopology() - for _, en := range ecNodes { - rackKey := string(en.dc) + ":" + string(en.rack) - node := topo.AddNode(en.info.Id, string(en.dc), rackKey, en.freeEcSlot) - // Group by physical machine (host) so shards spread across machines, not just - // nodes; the id stays the node identity used for moves. - node.SetHost(pb.NewServerAddressFromDataNode(en.info).ToHost()) - for diskId, d := range en.disks { - node.AddDisk(diskId, d.diskType, d.freeEcSlots, d.ecShardCount) - } - diskInfo, found := en.info.DiskInfos[string(diskType)] - if !found { - continue - } - for _, eci := range diskInfo.EcShardInfos { - if len(allowed) > 0 && !allowed[eci.Collection] { - continue - } - if volumeIds != nil && !volumeIds[eci.Id] { - continue - } - selected[eci.Id] = true - node.AddShards(eci.Id, eci.Collection, eci.DiskId, erasure_coding.ShardBits(eci.EcIndexBits)) - if d, p := ecbalancer.VolumeShardRatio(eci); d > 0 || p > 0 { - volRatios[volRatioKey{eci.Collection, eci.Id}] = [2]int{d, p} - } - } - } - - volumeRatio := func(collection string, vid uint32) (int, int) { - r := volRatios[volRatioKey{collection, vid}] - return r[0], r[1] - } - return topo, volumeRatio, selected -} - -// executeMoves carries out the planned moves. Phases run in order (a within-rack -// move can depend on a cross-rack move's result), and the independent moves -// within a phase run with up to maxParallelization concurrency. Apply mode does -// only the RPCs; dry-run mode runs sequentially and mutates the in-memory EcNode -// model so callers/tests can inspect the planned end state. -func (ecb *ecBalancer) executeMoves(moves []ecbalancer.Move) error { - byID := make(map[string]*EcNode, len(ecb.ecNodes)) - for _, en := range ecb.ecNodes { - byID[en.info.Id] = en - } - - // Plan emits moves grouped by phase; run each contiguous same-phase group - // together, waiting before the next so cross-phase dependencies hold. - for i := 0; i < len(moves); { - j := i - for j < len(moves) && moves[j].Phase == moves[i].Phase { - j++ - } - if err := ecb.executePhase(byID, moves[i:j]); err != nil { - return err - } - i = j - } - return nil -} - -func (ecb *ecBalancer) executePhase(byID map[string]*EcNode, moves []ecbalancer.Move) error { - if !ecb.applyBalancing { - // Dry-run: sequential so the in-memory model updates are race-free and - // reflect the full plan for inspection. - for _, m := range moves { - if err := ecb.executeMove(byID, m); err != nil { - return err - } - } - return nil - } - // Apply mode: parallelize across volumes, but run one volume's moves within a - // phase sequentially. Concurrent moves of the same volume to a node can race - // on its shared .ecx/.ecj/.vif sidecar files. - var order []uint32 - byVol := make(map[uint32][]ecbalancer.Move) - for _, m := range moves { - if _, ok := byVol[m.VolumeID]; !ok { - order = append(order, m.VolumeID) - } - byVol[m.VolumeID] = append(byVol[m.VolumeID], m) - } - taskGroups := make([][]ErrorWaitGroupTask, 0, len(order)) - for _, vid := range order { - movesForVolume := byVol[vid] - taskGroup := make([]ErrorWaitGroupTask, 0, len(movesForVolume)) - for _, move := range movesForVolume { - move := move - taskGroup = append(taskGroup, func() error { - return ecb.executeMove(byID, move) - }) - } - taskGroups = append(taskGroups, taskGroup) - } - return executeParallelTaskGroups(ecb.maxParallelization, taskGroups) -} - -// verifyEcShardOnKeepNode confirms the node a dedup move chose to keep actually -// holds the shard, so a duplicate is only removed when a real copy remains. An -// unreachable keep node is unknown, not confirmed, and blocks the delete — as -// does one that answers too slowly to be waited on, which is why this is -// bounded rather than left to hang the whole balance run. -// ecShardVerifyTimeout bounds the keep-node inventory query. A node that accepts -// the connection but never answers must not stall the whole balance run. -const ecShardVerifyTimeout = 30 * time.Second - -func verifyEcShardOnKeepNode(grpcDialOption grpc.DialOption, collection string, vid needle.VolumeId, keepNode string, shardId erasure_coding.ShardId) error { - if keepNode == "" { - return fmt.Errorf("refusing dedup delete of %d.%d: no keep node recorded", vid, shardId) - } - ctx, cancel := context.WithTimeout(context.Background(), ecShardVerifyTimeout) - defer cancel() - if err := erasure_coding.VerifyShardsOnServer(ctx, collection, uint32(vid), keepNode, - []uint32{uint32(shardId)}, grpcDialOption); err != nil { - return fmt.Errorf("refusing dedup delete: %w", err) - } - return nil -} - -func (ecb *ecBalancer) executeMove(byID map[string]*EcNode, m ecbalancer.Move) error { - src := byID[m.SourceNode] - if src == nil { - return nil - } - vid := needle.VolumeId(m.VolumeID) - shardId := erasure_coding.ShardId(m.ShardID) - shardIds := []erasure_coding.ShardId{shardId} - - if m.Phase == "dedup" { - fmt.Printf("dedup: delete ec shard %d.%d on %s\n", vid, shardId, m.SourceNode) - if !ecb.applyBalancing { - src.deleteEcVolumeShards(vid, shardIds, ecb.diskType) - return nil - } - grpcDialOption := ecb.commandEnv.option.GrpcDialOption - // Nothing is copied first, so the shard surviving elsewhere is the only - // thing making this safe -- and the plan saying so is not evidence. A - // topology entry can name a location holding nothing, and deleting on - // that basis removes the last copy. Confirm the keep node has it. - if err := verifyEcShardOnKeepNode(grpcDialOption, m.Collection, vid, m.KeepNode, shardId); err != nil { - return err - } - addr := pb.NewServerAddressFromDataNode(src.info) - if err := unmountEcShards(grpcDialOption, vid, addr, shardIds); err != nil { - return err - } - return sourceServerDeleteEcShards(grpcDialOption, m.Collection, vid, addr, shardIds) - } - - dst := byID[m.TargetNode] - if dst == nil { - return nil - } - if m.TargetDisk > 0 { - fmt.Printf("%s moves ec shard %d.%d to %s (disk %d)\n", m.SourceNode, vid, shardId, m.TargetNode, m.TargetDisk) - } else { - fmt.Printf("%s moves ec shard %d.%d to %s\n", m.SourceNode, vid, shardId, m.TargetNode) - } - if !ecb.applyBalancing { - // Dry-run: update the in-memory model only. - return moveMountedShardToEcNode(ecb.commandEnv, src, m.Collection, vid, shardId, dst, m.TargetDisk, false, ecb.diskType) - } - return ecb.applyShardMoveRPC(src, dst, m.Collection, vid, shardId, m.TargetDisk) -} - -// applyShardMoveRPC copies a shard to the destination disk, verifies the -// destination registered it, then unmounts and deletes it on the source. It -// does not touch the in-memory model, so it is safe to run concurrently across -// the moves of a phase. -func (ecb *ecBalancer) applyShardMoveRPC(src, dst *EcNode, collection string, vid needle.VolumeId, shardId erasure_coding.ShardId, destDiskId uint32) error { - srcAddr := pb.NewServerAddressFromDataNode(src.info) - dstAddr := pb.NewServerAddressFromDataNode(dst.info) - if volume_move.SameServer(srcAddr, dstAddr) { - // A same-server (cross-disk) move cannot be expressed with these RPCs; - // leave the shard where it is. - return nil - } - return volume_move.NewMover(ecb.commandEnv.option.GrpcDialOption).MoveEcShards(context.Background(), volume_move.EcShardMove{ - VolumeId: vid, - Collection: collection, - ShardIds: []erasure_coding.ShardId{shardId}, - Source: srcAddr, - Target: dstAddr, - TargetDisk: destDiskId, - }, volume_move.EcMoveOptions{IoBytePerSecond: ecb.ioBytePerSecond, Writer: os.Stdout}) -} - -// parseVolumeIdsFlag parses a comma-separated -volumeIds flag value, dropping -// duplicates and keeping the given order. -func parseVolumeIdsFlag(volumeIdsStr string) ([]needle.VolumeId, error) { - var volumeIds []needle.VolumeId - seen := make(map[needle.VolumeId]bool) - for _, part := range strings.Split(volumeIdsStr, ",") { - part = strings.TrimSpace(part) - if part == "" { - continue - } - vidValue, err := strconv.ParseUint(part, 10, 32) - if err != nil || vidValue == 0 { - return nil, fmt.Errorf("invalid volume id %q in -volumeIds", part) - } - // ParseUint with bitSize 32 bounds the value; convert through uint32 - // (matching the rest of the codebase) so the narrowing is provably safe. - vid := needle.VolumeId(uint32(vidValue)) - if seen[vid] { - continue - } - seen[vid] = true - volumeIds = append(volumeIds, vid) - } - if len(volumeIds) == 0 { - return nil, fmt.Errorf("-volumeIds does not contain any valid volume id") - } - return volumeIds, nil + return ec.EcBalance(commandEnv.ecEnv(), collections, dc, ecReplicaPlacement, diskType, maxParallelization, ioBytePerSecond, applyBalancing, excludeNodes, volumeIds) } // compileCollectionPattern compiles a regex pattern for collection matching. diff --git a/weed/shell/command_ec_decode.go b/weed/shell/command_ec_decode.go index a7135efc2..6ba88dc79 100644 --- a/weed/shell/command_ec_decode.go +++ b/weed/shell/command_ec_decode.go @@ -5,21 +5,11 @@ import ( "flag" "fmt" "io" - "strings" - "github.com/seaweedfs/seaweedfs/weed/glog" - "github.com/seaweedfs/seaweedfs/weed/pb" - "github.com/seaweedfs/seaweedfs/weed/storage/types" - - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - - "github.com/seaweedfs/seaweedfs/weed/operation" + "github.com/seaweedfs/seaweedfs/weed/ec" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" - "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" - "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/types" ) func init() { @@ -82,27 +72,29 @@ func (c *commandEcDecode) Do(args []string, commandEnv *CommandEnv, writer io.Wr vid := needle.VolumeId(*volumeId) diskType := types.ToDiskType(*diskTypeStr) + env := commandEnv.ecEnv() // collect topology information topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) if err != nil { return err } - var diskUsageState *decodeDiskUsageState + var diskUsageState *ec.DecodeDiskUsageState if *checkMinFreeSpace { - diskUsageState = newDecodeDiskUsageState(topologyInfo, diskType) + diskUsageState = ec.NewDecodeDiskUsageState(topologyInfo, diskType) } // volumeId is provided if vid != 0 { - return doEcDecode(commandEnv, topologyInfo, *collection, vid, diskType, *checkMinFreeSpace, diskUsageState) + return ec.DoEcDecode(env, topologyInfo, *collection, vid, diskType, *checkMinFreeSpace, diskUsageState) } // apply to all volumes in the collection - volumeIds, err := collectEcShardIds(topologyInfo, *collection, diskType) + collectionRegex, err := compileCollectionPattern(*collection) if err != nil { - return err + return fmt.Errorf("invalid collection pattern '%s': %v", *collection, err) } + volumeIds := ec.CollectEcShardIds(topologyInfo, collectionRegex, diskType) fmt.Printf("ec decode volumes: %v\n", volumeIds) batches := chunkVolumeIds(volumeIds, *batchSize) for i, batch := range batches { @@ -114,14 +106,14 @@ func (c *commandEcDecode) Do(args []string, commandEnv *CommandEnv, writer io.Wr return err } if *checkMinFreeSpace { - diskUsageState = newDecodeDiskUsageState(topologyInfo, diskType) + diskUsageState = ec.NewDecodeDiskUsageState(topologyInfo, diskType) } } if len(batches) > 1 { fmt.Printf("ec decode batch %d/%d: %v\n", i+1, len(batches), batch) } for _, vid := range batch { - if err = doEcDecode(commandEnv, topologyInfo, *collection, vid, diskType, *checkMinFreeSpace, diskUsageState); err != nil { + if err = ec.DoEcDecode(env, topologyInfo, *collection, vid, diskType, *checkMinFreeSpace, diskUsageState); err != nil { return err } } @@ -130,262 +122,6 @@ func (c *commandEcDecode) Do(args []string, commandEnv *CommandEnv, writer io.Wr return nil } -func doEcDecode(commandEnv *CommandEnv, topoInfo *master_pb.TopologyInfo, collection string, vid needle.VolumeId, diskType types.DiskType, checkMinFreeSpace bool, diskUsageState *decodeDiskUsageState) (err error) { - - if !commandEnv.isLocked() { - return fmt.Errorf("lock is lost") - } - - // find volume location - nodeToEcShardsInfo, dataShards := collectEcNodeShardsInfo(topoInfo, vid, diskType) - - fmt.Printf("ec volume %d shard locations: %+v\n", vid, nodeToEcShardsInfo) - - if len(nodeToEcShardsInfo) == 0 { - return fmt.Errorf("no EC shards found for volume %d (diskType %s)", vid, diskType.ReadableString()) - } - - var originalShardCounts map[pb.ServerAddress]int - if diskUsageState != nil { - originalShardCounts = make(map[pb.ServerAddress]int, len(nodeToEcShardsInfo)) - for location, si := range nodeToEcShardsInfo { - originalShardCounts[location] = si.Count() - } - } - - var eligibleTargets map[pb.ServerAddress]struct{} - if checkMinFreeSpace { - if diskUsageState == nil { - return fmt.Errorf("min free space checking requires disk usage state") - } - eligibleTargets = make(map[pb.ServerAddress]struct{}) - for location := range nodeToEcShardsInfo { - if freeCount, found := diskUsageState.freeVolumeCount(location); found && freeCount > 0 { - eligibleTargets[location] = struct{}{} - } - } - if len(eligibleTargets) == 0 { - return fmt.Errorf("no eligible target datanodes with free volume slots for volume %d (diskType %s); use -checkMinFreeSpace=false to override", vid, diskType.ReadableString()) - } - } - - // collect ec shards to the server with most space - targetNodeLocation, err := collectEcShards(commandEnv, nodeToEcShardsInfo, collection, vid, eligibleTargets, dataShards) - if err != nil { - return fmt.Errorf("collectEcShards for volume %d: %v", vid, err) - } - - // generate a normal volume - err = generateNormalVolume(commandEnv.option.GrpcDialOption, vid, collection, targetNodeLocation) - if err != nil { - // Special case: if the EC index has no live entries, decoding is a no-op. - // Just purge EC shards and return success without generating/mounting an empty volume. - if isEcDecodeEmptyVolumeErr(err) { - if err := unmountAndDeleteEcShards(commandEnv.option.GrpcDialOption, collection, nodeToEcShardsInfo, vid); err != nil { - return err - } - if diskUsageState != nil { - diskUsageState.applyDecode(targetNodeLocation, originalShardCounts, false) - } - return nil - } - return fmt.Errorf("generate normal volume %d on %s: %v", vid, targetNodeLocation, err) - } - - // mount the decoded volume after server-side offline compaction succeeded - err = mountDecodedVolume(commandEnv.option.GrpcDialOption, targetNodeLocation, vid) - if err != nil { - return fmt.Errorf("mount decoded volume %d on %s: %v", vid, targetNodeLocation, err) - } - - // Confirm the regenerated .dat is present and non-empty before destroying - // the shards. Without this gate, a silent failure in generate/mount could - // leave the cluster with neither shards nor volume. - if err := verifyDecodedVolumeBeforeDelete(commandEnv.option.GrpcDialOption, targetNodeLocation, vid); err != nil { - return fmt.Errorf("verify decoded volume %d on %s before deleting shards: %w", vid, targetNodeLocation, err) - } - - // delete the previous ec shards - err = unmountAndDeleteEcShardsWithPrefix("deleteDecodedEcShards", commandEnv.option.GrpcDialOption, collection, nodeToEcShardsInfo, vid) - if err != nil { - return fmt.Errorf("delete ec shards for volume %d: %v", vid, err) - } - if diskUsageState != nil { - diskUsageState.applyDecode(targetNodeLocation, originalShardCounts, true) - } - - return nil -} - -func isEcDecodeEmptyVolumeErr(err error) bool { - st, ok := status.FromError(err) - if !ok { - return false - } - if st.Code() != codes.FailedPrecondition { - return false - } - // Keep this robust against wording tweaks while still being specific. - return strings.Contains(st.Message(), erasure_coding.EcNoLiveEntriesSubstring) -} - -func unmountAndDeleteEcShards(grpcDialOption grpc.DialOption, collection string, nodeToShardsInfo map[pb.ServerAddress]*erasure_coding.ShardsInfo, vid needle.VolumeId) error { - return unmountAndDeleteEcShardsWithPrefix("unmountAndDeleteEcShards", grpcDialOption, collection, nodeToShardsInfo, vid) -} - -func unmountAndDeleteEcShardsWithPrefix(prefix string, grpcDialOption grpc.DialOption, collection string, nodeToShardsInfo map[pb.ServerAddress]*erasure_coding.ShardsInfo, vid needle.VolumeId) error { - ewg := NewErrorWaitGroup(len(nodeToShardsInfo)) - - // unmount and delete ec shards in parallel (one goroutine per location) - for location, si := range nodeToShardsInfo { - location, si := location, si // capture loop variables for goroutine - ewg.Add(func() error { - fmt.Printf("unmount ec volume %d on %s has shards: %+v\n", vid, location, si.Ids()) - if err := unmountEcShards(grpcDialOption, vid, location, si.Ids()); err != nil { - return fmt.Errorf("%s unmount ec volume %d on %s: %w", prefix, vid, location, err) - } - - fmt.Printf("delete ec volume %d on %s has shards: %+v\n", vid, location, si.Ids()) - if err := sourceServerDeleteEcShards(grpcDialOption, collection, vid, location, si.Ids()); err != nil { - return fmt.Errorf("%s delete ec volume %d on %s: %w", prefix, vid, location, err) - } - return nil - }) - } - return ewg.Wait() -} - -func verifyDecodedVolumeBeforeDelete(grpcDialOption grpc.DialOption, target pb.ServerAddress, vid needle.VolumeId) error { - var resp *volume_server_pb.ReadVolumeFileStatusResponse - if err := operation.WithVolumeServerClient(false, target, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { - r, e := client.ReadVolumeFileStatus(context.Background(), &volume_server_pb.ReadVolumeFileStatusRequest{ - VolumeId: uint32(vid), - }) - if e != nil { - return e - } - resp = r - return nil - }); err != nil { - return fmt.Errorf("read volume file status: %w", err) - } - if resp.DatFileSize == 0 { - return fmt.Errorf("decoded .dat is 0 bytes") - } - if resp.IdxFileSize == 0 { - return fmt.Errorf("decoded .idx is 0 bytes") - } - glog.V(0).Infof("ec decode verification ok for volume %d on %s: dat=%d idx=%d", vid, target, resp.DatFileSize, resp.IdxFileSize) - return nil -} - -func mountDecodedVolume(grpcDialOption grpc.DialOption, targetNodeLocation pb.ServerAddress, vid needle.VolumeId) error { - return operation.WithVolumeServerClient(false, targetNodeLocation, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { - _, mountErr := volumeServerClient.VolumeMount(context.Background(), &volume_server_pb.VolumeMountRequest{ - VolumeId: uint32(vid), - }) - return mountErr - }) -} - -func generateNormalVolume(grpcDialOption grpc.DialOption, vid needle.VolumeId, collection string, sourceVolumeServer pb.ServerAddress) error { - fmt.Printf("generateNormalVolume from ec volume %d on %s\n", vid, sourceVolumeServer) - - err := operation.WithVolumeServerClient(false, sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { - _, genErr := volumeServerClient.VolumeEcShardsToVolume(context.Background(), &volume_server_pb.VolumeEcShardsToVolumeRequest{ - VolumeId: uint32(vid), - Collection: collection, - }) - return genErr - }) - - return err - -} - -func collectEcShards(commandEnv *CommandEnv, nodeToShardsInfo map[pb.ServerAddress]*erasure_coding.ShardsInfo, collection string, vid needle.VolumeId, eligibleTargets map[pb.ServerAddress]struct{}, dataShards int) (targetNodeLocation pb.ServerAddress, err error) { - - maxShardCount := -1 - existingShardsInfo := erasure_coding.NewShardsInfo() - for loc, si := range nodeToShardsInfo { - if eligibleTargets != nil { - if _, ok := eligibleTargets[loc]; !ok { - continue - } - } - toBeCopiedShardCount := si.MinusParityShards(dataShards).Count() - if toBeCopiedShardCount > maxShardCount { - maxShardCount = toBeCopiedShardCount - targetNodeLocation = loc - existingShardsInfo = si - } - } - if targetNodeLocation == "" { - return "", fmt.Errorf("no eligible target datanodes available to decode volume %d", vid) - } - - fmt.Printf("collectEcShards: ec volume %d collect shards to %s from: %+v\n", vid, targetNodeLocation, nodeToShardsInfo) - - copiedShardsInfo := erasure_coding.NewShardsInfo() - for loc, si := range nodeToShardsInfo { - if loc == targetNodeLocation { - continue - } - - needToCopyShardsInfo := si.Minus(existingShardsInfo).MinusParityShards(dataShards) - - err = operation.WithVolumeServerClient(false, targetNodeLocation, commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { - - // Always collect .ecj from every shard location. Each server's .ecj - // only contains deletions for needles whose data resides in shards - // held by that server. Without merging all .ecj files, deletions - // recorded on other servers would be lost during decode. - if needToCopyShardsInfo.Count() > 0 { - fmt.Printf("copy %d.%v %s => %s\n", vid, needToCopyShardsInfo.Ids(), loc, targetNodeLocation) - } else { - fmt.Printf("collect ecj %d %s => %s\n", vid, loc, targetNodeLocation) - } - - _, copyErr := volumeServerClient.VolumeEcShardsCopy(context.Background(), &volume_server_pb.VolumeEcShardsCopyRequest{ - VolumeId: uint32(vid), - Collection: collection, - ShardIds: needToCopyShardsInfo.IdsUint32(), - CopyEcxFile: false, - CopyEcjFile: true, - CopyVifFile: needToCopyShardsInfo.Count() > 0, - SourceDataNode: string(loc), - }) - if copyErr != nil { - return fmt.Errorf("copy %d.%v %s => %s : %v\n", vid, needToCopyShardsInfo.Ids(), loc, targetNodeLocation, copyErr) - } - - if needToCopyShardsInfo.Count() > 0 { - fmt.Printf("mount %d.%v on %s\n", vid, needToCopyShardsInfo.Ids(), targetNodeLocation) - _, mountErr := volumeServerClient.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{ - VolumeId: uint32(vid), - Collection: collection, - ShardIds: needToCopyShardsInfo.IdsUint32(), - }) - if mountErr != nil { - return fmt.Errorf("mount %d.%v on %s : %v\n", vid, needToCopyShardsInfo.Ids(), targetNodeLocation, mountErr) - } - } - - return nil - }) - - if err != nil { - break - } - - copiedShardsInfo.Add(needToCopyShardsInfo) - } - - nodeToShardsInfo[targetNodeLocation] = existingShardsInfo.Plus(copiedShardsInfo) - - return targetNodeLocation, err -} - func lookupVolumeIds(commandEnv *CommandEnv, volumeIds []string) (volumeIdLocations []*master_pb.LookupVolumeResponse_VolumeIdLocation, err error) { var resp *master_pb.LookupVolumeResponse err = commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error { @@ -397,109 +133,3 @@ func lookupVolumeIds(commandEnv *CommandEnv, volumeIds []string) (volumeIdLocati } return resp.VolumeIdLocations, nil } - -func collectEcShardIds(topoInfo *master_pb.TopologyInfo, collectionPattern string, diskType types.DiskType) (vids []needle.VolumeId, err error) { - // compile regex pattern for collection matching - collectionRegex, err := compileCollectionPattern(collectionPattern) - if err != nil { - return nil, fmt.Errorf("invalid collection pattern '%s': %v", collectionPattern, err) - } - - vidMap := make(map[uint32]bool) - eachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) { - if diskInfo, found := dn.DiskInfos[string(diskType)]; found { - for _, v := range diskInfo.EcShardInfos { - if collectionRegex.MatchString(v.Collection) { - vidMap[v.Id] = true - } - } - } - }) - - for vid := range vidMap { - vids = append(vids, needle.VolumeId(vid)) - } - - return -} - -func collectEcNodeShardsInfo(topoInfo *master_pb.TopologyInfo, vid needle.VolumeId, diskType types.DiskType) (map[pb.ServerAddress]*erasure_coding.ShardsInfo, int) { - res := make(map[pb.ServerAddress]*erasure_coding.ShardsInfo) - eachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) { - if diskInfo, found := dn.DiskInfos[string(diskType)]; found { - // A node may report several EcShardInfos for one volume — one per - // physical disk holding shards of it (multi-disk nodes). Union them - // rather than overwriting, or only the last disk's shards survive and - // the node looks like it is missing shards it actually has. - for _, v := range diskInfo.EcShardInfos { - if v.Id == uint32(vid) { - addr := pb.NewServerAddressFromDataNode(dn) - si := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(v) - if existing, ok := res[addr]; ok { - existing.Add(si) - } else { - res[addr] = si - } - } - } - } - }) - - // OSS is always 10+4; the per-volume ratio override lives in the enterprise build. - return res, erasure_coding.DataShardsCount -} - -type decodeDiskUsageState struct { - byNode map[pb.ServerAddress]*decodeDiskUsageCounts -} - -type decodeDiskUsageCounts struct { - maxVolumeCount int64 - volumeCount int64 - remoteVolumeCount int64 - ecShardCount int64 -} - -func newDecodeDiskUsageState(topoInfo *master_pb.TopologyInfo, diskType types.DiskType) *decodeDiskUsageState { - state := &decodeDiskUsageState{byNode: make(map[pb.ServerAddress]*decodeDiskUsageCounts)} - eachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) { - if diskInfo, found := dn.DiskInfos[string(diskType)]; found { - state.byNode[pb.NewServerAddressFromDataNode(dn)] = &decodeDiskUsageCounts{ - maxVolumeCount: diskInfo.MaxVolumeCount, - volumeCount: diskInfo.VolumeCount, - remoteVolumeCount: diskInfo.RemoteVolumeCount, - ecShardCount: int64(countShards(diskInfo.EcShardInfos)), - } - } - }) - return state -} - -func (state *decodeDiskUsageState) freeVolumeCount(location pb.ServerAddress) (int64, bool) { - if state == nil { - return 0, false - } - usage, found := state.byNode[location] - if !found { - return 0, false - } - free := usage.maxVolumeCount - (usage.volumeCount - usage.remoteVolumeCount) - free -= (usage.ecShardCount + int64(erasure_coding.DataShardsCount) - 1) / int64(erasure_coding.DataShardsCount) - return free, true -} - -func (state *decodeDiskUsageState) applyDecode(targetNodeLocation pb.ServerAddress, shardCounts map[pb.ServerAddress]int, createdVolume bool) { - if state == nil { - return - } - for location, shardCount := range shardCounts { - if usage, found := state.byNode[location]; found { - usage.ecShardCount -= int64(shardCount) - } - } - if createdVolume { - if usage, found := state.byNode[targetNodeLocation]; found { - usage.volumeCount++ - } - } -} diff --git a/weed/shell/command_ec_encode.go b/weed/shell/command_ec_encode.go index a4bcc53f0..0af96dc7c 100644 --- a/weed/shell/command_ec_encode.go +++ b/weed/shell/command_ec_encode.go @@ -1,36 +1,19 @@ package shell import ( - "context" - "errors" "flag" "fmt" "io" - "regexp" - "slices" - "sort" - "strconv" "strings" - "sync" "time" "github.com/seaweedfs/seaweedfs/weed/storage/types" + "github.com/seaweedfs/seaweedfs/weed/ec" "github.com/seaweedfs/seaweedfs/weed/glog" - "github.com/seaweedfs/seaweedfs/weed/pb" - "github.com/seaweedfs/seaweedfs/weed/wdclient" - - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - - "github.com/seaweedfs/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" - "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" "github.com/seaweedfs/seaweedfs/weed/storage/needle" - "github.com/seaweedfs/seaweedfs/weed/storage/super_block" - "github.com/seaweedfs/seaweedfs/weed/storage/volume_replica" ) func init() { @@ -183,7 +166,7 @@ func (c *commandEcEncode) Do(args []string, commandEnv *CommandEnv, writer io.Wr return fmt.Errorf("-batchSize must be >= 0") } - batches := chunkVolumeIds(volumeIds, *batchSize) + batches := ec.ChunkVolumeIds(volumeIds, *batchSize) if len(batches) > 1 { fmt.Printf("Processing %d volumes in %d batch(es), batchSize=%d\n", len(volumeIds), len(batches), *batchSize) } @@ -191,7 +174,7 @@ func (c *commandEcEncode) Do(args []string, commandEnv *CommandEnv, writer io.Wr if len(batches) > 1 { fmt.Printf("Starting EC encoding batch %d/%d with %d volumes: %v\n", i+1, len(batches), len(batchVolumeIds), batchVolumeIds) } - if err := processEcEncodeBatch(commandEnv, writer, batchVolumeIds, rp, diskType, *maxParallelization, *applyBalancing, *collection); err != nil { + if err := ec.ProcessEcEncodeBatch(commandEnv.ecEnv(), writer, batchVolumeIds, rp, diskType, *maxParallelization, *applyBalancing, *collection); err != nil { return fmt.Errorf("ec encode batch %d/%d for volumes %v: %w", i+1, len(batches), batchVolumeIds, err) } } @@ -201,790 +184,6 @@ func (c *commandEcEncode) Do(args []string, commandEnv *CommandEnv, writer io.Wr return nil } - -func chunkVolumeIds(volumeIds []needle.VolumeId, batchSize int) [][]needle.VolumeId { - if batchSize <= 0 || len(volumeIds) == 0 { - return [][]needle.VolumeId{volumeIds} - } - var batches [][]needle.VolumeId - for start := 0; start < len(volumeIds); start += batchSize { - end := start + batchSize - if end > len(volumeIds) { - end = len(volumeIds) - } - batches = append(batches, volumeIds[start:end]) - } - return batches -} - -func processEcEncodeBatch(commandEnv *CommandEnv, writer io.Writer, volumeIds []needle.VolumeId, rp *super_block.ReplicaPlacement, diskType types.DiskType, maxParallelization int, applyBalancing bool, collectionForMessage string) (err error) { - topologyInfo, _, err := collectTopologyInfo(commandEnv, 0) - if err != nil { - return err - } - - // Refuse to encode a volume that is already EC (present only as shards): - // an EC volume has no .dat, so re-encoding it would tear down its only - // copy before failing. A regular volume (with a .dat) passes. This closes - // the operator-rerun / script-retry path; a worker racing the snapshot is - // handled by encode fencing, not here. - if err := assertEncodableRegularVolumes(topologyInfo, volumeIds); err != nil { - return err - } - - volumeIdToCollection := collectVolumeIdToCollection(topologyInfo, volumeIds) - balanceCollections := collectCollectionsForVolumeIds(topologyInfo, volumeIds) - - fmt.Printf("Collecting volume locations for %d volumes before EC encoding...\n", len(volumeIds)) - volumeLocationsMap, err := volumeLocations(commandEnv, volumeIds) - if err != nil { - return fmt.Errorf("failed to collect volume locations before EC encoding: %w", err) - } - - if err := checkEcEncodeCapacity(topologyInfo, len(volumeIds), diskType, collectionForMessage); err != nil { - return err - } - - // From here doEcEncode marks the volumes readonly and generates EC shards. - // If any step before the originals are deleted fails, roll the encode back: - // tear down the shards produced this run and restore the sources to writable, - // so a failed (and possibly abandoned) ec.encode does not strand volumes - // readonly or leave orphan EC shards behind. Once the shards are verified - // recoverable we are committed to the EC copy and must not roll back. - committed := false - defer func() { - if err != nil && !committed { - rollbackFailedEcEncode(commandEnv, writer, volumeIds, volumeIdToCollection, volumeLocationsMap, maxParallelization) - } - }() - - skippedNodes, err := doEcEncode(commandEnv, writer, volumeIdToCollection, volumeIds, maxParallelization, topologyInfo) - if err != nil { - return fmt.Errorf("ec encode for volumes %v: %w", volumeIds, err) - } - // Mounting the new shards notifies the master asynchronously, and EcBalance - // plans from a fresh topology snapshot: one taken before the mounts land - // shows no shards for these volumes, so the balance plans no moves and - // silently leaves every shard on the generation host. - if err := waitForEcShardsToRegister(commandEnv, volumeIds); err != nil { - return fmt.Errorf("wait for ec shards to register with the master: %w", err) - } - // EcBalance works at collection scope. In batch mode this intentionally - // rebalances each collection after every batch so source volumes can be - // safely verified and deleted without waiting for all batches to finish. - // skippedNodes are excluded so a recovered node's stale orphan is never - // paired with a new-generation shard. - if err := EcBalance(commandEnv, balanceCollections, "", rp, diskType, maxParallelization, 0, applyBalancing, skippedNodes, nil); err != nil { - return fmt.Errorf("re-balance ec shards for collection(s) %v: %w", balanceCollections, err) - } - if err := verifyEcShardsBeforeDelete(commandEnv, volumeIds, diskType, applyBalancing); err != nil { - return fmt.Errorf("verify EC shards before deleting originals: %w", err) - } - // Past verify the EC copy is recoverable; a delete failure below must not - // tear the shards down. - committed = true - fmt.Printf("Deleting original volumes after EC encoding...\n") - if err := doDeleteVolumesWithLocations(commandEnv, volumeIds, volumeLocationsMap, maxParallelization); err != nil { - return fmt.Errorf("delete original volumes after EC encoding: %w", err) - } - fmt.Printf("Successfully completed EC encoding for %d volumes\n", len(volumeIds)) - return nil -} - -// rollbackFailedEcEncode is a best-effort cleanup for an ec.encode that failed -// after marking volumes readonly / generating shards but before the originals -// were deleted. It tears down the EC shards this run produced (so they do not -// survive as orphans until the next encode) and restores the source volumes to -// writable (so a failed and possibly abandoned encode does not strand them -// readonly). Errors are logged, not returned — we are already on the failure -// path. Both operations are idempotent: clearPreexistingEcShards is a no-op when -// no shards were generated, and marking an already-writable volume is a no-op, -// so it is safe even for a failure before the volumes were marked readonly. -func rollbackFailedEcEncode(commandEnv *CommandEnv, writer io.Writer, volumeIds []needle.VolumeId, volumeIdToCollection map[needle.VolumeId]string, volumeLocationsMap map[needle.VolumeId][]wdclient.Location, maxParallelization int) { - fmt.Fprintf(writer, "rolling back failed EC encode for volumes %v...\n", volumeIds) - - // Tear down any EC shards this run produced. A fresh topology snapshot finds - // them wherever generate/balance left them; the teardown is blanket. - if topologyInfo, _, err := collectTopologyInfo(commandEnv, 0); err != nil { - fmt.Fprintf(writer, "rollback: collect topology to clear ec shards: %v\n", err) - } else if _, err := clearPreexistingEcShards(commandEnv, topologyInfo, volumeIds, volumeIdToCollection, maxParallelization); err != nil { - fmt.Fprintf(writer, "rollback: clear ec shards: %v\n", err) - } - - // Restore the source volumes to writable. doEcEncode re-reads the locations - // and marks every replica of that later snapshot readonly, so re-read here - // too: a replica added or moved between the batch's initial snapshot - // (volumeLocationsMap) and doEcEncode's readonly-marking would otherwise be - // left readonly. Fall back to the initial snapshot if the re-read fails. - locations := volumeLocationsMap - if fresh, err := volumeLocations(commandEnv, volumeIds); err != nil { - fmt.Fprintf(writer, "rollback: re-read volume locations (using pre-encode snapshot): %v\n", err) - } else { - locations = fresh - } - ewg := NewErrorWaitGroup(maxParallelization) - for _, vid := range volumeIds { - for _, l := range locations[vid] { - ewg.Add(func() error { - if err := markVolumeReplicaWritable(context.Background(), commandEnv.option.GrpcDialOption, vid, l, true, false); err != nil { - return fmt.Errorf("restore volume %d writable on %s: %w", vid, l.Url, err) - } - return nil - }) - } - } - if err := ewg.Wait(); err != nil { - fmt.Fprintf(writer, "rollback: %v\n", err) - } -} - -func checkEcEncodeCapacity(topologyInfo *master_pb.TopologyInfo, volumeCount int, diskType types.DiskType, collectionForMessage string) error { - // Pre-flight check: verify the target disk type has capacity for EC shards. - // This prevents encoding shards only to fail during rebalance. Reuse the - // caller's topology snapshot instead of issuing another VolumeList to the - // master per batch. - _, totalFreeEcSlots := collectEcVolumeServersByDc(topologyInfo, "", diskType) - - // Each volume needs TotalShardsCount (14) shards distributed. - requiredSlots := volumeCount * erasure_coding.TotalShardsCount - if totalFreeEcSlots < 1 { - if diskType != types.HardDriveType { - tryDiskTypeMessage := "Try passing -diskType=hdd, or omit -diskType to use the default (hdd)" - if collectionForMessage != "" { - tryDiskTypeMessage = fmt.Sprintf("Try:\n ec.encode -collection=%s -diskType=hdd\nOr omit -diskType to use the default (hdd)", collectionForMessage) - } - return fmt.Errorf("no free ec shard slots on disk type '%s'. The target disk type has no capacity.\n"+ - "Your volumes are likely on a different disk type. %s", diskType, tryDiskTypeMessage) - } - return fmt.Errorf("no free ec shard slots. only %d left on disk type '%s'", totalFreeEcSlots, diskType) - } - - if totalFreeEcSlots < requiredSlots { - fmt.Printf("Warning: limited EC shard capacity. Need %d slots for %d volumes, but only %d slots available on disk type '%s'.\n", - requiredSlots, volumeCount, totalFreeEcSlots, diskType) - fmt.Printf("Rebalancing may not achieve optimal distribution.\n") - } - return nil -} - -func volumeLocations(commandEnv *CommandEnv, volumeIds []needle.VolumeId) (map[needle.VolumeId][]wdclient.Location, error) { - res := map[needle.VolumeId][]wdclient.Location{} - for _, vid := range volumeIds { - ls, ok := commandEnv.MasterClient.GetLocationsClone(uint32(vid)) - if !ok { - return nil, fmt.Errorf("volume %d not found", vid) - } - res[vid] = ls - } - - return res, nil -} - -func doEcEncode(commandEnv *CommandEnv, writer io.Writer, volumeIdToCollection map[needle.VolumeId]string, volumeIds []needle.VolumeId, maxParallelization int, topologyInfo *master_pb.TopologyInfo) (skippedNodes map[pb.ServerAddress]struct{}, err error) { - if !commandEnv.isLocked() { - return nil, fmt.Errorf("lock is lost") - } - locations, err := volumeLocations(commandEnv, volumeIds) - if err != nil { - return nil, fmt.Errorf("failed to get volume locations for EC encoding: %w", err) - } - - // Clear EC shards left by a previous failed/partial encode so a retry - // starts clean and never mixes two encode runs. A node skipped here as - // unreachable is excluded from the later balance: it may still hold a stale - // orphan that, paired with a new-generation shard from a balance copy, would - // mix generations on that node. - skippedNodes, err = clearPreexistingEcShards(commandEnv, topologyInfo, volumeIds, volumeIdToCollection, maxParallelization) - if err != nil { - return nil, fmt.Errorf("clear pre-existing ec shards before encoding: %w", err) - } - - // Build a map of (volumeId, serverAddress) -> freeVolumeCount. - // Key by dn.Address so it matches wdclient.Location.Url. In deployments - // where dn.Id is a short name (e.g. Kubernetes StatefulSet pod name) - // while dn.Address is a FQDN:port, keying by dn.Id would never match the - // location Url during the health-check lookup below. - freeVolumeCountMap := make(map[string]int) // key: volumeId-serverAddress - eachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) { - addr := dn.Address - if addr == "" { - addr = dn.Id // older nodes use ip:port as id - } - for _, diskInfo := range dn.DiskInfos { - for _, v := range diskInfo.VolumeInfos { - key := fmt.Sprintf("%d-%s", v.Id, addr) - freeVolumeCountMap[key] = int(diskInfo.FreeVolumeCount) - } - } - }) - - // Filter replicas by free capacity BEFORE marking volumes readonly so that - // a failed health check does not strand volumes in readonly state. - filteredLocations := make(map[needle.VolumeId][]wdclient.Location) - for _, vid := range volumeIds { - var filteredLocs []wdclient.Location - for _, l := range locations[vid] { - key := fmt.Sprintf("%d-%s", vid, l.Url) - if freeCount, found := freeVolumeCountMap[key]; found && freeCount >= 2 { - filteredLocs = append(filteredLocs, l) - } - } - if len(filteredLocs) == 0 { - return nil, fmt.Errorf("no healthy replicas (FreeVolumeCount >= 2) found for volume %d to use as source for EC encoding", vid) - } - filteredLocations[vid] = filteredLocs - } - - // mark volumes as readonly - ewg := NewErrorWaitGroup(maxParallelization) - for _, vid := range volumeIds { - for _, l := range locations[vid] { - ewg.Add(func() error { - if err := markVolumeReplicaWritable(context.Background(), commandEnv.option.GrpcDialOption, vid, l, false, false); err != nil { - return fmt.Errorf("mark volume %d as readonly on %s: %v", vid, l.Url, err) - } - return nil - }) - } - } - if err := ewg.Wait(); err != nil { - return nil, err - } - - // Sync replicas and select the best one for each volume (with highest file count) - // This addresses data inconsistency risk in multi-replica volumes (issue #7797) - // by syncing missing entries between replicas before encoding - bestReplicas := make(map[needle.VolumeId]wdclient.Location) - for _, vid := range volumeIds { - collection := volumeIdToCollection[vid] - - // Sync missing entries between replicas, then select the best one - bestLoc, selectErr := volume_replica.SyncAndSelectBestReplica(commandEnv.option.GrpcDialOption, vid, collection, filteredLocations[vid], "", writer) - if selectErr != nil { - return nil, fmt.Errorf("failed to sync and select replica for volume %d: %v", vid, selectErr) - } - bestReplicas[vid] = bestLoc - } - - // Re-attempt the orphan sweep on the nodes skipped as unreachable, now that - // any node that recovered during readonly-marking and replica sync answers - // again. A node whose teardown now succeeds is clean (and the generation host - // re-wipes its own disks regardless), so it leaves the skipped set and can be - // a balance source/target — otherwise its shards would never distribute off - // it. A node that is still down stays skipped and excluded, preserving the - // leniency for a genuinely-down node; such a node also cannot be the - // generation host below, since VolumeEcShardsGenerate would fail to read .dat. - if err := resweepSkippedNodes(commandEnv, skippedNodes, volumeIds, volumeIdToCollection, maxParallelization); err != nil { - return nil, err - } - - // A selected generation host still in skippedNodes after the re-sweep was - // transport-down when we tried to clean it, so its stale orphans were never - // removed and EcBalance excludes it as both source and target. If it recovers - // just in time for generation, all shards land on a node we can neither clean - // nor balance off — a single point of failure that union-only verification - // still accepts, after which the originals are deleted. Abort instead. - for _, vid := range volumeIds { - genHost := bestReplicas[vid].ServerAddress() - if _, stillSkipped := skippedNodes[genHost]; stillSkipped { - return nil, fmt.Errorf("generate ec shards for volume %d aborted: selected source %s is still skipped after the orphan re-sweep", vid, genHost) - } - } - - // generate ec shards using the best replica for each volume - ewg.Reset() - for _, vid := range volumeIds { - target := bestReplicas[vid] - collection := volumeIdToCollection[vid] - ewg.Add(func() error { - if err := generateEcShards(commandEnv.option.GrpcDialOption, vid, collection, target.ServerAddress()); err != nil { - return fmt.Errorf("generate ec shards for volume %d on %s: %v", vid, target.Url, err) - } - return nil - }) - } - if err := ewg.Wait(); err != nil { - return nil, err - } - - // mount all ec shards for the converted volume - shardIds := erasure_coding.AllShardIds() - - ewg.Reset() - for _, vid := range volumeIds { - target := bestReplicas[vid] - collection := volumeIdToCollection[vid] - ewg.Add(func() error { - if err := mountEcShards(commandEnv.option.GrpcDialOption, collection, vid, target.ServerAddress(), shardIds); err != nil { - return fmt.Errorf("mount ec shards for volume %d on %s: %v", vid, target.Url, err) - } - return nil - }) - } - if err := ewg.Wait(); err != nil { - return nil, err - } - - return skippedNodes, nil -} - -// clearPreexistingEcShards removes EC shards and index files left over from a -// previous (failed or partial) encode of the given volume ids, on every node -// that still reports them, so a fresh encode regenerates from a clean slate. -// Scans all disk types. The normal .dat/.idx — the source of truth for this -// encode — is untouched; only orphaned EC artifacts are deleted. -// -// Returns the set of nodes skipped as unreachable. A skipped node may still hold -// an un-deleted orphan from a prior run; if it recovers it must be kept out of -// this encode's shard distribution, or the balance could install the new -// generation alongside the stale orphan and mix generations on one node. -func clearPreexistingEcShards(commandEnv *CommandEnv, topologyInfo *master_pb.TopologyInfo, volumeIds []needle.VolumeId, volumeIdToCollection map[needle.VolumeId]string, maxParallelization int) (skipped map[pb.ServerAddress]struct{}, err error) { - wanted := make(map[uint32]bool, len(volumeIds)) - for _, vid := range volumeIds { - wanted[uint32(vid)] = true - } - - // Note which (node, vid) pairs the topology already reports EC shards for: - // those are mounted leftovers and cleaning them is required (fatal on - // error). Every other (node, vid) is swept best-effort to catch UNMOUNTED - // orphans left by a failed copy — invisible to the heartbeat, so absent - // here. A node that is down or holds nothing is a harmless no-op; a node - // unreachable now also cannot receive this encode's new generation, so a - // surviving orphan there keeps its old identity and the read guard rejects - // it. Always delete the full shard-id range so a wider custom ratio's - // leftovers are covered too. - reportedKey := func(addr pb.ServerAddress, vid uint32) string { - return string(addr) + "\x00" + strconv.FormatUint(uint64(vid), 10) - } - reported := make(map[string]struct{}) - var nodes []pb.ServerAddress - eachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) { - addr := pb.NewServerAddressFromDataNode(dn) - nodes = append(nodes, addr) - for _, diskInfo := range dn.DiskInfos { - for _, ecInfo := range diskInfo.EcShardInfos { - if wanted[ecInfo.Id] { - reported[reportedKey(addr, ecInfo.Id)] = struct{}{} - } - } - } - }) - - allShardIds := make([]erasure_coding.ShardId, erasure_coding.MaxShardCount) - for i := range allShardIds { - allShardIds[i] = erasure_coding.ShardId(i) - } - - if len(reported) > 0 { - fmt.Printf("clearing stale EC shards reported for %d (node,volume) pair(s) before regenerating...\n", len(reported)) - } - // Nodes skipped as unreachable, accumulated across the concurrent sweep tasks. - skipped = make(map[pb.ServerAddress]struct{}) - var skippedMu sync.Mutex - ewg := NewErrorWaitGroup(maxParallelization) - for _, addr := range nodes { - for _, vid := range volumeIds { - fatal := false - if _, ok := reported[reportedKey(addr, uint32(vid))]; ok { - fatal = true - } - collection := volumeIdToCollection[vid] - ewg.Add(func() error { - if err := unmountAndDeleteEcShardsQuiet(commandEnv.option.GrpcDialOption, collection, vid, addr, allShardIds); err != nil { - // Surface a reachable node whose delete genuinely failed (its orphan would - // be re-stamped by a later copy installing the new .vif). A missing - // full_teardown ack from a reachable pre-upgrade node is fatal too: it may - // still hold an orphan a later copy would re-stamp into the new generation. - // Stay best-effort only for a node that is truly unreachable: codes.Unavailable - // alone is ambiguous — a genuinely-down node and a reachable Rust volume - // server in maintenance mode both return it (a Go server returns Unknown for - // maintenance, already fatal above). Confirm with a non-maintenance-gated Ping - // before skipping; skip only when the Ping itself transport-failed (nodeDown). - // A reachable maintenance node (nodeUp) CAN receive this generation, and an - // inconclusive Ping (nodeLivenessUnknown, e.g. a pre-Ping server returning - // Unimplemented — which means the node is up) does not prove the node is down, - // so both stay fatal rather than silently leaving a stale EC generation. - if fatal || errors.Is(err, errFullTeardownNotAcked) || !isNodeUnreachable(err) || - classifyNodeLiveness(pingVolumeServer(commandEnv.option.GrpcDialOption, addr)) != nodeDown { - return fmt.Errorf("clear stale ec shards for volume %d on %s: %w", vid, addr, err) - } - glog.V(1).Infof("orphan sweep: volume %d on %s skipped (unreachable): %v", vid, addr, err) - skippedMu.Lock() - skipped[addr] = struct{}{} - skippedMu.Unlock() - } - return nil - }) - } - } - if err := ewg.Wait(); err != nil { - return nil, err - } - return skipped, nil -} - -// resweepSkippedNodes re-attempts the orphan teardown on the nodes that the -// initial sweep skipped as unreachable, just before shard generation. A node -// that recovered in the meantime — and is therefore eligible to host this -// encode's generation — has its teardown retried; if it now fully succeeds it is -// removed from skipped so the rebalance can use it as a source and move its -// shards off, instead of stranding all shards on the single generation host and -// collapsing fault tolerance. A node still transport-down stays skipped (the -// same leniency the initial sweep grants), and a node that came back reachable -// but whose delete genuinely failed is fatal, exactly as in the initial sweep, -// so a stale generation is never silently left behind. Mutates skipped in place. -func resweepSkippedNodes(commandEnv *CommandEnv, skipped map[pb.ServerAddress]struct{}, volumeIds []needle.VolumeId, volumeIdToCollection map[needle.VolumeId]string, maxParallelization int) error { - if len(skipped) == 0 { - return nil - } - - allShardIds := make([]erasure_coding.ShardId, erasure_coding.MaxShardCount) - for i := range allShardIds { - allShardIds[i] = erasure_coding.ShardId(i) - } - - addrs := make([]pb.ServerAddress, 0, len(skipped)) - for addr := range skipped { - addrs = append(addrs, addr) - } - - fmt.Printf("re-checking %d node(s) skipped by the orphan sweep before generating shards...\n", len(addrs)) - - // A node still down on every retried vid stays skipped; one that fully - // succeeds is un-skipped. Track per-node whether any retry still failed - // (down) so a node whose state is mixed across vids never gets un-skipped. - stillDown := make(map[pb.ServerAddress]struct{}) - var mu sync.Mutex - ewg := NewErrorWaitGroup(maxParallelization) - for _, addr := range addrs { - for _, vid := range volumeIds { - collection := volumeIdToCollection[vid] - ewg.Add(func() error { - if err := unmountAndDeleteEcShardsQuiet(commandEnv.option.GrpcDialOption, collection, vid, addr, allShardIds); err != nil { - // Same decision as the initial sweep: a reachable node whose delete - // genuinely failed (or did not ack a full teardown, or whose liveness is - // inconclusive) is fatal, since it could hold an orphan a later copy - // re-stamps into this generation. Only a node still transport-down stays - // skipped. - if errors.Is(err, errFullTeardownNotAcked) || !isNodeUnreachable(err) || - classifyNodeLiveness(pingVolumeServer(commandEnv.option.GrpcDialOption, addr)) != nodeDown { - return fmt.Errorf("re-clear stale ec shards for volume %d on %s: %w", vid, addr, err) - } - glog.V(1).Infof("orphan re-sweep: volume %d on %s still skipped (unreachable): %v", vid, addr, err) - mu.Lock() - stillDown[addr] = struct{}{} - mu.Unlock() - } - return nil - }) - } - } - if err := ewg.Wait(); err != nil { - return err - } - - for _, addr := range addrs { - if _, down := stillDown[addr]; !down { - delete(skipped, addr) - glog.V(0).Infof("orphan re-sweep: node %s recovered and was cleaned; it will participate in the EC rebalance", addr) - } - } - return nil -} - -// isNodeUnreachable reports whether err means the volume server could not be -// reached at all, as opposed to an RPC that reached the node and failed. Only an -// unreachable node is safe to skip in the orphan sweep. A dead peer surfaces as -// a gRPC codes.Unavailable from the RPC (the dial is lazy, so it never fails at -// connect time); any non-status error reached node logic and is treated as -// reachable, so the sweep stays fatal rather than silently leaving stale state. -func isNodeUnreachable(err error) bool { - if err == nil { - return false - } - st, ok := status.FromError(err) - return ok && st.Code() == codes.Unavailable -} - -// nodeLiveness is the tri-state result of a pingVolumeServer probe. -type nodeLiveness int - -const ( - // nodeUp: Ping succeeded — the node is reachable (e.g. a Rust volume server - // in maintenance mode that fails the delete but answers Ping). - nodeUp nodeLiveness = iota - // nodeDown: Ping itself transport-failed with codes.Unavailable — the node is - // confirmed unreachable. The only state the orphan sweep may skip. - nodeDown - // nodeLivenessUnknown: Ping reached failing logic with any non-Unavailable - // code (Internal, ResourceExhausted, Unimplemented from a pre-Ping server, …) - // or a non-status error. This does NOT prove the node is down, so it is fatal. - nodeLivenessUnknown -) - -// classifyNodeLiveness maps a pingVolumeServer error into the tri-state. A nil -// error is nodeUp, a transport codes.Unavailable is nodeDown (reusing the same -// rule as isNodeUnreachable), and every other Ping failure is nodeLivenessUnknown. -func classifyNodeLiveness(pingErr error) nodeLiveness { - if pingErr == nil { - return nodeUp - } - if isNodeUnreachable(pingErr) { - return nodeDown - } - return nodeLivenessUnknown -} - -// collectEcShardBitsByNode returns, for one volume, the EC shard bits each node -// reports, unioned across all its disk types. Freshly generated shards sit on -// the disk that held the source .dat, which may differ from the balance target -// disk type, so visibility questions ("has the master heard about these shards -// at all?") must not filter by disk type. Only the newest encode generation -// (largest EncodeTsNs) counts: an orphaned older generation — a failed earlier -// encode on a node the pre-encode sweep could not reach but the master still -// hears from — must neither satisfy the registration wait nor pose as a second -// holder in the clump check. Entries without a timestamp form the legacy -// generation zero, so they only count when no stamped generation exists; -// dropping them otherwise errs toward keeping the source volume. -func collectEcShardBitsByNode(topoInfo *master_pb.TopologyInfo, vid needle.VolumeId) map[pb.ServerAddress]erasure_coding.ShardBits { - type shardEntry struct { - addr pb.ServerAddress - ts int64 - bits erasure_coding.ShardBits - } - var entries []shardEntry - var newestTs int64 - eachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) { - for _, diskInfo := range dn.DiskInfos { - for _, ecInfo := range diskInfo.EcShardInfos { - if ecInfo.Id != uint32(vid) { - continue - } - entries = append(entries, shardEntry{pb.NewServerAddressFromDataNode(dn), ecInfo.EncodeTsNs, erasure_coding.ShardBits(ecInfo.EcIndexBits)}) - if ecInfo.EncodeTsNs > newestTs { - newestTs = ecInfo.EncodeTsNs - } - } - } - }) - res := make(map[pb.ServerAddress]erasure_coding.ShardBits) - for _, e := range entries { - if e.ts == newestTs { - res[e.addr] |= e.bits - } - } - return res -} - -// waitForEcShardsToRegister polls the master topology until every given volume -// reports a full EC shard set. Mounting shards notifies the master -// asynchronously (mount -> NewEcShardsChan -> delta heartbeat), so a topology -// snapshot taken right after doEcEncode can predate the mounts. Failing after -// the retries keeps the source volumes, and a re-run of ec.encode starts clean. -func waitForEcShardsToRegister(commandEnv *CommandEnv, volumeIds []needle.VolumeId) error { - const maxAttempts = 10 - const retryInterval = 2 * time.Second - - var lastMissing []string - for attempt := 0; attempt < maxAttempts; attempt++ { - if attempt > 0 { - time.Sleep(retryInterval) - } - topoInfo, _, err := collectTopologyInfo(commandEnv, 0) - if err != nil { - return fmt.Errorf("fetch topology while waiting for ec shards to register: %w", err) - } - lastMissing = lastMissing[:0] - for _, vid := range volumeIds { - var union erasure_coding.ShardBits - for _, bits := range collectEcShardBitsByNode(topoInfo, vid) { - union |= bits - } - if union.Count() < erasure_coding.TotalShardsCount { - lastMissing = append(lastMissing, fmt.Sprintf("volume %d: %d/%d shards", vid, union.Count(), erasure_coding.TotalShardsCount)) - } - } - if len(lastMissing) == 0 { - return nil - } - glog.V(0).Infof("waiting for newly generated ec shards to register with the master (attempt %d/%d): %v", - attempt+1, maxAttempts, lastMissing) - } - return fmt.Errorf("newly generated ec shards did not register with the master after %d attempts: %v", maxAttempts, lastMissing) -} - -// ecShardsClumpedOnOneNode reports whether every EC shard the master sees for -// vid sits on a single node while at least one other node has free EC shard -// slots on the target disk type — i.e. the preceding rebalance could have -// spread the shards but did not. Zero visible shards is not a clump; the -// recoverability check owns that case. -func ecShardsClumpedOnOneNode(topoInfo *master_pb.TopologyInfo, vid needle.VolumeId, diskType types.DiskType) (holder pb.ServerAddress, clumped bool) { - byNode := collectEcShardBitsByNode(topoInfo, vid) - if len(byNode) != 1 { - return "", false - } - for addr := range byNode { - holder = addr - } - ecNodes, _ := collectEcVolumeServersByDc(topoInfo, "", diskType) - for _, en := range ecNodes { - if pb.NewServerAddressFromDataNode(en.info) == holder { - continue - } - if en.freeEcSlot > 0 { - return holder, true - } - } - return "", false -} - -// ecShardSummaryByNode says where a volume's shards are, one entry per node, -// sorted so the message is stable. It names the ids and not just the count: a -// set holding shards 0-9 and one holding 4-13 are both "10 shards", and which -// ones survived is what says whether the set is recoverable and from where. -func ecShardSummaryByNode(byNode map[pb.ServerAddress]erasure_coding.ShardBits) []string { - summary := make([]string, 0, len(byNode)) - for node, bits := range byNode { - summary = append(summary, fmt.Sprintf("%s=%d shards %v", node, bits.Count(), slices.Collect(bits.All()))) - } - sort.Strings(summary) - return summary -} - -func verifyEcShardsBeforeDelete(commandEnv *CommandEnv, volumeIds []needle.VolumeId, diskType types.DiskType, expectSpread bool) error { - // Shard relocations from the preceding EC balance reach the master via - // volume-server heartbeats, so freshly distributed shards may not all be - // visible in the master topology immediately. Poll a few times before - // concluding the shard set is incomplete, so a heartbeat-propagation lag is - // not mistaken for missing data. After the retries: a volume below the - // recoverable threshold (dataShards) aborts the deletion; a recoverable - // but degraded set proceeds with a warning, since the missing shards can - // be rebuilt from the survivors while keeping the source next to live - // shards is the more dangerous mixed state. When expectSpread is set (the - // rebalance ran in apply mode), a volume whose shards all still sit on one - // node while another node has free slots also aborts the deletion: losing - // that node would lose the volume, so the original is the safer copy. - const maxAttempts = 10 - const retryInterval = 2 * time.Second - - var lastErr error - var lastDegraded []string - var lastClumped []string - for attempt := 0; attempt < maxAttempts; attempt++ { - topoInfo, _, err := collectTopologyInfo(commandEnv, 0) - if err != nil { - return fmt.Errorf("fetch topology for shard verification: %w", err) - } - - lastErr = nil - lastDegraded = lastDegraded[:0] - lastClumped = lastClumped[:0] - for _, vid := range volumeIds { - // Count the shards wherever they landed, as waitForEcShardsToRegister - // above already does. generateEcShards writes them beside the source - // volume, so encoding a volume that lives on a non-default medium - // puts them on that medium while -diskType still says hdd. Counting - // only the -diskType bucket then reports a complete set as entirely - // missing and aborts an encode that in fact succeeded, leaving the - // volume as both a .dat and a full set of shards. - byNode := collectEcShardBitsByNode(topoInfo, vid) - - var union erasure_coding.ShardBits - for _, bits := range byNode { - union |= bits - } - - totalShards := erasure_coding.TotalShardsCount - degraded, err := erasure_coding.RequireRecoverableShardSet(uint32(vid), union, erasure_coding.DataShardsCount, totalShards) - if err != nil { - lastErr = fmt.Errorf("volume %d: %w (observed: %v)", vid, err, ecShardSummaryByNode(byNode)) - break - } - if expectSpread { - if holder, clumped := ecShardsClumpedOnOneNode(topoInfo, vid, diskType); clumped { - lastClumped = append(lastClumped, fmt.Sprintf("volume %d: all shards on %s", vid, holder)) - continue - } - } - if degraded { - lastDegraded = append(lastDegraded, fmt.Sprintf("volume %d: %d/%d shards", vid, union.Count(), totalShards)) - continue - } - - glog.V(0).Infof("EC shard verification ok for volume %d: %d/%d shards present across %d nodes", - vid, union.Count(), totalShards, len(byNode)) - } - - if lastErr == nil && len(lastDegraded) == 0 && len(lastClumped) == 0 { - return nil - } - if attempt < maxAttempts-1 { - glog.V(0).Infof("EC shard verification incomplete (attempt %d/%d), waiting for shard locations to propagate: %v %v %v", - attempt+1, maxAttempts, lastErr, lastDegraded, lastClumped) - time.Sleep(retryInterval) - } - } - - if lastErr != nil { - glog.Errorf("EC shard verification failed after %d attempts: %v", maxAttempts, lastErr) - return lastErr - } - if len(lastClumped) > 0 { - return fmt.Errorf("EC shards still sit on a single node after rebalance even though other nodes have free slots (%v); keeping the original volumes. Run ec.balance -apply, verify the spread, then delete the originals or re-run ec.encode", lastClumped) - } - glog.Warningf("EC shard set incomplete but recoverable after %d attempts, proceeding with source deletion (rebuild missing shards with ec.rebuild): %v", - maxAttempts, lastDegraded) - return nil -} - -// doDeleteVolumesWithLocations deletes volumes using pre-collected location information -// This avoids race conditions where master metadata is updated after EC encoding -func doDeleteVolumesWithLocations(commandEnv *CommandEnv, volumeIds []needle.VolumeId, volumeLocationsMap map[needle.VolumeId][]wdclient.Location, maxParallelization int) error { - if !commandEnv.isLocked() { - return fmt.Errorf("lock is lost") - } - - ewg := NewErrorWaitGroup(maxParallelization) - for _, vid := range volumeIds { - locations, found := volumeLocationsMap[vid] - if !found { - fmt.Printf("warning: no locations found for volume %d, skipping deletion\n", vid) - continue - } - - for _, l := range locations { - ewg.Add(func() error { - if err := deleteVolume(context.Background(), commandEnv.option.GrpcDialOption, vid, l.ServerAddress(), false, false); err != nil { - return fmt.Errorf("deleteVolume %s volume %d: %v", l.Url, vid, err) - } - fmt.Printf("deleted volume %d from %s\n", vid, l.Url) - return nil - }) - } - } - if err := ewg.Wait(); err != nil { - return err - } - - return nil -} - -func generateEcShards(grpcDialOption grpc.DialOption, volumeId needle.VolumeId, collection string, sourceVolumeServer pb.ServerAddress) error { - - fmt.Printf("generateEcShards %d (collection %q) on %s ...\n", volumeId, collection, sourceVolumeServer) - - err := operation.WithVolumeServerClient(false, sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { - _, genErr := volumeServerClient.VolumeEcShardsGenerate(context.Background(), &volume_server_pb.VolumeEcShardsGenerateRequest{ - VolumeId: uint32(volumeId), - Collection: collection, - }) - return genErr - }) - - return err - -} - func collectVolumeIdsForEcEncode(commandEnv *CommandEnv, collectionPattern string, sourceDiskType *types.DiskType, fullPercentage float64, quietPeriod time.Duration, verbose bool) (vids []needle.VolumeId, matchedCollections []string, err error) { // compile regex pattern for collection matching collectionRegex, err := compileCollectionPattern(collectionPattern) @@ -1003,151 +202,6 @@ func collectVolumeIdsForEcEncode(commandEnv *CommandEnv, collectionPattern strin fmt.Printf("collect volumes with collection pattern '%s', quiet for: %d seconds and %.1f%% full\n", collectionPattern, quietSeconds, fullPercentage) - vids, matchedCollections = selectVolumeIdsFromTopology(topologyInfo, volumeSizeLimitMb, collectionRegex, sourceDiskType, quietSeconds, nowUnixSeconds, fullPercentage, verbose) - return -} - -func selectVolumeIdsFromTopology(topologyInfo *master_pb.TopologyInfo, volumeSizeLimitMb uint64, collectionRegex *regexp.Regexp, sourceDiskType *types.DiskType, quietSeconds int64, nowUnixSeconds int64, fullPercentage float64, verbose bool) (vids []needle.VolumeId, matchedCollections []string) { - // Statistics for verbose mode - var ( - totalVolumes int - remoteVolumes int - wrongCollection int - wrongDiskType int - tooRecent int - tooSmall int - noFreeDisk int - ) - - vidMap := make(map[uint32]bool) - collectionSet := make(map[string]bool) - eachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) { - for _, diskInfo := range dn.DiskInfos { - for _, v := range diskInfo.VolumeInfos { - totalVolumes++ - - // ignore remote volumes - if v.RemoteStorageName != "" { - remoteVolumes++ - if verbose { - fmt.Printf("skip volume %d on %s: remote volume (storage: %s)\n", - v.Id, dn.Id, v.RemoteStorageName) - } - continue - } - - // check collection against regex pattern - if !collectionRegex.MatchString(v.Collection) { - wrongCollection++ - if verbose { - fmt.Printf("skip volume %d on %s: collection doesn't match pattern (pattern: %s, actual: %s)\n", - v.Id, dn.Id, collectionRegex.String(), v.Collection) - } - continue - } - - // track matched collection - collectionSet[v.Collection] = true - - // check disk type - if sourceDiskType != nil && types.ToDiskType(v.DiskType) != *sourceDiskType { - wrongDiskType++ - if verbose { - fmt.Printf("skip volume %d on %s: wrong disk type (expected: %s, actual: %s)\n", - v.Id, dn.Id, sourceDiskType.ReadableString(), types.ToDiskType(v.DiskType).ReadableString()) - } - continue - } - - // check quiet period - if v.ModifiedAtSecond+quietSeconds >= nowUnixSeconds { - tooRecent++ - if verbose { - fmt.Printf("skip volume %d on %s: too recently modified (last modified: %d seconds ago, required: %d seconds)\n", - v.Id, dn.Id, nowUnixSeconds-v.ModifiedAtSecond, quietSeconds) - } - continue - } - - // check size - sizeThreshold := fullPercentage / 100 * float64(volumeSizeLimitMb) * 1024 * 1024 - if float64(v.Size) <= sizeThreshold { - tooSmall++ - if verbose { - fmt.Printf("skip volume %d on %s: too small (size: %.1f MB, threshold: %.1f MB, %.1f%% full)\n", - v.Id, dn.Id, float64(v.Size)/(1024*1024), sizeThreshold/(1024*1024), - float64(v.Size)*100/(float64(volumeSizeLimitMb)*1024*1024)) - } - continue - } - - // check free disk space - if diskInfo.FreeVolumeCount < 2 { - glog.V(0).Infof("replica %s %d on %s has no free disk", v.Collection, v.Id, dn.Id) - if verbose { - fmt.Printf("skip replica of volume %d on %s: insufficient free disk space (free volumes: %d, required: 2)\n", - v.Id, dn.Id, diskInfo.FreeVolumeCount) - } - if _, found := vidMap[v.Id]; !found { - vidMap[v.Id] = false - } - } else { - if verbose { - fmt.Printf("selected volume %d on %s: size %.1f MB (%.1f%% full), last modified %d seconds ago, free volumes: %d\n", - v.Id, dn.Id, float64(v.Size)/(1024*1024), - float64(v.Size)*100/(float64(volumeSizeLimitMb)*1024*1024), - nowUnixSeconds-v.ModifiedAtSecond, diskInfo.FreeVolumeCount) - } - vidMap[v.Id] = true - } - } - } - }) - - for vid, good := range vidMap { - if good { - vids = append(vids, needle.VolumeId(vid)) - } else { - noFreeDisk++ - } - } - - // Convert collection set to slice - for collection := range collectionSet { - matchedCollections = append(matchedCollections, collection) - } - sort.Strings(matchedCollections) - - // Print summary statistics in verbose mode or when no volumes selected - if verbose || len(vids) == 0 { - fmt.Printf("\nVolume selection summary:\n") - fmt.Printf(" Total volumes examined: %d\n", totalVolumes) - fmt.Printf(" Selected for encoding: %d\n", len(vids)) - fmt.Printf(" Collections matched: %v\n", matchedCollections) - - if totalVolumes > 0 { - fmt.Printf("\nReasons for exclusion:\n") - if remoteVolumes > 0 { - fmt.Printf(" Remote volumes: %d\n", remoteVolumes) - } - if wrongCollection > 0 { - fmt.Printf(" Collection doesn't match pattern: %d\n", wrongCollection) - } - if wrongDiskType > 0 { - fmt.Printf(" Wrong disk type: %d\n", wrongDiskType) - } - if tooRecent > 0 { - fmt.Printf(" Too recently modified: %d\n", tooRecent) - } - if tooSmall > 0 { - fmt.Printf(" Too small (< %.1f%% full): %d\n", fullPercentage, tooSmall) - } - if noFreeDisk > 0 { - fmt.Printf(" Insufficient free disk space: %d\n", noFreeDisk) - } - } - fmt.Println() - } - + vids, matchedCollections = ec.SelectVolumeIdsFromTopology(topologyInfo, volumeSizeLimitMb, collectionRegex, sourceDiskType, quietSeconds, nowUnixSeconds, fullPercentage, verbose) return } diff --git a/weed/shell/command_ec_rebuild.go b/weed/shell/command_ec_rebuild.go index a18bcddc6..1237e1a16 100644 --- a/weed/shell/command_ec_rebuild.go +++ b/weed/shell/command_ec_rebuild.go @@ -1,19 +1,13 @@ package shell import ( - "context" "flag" "fmt" "io" - "slices" "strconv" "strings" - "sync" - "github.com/seaweedfs/seaweedfs/weed/operation" - "github.com/seaweedfs/seaweedfs/weed/pb" - "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" - "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" + "github.com/seaweedfs/seaweedfs/weed/ec" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/types" ) @@ -22,19 +16,6 @@ func init() { Commands = append(Commands, &commandEcRebuild{}) } -type ecRebuilder struct { - commandEnv *CommandEnv - ecNodes []*EcNode - writer io.Writer - applyChanges bool - collections []string - volumeIds []needle.VolumeId - diskType types.DiskType - - ewg *ErrorWaitGroup - ecNodesMu sync.Mutex -} - type commandEcRebuild struct { } @@ -142,401 +123,5 @@ func (c *commandEcRebuild) Do(args []string, commandEnv *CommandEnv, writer io.W } } - erb := &ecRebuilder{ - commandEnv: commandEnv, - ecNodes: allEcNodes, - writer: writer, - applyChanges: *applyChanges, - collections: collections, - volumeIds: volumeIds, - diskType: diskType, - - ewg: NewErrorWaitGroup(*maxParallelization), - } - - // Recover shards left unmounted by a missing .ecx index before planning: such - // shards never register with the master, so the rebuild below would treat the - // volume as short or unrepairable even though its data is intact (issue #10104). - erb.recoverMissingIndexes() - - fmt.Printf("rebuildEcVolumes for %d collection(s)\n", len(collections)) - for _, c := range collections { - erb.rebuildEcVolumes(c) - } - - return erb.ewg.Wait() -} - -func (erb *ecRebuilder) write(format string, a ...any) { - fmt.Fprintf(erb.writer, format, a...) -} - -func (erb *ecRebuilder) isLocked() bool { - return erb.commandEnv.isLocked() -} - -// matchesVolumeId verifies whether the rebuilder is targeted at a given volume ID. -func (erb *ecRebuilder) matchesVolumeId(vid needle.VolumeId) bool { - if len(erb.volumeIds) == 0 { - return true - } - - return slices.Contains(erb.volumeIds, vid) -} - -// countLocalShards returns the number of shards already present locally on the node for the given volume. -// Unions across all of the node's disks, like prepareDataToRecover, so slot -// accounting matches what the rebuild will actually treat as local. -func (erb *ecRebuilder) countLocalShards(node *EcNode, collection string, volumeId needle.VolumeId) int { - localShardsInfo := erasure_coding.NewShardsInfo() - for _, diskInfo := range node.info.DiskInfos { - for _, ecShardInfo := range diskInfo.EcShardInfos { - if ecShardInfo.Collection == collection && needle.VolumeId(ecShardInfo.Id) == volumeId { - localShardsInfo.Add(erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(ecShardInfo)) - } - } - } - return localShardsInfo.Count() -} - -// selectAndReserveRebuilder atomically selects a rebuilder node with sufficient free slots -// and reserves slots only for the non-local shards that need to be copied/generated. -func (erb *ecRebuilder) selectAndReserveRebuilder(collection string, volumeId needle.VolumeId) (*EcNode, int, error) { - erb.ecNodesMu.Lock() - defer erb.ecNodesMu.Unlock() - - if len(erb.ecNodes) == 0 { - return nil, 0, fmt.Errorf("no ec nodes available") - } - - // Find the node with the most free slots, considering local shards - var bestNode *EcNode - var bestSlotsNeeded int - var maxAvailableSlots int - var minSlotsNeeded int = erasure_coding.TotalShardsCount // Start with maximum possible - for _, node := range erb.ecNodes { - localShards := erb.countLocalShards(node, collection, volumeId) - slotsNeeded := erasure_coding.TotalShardsCount - localShards - if slotsNeeded < 0 { - slotsNeeded = 0 - } - - if node.freeEcSlot > maxAvailableSlots { - maxAvailableSlots = node.freeEcSlot - } - - if slotsNeeded < minSlotsNeeded { - minSlotsNeeded = slotsNeeded - } - - if node.freeEcSlot >= slotsNeeded { - if bestNode == nil || node.freeEcSlot > bestNode.freeEcSlot { - bestNode = node - bestSlotsNeeded = slotsNeeded - } - } - } - - if bestNode == nil { - return nil, 0, fmt.Errorf("no node has sufficient free slots for volume %d (need at least %d slots, max available: %d)", - volumeId, minSlotsNeeded, maxAvailableSlots) - } - - // Reserve slots only for non-local shards - bestNode.freeEcSlot -= bestSlotsNeeded - - return bestNode, bestSlotsNeeded, nil -} - -// releaseRebuilder releases the reserved slots back to the rebuilder node. -func (erb *ecRebuilder) releaseRebuilder(node *EcNode, slotsToRelease int) { - erb.ecNodesMu.Lock() - defer erb.ecNodesMu.Unlock() - - // Release slots by incrementing the free slot count - node.freeEcSlot += slotsToRelease -} - -func (erb *ecRebuilder) rebuildEcVolumes(collection string) { - fmt.Printf("rebuildEcVolumes for %q\n", collection) - - // collect vid => each shard locations, similar to ecShardMap in topology.go - ecShardMap := make(EcShardMap) - erb.ecNodesMu.Lock() - for _, ecNode := range erb.ecNodes { - ecShardMap.registerEcNode(ecNode, collection) - } - erb.ecNodesMu.Unlock() - - for vid, locations := range ecShardMap { - if !erb.matchesVolumeId(vid) { - continue - } - shardCount := locations.shardCount() - if shardCount == erasure_coding.TotalShardsCount { - continue - } - if shardCount < erasure_coding.DataShardsCount { - erb.write("ec volume %d is unrepairable with %d shards (need %d), skipping\n", vid, shardCount, erasure_coding.DataShardsCount) - continue - } - - // Capture variables for closure - vid := vid - locations := locations - - erb.ewg.Add(func() error { - // Select rebuilder and reserve slots atomically per volume - rebuilder, slotsToReserve, err := erb.selectAndReserveRebuilder(collection, vid) - if err != nil { - return fmt.Errorf("failed to select rebuilder for volume %d: %v", vid, err) - } - defer erb.releaseRebuilder(rebuilder, slotsToReserve) - - return erb.rebuildOneEcVolume(collection, vid, locations, rebuilder) - }) - } -} - -// recoverMissingIndexes asks every ec node to fetch a missing .ecx index from a -// peer and mount the on-disk shards it could not load on its own. Shards -// orphaned this way (index only on another server) are absent from the master -// topology, so without this pass ec.rebuild would regenerate or give up on -// shards whose data is actually present — and a volume whose every holder lacks -// the index would not appear in the topology at all. Each node therefore -// recovers all of its on-disk orphans (volume_id 0); an explicit -volumeIds -// list narrows that to the requested volumes. On apply it refreshes the topology -// so the rebuild planning sees the recovered shards (issue #10104). -func (erb *ecRebuilder) recoverMissingIndexes() { - erb.ecNodesMu.Lock() - nodes := append([]*EcNode(nil), erb.ecNodes...) - erb.ecNodesMu.Unlock() - if len(nodes) == 0 { - return - } - - // volume_id 0 means "recover every orphan on the node"; a -volumeIds list - // narrows recovery to those ids (each scanned across collections server-side). - vids := erb.volumeIds - if len(vids) == 0 { - vids = []needle.VolumeId{0} - } - - if !erb.applyChanges { - erb.write("would ask %d ec node(s) to recover EC shards left unmounted by a missing .ecx index\n", len(nodes)) - return - } - - for _, node := range nodes { - for _, vid := range vids { - err := operation.WithVolumeServerClient(false, pb.NewServerAddressFromDataNode(node.info), erb.commandEnv.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error { - _, mountErr := client.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{ - VolumeId: uint32(vid), - RecoverMissingIndex: true, - }) - return mountErr - }) - if err != nil { - erb.write("%s recover missing index (volume %d): %v\n", node.info.Id, vid, err) - } - } - } - - // Refresh topology so the rebuild planning sees shards the recovery registered. - refreshed, _, err := collectEcNodes(erb.commandEnv, erb.diskType) - if err != nil { - erb.write("failed to refresh ec nodes after index recovery: %v\n", err) - return - } - erb.ecNodesMu.Lock() - erb.ecNodes = refreshed - erb.ecNodesMu.Unlock() -} - -func (erb *ecRebuilder) rebuildOneEcVolume(collection string, volumeId needle.VolumeId, locations EcShardLocations, rebuilder *EcNode) error { - if !erb.isLocked() { - return fmt.Errorf("lock is lost") - } - - fmt.Printf("rebuildOneEcVolume %s %d\n", collection, volumeId) - - // collect shard files to rebuilder local disk - var generatedShardIds []erasure_coding.ShardId - copiedShardIds, _, err := erb.prepareDataToRecover(rebuilder, collection, volumeId, locations) - defer func() { - // Clean up the working copies this run actually made, even when the - // recoverability gate failed after some copies already succeeded: - // they are temp files on the rebuilder nothing else reclaims. Dry-run - // copies nothing (copiedShardIds is empty), so this issues no delete - // RPC. Use a local error so a cleanup failure cannot mask the return. - if !erb.applyChanges || len(copiedShardIds) == 0 { - return - } - if derr := sourceServerDeleteEcShards(erb.commandEnv.option.GrpcDialOption, collection, volumeId, pb.NewServerAddressFromDataNode(rebuilder.info), copiedShardIds); derr != nil { - erb.write("%s delete copied ec shards %s %d.%v: %v\n", rebuilder.info.Id, collection, volumeId, copiedShardIds, derr) - } - }() - if err != nil { - return err - } - - if !erb.applyChanges { - return nil - } - - // generate ec shards, and maybe ecx file - generatedShardIds, err = erb.generateMissingShards(collection, volumeId, pb.NewServerAddressFromDataNode(rebuilder.info)) - if err != nil { - return err - } - - // mount the generated shards - err = mountEcShards(erb.commandEnv.option.GrpcDialOption, collection, volumeId, pb.NewServerAddressFromDataNode(rebuilder.info), generatedShardIds) - if err != nil { - return err - } - - // ensure ECNode updates are atomic - erb.ecNodesMu.Lock() - defer erb.ecNodesMu.Unlock() - rebuilder.addEcVolumeShards(volumeId, collection, generatedShardIds, erb.diskType) - - return nil -} - -func (erb *ecRebuilder) generateMissingShards(collection string, volumeId needle.VolumeId, sourceLocation pb.ServerAddress) (rebuiltShardIds []erasure_coding.ShardId, err error) { - - err = operation.WithVolumeServerClient(false, sourceLocation, erb.commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { - resp, rebuildErr := volumeServerClient.VolumeEcShardsRebuild(context.Background(), &volume_server_pb.VolumeEcShardsRebuildRequest{ - VolumeId: uint32(volumeId), - Collection: collection, - }) - if rebuildErr == nil { - rebuiltShardIds = erasure_coding.Uint32ToShardIds(resp.RebuiltShardIds) - } - return rebuildErr - }) - return -} - -func (erb *ecRebuilder) prepareDataToRecover(rebuilder *EcNode, collection string, volumeId needle.VolumeId, locations EcShardLocations) (copiedShardIds []erasure_coding.ShardId, localShardIds []erasure_coding.ShardId, err error) { - - needEcxFile := true - localShardsInfo := erasure_coding.NewShardsInfo() - for _, diskInfo := range rebuilder.info.DiskInfos { - for _, ecShardInfo := range diskInfo.EcShardInfos { - if ecShardInfo.Collection == collection && needle.VolumeId(ecShardInfo.Id) == volumeId { - needEcxFile = false - // Union across disks: the rebuilder may hold this volume's - // shards on more than one disk. Overwriting per-disk would - // make a shard on a non-last disk look remote and get copied - // onto itself (O_TRUNC) and then node-wide deleted. - localShardsInfo.Add(erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(ecShardInfo)) - } - } - } - - targetShardCount := erasure_coding.TotalShardsCount - for i := erasure_coding.TotalShardsCount; i < len(locations); i++ { - if len(locations[i]) > 0 { - targetShardCount = i + 1 - } - } - - // recoverableRemoteShards counts remote shards that can contribute to the - // rebuild. Dry-run counts the plan; apply mode counts only successful copies. - recoverableRemoteShards := 0 - for i := 0; i < targetShardCount; i++ { - ecNodes := locations[i] - shardId := erasure_coding.ShardId(i) - if len(ecNodes) == 0 { - erb.write("missing shard %d.%d\n", volumeId, shardId) - continue - } - - if localShardsInfo.Has(shardId) { - localShardIds = append(localShardIds, shardId) - erb.write("use existing shard %d.%d\n", volumeId, shardId) - continue - } - - // The rebuilder is itself the only listed holder: never copy a shard - // onto itself (the in-place O_TRUNC would destroy it) nor schedule it - // for the post-rebuild delete. Treat it as already local. - if ecNodes[0].info.Id == rebuilder.info.Id { - localShardIds = append(localShardIds, shardId) - erb.write("use existing shard %d.%d (already on rebuilder)\n", volumeId, shardId) - continue - } - - if !erb.applyChanges { - recoverableRemoteShards++ - erb.write("would copy %d.%d from %s\n", volumeId, shardId, ecNodes[0].info.Id) - continue - } - - copyErr := operation.WithVolumeServerClient(false, pb.NewServerAddressFromDataNode(rebuilder.info), erb.commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { - _, copyErr := volumeServerClient.VolumeEcShardsCopy(context.Background(), &volume_server_pb.VolumeEcShardsCopyRequest{ - VolumeId: uint32(volumeId), - Collection: collection, - ShardIds: []uint32{uint32(shardId)}, - CopyEcxFile: needEcxFile, - CopyEcjFile: true, - CopyVifFile: needEcxFile, - SourceDataNode: string(pb.NewServerAddressFromDataNode(ecNodes[0].info)), - }) - return copyErr - }) - if copyErr != nil { - erb.write("%s failed to copy %d.%d from %s: %v\n", rebuilder.info.Id, volumeId, shardId, ecNodes[0].info.Id, copyErr) - continue - } - recoverableRemoteShards++ - if needEcxFile { - needEcxFile = false - } - erb.write("%s copied %d.%d from %s\n", rebuilder.info.Id, volumeId, shardId, ecNodes[0].info.Id) - // Only shards this run actually copied are temp working files to be - // deleted afterward; never a pre-existing local or remote shard. - copiedShardIds = append(copiedShardIds, shardId) - } - - if len(localShardIds)+recoverableRemoteShards >= erasure_coding.DataShardsCount { - return copiedShardIds, localShardIds, nil - } - - // Hand back what was copied so the caller deletes these orphaned working - // shards: recovery failed, but the temp files are already on the rebuilder. - return copiedShardIds, localShardIds, fmt.Errorf("%d shards are not enough to recover volume %d", len(localShardIds)+recoverableRemoteShards, volumeId) - -} - -type EcShardMap map[needle.VolumeId]EcShardLocations -type EcShardLocations [][]*EcNode - -func (ecShardMap EcShardMap) registerEcNode(ecNode *EcNode, collection string) { - for _, diskInfo := range ecNode.info.DiskInfos { - for _, shardInfo := range diskInfo.EcShardInfos { - if shardInfo.Collection == collection { - existing, found := ecShardMap[needle.VolumeId(shardInfo.Id)] - if !found { - // Use MaxShardCount (32) to support custom EC ratios - existing = make([][]*EcNode, erasure_coding.MaxShardCount) - ecShardMap[needle.VolumeId(shardInfo.Id)] = existing - } - for _, shardId := range erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(shardInfo).Ids() { - existing[shardId] = append(existing[shardId], ecNode) - } - } - } - } -} - -func (ecShardLocations EcShardLocations) shardCount() (count int) { - for _, locations := range ecShardLocations { - if len(locations) > 0 { - count++ - } - } - return + return ec.RebuildEcVolumes(commandEnv.ecEnv(), allEcNodes, writer, collections, volumeIds, diskType, *maxParallelization, *applyChanges) } diff --git a/weed/shell/command_ec_scrub.go b/weed/shell/command_ec_scrub.go index 13b6897ab..349bacc84 100644 --- a/weed/shell/command_ec_scrub.go +++ b/weed/shell/command_ec_scrub.go @@ -1,18 +1,15 @@ package shell import ( - "context" "flag" "fmt" "io" "strconv" "strings" - "sync" - "github.com/seaweedfs/seaweedfs/weed/operation" + "github.com/seaweedfs/seaweedfs/weed/ec" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" - "google.golang.org/grpc" ) func init() { @@ -20,12 +17,6 @@ func init() { } type commandEcVolumeScrub struct { - env *CommandEnv - volumeServerAddrs []pb.ServerAddress - volumeIDs []uint32 - mode volume_server_pb.VolumeScrubMode - forceDeletedNeedlesCheck bool - grpcDialOption grpc.DialOption } func (c *commandEcVolumeScrub) Name() string { @@ -62,10 +53,14 @@ func (c *commandEcVolumeScrub) Do(args []string, commandEnv *CommandEnv, writer return } - c.volumeServerAddrs = []pb.ServerAddress{} + volumeServerAddrs := []pb.ServerAddress{} if *nodesStr != "" { for _, addr := range strings.Split(*nodesStr, ",") { - c.volumeServerAddrs = append(c.volumeServerAddrs, pb.ServerAddress(addr)) + addr = strings.TrimSpace(addr) + if addr == "" { + continue + } + volumeServerAddrs = append(volumeServerAddrs, pb.ServerAddress(addr)) } } else { dns, err := collectDataNodes(commandEnv, 0) @@ -73,11 +68,11 @@ func (c *commandEcVolumeScrub) Do(args []string, commandEnv *CommandEnv, writer return err } for _, dn := range dns { - c.volumeServerAddrs = append(c.volumeServerAddrs, pb.ServerAddress(dn.Address)) + volumeServerAddrs = append(volumeServerAddrs, pb.ServerAddress(dn.Address)) } } - c.volumeIDs = []uint32{} + volumeIDs := []uint32{} if *volumeIDsStr != "" { for _, vids := range strings.Split(*volumeIDsStr, ",") { vids = strings.TrimSpace(vids) @@ -85,96 +80,30 @@ func (c *commandEcVolumeScrub) Do(args []string, commandEnv *CommandEnv, writer continue } if vid, err := strconv.ParseUint(vids, 10, 32); err == nil { - c.volumeIDs = append(c.volumeIDs, uint32(vid)) + volumeIDs = append(volumeIDs, uint32(vid)) } else { return fmt.Errorf("invalid volume ID %q", vids) } } } + var scrubMode volume_server_pb.VolumeScrubMode switch strings.ToUpper(*mode) { case "INDEX": - c.mode = volume_server_pb.VolumeScrubMode_INDEX + scrubMode = volume_server_pb.VolumeScrubMode_INDEX case "LOCAL": - c.mode = volume_server_pb.VolumeScrubMode_LOCAL + scrubMode = volume_server_pb.VolumeScrubMode_LOCAL case "FULL": - c.mode = volume_server_pb.VolumeScrubMode_FULL + scrubMode = volume_server_pb.VolumeScrubMode_FULL case "CHECKSUM": - c.mode = volume_server_pb.VolumeScrubMode_CHECKSUM + scrubMode = volume_server_pb.VolumeScrubMode_CHECKSUM default: return fmt.Errorf("unsupported scrubbing mode %q", *mode) } - fmt.Fprintf(writer, "using %s mode\n", c.mode.String()) - c.env = commandEnv - c.forceDeletedNeedlesCheck = *forceDeletedNeedlesCheck - if c.forceDeletedNeedlesCheck && c.mode != volume_server_pb.VolumeScrubMode_FULL { + fmt.Fprintf(writer, "using %s mode\n", scrubMode.String()) + if *forceDeletedNeedlesCheck && scrubMode != volume_server_pb.VolumeScrubMode_FULL { return fmt.Errorf("deleted needle checks are only supported for FULL scrubs") } - return c.scrubEcVolumes(writer, *maxParallelization, *showDetails) -} - -func (c *commandEcVolumeScrub) scrubEcVolumes(writer io.Writer, maxParallelization int, showDetails bool) error { - var brokenVolumesStr, brokenShardsStr []string - var details []string - var totalVolumes, brokenVolumes, brokenShards, totalFiles uint64 - var mu sync.Mutex - - ewg := NewErrorWaitGroup(maxParallelization) - count := 0 - for _, addr := range c.volumeServerAddrs { - ewg.Add(func() error { - mu.Lock() - count++ - fmt.Fprintf(writer, "Scrubbing %s (%d/%d)...\n", addr.String(), count, len(c.volumeServerAddrs)) - mu.Unlock() - - err := operation.WithVolumeServerClient(false, addr, c.env.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error { - res, err := volumeServerClient.ScrubEcVolume(context.Background(), &volume_server_pb.ScrubEcVolumeRequest{ - Mode: c.mode, - VolumeIds: c.volumeIDs, - ForceDeletedNeedlesCheck: c.forceDeletedNeedlesCheck, - }) - if err != nil { - return err - } - - mu.Lock() - defer mu.Unlock() - - totalVolumes += res.GetTotalVolumes() - totalFiles += res.GetTotalFiles() - brokenVolumes += uint64(len(res.GetBrokenVolumeIds())) - brokenShards += uint64(len(res.GetBrokenShardInfos())) - for _, d := range res.GetDetails() { - details = append(details, fmt.Sprintf("[%s] %s", addr, d)) - } - for _, vid := range res.GetBrokenVolumeIds() { - brokenVolumesStr = append(brokenVolumesStr, fmt.Sprintf("%s:%v", addr, vid)) - } - for _, si := range res.GetBrokenShardInfos() { - brokenShardsStr = append(brokenShardsStr, fmt.Sprintf("%s:%v:%v", addr, si.VolumeId, si.ShardId)) - } - - return nil - }) - return err - }) - } - if err := ewg.Wait(); err != nil { - return err - } - - fmt.Fprintf(writer, "Scrubbed %d EC files and %d volumes on %d nodes\n", totalFiles, totalVolumes, len(c.volumeServerAddrs)) - if brokenVolumes != 0 { - fmt.Fprintf(writer, "\nGot scrub failures on %d EC volumes and %d EC shards :(\n", brokenVolumes, brokenShards) - fmt.Fprintf(writer, "Affected volumes: %s\n", strings.Join(brokenVolumesStr, ", ")) - if len(brokenShardsStr) != 0 { - fmt.Fprintf(writer, "Affected shards: %s\n", strings.Join(brokenShardsStr, ", ")) - } - if showDetails && len(details) != 0 { - fmt.Fprintf(writer, "Details:\n\t%s\n", strings.Join(details, "\n\t")) - } - } - return nil + return ec.ScrubEcVolumes(commandEnv.ecEnv(), writer, volumeServerAddrs, volumeIDs, scrubMode, *forceDeletedNeedlesCheck, *maxParallelization, *showDetails) } diff --git a/weed/shell/command_ec_shard_unmount.go b/weed/shell/command_ec_shard_unmount.go index ee2659049..debffee54 100644 --- a/weed/shell/command_ec_shard_unmount.go +++ b/weed/shell/command_ec_shard_unmount.go @@ -1,76 +1,19 @@ package shell import ( - "context" "flag" "fmt" "io" "math" - "sort" - "strconv" - "strings" - "time" - "github.com/seaweedfs/seaweedfs/weed/operation" - "github.com/seaweedfs/seaweedfs/weed/pb" - "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" - "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" - "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" + "github.com/seaweedfs/seaweedfs/weed/ec" ) -const ( - ecShardActionTimeout = 1 * time.Minute -) - -type ecShard struct { - ShardID uint32 - Collection string - NodeAddress string -} - -func (s *ecShard) String() string { - if s.NodeAddress == "" { - return fmt.Sprintf("%d", s.ShardID) - } - return fmt.Sprintf("%d@%s", s.ShardID, s.NodeAddress) -} - -func ecShardsFromString(shards string) ([]*ecShard, error) { - res := []*ecShard{} - - for _, s := range strings.Split(shards, ",") { - s = strings.TrimSpace(s) - if s == "" { - return nil, fmt.Errorf("empty shard ID in %q", shards) - } - - // optional @ to pick one copy - idStr, addr, _ := strings.Cut(s, "@") - - id, err := strconv.Atoi(idStr) - if err != nil || id < 0 || id >= erasure_coding.MaxShardCount { - return nil, fmt.Errorf("invalid shard ID %q", s) - } - - res = append(res, &ecShard{ShardID: uint32(id), NodeAddress: addr}) - } - - return res, nil -} - func init() { Commands = append(Commands, &commandEcShardUnmount{}) } type commandEcShardUnmount struct { - env *CommandEnv - writer io.Writer - topology *master_pb.TopologyInfo - volumeID uint32 - delete bool - ignoreInvalid bool - apply bool - shards []*ecShard } func (c *commandEcShardUnmount) Name() string { @@ -107,7 +50,6 @@ func (c *commandEcShardUnmount) Do(args []string, commandEnv *CommandEnv, writer if handleHelpRequest(c, args, writer) { return nil } - ecShardUnmountCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) volumeID := ecShardUnmountCommand.Uint("volumeId", 0, "volume ID for the shards to process") shardIDsStr := ecShardUnmountCommand.String("shardId", "", "comma-separated EC shard IDs for the volume") @@ -133,7 +75,7 @@ func (c *commandEcShardUnmount) Do(args []string, commandEnv *CommandEnv, writer if *shardIDsStr == "" { return fmt.Errorf("missing shardId") } - shards, err := ecShardsFromString(*shardIDsStr) + shards, err := ec.ShardRefsFromString(*shardIDsStr) if err != nil { return err } @@ -144,136 +86,12 @@ func (c *commandEcShardUnmount) Do(args []string, commandEnv *CommandEnv, writer return err } - c.env = commandEnv - c.writer = writer - c.topology = topology - c.volumeID = uint32(*volumeID) - c.delete = *delete - c.ignoreInvalid = *ignoreInvalid - c.apply = *apply || *applyAlias - c.shards = shards - - return c.doShardsUnmount() -} - -func (c *commandEcShardUnmount) write(format string, a ...any) { - fmt.Fprintf(c.writer, format, a...) -} - -func (c *commandEcShardUnmount) liveShardsForVolume() []*ecShard { - shards := []*ecShard{} - - for _, dci := range c.topology.GetDataCenterInfos() { - for _, ri := range dci.GetRackInfos() { - for _, dni := range ri.GetDataNodeInfos() { - nodeAddress := dni.GetAddress() - for _, di := range dni.GetDiskInfos() { - for _, eci := range di.GetEcShardInfos() { - if eci.GetId() == c.volumeID { - sinfo := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(eci) - for _, sid := range sinfo.Ids() { - shards = append(shards, &ecShard{ - ShardID: uint32(sid), - Collection: eci.GetCollection(), - NodeAddress: nodeAddress, - }) - } - } - } - } - } - } - } - - sort.SliceStable(shards, func(i, j int) bool { return shards[i].ShardID < shards[j].ShardID }) - return shards -} - -func (c *commandEcShardUnmount) printShards(ss []*ecShard) { - for _, s := range ss { - c.write("\t%v\n", s) - } - c.write("\n") -} - -func (c *commandEcShardUnmount) doShardsUnmount() error { - liveShards := c.liveShardsForVolume() - c.write("Live shard topology for volume ID %d (%d shards):\n", c.volumeID, len(liveShards)) - c.printShards(liveShards) - - // resolve target shards against the live topology - targetShards := []*ecShard{} - for _, ps := range c.shards { - var result *ecShard - for _, ts := range liveShards { - if ts.ShardID == ps.ShardID { - if ps.NodeAddress == "" || ps.NodeAddress == ts.NodeAddress { - if result != nil { - return fmt.Errorf("shard %v is ambiguous", ps) - } - result = ts - } - } - } - if result == nil { - if !c.ignoreInvalid { - return fmt.Errorf("shard %v is invalid", ps) - } - c.write("!!! ignoring invalid shard %v\n", ps) - } else { - targetShards = append(targetShards, result) - } - } - if len(targetShards) == 0 { - return fmt.Errorf("got no shards to process") - } - - mode := "unmount" - if c.delete { - mode = "unmount + delete" - } - c.write("Will %s %d shard(s):\n", mode, len(targetShards)) - c.printShards(targetShards) - - if !c.apply { - c.write("Not proceeding in dry-run mode\n") - return nil - } - - for _, s := range targetShards { - if err := c.unmountShard(s); err != nil { - return err - } - } - - c.write("\nAll done!\n") - return nil -} - -func (c *commandEcShardUnmount) unmountShard(s *ecShard) error { - ctx, cancel := context.WithTimeout(context.Background(), ecShardActionTimeout) - defer cancel() - - return operation.WithVolumeServerClient(false, pb.ServerAddress(s.NodeAddress), c.env.option.GrpcDialOption, func(vsc volume_server_pb.VolumeServerClient) error { - c.write("Unmounting shard %v for volume ID %d...\n", s, c.volumeID) - if _, err := vsc.VolumeEcShardsUnmount(ctx, &volume_server_pb.VolumeEcShardsUnmountRequest{ - VolumeId: c.volumeID, - ShardIds: []uint32{s.ShardID}, - }); err != nil { - return err - } - - if c.delete { - c.write("Deleting shard %v for volume ID %d...\n", s, c.volumeID) - if _, err := vsc.VolumeEcShardsDelete(ctx, &volume_server_pb.VolumeEcShardsDeleteRequest{ - VolumeId: c.volumeID, - Collection: s.Collection, - ShardIds: []uint32{s.ShardID}, - }); err != nil { - return err - } - } - - return nil + return ec.UnmountShards(commandEnv.ecEnv(), writer, ec.ShardUnmountRequest{ + Topology: topology, + VolumeID: uint32(*volumeID), + Shards: shards, + Delete: *delete, + IgnoreInvalid: *ignoreInvalid, + Apply: *apply || *applyAlias, }) } diff --git a/weed/shell/command_volume_server_evacuate.go b/weed/shell/command_volume_server_evacuate.go index e8a9b885a..af0a99974 100644 --- a/weed/shell/command_volume_server_evacuate.go +++ b/weed/shell/command_volume_server_evacuate.go @@ -177,7 +177,7 @@ func (c *commandVolumeServerEvacuate) evacuateEcVolumes(commandEnv *CommandEnv, // move away ec volumes for this disk type for _, thisNode := range thisNodes { - diskInfo, found := thisNode.info.DiskInfos[string(diskType)] + diskInfo, found := thisNode.Info.DiskInfos[string(diskType)] if !found { continue } @@ -205,12 +205,12 @@ func (c *commandVolumeServerEvacuate) moveAwayOneEcVolume(commandEnv *CommandEnv // Sort by: 1) fewest shards of this volume, 2) most free EC slots // This ensures we prefer nodes with capacity and balanced shard distribution slices.SortFunc(otherNodes, func(a, b *EcNode) int { - aShards := a.localShardIdCount(ecShardInfo.Id) - bShards := b.localShardIdCount(ecShardInfo.Id) + aShards := a.LocalShardIdCount(ecShardInfo.Id) + bShards := b.LocalShardIdCount(ecShardInfo.Id) if aShards != bShards { return aShards - bShards // Prefer fewer shards } - return b.freeEcSlot - a.freeEcSlot // Then prefer more free slots + return b.FreeEcSlot - a.FreeEcSlot // Then prefer more free slots }) shardMoved := false @@ -219,7 +219,7 @@ func (c *commandVolumeServerEvacuate) moveAwayOneEcVolume(commandEnv *CommandEnv emptyNode := otherNodes[i] // Skip nodes with no free EC slots - if emptyNode.freeEcSlot <= 0 { + if emptyNode.FreeEcSlot <= 0 { skippedNodes++ continue } @@ -233,9 +233,9 @@ func (c *commandVolumeServerEvacuate) moveAwayOneEcVolume(commandEnv *CommandEnv // No anti-affinity needed for evacuation (dataShardCount=0) destDiskId := pickBestDiskOnNode(emptyNode, vid, diskType, false, shardId, 0) if destDiskId > 0 { - fmt.Fprintf(writer, "moving ec volume %s%d.%d %s => %s (disk %d)\n", collectionPrefix, ecShardInfo.Id, shardId, thisNode.info.Id, emptyNode.info.Id, destDiskId) + fmt.Fprintf(writer, "moving ec volume %s%d.%d %s => %s (disk %d)\n", collectionPrefix, ecShardInfo.Id, shardId, thisNode.Info.Id, emptyNode.Info.Id, destDiskId) } else { - fmt.Fprintf(writer, "moving ec volume %s%d.%d %s => %s\n", collectionPrefix, ecShardInfo.Id, shardId, thisNode.info.Id, emptyNode.info.Id) + fmt.Fprintf(writer, "moving ec volume %s%d.%d %s => %s\n", collectionPrefix, ecShardInfo.Id, shardId, thisNode.Info.Id, emptyNode.Info.Id) } err = moveMountedShardToEcNode(commandEnv, thisNode, ecShardInfo.Collection, vid, shardId, emptyNode, destDiskId, applyChange, diskType) if err != nil { @@ -245,7 +245,7 @@ func (c *commandVolumeServerEvacuate) moveAwayOneEcVolume(commandEnv *CommandEnv hasMoved = true shardMoved = true // Update the node's free slot count after successful move - emptyNode.freeEcSlot-- + emptyNode.FreeEcSlot-- break } } @@ -310,14 +310,14 @@ func (c *commandVolumeServerEvacuate) nodesOtherThan(volumeServers []*Node, this func (c *commandVolumeServerEvacuate) ecNodesOtherThan(volumeServers []*EcNode, thisServer string) (thisNodes []*EcNode, otherNodes []*EcNode) { for _, node := range volumeServers { - if node.info.Id == thisServer || (*c.volumeRack != "" && string(node.rack) == *c.volumeRack) { + if node.Info.Id == thisServer || (*c.volumeRack != "" && string(node.Rack) == *c.volumeRack) { thisNodes = append(thisNodes, node) continue } - if *c.volumeRack != "" && *c.volumeRack == string(node.rack) { + if *c.volumeRack != "" && *c.volumeRack == string(node.Rack) { continue } - if *c.targetServer != "" && *c.targetServer != node.info.Id { + if *c.targetServer != "" && *c.targetServer != node.Info.Id { continue } otherNodes = append(otherNodes, node) diff --git a/weed/shell/common.go b/weed/shell/common.go index 4dfac5003..147116c98 100644 --- a/weed/shell/common.go +++ b/weed/shell/common.go @@ -1,9 +1,7 @@ package shell import ( - "errors" - "fmt" - "sync" + "github.com/seaweedfs/seaweedfs/weed/util" ) var ( @@ -16,89 +14,9 @@ var ( CollectionDefault = "_default" ) -// ErrorWaitGroup implements a goroutine wait group which aggregates errors, if any. -type ErrorWaitGroup struct { - maxConcurrency int - wg *sync.WaitGroup - wgSem chan bool - errors []error - errorsMu sync.Mutex -} +// ErrorWaitGroup lives in weed/util so packages outside the shell can share it. +type ErrorWaitGroup = util.ErrorWaitGroup +type ErrorWaitGroupTask = util.ErrorWaitGroupTask -type ErrorWaitGroupTask func() error - -// executeParallelTaskGroups runs tasks from each group sequentially and runs -// independent groups with up to maxParallelization concurrency. EC balancing -// uses one group per volume because its shard sidecar files are shared; volume -// balancing uses one group per already-reserved volume move. -func executeParallelTaskGroups(maxParallelization int, taskGroups [][]ErrorWaitGroupTask) error { - ewg := NewErrorWaitGroup(maxParallelization) - for _, taskGroup := range taskGroups { - taskGroup := taskGroup - ewg.Add(func() error { - for _, task := range taskGroup { - if err := task(); err != nil { - return err - } - } - return nil - }) - } - return ewg.Wait() -} - -func NewErrorWaitGroup(maxConcurrency int) *ErrorWaitGroup { - if maxConcurrency <= 0 { - // no concurrency = one task at the time - maxConcurrency = 1 - } - return &ErrorWaitGroup{ - maxConcurrency: maxConcurrency, - wg: &sync.WaitGroup{}, - wgSem: make(chan bool, maxConcurrency), - } -} - -// Reset restarts an ErrorWaitGroup, keeping original settings. Errors and pending goroutines, if any, are flushed. -func (ewg *ErrorWaitGroup) Reset() { - close(ewg.wgSem) - - ewg.wg = &sync.WaitGroup{} - ewg.wgSem = make(chan bool, ewg.maxConcurrency) - ewg.errors = nil -} - -// Add queues an ErrorWaitGroupTask to be executed as a goroutine. -func (ewg *ErrorWaitGroup) Add(f ErrorWaitGroupTask) { - if ewg.maxConcurrency <= 1 { - // keep run order deterministic when parallelization is off - ewg.errors = append(ewg.errors, f()) - return - } - - ewg.wg.Add(1) - go func() { - ewg.wgSem <- true - - err := f() - ewg.errorsMu.Lock() - ewg.errors = append(ewg.errors, err) - ewg.errorsMu.Unlock() - - <-ewg.wgSem - ewg.wg.Done() - }() -} - -// AddErrorf adds an error to an ErrorWaitGroupTask result, without queueing any goroutines. -func (ewg *ErrorWaitGroup) AddErrorf(format string, a ...interface{}) { - ewg.errorsMu.Lock() - ewg.errors = append(ewg.errors, fmt.Errorf(format, a...)) - ewg.errorsMu.Unlock() -} - -// Wait sleeps until all ErrorWaitGroupTasks are completed, then returns errors for them. -func (ewg *ErrorWaitGroup) Wait() error { - ewg.wg.Wait() - return errors.Join(ewg.errors...) -} +var NewErrorWaitGroup = util.NewErrorWaitGroup +var executeParallelTaskGroups = util.ExecuteParallelTaskGroups diff --git a/weed/shell/common_test.go b/weed/shell/common_test.go index 284ad3acb..023bedfb8 100644 --- a/weed/shell/common_test.go +++ b/weed/shell/common_test.go @@ -2,9 +2,6 @@ package shell import ( _ "embed" - "errors" - "testing" - "time" ) //go:embed volume.list.txt @@ -21,90 +18,3 @@ var ( testTopology2 = parseOutput(topoData2) testTopologyEc = parseOutput(topoDataEc) ) - -func TestExecuteParallelTaskGroups(t *testing.T) { - firstTaskStarted := make(chan struct{}) - firstTaskRelease := make(chan struct{}) - secondTaskStarted := make(chan struct{}) - secondTaskInFirstGroupStarted := make(chan struct{}) - done := make(chan error, 1) - - go func() { - done <- executeParallelTaskGroups(2, [][]ErrorWaitGroupTask{ - { - func() error { - close(firstTaskStarted) - <-firstTaskRelease - return nil - }, - func() error { - close(secondTaskInFirstGroupStarted) - return nil - }, - }, - { - func() error { - close(secondTaskStarted) - return nil - }, - }, - }) - }() - - select { - case <-firstTaskStarted: - case <-time.After(time.Second): - t.Fatal("first task did not start") - } - select { - case <-secondTaskStarted: - case <-time.After(time.Second): - t.Fatal("independent task group did not start in parallel") - } - select { - case <-secondTaskInFirstGroupStarted: - t.Fatal("tasks in the same group ran in parallel") - default: - } - - close(firstTaskRelease) - select { - case err := <-done: - if err != nil { - t.Fatalf("execute parallel task groups: %v", err) - } - case <-time.After(time.Second): - t.Fatal("parallel task groups did not finish") - } -} - -func TestExecuteParallelTaskGroupsStopsOnlyFailedGroup(t *testing.T) { - expectedErr := errors.New("move failed") - failedGroupContinued := false - otherGroupRan := false - - err := executeParallelTaskGroups(1, [][]ErrorWaitGroupTask{ - { - func() error { return expectedErr }, - func() error { - failedGroupContinued = true - return nil - }, - }, - { - func() error { - otherGroupRan = true - return nil - }, - }, - }) - if !errors.Is(err, expectedErr) { - t.Fatalf("expected task error, got %v", err) - } - if failedGroupContinued { - t.Fatal("task after a failed task in the same group ran") - } - if !otherGroupRan { - t.Fatal("independent task group did not run") - } -} diff --git a/weed/shell/ec_proportional_rebalance.go b/weed/shell/ec_proportional_rebalance.go deleted file mode 100644 index 8d6b1c1b7..000000000 --- a/weed/shell/ec_proportional_rebalance.go +++ /dev/null @@ -1,38 +0,0 @@ -package shell - -import ( - "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" - "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/distribution" - "github.com/seaweedfs/seaweedfs/weed/storage/needle" - "github.com/seaweedfs/seaweedfs/weed/storage/super_block" - "github.com/seaweedfs/seaweedfs/weed/storage/types" -) - -// ECDistribution is an alias to the distribution package type for backward compatibility -type ECDistribution = distribution.ECDistribution - -// TopologyDistributionAnalysis holds the current shard distribution analysis -// This wraps the distribution package's TopologyAnalysis with shell-specific EcNode handling -type TopologyDistributionAnalysis struct { - inner *distribution.TopologyAnalysis - - // Shell-specific mappings - nodeMap map[string]*EcNode // nodeID -> EcNode -} - -// ECShardMove represents a planned shard move (shell-specific with EcNode references) -type ECShardMove struct { - VolumeId needle.VolumeId - ShardId erasure_coding.ShardId - SourceNode *EcNode - DestNode *EcNode - Reason string -} - -// ProportionalECRebalancer implements proportional shard distribution for shell commands -type ProportionalECRebalancer struct { - ecNodes []*EcNode - replicaPlacement *super_block.ReplicaPlacement - diskType types.DiskType - ecConfig distribution.ECConfig -} diff --git a/weed/util/error_wait_group.go b/weed/util/error_wait_group.go new file mode 100644 index 000000000..c76bc17aa --- /dev/null +++ b/weed/util/error_wait_group.go @@ -0,0 +1,94 @@ +package util + +import ( + "errors" + "fmt" + "sync" +) + +// ErrorWaitGroup implements a goroutine wait group which aggregates errors, if any. +type ErrorWaitGroup struct { + maxConcurrency int + wg *sync.WaitGroup + wgSem chan bool + errors []error + errorsMu sync.Mutex +} + +type ErrorWaitGroupTask func() error + +// ExecuteParallelTaskGroups runs tasks from each group sequentially and runs +// independent groups with up to maxParallelization concurrency. EC balancing +// uses one group per volume because its shard sidecar files are shared; volume +// balancing uses one group per already-reserved volume move. +func ExecuteParallelTaskGroups(maxParallelization int, taskGroups [][]ErrorWaitGroupTask) error { + ewg := NewErrorWaitGroup(maxParallelization) + for _, taskGroup := range taskGroups { + taskGroup := taskGroup + ewg.Add(func() error { + for _, task := range taskGroup { + if err := task(); err != nil { + return err + } + } + return nil + }) + } + return ewg.Wait() +} + +func NewErrorWaitGroup(maxConcurrency int) *ErrorWaitGroup { + if maxConcurrency <= 0 { + // no concurrency = one task at the time + maxConcurrency = 1 + } + return &ErrorWaitGroup{ + maxConcurrency: maxConcurrency, + wg: &sync.WaitGroup{}, + wgSem: make(chan bool, maxConcurrency), + } +} + +// Reset restarts an ErrorWaitGroup, keeping original settings. Errors and pending goroutines, if any, are flushed. +func (ewg *ErrorWaitGroup) Reset() { + close(ewg.wgSem) + + ewg.wg = &sync.WaitGroup{} + ewg.wgSem = make(chan bool, ewg.maxConcurrency) + ewg.errors = nil +} + +// Add queues an ErrorWaitGroupTask to be executed as a goroutine. +func (ewg *ErrorWaitGroup) Add(f ErrorWaitGroupTask) { + if ewg.maxConcurrency <= 1 { + // keep run order deterministic when parallelization is off + ewg.errors = append(ewg.errors, f()) + return + } + + ewg.wg.Add(1) + go func() { + ewg.wgSem <- true + + err := f() + ewg.errorsMu.Lock() + ewg.errors = append(ewg.errors, err) + ewg.errorsMu.Unlock() + + <-ewg.wgSem + ewg.wg.Done() + }() +} + +// AddErrorf adds an error to an ErrorWaitGroupTask result, without queueing any goroutines. +func (ewg *ErrorWaitGroup) AddErrorf(format string, a ...interface{}) { + ewg.errorsMu.Lock() + ewg.errors = append(ewg.errors, fmt.Errorf(format, a...)) + ewg.errorsMu.Unlock() +} + +// Wait sleeps until all ErrorWaitGroupTasks are completed, then returns errors for them. +func (ewg *ErrorWaitGroup) Wait() error { + ewg.wg.Wait() + return errors.Join(ewg.errors...) +} diff --git a/weed/util/error_wait_group_test.go b/weed/util/error_wait_group_test.go new file mode 100644 index 000000000..8068a92a3 --- /dev/null +++ b/weed/util/error_wait_group_test.go @@ -0,0 +1,94 @@ +package util + +import ( + "errors" + "testing" + "time" +) + +func TestExecuteParallelTaskGroups(t *testing.T) { + firstTaskStarted := make(chan struct{}) + firstTaskRelease := make(chan struct{}) + secondTaskStarted := make(chan struct{}) + secondTaskInFirstGroupStarted := make(chan struct{}) + done := make(chan error, 1) + + go func() { + done <- ExecuteParallelTaskGroups(2, [][]ErrorWaitGroupTask{ + { + func() error { + close(firstTaskStarted) + <-firstTaskRelease + return nil + }, + func() error { + close(secondTaskInFirstGroupStarted) + return nil + }, + }, + { + func() error { + close(secondTaskStarted) + return nil + }, + }, + }) + }() + + select { + case <-firstTaskStarted: + case <-time.After(time.Second): + t.Fatal("first task did not start") + } + select { + case <-secondTaskStarted: + case <-time.After(time.Second): + t.Fatal("independent task group did not start in parallel") + } + select { + case <-secondTaskInFirstGroupStarted: + t.Fatal("tasks in the same group ran in parallel") + default: + } + + close(firstTaskRelease) + select { + case err := <-done: + if err != nil { + t.Fatalf("execute parallel task groups: %v", err) + } + case <-time.After(time.Second): + t.Fatal("parallel task groups did not finish") + } +} + +func TestExecuteParallelTaskGroupsStopsOnlyFailedGroup(t *testing.T) { + expectedErr := errors.New("move failed") + failedGroupContinued := false + otherGroupRan := false + + err := ExecuteParallelTaskGroups(1, [][]ErrorWaitGroupTask{ + { + func() error { return expectedErr }, + func() error { + failedGroupContinued = true + return nil + }, + }, + { + func() error { + otherGroupRan = true + return nil + }, + }, + }) + if !errors.Is(err, expectedErr) { + t.Fatalf("expected task error, got %v", err) + } + if failedGroupContinued { + t.Fatal("task after a failed task in the same group ran") + } + if !otherGroupRan { + t.Fatal("independent task group did not run") + } +} diff --git a/weed/worker/tasks/erasure_coding/detection.go b/weed/worker/tasks/erasure_coding/detection.go index 571fa0132..63a602fdd 100644 --- a/weed/worker/tasks/erasure_coding/detection.go +++ b/weed/worker/tasks/erasure_coding/detection.go @@ -8,6 +8,7 @@ import ( "time" "github.com/seaweedfs/seaweedfs/weed/admin/topology" + "github.com/seaweedfs/seaweedfs/weed/ec" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/pb" @@ -757,50 +758,12 @@ func cleanupOrphanSourceReplicas(ctx context.Context, clusterInfo *types.Cluster return deleted, nil } -// countExistingEcShardsForVolume returns the number of distinct EC shard IDs -// for (volumeID, collection) present in the topology, counting only the single -// largest encode generation. Shards are grouped by encode_ts_ns (the per-encode -// identity from .vif), so two interrupted encode runs whose shard sets overlap -// are never unioned into a false-complete set that would wrongly trigger the -// orphaned-source delete. Walks every disk's EcIndexBits bitmap rather than -// trusting len(EcShardInfos), because a single info entry can carry multiple -// shards. Shards reporting encode_ts_ns==0 (pre-upgrade servers) form their own -// generation bucket. -// -// Limitation: the heartbeat carries one encode_ts_ns per (volume, disk), so this -// separates generations living on different disks; same-disk mixing is prevented -// upstream by the pre-encode artifact wipe and the cross-run read guard. +// countExistingEcShardsForVolume counts (volumeID, collection) EC shards in the +// topology, counting only the single largest encode generation; see +// ec.CountExistingEcShardsForVolume for the generation semantics. func countExistingEcShardsForVolume(activeTopology *topology.ActiveTopology, volumeID uint32, collection string) int { if activeTopology == nil { return 0 } - topologyInfo := activeTopology.GetTopologyInfo() - if topologyInfo == nil { - return 0 - } - perGeneration := make(map[int64]erasure_coding.ShardBits) - for _, dc := range topologyInfo.DataCenterInfos { - for _, rack := range dc.RackInfos { - for _, node := range rack.DataNodeInfos { - for _, diskInfo := range node.DiskInfos { - for _, ecShardInfo := range diskInfo.EcShardInfos { - if ecShardInfo == nil { - continue - } - if ecShardInfo.Id != volumeID || ecShardInfo.Collection != collection { - continue - } - perGeneration[ecShardInfo.EncodeTsNs] |= erasure_coding.ShardBits(ecShardInfo.EcIndexBits) - } - } - } - } - } - best := 0 - for _, bits := range perGeneration { - if c := bits.Count(); c > best { - best = c - } - } - return best + return ec.CountExistingEcShardsForVolume(activeTopology.GetTopologyInfo(), volumeID, collection) }