ec.decode: finish the cleanup an interrupted decode left behind (#10767)

A decode deletes the shards only after the regenerated volume is mounted and verified, so a run interrupted in that last phase leaves the volume in place with its shards partway through deletion. The re-run then finds both, tries to collect the shards again to rebuild a volume that already exists, and fails on the first shard the interrupted run had removed: generate normal volume 3 ...: ec volume 3 missing shard 6 Nothing recovers from there: the shard set is deliberately being destroyed, so every retry fails the same way while the decoded volume sits there, already complete. Finish that cleanup instead. A volume beside the shards is not enough to act on -- an encode interrupted before it deleted the original leaves the same shape, as does a decode killed while generating, whose volume may be half written -- so require a data shard to be gone. Only the deletion phase removes one, and it is also exactly the state no decode can recover from, so finishing is the only move left rather than a choice between two. The deletion still runs behind verifyDecodedVolumeBeforeDelete, the check that guards it in a normal run.
2026-08-16 20:26:45 +00:00 · 2026-08-15 13:04:37 -07:00
parent 97a155d14d
commit 829064af71
2 changed files with 156 additions and 0 deletions
@@ -35,6 +35,29 @@ func DoEcDecode(env *Env, topoInfo *master_pb.TopologyInfo, collection string, v
 		return fmt.Errorf("no EC shards found for volume %d", vid)
 	}

+	// A decode interrupted while deleting the shards leaves the regenerated
+	// volume in place with the set half gone, so a re-run finds both and fails
+	// re-collecting the first shard the interrupted run removed.
+	//
+	// Finding a volume beside the shards is not enough to act on: an encode
+	// interrupted before it deleted the original leaves the same shape, as does
+	// a decode killed while generating, whose volume may be half written. Both
+	// of those leave the shard set COMPLETE. Only the deletion phase can remove
+	// a data shard, so require one to be gone -- that is also exactly the state
+	// no decode can recover from, which makes finishing the cleanup the only
+	// move left rather than a choice between two.
+	if _, found := missingDataShard(nodeToEcShardsInfo, dataShards); found {
+		holder, hasVolume := regularVolumeHolder(topoInfo, vid)
+		if !hasVolume {
+			return fmt.Errorf("volume %d cannot be decoded: data shards are missing and no decoded volume exists to finish", vid)
+		}
+		if err := verifyDecodedVolumeBeforeDelete(env.GrpcDialOption, holder, vid); err != nil {
+			return fmt.Errorf("volume %d is already decoded on %s but did not verify, keeping its ec shards: %v", vid, holder, err)
+		}
+		fmt.Printf("volume %d is already decoded on %s; deleting the ec shards the interrupted run left behind\n", vid, holder)
+		return unmountAndDeleteEcShardsWithPrefix("deleteDecodedEcShards", env.GrpcDialOption, collection, nodeToEcShardsInfo, vid)
+	}
+
 	var originalShardCounts map[pb.ServerAddress]int
 	if diskUsageState != nil {
 		originalShardCounts = make(map[pb.ServerAddress]int, len(nodeToEcShardsInfo))
@@ -145,6 +168,48 @@ func unmountAndDeleteEcShardsWithPrefix(prefix string, grpcDialOption grpc.DialO
 	return ewg.Wait()
 }

+// missingDataShard reports the first data shard absent from every holder.
+// Parity shards are not enough to answer this: the decode rebuilds the volume
+// from the data shards, so one of those going missing is what makes a re-run
+// impossible.
+func missingDataShard(nodeToShardsInfo map[pb.ServerAddress]*erasure_coding.ShardsInfo, dataShards int) (erasure_coding.ShardId, bool) {
+	var present erasure_coding.ShardBits
+	for _, si := range nodeToShardsInfo {
+		for _, id := range si.Ids() {
+			present = present.Set(id)
+		}
+	}
+	for id := 0; id < dataShards; id++ {
+		if !present.Has(erasure_coding.ShardId(id)) {
+			return erasure_coding.ShardId(id), true
+		}
+	}
+	return 0, false
+}
+
+// regularVolumeHolder returns a server already serving vid as a regular
+// volume. A decode only finds one when an earlier run was interrupted between
+// regenerating the volume and deleting the shards it came from.
+func regularVolumeHolder(topoInfo *master_pb.TopologyInfo, vid needle.VolumeId) (pb.ServerAddress, bool) {
+	var holder pb.ServerAddress
+	found := false
+	EachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
+		if found {
+			return
+		}
+		for _, diskInfo := range dn.DiskInfos {
+			for _, vi := range diskInfo.VolumeInfos {
+				if needle.VolumeId(vi.Id) == vid {
+					holder = pb.NewServerAddressFromDataNode(dn)
+					found = true
+					return
+				}
+			}
+		}
+	})
+	return holder, found
+}
+
 func verifyDecodedVolumeBeforeDelete(grpcDialOption grpc.DialOption, target pb.ServerAddress, vid needle.VolumeId) error {
 	var resp *volume_server_pb.ReadVolumeFileStatusResponse
 	if err := operation.WithVolumeServerClient(false, target, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
@@ -0,0 +1,91 @@
+package ec
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
+)
+
+// shardsOn builds one holder's inventory.
+func shardsOn(ids ...int) *erasure_coding.ShardsInfo {
+	si := erasure_coding.NewShardsInfo()
+	for _, id := range ids {
+		si.Set(erasure_coding.NewShardInfo(erasure_coding.ShardId(id), 1024))
+	}
+	return si
+}
+
+// missingDataShard decides whether a decode may delete the shards of a volume
+// that already exists elsewhere. Only the deletion phase of a decode removes a
+// data shard, so the answer separates "an earlier decode was interrupted while
+// cleaning up" from the two states that look the same from the outside: an
+// encode interrupted before it deleted the original, and a decode killed while
+// generating. Both of those leave every shard in place, and the volume beside
+// them may be the untouched original or a half-written one -- neither is safe
+// to trade for the shards.
+func TestMissingDataShard(t *testing.T) {
+	const dataShards = 10
+
+	tests := []struct {
+		name       string
+		holders    map[pb.ServerAddress]*erasure_coding.ShardsInfo
+		wantID     erasure_coding.ShardId
+		wantMissed bool
+	}{
+		{
+			name: "complete set on one holder decodes, so nothing is finished",
+			holders: map[pb.ServerAddress]*erasure_coding.ShardsInfo{
+				"server1:8080": shardsOn(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13),
+			},
+		},
+		{
+			name: "complete set spread across holders is still complete",
+			holders: map[pb.ServerAddress]*erasure_coding.ShardsInfo{
+				"server1:8080": shardsOn(0, 1, 2, 3, 4),
+				"server2:8080": shardsOn(5, 6, 7, 8, 9),
+				"server3:8080": shardsOn(10, 11, 12, 13),
+			},
+		},
+		{
+			name: "parity gone is not the deletion phase: the volume still decodes",
+			holders: map[pb.ServerAddress]*erasure_coding.ShardsInfo{
+				"server1:8080": shardsOn(0, 1, 2, 3, 4, 5, 6, 7, 8, 9),
+			},
+		},
+		{
+			name: "a data shard gone is a decode that was interrupted mid-cleanup",
+			holders: map[pb.ServerAddress]*erasure_coding.ShardsInfo{
+				"server1:8080": shardsOn(0, 1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 12, 13),
+			},
+			wantID:     6,
+			wantMissed: true,
+		},
+		{
+			name: "the first gap is reported, not the last",
+			holders: map[pb.ServerAddress]*erasure_coding.ShardsInfo{
+				"server1:8080": shardsOn(0, 3, 4, 5, 6, 7, 8, 9),
+			},
+			wantID:     1,
+			wantMissed: true,
+		},
+		{
+			name:       "no holders at all leaves every data shard missing",
+			holders:    map[pb.ServerAddress]*erasure_coding.ShardsInfo{},
+			wantID:     0,
+			wantMissed: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			id, missed := missingDataShard(tt.holders, dataShards)
+			if missed != tt.wantMissed {
+				t.Fatalf("missingDataShard = %v, want %v", missed, tt.wantMissed)
+			}
+			if missed && id != tt.wantID {
+				t.Errorf("missing shard id = %d, want %d", id, tt.wantID)
+			}
+		})
+	}
+}