diff --git a/weed/ec/ec_decode.go b/weed/ec/ec_decode.go index c236b35d8..a20830529 100644 --- a/weed/ec/ec_decode.go +++ b/weed/ec/ec_decode.go @@ -35,6 +35,29 @@ func DoEcDecode(env *Env, topoInfo *master_pb.TopologyInfo, collection string, v return fmt.Errorf("no EC shards found for volume %d", vid) } + // A decode interrupted while deleting the shards leaves the regenerated + // volume in place with the set half gone, so a re-run finds both and fails + // re-collecting the first shard the interrupted run removed. + // + // Finding a volume beside the shards is not enough to act on: an encode + // interrupted before it deleted the original leaves the same shape, as does + // a decode killed while generating, whose volume may be half written. Both + // of those leave the shard set COMPLETE. Only the deletion phase can remove + // a data shard, so require one to be gone -- that is also exactly the state + // no decode can recover from, which makes finishing the cleanup the only + // move left rather than a choice between two. + if _, found := missingDataShard(nodeToEcShardsInfo, dataShards); found { + holder, hasVolume := regularVolumeHolder(topoInfo, vid) + if !hasVolume { + return fmt.Errorf("volume %d cannot be decoded: data shards are missing and no decoded volume exists to finish", vid) + } + if err := verifyDecodedVolumeBeforeDelete(env.GrpcDialOption, holder, vid); err != nil { + return fmt.Errorf("volume %d is already decoded on %s but did not verify, keeping its ec shards: %v", vid, holder, err) + } + fmt.Printf("volume %d is already decoded on %s; deleting the ec shards the interrupted run left behind\n", vid, holder) + return unmountAndDeleteEcShardsWithPrefix("deleteDecodedEcShards", env.GrpcDialOption, collection, nodeToEcShardsInfo, vid) + } + var originalShardCounts map[pb.ServerAddress]int if diskUsageState != nil { originalShardCounts = make(map[pb.ServerAddress]int, len(nodeToEcShardsInfo)) @@ -145,6 +168,48 @@ func unmountAndDeleteEcShardsWithPrefix(prefix string, grpcDialOption grpc.DialO return ewg.Wait() } +// missingDataShard reports the first data shard absent from every holder. +// Parity shards are not enough to answer this: the decode rebuilds the volume +// from the data shards, so one of those going missing is what makes a re-run +// impossible. +func missingDataShard(nodeToShardsInfo map[pb.ServerAddress]*erasure_coding.ShardsInfo, dataShards int) (erasure_coding.ShardId, bool) { + var present erasure_coding.ShardBits + for _, si := range nodeToShardsInfo { + for _, id := range si.Ids() { + present = present.Set(id) + } + } + for id := 0; id < dataShards; id++ { + if !present.Has(erasure_coding.ShardId(id)) { + return erasure_coding.ShardId(id), true + } + } + return 0, false +} + +// regularVolumeHolder returns a server already serving vid as a regular +// volume. A decode only finds one when an earlier run was interrupted between +// regenerating the volume and deleting the shards it came from. +func regularVolumeHolder(topoInfo *master_pb.TopologyInfo, vid needle.VolumeId) (pb.ServerAddress, bool) { + var holder pb.ServerAddress + found := false + EachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) { + if found { + return + } + for _, diskInfo := range dn.DiskInfos { + for _, vi := range diskInfo.VolumeInfos { + if needle.VolumeId(vi.Id) == vid { + holder = pb.NewServerAddressFromDataNode(dn) + found = true + return + } + } + } + }) + return holder, found +} + func verifyDecodedVolumeBeforeDelete(grpcDialOption grpc.DialOption, target pb.ServerAddress, vid needle.VolumeId) error { var resp *volume_server_pb.ReadVolumeFileStatusResponse if err := operation.WithVolumeServerClient(false, target, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { diff --git a/weed/ec/ec_decode_test.go b/weed/ec/ec_decode_test.go new file mode 100644 index 000000000..bf73126e8 --- /dev/null +++ b/weed/ec/ec_decode_test.go @@ -0,0 +1,91 @@ +package ec + +import ( + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" +) + +// shardsOn builds one holder's inventory. +func shardsOn(ids ...int) *erasure_coding.ShardsInfo { + si := erasure_coding.NewShardsInfo() + for _, id := range ids { + si.Set(erasure_coding.NewShardInfo(erasure_coding.ShardId(id), 1024)) + } + return si +} + +// missingDataShard decides whether a decode may delete the shards of a volume +// that already exists elsewhere. Only the deletion phase of a decode removes a +// data shard, so the answer separates "an earlier decode was interrupted while +// cleaning up" from the two states that look the same from the outside: an +// encode interrupted before it deleted the original, and a decode killed while +// generating. Both of those leave every shard in place, and the volume beside +// them may be the untouched original or a half-written one -- neither is safe +// to trade for the shards. +func TestMissingDataShard(t *testing.T) { + const dataShards = 10 + + tests := []struct { + name string + holders map[pb.ServerAddress]*erasure_coding.ShardsInfo + wantID erasure_coding.ShardId + wantMissed bool + }{ + { + name: "complete set on one holder decodes, so nothing is finished", + holders: map[pb.ServerAddress]*erasure_coding.ShardsInfo{ + "server1:8080": shardsOn(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13), + }, + }, + { + name: "complete set spread across holders is still complete", + holders: map[pb.ServerAddress]*erasure_coding.ShardsInfo{ + "server1:8080": shardsOn(0, 1, 2, 3, 4), + "server2:8080": shardsOn(5, 6, 7, 8, 9), + "server3:8080": shardsOn(10, 11, 12, 13), + }, + }, + { + name: "parity gone is not the deletion phase: the volume still decodes", + holders: map[pb.ServerAddress]*erasure_coding.ShardsInfo{ + "server1:8080": shardsOn(0, 1, 2, 3, 4, 5, 6, 7, 8, 9), + }, + }, + { + name: "a data shard gone is a decode that was interrupted mid-cleanup", + holders: map[pb.ServerAddress]*erasure_coding.ShardsInfo{ + "server1:8080": shardsOn(0, 1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 12, 13), + }, + wantID: 6, + wantMissed: true, + }, + { + name: "the first gap is reported, not the last", + holders: map[pb.ServerAddress]*erasure_coding.ShardsInfo{ + "server1:8080": shardsOn(0, 3, 4, 5, 6, 7, 8, 9), + }, + wantID: 1, + wantMissed: true, + }, + { + name: "no holders at all leaves every data shard missing", + holders: map[pb.ServerAddress]*erasure_coding.ShardsInfo{}, + wantID: 0, + wantMissed: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + id, missed := missingDataShard(tt.holders, dataShards) + if missed != tt.wantMissed { + t.Fatalf("missingDataShard = %v, want %v", missed, tt.wantMissed) + } + if missed && id != tt.wantID { + t.Errorf("missing shard id = %d, want %d", id, tt.wantID) + } + }) + } +}