mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-08-16 12:16:36 +00:00
Chris LuandGitHub
5ec813b4f1
* topology: follow a volume that moved between a server's disks The heartbeat diff asked only whether a volume id was reported anywhere on the node, so a volume that moved to a disk of another type stayed on the disk it left as well. The master then held two copies of it forever: the volume count was overstated, and GetVolumesById returned whichever disk the map iterated first, so lookups could hand back the disk the volume had already left. Track which disk types the heartbeat named each volume on, and treat a volume named on another disk as absent from this one. Disk types are interned to an index because a server reports a handful of them across hundreds of thousands of volumes. A volume named on two disks at once is a stale twin rather than a move, and is still kept on both -- dropping one would tell the master a replica vanished. Only a volume named twice on one disk type is unrepresentable, so that is now what marks the node, rather than any repeat of an id. * master: do not tell clients a moved volume left the node A volume moved between a node's disks is removed from one and added to the other, so it lands in both lists of the same heartbeat. Clients apply additions before deletions, so the removal wins and they end up with no location for a volume that never went anywhere. Skip removals for volumes the node still holds, as the ec shard paths already do, and update the topology before judging the delta removals so an unmount that really did happen is still reported. * trim the comments on this change to the parts that are not evident * master: judge a volume removal on normal replicas alone HasVolumesById answers for ec shards as well, so a replica encoded into ec shards looked like it was still on the node and clients were never told the normal location had gone. They hold normal and ec locations separately and prefer the normal one from the same generation, so that location would have gone on shadowing the shards.
99 lines
4.0 KiB
Go
99 lines
4.0 KiB
Go
package weed_server
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/sequence"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
"github.com/seaweedfs/seaweedfs/weed/topology"
|
|
)
|
|
|
|
func moveTestNode(t *testing.T) (*topology.Topology, *topology.DataNode) {
|
|
t.Helper()
|
|
topo := topology.NewTopology("test", sequence.NewMemorySequencer(), 32*1024*1024*1024, 5, false)
|
|
dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
|
GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100, "ssd": 100})
|
|
return topo, dn
|
|
}
|
|
|
|
func moveTestVolume(diskType string, diskId uint32) *master_pb.VolumeInformationMessage {
|
|
return &master_pb.VolumeInformationMessage{
|
|
Id: 1, Size: 1024, Collection: "c", Version: 3, DiskType: diskType, DiskId: diskId,
|
|
}
|
|
}
|
|
|
|
// Clients apply additions before deletions, so a move reported as both would
|
|
// leave them with no location for a volume that never went anywhere.
|
|
func TestVolumeMovedBetweenDisksIsNotBroadcastAsRemoved(t *testing.T) {
|
|
topo, dn := moveTestNode(t)
|
|
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{moveTestVolume("", 0)}, dn)
|
|
|
|
_, deleted := topo.SyncDataNodeRegistration(
|
|
[]*master_pb.VolumeInformationMessage{moveTestVolume("ssd", 1)}, dn)
|
|
|
|
if len(deleted) != 1 {
|
|
t.Fatalf("expected the move to remove the volume from the disk it left, got %d removals", len(deleted))
|
|
}
|
|
if shouldBroadcastVolumeRemoval(dn, needle.VolumeId(1)) {
|
|
t.Error("clients would be told a volume left a node that still has it")
|
|
}
|
|
}
|
|
|
|
func TestVolumeGoneFromTheNodeIsBroadcastAsRemoved(t *testing.T) {
|
|
topo, dn := moveTestNode(t)
|
|
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{moveTestVolume("", 0)}, dn)
|
|
|
|
if _, deleted := topo.SyncDataNodeRegistration(nil, dn); len(deleted) != 1 {
|
|
t.Fatalf("expected the volume to be removed, got %d removals", len(deleted))
|
|
}
|
|
if !shouldBroadcastVolumeRemoval(dn, needle.VolumeId(1)) {
|
|
t.Error("clients were not told about a volume that really did leave the node")
|
|
}
|
|
}
|
|
|
|
func TestVolumeRemountedOnAnotherDiskIsNotBroadcastAsRemoved(t *testing.T) {
|
|
topo, dn := moveTestNode(t)
|
|
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{moveTestVolume("", 0)}, dn)
|
|
|
|
topo.IncrementalSyncDataNodeRegistration(
|
|
[]*master_pb.VolumeShortInformationMessage{{Id: 1, Collection: "c", DiskType: "ssd"}},
|
|
[]*master_pb.VolumeShortInformationMessage{{Id: 1, Collection: "c", DiskType: ""}},
|
|
dn)
|
|
|
|
if shouldBroadcastVolumeRemoval(dn, needle.VolumeId(1)) {
|
|
t.Error("clients would be told a volume left a node that still has it on another disk")
|
|
}
|
|
}
|
|
|
|
// Fails if the topology update stops running before the removals are judged.
|
|
func TestVolumeUnmountedViaDeltaIsBroadcastAsRemoved(t *testing.T) {
|
|
topo, dn := moveTestNode(t)
|
|
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{moveTestVolume("", 0)}, dn)
|
|
|
|
topo.IncrementalSyncDataNodeRegistration(nil,
|
|
[]*master_pb.VolumeShortInformationMessage{{Id: 1, Collection: "c", DiskType: ""}}, dn)
|
|
|
|
if !shouldBroadcastVolumeRemoval(dn, needle.VolumeId(1)) {
|
|
t.Error("clients were not told about a volume that really was unmounted")
|
|
}
|
|
}
|
|
|
|
// Clients track normal and ec locations separately, and prefer the normal one
|
|
// when both come from the same generation. A replica that became ec shards has
|
|
// genuinely left, so the removal must still go out.
|
|
func TestVolumeReplacedByEcShardsIsBroadcastAsRemoved(t *testing.T) {
|
|
topo, dn := moveTestNode(t)
|
|
topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{moveTestVolume("", 0)}, dn)
|
|
topo.SyncDataNodeEcShards([]*master_pb.VolumeEcShardInformationMessage{
|
|
{Id: 1, Collection: "c", EcIndexBits: 0x3fff},
|
|
}, dn)
|
|
|
|
if _, deleted := topo.SyncDataNodeRegistration(nil, dn); len(deleted) != 1 {
|
|
t.Fatalf("expected the normal volume to be removed, got %d removals", len(deleted))
|
|
}
|
|
if !shouldBroadcastVolumeRemoval(dn, needle.VolumeId(1)) {
|
|
t.Error("clients kept a normal-volume location for a replica that became ec shards")
|
|
}
|
|
}
|