mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-26 01:44:48 +00:00
refactor: close bounded phase 16 restart truth seams
Bind non-authoritative inventory, restart primary-truth rebasing, and sparse replica readiness retention into the heartbeat/master seam, and package the bounded finish-line checkpoint with explicit claims, non-claims, and proof commands. Made-with: Cursor
This commit is contained in:
@@ -24,20 +24,21 @@ const (
|
||||
|
||||
// ReplicaInfo tracks one replica of a block volume (CP8-2).
|
||||
type ReplicaInfo struct {
|
||||
Server string // replica VS address
|
||||
Path string // file path on replica VS
|
||||
ISCSIAddr string // iSCSI target address
|
||||
IQN string // iSCSI qualified name
|
||||
NvmeAddr string // NVMe/TCP target address (ip:port), empty if NVMe disabled
|
||||
NQN string // NVMe subsystem NQN, empty if NVMe disabled
|
||||
DataAddr string // WAL receiver data listen addr
|
||||
CtrlAddr string // WAL receiver ctrl listen addr
|
||||
Ready bool // receiver/publish readiness confirmed by replica heartbeat
|
||||
HealthScore float64 // from heartbeat (0.0-1.0)
|
||||
WALHeadLSN uint64 // from heartbeat
|
||||
WALLag uint64 // computed: primary.WALHeadLSN - replica.WALHeadLSN
|
||||
LastHeartbeat time.Time // last heartbeat received from this replica
|
||||
Role uint32 // replica role (RoleReplica, RoleRebuilding, etc.)
|
||||
Server string // replica VS address
|
||||
Path string // file path on replica VS
|
||||
ISCSIAddr string // iSCSI target address
|
||||
IQN string // iSCSI qualified name
|
||||
NvmeAddr string // NVMe/TCP target address (ip:port), empty if NVMe disabled
|
||||
NQN string // NVMe subsystem NQN, empty if NVMe disabled
|
||||
DataAddr string // WAL receiver data listen addr
|
||||
CtrlAddr string // WAL receiver ctrl listen addr
|
||||
Ready bool // receiver/publish readiness confirmed by replica heartbeat
|
||||
HasExplicitReady bool // whether replica readiness was carried explicitly on heartbeat
|
||||
HealthScore float64 // from heartbeat (0.0-1.0)
|
||||
WALHeadLSN uint64 // from heartbeat
|
||||
WALLag uint64 // computed: primary.WALHeadLSN - replica.WALHeadLSN
|
||||
LastHeartbeat time.Time // last heartbeat received from this replica
|
||||
Role uint32 // replica role (RoleReplica, RoleRebuilding, etc.)
|
||||
}
|
||||
|
||||
const (
|
||||
@@ -485,6 +486,10 @@ type HeartbeatResult struct {
|
||||
}
|
||||
|
||||
func (r *BlockVolumeRegistry) UpdateFullHeartbeat(server string, infos []*master_pb.BlockVolumeInfoMessage, nvmeAddr string) HeartbeatResult {
|
||||
return r.UpdateFullHeartbeatWithInventoryAuthority(server, infos, nvmeAddr, true)
|
||||
}
|
||||
|
||||
func (r *BlockVolumeRegistry) UpdateFullHeartbeatWithInventoryAuthority(server string, infos []*master_pb.BlockVolumeInfoMessage, nvmeAddr string, blockInventoryAuthoritative bool) HeartbeatResult {
|
||||
var result HeartbeatResult
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
@@ -499,44 +504,46 @@ func (r *BlockVolumeRegistry) UpdateFullHeartbeat(server string, infos []*master
|
||||
}
|
||||
|
||||
// Find entries for this server that are NOT reported -> reconcile.
|
||||
if names, ok := r.byServer[server]; ok {
|
||||
for name := range names {
|
||||
entry := r.volumes[name]
|
||||
if entry == nil {
|
||||
continue
|
||||
}
|
||||
if entry.VolumeServer == server {
|
||||
// Server is the primary: check if primary path is reported.
|
||||
if _, found := reported[entry.Path]; !found {
|
||||
// B-10: Do not delete entries with a coordinated expand in flight.
|
||||
// The primary may have restarted mid-expand; deleting the entry
|
||||
// would orphan the volume and strand the expand coordinator.
|
||||
if entry.ExpandInProgress {
|
||||
glog.Warningf("block registry: skipping stale-cleanup for %q (ExpandInProgress=true, server=%s)",
|
||||
name, server)
|
||||
continue
|
||||
}
|
||||
delete(r.volumes, name)
|
||||
delete(names, name)
|
||||
// Also clean up replica entries from byServer.
|
||||
for _, ri := range entry.Replicas {
|
||||
r.removeFromServer(ri.Server, name)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Server is a replica: check if replica path is reported.
|
||||
ri := entry.ReplicaByServer(server)
|
||||
if ri == nil {
|
||||
// No replica record — stale byServer index, just clean up.
|
||||
delete(names, name)
|
||||
if blockInventoryAuthoritative {
|
||||
if names, ok := r.byServer[server]; ok {
|
||||
for name := range names {
|
||||
entry := r.volumes[name]
|
||||
if entry == nil {
|
||||
continue
|
||||
}
|
||||
if _, found := reported[ri.Path]; !found {
|
||||
// Replica path not reported — remove this replica, NOT the whole volume.
|
||||
r.removeReplicaLocked(entry, server, name)
|
||||
delete(names, name)
|
||||
glog.V(0).Infof("block registry: removed stale replica %s for %q (path %s not in heartbeat)",
|
||||
server, name, ri.Path)
|
||||
if entry.VolumeServer == server {
|
||||
// Server is the primary: check if primary path is reported.
|
||||
if _, found := reported[entry.Path]; !found {
|
||||
// B-10: Do not delete entries with a coordinated expand in flight.
|
||||
// The primary may have restarted mid-expand; deleting the entry
|
||||
// would orphan the volume and strand the expand coordinator.
|
||||
if entry.ExpandInProgress {
|
||||
glog.Warningf("block registry: skipping stale-cleanup for %q (ExpandInProgress=true, server=%s)",
|
||||
name, server)
|
||||
continue
|
||||
}
|
||||
delete(r.volumes, name)
|
||||
delete(names, name)
|
||||
// Also clean up replica entries from byServer.
|
||||
for _, ri := range entry.Replicas {
|
||||
r.removeFromServer(ri.Server, name)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Server is a replica: check if replica path is reported.
|
||||
ri := entry.ReplicaByServer(server)
|
||||
if ri == nil {
|
||||
// No replica record — stale byServer index, just clean up.
|
||||
delete(names, name)
|
||||
continue
|
||||
}
|
||||
if _, found := reported[ri.Path]; !found {
|
||||
// Replica path not reported — remove this replica, NOT the whole volume.
|
||||
r.removeReplicaLocked(entry, server, name)
|
||||
delete(names, name)
|
||||
glog.V(0).Infof("block registry: removed stale replica %s for %q (path %s not in heartbeat)",
|
||||
server, name, ri.Path)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -606,20 +613,32 @@ func (r *BlockVolumeRegistry) UpdateFullHeartbeat(server string, infos []*master
|
||||
existing, dup := r.volumes[name]
|
||||
if !dup {
|
||||
entry := &BlockVolumeEntry{
|
||||
Name: name,
|
||||
VolumeServer: server,
|
||||
Path: info.Path,
|
||||
SizeBytes: info.VolumeSize,
|
||||
Epoch: info.Epoch,
|
||||
Role: info.Role,
|
||||
Status: StatusActive,
|
||||
LastLeaseGrant: time.Now(),
|
||||
LeaseTTL: 30 * time.Second,
|
||||
HealthScore: info.HealthScore,
|
||||
TransportDegraded: info.ReplicaDegraded,
|
||||
WALHeadLSN: info.WalHeadLsn,
|
||||
DurabilityMode: info.DurabilityMode,
|
||||
Name: name,
|
||||
VolumeServer: server,
|
||||
Path: info.Path,
|
||||
SizeBytes: info.VolumeSize,
|
||||
Epoch: info.Epoch,
|
||||
Role: info.Role,
|
||||
Status: StatusActive,
|
||||
LastLeaseGrant: time.Now(),
|
||||
LeaseTTL: 30 * time.Second,
|
||||
HealthScore: info.HealthScore,
|
||||
TransportDegraded: info.ReplicaDegraded,
|
||||
NeedsRebuild: false,
|
||||
HasNeedsRebuild: false,
|
||||
PublishHealthy: false,
|
||||
HasPublishHealthy: false,
|
||||
HeartbeatVolumeMode: "",
|
||||
HasHeartbeatVolumeMode: false,
|
||||
HeartbeatVolumeReason: "",
|
||||
HasHeartbeatVolumeReason: false,
|
||||
WALHeadLSN: info.WalHeadLsn,
|
||||
DurabilityMode: info.DurabilityMode,
|
||||
}
|
||||
entry.NeedsRebuild, entry.HasNeedsRebuild = primaryNeedsRebuildObservedFromHeartbeat(info)
|
||||
entry.PublishHealthy, entry.HasPublishHealthy = primaryPublishHealthyObservedFromHeartbeat(info)
|
||||
entry.HeartbeatVolumeMode, entry.HasHeartbeatVolumeMode = primaryVolumeModeObservedFromHeartbeat(info)
|
||||
entry.HeartbeatVolumeReason, entry.HasHeartbeatVolumeReason = primaryVolumeReasonObservedFromHeartbeat(info)
|
||||
if info.ReplicaDataAddr != "" {
|
||||
entry.ReplicaDataAddr = info.ReplicaDataAddr
|
||||
}
|
||||
@@ -662,10 +681,7 @@ func (r *BlockVolumeRegistry) applyPrimaryHeartbeatObservation(existing *BlockVo
|
||||
existing.LastLeaseGrant = time.Now()
|
||||
existing.HealthScore = info.HealthScore
|
||||
existing.TransportDegraded = info.ReplicaDegraded
|
||||
existing.NeedsRebuild, existing.HasNeedsRebuild = primaryNeedsRebuildObservedFromHeartbeat(info)
|
||||
existing.PublishHealthy, existing.HasPublishHealthy = primaryPublishHealthyObservedFromHeartbeat(info)
|
||||
existing.HeartbeatVolumeMode, existing.HasHeartbeatVolumeMode = primaryVolumeModeObservedFromHeartbeat(info)
|
||||
existing.HeartbeatVolumeReason, existing.HasHeartbeatVolumeReason = primaryVolumeReasonObservedFromHeartbeat(info)
|
||||
applyExplicitPrimaryTruthFromHeartbeat(existing, info, true)
|
||||
existing.WALHeadLSN = info.WalHeadLsn
|
||||
// F3: only update DurabilityMode when non-empty (prevents older VS from clearing strict mode).
|
||||
if info.DurabilityMode != "" {
|
||||
@@ -707,7 +723,7 @@ func (r *BlockVolumeRegistry) applyReplicaHeartbeatObservation(existing *BlockVo
|
||||
existing.Replicas[i].Role = blockvol.RoleToWire(blockvol.RoleReplica)
|
||||
existing.Replicas[i].NvmeAddr = info.NvmeAddr
|
||||
existing.Replicas[i].NQN = info.Nqn
|
||||
existing.Replicas[i].Ready = replicaReadyObservedFromHeartbeat(info)
|
||||
applyReplicaReadyFromHeartbeat(&existing.Replicas[i], info, true)
|
||||
if existing.WALHeadLSN > info.WalHeadLsn {
|
||||
existing.Replicas[i].WALLag = existing.WALHeadLSN - info.WalHeadLsn
|
||||
} else {
|
||||
@@ -743,6 +759,22 @@ func (r *BlockVolumeRegistry) applyReplicaHeartbeatObservation(existing *BlockVo
|
||||
existing.recomputeReplicaState()
|
||||
}
|
||||
|
||||
func applyReplicaReadyFromHeartbeat(replica *ReplicaInfo, info *master_pb.BlockVolumeInfoMessage, preserveWhenAbsent bool) {
|
||||
if replica == nil || info == nil {
|
||||
return
|
||||
}
|
||||
if info.ReplicaReady != nil {
|
||||
replica.Ready = info.GetReplicaReady()
|
||||
replica.HasExplicitReady = true
|
||||
return
|
||||
}
|
||||
if preserveWhenAbsent && replica.HasExplicitReady {
|
||||
return
|
||||
}
|
||||
replica.Ready = info.ReplicaDataAddr != "" && info.ReplicaCtrlAddr != ""
|
||||
replica.HasExplicitReady = false
|
||||
}
|
||||
|
||||
func replicaReadyObservedFromHeartbeat(info *master_pb.BlockVolumeInfoMessage) bool {
|
||||
if info == nil {
|
||||
return false
|
||||
@@ -793,6 +825,40 @@ func primaryVolumeReasonObservedFromHeartbeat(info *master_pb.BlockVolumeInfoMes
|
||||
return "", false
|
||||
}
|
||||
|
||||
func applyExplicitPrimaryTruthFromHeartbeat(existing *BlockVolumeEntry, info *master_pb.BlockVolumeInfoMessage, preserveWhenAbsent bool) {
|
||||
if existing == nil || info == nil {
|
||||
return
|
||||
}
|
||||
if needsRebuild, ok := primaryNeedsRebuildObservedFromHeartbeat(info); ok {
|
||||
existing.NeedsRebuild = needsRebuild
|
||||
existing.HasNeedsRebuild = true
|
||||
} else if !preserveWhenAbsent {
|
||||
existing.NeedsRebuild = false
|
||||
existing.HasNeedsRebuild = false
|
||||
}
|
||||
if publishHealthy, ok := primaryPublishHealthyObservedFromHeartbeat(info); ok {
|
||||
existing.PublishHealthy = publishHealthy
|
||||
existing.HasPublishHealthy = true
|
||||
} else if !preserveWhenAbsent {
|
||||
existing.PublishHealthy = false
|
||||
existing.HasPublishHealthy = false
|
||||
}
|
||||
if mode, ok := primaryVolumeModeObservedFromHeartbeat(info); ok {
|
||||
existing.HeartbeatVolumeMode = mode
|
||||
existing.HasHeartbeatVolumeMode = true
|
||||
} else if !preserveWhenAbsent {
|
||||
existing.HeartbeatVolumeMode = ""
|
||||
existing.HasHeartbeatVolumeMode = false
|
||||
}
|
||||
if reason, ok := primaryVolumeReasonObservedFromHeartbeat(info); ok {
|
||||
existing.HeartbeatVolumeReason = reason
|
||||
existing.HasHeartbeatVolumeReason = true
|
||||
} else if !preserveWhenAbsent {
|
||||
existing.HeartbeatVolumeReason = ""
|
||||
existing.HasHeartbeatVolumeReason = false
|
||||
}
|
||||
}
|
||||
|
||||
func validHeartbeatVolumeMode(mode string) bool {
|
||||
switch mode {
|
||||
case "allocated_only", "bootstrap_pending", "publish_healthy", "degraded", "needs_rebuild":
|
||||
@@ -907,6 +973,7 @@ func (r *BlockVolumeRegistry) demoteExistingToReplica(name string, existing *Blo
|
||||
}
|
||||
existing.NvmeAddr = info.NvmeAddr
|
||||
existing.NQN = info.Nqn
|
||||
applyExplicitPrimaryTruthFromHeartbeat(existing, info, false)
|
||||
|
||||
// Add old primary as replica.
|
||||
existing.Replicas = append(existing.Replicas, oldReplica)
|
||||
@@ -917,6 +984,7 @@ func (r *BlockVolumeRegistry) demoteExistingToReplica(name string, existing *Blo
|
||||
existing.ReplicaServer = oldReplica.Server
|
||||
existing.ReplicaPath = oldReplica.Path
|
||||
}
|
||||
existing.recomputeReplicaState()
|
||||
}
|
||||
|
||||
// upsertServerAsReplica adds or updates the server as a replica for the existing entry.
|
||||
@@ -944,6 +1012,7 @@ func (r *BlockVolumeRegistry) upsertServerAsReplica(name string, existing *Block
|
||||
existing.Replicas[i].Role = replicaRole
|
||||
existing.Replicas[i].NvmeAddr = info.NvmeAddr
|
||||
existing.Replicas[i].NQN = info.Nqn
|
||||
applyReplicaReadyFromHeartbeat(&existing.Replicas[i], info, true)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -962,6 +1031,7 @@ func (r *BlockVolumeRegistry) upsertServerAsReplica(name string, existing *Block
|
||||
NvmeAddr: info.NvmeAddr,
|
||||
NQN: info.Nqn,
|
||||
}
|
||||
applyReplicaReadyFromHeartbeat(&ri, info, false)
|
||||
existing.Replicas = append(existing.Replicas, ri)
|
||||
r.addToServer(newServer, name)
|
||||
if len(existing.Replicas) == 1 {
|
||||
|
||||
@@ -114,6 +114,28 @@ func TestRegistry_UpdateFullHeartbeat(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistry_UpdateFullHeartbeatWithInventoryAuthority_NonAuthoritativeEmptyDoesNotDelete(t *testing.T) {
|
||||
r := NewBlockVolumeRegistry()
|
||||
r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk", Status: StatusActive})
|
||||
|
||||
r.UpdateFullHeartbeatWithInventoryAuthority("s1", nil, "", false)
|
||||
|
||||
if _, ok := r.Lookup("vol1"); !ok {
|
||||
t.Fatal("non-authoritative empty full heartbeat should not delete vol1")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistry_UpdateFullHeartbeatWithInventoryAuthority_AuthoritativeEmptyStillDeletes(t *testing.T) {
|
||||
r := NewBlockVolumeRegistry()
|
||||
r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk", Status: StatusActive})
|
||||
|
||||
r.UpdateFullHeartbeatWithInventoryAuthority("s1", nil, "", true)
|
||||
|
||||
if _, ok := r.Lookup("vol1"); ok {
|
||||
t.Fatal("authoritative empty full heartbeat should still delete vol1")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistry_UpdateDeltaHeartbeat(t *testing.T) {
|
||||
r := NewBlockVolumeRegistry()
|
||||
r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk", Status: StatusPending})
|
||||
@@ -1663,6 +1685,110 @@ func TestMasterRestart_HigherEpochWins(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMasterRestart_HigherEpochRebasesExplicitPrimaryTruth(t *testing.T) {
|
||||
r := NewBlockVolumeRegistry()
|
||||
|
||||
oldNeedsRebuild := false
|
||||
oldPublishHealthy := true
|
||||
oldMode := "publish_healthy"
|
||||
oldReason := ""
|
||||
r.UpdateFullHeartbeat("vs1:9333", []*master_pb.BlockVolumeInfoMessage{{
|
||||
Path: "/data/vol1.blk",
|
||||
Epoch: 5,
|
||||
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
||||
WalHeadLsn: 100,
|
||||
PublishHealthy: &oldPublishHealthy,
|
||||
NeedsRebuild: &oldNeedsRebuild,
|
||||
VolumeMode: &oldMode,
|
||||
VolumeModeReason: &oldReason,
|
||||
VolumeSize: 1 << 30,
|
||||
}}, "")
|
||||
|
||||
newNeedsRebuild := true
|
||||
newPublishHealthy := false
|
||||
newMode := "needs_rebuild"
|
||||
newReason := "gap_too_large"
|
||||
r.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{{
|
||||
Path: "/data/vol1.blk",
|
||||
Epoch: 6,
|
||||
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
||||
WalHeadLsn: 150,
|
||||
PublishHealthy: &newPublishHealthy,
|
||||
NeedsRebuild: &newNeedsRebuild,
|
||||
VolumeMode: &newMode,
|
||||
VolumeModeReason: &newReason,
|
||||
VolumeSize: 1 << 30,
|
||||
}}, "")
|
||||
|
||||
entry, _ := r.Lookup("vol1")
|
||||
if entry.VolumeServer != "vs2:9333" {
|
||||
t.Fatalf("expected vs2 as primary after rebase, got %q", entry.VolumeServer)
|
||||
}
|
||||
if !entry.HasNeedsRebuild || !entry.NeedsRebuild {
|
||||
t.Fatalf("expected new primary explicit needs_rebuild truth, entry=%+v", entry)
|
||||
}
|
||||
if !entry.HasPublishHealthy || entry.PublishHealthy {
|
||||
t.Fatalf("expected new primary explicit false publish_healthy truth, entry=%+v", entry)
|
||||
}
|
||||
if !entry.HasHeartbeatVolumeMode || entry.HeartbeatVolumeMode != "needs_rebuild" {
|
||||
t.Fatalf("expected new primary explicit volume_mode truth, entry=%+v", entry)
|
||||
}
|
||||
if !entry.HasHeartbeatVolumeReason || entry.HeartbeatVolumeReason != "gap_too_large" {
|
||||
t.Fatalf("expected new primary explicit volume_mode_reason truth, entry=%+v", entry)
|
||||
}
|
||||
if entry.VolumeMode != "needs_rebuild" {
|
||||
t.Fatalf("expected outward mode to follow winning primary truth, got %q", entry.VolumeMode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMasterRestart_HigherEpochSparsePrimaryClearsOldExplicitTruth(t *testing.T) {
|
||||
r := NewBlockVolumeRegistry()
|
||||
|
||||
oldNeedsRebuild := false
|
||||
oldPublishHealthy := true
|
||||
oldMode := "publish_healthy"
|
||||
oldReason := ""
|
||||
r.UpdateFullHeartbeat("vs1:9333", []*master_pb.BlockVolumeInfoMessage{{
|
||||
Path: "/data/vol1.blk",
|
||||
Epoch: 5,
|
||||
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
||||
WalHeadLsn: 100,
|
||||
PublishHealthy: &oldPublishHealthy,
|
||||
NeedsRebuild: &oldNeedsRebuild,
|
||||
VolumeMode: &oldMode,
|
||||
VolumeModeReason: &oldReason,
|
||||
VolumeSize: 1 << 30,
|
||||
}}, "")
|
||||
|
||||
r.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{{
|
||||
Path: "/data/vol1.blk",
|
||||
Epoch: 6,
|
||||
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
||||
WalHeadLsn: 150,
|
||||
VolumeSize: 1 << 30,
|
||||
}}, "")
|
||||
|
||||
entry, _ := r.Lookup("vol1")
|
||||
if entry.VolumeServer != "vs2:9333" {
|
||||
t.Fatalf("expected vs2 as primary after sparse rebase, got %q", entry.VolumeServer)
|
||||
}
|
||||
if entry.HasNeedsRebuild || entry.NeedsRebuild {
|
||||
t.Fatalf("sparse new primary should clear old explicit needs_rebuild truth, entry=%+v", entry)
|
||||
}
|
||||
if entry.HasPublishHealthy || entry.PublishHealthy {
|
||||
t.Fatalf("sparse new primary should clear old explicit publish_healthy truth, entry=%+v", entry)
|
||||
}
|
||||
if entry.HasHeartbeatVolumeMode || entry.HeartbeatVolumeMode != "" {
|
||||
t.Fatalf("sparse new primary should clear old explicit volume_mode truth, entry=%+v", entry)
|
||||
}
|
||||
if entry.HasHeartbeatVolumeReason || entry.HeartbeatVolumeReason != "" {
|
||||
t.Fatalf("sparse new primary should clear old explicit volume_mode_reason truth, entry=%+v", entry)
|
||||
}
|
||||
if entry.VolumeMode == "publish_healthy" {
|
||||
t.Fatalf("sparse new primary should not retain stale publish_healthy mode, entry=%+v", entry)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMasterRestart_LowerEpochBecomesReplica(t *testing.T) {
|
||||
r := NewBlockVolumeRegistry()
|
||||
|
||||
@@ -2180,6 +2306,80 @@ func TestRegistry_UpdateFullHeartbeat_ReplicaReadyFallsBackToAddressesWhenFieldA
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistry_UpdateFullHeartbeat_ReplicaReadyMissingFieldPreservesAcceptedExplicitTruth(t *testing.T) {
|
||||
r := NewBlockVolumeRegistry()
|
||||
if err := r.Register(&BlockVolumeEntry{
|
||||
Name: "vol-master-ready-preserve-explicit",
|
||||
VolumeServer: "primary-server:8080",
|
||||
Path: "/blocks/vol-master-ready-preserve-explicit-primary.blk",
|
||||
Status: StatusActive,
|
||||
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
||||
ReplicaFactor: 2,
|
||||
Replicas: []ReplicaInfo{{
|
||||
Server: "replica-server:8080",
|
||||
Path: "/blocks/vol-master-ready-preserve-explicit.blk",
|
||||
}},
|
||||
}); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
|
||||
replicaReady := false
|
||||
r.UpdateFullHeartbeat("replica-server:8080", []*master_pb.BlockVolumeInfoMessage{{
|
||||
Path: "/blocks/vol-master-ready-preserve-explicit.blk",
|
||||
ReplicaDataAddr: "10.0.0.2:4260",
|
||||
ReplicaCtrlAddr: "10.0.0.2:4261",
|
||||
ReplicaReady: &replicaReady,
|
||||
}}, "")
|
||||
r.UpdateFullHeartbeat("replica-server:8080", []*master_pb.BlockVolumeInfoMessage{{
|
||||
Path: "/blocks/vol-master-ready-preserve-explicit.blk",
|
||||
ReplicaDataAddr: "10.0.0.2:4260",
|
||||
ReplicaCtrlAddr: "10.0.0.2:4261",
|
||||
}}, "")
|
||||
|
||||
entry, _ := r.Lookup("vol-master-ready-preserve-explicit")
|
||||
if !entry.Replicas[0].HasExplicitReady {
|
||||
t.Fatalf("expected accepted explicit replica readiness to remain marked explicit, entry=%+v", entry)
|
||||
}
|
||||
if entry.Replicas[0].Ready {
|
||||
t.Fatalf("missing-field replica heartbeat should preserve explicit false ready truth, entry=%+v", entry)
|
||||
}
|
||||
if entry.ReplicaReady {
|
||||
t.Fatalf("aggregate replica ready should remain false after sparse heartbeat, entry=%+v", entry)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistry_UpdateFullHeartbeat_ReplicaReadyMissingFieldFreshEntryStillFallsBack(t *testing.T) {
|
||||
r := NewBlockVolumeRegistry()
|
||||
if err := r.Register(&BlockVolumeEntry{
|
||||
Name: "vol-master-ready-fresh-fallback",
|
||||
VolumeServer: "primary-server:8080",
|
||||
Path: "/blocks/vol-master-ready-fresh-fallback-primary.blk",
|
||||
Status: StatusActive,
|
||||
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
||||
ReplicaFactor: 2,
|
||||
Replicas: []ReplicaInfo{{
|
||||
Server: "replica-server:8080",
|
||||
Path: "/blocks/vol-master-ready-fresh-fallback.blk",
|
||||
}},
|
||||
}); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
|
||||
r.UpdateFullHeartbeat("replica-server:8080", []*master_pb.BlockVolumeInfoMessage{{
|
||||
Path: "/blocks/vol-master-ready-fresh-fallback.blk",
|
||||
ReplicaDataAddr: "10.0.0.2:4260",
|
||||
ReplicaCtrlAddr: "10.0.0.2:4261",
|
||||
}}, "")
|
||||
|
||||
entry, _ := r.Lookup("vol-master-ready-fresh-fallback")
|
||||
if entry.Replicas[0].HasExplicitReady {
|
||||
t.Fatalf("fresh missing-field replica heartbeat should not invent explicit readiness, entry=%+v", entry)
|
||||
}
|
||||
if !entry.Replicas[0].Ready || !entry.ReplicaReady {
|
||||
t.Fatalf("fresh missing-field replica heartbeat should still fall back to addresses, entry=%+v", entry)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistry_UpdateFullHeartbeat_ConsumesExplicitNeedsRebuildFromPrimaryHeartbeat(t *testing.T) {
|
||||
r := NewBlockVolumeRegistry()
|
||||
if err := r.Register(&BlockVolumeEntry{
|
||||
@@ -2479,3 +2679,175 @@ func TestRegistry_UpdateFullHeartbeat_VolumeModeFallsBackWhenFieldAbsent(t *test
|
||||
t.Fatalf("expected fallback reconstructed degraded mode, got %q", entry.VolumeMode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistry_UpdateFullHeartbeat_AutoRegisterPreservesExplicitPrimaryTruthOnRestart(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
needsRebuild bool
|
||||
publishHealthy bool
|
||||
mode string
|
||||
reason string
|
||||
wantMode string
|
||||
wantReason string
|
||||
}{
|
||||
{
|
||||
name: "publish_healthy",
|
||||
needsRebuild: false,
|
||||
publishHealthy: true,
|
||||
mode: "publish_healthy",
|
||||
reason: "",
|
||||
wantMode: "publish_healthy",
|
||||
wantReason: "",
|
||||
},
|
||||
{
|
||||
name: "needs_rebuild",
|
||||
needsRebuild: true,
|
||||
publishHealthy: false,
|
||||
mode: "needs_rebuild",
|
||||
reason: "gap_too_large",
|
||||
wantMode: "needs_rebuild",
|
||||
wantReason: "gap_too_large",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := NewBlockVolumeRegistry()
|
||||
r.MarkBlockCapable("primary-server:8080")
|
||||
|
||||
mode := tt.mode
|
||||
reason := tt.reason
|
||||
needsRebuild := tt.needsRebuild
|
||||
publishHealthy := tt.publishHealthy
|
||||
r.UpdateFullHeartbeat("primary-server:8080", []*master_pb.BlockVolumeInfoMessage{{
|
||||
Path: "/blocks/vol-restart-" + tt.name + ".blk",
|
||||
VolumeSize: 1 << 30,
|
||||
BlockSize: 4096,
|
||||
Epoch: 7,
|
||||
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
||||
NeedsRebuild: &needsRebuild,
|
||||
PublishHealthy: &publishHealthy,
|
||||
VolumeMode: &mode,
|
||||
VolumeModeReason: &reason,
|
||||
}}, "")
|
||||
|
||||
entry, ok := r.Lookup("vol-restart-" + tt.name)
|
||||
if !ok {
|
||||
t.Fatalf("expected auto-registered entry for %q", tt.name)
|
||||
}
|
||||
if !entry.HasNeedsRebuild || entry.NeedsRebuild != tt.needsRebuild {
|
||||
t.Fatalf("needs_rebuild truth lost during auto-register, entry=%+v", entry)
|
||||
}
|
||||
if !entry.HasPublishHealthy || entry.PublishHealthy != tt.publishHealthy {
|
||||
t.Fatalf("publish_healthy truth lost during auto-register, entry=%+v", entry)
|
||||
}
|
||||
if !entry.HasHeartbeatVolumeMode || entry.HeartbeatVolumeMode != tt.mode {
|
||||
t.Fatalf("volume_mode truth lost during auto-register, entry=%+v", entry)
|
||||
}
|
||||
if !entry.HasHeartbeatVolumeReason || entry.HeartbeatVolumeReason != tt.reason {
|
||||
t.Fatalf("volume_mode_reason truth lost during auto-register, entry=%+v", entry)
|
||||
}
|
||||
if entry.VolumeMode != tt.wantMode {
|
||||
t.Fatalf("expected outward volume_mode %q after auto-register, got %q", tt.wantMode, entry.VolumeMode)
|
||||
}
|
||||
|
||||
info := entryToVolumeInfo(&entry, true)
|
||||
if info.VolumeMode != tt.wantMode {
|
||||
t.Fatalf("expected outward API volume_mode %q after auto-register, got %q", tt.wantMode, info.VolumeMode)
|
||||
}
|
||||
if info.VolumeModeReason != tt.wantReason {
|
||||
t.Fatalf("expected outward API volume_mode_reason %q after auto-register, got %q", tt.wantReason, info.VolumeModeReason)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistry_UpdateFullHeartbeat_MissingFieldsPreserveAcceptedExplicitPrimaryTruth(t *testing.T) {
|
||||
r := NewBlockVolumeRegistry()
|
||||
if err := r.Register(&BlockVolumeEntry{
|
||||
Name: "vol-master-preserve-explicit-truth",
|
||||
VolumeServer: "primary-server:8080",
|
||||
Path: "/blocks/vol-master-preserve-explicit-truth-primary.blk",
|
||||
Status: StatusActive,
|
||||
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
||||
ReplicaFactor: 2,
|
||||
Replicas: []ReplicaInfo{{
|
||||
Server: "replica-server:8080",
|
||||
Path: "/blocks/vol-master-preserve-explicit-truth-replica.blk",
|
||||
Ready: true,
|
||||
Role: blockvol.RoleToWire(blockvol.RoleRebuilding),
|
||||
}},
|
||||
}); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
|
||||
needsRebuild := false
|
||||
publishHealthy := false
|
||||
mode := "degraded"
|
||||
reason := "barrier_timeout"
|
||||
r.UpdateFullHeartbeat("primary-server:8080", []*master_pb.BlockVolumeInfoMessage{{
|
||||
Path: "/blocks/vol-master-preserve-explicit-truth-primary.blk",
|
||||
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
||||
ReplicaDegraded: true,
|
||||
NeedsRebuild: &needsRebuild,
|
||||
PublishHealthy: &publishHealthy,
|
||||
VolumeMode: &mode,
|
||||
VolumeModeReason: &reason,
|
||||
}}, "")
|
||||
|
||||
r.UpdateFullHeartbeat("primary-server:8080", []*master_pb.BlockVolumeInfoMessage{{
|
||||
Path: "/blocks/vol-master-preserve-explicit-truth-primary.blk",
|
||||
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
||||
ReplicaDegraded: true,
|
||||
}}, "")
|
||||
|
||||
entry, _ := r.Lookup("vol-master-preserve-explicit-truth")
|
||||
if !entry.HasNeedsRebuild || entry.NeedsRebuild {
|
||||
t.Fatalf("missing-field heartbeat should preserve explicit false needs_rebuild truth, entry=%+v", entry)
|
||||
}
|
||||
if !entry.HasPublishHealthy || entry.PublishHealthy {
|
||||
t.Fatalf("missing-field heartbeat should preserve explicit false publish_healthy truth, entry=%+v", entry)
|
||||
}
|
||||
if !entry.HasHeartbeatVolumeMode || entry.HeartbeatVolumeMode != "degraded" {
|
||||
t.Fatalf("missing-field heartbeat should preserve explicit volume_mode truth, entry=%+v", entry)
|
||||
}
|
||||
if !entry.HasHeartbeatVolumeReason || entry.HeartbeatVolumeReason != "barrier_timeout" {
|
||||
t.Fatalf("missing-field heartbeat should preserve explicit volume_mode_reason truth, entry=%+v", entry)
|
||||
}
|
||||
if entry.VolumeMode != "degraded" {
|
||||
t.Fatalf("missing-field heartbeat should preserve outward degraded mode, got %q", entry.VolumeMode)
|
||||
}
|
||||
|
||||
info := entryToVolumeInfo(&entry, true)
|
||||
if info.VolumeMode != "degraded" {
|
||||
t.Fatalf("expected outward API volume_mode=degraded, got %q", info.VolumeMode)
|
||||
}
|
||||
if info.VolumeModeReason != "barrier_timeout" {
|
||||
t.Fatalf("expected outward API volume_mode_reason=barrier_timeout, got %q", info.VolumeModeReason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistry_UpdateFullHeartbeat_MissingFieldsDoNotInventExplicitTruthOnFreshEntry(t *testing.T) {
|
||||
r := NewBlockVolumeRegistry()
|
||||
r.MarkBlockCapable("primary-server:8080")
|
||||
|
||||
r.UpdateFullHeartbeat("primary-server:8080", []*master_pb.BlockVolumeInfoMessage{{
|
||||
Path: "/blocks/vol-master-fresh-fallback-primary.blk",
|
||||
VolumeSize: 1 << 30,
|
||||
BlockSize: 4096,
|
||||
Epoch: 3,
|
||||
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
||||
ReplicaDegraded: true,
|
||||
}}, "")
|
||||
|
||||
entry, ok := r.Lookup("vol-master-fresh-fallback-primary")
|
||||
if !ok {
|
||||
t.Fatal("expected fresh entry from auto-register")
|
||||
}
|
||||
if entry.HasNeedsRebuild || entry.HasPublishHealthy || entry.HasHeartbeatVolumeMode || entry.HasHeartbeatVolumeReason {
|
||||
t.Fatalf("fresh missing-field heartbeat should not invent explicit truth, entry=%+v", entry)
|
||||
}
|
||||
if entry.VolumeMode != "allocated_only" {
|
||||
t.Fatalf("expected fresh fallback outward mode allocated_only without explicit truth, got %q", entry.VolumeMode)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -277,7 +277,16 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
|
||||
// (BlockVolumeInfos on first heartbeat) or deltas (NewBlockVolumes/DeletedBlockVolumes
|
||||
// on subsequent heartbeats), never both in the same message.
|
||||
if len(heartbeat.BlockVolumeInfos) > 0 || heartbeat.HasNoBlockVolumes {
|
||||
hbResult := ms.blockRegistry.UpdateFullHeartbeat(dn.Url(), heartbeat.BlockVolumeInfos, heartbeat.BlockNvmeAddr)
|
||||
blockInventoryAuthoritative := true
|
||||
if heartbeat.BlockVolumeInventoryAuthoritative != nil {
|
||||
blockInventoryAuthoritative = heartbeat.GetBlockVolumeInventoryAuthoritative()
|
||||
}
|
||||
hbResult := ms.blockRegistry.UpdateFullHeartbeatWithInventoryAuthority(
|
||||
dn.Url(),
|
||||
heartbeat.BlockVolumeInfos,
|
||||
heartbeat.BlockNvmeAddr,
|
||||
blockInventoryAuthoritative,
|
||||
)
|
||||
// CP13-8: If a replica's receiver address changed (e.g., restart with port conflict),
|
||||
// immediately refresh the primary's assignment with the new addresses.
|
||||
for _, ac := range hbResult.AddrChanges {
|
||||
|
||||
@@ -349,16 +349,18 @@ func (vs *VolumeServer) doHeartbeatWithRetry(masterAddress pb.ServerAddress, grp
|
||||
return
|
||||
case <-vs.stopChan:
|
||||
var volumeMessages []*master_pb.VolumeInformationMessage
|
||||
blockInventoryAuthoritative := true
|
||||
emptyBeat := &master_pb.Heartbeat{
|
||||
Ip: ip,
|
||||
Port: port,
|
||||
PublicUrl: vs.store.PublicUrl,
|
||||
MaxFileKey: uint64(0),
|
||||
DataCenter: dataCenter,
|
||||
Rack: rack,
|
||||
Volumes: volumeMessages,
|
||||
HasNoVolumes: len(volumeMessages) == 0,
|
||||
HasNoBlockVolumes: vs.blockService != nil,
|
||||
Ip: ip,
|
||||
Port: port,
|
||||
PublicUrl: vs.store.PublicUrl,
|
||||
MaxFileKey: uint64(0),
|
||||
DataCenter: dataCenter,
|
||||
Rack: rack,
|
||||
Volumes: volumeMessages,
|
||||
HasNoVolumes: len(volumeMessages) == 0,
|
||||
HasNoBlockVolumes: vs.blockService != nil,
|
||||
BlockVolumeInventoryAuthoritative: &blockInventoryAuthoritative,
|
||||
}
|
||||
glog.V(1).Infof("volume server %s:%d stops and deletes all volumes", vs.store.Ip, vs.store.Port)
|
||||
if err = stream.Send(emptyBeat); err != nil {
|
||||
@@ -374,13 +376,15 @@ func (vs *VolumeServer) doHeartbeatWithRetry(masterAddress pb.ServerAddress, grp
|
||||
// Uses BlockService.CollectBlockVolumeHeartbeat which includes replication addresses (R1-4).
|
||||
func (vs *VolumeServer) collectBlockVolumeHeartbeat(ip string, port uint32, dc, rack string) *master_pb.Heartbeat {
|
||||
msgs := vs.blockService.CollectBlockVolumeHeartbeat()
|
||||
blockInventoryAuthoritative := vs.blockService.BlockInventoryAuthoritative()
|
||||
return &master_pb.Heartbeat{
|
||||
Ip: ip,
|
||||
Port: port,
|
||||
DataCenter: dc,
|
||||
Rack: rack,
|
||||
BlockVolumeInfos: blockvol.InfoMessagesToProto(msgs),
|
||||
HasNoBlockVolumes: len(msgs) == 0,
|
||||
BlockNvmeAddr: vs.blockService.NvmeListenAddr(),
|
||||
Ip: ip,
|
||||
Port: port,
|
||||
DataCenter: dc,
|
||||
Rack: rack,
|
||||
BlockVolumeInfos: blockvol.InfoMessagesToProto(msgs),
|
||||
HasNoBlockVolumes: len(msgs) == 0,
|
||||
BlockNvmeAddr: vs.blockService.NvmeListenAddr(),
|
||||
BlockVolumeInventoryAuthoritative: &blockInventoryAuthoritative,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,6 +97,10 @@ type BlockService struct {
|
||||
// routable host:port. This is the -ip value (IP or resolvable hostname),
|
||||
// never an opaque server identity from -id.
|
||||
advertisedHost string
|
||||
// blockInventoryAuthoritative reports whether the in-memory block inventory is
|
||||
// authoritative enough to drive master-side stale cleanup from a full
|
||||
// heartbeat. It becomes false when startup inventory scan fails.
|
||||
blockInventoryAuthoritative bool
|
||||
|
||||
// TestHook: if set, invoked when the legacy direct rebuild starter is used.
|
||||
onLegacyStartRebuild func(path, rebuildAddr string, epoch uint64)
|
||||
@@ -112,6 +116,15 @@ func (bs *BlockService) V2Core() *engine.CoreEngine {
|
||||
return bs.v2Core
|
||||
}
|
||||
|
||||
// BlockInventoryAuthoritative reports whether the current block inventory can be
|
||||
// treated as authoritative for full-heartbeat stale cleanup.
|
||||
func (bs *BlockService) BlockInventoryAuthoritative() bool {
|
||||
if bs == nil {
|
||||
return false
|
||||
}
|
||||
return bs.blockInventoryAuthoritative
|
||||
}
|
||||
|
||||
// CoreProjection returns the latest adapter-cached projection emitted by the
|
||||
// explicit V2 core on the narrow live path.
|
||||
func (bs *BlockService) CoreProjection(path string) (engine.PublicationProjection, bool) {
|
||||
@@ -210,17 +223,18 @@ func StartBlockService(listenAddr, blockDir, iqnPrefix, portalAddr string, nvmeC
|
||||
}
|
||||
|
||||
bs := &BlockService{
|
||||
blockStore: storage.NewBlockVolumeStore(),
|
||||
iqnPrefix: iqnPrefix,
|
||||
nqnPrefix: nqnPrefix,
|
||||
blockDir: blockDir,
|
||||
listenAddr: listenAddr,
|
||||
nvmeListenAddr: nvmeCfg.ListenAddr,
|
||||
v2Bridge: v2bridge.NewControlBridge(),
|
||||
v2Orchestrator: engine.NewRecoveryOrchestrator(),
|
||||
v2Core: engine.NewCoreEngine(),
|
||||
localServerID: listenAddr, // INTERIM: transport-shaped, see field doc
|
||||
coreProj: make(map[string]engine.PublicationProjection),
|
||||
blockStore: storage.NewBlockVolumeStore(),
|
||||
iqnPrefix: iqnPrefix,
|
||||
nqnPrefix: nqnPrefix,
|
||||
blockDir: blockDir,
|
||||
listenAddr: listenAddr,
|
||||
nvmeListenAddr: nvmeCfg.ListenAddr,
|
||||
v2Bridge: v2bridge.NewControlBridge(),
|
||||
v2Orchestrator: engine.NewRecoveryOrchestrator(),
|
||||
v2Core: engine.NewCoreEngine(),
|
||||
localServerID: listenAddr, // INTERIM: transport-shaped, see field doc
|
||||
coreProj: make(map[string]engine.PublicationProjection),
|
||||
blockInventoryAuthoritative: false,
|
||||
}
|
||||
bs.v2Recovery = NewRecoveryManager(bs)
|
||||
|
||||
@@ -290,6 +304,7 @@ func StartBlockService(listenAddr, blockDir, iqnPrefix, portalAddr string, nvmeC
|
||||
name := strings.TrimSuffix(entry.Name(), ".blk")
|
||||
bs.registerVolume(vol, name)
|
||||
}
|
||||
bs.blockInventoryAuthoritative = true
|
||||
|
||||
// Start iSCSI target in background.
|
||||
go func() {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package weed_server
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
@@ -67,3 +68,47 @@ func TestMaintenanceMode(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectBlockVolumeHeartbeat_IncludesInventoryAuthority(t *testing.T) {
|
||||
vs := &VolumeServer{
|
||||
store: &storage.Store{},
|
||||
blockService: &BlockService{
|
||||
blockStore: storage.NewBlockVolumeStore(),
|
||||
blockInventoryAuthoritative: false,
|
||||
},
|
||||
}
|
||||
|
||||
hb := vs.collectBlockVolumeHeartbeat("127.0.0.1", 18080, "dc1", "rack1")
|
||||
if hb.BlockVolumeInventoryAuthoritative == nil {
|
||||
t.Fatal("expected block inventory authority bit to be present")
|
||||
}
|
||||
if hb.GetBlockVolumeInventoryAuthoritative() {
|
||||
t.Fatal("expected non-authoritative block inventory bit on test heartbeat")
|
||||
}
|
||||
if !hb.HasNoBlockVolumes {
|
||||
t.Fatal("expected empty heartbeat to report has_no_block_volumes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartBlockService_ScanFailureEmitsNonAuthoritativeInventory(t *testing.T) {
|
||||
missingDir := filepath.Join(t.TempDir(), "missing-block-dir")
|
||||
bs := StartBlockService("127.0.0.1:3260", missingDir, "", "", NVMeConfig{})
|
||||
if bs == nil {
|
||||
t.Fatal("expected block service even when startup scan fails")
|
||||
}
|
||||
if bs.BlockInventoryAuthoritative() {
|
||||
t.Fatal("startup scan failure should leave block inventory non-authoritative")
|
||||
}
|
||||
|
||||
vs := &VolumeServer{
|
||||
store: &storage.Store{},
|
||||
blockService: bs,
|
||||
}
|
||||
hb := vs.collectBlockVolumeHeartbeat("127.0.0.1", 18080, "dc1", "rack1")
|
||||
if hb.BlockVolumeInventoryAuthoritative == nil {
|
||||
t.Fatal("expected inventory authority bit on startup-scan-failure heartbeat")
|
||||
}
|
||||
if hb.GetBlockVolumeInventoryAuthoritative() {
|
||||
t.Fatal("startup-scan-failure heartbeat should be non-authoritative")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user