refactor: close bounded phase 16 restart truth seams

Bind non-authoritative inventory, restart primary-truth rebasing, and sparse replica readiness retention into the heartbeat/master seam, and package the bounded finish-line checkpoint with explicit claims, non-claims, and proof commands.

Made-with: Cursor
This commit is contained in:
pingqiu
2026-04-04 16:13:06 -07:00
parent 10833c8b68
commit 0f72c8d062
12 changed files with 1491 additions and 105 deletions
+138 -68
View File
@@ -24,20 +24,21 @@ const (
// ReplicaInfo tracks one replica of a block volume (CP8-2).
type ReplicaInfo struct {
Server string // replica VS address
Path string // file path on replica VS
ISCSIAddr string // iSCSI target address
IQN string // iSCSI qualified name
NvmeAddr string // NVMe/TCP target address (ip:port), empty if NVMe disabled
NQN string // NVMe subsystem NQN, empty if NVMe disabled
DataAddr string // WAL receiver data listen addr
CtrlAddr string // WAL receiver ctrl listen addr
Ready bool // receiver/publish readiness confirmed by replica heartbeat
HealthScore float64 // from heartbeat (0.0-1.0)
WALHeadLSN uint64 // from heartbeat
WALLag uint64 // computed: primary.WALHeadLSN - replica.WALHeadLSN
LastHeartbeat time.Time // last heartbeat received from this replica
Role uint32 // replica role (RoleReplica, RoleRebuilding, etc.)
Server string // replica VS address
Path string // file path on replica VS
ISCSIAddr string // iSCSI target address
IQN string // iSCSI qualified name
NvmeAddr string // NVMe/TCP target address (ip:port), empty if NVMe disabled
NQN string // NVMe subsystem NQN, empty if NVMe disabled
DataAddr string // WAL receiver data listen addr
CtrlAddr string // WAL receiver ctrl listen addr
Ready bool // receiver/publish readiness confirmed by replica heartbeat
HasExplicitReady bool // whether replica readiness was carried explicitly on heartbeat
HealthScore float64 // from heartbeat (0.0-1.0)
WALHeadLSN uint64 // from heartbeat
WALLag uint64 // computed: primary.WALHeadLSN - replica.WALHeadLSN
LastHeartbeat time.Time // last heartbeat received from this replica
Role uint32 // replica role (RoleReplica, RoleRebuilding, etc.)
}
const (
@@ -485,6 +486,10 @@ type HeartbeatResult struct {
}
func (r *BlockVolumeRegistry) UpdateFullHeartbeat(server string, infos []*master_pb.BlockVolumeInfoMessage, nvmeAddr string) HeartbeatResult {
return r.UpdateFullHeartbeatWithInventoryAuthority(server, infos, nvmeAddr, true)
}
func (r *BlockVolumeRegistry) UpdateFullHeartbeatWithInventoryAuthority(server string, infos []*master_pb.BlockVolumeInfoMessage, nvmeAddr string, blockInventoryAuthoritative bool) HeartbeatResult {
var result HeartbeatResult
r.mu.Lock()
defer r.mu.Unlock()
@@ -499,44 +504,46 @@ func (r *BlockVolumeRegistry) UpdateFullHeartbeat(server string, infos []*master
}
// Find entries for this server that are NOT reported -> reconcile.
if names, ok := r.byServer[server]; ok {
for name := range names {
entry := r.volumes[name]
if entry == nil {
continue
}
if entry.VolumeServer == server {
// Server is the primary: check if primary path is reported.
if _, found := reported[entry.Path]; !found {
// B-10: Do not delete entries with a coordinated expand in flight.
// The primary may have restarted mid-expand; deleting the entry
// would orphan the volume and strand the expand coordinator.
if entry.ExpandInProgress {
glog.Warningf("block registry: skipping stale-cleanup for %q (ExpandInProgress=true, server=%s)",
name, server)
continue
}
delete(r.volumes, name)
delete(names, name)
// Also clean up replica entries from byServer.
for _, ri := range entry.Replicas {
r.removeFromServer(ri.Server, name)
}
}
} else {
// Server is a replica: check if replica path is reported.
ri := entry.ReplicaByServer(server)
if ri == nil {
// No replica record — stale byServer index, just clean up.
delete(names, name)
if blockInventoryAuthoritative {
if names, ok := r.byServer[server]; ok {
for name := range names {
entry := r.volumes[name]
if entry == nil {
continue
}
if _, found := reported[ri.Path]; !found {
// Replica path not reported — remove this replica, NOT the whole volume.
r.removeReplicaLocked(entry, server, name)
delete(names, name)
glog.V(0).Infof("block registry: removed stale replica %s for %q (path %s not in heartbeat)",
server, name, ri.Path)
if entry.VolumeServer == server {
// Server is the primary: check if primary path is reported.
if _, found := reported[entry.Path]; !found {
// B-10: Do not delete entries with a coordinated expand in flight.
// The primary may have restarted mid-expand; deleting the entry
// would orphan the volume and strand the expand coordinator.
if entry.ExpandInProgress {
glog.Warningf("block registry: skipping stale-cleanup for %q (ExpandInProgress=true, server=%s)",
name, server)
continue
}
delete(r.volumes, name)
delete(names, name)
// Also clean up replica entries from byServer.
for _, ri := range entry.Replicas {
r.removeFromServer(ri.Server, name)
}
}
} else {
// Server is a replica: check if replica path is reported.
ri := entry.ReplicaByServer(server)
if ri == nil {
// No replica record — stale byServer index, just clean up.
delete(names, name)
continue
}
if _, found := reported[ri.Path]; !found {
// Replica path not reported — remove this replica, NOT the whole volume.
r.removeReplicaLocked(entry, server, name)
delete(names, name)
glog.V(0).Infof("block registry: removed stale replica %s for %q (path %s not in heartbeat)",
server, name, ri.Path)
}
}
}
}
@@ -606,20 +613,32 @@ func (r *BlockVolumeRegistry) UpdateFullHeartbeat(server string, infos []*master
existing, dup := r.volumes[name]
if !dup {
entry := &BlockVolumeEntry{
Name: name,
VolumeServer: server,
Path: info.Path,
SizeBytes: info.VolumeSize,
Epoch: info.Epoch,
Role: info.Role,
Status: StatusActive,
LastLeaseGrant: time.Now(),
LeaseTTL: 30 * time.Second,
HealthScore: info.HealthScore,
TransportDegraded: info.ReplicaDegraded,
WALHeadLSN: info.WalHeadLsn,
DurabilityMode: info.DurabilityMode,
Name: name,
VolumeServer: server,
Path: info.Path,
SizeBytes: info.VolumeSize,
Epoch: info.Epoch,
Role: info.Role,
Status: StatusActive,
LastLeaseGrant: time.Now(),
LeaseTTL: 30 * time.Second,
HealthScore: info.HealthScore,
TransportDegraded: info.ReplicaDegraded,
NeedsRebuild: false,
HasNeedsRebuild: false,
PublishHealthy: false,
HasPublishHealthy: false,
HeartbeatVolumeMode: "",
HasHeartbeatVolumeMode: false,
HeartbeatVolumeReason: "",
HasHeartbeatVolumeReason: false,
WALHeadLSN: info.WalHeadLsn,
DurabilityMode: info.DurabilityMode,
}
entry.NeedsRebuild, entry.HasNeedsRebuild = primaryNeedsRebuildObservedFromHeartbeat(info)
entry.PublishHealthy, entry.HasPublishHealthy = primaryPublishHealthyObservedFromHeartbeat(info)
entry.HeartbeatVolumeMode, entry.HasHeartbeatVolumeMode = primaryVolumeModeObservedFromHeartbeat(info)
entry.HeartbeatVolumeReason, entry.HasHeartbeatVolumeReason = primaryVolumeReasonObservedFromHeartbeat(info)
if info.ReplicaDataAddr != "" {
entry.ReplicaDataAddr = info.ReplicaDataAddr
}
@@ -662,10 +681,7 @@ func (r *BlockVolumeRegistry) applyPrimaryHeartbeatObservation(existing *BlockVo
existing.LastLeaseGrant = time.Now()
existing.HealthScore = info.HealthScore
existing.TransportDegraded = info.ReplicaDegraded
existing.NeedsRebuild, existing.HasNeedsRebuild = primaryNeedsRebuildObservedFromHeartbeat(info)
existing.PublishHealthy, existing.HasPublishHealthy = primaryPublishHealthyObservedFromHeartbeat(info)
existing.HeartbeatVolumeMode, existing.HasHeartbeatVolumeMode = primaryVolumeModeObservedFromHeartbeat(info)
existing.HeartbeatVolumeReason, existing.HasHeartbeatVolumeReason = primaryVolumeReasonObservedFromHeartbeat(info)
applyExplicitPrimaryTruthFromHeartbeat(existing, info, true)
existing.WALHeadLSN = info.WalHeadLsn
// F3: only update DurabilityMode when non-empty (prevents older VS from clearing strict mode).
if info.DurabilityMode != "" {
@@ -707,7 +723,7 @@ func (r *BlockVolumeRegistry) applyReplicaHeartbeatObservation(existing *BlockVo
existing.Replicas[i].Role = blockvol.RoleToWire(blockvol.RoleReplica)
existing.Replicas[i].NvmeAddr = info.NvmeAddr
existing.Replicas[i].NQN = info.Nqn
existing.Replicas[i].Ready = replicaReadyObservedFromHeartbeat(info)
applyReplicaReadyFromHeartbeat(&existing.Replicas[i], info, true)
if existing.WALHeadLSN > info.WalHeadLsn {
existing.Replicas[i].WALLag = existing.WALHeadLSN - info.WalHeadLsn
} else {
@@ -743,6 +759,22 @@ func (r *BlockVolumeRegistry) applyReplicaHeartbeatObservation(existing *BlockVo
existing.recomputeReplicaState()
}
func applyReplicaReadyFromHeartbeat(replica *ReplicaInfo, info *master_pb.BlockVolumeInfoMessage, preserveWhenAbsent bool) {
if replica == nil || info == nil {
return
}
if info.ReplicaReady != nil {
replica.Ready = info.GetReplicaReady()
replica.HasExplicitReady = true
return
}
if preserveWhenAbsent && replica.HasExplicitReady {
return
}
replica.Ready = info.ReplicaDataAddr != "" && info.ReplicaCtrlAddr != ""
replica.HasExplicitReady = false
}
func replicaReadyObservedFromHeartbeat(info *master_pb.BlockVolumeInfoMessage) bool {
if info == nil {
return false
@@ -793,6 +825,40 @@ func primaryVolumeReasonObservedFromHeartbeat(info *master_pb.BlockVolumeInfoMes
return "", false
}
func applyExplicitPrimaryTruthFromHeartbeat(existing *BlockVolumeEntry, info *master_pb.BlockVolumeInfoMessage, preserveWhenAbsent bool) {
if existing == nil || info == nil {
return
}
if needsRebuild, ok := primaryNeedsRebuildObservedFromHeartbeat(info); ok {
existing.NeedsRebuild = needsRebuild
existing.HasNeedsRebuild = true
} else if !preserveWhenAbsent {
existing.NeedsRebuild = false
existing.HasNeedsRebuild = false
}
if publishHealthy, ok := primaryPublishHealthyObservedFromHeartbeat(info); ok {
existing.PublishHealthy = publishHealthy
existing.HasPublishHealthy = true
} else if !preserveWhenAbsent {
existing.PublishHealthy = false
existing.HasPublishHealthy = false
}
if mode, ok := primaryVolumeModeObservedFromHeartbeat(info); ok {
existing.HeartbeatVolumeMode = mode
existing.HasHeartbeatVolumeMode = true
} else if !preserveWhenAbsent {
existing.HeartbeatVolumeMode = ""
existing.HasHeartbeatVolumeMode = false
}
if reason, ok := primaryVolumeReasonObservedFromHeartbeat(info); ok {
existing.HeartbeatVolumeReason = reason
existing.HasHeartbeatVolumeReason = true
} else if !preserveWhenAbsent {
existing.HeartbeatVolumeReason = ""
existing.HasHeartbeatVolumeReason = false
}
}
func validHeartbeatVolumeMode(mode string) bool {
switch mode {
case "allocated_only", "bootstrap_pending", "publish_healthy", "degraded", "needs_rebuild":
@@ -907,6 +973,7 @@ func (r *BlockVolumeRegistry) demoteExistingToReplica(name string, existing *Blo
}
existing.NvmeAddr = info.NvmeAddr
existing.NQN = info.Nqn
applyExplicitPrimaryTruthFromHeartbeat(existing, info, false)
// Add old primary as replica.
existing.Replicas = append(existing.Replicas, oldReplica)
@@ -917,6 +984,7 @@ func (r *BlockVolumeRegistry) demoteExistingToReplica(name string, existing *Blo
existing.ReplicaServer = oldReplica.Server
existing.ReplicaPath = oldReplica.Path
}
existing.recomputeReplicaState()
}
// upsertServerAsReplica adds or updates the server as a replica for the existing entry.
@@ -944,6 +1012,7 @@ func (r *BlockVolumeRegistry) upsertServerAsReplica(name string, existing *Block
existing.Replicas[i].Role = replicaRole
existing.Replicas[i].NvmeAddr = info.NvmeAddr
existing.Replicas[i].NQN = info.Nqn
applyReplicaReadyFromHeartbeat(&existing.Replicas[i], info, true)
return
}
}
@@ -962,6 +1031,7 @@ func (r *BlockVolumeRegistry) upsertServerAsReplica(name string, existing *Block
NvmeAddr: info.NvmeAddr,
NQN: info.Nqn,
}
applyReplicaReadyFromHeartbeat(&ri, info, false)
existing.Replicas = append(existing.Replicas, ri)
r.addToServer(newServer, name)
if len(existing.Replicas) == 1 {
+372
View File
@@ -114,6 +114,28 @@ func TestRegistry_UpdateFullHeartbeat(t *testing.T) {
}
}
func TestRegistry_UpdateFullHeartbeatWithInventoryAuthority_NonAuthoritativeEmptyDoesNotDelete(t *testing.T) {
r := NewBlockVolumeRegistry()
r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk", Status: StatusActive})
r.UpdateFullHeartbeatWithInventoryAuthority("s1", nil, "", false)
if _, ok := r.Lookup("vol1"); !ok {
t.Fatal("non-authoritative empty full heartbeat should not delete vol1")
}
}
func TestRegistry_UpdateFullHeartbeatWithInventoryAuthority_AuthoritativeEmptyStillDeletes(t *testing.T) {
r := NewBlockVolumeRegistry()
r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk", Status: StatusActive})
r.UpdateFullHeartbeatWithInventoryAuthority("s1", nil, "", true)
if _, ok := r.Lookup("vol1"); ok {
t.Fatal("authoritative empty full heartbeat should still delete vol1")
}
}
func TestRegistry_UpdateDeltaHeartbeat(t *testing.T) {
r := NewBlockVolumeRegistry()
r.Register(&BlockVolumeEntry{Name: "vol1", VolumeServer: "s1", Path: "/v1.blk", Status: StatusPending})
@@ -1663,6 +1685,110 @@ func TestMasterRestart_HigherEpochWins(t *testing.T) {
}
}
func TestMasterRestart_HigherEpochRebasesExplicitPrimaryTruth(t *testing.T) {
r := NewBlockVolumeRegistry()
oldNeedsRebuild := false
oldPublishHealthy := true
oldMode := "publish_healthy"
oldReason := ""
r.UpdateFullHeartbeat("vs1:9333", []*master_pb.BlockVolumeInfoMessage{{
Path: "/data/vol1.blk",
Epoch: 5,
Role: blockvol.RoleToWire(blockvol.RolePrimary),
WalHeadLsn: 100,
PublishHealthy: &oldPublishHealthy,
NeedsRebuild: &oldNeedsRebuild,
VolumeMode: &oldMode,
VolumeModeReason: &oldReason,
VolumeSize: 1 << 30,
}}, "")
newNeedsRebuild := true
newPublishHealthy := false
newMode := "needs_rebuild"
newReason := "gap_too_large"
r.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{{
Path: "/data/vol1.blk",
Epoch: 6,
Role: blockvol.RoleToWire(blockvol.RolePrimary),
WalHeadLsn: 150,
PublishHealthy: &newPublishHealthy,
NeedsRebuild: &newNeedsRebuild,
VolumeMode: &newMode,
VolumeModeReason: &newReason,
VolumeSize: 1 << 30,
}}, "")
entry, _ := r.Lookup("vol1")
if entry.VolumeServer != "vs2:9333" {
t.Fatalf("expected vs2 as primary after rebase, got %q", entry.VolumeServer)
}
if !entry.HasNeedsRebuild || !entry.NeedsRebuild {
t.Fatalf("expected new primary explicit needs_rebuild truth, entry=%+v", entry)
}
if !entry.HasPublishHealthy || entry.PublishHealthy {
t.Fatalf("expected new primary explicit false publish_healthy truth, entry=%+v", entry)
}
if !entry.HasHeartbeatVolumeMode || entry.HeartbeatVolumeMode != "needs_rebuild" {
t.Fatalf("expected new primary explicit volume_mode truth, entry=%+v", entry)
}
if !entry.HasHeartbeatVolumeReason || entry.HeartbeatVolumeReason != "gap_too_large" {
t.Fatalf("expected new primary explicit volume_mode_reason truth, entry=%+v", entry)
}
if entry.VolumeMode != "needs_rebuild" {
t.Fatalf("expected outward mode to follow winning primary truth, got %q", entry.VolumeMode)
}
}
func TestMasterRestart_HigherEpochSparsePrimaryClearsOldExplicitTruth(t *testing.T) {
r := NewBlockVolumeRegistry()
oldNeedsRebuild := false
oldPublishHealthy := true
oldMode := "publish_healthy"
oldReason := ""
r.UpdateFullHeartbeat("vs1:9333", []*master_pb.BlockVolumeInfoMessage{{
Path: "/data/vol1.blk",
Epoch: 5,
Role: blockvol.RoleToWire(blockvol.RolePrimary),
WalHeadLsn: 100,
PublishHealthy: &oldPublishHealthy,
NeedsRebuild: &oldNeedsRebuild,
VolumeMode: &oldMode,
VolumeModeReason: &oldReason,
VolumeSize: 1 << 30,
}}, "")
r.UpdateFullHeartbeat("vs2:9333", []*master_pb.BlockVolumeInfoMessage{{
Path: "/data/vol1.blk",
Epoch: 6,
Role: blockvol.RoleToWire(blockvol.RolePrimary),
WalHeadLsn: 150,
VolumeSize: 1 << 30,
}}, "")
entry, _ := r.Lookup("vol1")
if entry.VolumeServer != "vs2:9333" {
t.Fatalf("expected vs2 as primary after sparse rebase, got %q", entry.VolumeServer)
}
if entry.HasNeedsRebuild || entry.NeedsRebuild {
t.Fatalf("sparse new primary should clear old explicit needs_rebuild truth, entry=%+v", entry)
}
if entry.HasPublishHealthy || entry.PublishHealthy {
t.Fatalf("sparse new primary should clear old explicit publish_healthy truth, entry=%+v", entry)
}
if entry.HasHeartbeatVolumeMode || entry.HeartbeatVolumeMode != "" {
t.Fatalf("sparse new primary should clear old explicit volume_mode truth, entry=%+v", entry)
}
if entry.HasHeartbeatVolumeReason || entry.HeartbeatVolumeReason != "" {
t.Fatalf("sparse new primary should clear old explicit volume_mode_reason truth, entry=%+v", entry)
}
if entry.VolumeMode == "publish_healthy" {
t.Fatalf("sparse new primary should not retain stale publish_healthy mode, entry=%+v", entry)
}
}
func TestMasterRestart_LowerEpochBecomesReplica(t *testing.T) {
r := NewBlockVolumeRegistry()
@@ -2180,6 +2306,80 @@ func TestRegistry_UpdateFullHeartbeat_ReplicaReadyFallsBackToAddressesWhenFieldA
}
}
func TestRegistry_UpdateFullHeartbeat_ReplicaReadyMissingFieldPreservesAcceptedExplicitTruth(t *testing.T) {
r := NewBlockVolumeRegistry()
if err := r.Register(&BlockVolumeEntry{
Name: "vol-master-ready-preserve-explicit",
VolumeServer: "primary-server:8080",
Path: "/blocks/vol-master-ready-preserve-explicit-primary.blk",
Status: StatusActive,
Role: blockvol.RoleToWire(blockvol.RolePrimary),
ReplicaFactor: 2,
Replicas: []ReplicaInfo{{
Server: "replica-server:8080",
Path: "/blocks/vol-master-ready-preserve-explicit.blk",
}},
}); err != nil {
t.Fatalf("register: %v", err)
}
replicaReady := false
r.UpdateFullHeartbeat("replica-server:8080", []*master_pb.BlockVolumeInfoMessage{{
Path: "/blocks/vol-master-ready-preserve-explicit.blk",
ReplicaDataAddr: "10.0.0.2:4260",
ReplicaCtrlAddr: "10.0.0.2:4261",
ReplicaReady: &replicaReady,
}}, "")
r.UpdateFullHeartbeat("replica-server:8080", []*master_pb.BlockVolumeInfoMessage{{
Path: "/blocks/vol-master-ready-preserve-explicit.blk",
ReplicaDataAddr: "10.0.0.2:4260",
ReplicaCtrlAddr: "10.0.0.2:4261",
}}, "")
entry, _ := r.Lookup("vol-master-ready-preserve-explicit")
if !entry.Replicas[0].HasExplicitReady {
t.Fatalf("expected accepted explicit replica readiness to remain marked explicit, entry=%+v", entry)
}
if entry.Replicas[0].Ready {
t.Fatalf("missing-field replica heartbeat should preserve explicit false ready truth, entry=%+v", entry)
}
if entry.ReplicaReady {
t.Fatalf("aggregate replica ready should remain false after sparse heartbeat, entry=%+v", entry)
}
}
func TestRegistry_UpdateFullHeartbeat_ReplicaReadyMissingFieldFreshEntryStillFallsBack(t *testing.T) {
r := NewBlockVolumeRegistry()
if err := r.Register(&BlockVolumeEntry{
Name: "vol-master-ready-fresh-fallback",
VolumeServer: "primary-server:8080",
Path: "/blocks/vol-master-ready-fresh-fallback-primary.blk",
Status: StatusActive,
Role: blockvol.RoleToWire(blockvol.RolePrimary),
ReplicaFactor: 2,
Replicas: []ReplicaInfo{{
Server: "replica-server:8080",
Path: "/blocks/vol-master-ready-fresh-fallback.blk",
}},
}); err != nil {
t.Fatalf("register: %v", err)
}
r.UpdateFullHeartbeat("replica-server:8080", []*master_pb.BlockVolumeInfoMessage{{
Path: "/blocks/vol-master-ready-fresh-fallback.blk",
ReplicaDataAddr: "10.0.0.2:4260",
ReplicaCtrlAddr: "10.0.0.2:4261",
}}, "")
entry, _ := r.Lookup("vol-master-ready-fresh-fallback")
if entry.Replicas[0].HasExplicitReady {
t.Fatalf("fresh missing-field replica heartbeat should not invent explicit readiness, entry=%+v", entry)
}
if !entry.Replicas[0].Ready || !entry.ReplicaReady {
t.Fatalf("fresh missing-field replica heartbeat should still fall back to addresses, entry=%+v", entry)
}
}
func TestRegistry_UpdateFullHeartbeat_ConsumesExplicitNeedsRebuildFromPrimaryHeartbeat(t *testing.T) {
r := NewBlockVolumeRegistry()
if err := r.Register(&BlockVolumeEntry{
@@ -2479,3 +2679,175 @@ func TestRegistry_UpdateFullHeartbeat_VolumeModeFallsBackWhenFieldAbsent(t *test
t.Fatalf("expected fallback reconstructed degraded mode, got %q", entry.VolumeMode)
}
}
func TestRegistry_UpdateFullHeartbeat_AutoRegisterPreservesExplicitPrimaryTruthOnRestart(t *testing.T) {
tests := []struct {
name string
needsRebuild bool
publishHealthy bool
mode string
reason string
wantMode string
wantReason string
}{
{
name: "publish_healthy",
needsRebuild: false,
publishHealthy: true,
mode: "publish_healthy",
reason: "",
wantMode: "publish_healthy",
wantReason: "",
},
{
name: "needs_rebuild",
needsRebuild: true,
publishHealthy: false,
mode: "needs_rebuild",
reason: "gap_too_large",
wantMode: "needs_rebuild",
wantReason: "gap_too_large",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
r := NewBlockVolumeRegistry()
r.MarkBlockCapable("primary-server:8080")
mode := tt.mode
reason := tt.reason
needsRebuild := tt.needsRebuild
publishHealthy := tt.publishHealthy
r.UpdateFullHeartbeat("primary-server:8080", []*master_pb.BlockVolumeInfoMessage{{
Path: "/blocks/vol-restart-" + tt.name + ".blk",
VolumeSize: 1 << 30,
BlockSize: 4096,
Epoch: 7,
Role: blockvol.RoleToWire(blockvol.RolePrimary),
NeedsRebuild: &needsRebuild,
PublishHealthy: &publishHealthy,
VolumeMode: &mode,
VolumeModeReason: &reason,
}}, "")
entry, ok := r.Lookup("vol-restart-" + tt.name)
if !ok {
t.Fatalf("expected auto-registered entry for %q", tt.name)
}
if !entry.HasNeedsRebuild || entry.NeedsRebuild != tt.needsRebuild {
t.Fatalf("needs_rebuild truth lost during auto-register, entry=%+v", entry)
}
if !entry.HasPublishHealthy || entry.PublishHealthy != tt.publishHealthy {
t.Fatalf("publish_healthy truth lost during auto-register, entry=%+v", entry)
}
if !entry.HasHeartbeatVolumeMode || entry.HeartbeatVolumeMode != tt.mode {
t.Fatalf("volume_mode truth lost during auto-register, entry=%+v", entry)
}
if !entry.HasHeartbeatVolumeReason || entry.HeartbeatVolumeReason != tt.reason {
t.Fatalf("volume_mode_reason truth lost during auto-register, entry=%+v", entry)
}
if entry.VolumeMode != tt.wantMode {
t.Fatalf("expected outward volume_mode %q after auto-register, got %q", tt.wantMode, entry.VolumeMode)
}
info := entryToVolumeInfo(&entry, true)
if info.VolumeMode != tt.wantMode {
t.Fatalf("expected outward API volume_mode %q after auto-register, got %q", tt.wantMode, info.VolumeMode)
}
if info.VolumeModeReason != tt.wantReason {
t.Fatalf("expected outward API volume_mode_reason %q after auto-register, got %q", tt.wantReason, info.VolumeModeReason)
}
})
}
}
func TestRegistry_UpdateFullHeartbeat_MissingFieldsPreserveAcceptedExplicitPrimaryTruth(t *testing.T) {
r := NewBlockVolumeRegistry()
if err := r.Register(&BlockVolumeEntry{
Name: "vol-master-preserve-explicit-truth",
VolumeServer: "primary-server:8080",
Path: "/blocks/vol-master-preserve-explicit-truth-primary.blk",
Status: StatusActive,
Role: blockvol.RoleToWire(blockvol.RolePrimary),
ReplicaFactor: 2,
Replicas: []ReplicaInfo{{
Server: "replica-server:8080",
Path: "/blocks/vol-master-preserve-explicit-truth-replica.blk",
Ready: true,
Role: blockvol.RoleToWire(blockvol.RoleRebuilding),
}},
}); err != nil {
t.Fatalf("register: %v", err)
}
needsRebuild := false
publishHealthy := false
mode := "degraded"
reason := "barrier_timeout"
r.UpdateFullHeartbeat("primary-server:8080", []*master_pb.BlockVolumeInfoMessage{{
Path: "/blocks/vol-master-preserve-explicit-truth-primary.blk",
Role: blockvol.RoleToWire(blockvol.RolePrimary),
ReplicaDegraded: true,
NeedsRebuild: &needsRebuild,
PublishHealthy: &publishHealthy,
VolumeMode: &mode,
VolumeModeReason: &reason,
}}, "")
r.UpdateFullHeartbeat("primary-server:8080", []*master_pb.BlockVolumeInfoMessage{{
Path: "/blocks/vol-master-preserve-explicit-truth-primary.blk",
Role: blockvol.RoleToWire(blockvol.RolePrimary),
ReplicaDegraded: true,
}}, "")
entry, _ := r.Lookup("vol-master-preserve-explicit-truth")
if !entry.HasNeedsRebuild || entry.NeedsRebuild {
t.Fatalf("missing-field heartbeat should preserve explicit false needs_rebuild truth, entry=%+v", entry)
}
if !entry.HasPublishHealthy || entry.PublishHealthy {
t.Fatalf("missing-field heartbeat should preserve explicit false publish_healthy truth, entry=%+v", entry)
}
if !entry.HasHeartbeatVolumeMode || entry.HeartbeatVolumeMode != "degraded" {
t.Fatalf("missing-field heartbeat should preserve explicit volume_mode truth, entry=%+v", entry)
}
if !entry.HasHeartbeatVolumeReason || entry.HeartbeatVolumeReason != "barrier_timeout" {
t.Fatalf("missing-field heartbeat should preserve explicit volume_mode_reason truth, entry=%+v", entry)
}
if entry.VolumeMode != "degraded" {
t.Fatalf("missing-field heartbeat should preserve outward degraded mode, got %q", entry.VolumeMode)
}
info := entryToVolumeInfo(&entry, true)
if info.VolumeMode != "degraded" {
t.Fatalf("expected outward API volume_mode=degraded, got %q", info.VolumeMode)
}
if info.VolumeModeReason != "barrier_timeout" {
t.Fatalf("expected outward API volume_mode_reason=barrier_timeout, got %q", info.VolumeModeReason)
}
}
func TestRegistry_UpdateFullHeartbeat_MissingFieldsDoNotInventExplicitTruthOnFreshEntry(t *testing.T) {
r := NewBlockVolumeRegistry()
r.MarkBlockCapable("primary-server:8080")
r.UpdateFullHeartbeat("primary-server:8080", []*master_pb.BlockVolumeInfoMessage{{
Path: "/blocks/vol-master-fresh-fallback-primary.blk",
VolumeSize: 1 << 30,
BlockSize: 4096,
Epoch: 3,
Role: blockvol.RoleToWire(blockvol.RolePrimary),
ReplicaDegraded: true,
}}, "")
entry, ok := r.Lookup("vol-master-fresh-fallback-primary")
if !ok {
t.Fatal("expected fresh entry from auto-register")
}
if entry.HasNeedsRebuild || entry.HasPublishHealthy || entry.HasHeartbeatVolumeMode || entry.HasHeartbeatVolumeReason {
t.Fatalf("fresh missing-field heartbeat should not invent explicit truth, entry=%+v", entry)
}
if entry.VolumeMode != "allocated_only" {
t.Fatalf("expected fresh fallback outward mode allocated_only without explicit truth, got %q", entry.VolumeMode)
}
}
+10 -1
View File
@@ -277,7 +277,16 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
// (BlockVolumeInfos on first heartbeat) or deltas (NewBlockVolumes/DeletedBlockVolumes
// on subsequent heartbeats), never both in the same message.
if len(heartbeat.BlockVolumeInfos) > 0 || heartbeat.HasNoBlockVolumes {
hbResult := ms.blockRegistry.UpdateFullHeartbeat(dn.Url(), heartbeat.BlockVolumeInfos, heartbeat.BlockNvmeAddr)
blockInventoryAuthoritative := true
if heartbeat.BlockVolumeInventoryAuthoritative != nil {
blockInventoryAuthoritative = heartbeat.GetBlockVolumeInventoryAuthoritative()
}
hbResult := ms.blockRegistry.UpdateFullHeartbeatWithInventoryAuthority(
dn.Url(),
heartbeat.BlockVolumeInfos,
heartbeat.BlockNvmeAddr,
blockInventoryAuthoritative,
)
// CP13-8: If a replica's receiver address changed (e.g., restart with port conflict),
// immediately refresh the primary's assignment with the new addresses.
for _, ac := range hbResult.AddrChanges {
+20 -16
View File
@@ -349,16 +349,18 @@ func (vs *VolumeServer) doHeartbeatWithRetry(masterAddress pb.ServerAddress, grp
return
case <-vs.stopChan:
var volumeMessages []*master_pb.VolumeInformationMessage
blockInventoryAuthoritative := true
emptyBeat := &master_pb.Heartbeat{
Ip: ip,
Port: port,
PublicUrl: vs.store.PublicUrl,
MaxFileKey: uint64(0),
DataCenter: dataCenter,
Rack: rack,
Volumes: volumeMessages,
HasNoVolumes: len(volumeMessages) == 0,
HasNoBlockVolumes: vs.blockService != nil,
Ip: ip,
Port: port,
PublicUrl: vs.store.PublicUrl,
MaxFileKey: uint64(0),
DataCenter: dataCenter,
Rack: rack,
Volumes: volumeMessages,
HasNoVolumes: len(volumeMessages) == 0,
HasNoBlockVolumes: vs.blockService != nil,
BlockVolumeInventoryAuthoritative: &blockInventoryAuthoritative,
}
glog.V(1).Infof("volume server %s:%d stops and deletes all volumes", vs.store.Ip, vs.store.Port)
if err = stream.Send(emptyBeat); err != nil {
@@ -374,13 +376,15 @@ func (vs *VolumeServer) doHeartbeatWithRetry(masterAddress pb.ServerAddress, grp
// Uses BlockService.CollectBlockVolumeHeartbeat which includes replication addresses (R1-4).
func (vs *VolumeServer) collectBlockVolumeHeartbeat(ip string, port uint32, dc, rack string) *master_pb.Heartbeat {
msgs := vs.blockService.CollectBlockVolumeHeartbeat()
blockInventoryAuthoritative := vs.blockService.BlockInventoryAuthoritative()
return &master_pb.Heartbeat{
Ip: ip,
Port: port,
DataCenter: dc,
Rack: rack,
BlockVolumeInfos: blockvol.InfoMessagesToProto(msgs),
HasNoBlockVolumes: len(msgs) == 0,
BlockNvmeAddr: vs.blockService.NvmeListenAddr(),
Ip: ip,
Port: port,
DataCenter: dc,
Rack: rack,
BlockVolumeInfos: blockvol.InfoMessagesToProto(msgs),
HasNoBlockVolumes: len(msgs) == 0,
BlockNvmeAddr: vs.blockService.NvmeListenAddr(),
BlockVolumeInventoryAuthoritative: &blockInventoryAuthoritative,
}
}
+26 -11
View File
@@ -97,6 +97,10 @@ type BlockService struct {
// routable host:port. This is the -ip value (IP or resolvable hostname),
// never an opaque server identity from -id.
advertisedHost string
// blockInventoryAuthoritative reports whether the in-memory block inventory is
// authoritative enough to drive master-side stale cleanup from a full
// heartbeat. It becomes false when startup inventory scan fails.
blockInventoryAuthoritative bool
// TestHook: if set, invoked when the legacy direct rebuild starter is used.
onLegacyStartRebuild func(path, rebuildAddr string, epoch uint64)
@@ -112,6 +116,15 @@ func (bs *BlockService) V2Core() *engine.CoreEngine {
return bs.v2Core
}
// BlockInventoryAuthoritative reports whether the current block inventory can be
// treated as authoritative for full-heartbeat stale cleanup.
func (bs *BlockService) BlockInventoryAuthoritative() bool {
if bs == nil {
return false
}
return bs.blockInventoryAuthoritative
}
// CoreProjection returns the latest adapter-cached projection emitted by the
// explicit V2 core on the narrow live path.
func (bs *BlockService) CoreProjection(path string) (engine.PublicationProjection, bool) {
@@ -210,17 +223,18 @@ func StartBlockService(listenAddr, blockDir, iqnPrefix, portalAddr string, nvmeC
}
bs := &BlockService{
blockStore: storage.NewBlockVolumeStore(),
iqnPrefix: iqnPrefix,
nqnPrefix: nqnPrefix,
blockDir: blockDir,
listenAddr: listenAddr,
nvmeListenAddr: nvmeCfg.ListenAddr,
v2Bridge: v2bridge.NewControlBridge(),
v2Orchestrator: engine.NewRecoveryOrchestrator(),
v2Core: engine.NewCoreEngine(),
localServerID: listenAddr, // INTERIM: transport-shaped, see field doc
coreProj: make(map[string]engine.PublicationProjection),
blockStore: storage.NewBlockVolumeStore(),
iqnPrefix: iqnPrefix,
nqnPrefix: nqnPrefix,
blockDir: blockDir,
listenAddr: listenAddr,
nvmeListenAddr: nvmeCfg.ListenAddr,
v2Bridge: v2bridge.NewControlBridge(),
v2Orchestrator: engine.NewRecoveryOrchestrator(),
v2Core: engine.NewCoreEngine(),
localServerID: listenAddr, // INTERIM: transport-shaped, see field doc
coreProj: make(map[string]engine.PublicationProjection),
blockInventoryAuthoritative: false,
}
bs.v2Recovery = NewRecoveryManager(bs)
@@ -290,6 +304,7 @@ func StartBlockService(listenAddr, blockDir, iqnPrefix, portalAddr string, nvmeC
name := strings.TrimSuffix(entry.Name(), ".blk")
bs.registerVolume(vol, name)
}
bs.blockInventoryAuthoritative = true
// Start iSCSI target in background.
go func() {
+45
View File
@@ -1,6 +1,7 @@
package weed_server
import (
"path/filepath"
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
@@ -67,3 +68,47 @@ func TestMaintenanceMode(t *testing.T) {
})
}
}
func TestCollectBlockVolumeHeartbeat_IncludesInventoryAuthority(t *testing.T) {
vs := &VolumeServer{
store: &storage.Store{},
blockService: &BlockService{
blockStore: storage.NewBlockVolumeStore(),
blockInventoryAuthoritative: false,
},
}
hb := vs.collectBlockVolumeHeartbeat("127.0.0.1", 18080, "dc1", "rack1")
if hb.BlockVolumeInventoryAuthoritative == nil {
t.Fatal("expected block inventory authority bit to be present")
}
if hb.GetBlockVolumeInventoryAuthoritative() {
t.Fatal("expected non-authoritative block inventory bit on test heartbeat")
}
if !hb.HasNoBlockVolumes {
t.Fatal("expected empty heartbeat to report has_no_block_volumes")
}
}
func TestStartBlockService_ScanFailureEmitsNonAuthoritativeInventory(t *testing.T) {
missingDir := filepath.Join(t.TempDir(), "missing-block-dir")
bs := StartBlockService("127.0.0.1:3260", missingDir, "", "", NVMeConfig{})
if bs == nil {
t.Fatal("expected block service even when startup scan fails")
}
if bs.BlockInventoryAuthoritative() {
t.Fatal("startup scan failure should leave block inventory non-authoritative")
}
vs := &VolumeServer{
store: &storage.Store{},
blockService: bs,
}
hb := vs.collectBlockVolumeHeartbeat("127.0.0.1", 18080, "dc1", "rack1")
if hb.BlockVolumeInventoryAuthoritative == nil {
t.Fatal("expected inventory authority bit on startup-scan-failure heartbeat")
}
if hb.GetBlockVolumeInventoryAuthoritative() {
t.Fatal("startup-scan-failure heartbeat should be non-authoritative")
}
}