diff --git a/sw-block/.private/phase/phase-16-log.md b/sw-block/.private/phase/phase-16-log.md index 92a134c15..e979d9b37 100644 --- a/sw-block/.private/phase/phase-16-log.md +++ b/sw-block/.private/phase/phase-16-log.md @@ -1340,3 +1340,72 @@ Conclusion: explicit field still fall back to the previous heuristic 3. this slice still does not claim full `VolumeMode` heartbeat ownership or broad failover closure by itself + +--- + +#### `16O` Start Note Rev 1 + +Date: 2026-04-04 +Scope: bounded explicit `publish_healthy` preservation on the heartbeat/master seam + +Why this slice exists: + +1. `16M` made replica readiness explicit on the heartbeat/master seam +2. `16N` preserved explicit `needs_rebuild` truth on the same seam +3. but master-side outward healthy publication is still reconstructed from + secondary readiness/degraded heuristics rather than explicitly consumed from + the current core-owned publication owner + +Chosen implementation rule: + +1. widen the heartbeat wire additively with an explicit `publish_healthy` field +2. emit it from the current bounded core publication truth on the core-present + path +3. make master-side consume prefer explicit healthy publication truth and retain + the previous reconstruction only as backward-compatible fallback +4. do not broaden this slice into full `VolumeMode` heartbeat ownership + +--- + +#### `16O` Delivery Note Rev 1 + +Date: 2026-04-04 +Scope: bounded explicit `publish_healthy` preservation on the heartbeat/master seam + +What changed: + +1. `weed/pb/master.proto` + - added additive optional `publish_healthy` to `BlockVolumeInfoMessage` +2. `weed/pb/master_pb/master.pb.go` + - regenerated so heartbeat wire presence is represented as `*bool` +3. `weed/storage/blockvol/block_heartbeat.go` + - heartbeat wire struct now carries explicit `PublishHealthy` +4. `weed/storage/blockvol/block_heartbeat_proto.go` + - heartbeat conversion now writes and reads `PublishHealthy` +5. `weed/server/volume_server_block.go` + - heartbeat emission now preserves explicit bounded healthy-publication truth + from the current core publication owner +6. `weed/server/master_block_registry.go` + - registry consume now prefers explicit heartbeat `publish_healthy` truth and + keeps the older reconstruction only when the field is absent +7. focused tests in `block_heartbeat_proto_test.go`, + `volume_server_block_test.go`, and `master_block_registry_test.go` + - now prove explicit healthy-publication preservation and backward-compatible + fallback + +Proof / evidence: + +1. `go test ./weed/storage/blockvol/ -count=1 -run "TestInfoMessage_(Replica|NeedsRebuild|PublishHealthy)"` +2. `go test ./weed/server/ -count=1 -timeout 120s -run "Test(BlockService_CollectBlockVolumeHeartbeat_PrimaryPublishHealthyUsesCoreTruth|Registry_UpdateFullHeartbeat_(ConsumesExplicitPublishHealthyFromPrimaryHeartbeat|ExplicitUnhealthySuppressesStalePublishHealthyHeuristic)|BlockService_CollectBlockVolumeHeartbeat_PrimaryNeedsRebuildUsesCoreMode|Registry_UpdateFullHeartbeat_ConsumesExplicitNeedsRebuildFromPrimaryHeartbeat)"` +3. `go test ./weed/server/ -count=1 -timeout 120s -run "Test(BlockService_ApplyAssignments_|P16B_|P4_|Registry_UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaReady|Registry_UpdateFullHeartbeat_ReplicaReadyFallsBackToAddressesWhenFieldAbsent)"` +4. result: `PASS` + +Conclusion: + +1. the heartbeat/master seam no longer reconstructs healthy publication only + from secondary readiness/degraded signals when explicit core publication + truth is available +2. backward compatibility is preserved because older heartbeats without the + explicit field still fall back to the previous reconstruction +3. this slice still does not claim full `VolumeMode` heartbeat ownership or + broad failover closure by itself diff --git a/sw-block/.private/phase/phase-16.md b/sw-block/.private/phase/phase-16.md index 9887cc164..845278050 100644 --- a/sw-block/.private/phase/phase-16.md +++ b/sw-block/.private/phase/phase-16.md @@ -605,6 +605,62 @@ Evidence: 1. focused working-tree change after `16M` closeout +### `16O`: PublishHealthy Heartbeat Mode Preservation + +Goal: + +1. close one bounded failover/publication seam by preserving explicit + `publish_healthy` truth across the primary heartbeat/master consume boundary + instead of reconstructing that mode only from secondary readiness/degraded + heuristics +2. keep the slice limited to publication-health preservation on the current + heartbeat wire and master-registry consume path, not broad full + `VolumeMode` heartbeat ownership + +Acceptance object: + +1. `BlockVolumeInfoMessage` carries an additive explicit `publish_healthy` bit + on the heartbeat wire +2. `weed/server` heartbeat emission sets that bit from the current core-owned + publication truth on the core-present path +3. `master_block_registry` consumes explicit heartbeat `publish_healthy` truth + before reconstructing healthy publication from secondary heuristics +4. focused proofs show primary healthy publication survives heartbeat/master + consume as explicit publication truth +5. this slice still does not yet claim full `VolumeMode` heartbeat ownership or + broad failover closure + +Current chosen path: + +1. widen `master.proto` / heartbeat conversion with an additive + `publish_healthy` field +2. emit that field from `CollectBlockVolumeHeartbeat` using the bounded core + publication truth on the primary path +3. let master consume prefer explicit healthy publication truth while retaining + the previous reconstruction as backward-compatible fallback + +Status: + +1. delivered + +Delivered result: + +1. `BlockVolumeInfoMessage` now carries additive explicit `publish_healthy` + heartbeat truth on the wire +2. `weed/server` heartbeat emission now preserves explicit bounded + healthy-publication truth from the core-owned publication owner on the + current core-present path +3. `master_block_registry` now prefers explicit heartbeat `publish_healthy` + truth over secondary readiness/degraded reconstruction while keeping the + previous reconstruction as backward-compatible fallback when the field is + absent +4. focused proofs now show primary healthy publication survives heartbeat/master + consume as explicit publication truth rather than only as reconstructed mode + +Evidence: + +1. focused working-tree change after `16N` closeout + ## Current Checkpoint Review Target The current review target is the current widened bounded runtime checkpoint @@ -686,6 +742,10 @@ boundary: - primary heartbeat/master consume now preserves explicit bounded `needs_rebuild` truth with backward-compatible fallback for older heartbeats +16. `16O` delivered: + - primary heartbeat/master consume now preserves explicit bounded + healthy-publication truth with backward-compatible fallback for older + heartbeats After this checkpoint: diff --git a/weed/pb/master.proto b/weed/pb/master.proto index fec53dcd7..68f32b348 100644 --- a/weed/pb/master.proto +++ b/weed/pb/master.proto @@ -514,6 +514,7 @@ message BlockVolumeInfoMessage { string nqn = 18; optional bool replica_ready = 19; optional bool needs_rebuild = 20; + optional bool publish_healthy = 21; } message BlockVolumeShortInfoMessage { diff --git a/weed/pb/master_pb/master.pb.go b/weed/pb/master_pb/master.pb.go index 194534119..550f22f30 100644 --- a/weed/pb/master_pb/master.pb.go +++ b/weed/pb/master_pb/master.pb.go @@ -3913,6 +3913,7 @@ type BlockVolumeInfoMessage struct { Nqn string `protobuf:"bytes,18,opt,name=nqn,proto3" json:"nqn,omitempty"` ReplicaReady *bool `protobuf:"varint,19,opt,name=replica_ready,json=replicaReady,proto3,oneof" json:"replica_ready,omitempty"` NeedsRebuild *bool `protobuf:"varint,20,opt,name=needs_rebuild,json=needsRebuild,proto3,oneof" json:"needs_rebuild,omitempty"` + PublishHealthy *bool `protobuf:"varint,21,opt,name=publish_healthy,json=publishHealthy,proto3,oneof" json:"publish_healthy,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -4087,6 +4088,13 @@ func (x *BlockVolumeInfoMessage) GetNeedsRebuild() bool { return false } +func (x *BlockVolumeInfoMessage) GetPublishHealthy() bool { + if x != nil && x.PublishHealthy != nil { + return *x.PublishHealthy + } + return false +} + type BlockVolumeShortInfoMessage struct { state protoimpl.MessageState `protogen:"open.v1"` Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` @@ -5975,7 +5983,7 @@ const file_weed_pb_master_proto_rawDesc = "" + "\x0fprevious_leader\x18\x01 \x01(\tR\x0epreviousLeader\x12\x1d\n" + "\n" + "new_leader\x18\x02 \x01(\tR\tnewLeader\"\x14\n" + - "\x12VolumeGrowResponse\"\xda\x05\n" + + "\x12VolumeGrowResponse\"\x9c\x06\n" + "\x16BlockVolumeInfoMessage\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x1f\n" + "\vvolume_size\x18\x02 \x01(\x04R\n" + @@ -6000,9 +6008,11 @@ const file_weed_pb_master_proto_rawDesc = "" + "\tnvme_addr\x18\x11 \x01(\tR\bnvmeAddr\x12\x10\n" + "\x03nqn\x18\x12 \x01(\tR\x03nqn\x12(\n" + "\rreplica_ready\x18\x13 \x01(\bH\x00R\freplicaReady\x88\x01\x01\x12(\n" + - "\rneeds_rebuild\x18\x14 \x01(\bH\x01R\fneedsRebuild\x88\x01\x01B\x10\n" + + "\rneeds_rebuild\x18\x14 \x01(\bH\x01R\fneedsRebuild\x88\x01\x01\x12,\n" + + "\x0fpublish_healthy\x18\x15 \x01(\bH\x02R\x0epublishHealthy\x88\x01\x01B\x10\n" + "\x0e_replica_readyB\x10\n" + - "\x0e_needs_rebuild\"\x8e\x01\n" + + "\x0e_needs_rebuildB\x12\n" + + "\x10_publish_healthy\"\x8e\x01\n" + "\x1bBlockVolumeShortInfoMessage\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x1f\n" + "\vvolume_size\x18\x02 \x01(\x04R\n" + diff --git a/weed/server/master_block_registry.go b/weed/server/master_block_registry.go index 6f19d3152..56979bfed 100644 --- a/weed/server/master_block_registry.go +++ b/weed/server/master_block_registry.go @@ -79,6 +79,8 @@ type BlockVolumeEntry struct { TransportDegraded bool // primary reports degraded replicas NeedsRebuild bool // explicit primary heartbeat needs_rebuild truth when present HasNeedsRebuild bool // whether the current primary heartbeat carried explicit needs_rebuild truth + PublishHealthy bool // explicit primary heartbeat publish_healthy truth when present + HasPublishHealthy bool // whether the current primary heartbeat carried explicit publish_healthy truth // CP13-9: Normalized volume mode for external surfaces. // Computed by recomputeReplicaState from the current entry state. @@ -195,6 +197,11 @@ func (e *BlockVolumeEntry) computeVolumeMode() string { } } + // Prefer explicit primary heartbeat publish_healthy truth when present. + if e.HasPublishHealthy && e.PublishHealthy { + return "publish_healthy" + } + // Replicas exist but not all ready. if !e.ReplicaReady { return "bootstrap_pending" @@ -205,6 +212,10 @@ func (e *BlockVolumeEntry) computeVolumeMode() string { return "degraded" } + if e.HasPublishHealthy { + return "bootstrap_pending" + } + return "publish_healthy" } @@ -644,6 +655,7 @@ func (r *BlockVolumeRegistry) applyPrimaryHeartbeatObservation(existing *BlockVo existing.HealthScore = info.HealthScore existing.TransportDegraded = info.ReplicaDegraded existing.NeedsRebuild, existing.HasNeedsRebuild = primaryNeedsRebuildObservedFromHeartbeat(info) + existing.PublishHealthy, existing.HasPublishHealthy = primaryPublishHealthyObservedFromHeartbeat(info) existing.WALHeadLSN = info.WalHeadLsn // F3: only update DurabilityMode when non-empty (prevents older VS from clearing strict mode). if info.DurabilityMode != "" { @@ -741,6 +753,16 @@ func primaryNeedsRebuildObservedFromHeartbeat(info *master_pb.BlockVolumeInfoMes return false, false } +func primaryPublishHealthyObservedFromHeartbeat(info *master_pb.BlockVolumeInfoMessage) (bool, bool) { + if info == nil { + return false, false + } + if info.PublishHealthy != nil { + return info.GetPublishHealthy(), true + } + return false, false +} + // reconcileOnRestart handles the case where a second server reports a volume // name that already exists in the registry during master restart reconstruction. // Uses epoch-based tie-breaking to determine who is the real primary. diff --git a/weed/server/master_block_registry_test.go b/weed/server/master_block_registry_test.go index 115fbe202..d55dc0b24 100644 --- a/weed/server/master_block_registry_test.go +++ b/weed/server/master_block_registry_test.go @@ -2284,3 +2284,71 @@ func TestRegistry_UpdateFullHeartbeat_ExplicitHealthySuppressesStaleNeedsRebuild t.Fatalf("expected explicit false to suppress stale needs_rebuild heuristic, got %q", entry.VolumeMode) } } + +func TestRegistry_UpdateFullHeartbeat_ConsumesExplicitPublishHealthyFromPrimaryHeartbeat(t *testing.T) { + r := NewBlockVolumeRegistry() + if err := r.Register(&BlockVolumeEntry{ + Name: "vol-master-publish-healthy", + VolumeServer: "primary-server:8080", + Path: "/blocks/vol-master-publish-healthy-primary.blk", + Status: StatusActive, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + ReplicaFactor: 2, + Replicas: []ReplicaInfo{{ + Server: "replica-server:8080", + Path: "/blocks/vol-master-publish-healthy-replica.blk", + Ready: false, + }}, + }); err != nil { + t.Fatalf("register: %v", err) + } + + publishHealthy := true + r.UpdateFullHeartbeat("primary-server:8080", []*master_pb.BlockVolumeInfoMessage{{ + Path: "/blocks/vol-master-publish-healthy-primary.blk", + Role: blockvol.RoleToWire(blockvol.RolePrimary), + PublishHealthy: &publishHealthy, + }}, "") + + entry, _ := r.Lookup("vol-master-publish-healthy") + if !entry.PublishHealthy || !entry.HasPublishHealthy { + t.Fatalf("expected explicit publish_healthy truth on entry, entry=%+v", entry) + } + if entry.VolumeMode != "publish_healthy" { + t.Fatalf("expected publish_healthy from explicit primary heartbeat truth, got %q", entry.VolumeMode) + } +} + +func TestRegistry_UpdateFullHeartbeat_ExplicitUnhealthySuppressesStalePublishHealthyHeuristic(t *testing.T) { + r := NewBlockVolumeRegistry() + if err := r.Register(&BlockVolumeEntry{ + Name: "vol-master-publish-healthy-explicit-false", + VolumeServer: "primary-server:8080", + Path: "/blocks/vol-master-publish-healthy-explicit-false-primary.blk", + Status: StatusActive, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + ReplicaFactor: 2, + Replicas: []ReplicaInfo{{ + Server: "replica-server:8080", + Path: "/blocks/vol-master-publish-healthy-explicit-false-replica.blk", + Ready: true, + }}, + }); err != nil { + t.Fatalf("register: %v", err) + } + + publishHealthy := false + r.UpdateFullHeartbeat("primary-server:8080", []*master_pb.BlockVolumeInfoMessage{{ + Path: "/blocks/vol-master-publish-healthy-explicit-false-primary.blk", + Role: blockvol.RoleToWire(blockvol.RolePrimary), + PublishHealthy: &publishHealthy, + }}, "") + + entry, _ := r.Lookup("vol-master-publish-healthy-explicit-false") + if !entry.HasPublishHealthy || entry.PublishHealthy { + t.Fatalf("expected explicit false publish_healthy truth on entry, entry=%+v", entry) + } + if entry.VolumeMode != "bootstrap_pending" { + t.Fatalf("expected explicit false to suppress stale publish_healthy heuristic, got %q", entry.VolumeMode) + } +} diff --git a/weed/server/volume_server_block.go b/weed/server/volume_server_block.go index 43a483fed..af3b52a40 100644 --- a/weed/server/volume_server_block.go +++ b/weed/server/volume_server_block.go @@ -967,6 +967,7 @@ func (bs *BlockService) CollectBlockVolumeHeartbeat() []blockvol.BlockVolumeInfo msgs[i].ReplicaDataAddr, msgs[i].ReplicaCtrlAddr = bs.heartbeatReplicaAddrs(msgs[i].Path, s) msgs[i].ReplicaReady = bs.heartbeatReplicaReady(msgs[i].Path, s) msgs[i].NeedsRebuild = bs.heartbeatNeedsRebuild(msgs[i].Path, s) + msgs[i].PublishHealthy = bs.heartbeatPublishHealthy(msgs[i].Path, s) } msgs[i].ReplicaDegraded = bs.heartbeatReplicaDegraded(msgs[i].Path, msgs[i].ReplicaDegraded) // NVMe publication: report nvme_addr and nqn if NVMe target is running. @@ -1036,6 +1037,21 @@ func (bs *BlockService) heartbeatNeedsRebuild(path string, state *volReplState) return false } +// heartbeatPublishHealthy returns the explicit publish_healthy truth that should +// be exposed on the current heartbeat surface. On the core-present path it +// preserves the core-owned publication truth rather than reconstructing it from +// secondary readiness/degraded signals. Older paths fall back to the existing +// adapter-local publish flag for compatibility. +func (bs *BlockService) heartbeatPublishHealthy(path string, state *volReplState) bool { + if state == nil { + return false + } + if proj, ok := bs.CoreProjection(path); ok { + return proj.Publication.Healthy + } + return state.publishHealthy +} + // heartbeatReplicaDegraded returns the bounded degraded bit for the current // heartbeat surface. On the Phase 15 live path it prefers the core mode when // present, then falls back to the runtime-local status bit. diff --git a/weed/server/volume_server_block_test.go b/weed/server/volume_server_block_test.go index a4c15290f..4152bba76 100644 --- a/weed/server/volume_server_block_test.go +++ b/weed/server/volume_server_block_test.go @@ -1313,6 +1313,37 @@ func TestBlockService_CollectBlockVolumeHeartbeat_PrimaryNeedsRebuildUsesCoreMod } } +func TestBlockService_CollectBlockVolumeHeartbeat_PrimaryPublishHealthyUsesCoreTruth(t *testing.T) { + bs := newTestBlockServiceDirect(t) + path := createTestVolDirect(t, bs, "vol-heartbeat-publish-healthy-primary") + + errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{ + { + Path: path, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + LeaseTtlMs: 30000, + ReplicaServerID: "vs-2", + ReplicaDataAddr: "10.0.0.2:4260", + ReplicaCtrlAddr: "10.0.0.2:4261", + }, + }) + if len(errs) != 1 || errs[0] != nil { + t.Fatalf("apply assignment errs=%v", errs) + } + + bs.applyCoreEvent(engine.ShipperConnectedObserved{ID: path}) + bs.applyCoreEvent(engine.BarrierAccepted{ID: path, FlushedLSN: 12}) + + msg := findHeartbeatMsg(bs.CollectBlockVolumeHeartbeat(), path) + if msg == nil { + t.Fatal("volume missing from heartbeat") + } + if !msg.PublishHealthy { + t.Fatalf("expected explicit publish_healthy truth on heartbeat, msg=%+v", msg) + } +} + func TestBlockService_HeartbeatIncludesReplicaAddrs(t *testing.T) { bs := newTestBlockServiceDirect(t) path := createTestVolDirect(t, bs, "vol1") diff --git a/weed/storage/blockvol/block_heartbeat.go b/weed/storage/blockvol/block_heartbeat.go index fc0e3c65b..09729acd2 100644 --- a/weed/storage/blockvol/block_heartbeat.go +++ b/weed/storage/blockvol/block_heartbeat.go @@ -21,6 +21,7 @@ type BlockVolumeInfoMessage struct { ReplicaCtrlAddr string // receiver ctrl listen addr ReplicaReady bool // explicit replica readiness truth on the heartbeat seam NeedsRebuild bool // explicit needs_rebuild truth on the heartbeat seam + PublishHealthy bool // explicit publish_healthy truth on the heartbeat seam HealthScore float64 // CP8-2: 0.0-1.0 ScrubErrors int64 // CP8-2: lifetime scrub error count LastScrubTime int64 // CP8-2: unix seconds diff --git a/weed/storage/blockvol/block_heartbeat_proto.go b/weed/storage/blockvol/block_heartbeat_proto.go index 53943669e..a3a099eb4 100644 --- a/weed/storage/blockvol/block_heartbeat_proto.go +++ b/weed/storage/blockvol/block_heartbeat_proto.go @@ -20,6 +20,7 @@ func InfoMessageToProto(m BlockVolumeInfoMessage) *master_pb.BlockVolumeInfoMess ReplicaCtrlAddr: m.ReplicaCtrlAddr, ReplicaReady: &m.ReplicaReady, NeedsRebuild: &m.NeedsRebuild, + PublishHealthy: &m.PublishHealthy, HealthScore: m.HealthScore, ScrubErrors: m.ScrubErrors, LastScrubTime: m.LastScrubTime, @@ -49,6 +50,7 @@ func InfoMessageFromProto(p *master_pb.BlockVolumeInfoMessage) BlockVolumeInfoMe ReplicaCtrlAddr: p.ReplicaCtrlAddr, ReplicaReady: p.GetReplicaReady(), NeedsRebuild: p.GetNeedsRebuild(), + PublishHealthy: p.GetPublishHealthy(), HealthScore: p.HealthScore, ScrubErrors: p.ScrubErrors, LastScrubTime: p.LastScrubTime, diff --git a/weed/storage/blockvol/block_heartbeat_proto_test.go b/weed/storage/blockvol/block_heartbeat_proto_test.go index a41775840..582aaa273 100644 --- a/weed/storage/blockvol/block_heartbeat_proto_test.go +++ b/weed/storage/blockvol/block_heartbeat_proto_test.go @@ -287,6 +287,26 @@ func TestInfoMessage_NeedsRebuildRoundTrip(t *testing.T) { } } +func TestInfoMessage_PublishHealthyRoundTrip(t *testing.T) { + orig := BlockVolumeInfoMessage{ + Path: "/data/vol.blk", + Epoch: 1, + PublishHealthy: true, + } + pb := InfoMessageToProto(orig) + back := InfoMessageFromProto(pb) + if !back.PublishHealthy { + t.Fatal("PublishHealthy should be true after round-trip") + } + + orig.PublishHealthy = false + pb = InfoMessageToProto(orig) + back = InfoMessageFromProto(pb) + if back.PublishHealthy { + t.Fatal("PublishHealthy should be false after round-trip") + } +} + func TestAssignment_MultiReplicaRoundTrip(t *testing.T) { orig := BlockVolumeAssignment{ Path: "/data/vol.blk",