From 6cad5bb8e1197c0b2c094f88f5376e0da4e7017a Mon Sep 17 00:00:00 2001 From: pingqiu Date: Sat, 4 Apr 2026 13:56:41 -0700 Subject: [PATCH] refactor: rebind bounded volume mode heartbeat truth Make the heartbeat/master boundary preserve explicit volume_mode truth so master consume no longer reconstructs outward mode only from secondary heartbeat signals. Keep backward compatibility by falling back to the previous reconstruction when older heartbeats do not send the field. Made-with: Cursor --- sw-block/.private/phase/phase-16-log.md | 68 +++++++++++ sw-block/.private/phase/phase-16.md | 59 +++++++++ weed/pb/master.proto | 1 + weed/pb/master_pb/master.pb.go | 17 ++- weed/server/master_block_registry.go | 46 +++++-- weed/server/master_block_registry_test.go | 115 ++++++++++++++++++ weed/server/volume_server_block.go | 15 +++ weed/server/volume_server_block_test.go | 39 ++++++ weed/storage/blockvol/block_heartbeat.go | 1 + .../storage/blockvol/block_heartbeat_proto.go | 9 ++ .../blockvol/block_heartbeat_proto_test.go | 13 ++ 11 files changed, 370 insertions(+), 13 deletions(-) diff --git a/sw-block/.private/phase/phase-16-log.md b/sw-block/.private/phase/phase-16-log.md index e979d9b37..0a2b33db0 100644 --- a/sw-block/.private/phase/phase-16-log.md +++ b/sw-block/.private/phase/phase-16-log.md @@ -1409,3 +1409,71 @@ Conclusion: explicit field still fall back to the previous reconstruction 3. this slice still does not claim full `VolumeMode` heartbeat ownership or broad failover closure by itself + +--- + +#### `16P` Start Note Rev 1 + +Date: 2026-04-04 +Scope: bounded explicit `volume_mode` preservation on the heartbeat/master seam + +Why this slice exists: + +1. `16M`, `16N`, and `16O` made the main heartbeat mode ingredients explicit +2. but master-side outward `VolumeMode` is still reconstructed from those + secondary signals instead of directly consuming the core-owned mode truth +3. that leaves one final bounded outward mode seam at the heartbeat/master + boundary even when the core already knows the exact chosen mode + +Chosen implementation rule: + +1. widen the heartbeat wire additively with an explicit `volume_mode` field +2. emit it from the current bounded core mode on the core-present path +3. make master outward mode prefer explicit heartbeat mode truth and retain the + previous reconstruction only as backward-compatible fallback +4. do not broaden this slice into publication reason or restart/disturbance + closure + +--- + +#### `16P` Delivery Note Rev 1 + +Date: 2026-04-04 +Scope: bounded explicit `volume_mode` preservation on the heartbeat/master seam + +What changed: + +1. `weed/pb/master.proto` + - added additive optional `volume_mode` to `BlockVolumeInfoMessage` +2. `weed/pb/master_pb/master.pb.go` + - regenerated so heartbeat wire presence is represented as `*string` +3. `weed/storage/blockvol/block_heartbeat.go` + - heartbeat wire struct now carries explicit `VolumeMode` +4. `weed/storage/blockvol/block_heartbeat_proto.go` + - heartbeat conversion now writes and reads `VolumeMode` +5. `weed/server/volume_server_block.go` + - heartbeat emission now preserves explicit bounded core mode truth directly +6. `weed/server/master_block_registry.go` + - outward `VolumeMode` now prefers explicit heartbeat mode truth and falls + back to the previous reconstruction when the field is absent +7. focused tests in `block_heartbeat_proto_test.go`, + `volume_server_block_test.go`, and `master_block_registry_test.go` + - now prove explicit outward mode preservation and backward-compatible + fallback + +Proof / evidence: + +1. `go test ./weed/storage/blockvol/ -count=1 -run "TestInfoMessage_(Replica|NeedsRebuild|PublishHealthy|VolumeMode)"` +2. `go test ./weed/server/ -count=1 -timeout 120s -run "Test(BlockService_CollectBlockVolumeHeartbeat_(PrimaryUsesCoreReadinessGate|PrimaryNeedsRebuildUsesCoreMode|PrimaryPublishHealthyUsesCoreTruth|PrimaryDegradedUsesCoreModeTruth)|Registry_UpdateFullHeartbeat_(ConsumesExplicitVolumeModeFromPrimaryHeartbeat|VolumeModeFallsBackWhenFieldAbsent|ConsumesExplicitPublishHealthyFromPrimaryHeartbeat|ConsumesExplicitNeedsRebuildFromPrimaryHeartbeat))"` +3. `go test ./weed/server/ -count=1 -timeout 120s -run "Test(BlockService_ApplyAssignments_|P16B_|P4_|Registry_UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaReady|Registry_UpdateFullHeartbeat_ReplicaReadyFallsBackToAddressesWhenFieldAbsent|Registry_UpdateFullHeartbeat_ExplicitUnhealthySuppressesStalePublishHealthyHeuristic)"` +4. result: `PASS` + +Conclusion: + +1. the heartbeat/master seam no longer reconstructs outward `VolumeMode` only + from secondary heartbeat signals when explicit core-owned mode truth is + available +2. backward compatibility is preserved because older heartbeats without the + explicit field still fall back to the previous reconstruction +3. this slice still does not claim publication reason, restart/disturbance, or + broad failover closure by itself diff --git a/sw-block/.private/phase/phase-16.md b/sw-block/.private/phase/phase-16.md index 845278050..46de3e93c 100644 --- a/sw-block/.private/phase/phase-16.md +++ b/sw-block/.private/phase/phase-16.md @@ -661,6 +661,62 @@ Evidence: 1. focused working-tree change after `16N` closeout +### `16P`: VolumeMode Heartbeat Truth Rebinding + +Goal: + +1. close one bounded failover/publication seam by preserving explicit + `volume_mode` truth across the heartbeat/master boundary instead of + reconstructing outward mode from multiple secondary heartbeat bits +2. keep the slice limited to outward `VolumeMode` preservation on the current + heartbeat wire and master-registry consume path, not broader restart/failover + closure + +Acceptance object: + +1. `BlockVolumeInfoMessage` carries an additive explicit `volume_mode` field on + the heartbeat wire +2. `weed/server` heartbeat emission sets that field from the current core-owned + mode truth on the core-present path +3. `master_block_registry` prefers explicit heartbeat `volume_mode` truth for + outward mode while retaining the previous reconstruction as backward-compatible + fallback when the field is absent +4. focused proofs show primary `bootstrap_pending`, `degraded`, + `needs_rebuild`, and `publish_healthy` survive heartbeat/master consume as + explicit outward mode truth +5. this slice still does not yet claim broad restart/disturbance or launch + closure + +Current chosen path: + +1. widen `master.proto` / heartbeat conversion with an additive `volume_mode` + field +2. emit that field from `CollectBlockVolumeHeartbeat` using the bounded core + mode on the core-present path +3. let master outward `VolumeMode` prefer explicit heartbeat mode truth and keep + the previous reconstruction only as backward-compatible fallback + +Status: + +1. delivered + +Delivered result: + +1. `BlockVolumeInfoMessage` now carries additive explicit `volume_mode` truth on + the heartbeat wire +2. `weed/server` heartbeat emission now preserves explicit bounded + core-owned `VolumeMode` truth on the current core-present path +3. `master_block_registry` now prefers explicit heartbeat `volume_mode` for + outward mode while keeping the previous reconstruction as backward-compatible + fallback when the field is absent +4. focused proofs now show primary `bootstrap_pending`, `degraded`, + `needs_rebuild`, and `publish_healthy` survive heartbeat/master consume as + explicit outward mode truth + +Evidence: + +1. focused working-tree change after `16O` closeout + ## Current Checkpoint Review Target The current review target is the current widened bounded runtime checkpoint @@ -746,6 +802,9 @@ boundary: - primary heartbeat/master consume now preserves explicit bounded healthy-publication truth with backward-compatible fallback for older heartbeats +17. `16P` delivered: + - heartbeat/master consume now preserves explicit bounded `VolumeMode` truth + with backward-compatible fallback for older heartbeats After this checkpoint: diff --git a/weed/pb/master.proto b/weed/pb/master.proto index 68f32b348..b415a8337 100644 --- a/weed/pb/master.proto +++ b/weed/pb/master.proto @@ -515,6 +515,7 @@ message BlockVolumeInfoMessage { optional bool replica_ready = 19; optional bool needs_rebuild = 20; optional bool publish_healthy = 21; + optional string volume_mode = 22; } message BlockVolumeShortInfoMessage { diff --git a/weed/pb/master_pb/master.pb.go b/weed/pb/master_pb/master.pb.go index 550f22f30..8c6e8b978 100644 --- a/weed/pb/master_pb/master.pb.go +++ b/weed/pb/master_pb/master.pb.go @@ -3914,6 +3914,7 @@ type BlockVolumeInfoMessage struct { ReplicaReady *bool `protobuf:"varint,19,opt,name=replica_ready,json=replicaReady,proto3,oneof" json:"replica_ready,omitempty"` NeedsRebuild *bool `protobuf:"varint,20,opt,name=needs_rebuild,json=needsRebuild,proto3,oneof" json:"needs_rebuild,omitempty"` PublishHealthy *bool `protobuf:"varint,21,opt,name=publish_healthy,json=publishHealthy,proto3,oneof" json:"publish_healthy,omitempty"` + VolumeMode *string `protobuf:"bytes,22,opt,name=volume_mode,json=volumeMode,proto3,oneof" json:"volume_mode,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -4095,6 +4096,13 @@ func (x *BlockVolumeInfoMessage) GetPublishHealthy() bool { return false } +func (x *BlockVolumeInfoMessage) GetVolumeMode() string { + if x != nil && x.VolumeMode != nil { + return *x.VolumeMode + } + return "" +} + type BlockVolumeShortInfoMessage struct { state protoimpl.MessageState `protogen:"open.v1"` Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` @@ -5983,7 +5991,7 @@ const file_weed_pb_master_proto_rawDesc = "" + "\x0fprevious_leader\x18\x01 \x01(\tR\x0epreviousLeader\x12\x1d\n" + "\n" + "new_leader\x18\x02 \x01(\tR\tnewLeader\"\x14\n" + - "\x12VolumeGrowResponse\"\x9c\x06\n" + + "\x12VolumeGrowResponse\"\xd2\x06\n" + "\x16BlockVolumeInfoMessage\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x1f\n" + "\vvolume_size\x18\x02 \x01(\x04R\n" + @@ -6009,10 +6017,13 @@ const file_weed_pb_master_proto_rawDesc = "" + "\x03nqn\x18\x12 \x01(\tR\x03nqn\x12(\n" + "\rreplica_ready\x18\x13 \x01(\bH\x00R\freplicaReady\x88\x01\x01\x12(\n" + "\rneeds_rebuild\x18\x14 \x01(\bH\x01R\fneedsRebuild\x88\x01\x01\x12,\n" + - "\x0fpublish_healthy\x18\x15 \x01(\bH\x02R\x0epublishHealthy\x88\x01\x01B\x10\n" + + "\x0fpublish_healthy\x18\x15 \x01(\bH\x02R\x0epublishHealthy\x88\x01\x01\x12$\n" + + "\vvolume_mode\x18\x16 \x01(\tH\x03R\n" + + "volumeMode\x88\x01\x01B\x10\n" + "\x0e_replica_readyB\x10\n" + "\x0e_needs_rebuildB\x12\n" + - "\x10_publish_healthy\"\x8e\x01\n" + + "\x10_publish_healthyB\x0e\n" + + "\f_volume_mode\"\x8e\x01\n" + "\x1bBlockVolumeShortInfoMessage\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x1f\n" + "\vvolume_size\x18\x02 \x01(\x04R\n" + diff --git a/weed/server/master_block_registry.go b/weed/server/master_block_registry.go index 56979bfed..89c036df4 100644 --- a/weed/server/master_block_registry.go +++ b/weed/server/master_block_registry.go @@ -71,16 +71,18 @@ type BlockVolumeEntry struct { RebuildListenAddr string // rebuild server listen addr on primary // CP8-2: Multi-replica support. - ReplicaFactor int // 2 or 3 (default 2) - Replicas []ReplicaInfo // one per replica (RF-1 entries) - HealthScore float64 // primary health score from heartbeat - ReplicaReady bool // all configured replicas are ready for publication - ReplicaDegraded bool // aggregate: transport degraded OR not ready - TransportDegraded bool // primary reports degraded replicas - NeedsRebuild bool // explicit primary heartbeat needs_rebuild truth when present - HasNeedsRebuild bool // whether the current primary heartbeat carried explicit needs_rebuild truth - PublishHealthy bool // explicit primary heartbeat publish_healthy truth when present - HasPublishHealthy bool // whether the current primary heartbeat carried explicit publish_healthy truth + ReplicaFactor int // 2 or 3 (default 2) + Replicas []ReplicaInfo // one per replica (RF-1 entries) + HealthScore float64 // primary health score from heartbeat + ReplicaReady bool // all configured replicas are ready for publication + ReplicaDegraded bool // aggregate: transport degraded OR not ready + TransportDegraded bool // primary reports degraded replicas + NeedsRebuild bool // explicit primary heartbeat needs_rebuild truth when present + HasNeedsRebuild bool // whether the current primary heartbeat carried explicit needs_rebuild truth + PublishHealthy bool // explicit primary heartbeat publish_healthy truth when present + HasPublishHealthy bool // whether the current primary heartbeat carried explicit publish_healthy truth + HeartbeatVolumeMode string // explicit primary heartbeat outward volume_mode truth when present + HasHeartbeatVolumeMode bool // whether the current primary heartbeat carried explicit outward volume_mode truth // CP13-9: Normalized volume mode for external surfaces. // Computed by recomputeReplicaState from the current entry state. @@ -167,6 +169,10 @@ func (e *BlockVolumeEntry) recomputeReplicaState() { // - "degraded": replication was healthy but is now impaired (transient) // - "needs_rebuild": one or more replicas have unrecoverable gap func (e *BlockVolumeEntry) computeVolumeMode() string { + if e.HasHeartbeatVolumeMode && validHeartbeatVolumeMode(e.HeartbeatVolumeMode) { + return e.HeartbeatVolumeMode + } + rf := e.ReplicaFactor if rf == 0 { rf = 1 @@ -656,6 +662,7 @@ func (r *BlockVolumeRegistry) applyPrimaryHeartbeatObservation(existing *BlockVo existing.TransportDegraded = info.ReplicaDegraded existing.NeedsRebuild, existing.HasNeedsRebuild = primaryNeedsRebuildObservedFromHeartbeat(info) existing.PublishHealthy, existing.HasPublishHealthy = primaryPublishHealthyObservedFromHeartbeat(info) + existing.HeartbeatVolumeMode, existing.HasHeartbeatVolumeMode = primaryVolumeModeObservedFromHeartbeat(info) existing.WALHeadLSN = info.WalHeadLsn // F3: only update DurabilityMode when non-empty (prevents older VS from clearing strict mode). if info.DurabilityMode != "" { @@ -763,6 +770,25 @@ func primaryPublishHealthyObservedFromHeartbeat(info *master_pb.BlockVolumeInfoM return false, false } +func primaryVolumeModeObservedFromHeartbeat(info *master_pb.BlockVolumeInfoMessage) (string, bool) { + if info == nil { + return "", false + } + if info.VolumeMode != nil { + return info.GetVolumeMode(), true + } + return "", false +} + +func validHeartbeatVolumeMode(mode string) bool { + switch mode { + case "allocated_only", "bootstrap_pending", "publish_healthy", "degraded", "needs_rebuild": + return true + default: + return false + } +} + // reconcileOnRestart handles the case where a second server reports a volume // name that already exists in the registry during master restart reconstruction. // Uses epoch-based tie-breaking to determine who is the real primary. diff --git a/weed/server/master_block_registry_test.go b/weed/server/master_block_registry_test.go index d55dc0b24..ce4832b86 100644 --- a/weed/server/master_block_registry_test.go +++ b/weed/server/master_block_registry_test.go @@ -2352,3 +2352,118 @@ func TestRegistry_UpdateFullHeartbeat_ExplicitUnhealthySuppressesStalePublishHea t.Fatalf("expected explicit false to suppress stale publish_healthy heuristic, got %q", entry.VolumeMode) } } + +func TestRegistry_UpdateFullHeartbeat_ConsumesExplicitVolumeModeFromPrimaryHeartbeat(t *testing.T) { + tests := []struct { + name string + mode string + replicas []ReplicaInfo + wantMode string + }{ + { + name: "bootstrap_pending", + mode: "bootstrap_pending", + replicas: []ReplicaInfo{{ + Server: "replica-server:8080", + Path: "/blocks/vol-master-mode-bootstrap-replica.blk", + Ready: true, + }}, + wantMode: "bootstrap_pending", + }, + { + name: "degraded", + mode: "degraded", + replicas: []ReplicaInfo{{ + Server: "replica-server:8080", + Path: "/blocks/vol-master-mode-degraded-replica.blk", + Ready: true, + }}, + wantMode: "degraded", + }, + { + name: "needs_rebuild", + mode: "needs_rebuild", + replicas: []ReplicaInfo{{ + Server: "replica-server:8080", + Path: "/blocks/vol-master-mode-needs-rebuild-replica.blk", + Ready: true, + }}, + wantMode: "needs_rebuild", + }, + { + name: "publish_healthy", + mode: "publish_healthy", + replicas: []ReplicaInfo{{ + Server: "replica-server:8080", + Path: "/blocks/vol-master-mode-publish-healthy-replica.blk", + Ready: false, + }}, + wantMode: "publish_healthy", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := NewBlockVolumeRegistry() + if err := r.Register(&BlockVolumeEntry{ + Name: "vol-master-mode-" + tt.name, + VolumeServer: "primary-server:8080", + Path: "/blocks/vol-master-mode-" + tt.name + "-primary.blk", + Status: StatusActive, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + ReplicaFactor: 2, + Replicas: tt.replicas, + }); err != nil { + t.Fatalf("register: %v", err) + } + + mode := tt.mode + r.UpdateFullHeartbeat("primary-server:8080", []*master_pb.BlockVolumeInfoMessage{{ + Path: "/blocks/vol-master-mode-" + tt.name + "-primary.blk", + Role: blockvol.RoleToWire(blockvol.RolePrimary), + VolumeMode: &mode, + }}, "") + + entry, _ := r.Lookup("vol-master-mode-" + tt.name) + if !entry.HasHeartbeatVolumeMode || entry.HeartbeatVolumeMode != tt.mode { + t.Fatalf("expected explicit volume_mode truth on entry, entry=%+v", entry) + } + if entry.VolumeMode != tt.wantMode { + t.Fatalf("expected explicit volume_mode %q, got %q", tt.wantMode, entry.VolumeMode) + } + }) + } +} + +func TestRegistry_UpdateFullHeartbeat_VolumeModeFallsBackWhenFieldAbsent(t *testing.T) { + r := NewBlockVolumeRegistry() + if err := r.Register(&BlockVolumeEntry{ + Name: "vol-master-mode-fallback", + VolumeServer: "primary-server:8080", + Path: "/blocks/vol-master-mode-fallback-primary.blk", + Status: StatusActive, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + ReplicaFactor: 2, + Replicas: []ReplicaInfo{{ + Server: "replica-server:8080", + Path: "/blocks/vol-master-mode-fallback-replica.blk", + Ready: true, + }}, + }); err != nil { + t.Fatalf("register: %v", err) + } + + r.UpdateFullHeartbeat("primary-server:8080", []*master_pb.BlockVolumeInfoMessage{{ + Path: "/blocks/vol-master-mode-fallback-primary.blk", + Role: blockvol.RoleToWire(blockvol.RolePrimary), + ReplicaDegraded: true, + }}, "") + + entry, _ := r.Lookup("vol-master-mode-fallback") + if entry.HasHeartbeatVolumeMode { + t.Fatalf("did not expect explicit volume_mode when field absent, entry=%+v", entry) + } + if entry.VolumeMode != "degraded" { + t.Fatalf("expected fallback reconstructed degraded mode, got %q", entry.VolumeMode) + } +} diff --git a/weed/server/volume_server_block.go b/weed/server/volume_server_block.go index af3b52a40..a46606551 100644 --- a/weed/server/volume_server_block.go +++ b/weed/server/volume_server_block.go @@ -968,6 +968,7 @@ func (bs *BlockService) CollectBlockVolumeHeartbeat() []blockvol.BlockVolumeInfo msgs[i].ReplicaReady = bs.heartbeatReplicaReady(msgs[i].Path, s) msgs[i].NeedsRebuild = bs.heartbeatNeedsRebuild(msgs[i].Path, s) msgs[i].PublishHealthy = bs.heartbeatPublishHealthy(msgs[i].Path, s) + msgs[i].VolumeMode = bs.heartbeatVolumeMode(msgs[i].Path, s) } msgs[i].ReplicaDegraded = bs.heartbeatReplicaDegraded(msgs[i].Path, msgs[i].ReplicaDegraded) // NVMe publication: report nvme_addr and nqn if NVMe target is running. @@ -1052,6 +1053,20 @@ func (bs *BlockService) heartbeatPublishHealthy(path string, state *volReplState return state.publishHealthy } +// heartbeatVolumeMode returns the explicit outward volume mode that should be +// exposed on the current heartbeat surface. On the core-present path it +// preserves the core-owned mode directly. Older paths return empty so the +// master can fall back to previous reconstruction logic. +func (bs *BlockService) heartbeatVolumeMode(path string, state *volReplState) string { + if state == nil { + return "" + } + if proj, ok := bs.CoreProjection(path); ok { + return string(proj.Mode.Name) + } + return "" +} + // heartbeatReplicaDegraded returns the bounded degraded bit for the current // heartbeat surface. On the Phase 15 live path it prefers the core mode when // present, then falls back to the runtime-local status bit. diff --git a/weed/server/volume_server_block_test.go b/weed/server/volume_server_block_test.go index 4152bba76..d3401f095 100644 --- a/weed/server/volume_server_block_test.go +++ b/weed/server/volume_server_block_test.go @@ -1067,6 +1067,9 @@ func TestBlockService_CollectBlockVolumeHeartbeat_PrimaryUsesCoreReadinessGate(t if msg.ReplicaReady { t.Fatalf("primary heartbeat must not claim replica-ready truth, msg=%+v", msg) } + if msg.VolumeMode != "bootstrap_pending" { + t.Fatalf("expected explicit bootstrap_pending volume_mode on heartbeat, msg=%+v", msg) + } } func TestBlockService_CollectBlockVolumeHeartbeat_ReplicaUsesCoreReadinessGate(t *testing.T) { @@ -1311,6 +1314,9 @@ func TestBlockService_CollectBlockVolumeHeartbeat_PrimaryNeedsRebuildUsesCoreMod if !msg.ReplicaDegraded { t.Fatalf("expected degraded bit to remain true on needs_rebuild path, msg=%+v", msg) } + if msg.VolumeMode != "needs_rebuild" { + t.Fatalf("expected explicit needs_rebuild volume_mode on heartbeat, msg=%+v", msg) + } } func TestBlockService_CollectBlockVolumeHeartbeat_PrimaryPublishHealthyUsesCoreTruth(t *testing.T) { @@ -1342,6 +1348,39 @@ func TestBlockService_CollectBlockVolumeHeartbeat_PrimaryPublishHealthyUsesCoreT if !msg.PublishHealthy { t.Fatalf("expected explicit publish_healthy truth on heartbeat, msg=%+v", msg) } + if msg.VolumeMode != "publish_healthy" { + t.Fatalf("expected explicit publish_healthy volume_mode on heartbeat, msg=%+v", msg) + } +} + +func TestBlockService_CollectBlockVolumeHeartbeat_PrimaryDegradedUsesCoreModeTruth(t *testing.T) { + bs := newTestBlockServiceDirect(t) + path := createTestVolDirect(t, bs, "vol-heartbeat-degraded-primary") + + errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{ + { + Path: path, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + LeaseTtlMs: 30000, + ReplicaServerID: "vs-2", + ReplicaDataAddr: "10.0.0.2:4260", + ReplicaCtrlAddr: "10.0.0.2:4261", + }, + }) + if len(errs) != 1 || errs[0] != nil { + t.Fatalf("apply assignment errs=%v", errs) + } + + bs.applyCoreEvent(engine.BarrierRejected{ID: path, Reason: "barrier_timeout"}) + + msg := findHeartbeatMsg(bs.CollectBlockVolumeHeartbeat(), path) + if msg == nil { + t.Fatal("volume missing from heartbeat") + } + if msg.VolumeMode != "degraded" { + t.Fatalf("expected explicit degraded volume_mode on heartbeat, msg=%+v", msg) + } } func TestBlockService_HeartbeatIncludesReplicaAddrs(t *testing.T) { diff --git a/weed/storage/blockvol/block_heartbeat.go b/weed/storage/blockvol/block_heartbeat.go index 09729acd2..af862f236 100644 --- a/weed/storage/blockvol/block_heartbeat.go +++ b/weed/storage/blockvol/block_heartbeat.go @@ -22,6 +22,7 @@ type BlockVolumeInfoMessage struct { ReplicaReady bool // explicit replica readiness truth on the heartbeat seam NeedsRebuild bool // explicit needs_rebuild truth on the heartbeat seam PublishHealthy bool // explicit publish_healthy truth on the heartbeat seam + VolumeMode string // explicit outward volume_mode truth on the heartbeat seam HealthScore float64 // CP8-2: 0.0-1.0 ScrubErrors int64 // CP8-2: lifetime scrub error count LastScrubTime int64 // CP8-2: unix seconds diff --git a/weed/storage/blockvol/block_heartbeat_proto.go b/weed/storage/blockvol/block_heartbeat_proto.go index a3a099eb4..0a513f58a 100644 --- a/weed/storage/blockvol/block_heartbeat_proto.go +++ b/weed/storage/blockvol/block_heartbeat_proto.go @@ -4,6 +4,13 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" ) +func optionalStringPtr(v string) *string { + if v == "" { + return nil + } + return &v +} + // InfoMessageToProto converts a Go wire type to proto. func InfoMessageToProto(m BlockVolumeInfoMessage) *master_pb.BlockVolumeInfoMessage { return &master_pb.BlockVolumeInfoMessage{ @@ -21,6 +28,7 @@ func InfoMessageToProto(m BlockVolumeInfoMessage) *master_pb.BlockVolumeInfoMess ReplicaReady: &m.ReplicaReady, NeedsRebuild: &m.NeedsRebuild, PublishHealthy: &m.PublishHealthy, + VolumeMode: optionalStringPtr(m.VolumeMode), HealthScore: m.HealthScore, ScrubErrors: m.ScrubErrors, LastScrubTime: m.LastScrubTime, @@ -51,6 +59,7 @@ func InfoMessageFromProto(p *master_pb.BlockVolumeInfoMessage) BlockVolumeInfoMe ReplicaReady: p.GetReplicaReady(), NeedsRebuild: p.GetNeedsRebuild(), PublishHealthy: p.GetPublishHealthy(), + VolumeMode: p.GetVolumeMode(), HealthScore: p.HealthScore, ScrubErrors: p.ScrubErrors, LastScrubTime: p.LastScrubTime, diff --git a/weed/storage/blockvol/block_heartbeat_proto_test.go b/weed/storage/blockvol/block_heartbeat_proto_test.go index 582aaa273..9088cf5e0 100644 --- a/weed/storage/blockvol/block_heartbeat_proto_test.go +++ b/weed/storage/blockvol/block_heartbeat_proto_test.go @@ -307,6 +307,19 @@ func TestInfoMessage_PublishHealthyRoundTrip(t *testing.T) { } } +func TestInfoMessage_VolumeModeRoundTrip(t *testing.T) { + orig := BlockVolumeInfoMessage{ + Path: "/data/vol.blk", + Epoch: 1, + VolumeMode: "bootstrap_pending", + } + pb := InfoMessageToProto(orig) + back := InfoMessageFromProto(pb) + if back.VolumeMode != "bootstrap_pending" { + t.Fatalf("VolumeMode: got %q, want %q", back.VolumeMode, "bootstrap_pending") + } +} + func TestAssignment_MultiReplicaRoundTrip(t *testing.T) { orig := BlockVolumeAssignment{ Path: "/data/vol.blk",