From 8c2485e0e9d8acce15110d35ed5403d31e242e7b Mon Sep 17 00:00:00 2001 From: pingqiu Date: Fri, 3 Apr 2026 20:58:12 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20Phase=2015=20+=20Phase=2016A/B=20?= =?UTF-8?q?=E2=80=94=20V2=20core=20integration=20+=20checkpoint=20review?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 15: V2 core wired into BlockService - volume_server_block.go: v2Core field, applyCoreAssignmentEvent, core command executors (ApplyRole, StartReceiver, ConfigureShipper, InvalidateSession, StartCatchUp, StartRebuild, PublishProjection) - Assignment processing now goes through core engine → command emission → bounded execution, replacing direct V1 replication setup - master_block_registry.go: ClusterHealthSummary, VolumeMode in entries - master_server_handlers_block.go: blockStatusHandler, entryToVolumeInfo refactored with entryReplicaSurface Phase 16A: Core projection surfaces Phase 16B: Bounded closure (checkpoint review ready) Test fixes: add v2Core to manually-constructed BlockService in idempotence, convergence, soak, and CP13-8A tests (required because V1 replication setup paths now delegate to core engine). All tests pass (21s regression). Co-Authored-By: Claude Opus 4.6 (1M context) --- sw-block/.private/phase/phase-15-log.md | 879 ++++++++++++++++++ sw-block/.private/phase/phase-15.md | 101 ++ .../phase/phase-16-checkpoint-review.md | 155 +++ sw-block/.private/phase/phase-16-log.md | 429 +++++++++ sw-block/.private/phase/phase-16.md | 139 +++ sw-block/design/v2-protocol-truths.md | 442 ++++++++- weed/server/block_recovery.go | 215 ++++- weed/server/block_recovery_test.go | 149 +++ .../server/master_block_observability_test.go | 244 +++++ weed/server/master_block_registry.go | 238 ++--- weed/server/master_block_registry_test.go | 134 ++- weed/server/master_grpc_server_block.go | 8 +- weed/server/master_grpc_server_block_test.go | 41 + weed/server/master_server_handlers_block.go | 32 +- .../master_server_handlers_block_test.go | 140 +++ weed/server/qa_block_convergence_test.go | 1 + weed/server/qa_block_idempotence_test.go | 1 + weed/server/qa_block_soak_test.go | 2 + weed/server/volume_server_block.go | 428 ++++++++- weed/server/volume_server_block_debug.go | 53 +- weed/server/volume_server_block_test.go | 632 ++++++++++++- 21 files changed, 4246 insertions(+), 217 deletions(-) create mode 100644 sw-block/.private/phase/phase-15-log.md create mode 100644 sw-block/.private/phase/phase-15.md create mode 100644 sw-block/.private/phase/phase-16-checkpoint-review.md create mode 100644 sw-block/.private/phase/phase-16-log.md create mode 100644 sw-block/.private/phase/phase-16.md diff --git a/sw-block/.private/phase/phase-15-log.md b/sw-block/.private/phase/phase-15-log.md new file mode 100644 index 000000000..39730dba5 --- /dev/null +++ b/sw-block/.private/phase/phase-15-log.md @@ -0,0 +1,879 @@ +Purpose: append-only technical pack and delivery log for `Phase 15` adapter +hook and projection rebinding work. + +--- + +### `15A` Technical Pack + +Date: 2026-04-03 +Goal: connect one narrow live path from `weed/` into the explicit `V2 core` +and one bounded command/projection path back out, without attempting broad +runtime cutover + +#### Layer 1: Semantic Core + +`15A` accepts one bounded thing: + +1. the explicit core is no longer isolated from the integrated path + +It does not accept: + +1. live runtime cutover +2. registry/lookup rebinding +3. broad product-surface migration + +#### Narrow path chosen + +Ingress: + +1. `weed/server/volume_server_block.go` +2. `BlockService.ApplyAssignments()` + +Egress: + +1. `PublishProjectionCommand` +2. adapter-local projection cache on `BlockService` + +Reason: + +1. this is the narrowest stable live path after heartbeat delivery +2. it already owns assignment apply / receiver / shipper setup +3. it allows a real in-process `weed -> core -> adapter` loop without reopening + master registry or product surfaces yet + +#### `15A` Delivery Note Rev 1 + +Date: 2026-04-03 +Scope: wire the explicit core into `BlockService.ApplyAssignments()` on one +narrow live path + +What changed: + +1. `BlockService` now owns an explicit `v2Core` and adapter-local core + projection cache +2. `ApplyAssignments()` now sends bounded assignment and local observation + events into the explicit core: + - `AssignmentDelivered` + - `RoleApplied` + - `ReceiverReadyObserved` + - `ShipperConfiguredObserved` + - bounded `ShipperConnectedObserved` when observable +3. `PublishProjectionCommand` now has one real egress path back into `weed/` + through the adapter-local core projection cache + +Files changed: + +1. `weed/server/volume_server_block.go` + - added `v2Core` + - added adapter-local projection cache + - added narrow assignment-event delivery into the explicit core + - cached `PublishProjectionCommand` output for live-path inspection +2. `weed/server/volume_server_block_test.go` + - added narrow-path proofs for replica and primary assignment delivery +3. `sw-block/.private/phase/phase-15.md` + - added phase/slice framing + +Proofs added: + +1. replica assignment narrow-path proof + - live `ApplyAssignments()` updates core projection cache + - resulting projection is `replica_ready` + - publication stays non-healthy with reason `replica_not_primary` +2. primary assignment narrow-path proof + - live `ApplyAssignments()` updates core projection cache + - resulting projection carries applied role and shipper-configured truth + - publication does not overclaim healthy without durable boundary closure + +Validation: + +1. targeted `weed/server` tests for the new narrow path +2. existing `sw-block/engine/replication` package tests stay green + +Constraint / overclaim / proof review: + +1. semantic constraint satisfied + - one real adapter ingress now reaches the explicit core owner +2. overclaim avoided + - this is not broad surface rebinding + - the cache is adapter-local, not yet a product truth store +3. proof preserved + - `Phase 14` core shell remains the semantic owner + +--- + +#### `15A` Delivery Note Rev 2 + +Date: 2026-04-03 +Scope: prove the adapter-local projection cache does not split from the explicit +core on the narrow live path + +What changed: + +1. extracted adapter command egress into a dedicated helper: + - `applyCoreCommands` +2. added focused proofs that: + - adapter-local projection cache equals the explicit core projection + - repeated unchanged assignment does not make adapter cache and core diverge + +Files changed: + +1. `weed/server/volume_server_block.go` + - extracted command egress helper for `PublishProjectionCommand` +2. `weed/server/volume_server_block_test.go` + - strengthened replica/primary narrow-path tests with cache-vs-core equality + - added unchanged-assignment consistency proof + +Proofs strengthened: + +1. adapter/core projection coherence + - after live `ApplyAssignments()`, cached projection equals + `bs.V2Core().Projection(path)` +2. unchanged-assignment coherence + - repeated identical assignment keeps cache and core aligned + - repeated identical assignment does not mutate the cached outward truth + +Validation: + +1. `go test ./weed/server -run "TestBlockService_ApplyAssignments_(UpdatesCoreProjection|RepeatedUnchangedStaysInSyncWithCore)"` +2. `go test ./...` in `sw-block/engine/replication` +3. result: `PASS` + +Constraint / overclaim / proof review: + +1. semantic constraint satisfied + - the narrow adapter egress is now proven coherent with the explicit core +2. overclaim avoided + - adapter-local cache is no longer merely assumed to reflect core truth +3. proof preserved + - `15A Rev 1` ingress/egress proof remains intact + +--- + +#### `15A` Delivery Note Rev 3 + +Date: 2026-04-03 +Scope: make narrow-path adapter/core coherence explicitly checkable and record +the remaining semantic boundary around adapter-local `PublishHealthy` + +What changed: + +1. added `CoreProjectionMismatches(path)` on `BlockService` + - compares only the fields that should already agree on the narrow `15A` + path + - intentionally excludes adapter-local `ReadinessSnapshot.PublishHealthy` +2. documented that `BlockReadinessSnapshot.PublishHealthy` is still an + adapter-local bit and not the semantic owner for Phase 14 core publication + health +3. strengthened the narrow-path tests to require zero adapter/core mismatches + +Files changed: + +1. `weed/server/volume_server_block.go` + - added `CoreProjectionMismatches` + - clarified `BlockReadinessSnapshot.PublishHealthy` semantics +2. `weed/server/volume_server_block_test.go` + - replica narrow-path proof now asserts zero mismatches + - primary narrow-path proof now asserts zero mismatches + - repeated unchanged assignment proof now asserts zero mismatches + +Proofs strengthened: + +1. narrow-path aligned subset is now explicitly machine-checked +2. remaining semantic split is documented rather than hidden: + - core publication owner = `engine.PublicationView` + - adapter-local `PublishHealthy` remains a current-surface bit pending later + rebinding + +Validation: + +1. `go test ./weed/server -run "TestBlockService_ApplyAssignments_(UpdatesCoreProjection|RepeatedUnchangedStaysInSyncWithCore)"` +2. `go test ./...` in `sw-block/engine/replication` +3. result: `PASS` + +Constraint / overclaim / proof review: + +1. semantic constraint satisfied + - the narrow live path now has an explicit consistency oracle +2. overclaim avoided + - we no longer imply that all adapter-local fields are already rebound +3. proof preserved + - `15A Rev 1` and `Rev 2` proofs still pass + +--- + +### `15B` Technical Pack + +Date: 2026-04-04 +Goal: make one existing `weed/` outward surface consume core-owned projection +truth instead of only adapter-local readiness bits + +#### Layer 1: Semantic Core + +`15B` accepts one bounded thing: + +1. one real `weed/` read surface now prefers the explicit core projection when + that projection exists on the live path + +It does not accept: + +1. master registry rebinding +2. master lookup/public API rebinding +3. broad runtime cutover +4. removal of all adapter-local convenience state + +#### Chosen surface + +Surface: + +1. `weed/server/volume_server_block_debug.go` +2. `/debug/block/shipper` + +Reason: + +1. it is an existing explicit read-only `weed/` surface +2. it already exposes readiness/publication-adjacent fields +3. it is narrow enough to rebind without reopening master or product surfaces + +#### `15B` Delivery Note Rev 1 + +Date: 2026-04-04 +Scope: rebind one VS debug surface so it consumes core-owned projection truth on +the narrow live path + +What changed: + +1. added `BlockService.DebugInfoForVolume(path, vol)` + - builds the outward debug view for one volume + - prefers `CoreProjection(path)` when present + - falls back to adapter-local readiness only when the core projection does + not exist yet +2. `/debug/block/shipper` now uses that helper instead of assembling the + surface directly from adapter-local readiness flags +3. the debug surface now carries bounded core-owned outward meaning: + - `mode` + - `publish_healthy` + - `publication_reason` + +Files changed: + +1. `weed/server/volume_server_block_debug.go` + - added `DebugInfoForVolume` + - rebound debug surface assembly to core projection + - added `mode` and `publication_reason` fields +2. `weed/server/volume_server_block_test.go` + - added primary-path proof that debug `publish_healthy` follows core + publication truth, not adapter-local convenience truth + - added replica-path proof that debug role/mode/readiness/publication align + with the cached core projection +3. `sw-block/.private/phase/phase-15.md` + - marked `15A` delivered and `15B` active + +Proofs added: + +1. primary-path publication overclaim blocked on the real `weed/` surface + - adapter-local readiness may still say `PublishHealthy=true` + - debug surface now reports the core-owned publication result instead + - this proves `assignment delivered != publish healthy` on the live path +2. replica-path projection rebinding + - debug role/mode/readiness/publication now match the cached core projection + - this proves one outward `weed/` surface is consuming core-owned truth + +Validation: + +1. `go test ./weed/server -run "TestBlockService_(ApplyAssignments|DebugInfoForVolume)"` +2. result: `PASS` + +Constraint / overclaim / proof review: + +1. semantic constraint satisfied + - one existing `weed/` surface now consumes explicit core projection truth +2. overclaim avoided + - this is not yet registry/lookup rebinding + - adapter-local readiness still exists as fallback and for unrebound paths +3. proof preserved + - `15A` narrow ingress/egress/cache-coherence proofs still pass + +--- + +#### `15B` Delivery Note Rev 2 + +Date: 2026-04-04 +Scope: rebind the VS heartbeat address-publication gate so it consumes +core-owned readiness projection instead of adapter-local `publishHealthy` + +What changed: + +1. `CollectBlockVolumeHeartbeat()` no longer gates scalar replica transport + addresses on adapter-local `publishHealthy` alone +2. added `heartbeatReplicaAddrs(path, state)` + - prefers `CoreProjection(path)` when present + - on primary path, heartbeat address publication follows core + `Readiness.ShipperConfigured` + - on replica path, heartbeat address publication follows core + `Readiness.ReceiverReady` + - falls back to legacy adapter-local behavior only when the core projection + does not exist yet +3. added focused differential proofs that heartbeat still reports the correct + addresses even when adapter-local `publishHealthy` is manually cleared + +Files changed: + +1. `weed/server/volume_server_block.go` + - rebound heartbeat scalar address publication to core readiness projection +2. `weed/server/volume_server_block_test.go` + - added primary-path heartbeat proof + - added replica-path heartbeat proof + +Proofs added: + +1. primary-path heartbeat rebinding + - core projection says `ShipperConfigured=true` + - core publication still remains unhealthy + - adapter-local `publishHealthy` is forcibly cleared in test + - heartbeat still reports replica addresses, proving it no longer depends on + adapter-local publication convenience truth +2. replica-path heartbeat rebinding + - core projection says `ReceiverReady=true` + - core publication remains unhealthy because replica is not the publication + owner + - adapter-local `publishHealthy` is forcibly cleared in test + - heartbeat still reports receiver addresses, proving it follows the core + readiness projection on the narrow live path + +Validation: + +1. `go test ./weed/server -run "TestBlockService_(ApplyAssignments|DebugInfoForVolume|CollectBlockVolumeHeartbeat)"` +2. result: `PASS` + +Constraint / overclaim / proof review: + +1. semantic constraint satisfied + - one real report path from `weed/` to master now consumes core projection + truth +2. overclaim avoided + - heartbeat proto is not yet widened to carry full mode/publication objects + - master registry/lookup are not yet rebound +3. proof preserved + - `15B Rev 1` debug-surface rebinding still passes + +--- + +#### `15B` Delivery Note Rev 3 + +Date: 2026-04-04 +Scope: rebind the shared VS-side readiness snapshot so aligned fields prefer the +explicit core projection instead of adapter-local readiness state + +What changed: + +1. `ReadinessSnapshot(path)` now prefers `CoreProjection(path)` for the aligned + readiness subset when the narrow Phase 15 path has already produced a + projection: + - `role_applied` + - `receiver_ready` + - `shipper_configured` + - `shipper_connected` + - `replica_eligible` +2. `PublishHealthy` remains adapter-local on `ReadinessSnapshot` + - this keeps the publication ownership boundary explicit instead of silently + rebinding it through a convenience struct +3. added focused proofs that manually corrupt adapter-local readiness state and + show `ReadinessSnapshot()` still returns the core-owned aligned fields + +Files changed: + +1. `weed/server/volume_server_block.go` + - rebound `ReadinessSnapshot()` aligned subset to core projection + - clarified snapshot ownership boundary in comments +2. `weed/server/volume_server_block_test.go` + - added primary-path readiness snapshot proof + - added replica-path readiness snapshot proof + +Proofs added: + +1. primary-path shared snapshot rebinding + - adapter-local `roleApplied` and `shipperConfigured` are forcibly cleared + - `ReadinessSnapshot()` still returns them as true from the core projection + - `PublishHealthy` stays false in the snapshot, proving publication was not + silently rebound +2. replica-path shared snapshot rebinding + - adapter-local `receiverReady` and `replicaEligible` are forcibly cleared + - `ReadinessSnapshot()` still returns them as true from the core projection + - `PublishHealthy` stays false in the snapshot, preserving the ownership + boundary + +Validation: + +1. `go test ./weed/server -run "TestBlockService_(ApplyAssignments|DebugInfoForVolume|CollectBlockVolumeHeartbeat|ReadinessSnapshot)"` +2. result: `PASS` + +Constraint / overclaim / proof review: + +1. semantic constraint satisfied + - the shared VS-side readiness snapshot now consumes the explicit core + projection on the narrow live path +2. overclaim avoided + - publication ownership still remains outside `ReadinessSnapshot` + - master registry/lookup are still not rebound +3. proof preserved + - `15B Rev 1` debug and `Rev 2` heartbeat rebinding proofs still pass + +--- + +#### `15B` Delivery Note Rev 4 + +Date: 2026-04-04 +Scope: rebind the heartbeat `replica_degraded` producer bit to the explicit core +mode and prove the master registry consume path accepts that rebinding + +What changed: + +1. `CollectBlockVolumeHeartbeat()` now also prefers the explicit core + projection for the bounded degraded bit +2. added `heartbeatReplicaDegraded(path, current)` + - maps `ModeDegraded` and `ModeNeedsRebuild` to heartbeat + `ReplicaDegraded=true` + - maps all other core modes to `false` + - falls back to the runtime-local status bit when no core projection exists +3. added a producer-side proof that `heartbeatReplicaDegraded(..., false)` still + returns `true` when the core projection enters `needs_rebuild` +4. added a minimal master-consume proof: + - a `BlockService` heartbeat is produced after core degraded transition + - `BlockVolumeRegistry.UpdateFullHeartbeat()` consumes that heartbeat + - registry truth becomes `TransportDegraded=true`, `ReplicaDegraded=true`, + `VolumeMode="degraded"` + +Files changed: + +1. `weed/server/volume_server_block.go` + - rebound heartbeat degraded bit to explicit core mode +2. `weed/server/volume_server_block_test.go` + - added bounded producer proof for core-driven degraded mapping +3. `weed/server/master_block_registry_test.go` + - added bounded consume proof for registry ingest of the core-influenced + heartbeat degraded bit + +Proofs added: + +1. producer degraded-bit rebinding + - replica path enters core `needs_rebuild` + - helper returns degraded even when the input `current` bit is `false` + - this proves the heartbeat producer is no longer only echoing the runtime + bit on the narrow live path +2. master consume closure + - primary path enters core `degraded` + - heartbeat exports `ReplicaDegraded=true` + - registry consume derives degraded transport and degraded volume mode from + that heartbeat + +Validation: + +1. `go test ./weed/server -run "Test(BlockService_(ApplyAssignments|DebugInfoForVolume|CollectBlockVolumeHeartbeat|ReadinessSnapshot|HeartbeatReplicaDegraded)|Registry_(ReplicaReadyRequiresReplicaHeartbeat|UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaDegraded))"` +2. result: `PASS` + +Constraint / overclaim / proof review: + +1. semantic constraint satisfied + - the first bounded master-consume path now accepts a core-influenced + heartbeat bit +2. overclaim avoided + - registry mode derivation itself is not yet replaced by core-owned mode + - lookup/public API surfaces are still not rebound +3. proof preserved + - `15B Rev 1-3` VS-side rebinding proofs still pass + +--- + +#### `15B` Delivery Note Rev 5 + +Date: 2026-04-04 +Scope: close the other half of the first master-consume boundary by proving the +registry also consumes core-influenced ready heartbeats, not only degraded ones + +What changed: + +1. added a bounded ready-path consume proof in `master_block_registry_test.go` +2. the proof uses a real `BlockService` replica assignment path to produce a + heartbeat whose replica addresses still publish even after adapter-local + `publishHealthy` is manually cleared +3. `BlockVolumeRegistry.UpdateFullHeartbeat()` then consumes that heartbeat and + closes the ready half of the contract: + - replica detail becomes `Ready=true` + - aggregate `ReplicaReady=true` + - aggregate `ReplicaDegraded=false` + - normalized `VolumeMode="publish_healthy"` + +Files changed: + +1. `weed/server/master_block_registry_test.go` + - added `TestRegistry_UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaReady` + +Proofs added: + +1. master consume ready closure + - VS producer emits replica addresses from the core-influenced ready path + even after adapter-local publication convenience truth is cleared + - registry consume converts that heartbeat into ready aggregate truth and + `publish_healthy` outward mode +2. together with `Rev 4`, the first bounded master-consume edge now has both + sides covered: + - degraded consume + - ready consume + +Validation: + +1. `go test ./weed/server -run "TestRegistry_(UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaDegraded|UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaReady)"` +2. result: `PASS` + +Constraint / overclaim / proof review: + +1. semantic constraint satisfied + - the first bounded master-consume edge now has explicit proof for both ready + and degraded outcomes +2. overclaim avoided + - registry is still consuming heartbeat-derived booleans/addresses, not full + core mode/publication objects + - lookup/public API remain unrebound +3. proof preserved + - `15B Rev 4` degraded consume proof still passes unchanged + +--- + +#### `15B` Delivery Note Rev 6 + +Date: 2026-04-04 +Scope: extract the first explicit master-side consume helpers so registry +heartbeat semantics are no longer embedded only as inline logic inside +`UpdateFullHeartbeat()` + +What changed: + +1. extracted `applyPrimaryHeartbeatObservation(existing, info)` + - names the primary-heartbeat -> registry consume contract +2. extracted `applyReplicaHeartbeatObservation(existing, server, existingName, info, result)` + - names the replica-heartbeat -> registry consume contract +3. extracted `replicaReadyObservedFromHeartbeat(info)` + - makes the current ready gate explicit: + published replica receiver addresses => `Ready=true` +4. `UpdateFullHeartbeat()` now delegates to those helpers instead of carrying + the full consume mapping inline + +Files changed: + +1. `weed/server/master_block_registry.go` + - extracted explicit consume helpers from `UpdateFullHeartbeat()` + +Proof / validation posture: + +1. no new behavior claim + - this revision is an extraction/clarification step, not a semantics change +2. existing master consume proofs remain the acceptance object: + - `ReplicaReadyRequiresReplicaHeartbeat` + - `ConsumesCoreInfluencedReplicaDegraded` + - `ConsumesCoreInfluencedReplicaReady` + +Validation: + +1. `go test ./weed/server -run "TestRegistry_(ReplicaReadyRequiresReplicaHeartbeat|UpdateFullHeartbeat|UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaDegraded|UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaReady)"` +2. result: `PASS` + +Constraint / overclaim / proof review: + +1. semantic constraint satisfied + - the first master consume edge is now explicit in code, not only in tests +2. overclaim avoided + - registry derivation semantics are not replaced yet + - lookup/public API are still unrebound +3. proof preserved + - `15B Rev 4-5` consume proofs still pass after extraction + +--- + +#### `15B` Delivery Note Rev 7 + +Date: 2026-04-04 +Scope: push the first bounded closure from master consume into an outward +master read surface + +What changed: + +1. extracted `entryReplicaSurfaceInfo(e, primaryAlive)` in + `master_server_handlers_block.go` + - makes the current registry -> outward surface mapping explicit for: + `ReplicaReady`, `ReplicaDegraded`, `VolumeMode`, `HealthState` +2. `entryToVolumeInfo()` now reads those outward replica-surface fields through + the helper instead of inlining them +3. added two end-to-end outward-surface proofs: + - core-influenced ready consume -> `entryToVolumeInfo()` + - core-influenced degraded consume -> `entryToVolumeInfo()` + +Files changed: + +1. `weed/server/master_server_handlers_block.go` + - added `entryReplicaSurfaceInfo` + - rebound `entryToVolumeInfo` to the explicit outward surface helper +2. `weed/server/master_block_observability_test.go` + - added ready-path outward closure proof + - added degraded-path outward closure proof + +Proofs added: + +1. ready outward closure + - VS emits a core-influenced ready heartbeat + - registry consumes it into ready aggregate truth + - `entryToVolumeInfo()` exposes: + `ReplicaReady=true`, `ReplicaDegraded=false`, + `VolumeMode=publish_healthy`, `HealthState=healthy` +2. degraded outward closure + - VS emits a core-influenced degraded heartbeat + - registry consumes it into degraded aggregate truth + - `entryToVolumeInfo()` exposes: + `ReplicaDegraded=true`, `VolumeMode=degraded`, + `HealthState=degraded` + +Validation: + +1. `go test ./weed/server -run "Test(Registry_(UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaDegraded|UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaReady)|EntryToVolumeInfo_(IncludesHealthState|ReflectsCoreInfluencedReadyConsume|ReflectsCoreInfluencedDegradedConsume))"` +2. result: `PASS` + +Constraint / overclaim / proof review: + +1. semantic constraint satisfied + - one bounded master outward read path now explicitly reflects the + core-influenced consume chain +2. overclaim avoided + - this is `entryToVolumeInfo()` closure only, not full REST/gRPC surface + rebinding + - lookup/public API transport remains otherwise unchanged +3. proof preserved + - `15B Rev 4-6` producer/consume/extraction proofs remain valid + +--- + +#### `15B` Delivery Note Rev 8 + +Date: 2026-04-04 +Scope: close the first real HTTP handler proofs above the master outward helper + +What changed: + +1. added handler-level proof for `GET /block/volume/{name}` + - proves lookup handler reflects the core-influenced ready path +2. added handler-level proof for `GET /block/volumes` + - proves list handler reflects the core-influenced degraded path +3. both proofs reuse the same bounded chain already established in earlier + revisions: + - `BlockService` emits core-influenced heartbeat + - `BlockVolumeRegistry.UpdateFullHeartbeat()` consumes it + - outward handler returns the resulting truth + +Files changed: + +1. `weed/server/master_server_handlers_block_test.go` + - added lookup-handler ready closure proof + - added list-handler degraded closure proof + +Proofs added: + +1. lookup handler ready closure + - replica assignment path produces a core-influenced ready heartbeat + - registry consumes it + - `GET /block/volume/{name}` returns: + `ReplicaReady=true`, `ReplicaDegraded=false`, + `VolumeMode=publish_healthy` +2. list handler degraded closure + - primary path produces a core-influenced degraded heartbeat + - registry consumes it + - `GET /block/volumes` returns: + `ReplicaDegraded=true`, `VolumeMode=degraded` + +Validation: + +1. `go test ./weed/server -run "TestBlockVolume(LookupHandler_ReflectsCoreInfluencedReadyConsume|ListHandler_ReflectsCoreInfluencedDegradedConsume)"` +2. result: `PASS` + +Constraint / overclaim / proof review: + +1. semantic constraint satisfied + - the bounded closure now reaches real HTTP handler surfaces +2. overclaim avoided + - only two handler paths are proven so far + - gRPC lookup response remains a separate surface +3. proof preserved + - `15B Rev 7` outward helper closure remains the underlying contract + +--- + +#### `15B` Delivery Note Rev 10 + +Date: 2026-04-04 +Scope: extend the bounded closure from per-volume outward surfaces to the first +cluster-level aggregate outward surface + +What changed: + +1. added `TestBlockStatusHandler_ReflectsCoreInfluencedConsumeCounts` +2. the proof constructs two real bounded chains: + - ready path: replica-side core-influenced heartbeat -> registry consume + - degraded path: primary-side core-influenced heartbeat -> registry consume +3. `GET /block/status` is then verified to expose the resulting aggregate truth: + - `VolumeCount=2` + - `HealthyCount=1` + - `DegradedCount=1` + - `RebuildingCount=0` + - `UnsafeCount=0` + +Files changed: + +1. `weed/server/master_block_observability_test.go` + - added cluster-level status closure proof + +Proofs added: + +1. status-handler aggregate closure + - two independent core-influenced consume chains are materialized in the + registry + - `blockStatusHandler` reports the expected aggregate health counts + - this proves the bounded closure now reaches a cluster-level outward read + surface, not only per-volume lookup/list surfaces + +Validation: + +1. `go test ./weed/server -run "TestBlockStatusHandler_(IncludesHealthCounts|ReflectsCoreInfluencedConsumeCounts)"` +2. result: `PASS` + +Constraint / overclaim / proof review: + +1. semantic constraint satisfied + - a cluster-level outward aggregate now reflects the same bounded + core-influenced consume chain +2. overclaim avoided + - only the status-count surface is proven here + - no broader dashboard/runbook claims are added by this revision +3. proof preserved + - `15B Rev 8-9` per-volume outward surface proofs remain valid + +--- + +#### `15B` Delivery Note Rev 11 + +Date: 2026-04-04 +Scope: extract the first explicit cluster-level outward response helper + +What changed: + +1. extracted `statusResponseFromRegistry()` from `blockStatusHandler` +2. `blockStatusHandler` now delegates to that helper instead of assembling the + aggregate response inline +3. this makes the current cluster-level outward mapping explicit for: + - volume/server counts + - promotion/barrier/queue aggregates + - healthy/degraded/rebuilding/unsafe counts + - NVMe-capable server count + +Files changed: + +1. `weed/server/master_server_handlers_block.go` + - added `statusResponseFromRegistry()` + - rebound `blockStatusHandler` to the helper + +Proof / validation posture: + +1. no new behavior claim + - this revision is a contract extraction step for the status surface +2. existing status closure proof remains the acceptance object: + - `TestBlockStatusHandler_ReflectsCoreInfluencedConsumeCounts` + +Validation: + +1. `go test ./weed/server -run "TestBlockStatusHandler_(IncludesHealthCounts|ReflectsCoreInfluencedConsumeCounts)"` +2. result: `PASS` + +Constraint / overclaim / proof review: + +1. semantic constraint satisfied + - the cluster-level outward aggregate now has an explicit code-level contract +2. overclaim avoided + - no new status semantics are introduced in this revision +3. proof preserved + - `15B Rev 10` status-handler closure proof still passes after extraction + +--- + +#### `15B` Closeout Note + +Date: 2026-04-04 + +Closeout judgment: + +1. `15A` + `15B` are now treated as delivered +2. `weed/` now has one bounded integrated path where: + - core-owned events enter from the live adapter path + - bounded command/projection egress returns to the adapter + - projection/store/outward surfaces consume core-owned truth on the selected + path + +Final focused validation sweep: + +1. `go test ./weed/server -run "Test(BlockService_(ApplyAssignments|DebugInfoForVolume|CollectBlockVolumeHeartbeat|ReadinessSnapshot|HeartbeatReplicaDegraded)|Registry_(ReplicaReadyRequiresReplicaHeartbeat|UpdateFullHeartbeat|UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaDegraded|UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaReady)|EntryToVolumeInfo_(IncludesHealthState|ReflectsCoreInfluencedReadyConsume|ReflectsCoreInfluencedDegradedConsume)|BlockVolume(LookupHandler_ReflectsCoreInfluencedReadyConsume|ListHandler_ReflectsCoreInfluencedDegradedConsume)|BlockStatusHandler_(IncludesHealthCounts|ReflectsCoreInfluencedConsumeCounts)|LookupResponseFromEntry_PublicationMinimalSurface)"` +2. result: `PASS` + +Next phase handoff: + +1. move to `Phase 16` +2. stop widening surface rebinding by default +3. start replacing one adapter-owned runtime-driving path with core-driven + command ownership + +--- + +#### `15B` Delivery Note Rev 9 + +Date: 2026-04-04 +Scope: make the parallel gRPC lookup surface explicit as its own bounded outward +contract + +What changed: + +1. extracted `lookupResponseFromEntry(entry)` in + `master_grpc_server_block.go` + - this names the current `BlockVolumeEntry -> LookupBlockVolumeResponse` + mapping explicitly instead of leaving it inline inside the gRPC handler +2. `LookupBlockVolume()` now delegates to that helper +3. added a focused test that proves the helper remains a publication-minimal + outward surface: + - it returns server/transport/capacity/replica-set/durability/NVMe fields + - it does not attempt to become a second semantic owner for mode/readiness + +Files changed: + +1. `weed/server/master_grpc_server_block.go` + - added `lookupResponseFromEntry` + - rebound `LookupBlockVolume()` to the helper +2. `weed/server/master_grpc_server_block_test.go` + - added `TestLookupResponseFromEntry_PublicationMinimalSurface` + +Proofs added: + +1. gRPC lookup outward contract + - response helper preserves the current exposed fields: + `VolumeServer`, `IscsiAddr`, `CapacityBytes`, + `ReplicaServer`, `ReplicaFactor`, `ReplicaServers`, + `DurabilityMode`, `NvmeAddr`, `Nqn` + - response remains intentionally publication-minimal rather than trying to + mirror the richer HTTP mode/readiness surface + +Validation: + +1. `go test ./weed/server -run "Test(Master_LookupBlockVolume|LookupResponseFromEntry_PublicationMinimalSurface|Master_LookupResponse_)"` +2. result: `PASS` + +Constraint / overclaim / proof review: + +1. semantic constraint satisfied + - the parallel gRPC outward path now has an explicit code-level contract +2. overclaim avoided + - gRPC lookup schema is not widened in this revision + - mode/readiness/publication truth stay on the HTTP/helper side for now +3. proof preserved + - `15B Rev 8` handler-level closures remain valid diff --git a/sw-block/.private/phase/phase-15.md b/sw-block/.private/phase/phase-15.md new file mode 100644 index 000000000..860453a63 --- /dev/null +++ b/sw-block/.private/phase/phase-15.md @@ -0,0 +1,101 @@ +# Phase 15 + +Date: 2026-04-03 +Status: delivered +Purpose: connect the explicit `V2 core` to one narrow live adapter path so the +repo starts proving semantic ownership on the integrated path, not only inside +`sw-block/engine/replication` + +## Why This Phase Exists + +`Phase 14` delivered the first bounded explicit core shell: + +1. `14A`: mode / readiness / publication shell closure +2. `14B`: command-sequence closure +3. `14C`: boundary / recovery closure + +That shell is real, but it still mostly lives as an internal owner inside +`sw-block/engine/replication`. + +`Phase 15` exists to connect one narrow live path from `weed/` into that owner +without broad rebinding or runtime cutover. + +## Phase Goal + +Connect one bounded adapter ingress/egress path between `weed/` and the explicit +`V2 core`, then prove the path does not silently split semantic truth. + +## Scope + +### In scope + +1. one narrow event ingress from a live `weed/` path into the explicit core +2. one bounded command/projection egress back to the adapter layer +3. focused proof that the narrow path carries explicit core-owned truth + +### Out of scope + +1. no broad registry rewrite yet +2. no product-surface rebinding yet +3. no broad runtime cutover +4. no transport redesign + +## Phase 15 Slices + +### `15A`: Minimal Adapter Hook + +Goal: + +1. connect one narrow adapter ingress to the new core + +Acceptance object: + +1. one real event path from `weed/` into `sw-block/engine/replication` +2. one bounded command/projection path back out +3. structural proof that the narrow path updates core-owned projection truth on + the live code path + +Status: + +1. delivered + +### `15B`: Projection-Store Rebinding + +Goal: + +1. make `weed/` projection/state surfaces consume core-owned projection truth + +Acceptance object: + +1. bounded rebinding of one or more real `weed/` surfaces to core-owned projection truth +2. proof that assignment delivered != ready != publish healthy on the real path + +Current chosen paths: + +1. `weed/server/volume_server_block_debug.go` +2. `/debug/block/shipper` +3. `BlockService.CollectBlockVolumeHeartbeat()` +4. `BlockVolumeRegistry.UpdateFullHeartbeat()` +5. `entryToVolumeInfo()` in `master_server_handlers_block.go` +6. `blockVolumeLookupHandler()` and `blockVolumeListHandler()` +7. `LookupBlockVolume()` in `master_grpc_server_block.go` +8. `blockStatusHandler()` aggregate counts +9. core projection preferred when present; adapter-local readiness only as fallback + +Status: + +1. delivered + +## Immediate Next Step + +Start `Phase 16` from the first bounded runtime-driving path: + +1. replace one adapter-owned execution decision path with core-driven command + ownership +2. keep reusing `blockvol` as execution backend, but stop letting adapter-local + execution branching remain the semantic owner + +This is the next natural step after `15B`: outward surfaces now consume +core-owned truth on a bounded path; `Phase 16` must make one bounded integrated +runtime path behave as a `V2`-owned runtime rather than constrained-`V1` +semantics plus rebinding. diff --git a/sw-block/.private/phase/phase-16-checkpoint-review.md b/sw-block/.private/phase/phase-16-checkpoint-review.md new file mode 100644 index 000000000..7e350c063 --- /dev/null +++ b/sw-block/.private/phase/phase-16-checkpoint-review.md @@ -0,0 +1,155 @@ +# Phase 16 Checkpoint Review + +Date: 2026-04-04 +Status: ready for review + +## Review Object + +Review the current bounded checkpoint as: + +1. `Phase 15` delivered +2. `16A` delivered +3. `16B` current bounded closure + +This checkpoint should be judged as the first bounded integrated runtime +checkpoint after `Phase 15` closeout. + +## What Is In Scope + +### `Phase 15` closeout + +1. bounded surface/store/outward consume-chain rebinding to core-owned truth +2. cluster-level status surface extraction and closure proof preserved + +### `16A` delivered + +Bounded command-driven adapter ownership now covers: + +1. `apply_role` +2. `start_receiver` +3. `configure_shipper` +4. `invalidate_session` + +Expected judgment: + +1. these paths execute because the core emitted commands +2. the adapter is executor, not semantic owner + +### `16B` current bounded closure + +Bounded live recovery closure now covers: + +1. live recovery observations return into the core on catch-up / rebuild + entry/exit points +2. bounded catch-up execution runs from `StartCatchUpCommand` +3. old no-core path compatibility remains preserved + +Expected judgment: + +1. this is a real bounded runtime closure step +2. it is not yet full recovery-loop ownership + +## What Is Explicitly Out Of Scope + +Do NOT review this checkpoint as claiming: + +1. `start_rebuild` execution ownership +2. full rebuild runtime closure +3. full recovery-loop closure +4. broad multi-replica runtime ownership +5. launch / rollout readiness + +## Primary Files + +Phase tracking: + +1. `sw-block/.private/phase/phase-15.md` +2. `sw-block/.private/phase/phase-15-log.md` +3. `sw-block/.private/phase/phase-16.md` +4. `sw-block/.private/phase/phase-16-log.md` + +Integrated runtime code: + +1. `weed/server/volume_server_block.go` +2. `weed/server/volume_server_block_test.go` +3. `weed/server/master_server_handlers_block.go` +4. `weed/server/master_block_observability_test.go` +5. `weed/server/block_recovery.go` +6. `weed/server/block_recovery_test.go` + +## Evidence Summary + +### Surface/store closure preserved + +Focused proof suite: + +1. `go test ./weed/server -run "Test(BlockService_(ApplyAssignments|BarrierRejected|DebugInfoForVolume|CollectBlockVolumeHeartbeat|ReadinessSnapshot|HeartbeatReplicaDegraded)|Registry_(ReplicaReadyRequiresReplicaHeartbeat|UpdateFullHeartbeat|UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaDegraded|UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaReady)|EntryToVolumeInfo_(IncludesHealthState|ReflectsCoreInfluencedReadyConsume|ReflectsCoreInfluencedDegradedConsume)|BlockVolume(LookupHandler_ReflectsCoreInfluencedReadyConsume|ListHandler_ReflectsCoreInfluencedDegradedConsume)|BlockStatusHandler_(IncludesHealthCounts|ReflectsCoreInfluencedConsumeCounts)|LookupResponseFromEntry_PublicationMinimalSurface)"` +2. result: `PASS` + +### Recovery closure + +Focused recovery proof suite: + +1. `go test ./weed/server -run "TestP(4_LivePath_RealVol_ReachesPlan|16B_RunCatchUp_)"` +2. result: `PASS` + +## Review Questions + +### For `sw` + +Please check implementation correctness and commit-readiness: + +1. Is the suggested commit boundary coherent as one checkpoint? +2. Are the file changes internally consistent for: + - `Phase 15` closeout + - `16A` delivered + - `16B` current closure +3. Are there any obvious cleanup/refactor issues that should be fixed before + commit, without broadening scope? + +Suggested commit boundary if accepted: + +1. `sw-block/.private/phase/phase-15.md` +2. `sw-block/.private/phase/phase-15-log.md` +3. `sw-block/.private/phase/phase-16.md` +4. `sw-block/.private/phase/phase-16-log.md` +5. `weed/server/volume_server_block.go` +6. `weed/server/volume_server_block_test.go` +7. `weed/server/master_server_handlers_block.go` +8. `weed/server/master_block_observability_test.go` +9. `weed/server/block_recovery.go` +10. `weed/server/block_recovery_test.go` + +### For `tester` + +Please challenge the proof posture: + +1. Does `16A` really prove command-driven ownership, or only show refactored + call placement? +2. Does `16B Rev 2` really prove `start_catchup` is command-driven on the live + path? +3. Are there any remaining surfaces where adapter-local truth could still + contradict the core on the bounded path? +4. Are any of the current tests proving implementation shape only, rather than + semantic claim? + +### For `manager` + +Please challenge boundaries and overclaim: + +1. Are `16A` and `16B` still cleanly separated? +2. Is `16B Rev 2` still a bounded catch-up slice, rather than silently becoming + full recovery-loop closure? +3. Does the checkpoint wording stay disciplined about what is NOT yet claimed? +4. Is the proposed commit boundary a good stage checkpoint? + +## Requested Output Shape + +Please reply with one of: + +1. `ACCEPT` +2. `ACCEPT WITH MINOR FIXES` +3. `REJECT` + +If not `ACCEPT`, list findings ordered by severity and keep them bounded to this +checkpoint's actual claim set. diff --git a/sw-block/.private/phase/phase-16-log.md b/sw-block/.private/phase/phase-16-log.md new file mode 100644 index 000000000..9637c925f --- /dev/null +++ b/sw-block/.private/phase/phase-16-log.md @@ -0,0 +1,429 @@ +Purpose: append-only technical pack and delivery log for `Phase 16` +`V2`-native runtime closure work. + +--- + +### `16A` Technical Pack + +Date: 2026-04-04 +Goal: replace one adapter-owned execution decision path with explicit +core-driven command ownership while keeping `blockvol` as the execution backend + +#### Layer 1: Semantic Core + +`16A` accepts one bounded thing: + +1. one real integrated execution path now runs because the core emitted the + command, not because adapter-local branching independently decided it + +It does not accept: + +1. full replacement of `blockvol` async runtime loops +2. broad runtime cutover across all adapter paths +3. all recovery/failover paths becoming `V2`-native at once + +#### Candidate runtime-driving paths + +Candidate commands already emitted by the explicit core: + +1. `apply_role` +2. `start_receiver` +3. `configure_shipper` +4. `start_catchup` +5. `start_rebuild` + +Selection rule: + +1. prefer the narrowest path that is already real on the integrated `weed/` + path +2. prefer a path where current execution is still primarily adapter-owned +3. prefer a path with direct proofable observation back into the core + +#### Initial implementation rule + +For the first `16A` slice: + +1. do not rewrite `blockvol` internals +2. do not change flusher/shipper goroutine architecture yet +3. move runtime-driving ownership one step upward: + - core emits command + - adapter executes command + - execution observation returns as explicit core event + +#### Validation target + +The first accepted `16A` path must prove: + +1. the command was emitted by the core +2. the adapter executed because of that command +3. the resulting observation fed back into the core +4. outward surfaces stayed coherent with that same path + +--- + +#### `16A` Delivery Note Rev 1 + +Date: 2026-04-04 +Scope: first bounded runtime-driving command path on the integrated adapter + +What changed: + +1. `ApplyAssignments()` no longer performs role application as adapter-local + branching when the explicit core is present +2. assignment delivery now executes `apply_role` from core command egress +3. replica assignment path also executes `start_receiver` from the same bounded + command path +4. `publish_projection` caching now prefers the latest core projection so + command-chain observations cannot be overwritten by an older inline + projection snapshot + +Files changed: + +1. `weed/server/volume_server_block.go` + - role application moved to `ApplyRoleCommand` execution + - replica receiver startup moved to `StartReceiverCommand` execution + - added bounded executed-command trace for focused runtime-ownership proofs + - projection cache now prefers latest `v2Core.Projection()` +2. `weed/server/volume_server_block_test.go` + - added focused proof for primary `apply_role` command ownership + - added focused proof for replica `apply_role + start_receiver` ownership + +Bounded contract: + +`16A Rev 1` accepts only this: + +1. one integrated assignment path now executes from core command egress +2. the adapter remains the executor, not the semantic owner +3. outward/readiness projections remain aligned after the command chain + +It does not yet accept: + +1. primary shipper configuration becoming fully core-command-driven +2. rebuild / catch-up / failover runtime closure +3. broad `Phase 16` completion + +Validation: + +1. `go test ./weed/server -run "TestBlockService_(ApplyAssignments|DebugInfoForVolume|CollectBlockVolumeHeartbeat|ReadinessSnapshot|HeartbeatReplicaDegraded)"` +2. `go test ./weed/server -run "Test(BlockService_(ApplyAssignments|DebugInfoForVolume|CollectBlockVolumeHeartbeat|ReadinessSnapshot|HeartbeatReplicaDegraded)|Registry_(ReplicaReadyRequiresReplicaHeartbeat|UpdateFullHeartbeat|UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaDegraded|UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaReady)|EntryToVolumeInfo_(IncludesHealthState|ReflectsCoreInfluencedReadyConsume|ReflectsCoreInfluencedDegradedConsume)|BlockVolume(LookupHandler_ReflectsCoreInfluencedReadyConsume|ListHandler_ReflectsCoreInfluencedDegradedConsume)|BlockStatusHandler_(IncludesHealthCounts|ReflectsCoreInfluencedConsumeCounts)|LookupResponseFromEntry_PublicationMinimalSurface)"` +3. result: `PASS` + +Constraint / overclaim / proof review: + +1. semantic constraint satisfied + - `Phase 16` requires one bounded path where core-owned semantics drive + adapter execution rather than adapter-local truth +2. overclaim avoided + - this revision does not claim full async runtime replacement or full + failover/rebuild closure +3. proof preserved + - existing `15A/15B` projection/store closure tests still pass after the new + command-driven assignment path + +--- + +#### `16A` Delivery Note Rev 2 + +Date: 2026-04-04 +Scope: extend the same bounded assignment-side command path to primary shipper +configuration + +What changed: + +1. `ApplyAssignments()` no longer configures primary replication directly when + the explicit core is present +2. primary shipper wiring now executes only from + `engine.ConfigureShipperCommand` +3. the executed-command trace now proves a full bounded assignment-side command + chain: + - primary: `apply_role -> configure_shipper` + - replica: `apply_role -> start_receiver` + +Files changed: + +1. `weed/server/volume_server_block.go` + - added `executeConfigureShipperCommand()` + - removed adapter-local primary replication setup from `ApplyAssignments()` +2. `weed/server/volume_server_block_test.go` + - strengthened primary command-ownership proof to require + `apply_role + configure_shipper` + +Bounded contract: + +`16A Rev 2` accepts only this: + +1. the full assignment-side command chain is now core-command-driven on the + integrated path +2. primary and replica setup no longer rely on adapter-local branching for the + bounded assignment path +3. existing projection/store/outward-surface proofs remain intact + +It does not yet accept: + +1. catch-up runtime ownership +2. rebuild runtime ownership +3. invalidation / recovery-loop runtime closure + +Validation: + +1. `go test ./weed/server -run "TestBlockService_(ApplyAssignments|DebugInfoForVolume|CollectBlockVolumeHeartbeat|ReadinessSnapshot|HeartbeatReplicaDegraded)"` +2. `go test ./weed/server -run "Test(BlockService_(ApplyAssignments|DebugInfoForVolume|CollectBlockVolumeHeartbeat|ReadinessSnapshot|HeartbeatReplicaDegraded)|Registry_(ReplicaReadyRequiresReplicaHeartbeat|UpdateFullHeartbeat|UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaDegraded|UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaReady)|EntryToVolumeInfo_(IncludesHealthState|ReflectsCoreInfluencedReadyConsume|ReflectsCoreInfluencedDegradedConsume)|BlockVolume(LookupHandler_ReflectsCoreInfluencedReadyConsume|ListHandler_ReflectsCoreInfluencedDegradedConsume)|BlockStatusHandler_(IncludesHealthCounts|ReflectsCoreInfluencedConsumeCounts)|LookupResponseFromEntry_PublicationMinimalSurface)"` +3. result: `PASS` + +Constraint / overclaim / proof review: + +1. semantic constraint satisfied + - `Phase 16` requires one bounded runtime path where command emission from + the explicit core drives adapter execution +2. overclaim avoided + - this revision still does not claim recovery-side runtime closure +3. proof preserved + - `Phase 15B` surface consume-chain proofs remain green after moving primary + shipper setup behind core command ownership + +--- + +#### `16A` Delivery Note Rev 3 + +Date: 2026-04-04 +Scope: first bounded recovery-side command path on the integrated runtime + +What changed: + +1. `InvalidateSessionCommand` is now executed on the live adapter path +2. when the core emits invalidation for a volume, the adapter invalidates the + corresponding integrated sender sessions through the orchestrator registry +3. repeated identical failure transitions still remain bounded: + one new failure transition triggers one invalidation command; repeated same + failure does not re-execute it + +Files changed: + +1. `weed/server/volume_server_block.go` + - added `executeInvalidateSessionCommand()` + - wired `InvalidateSessionCommand` into command execution +2. `weed/server/volume_server_block_test.go` + - added proof that `BarrierRejected` invalidates an active sender session + - added proof that repeated same-reason rejection does not re-execute + invalidation + +Bounded contract: + +`16A Rev 3` accepts only this: + +1. one integrated failure-side path is now also core-command-driven +2. sender-session invalidation on the bounded path no longer depends on + adapter-local branching alone +3. assignment-side command ownership and outward surface closure remain intact + +It does not yet accept: + +1. catch-up execution ownership +2. rebuild execution ownership +3. full recovery-loop runtime closure + +Validation: + +1. `go test ./weed/server -run "TestBlockService_(ApplyAssignments|BarrierRejected|DebugInfoForVolume|CollectBlockVolumeHeartbeat|ReadinessSnapshot|HeartbeatReplicaDegraded)"` +2. `go test ./weed/server -run "Test(BlockService_(ApplyAssignments|BarrierRejected|DebugInfoForVolume|CollectBlockVolumeHeartbeat|ReadinessSnapshot|HeartbeatReplicaDegraded)|Registry_(ReplicaReadyRequiresReplicaHeartbeat|UpdateFullHeartbeat|UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaDegraded|UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaReady)|EntryToVolumeInfo_(IncludesHealthState|ReflectsCoreInfluencedReadyConsume|ReflectsCoreInfluencedDegradedConsume)|BlockVolume(LookupHandler_ReflectsCoreInfluencedReadyConsume|ListHandler_ReflectsCoreInfluencedDegradedConsume)|BlockStatusHandler_(IncludesHealthCounts|ReflectsCoreInfluencedConsumeCounts)|LookupResponseFromEntry_PublicationMinimalSurface)"` +3. result: `PASS` + +Constraint / overclaim / proof review: + +1. semantic constraint satisfied + - `Phase 14` command semantics already required invalidation only on a new + failure transition; `Phase 16` now makes that command real on the bounded + integrated path +2. overclaim avoided + - this revision invalidates sessions, but does not yet claim catch-up or + rebuild execution ownership +3. proof preserved + - `15B` projection/store/outward-surface proofs remain green after adding the + failure-side command path + +--- + +#### `16A` Closeout Note + +Date: 2026-04-04 + +Closeout judgment: + +1. `16A` is treated as delivered +2. the bounded integrated command-driven path now includes: + - `apply_role` + - `start_receiver` + - `configure_shipper` + - `invalidate_session` +3. this is enough to say one real runtime-driving command chain is now + core-owned on the integrated path + +Residual non-claims: + +1. active recovery execution (`start_catchup`, `start_rebuild`) is not yet + core-command-driven +2. `16A` does not close the full recovery loop by itself + +--- + +#### `16B` Delivery Note Rev 1 + +Date: 2026-04-04 +Scope: first live recovery observation path back into the explicit core + +What changed: + +1. `RecoveryManager.runCatchUp()` now feeds live recovery planning/escalation + back into the core: + - `CatchUpPlanned` + - `NeedsRebuildObserved` + - `CatchUpCompleted` +2. `RecoveryManager.runRebuild()` now feeds live rebuild lifecycle milestones + back into the core: + - `RebuildStarted` + - `RebuildCommitted` +3. this makes the integrated recovery path update core-owned recovery/boundary + truth instead of leaving those states test-only or documentation-only + +Files changed: + +1. `weed/server/block_recovery.go` + - emit recovery events into `v2Core` from live catch-up/rebuild execution +2. `weed/server/block_recovery_test.go` + - added focused proof that live catch-up updates core projection boundary + - added focused proof that live needs-rebuild escalation updates core mode + +Bounded contract: + +`16B Rev 1` accepts only this: + +1. one live recovery observation path now closes back into the core +2. core-owned boundary/recovery/mode fields update from real recovery work +3. existing `15B/16A` surface and command closure remain intact + +It does not yet accept: + +1. `start_catchup` command execution ownership +2. `start_rebuild` command execution ownership +3. full recovery-loop execution becoming core-command-driven + +Validation: + +1. `go test ./weed/server -run "TestP(4_LivePath_RealVol_ReachesPlan|16B_RunCatchUp_)"` +2. `go test ./weed/server -run "Test(P4_|P16B_|BlockService_(ApplyAssignments|BarrierRejected|DebugInfoForVolume|CollectBlockVolumeHeartbeat|ReadinessSnapshot|HeartbeatReplicaDegraded)|Registry_(ReplicaReadyRequiresReplicaHeartbeat|UpdateFullHeartbeat|UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaDegraded|UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaReady)|EntryToVolumeInfo_(IncludesHealthState|ReflectsCoreInfluencedReadyConsume|ReflectsCoreInfluencedDegradedConsume)|BlockVolume(LookupHandler_ReflectsCoreInfluencedReadyConsume|ListHandler_ReflectsCoreInfluencedDegradedConsume)|BlockStatusHandler_(IncludesHealthCounts|ReflectsCoreInfluencedConsumeCounts)|LookupResponseFromEntry_PublicationMinimalSurface)"` +3. result: `PASS` + +Constraint / overclaim / proof review: + +1. semantic constraint satisfied + - `Phase 16` requires runtime observations to close back into core-owned + truth instead of remaining adapter-local/runtime-local facts +2. overclaim avoided + - this revision does not yet claim that recovery execution itself is + core-command-driven +3. proof preserved + - `15B` outward consume-chain proofs and `16A` command-driven proofs remain + green after adding recovery event ingress + +--- + +#### `16B` Delivery Note Rev 2 + +Date: 2026-04-04 +Scope: bounded catch-up execution ownership on the live recovery path + +What changed: + +1. `RecoveryManager.runCatchUp()` no longer directly executes the catch-up plan + when the explicit core is present +2. catch-up plans are now cached as bounded pending executions and consumed only + from `StartCatchUpCommand` +3. if the core does not emit/consume `StartCatchUpCommand`, the pending plan is + cancelled fail-closed instead of executing implicitly +4. compatibility is preserved for the old no-core path: + legacy `P4` recovery tests still execute directly when `v2Core` is absent + +Files changed: + +1. `weed/server/block_recovery.go` + - added bounded pending recovery execution cache + - `runCatchUp()` now plans + emits core event, then waits for command + consumption on the core-present path + - added `ExecutePendingCatchUp()` and shared execution helpers +2. `weed/server/volume_server_block.go` + - wired `StartCatchUpCommand` into command execution +3. `weed/server/block_recovery_test.go` + - strengthened catch-up proof to require live `start_catchup` execution + - preserved old `P4` live-path proof + +Bounded contract: + +`16B Rev 2` accepts only this: + +1. one active recovery execution path (`catch-up`) is now core-command-driven on + the integrated runtime +2. the corresponding live recovery observations still return back into the core +3. old no-core compatibility remains preserved for previously accepted tests + +It does not yet accept: + +1. `start_rebuild` execution ownership +2. full recovery-loop execution closure +3. broad multi-replica recovery ownership beyond the current bounded path + +Validation: + +1. `go test ./weed/server -run "TestP(4_LivePath_RealVol_ReachesPlan|16B_RunCatchUp_)"` +2. `go test ./weed/server -run "Test(P4_|P16B_|BlockService_(ApplyAssignments|BarrierRejected|DebugInfoForVolume|CollectBlockVolumeHeartbeat|ReadinessSnapshot|HeartbeatReplicaDegraded)|Registry_(ReplicaReadyRequiresReplicaHeartbeat|UpdateFullHeartbeat|UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaDegraded|UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaReady)|EntryToVolumeInfo_(IncludesHealthState|ReflectsCoreInfluencedReadyConsume|ReflectsCoreInfluencedDegradedConsume)|BlockVolume(LookupHandler_ReflectsCoreInfluencedReadyConsume|ListHandler_ReflectsCoreInfluencedDegradedConsume)|BlockStatusHandler_(IncludesHealthCounts|ReflectsCoreInfluencedConsumeCounts)|LookupResponseFromEntry_PublicationMinimalSurface)"` +3. result: `PASS` + +Constraint / overclaim / proof review: + +1. semantic constraint satisfied + - `Phase 16` requires not only observation closure, but also one bounded + active recovery execution path to run because the core emitted the command +2. overclaim avoided + - only catch-up is moved behind core command ownership in this revision; + rebuild remains explicitly outside the accepted closure +3. proof preserved + - accepted `P4` no-core recovery tests still pass, and `15B/16A/16B` bounded + consume-chain proofs remain green + +--- + +#### `16` Checkpoint Review Note + +Date: 2026-04-04 + +Current checkpoint judgment: + +1. this is the first bounded integrated runtime checkpoint after `Phase 15` + closeout +2. `16A` is delivered +3. `16B` has one accepted current closure: + - live recovery observations close back into the core + - bounded catch-up execution is core-command-driven + +Recommended review target: + +1. review this checkpoint as: + - `Phase 15 delivered` + - `16A delivered` + - `16B current bounded closure` +2. do not review it as: + - full rebuild execution ownership + - full recovery-loop closure + - launch / rollout readiness + +Suggested commit boundary if review is accepted: + +1. `sw-block/.private/phase/phase-15.md` +2. `sw-block/.private/phase/phase-15-log.md` +3. `sw-block/.private/phase/phase-16.md` +4. `sw-block/.private/phase/phase-16-log.md` +5. `weed/server/volume_server_block.go` +6. `weed/server/volume_server_block_test.go` +7. `weed/server/master_server_handlers_block.go` +8. `weed/server/master_block_observability_test.go` +9. `weed/server/block_recovery.go` +10. `weed/server/block_recovery_test.go` diff --git a/sw-block/.private/phase/phase-16.md b/sw-block/.private/phase/phase-16.md new file mode 100644 index 000000000..bced64fff --- /dev/null +++ b/sw-block/.private/phase/phase-16.md @@ -0,0 +1,139 @@ +# Phase 16 + +Date: 2026-04-04 +Status: active +Purpose: close one bounded `V2`-native runtime path where the explicit core +owns runtime-driving semantics and `blockvol` remains only the execution backend + +## Why This Phase Exists + +`Phase 14` made the explicit core real. + +`Phase 15` then rebound one bounded set of integrated `weed/` surfaces so they +consume core-owned truth instead of silently inheriting adapter-local semantics. + +That means the repo now has: + +1. explicit core-owned state / command / projection semantics +2. bounded integrated surface rebinding across VS, registry, HTTP, gRPC, and + cluster status + +But it still does not yet have one bounded path where runtime-driving execution +ownership itself is `V2`-native. + +## Phase Goal + +Close one bounded integrated runtime path where: + +1. the explicit core decides the runtime-driving command sequence +2. the adapter executes those commands against `blockvol` +3. runtime observations return back into the core +4. outward surfaces continue to reflect that core-owned runtime path + +## Scope + +### In scope + +1. one bounded command-driven adapter execution path +2. one bounded observation-feedback path from execution back into the core +3. end-to-end proof that the bounded path behaves as a `V2`-owned runtime path + +### Out of scope + +1. no full replacement of all `blockvol` async executors +2. no broad runtime cutover across every `weed/` path +3. no protocol rediscovery +4. no launch / rollout approval + +## Phase 16 Slices + +### `16A`: Command-Driven Adapter Ownership + +Goal: + +1. replace one adapter-owned execution decision path with core-driven command + ownership + +Acceptance object: + +1. one real integrated path executes because the core emitted the command +2. the adapter no longer decides that path only from its local branching +3. proof that command emission and command execution stay aligned + +Current chosen path: + +1. assignment-driven `apply_role` execution now runs from core command egress +2. replica-path `start_receiver` execution follows the same bounded command path +3. primary-path `configure_shipper` execution now also follows the bounded + command path +4. failure-side `invalidate_session` execution now follows the bounded command + path for the integrated sender path +5. catch-up / rebuild remain outside the current `16A` closure + +Status: + +1. delivered + +### `16B`: Runtime Observation Closure + +Goal: + +1. make the bounded runtime path close back into the core through explicit + observation semantics + +Acceptance object: + +1. one end-to-end failover/recovery/publication scenario runs on the + core-driven path +2. proof that outward surfaces remain consistent with the same bounded runtime + path + +Current chosen path: + +1. live recovery observations now return into the core on catch-up and rebuild + entry/exit points +2. bounded catch-up execution now runs from `StartCatchUpCommand` +3. rebuild execution is the next likely runtime-driving candidate on the same + path + +Status: + +1. active + +## Current Checkpoint Review Target + +The current review target is the first bounded integrated runtime checkpoint +after `Phase 15` closeout: + +1. `Phase 15` delivered: + - bounded surface/store/outward consume-chain rebinding +2. `16A` delivered: + - bounded command-driven adapter ownership for: + - `apply_role` + - `start_receiver` + - `configure_shipper` + - `invalidate_session` +3. `16B` active with accepted current closure: + - live recovery observations return into the core + - bounded catch-up execution runs from `StartCatchUpCommand` + +This checkpoint is intentionally still bounded: + +1. `start_rebuild` execution ownership is not yet in scope +2. broad recovery-loop closure is not yet claimed +3. launch / rollout readiness is not claimed + +## Immediate Next Step + +Start `16A` with the narrowest runtime-driving execution decision still owned by +adapter branching: + +1. choose one path where the core already emits a bounded command +2. let the adapter execute from that command instead of from implicit local + control flow +3. keep `blockvol` as execution backend and treat its local state only as + observation input + +The next likely engineering target is the matching rebuild execution path, +unless the current `Phase 15 + 16A + 16B` boundary is accepted as the next +commit checkpoint first. diff --git a/sw-block/design/v2-protocol-truths.md b/sw-block/design/v2-protocol-truths.md index 5cdf29e39..ebdf38cbf 100644 --- a/sw-block/design/v2-protocol-truths.md +++ b/sw-block/design/v2-protocol-truths.md @@ -429,13 +429,15 @@ Evidence anchor: ## Current Strongest Evidence By Layer -| Layer | Main value | -|------|------------| -| `FSM` / design | define truth and non-goals | -| simulator | prove protocol truth and failure-class closure cheaply | -| prototype | prove implementation-shape and authority semantics cheaply | -| engine | prove the accepted contracts survive real implementation structure | -| service slice / runner | prove truth survives real control/storage/system reality | + +| Layer | Main value | +| ---------------------- | ------------------------------------------------------------------ | +| `FSM` / design | define truth and non-goals | +| simulator | prove protocol truth and failure-class closure cheaply | +| prototype | prove implementation-shape and authority semantics cheaply | +| engine | prove the accepted contracts survive real implementation structure | +| service slice / runner | prove truth survives real control/storage/system reality | + ## Phase Conformance Notes @@ -594,16 +596,20 @@ Main review risk: When a later feature expands the protocol surface (for example `SmartWAL` or a new rebuild optimization), the order should be: 1. `FSM / design` + - define the new semantics and non-goals -2. `Truth update` +1. `Truth update` + - either attach the feature to an existing truth - or add a new protocol truth if the feature creates a new long-lived invariant -3. `Phase alignment` +1. `Phase alignment` + - define which later phases strengthen or validate that truth -4. `Evidence ladder` +1. `Evidence ladder` + - simulator, prototype, engine, service slice as needed Do not start feature implementation by editing engine or service glue first and only later trying to explain what truth changed. @@ -618,3 +624,419 @@ For any future feature, later reviews should ask: 4. does the feature weaken any existing truth? This keeps feature growth aligned with protocol truth instead of letting implementation convenience define semantics. + +## 继承图 + +最短可以看成三层： + +```text +Layer 1: 已接受的语义与证据 +----------------------------------------- +Phase 08-13 + -> protocol truths + -> claim / evidence + -> envelope / non-claim + -> bounded proofs + -> V1-under-V2-constraints test results + + || + || 继承“语义约束、证明边界、不能 overclaim 的规则” + \/ + +Layer 2: V2 core 原型与显式自动机 +----------------------------------------- +Phase 14 + -> small automata + - assignment + - recovery + - boundary + - mode + - publication + -> explicit state owner + -> transition rules + -> command emission rules + -> projection contracts + + || + || 继承“状态机结构、命令规则、对外投影规则” + \/ + +Layer 3: 集成运行时 +----------------------------------------- +Phase 15-16 + -> weed/ adapter + -> blockvol executor + -> master / VS / heartbeat / lookup / debug / status + -> real command-driven path + -> real observation feedback path + +目标： + V2 core owns semantics + adapter translates + blockvol executes +``` + +如果换成“什么被继承、什么不被继承”，就是这个图： + +```text +前面 phase 资产 +| +|-- 继承 +| |-- truth constraints +| |-- FSM/state-transition meaning +| |-- command emission conditions +| |-- projection/publication rules +| |-- accepted evidence and non-claims +| +|-- 不直接继承 +| |-- V1 runtime 的状态所有权 +| |-- V1 内联控制分支就是最终 truth +| |-- “测试跑通了所以产品语义成立” 这种隐式结论 +| +v +Phase 14 core +| +|-- 输出 +| |-- explicit core engine +| |-- owned state +| |-- explicit commands +| |-- explicit projections +| +v +Phase 15-16 integration +| +|-- 复用 +| |-- blockvol flusher +| |-- shipper / receiver +| |-- rebuild server +| |-- existing transport/backend machinery +| +|-- 替换/上提 +| |-- 谁决定 apply_role +| |-- 谁决定 start_receiver +| |-- 后面继续上提 configure_shipper 等 runtime-driving decision +``` + +再给你一个“现在走到哪里”的定位图： + +```text +[Phase 08-13] +约束 / 证据 / envelope / prototype input + | + v +[Phase 14] +V2 core shell 成立 + | + v +[Phase 15] +surface consume core truth + | + v +[Phase 16] +runtime-driving decisions move from adapter -> core + | + v +[未来] +one bounded V2-native runtime path +``` + +一句话总结： + +- `Phase 08-13` 留下的是“规则和证据” +- `Phase 14` 把它们变成“显式 core” +- `Phase 15-16` 把 core 接到真实 runtime，上提语义控制权 +- `V1` 主要保留为执行 backend，不再保留为语义 owner + +## 逻辑 +可以。我换成更“工程视角”的讲法，不只是图，而是讲这套继承到底怎么落到代码和后续开发上。 + +### 核心思路 + +这个项目现在不是一条线直接从“协议文档”走到“最终产品”，而是四层： + +1. `truth / claim / envelope` +2. `core semantic model` +3. `adapter integration` +4. `backend execution reuse` + +它们的关系不是谁替代谁，而是“上层定义意义，下层负责承载”。 + +最重要的一点是： + +- 上层决定“什么算真” +- 下层只负责“怎么做出来” + +这就是为什么我一直在强调： +`V2 owns semantics, V1/blockvol owns execution` + +### 四层继承图 + +```text +Layer A: 语义约束层 +-------------------------------- +sw-block/design/v2-protocol-truths.md +sw-block/design/v2-protocol-claim-and-evidence.md +sw-block/design/v2-protocol-closure-map.zh.md + +定义： +- 什么叫 assigned / ready / durable / degraded / publish healthy +- 哪些 claim 已经接受 +- 哪些 envelope 是允许宣称的 +- 哪些 non-claim 不能偷着扩张 + + || + || 把“正确性的标准”传下去 + \/ + +Layer B: V2 core 语义层 +-------------------------------- +sw-block/design/v2_mini_core_design.md +sw-block/design/v2-phase14plus-semantic-framework.md +sw-block/engine/replication/ + +定义： +- 小自动机怎么拆 +- state 谁拥有 +- transition 怎么发生 +- command 什么时候发 +- projection 允许怎么对外表达 + + || + || 把“标准”变成“可执行语义模型” + \/ + +Layer C: Adapter 集成层 +-------------------------------- +weed/server/... +master / volume server / heartbeat / lookup / status / debug + +职责： +- 把 live assignment / heartbeat / observation 送进 core +- 把 core 的 command / projection 接回真实系统 +- 对外 surface 尽量只消费 core truth + + || + || 把 core 接到真实运行时 + \/ + +Layer D: Backend 执行层 +-------------------------------- +weed/storage/blockvol/... + +职责： +- flusher +- shipper +- receiver +- rebuild server +- checkpoint / WAL / async execution + +只负责： +- 执行 +- 持久化 +- 传输 +- 提供 observation + +不负责： +- 最终语义定义 +``` + +### 每层分别继承了什么 + +#### 1. 从前面 phases 继承了什么 + +前面 `Phase 08-13` 不是白做的，它们沉淀了三类资产： + +1. 语义定义 +2. 证明边界 +3. 已接受证据 + +比如： +- durable progress 该看什么，不该看什么 +- publication healthy 不能等同于 replica ready +- degraded / needs_rebuild 不能乱宣称 +- 哪些测试只是 witness，哪些已经是 proof +- 哪些 envelope 是当前 chosen path，不能擅自放大成“通用产品能力” + +所以现在做 `Phase 15/16`，不是重新设计世界，而是在问： + +- 这个 live path 是否遵守了已经接受的 truth？ +- 这个 adapter 是否把语义 owner 放错地方了？ +- 这个 surface 是否产生 overclaim？ + +#### 2. `Phase 14` 继承并固化了什么 + +`Phase 14` 的价值，不是“写了一个新包”，而是把前面松散的规则固化成 core 结构。 + +它固定了四件事： + +1. state owner +2. transition rule +3. command emission rule +4. projection contract + +这意味着后面就不是“看到一个 bug 再 patch 一层”，而是可以用统一问题来判断： + +- 这是 state 问题？ +- 这是 transition 问题？ +- 这是 command 发错时机？ +- 还是 projection overclaim？ + +这就是原来你说的那种目标： +“以后每个算法选择都能回答：满足哪个语义约束，避免哪个 overclaim，保住哪个 proof。” + +#### 3. `Phase 15` 继承了什么 + +`Phase 15` 不是在发明新语义，而是在做一个很重要的事情： + +让 `weed/` 的真实 surface 开始消费 core-owned truth。 + +也就是把过去散落在 adapter 里的这些东西逐步收回： + +- debug +- heartbeat +- readiness snapshot +- master registry consume +- lookup/list/status outward surface + +`Phase 15` 的主题不是 runtime control，而是 surface rebinding。 + +意思是： + +- 先不急着改谁驱动执行 +- 先把谁有资格对外说话这件事收紧 + +所以 `Phase 15` 更像是： +“让嘴巴先归 core 管” + +#### 4. `Phase 16` 继承了什么 + +`Phase 16` 才开始继续向下推进，去拿“手脚”。 + +也就是从： + +- core 只是解释系统状态 +- adapter 还在自己决定很多执行动作 + +变成： + +- core 发 command +- adapter 只是执行 command +- backend 执行后返回 observation +- core 再更新 state / projection + +这就是为什么我刚刚在 `16A` 先把： + +- `apply_role` +- `start_receiver` + +这两条路径接成 command-driven。 + +这一步的意义不在于“功能变多了”，而在于 runtime ownership 往上提了。 + +### 一张更贴 repo 的图 + +```text +v2-protocol-truths.md +v2-protocol-claim-and-evidence.md +v2-protocol-closure-map.zh.md + | + v +v2_mini_core_design.md +v2-phase14plus-semantic-framework.md + | + v +sw-block/engine/replication/ + - state + - event + - command + - projection + | + v +weed/server/ + - ApplyAssignments() + - heartbeat consume / produce + - lookup/list/status/debug + | + v +weed/storage/blockvol/ + - WAL + - flusher + - shipper + - receiver + - rebuild +``` + +可以把它理解成： + +- 文档定义语义世界 +- `engine/replication` 把语义世界程序化 +- `weed/server` 把程序化语义接到 live system +- `blockvol` 提供真正的执行能力 + +### 为什么要这样分，而不是继续修 V1 + +这是你前面感觉最敏锐的地方： + +如果一直在 `V1` 上补测试、补约束、补文档，最后容易变成： + +- 我们越来越懂 `V1` +- 但没有真正得到一个 `V2` owner + +那样后面每加一个功能都要重新回答： +“这次到底是谁说了算？” + +现在这种分层，就是为了避免未来每做一次事都重新付出一大轮认知成本。 + +理想结果是： + +- 新功能先接到 core 语义层 +- adapter 只做接线 +- backend 只做执行 +- proofs 可以复用 +- envelope 不需要每次重谈 + +也就是说，前期看起来“多做了一层”，但长期是为了让后面的功能开发更便宜。 + +### 你可以怎么理解现在的进度 + +如果用一句最直白的话： + +- `Phase 08-13`：知道什么是对的，并拿到了很多边界证据 +- `Phase 14`：把“什么是对的”变成显式 core +- `Phase 15`：让对外说法开始归 core +- `Phase 16`：让真实执行决策开始归 core + +所以现在不是从零开始，也不是还在原地测 `V1`。 +而是在把以前的知识资产，逐步变成真正可接管 runtime 的结构。 + +### 你后面看代码时可以用的判断法 + +你如果想快速判断某一段代码属于哪一层，可以问这三个问题： + +1. 它是在定义“什么是真的”吗？ +如果是，那它应该靠近 `design` 或 `engine/replication` + +2. 它是在把外部输入翻译给 core，或者把 core 输出翻回系统吗？ +如果是，那它应该在 `weed/server` + +3. 它是在真正写 WAL、flush、ship、rebuild、listen socket 吗？ +如果是，那它应该属于 `blockvol` 执行层，不该拥有最终语义 + +这个判断法很有用，因为以后 review 时会一直遇到“这段逻辑该放哪里”。 + +## 当前最关键的设计纪律 + +现在最需要守住的纪律其实就两条： + +1. 不让 `V1` 本地状态重新成为 semantic owner +2. 不让新的 outward surface 绕开 core 再发明一套 truth + +只要守住这两条，前面 phases 的资产就不是散掉，而是在持续被吸收到最终架构里。 + +如果你想，我下一条可以继续讲一个更具体的话题： + +1. “五个小自动机分别管什么” +2. “为什么 `publication` 必须单独拥有 authority” +3. “为什么 `blockvol` 的异步线程还能保留，但 semantic ownership 不能保留” + +你选一个，我可以继续往下展开。 \ No newline at end of file diff --git a/weed/server/block_recovery.go b/weed/server/block_recovery.go index 8691d52a4..59789f3eb 100644 --- a/weed/server/block_recovery.go +++ b/weed/server/block_recovery.go @@ -31,19 +31,29 @@ type recoveryTask struct { type RecoveryManager struct { bs *BlockService - mu sync.Mutex - tasks map[string]*recoveryTask - wg sync.WaitGroup + mu sync.Mutex + tasks map[string]*recoveryTask + pending map[string]*pendingRecoveryExecution + wg sync.WaitGroup // TestHook: if set, called before execution starts. Tests use this // to hold the goroutine alive for serialized-replacement proofs. OnBeforeExecute func(replicaID string) } +type pendingRecoveryExecution struct { + volumeID string + replicaID string + driver *engine.RecoveryDriver + plan *engine.RecoveryPlan + io *v2bridge.Executor +} + func NewRecoveryManager(bs *BlockService) *RecoveryManager { return &RecoveryManager{ - bs: bs, - tasks: make(map[string]*recoveryTask), + bs: bs, + tasks: make(map[string]*recoveryTask), + pending: make(map[string]*pendingRecoveryExecution), } } @@ -130,6 +140,12 @@ func (rm *RecoveryManager) Shutdown() { } } rm.tasks = make(map[string]*recoveryTask) + for volumeID, pending := range rm.pending { + if pending != nil && pending.driver != nil && pending.plan != nil { + pending.driver.CancelPlan(pending.plan, "recovery_shutdown") + } + delete(rm.pending, volumeID) + } rm.mu.Unlock() rm.wg.Wait() } @@ -249,25 +265,43 @@ func (rm *RecoveryManager) runCatchUp(ctx context.Context, replicaID, rebuildAdd glog.Warningf("recovery: plan failed for %s: %v", replicaID, err) return } + switch plan.Outcome { + case engine.OutcomeCatchUp: + if bs.v2Core == nil { + if err := rm.executeCatchUpPlan(volPath, replicaID, driver, plan, executor); err != nil { + if ctx.Err() != nil { + glog.V(1).Infof("recovery: catch-up cancelled for %s: %v", replicaID, err) + } else { + glog.Warningf("recovery: catch-up execution failed for %s: %v", replicaID, err) + } + } + return + } + rm.storePendingExecution(volPath, &pendingRecoveryExecution{ + volumeID: volPath, + replicaID: replicaID, + driver: driver, + plan: plan, + io: executor, + }) + bs.applyCoreEvent(engine.CatchUpPlanned{ID: volPath, TargetLSN: plan.CatchUpTarget}) + if rm.hasPendingExecution(volPath) { + rm.cancelPendingExecution(volPath, "start_catchup_not_emitted") + return + } + case engine.OutcomeNeedsRebuild: + reason := "needs_rebuild" + if plan.Proof != nil && plan.Proof.Reason != "" { + reason = plan.Proof.Reason + } + bs.applyCoreEvent(engine.NeedsRebuildObserved{ID: volPath, Reason: reason}) + return + } if ctx.Err() != nil { driver.CancelPlan(plan, "context_cancelled") return } - - exec := engine.NewCatchUpExecutor(driver, plan) - exec.IO = executor - - if execErr := exec.Execute(nil, 0); execErr != nil { - if ctx.Err() != nil { - glog.V(1).Infof("recovery: catch-up cancelled for %s: %v", replicaID, execErr) - } else { - glog.Warningf("recovery: catch-up execution failed for %s: %v", replicaID, execErr) - } - return - } - - glog.V(0).Infof("recovery: catch-up completed for %s", replicaID) } func (rm *RecoveryManager) runRebuild(ctx context.Context, replicaID, rebuildAddr string) { @@ -306,25 +340,140 @@ func (rm *RecoveryManager) runRebuild(ctx context.Context, replicaID, rebuildAdd glog.Warningf("recovery: rebuild plan failed for %s: %v", replicaID, err) return } - - if ctx.Err() != nil { - driver.CancelPlan(plan, "context_cancelled") - return - } - - exec := engine.NewRebuildExecutor(driver, plan) - exec.IO = executor - - if execErr := exec.Execute(); execErr != nil { - if ctx.Err() != nil { - glog.V(1).Infof("recovery: rebuild cancelled for %s: %v", replicaID, execErr) - } else { - glog.Warningf("recovery: rebuild execution failed for %s: %v", replicaID, execErr) + if bs.v2Core == nil { + if err := rm.executeRebuildPlan(volPath, replicaID, driver, plan, executor); err != nil { + if ctx.Err() != nil { + glog.V(1).Infof("recovery: rebuild cancelled for %s: %v", replicaID, err) + } else { + glog.Warningf("recovery: rebuild execution failed for %s: %v", replicaID, err) + } } return } + rm.storePendingExecution(volPath, &pendingRecoveryExecution{ + volumeID: volPath, + replicaID: replicaID, + driver: driver, + plan: plan, + io: executor, + }) + bs.applyCoreEvent(engine.RebuildStarted{ID: volPath, TargetLSN: plan.RebuildTargetLSN}) + if rm.hasPendingExecution(volPath) { + rm.cancelPendingExecution(volPath, "start_rebuild_not_emitted") + } +} +func (rm *RecoveryManager) storePendingExecution(volumeID string, pending *pendingRecoveryExecution) { + rm.mu.Lock() + defer rm.mu.Unlock() + if rm.pending == nil { + rm.pending = make(map[string]*pendingRecoveryExecution) + } + rm.pending[volumeID] = pending +} + +func (rm *RecoveryManager) takePendingExecution(volumeID string) (*pendingRecoveryExecution, bool) { + rm.mu.Lock() + defer rm.mu.Unlock() + pending, ok := rm.pending[volumeID] + if ok { + delete(rm.pending, volumeID) + } + return pending, ok +} + +func (rm *RecoveryManager) hasPendingExecution(volumeID string) bool { + rm.mu.Lock() + defer rm.mu.Unlock() + _, ok := rm.pending[volumeID] + return ok +} + +func (rm *RecoveryManager) cancelPendingExecution(volumeID, reason string) { + pending, ok := rm.takePendingExecution(volumeID) + if !ok || pending == nil || pending.driver == nil || pending.plan == nil { + return + } + pending.driver.CancelPlan(pending.plan, reason) +} + +func (rm *RecoveryManager) ExecutePendingCatchUp(volumeID string, targetLSN uint64) error { + pending, ok := rm.takePendingExecution(volumeID) + if !ok || pending == nil || pending.plan == nil || pending.driver == nil { + return nil + } + if pending.plan.CatchUpTarget != targetLSN { + pending.driver.CancelPlan(pending.plan, "start_catchup_target_mismatch") + return nil + } + return rm.executeCatchUpPlan(volumeID, pending.replicaID, pending.driver, pending.plan, pending.io) +} + +func (rm *RecoveryManager) ExecutePendingRebuild(volumeID string, targetLSN uint64) error { + pending, ok := rm.takePendingExecution(volumeID) + if !ok || pending == nil || pending.plan == nil || pending.driver == nil { + return nil + } + if pending.plan.RebuildTargetLSN != targetLSN { + pending.driver.CancelPlan(pending.plan, "start_rebuild_target_mismatch") + return nil + } + return rm.executeRebuildPlan(volumeID, pending.replicaID, pending.driver, pending.plan, pending.io) +} + +func (rm *RecoveryManager) executeCatchUpPlan(volumeID, replicaID string, driver *engine.RecoveryDriver, plan *engine.RecoveryPlan, io *v2bridge.Executor) error { + exec := engine.NewCatchUpExecutor(driver, plan) + exec.IO = io + if err := exec.Execute(nil, 0); err != nil { + return err + } + glog.V(0).Infof("recovery: catch-up completed for %s", replicaID) + if rm.bs != nil && rm.bs.v2Core != nil { + achievedLSN := plan.CatchUpTarget + if achievedLSN == 0 { + achievedLSN = plan.CatchUpStartLSN + } + rm.bs.applyCoreEvent(engine.CatchUpCompleted{ID: volumeID, AchievedLSN: achievedLSN}) + } + return nil +} + +func (rm *RecoveryManager) executeRebuildPlan(volumeID, replicaID string, driver *engine.RecoveryDriver, plan *engine.RecoveryPlan, io *v2bridge.Executor) error { + exec := engine.NewRebuildExecutor(driver, plan) + exec.IO = io + if err := exec.Execute(); err != nil { + return err + } glog.V(0).Infof("recovery: rebuild completed for %s", replicaID) + if rm.bs == nil || rm.bs.v2Core == nil { + return nil + } + var snap blockvol.V2StatusSnapshot + if err := rm.bs.blockStore.WithVolume(volumeID, func(vol *blockvol.BlockVol) error { + snap = vol.StatusSnapshot() + return nil + }); err != nil { + glog.Warningf("recovery: cannot read status snapshot for %s after rebuild: %v", volumeID, err) + } + flushedLSN := snap.CommittedLSN + if flushedLSN == 0 { + flushedLSN = plan.RebuildTargetLSN + } + checkpointLSN := snap.CheckpointLSN + if checkpointLSN == 0 { + checkpointLSN = plan.RebuildTargetLSN + } + achievedLSN := flushedLSN + if checkpointLSN > achievedLSN { + achievedLSN = checkpointLSN + } + rm.bs.applyCoreEvent(engine.RebuildCommitted{ + ID: volumeID, + AchievedLSN: achievedLSN, + FlushedLSN: flushedLSN, + CheckpointLSN: checkpointLSN, + }) + return nil } func (rm *RecoveryManager) deriveRebuildAddr(replicaID string, assignments []blockvol.BlockVolumeAssignment) string { diff --git a/weed/server/block_recovery_test.go b/weed/server/block_recovery_test.go index fede33635..4ae8d2b8d 100644 --- a/weed/server/block_recovery_test.go +++ b/weed/server/block_recovery_test.go @@ -1,6 +1,7 @@ package weed_server import ( + "context" "path/filepath" "testing" "time" @@ -62,6 +63,45 @@ func createTestBlockServiceWithVol(t *testing.T) (*BlockService, string) { return bs, volPath } +func createTestBlockServiceWithVolCoreNoRecovery(t *testing.T) (*BlockService, string) { + t.Helper() + dir := t.TempDir() + + volPath := filepath.Join(dir, "vol1.blk") + vol, err := blockvol.CreateBlockVol(volPath, blockvol.CreateOptions{ + VolumeSize: 1 * 1024 * 1024, + BlockSize: 4096, + WALSize: 256 * 1024, + }) + if err != nil { + t.Fatalf("CreateBlockVol: %v", err) + } + vol.Close() + + store := storage.NewBlockVolumeStore() + if _, err := store.AddBlockVolume(volPath, ""); err != nil { + t.Fatalf("AddBlockVolume: %v", err) + } + + bs := &BlockService{ + blockStore: store, + blockDir: dir, + listenAddr: "127.0.0.1:3260", + localServerID: "test-server-1", + v2Bridge: v2bridge.NewControlBridge(), + v2Orchestrator: engine.NewRecoveryOrchestrator(), + v2Core: engine.NewCoreEngine(), + coreProj: make(map[string]engine.PublicationProjection), + replStates: make(map[string]*volReplState), + } + + t.Cleanup(func() { + store.Close() + }) + + return bs, volPath +} + // --- Live-path with real vol: reaches planning --- func TestP4_LivePath_RealVol_ReachesPlan(t *testing.T) { @@ -125,6 +165,115 @@ func TestP4_LivePath_RealVol_ReachesPlan(t *testing.T) { t.Log("P4 live-path: ProcessAssignments → plan_catchup → exec_catchup_started → exec_completed → in_sync") } +func TestP16B_RunCatchUp_UpdatesCoreProjectionFromLiveRecovery(t *testing.T) { + bs, volPath := createTestBlockServiceWithVolCoreNoRecovery(t) + + if err := bs.blockStore.WithVolume(volPath, func(vol *blockvol.BlockVol) error { + for i := 0; i < 5; i++ { + if err := vol.WriteLBA(uint64(i), make([]byte, 4096)); err != nil { + return err + } + } + return nil + }); err != nil { + t.Fatalf("write: %v", err) + } + + bs.ProcessAssignments([]blockvol.BlockVolumeAssignment{ + { + Path: volPath, + Epoch: 1, + Role: uint32(blockvol.RolePrimary), + ReplicaServerID: "vs2", + ReplicaDataAddr: "10.0.0.2:9333", + ReplicaCtrlAddr: "10.0.0.2:9334", + }, + }) + + replicaID := volPath + "/vs2" + sender := bs.v2Orchestrator.Registry.Sender(replicaID) + if sender == nil || !sender.HasActiveSession() { + t.Fatal("expected active sender session before catch-up") + } + + rm := NewRecoveryManager(bs) + bs.v2Recovery = rm + rm.runCatchUp(context.Background(), replicaID, "") + + proj, ok := bs.CoreProjection(volPath) + if !ok { + t.Fatal("expected cached core projection after live catch-up") + } + if proj.Boundary.TargetLSN == 0 { + t.Fatalf("target_lsn=%d", proj.Boundary.TargetLSN) + } + if proj.Boundary.AchievedLSN == 0 { + t.Fatalf("achieved_lsn=%d", proj.Boundary.AchievedLSN) + } + if proj.Boundary.DurableLSN == 0 { + t.Fatalf("durable_lsn=%d", proj.Boundary.DurableLSN) + } + if proj.Recovery.Phase != engine.RecoveryIdle { + t.Fatalf("recovery_phase=%s", proj.Recovery.Phase) + } + if got := bs.ExecutedCoreCommands(volPath); len(got) == 0 || got[len(got)-1] != "start_catchup" { + t.Fatalf("expected start_catchup execution, got %v", got) + } +} + +func TestP16B_RunCatchUp_EscalatesNeedsRebuildIntoCoreProjection(t *testing.T) { + bs, volPath := createTestBlockServiceWithVolCoreNoRecovery(t) + + if err := bs.blockStore.WithVolume(volPath, func(vol *blockvol.BlockVol) error { + for i := 0; i < 5; i++ { + if err := vol.WriteLBA(uint64(i), make([]byte, 4096)); err != nil { + return err + } + } + return vol.ForceFlush() + }); err != nil { + t.Fatalf("write+flush: %v", err) + } + + bs.ProcessAssignments([]blockvol.BlockVolumeAssignment{ + { + Path: volPath, + Epoch: 1, + Role: uint32(blockvol.RolePrimary), + ReplicaServerID: "vs2", + ReplicaDataAddr: "10.0.0.2:9333", + ReplicaCtrlAddr: "10.0.0.2:9334", + }, + }) + + replicaID := volPath + "/vs2" + sender := bs.v2Orchestrator.Registry.Sender(replicaID) + if sender == nil || !sender.HasActiveSession() { + t.Fatal("expected active sender session before needs_rebuild planning") + } + + rm := NewRecoveryManager(bs) + bs.v2Recovery = rm + rm.runCatchUp(context.Background(), replicaID, "") + + proj, ok := bs.CoreProjection(volPath) + if !ok { + t.Fatal("expected cached core projection after needs_rebuild escalation") + } + if proj.Mode.Name != engine.ModeNeedsRebuild { + t.Fatalf("mode=%s", proj.Mode.Name) + } + if proj.Recovery.Phase != engine.RecoveryNeedsRebuild { + t.Fatalf("recovery_phase=%s", proj.Recovery.Phase) + } + if proj.Publication.Reason == "" { + t.Fatal("expected needs_rebuild reason") + } + if got := bs.ExecutedCoreCommands(volPath); len(got) != 3 { + t.Fatalf("needs_rebuild path should not execute start_catchup, got %v", got) + } +} + // --- Serialized replacement: old drained before new starts --- func TestP4_SerializedReplacement_DrainsBeforeStart(t *testing.T) { diff --git a/weed/server/master_block_observability_test.go b/weed/server/master_block_observability_test.go index a58fe738e..691eaa8aa 100644 --- a/weed/server/master_block_observability_test.go +++ b/weed/server/master_block_observability_test.go @@ -10,6 +10,7 @@ import ( "strings" "testing" + engine "github.com/seaweedfs/seaweedfs/sw-block/engine/replication" "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/blockapi" ) @@ -217,6 +218,126 @@ func TestBlockStatusHandler_IncludesHealthCounts(t *testing.T) { } } +func TestBlockStatusHandler_ReflectsCoreInfluencedConsumeCounts(t *testing.T) { + ms := qaPlanMaster(t) + + // Ready path: replica heartbeat publishes receiver addrs from the + // core-influenced ready path, then registry consumes it into a healthy volume. + readyBS := newTestBlockServiceDirect(t) + readyPath := createTestVolDirect(t, readyBS, "status-ready") + errs := readyBS.ApplyAssignments([]blockvol.BlockVolumeAssignment{{ + Path: readyPath, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RoleReplica), + LeaseTtlMs: 30000, + ReplicaDataAddr: "127.0.0.1:0", + ReplicaCtrlAddr: "127.0.0.1:0", + }}) + if len(errs) != 1 || errs[0] != nil { + t.Fatalf("ready apply errs=%v", errs) + } + readyBS.replMu.Lock() + readyState := readyBS.replStates[readyPath] + if readyState == nil { + readyBS.replMu.Unlock() + t.Fatal("missing ready repl state") + } + readyState.publishHealthy = false + readyBS.replMu.Unlock() + readyHB := findHeartbeatMsg(readyBS.CollectBlockVolumeHeartbeat(), readyPath) + if readyHB == nil { + t.Fatal("ready heartbeat missing") + } + + if err := ms.blockRegistry.Register(&BlockVolumeEntry{ + Name: "status-ready", + VolumeServer: "vs1:9333", + Path: "/blocks/status-ready-primary.blk", + Epoch: 1, + Status: StatusActive, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + ReplicaFactor: 2, + Replicas: []ReplicaInfo{{ + Server: "vs2:9333", + Path: readyPath, + }}, + }); err != nil { + t.Fatalf("register ready: %v", err) + } + ms.blockRegistry.mu.Lock() + ms.blockRegistry.addToServer("vs2:9333", "status-ready") + ms.blockRegistry.mu.Unlock() + ms.blockRegistry.UpdateFullHeartbeat("vs2:9333", blockvol.InfoMessagesToProto([]blockvol.BlockVolumeInfoMessage{*readyHB}), "") + + // Degraded path: primary heartbeat publishes degraded bit from the + // core-influenced degraded path, then registry consumes it into a degraded volume. + degradedBS := newTestBlockServiceDirect(t) + degradedPath := createTestVolDirect(t, degradedBS, "status-degraded") + errs = degradedBS.ApplyAssignments([]blockvol.BlockVolumeAssignment{{ + Path: degradedPath, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + LeaseTtlMs: 30000, + ReplicaServerID: "vs1:9333", + ReplicaDataAddr: "10.0.0.3:4260", + ReplicaCtrlAddr: "10.0.0.3:4261", + }}) + if len(errs) != 1 || errs[0] != nil { + t.Fatalf("degraded apply errs=%v", errs) + } + degradedBS.applyCoreEvent(engine.BarrierRejected{ID: degradedPath, Reason: "barrier_timeout"}) + degradedHB := findHeartbeatMsg(degradedBS.CollectBlockVolumeHeartbeat(), degradedPath) + if degradedHB == nil { + t.Fatal("degraded heartbeat missing") + } + + if err := ms.blockRegistry.Register(&BlockVolumeEntry{ + Name: "status-degraded", + VolumeServer: "vs3:9333", + Path: degradedPath, + Epoch: 1, + Status: StatusActive, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + ReplicaFactor: 2, + Replicas: []ReplicaInfo{{ + Server: "vs1:9333", + Path: "/blocks/status-degraded-replica.blk", + Ready: true, + }}, + }); err != nil { + t.Fatalf("register degraded: %v", err) + } + ms.blockRegistry.UpdateFullHeartbeat("vs3:9333", blockvol.InfoMessagesToProto([]blockvol.BlockVolumeInfoMessage{*degradedHB}), "") + + req := httptest.NewRequest(http.MethodGet, "/block/status", nil) + w := httptest.NewRecorder() + ms.blockStatusHandler(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", w.Code) + } + + var resp blockapi.BlockStatusResponse + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("decode: %v", err) + } + if resp.VolumeCount != 2 { + t.Fatalf("expected volume_count=2, got %d", resp.VolumeCount) + } + if resp.HealthyCount != 1 { + t.Fatalf("expected healthy_count=1, got %d", resp.HealthyCount) + } + if resp.DegradedCount != 1 { + t.Fatalf("expected degraded_count=1, got %d", resp.DegradedCount) + } + if resp.RebuildingCount != 0 { + t.Fatalf("expected rebuilding_count=0, got %d", resp.RebuildingCount) + } + if resp.UnsafeCount != 0 { + t.Fatalf("expected unsafe_count=0, got %d", resp.UnsafeCount) + } +} + // --- VolumeInfo includes health_state --- func TestEntryToVolumeInfo_IncludesHealthState(t *testing.T) { @@ -234,6 +355,129 @@ func TestEntryToVolumeInfo_IncludesHealthState(t *testing.T) { } } +func TestEntryToVolumeInfo_ReflectsCoreInfluencedReadyConsume(t *testing.T) { + bs := newTestBlockServiceDirect(t) + path := createTestVolDirect(t, bs, "vol-info-ready") + + errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{ + { + Path: path, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RoleReplica), + LeaseTtlMs: 30000, + ReplicaDataAddr: "127.0.0.1:0", + ReplicaCtrlAddr: "127.0.0.1:0", + }, + }) + if len(errs) != 1 || errs[0] != nil { + t.Fatalf("apply assignment errs=%v", errs) + } + + bs.replMu.Lock() + state := bs.replStates[path] + if state == nil { + bs.replMu.Unlock() + t.Fatal("missing repl state") + } + state.publishHealthy = false + bs.replMu.Unlock() + + hb := findHeartbeatMsg(bs.CollectBlockVolumeHeartbeat(), path) + if hb == nil { + t.Fatal("heartbeat volume missing") + } + + r := NewBlockVolumeRegistry() + r.MarkBlockCapable("primary-server:8080") + if err := r.Register(&BlockVolumeEntry{ + Name: "vol-info-ready", + VolumeServer: "primary-server:8080", + Path: "/blocks/vol-info-ready-primary.blk", + Status: StatusActive, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + ReplicaFactor: 2, + Replicas: []ReplicaInfo{{ + Server: "replica-server:8080", + Path: path, + }}, + }); err != nil { + t.Fatalf("register: %v", err) + } + r.UpdateFullHeartbeat("replica-server:8080", blockvol.InfoMessagesToProto([]blockvol.BlockVolumeInfoMessage{*hb}), "") + + entry, _ := r.Lookup("vol-info-ready") + info := entryToVolumeInfo(&entry, true) + if !info.ReplicaReady { + t.Fatalf("expected outward ReplicaReady=true, info=%+v", info) + } + if info.ReplicaDegraded { + t.Fatalf("expected outward ReplicaDegraded=false, info=%+v", info) + } + if info.VolumeMode != "publish_healthy" { + t.Fatalf("expected outward VolumeMode=publish_healthy, got %q", info.VolumeMode) + } + if info.HealthState != HealthStateHealthy { + t.Fatalf("expected outward HealthState=%q, got %q", HealthStateHealthy, info.HealthState) + } +} + +func TestEntryToVolumeInfo_ReflectsCoreInfluencedDegradedConsume(t *testing.T) { + bs := newTestBlockServiceDirect(t) + path := createTestVolDirect(t, bs, "vol-info-degraded") + + errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{ + { + Path: path, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + LeaseTtlMs: 30000, + ReplicaServerID: "replica-server:8080", + ReplicaDataAddr: "10.0.0.2:4260", + ReplicaCtrlAddr: "10.0.0.2:4261", + }, + }) + if len(errs) != 1 || errs[0] != nil { + t.Fatalf("apply assignment errs=%v", errs) + } + bs.applyCoreEvent(engine.BarrierRejected{ID: path, Reason: "barrier_timeout"}) + + hb := findHeartbeatMsg(bs.CollectBlockVolumeHeartbeat(), path) + if hb == nil { + t.Fatal("heartbeat volume missing") + } + + r := NewBlockVolumeRegistry() + r.MarkBlockCapable("primary-server:8080") + if err := r.Register(&BlockVolumeEntry{ + Name: "vol-info-degraded", + VolumeServer: "primary-server:8080", + Path: path, + Status: StatusActive, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + ReplicaFactor: 2, + Replicas: []ReplicaInfo{{ + Server: "replica-server:8080", + Path: "/blocks/vol-info-degraded-replica.blk", + Ready: true, + }}, + }); err != nil { + t.Fatalf("register: %v", err) + } + r.UpdateFullHeartbeat("primary-server:8080", blockvol.InfoMessagesToProto([]blockvol.BlockVolumeInfoMessage{*hb}), "") + + entry, _ := r.Lookup("vol-info-degraded") + info := entryToVolumeInfo(&entry, true) + if !info.ReplicaDegraded { + t.Fatalf("expected outward ReplicaDegraded=true, info=%+v", info) + } + if info.VolumeMode != "degraded" { + t.Fatalf("expected outward VolumeMode=degraded, got %q", info.VolumeMode) + } + if info.HealthState != HealthStateDegraded { + t.Fatalf("expected outward HealthState=%q, got %q", HealthStateDegraded, info.HealthState) + } +} + func TestEntryToVolumeInfo_PrimaryDead_Unsafe(t *testing.T) { entry := &BlockVolumeEntry{ Name: "dead-primary-vol", diff --git a/weed/server/master_block_registry.go b/weed/server/master_block_registry.go index 9151895a7..3b26755ed 100644 --- a/weed/server/master_block_registry.go +++ b/weed/server/master_block_registry.go @@ -71,23 +71,23 @@ type BlockVolumeEntry struct { RebuildListenAddr string // rebuild server listen addr on primary // CP8-2: Multi-replica support. - ReplicaFactor int // 2 or 3 (default 2) - Replicas []ReplicaInfo // one per replica (RF-1 entries) - HealthScore float64 // primary health score from heartbeat - ReplicaReady bool // all configured replicas are ready for publication - ReplicaDegraded bool // aggregate: transport degraded OR not ready - TransportDegraded bool // primary reports degraded replicas + ReplicaFactor int // 2 or 3 (default 2) + Replicas []ReplicaInfo // one per replica (RF-1 entries) + HealthScore float64 // primary health score from heartbeat + ReplicaReady bool // all configured replicas are ready for publication + ReplicaDegraded bool // aggregate: transport degraded OR not ready + TransportDegraded bool // primary reports degraded replicas // CP13-9: Normalized volume mode for external surfaces. // Computed by recomputeReplicaState from the current entry state. - VolumeMode string // "allocated_only", "bootstrap_pending", "publish_healthy", "degraded", "needs_rebuild" - WALHeadLSN uint64 // primary WAL head LSN from heartbeat + VolumeMode string // "allocated_only", "bootstrap_pending", "publish_healthy", "degraded", "needs_rebuild" + WALHeadLSN uint64 // primary WAL head LSN from heartbeat // CP8-3-1: Durability mode. - DurabilityMode string // "best_effort", "sync_all", "sync_quorum" + DurabilityMode string // "best_effort", "sync_all", "sync_quorum" // CP11B-1: Provisioning preset (control-plane metadata only). - Preset string // "database", "general", "throughput", or "" + Preset string // "database", "general", "throughput", or "" // Lease tracking for failover (CP6-3 F2). LastLeaseGrant time.Time @@ -95,7 +95,7 @@ type BlockVolumeEntry struct { // CP11A-2: Coordinated expand tracking. ExpandInProgress bool - ExpandFailed bool // true = primary committed but replica(s) failed; size suppressed + ExpandFailed bool // true = primary committed but replica(s) failed; size suppressed PendingExpandSize uint64 ExpandEpoch uint64 @@ -256,7 +256,6 @@ type PlacementCandidateInfo struct { NvmeCapable bool } - // NewBlockVolumeRegistry creates an empty registry. func NewBlockVolumeRegistry() *BlockVolumeRegistry { return &BlockVolumeRegistry{ @@ -452,7 +451,7 @@ type ReplicaAddrChange struct { // HeartbeatResult holds the side effects from UpdateFullHeartbeat that the // caller (heartbeat handler) must process. type HeartbeatResult struct { - AddrChanges []ReplicaAddrChange + AddrChanges []ReplicaAddrChange PrimaryRefreshNeeded []BlockVolumeEntry // CP13-8A: entries needing primary assignment refresh } @@ -551,88 +550,9 @@ func (r *BlockVolumeRegistry) UpdateFullHeartbeat(server string, infos []*master isReplica := existing.ReplicaByServer(server) != nil if isPrimary { - // Primary heartbeat: update primary fields. - // CP11A-2: skip size update during coordinated expand. - if !existing.ExpandInProgress { - existing.SizeBytes = info.VolumeSize - } - existing.Epoch = info.Epoch - existing.Role = info.Role - existing.Status = StatusActive - existing.LastLeaseGrant = time.Now() - existing.HealthScore = info.HealthScore - existing.TransportDegraded = info.ReplicaDegraded - existing.WALHeadLSN = info.WalHeadLsn - // F3: only update DurabilityMode when non-empty (prevents older VS from clearing strict mode). - if info.DurabilityMode != "" { - existing.DurabilityMode = info.DurabilityMode - } - // F5: update replica addresses from heartbeat info. - if info.ReplicaDataAddr != "" { - existing.ReplicaDataAddr = info.ReplicaDataAddr - } - if info.ReplicaCtrlAddr != "" { - existing.ReplicaCtrlAddr = info.ReplicaCtrlAddr - } - // NVMe publication: update NVMe fields from heartbeat. - // Required for master restart reconstruction and NVMe enable/disable. - existing.NvmeAddr = info.NvmeAddr - existing.NQN = info.Nqn - // Sync first replica's data addrs to Replicas[]. - if info.ReplicaDataAddr != "" && len(existing.Replicas) > 0 { - existing.Replicas[0].DataAddr = info.ReplicaDataAddr - existing.Replicas[0].CtrlAddr = info.ReplicaCtrlAddr - } - existing.recomputeReplicaState() + r.applyPrimaryHeartbeatObservation(existing, info) } else if isReplica { - // Replica heartbeat: update ReplicaInfo fields. - for i := range existing.Replicas { - if existing.Replicas[i].Server == server { - existing.Replicas[i].Path = info.Path - existing.Replicas[i].WALHeadLSN = info.WalHeadLsn - existing.Replicas[i].HealthScore = info.HealthScore - existing.Replicas[i].LastHeartbeat = time.Now() - // Keep role as RoleReplica — the VS may report a stale - // primary role if it hasn't received its demotion assignment yet. - // The registry's decision (lower epoch = replica) is authoritative. - existing.Replicas[i].Role = blockvol.RoleToWire(blockvol.RoleReplica) - existing.Replicas[i].NvmeAddr = info.NvmeAddr - existing.Replicas[i].NQN = info.Nqn - existing.Replicas[i].Ready = info.ReplicaDataAddr != "" && info.ReplicaCtrlAddr != "" - if existing.WALHeadLSN > info.WalHeadLsn { - existing.Replicas[i].WALLag = existing.WALHeadLSN - info.WalHeadLsn - } else { - existing.Replicas[i].WALLag = 0 - } - // CP13-8: detect address change on replica restart. - // If either the data or control address changed, the primary's - // shipper has a stale endpoint. Queue a Primary refresh. - if info.ReplicaDataAddr != "" || info.ReplicaCtrlAddr != "" { - oldData := existing.Replicas[i].DataAddr - oldCtrl := existing.Replicas[i].CtrlAddr - dataChanged := info.ReplicaDataAddr != "" && oldData != "" && oldData != info.ReplicaDataAddr - ctrlChanged := info.ReplicaCtrlAddr != "" && oldCtrl != "" && oldCtrl != info.ReplicaCtrlAddr - if dataChanged || ctrlChanged { - result.AddrChanges = append(result.AddrChanges, ReplicaAddrChange{ - VolumeName: existingName, - PrimaryServer: existing.VolumeServer, - OldDataAddr: oldData, - OldCtrlAddr: oldCtrl, - NewDataAddr: info.ReplicaDataAddr, - NewCtrlAddr: info.ReplicaCtrlAddr, - }) - } - if info.ReplicaDataAddr != "" { - existing.Replicas[i].DataAddr = info.ReplicaDataAddr - } - if info.ReplicaCtrlAddr != "" { - existing.Replicas[i].CtrlAddr = info.ReplicaCtrlAddr - } - } - break - } - } - existing.recomputeReplicaState() + r.applyReplicaHeartbeatObservation(existing, server, existingName, info, &result) } else { // Server reports a volume that exists but has no record of this server. // This happens after master restart. Use epoch-based reconciliation @@ -657,19 +577,19 @@ func (r *BlockVolumeRegistry) UpdateFullHeartbeat(server string, infos []*master existing, dup := r.volumes[name] if !dup { entry := &BlockVolumeEntry{ - Name: name, - VolumeServer: server, - Path: info.Path, - SizeBytes: info.VolumeSize, - Epoch: info.Epoch, - Role: info.Role, - Status: StatusActive, - LastLeaseGrant: time.Now(), - LeaseTTL: 30 * time.Second, - HealthScore: info.HealthScore, + Name: name, + VolumeServer: server, + Path: info.Path, + SizeBytes: info.VolumeSize, + Epoch: info.Epoch, + Role: info.Role, + Status: StatusActive, + LastLeaseGrant: time.Now(), + LeaseTTL: 30 * time.Second, + HealthScore: info.HealthScore, TransportDegraded: info.ReplicaDegraded, - WALHeadLSN: info.WalHeadLsn, - DurabilityMode: info.DurabilityMode, + WALHeadLSN: info.WalHeadLsn, + DurabilityMode: info.DurabilityMode, } if info.ReplicaDataAddr != "" { entry.ReplicaDataAddr = info.ReplicaDataAddr @@ -700,6 +620,103 @@ func (r *BlockVolumeRegistry) UpdateFullHeartbeat(server string, infos []*master return result } +// applyPrimaryHeartbeatObservation consumes one primary-side heartbeat into the +// registry entry. Caller must hold r.mu. +func (r *BlockVolumeRegistry) applyPrimaryHeartbeatObservation(existing *BlockVolumeEntry, info *master_pb.BlockVolumeInfoMessage) { + // CP11A-2: skip size update during coordinated expand. + if !existing.ExpandInProgress { + existing.SizeBytes = info.VolumeSize + } + existing.Epoch = info.Epoch + existing.Role = info.Role + existing.Status = StatusActive + existing.LastLeaseGrant = time.Now() + existing.HealthScore = info.HealthScore + existing.TransportDegraded = info.ReplicaDegraded + existing.WALHeadLSN = info.WalHeadLsn + // F3: only update DurabilityMode when non-empty (prevents older VS from clearing strict mode). + if info.DurabilityMode != "" { + existing.DurabilityMode = info.DurabilityMode + } + // F5: update replica addresses from heartbeat info. + if info.ReplicaDataAddr != "" { + existing.ReplicaDataAddr = info.ReplicaDataAddr + } + if info.ReplicaCtrlAddr != "" { + existing.ReplicaCtrlAddr = info.ReplicaCtrlAddr + } + // NVMe publication: update NVMe fields from heartbeat. + // Required for master restart reconstruction and NVMe enable/disable. + existing.NvmeAddr = info.NvmeAddr + existing.NQN = info.Nqn + // Sync first replica's data addrs to Replicas[]. + if info.ReplicaDataAddr != "" && len(existing.Replicas) > 0 { + existing.Replicas[0].DataAddr = info.ReplicaDataAddr + existing.Replicas[0].CtrlAddr = info.ReplicaCtrlAddr + } + existing.recomputeReplicaState() +} + +// applyReplicaHeartbeatObservation consumes one replica-side heartbeat into the +// registry entry. Caller must hold r.mu. +func (r *BlockVolumeRegistry) applyReplicaHeartbeatObservation(existing *BlockVolumeEntry, server, existingName string, info *master_pb.BlockVolumeInfoMessage, result *HeartbeatResult) { + for i := range existing.Replicas { + if existing.Replicas[i].Server != server { + continue + } + existing.Replicas[i].Path = info.Path + existing.Replicas[i].WALHeadLSN = info.WalHeadLsn + existing.Replicas[i].HealthScore = info.HealthScore + existing.Replicas[i].LastHeartbeat = time.Now() + // Keep role as RoleReplica — the VS may report a stale + // primary role if it hasn't received its demotion assignment yet. + // The registry's decision (lower epoch = replica) is authoritative. + existing.Replicas[i].Role = blockvol.RoleToWire(blockvol.RoleReplica) + existing.Replicas[i].NvmeAddr = info.NvmeAddr + existing.Replicas[i].NQN = info.Nqn + existing.Replicas[i].Ready = replicaReadyObservedFromHeartbeat(info) + if existing.WALHeadLSN > info.WalHeadLsn { + existing.Replicas[i].WALLag = existing.WALHeadLSN - info.WalHeadLsn + } else { + existing.Replicas[i].WALLag = 0 + } + // CP13-8: detect address change on replica restart. + // If either the data or control address changed, the primary's + // shipper has a stale endpoint. Queue a Primary refresh. + if info.ReplicaDataAddr != "" || info.ReplicaCtrlAddr != "" { + oldData := existing.Replicas[i].DataAddr + oldCtrl := existing.Replicas[i].CtrlAddr + dataChanged := info.ReplicaDataAddr != "" && oldData != "" && oldData != info.ReplicaDataAddr + ctrlChanged := info.ReplicaCtrlAddr != "" && oldCtrl != "" && oldCtrl != info.ReplicaCtrlAddr + if dataChanged || ctrlChanged { + result.AddrChanges = append(result.AddrChanges, ReplicaAddrChange{ + VolumeName: existingName, + PrimaryServer: existing.VolumeServer, + OldDataAddr: oldData, + OldCtrlAddr: oldCtrl, + NewDataAddr: info.ReplicaDataAddr, + NewCtrlAddr: info.ReplicaCtrlAddr, + }) + } + if info.ReplicaDataAddr != "" { + existing.Replicas[i].DataAddr = info.ReplicaDataAddr + } + if info.ReplicaCtrlAddr != "" { + existing.Replicas[i].CtrlAddr = info.ReplicaCtrlAddr + } + } + break + } + existing.recomputeReplicaState() +} + +func replicaReadyObservedFromHeartbeat(info *master_pb.BlockVolumeInfoMessage) bool { + if info == nil { + return false + } + return info.ReplicaDataAddr != "" && info.ReplicaCtrlAddr != "" +} + // reconcileOnRestart handles the case where a second server reports a volume // name that already exists in the registry during master restart reconstruction. // Uses epoch-based tie-breaking to determine who is the real primary. @@ -1244,11 +1261,11 @@ type PromotionRejection struct { // Used by auto-promotion, manual promote API, preflight status, and logging. type PromotionPreflightResult struct { VolumeName string - Promotable bool // true if a candidate was found - Candidate *ReplicaInfo // best candidate (nil if !Promotable) - CandidateIdx int // index in Replicas[] (-1 if !Promotable) + Promotable bool // true if a candidate was found + Candidate *ReplicaInfo // best candidate (nil if !Promotable) + CandidateIdx int // index in Replicas[] (-1 if !Promotable) Rejections []PromotionRejection // why each non-candidate was rejected - Reason string // human-readable summary when !Promotable + Reason string // human-readable summary when !Promotable } // evaluatePromotionLocked evaluates promotion candidates for a volume. @@ -1847,4 +1864,3 @@ func nameFromPath(path string) string { } return base } - diff --git a/weed/server/master_block_registry_test.go b/weed/server/master_block_registry_test.go index 14b20789e..4ba9fc962 100644 --- a/weed/server/master_block_registry_test.go +++ b/weed/server/master_block_registry_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + engine "github.com/seaweedfs/seaweedfs/sw-block/engine/replication" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" ) @@ -1391,7 +1392,7 @@ func TestRegistry_VolumesWithDeadPrimary_Basic(t *testing.T) { r.Register(&BlockVolumeEntry{ Name: "vol1", VolumeServer: "vs1", Path: "/data/vol1.blk", SizeBytes: 1 << 30, Epoch: 1, Role: blockvol.RoleToWire(blockvol.RolePrimary), - Status: StatusActive, + Status: StatusActive, Replicas: []ReplicaInfo{{Server: "vs2", Path: "/data/vol1.blk"}}, }) @@ -2007,3 +2008,134 @@ func TestRegistry_ReplicaReadyRequiresReplicaHeartbeat(t *testing.T) { t.Fatal("aggregate replica readiness should become true after replica heartbeat") } } + +func TestRegistry_UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaDegraded(t *testing.T) { + bs := newTestBlockServiceDirect(t) + path := createTestVolDirect(t, bs, "vol-master-consume") + + errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{ + { + Path: path, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + LeaseTtlMs: 30000, + ReplicaServerID: "replica-server:8080", + ReplicaDataAddr: "10.0.0.2:4260", + ReplicaCtrlAddr: "10.0.0.2:4261", + }, + }) + if len(errs) != 1 || errs[0] != nil { + t.Fatalf("apply assignment errs=%v", errs) + } + + bs.applyCoreEvent(engine.BarrierRejected{ID: path, Reason: "barrier_timeout"}) + + hb := findHeartbeatMsg(bs.CollectBlockVolumeHeartbeat(), path) + if hb == nil { + t.Fatal("heartbeat volume missing") + } + if !hb.ReplicaDegraded { + t.Fatalf("expected core-influenced heartbeat degraded bit, hb=%+v", hb) + } + + r := NewBlockVolumeRegistry() + if err := r.Register(&BlockVolumeEntry{ + Name: "vol-master-consume", + VolumeServer: "primary-server:8080", + Path: path, + Status: StatusActive, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + ReplicaFactor: 2, + Replicas: []ReplicaInfo{{ + Server: "replica-server:8080", + Path: "/blocks/vol-master-consume.blk", + Ready: true, + }}, + }); err != nil { + t.Fatalf("register: %v", err) + } + + r.UpdateFullHeartbeat("primary-server:8080", blockvol.InfoMessagesToProto([]blockvol.BlockVolumeInfoMessage{*hb}), "") + + entry, _ := r.Lookup("vol-master-consume") + if !entry.TransportDegraded { + t.Fatalf("expected registry transport degraded from heartbeat, entry=%+v", entry) + } + if !entry.ReplicaDegraded { + t.Fatalf("expected registry aggregate degraded from heartbeat, entry=%+v", entry) + } + if entry.VolumeMode != "degraded" { + t.Fatalf("expected degraded volume mode after consume, got %q", entry.VolumeMode) + } +} + +func TestRegistry_UpdateFullHeartbeat_ConsumesCoreInfluencedReplicaReady(t *testing.T) { + bs := newTestBlockServiceDirect(t) + path := createTestVolDirect(t, bs, "vol-master-ready") + + errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{ + { + Path: path, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RoleReplica), + LeaseTtlMs: 30000, + ReplicaDataAddr: "127.0.0.1:0", + ReplicaCtrlAddr: "127.0.0.1:0", + }, + }) + if len(errs) != 1 || errs[0] != nil { + t.Fatalf("apply assignment errs=%v", errs) + } + + bs.replMu.Lock() + state := bs.replStates[path] + if state == nil { + bs.replMu.Unlock() + t.Fatal("missing repl state") + } + state.publishHealthy = false + bs.replMu.Unlock() + + hb := findHeartbeatMsg(bs.CollectBlockVolumeHeartbeat(), path) + if hb == nil { + t.Fatal("heartbeat volume missing") + } + if hb.ReplicaDataAddr == "" || hb.ReplicaCtrlAddr == "" { + t.Fatalf("expected core-influenced replica addresses on heartbeat, hb=%+v", hb) + } + if hb.ReplicaDegraded { + t.Fatalf("did not expect degraded heartbeat on ready path, hb=%+v", hb) + } + + r := NewBlockVolumeRegistry() + if err := r.Register(&BlockVolumeEntry{ + Name: "vol-master-ready", + VolumeServer: "primary-server:8080", + Path: "/blocks/vol-master-ready-primary.blk", + Status: StatusActive, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + ReplicaFactor: 2, + Replicas: []ReplicaInfo{{ + Server: "replica-server:8080", + Path: path, + }}, + }); err != nil { + t.Fatalf("register: %v", err) + } + + r.UpdateFullHeartbeat("replica-server:8080", blockvol.InfoMessagesToProto([]blockvol.BlockVolumeInfoMessage{*hb}), "") + + entry, _ := r.Lookup("vol-master-ready") + if !entry.Replicas[0].Ready { + t.Fatalf("expected replica ready from heartbeat addresses, entry=%+v", entry) + } + if !entry.ReplicaReady { + t.Fatalf("expected aggregate replica ready after consume, entry=%+v", entry) + } + if entry.ReplicaDegraded { + t.Fatalf("did not expect aggregate degraded after ready consume, entry=%+v", entry) + } + if entry.VolumeMode != "publish_healthy" { + t.Fatalf("expected publish_healthy after ready consume, got %q", entry.VolumeMode) + } +} diff --git a/weed/server/master_grpc_server_block.go b/weed/server/master_grpc_server_block.go index a9c549fa8..d650fedd9 100644 --- a/weed/server/master_grpc_server_block.go +++ b/weed/server/master_grpc_server_block.go @@ -250,7 +250,11 @@ func (ms *MasterServer) LookupBlockVolume(ctx context.Context, req *master_pb.Lo return nil, fmt.Errorf("block volume %q not found", req.Name) } - replicaServers := replicaServerList(&entry) + return lookupResponseFromEntry(&entry), nil +} + +func lookupResponseFromEntry(entry *BlockVolumeEntry) *master_pb.LookupBlockVolumeResponse { + replicaServers := replicaServerList(entry) rf := entry.ReplicaFactor if rf == 0 { rf = 2 // default for pre-CP8-2 entries @@ -270,7 +274,7 @@ func (ms *MasterServer) LookupBlockVolume(ctx context.Context, req *master_pb.Lo DurabilityMode: durModeStr, NvmeAddr: entry.NvmeAddr, Nqn: entry.NQN, - }, nil + } } // tryCreateOneReplica attempts to create one replica volume on a different server. diff --git a/weed/server/master_grpc_server_block_test.go b/weed/server/master_grpc_server_block_test.go index ed1abf80f..3893dd97d 100644 --- a/weed/server/master_grpc_server_block_test.go +++ b/weed/server/master_grpc_server_block_test.go @@ -665,6 +665,47 @@ func TestMaster_LookupBlockVolume(t *testing.T) { } } +func TestLookupResponseFromEntry_PublicationMinimalSurface(t *testing.T) { + resp := lookupResponseFromEntry(&BlockVolumeEntry{ + Name: "lookup-minimal", + VolumeServer: "vs1:9333", + ISCSIAddr: "10.0.0.1:3260", + IQN: "iqn.2024.test:lookup-minimal", + SizeBytes: 1 << 30, + ReplicaServer: "vs2:9333", + ReplicaFactor: 2, + Replicas: []ReplicaInfo{{Server: "vs2:9333"}}, + DurabilityMode: "sync_all", + NvmeAddr: "10.0.0.1:4420", + NQN: "nqn.2024-01.com.seaweedfs:vol.lookup-minimal", + ReplicaReady: true, + ReplicaDegraded: false, + VolumeMode: "publish_healthy", + }) + + if resp.VolumeServer != "vs1:9333" { + t.Fatalf("VolumeServer=%q", resp.VolumeServer) + } + if resp.IscsiAddr != "10.0.0.1:3260" { + t.Fatalf("IscsiAddr=%q", resp.IscsiAddr) + } + if resp.CapacityBytes != 1<<30 { + t.Fatalf("CapacityBytes=%d", resp.CapacityBytes) + } + if resp.ReplicaFactor != 2 { + t.Fatalf("ReplicaFactor=%d", resp.ReplicaFactor) + } + if len(resp.ReplicaServers) != 1 || resp.ReplicaServers[0] != "vs2:9333" { + t.Fatalf("ReplicaServers=%v", resp.ReplicaServers) + } + if resp.DurabilityMode != "sync_all" { + t.Fatalf("DurabilityMode=%q", resp.DurabilityMode) + } + if resp.NvmeAddr != "10.0.0.1:4420" || resp.Nqn != "nqn.2024-01.com.seaweedfs:vol.lookup-minimal" { + t.Fatalf("NVMe fields addr=%q nqn=%q", resp.NvmeAddr, resp.Nqn) + } +} + // ============================================================ // CP8-2 T9: Multi-Replica Create/Delete/Assign Tests // ============================================================ diff --git a/weed/server/master_server_handlers_block.go b/weed/server/master_server_handlers_block.go index 4b118966f..bf566ae70 100644 --- a/weed/server/master_server_handlers_block.go +++ b/weed/server/master_server_handlers_block.go @@ -251,8 +251,12 @@ func (ms *MasterServer) blockVolumeExpandHandler(w http.ResponseWriter, r *http. // blockStatusHandler handles GET /block/status — cluster summary with health counts. func (ms *MasterServer) blockStatusHandler(w http.ResponseWriter, r *http.Request) { + writeJsonQuiet(w, r, http.StatusOK, ms.statusResponseFromRegistry()) +} + +func (ms *MasterServer) statusResponseFromRegistry() blockapi.BlockStatusResponse { healthSummary := ms.blockRegistry.ComputeClusterHealthSummary() - status := blockapi.BlockStatusResponse{ + return blockapi.BlockStatusResponse{ VolumeCount: len(ms.blockRegistry.ListAll()), ServerCount: len(ms.blockRegistry.BlockCapableServers()), PromotionLSNTolerance: ms.blockRegistry.PromotionLSNTolerance(), @@ -267,7 +271,6 @@ func (ms *MasterServer) blockStatusHandler(w http.ResponseWriter, r *http.Reques UnsafeCount: healthSummary.Unsafe, NvmeCapableServers: ms.blockRegistry.NvmeCapableServerCount(), } - writeJsonQuiet(w, r, http.StatusOK, status) } // blockVolumePreflightHandler handles GET /block/volume/{name}/preflight. @@ -365,6 +368,22 @@ func (ms *MasterServer) blockVolumePromoteHandler(w http.ResponseWriter, r *http // entryToVolumeInfo converts a BlockVolumeEntry to a blockapi.VolumeInfo. // primaryAlive indicates whether the primary server is alive (in blockServers set). +type entryReplicaSurface struct { + ReplicaReady bool + ReplicaDegraded bool + VolumeMode string + HealthState string +} + +func entryReplicaSurfaceInfo(e *BlockVolumeEntry, primaryAlive bool) entryReplicaSurface { + return entryReplicaSurface{ + ReplicaReady: e.ReplicaReady, + ReplicaDegraded: e.ReplicaDegraded, + VolumeMode: e.VolumeMode, + HealthState: deriveHealthStateWithLiveness(e, primaryAlive), + } +} + func entryToVolumeInfo(e *BlockVolumeEntry, primaryAlive bool) blockapi.VolumeInfo { status := "pending" if e.Status == StatusActive { @@ -378,6 +397,7 @@ func entryToVolumeInfo(e *BlockVolumeEntry, primaryAlive bool) blockapi.VolumeIn if durMode == "" { durMode = "best_effort" } + surface := entryReplicaSurfaceInfo(e, primaryAlive) info := blockapi.VolumeInfo{ Name: e.Name, VolumeServer: e.VolumeServer, @@ -394,15 +414,15 @@ func entryToVolumeInfo(e *BlockVolumeEntry, primaryAlive bool) blockapi.VolumeIn ReplicaDataAddr: e.ReplicaDataAddr, ReplicaCtrlAddr: e.ReplicaCtrlAddr, ReplicaFactor: rf, - ReplicaReady: e.ReplicaReady, + ReplicaReady: surface.ReplicaReady, HealthScore: e.HealthScore, - ReplicaDegraded: e.ReplicaDegraded, + ReplicaDegraded: surface.ReplicaDegraded, DurabilityMode: durMode, Preset: e.Preset, NvmeAddr: e.NvmeAddr, NQN: e.NQN, - HealthState: deriveHealthStateWithLiveness(e, primaryAlive), - VolumeMode: e.VolumeMode, + HealthState: surface.HealthState, + VolumeMode: surface.VolumeMode, } for _, ri := range e.Replicas { info.Replicas = append(info.Replicas, blockapi.ReplicaDetail{ diff --git a/weed/server/master_server_handlers_block_test.go b/weed/server/master_server_handlers_block_test.go index 89395ba37..72291465c 100644 --- a/weed/server/master_server_handlers_block_test.go +++ b/weed/server/master_server_handlers_block_test.go @@ -10,6 +10,8 @@ import ( "testing" "github.com/gorilla/mux" + engine "github.com/seaweedfs/seaweedfs/sw-block/engine/replication" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/blockapi" ) @@ -154,6 +156,77 @@ func TestBlockVolumeLookupHandler(t *testing.T) { } } +func TestBlockVolumeLookupHandler_ReflectsCoreInfluencedReadyConsume(t *testing.T) { + ms, ts := blockTestServer(t) + bs := newTestBlockServiceDirect(t) + path := createTestVolDirect(t, bs, "vol-handler-ready") + + errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{{ + Path: path, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RoleReplica), + LeaseTtlMs: 30000, + ReplicaDataAddr: "127.0.0.1:0", + ReplicaCtrlAddr: "127.0.0.1:0", + }}) + if len(errs) != 1 || errs[0] != nil { + t.Fatalf("apply assignment errs=%v", errs) + } + bs.replMu.Lock() + state := bs.replStates[path] + if state == nil { + bs.replMu.Unlock() + t.Fatal("missing repl state") + } + state.publishHealthy = false + bs.replMu.Unlock() + + hb := findHeartbeatMsg(bs.CollectBlockVolumeHeartbeat(), path) + if hb == nil { + t.Fatal("heartbeat volume missing") + } + + ms.blockRegistry.MarkBlockCapable("primary-server:8080") + if err := ms.blockRegistry.Register(&BlockVolumeEntry{ + Name: "vol-handler-ready", + VolumeServer: "primary-server:8080", + Path: "/blocks/vol-handler-ready-primary.blk", + Status: StatusActive, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + ReplicaFactor: 2, + Replicas: []ReplicaInfo{{ + Server: "replica-server:8080", + Path: path, + }}, + }); err != nil { + t.Fatalf("register: %v", err) + } + ms.blockRegistry.UpdateFullHeartbeat("replica-server:8080", blockvol.InfoMessagesToProto([]blockvol.BlockVolumeInfoMessage{*hb}), "") + + resp, err := http.Get(ts.URL + "/block/volume/vol-handler-ready") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + + var info blockapi.VolumeInfo + if err := json.NewDecoder(resp.Body).Decode(&info); err != nil { + t.Fatal(err) + } + if !info.ReplicaReady { + t.Fatalf("expected outward ReplicaReady=true, info=%+v", info) + } + if info.ReplicaDegraded { + t.Fatalf("expected outward ReplicaDegraded=false, info=%+v", info) + } + if info.VolumeMode != "publish_healthy" { + t.Fatalf("expected outward VolumeMode=publish_healthy, got %q", info.VolumeMode) + } +} + func TestBlockVolumeDeleteHandler(t *testing.T) { ms, ts := blockTestServer(t) @@ -180,6 +253,73 @@ func TestBlockVolumeDeleteHandler(t *testing.T) { } } +func TestBlockVolumeListHandler_ReflectsCoreInfluencedDegradedConsume(t *testing.T) { + ms, ts := blockTestServer(t) + bs := newTestBlockServiceDirect(t) + path := createTestVolDirect(t, bs, "vol-handler-degraded") + + errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{{ + Path: path, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + LeaseTtlMs: 30000, + ReplicaServerID: "replica-server:8080", + ReplicaDataAddr: "10.0.0.2:4260", + ReplicaCtrlAddr: "10.0.0.2:4261", + }}) + if len(errs) != 1 || errs[0] != nil { + t.Fatalf("apply assignment errs=%v", errs) + } + bs.applyCoreEvent(engine.BarrierRejected{ID: path, Reason: "barrier_timeout"}) + + hb := findHeartbeatMsg(bs.CollectBlockVolumeHeartbeat(), path) + if hb == nil { + t.Fatal("heartbeat volume missing") + } + + ms.blockRegistry.MarkBlockCapable("primary-server:8080") + if err := ms.blockRegistry.Register(&BlockVolumeEntry{ + Name: "vol-handler-degraded", + VolumeServer: "primary-server:8080", + Path: path, + Status: StatusActive, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + ReplicaFactor: 2, + Replicas: []ReplicaInfo{{ + Server: "replica-server:8080", + Path: "/blocks/vol-handler-degraded-replica.blk", + Ready: true, + }}, + }); err != nil { + t.Fatalf("register: %v", err) + } + ms.blockRegistry.UpdateFullHeartbeat("primary-server:8080", blockvol.InfoMessagesToProto([]blockvol.BlockVolumeInfoMessage{*hb}), "") + + resp, err := http.Get(ts.URL + "/block/volumes") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + + var infos []blockapi.VolumeInfo + if err := json.NewDecoder(resp.Body).Decode(&infos); err != nil { + t.Fatal(err) + } + if len(infos) != 1 { + t.Fatalf("expected 1 volume, got %d", len(infos)) + } + info := infos[0] + if !info.ReplicaDegraded { + t.Fatalf("expected outward ReplicaDegraded=true, info=%+v", info) + } + if info.VolumeMode != "degraded" { + t.Fatalf("expected outward VolumeMode=degraded, got %q", info.VolumeMode) + } +} + func TestBlockAssignHandler(t *testing.T) { ms, ts := blockTestServer(t) diff --git a/weed/server/qa_block_convergence_test.go b/weed/server/qa_block_convergence_test.go index 272fef20d..ac79e1bb4 100644 --- a/weed/server/qa_block_convergence_test.go +++ b/weed/server/qa_block_convergence_test.go @@ -48,6 +48,7 @@ func createP2BlockService(t *testing.T) (*BlockService, string) { localServerID: "vs1-node:18080", v2Bridge: v2bridge.NewControlBridge(), v2Orchestrator: engine.NewRecoveryOrchestrator(), + v2Core: engine.NewCoreEngine(), replStates: make(map[string]*volReplState), } bs.v2Recovery = NewRecoveryManager(bs) diff --git a/weed/server/qa_block_idempotence_test.go b/weed/server/qa_block_idempotence_test.go index 6e48d9174..cfd95b91a 100644 --- a/weed/server/qa_block_idempotence_test.go +++ b/weed/server/qa_block_idempotence_test.go @@ -47,6 +47,7 @@ func createP3BlockService(t *testing.T) (*BlockService, string) { localServerID: "vs1-node:18080", v2Bridge: v2bridge.NewControlBridge(), v2Orchestrator: engine.NewRecoveryOrchestrator(), + v2Core: engine.NewCoreEngine(), replStates: make(map[string]*volReplState), } bs.v2Recovery = NewRecoveryManager(bs) diff --git a/weed/server/qa_block_soak_test.go b/weed/server/qa_block_soak_test.go index 3e873cddc..7d685f386 100644 --- a/weed/server/qa_block_soak_test.go +++ b/weed/server/qa_block_soak_test.go @@ -95,6 +95,7 @@ func newSoakSetup(t *testing.T) *soakSetup { advertisedHost: "127.0.0.1", v2Bridge: v2bridge.NewControlBridge(), v2Orchestrator: engine.NewRecoveryOrchestrator(), + v2Core: engine.NewCoreEngine(), replStates: make(map[string]*volReplState), } bs.v2Recovery = NewRecoveryManager(bs) @@ -156,6 +157,7 @@ func TestCP13_2_BlockService_AdvertisedHost_NotOpaqueID(t *testing.T) { advertisedHost: "10.0.0.42", // routable v2Bridge: v2bridge.NewControlBridge(), v2Orchestrator: engine.NewRecoveryOrchestrator(), + v2Core: engine.NewCoreEngine(), replStates: make(map[string]*volReplState), } diff --git a/weed/server/volume_server_block.go b/weed/server/volume_server_block.go index ceb46fc8c..7fca2286c 100644 --- a/weed/server/volume_server_block.go +++ b/weed/server/volume_server_block.go @@ -9,9 +9,9 @@ import ( "strings" "sync" + engine "github.com/seaweedfs/seaweedfs/sw-block/engine/replication" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/storage" - engine "github.com/seaweedfs/seaweedfs/sw-block/engine/replication" "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/iscsi" "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/nvme" @@ -23,17 +23,22 @@ type volReplState struct { replicaDataAddr string replicaCtrlAddr string // allReplicas stores the full replica set for multi-replica idempotence. - allReplicas []blockvol.ReplicaAddr - roleApplied bool - receiverReady bool + allReplicas []blockvol.ReplicaAddr + roleApplied bool + receiverReady bool shipperConfigured bool - replicaEligible bool - publishHealthy bool + replicaEligible bool + publishHealthy bool } // BlockReadinessSnapshot names the assignment-to-publication closure at the // BlockService boundary. These flags are owned by the service/adapter layer, // not by blockvol's local storage mechanics. +// +// Important: +// PublishHealthy here is still an adapter-local publication bit used by current +// `weed/server` surfaces. It is NOT the semantic owner for Phase 14 core +// publication health; that owner is `engine.PublicationView`. type BlockReadinessSnapshot struct { RoleApplied bool ReceiverReady bool @@ -70,7 +75,12 @@ type BlockService struct { // V2 engine bridge (Phase 08 P1). v2Bridge *v2bridge.ControlBridge v2Orchestrator *engine.RecoveryOrchestrator + v2Core *engine.CoreEngine v2Recovery *RecoveryManager + coreProjMu sync.RWMutex + coreProj map[string]engine.PublicationProjection + coreExecMu sync.RWMutex + coreExec map[string][]string // P3: last-applied assignment per volume path for idempotence. lastAssignMu sync.RWMutex @@ -92,6 +102,63 @@ func (bs *BlockService) V2Orchestrator() *engine.RecoveryOrchestrator { return bs.v2Orchestrator } +// V2Core returns the explicit Phase 14/15 core shell if wired. +func (bs *BlockService) V2Core() *engine.CoreEngine { + return bs.v2Core +} + +// CoreProjection returns the latest adapter-cached projection emitted by the +// explicit V2 core on the narrow live path. +func (bs *BlockService) CoreProjection(path string) (engine.PublicationProjection, bool) { + bs.coreProjMu.RLock() + defer bs.coreProjMu.RUnlock() + if bs.coreProj == nil { + return engine.PublicationProjection{}, false + } + proj, ok := bs.coreProj[path] + return proj, ok +} + +// ExecutedCoreCommands returns the bounded list of core commands executed on the +// current integrated path for one volume. Intended for focused runtime-ownership +// proofs in Phase 16. +func (bs *BlockService) ExecutedCoreCommands(path string) []string { + bs.coreExecMu.RLock() + defer bs.coreExecMu.RUnlock() + if bs.coreExec == nil { + return nil + } + cmds := bs.coreExec[path] + out := make([]string, len(cmds)) + copy(out, cmds) + return out +} + +// CoreProjectionMismatches reports fields that should already agree on the +// narrow Phase 15A path but do not. It intentionally excludes adapter-local +// `PublishHealthy`, which is not yet rebound to the core publication owner. +func (bs *BlockService) CoreProjectionMismatches(path string) []string { + proj, ok := bs.CoreProjection(path) + if !ok { + return []string{"missing_core_projection"} + } + readiness := bs.ReadinessSnapshot(path) + var mismatches []string + if readiness.RoleApplied != proj.Readiness.RoleApplied { + mismatches = append(mismatches, "role_applied") + } + if readiness.ReceiverReady != proj.Readiness.ReceiverReady { + mismatches = append(mismatches, "receiver_ready") + } + if readiness.ShipperConfigured != proj.Readiness.ShipperConfigured { + mismatches = append(mismatches, "shipper_configured") + } + if readiness.ShipperConnected != proj.Readiness.ShipperConnected { + mismatches = append(mismatches, "shipper_connected") + } + return mismatches +} + // SetServerID sets the stable server identity for V2 control semantics. // This may be an opaque string (from -id flag) — not guaranteed routable. func (bs *BlockService) SetServerID(id string) { @@ -144,7 +211,9 @@ func StartBlockService(listenAddr, blockDir, iqnPrefix, portalAddr string, nvmeC nvmeListenAddr: nvmeCfg.ListenAddr, v2Bridge: v2bridge.NewControlBridge(), v2Orchestrator: engine.NewRecoveryOrchestrator(), + v2Core: engine.NewCoreEngine(), localServerID: listenAddr, // INTERIM: transport-shaped, see field doc + coreProj: make(map[string]engine.PublicationProjection), } bs.v2Recovery = NewRecoveryManager(bs) @@ -430,37 +499,17 @@ func (bs *BlockService) ApplyAssignments(assignments []blockvol.BlockVolumeAssig } for i, a := range assignments { role := blockvol.RoleFromWire(a.Role) - ttl := blockvol.LeaseTTLFromWire(a.LeaseTtlMs) - - // 1. Apply role/epoch/lease. - if err := bs.blockStore.WithVolume(a.Path, func(vol *blockvol.BlockVol) error { - return vol.HandleAssignment(a.Epoch, role, ttl) - }); err != nil { + bs.recordAppliedAssignment(a) + if err := bs.applyCoreAssignmentEvent(a); err != nil { errs[i] = err glog.Warningf("block service: assignment %s epoch=%d role=%s: %v", a.Path, a.Epoch, role, err) continue } - bs.noteRoleApplied(a.Path, role) // 2. Replication setup based on role + addresses. switch role { case blockvol.RolePrimary: - // CP8-2: ReplicaAddrs (multi-replica) takes precedence over scalar fields. - if len(a.ReplicaAddrs) > 0 { - if err := bs.setupPrimaryReplicationMulti(a.Path, a.ReplicaAddrs); err != nil { - errs[i] = err - } - } else if a.ReplicaDataAddr != "" && a.ReplicaCtrlAddr != "" { - if err := bs.setupPrimaryReplication(a.Path, a.ReplicaDataAddr, a.ReplicaCtrlAddr); err != nil { - errs[i] = err - } - } case blockvol.RoleReplica: - if a.ReplicaDataAddr != "" && a.ReplicaCtrlAddr != "" { - if err := bs.setupReplicaReceiver(a.Path, a.ReplicaDataAddr, a.ReplicaCtrlAddr); err != nil { - errs[i] = err - } - } case blockvol.RoleRebuilding: if a.RebuildAddr != "" { bs.startRebuild(a.Path, a.RebuildAddr, a.Epoch) @@ -470,6 +519,263 @@ func (bs *BlockService) ApplyAssignments(assignments []blockvol.BlockVolumeAssig return errs } +func (bs *BlockService) applyCoreAssignmentEvent(a blockvol.BlockVolumeAssignment) error { + if bs == nil || bs.v2Core == nil { + return bs.applyRoleAssignment(a) + } + ev, ok := bs.coreAssignmentEvent(a) + if !ok { + return nil + } + result := bs.v2Core.ApplyEvent(ev) + return bs.applyCoreCommandsWithAssignment(result.Commands, &a) +} + +func (bs *BlockService) applyCoreEvent(ev engine.Event) { + if bs == nil || bs.v2Core == nil { + return + } + result := bs.v2Core.ApplyEvent(ev) + bs.applyCoreCommands(result.Commands) +} + +func (bs *BlockService) applyCoreCommands(cmds []engine.Command) { + _ = bs.applyCoreCommandsWithAssignment(cmds, nil) +} + +func (bs *BlockService) applyCoreCommandsWithAssignment(cmds []engine.Command, assignment *blockvol.BlockVolumeAssignment) error { + for _, cmd := range cmds { + switch v := cmd.(type) { + case engine.ApplyRoleCommand: + if err := bs.executeApplyRoleCommand(v, assignment); err != nil { + return err + } + case engine.StartReceiverCommand: + if err := bs.executeStartReceiverCommand(v, assignment); err != nil { + return err + } + case engine.ConfigureShipperCommand: + if err := bs.executeConfigureShipperCommand(v); err != nil { + return err + } + case engine.InvalidateSessionCommand: + if err := bs.executeInvalidateSessionCommand(v); err != nil { + return err + } + case engine.StartCatchUpCommand: + if err := bs.executeStartCatchUpCommand(v); err != nil { + return err + } + case engine.StartRebuildCommand: + if err := bs.executeStartRebuildCommand(v); err != nil { + return err + } + case engine.PublishProjectionCommand: + proj := v.Projection + if latest, ok := bs.V2Core().Projection(v.VolumeID); ok { + proj = latest + } + bs.coreProjMu.Lock() + if bs.coreProj == nil { + bs.coreProj = make(map[string]engine.PublicationProjection) + } + bs.coreProj[v.VolumeID] = proj + bs.coreProjMu.Unlock() + } + } + return nil +} + +func (bs *BlockService) executeApplyRoleCommand(cmd engine.ApplyRoleCommand, assignment *blockvol.BlockVolumeAssignment) error { + if assignment == nil { + return nil + } + if assignment.Path != cmd.VolumeID { + return fmt.Errorf("block service: core apply_role path mismatch %q != %q", assignment.Path, cmd.VolumeID) + } + if err := bs.applyRoleAssignment(*assignment); err != nil { + return err + } + bs.recordExecutedCoreCommand(cmd.VolumeID, "apply_role") + return nil +} + +func (bs *BlockService) executeStartReceiverCommand(cmd engine.StartReceiverCommand, assignment *blockvol.BlockVolumeAssignment) error { + if assignment == nil { + return nil + } + if assignment.Path != cmd.VolumeID { + return fmt.Errorf("block service: core start_receiver path mismatch %q != %q", assignment.Path, cmd.VolumeID) + } + if assignment.ReplicaDataAddr == "" || assignment.ReplicaCtrlAddr == "" { + return nil + } + if err := bs.setupReplicaReceiver(assignment.Path, assignment.ReplicaDataAddr, assignment.ReplicaCtrlAddr); err != nil { + return err + } + bs.recordExecutedCoreCommand(cmd.VolumeID, "start_receiver") + bs.applyCoreEvent(engine.ReceiverReadyObserved{ID: cmd.VolumeID}) + return nil +} + +func (bs *BlockService) executeConfigureShipperCommand(cmd engine.ConfigureShipperCommand) error { + addrs := make([]blockvol.ReplicaAddr, 0, len(cmd.Replicas)) + for _, replica := range cmd.Replicas { + if replica.Endpoint.DataAddr == "" || replica.Endpoint.CtrlAddr == "" { + continue + } + addrs = append(addrs, blockvol.ReplicaAddr{ + ServerID: replica.ReplicaID, + DataAddr: replica.Endpoint.DataAddr, + CtrlAddr: replica.Endpoint.CtrlAddr, + }) + } + if len(addrs) == 0 { + return nil + } + if len(addrs) == 1 { + if err := bs.setupPrimaryReplication(cmd.VolumeID, addrs[0].DataAddr, addrs[0].CtrlAddr); err != nil { + return err + } + } else { + if err := bs.setupPrimaryReplicationMulti(cmd.VolumeID, addrs); err != nil { + return err + } + } + bs.recordExecutedCoreCommand(cmd.VolumeID, "configure_shipper") + bs.applyCoreEvent(engine.ShipperConfiguredObserved{ID: cmd.VolumeID}) + if bs.isPrimaryShipperConnected(cmd.VolumeID) { + bs.applyCoreEvent(engine.ShipperConnectedObserved{ID: cmd.VolumeID}) + } + return nil +} + +func (bs *BlockService) executeInvalidateSessionCommand(cmd engine.InvalidateSessionCommand) error { + if bs == nil || bs.v2Orchestrator == nil || bs.v2Core == nil { + return nil + } + proj, ok := bs.v2Core.Projection(cmd.VolumeID) + if !ok { + return nil + } + for _, replicaID := range proj.ReplicaIDs { + sender := bs.v2Orchestrator.Registry.Sender(replicaID) + if sender == nil { + continue + } + sender.InvalidateSession(cmd.Reason, engine.StateDisconnected) + } + bs.recordExecutedCoreCommand(cmd.VolumeID, "invalidate_session") + return nil +} + +func (bs *BlockService) executeStartCatchUpCommand(cmd engine.StartCatchUpCommand) error { + if bs == nil || bs.v2Recovery == nil { + return nil + } + if err := bs.v2Recovery.ExecutePendingCatchUp(cmd.VolumeID, cmd.TargetLSN); err != nil { + return err + } + bs.recordExecutedCoreCommand(cmd.VolumeID, "start_catchup") + return nil +} + +func (bs *BlockService) executeStartRebuildCommand(cmd engine.StartRebuildCommand) error { + if bs == nil || bs.v2Recovery == nil { + return nil + } + if err := bs.v2Recovery.ExecutePendingRebuild(cmd.VolumeID, cmd.TargetLSN); err != nil { + return err + } + bs.recordExecutedCoreCommand(cmd.VolumeID, "start_rebuild") + return nil +} + +func (bs *BlockService) applyRoleAssignment(a blockvol.BlockVolumeAssignment) error { + if bs == nil || bs.blockStore == nil { + return nil + } + role := blockvol.RoleFromWire(a.Role) + ttl := blockvol.LeaseTTLFromWire(a.LeaseTtlMs) + if err := bs.blockStore.WithVolume(a.Path, func(vol *blockvol.BlockVol) error { + return vol.HandleAssignment(a.Epoch, role, ttl) + }); err != nil { + return err + } + bs.noteRoleApplied(a.Path, role) + bs.applyCoreEvent(engine.RoleApplied{ID: a.Path}) + return nil +} + +func (bs *BlockService) recordExecutedCoreCommand(path, name string) { + bs.coreExecMu.Lock() + defer bs.coreExecMu.Unlock() + if bs.coreExec == nil { + bs.coreExec = make(map[string][]string) + } + bs.coreExec[path] = append(bs.coreExec[path], name) +} + +func (bs *BlockService) coreAssignmentEvent(a blockvol.BlockVolumeAssignment) (engine.AssignmentDelivered, bool) { + role := blockvol.RoleFromWire(a.Role) + ev := engine.AssignmentDelivered{ + ID: a.Path, + Epoch: a.Epoch, + } + switch role { + case blockvol.RolePrimary: + ev.Role = engine.RolePrimary + if len(a.ReplicaAddrs) > 0 { + ev.Replicas = make([]engine.ReplicaAssignment, 0, len(a.ReplicaAddrs)) + for _, ra := range a.ReplicaAddrs { + if ra.ServerID == "" { + continue + } + ev.Replicas = append(ev.Replicas, engine.ReplicaAssignment{ + ReplicaID: fmt.Sprintf("%s/%s", a.Path, ra.ServerID), + Endpoint: engine.Endpoint{ + DataAddr: ra.DataAddr, + CtrlAddr: ra.CtrlAddr, + }, + }) + } + } else if a.ReplicaServerID != "" && a.ReplicaDataAddr != "" { + ev.Replicas = []engine.ReplicaAssignment{{ + ReplicaID: fmt.Sprintf("%s/%s", a.Path, a.ReplicaServerID), + Endpoint: engine.Endpoint{ + DataAddr: a.ReplicaDataAddr, + CtrlAddr: a.ReplicaCtrlAddr, + }, + }} + } + return ev, true + case blockvol.RoleReplica: + ev.Role = engine.RoleReplica + ev.Replicas = []engine.ReplicaAssignment{{ + ReplicaID: fmt.Sprintf("%s/%s", a.Path, bs.localServerID), + Endpoint: engine.Endpoint{ + DataAddr: a.ReplicaDataAddr, + CtrlAddr: a.ReplicaCtrlAddr, + }, + }} + return ev, true + default: + return engine.AssignmentDelivered{}, false + } +} + +func (bs *BlockService) isPrimaryShipperConnected(path string) bool { + if bs == nil || bs.blockStore == nil { + return false + } + connected := false + _ = bs.blockStore.WithVolume(path, func(vol *blockvol.BlockVol) error { + connected = len(vol.ReplicaShipperStates()) > 0 && !vol.Status().ReplicaDegraded + return nil + }) + return connected +} + // setupPrimaryReplication configures WAL shipping from primary to replica // and starts the rebuild server (R1-2). func (bs *BlockService) setupPrimaryReplication(path, replicaDataAddr, replicaCtrlAddr string) error { @@ -732,11 +1038,9 @@ func (bs *BlockService) CollectBlockVolumeHeartbeat() []blockvol.BlockVolumeInfo defer bs.replMu.RUnlock() for i := range msgs { if s, ok := bs.replStates[msgs[i].Path]; ok { - if s.publishHealthy { - msgs[i].ReplicaDataAddr = s.replicaDataAddr - msgs[i].ReplicaCtrlAddr = s.replicaCtrlAddr - } + msgs[i].ReplicaDataAddr, msgs[i].ReplicaCtrlAddr = bs.heartbeatReplicaAddrs(msgs[i].Path, s) } + msgs[i].ReplicaDegraded = bs.heartbeatReplicaDegraded(msgs[i].Path, msgs[i].ReplicaDegraded) // NVMe publication: report nvme_addr and nqn if NVMe target is running. if bs.nvmeListenAddr != "" { msgs[i].NvmeAddr = bs.nvmeListenAddr @@ -749,6 +1053,48 @@ func (bs *BlockService) CollectBlockVolumeHeartbeat() []blockvol.BlockVolumeInfo return msgs } +// heartbeatReplicaAddrs returns the scalar replica transport addresses that +// should be exposed on the current heartbeat surface. On the Phase 15 live path +// it prefers the explicit core projection when present, while preserving the +// older adapter-local fallback for unrebound paths. +func (bs *BlockService) heartbeatReplicaAddrs(path string, state *volReplState) (string, string) { + if state == nil { + return "", "" + } + if proj, ok := bs.CoreProjection(path); ok { + switch proj.Role { + case engine.RolePrimary: + if proj.Readiness.ShipperConfigured { + return state.replicaDataAddr, state.replicaCtrlAddr + } + case engine.RoleReplica: + if proj.Readiness.ReceiverReady { + return state.replicaDataAddr, state.replicaCtrlAddr + } + } + return "", "" + } + if state.publishHealthy { + return state.replicaDataAddr, state.replicaCtrlAddr + } + return "", "" +} + +// heartbeatReplicaDegraded returns the bounded degraded bit for the current +// heartbeat surface. On the Phase 15 live path it prefers the core mode when +// present, then falls back to the runtime-local status bit. +func (bs *BlockService) heartbeatReplicaDegraded(path string, current bool) bool { + if proj, ok := bs.CoreProjection(path); ok { + switch proj.Mode.Name { + case engine.ModeDegraded, engine.ModeNeedsRebuild: + return true + default: + return false + } + } + return current +} + // multiReplicaUnchanged checks if the full replica set is unchanged. func (bs *BlockService) multiReplicaUnchanged(path string, addrs []blockvol.ReplicaAddr) bool { bs.replMu.RLock() @@ -849,7 +1195,9 @@ func (bs *BlockService) markReceiverReady(path, dataAddr, ctrlAddr string) { } // ReadinessSnapshot reports the service-owned assignment/readiness closure for -// one volume. It keeps v2 publication truth above blockvol's local mechanics. +// one volume. On the Phase 15 live path it prefers the explicit core projection +// for the aligned readiness subset, while `PublishHealthy` remains adapter-local +// until publication ownership is fully rebound. func (bs *BlockService) ReadinessSnapshot(path string) BlockReadinessSnapshot { snap := BlockReadinessSnapshot{} bs.replMu.RLock() @@ -863,12 +1211,26 @@ func (bs *BlockService) ReadinessSnapshot(path string) BlockReadinessSnapshot { } bs.replMu.RUnlock() if !snap.ShipperConfigured || bs.blockStore == nil { + if proj, ok := bs.CoreProjection(path); ok { + snap.RoleApplied = proj.Readiness.RoleApplied + snap.ReceiverReady = proj.Readiness.ReceiverReady + snap.ShipperConfigured = proj.Readiness.ShipperConfigured + snap.ShipperConnected = proj.Readiness.ShipperConnected + snap.ReplicaEligible = proj.Readiness.ReplicaReady + } return snap } _ = bs.blockStore.WithVolume(path, func(vol *blockvol.BlockVol) error { snap.ShipperConnected = len(vol.ReplicaShipperStates()) > 0 && !vol.Status().ReplicaDegraded return nil }) + if proj, ok := bs.CoreProjection(path); ok { + snap.RoleApplied = proj.Readiness.RoleApplied + snap.ReceiverReady = proj.Readiness.ReceiverReady + snap.ShipperConfigured = proj.Readiness.ShipperConfigured + snap.ShipperConnected = proj.Readiness.ShipperConnected + snap.ReplicaEligible = proj.Readiness.ReplicaReady + } return snap } diff --git a/weed/server/volume_server_block_debug.go b/weed/server/volume_server_block_debug.go index 7c5e16df4..42a2282ce 100644 --- a/weed/server/volume_server_block_debug.go +++ b/weed/server/volume_server_block_debug.go @@ -19,6 +19,7 @@ type ShipperDebugInfo struct { type BlockVolumeDebugInfo struct { Path string `json:"path"` Role string `json:"role"` + Mode string `json:"mode,omitempty"` Epoch uint64 `json:"epoch"` HeadLSN uint64 `json:"head_lsn"` Degraded bool `json:"degraded"` @@ -28,10 +29,45 @@ type BlockVolumeDebugInfo struct { ShipperConnected bool `json:"shipper_connected"` ReplicaEligible bool `json:"replica_eligible"` PublishHealthy bool `json:"publish_healthy"` + PublicationReason string `json:"publication_reason,omitempty"` Shippers []ShipperDebugInfo `json:"shippers,omitempty"` Timestamp string `json:"timestamp"` } +// DebugInfoForVolume returns the current debug surface for one volume. When the +// Phase 15 core projection exists on the live path, this surface prefers the +// core-owned projection truth over adapter-local convenience flags. +func (bs *BlockService) DebugInfoForVolume(path string, vol *blockvol.BlockVol) BlockVolumeDebugInfo { + status := vol.Status() + readiness := bs.ReadinessSnapshot(path) + info := BlockVolumeDebugInfo{ + Path: path, + Role: status.Role.String(), + Epoch: status.Epoch, + HeadLSN: status.WALHeadLSN, + Degraded: status.ReplicaDegraded, + RoleApplied: readiness.RoleApplied, + ReceiverReady: readiness.ReceiverReady, + ShipperConfigured: readiness.ShipperConfigured, + ShipperConnected: readiness.ShipperConnected, + ReplicaEligible: readiness.ReplicaEligible, + PublishHealthy: readiness.PublishHealthy, + Timestamp: time.Now().UTC().Format(time.RFC3339Nano), + } + if proj, ok := bs.CoreProjection(path); ok { + info.Role = string(proj.Role) + info.Mode = string(proj.Mode.Name) + info.RoleApplied = proj.Readiness.RoleApplied + info.ReceiverReady = proj.Readiness.ReceiverReady + info.ShipperConfigured = proj.Readiness.ShipperConfigured + info.ShipperConnected = proj.Readiness.ShipperConnected + info.ReplicaEligible = proj.Readiness.ReplicaReady + info.PublishHealthy = proj.Publication.Healthy + info.PublicationReason = proj.Publication.Reason + } + return info +} + // debugBlockShipperHandler returns real-time shipper state for all block volumes. // Unlike the master's replica_degraded (heartbeat-lagged), this reads directly // from the shipper's atomic state field — no heartbeat delay. @@ -53,22 +89,7 @@ func (vs *VolumeServer) debugBlockShipperHandler(w http.ResponseWriter, r *http. var infos []BlockVolumeDebugInfo store.IterateBlockVolumes(func(path string, vol *blockvol.BlockVol) { - status := vol.Status() - readiness := vs.blockService.ReadinessSnapshot(path) - info := BlockVolumeDebugInfo{ - Path: path, - Role: status.Role.String(), - Epoch: status.Epoch, - HeadLSN: status.WALHeadLSN, - Degraded: status.ReplicaDegraded, - RoleApplied: readiness.RoleApplied, - ReceiverReady: readiness.ReceiverReady, - ShipperConfigured: readiness.ShipperConfigured, - ShipperConnected: readiness.ShipperConnected, - ReplicaEligible: readiness.ReplicaEligible, - PublishHealthy: readiness.PublishHealthy, - Timestamp: time.Now().UTC().Format(time.RFC3339Nano), - } + info := vs.blockService.DebugInfoForVolume(path, vol) // Get per-shipper state from ShipperGroup if available. sg := vol.GetShipperGroup() diff --git a/weed/server/volume_server_block_test.go b/weed/server/volume_server_block_test.go index 44841cd73..6fca44fde 100644 --- a/weed/server/volume_server_block_test.go +++ b/weed/server/volume_server_block_test.go @@ -2,8 +2,10 @@ package weed_server import ( "path/filepath" + "reflect" "testing" + engine "github.com/seaweedfs/seaweedfs/sw-block/engine/replication" "github.com/seaweedfs/seaweedfs/weed/storage" "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" ) @@ -66,11 +68,14 @@ func newTestBlockServiceDirect(t *testing.T) *BlockService { store := storage.NewBlockVolumeStore() t.Cleanup(func() { store.Close() }) return &BlockService{ - blockStore: store, - blockDir: dir, - listenAddr: "0.0.0.0:3260", - iqnPrefix: "iqn.2024-01.com.seaweedfs:vol.", - replStates: make(map[string]*volReplState), + blockStore: store, + blockDir: dir, + listenAddr: "0.0.0.0:3260", + iqnPrefix: "iqn.2024-01.com.seaweedfs:vol.", + replStates: make(map[string]*volReplState), + v2Core: engine.NewCoreEngine(), + coreProj: make(map[string]engine.PublicationProjection), + localServerID: "vs-test", } } @@ -170,6 +175,623 @@ func TestBlockService_ProcessAssignment_WithReplicaAddrs(t *testing.T) { } } +func TestBlockService_ApplyAssignments_UpdatesCoreProjectionReplicaPath(t *testing.T) { + bs := newTestBlockServiceDirect(t) + path := createTestVolDirect(t, bs, "vol-core-replica") + + errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{ + { + Path: path, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RoleReplica), + LeaseTtlMs: 30000, + ReplicaDataAddr: "127.0.0.1:0", + ReplicaCtrlAddr: "127.0.0.1:0", + }, + }) + if len(errs) != 1 { + t.Fatalf("errs len=%d", len(errs)) + } + if errs[0] != nil { + t.Fatalf("apply assignment: %v", errs[0]) + } + + proj, ok := bs.CoreProjection(path) + if !ok { + t.Fatal("expected core projection to be cached on narrow live path") + } + if proj.Role != engine.RoleReplica { + t.Fatalf("role=%s", proj.Role) + } + if proj.Mode.Name != engine.ModeReplicaReady { + t.Fatalf("mode=%s", proj.Mode.Name) + } + if !proj.Readiness.RoleApplied || !proj.Readiness.ReceiverReady { + t.Fatalf("readiness=%+v", proj.Readiness) + } + if proj.Publication.Healthy { + t.Fatal("replica-ready narrow path must not overclaim publish healthy") + } + if proj.Publication.Reason != "replica_not_primary" { + t.Fatalf("publication_reason=%q", proj.Publication.Reason) + } + + coreProj, ok := bs.V2Core().Projection(path) + if !ok { + t.Fatal("expected core projection from explicit V2 core") + } + if !reflect.DeepEqual(proj, coreProj) { + t.Fatalf("adapter cache diverged from core projection:\ncache=%+v\ncore=%+v", proj, coreProj) + } + if mismatches := bs.CoreProjectionMismatches(path); len(mismatches) != 0 { + t.Fatalf("readiness/core mismatches=%v", mismatches) + } +} + +func TestBlockService_ApplyAssignments_UpdatesCoreProjectionPrimaryPath(t *testing.T) { + bs := newTestBlockServiceDirect(t) + path := createTestVolDirect(t, bs, "vol-core-primary") + + errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{ + { + Path: path, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + LeaseTtlMs: 30000, + ReplicaServerID: "vs-2", + ReplicaDataAddr: "10.0.0.2:4260", + ReplicaCtrlAddr: "10.0.0.2:4261", + }, + }) + if len(errs) != 1 { + t.Fatalf("errs len=%d", len(errs)) + } + if errs[0] != nil { + t.Fatalf("apply assignment: %v", errs[0]) + } + + proj, ok := bs.CoreProjection(path) + if !ok { + t.Fatal("expected core projection to be cached on narrow live path") + } + if proj.Role != engine.RolePrimary { + t.Fatalf("role=%s", proj.Role) + } + if proj.Readiness.RoleApplied != true { + t.Fatalf("readiness=%+v", proj.Readiness) + } + if !proj.Readiness.ShipperConfigured { + t.Fatalf("readiness=%+v", proj.Readiness) + } + if proj.Publication.Healthy { + t.Fatal("primary assignment without durable boundary must not overclaim publish healthy") + } + + coreProj, ok := bs.V2Core().Projection(path) + if !ok { + t.Fatal("expected core projection from explicit V2 core") + } + if !reflect.DeepEqual(proj, coreProj) { + t.Fatalf("adapter cache diverged from core projection:\ncache=%+v\ncore=%+v", proj, coreProj) + } + if mismatches := bs.CoreProjectionMismatches(path); len(mismatches) != 0 { + t.Fatalf("readiness/core mismatches=%v", mismatches) + } +} + +func TestBlockService_ApplyAssignments_RepeatedUnchangedStaysInSyncWithCore(t *testing.T) { + bs := newTestBlockServiceDirect(t) + path := createTestVolDirect(t, bs, "vol-core-repeat") + + assign := blockvol.BlockVolumeAssignment{ + Path: path, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + LeaseTtlMs: 30000, + ReplicaServerID: "vs-2", + ReplicaDataAddr: "10.0.0.2:4260", + ReplicaCtrlAddr: "10.0.0.2:4261", + } + + errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{assign}) + if len(errs) != 1 || errs[0] != nil { + t.Fatalf("first apply errs=%v", errs) + } + firstCache, ok := bs.CoreProjection(path) + if !ok { + t.Fatal("expected cached projection after first apply") + } + firstCore, ok := bs.V2Core().Projection(path) + if !ok { + t.Fatal("expected core projection after first apply") + } + if !reflect.DeepEqual(firstCache, firstCore) { + t.Fatalf("first cache/core mismatch:\ncache=%+v\ncore=%+v", firstCache, firstCore) + } + + errs = bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{assign}) + if len(errs) != 1 || errs[0] != nil { + t.Fatalf("second apply errs=%v", errs) + } + secondCache, ok := bs.CoreProjection(path) + if !ok { + t.Fatal("expected cached projection after second apply") + } + secondCore, ok := bs.V2Core().Projection(path) + if !ok { + t.Fatal("expected core projection after second apply") + } + if !reflect.DeepEqual(secondCache, secondCore) { + t.Fatalf("second cache/core mismatch:\ncache=%+v\ncore=%+v", secondCache, secondCore) + } + if !reflect.DeepEqual(firstCache, secondCache) { + t.Fatalf("unchanged assignment should not split cached projection:\nfirst=%+v\nsecond=%+v", firstCache, secondCache) + } + if mismatches := bs.CoreProjectionMismatches(path); len(mismatches) != 0 { + t.Fatalf("readiness/core mismatches=%v", mismatches) + } +} + +func TestBlockService_ApplyAssignments_ExecutesCoreCommands_PrimaryRoleApplyAndConfigureShipper(t *testing.T) { + bs := newTestBlockServiceDirect(t) + path := createTestVolDirect(t, bs, "vol-core-cmd-primary") + + assign := blockvol.BlockVolumeAssignment{ + Path: path, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + LeaseTtlMs: 30000, + ReplicaServerID: "vs-2", + ReplicaDataAddr: "10.0.0.2:4260", + ReplicaCtrlAddr: "10.0.0.2:4261", + } + + errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{assign}) + if len(errs) != 1 || errs[0] != nil { + t.Fatalf("apply errs=%v", errs) + } + + vol, ok := bs.blockStore.GetBlockVolume(path) + if !ok { + t.Fatal("volume not found") + } + status := vol.Status() + if status.Role != blockvol.RolePrimary || status.Epoch != 1 { + t.Fatalf("status=%+v", status) + } + if got := bs.ExecutedCoreCommands(path); !reflect.DeepEqual(got, []string{"apply_role", "configure_shipper"}) { + t.Fatalf("executed commands=%v", got) + } + + errs = bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{assign}) + if len(errs) != 1 || errs[0] != nil { + t.Fatalf("second apply errs=%v", errs) + } + if got := bs.ExecutedCoreCommands(path); !reflect.DeepEqual(got, []string{"apply_role", "configure_shipper"}) { + t.Fatalf("unchanged assignment should not re-execute command chain, got %v", got) + } +} + +func TestBlockService_ApplyAssignments_ExecutesCoreCommands_ReplicaRoleAndReceiver(t *testing.T) { + bs := newTestBlockServiceDirect(t) + path := createTestVolDirect(t, bs, "vol-core-cmd-replica") + + errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{{ + Path: path, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RoleReplica), + LeaseTtlMs: 30000, + ReplicaDataAddr: "127.0.0.1:0", + ReplicaCtrlAddr: "127.0.0.1:0", + }}) + if len(errs) != 1 || errs[0] != nil { + t.Fatalf("apply errs=%v", errs) + } + + vol, ok := bs.blockStore.GetBlockVolume(path) + if !ok { + t.Fatal("volume not found") + } + status := vol.Status() + if status.Role != blockvol.RoleReplica || status.Epoch != 1 { + t.Fatalf("status=%+v", status) + } + if got := bs.ExecutedCoreCommands(path); !reflect.DeepEqual(got, []string{"apply_role", "start_receiver"}) { + t.Fatalf("executed commands=%v", got) + } + readiness := bs.ReadinessSnapshot(path) + if !readiness.RoleApplied || !readiness.ReceiverReady { + t.Fatalf("readiness=%+v", readiness) + } +} + +func TestBlockService_BarrierRejected_ExecutesCoreInvalidateSession(t *testing.T) { + bs := newTestBlockServiceDirect(t) + bs.v2Bridge = newTestControlBridge() + bs.v2Orchestrator = newTestOrchestrator() + path := createTestVolDirect(t, bs, "vol-core-cmd-invalidate") + + errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{{ + Path: path, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + LeaseTtlMs: 30000, + ReplicaServerID: "vs-2", + ReplicaDataAddr: "10.0.0.2:4260", + ReplicaCtrlAddr: "10.0.0.2:4261", + }}) + if len(errs) != 1 || errs[0] != nil { + t.Fatalf("apply errs=%v", errs) + } + + replicaID := path + "/vs-2" + sender := bs.v2Orchestrator.Registry.Sender(replicaID) + if sender == nil { + t.Fatal("sender not found") + } + if !sender.HasActiveSession() { + t.Fatal("sender should start with active session") + } + + bs.applyCoreEvent(engine.BarrierRejected{ID: path, Reason: "timeout"}) + + if sender.HasActiveSession() { + t.Fatal("sender session should be invalidated by core command") + } + if got := bs.ExecutedCoreCommands(path); !reflect.DeepEqual(got, []string{"apply_role", "configure_shipper", "invalidate_session"}) { + t.Fatalf("executed commands=%v", got) + } +} + +func TestBlockService_BarrierRejected_DoesNotReexecuteInvalidateOnSameReason(t *testing.T) { + bs := newTestBlockServiceDirect(t) + bs.v2Bridge = newTestControlBridge() + bs.v2Orchestrator = newTestOrchestrator() + path := createTestVolDirect(t, bs, "vol-core-cmd-invalidate-repeat") + + errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{{ + Path: path, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + LeaseTtlMs: 30000, + ReplicaServerID: "vs-2", + ReplicaDataAddr: "10.0.0.2:4260", + ReplicaCtrlAddr: "10.0.0.2:4261", + }}) + if len(errs) != 1 || errs[0] != nil { + t.Fatalf("apply errs=%v", errs) + } + + bs.applyCoreEvent(engine.BarrierRejected{ID: path, Reason: "timeout"}) + first := bs.ExecutedCoreCommands(path) + bs.applyCoreEvent(engine.BarrierRejected{ID: path, Reason: "timeout"}) + second := bs.ExecutedCoreCommands(path) + + if !reflect.DeepEqual(first, []string{"apply_role", "configure_shipper", "invalidate_session"}) { + t.Fatalf("first executed commands=%v", first) + } + if !reflect.DeepEqual(second, first) { + t.Fatalf("repeated failure should not re-execute invalidate_session: first=%v second=%v", first, second) + } +} + +func TestBlockService_DebugInfoForVolume_UsesCoreProjectionPrimaryPath(t *testing.T) { + bs := newTestBlockServiceDirect(t) + path := createTestVolDirect(t, bs, "vol-debug-primary") + + errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{ + { + Path: path, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + LeaseTtlMs: 30000, + ReplicaServerID: "vs-2", + ReplicaDataAddr: "10.0.0.2:4260", + ReplicaCtrlAddr: "10.0.0.2:4261", + }, + }) + if len(errs) != 1 || errs[0] != nil { + t.Fatalf("apply assignment errs=%v", errs) + } + + readiness := bs.ReadinessSnapshot(path) + if !readiness.PublishHealthy { + t.Fatalf("expected adapter-local readiness to still report publish healthy, got %+v", readiness) + } + + vol, ok := bs.blockStore.GetBlockVolume(path) + if !ok { + t.Fatal("volume not found") + } + info := bs.DebugInfoForVolume(path, vol) + proj, ok := bs.CoreProjection(path) + if !ok { + t.Fatal("expected core projection for debug surface") + } + if info.Role != string(proj.Role) { + t.Fatalf("role=%q projection_role=%q", info.Role, proj.Role) + } + if info.Mode != string(proj.Mode.Name) { + t.Fatalf("mode=%q projection_mode=%q", info.Mode, proj.Mode.Name) + } + if info.PublishHealthy != proj.Publication.Healthy { + t.Fatalf("publish_healthy=%v projection=%v", info.PublishHealthy, proj.Publication.Healthy) + } + if info.PublicationReason != proj.Publication.Reason { + t.Fatalf("publication_reason=%q projection_reason=%q", info.PublicationReason, proj.Publication.Reason) + } + if info.PublishHealthy { + t.Fatalf("debug surface must not overclaim healthy on primary path without durable boundary: %+v", info) + } +} + +func TestBlockService_DebugInfoForVolume_UsesCoreProjectionReplicaPath(t *testing.T) { + bs := newTestBlockServiceDirect(t) + path := createTestVolDirect(t, bs, "vol-debug-replica") + + errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{ + { + Path: path, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RoleReplica), + LeaseTtlMs: 30000, + ReplicaDataAddr: "127.0.0.1:0", + ReplicaCtrlAddr: "127.0.0.1:0", + }, + }) + if len(errs) != 1 || errs[0] != nil { + t.Fatalf("apply assignment errs=%v", errs) + } + + vol, ok := bs.blockStore.GetBlockVolume(path) + if !ok { + t.Fatal("volume not found") + } + info := bs.DebugInfoForVolume(path, vol) + proj, ok := bs.CoreProjection(path) + if !ok { + t.Fatal("expected core projection for debug surface") + } + if info.Role != string(proj.Role) { + t.Fatalf("role=%q projection_role=%q", info.Role, proj.Role) + } + if info.Mode != string(proj.Mode.Name) { + t.Fatalf("mode=%q projection_mode=%q", info.Mode, proj.Mode.Name) + } + if info.ReceiverReady != proj.Readiness.ReceiverReady { + t.Fatalf("receiver_ready=%v projection=%v", info.ReceiverReady, proj.Readiness.ReceiverReady) + } + if info.ReplicaEligible != proj.Readiness.ReplicaReady { + t.Fatalf("replica_eligible=%v projection_ready=%v", info.ReplicaEligible, proj.Readiness.ReplicaReady) + } + if info.PublishHealthy != proj.Publication.Healthy { + t.Fatalf("publish_healthy=%v projection=%v", info.PublishHealthy, proj.Publication.Healthy) + } + if info.PublicationReason != proj.Publication.Reason { + t.Fatalf("publication_reason=%q projection_reason=%q", info.PublicationReason, proj.Publication.Reason) + } +} + +func TestBlockService_CollectBlockVolumeHeartbeat_PrimaryUsesCoreReadinessGate(t *testing.T) { + bs := newTestBlockServiceDirect(t) + path := createTestVolDirect(t, bs, "vol-heartbeat-primary") + + errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{ + { + Path: path, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + LeaseTtlMs: 30000, + ReplicaServerID: "vs-2", + ReplicaDataAddr: "10.0.0.2:4260", + ReplicaCtrlAddr: "10.0.0.2:4261", + }, + }) + if len(errs) != 1 || errs[0] != nil { + t.Fatalf("apply assignment errs=%v", errs) + } + + proj, ok := bs.CoreProjection(path) + if !ok { + t.Fatal("expected core projection for heartbeat surface") + } + if !proj.Readiness.ShipperConfigured { + t.Fatalf("projection readiness=%+v", proj.Readiness) + } + if proj.Publication.Healthy { + t.Fatalf("primary projection must still be publication-unhealthy here: %+v", proj.Publication) + } + + bs.replMu.Lock() + state := bs.replStates[path] + if state == nil { + bs.replMu.Unlock() + t.Fatal("missing repl state") + } + state.publishHealthy = false + bs.replMu.Unlock() + + msgs := bs.CollectBlockVolumeHeartbeat() + msg := findHeartbeatMsg(msgs, path) + if msg == nil { + t.Fatal("volume missing from heartbeat") + } + if msg.ReplicaDataAddr != "10.0.0.2:4260" { + t.Fatalf("ReplicaDataAddr=%q", msg.ReplicaDataAddr) + } + if msg.ReplicaCtrlAddr != "10.0.0.2:4261" { + t.Fatalf("ReplicaCtrlAddr=%q", msg.ReplicaCtrlAddr) + } +} + +func TestBlockService_CollectBlockVolumeHeartbeat_ReplicaUsesCoreReadinessGate(t *testing.T) { + bs := newTestBlockServiceDirect(t) + path := createTestVolDirect(t, bs, "vol-heartbeat-replica") + + errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{ + { + Path: path, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RoleReplica), + LeaseTtlMs: 30000, + ReplicaDataAddr: "127.0.0.1:0", + ReplicaCtrlAddr: "127.0.0.1:0", + }, + }) + if len(errs) != 1 || errs[0] != nil { + t.Fatalf("apply assignment errs=%v", errs) + } + + proj, ok := bs.CoreProjection(path) + if !ok { + t.Fatal("expected core projection for heartbeat surface") + } + if !proj.Readiness.ReceiverReady { + t.Fatalf("projection readiness=%+v", proj.Readiness) + } + if proj.Publication.Healthy { + t.Fatalf("replica projection must not own healthy publication: %+v", proj.Publication) + } + + bs.replMu.Lock() + state := bs.replStates[path] + if state == nil { + bs.replMu.Unlock() + t.Fatal("missing repl state") + } + expectedData := state.replicaDataAddr + expectedCtrl := state.replicaCtrlAddr + state.publishHealthy = false + bs.replMu.Unlock() + + msgs := bs.CollectBlockVolumeHeartbeat() + msg := findHeartbeatMsg(msgs, path) + if msg == nil { + t.Fatal("volume missing from heartbeat") + } + if msg.ReplicaDataAddr != expectedData { + t.Fatalf("ReplicaDataAddr=%q expected=%q", msg.ReplicaDataAddr, expectedData) + } + if msg.ReplicaCtrlAddr != expectedCtrl { + t.Fatalf("ReplicaCtrlAddr=%q expected=%q", msg.ReplicaCtrlAddr, expectedCtrl) + } +} + +func TestBlockService_ReadinessSnapshot_PrefersCoreProjectionPrimaryFields(t *testing.T) { + bs := newTestBlockServiceDirect(t) + path := createTestVolDirect(t, bs, "vol-readiness-primary") + + errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{ + { + Path: path, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RolePrimary), + LeaseTtlMs: 30000, + ReplicaServerID: "vs-2", + ReplicaDataAddr: "10.0.0.2:4260", + ReplicaCtrlAddr: "10.0.0.2:4261", + }, + }) + if len(errs) != 1 || errs[0] != nil { + t.Fatalf("apply assignment errs=%v", errs) + } + + bs.replMu.Lock() + state := bs.replStates[path] + if state == nil { + bs.replMu.Unlock() + t.Fatal("missing repl state") + } + state.roleApplied = false + state.shipperConfigured = false + state.publishHealthy = false + bs.replMu.Unlock() + + snap := bs.ReadinessSnapshot(path) + if !snap.RoleApplied { + t.Fatalf("expected snapshot role_applied from core projection, got %+v", snap) + } + if !snap.ShipperConfigured { + t.Fatalf("expected snapshot shipper_configured from core projection, got %+v", snap) + } + if snap.PublishHealthy { + t.Fatalf("publish_healthy should remain adapter-local on readiness snapshot, got %+v", snap) + } +} + +func TestBlockService_ReadinessSnapshot_PrefersCoreProjectionReplicaFields(t *testing.T) { + bs := newTestBlockServiceDirect(t) + path := createTestVolDirect(t, bs, "vol-readiness-replica") + + errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{ + { + Path: path, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RoleReplica), + LeaseTtlMs: 30000, + ReplicaDataAddr: "127.0.0.1:0", + ReplicaCtrlAddr: "127.0.0.1:0", + }, + }) + if len(errs) != 1 || errs[0] != nil { + t.Fatalf("apply assignment errs=%v", errs) + } + + bs.replMu.Lock() + state := bs.replStates[path] + if state == nil { + bs.replMu.Unlock() + t.Fatal("missing repl state") + } + state.receiverReady = false + state.replicaEligible = false + state.publishHealthy = false + bs.replMu.Unlock() + + snap := bs.ReadinessSnapshot(path) + if !snap.ReceiverReady { + t.Fatalf("expected snapshot receiver_ready from core projection, got %+v", snap) + } + if !snap.ReplicaEligible { + t.Fatalf("expected snapshot replica_eligible from core projection, got %+v", snap) + } + if snap.PublishHealthy { + t.Fatalf("publish_healthy should remain adapter-local on readiness snapshot, got %+v", snap) + } +} + +func TestBlockService_HeartbeatReplicaDegraded_UsesCoreMode(t *testing.T) { + bs := newTestBlockServiceDirect(t) + path := createTestVolDirect(t, bs, "vol-heartbeat-degraded") + + errs := bs.ApplyAssignments([]blockvol.BlockVolumeAssignment{ + { + Path: path, + Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RoleReplica), + LeaseTtlMs: 30000, + ReplicaDataAddr: "127.0.0.1:0", + ReplicaCtrlAddr: "127.0.0.1:0", + }, + }) + if len(errs) != 1 || errs[0] != nil { + t.Fatalf("apply assignment errs=%v", errs) + } + + bs.applyCoreEvent(engine.NeedsRebuildObserved{ID: path, Reason: "gap_too_large"}) + + proj, ok := bs.CoreProjection(path) + if !ok { + t.Fatal("expected core projection for heartbeat degraded mapping") + } + if proj.Mode.Name != engine.ModeNeedsRebuild { + t.Fatalf("mode=%s", proj.Mode.Name) + } + if !bs.heartbeatReplicaDegraded(path, false) { + t.Fatalf("expected degraded bit from core mode even when current=false, projection=%+v", proj) + } +} + func TestBlockService_HeartbeatIncludesReplicaAddrs(t *testing.T) { bs := newTestBlockServiceDirect(t) path := createTestVolDirect(t, bs, "vol1")