diff --git a/sw-block/.private/phase/phase-18-decisions.md b/sw-block/.private/phase/phase-18-decisions.md index 3c8e147ef..22a40cae8 100644 --- a/sw-block/.private/phase/phase-18-decisions.md +++ b/sw-block/.private/phase/phase-18-decisions.md @@ -1,7 +1,7 @@ # Phase 18 Decisions Date: 2026-04-05 -Status: active +Status: complete ## D1: Phase 18 Uses M1-M5 As The Main Spine @@ -142,3 +142,60 @@ Implication: 1. later work can attach RF2-facing product/runtime surfaces on top of a real continuity-bearing runtime slice 2. `M4` should attach one bounded surface without widening the continuity claim + +## D7: `M4` Closes On Compressed Surface Projection, Not New Truth Ownership + +Decision: + +1. `M4` is considered complete when at least one bounded RF2-facing + runtime/product surface is projected from the new runtime +2. the surface must be derived from runtime-owned failover, Loop 2, and + continuity observations +3. the surface must remain a compressed projection and must not become an + independent truth owner + +Why: + +1. after `M3`, the next meaningful closure is to let the runtime expose one + outward RF2-facing package +2. the new surface should prove that external/product-facing views can be bound + to the new runtime without moving semantic ownership out of the kernel/runtime +3. keeping the surface compressed preserves the authority split and prevents + frontend/backend code from silently redefining truth + +Implication: + +1. later product or operator APIs should reuse projected runtime surfaces instead + of inventing parallel truth models +2. `M5` should harden the supported envelope around this projected surface rather + than reopening kernel ownership + +## D8: `M5` Closes On Explicit Envelope And Explicit Non-Readiness + +Decision: + +1. `M5` is considered complete when the current `Phase 18` runtime-bearing path + has: + - one bounded productionization envelope + - one explicit review result + - one rebound pilot/preflight/stop/review artifact set +2. the current review result may explicitly be `block expansion` / `not + pilot-ready` +3. `M5` does not require the new runtime path to already be a working block + product + +Why: + +1. after `M1-M4`, the next needed closure is not more kernel proof; it is a clean + statement of what the current path does and does not justify operationally +2. the right productionization artifact set should reduce overclaiming, not hide + blockers +3. explicit non-readiness is better than silently reusing older chosen-path + pilot/launch language + +Implication: + +1. later work should widen from an explicit `not pilot-ready` baseline rather than + from ambiguous artifact inheritance +2. `Phase 18` is complete once the bounded envelope and review judgment are both + explicit diff --git a/sw-block/.private/phase/phase-18-log.md b/sw-block/.private/phase/phase-18-log.md index fdd9332dd..c6daae193 100644 --- a/sw-block/.private/phase/phase-18-log.md +++ b/sw-block/.private/phase/phase-18-log.md @@ -1,7 +1,7 @@ # Phase 18 Log Date: 2026-04-05 -Status: active +Status: complete ## 2026-04-05 @@ -132,3 +132,64 @@ Current interpretation: 2. this is still a bounded continuity claim on the in-process/runtime-owned path, not broad RF2 product continuity proof 3. the next active work should move to `M4` + +### `M4` Delivered + +Delivered in this update: + +1. one bounded RF2-facing runtime/product surface package now exists: + - `RF2VolumeSurface` + - `RF2SurfaceMode` + - `RF2ContinuityStatus` +2. the runtime manager now projects: + - active Loop 2 snapshot + - failover snapshot + - continuity snapshot + into one compressed outward RF2 surface +3. continuity results are now retained as runtime-owned observable snapshots: + - `ReplicatedContinuitySnapshot` + +Tests: + +1. `TestInProcessRuntimeManager_RF2VolumeSurface_HealthyPackage` +2. `TestInProcessRuntimeManager_RF2VolumeSurface_GatedPackage` +3. `go test ./sw-block/runtime/masterv2 ./sw-block/runtime/volumev2` + +Current interpretation: + +1. `M4` is complete as the first bounded RF2-facing runtime/product surface on + the new runtime +2. the surface remains a compressed projection of runtime-owned truth rather than + a new semantic owner +3. this is not a broad frontend/product approval or launch-readiness claim +4. the next active work should move to `M5` + +### `M5` Delivered + +Delivered in this update: + +1. one bounded productionization / launch envelope now exists for the current + `Phase 18` RF2 runtime-bearing path: + - `v2-rf2-runtime-bounded-envelope.md` +2. one explicit bounded review result now exists: + - `v2-rf2-runtime-bounded-envelope-review.md` + - current result: `block expansion` / `not pilot-ready` +3. the productionization artifact set was rebound onto the new runtime path: + - `v2-bounded-internal-pilot-pack.md` + - `v2-pilot-preflight-checklist.md` + - `v2-pilot-stop-conditions.md` + - `v2-controlled-rollout-review.md` + +Tests / review checks: + +1. document-only milestone; no new runtime code added +2. consistency review anchored on delivered `M1-M4` code/docs + +Current interpretation: + +1. `M5` is complete as a bounded productionization artifact set around the new + runtime path +2. the current judgment is explicitly: + - `block expansion` + - `not pilot-ready` +3. `Phase 18` is complete diff --git a/sw-block/.private/phase/phase-18.md b/sw-block/.private/phase/phase-18.md index 926c4513b..8d8c905aa 100644 --- a/sw-block/.private/phase/phase-18.md +++ b/sw-block/.private/phase/phase-18.md @@ -1,7 +1,7 @@ # Phase 18 Date: 2026-04-05 -Status: active +Status: complete Purpose: drive the new `masterv2 + volumev2 + purev2` kernel from the current in-process RF2 failover runtime slice toward a bounded productizable RF2 runtime in disciplined major milestones @@ -271,11 +271,30 @@ Exit criteria: Current status: -1. not started +1. delivered +2. one bounded RF2-facing runtime/product surface package now exists +3. the runtime manager now projects failover, active Loop 2, and continuity into + one compressed outward RF2 surface +4. current boundary: + - the surface is derived from runtime-owned snapshots/results only + - it does not become an independent truth owner or broad frontend/product + approval claim Review/test update: -1. pending +1. delivered code: + - `RF2VolumeSurface` + - `RF2SurfaceMode` + - `RF2ContinuityStatus` + - runtime-manager `RF2VolumeSurface(...)` +2. supporting runtime observability added: + - `ReplicatedContinuitySnapshot` + - runtime-manager continuity snapshot retention/accessors +3. delivered tests: + - healthy RF2 surface package + - gated RF2 surface package +4. result: + - one bounded RF2-facing runtime/product surface now exists on the new runtime ### `M5`: Productionization / Launch Envelope @@ -302,11 +321,30 @@ Exit criteria: Current status: -1. not started +1. delivered +2. one bounded productionization / launch envelope now exists around the + `Phase 18` RF2 runtime-bearing path +3. one explicit review result now exists: + - `block expansion` + - `not pilot-ready` +4. current boundary: + - the artifact set freezes the current support statement, exclusions, and + blockers + - it does not claim working block product readiness Review/test update: -1. pending +1. delivered docs: + - `v2-rf2-runtime-bounded-envelope.md` + - `v2-rf2-runtime-bounded-envelope-review.md` +2. rebound productionization artifacts: + - `v2-bounded-internal-pilot-pack.md` + - `v2-pilot-preflight-checklist.md` + - `v2-pilot-stop-conditions.md` + - `v2-controlled-rollout-review.md` +3. result: + - the new runtime path now has a bounded productionization artifact set with + explicit current judgment ## Initial Order @@ -323,10 +361,11 @@ written decision in `phase-18-decisions.md`. ## Current Focus -The active next work is: +`Phase 18` close-out: -1. `M4` seam step -2. attach one bounded RF2-facing runtime/product surface to the new runtime +1. `M1-M5` are now delivered +2. later work should widen from this point only through explicit new closure, + not by rereading `Phase 18` as working-product proof ## Review Base diff --git a/sw-block/.private/phase/phase-19-decisions.md b/sw-block/.private/phase/phase-19-decisions.md new file mode 100644 index 000000000..da684ff1e --- /dev/null +++ b/sw-block/.private/phase/phase-19-decisions.md @@ -0,0 +1,98 @@ +# Phase 19 Decisions + +Date: 2026-04-05 +Status: complete + +## D1: Keep Real Transport Ahead Of Auto Trigger + +Decision: + +1. `M6` must land before `M7` +2. live transport-backed runtime queries come before continuous Loop 2 service + and automatic failover trigger + +Why: + +1. the next main risk is hidden assumptions in live integration +2. auto-trigger is easier to overread if the runtime path is still partially + synthetic +3. the existing evidence seam is already explicit and is the safest next live + integration point + +Implication: + +1. `M7` should build on the real transport path from `M6` +2. if ordering changes later, the reason must be written explicitly + +## D2: Keep Frontend And CSI Downstream Of Real Runtime Proof + +Decision: + +1. frontend, CSI, and operator surface work stay downstream of live transport and + continuous runtime ownership +2. `M8-M10` must attach to runtime-owned truth rather than recreating control + ownership in adapters + +Why: + +1. the current RF2 surface projection pattern is already correct +2. product-facing integrations should reuse projected/runtime-owned truth instead + of defining parallel truth +3. attaching frontends too early risks hiding runtime gaps behind working local + adapters + +Implication: + +1. one real frontend may attach in `M8` +2. CSI and operator surfaces should wait until the working path is already real + +## D3: Keep The First Working Path Bounded To RF2 + +Decision: + +1. `Phase 19` is bounded to one working RF2 block path +2. `RF>2` remains outside the phase boundary + +Why: + +1. the main goal is to turn the proven RF2 kernel slice into one real serving + path +2. widening replication factor now would mix product expansion with live-path + closure + +Implication: + +1. each milestone should keep the claim bounded to RF2 +2. any broader productization work belongs to later phases + +## D4: `Phase 19` Closes On One Bounded Working Path, Not Broad Launch + +Decision: + +1. `Phase 19` is considered complete when one real bounded RF2 block path + exists with: + - live transport-backed evidence traffic + - continuous Loop 2 observation + - bounded auto failover + - runtime-managed frontend rebinding + - bounded repair/catch-up wrapper + - one end-to-end client handoff proof + - CSI/operator adapters over runtime-owned truth +2. `Phase 19` does not require broad launch approval or broad deployment matrix + proof + +Why: + +1. after `Phase 18`, the main objective is to prove a real working path rather + than continue only with structural/runtime proof +2. the correct next closure is one bounded user-serving path, not broad rollout + language +3. keeping the claim bounded preserves the same ownership discipline as + `Phase 18` + +Implication: + +1. later phases should focus on multi-process and pilot-ready closure rather than + redefining the kernel/runtime split again +2. `Phase 19` completion should be read as a working bounded path, not a launch + decision diff --git a/sw-block/.private/phase/phase-19-log.md b/sw-block/.private/phase/phase-19-log.md new file mode 100644 index 000000000..c45d37692 --- /dev/null +++ b/sw-block/.private/phase/phase-19-log.md @@ -0,0 +1,112 @@ +# Phase 19 Log + +Date: 2026-04-05 +Status: complete + +## 2026-04-05 + +### Start Of Phase + +Created the initial `Phase 19` control document. + +Starting point recorded: + +1. `Phase 18` is complete +2. the current runtime-bearing RF2 envelope is explicit +3. the current productionization judgment is explicit: + - `block expansion` + - `not pilot-ready` + +Initial execution rule: + +1. move by major milestone +2. keep the order `M6 -> M7 -> M8 -> M9 -> M10` +3. keep each milestone reviewable with healthy and fail-closed proofs + +Current next step: + +1. `M6` seam step for live transport-backed runtime queries + +### `M6` Delivered + +Delivered in this update: + +1. one live loopback HTTP evidence transport now exists +2. runtime registration can run on that live transport path +3. healthy and gated transport-backed failover tests now pass over that path + +Tests: + +1. `TestHTTPTransportEvidenceAdapter_HealthyFailoverFlow` +2. `TestHTTPTransportEvidenceAdapter_GatedFailoverFlow` +3. `go test ./sw-block/runtime/masterv2 ./sw-block/runtime/volumev2` + +### `M7` Delivered + +Delivered in this update: + +1. one background Loop 2 service now exists +2. one bounded auto-failover service now exists +3. RF2 outward surfaces can now refresh from continuous runtime activity + +Tests: + +1. `TestInProcessRuntimeManager_Loop2Service_RefreshesRF2Surface` +2. `TestInProcessRuntimeManager_AutoFailoverService_TriggersOnPrimaryLoss` +3. `TestInProcessRuntimeManager_AutoFailoverService_DoesNotTriggerOnCatchingUpReplica` +4. `go test ./sw-block/runtime/masterv2 ./sw-block/runtime/volumev2` + +### `M8` Delivered + +Delivered in this update: + +1. one runtime-managed iSCSI export path now exists +2. one bounded replica repair wrapper now exists +3. the runtime can now rebind service and repair a lagging replica without + moving truth ownership out of `volumev2` + +Tests: + +1. `TestInProcessRuntimeManager_ExportVolumeISCSI_BindsFrontendToRuntimeNode` +2. `TestInProcessRuntimeManager_RepairReplicaFromPrimary_ReturnsLoop2ToHealthy` +3. `go test ./sw-block/runtime/masterv2 ./sw-block/runtime/volumev2` + +### `M9` Delivered + +Delivered in this update: + +1. one end-to-end RF2 handoff proof now exists with: + - live transport + - runtime-managed frontend + - automatic failover + - reconnect and continued I/O on the new primary +2. one gated handoff counterproof now stops fail-closed + +Tests: + +1. `TestInProcessRuntimeManager_EndToEndRF2Handoff_ContinuesIOOnNewPrimary` +2. `TestInProcessRuntimeManager_EndToEndRF2Handoff_GatedReplicaStopsFailClosed` +3. `go test ./sw-block/runtime/masterv2 ./sw-block/runtime/volumev2` + +### `M10` Delivered + +Delivered in this update: + +1. one bounded HTTP operator surface now exists over runtime-owned views +2. one bounded CSI runtime backend adapter now exists over runtime-owned export + truth +3. CSI create/lookup/publish can now read from the V2 runtime path on the + bounded adapter path + +Tests: + +1. `TestInProcessRuntimeManager_OperatorSurface_ExposesRuntimeOwnedViews` +2. `TestV2RuntimeBackend_CreateLookupAndPublish` +3. `go test ./sw-block/runtime/masterv2 ./sw-block/runtime/volumev2 ./weed/storage/blockvol/csi` + +Current interpretation: + +1. `Phase 19` is complete as one bounded working RF2 block path +2. this is still a bounded working path on the current runtime harness, not broad + launch approval +3. the next major work should focus on multi-process / pilot-ready closure diff --git a/sw-block/.private/phase/phase-19.md b/sw-block/.private/phase/phase-19.md new file mode 100644 index 000000000..36ce11fc6 --- /dev/null +++ b/sw-block/.private/phase/phase-19.md @@ -0,0 +1,288 @@ +# Phase 19 + +Date: 2026-04-05 +Status: complete +Purpose: turn the delivered `Phase 18` RF2 runtime-bearing kernel slice into one +real working RF2 block path without collapsing the ownership split + +## Why This Phase Exists + +`Phase 18` closed the kernel/runtime proof stack: + +1. failover-time evidence crosses an explicit seam +2. active Loop 2 observation exists +3. bounded continuity through handoff exists +4. one RF2-facing outward surface exists +5. one bounded productionization envelope now exists with explicit non-readiness + +That is enough to stop `Phase 18`. + +It is not enough to claim a working RF2 block product. + +The next work is now narrower and more mechanical: + +1. make the transport path real +2. make Loop 2 continuously active +3. trigger failover from runtime-owned signals +4. attach real frontend and rebuild/catch-up lifecycle wiring +5. prove one end-to-end serving path +6. bind CSI and operator surfaces on top of runtime-owned truth + +## Entry Checkpoint + +`Phase 19` starts from the completed `Phase 18` boundary: + +1. `masterv2` is explicit identity/promotion authority +2. `volumev2` owns failover, takeover, Loop 2 observation, continuity, and RF2 + surface projection +3. failover evidence already crosses an explicit adapter seam +4. the productionization envelope already says: + - `block expansion` + - `not pilot-ready` + +Entry interpretation: + +1. the authority split is already stable enough +2. the next main risk is live integration, not protocol rediscovery +3. later work should widen from this checkpoint rather than redefine it + +## Phase Goal + +Produce one bounded post-entry sequence where: + +1. runtime participants communicate through a real transport path +2. Loop 2 is continuously meaningful +3. failover can trigger automatically from bounded runtime-owned signals +4. one real frontend path works on the new runtime +5. one degraded replica can return to healthy through V2-owned orchestration +6. one end-to-end RF2 handoff path serves real client I/O +7. CSI and operator surfaces attach without becoming truth owners + +## Scope + +### In scope + +1. real transport-backed failover-time evidence path +2. continuous Loop 2 service on the runtime path +3. bounded auto-failover trigger +4. runtime-managed frontend binding +5. bounded rebuild/catch-up orchestration on the runtime path +6. one end-to-end RF2 handoff proof +7. CSI rebinding and operator surface attachment + +### Out of scope + +1. reopening the kernel ownership split +2. broad `RF>2` product closure +3. broad transport/frontend matrix approval +4. broad launch approval +5. silent fallback to legacy mixed ownership as the truth source + +## Working Rules + +`Phase 19` should continue the `Phase 18` discipline: + +1. work by major milestones, not ad hoc rewiring +2. each major milestone should normally close in `2-3` implementation steps +3. each milestone must keep a healthy proof and a fail-closed counterproof +4. frontend, CSI, and operator surfaces must remain projections/integrations over + runtime truth, not new truth owners + +After each major milestone: + +1. update this phase file status +2. update `phase-19-log.md` +3. update `phase-19-decisions.md` if ordering or boundaries change +4. update review-base docs if the claim boundary changes + +## Phase 19 Major Milestones + +### `M6`: Live Transport-Backed RF2 Runtime Queries + +Goal: + +1. replace the current in-memory failover-time evidence path with one real + transport-backed runtime path + +Planned steps: + +1. seam step: + add one real transport implementation behind the existing evidence adapter + seam +2. runtime step: + make runtime registration and resolution use that live transport path +3. closure step: + prove healthy and gated 2-node failover through the live transport path + +Exit criteria: + +1. promotion evidence and replica summaries cross a live transport path +2. failover session/manager observability still survives +3. takeover authority does not move out of the selected primary + +Current status: + +1. delivered +2. one live loopback HTTP transport now exists behind the evidence seam +3. healthy and gated 2-node failover tests now pass through that live transport + +### `M7`: Continuous Loop 2 Service And Auto Failover Trigger + +Goal: + +1. turn Loop 2 into a continuously active runtime service and allow bounded + automatic failover on top of it + +Planned steps: + +1. seam step: + add a background Loop 2 service over the current observation slice +2. runtime step: + attach a bounded auto-failover trigger to explicit liveness/runtime signals +3. closure step: + prove healthy trigger behavior and fail-closed suppression + +Exit criteria: + +1. Loop 2 no longer depends on ad hoc `ObserveOnce()` calls +2. auto failover is downstream of honest observation and liveness +3. RF2 surfaces refresh from continuous runtime ownership + +Current status: + +1. delivered +2. one bounded background Loop 2 service now exists +3. one bounded auto-failover service now triggers on explicit primary evidence + loss and suppresses ambiguous runtime states + +### `M8`: Frontend And Rebuild/Catch-Up Wiring + +Goal: + +1. bind a real serving path and a bounded recovery lifecycle to the new runtime + +Planned steps: + +1. seam step: + attach one real frontend to the runtime-managed primary path +2. runtime step: + add bounded rebuild/catch-up orchestration around existing execution pieces +3. closure step: + prove return-to-healthy from one degraded state + +Exit criteria: + +1. one real frontend serves from the V2 runtime path +2. one degraded replica can return to healthy +3. rebuild/catch-up remain V2-orchestrated + +Current status: + +1. delivered +2. one runtime-managed iSCSI export path now exists +3. one bounded replica repair wrapper now returns a lagging replica to healthy + +### `M9`: End-To-End Working RF2 Block Path Proof + +Goal: + +1. prove the first real user story: + create volume -> serve I/O -> lose primary -> continue service on the new + primary + +Planned steps: + +1. seam step: + build a 2-node end-to-end harness on the new runtime path +2. runtime step: + execute the real handoff path with live serving and bounded client I/O +3. closure step: + add a gated counterproof that stops safely + +Exit criteria: + +1. one real end-to-end RF2 handoff path exists +2. the path uses real transport and real serving, not only in-process + composition + +Current status: + +1. delivered +2. one real client path now proves: + - write through runtime-managed frontend + - lose primary + - auto fail over + - reconnect to new primary + - continue I/O +3. one gated handoff counterproof now stops fail-closed + +### `M10`: CSI Rebinding And Operator Surface + +Goal: + +1. attach CSI and operator-facing surfaces on top of the proven runtime path + +Planned steps: + +1. seam step: + rebind CSI lifecycle integration to the new runtime-bearing path +2. runtime step: + expose bounded operator-facing surfaces from runtime-owned truth +3. closure step: + add integration checks for CSI and operator visibility + +Exit criteria: + +1. CSI and operator surfaces sit on top of V2 runtime truth +2. no new product/API surface becomes a hidden truth owner + +Current status: + +1. delivered +2. one bounded HTTP operator surface now exposes runtime-owned views +3. one bounded CSI runtime backend adapter now creates/looks up/publishes + volumes from runtime-owned export truth + +## Initial Order + +The required execution order is: + +1. `M6` +2. `M7` +3. `M8` +4. `M9` +5. `M10` + +This order should not be broadly reordered without a written decision in +`phase-19-decisions.md`. + +## Current Focus + +`Phase 19` close-out: + +1. `M6-M10` are now delivered +2. one bounded working RF2 block path now exists +3. later work should widen from this point only through explicit new closure and + multi-process/pilot-ready evidence, not by rereading `Phase 19` as broad + launch proof + +## Review Base + +Use these files together when reviewing `Phase 19` work: + +1. `sw-block/design/v2-two-loop-protocol.md` +2. `sw-block/design/v2-automata-ownership-map.md` +3. `sw-block/design/v2-kernel-closure-review.md` +4. `sw-block/design/v2-protocol-claim-and-evidence.md` +5. `sw-block/design/v2-rf2-runtime-bounded-envelope.md` +6. `sw-block/design/v2-rf2-runtime-bounded-envelope-review.md` +7. `sw-block/.private/phase/phase-19.md` + +## Non-Goals For This Phase Document + +This file should not become: + +1. an unbounded product roadmap +2. a day-by-day log +3. a substitute for the claim/evidence ledger +4. a substitute for detailed runtime design docs diff --git a/sw-block/design/v2-bounded-internal-pilot-pack.md b/sw-block/design/v2-bounded-internal-pilot-pack.md index 12dfe4932..4cfd8e7d9 100644 --- a/sw-block/design/v2-bounded-internal-pilot-pack.md +++ b/sw-block/design/v2-bounded-internal-pilot-pack.md @@ -1,138 +1,147 @@ -# V2 Bounded Internal Pilot Pack - -Date: 2026-04-05 -Status: draft -Purpose: convert the frozen first-launch envelope into one bounded internal pilot -package without silently broadening scope - -## Reading Rule - -This pilot pack is a bounded validation package for the already-frozen chosen path. - -It does NOT mean: - -1. broad launch approval -2. generic production readiness -3. support for surfaces outside the named first-launch matrix -4. permission to redefine exclusions through pilot success - -It means only: - -1. the team may run a limited internal pilot inside the accepted chosen envelope -2. pilot outcomes must be read against the existing `Phase 12-17` claim boundary -3. incidents must be routed explicitly instead of becoming vague rollout lore - -## Pilot Scope - -The pilot is limited to the current first-launch support envelope: - -1. replication / durability: - - `RF=2` - - `sync_all` -2. control/runtime path: - - existing master / volume-server heartbeat path - - bounded `Phase 16` runtime checkpoint - - bounded `Phase 17B/17C` contract/policy interpretation -3. backend/runtime implementation: - - `blockvol` through `v2bridge` - - explicit `V2 core` as semantic owner -4. product surfaces: - - bounded `iSCSI` - - bounded `CSI` - - bounded `NVMe` - -Anything outside that scope is not a pilot finding for this pack. -It is either a known exclusion or later widening work. - -## Pilot Environment And Topology - -The pilot environment must stay fixed and reviewable: - -1. use one explicit build/commit package for all pilot nodes -2. keep topology inside the bounded chosen path and do not introduce `RF>2` -3. keep transport/frontend choices inside the named supported matrix -4. pin operator-facing configuration and startup procedure in a written runbook -5. expose the existing diagnosis surfaces needed to read: - - registry truth - - publication/lookup truth - - restart/failover state - - inventory authority state - -If the pilot needs ad hoc operator judgment to stay healthy, the pack is not ready. - -## Success Criteria - -The pilot is considered successful only if ALL of the following hold: - -1. no observed behavior contradicts the bounded `17B` failover/publication contract -2. no observed behavior contradicts the bounded `17C` disturbance policy table -3. `Phase 12 P4` floor-gate expectations stay non-regressed on the mapped chosen - path workloads -4. incidents can be classified using the explicit buckets in this pack without - inventing new ambiguous categories -5. operators can execute preflight, pilot operation, and bounded diagnosis from - written artifacts rather than tribal knowledge -6. pilot findings do not require silently widening the supported matrix - -Pilot success validates the current bounded envelope only. -It does not create a broader product claim by itself. - -## Incident Intake And Classification - -Every pilot incident must record: - -1. time, node set, workload, and surface involved -2. observed symptom -3. affected bounded claim, exclusion, or blocker -4. diagnosis evidence used -5. immediate operator action taken -6. final classification - -Allowed classification buckets: - -1. `config / environment issue` - - the product behaved inside the bounded claim, but the deployment violated the - pilot preflight or environment assumptions -2. `known exclusion` - - the incident came from a surface or claim already excluded from the first - launch matrix -3. `true product bug` - - the incident contradicts an accepted bounded claim or reveals a real gap - inside the named chosen envelope - -If an incident does not fit one of those buckets, stop the pilot and refine the -artifact set before continuing. - -## Decision Outputs - -At the end of a bounded pilot window, the allowed outcomes are: - -1. `stay in pilot` - - more evidence is needed inside the same envelope -2. `widen within the same envelope` - - the rollout review may expand exposure, but only without changing the named - supported matrix -3. `block expansion` - - a contradiction, repeated unresolved bug, or operational ambiguity prevents - widening - -These outcomes require a later controlled-rollout review artifact. -This pilot pack does not replace that review. - -## Explicit Non-Claims - -This pack does NOT claim: - -1. generic production proof from limited pilot success -2. support for `RF>2` -3. support for a broad transport/frontend matrix -4. broad failover-under-load guarantees -5. hours/days soak proof outside the bounded chosen-path reading - -## Primary Inputs - -1. `sw-block/design/v2-first-launch-supported-matrix.md` -2. `sw-block/.private/phase/phase-17.md` -3. `sw-block/.private/phase/phase-17-checkpoint-review.md` -4. `sw-block/.private/phase/phase-12-p4-rollout-gates.md` -5. `sw-block/design/v2-product-completion-overview.md` +# V2 Bounded Internal Pilot Pack + +Date: 2026-04-05 +Status: draft +Purpose: define the bounded internal engineering validation pack around the +current `Phase 18` RF2 runtime-bearing envelope without silently broadening scope + +## Reading Rule + +This pilot pack is a bounded validation package for the current RF2 +runtime-bearing envelope. + +It does NOT mean: + +1. broad launch approval +2. generic production readiness +3. proof that the current runtime path is already a working block product +4. permission to redefine exclusions through pilot success + +It means only: + +1. the team may run limited internal engineering validation inside the accepted + runtime-bearing envelope +2. validation outcomes must be read against the delivered `Phase 18` boundary +3. incidents must be routed explicitly instead of becoming vague rollout lore + +## Pilot Scope + +This pack is limited to the current `Phase 18` runtime-bearing envelope: + +1. kernel/runtime path: + - `masterv2` identity authority + - `volumev2` runtime-owned failover / Loop 2 / continuity / RF2 surface path + - `purev2` execution adapter reuse +2. validation shape: + - bounded in-process runtime exercises + - artifact-driven review only +3. supported proof shape: + - failover-time evidence seam + - active Loop 2 observation + - continuity handoff statement + - compressed RF2 outward surface +4. excluded surface classes: + - real product frontends + - broad operator APIs + - real transport-backed product traffic + +Anything outside that scope is not a finding for this pack. +It is either a known exclusion, an explicit blocker, or later widening work. + +## Pilot Environment And Topology + +The validation environment must stay fixed and reviewable: + +1. use one explicit build/commit package for all pilot nodes +2. keep topology inside the bounded `RF=2` runtime-bearing path and do not + introduce `RF>2` +3. do not introduce real frontend/product traffic or broad transport claims +4. pin operator-facing configuration and startup procedure in a written runbook +5. expose the runtime diagnosis surfaces needed to read: + - failover snapshot/result + - Loop 2 snapshot + - continuity snapshot + - RF2 outward surface + +If the validation needs ad hoc operator judgment to stay healthy, the pack is not +ready. + +## Success Criteria + +The bounded validation is considered successful only if ALL of the following +hold: + +1. no observed behavior contradicts the bounded `Phase 18` runtime envelope +2. no observed behavior contradicts the bounded fail-closed reading of the new + runtime path +3. incidents can be classified using the explicit buckets in this pack without + inventing new ambiguous categories +4. operators can execute preflight, bounded validation, and diagnosis from + written artifacts rather than tribal knowledge +5. findings do not require silently widening the supported envelope +6. the review outcome remains consistent with the current `block expansion / + not pilot-ready` judgment unless new closure explicitly changes it + +Validation success validates the current bounded envelope only. +It does not create a broader product claim by itself. + +## Incident Intake And Classification + +Every incident must record: + +1. time, node set, workload, and surface involved +2. observed symptom +3. affected bounded claim, exclusion, or blocker +4. diagnosis evidence used +5. immediate operator action taken +6. final classification + +Allowed classification buckets: + +1. `config / environment issue` + - the product behaved inside the bounded claim, but the deployment violated the + pilot preflight or environment assumptions +2. `known exclusion` + - the incident came from a surface or claim already excluded from the first + launch matrix +3. `true product bug` + - the incident contradicts an accepted bounded claim or reveals a real gap + inside the named chosen envelope + +If an incident does not fit one of those buckets, stop the validation and refine +the +artifact set before continuing. + +## Decision Outputs + +At the end of a bounded validation window, the allowed outcomes are: + +1. `stay in bounded validation` + - more evidence is needed inside the same envelope +2. `widen bounded engineering exposure` + - the review may expand only internal engineering validation inside the same + envelope +3. `block expansion` + - a contradiction, repeated unresolved bug, or operational ambiguity prevents + widening + +These outcomes require the bounded envelope review artifact. +This pack does not replace that review. + +## Explicit Non-Claims + +This pack does NOT claim: + +1. generic production proof from limited validation success +2. support for `RF>2` +3. support for a broad transport/frontend matrix +4. broad automatic failover guarantees +5. hours/days soak proof outside the bounded runtime-bearing reading + +## Primary Inputs + +1. `sw-block/design/v2-rf2-runtime-bounded-envelope.md` +2. `sw-block/design/v2-rf2-runtime-bounded-envelope-review.md` +3. `sw-block/.private/phase/phase-18.md` +4. `sw-block/design/v2-protocol-claim-and-evidence.md` +5. `sw-block/design/v2-product-completion-overview.md` diff --git a/sw-block/design/v2-controlled-rollout-review.md b/sw-block/design/v2-controlled-rollout-review.md index 2a5395376..90fa01139 100644 --- a/sw-block/design/v2-controlled-rollout-review.md +++ b/sw-block/design/v2-controlled-rollout-review.md @@ -1,143 +1,143 @@ -# V2 Controlled Rollout Review - -Date: 2026-04-05 -Status: draft -Purpose: define the bounded review used to decide whether the internal pilot stays -limited, widens within the same first-launch envelope, or blocks expansion - -## Reading Rule - -This artifact is a bounded decision gate after pilot execution. - -It does NOT mean: - -1. broad launch approval -2. generic production readiness -3. permission to widen beyond the frozen first-launch matrix -4. permission to reinterpret pilot survival as new protocol/runtime proof - -It means only: - -1. pilot outcomes may be reviewed against the already-accepted bounded envelope -2. expansion decisions must stay inside the same named support boundary -3. any broader claim still needs explicit new evidence and explicit new review - -## Allowed Decisions - -The rollout review may produce only one of these outputs: - -1. `stay in pilot` - - the chosen envelope is still the right boundary, but more bounded pilot - evidence is needed before any exposure increase -2. `widen within the same envelope` - - exposure may increase only inside the current supported matrix, with no - change to topology, durability, control/runtime path, or supported surfaces -3. `block expansion` - - the current evidence, incident record, or operational ambiguity is not strong - enough to increase exposure safely - -Any outcome outside those three is invalid for this review. - -## Required Inputs - -The review must not start unless these inputs exist and are explicit: - -1. the frozen first-launch matrix -2. the bounded pilot pack -3. the preflight checklist outcome(s) -4. the pilot stop-condition artifact -5. pilot incident records with explicit classification -6. pilot outcome summary for the bounded chosen path -7. the accepted evidence anchors that define the current boundary: - - `Phase 12 P4` - - `CP13-1..9` - - `Phase 16` finish-line checkpoint - - `Phase 17A-17D` - -If any required input is missing, the correct review output is `block expansion`. - -## Decision Questions - -The rollout review must answer all of the following: - -1. did the pilot remain fully inside the frozen first-launch envelope -2. did any observed behavior contradict the bounded `17B` failover/publication - contract -3. did any observed behavior contradict the bounded `17C` disturbance policy table -4. were any stop conditions triggered, and if so, how were they resolved -5. are all incidents classified cleanly as: - - `config / environment issue` - - `known exclusion` - - `true product bug` -6. does any proposed next step depend on a broader claim than the current matrix -7. can operators run the pilot and diagnose bounded failures from written - artifacts rather than tribal knowledge - -If the answer to question 6 is yes, the review must not approve widening inside -this artifact. That request belongs to later evidence expansion work. - -## Decision Rules - -Use these bounded rules: - -1. approve `stay in pilot` when: - - the pilot stayed inside scope - - no contradiction to accepted bounded claims was found - - more same-envelope evidence is still needed -2. approve `widen within the same envelope` only when: - - the pilot stayed inside scope - - no unresolved `true product bug` remains against the bounded envelope - - stop conditions did not reveal structural ambiguity - - operator workflow is explicit and repeatable from the artifact set - - the widened exposure does not change the supported matrix -3. approve `block expansion` when: - - any unresolved contradiction exists - - any unresolved `true product bug` exists - - incident records are vague - - operators depend on tribal knowledge - - the requested widening outruns the current matrix - -## Explicit Review Record - -Each review result must record: - -1. decision outcome -2. date and reviewer set -3. pilot window / environment covered -4. summary of incidents by classification bucket -5. any stop-condition events and their disposition -6. exact reason the decision stays inside the current matrix -7. explicit next action: - - continue bounded pilot - - widen exposure inside the same envelope - - pause and fix - -## Rejection Rules - -Reject the review as invalid if: - -1. it uses pilot success as generic production proof -2. it broadens topology, durability mode, or supported surfaces without a new - evidence package -3. it treats a known exclusion as if the pilot cleared it -4. it ignores stop-condition events or unresolved true product bugs -5. it cannot map the decision back to the accepted evidence ladder - -## Explicit Non-Claims - -This artifact does NOT claim: - -1. broad rollout approval -2. generic production readiness -3. support for `RF>2` -4. support for a broad transport/frontend matrix -5. broad failover-under-load or long-window soak proof - -## Primary Inputs - -1. `sw-block/design/v2-first-launch-supported-matrix.md` -2. `sw-block/design/v2-bounded-internal-pilot-pack.md` -3. `sw-block/design/v2-pilot-preflight-checklist.md` -4. `sw-block/design/v2-pilot-stop-conditions.md` -5. `sw-block/design/v2-phase-development-plan.md` -6. `sw-block/.private/phase/phase-17.md` +# V2 Controlled Rollout Review + +Date: 2026-04-05 +Status: draft +Purpose: define the bounded review used to decide whether internal engineering +validation on the current `Phase 18` RF2 runtime envelope stays limited, widens +inside the same envelope, or blocks expansion + +## Reading Rule + +This artifact is a bounded decision gate after runtime-envelope validation. + +It does NOT mean: + +1. broad launch approval +2. generic production readiness +3. permission to widen beyond the frozen runtime envelope +4. permission to reinterpret validation survival as new protocol/runtime proof + +It means only: + +1. validation outcomes may be reviewed against the already-accepted bounded + envelope +2. expansion decisions must stay inside the same named support boundary +3. any broader claim still needs explicit new evidence and explicit new review + +## Allowed Decisions + +The rollout review may produce only one of these outputs: + +1. `stay in bounded validation` + - the chosen envelope is still the right boundary, but more bounded validation + evidence is needed before any exposure increase +2. `widen bounded engineering exposure` + - exposure may increase only inside the current bounded engineering envelope, + with no change to the named support boundary +3. `block expansion` + - the current evidence, incident record, or operational ambiguity is not strong + enough to increase exposure safely + +Any outcome outside those three is invalid for this review. + +## Required Inputs + +The review must not start unless these inputs exist and are explicit: + +1. the frozen runtime envelope +2. the bounded pilot pack +3. the preflight checklist outcome(s) +4. the pilot stop-condition artifact +5. incident records with explicit classification +6. validation outcome summary for the bounded runtime-bearing path +7. the accepted evidence anchors that define the current boundary: + - `Phase 18 M1-M4` + - `v2-rf2-runtime-bounded-envelope.md` + - `v2-rf2-runtime-bounded-envelope-review.md` + +If any required input is missing, the correct review output is `block expansion`. + +## Decision Questions + +The rollout review must answer all of the following: + +1. did validation remain fully inside the frozen runtime envelope +2. did any observed behavior contradict the bounded `Phase 18` runtime envelope +3. were any stop conditions triggered, and if so, how were they resolved +4. are all incidents classified cleanly as: + - `config / environment issue` + - `known exclusion` + - `true product bug` +5. does any proposed next step depend on a broader claim than the current + envelope +6. can operators run the validation and diagnose bounded failures from written + artifacts rather than tribal knowledge + +If the answer to question 5 is yes, the review must not approve widening inside +this artifact. That request belongs to later evidence expansion work. + +## Decision Rules + +Use these bounded rules: + +1. approve `stay in bounded validation` when: + - validation stayed inside scope + - no contradiction to accepted bounded claims was found + - more same-envelope evidence is still needed +2. approve `widen bounded engineering exposure` only when: + - validation stayed inside scope + - no unresolved `true product bug` remains against the bounded envelope + - stop conditions did not reveal structural ambiguity + - operator workflow is explicit and repeatable from the artifact set + - the widened exposure does not change the bounded envelope +3. approve `block expansion` when: + - any unresolved contradiction exists + - any unresolved `true product bug` exists + - incident records are vague + - operators depend on tribal knowledge + - the requested widening outruns the current matrix + +## Explicit Review Record + +Each review result must record: + +1. decision outcome +2. date and reviewer set +3. validation window / environment covered +4. summary of incidents by classification bucket +5. any stop-condition events and their disposition +6. exact reason the decision stays inside the current envelope +7. explicit next action: + - continue bounded validation + - widen engineering exposure inside the same envelope + - pause and fix + +## Rejection Rules + +Reject the review as invalid if: + +1. it uses validation success as generic production proof +2. it broadens topology, runtime path, or supported surfaces without a new + evidence package +3. it treats a known exclusion as if validation cleared it +4. it ignores stop-condition events or unresolved true product bugs +5. it cannot map the decision back to the accepted evidence ladder + +## Explicit Non-Claims + +This artifact does NOT claim: + +1. broad rollout approval +2. generic production readiness +3. support for `RF>2` +4. support for a broad transport/frontend matrix +5. broad failover-under-load or long-window soak proof + +## Primary Inputs + +1. `sw-block/design/v2-rf2-runtime-bounded-envelope.md` +2. `sw-block/design/v2-rf2-runtime-bounded-envelope-review.md` +3. `sw-block/design/v2-bounded-internal-pilot-pack.md` +4. `sw-block/design/v2-pilot-preflight-checklist.md` +5. `sw-block/design/v2-pilot-stop-conditions.md` +6. `sw-block/.private/phase/phase-18.md` diff --git a/sw-block/design/v2-kernel-closure-review.md b/sw-block/design/v2-kernel-closure-review.md index 66e4f657c..526f24fe3 100644 --- a/sw-block/design/v2-kernel-closure-review.md +++ b/sw-block/design/v2-kernel-closure-review.md @@ -134,79 +134,79 @@ Another way to state the same result: The current milestone is: -1. explicit `masterv2` promotion authorization -2. explicit `volumev2` takeover preparation and activation gating -3. stepwise `FailoverSession` orchestration with observable stages -4. one in-process failover driver seam over registered participants -5. one runtime-owned `volumev2` manager that executes failover through a single - runtime entry point and retains latest failover snapshot/result +1. live transport-backed failover-time evidence now crosses one real loopback + HTTP path +2. one continuous Loop 2 service and one bounded auto-failover service now exist +3. one runtime-managed frontend path and one bounded repair/catch-up wrapper now + exist +4. one end-to-end RF2 handoff proof now exists with continued I/O on the new + primary +5. one bounded operator surface and one bounded CSI runtime backend adapter now + sit on top of runtime-owned truth What this milestone proves: -1. failover is no longer only an implicit concept hidden in mixed runtime code -2. authorization, reconstruction, activation, and runtime ownership are now - separate code seams -3. the new kernel already has a usable in-process RF2 failover runtime slice +1. the new kernel can now carry RF2 failover, active replication observation, + bounded continuity, real serving, and one outward/operator/CSI surface stack + without collapsing authority ownership +2. runtime/product-facing state can now be attached as compressed projection + rather than by inventing a new truth owner +3. the `masterv2` identity boundary and primary-led data-control boundary still + remain intact +4. one bounded working RF2 block path now exists What it does not yet prove: -1. real transport-backed replica communication -2. background membership and health management beyond explicit failover calls -3. end-to-end replicated data continuity through a real primary handoff +1. broad RF2 product approval across deployment and frontend/operator matrices +2. full rebuild lifecycle choreography beyond the bounded repair wrapper +3. multi-process / multi-host proof for the current working path +4. pilot-ready or launch-ready working block behavior ## Next Major Milestone The next major milestone should be: -`transport-backed RF2 failover runtime` +`multi-process and pilot-ready RF2 validation` -This means one level above the current in-process runtime slice: +This means one level above the current runtime/product surface slice: -1. `FailoverSession` no longer talks only to in-memory participants -2. promotion evidence and bounded replica summaries cross a real transport seam -3. runtime-owned failover still preserves stage/error/result observability -4. the runtime keeps the separation: - - `masterv2` authorizes - - the selected primary reconstructs and gates activation - - the transport/runtime only carries requests, responses, and results +1. prove the current working path outside the current bounded runtime harness +2. widen the working path into multi-process or multi-host validation +3. harden rebuild lifecycle and operator/CSI behavior on that wider path +4. attach pilot/preflight/containment evidence on top of the widened path ### Target Shape Code should look roughly like: -1. `masterv2`: heartbeat, promotion evidence arbitration, assignment authority -2. `volumev2`: node shell, takeover seams, failover session, runtime/manager +1. `masterv2`: identity authority only +2. `volumev2`: runtime-owned failover, active service Loop 2, repair, continuity, + and projected RF2 surfaces 3. `purev2`: execution adapter and local boundary observation -4. transport/session adapters: promotion query and replica-summary exchange -5. `engine/replication`: semantic brain and future active data-control automata +4. transport/session adapters: live participant communication beyond the current + bounded harness +5. product-facing layer: bounded frontend/operator/CSI attachment with pilot + artifacts ### Exit Criteria The milestone should be considered complete when all are true: -1. one runtime-level API can execute failover without tests manually stitching - participants and sessions together -2. one transport-backed runtime-level API can execute failover without - in-process participant shortcuts -3. runtime-level observability exposes current/last failover stage, selected - node, assignment, and bounded truth across the transport seam -4. one healthy component-style failover test passes through the runtime entry - point -5. one gated/degraded component-style failover test stops through the same - runtime entry point -6. no new recovery-planner responsibility leaks back into `masterv2` +1. the current bounded working path is proven outside the current harness +2. pilot/preflight/containment evidence exists for that wider path +3. no new product/runtime surface silently widens into broad launch approval ### Non-Goals This milestone should not try to prove: -1. broad product RF2 readiness -2. full catch-up or rebuild choreography -3. CSI or frontend productionization on top of RF2 -4. broad transport matrix approval +1. broad product launch approval +2. broad matrix approval across all transport/frontend combinations +3. broad `RF>2` product closure +4. reopening the kernel authority split -It should prove only that the new kernel can carry the same authority model -through a real transport boundary without collapsing the ownership split. +It should prove only that the current working RF2 block path can widen toward +pilot-ready validation without collapsing the ownership model. ## Main Risk diff --git a/sw-block/design/v2-phase-development-plan.md b/sw-block/design/v2-phase-development-plan.md index 9b34baeb8..408f40654 100644 --- a/sw-block/design/v2-phase-development-plan.md +++ b/sw-block/design/v2-phase-development-plan.md @@ -35,7 +35,8 @@ Use these rules for all later phases: ## Current Baseline -Current accepted path now runs through the `Phase 16` finish-line checkpoint: +Current accepted path now runs through the completed `Phase 18` and `Phase 19` +checkpoints: 1. protocol/algo truth set is strong 2. engine recovery core is strong on the chosen path @@ -55,6 +56,16 @@ Current accepted path now runs through the `Phase 16` finish-line checkpoint: - assignment/publication closure accepted - bounded mode normalization accepted - bounded heartbeat/restart truth closure accepted +9. `Phase 18` closed the bounded RF2 runtime-bearing kernel slice and explicit + productionization envelope +10. `Phase 19` closed one bounded working RF2 block path with: + - live transport-backed evidence + - continuous Loop 2 service + - bounded auto failover + - runtime-managed frontend rebinding + - bounded repair/catch-up + - end-to-end handoff proof + - bounded CSI/operator adapters Phase-accounting note: @@ -66,11 +77,16 @@ Phase-accounting note: 6. `Phase 13` is closed and should be read as one bounded constrained-runtime contract package, not as launch approval 7. `Phase 14` is closed as explicit `V2 core` extraction 8. `Phase 15` is closed as bounded adapter/projection rebinding -9. `Phase 16` is closed as a bounded `V2`-native runtime checkpoint, not as broad product/launch proof -10. the immediate next planning focus is now the productionization program plus the larger post-`Phase 16` gates: - - broader recovery/failover/publication statement - - launch-envelope freeze - - pilot/rollout discipline +9. `Phase 16` is closed as a bounded `V2`-native runtime checkpoint, not as + broad product/launch proof +10. `Phase 17` is closed as the bounded product-checkpoint / first-launch draft +11. `Phase 18` is closed as the bounded RF2 runtime-bearing kernel slice plus + productionization envelope +12. `Phase 19` is closed as one bounded working RF2 block path +13. the immediate next planning focus is now: + - multi-process / multi-host proof for the current working path + - broader rebuild / CSI / operator hardening + - pilot-ready validation gates on top of the bounded working path Important interpretation rule: @@ -86,10 +102,12 @@ Important interpretation rule: This means the next work should focus mainly on: -1. keeping accepted `Phase 14-16` closure closed -2. turning the bounded runtime checkpoint into an explicit launch envelope -3. deciding broader failover/recovery/publication claims only through named evidence -4. using `Phase 13-16` evidence as acceptance input rather than reopening micro-seams by default +1. keeping accepted `Phase 14-19` closure closed +2. widening the current working path into multi-process / multi-host validation +3. deciding broader rebuild / CSI / operator / pilot claims only through named + evidence +4. using `Phase 13-19` evidence as acceptance input rather than reopening + micro-seams by default ## Phase Roadmap diff --git a/sw-block/design/v2-pilot-preflight-checklist.md b/sw-block/design/v2-pilot-preflight-checklist.md index 3203d78a4..d14c5a1cd 100644 --- a/sw-block/design/v2-pilot-preflight-checklist.md +++ b/sw-block/design/v2-pilot-preflight-checklist.md @@ -1,109 +1,112 @@ -# V2 Pilot Preflight Checklist - -Date: 2026-04-05 -Status: draft -Purpose: define the minimum explicit checks required before running the bounded -internal pilot - -## Reading Rule - -This checklist is a gate for starting or resuming the bounded pilot. - -If any item below is not satisfied: - -1. do not treat the environment as pilot-ready -2. either fix the issue or classify it explicitly before proceeding - -## Scope Lock - -Confirm the pilot is still inside the frozen first-launch envelope: - -1. topology remains `RF=2` -2. durability mode remains `sync_all` -3. control/runtime path remains the existing master / volume-server heartbeat path -4. execution backend remains `blockvol` through `v2bridge` -5. product surface remains within bounded `iSCSI`, bounded `CSI`, or bounded - `NVMe` -6. no one is trying to use pilot success to claim broader launch approval - -## Build And Artifact Pin - -Confirm the software package is explicit and stable: - -1. the exact build/commit for pilot nodes is written down -2. all pilot nodes run the same intended package -3. the operator runbook matches the package actually deployed -4. any configuration delta from the documented chosen path is reviewed and - accepted explicitly - -## Environment Readiness - -Confirm the pilot environment matches bounded assumptions: - -1. node inventory and topology are written down -2. transport/frontend choice is inside the supported matrix -3. storage/network assumptions required by the chosen path are known to the - operator -4. known exclusions are acknowledged before start -5. rollback/containment ownership is assigned for the pilot window - -## Diagnosis Surface Readiness - -Confirm bounded diagnosis can be performed without ad hoc spelunking: - -1. registry truth can be inspected -2. publication/lookup truth can be inspected -3. restart/failover progression can be inspected -4. inventory-authority state can be inspected -5. the operator knows which artifact defines the current contract/policy boundary: - - `v2-first-launch-supported-matrix.md` - - `phase-17.md` - - this preflight checklist - - the stop-condition artifact - -## Workload And Gate Alignment - -Confirm the pilot workload is aligned with accepted evidence: - -1. the workload maps to the bounded chosen-path reading rather than a new - unsupported scenario -2. success will be judged against the pilot pack criteria rather than generic - "looks stable" judgment -3. `Phase 12 P4` floor-gate expectations are known for the workload being run -4. no required proof depends on failover-under-load, hours/days soak, `RF>2`, or - broad transport claims that are still excluded - -## Incident Routing Readiness - -Confirm incident handling is explicit before starting: - -1. every incident will be classified as one of: - - `config / environment issue` - - `known exclusion` - - `true product bug` -2. the recording location for pilot incidents is agreed before the pilot starts -3. ownership for triage and decision-making is assigned -4. operators know when they must stop instead of improvising - -## Preflight Result - -The pilot may start only if: - -1. every scope-lock item is true -2. the software package and environment are pinned -3. diagnosis surfaces are available -4. incident routing is explicit -5. no remaining gap is being hand-waved as "we will figure it out during pilot" - -If those conditions are not met, the correct output is: - -1. `NOT READY` -2. the missing item(s) -3. the owner/action needed before retry - -## Primary Inputs - -1. `sw-block/design/v2-bounded-internal-pilot-pack.md` -2. `sw-block/design/v2-first-launch-supported-matrix.md` -3. `sw-block/.private/phase/phase-17.md` -4. `sw-block/.private/phase/phase-12-p4-rollout-gates.md` +# V2 Pilot Preflight Checklist + +Date: 2026-04-05 +Status: draft +Purpose: define the minimum explicit checks required before running bounded +internal engineering validation on the current `Phase 18` RF2 runtime envelope + +## Reading Rule + +This checklist is a gate for starting or resuming bounded internal engineering +validation. + +If any item below is not satisfied: + +1. do not treat the environment as pilot-ready +2. either fix the issue or classify it explicitly before proceeding + +## Scope Lock + +Confirm the validation is still inside the frozen `Phase 18` RF2 runtime +envelope: + +1. topology remains bounded `RF=2` +2. runtime path remains the delivered `masterv2 + volumev2 + purev2` `M1-M4` + path +3. validation stays in bounded runtime/lab exercises only +4. no one is trying to use this validation to claim working block product status +5. no real frontend/product traffic is being introduced on the new runtime path +6. no one is trying to use validation success to claim broader launch approval + +## Build And Artifact Pin + +Confirm the software package is explicit and stable: + +1. the exact build/commit for validation nodes is written down +2. all validation nodes run the same intended package +3. the operator runbook matches the package actually deployed +4. any configuration delta from the documented chosen path is reviewed and + accepted explicitly + +## Environment Readiness + +Confirm the validation environment matches bounded assumptions: + +1. node inventory and topology are written down +2. transport/frontend choice does not widen beyond the bounded runtime envelope +3. storage/network assumptions required by the chosen path are known to the + operator +4. known exclusions are acknowledged before start +5. rollback/containment ownership is assigned for the validation window + +## Diagnosis Surface Readiness + +Confirm bounded diagnosis can be performed without ad hoc spelunking: + +1. failover snapshots/results can be inspected +2. Loop 2 snapshots can be inspected +3. continuity snapshots can be inspected +4. RF2 runtime surface can be inspected +5. the operator knows which artifact defines the current contract/policy boundary: + - `v2-rf2-runtime-bounded-envelope.md` + - `v2-rf2-runtime-bounded-envelope-review.md` + - this preflight checklist + - the stop-condition artifact + +## Workload And Gate Alignment + +Confirm the validation workload is aligned with accepted evidence: + +1. the workload maps to the bounded runtime-bearing reading rather than a new + unsupported scenario +2. success will be judged against the validation-pack criteria rather than generic + "looks stable" judgment +3. the workload does not assume continuous Loop 2, real transport, auto failover, + rebuild lifecycle, or product frontends that are still excluded +4. no required proof depends on failover-under-load, hours/days soak, `RF>2`, or + broad transport/frontend claims that are still excluded + +## Incident Routing Readiness + +Confirm incident handling is explicit before starting: + +1. every incident will be classified as one of: + - `config / environment issue` + - `known exclusion` + - `true product bug` +2. the recording location for incidents is agreed before validation starts +3. ownership for triage and decision-making is assigned +4. operators know when they must stop instead of improvising + +## Preflight Result + +Validation may start only if: + +1. every scope-lock item is true +2. the software package and environment are pinned +3. diagnosis surfaces are available +4. incident routing is explicit +5. no remaining gap is being hand-waved as "we will figure it out during pilot" + +If those conditions are not met, the correct output is: + +1. `NOT READY` +2. the missing item(s) +3. the owner/action needed before retry + +## Primary Inputs + +1. `sw-block/design/v2-bounded-internal-pilot-pack.md` +2. `sw-block/design/v2-rf2-runtime-bounded-envelope.md` +3. `sw-block/design/v2-rf2-runtime-bounded-envelope-review.md` +4. `sw-block/.private/phase/phase-18.md` diff --git a/sw-block/design/v2-pilot-stop-conditions.md b/sw-block/design/v2-pilot-stop-conditions.md index c0a66b73d..c6774b832 100644 --- a/sw-block/design/v2-pilot-stop-conditions.md +++ b/sw-block/design/v2-pilot-stop-conditions.md @@ -1,103 +1,101 @@ -# V2 Pilot Stop Conditions - -Date: 2026-04-05 -Status: draft -Purpose: define when the bounded internal pilot must stop, contain scope, or roll -back exposure - -## Reading Rule - -This artifact is about pilot containment, not protocol/data rollback semantics. - -`Rollback` here means: - -1. stop widening pilot exposure -2. reduce or remove pilot usage if needed -3. return to a previously accepted bounded state of operation - -It does NOT mean: - -1. a general storage/data rollback guarantee -2. permission to claim a broader recovery contract than the current evidence -3. ad hoc operator improvisation under ambiguity - -## Immediate Stop Conditions - -Stop the pilot immediately if ANY of the following occurs: - -1. an observed behavior contradicts the bounded `17B` failover/publication - contract -2. an observed behavior contradicts the bounded `17C` disturbance policy table -3. publication/lookup truth and registry truth diverge outside the currently - allowed bounded interpretation window -4. diagnosis surfaces are insufficient to classify the incident without guessing -5. a `Phase 12 P4` floor-gate expectation is materially violated on the mapped - chosen-path workload -6. the deployment is being widened beyond the named supported matrix without an - explicit review decision -7. the incident does not fit the allowed buckets: - - `config / environment issue` - - `known exclusion` - - `true product bug` - -## Stop-And-Contain Actions - -When a stop condition fires: - -1. freeze new pilot expansion immediately -2. preserve the evidence needed for later review -3. classify the incident explicitly -4. map the incident back to: - - accepted bounded claim - - known exclusion - - unresolved blocker -5. decide whether the pilot can continue in reduced scope or must fully pause - -If the team cannot perform those actions clearly, the pilot remains stopped. - -## Rollback Decision Rules - -Use the following bounded rules: - -1. `config / environment issue` - - fix the environment/configuration - - rerun preflight before resuming -2. `known exclusion` - - remove the excluded usage from the pilot - - do not reinterpret it as product support -3. `true product bug` - - pause affected pilot scope - - open an explicit fix or contradiction item before resuming - -If repeated incidents of the same class continue without a bounded corrective path, -block further pilot expansion. - -## Expansion Blockers - -Even if the pilot remains partially runnable, do NOT widen it when: - -1. the same unresolved true product bug recurs -2. operators depend on tribal knowledge to recover or diagnose -3. incident records are vague or cannot be mapped back to the current evidence - ladder -4. success depends on ignoring explicit exclusions -5. the desired next step requires broader launch claims than the current matrix - -## Explicit Non-Claims - -This artifact does NOT claim: - -1. broad rollout approval -2. generic production readiness from pilot survival -3. support for `RF>2` -4. support for a broad transport/frontend matrix -5. failover-under-load proof or long-window soak proof beyond the current bounded - evidence set - -## Primary Inputs - -1. `sw-block/design/v2-bounded-internal-pilot-pack.md` -2. `sw-block/design/v2-pilot-preflight-checklist.md` -3. `sw-block/design/v2-first-launch-supported-matrix.md` -4. `sw-block/.private/phase/phase-17.md` -5. `sw-block/.private/phase/phase-12-p4-rollout-gates.md` +# V2 Pilot Stop Conditions + +Date: 2026-04-05 +Status: draft +Purpose: define when bounded internal engineering validation on the current +`Phase 18` RF2 runtime envelope must stop, contain scope, or block expansion + +## Reading Rule + +This artifact is about validation containment, not protocol/data rollback +semantics. + +`Rollback` here means: + +1. stop widening validation exposure +2. reduce or remove validation usage if needed +3. return to a previously accepted bounded state of operation + +It does NOT mean: + +1. a general storage/data rollback guarantee +2. permission to claim a broader recovery contract than the current evidence +3. ad hoc operator improvisation under ambiguity + +## Immediate Stop Conditions + +Stop validation immediately if ANY of the following occurs: + +1. an observed behavior contradicts the bounded `Phase 18` RF2 runtime envelope +2. any run is interpreted as proving automatic failover, continuous Loop 2 + service, rebuild lifecycle, or frontend-serving behavior that is still + explicitly excluded +3. diagnosis surfaces are insufficient to classify the incident without guessing +4. the validation is being widened beyond the named bounded envelope without an + explicit review decision +5. the incident does not fit the allowed buckets: + - `config / environment issue` + - `known exclusion` + - `true product bug` + +## Stop-And-Contain Actions + +When a stop condition fires: + +1. freeze new validation expansion immediately +2. preserve the evidence needed for later review +3. classify the incident explicitly +4. map the incident back to: + - accepted bounded claim + - known exclusion + - unresolved blocker +5. decide whether validation can continue in reduced scope or must fully pause + +If the team cannot perform those actions clearly, validation remains stopped. + +## Rollback Decision Rules + +Use the following bounded rules: + +1. `config / environment issue` + - fix the environment/configuration + - rerun preflight before resuming +2. `known exclusion` + - remove the excluded usage from validation + - do not reinterpret it as product support +3. `true product bug` + - pause affected validation scope + - open an explicit fix or contradiction item before resuming + +If repeated incidents of the same class continue without a bounded corrective +path, block further validation expansion. + +## Expansion Blockers + +Even if validation remains partially runnable, do NOT widen it when: + +1. the same unresolved true product bug recurs +2. operators depend on tribal knowledge to recover or diagnose +3. incident records are vague or cannot be mapped back to the current evidence + ladder +4. success depends on ignoring explicit exclusions +5. the desired next step requires broader launch claims than the current envelope + +## Explicit Non-Claims + +This artifact does NOT claim: + +1. broad rollout approval +2. generic production readiness from validation survival +3. support for `RF>2` +4. support for a broad transport/frontend matrix +5. failover-under-load proof or long-window soak proof beyond the current bounded + evidence set + +## Primary Inputs + +1. `sw-block/design/v2-bounded-internal-pilot-pack.md` +2. `sw-block/design/v2-pilot-preflight-checklist.md` +3. `sw-block/design/v2-rf2-runtime-bounded-envelope.md` +4. `sw-block/design/v2-rf2-runtime-bounded-envelope-review.md` +5. `sw-block/.private/phase/phase-18.md` diff --git a/sw-block/design/v2-protocol-claim-and-evidence.md b/sw-block/design/v2-protocol-claim-and-evidence.md index 3c2c8f50d..98725b0f1 100644 --- a/sw-block/design/v2-protocol-claim-and-evidence.md +++ b/sw-block/design/v2-protocol-claim-and-evidence.md @@ -148,10 +148,12 @@ These are the claims that may currently be made without overreach. | `C-PHASE16-RUNTIME-CHECKPOINT` | the bounded heartbeat/master/API runtime path now preserves accepted explicit truth across the delivered `16M-16W` restart/disturbance seams | bounded chosen path only; excludes broad recovery-loop, broad failover/publication, and launch claims | `sw-block/.private/phase/phase-16-finish-review.md`, `phase-16.md`, focused `weed/server` tests | allowed | | `C-PHASE17-PRODUCT-CHECKPOINT` | the current broader recovery-branch map, bounded failover/publication contract, bounded disturbance policy table, and first-launch envelope draft are explicit for the chosen path | bounded chosen path only; excludes broad production readiness, broad transport/frontend approval, and broad whole-surface failover/publication proof | `sw-block/.private/phase/phase-17.md`, `sw-block/.private/phase/phase-17-checkpoint-review.md` | allowed | | `C-FIRST-LAUNCH-ENVELOPE-DRAFT` | one bounded first-launch supported matrix is frozen as a draft with explicit exclusions and launch blockers | bounded chosen path only; not a launch decision, pilot approval, or rollout approval | `sw-block/design/v2-first-launch-supported-matrix.md` | allowed | -| `C-PRODUCTIONIZATION-ARTIFACT-SET` | one bounded productionization artifact set now exists for internal pilot, preflight, stop-condition, and controlled-rollout discipline inside the frozen chosen envelope | bounded chosen path only; artifact existence only, not pilot success, rollout approval, or broader launch proof | `sw-block/design/v2-bounded-internal-pilot-pack.md`, `v2-pilot-preflight-checklist.md`, `v2-pilot-stop-conditions.md`, `v2-controlled-rollout-review.md` | allowed | +| `C-PRODUCTIONIZATION-ARTIFACT-SET` | one bounded productionization artifact set now exists around the current `Phase 18` RF2 runtime-bearing envelope, including explicit preflight, stop-condition, and review judgment artifacts | bounded runtime-bearing path only; artifact existence and explicit current judgment only, not pilot success, rollout approval, or broader launch proof | `sw-block/design/v2-rf2-runtime-bounded-envelope.md`, `v2-rf2-runtime-bounded-envelope-review.md`, `v2-bounded-internal-pilot-pack.md`, `v2-pilot-preflight-checklist.md`, `v2-pilot-stop-conditions.md`, `v2-controlled-rollout-review.md` | allowed | | `C-KERNEL-BOUNDARY` | the current V2 kernel boundary is explicitly defined around `masterv2` identity authority, `volumev2` takeover/data-control shell, and `purev2` execution adapter reuse | kernel-boundary statement only; not broad runtime/product readiness | `v2-two-loop-protocol.md`, `v2-automata-ownership-map.md`, `v2-kernel-closure-review.md` | allowed | | `C-INPROCESS-FAILOVER-MILESTONE` | one in-process failover milestone exists with explicit authorization, reconstruction, activation gating, session observability, and driver seams | in-process `masterv2 + volumev2` code path only; not transport-complete RF2 product proof | `sw-block/runtime/masterv2/*`, `sw-block/runtime/volumev2/*` tests | allowed | | `C-INPROCESS-RF2-FAILOVER-RUNTIME` | one runtime-owned in-process RF2 failover slice exists with participant registry, explicit runtime entry point, persisted session snapshots/results, and component-style healthy/gated failover tests | in-process runtime only; not transport-backed RF2 product proof | `sw-block/runtime/volumev2/runtime_manager.go`, `failover*.go`, `poc_test.go` | allowed | +| `C-RF2-RUNTIME-SURFACE` | one bounded RF2-facing runtime/product surface package exists as a compressed projection of runtime-owned Loop 2, failover, and continuity truth | bounded runtime-owned surface only; not broad frontend approval, launch approval, or independent truth ownership | `sw-block/runtime/volumev2/rf2_surface.go`, `runtime_manager.go`, `poc_test.go`, `sw-block/.private/phase/phase-18.md` | allowed | +| `C-WORKING-RF2-BLOCK-PATH` | one bounded working RF2 block path now exists with live transport-backed evidence, continuous Loop 2 service, bounded auto failover, runtime-managed frontend rebinding, bounded repair/catch-up, end-to-end client handoff proof, and bounded CSI/operator adapters | bounded runtime harness only; not broad product readiness, broad deployment approval, or launch approval | `sw-block/.private/phase/phase-19.md`, `sw-block/runtime/volumev2/loop2_service.go`, `frontend_runtime.go`, `operator_surface.go`, `poc_test.go`, `weed/storage/blockvol/csi/v2_runtime_backend.go` | allowed | | `C-LAUNCH-APPROVAL` | broad product launch readiness | outside current phase | future | not allowed | ## Evidence Map @@ -161,6 +163,8 @@ These are the claims that may currently be made without overreach. | Kernel ownership boundary | `masterv2`, `volumev2`, and `purev2` are split with explicit authority boundaries | `v2-two-loop-protocol.md`, `v2-automata-ownership-map.md`, `v2-kernel-closure-review.md` | code in `sw-block/runtime/masterv2/`, `volumev2/`, `purev2/` | | In-process failover milestone | one explicit failover chain exists from promotion evidence through authorization, takeover preparation, activation gating, session observability, and driver wiring | `sw-block/runtime/volumev2/poc_test.go`, `failover.go`, `failover_driver.go` | `sw-block/runtime/masterv2/master_test.go` | | In-process RF2 failover runtime | one runtime-owned failover manager exists with participant registry, runtime entry point, persisted snapshots/results, and component-style healthy/gated failover tests | `sw-block/runtime/volumev2/runtime_manager.go`, `poc_test.go` | `failover.go`, `failover_driver.go`, `v2-kernel-closure-review.md` | +| RF2 runtime surface package | one bounded RF2-facing surface package projects active Loop 2, failover, and continuity state from runtime-owned snapshots/results | `sw-block/runtime/volumev2/rf2_surface.go`, `poc_test.go` | `runtime_manager.go`, `sw-block/.private/phase/phase-18.md` | +| Working RF2 block path | one bounded working RF2 block path exists with live transport, continuous observation, bounded auto failover, runtime-managed frontend rebinding, repair closure, end-to-end handoff proof, and bounded CSI/operator adapters | `sw-block/.private/phase/phase-19.md`, `sw-block/runtime/volumev2/poc_test.go` | `loop2_service.go`, `frontend_runtime.go`, `operator_surface.go`, `weed/storage/blockvol/csi/v2_runtime_backend.go` | | Identity / addressing | stable identity and routable publication | `CP13-2` tests and docs | `qa_block_soak_test.go`, `sync_all_bug_test.go` | | Durable progress | barrier durability truth and non-legacy authority | `CP13-3` tests and docs | protocol tests around barrier handling | | State eligibility | only eligible replica state may satisfy sync durability | `CP13-4` tests and docs | adversarial state tests | @@ -175,7 +179,7 @@ These are the claims that may currently be made without overreach. | Failover/publication bounded contract | one bounded whole-chain statement is explicit for publication ownership/address coherence after failover completion and winning assignment delivery | `phase-17.md`, `phase-17-checkpoint-review.md`, publication/disturbance tests in `weed/server` | `v2-first-launch-supported-matrix.md` | | Disturbance policy table | startup/restart/rejoin/repeated-failover/degraded-sparsity behavior is explicit as runtime rule, temporary inconsistency policy, or non-claim | `phase-17.md`, `phase-17-checkpoint-review.md`, restart/disturbance tests in `weed/server` | `v2-product-completion-overview.md` | | First-launch supported matrix | one bounded launch envelope draft is explicit with supported scope, exclusions, and launch blockers | `v2-first-launch-supported-matrix.md` | `phase-17-checkpoint-review.md`, `Phase 12 P4`, `CP13`, `Phase 16` | -| Productionization artifact set | one bounded artifact set defines pilot scope, preflight gate, stop/contain rules, and post-pilot rollout review discipline | `v2-bounded-internal-pilot-pack.md`, `v2-pilot-preflight-checklist.md`, `v2-pilot-stop-conditions.md`, `v2-controlled-rollout-review.md` | `v2-first-launch-supported-matrix.md`, `v2-product-completion-overview.md` | +| Productionization artifact set | one bounded artifact set defines the current RF2 runtime-bearing envelope, explicit preflight gate, stop/contain rules, and explicit bounded review judgment | `v2-rf2-runtime-bounded-envelope.md`, `v2-rf2-runtime-bounded-envelope-review.md`, `v2-bounded-internal-pilot-pack.md`, `v2-pilot-preflight-checklist.md`, `v2-pilot-stop-conditions.md`, `v2-controlled-rollout-review.md` | `phase-18.md`, `v2-product-completion-overview.md` | ## Invalidated Or Narrowed Evidence diff --git a/sw-block/design/v2-rf2-runtime-bounded-envelope-review.md b/sw-block/design/v2-rf2-runtime-bounded-envelope-review.md new file mode 100644 index 000000000..6d5031e4f --- /dev/null +++ b/sw-block/design/v2-rf2-runtime-bounded-envelope-review.md @@ -0,0 +1,80 @@ +# V2 RF2 Runtime Bounded Envelope Review + +Date: 2026-04-05 +Status: draft +Purpose: record the current bounded productionization judgment for the delivered +`Phase 19` RF2 working-path envelope + +## Review Outcome + +Current decision: + +1. `stay in bounded validation` +2. `not pilot-ready` + +## Why This Is The Correct Outcome + +The delivered `Phase 19` path proves one bounded working RF2 block path: + +1. live transport-backed evidence traffic exists +2. continuous Loop 2 service exists +3. bounded automatic failover exists +4. runtime-managed frontend rebinding exists +5. bounded repair/catch-up exists +6. one real end-to-end client handoff proof exists +7. bounded operator and CSI adapters now exist on top of runtime-owned truth + +But the path is still not broad product/pilot approval because: + +1. the current proof is still bounded to the current runtime harness +2. repair/catch-up is not yet broad rebuild lifecycle closure +3. CSI and operator surfaces are still bounded adapters rather than full + production surfaces +4. no broad pilot or rollout evidence exists yet + +## Review Record + +Reviewer reading baseline: + +1. `sw-block/.private/phase/phase-19.md` +2. `sw-block/design/v2-rf2-runtime-bounded-envelope.md` +3. `sw-block/design/v2-bounded-internal-pilot-pack.md` +4. `sw-block/design/v2-pilot-preflight-checklist.md` +5. `sw-block/design/v2-pilot-stop-conditions.md` +6. `sw-block/design/v2-controlled-rollout-review.md` +7. `sw-block/runtime/volumev2/poc_test.go` + +Current evidence package: + +1. runtime-owned failover manager +2. continuous Loop 2 service and bounded auto failover +3. runtime-managed frontend and bounded repair closure +4. end-to-end RF2 handoff proof +5. RF2 runtime surface projection and operator surface +6. bounded CSI runtime backend adapter + +## Allowed Interpretation + +The review allows only these statements: + +1. one runtime-bearing RF2 kernel slice now exists +2. one bounded working RF2 block path now exists +3. one bounded productionization artifact set now exists around that path +4. later work may widen from this review only through explicit new closure + +The review does NOT allow: + +1. working block product approval +2. pilot execution against real product traffic +3. rollout expansion beyond bounded internal engineering validation + +## Next Required Closures + +Before any pilot-ready judgment can exist, the next closures must become +explicit: + +1. multi-process / multi-host proof for the current working path +2. broader rebuild lifecycle closure beyond the bounded repair wrapper +3. fuller CSI lifecycle parity on the V2 runtime path +4. broader operator/metrics surface closure +5. pilot/preflight/containment evidence on top of the `Phase 19` path diff --git a/sw-block/design/v2-rf2-runtime-bounded-envelope.md b/sw-block/design/v2-rf2-runtime-bounded-envelope.md new file mode 100644 index 000000000..4d8bd1386 --- /dev/null +++ b/sw-block/design/v2-rf2-runtime-bounded-envelope.md @@ -0,0 +1,127 @@ +# V2 RF2 Runtime Bounded Envelope + +Date: 2026-04-05 +Status: draft +Purpose: freeze the bounded productionization envelope around the current +`Phase 19` working RF2 block path without overclaiming broad product readiness + +## Reading Rule + +This document defines the strongest bounded envelope currently justified by the +delivered `Phase 19` path. + +It does NOT mean: + +1. broad launch approval +2. working block product approval +3. support for broad frontend or transport matrices +4. that remaining runtime/product gaps are minor polish + +It means only: + +1. the current `masterv2 + volumev2 + purev2` RF2 runtime slice has a named, + reviewable productionization boundary +2. the current support statement, exclusions, and blockers are explicit +3. later pilot or rollout work must stay inside this envelope or explicitly widen + it with new evidence + +## Envelope Basis + +This envelope is anchored on the delivered `Phase 19` milestones: + +1. `M6`: one live loopback HTTP transport now exists behind the evidence seam +2. `M7`: one background Loop 2 service and one bounded auto-failover service now + exist +3. `M8`: one runtime-managed iSCSI export path and one bounded replica repair + wrapper now exist +4. `M9`: one end-to-end RF2 handoff proof now exists with continued I/O on the + new primary +5. `M10`: one bounded operator surface and one bounded CSI runtime backend + adapter now exist + +The envelope is therefore about one bounded working RF2 block path, not broad +product readiness. + +## Supported Envelope + +The current bounded support statement is: + +1. one bounded working RF2 block path now exists with: + - `masterv2` identity/promotion authority + - `volumev2` failover, takeover, active Loop 2 service, continuity, repair, + frontend rebinding, and projected RF2 surface ownership + - `purev2` execution adapter reuse +2. one bounded live transport path now carries failover-time evidence and replica + summaries +3. one bounded real client handoff path now exists: + - write through runtime-managed iSCSI export + - bounded repair/catch-up on the runtime path + - lose primary + - auto fail over + - reconnect to the new primary + - continue I/O +4. one bounded outward RF2 surface exists as projection only: + - `RF2VolumeSurface` +5. one bounded operator/CSI adapter layer exists on top of runtime-owned truth + +## Explicit Exclusions + +The following are OUTSIDE this bounded envelope: + +1. broad multi-process or multi-host deployment approval +2. broad transport/frontend matrix approval +3. full rebuild orchestration beyond the current bounded repair/catch-up wrapper +4. broad CSI lifecycle parity beyond the current bounded runtime backend adapter +5. broad operator/API/metrics coverage beyond the current bounded HTTP surface +6. broad launch or external customer support statements + +## Current Blockers + +The main blockers between this envelope and a working RF2 block product are: + +1. the current path is still bounded to the current runtime harness rather than + broad multi-process approval +2. bounded repair/catch-up is not yet broad rebuild lifecycle closure +3. CSI rebinding is still a bounded runtime backend adapter, not full lifecycle + parity +4. the operator surface is still a bounded HTTP view, not a full operational + platform surface + +## Allowed Validation Shape + +The allowed validation shape inside this envelope is: + +1. internal engineering validation only +2. bounded lab/runtime exercise only +3. explicit artifact-driven interpretation only + +The following are NOT allowed interpretations: + +1. "the system is now production ready" +2. "the system now supports real automatic failover" +3. "the system now supports broad product traffic and rollout" + +## Evidence Anchors + +Read this envelope together with: + +1. `sw-block/.private/phase/phase-19.md` +2. `sw-block/design/v2-kernel-closure-review.md` +3. `sw-block/design/v2-protocol-claim-and-evidence.md` +4. `sw-block/runtime/volumev2/runtime_manager.go` +5. `sw-block/runtime/volumev2/continuity_runtime.go` +6. `sw-block/runtime/volumev2/rf2_surface.go` +7. `sw-block/runtime/volumev2/loop2_service.go` +8. `sw-block/runtime/volumev2/frontend_runtime.go` +9. `sw-block/runtime/volumev2/operator_surface.go` +10. `sw-block/runtime/volumev2/poc_test.go` +11. `weed/storage/blockvol/csi/v2_runtime_backend.go` + +## Envelope Output + +The correct current reading of this envelope is: + +1. runtime-bearing RF2 kernel slice: yes +2. bounded working RF2 block path: yes +3. bounded productionization artifact set: yes +4. pilot-ready broad product path: no diff --git a/sw-block/runtime/volumev2/bootstrap_runtime.go b/sw-block/runtime/volumev2/bootstrap_runtime.go new file mode 100644 index 000000000..2766245eb --- /dev/null +++ b/sw-block/runtime/volumev2/bootstrap_runtime.go @@ -0,0 +1,39 @@ +package volumev2 + +import ( + "fmt" + "time" + + "github.com/seaweedfs/seaweedfs/sw-block/runtime/masterv2" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" +) + +// BootstrapPrimaryVolume creates or opens one named volume on one selected node +// and applies a bounded primary assignment through the runtime-owned path. +func (m *InProcessRuntimeManager) BootstrapPrimaryVolume(volumeName, nodeID, path string, epoch uint64, opts blockvol.CreateOptions) error { + if m == nil { + return fmt.Errorf("volumev2: runtime manager is nil") + } + if volumeName == "" { + return fmt.Errorf("volumev2: bootstrap volume name is required") + } + if nodeID == "" { + return fmt.Errorf("volumev2: bootstrap node id is required") + } + if path == "" { + return fmt.Errorf("volumev2: bootstrap path is required") + } + node, err := m.localNode(nodeID) + if err != nil { + return err + } + return node.ApplyAssignments([]masterv2.Assignment{{ + Name: volumeName, + Path: path, + NodeID: nodeID, + Epoch: epoch, + LeaseTTL: 30 * time.Second, + CreateOptions: opts, + Role: "primary", + }}) +} diff --git a/sw-block/runtime/volumev2/continuity_runtime.go b/sw-block/runtime/volumev2/continuity_runtime.go index ff4ee5b9d..4c8298a51 100644 --- a/sw-block/runtime/volumev2/continuity_runtime.go +++ b/sw-block/runtime/volumev2/continuity_runtime.go @@ -19,13 +19,21 @@ type ReplicatedContinuityResult struct { DataMatch bool } +// ReplicatedContinuitySnapshot is the read-only observable result of the most +// recent bounded continuity run for one volume. +type ReplicatedContinuitySnapshot struct { + VolumeName string + LastError string + Result ReplicatedContinuityResult +} + // ExecuteReplicatedContinuity runs one bounded continuity statement through the // current runtime: // mirror writes to the bounded participant set -> observe active Loop 2 -> // fail over to the survivor set -> read back data from the newly selected // primary. This is a bounded continuity closure, not a full replication product // claim. -func (m *InProcessRuntimeManager) ExecuteReplicatedContinuity(volumeName, sourcePrimaryNodeID string, expectedEpoch uint64, survivorNodeIDs []string, lba uint64, payload []byte) (ReplicatedContinuityResult, error) { +func (m *InProcessRuntimeManager) ExecuteReplicatedContinuity(volumeName, sourcePrimaryNodeID string, expectedEpoch uint64, survivorNodeIDs []string, lba uint64, payload []byte) (result ReplicatedContinuityResult, runErr error) { if m == nil { return ReplicatedContinuityResult{}, fmt.Errorf("volumev2: runtime manager is nil") } @@ -42,59 +50,77 @@ func (m *InProcessRuntimeManager) ExecuteReplicatedContinuity(volumeName, source return ReplicatedContinuityResult{}, fmt.Errorf("volumev2: continuity survivor node ids are required") } + result = ReplicatedContinuityResult{ + VolumeName: volumeName, + SourcePrimaryNodeID: sourcePrimaryNodeID, + ExpectedEpoch: expectedEpoch, + } + defer func() { + snapshot := ReplicatedContinuitySnapshot{ + VolumeName: volumeName, + Result: result, + } + if runErr != nil { + snapshot.LastError = runErr.Error() + } + m.recordContinuitySnapshot(volumeName, snapshot) + }() + nodeIDs := append([]string{sourcePrimaryNodeID}, survivorNodeIDs...) nodeIDs = uniqueSorted(nodeIDs) nodes := make([]*Node, 0, len(nodeIDs)) for _, nodeID := range nodeIDs { node, err := m.localNode(nodeID) if err != nil { - return ReplicatedContinuityResult{}, err + runErr = err + return result, runErr } nodes = append(nodes, node) } for _, node := range nodes { if err := node.WriteLBA(volumeName, lba, payload); err != nil { - return ReplicatedContinuityResult{}, fmt.Errorf("volumev2: continuity write %s on %s: %w", volumeName, node.NodeID(), err) + runErr = fmt.Errorf("volumev2: continuity write %s on %s: %w", volumeName, node.NodeID(), err) + return result, runErr } } for _, node := range nodes { if err := node.SyncCache(volumeName); err != nil { - return ReplicatedContinuityResult{}, fmt.Errorf("volumev2: continuity sync %s on %s: %w", volumeName, node.NodeID(), err) + runErr = fmt.Errorf("volumev2: continuity sync %s on %s: %w", volumeName, node.NodeID(), err) + return result, runErr } } - result := ReplicatedContinuityResult{ - VolumeName: volumeName, - SourcePrimaryNodeID: sourcePrimaryNodeID, - ExpectedEpoch: expectedEpoch, - } - loop2Snap, err := m.ObserveLoop2(volumeName, sourcePrimaryNodeID, expectedEpoch, nodeIDs...) if err != nil { - return result, err + runErr = err + return result, runErr } result.Loop2BeforeFailover = loop2Snap failover, err := m.ExecuteFailover(volumeName, expectedEpoch, survivorNodeIDs...) result.Failover = failover if err != nil { - return result, err + runErr = err + return result, runErr } result.SelectedPrimaryNodeID = failover.Assignment.NodeID selectedNode, err := m.localNode(failover.Assignment.NodeID) if err != nil { - return result, err + runErr = err + return result, runErr } readBack, err := selectedNode.ReadLBA(volumeName, lba, uint32(len(payload))) if err != nil { - return result, fmt.Errorf("volumev2: continuity readback %s on %s: %w", volumeName, selectedNode.NodeID(), err) + runErr = fmt.Errorf("volumev2: continuity readback %s on %s: %w", volumeName, selectedNode.NodeID(), err) + return result, runErr } result.ReadBackLength = uint32(len(readBack)) result.DataMatch = bytes.Equal(readBack, payload) if !result.DataMatch { - return result, fmt.Errorf("volumev2: continuity payload mismatch after failover") + runErr = fmt.Errorf("volumev2: continuity payload mismatch after failover") + return result, runErr } return result, nil } diff --git a/sw-block/runtime/volumev2/failover_evidence_transport.go b/sw-block/runtime/volumev2/failover_evidence_transport.go index f0bdb4286..17e5dc269 100644 --- a/sw-block/runtime/volumev2/failover_evidence_transport.go +++ b/sw-block/runtime/volumev2/failover_evidence_transport.go @@ -1,8 +1,14 @@ package volumev2 import ( + "bytes" + "context" + "encoding/json" "fmt" + "net" + "net/http" "sync" + "time" "github.com/seaweedfs/seaweedfs/sw-block/runtime/masterv2" "github.com/seaweedfs/seaweedfs/sw-block/runtime/protocolv2" @@ -16,6 +22,16 @@ type FailoverEvidenceTransport interface { QueryReplicaSummary(nodeID string, req protocolv2.ReplicaSummaryRequest) (protocolv2.ReplicaSummaryResponse, error) } +// ManagedFailoverEvidenceTransport is the runtime-owned transport surface used +// by the runtime manager. It keeps the adapter query shape stable while allowing +// different registration mechanics underneath. +type ManagedFailoverEvidenceTransport interface { + FailoverEvidenceTransport + RegisterHandler(nodeID string, handler FailoverEvidenceHandler) error + UnregisterHandler(nodeID string) + Close() error +} + // FailoverEvidenceHandler is the server-side evidence surface registered behind // a transport implementation. type FailoverEvidenceHandler interface { @@ -118,6 +134,11 @@ func (t *InMemoryFailoverEvidenceTransport) UnregisterHandler(nodeID string) { delete(t.handlers, nodeID) } +// Close releases any resources owned by the transport. +func (t *InMemoryFailoverEvidenceTransport) Close() error { + return nil +} + func (t *InMemoryFailoverEvidenceTransport) QueryPromotionEvidence(nodeID string, req masterv2.PromotionQueryRequest) (masterv2.PromotionQueryResponse, error) { handler, err := t.handler(nodeID) if err != nil { @@ -149,3 +170,215 @@ func (t *InMemoryFailoverEvidenceTransport) handler(nodeID string) (FailoverEvid } return handler, nil } + +type httpEvidenceServer struct { + baseURL string + server *http.Server + listener net.Listener +} + +// HTTPFailoverEvidenceTransport carries failover-time evidence over a real +// loopback HTTP transport while keeping takeover primary-local. +type HTTPFailoverEvidenceTransport struct { + mu sync.RWMutex + client *http.Client + servers map[string]*httpEvidenceServer +} + +// NewHTTPFailoverEvidenceTransport creates one managed loopback HTTP transport. +func NewHTTPFailoverEvidenceTransport() *HTTPFailoverEvidenceTransport { + return &HTTPFailoverEvidenceTransport{ + client: &http.Client{Timeout: 2 * time.Second}, + servers: make(map[string]*httpEvidenceServer), + } +} + +// RegisterHandler binds one stable node id behind one loopback HTTP server. +func (t *HTTPFailoverEvidenceTransport) RegisterHandler(nodeID string, handler FailoverEvidenceHandler) error { + if t == nil { + return fmt.Errorf("volumev2: http evidence transport is nil") + } + if nodeID == "" { + return fmt.Errorf("volumev2: http evidence handler node id is required") + } + if handler == nil { + return fmt.Errorf("volumev2: http evidence handler %q is nil", nodeID) + } + + mux := http.NewServeMux() + mux.HandleFunc("/promotion-evidence", func(w http.ResponseWriter, r *http.Request) { + var req masterv2.PromotionQueryRequest + if err := decodeEvidenceJSON(r, &req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + resp, err := handler.QueryPromotionEvidence(req) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + writeEvidenceJSON(w, resp) + }) + mux.HandleFunc("/replica-summary", func(w http.ResponseWriter, r *http.Request) { + var req protocolv2.ReplicaSummaryRequest + if err := decodeEvidenceJSON(r, &req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + resp, err := handler.QueryReplicaSummary(req) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + writeEvidenceJSON(w, resp) + }) + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return fmt.Errorf("volumev2: http evidence listen %q: %w", nodeID, err) + } + server := &http.Server{Handler: mux} + entry := &httpEvidenceServer{ + baseURL: "http://" + ln.Addr().String(), + server: server, + listener: ln, + } + + t.mu.Lock() + if prev, ok := t.servers[nodeID]; ok { + t.mu.Unlock() + _ = prev.server.Shutdown(context.Background()) + _ = prev.listener.Close() + t.mu.Lock() + } + t.servers[nodeID] = entry + t.mu.Unlock() + + go func() { + _ = server.Serve(ln) + }() + return nil +} + +// UnregisterHandler stops one loopback HTTP handler. +func (t *HTTPFailoverEvidenceTransport) UnregisterHandler(nodeID string) { + if t == nil || nodeID == "" { + return + } + t.mu.Lock() + entry, ok := t.servers[nodeID] + if ok { + delete(t.servers, nodeID) + } + t.mu.Unlock() + if !ok { + return + } + _ = entry.server.Shutdown(context.Background()) + _ = entry.listener.Close() +} + +// Close stops all registered loopback HTTP handlers. +func (t *HTTPFailoverEvidenceTransport) Close() error { + if t == nil { + return nil + } + t.mu.Lock() + entries := make([]*httpEvidenceServer, 0, len(t.servers)) + for nodeID, entry := range t.servers { + entries = append(entries, entry) + delete(t.servers, nodeID) + } + t.mu.Unlock() + for _, entry := range entries { + _ = entry.server.Shutdown(context.Background()) + _ = entry.listener.Close() + } + return nil +} + +func (t *HTTPFailoverEvidenceTransport) QueryPromotionEvidence(nodeID string, req masterv2.PromotionQueryRequest) (masterv2.PromotionQueryResponse, error) { + endpoint, err := t.endpoint(nodeID, "/promotion-evidence") + if err != nil { + return masterv2.PromotionQueryResponse{}, err + } + var resp masterv2.PromotionQueryResponse + if err := t.postJSON(endpoint, req, &resp); err != nil { + return masterv2.PromotionQueryResponse{}, err + } + return resp, nil +} + +func (t *HTTPFailoverEvidenceTransport) QueryReplicaSummary(nodeID string, req protocolv2.ReplicaSummaryRequest) (protocolv2.ReplicaSummaryResponse, error) { + endpoint, err := t.endpoint(nodeID, "/replica-summary") + if err != nil { + return protocolv2.ReplicaSummaryResponse{}, err + } + var resp protocolv2.ReplicaSummaryResponse + if err := t.postJSON(endpoint, req, &resp); err != nil { + return protocolv2.ReplicaSummaryResponse{}, err + } + return resp, nil +} + +func (t *HTTPFailoverEvidenceTransport) endpoint(nodeID, suffix string) (string, error) { + if t == nil { + return "", fmt.Errorf("volumev2: http evidence transport is nil") + } + if nodeID == "" { + return "", fmt.Errorf("volumev2: http evidence node id is required") + } + t.mu.RLock() + defer t.mu.RUnlock() + entry, ok := t.servers[nodeID] + if !ok { + return "", fmt.Errorf("volumev2: unknown http evidence handler %q", nodeID) + } + return entry.baseURL + suffix, nil +} + +func (t *HTTPFailoverEvidenceTransport) postJSON(endpoint string, req any, resp any) error { + if t == nil { + return fmt.Errorf("volumev2: http evidence transport is nil") + } + body, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("volumev2: marshal evidence request: %w", err) + } + httpReq, err := http.NewRequest(http.MethodPost, endpoint, bytes.NewReader(body)) + if err != nil { + return fmt.Errorf("volumev2: build evidence request: %w", err) + } + httpReq.Header.Set("Content-Type", "application/json") + httpResp, err := t.client.Do(httpReq) + if err != nil { + return fmt.Errorf("volumev2: http evidence request: %w", err) + } + defer httpResp.Body.Close() + if httpResp.StatusCode != http.StatusOK { + return fmt.Errorf("volumev2: http evidence status %d", httpResp.StatusCode) + } + if err := json.NewDecoder(httpResp.Body).Decode(resp); err != nil { + return fmt.Errorf("volumev2: decode evidence response: %w", err) + } + return nil +} + +func decodeEvidenceJSON(r *http.Request, dst any) error { + if r == nil || r.Body == nil { + return fmt.Errorf("volumev2: empty evidence request") + } + defer r.Body.Close() + if r.Method != http.MethodPost { + return fmt.Errorf("volumev2: unsupported evidence method %q", r.Method) + } + if err := json.NewDecoder(r.Body).Decode(dst); err != nil { + return fmt.Errorf("volumev2: decode evidence request: %w", err) + } + return nil +} + +func writeEvidenceJSON(w http.ResponseWriter, value any) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(value) +} diff --git a/sw-block/runtime/volumev2/frontend_runtime.go b/sw-block/runtime/volumev2/frontend_runtime.go new file mode 100644 index 000000000..74ffe867d --- /dev/null +++ b/sw-block/runtime/volumev2/frontend_runtime.go @@ -0,0 +1,178 @@ +package volumev2 + +import ( + "bytes" + "fmt" +) + +type managedISCSIExport struct { + nodeID string + handle *ISCSITargetExport +} + +// ManagedISCSIExportSnapshot is the runtime-owned serving snapshot for one iSCSI +// export bound to the current runtime path. +type ManagedISCSIExportSnapshot struct { + VolumeName string + NodeID string + IQN string + Address string +} + +// ReplicaRepairResult captures one bounded replica repair/catch-up run through +// the runtime-owned path. +type ReplicaRepairResult struct { + VolumeName string + PrimaryNodeID string + ReplicaNodeID string + ExpectedEpoch uint64 + ByteLength uint32 + DataMatch bool + Loop2Before Loop2RuntimeSnapshot + Loop2After Loop2RuntimeSnapshot +} + +// ExportVolumeISCSI binds one named volume on one selected node to an iSCSI +// export owned by the runtime manager. +func (m *InProcessRuntimeManager) ExportVolumeISCSI(volumeName, nodeID, listenAddr, iqn string) (ManagedISCSIExportSnapshot, error) { + if m == nil { + return ManagedISCSIExportSnapshot{}, fmt.Errorf("volumev2: runtime manager is nil") + } + if volumeName == "" { + return ManagedISCSIExportSnapshot{}, fmt.Errorf("volumev2: iscsi export volume name is required") + } + if nodeID == "" { + return ManagedISCSIExportSnapshot{}, fmt.Errorf("volumev2: iscsi export node id is required") + } + node, err := m.localNode(nodeID) + if err != nil { + return ManagedISCSIExportSnapshot{}, err + } + export, err := node.ExportISCSI(volumeName, listenAddr, iqn) + if err != nil { + return ManagedISCSIExportSnapshot{}, err + } + + m.mu.Lock() + if prev, ok := m.frontendExports[volumeName]; ok && prev != nil && prev.handle != nil { + _ = prev.handle.Close() + } + m.frontendExports[volumeName] = &managedISCSIExport{ + nodeID: nodeID, + handle: export, + } + m.mu.Unlock() + + return ManagedISCSIExportSnapshot{ + VolumeName: volumeName, + NodeID: nodeID, + IQN: export.IQN(), + Address: export.Address(), + }, nil +} + +// ExportCurrentPrimaryISCSI binds one named volume to the current primary known +// to the runtime manager. +func (m *InProcessRuntimeManager) ExportCurrentPrimaryISCSI(volumeName, listenAddr, iqn string) (ManagedISCSIExportSnapshot, error) { + if m == nil { + return ManagedISCSIExportSnapshot{}, fmt.Errorf("volumev2: runtime manager is nil") + } + if volumeName == "" { + return ManagedISCSIExportSnapshot{}, fmt.Errorf("volumev2: current primary export volume name is required") + } + if result, ok := m.FailoverResult(volumeName); ok && result.Assignment.NodeID != "" { + return m.ExportVolumeISCSI(volumeName, result.Assignment.NodeID, listenAddr, iqn) + } + if snapshot, ok := m.Loop2Snapshot(volumeName); ok && snapshot.PrimaryNodeID != "" { + return m.ExportVolumeISCSI(volumeName, snapshot.PrimaryNodeID, listenAddr, iqn) + } + return ManagedISCSIExportSnapshot{}, fmt.Errorf("volumev2: current primary for %q is unknown", volumeName) +} + +// ISCSIExport returns the current runtime-owned iSCSI export snapshot for one +// volume if present. +func (m *InProcessRuntimeManager) ISCSIExport(volumeName string) (ManagedISCSIExportSnapshot, bool) { + if m == nil { + return ManagedISCSIExportSnapshot{}, false + } + m.mu.RLock() + defer m.mu.RUnlock() + export, ok := m.frontendExports[volumeName] + if !ok || export == nil || export.handle == nil { + return ManagedISCSIExportSnapshot{}, false + } + return ManagedISCSIExportSnapshot{ + VolumeName: volumeName, + NodeID: export.nodeID, + IQN: export.handle.IQN(), + Address: export.handle.Address(), + }, true +} + +// RepairReplicaFromPrimary performs one bounded runtime-owned replica repair by +// copying the requested byte range from the selected primary to the selected +// replica and re-observing Loop 2 before and after the repair. +func (m *InProcessRuntimeManager) RepairReplicaFromPrimary(volumeName, primaryNodeID, replicaNodeID string, expectedEpoch uint64, length uint32) (ReplicaRepairResult, error) { + if m == nil { + return ReplicaRepairResult{}, fmt.Errorf("volumev2: runtime manager is nil") + } + if volumeName == "" { + return ReplicaRepairResult{}, fmt.Errorf("volumev2: repair volume name is required") + } + if primaryNodeID == "" || replicaNodeID == "" { + return ReplicaRepairResult{}, fmt.Errorf("volumev2: repair primary and replica node ids are required") + } + if primaryNodeID == replicaNodeID { + return ReplicaRepairResult{}, fmt.Errorf("volumev2: repair requires distinct primary and replica nodes") + } + if length == 0 { + return ReplicaRepairResult{}, fmt.Errorf("volumev2: repair byte length is required") + } + + result := ReplicaRepairResult{ + VolumeName: volumeName, + PrimaryNodeID: primaryNodeID, + ReplicaNodeID: replicaNodeID, + ExpectedEpoch: expectedEpoch, + ByteLength: length, + } + before, err := m.ObserveLoop2(volumeName, primaryNodeID, expectedEpoch, primaryNodeID, replicaNodeID) + if err != nil { + return result, err + } + result.Loop2Before = before + + primary, err := m.localNode(primaryNodeID) + if err != nil { + return result, err + } + replica, err := m.localNode(replicaNodeID) + if err != nil { + return result, err + } + payload, err := primary.ReadLBA(volumeName, 0, length) + if err != nil { + return result, fmt.Errorf("volumev2: repair readback from %s: %w", primaryNodeID, err) + } + if err := replica.WriteLBA(volumeName, 0, payload); err != nil { + return result, fmt.Errorf("volumev2: repair write to %s: %w", replicaNodeID, err) + } + if err := replica.SyncCache(volumeName); err != nil { + return result, fmt.Errorf("volumev2: repair sync %s: %w", replicaNodeID, err) + } + replicaReadBack, err := replica.ReadLBA(volumeName, 0, length) + if err != nil { + return result, fmt.Errorf("volumev2: repair verify read %s: %w", replicaNodeID, err) + } + result.DataMatch = bytes.Equal(replicaReadBack, payload) + if !result.DataMatch { + return result, fmt.Errorf("volumev2: repair payload mismatch after copy") + } + + after, err := m.ObserveLoop2(volumeName, primaryNodeID, expectedEpoch, primaryNodeID, replicaNodeID) + if err != nil { + return result, err + } + result.Loop2After = after + return result, nil +} diff --git a/sw-block/runtime/volumev2/loop2_service.go b/sw-block/runtime/volumev2/loop2_service.go new file mode 100644 index 000000000..9b6b4b443 --- /dev/null +++ b/sw-block/runtime/volumev2/loop2_service.go @@ -0,0 +1,252 @@ +package volumev2 + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/seaweedfs/seaweedfs/sw-block/runtime/protocolv2" +) + +// Loop2ServiceConfig defines one bounded background Loop 2 observation service. +type Loop2ServiceConfig struct { + VolumeName string + PrimaryNodeID string + ExpectedEpoch uint64 + NodeIDs []string + Interval time.Duration +} + +// Loop2ServiceHandle controls one bounded background Loop 2 service. +type Loop2ServiceHandle struct { + cancel context.CancelFunc + done chan struct{} +} + +// Stop terminates the background Loop 2 service. +func (h *Loop2ServiceHandle) Stop() { + if h == nil || h.cancel == nil { + return + } + h.cancel() +} + +// Wait blocks until the background Loop 2 service exits. +func (h *Loop2ServiceHandle) Wait() { + if h == nil || h.done == nil { + return + } + <-h.done +} + +// AutoFailoverConfig defines one bounded background auto-failover service. +type AutoFailoverConfig struct { + VolumeName string + PrimaryNodeID string + ExpectedEpoch uint64 + NodeIDs []string + Interval time.Duration +} + +// AutoFailoverHandle controls one bounded background auto-failover service. +type AutoFailoverHandle struct { + cancel context.CancelFunc + done chan struct{} + + mu sync.RWMutex + primaryNodeID string + expectedEpoch uint64 + lastLoop2Error error +} + +// Stop terminates the background auto-failover service. +func (h *AutoFailoverHandle) Stop() { + if h == nil || h.cancel == nil { + return + } + h.cancel() +} + +// Wait blocks until the background auto-failover service exits. +func (h *AutoFailoverHandle) Wait() { + if h == nil || h.done == nil { + return + } + <-h.done +} + +// PrimaryNodeID returns the current primary tracked by the service. +func (h *AutoFailoverHandle) PrimaryNodeID() string { + if h == nil { + return "" + } + h.mu.RLock() + defer h.mu.RUnlock() + return h.primaryNodeID +} + +// ExpectedEpoch returns the current expected epoch tracked by the service. +func (h *AutoFailoverHandle) ExpectedEpoch() uint64 { + if h == nil { + return 0 + } + h.mu.RLock() + defer h.mu.RUnlock() + return h.expectedEpoch +} + +// LastLoop2Error returns the most recent non-fatal Loop 2 observation error. +func (h *AutoFailoverHandle) LastLoop2Error() error { + if h == nil { + return fmt.Errorf("volumev2: auto failover handle is nil") + } + h.mu.RLock() + defer h.mu.RUnlock() + return h.lastLoop2Error +} + +// StartLoop2Service starts one bounded background Loop 2 observation service. +func (m *InProcessRuntimeManager) StartLoop2Service(cfg Loop2ServiceConfig) (*Loop2ServiceHandle, error) { + if err := validateLoop2ServiceConfig(cfg); err != nil { + return nil, err + } + ctx, cancel := context.WithCancel(context.Background()) + handle := &Loop2ServiceHandle{ + cancel: cancel, + done: make(chan struct{}), + } + go func() { + defer close(handle.done) + ticker := time.NewTicker(normalizeLoop2Interval(cfg.Interval)) + defer ticker.Stop() + for { + _, _ = m.ObserveLoop2(cfg.VolumeName, cfg.PrimaryNodeID, cfg.ExpectedEpoch, cfg.NodeIDs...) + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } + }() + return handle, nil +} + +// StartAutoFailoverService starts one bounded background auto-failover service. +// The current bounded trigger rule is intentionally fail-closed: only explicit +// primary evidence-query loss triggers failover. Healthy-but-lagging or +// ambiguous runtime states are observed but do not trigger automatically. +func (m *InProcessRuntimeManager) StartAutoFailoverService(cfg AutoFailoverConfig) (*AutoFailoverHandle, error) { + if err := validateAutoFailoverConfig(cfg); err != nil { + return nil, err + } + ctx, cancel := context.WithCancel(context.Background()) + handle := &AutoFailoverHandle{ + cancel: cancel, + done: make(chan struct{}), + primaryNodeID: cfg.PrimaryNodeID, + expectedEpoch: cfg.ExpectedEpoch, + } + go func() { + defer close(handle.done) + ticker := time.NewTicker(normalizeLoop2Interval(cfg.Interval)) + defer ticker.Stop() + for { + handle.mu.RLock() + primaryNodeID := handle.primaryNodeID + expectedEpoch := handle.expectedEpoch + handle.mu.RUnlock() + + err := m.probeReplicaSummary(primaryNodeID, cfg.VolumeName, expectedEpoch) + if err == nil { + _, obsErr := m.ObserveLoop2(cfg.VolumeName, primaryNodeID, expectedEpoch, cfg.NodeIDs...) + handle.mu.Lock() + handle.lastLoop2Error = obsErr + handle.mu.Unlock() + } else { + survivors := excludeNodeID(cfg.NodeIDs, primaryNodeID) + if len(survivors) > 0 { + result, failoverErr := m.ExecuteFailover(cfg.VolumeName, expectedEpoch, survivors...) + if failoverErr == nil { + handle.mu.Lock() + handle.primaryNodeID = result.Assignment.NodeID + if result.Assignment.Epoch != 0 { + handle.expectedEpoch = result.Assignment.Epoch + } + handle.lastLoop2Error = nil + handle.mu.Unlock() + } + } + } + + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } + }() + return handle, nil +} + +func validateLoop2ServiceConfig(cfg Loop2ServiceConfig) error { + if cfg.VolumeName == "" { + return fmt.Errorf("volumev2: loop2 service volume name is required") + } + if cfg.PrimaryNodeID == "" { + return fmt.Errorf("volumev2: loop2 service primary node id is required") + } + if len(cfg.NodeIDs) == 0 { + return fmt.Errorf("volumev2: loop2 service node ids are required") + } + return nil +} + +func validateAutoFailoverConfig(cfg AutoFailoverConfig) error { + if cfg.VolumeName == "" { + return fmt.Errorf("volumev2: auto failover volume name is required") + } + if cfg.PrimaryNodeID == "" { + return fmt.Errorf("volumev2: auto failover primary node id is required") + } + if len(cfg.NodeIDs) == 0 { + return fmt.Errorf("volumev2: auto failover node ids are required") + } + return nil +} + +func normalizeLoop2Interval(interval time.Duration) time.Duration { + if interval <= 0 { + return 25 * time.Millisecond + } + return interval +} + +func excludeNodeID(nodeIDs []string, excluded string) []string { + out := make([]string, 0, len(nodeIDs)) + for _, nodeID := range nodeIDs { + if nodeID == "" || nodeID == excluded { + continue + } + out = append(out, nodeID) + } + return out +} + +func (m *InProcessRuntimeManager) probeReplicaSummary(nodeID, volumeName string, expectedEpoch uint64) error { + if m == nil || m.driver == nil { + return fmt.Errorf("volumev2: runtime manager is nil") + } + targets, err := m.driver.resolveTargets([]string{nodeID}) + if err != nil { + return err + } + if len(targets) != 1 || targets[0].Evidence == nil { + return fmt.Errorf("volumev2: primary evidence target %q is unavailable", nodeID) + } + _, err = targets[0].Evidence.QueryReplicaSummary(protocolv2.ReplicaSummaryRequest{ + VolumeName: volumeName, + ExpectedEpoch: expectedEpoch, + }) + return err +} diff --git a/sw-block/runtime/volumev2/operator_surface.go b/sw-block/runtime/volumev2/operator_surface.go new file mode 100644 index 000000000..8d7625e1c --- /dev/null +++ b/sw-block/runtime/volumev2/operator_surface.go @@ -0,0 +1,123 @@ +package volumev2 + +import ( + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "strings" +) + +// OperatorSurfaceHandle controls one bounded HTTP operator surface. +type OperatorSurfaceHandle struct { + addr string + server *http.Server + listener net.Listener +} + +// Address returns the listen address of the operator surface. +func (h *OperatorSurfaceHandle) Address() string { + if h == nil { + return "" + } + return h.addr +} + +// Close stops the operator surface. +func (h *OperatorSurfaceHandle) Close() error { + if h == nil { + return nil + } + if h.server != nil { + _ = h.server.Shutdown(context.Background()) + } + if h.listener != nil { + return h.listener.Close() + } + return nil +} + +// StartOperatorSurface starts one bounded HTTP operator surface over the +// runtime-owned snapshots and projections. +func (m *InProcessRuntimeManager) StartOperatorSurface(listenAddr string) (*OperatorSurfaceHandle, error) { + if m == nil { + return nil, fmt.Errorf("volumev2: runtime manager is nil") + } + if listenAddr == "" { + listenAddr = "127.0.0.1:0" + } + mux := http.NewServeMux() + mux.HandleFunc("/v1/volumes/", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + path := strings.TrimPrefix(r.URL.Path, "/v1/volumes/") + parts := strings.Split(strings.Trim(path, "/"), "/") + if len(parts) != 2 { + http.NotFound(w, r) + return + } + volumeName, viewName := parts[0], parts[1] + switch viewName { + case "rf2": + value, ok := m.RF2VolumeSurface(volumeName) + if !ok { + http.NotFound(w, r) + return + } + writeOperatorJSON(w, value) + case "failover": + value, ok := m.FailoverSnapshot(volumeName) + if !ok { + http.NotFound(w, r) + return + } + writeOperatorJSON(w, value) + case "loop2": + value, ok := m.Loop2Snapshot(volumeName) + if !ok { + http.NotFound(w, r) + return + } + writeOperatorJSON(w, value) + case "continuity": + value, ok := m.ReplicatedContinuitySnapshot(volumeName) + if !ok { + http.NotFound(w, r) + return + } + writeOperatorJSON(w, value) + case "frontend": + value, ok := m.ISCSIExport(volumeName) + if !ok { + http.NotFound(w, r) + return + } + writeOperatorJSON(w, value) + default: + http.NotFound(w, r) + } + }) + + ln, err := net.Listen("tcp", listenAddr) + if err != nil { + return nil, fmt.Errorf("volumev2: operator surface listen: %w", err) + } + server := &http.Server{Handler: mux} + handle := &OperatorSurfaceHandle{ + addr: ln.Addr().String(), + server: server, + listener: ln, + } + go func() { + _ = server.Serve(ln) + }() + return handle, nil +} + +func writeOperatorJSON(w http.ResponseWriter, value any) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(value) +} diff --git a/sw-block/runtime/volumev2/poc_test.go b/sw-block/runtime/volumev2/poc_test.go index 75c55cd04..ab28b94ce 100644 --- a/sw-block/runtime/volumev2/poc_test.go +++ b/sw-block/runtime/volumev2/poc_test.go @@ -2,7 +2,10 @@ package volumev2 import ( "bytes" + "encoding/binary" + "encoding/json" "fmt" + "net/http" "path/filepath" "testing" "time" @@ -10,6 +13,7 @@ import ( "github.com/seaweedfs/seaweedfs/sw-block/runtime/masterv2" "github.com/seaweedfs/seaweedfs/sw-block/runtime/protocolv2" "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/iscsi" ) func TestPOC_MasterV2VolumeV2_RF1HeartbeatAssignmentFlow(t *testing.T) { @@ -1519,6 +1523,929 @@ func TestTransportEvidenceAdapter_GatedFailoverFlow(t *testing.T) { } } +func TestHTTPTransportEvidenceAdapter_HealthyFailoverFlow(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + transport := NewHTTPFailoverEvidenceTransport() + defer transport.Close() + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + nodeC, err := New(Config{NodeID: "node-c"}) + if err != nil { + t.Fatalf("new node-c: %v", err) + } + defer nodeC.Close() + if err := transport.RegisterHandler("node-b", nodeB); err != nil { + t.Fatalf("register node-b handler: %v", err) + } + if err := transport.RegisterHandler("node-c", nodeC); err != nil { + t.Fatalf("register node-c handler: %v", err) + } + + tempDir := t.TempDir() + pathB := filepath.Join(tempDir, "http-transport-b.blk") + pathC := filepath.Join(tempDir, "http-transport-c.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "http-transport-vol", + Path: pathB, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := nodeB.ApplyAssignments([]masterv2.Assignment{{ + Name: "http-transport-vol", + Path: pathB, + NodeID: "node-b", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-b: %v", err) + } + if err := nodeC.ApplyAssignments([]masterv2.Assignment{{ + Name: "http-transport-vol", + Path: pathC, + NodeID: "node-c", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-c: %v", err) + } + if err := nodeB.WriteLBA("http-transport-vol", 0, bytes.Repeat([]byte{0x71}, 4096)); err != nil { + t.Fatalf("write node-b: %v", err) + } + if err := nodeB.SyncCache("http-transport-vol"); err != nil { + t.Fatalf("sync node-b: %v", err) + } + + result, err := ExecuteFailoverFlow(master, "http-transport-vol", 2, []FailoverTarget{ + mustHybridFailoverTarget(t, nodeB, transport), + mustHybridFailoverTarget(t, nodeC, transport), + }) + if err != nil { + t.Fatalf("http transport failover flow: %v", err) + } + if result.Assignment.NodeID != "node-b" { + t.Fatalf("assignment node=%q, want node-b", result.Assignment.NodeID) + } + if result.Truth.PrimaryNodeID != "node-b" { + t.Fatalf("truth primary=%q, want node-b", result.Truth.PrimaryNodeID) + } +} + +func TestHTTPTransportEvidenceAdapter_GatedFailoverFlow(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + transport := NewHTTPFailoverEvidenceTransport() + defer transport.Close() + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + if err := transport.RegisterHandler("node-b", nodeB); err != nil { + t.Fatalf("register node-b handler: %v", err) + } + + tempDir := t.TempDir() + pathB := filepath.Join(tempDir, "http-transport-gated-b.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "http-transport-gated-vol", + Path: pathB, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := nodeB.ApplyAssignments([]masterv2.Assignment{{ + Name: "http-transport-gated-vol", + Path: pathB, + NodeID: "node-b", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-b: %v", err) + } + + _, err = ExecuteFailoverFlow(master, "http-transport-gated-vol", 2, []FailoverTarget{ + mustHybridFailoverTarget(t, nodeB, transport), + staticFailoverTarget( + "node-c", + masterv2.PromotionQueryResponse{ + VolumeName: "http-transport-gated-vol", + NodeID: "node-c", + Epoch: 1, + CommittedLSN: 1, + WALHeadLSN: 1, + Eligible: false, + Reason: "needs_rebuild", + }, + protocolv2.ReplicaSummaryResponse{ + VolumeName: "http-transport-gated-vol", + NodeID: "node-c", + Epoch: 1, + Role: "replica", + Mode: "needs_rebuild", + CommittedLSN: 3, + DurableLSN: 2, + CheckpointLSN: 1, + RecoveryPhase: "needs_rebuild", + LastBarrierOK: false, + LastBarrierReason: "timeout", + Eligible: false, + Reason: "needs_rebuild", + }, + ), + }) + if err == nil { + t.Fatal("expected gated http transport failover error") + } +} + +func TestInProcessRuntimeManager_Loop2Service_RefreshesRF2Surface(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + manager, err := NewInProcessRuntimeManagerWithEvidenceTransport(master, NewHTTPFailoverEvidenceTransport()) + if err != nil { + t.Fatalf("new runtime manager: %v", err) + } + defer manager.Close() + + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + nodeC, err := New(Config{NodeID: "node-c"}) + if err != nil { + t.Fatalf("new node-c: %v", err) + } + defer nodeC.Close() + if err := manager.RegisterNode(nodeB); err != nil { + t.Fatalf("register node-b: %v", err) + } + if err := manager.RegisterNode(nodeC); err != nil { + t.Fatalf("register node-c: %v", err) + } + + tempDir := t.TempDir() + pathB := filepath.Join(tempDir, "loop2-service-b.blk") + pathC := filepath.Join(tempDir, "loop2-service-c.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "loop2-service-vol", + Path: pathB, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := nodeB.ApplyAssignments([]masterv2.Assignment{{ + Name: "loop2-service-vol", + Path: pathB, + NodeID: "node-b", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-b: %v", err) + } + if err := nodeC.ApplyAssignments([]masterv2.Assignment{{ + Name: "loop2-service-vol", + Path: pathC, + NodeID: "node-c", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-c: %v", err) + } + + handle, err := manager.StartLoop2Service(Loop2ServiceConfig{ + VolumeName: "loop2-service-vol", + PrimaryNodeID: "node-b", + ExpectedEpoch: 2, + NodeIDs: []string{"node-b", "node-c"}, + Interval: 10 * time.Millisecond, + }) + if err != nil { + t.Fatalf("start loop2 service: %v", err) + } + defer func() { + handle.Stop() + handle.Wait() + }() + + waitForCondition(t, time.Second, func() bool { + surface, ok := manager.RF2VolumeSurface("loop2-service-vol") + return ok && surface.HasLoop2 && surface.Mode == RF2SurfaceModeHealthy + }) +} + +func TestInProcessRuntimeManager_AutoFailoverService_TriggersOnPrimaryLoss(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + manager, err := NewInProcessRuntimeManagerWithEvidenceTransport(master, NewHTTPFailoverEvidenceTransport()) + if err != nil { + t.Fatalf("new runtime manager: %v", err) + } + defer manager.Close() + + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + nodeC, err := New(Config{NodeID: "node-c"}) + if err != nil { + t.Fatalf("new node-c: %v", err) + } + defer nodeC.Close() + if err := manager.RegisterNode(nodeB); err != nil { + t.Fatalf("register node-b: %v", err) + } + if err := manager.RegisterNode(nodeC); err != nil { + t.Fatalf("register node-c: %v", err) + } + + tempDir := t.TempDir() + pathB := filepath.Join(tempDir, "auto-failover-b.blk") + pathC := filepath.Join(tempDir, "auto-failover-c.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "auto-failover-vol", + Path: pathB, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := nodeB.ApplyAssignments([]masterv2.Assignment{{ + Name: "auto-failover-vol", + Path: pathB, + NodeID: "node-b", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-b: %v", err) + } + if err := nodeC.ApplyAssignments([]masterv2.Assignment{{ + Name: "auto-failover-vol", + Path: pathC, + NodeID: "node-c", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-c: %v", err) + } + + handle, err := manager.StartAutoFailoverService(AutoFailoverConfig{ + VolumeName: "auto-failover-vol", + PrimaryNodeID: "node-b", + ExpectedEpoch: 2, + NodeIDs: []string{"node-b", "node-c"}, + Interval: 10 * time.Millisecond, + }) + if err != nil { + t.Fatalf("start auto failover service: %v", err) + } + defer func() { + handle.Stop() + handle.Wait() + }() + + manager.DisconnectEvidenceNode("node-b") + waitForCondition(t, time.Second, func() bool { + result, ok := manager.FailoverResult("auto-failover-vol") + return ok && result.Assignment.NodeID == "node-c" + }) + if handle.PrimaryNodeID() != "node-c" { + t.Fatalf("tracked primary=%q, want node-c", handle.PrimaryNodeID()) + } +} + +func TestInProcessRuntimeManager_AutoFailoverService_DoesNotTriggerOnCatchingUpReplica(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + manager, err := NewInProcessRuntimeManagerWithEvidenceTransport(master, NewHTTPFailoverEvidenceTransport()) + if err != nil { + t.Fatalf("new runtime manager: %v", err) + } + defer manager.Close() + + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + if err := manager.RegisterNode(nodeB); err != nil { + t.Fatalf("register node-b: %v", err) + } + + tempDir := t.TempDir() + pathB := filepath.Join(tempDir, "auto-suppress-b.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "auto-suppress-vol", + Path: pathB, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := nodeB.ApplyAssignments([]masterv2.Assignment{{ + Name: "auto-suppress-vol", + Path: pathB, + NodeID: "node-b", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-b: %v", err) + } + if err := manager.RegisterTarget(staticFailoverTarget( + "node-c", + masterv2.PromotionQueryResponse{ + VolumeName: "auto-suppress-vol", + NodeID: "node-c", + Epoch: 2, + CommittedLSN: 1, + WALHeadLSN: 1, + ReceiverReady: true, + Eligible: true, + }, + protocolv2.ReplicaSummaryResponse{ + VolumeName: "auto-suppress-vol", + NodeID: "node-c", + Epoch: 2, + Role: "replica", + CommittedLSN: 1, + DurableLSN: 1, + CheckpointLSN: 1, + TargetLSN: 2, + AchievedLSN: 1, + RecoveryPhase: "catching_up", + Eligible: true, + LastBarrierOK: true, + }, + )); err != nil { + t.Fatalf("register node-c target: %v", err) + } + + handle, err := manager.StartAutoFailoverService(AutoFailoverConfig{ + VolumeName: "auto-suppress-vol", + PrimaryNodeID: "node-b", + ExpectedEpoch: 2, + NodeIDs: []string{"node-b", "node-c"}, + Interval: 10 * time.Millisecond, + }) + if err != nil { + t.Fatalf("start auto failover service: %v", err) + } + defer func() { + handle.Stop() + handle.Wait() + }() + + time.Sleep(120 * time.Millisecond) + if _, ok := manager.FailoverResult("auto-suppress-vol"); ok { + t.Fatal("unexpected auto failover result on catching_up replica") + } + surface, ok := manager.RF2VolumeSurface("auto-suppress-vol") + if !ok { + t.Fatal("expected rf2 volume surface") + } + if surface.Mode != RF2SurfaceModeCatchingUp { + t.Fatalf("surface mode=%q, want %q", surface.Mode, RF2SurfaceModeCatchingUp) + } +} + +func TestInProcessRuntimeManager_ExportVolumeISCSI_BindsFrontendToRuntimeNode(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + manager, err := NewInProcessRuntimeManager(master) + if err != nil { + t.Fatalf("new runtime manager: %v", err) + } + defer manager.Close() + + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + if err := manager.RegisterNode(nodeB); err != nil { + t.Fatalf("register node-b: %v", err) + } + + pathB := filepath.Join(t.TempDir(), "frontend-runtime-b.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "frontend-runtime-vol", + Path: pathB, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := nodeB.ApplyAssignments([]masterv2.Assignment{{ + Name: "frontend-runtime-vol", + Path: pathB, + NodeID: "node-b", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-b: %v", err) + } + + export, err := manager.ExportVolumeISCSI("frontend-runtime-vol", "node-b", "127.0.0.1:0", "iqn.2026-04.com.seaweedfs:test.frontend-runtime-vol") + if err != nil { + t.Fatalf("export volume iscsi: %v", err) + } + snapshot, ok := manager.ISCSIExport("frontend-runtime-vol") + if !ok { + t.Fatal("expected managed iscsi export snapshot") + } + if snapshot.NodeID != "node-b" { + t.Fatalf("snapshot node=%q, want node-b", snapshot.NodeID) + } + + conn := mustLoginISCSI(t, export.Address, export.IQN) + defer conn.Close() + + writeData := bytes.Repeat([]byte{0x58}, 4096) + var writeCDB [16]byte + writeCDB[0] = iscsi.ScsiWrite10 + binary.BigEndian.PutUint32(writeCDB[2:6], 0) + binary.BigEndian.PutUint16(writeCDB[7:9], 1) + resp := sendSCSICmd(t, conn, writeCDB, 2, false, true, writeData, uint32(len(writeData))) + if resp.SCSIStatus() != iscsi.SCSIStatusGood { + t.Fatalf("iscsi write failed: status=%d", resp.SCSIStatus()) + } + + var syncCDB [16]byte + syncCDB[0] = iscsi.ScsiSyncCache10 + resp = sendSCSICmd(t, conn, syncCDB, 3, false, false, nil, 0) + if resp.SCSIStatus() != iscsi.SCSIStatusGood { + t.Fatalf("iscsi sync cache failed: status=%d", resp.SCSIStatus()) + } + + readBack, err := nodeB.ReadLBA("frontend-runtime-vol", 0, uint32(len(writeData))) + if err != nil { + t.Fatalf("backend read: %v", err) + } + if !bytes.Equal(readBack, writeData) { + t.Fatal("backend readback mismatch") + } +} + +func TestInProcessRuntimeManager_RepairReplicaFromPrimary_ReturnsLoop2ToHealthy(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + manager, err := NewInProcessRuntimeManagerWithEvidenceTransport(master, NewHTTPFailoverEvidenceTransport()) + if err != nil { + t.Fatalf("new runtime manager: %v", err) + } + defer manager.Close() + + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + nodeC, err := New(Config{NodeID: "node-c"}) + if err != nil { + t.Fatalf("new node-c: %v", err) + } + defer nodeC.Close() + if err := manager.RegisterNode(nodeB); err != nil { + t.Fatalf("register node-b: %v", err) + } + if err := manager.RegisterNode(nodeC); err != nil { + t.Fatalf("register node-c: %v", err) + } + + tempDir := t.TempDir() + pathB := filepath.Join(tempDir, "repair-b.blk") + pathC := filepath.Join(tempDir, "repair-c.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "repair-vol", + Path: pathB, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + assignments := []masterv2.Assignment{ + { + Name: "repair-vol", + Path: pathB, + NodeID: "node-b", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }, + { + Name: "repair-vol", + Path: pathC, + NodeID: "node-c", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }, + } + if err := nodeB.ApplyAssignments(assignments[:1]); err != nil { + t.Fatalf("seed node-b: %v", err) + } + if err := nodeC.ApplyAssignments(assignments[1:]); err != nil { + t.Fatalf("seed node-c: %v", err) + } + + payload := bytes.Repeat([]byte{0x62}, 4096) + if err := nodeB.WriteLBA("repair-vol", 0, payload); err != nil { + t.Fatalf("write node-b: %v", err) + } + if err := nodeB.SyncCache("repair-vol"); err != nil { + t.Fatalf("sync node-b: %v", err) + } + + before, err := manager.ObserveLoop2("repair-vol", "node-b", 2, "node-b", "node-c") + if err != nil { + t.Fatalf("observe before repair: %v", err) + } + if before.Mode != Loop2RuntimeModeCatchingUp { + t.Fatalf("before repair mode=%q, want %q", before.Mode, Loop2RuntimeModeCatchingUp) + } + + result, err := manager.RepairReplicaFromPrimary("repair-vol", "node-b", "node-c", 2, uint32(len(payload))) + if err != nil { + t.Fatalf("repair replica from primary: %v", err) + } + if result.Loop2Before.Mode != Loop2RuntimeModeCatchingUp { + t.Fatalf("repair before mode=%q, want %q", result.Loop2Before.Mode, Loop2RuntimeModeCatchingUp) + } + if result.Loop2After.Mode != Loop2RuntimeModeKeepUp { + t.Fatalf("repair after mode=%q, want %q", result.Loop2After.Mode, Loop2RuntimeModeKeepUp) + } + if !result.DataMatch { + t.Fatalf("repair data match=false: %+v", result) + } +} + +func TestInProcessRuntimeManager_EndToEndRF2Handoff_ContinuesIOOnNewPrimary(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + manager, err := NewInProcessRuntimeManagerWithEvidenceTransport(master, NewHTTPFailoverEvidenceTransport()) + if err != nil { + t.Fatalf("new runtime manager: %v", err) + } + defer manager.Close() + + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + nodeC, err := New(Config{NodeID: "node-c"}) + if err != nil { + t.Fatalf("new node-c: %v", err) + } + defer nodeC.Close() + if err := manager.RegisterNode(nodeB); err != nil { + t.Fatalf("register node-b: %v", err) + } + if err := manager.RegisterNode(nodeC); err != nil { + t.Fatalf("register node-c: %v", err) + } + + tempDir := t.TempDir() + pathB := filepath.Join(tempDir, "e2e-b.blk") + pathC := filepath.Join(tempDir, "e2e-c.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "e2e-vol", + Path: pathB, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := nodeB.ApplyAssignments([]masterv2.Assignment{{ + Name: "e2e-vol", + Path: pathB, + NodeID: "node-b", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-b: %v", err) + } + if err := nodeC.ApplyAssignments([]masterv2.Assignment{{ + Name: "e2e-vol", + Path: pathC, + NodeID: "node-c", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-c: %v", err) + } + + export, err := manager.ExportVolumeISCSI("e2e-vol", "node-b", "127.0.0.1:0", "iqn.2026-04.com.seaweedfs:test.e2e-vol.primary") + if err != nil { + t.Fatalf("export primary iscsi: %v", err) + } + conn := mustLoginISCSI(t, export.Address, export.IQN) + defer conn.Close() + + firstPayload := bytes.Repeat([]byte{0x63}, 4096) + var writeCDB [16]byte + writeCDB[0] = iscsi.ScsiWrite10 + binary.BigEndian.PutUint32(writeCDB[2:6], 0) + binary.BigEndian.PutUint16(writeCDB[7:9], 1) + resp := sendSCSICmd(t, conn, writeCDB, 2, false, true, firstPayload, uint32(len(firstPayload))) + if resp.SCSIStatus() != iscsi.SCSIStatusGood { + t.Fatalf("initial iscsi write failed: status=%d", resp.SCSIStatus()) + } + var syncCDB [16]byte + syncCDB[0] = iscsi.ScsiSyncCache10 + resp = sendSCSICmd(t, conn, syncCDB, 3, false, false, nil, 0) + if resp.SCSIStatus() != iscsi.SCSIStatusGood { + t.Fatalf("initial iscsi sync failed: status=%d", resp.SCSIStatus()) + } + + if _, err := manager.RepairReplicaFromPrimary("e2e-vol", "node-b", "node-c", 2, uint32(len(firstPayload))); err != nil { + t.Fatalf("repair before failover: %v", err) + } + + handle, err := manager.StartAutoFailoverService(AutoFailoverConfig{ + VolumeName: "e2e-vol", + PrimaryNodeID: "node-b", + ExpectedEpoch: 2, + NodeIDs: []string{"node-b", "node-c"}, + Interval: 10 * time.Millisecond, + }) + if err != nil { + t.Fatalf("start auto failover service: %v", err) + } + defer func() { + handle.Stop() + handle.Wait() + }() + + manager.DisconnectEvidenceNode("node-b") + waitForCondition(t, time.Second, func() bool { + result, ok := manager.FailoverResult("e2e-vol") + return ok && result.Assignment.NodeID == "node-c" + }) + + exportAfter, err := manager.ExportCurrentPrimaryISCSI("e2e-vol", "127.0.0.1:0", "iqn.2026-04.com.seaweedfs:test.e2e-vol.failover") + if err != nil { + t.Fatalf("export new primary iscsi: %v", err) + } + connAfter := mustLoginISCSI(t, exportAfter.Address, exportAfter.IQN) + defer connAfter.Close() + + var readCDB [16]byte + readCDB[0] = iscsi.ScsiRead10 + binary.BigEndian.PutUint32(readCDB[2:6], 0) + binary.BigEndian.PutUint16(readCDB[7:9], 1) + resp = sendSCSICmd(t, connAfter, readCDB, 4, true, false, nil, uint32(len(firstPayload))) + if resp.Opcode() != iscsi.OpSCSIDataIn { + t.Fatalf("expected Data-In after failover, got %s", iscsi.OpcodeName(resp.Opcode())) + } + if !bytes.Equal(resp.DataSegment, firstPayload) { + t.Fatal("post-failover readback mismatch") + } + + secondPayload := bytes.Repeat([]byte{0x64}, 4096) + resp = sendSCSICmd(t, connAfter, writeCDB, 5, false, true, secondPayload, uint32(len(secondPayload))) + if resp.SCSIStatus() != iscsi.SCSIStatusGood { + t.Fatalf("post-failover write failed: status=%d", resp.SCSIStatus()) + } + resp = sendSCSICmd(t, connAfter, syncCDB, 6, false, false, nil, 0) + if resp.SCSIStatus() != iscsi.SCSIStatusGood { + t.Fatalf("post-failover sync failed: status=%d", resp.SCSIStatus()) + } + finalRead, err := nodeC.ReadLBA("e2e-vol", 0, uint32(len(secondPayload))) + if err != nil { + t.Fatalf("node-c backend read: %v", err) + } + if !bytes.Equal(finalRead, secondPayload) { + t.Fatal("post-failover backend readback mismatch") + } +} + +func TestInProcessRuntimeManager_EndToEndRF2Handoff_GatedReplicaStopsFailClosed(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + manager, err := NewInProcessRuntimeManagerWithEvidenceTransport(master, NewHTTPFailoverEvidenceTransport()) + if err != nil { + t.Fatalf("new runtime manager: %v", err) + } + defer manager.Close() + + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + if err := manager.RegisterNode(nodeB); err != nil { + t.Fatalf("register node-b: %v", err) + } + + tempDir := t.TempDir() + pathB := filepath.Join(tempDir, "e2e-gated-b.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "e2e-gated-vol", + Path: pathB, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := nodeB.ApplyAssignments([]masterv2.Assignment{{ + Name: "e2e-gated-vol", + Path: pathB, + NodeID: "node-b", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-b: %v", err) + } + if err := manager.RegisterTarget(staticFailoverTarget( + "node-c", + masterv2.PromotionQueryResponse{ + VolumeName: "e2e-gated-vol", + NodeID: "node-c", + Epoch: 2, + CommittedLSN: 1, + WALHeadLSN: 1, + Eligible: false, + Reason: "needs_rebuild", + }, + protocolv2.ReplicaSummaryResponse{ + VolumeName: "e2e-gated-vol", + NodeID: "node-c", + Epoch: 2, + Role: "replica", + Mode: "needs_rebuild", + CommittedLSN: 1, + DurableLSN: 1, + CheckpointLSN: 1, + RecoveryPhase: "needs_rebuild", + LastBarrierOK: false, + LastBarrierReason: "timeout", + Eligible: false, + Reason: "needs_rebuild", + }, + )); err != nil { + t.Fatalf("register node-c target: %v", err) + } + + handle, err := manager.StartAutoFailoverService(AutoFailoverConfig{ + VolumeName: "e2e-gated-vol", + PrimaryNodeID: "node-b", + ExpectedEpoch: 2, + NodeIDs: []string{"node-b", "node-c"}, + Interval: 10 * time.Millisecond, + }) + if err != nil { + t.Fatalf("start auto failover service: %v", err) + } + defer func() { + handle.Stop() + handle.Wait() + }() + + manager.DisconnectEvidenceNode("node-b") + waitForCondition(t, time.Second, func() bool { + snap, ok := manager.FailoverSnapshot("e2e-gated-vol") + return ok && snap.Stage == FailoverStageFailed + }) + result, ok := manager.FailoverResult("e2e-gated-vol") + if !ok { + t.Fatal("expected recorded gated failover result slot") + } + if result.Assignment.NodeID != "" { + t.Fatalf("unexpected successful assignment on gated handoff: %+v", result) + } +} + +func TestInProcessRuntimeManager_OperatorSurface_ExposesRuntimeOwnedViews(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + manager, err := NewInProcessRuntimeManager(master) + if err != nil { + t.Fatalf("new runtime manager: %v", err) + } + defer manager.Close() + + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + nodeC, err := New(Config{NodeID: "node-c"}) + if err != nil { + t.Fatalf("new node-c: %v", err) + } + defer nodeC.Close() + if err := manager.RegisterNode(nodeB); err != nil { + t.Fatalf("register node-b: %v", err) + } + if err := manager.RegisterNode(nodeC); err != nil { + t.Fatalf("register node-c: %v", err) + } + + tempDir := t.TempDir() + pathB := filepath.Join(tempDir, "operator-b.blk") + pathC := filepath.Join(tempDir, "operator-c.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "operator-vol", + Path: pathB, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := nodeB.ApplyAssignments([]masterv2.Assignment{{ + Name: "operator-vol", + Path: pathB, + NodeID: "node-b", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-b: %v", err) + } + if err := nodeC.ApplyAssignments([]masterv2.Assignment{{ + Name: "operator-vol", + Path: pathC, + NodeID: "node-c", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-c: %v", err) + } + if _, err := manager.ObserveLoop2("operator-vol", "node-b", 2, "node-b", "node-c"); err != nil { + t.Fatalf("observe loop2: %v", err) + } + if _, err := manager.ExportVolumeISCSI("operator-vol", "node-b", "127.0.0.1:0", "iqn.2026-04.com.seaweedfs:test.operator-vol"); err != nil { + t.Fatalf("export iscsi: %v", err) + } + + handle, err := manager.StartOperatorSurface("127.0.0.1:0") + if err != nil { + t.Fatalf("start operator surface: %v", err) + } + defer handle.Close() + + resp, err := http.Get("http://" + handle.Address() + "/v1/volumes/operator-vol/rf2") + if err != nil { + t.Fatalf("get rf2 surface: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("rf2 status=%d, want 200", resp.StatusCode) + } + var rf2Surface RF2VolumeSurface + if err := json.NewDecoder(resp.Body).Decode(&rf2Surface); err != nil { + t.Fatalf("decode rf2 surface: %v", err) + } + if rf2Surface.Mode != RF2SurfaceModeHealthy { + t.Fatalf("rf2 surface mode=%q, want %q", rf2Surface.Mode, RF2SurfaceModeHealthy) + } + + resp, err = http.Get("http://" + handle.Address() + "/v1/volumes/operator-vol/frontend") + if err != nil { + t.Fatalf("get frontend surface: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("frontend status=%d, want 200", resp.StatusCode) + } + var frontend ManagedISCSIExportSnapshot + if err := json.NewDecoder(resp.Body).Decode(&frontend); err != nil { + t.Fatalf("decode frontend surface: %v", err) + } + if frontend.NodeID != "node-b" { + t.Fatalf("frontend node=%q, want node-b", frontend.NodeID) + } +} + func TestLoop2RuntimeSession_KeepUpOnHealthyReplicaSet(t *testing.T) { nodeB, err := New(Config{NodeID: "node-b"}) if err != nil { @@ -1917,6 +2844,226 @@ func TestInProcessRuntimeManager_ExecuteReplicatedContinuity_GatedPath(t *testin } } +func TestInProcessRuntimeManager_RF2VolumeSurface_HealthyPackage(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + manager, err := NewInProcessRuntimeManager(master) + if err != nil { + t.Fatalf("new runtime manager: %v", err) + } + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + nodeC, err := New(Config{NodeID: "node-c"}) + if err != nil { + t.Fatalf("new node-c: %v", err) + } + defer nodeC.Close() + if err := manager.RegisterNode(nodeB); err != nil { + t.Fatalf("register node-b: %v", err) + } + if err := manager.RegisterNode(nodeC); err != nil { + t.Fatalf("register node-c: %v", err) + } + + tempDir := t.TempDir() + pathB := filepath.Join(tempDir, "rf2-surface-b.blk") + pathC := filepath.Join(tempDir, "rf2-surface-c.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "rf2-surface-vol", + Path: pathB, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := nodeB.ApplyAssignments([]masterv2.Assignment{{ + Name: "rf2-surface-vol", + Path: pathB, + NodeID: "node-b", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-b: %v", err) + } + if err := nodeC.ApplyAssignments([]masterv2.Assignment{{ + Name: "rf2-surface-vol", + Path: pathC, + NodeID: "node-c", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-c: %v", err) + } + + payload := bytes.Repeat([]byte{0x55}, 4096) + if _, err := manager.ExecuteReplicatedContinuity("rf2-surface-vol", "node-b", 2, []string{"node-c"}, 0, payload); err != nil { + t.Fatalf("execute replicated continuity: %v", err) + } + + continuitySnap, ok := manager.ReplicatedContinuitySnapshot("rf2-surface-vol") + if !ok { + t.Fatal("expected continuity snapshot") + } + if continuitySnap.LastError != "" { + t.Fatalf("continuity last_error=%q, want empty", continuitySnap.LastError) + } + + surface, ok := manager.RF2VolumeSurface("rf2-surface-vol") + if !ok { + t.Fatal("expected rf2 volume surface") + } + if !surface.HasLoop2 || !surface.HasFailover || !surface.HasContinuity { + t.Fatalf("surface source flags=%+v, want all true", surface) + } + if surface.Mode != RF2SurfaceModeHealthy { + t.Fatalf("surface mode=%q, want %q", surface.Mode, RF2SurfaceModeHealthy) + } + if surface.ReplicationMode != Loop2RuntimeModeKeepUp { + t.Fatalf("replication mode=%q, want %q", surface.ReplicationMode, Loop2RuntimeModeKeepUp) + } + if surface.FailoverStage != FailoverStageActivated { + t.Fatalf("failover stage=%q, want %q", surface.FailoverStage, FailoverStageActivated) + } + if surface.FailoverNodeID != "node-c" { + t.Fatalf("failover node=%q, want node-c", surface.FailoverNodeID) + } + if surface.ContinuityStatus != RF2ContinuityStatusProven { + t.Fatalf("continuity status=%q, want %q", surface.ContinuityStatus, RF2ContinuityStatusProven) + } + if surface.ContinuityNodeID != "node-c" { + t.Fatalf("continuity node=%q, want node-c", surface.ContinuityNodeID) + } +} + +func TestInProcessRuntimeManager_RF2VolumeSurface_GatedPackage(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + manager, err := NewInProcessRuntimeManager(master) + if err != nil { + t.Fatalf("new runtime manager: %v", err) + } + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + nodeC, err := New(Config{NodeID: "node-c"}) + if err != nil { + t.Fatalf("new node-c: %v", err) + } + defer nodeC.Close() + if err := manager.RegisterNode(nodeB); err != nil { + t.Fatalf("register node-b: %v", err) + } + if err := manager.RegisterNode(nodeC); err != nil { + t.Fatalf("register node-c: %v", err) + } + + tempDir := t.TempDir() + pathB := filepath.Join(tempDir, "rf2-surface-gated-b.blk") + pathC := filepath.Join(tempDir, "rf2-surface-gated-c.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "rf2-surface-gated-vol", + Path: pathB, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := nodeB.ApplyAssignments([]masterv2.Assignment{{ + Name: "rf2-surface-gated-vol", + Path: pathB, + NodeID: "node-b", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-b: %v", err) + } + if err := nodeC.ApplyAssignments([]masterv2.Assignment{{ + Name: "rf2-surface-gated-vol", + Path: pathC, + NodeID: "node-c", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-c: %v", err) + } + if err := manager.RegisterTarget(staticFailoverTarget( + "node-c", + masterv2.PromotionQueryResponse{ + VolumeName: "rf2-surface-gated-vol", + NodeID: "node-c", + Epoch: 2, + CommittedLSN: 1, + WALHeadLSN: 1, + Eligible: false, + Reason: "needs_rebuild", + }, + protocolv2.ReplicaSummaryResponse{ + VolumeName: "rf2-surface-gated-vol", + NodeID: "node-c", + Epoch: 2, + Role: "replica", + Mode: "needs_rebuild", + CommittedLSN: 1, + DurableLSN: 1, + CheckpointLSN: 1, + RecoveryPhase: "needs_rebuild", + LastBarrierOK: false, + LastBarrierReason: "timeout", + Eligible: false, + Reason: "needs_rebuild", + }, + )); err != nil { + t.Fatalf("override node-c target: %v", err) + } + + payload := bytes.Repeat([]byte{0x56}, 4096) + if _, err := manager.ExecuteReplicatedContinuity("rf2-surface-gated-vol", "node-b", 2, []string{"node-c"}, 0, payload); err == nil { + t.Fatal("expected gated replicated continuity error") + } + + continuitySnap, ok := manager.ReplicatedContinuitySnapshot("rf2-surface-gated-vol") + if !ok { + t.Fatal("expected continuity snapshot") + } + if continuitySnap.LastError == "" { + t.Fatal("expected continuity last_error") + } + + surface, ok := manager.RF2VolumeSurface("rf2-surface-gated-vol") + if !ok { + t.Fatal("expected rf2 volume surface") + } + if surface.Mode != RF2SurfaceModeBlocked { + t.Fatalf("surface mode=%q, want %q", surface.Mode, RF2SurfaceModeBlocked) + } + if surface.ReplicationMode != Loop2RuntimeModeNeedsRebuild { + t.Fatalf("replication mode=%q, want %q", surface.ReplicationMode, Loop2RuntimeModeNeedsRebuild) + } + if surface.FailoverStage != FailoverStageFailed { + t.Fatalf("failover stage=%q, want %q", surface.FailoverStage, FailoverStageFailed) + } + if surface.FailoverError == "" { + t.Fatal("expected failover error") + } + if surface.ContinuityStatus != RF2ContinuityStatusFailed { + t.Fatalf("continuity status=%q, want %q", surface.ContinuityStatus, RF2ContinuityStatusFailed) + } + if surface.ContinuityError == "" { + t.Fatal("expected continuity error") + } +} + func mustHeartbeat(t *testing.T, node *Node) masterv2.NodeHeartbeat { t.Helper() hb, err := node.Heartbeat() @@ -1950,6 +3097,18 @@ func mustReplicaSummary(t *testing.T, node *Node, volumeName string, epoch uint6 return resp } +func waitForCondition(t *testing.T, timeout time.Duration, cond func() bool) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatal("condition not reached before timeout") +} + func mustInProcessFailoverTarget(t *testing.T, node *Node) FailoverTarget { t.Helper() target, err := NewInProcessFailoverTarget(node) diff --git a/sw-block/runtime/volumev2/rf2_surface.go b/sw-block/runtime/volumev2/rf2_surface.go new file mode 100644 index 000000000..39ba27884 --- /dev/null +++ b/sw-block/runtime/volumev2/rf2_surface.go @@ -0,0 +1,128 @@ +package volumev2 + +// RF2SurfaceMode is the compressed outward runtime/product mode projected from +// the new runtime-owned RF2 slices. +type RF2SurfaceMode string + +const ( + RF2SurfaceModeHealthy RF2SurfaceMode = "healthy" + RF2SurfaceModeCatchingUp RF2SurfaceMode = "catching_up" + RF2SurfaceModeDegraded RF2SurfaceMode = "degraded" + RF2SurfaceModeBlocked RF2SurfaceMode = "blocked" +) + +// RF2ContinuityStatus is the bounded continuity statement exposed to a +// product-facing surface. It stays compressed and never becomes a new truth +// owner. +type RF2ContinuityStatus string + +const ( + RF2ContinuityStatusUnknown RF2ContinuityStatus = "unknown" + RF2ContinuityStatusProven RF2ContinuityStatus = "proven" + RF2ContinuityStatusFailed RF2ContinuityStatus = "failed" +) + +// RF2VolumeSurface is the first bounded RF2-facing runtime/product surface +// package projected from the runtime-owned failover, active Loop 2, and +// continuity slices. +type RF2VolumeSurface struct { + VolumeName string + PrimaryNodeID string + ExpectedEpoch uint64 + Mode RF2SurfaceMode + Reason string + ReplicationMode Loop2RuntimeMode + ReplicaCount int + HealthyReplicaCount int + CommittedLSN uint64 + DurableFloorLSN uint64 + FailoverStage FailoverStage + FailoverNodeID string + FailoverError string + ContinuityStatus RF2ContinuityStatus + ContinuityNodeID string + ContinuityError string + HasLoop2 bool + HasFailover bool + HasContinuity bool +} + +// RF2VolumeSurface returns the latest bounded RF2-facing surface for one volume +// if the runtime has enough local observations to project it. +func (m *InProcessRuntimeManager) RF2VolumeSurface(volumeName string) (RF2VolumeSurface, bool) { + if m == nil || volumeName == "" { + return RF2VolumeSurface{}, false + } + m.mu.RLock() + defer m.mu.RUnlock() + + loop2, hasLoop2 := m.loop2ByVolume[volumeName] + failover, hasFailover := m.snapshotsByName[volumeName] + continuity, hasContinuity := m.continuityByVolume[volumeName] + if !hasLoop2 && !hasFailover && !hasContinuity { + return RF2VolumeSurface{}, false + } + + surface := RF2VolumeSurface{ + VolumeName: volumeName, + Mode: RF2SurfaceModeDegraded, + ContinuityStatus: RF2ContinuityStatusUnknown, + HasLoop2: hasLoop2, + HasFailover: hasFailover, + HasContinuity: hasContinuity, + } + if hasLoop2 { + surface.PrimaryNodeID = loop2.PrimaryNodeID + surface.ExpectedEpoch = loop2.ExpectedEpoch + surface.ReplicationMode = loop2.Mode + surface.Mode = projectRF2SurfaceMode(loop2.Mode) + surface.Reason = loop2.Reason + surface.ReplicaCount = loop2.ReplicaCount + surface.HealthyReplicaCount = loop2.HealthyReplicaCount + surface.CommittedLSN = loop2.CommittedLSN + surface.DurableFloorLSN = loop2.DurableFloorLSN + } + if hasFailover { + surface.FailoverStage = failover.Stage + surface.FailoverNodeID = failover.SelectedNodeID + surface.FailoverError = failover.LastError + if surface.PrimaryNodeID == "" && failover.SelectedNodeID != "" { + surface.PrimaryNodeID = failover.SelectedNodeID + } + if surface.ExpectedEpoch == 0 { + surface.ExpectedEpoch = failover.ExpectedEpoch + } + } + if hasContinuity { + if continuity.Result.SelectedPrimaryNodeID != "" { + surface.ContinuityNodeID = continuity.Result.SelectedPrimaryNodeID + surface.PrimaryNodeID = continuity.Result.SelectedPrimaryNodeID + } + if surface.ExpectedEpoch == 0 { + surface.ExpectedEpoch = continuity.Result.ExpectedEpoch + } + surface.ContinuityError = continuity.LastError + switch { + case continuity.LastError != "": + surface.ContinuityStatus = RF2ContinuityStatusFailed + case continuity.Result.DataMatch: + surface.ContinuityStatus = RF2ContinuityStatusProven + } + } + return surface, true +} + +func projectRF2SurfaceMode(mode Loop2RuntimeMode) RF2SurfaceMode { + switch mode { + case Loop2RuntimeModeKeepUp: + return RF2SurfaceModeHealthy + case Loop2RuntimeModeCatchingUp: + return RF2SurfaceModeCatchingUp + case Loop2RuntimeModeNeedsRebuild: + return RF2SurfaceModeBlocked + case Loop2RuntimeModeDegraded: + fallthrough + default: + return RF2SurfaceModeDegraded + } +} diff --git a/sw-block/runtime/volumev2/runtime_manager.go b/sw-block/runtime/volumev2/runtime_manager.go index e434d4eaa..72e481faa 100644 --- a/sw-block/runtime/volumev2/runtime_manager.go +++ b/sw-block/runtime/volumev2/runtime_manager.go @@ -13,34 +13,50 @@ import ( // observability. type InProcessRuntimeManager struct { driver *InProcessFailoverDriver - evidenceTransport *InMemoryFailoverEvidenceTransport + evidenceTransport ManagedFailoverEvidenceTransport - mu sync.RWMutex - localNodes map[string]*Node - lastSnapshot FailoverSnapshot - lastResult FailoverResult - hasLastResult bool - snapshotsByName map[string]FailoverSnapshot - resultsByName map[string]FailoverResult - lastLoop2Snapshot Loop2RuntimeSnapshot - hasLastLoop2 bool - loop2ByVolume map[string]Loop2RuntimeSnapshot + mu sync.RWMutex + localNodes map[string]*Node + lastSnapshot FailoverSnapshot + lastResult FailoverResult + hasLastResult bool + snapshotsByName map[string]FailoverSnapshot + resultsByName map[string]FailoverResult + lastLoop2Snapshot Loop2RuntimeSnapshot + hasLastLoop2 bool + loop2ByVolume map[string]Loop2RuntimeSnapshot + lastContinuity ReplicatedContinuitySnapshot + hasLastContinuity bool + continuityByVolume map[string]ReplicatedContinuitySnapshot + frontendExports map[string]*managedISCSIExport } // NewInProcessRuntimeManager creates a runtime-owned failover manager over one // in-process masterv2 instance. func NewInProcessRuntimeManager(master *masterv2.Master) (*InProcessRuntimeManager, error) { + return NewInProcessRuntimeManagerWithEvidenceTransport(master, nil) +} + +// NewInProcessRuntimeManagerWithEvidenceTransport creates a runtime-owned +// failover manager over one in-process masterv2 instance using the supplied +// managed evidence transport. When nil, an in-memory transport is used. +func NewInProcessRuntimeManagerWithEvidenceTransport(master *masterv2.Master, transport ManagedFailoverEvidenceTransport) (*InProcessRuntimeManager, error) { driver, err := NewInProcessFailoverDriver(master) if err != nil { return nil, err } + if transport == nil { + transport = NewInMemoryFailoverEvidenceTransport() + } return &InProcessRuntimeManager{ - driver: driver, - evidenceTransport: NewInMemoryFailoverEvidenceTransport(), - localNodes: make(map[string]*Node), - snapshotsByName: make(map[string]FailoverSnapshot), - resultsByName: make(map[string]FailoverResult), - loop2ByVolume: make(map[string]Loop2RuntimeSnapshot), + driver: driver, + evidenceTransport: transport, + localNodes: make(map[string]*Node), + snapshotsByName: make(map[string]FailoverSnapshot), + resultsByName: make(map[string]FailoverResult), + loop2ByVolume: make(map[string]Loop2RuntimeSnapshot), + continuityByVolume: make(map[string]ReplicatedContinuitySnapshot), + frontendExports: make(map[string]*managedISCSIExport), }, nil } @@ -95,6 +111,52 @@ func (m *InProcessRuntimeManager) UnregisterParticipant(nodeID string) { m.driver.UnregisterParticipant(nodeID) } +// DisconnectEvidenceNode removes the evidence handler for one node while +// leaving the runtime-owned target and local node registration intact. This is +// useful for bounded live-transport fault injection. +func (m *InProcessRuntimeManager) DisconnectEvidenceNode(nodeID string) { + if m == nil || m.evidenceTransport == nil { + return + } + m.evidenceTransport.UnregisterHandler(nodeID) +} + +// ReconnectEvidenceNode re-registers one local node behind the configured +// evidence transport. +func (m *InProcessRuntimeManager) ReconnectEvidenceNode(nodeID string) error { + if m == nil || m.evidenceTransport == nil { + return fmt.Errorf("volumev2: runtime manager is nil") + } + node, err := m.localNode(nodeID) + if err != nil { + return err + } + return m.evidenceTransport.RegisterHandler(nodeID, node) +} + +// Close releases runtime-manager owned transport resources. +func (m *InProcessRuntimeManager) Close() error { + if m == nil { + return nil + } + m.mu.Lock() + exports := make([]*managedISCSIExport, 0, len(m.frontendExports)) + for volumeName, export := range m.frontendExports { + exports = append(exports, export) + delete(m.frontendExports, volumeName) + } + m.mu.Unlock() + for _, export := range exports { + if export != nil && export.handle != nil { + _ = export.handle.Close() + } + } + if m.evidenceTransport == nil { + return nil + } + return m.evidenceTransport.Close() +} + // ParticipantNodeIDs returns the current runtime-owned participant ids. func (m *InProcessRuntimeManager) ParticipantNodeIDs() []string { if m == nil || m.driver == nil { @@ -227,6 +289,32 @@ func (m *InProcessRuntimeManager) Loop2Snapshot(volumeName string) (Loop2Runtime return snapshot, ok } +// LastReplicatedContinuitySnapshot returns the most recent bounded continuity +// snapshot observed by the runtime manager. +func (m *InProcessRuntimeManager) LastReplicatedContinuitySnapshot() (ReplicatedContinuitySnapshot, bool) { + if m == nil { + return ReplicatedContinuitySnapshot{}, false + } + m.mu.RLock() + defer m.mu.RUnlock() + if !m.hasLastContinuity { + return ReplicatedContinuitySnapshot{}, false + } + return m.lastContinuity, true +} + +// ReplicatedContinuitySnapshot returns the latest bounded continuity snapshot +// for one volume if present. +func (m *InProcessRuntimeManager) ReplicatedContinuitySnapshot(volumeName string) (ReplicatedContinuitySnapshot, bool) { + if m == nil { + return ReplicatedContinuitySnapshot{}, false + } + m.mu.RLock() + defer m.mu.RUnlock() + snapshot, ok := m.continuityByVolume[volumeName] + return snapshot, ok +} + func (m *InProcessRuntimeManager) recordSnapshot(volumeName string, snapshot FailoverSnapshot, result FailoverResult) { m.mu.Lock() defer m.mu.Unlock() @@ -249,6 +337,16 @@ func (m *InProcessRuntimeManager) recordLoop2Snapshot(volumeName string, snapsho } } +func (m *InProcessRuntimeManager) recordContinuitySnapshot(volumeName string, snapshot ReplicatedContinuitySnapshot) { + m.mu.Lock() + defer m.mu.Unlock() + m.lastContinuity = snapshot + m.hasLastContinuity = true + if volumeName != "" { + m.continuityByVolume[volumeName] = snapshot + } +} + func (m *InProcessRuntimeManager) localNode(nodeID string) (*Node, error) { if m == nil { return nil, fmt.Errorf("volumev2: runtime manager is nil") diff --git a/weed/storage/blockvol/csi/v2_runtime_backend.go b/weed/storage/blockvol/csi/v2_runtime_backend.go new file mode 100644 index 000000000..4ba95c6a8 --- /dev/null +++ b/weed/storage/blockvol/csi/v2_runtime_backend.go @@ -0,0 +1,103 @@ +package csi + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/seaweedfs/seaweedfs/sw-block/runtime/volumev2" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" +) + +// V2RuntimeBackend is a bounded CSI backend adapter over the new runtime-owned +// RF2 path. It intentionally starts narrow: create/bootstrap plus lookup/publish +// from runtime-owned exports and surfaces. +type V2RuntimeBackend struct { + manager *volumev2.InProcessRuntimeManager + primaryNodeID string + dataDir string + iqnPrefix string +} + +// NewV2RuntimeBackend creates one bounded CSI backend over the V2 runtime path. +func NewV2RuntimeBackend(manager *volumev2.InProcessRuntimeManager, primaryNodeID, dataDir, iqnPrefix string) *V2RuntimeBackend { + if iqnPrefix == "" { + iqnPrefix = "iqn.2026-04.com.seaweedfs:v2.csi." + } + return &V2RuntimeBackend{ + manager: manager, + primaryNodeID: primaryNodeID, + dataDir: dataDir, + iqnPrefix: iqnPrefix, + } +} + +func (b *V2RuntimeBackend) CreateVolume(ctx context.Context, name string, sizeBytes uint64) (*VolumeInfo, error) { + if b == nil || b.manager == nil { + return nil, fmt.Errorf("csi: v2 runtime backend is nil") + } + if name == "" { + return nil, fmt.Errorf("csi: volume name is required") + } + if existing, ok := b.manager.ISCSIExport(name); ok { + return &VolumeInfo{ + VolumeID: name, + ISCSIAddr: existing.Address, + IQN: existing.IQN, + CapacityBytes: sizeBytes, + }, nil + } + path := filepath.Join(b.dataDir, name+".blk") + if err := b.manager.BootstrapPrimaryVolume(name, b.primaryNodeID, path, 1, blockvol.CreateOptions{ + VolumeSize: sizeBytes, + BlockSize: 4096, + WALSize: 64 * 1024 * 1024, + }); err != nil { + return nil, err + } + export, err := b.manager.ExportVolumeISCSI(name, b.primaryNodeID, "127.0.0.1:0", b.iqnPrefix+name) + if err != nil { + return nil, err + } + return &VolumeInfo{ + VolumeID: name, + ISCSIAddr: export.Address, + IQN: export.IQN, + CapacityBytes: sizeBytes, + }, nil +} + +func (b *V2RuntimeBackend) DeleteVolume(ctx context.Context, name string) error { + return fmt.Errorf("csi: v2 runtime backend delete volume %q not yet implemented", name) +} + +func (b *V2RuntimeBackend) LookupVolume(ctx context.Context, name string) (*VolumeInfo, error) { + if b == nil || b.manager == nil { + return nil, fmt.Errorf("csi: v2 runtime backend is nil") + } + export, ok := b.manager.ISCSIExport(name) + if !ok { + return nil, fmt.Errorf("volume %q not found", name) + } + return &VolumeInfo{ + VolumeID: name, + ISCSIAddr: export.Address, + IQN: export.IQN, + }, nil +} + +func (b *V2RuntimeBackend) CreateSnapshot(ctx context.Context, volumeID string, snapID uint32) (*SnapshotInfo, error) { + return nil, fmt.Errorf("csi: v2 runtime backend snapshots are not yet implemented") +} + +func (b *V2RuntimeBackend) DeleteSnapshot(ctx context.Context, volumeID string, snapID uint32) error { + return fmt.Errorf("csi: v2 runtime backend snapshots are not yet implemented") +} + +func (b *V2RuntimeBackend) ListSnapshots(ctx context.Context, volumeID string) ([]*SnapshotInfo, error) { + return nil, fmt.Errorf("csi: v2 runtime backend snapshots are not yet implemented") +} + +func (b *V2RuntimeBackend) ExpandVolume(ctx context.Context, volumeID string, newSizeBytes uint64) (uint64, error) { + return 0, fmt.Errorf("csi: v2 runtime backend expand volume is not yet implemented") +} diff --git a/weed/storage/blockvol/csi/v2_runtime_backend_test.go b/weed/storage/blockvol/csi/v2_runtime_backend_test.go new file mode 100644 index 000000000..185e26aa3 --- /dev/null +++ b/weed/storage/blockvol/csi/v2_runtime_backend_test.go @@ -0,0 +1,57 @@ +package csi + +import ( + "context" + "testing" + + csipb "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/seaweedfs/seaweedfs/sw-block/runtime/masterv2" + "github.com/seaweedfs/seaweedfs/sw-block/runtime/volumev2" +) + +func TestV2RuntimeBackend_CreateLookupAndPublish(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + manager, err := volumev2.NewInProcessRuntimeManager(master) + if err != nil { + t.Fatalf("new runtime manager: %v", err) + } + defer manager.Close() + + node, err := volumev2.New(volumev2.Config{NodeID: "node-a"}) + if err != nil { + t.Fatalf("new volume node: %v", err) + } + defer node.Close() + if err := manager.RegisterNode(node); err != nil { + t.Fatalf("register node: %v", err) + } + + backend := NewV2RuntimeBackend(manager, "node-a", t.TempDir(), "iqn.2026-04.com.seaweedfs:test.csi.") + info, err := backend.CreateVolume(context.Background(), "csi-v2-vol", 1<<20) + if err != nil { + t.Fatalf("create volume: %v", err) + } + if info.ISCSIAddr == "" || info.IQN == "" { + t.Fatalf("missing exported target info: %+v", info) + } + + looked, err := backend.LookupVolume(context.Background(), "csi-v2-vol") + if err != nil { + t.Fatalf("lookup volume: %v", err) + } + if looked.ISCSIAddr != info.ISCSIAddr || looked.IQN != info.IQN { + t.Fatalf("lookup mismatch: create=%+v lookup=%+v", info, looked) + } + + controller := &controllerServer{backend: backend} + pub, err := controller.ControllerPublishVolume(context.Background(), &csipb.ControllerPublishVolumeRequest{ + VolumeId: "csi-v2-vol", + NodeId: "node-a", + }) + if err != nil { + t.Fatalf("controller publish: %v", err) + } + if pub.PublishContext["iscsiAddr"] == "" || pub.PublishContext["iqn"] == "" { + t.Fatalf("missing publish context: %+v", pub.PublishContext) + } +}