diff --git a/sw-block/.private/phase/phase-18-decisions.md b/sw-block/.private/phase/phase-18-decisions.md new file mode 100644 index 000000000..518c41989 --- /dev/null +++ b/sw-block/.private/phase/phase-18-decisions.md @@ -0,0 +1,95 @@ +# Phase 18 Decisions + +Date: 2026-04-05 +Status: active + +## D1: Phase 18 Uses M1-M5 As The Main Spine + +Decision: + +1. `Phase 18` will be the main control phase for the next kernel/runtime climb +2. the five major milestones (`M1-M5`) are the primary structure inside it +3. each major milestone should normally close in `2-3` implementation steps + +Why: + +1. we are no longer in pure exploration mode +2. the kernel boundary is now stable enough to support larger development slices +3. milestone-level review is more efficient than micro-slice review + +Implication: + +1. later work should be grouped into larger reviewable packages +2. helper-level or naming-level pauses should be minimized unless they affect + architecture + +## D2: Preserve Current In-Process Runtime As The Reference Slice + +Decision: + +1. the current in-process RF2 failover runtime remains the reference slice while + `M1` introduces the transport/session seam + +Why: + +1. it already proves the current authority split in executable form +2. it gives a stable baseline for transport-backed migration +3. it reduces the risk of confusing transport mechanics with ownership + +Implication: + +1. `M1` should introduce adapter seams first +2. the existing in-process path should remain valid until the transport-backed + slice closes + +## D3: Make Adapter-Backed Targets The Primary Failover Contract + +Decision: + +1. the primary failover contract is now `FailoverTarget` +2. `FailoverTarget` is split into: + - `FailoverEvidenceAdapter` + - `FailoverTakeoverAdapter` +3. the old all-in-one `FailoverParticipant` remains only as a compatibility + wrapper + +Why: + +1. failover-time query traffic and takeover execution are different boundary + types +2. the transport seam should be explicit before any real remote adapter is added +3. the runtime/driver/session should depend on adapters, not on concrete + `*Node` coupling + +Implication: + +1. future remote work should implement adapter contracts rather than widening + direct node ownership +2. current in-process tests and runtime remain valid through the in-process + adapter implementation + +## D4: `M1` Closes On Failover-Time Evidence Transport, Not Remote Takeover + +Decision: + +1. `M1` is considered complete when `PromotionQuery` and `ReplicaSummary` + traffic cross an explicit transport/session adapter seam +2. `M1` does not require remote takeover execution +3. takeover remains primary-local in this milestone + +Why: + +1. the `M1` goal is to remove direct failover-time evidence coupling from the + orchestration path +2. the selected primary should remain the owner of reconstruction and activation + gating +3. forcing remote takeover too early would risk mixing transport mechanics with + ownership changes + +Implication: + +1. the first transport-backed slice is: + - transport/session-backed evidence + - primary-local takeover +2. later transport work may widen execution transport, but only without changing + the authority split diff --git a/sw-block/.private/phase/phase-18-log.md b/sw-block/.private/phase/phase-18-log.md new file mode 100644 index 000000000..d316a9a20 --- /dev/null +++ b/sw-block/.private/phase/phase-18-log.md @@ -0,0 +1,78 @@ +# Phase 18 Log + +Date: 2026-04-05 +Status: active + +## 2026-04-05 + +### Start of Phase + +Created the initial `Phase 18` control document. + +Starting point recorded: + +1. in-process RF2 failover runtime slice exists +2. `FailoverSession`, in-process driver, and runtime manager exist +3. review-base docs already reflect the current kernel boundary and current + milestone + +Initial execution rule: + +1. move by major milestone +2. target `2-3` implementation steps per major milestone +3. update phase, log, decisions, and review-base docs after each major step + +Current next step: + +1. `M1` seam step for transport/session adapter boundary + +### `M1` Adapter-Seam Package + +Delivered in this update: + +1. explicit failover adapter seam introduced in code: + - `FailoverEvidenceAdapter` + - `FailoverTakeoverAdapter` + - `FailoverTarget` +2. first in-process adapter implementation delivered: + - `NewInProcessFailoverTarget(...)` +3. `FailoverSession` now uses explicit targets as the primary path +4. failover driver and runtime manager now register/resolve targets as the + primary path +5. existing healthy/gated runtime failover tests were moved onto the new target + seam + +Tests: + +1. `go test ./sw-block/runtime/masterv2 ./sw-block/runtime/volumev2` + +Current interpretation: + +1. the transport/session adapter seam is now real in code +2. the in-process path is still the reference implementation behind that seam +3. the first non-in-process adapter remains the next required slice before `M1` + can be treated as fully closed + +### `M1` Delivered + +Delivered in this update: + +1. the failover-time query path now crosses a transport/session adapter boundary +2. `PromotionQuery` and `ReplicaSummary` no longer depend on direct orchestrator + calls to `*Node` as the only implementation path +3. the first transport/session implementation is `InMemoryFailoverEvidenceTransport` +4. the runtime manager now registers nodes behind the evidence transport and + executes failover through the transport-backed evidence path + +Tests: + +1. `TestTransportEvidenceAdapter_HealthyFailoverFlow` +2. `TestTransportEvidenceAdapter_GatedFailoverFlow` +3. `go test ./sw-block/runtime/masterv2 ./sw-block/runtime/volumev2` + +Current interpretation: + +1. `M1` is complete as a transport/session-backed failover-time evidence slice +2. this is still a bounded request/response transport implementation, not broad + network-product proof +3. the next active work should move to `M2` diff --git a/sw-block/.private/phase/phase-18.md b/sw-block/.private/phase/phase-18.md new file mode 100644 index 000000000..f20f94545 --- /dev/null +++ b/sw-block/.private/phase/phase-18.md @@ -0,0 +1,316 @@ +# Phase 18 + +Date: 2026-04-05 +Status: active +Purpose: drive the new `masterv2 + volumev2 + purev2` kernel from the current +in-process RF2 failover runtime slice toward a bounded productizable RF2 runtime +in disciplined major milestones + +## Why This Phase Exists + +The current kernel has crossed the first important threshold: + +1. `masterv2` now behaves like explicit identity authority +2. `volumev2` now has explicit takeover preparation and activation gating +3. failover now exists as a runtime-owned slice rather than only implicit mixed + runtime behavior + +That is enough to stop the current milestone. + +It is not enough to claim a transport-backed RF2 runtime, continuous +replication-runtime ownership, or product-ready RF2 surfaces. + +The next work is larger than micro-slicing: + +1. the failover seam must cross a real transport/session boundary +2. the primary-led Loop 2 runtime must become continuously active +3. data continuity must be closed through real handoff paths +4. product/runtime surfaces must be attached without breaking authority split +5. productionization evidence must be bounded and explicit + +This phase exists to package those larger objects as one ordered program rather +than continuing disconnected local improvements. + +## Entry Checkpoint + +`Phase 18` starts from the current completed kernel/runtime slice: + +1. explicit `masterv2` promotion authorization +2. explicit `volumev2` takeover prepare/gate seams +3. stepwise `FailoverSession` with observable stages and failure snapshots +4. in-process failover driver seam +5. runtime-owned in-process RF2 failover manager entry point + +Entry interpretation: + +1. this is a real kernel/runtime checkpoint +2. this is not yet transport-backed RF2 closure +3. this is not yet active Loop 2 runtime closure +4. this is not yet RF2 product or production closure + +## Phase Goal + +Produce one bounded post-entry sequence where: + +1. the failover runtime crosses a real transport/session seam without changing + authority ownership +2. the primary-led Loop 2 runtime becomes continuously meaningful rather than + appearing only at failover boundaries +3. one replicated continuity statement is supported through real handoff +4. one bounded RF2 product/runtime surface exists on top of the new runtime +5. one bounded productionization envelope is explicit and reviewable + +## Scope + +### In scope + +1. transport/session seam for failover-time evidence and bounded replica + summaries +2. runtime-owned RF2 failover flow beyond in-process direct calls +3. primary-led active Loop 2 runtime growth +4. replicated continuity closure +5. RF2 runtime/product surface attachment +6. bounded pilot/productionization review package + +### Out of scope + +1. broad protocol rediscovery +2. silent return to `weed/server` ownership +3. premature `RF>2` product closure +4. broad transport/frontend matrix approval before bounded RF2 runtime closure +5. broad launch claim before explicit productionization evidence + +## Working Rules + +`Phase 18` should be executed in major milestones, not micro-patches. + +Each major milestone should normally complete in `2-3` implementation steps: + +1. seam step +2. runtime step +3. closure/review step + +After each major milestone: + +1. update this phase file status +2. update `phase-18-log.md` with what changed and what was tested +3. update `phase-18-decisions.md` if any boundary or tradeoff changed +4. update review-base docs if the claim boundary changed + +## Phase 18 Major Milestones + +### `M1`: Transport-Backed RF2 Failover Runtime + +Goal: + +1. replace the current in-process participant shortcut with an explicit + transport/session adapter seam for promotion evidence and bounded replica + summary exchange + +Planned steps: + +1. seam step: + define transport/session adapter contracts while keeping current authority + split intact +2. runtime step: + make runtime-owned failover use adapter-backed participants instead of direct + in-process coupling +3. closure step: + prove healthy and gated failover through the runtime entry point across the + adapter seam + +Exit criteria: + +1. runtime failover no longer depends on direct `*Node` method calls as the only + implementation path +2. stage/error/result observability survives across the transport/session seam +3. no recovery-planner responsibility leaks into `masterv2` + +Current status: + +1. delivered +2. failover-time evidence now crosses an explicit transport/session adapter seam +3. runtime-owned failover still preserves stage/error/result observability +4. current implementation uses an in-memory request/response transport, not a + network transport matrix claim + +Review/test update: + +1. adapter seam introduced: + - `FailoverEvidenceAdapter` + - `FailoverTakeoverAdapter` + - `FailoverTarget` +2. first in-process adapter implementation delivered: + - `NewInProcessFailoverTarget(...)` +3. first transport-backed evidence implementation delivered: + - `InMemoryFailoverEvidenceTransport` + - `NewTransportEvidenceAdapter(...)` + - `NewHybridInProcessFailoverTarget(...)` +4. failover session/driver/runtime manager now use explicit targets instead of + direct `*Node` coupling as the primary path +5. healthy and gated failover tests now pass with promotion evidence and replica + summary traffic crossing the transport/session seam + +### `M2`: Active Loop 2 Replication Runtime + +Goal: + +1. turn primary-led Loop 2 from bounded takeover semantics into a continuously + active replication/runtime owner + +Planned steps: + +1. seam step: + define the minimum active Loop 2 runtime contracts for keepup/catchup/rebuild + progression +2. runtime step: + connect the active Loop 2 runtime to primary-side runtime ownership and + boundary observation +3. closure step: + prove at least one bounded active progression path beyond failover-only logic + +Exit criteria: + +1. Loop 2 has runtime-owned meaning outside failover +2. keepup/catchup/rebuild are not merely comments or future placeholders +3. outward mode remains a compressed projection, not the full runtime automaton + +Current status: + +1. not started + +Review/test update: + +1. pending + +### `M3`: Replicated Data Continuity Closure + +Goal: + +1. prove one bounded replicated continuity statement through real primary handoff + +Planned steps: + +1. seam step: + define the exact continuity contract to be claimed +2. runtime step: + run the handoff path through the new runtime instead of ad hoc proof-only + slices +3. closure step: + verify write -> progress -> failover -> continued service/data continuity + +Exit criteria: + +1. one healthy continuity path is explicit and repeatable +2. one degraded/gated path fails closed through the same runtime +3. the claim is bounded and does not silently widen into generic RF2 product + proof + +Current status: + +1. not started + +Review/test update: + +1. pending + +### `M4`: RF2 Product Runtime Surfaces + +Goal: + +1. attach bounded product/runtime surfaces to the new RF2 runtime without + breaking the ownership split + +Planned steps: + +1. seam step: + choose the first bounded RF2-facing product/runtime surfaces +2. runtime step: + attach them to the new runtime rather than to legacy mixed ownership +3. closure step: + prove one bounded product/runtime surface package on the new runtime + +Exit criteria: + +1. at least one RF2-facing runtime/product surface works on the new runtime +2. surface truth is still derived from the kernel/runtime authority model +3. no frontend/backend code becomes the hidden truth owner + +Current status: + +1. not started + +Review/test update: + +1. pending + +### `M5`: Productionization / Launch Envelope + +Goal: + +1. freeze one bounded productionization envelope for the new RF2 runtime path + +Planned steps: + +1. seam step: + define the explicit supported envelope and exclusions +2. runtime step: + collect the bounded pilot/preflight/stop-condition artifacts around the new + runtime path +3. closure step: + produce the review package for bounded productionization judgment + +Exit criteria: + +1. supported envelope, exclusions, and blockers are explicit +2. pilot/preflight/stop-condition artifacts exist for the bounded path +3. the result is reviewable as bounded productionization, not broad launch + approval + +Current status: + +1. not started + +Review/test update: + +1. pending + +## Initial Order + +The required execution order is: + +1. `M1` +2. `M2` +3. `M3` +4. `M4` +5. `M5` + +This order may be refined locally, but should not be broadly reordered without a +written decision in `phase-18-decisions.md`. + +## Current Focus + +The active next work is: + +1. `M2` seam step +2. keep the new transport/session-backed failover path as the reference closure + while active Loop 2 runtime is introduced + +## Review Base + +Use these files together when reviewing `Phase 18` work: + +1. `sw-block/design/v2-two-loop-protocol.md` +2. `sw-block/design/v2-automata-ownership-map.md` +3. `sw-block/design/v2-kernel-closure-review.md` +4. `sw-block/design/v2-protocol-claim-and-evidence.md` +5. `sw-block/.private/phase/phase-18.md` + +## Non-Goals For This Phase Document + +This file should not become: + +1. an unbounded idea dump +2. a day-by-day development log +3. a substitute for the claim/evidence ledger +4. a substitute for detailed kernel boundary documents diff --git a/sw-block/design/v2-kernel-closure-review.md b/sw-block/design/v2-kernel-closure-review.md index 700eee82b..66e4f657c 100644 --- a/sw-block/design/v2-kernel-closure-review.md +++ b/sw-block/design/v2-kernel-closure-review.md @@ -130,6 +130,84 @@ Another way to state the same result: 3. the selected primary inside that shell owns data-control truth and recovery choreography +## Current Milestone + +The current milestone is: + +1. explicit `masterv2` promotion authorization +2. explicit `volumev2` takeover preparation and activation gating +3. stepwise `FailoverSession` orchestration with observable stages +4. one in-process failover driver seam over registered participants +5. one runtime-owned `volumev2` manager that executes failover through a single + runtime entry point and retains latest failover snapshot/result + +What this milestone proves: + +1. failover is no longer only an implicit concept hidden in mixed runtime code +2. authorization, reconstruction, activation, and runtime ownership are now + separate code seams +3. the new kernel already has a usable in-process RF2 failover runtime slice + +What it does not yet prove: + +1. real transport-backed replica communication +2. background membership and health management beyond explicit failover calls +3. end-to-end replicated data continuity through a real primary handoff + +## Next Major Milestone + +The next major milestone should be: + +`transport-backed RF2 failover runtime` + +This means one level above the current in-process runtime slice: + +1. `FailoverSession` no longer talks only to in-memory participants +2. promotion evidence and bounded replica summaries cross a real transport seam +3. runtime-owned failover still preserves stage/error/result observability +4. the runtime keeps the separation: + - `masterv2` authorizes + - the selected primary reconstructs and gates activation + - the transport/runtime only carries requests, responses, and results + +### Target Shape + +Code should look roughly like: + +1. `masterv2`: heartbeat, promotion evidence arbitration, assignment authority +2. `volumev2`: node shell, takeover seams, failover session, runtime/manager +3. `purev2`: execution adapter and local boundary observation +4. transport/session adapters: promotion query and replica-summary exchange +5. `engine/replication`: semantic brain and future active data-control automata + +### Exit Criteria + +The milestone should be considered complete when all are true: + +1. one runtime-level API can execute failover without tests manually stitching + participants and sessions together +2. one transport-backed runtime-level API can execute failover without + in-process participant shortcuts +3. runtime-level observability exposes current/last failover stage, selected + node, assignment, and bounded truth across the transport seam +4. one healthy component-style failover test passes through the runtime entry + point +5. one gated/degraded component-style failover test stops through the same + runtime entry point +6. no new recovery-planner responsibility leaks back into `masterv2` + +### Non-Goals + +This milestone should not try to prove: + +1. broad product RF2 readiness +2. full catch-up or rebuild choreography +3. CSI or frontend productionization on top of RF2 +4. broad transport matrix approval + +It should prove only that the new kernel can carry the same authority model +through a real transport boundary without collapsing the ownership split. + ## Main Risk The main risk is not iSCSI or local I/O. The main risk is semantic leakage: diff --git a/sw-block/design/v2-protocol-claim-and-evidence.md b/sw-block/design/v2-protocol-claim-and-evidence.md index be956d682..3c2c8f50d 100644 --- a/sw-block/design/v2-protocol-claim-and-evidence.md +++ b/sw-block/design/v2-protocol-claim-and-evidence.md @@ -2,12 +2,15 @@ Date: 2026-04-05 Status: active -Purpose: keep one centralized ledger for the current chosen envelope, accepted claims, supporting evidence, invalidated evidence, and rerun obligations +Purpose: keep one centralized ledger for the current chosen envelope, accepted claims, supporting evidence, invalidated evidence, rerun obligations, and review baseline references ## Why This Document Exists `v2-protocol-truths.md` records stable protocol truths. `v2-protocol-closure-map.zh.md` records the structural closure model. +`v2-two-loop-protocol.md` records the current kernel protocol boundary. +`v2-automata-ownership-map.md` records ownership of identity/control/data semantics. +`v2-kernel-closure-review.md` records the current kernel closure standard. What they do not track in one place is the current operational contract: @@ -19,6 +22,23 @@ What they do not track in one place is the current operational contract: This document is that ledger. +It is not the only architectural source of truth. +It is the review base ledger that must be read together with the current kernel +boundary documents. + +## Review Base + +When reviewing a V2 change, use the following stack together: + +1. `v2-two-loop-protocol.md` for protocol boundary and authority rules +2. `v2-automata-ownership-map.md` for ownership of facts, events, and commands +3. `v2-kernel-closure-review.md` for current kernel milestone and closure standard +4. `v2-protocol-claim-and-evidence.md` for what may currently be claimed and + what evidence is still valid + +Use this document as the claim/evidence ledger and review entry point, not as +the only place where kernel structure is defined. + ## How To Use It When reviewing any new slice, bug fix, workload run, or delivery note, ask: @@ -32,24 +52,30 @@ If the answer changes the current state of the product, update this ledger in th ## Interpretation Rule For Current Integrated Evidence -Until an explicit `V2 core` exists as a real code structure and live event/command owner, -current integrated evidence should be interpreted as: +Current integrated evidence should be interpreted in two layers: -1. validation of current `V1` runtime behavior under `V2` constraints -2. not proof that a completed `V2 runtime` already exists +1. kernel-boundary evidence for the emerging `masterv2 + volumev2 + purev2` + structure +2. validation of constrained current-runtime behavior where the evidence still + comes from the older integrated path This means: -1. protocol truths and claim boundaries may already be `V2`-owned -2. workload and integration passes may still be about the constrained current runtime -3. later phases must keep separating: +1. protocol truths and kernel ownership boundaries may already be `V2`-owned +2. some workload and integration passes may still be about the constrained + current runtime +3. reviews must keep separating: - semantic authority + - new kernel closure evidence - constrained current-runtime validation - future pure-core extraction +If these layers are mixed, the review should be treated as incomplete. + ## Current Chosen Envelope -This is the bounded envelope currently allowed for active V2 claims: +This is the bounded legacy/integrated envelope currently allowed for active +runtime/product claims: | Item | Current value | Source | |------|---------------|--------| @@ -68,6 +94,13 @@ Current explicit exclusions: 4. broad transport matrix claims outside explicitly named evidence 5. treating synthetic benchmarks as substitutes for real workload validation +This envelope does not limit kernel-boundary reviews. +Kernel-boundary reviews should instead follow: + +1. `v2-two-loop-protocol.md` +2. `v2-automata-ownership-map.md` +3. `v2-kernel-closure-review.md` + ## Active Protocol Constraints These are the currently binding constraints that later work must preserve. @@ -86,6 +119,10 @@ These are the currently binding constraints that later work must preserve. | `CP13-7` | unrecoverable gap must escalate to `NeedsRebuild` and block normal paths | `Phase 13` | active | | `CP13-8A` | assignment delivered != receiver ready != publish healthy | `Phase 13` | active | | `CP13-9` | bounded external mode meaning must stay explicit and surface-consistent on the constrained current path | `Phase 13` | active | +| `K1` | `masterv2` is identity authority, not continuous recovery planner | `v2-two-loop-protocol.md`, `v2-automata-ownership-map.md` | active | +| `K2` | each volume is treated as a micro-cluster whose selected primary owns data-control truth | `v2-two-loop-protocol.md`, `v2-automata-ownership-map.md` | active | +| `K3` | takeover authorization belongs to `masterv2`, but reconstruction and activation gating belong to the new primary | `v2-two-loop-protocol.md`, `v2-automata-ownership-map.md` | active | +| `K4` | `Loop 1` and `Loop 2` must not collapse into one heartbeat or one state owner | `v2-two-loop-protocol.md` | active | ## Accepted Baselines @@ -112,12 +149,18 @@ These are the claims that may currently be made without overreach. | `C-PHASE17-PRODUCT-CHECKPOINT` | the current broader recovery-branch map, bounded failover/publication contract, bounded disturbance policy table, and first-launch envelope draft are explicit for the chosen path | bounded chosen path only; excludes broad production readiness, broad transport/frontend approval, and broad whole-surface failover/publication proof | `sw-block/.private/phase/phase-17.md`, `sw-block/.private/phase/phase-17-checkpoint-review.md` | allowed | | `C-FIRST-LAUNCH-ENVELOPE-DRAFT` | one bounded first-launch supported matrix is frozen as a draft with explicit exclusions and launch blockers | bounded chosen path only; not a launch decision, pilot approval, or rollout approval | `sw-block/design/v2-first-launch-supported-matrix.md` | allowed | | `C-PRODUCTIONIZATION-ARTIFACT-SET` | one bounded productionization artifact set now exists for internal pilot, preflight, stop-condition, and controlled-rollout discipline inside the frozen chosen envelope | bounded chosen path only; artifact existence only, not pilot success, rollout approval, or broader launch proof | `sw-block/design/v2-bounded-internal-pilot-pack.md`, `v2-pilot-preflight-checklist.md`, `v2-pilot-stop-conditions.md`, `v2-controlled-rollout-review.md` | allowed | +| `C-KERNEL-BOUNDARY` | the current V2 kernel boundary is explicitly defined around `masterv2` identity authority, `volumev2` takeover/data-control shell, and `purev2` execution adapter reuse | kernel-boundary statement only; not broad runtime/product readiness | `v2-two-loop-protocol.md`, `v2-automata-ownership-map.md`, `v2-kernel-closure-review.md` | allowed | +| `C-INPROCESS-FAILOVER-MILESTONE` | one in-process failover milestone exists with explicit authorization, reconstruction, activation gating, session observability, and driver seams | in-process `masterv2 + volumev2` code path only; not transport-complete RF2 product proof | `sw-block/runtime/masterv2/*`, `sw-block/runtime/volumev2/*` tests | allowed | +| `C-INPROCESS-RF2-FAILOVER-RUNTIME` | one runtime-owned in-process RF2 failover slice exists with participant registry, explicit runtime entry point, persisted session snapshots/results, and component-style healthy/gated failover tests | in-process runtime only; not transport-backed RF2 product proof | `sw-block/runtime/volumev2/runtime_manager.go`, `failover*.go`, `poc_test.go` | allowed | | `C-LAUNCH-APPROVAL` | broad product launch readiness | outside current phase | future | not allowed | ## Evidence Map | Evidence area | What it proves | Primary evidence | Support evidence | |---------------|----------------|------------------|------------------| +| Kernel ownership boundary | `masterv2`, `volumev2`, and `purev2` are split with explicit authority boundaries | `v2-two-loop-protocol.md`, `v2-automata-ownership-map.md`, `v2-kernel-closure-review.md` | code in `sw-block/runtime/masterv2/`, `volumev2/`, `purev2/` | +| In-process failover milestone | one explicit failover chain exists from promotion evidence through authorization, takeover preparation, activation gating, session observability, and driver wiring | `sw-block/runtime/volumev2/poc_test.go`, `failover.go`, `failover_driver.go` | `sw-block/runtime/masterv2/master_test.go` | +| In-process RF2 failover runtime | one runtime-owned failover manager exists with participant registry, runtime entry point, persisted snapshots/results, and component-style healthy/gated failover tests | `sw-block/runtime/volumev2/runtime_manager.go`, `poc_test.go` | `failover.go`, `failover_driver.go`, `v2-kernel-closure-review.md` | | Identity / addressing | stable identity and routable publication | `CP13-2` tests and docs | `qa_block_soak_test.go`, `sync_all_bug_test.go` | | Durable progress | barrier durability truth and non-legacy authority | `CP13-3` tests and docs | protocol tests around barrier handling | | State eligibility | only eligible replica state may satisfy sync durability | `CP13-4` tests and docs | adversarial state tests | @@ -165,4 +208,5 @@ No active `Phase 13` blocker currently remains inside the accepted bounded chose 1. do not add a new claim anywhere else without adding or updating the corresponding row here 2. when a bug narrows evidence, record the invalidation here in the same change 3. when a rerun restores a claim, move the row from `Invalidated Or Narrowed Evidence` to `Allowed Claims` or update its status -4. keep this document bounded to the active chosen path; do not turn it into a future roadmap +4. use this document as the review ledger, but keep kernel structure and authority rules in the dedicated kernel documents +5. keep this document bounded to active reviewable claims; do not turn it into a future roadmap diff --git a/sw-block/runtime/volumev2/failover.go b/sw-block/runtime/volumev2/failover.go index 4c9758065..e0a62791b 100644 --- a/sw-block/runtime/volumev2/failover.go +++ b/sw-block/runtime/volumev2/failover.go @@ -6,13 +6,13 @@ import ( "github.com/seaweedfs/seaweedfs/sw-block/runtime/masterv2" ) -// FailoverParticipant is the minimal surface needed to execute one failover -// flow across Loop 1 authorization and Loop 2 takeover preparation. +// FailoverParticipant is the legacy in-process convenience surface that +// predates the explicit failover adapter seam. It remains as a compatibility +// wrapper for tests and local callers while the adapter-backed target becomes +// the primary contract. type FailoverParticipant interface { - QueryPromotionEvidence(masterv2.PromotionQueryRequest) (masterv2.PromotionQueryResponse, error) - QueryReplicaSummarySource - PreparePrimaryTakeover(PrimaryTakeoverPlan) (ReconstructedPrimaryTruth, error) - GatePrimaryActivation(volumeName string, truth ReconstructedPrimaryTruth) error + FailoverEvidenceAdapter + FailoverTakeoverAdapter } // QueryReplicaSummarySource aliases the peer summary surface so the failover @@ -60,49 +60,76 @@ type FailoverSession struct { master *masterv2.Master volumeName string expectedEpoch uint64 - participants []FailoverParticipant + targets []FailoverTarget responses []masterv2.PromotionQueryResponse - byNode map[string]FailoverParticipant + byNode map[string]FailoverTarget result FailoverResult stage FailoverStage lastErr error } -// NewFailoverSession validates the narrow failover inputs and returns a +// NewFailoverSession validates the adapter-backed failover inputs and returns a // stepwise orchestration session. -func NewFailoverSession(master *masterv2.Master, volumeName string, expectedEpoch uint64, participants []FailoverParticipant) (*FailoverSession, error) { +func NewFailoverSession(master *masterv2.Master, volumeName string, expectedEpoch uint64, targets []FailoverTarget) (*FailoverSession, error) { if master == nil { return nil, fmt.Errorf("volumev2: master is nil") } if volumeName == "" { return nil, fmt.Errorf("volumev2: volume name is required") } - if len(participants) == 0 { - return nil, fmt.Errorf("volumev2: failover participants are required") + if len(targets) == 0 { + return nil, fmt.Errorf("volumev2: failover targets are required") } return &FailoverSession{ master: master, volumeName: volumeName, expectedEpoch: expectedEpoch, - participants: participants, + targets: targets, stage: FailoverStageNew, }, nil } +// NewFailoverSessionFromParticipants adapts the legacy participant surface into +// the explicit target seam for compatibility with older in-process tests. +func NewFailoverSessionFromParticipants(master *masterv2.Master, volumeName string, expectedEpoch uint64, participants []FailoverParticipant) (*FailoverSession, error) { + targets := make([]FailoverTarget, 0, len(participants)) + for _, participant := range participants { + if participant == nil { + continue + } + resp, err := participant.QueryPromotionEvidence(masterv2.PromotionQueryRequest{ + VolumeName: volumeName, + ExpectedEpoch: expectedEpoch, + }) + if err != nil { + return nil, fmt.Errorf("volumev2: participant target discovery %s: %w", volumeName, err) + } + if resp.NodeID == "" { + return nil, fmt.Errorf("volumev2: participant target discovery for %s missing node id", volumeName) + } + targets = append(targets, FailoverTarget{ + NodeID: resp.NodeID, + Evidence: participant, + Takeover: participant, + }) + } + return NewFailoverSession(master, volumeName, expectedEpoch, targets) +} + // CollectPromotionEvidence gathers fresh promotion responses from all // configured participants. func (s *FailoverSession) CollectPromotionEvidence() ([]masterv2.PromotionQueryResponse, error) { if s == nil { return nil, fmt.Errorf("volumev2: failover session is nil") } - responses := make([]masterv2.PromotionQueryResponse, 0, len(s.participants)) - byNode := make(map[string]FailoverParticipant, len(s.participants)) - for _, participant := range s.participants { - if participant == nil { + responses := make([]masterv2.PromotionQueryResponse, 0, len(s.targets)) + byNode := make(map[string]FailoverTarget, len(s.targets)) + for _, target := range s.targets { + if target.NodeID == "" || target.Evidence == nil { continue } - resp, err := participant.QueryPromotionEvidence(masterv2.PromotionQueryRequest{ + resp, err := target.Evidence.QueryPromotionEvidence(masterv2.PromotionQueryRequest{ VolumeName: s.volumeName, ExpectedEpoch: s.expectedEpoch, }) @@ -113,7 +140,7 @@ func (s *FailoverSession) CollectPromotionEvidence() ([]masterv2.PromotionQueryR return nil, s.failf("volumev2: promotion evidence for %s missing node id", s.volumeName) } responses = append(responses, resp) - byNode[resp.NodeID] = participant + byNode[resp.NodeID] = target } if len(responses) == 0 { return nil, s.failf("volumev2: no failover evidence collected for %s", s.volumeName) @@ -167,13 +194,18 @@ func (s *FailoverSession) PrepareTakeover() (ReconstructedPrimaryTruth, error) { return ReconstructedPrimaryTruth{}, s.failf("volumev2: authorized node %q missing participant", s.result.Assignment.NodeID) } peers := make([]ReplicaSummarySource, 0, len(s.byNode)-1) - for nodeID, participant := range s.byNode { + for nodeID, target := range s.byNode { if nodeID == s.result.Assignment.NodeID { continue } - peers = append(peers, participant) + if target.Evidence != nil { + peers = append(peers, target.Evidence) + } } - truth, err := selected.PreparePrimaryTakeover(PrimaryTakeoverPlan{ + if selected.Takeover == nil { + return ReconstructedPrimaryTruth{}, s.failf("volumev2: authorized node %q missing takeover adapter", s.result.Assignment.NodeID) + } + truth, err := selected.Takeover.PreparePrimaryTakeover(PrimaryTakeoverPlan{ Assignment: s.result.Assignment, Peers: peers, }) @@ -205,7 +237,10 @@ func (s *FailoverSession) Activate() error { if !ok { return s.failf("volumev2: authorized node %q missing participant", s.result.Assignment.NodeID) } - if err := selected.GatePrimaryActivation(s.volumeName, s.result.Truth); err != nil { + if selected.Takeover == nil { + return s.failf("volumev2: authorized node %q missing takeover adapter", s.result.Assignment.NodeID) + } + if err := selected.Takeover.GatePrimaryActivation(s.volumeName, s.result.Truth); err != nil { return s.fail(err) } s.stage = FailoverStageActivated @@ -283,8 +318,18 @@ func (s *FailoverSession) Run() (FailoverResult, error) { // ExecuteFailoverFlow runs the narrow failover path: // fresh promotion evidence -> master authorization -> takeover preparation -> // activation gate. It intentionally does not choreograph catch-up or rebuild. -func ExecuteFailoverFlow(master *masterv2.Master, volumeName string, expectedEpoch uint64, participants []FailoverParticipant) (FailoverResult, error) { - session, err := NewFailoverSession(master, volumeName, expectedEpoch, participants) +func ExecuteFailoverFlow(master *masterv2.Master, volumeName string, expectedEpoch uint64, targets []FailoverTarget) (FailoverResult, error) { + session, err := NewFailoverSession(master, volumeName, expectedEpoch, targets) + if err != nil { + return FailoverResult{}, err + } + return session.Run() +} + +// ExecuteFailoverFlowFromParticipants preserves the old in-process participant +// entry point while the explicit target seam becomes primary. +func ExecuteFailoverFlowFromParticipants(master *masterv2.Master, volumeName string, expectedEpoch uint64, participants []FailoverParticipant) (FailoverResult, error) { + session, err := NewFailoverSessionFromParticipants(master, volumeName, expectedEpoch, participants) if err != nil { return FailoverResult{}, err } diff --git a/sw-block/runtime/volumev2/failover_adapter.go b/sw-block/runtime/volumev2/failover_adapter.go new file mode 100644 index 000000000..6a17d3cc2 --- /dev/null +++ b/sw-block/runtime/volumev2/failover_adapter.go @@ -0,0 +1,84 @@ +package volumev2 + +import ( + "fmt" + + "github.com/seaweedfs/seaweedfs/sw-block/runtime/masterv2" + "github.com/seaweedfs/seaweedfs/sw-block/runtime/protocolv2" +) + +// FailoverEvidenceAdapter is the transport/session-facing query surface used by +// failover orchestration. Future remote implementations should satisfy this +// contract without changing failover session logic. +type FailoverEvidenceAdapter interface { + QueryPromotionEvidence(masterv2.PromotionQueryRequest) (masterv2.PromotionQueryResponse, error) + QueryReplicaSummary(protocolv2.ReplicaSummaryRequest) (protocolv2.ReplicaSummaryResponse, error) +} + +// FailoverTakeoverAdapter is the selected-primary execution surface used after +// masterv2 authorizes promotion. Future implementations may be local or remote, +// but takeover ownership remains on the selected primary side. +type FailoverTakeoverAdapter interface { + PreparePrimaryTakeover(PrimaryTakeoverPlan) (ReconstructedPrimaryTruth, error) + GatePrimaryActivation(volumeName string, truth ReconstructedPrimaryTruth) error +} + +// FailoverTarget binds one stable node id to the evidence and takeover adapters +// used by failover orchestration. +type FailoverTarget struct { + NodeID string + Evidence FailoverEvidenceAdapter + Takeover FailoverTakeoverAdapter +} + +// NewInProcessFailoverTarget builds the first adapter-backed target from one +// in-process volumev2 node. +func NewInProcessFailoverTarget(node *Node) (FailoverTarget, error) { + if node == nil { + return FailoverTarget{}, fmt.Errorf("volumev2: node is nil") + } + if node.NodeID() == "" { + return FailoverTarget{}, fmt.Errorf("volumev2: node id is required") + } + return FailoverTarget{ + NodeID: node.NodeID(), + Evidence: inProcessFailoverEvidenceAdapter{node: node}, + Takeover: inProcessFailoverTakeoverAdapter{node: node}, + }, nil +} + +type inProcessFailoverEvidenceAdapter struct { + node *Node +} + +func (a inProcessFailoverEvidenceAdapter) QueryPromotionEvidence(req masterv2.PromotionQueryRequest) (masterv2.PromotionQueryResponse, error) { + if a.node == nil { + return masterv2.PromotionQueryResponse{}, fmt.Errorf("volumev2: in-process evidence node is nil") + } + return a.node.QueryPromotionEvidence(req) +} + +func (a inProcessFailoverEvidenceAdapter) QueryReplicaSummary(req protocolv2.ReplicaSummaryRequest) (protocolv2.ReplicaSummaryResponse, error) { + if a.node == nil { + return protocolv2.ReplicaSummaryResponse{}, fmt.Errorf("volumev2: in-process evidence node is nil") + } + return a.node.QueryReplicaSummary(req) +} + +type inProcessFailoverTakeoverAdapter struct { + node *Node +} + +func (a inProcessFailoverTakeoverAdapter) PreparePrimaryTakeover(plan PrimaryTakeoverPlan) (ReconstructedPrimaryTruth, error) { + if a.node == nil { + return ReconstructedPrimaryTruth{}, fmt.Errorf("volumev2: in-process takeover node is nil") + } + return a.node.PreparePrimaryTakeover(plan) +} + +func (a inProcessFailoverTakeoverAdapter) GatePrimaryActivation(volumeName string, truth ReconstructedPrimaryTruth) error { + if a.node == nil { + return fmt.Errorf("volumev2: in-process takeover node is nil") + } + return a.node.GatePrimaryActivation(volumeName, truth) +} diff --git a/sw-block/runtime/volumev2/failover_driver.go b/sw-block/runtime/volumev2/failover_driver.go index 4e00ca426..2fde6b1cd 100644 --- a/sw-block/runtime/volumev2/failover_driver.go +++ b/sw-block/runtime/volumev2/failover_driver.go @@ -14,8 +14,8 @@ import ( type InProcessFailoverDriver struct { master *masterv2.Master - mu sync.RWMutex - participants map[string]FailoverParticipant + mu sync.RWMutex + targets map[string]FailoverTarget } // NewInProcessFailoverDriver creates a driver for one in-process masterv2. @@ -24,28 +24,41 @@ func NewInProcessFailoverDriver(master *masterv2.Master) (*InProcessFailoverDriv return nil, fmt.Errorf("volumev2: master is nil") } return &InProcessFailoverDriver{ - master: master, - participants: make(map[string]FailoverParticipant), + master: master, + targets: make(map[string]FailoverTarget), }, nil } -// RegisterParticipant binds one stable node id to one failover participant. -func (d *InProcessFailoverDriver) RegisterParticipant(nodeID string, participant FailoverParticipant) error { +// RegisterTarget binds one stable node id to one explicit failover target. +func (d *InProcessFailoverDriver) RegisterTarget(target FailoverTarget) error { if d == nil { return fmt.Errorf("volumev2: failover driver is nil") } - if nodeID == "" { - return fmt.Errorf("volumev2: participant node id is required") + if target.NodeID == "" { + return fmt.Errorf("volumev2: target node id is required") } - if participant == nil { - return fmt.Errorf("volumev2: participant %q is nil", nodeID) + if target.Evidence == nil { + return fmt.Errorf("volumev2: target %q missing evidence adapter", target.NodeID) } d.mu.Lock() defer d.mu.Unlock() - d.participants[nodeID] = participant + d.targets[target.NodeID] = target return nil } +// RegisterParticipant preserves the old convenience path by wrapping one +// legacy failover participant into an explicit target. +func (d *InProcessFailoverDriver) RegisterParticipant(nodeID string, participant FailoverParticipant) error { + if participant == nil { + return fmt.Errorf("volumev2: participant %q is nil", nodeID) + } + return d.RegisterTarget(FailoverTarget{ + NodeID: nodeID, + Evidence: participant, + Takeover: participant, + }) +} + // UnregisterParticipant removes one node from the in-process driver. func (d *InProcessFailoverDriver) UnregisterParticipant(nodeID string) { if d == nil || nodeID == "" { @@ -53,7 +66,7 @@ func (d *InProcessFailoverDriver) UnregisterParticipant(nodeID string) { } d.mu.Lock() defer d.mu.Unlock() - delete(d.participants, nodeID) + delete(d.targets, nodeID) } // ParticipantNodeIDs returns the currently registered node ids in stable order. @@ -63,8 +76,8 @@ func (d *InProcessFailoverDriver) ParticipantNodeIDs() []string { } d.mu.RLock() defer d.mu.RUnlock() - nodeIDs := make([]string, 0, len(d.participants)) - for nodeID := range d.participants { + nodeIDs := make([]string, 0, len(d.targets)) + for nodeID := range d.targets { nodeIDs = append(nodeIDs, nodeID) } slices.Sort(nodeIDs) @@ -77,11 +90,11 @@ func (d *InProcessFailoverDriver) NewSession(volumeName string, expectedEpoch ui if d == nil { return nil, fmt.Errorf("volumev2: failover driver is nil") } - participants, err := d.resolveParticipants(nodeIDs) + targets, err := d.resolveTargets(nodeIDs) if err != nil { return nil, err } - return NewFailoverSession(d.master, volumeName, expectedEpoch, participants) + return NewFailoverSession(d.master, volumeName, expectedEpoch, targets) } // Execute runs one failover using the resolved participant set. @@ -93,30 +106,30 @@ func (d *InProcessFailoverDriver) Execute(volumeName string, expectedEpoch uint6 return session.Run() } -func (d *InProcessFailoverDriver) resolveParticipants(nodeIDs []string) ([]FailoverParticipant, error) { +func (d *InProcessFailoverDriver) resolveTargets(nodeIDs []string) ([]FailoverTarget, error) { d.mu.RLock() defer d.mu.RUnlock() - if len(d.participants) == 0 { - return nil, fmt.Errorf("volumev2: no failover participants registered") + if len(d.targets) == 0 { + return nil, fmt.Errorf("volumev2: no failover targets registered") } resolvedIDs := nodeIDs if len(resolvedIDs) == 0 { - resolvedIDs = make([]string, 0, len(d.participants)) - for nodeID := range d.participants { + resolvedIDs = make([]string, 0, len(d.targets)) + for nodeID := range d.targets { resolvedIDs = append(resolvedIDs, nodeID) } slices.Sort(resolvedIDs) } - participants := make([]FailoverParticipant, 0, len(resolvedIDs)) + targets := make([]FailoverTarget, 0, len(resolvedIDs)) for _, nodeID := range resolvedIDs { - participant, ok := d.participants[nodeID] + target, ok := d.targets[nodeID] if !ok { - return nil, fmt.Errorf("volumev2: unknown failover participant %q", nodeID) + return nil, fmt.Errorf("volumev2: unknown failover target %q", nodeID) } - participants = append(participants, participant) + targets = append(targets, target) } - return participants, nil + return targets, nil } diff --git a/sw-block/runtime/volumev2/failover_evidence_transport.go b/sw-block/runtime/volumev2/failover_evidence_transport.go new file mode 100644 index 000000000..f0bdb4286 --- /dev/null +++ b/sw-block/runtime/volumev2/failover_evidence_transport.go @@ -0,0 +1,151 @@ +package volumev2 + +import ( + "fmt" + "sync" + + "github.com/seaweedfs/seaweedfs/sw-block/runtime/masterv2" + "github.com/seaweedfs/seaweedfs/sw-block/runtime/protocolv2" +) + +// FailoverEvidenceTransport carries failover-time query traffic across a +// transport/session boundary. This is the first transport seam needed for M1: +// promotion evidence and bounded replica summaries. +type FailoverEvidenceTransport interface { + QueryPromotionEvidence(nodeID string, req masterv2.PromotionQueryRequest) (masterv2.PromotionQueryResponse, error) + QueryReplicaSummary(nodeID string, req protocolv2.ReplicaSummaryRequest) (protocolv2.ReplicaSummaryResponse, error) +} + +// FailoverEvidenceHandler is the server-side evidence surface registered behind +// a transport implementation. +type FailoverEvidenceHandler interface { + QueryPromotionEvidence(masterv2.PromotionQueryRequest) (masterv2.PromotionQueryResponse, error) + QueryReplicaSummary(protocolv2.ReplicaSummaryRequest) (protocolv2.ReplicaSummaryResponse, error) +} + +// NewTransportEvidenceAdapter wraps one stable node id behind the supplied +// failover evidence transport. +func NewTransportEvidenceAdapter(nodeID string, transport FailoverEvidenceTransport) (FailoverEvidenceAdapter, error) { + if nodeID == "" { + return nil, fmt.Errorf("volumev2: evidence adapter node id is required") + } + if transport == nil { + return nil, fmt.Errorf("volumev2: evidence transport is nil") + } + return transportFailoverEvidenceAdapter{ + nodeID: nodeID, + transport: transport, + }, nil +} + +// NewHybridInProcessFailoverTarget builds the first transport-backed failover +// target shape: evidence crosses a transport/session seam, while takeover +// remains primary-local on the selected node. +func NewHybridInProcessFailoverTarget(node *Node, transport FailoverEvidenceTransport) (FailoverTarget, error) { + if node == nil { + return FailoverTarget{}, fmt.Errorf("volumev2: node is nil") + } + evidence, err := NewTransportEvidenceAdapter(node.NodeID(), transport) + if err != nil { + return FailoverTarget{}, err + } + return FailoverTarget{ + NodeID: node.NodeID(), + Evidence: evidence, + Takeover: inProcessFailoverTakeoverAdapter{node: node}, + }, nil +} + +type transportFailoverEvidenceAdapter struct { + nodeID string + transport FailoverEvidenceTransport +} + +func (a transportFailoverEvidenceAdapter) QueryPromotionEvidence(req masterv2.PromotionQueryRequest) (masterv2.PromotionQueryResponse, error) { + if a.transport == nil { + return masterv2.PromotionQueryResponse{}, fmt.Errorf("volumev2: transport evidence adapter is nil") + } + return a.transport.QueryPromotionEvidence(a.nodeID, req) +} + +func (a transportFailoverEvidenceAdapter) QueryReplicaSummary(req protocolv2.ReplicaSummaryRequest) (protocolv2.ReplicaSummaryResponse, error) { + if a.transport == nil { + return protocolv2.ReplicaSummaryResponse{}, fmt.Errorf("volumev2: transport evidence adapter is nil") + } + return a.transport.QueryReplicaSummary(a.nodeID, req) +} + +// InMemoryFailoverEvidenceTransport is the first request/response transport-style +// adapter. It is still in-process, but it eliminates direct orchestration-time +// calls to `*Node` for promotion evidence and replica-summary queries. +type InMemoryFailoverEvidenceTransport struct { + mu sync.RWMutex + handlers map[string]FailoverEvidenceHandler +} + +// NewInMemoryFailoverEvidenceTransport creates an in-memory request/response +// transport for failover evidence. +func NewInMemoryFailoverEvidenceTransport() *InMemoryFailoverEvidenceTransport { + return &InMemoryFailoverEvidenceTransport{ + handlers: make(map[string]FailoverEvidenceHandler), + } +} + +// RegisterHandler binds one stable node id to one evidence handler. +func (t *InMemoryFailoverEvidenceTransport) RegisterHandler(nodeID string, handler FailoverEvidenceHandler) error { + if t == nil { + return fmt.Errorf("volumev2: evidence transport is nil") + } + if nodeID == "" { + return fmt.Errorf("volumev2: evidence handler node id is required") + } + if handler == nil { + return fmt.Errorf("volumev2: evidence handler %q is nil", nodeID) + } + t.mu.Lock() + defer t.mu.Unlock() + t.handlers[nodeID] = handler + return nil +} + +// UnregisterHandler removes one evidence handler from the in-memory transport. +func (t *InMemoryFailoverEvidenceTransport) UnregisterHandler(nodeID string) { + if t == nil || nodeID == "" { + return + } + t.mu.Lock() + defer t.mu.Unlock() + delete(t.handlers, nodeID) +} + +func (t *InMemoryFailoverEvidenceTransport) QueryPromotionEvidence(nodeID string, req masterv2.PromotionQueryRequest) (masterv2.PromotionQueryResponse, error) { + handler, err := t.handler(nodeID) + if err != nil { + return masterv2.PromotionQueryResponse{}, err + } + return handler.QueryPromotionEvidence(req) +} + +func (t *InMemoryFailoverEvidenceTransport) QueryReplicaSummary(nodeID string, req protocolv2.ReplicaSummaryRequest) (protocolv2.ReplicaSummaryResponse, error) { + handler, err := t.handler(nodeID) + if err != nil { + return protocolv2.ReplicaSummaryResponse{}, err + } + return handler.QueryReplicaSummary(req) +} + +func (t *InMemoryFailoverEvidenceTransport) handler(nodeID string) (FailoverEvidenceHandler, error) { + if t == nil { + return nil, fmt.Errorf("volumev2: evidence transport is nil") + } + if nodeID == "" { + return nil, fmt.Errorf("volumev2: evidence node id is required") + } + t.mu.RLock() + defer t.mu.RUnlock() + handler, ok := t.handlers[nodeID] + if !ok { + return nil, fmt.Errorf("volumev2: unknown evidence handler %q", nodeID) + } + return handler, nil +} diff --git a/sw-block/runtime/volumev2/poc_test.go b/sw-block/runtime/volumev2/poc_test.go index 58fbfb1cc..0d908c3c8 100644 --- a/sw-block/runtime/volumev2/poc_test.go +++ b/sw-block/runtime/volumev2/poc_test.go @@ -730,7 +730,10 @@ func TestFailoverFlow_AuthorizesAndActivatesHealthyCandidate(t *testing.T) { t.Fatalf("sync node-b: %v", err) } - result, err := ExecuteFailoverFlow(master, "flow-vol", 2, []FailoverParticipant{nodeB, nodeC}) + result, err := ExecuteFailoverFlow(master, "flow-vol", 2, []FailoverTarget{ + mustInProcessFailoverTarget(t, nodeB), + mustInProcessFailoverTarget(t, nodeC), + }) if err != nil { t.Fatalf("execute failover flow: %v", err) } @@ -792,10 +795,11 @@ func TestFailoverFlow_StopsAtActivationGate(t *testing.T) { t.Fatalf("seed node-b: %v", err) } - result, err := ExecuteFailoverFlow(master, "flow-gated-vol", 2, []FailoverParticipant{ - nodeB, - staticFailoverParticipant{ - promotion: masterv2.PromotionQueryResponse{ + result, err := ExecuteFailoverFlow(master, "flow-gated-vol", 2, []FailoverTarget{ + mustInProcessFailoverTarget(t, nodeB), + staticFailoverTarget( + "node-c", + masterv2.PromotionQueryResponse{ VolumeName: "flow-gated-vol", NodeID: "node-c", Epoch: 1, @@ -804,7 +808,7 @@ func TestFailoverFlow_StopsAtActivationGate(t *testing.T) { Eligible: false, Reason: "needs_rebuild", }, - summary: protocolv2.ReplicaSummaryResponse{ + protocolv2.ReplicaSummaryResponse{ VolumeName: "flow-gated-vol", NodeID: "node-c", Epoch: 1, @@ -819,7 +823,7 @@ func TestFailoverFlow_StopsAtActivationGate(t *testing.T) { Eligible: false, Reason: "needs_rebuild", }, - }, + ), }) if err == nil { t.Fatal("expected failover gate error") @@ -888,7 +892,10 @@ func TestFailoverSession_StepwiseStagesExposeIntermediateState(t *testing.T) { t.Fatalf("sync node-b: %v", err) } - session, err := NewFailoverSession(master, "session-vol", 2, []FailoverParticipant{nodeB, nodeC}) + session, err := NewFailoverSession(master, "session-vol", 2, []FailoverTarget{ + mustInProcessFailoverTarget(t, nodeB), + mustInProcessFailoverTarget(t, nodeC), + }) if err != nil { t.Fatalf("new failover session: %v", err) } @@ -981,10 +988,11 @@ func TestFailoverSession_SnapshotCapturesFailureState(t *testing.T) { t.Fatalf("seed node-b: %v", err) } - session, err := NewFailoverSession(master, "snapshot-gated-vol", 2, []FailoverParticipant{ - nodeB, - staticFailoverParticipant{ - promotion: masterv2.PromotionQueryResponse{ + session, err := NewFailoverSession(master, "snapshot-gated-vol", 2, []FailoverTarget{ + mustInProcessFailoverTarget(t, nodeB), + staticFailoverTarget( + "node-c", + masterv2.PromotionQueryResponse{ VolumeName: "snapshot-gated-vol", NodeID: "node-c", Epoch: 1, @@ -993,7 +1001,7 @@ func TestFailoverSession_SnapshotCapturesFailureState(t *testing.T) { Eligible: false, Reason: "needs_rebuild", }, - summary: protocolv2.ReplicaSummaryResponse{ + protocolv2.ReplicaSummaryResponse{ VolumeName: "snapshot-gated-vol", NodeID: "node-c", Epoch: 1, @@ -1008,7 +1016,7 @@ func TestFailoverSession_SnapshotCapturesFailureState(t *testing.T) { Eligible: false, Reason: "needs_rebuild", }, - }, + ), }) if err != nil { t.Fatalf("new failover session: %v", err) @@ -1051,10 +1059,10 @@ func TestInProcessFailoverDriver_ExecuteHealthyFailover(t *testing.T) { t.Fatalf("new node-c: %v", err) } defer nodeC.Close() - if err := driver.RegisterParticipant("node-b", nodeB); err != nil { + if err := driver.RegisterTarget(mustInProcessFailoverTarget(t, nodeB)); err != nil { t.Fatalf("register node-b: %v", err) } - if err := driver.RegisterParticipant("node-c", nodeC); err != nil { + if err := driver.RegisterTarget(mustInProcessFailoverTarget(t, nodeC)); err != nil { t.Fatalf("register node-c: %v", err) } @@ -1121,11 +1129,12 @@ func TestInProcessFailoverDriver_ExecuteStopsOnGate(t *testing.T) { t.Fatalf("new node-b: %v", err) } defer nodeB.Close() - if err := driver.RegisterParticipant("node-b", nodeB); err != nil { + if err := driver.RegisterTarget(mustInProcessFailoverTarget(t, nodeB)); err != nil { t.Fatalf("register node-b: %v", err) } - if err := driver.RegisterParticipant("node-c", staticFailoverParticipant{ - promotion: masterv2.PromotionQueryResponse{ + if err := driver.RegisterTarget(staticFailoverTarget( + "node-c", + masterv2.PromotionQueryResponse{ VolumeName: "driver-gated-vol", NodeID: "node-c", Epoch: 1, @@ -1134,7 +1143,7 @@ func TestInProcessFailoverDriver_ExecuteStopsOnGate(t *testing.T) { Eligible: false, Reason: "needs_rebuild", }, - summary: protocolv2.ReplicaSummaryResponse{ + protocolv2.ReplicaSummaryResponse{ VolumeName: "driver-gated-vol", NodeID: "node-c", Epoch: 1, @@ -1149,7 +1158,7 @@ func TestInProcessFailoverDriver_ExecuteStopsOnGate(t *testing.T) { Eligible: false, Reason: "needs_rebuild", }, - }); err != nil { + )); err != nil { t.Fatalf("register node-c: %v", err) } @@ -1181,6 +1190,335 @@ func TestInProcessFailoverDriver_ExecuteStopsOnGate(t *testing.T) { } } +func TestInProcessRuntimeManager_ExecuteHealthyFailoverAndPersistSnapshot(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + manager, err := NewInProcessRuntimeManager(master) + if err != nil { + t.Fatalf("new runtime manager: %v", err) + } + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + nodeC, err := New(Config{NodeID: "node-c"}) + if err != nil { + t.Fatalf("new node-c: %v", err) + } + defer nodeC.Close() + if err := manager.RegisterNode(nodeB); err != nil { + t.Fatalf("register node-b: %v", err) + } + if err := manager.RegisterNode(nodeC); err != nil { + t.Fatalf("register node-c: %v", err) + } + + tempDir := t.TempDir() + pathB := filepath.Join(tempDir, "manager-b.blk") + pathC := filepath.Join(tempDir, "manager-c.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "manager-vol", + Path: pathB, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := nodeB.ApplyAssignments([]masterv2.Assignment{{ + Name: "manager-vol", + Path: pathB, + NodeID: "node-b", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-b: %v", err) + } + if err := nodeC.ApplyAssignments([]masterv2.Assignment{{ + Name: "manager-vol", + Path: pathC, + NodeID: "node-c", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-c: %v", err) + } + if err := nodeB.WriteLBA("manager-vol", 0, bytes.Repeat([]byte{0x52}, 4096)); err != nil { + t.Fatalf("write node-b: %v", err) + } + if err := nodeB.SyncCache("manager-vol"); err != nil { + t.Fatalf("sync node-b: %v", err) + } + + result, err := manager.ExecuteFailover("manager-vol", 2) + if err != nil { + t.Fatalf("manager execute failover: %v", err) + } + if result.Assignment.NodeID != "node-b" { + t.Fatalf("assignment node=%q, want node-b", result.Assignment.NodeID) + } + + lastSnap, ok := manager.LastFailoverSnapshot() + if !ok { + t.Fatal("expected last failover snapshot") + } + if lastSnap.Stage != FailoverStageActivated { + t.Fatalf("snapshot stage=%q, want %q", lastSnap.Stage, FailoverStageActivated) + } + if lastSnap.SelectedNodeID != "node-b" { + t.Fatalf("snapshot selected node=%q, want node-b", lastSnap.SelectedNodeID) + } + + perVolSnap, ok := manager.FailoverSnapshot("manager-vol") + if !ok { + t.Fatal("expected per-volume failover snapshot") + } + if perVolSnap.Result.Truth.PrimaryNodeID != "node-b" { + t.Fatalf("per-volume truth primary=%q, want node-b", perVolSnap.Result.Truth.PrimaryNodeID) + } + + lastResult, ok := manager.LastFailoverResult() + if !ok { + t.Fatal("expected last failover result") + } + if lastResult.Assignment.NodeID != "node-b" { + t.Fatalf("last result assignment=%q, want node-b", lastResult.Assignment.NodeID) + } +} + +func TestInProcessRuntimeManager_ExecuteStopsOnGateAndPersistsFailureSnapshot(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + manager, err := NewInProcessRuntimeManager(master) + if err != nil { + t.Fatalf("new runtime manager: %v", err) + } + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + if err := manager.RegisterNode(nodeB); err != nil { + t.Fatalf("register node-b: %v", err) + } + if err := manager.RegisterTarget(staticFailoverTarget( + "node-c", + masterv2.PromotionQueryResponse{ + VolumeName: "manager-gated-vol", + NodeID: "node-c", + Epoch: 1, + CommittedLSN: 1, + WALHeadLSN: 1, + Eligible: false, + Reason: "needs_rebuild", + }, + protocolv2.ReplicaSummaryResponse{ + VolumeName: "manager-gated-vol", + NodeID: "node-c", + Epoch: 1, + Role: "replica", + Mode: "needs_rebuild", + CommittedLSN: 3, + DurableLSN: 2, + CheckpointLSN: 1, + RecoveryPhase: "needs_rebuild", + LastBarrierOK: false, + LastBarrierReason: "timeout", + Eligible: false, + Reason: "needs_rebuild", + }, + )); err != nil { + t.Fatalf("register node-c: %v", err) + } + + tempDir := t.TempDir() + pathB := filepath.Join(tempDir, "manager-gated-b.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "manager-gated-vol", + Path: pathB, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := nodeB.ApplyAssignments([]masterv2.Assignment{{ + Name: "manager-gated-vol", + Path: pathB, + NodeID: "node-b", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-b: %v", err) + } + + _, err = manager.ExecuteFailover("manager-gated-vol", 2) + if err == nil { + t.Fatal("expected manager failover gate error") + } + + lastSnap, ok := manager.LastFailoverSnapshot() + if !ok { + t.Fatal("expected last failover snapshot") + } + if lastSnap.Stage != FailoverStageFailed { + t.Fatalf("snapshot stage=%q, want %q", lastSnap.Stage, FailoverStageFailed) + } + if lastSnap.LastError == "" { + t.Fatal("expected snapshot last error") + } + if !lastSnap.Result.Truth.NeedsRebuild { + t.Fatalf("expected needs_rebuild truth in snapshot: %+v", lastSnap.Result.Truth) + } +} + +func TestTransportEvidenceAdapter_HealthyFailoverFlow(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + transport := NewInMemoryFailoverEvidenceTransport() + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + nodeC, err := New(Config{NodeID: "node-c"}) + if err != nil { + t.Fatalf("new node-c: %v", err) + } + defer nodeC.Close() + if err := transport.RegisterHandler("node-b", nodeB); err != nil { + t.Fatalf("register node-b handler: %v", err) + } + if err := transport.RegisterHandler("node-c", nodeC); err != nil { + t.Fatalf("register node-c handler: %v", err) + } + + tempDir := t.TempDir() + pathB := filepath.Join(tempDir, "transport-b.blk") + pathC := filepath.Join(tempDir, "transport-c.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "transport-vol", + Path: pathB, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := nodeB.ApplyAssignments([]masterv2.Assignment{{ + Name: "transport-vol", + Path: pathB, + NodeID: "node-b", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-b: %v", err) + } + if err := nodeC.ApplyAssignments([]masterv2.Assignment{{ + Name: "transport-vol", + Path: pathC, + NodeID: "node-c", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-c: %v", err) + } + if err := nodeB.WriteLBA("transport-vol", 0, bytes.Repeat([]byte{0x61}, 4096)); err != nil { + t.Fatalf("write node-b: %v", err) + } + if err := nodeB.SyncCache("transport-vol"); err != nil { + t.Fatalf("sync node-b: %v", err) + } + + result, err := ExecuteFailoverFlow(master, "transport-vol", 2, []FailoverTarget{ + mustHybridFailoverTarget(t, nodeB, transport), + mustHybridFailoverTarget(t, nodeC, transport), + }) + if err != nil { + t.Fatalf("transport-backed failover flow: %v", err) + } + if result.Assignment.NodeID != "node-b" { + t.Fatalf("assignment node=%q, want node-b", result.Assignment.NodeID) + } + if result.Truth.PrimaryNodeID != "node-b" { + t.Fatalf("truth primary=%q, want node-b", result.Truth.PrimaryNodeID) + } +} + +func TestTransportEvidenceAdapter_GatedFailoverFlow(t *testing.T) { + master := masterv2.New(masterv2.Config{}) + transport := NewInMemoryFailoverEvidenceTransport() + nodeB, err := New(Config{NodeID: "node-b"}) + if err != nil { + t.Fatalf("new node-b: %v", err) + } + defer nodeB.Close() + if err := transport.RegisterHandler("node-b", nodeB); err != nil { + t.Fatalf("register node-b handler: %v", err) + } + + tempDir := t.TempDir() + pathB := filepath.Join(tempDir, "transport-gated-b.blk") + if err := master.DeclarePrimary(masterv2.VolumeSpec{ + Name: "transport-gated-vol", + Path: pathB, + PrimaryNodeID: "node-a", + CreateOptions: testCreateOptions(), + }); err != nil { + t.Fatalf("declare primary: %v", err) + } + if err := nodeB.ApplyAssignments([]masterv2.Assignment{{ + Name: "transport-gated-vol", + Path: pathB, + NodeID: "node-b", + Epoch: 2, + LeaseTTL: 30 * time.Second, + CreateOptions: testCreateOptions(), + Role: "primary", + }}); err != nil { + t.Fatalf("seed node-b: %v", err) + } + + _, err = ExecuteFailoverFlow(master, "transport-gated-vol", 2, []FailoverTarget{ + mustHybridFailoverTarget(t, nodeB, transport), + staticFailoverTarget( + "node-c", + masterv2.PromotionQueryResponse{ + VolumeName: "transport-gated-vol", + NodeID: "node-c", + Epoch: 1, + CommittedLSN: 1, + WALHeadLSN: 1, + Eligible: false, + Reason: "needs_rebuild", + }, + protocolv2.ReplicaSummaryResponse{ + VolumeName: "transport-gated-vol", + NodeID: "node-c", + Epoch: 1, + Role: "replica", + Mode: "needs_rebuild", + CommittedLSN: 3, + DurableLSN: 2, + CheckpointLSN: 1, + RecoveryPhase: "needs_rebuild", + LastBarrierOK: false, + LastBarrierReason: "timeout", + Eligible: false, + Reason: "needs_rebuild", + }, + ), + }) + if err == nil { + t.Fatal("expected gated transport-backed failover error") + } +} + func mustHeartbeat(t *testing.T, node *Node) masterv2.NodeHeartbeat { t.Helper() hb, err := node.Heartbeat() @@ -1202,6 +1540,24 @@ func mustPromotionEvidence(t *testing.T, node *Node, volumeName string, epoch ui return resp } +func mustInProcessFailoverTarget(t *testing.T, node *Node) FailoverTarget { + t.Helper() + target, err := NewInProcessFailoverTarget(node) + if err != nil { + t.Fatalf("new in-process failover target: %v", err) + } + return target +} + +func mustHybridFailoverTarget(t *testing.T, node *Node, transport FailoverEvidenceTransport) FailoverTarget { + t.Helper() + target, err := NewHybridInProcessFailoverTarget(node, transport) + if err != nil { + t.Fatalf("new hybrid failover target: %v", err) + } + return target +} + type staticReplicaSummarySource struct { resp protocolv2.ReplicaSummaryResponse err error @@ -1211,25 +1567,37 @@ func (s staticReplicaSummarySource) QueryReplicaSummary(protocolv2.ReplicaSummar return s.resp, s.err } -type staticFailoverParticipant struct { +type staticFailoverAdapter struct { promotion masterv2.PromotionQueryResponse summary protocolv2.ReplicaSummaryResponse err error } -func (s staticFailoverParticipant) QueryPromotionEvidence(masterv2.PromotionQueryRequest) (masterv2.PromotionQueryResponse, error) { +func staticFailoverTarget(nodeID string, promotion masterv2.PromotionQueryResponse, summary protocolv2.ReplicaSummaryResponse) FailoverTarget { + adapter := staticFailoverAdapter{ + promotion: promotion, + summary: summary, + } + return FailoverTarget{ + NodeID: nodeID, + Evidence: adapter, + Takeover: adapter, + } +} + +func (s staticFailoverAdapter) QueryPromotionEvidence(masterv2.PromotionQueryRequest) (masterv2.PromotionQueryResponse, error) { return s.promotion, s.err } -func (s staticFailoverParticipant) QueryReplicaSummary(protocolv2.ReplicaSummaryRequest) (protocolv2.ReplicaSummaryResponse, error) { +func (s staticFailoverAdapter) QueryReplicaSummary(protocolv2.ReplicaSummaryRequest) (protocolv2.ReplicaSummaryResponse, error) { return s.summary, s.err } -func (s staticFailoverParticipant) PreparePrimaryTakeover(PrimaryTakeoverPlan) (ReconstructedPrimaryTruth, error) { +func (s staticFailoverAdapter) PreparePrimaryTakeover(PrimaryTakeoverPlan) (ReconstructedPrimaryTruth, error) { return ReconstructedPrimaryTruth{}, fmt.Errorf("static failover participant cannot prepare takeover") } -func (s staticFailoverParticipant) GatePrimaryActivation(string, ReconstructedPrimaryTruth) error { +func (s staticFailoverAdapter) GatePrimaryActivation(string, ReconstructedPrimaryTruth) error { return fmt.Errorf("static failover participant cannot gate activation") } diff --git a/sw-block/runtime/volumev2/runtime_manager.go b/sw-block/runtime/volumev2/runtime_manager.go new file mode 100644 index 000000000..f6d93a439 --- /dev/null +++ b/sw-block/runtime/volumev2/runtime_manager.go @@ -0,0 +1,175 @@ +package volumev2 + +import ( + "fmt" + "sync" + + "github.com/seaweedfs/seaweedfs/sw-block/runtime/masterv2" +) + +// InProcessRuntimeManager is the first runtime-owned failover manager for the +// new kernel. It wraps the in-process failover driver, owns participant +// registration, and retains the latest failover snapshots/results for +// observability. +type InProcessRuntimeManager struct { + driver *InProcessFailoverDriver + evidenceTransport *InMemoryFailoverEvidenceTransport + + mu sync.RWMutex + lastSnapshot FailoverSnapshot + lastResult FailoverResult + hasLastResult bool + snapshotsByName map[string]FailoverSnapshot + resultsByName map[string]FailoverResult +} + +// NewInProcessRuntimeManager creates a runtime-owned failover manager over one +// in-process masterv2 instance. +func NewInProcessRuntimeManager(master *masterv2.Master) (*InProcessRuntimeManager, error) { + driver, err := NewInProcessFailoverDriver(master) + if err != nil { + return nil, err + } + return &InProcessRuntimeManager{ + driver: driver, + evidenceTransport: NewInMemoryFailoverEvidenceTransport(), + snapshotsByName: make(map[string]FailoverSnapshot), + resultsByName: make(map[string]FailoverResult), + }, nil +} + +// RegisterNode registers one concrete volumev2 node under its stable node id. +func (m *InProcessRuntimeManager) RegisterNode(node *Node) error { + if m == nil { + return fmt.Errorf("volumev2: runtime manager is nil") + } + if node == nil { + return fmt.Errorf("volumev2: node is nil") + } + if err := m.evidenceTransport.RegisterHandler(node.NodeID(), node); err != nil { + return err + } + target, err := NewHybridInProcessFailoverTarget(node, m.evidenceTransport) + if err != nil { + return err + } + return m.RegisterTarget(target) +} + +// RegisterTarget registers one explicit failover target. +func (m *InProcessRuntimeManager) RegisterTarget(target FailoverTarget) error { + if m == nil || m.driver == nil { + return fmt.Errorf("volumev2: runtime manager is nil") + } + return m.driver.RegisterTarget(target) +} + +// RegisterParticipant registers one failover-capable participant explicitly. +func (m *InProcessRuntimeManager) RegisterParticipant(nodeID string, participant FailoverParticipant) error { + if m == nil || m.driver == nil { + return fmt.Errorf("volumev2: runtime manager is nil") + } + return m.driver.RegisterParticipant(nodeID, participant) +} + +// UnregisterParticipant removes one participant from the runtime-owned driver. +func (m *InProcessRuntimeManager) UnregisterParticipant(nodeID string) { + if m == nil || m.driver == nil { + return + } + if m.evidenceTransport != nil { + m.evidenceTransport.UnregisterHandler(nodeID) + } + m.driver.UnregisterParticipant(nodeID) +} + +// ParticipantNodeIDs returns the current runtime-owned participant ids. +func (m *InProcessRuntimeManager) ParticipantNodeIDs() []string { + if m == nil || m.driver == nil { + return nil + } + return m.driver.ParticipantNodeIDs() +} + +// NewFailoverSession resolves participants through the runtime-owned registry. +func (m *InProcessRuntimeManager) NewFailoverSession(volumeName string, expectedEpoch uint64, nodeIDs ...string) (*FailoverSession, error) { + if m == nil || m.driver == nil { + return nil, fmt.Errorf("volumev2: runtime manager is nil") + } + return m.driver.NewSession(volumeName, expectedEpoch, nodeIDs...) +} + +// ExecuteFailover runs one runtime-owned failover and persists the latest +// observable snapshot/result for the volume and the manager as a whole. +func (m *InProcessRuntimeManager) ExecuteFailover(volumeName string, expectedEpoch uint64, nodeIDs ...string) (FailoverResult, error) { + if m == nil { + return FailoverResult{}, fmt.Errorf("volumev2: runtime manager is nil") + } + session, err := m.NewFailoverSession(volumeName, expectedEpoch, nodeIDs...) + if err != nil { + return FailoverResult{}, err + } + result, runErr := session.Run() + m.recordSnapshot(volumeName, session.Snapshot(), result) + return result, runErr +} + +// LastFailoverSnapshot returns the most recent runtime-owned failover snapshot. +func (m *InProcessRuntimeManager) LastFailoverSnapshot() (FailoverSnapshot, bool) { + if m == nil { + return FailoverSnapshot{}, false + } + m.mu.RLock() + defer m.mu.RUnlock() + if m.lastSnapshot.VolumeName == "" { + return FailoverSnapshot{}, false + } + return m.lastSnapshot, true +} + +// FailoverSnapshot returns the latest snapshot for one volume if present. +func (m *InProcessRuntimeManager) FailoverSnapshot(volumeName string) (FailoverSnapshot, bool) { + if m == nil { + return FailoverSnapshot{}, false + } + m.mu.RLock() + defer m.mu.RUnlock() + snap, ok := m.snapshotsByName[volumeName] + return snap, ok +} + +// LastFailoverResult returns the most recent runtime-owned failover result. +func (m *InProcessRuntimeManager) LastFailoverResult() (FailoverResult, bool) { + if m == nil { + return FailoverResult{}, false + } + m.mu.RLock() + defer m.mu.RUnlock() + if !m.hasLastResult { + return FailoverResult{}, false + } + return m.lastResult, true +} + +// FailoverResult returns the latest result for one volume if present. +func (m *InProcessRuntimeManager) FailoverResult(volumeName string) (FailoverResult, bool) { + if m == nil { + return FailoverResult{}, false + } + m.mu.RLock() + defer m.mu.RUnlock() + result, ok := m.resultsByName[volumeName] + return result, ok +} + +func (m *InProcessRuntimeManager) recordSnapshot(volumeName string, snapshot FailoverSnapshot, result FailoverResult) { + m.mu.Lock() + defer m.mu.Unlock() + m.lastSnapshot = snapshot + m.lastResult = result + m.hasLastResult = true + if volumeName != "" { + m.snapshotsByName[volumeName] = snapshot + m.resultsByName[volumeName] = result + } +}