From bdf83e350ead48186baff9c6798e6e8104ea6ff0 Mon Sep 17 00:00:00 2001 From: pingqiu Date: Thu, 2 Apr 2026 16:25:58 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20Phase=2011=20=E2=80=94=20product-surfac?= =?UTF-8?q?e=20rebinding=20(snapshot,=20CSI,=20publication,=20restore)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P1 Snapshots: CoW snapshot lifecycle through V2 engine path, create/list/delete via master RPC, BaseLSN tracking in manifest, ImportSnapshotForRebuild P2 CSI Lifecycle: masterServerBackend calling real MasterServer in-process, CreateVolume/DeleteVolume/ExpandVolume through CSI → master → VS flow, ExportedControllerServer/ExportedNodeServer for cross-package testing P3 Publication: LookupBlockVolume coherence across failover, iSCSI + NVMe address switching on promotion, repeated lookup self-consistency P4 Restore: RestoreBlockSnapshot RPC through master and volume server, snapshot restore with runtime convergence, epoch/role validation Co-Authored-By: Claude Opus 4.6 (1M context) --- sw-block/.private/phase/phase-11-log.md | 1450 +++++++++++++++++ sw-block/.private/phase/phase-11.md | 488 ++++++ weed/server/qa_block_csi_lifecycle_test.go | 340 ++++ weed/server/qa_block_publication_test.go | 303 ++++ weed/server/qa_block_restore_test.go | 240 +++ weed/server/qa_block_snapshot_product_test.go | 342 ++++ .../blockvol/csi/export_test_helpers.go | 86 + 7 files changed, 3249 insertions(+) create mode 100644 sw-block/.private/phase/phase-11-log.md create mode 100644 sw-block/.private/phase/phase-11.md create mode 100644 weed/server/qa_block_csi_lifecycle_test.go create mode 100644 weed/server/qa_block_publication_test.go create mode 100644 weed/server/qa_block_restore_test.go create mode 100644 weed/server/qa_block_snapshot_product_test.go create mode 100644 weed/storage/blockvol/csi/export_test_helpers.go diff --git a/sw-block/.private/phase/phase-11-log.md b/sw-block/.private/phase/phase-11-log.md new file mode 100644 index 000000000..647c07c2c --- /dev/null +++ b/sw-block/.private/phase/phase-11-log.md @@ -0,0 +1,1450 @@ +# Phase 11 Log + +Date: 2026-04-02 +Status: active +Purpose: record the technical pack and design notes for the first bounded product-surface rebinding slice after accepted `Phase 10` control-plane closure + +--- + +### P0 Technical Pack + +Date: 2026-04-02 +Goal: convert `Phase 11` from a broad roadmap label into one bounded first product-surface package with explicit steps, hard indicators, and reject shapes + +#### Step breakdown + +##### Step 1: First-surface contract freeze + +Goal: + +- decide the first surface and freeze what the first slice claims + +Decision: + +1. the first surface is `snapshot product path` +2. the first slice is limited to: + - create + - list + - delete +3. the first slice does not claim: + - restore + - clone + - broad snapshot workflow readiness + - `CSI` / `NVMe` / `iSCSI` rebinding + +Hard indicators: + +1. one explicit first-slice statement exists in `phase-11.md` +2. excluded surfaces/workflows are written down explicitly +3. no active planning text still says the first slice is ambiguous + +Reject if: + +1. the first slice still includes multiple product surfaces +2. clone/restore are implied without an explicit proof plan +3. the package is broad enough that assignment cannot stay bounded + +##### Step 2: Product-truth mapping + +Goal: + +- define exactly how product-visible snapshot behavior maps to accepted backend snapshot truth + +Must make explicit: + +1. what product-visible create means on the chosen path +2. what product-visible list must reflect +3. what delete removes and how visibility changes afterward +4. which backend surfaces remain authoritative for snapshot truth +5. which unsupported cases must fail closed + +Hard indicators: + +1. one authoritative truth chain is written: + - product request + - master/volume-server action + - backend snapshot truth + - visible product metadata/result +2. one explicit no-drift statement exists: + - visible snapshot state must match backend snapshot state +3. one explicit unsupported set exists: + - restore/clone or any deferred workflow + +Reject if: + +1. product-visible semantics are described only in backend terms +2. visible metadata ownership is ambiguous between master and volume server +3. unsupported cases are left implicit + +##### Step 3: Proof package definition + +Goal: + +- define the minimum test/evidence ladder that can accept the first slice + +Required proofs: + +1. create proof + - product-visible snapshot create succeeds on the chosen path + - created snapshot is visible through list/readback metadata +2. delete proof + - deleted snapshot disappears from the visible set +3. list coherence proof + - visible snapshot metadata matches backend truth +4. fail-closed proof + - invalid or unsupported cases do not imply false success +5. boundedness proof + - the slice does not overclaim restore/clone/full workflow support + +Hard indicators: + +1. at least one focused create/list/delete integration package is identified +2. visible-state assertions exist, not only RPC success assertions +3. no-overclaim wording is part of the acceptance text + +Reject if: + +1. tests prove only local helper logic +2. metadata/reporting truth is not asserted +3. broader workflow claims appear without evidence + +#### Recommended first-cut implementation boundary + +Prefer touching only the smallest surfaces that define product-visible snapshot truth: + +1. master snapshot RPC path +2. volume-server snapshot/list/delete path +3. focused snapshot integration tests + +Reference only unless the proof exposes a real mismatch: + +1. accepted V2 executor/backend snapshot machinery +2. accepted control-loop closure from `Phase 10` +3. unrelated front-end surfaces (`CSI`, `NVMe`, `iSCSI`) + +#### Assignment For `sw` + +1. Goal + - deliver `P1` snapshot product-path rebinding on the chosen path + +2. Required outputs + - one bounded contract statement for create/list/delete + - one create/list/delete proof package + - one fail-closed package for invalid/unsupported cases + - short reuse note: + - files updated in place + - files used as references only + - any copied code and why + +3. Hard rules + - do not reopen accepted `Phase 09` backend execution semantics + - do not reopen accepted `Phase 10` control-plane closure + - do not absorb `CSI`, `NVMe`, `iSCSI`, restore, or clone into the first slice + - keep claims bounded to snapshot create/list/delete on the chosen path + +#### Assignment For `tester` + +1. Goal + - validate that `P1` proves product-visible snapshot rebinding rather than backend-only snapshot behavior + +2. Validate + - create/list/delete visible behavior + - metadata coherence with backend truth + - fail-closed unsupported behavior + - no-overclaim around deferred workflows + +3. Reject if + - evidence is backend-only + - visible metadata truth is not checked + - clone/restore/full workflow claims appear without proof + +#### Short judgment + +`P0` is complete when: + +1. the first surface is fixed +2. the first slice steps are explicit +3. hard acceptance indicators are written down +4. `sw` and `tester` can start `P1` without re-deciding scope + +--- + +### P1 Technical Pack + +Date: 2026-04-02 +Goal: deliver bounded snapshot product-path rebinding on the chosen path so product-visible snapshot create/list/delete behavior is backed by accepted V2 execution/control truth without semantic drift or overclaim + +#### Layer 1: Semantic Core + +##### Problem statement + +`Phase 09` accepted real snapshot execution on the chosen path. +`Phase 10` accepted bounded master-driven control-loop closure on that same path. + +What is still not accepted is not backend snapshot ability itself. +It is whether the product-visible snapshot path can reuse the existing master/volume-server surfaces without letting older V1-facing wrappers silently become the source of truth. + +The first `Phase 11` slice should close that gap for one bounded snapshot surface: + +1. create +2. list +3. delete + +It should not turn into: + +1. restore +2. clone +3. full snapshot workflow productization +4. `CSI` / `NVMe` / `iSCSI` rebinding + +##### System reality + +Current reality already includes these surfaces: + +1. master-facing snapshot RPC wrappers: + - `MasterServer.CreateBlockSnapshot()` + - `MasterServer.DeleteBlockSnapshot()` + - `MasterServer.ListBlockSnapshots()` +2. volume-server execution wrappers: + - `BlockService.SnapshotBlockVol()` + - `BlockService.DeleteBlockSnapshot()` + - `BlockService.ListBlockSnapshots()` +3. backend execution reality under `blockvol`: + - `CreateSnapshot` + - `DeleteSnapshot` + - `ListSnapshots` + +The phase question is therefore: + +1. which surfaces are reused +2. which surfaces are updated +3. how visible product truth is kept aligned with accepted backend truth +4. how no-overclaim is enforced around unsupported workflow breadth + +##### High-level algorithm + +Use this rebinding model: + +1. product-visible snapshot request enters through the master snapshot RPC surface +2. master resolves the authoritative chosen-path volume entry and delegates to the owning volume server +3. volume server executes the snapshot operation on the accepted block volume path +4. product-visible response/listing reflect the same backend snapshot truth +5. unsupported or invalid cases fail closed rather than implying broader snapshot readiness + +##### Pseudo code + +```text +on snapshot create/list/delete request: + validate request at product-facing RPC boundary + resolve authoritative volume entry from master registry + call the owning volume-server snapshot surface + execute against accepted blockvol snapshot machinery + return only the product-visible metadata supported by the bounded slice + +after operation: + visible snapshot set matches backend snapshot truth + repeated delete is bounded by explicit idempotent or fail-closed contract + no restore/clone/full-workflow support is implied +``` + +##### State / contract + +`P1` must make these truths explicit: + +1. product-visible snapshot create/list/delete are now part of the accepted chosen-path product proof +2. V1-facing snapshot wrappers may be reused, but they are bounded adapters, not semantic owners +3. accepted backend snapshot truth remains the correctness source underneath the product surface +4. visible snapshot metadata must match backend truth for the chosen path +5. restore/clone/full workflow readiness remain deferred unless a later slice accepts them explicitly + +##### Reject shapes + +Reject before implementation if the slice: + +1. proves only backend snapshot mechanics without product-visible create/list/delete behavior +2. reuses V1 wrappers without explicitly bounding their role +3. silently turns create/list/delete into broader snapshot workflow claims +4. makes master-visible and volume-server-visible snapshot truth diverge +5. absorbs `CSI`, `NVMe`, `iSCSI`, restore, or clone work + +#### Layer 2: Execution Core + +##### Current gaps `P1` must close + +1. current snapshot tests mostly prove master wrapper behavior and volume-server local behavior separately +2. there is not yet one accepted bounded statement that these surfaces together form the first product-visible snapshot slice +3. V1 surface reuse is present in code but not yet explicitly bounded in phase accounting +4. no-overclaim around restore/clone/full workflow support is not yet part of the first-slice acceptance package + +##### Reuse / update instructions + +1. `weed/server/master_grpc_server_block.go` + - `update in place` + - this is the primary product-facing snapshot RPC surface for the bounded slice + - keep it as a wrapper/resolution layer; do not let it redefine snapshot semantics + - V1 reuse role: + - `reuse as bounded adapter` + +2. `weed/server/volume_server_block.go` + - `update in place` + - this is the primary volume-server execution surface for create/list/delete + - keep it as the execution adapter over accepted `blockvol` snapshot reality + - V1 reuse role: + - `reuse as bounded adapter` + +3. `weed/server/master_grpc_server_block_test.go` + - `update in place` + - extend/reshape tests so the first slice proves bounded product-visible RPC behavior rather than wrapper-only success + +4. `weed/server/volume_grpc_block_test.go` + - `update in place` + - keep focused create/list/delete execution checks on the volume-server side + +5. `weed/server/*snapshot*test.go` + - `update in place` if a dedicated bounded integration package is cleaner than overloading existing wrapper tests + - prefer one focused create/list/delete product-path package rather than many scattered small tests + +6. `weed/storage/blockvol/*` + - `reference only` unless the first product proof exposes a real snapshot-execution mismatch + - accepted snapshot execution reality should not be casually reopened + - V1 reuse role: + - `reuse as execution reality only` + +7. `weed/storage/blockvol/v2bridge/*` + - `reference only` + - no new bridge semantics should be introduced for this first snapshot slice unless a real mismatch is exposed + +8. copy guidance + - prefer `update in place` + - no parallel snapshot path + - every reused V1 surface must be named explicitly in delivery notes + +##### Validation focus + +Required proofs: + +1. create proof + - product-visible create succeeds on the chosen path + - response metadata is coherent with the created backend snapshot + +2. list coherence proof + - listed snapshot IDs/metadata match backend truth after create and after delete + +3. delete proof + - delete removes the snapshot from the visible set + - repeated delete behavior is explicit and bounded + +4. fail-closed proof + - nonexistent volume, missing snapshot, or unsupported preconditions do not imply false success + +5. reuse-boundary proof + - delivery notes identify every reused V1-facing surface and how it is bounded + +6. no-overclaim proof + - acceptance wording does not imply restore/clone/full workflow readiness + +Reject if: + +1. evidence remains split into unrelated wrapper-local tests with no bounded product statement +2. visible metadata truth is not asserted after create/delete/list +3. slice claims broader snapshot workflow support than it proves +4. reuse of V1 surfaces is left implicit + +##### Suggested first cut + +1. freeze the bounded product contract for create/list/delete +2. identify the smallest product-visible create/list/delete proof path through: + - master RPC + - volume-server execution + - backend snapshot truth +3. add or reshape tests so metadata truth is checked after create/list/delete +4. write explicit no-overclaim wording for restore/clone/full workflow deferral + +##### Assignment For `sw` + +1. Goal + - deliver `P1` snapshot product-path rebinding on the chosen path + +2. Required outputs + - one bounded contract statement for snapshot create/list/delete + - one focused create/list/delete proof package + - one fail-closed package for invalid/unsupported cases + - one explicit V1 reuse note listing: + - master-facing reused surfaces + - volume-server reused surfaces + - backend execution surfaces referenced only + - short reuse note: + - files updated in place + - files used as references only + - any copied code and why + +3. Hard rules + - do not reopen accepted `Phase 09` snapshot execution semantics casually + - do not reopen accepted `Phase 10` control-plane closure + - do not absorb restore, clone, `CSI`, `NVMe`, or `iSCSI` into the first slice + - keep all V1 reuse explicit and bounded + +##### Assignment For `tester` + +1. Goal + - validate that `P1` proves bounded product-visible snapshot rebinding rather than wrapper-only or backend-only behavior + +2. Validate + - create/list/delete visible behavior + - metadata coherence with backend truth + - fail-closed unsupported behavior + - explicit V1 reuse boundaries in delivery notes + - no-overclaim around restore/clone/full workflow support + +3. Reject if + - evidence is backend-only or wrapper-only + - visible metadata truth is not checked + - V1 reuse is not explicitly bounded + - restore/clone/full workflow claims appear without proof + +#### Short judgment + +`P1` is acceptable when: + +1. product-visible snapshot create/list/delete are proven on the chosen path +2. visible metadata matches backend truth +3. fail-closed cases are explicit +4. all V1 reuse surfaces are named and bounded +5. broader snapshot workflow support is still explicitly deferred + +--- + +### P1 Completion Record + +Date: 2026-04-02 +Status: accepted + +Accepted contract: + +1. `P1` proves bounded snapshot product-path rebinding for: + - create + - list + - delete +2. accepted proof is product-visible and routed through the real master and volume-server adapter chain +3. `P1` does not claim: + - restore + - clone + - full snapshot workflow readiness + - `CSI`/transport rebinding + +Accepted evidence: + +1. `weed/server/qa_block_snapshot_product_test.go` + - `TestP11P1_SnapshotCreate` + - `TestP11P1_SnapshotListCoherence` + - `TestP11P1_SnapshotDelete` + - `TestP11P1_SnapshotFailClosed` +2. the accepted chain includes: + - master snapshot RPC entry + - real block registry resolution + - real volume-server snapshot adapter methods + - accepted `blockvol` snapshot execution +3. metadata coherence is explicitly checked between create response and list/readback truth + +Reuse note: + +1. `weed/server/master_grpc_server_block.go` + - `reuse as bounded adapter` +2. `weed/server/volume_server_block.go` + - `reuse as bounded adapter` +3. `weed/storage/blockvol/*` + - `reuse as execution reality only` +4. `weed/storage/blockvol/v2bridge/*` + - `reference only` + +Residual notes: + +1. `P1` closure does not reduce the need for `P2`; it only proves snapshot product rebinding +2. `CSI` snapshot, expand, and transport-specific publish behavior remain outside accepted `P1` scope + +--- + +### P2 Technical Pack + +Date: 2026-04-02 +Goal: deliver bounded `CSI` controller/node rebinding on the chosen path so product-visible CSI volume lifecycle behavior is backed by accepted V2 execution/control truth without letting older CSI/V1 wrappers silently become semantic owners + +#### Layer 1: Semantic Core + +##### Problem statement + +`Phase 09` accepted execution truth on the chosen path. +`Phase 10` accepted bounded control-plane closure on that same path. +`P1` accepted a first product-visible rebinding slice for snapshots. + +What is still not accepted is whether the `CSI` product surface for the main volume lifecycle is coherently bound to that same chosen path. + +The first `P2` slice should close that gap for one bounded `CSI` surface: + +1. `CreateVolume` +2. `DeleteVolume` +3. `ControllerPublishVolume` +4. `NodeStageVolume` +5. `NodePublishVolume` +6. `NodeUnpublishVolume` +7. `NodeUnstageVolume` + +It should not turn into: + +1. `CSI` snapshot closure +2. `CSI` expand closure +3. `NVMe` transport preference or failover closure +4. broad Kubernetes readiness/productization +5. multi-node/topology breadth beyond the chosen bounded path + +##### System reality + +Current reality already includes these surfaces: + +1. CSI controller wrapper surface: + - `weed/storage/blockvol/csi/controller.go` +2. CSI node wrapper surface: + - `weed/storage/blockvol/csi/node.go` +3. CSI driver/server assembly: + - `weed/storage/blockvol/csi/server.go` +4. CSI backend bridge: + - `weed/storage/blockvol/csi/volume_backend.go` +5. local execution helper: + - `weed/storage/blockvol/csi/volume_manager.go` +6. accepted master-backed block product/control surfaces underneath the backend bridge: + - `CreateBlockVolume` + - `DeleteBlockVolume` + - `LookupBlockVolume` + +The phase question is therefore: + +1. which of these CSI/V1 surfaces are only wrappers +2. which of them are allowed to remain bridges/adapters +3. how controller-visible truth and node-visible truth are kept aligned with accepted chosen-path volume truth +4. how no-overclaim is enforced around unproven CSI breadth + +##### High-level algorithm + +Use this rebinding model: + +1. CSI controller receives create/delete/publish requests +2. controller delegates through the backend bridge to the accepted master-backed chosen-path volume surface +3. master returns authoritative target information for the chosen path +4. CSI node consumes only that published access truth and performs bounded stage/publish work +5. teardown uses the same bounded truth and removes staged/published state coherently +6. unsupported or incomplete publish/volume context fails closed rather than implying readiness + +##### Pseudo code + +```text +on CSI controller create/delete/publish: + validate bounded CSI request contract + call backend bridge + backend bridge calls accepted master-backed volume surface + return only the target and metadata truth proven by this slice + +on CSI node stage/publish/unstage/unpublish: + consume publish_context or volume_context from the bounded controller truth + resolve transport/device only for the chosen bounded path + stage and publish coherently + teardown coherently and preserve retry semantics on failure + +after operation: + controller-visible truth and node-visible truth match accepted chosen-path volume truth + missing or partial target info fails closed + no snapshot/expand/NVMe/K8s-breadth claim is implied +``` + +##### State / contract + +`P2` must make these truths explicit: + +1. CSI controller/node lifecycle is now the active product-surface rebinding target +2. CSI wrappers may be reused, but only as bounded adapters over accepted chosen-path truth +3. `volume_backend.go` is a bounded bridge, not a semantic owner +4. local `volume_manager.go` behavior may be used as execution reality only where the chosen proof explicitly allows it +5. node-visible staging/publishing must not drift from controller/master-provided target truth +6. CSI snapshot, expand, NVMe preference, and broader cluster readiness remain deferred unless a later slice accepts them explicitly + +##### Reject shapes + +Reject before implementation if the slice: + +1. proves only controller-local or node-local helper behavior with no chosen-path coherence statement +2. lets local manager shortcuts replace accepted master-backed truth for the active slice +3. silently turns bounded CSI lifecycle proof into snapshot/expand/NVMe/K8s readiness claims +4. leaves publish-context ownership or fail-closed behavior implicit + +#### Layer 2: Execution Core + +##### Current gaps `P2` must close + +1. current CSI tests are broad and mixed across controller, node, local mode, NVMe, snapshot, and adversarial cases +2. there is not yet one accepted bounded statement that the main CSI controller/node lifecycle is rebound to the accepted chosen path +3. reuse of CSI/V1 surfaces is present in code but not yet explicitly bounded in phase accounting +4. no-overclaim around snapshot/expand/NVMe/K8s breadth is not yet part of an accepted `P2` package + +##### Reuse / update instructions + +1. `weed/storage/blockvol/csi/controller.go` + - `update in place` + - this is the primary CSI controller product surface for the bounded slice + - keep it as a wrapper over the backend bridge; do not let it define backend truth + - V1 reuse role: + - `reuse as bounded adapter` + +2. `weed/storage/blockvol/csi/node.go` + - `update in place` + - this is the primary CSI node product surface for stage/publish lifecycle proof + - keep retry/fail-closed behavior explicit; do not let local fallback silently replace chosen-path truth + - V1 reuse role: + - `reuse as bounded adapter` + +3. `weed/storage/blockvol/csi/server.go` + - `update in place` + - driver assembly and mode wiring may be reused + - do not let mode convenience become product-scope inflation + - V1 reuse role: + - `reuse as bounded adapter` + +4. `weed/storage/blockvol/csi/volume_backend.go` + - `update in place` + - this is the key backend bridge from CSI to accepted master-backed volume truth + - keep the bridge explicit; do not let it become a second semantic owner + - V1 reuse role: + - `reuse as bounded bridge` + +5. `weed/storage/blockvol/csi/volume_manager.go` + - `reference only` unless the bounded proof exposes a real execution mismatch + - local manager behavior may support tests or local execution reality, but should not redefine the active chosen-path proof + - V1 reuse role: + - `reuse as execution reality only` + +6. `weed/server/master_grpc_server_block.go` + - `reference only` unless the bounded CSI proof exposes a real mismatch + - accepted master-facing volume truth already exists underneath the CSI bridge + - V1 reuse role: + - `reuse as bounded adapter` + +7. `weed/server/volume_server_block.go` + - `reference only` unless a real mismatch is exposed through CSI publication or chosen-path target truth + - do not reopen accepted control/execution surfaces casually + - V1 reuse role: + - `reuse as bounded adapter` + +8. `weed/storage/blockvol/csi/*test.go` + - `update in place` + - prefer one focused bounded lifecycle proof package rather than scattering acceptance across many unrelated tests + +9. copy guidance + - prefer `update in place` + - no parallel CSI path + - every reused V1 surface must be named explicitly in delivery notes + +##### Validation focus + +Required proofs: + +1. controller create/publish proof + - controller create succeeds on the chosen path + - publish context reflects accepted master-backed target truth + +2. node stage/publish proof + - node stages and publishes using that controller/master-provided truth + - chosen-path access contract is coherent end-to-end + +3. teardown proof + - unpublish/unstage/delete complete coherently + - retry semantics remain bounded and explicit on failure + +4. fail-closed proof + - missing or partial publish/volume context does not imply false success + - invalid or missing target information remains explicit + +5. reuse-boundary proof + - delivery notes identify every reused CSI/V1-facing surface and how it is bounded + +6. no-overclaim proof + - acceptance wording does not imply CSI snapshot, expand, NVMe preference, or broad K8s readiness + +Reject if: + +1. evidence remains split into unrelated unit tests with no bounded product statement +2. controller-visible truth and node-visible truth are not asserted against the same chosen-path source +3. local-manager convenience silently becomes the active proof path +4. slice claims broader CSI readiness than it proves + +##### Suggested first cut + +1. freeze the bounded CSI lifecycle contract for create/delete/publish/stage/publish/unstage/unpublish +2. identify the smallest chosen-path proof chain through: + - CSI controller + - backend bridge + - accepted master-backed target truth + - CSI node +3. add or reshape tests so controller-visible target info and node-consumed target info are asserted together +4. write explicit no-overclaim wording for CSI snapshot/expand/NVMe/K8s deferral + +##### Assignment For `sw` + +1. Goal + - deliver bounded `P2` CSI controller/node rebinding on the chosen path + +2. Required outputs + - one bounded contract statement for CSI lifecycle in scope + - one focused controller/node lifecycle proof package + - one fail-closed package for missing/partial target information + - one explicit V1 reuse note listing: + - CSI wrapper surfaces + - backend bridge surfaces + - execution-reality-only surfaces + - short reuse note: + - files updated in place + - files used as references only + - any copied code and why + +3. Hard rules + - do not reopen accepted `Phase 09`, `Phase 10`, or accepted `P1` semantics casually + - do not absorb CSI snapshot, expand, `NVMe`, or broad K8s/productization work into the first CSI slice + - keep all V1/CSI reuse explicit and bounded + +##### Assignment For `tester` + +1. Goal + - validate that `P2` proves bounded product-visible CSI rebinding rather than local-manager-only or wrapper-only behavior + +2. Validate + - controller/node lifecycle coherence + - publish-context coherence with accepted chosen-path truth + - fail-closed handling for missing/partial target information + - explicit V1 reuse boundaries in delivery notes + - no-overclaim around CSI snapshot, expand, NVMe, and broader K8s readiness + +3. Reject if + - evidence is local-only or wrapper-only + - controller truth and node truth are not checked together + - V1/CSI reuse is not explicitly bounded + - broader CSI readiness claims appear without proof + +#### Short judgment + +`P2` is acceptable when: + +1. controller create/publish and node stage/publish/teardown are proven on the chosen path +2. controller-visible target truth and node-consumed target truth match +3. fail-closed cases are explicit +4. all reused CSI/V1 surfaces are named and bounded +5. broader CSI snapshot/expand/NVMe/K8s readiness is still explicitly deferred + +--- + +### P2 Completion Record + +Date: 2026-04-02 +Status: accepted + +Accepted contract: + +1. `P2` proves bounded CSI controller/node lifecycle rebinding for: + - `CreateVolume` + - `DeleteVolume` + - `ControllerPublishVolume` + - `NodeStageVolume` + - `NodePublishVolume` + - `NodeUnpublishVolume` + - `NodeUnstageVolume` +2. accepted proof uses the real master-backed create/lookup/delete path for the controller side +3. accepted proof uses `mgr=nil` on the node side so published target truth is the only viable source +4. `P2` does not claim: + - CSI snapshot closure + - CSI expand closure + - NVMe preference/failover closure + - broad Kubernetes or product readiness + +Accepted evidence: + +1. `weed/server/qa_block_csi_lifecycle_test.go` + - `TestP11P2_CSI_MasterProduced_PublishContext` + - `TestP11P2_CSI_FullLifecycle` + - `TestP11P2_CSI_FailClosed_NoContext` + - `TestP11P2_CSI_FailClosed_MissingVolume` +2. accepted chain includes: + - CSI controller wrapper entry + - backend bridge calling real `MasterServer.CreateBlockVolume()` + - real `MasterServer.LookupBlockVolume()` + - real `MasterServer.DeleteBlockVolume()` + - real block registry and allocation path + - node-side consumption of published target truth with `mgr=nil` +3. publish-context proof explicitly asserts: + - master-produced `iscsiAddr` + - master-produced `iqn` +4. lifecycle proof explicitly asserts: + - `isLocal=false` + - staged address matches published address + - teardown errors are checked + - real `.blk` removal after delete +5. fail-closed proof explicitly asserts: + - `FailedPrecondition` with missing context on `mgr=nil` node + - `NotFound` on missing volume publish + - idempotent delete on missing volume + +Reuse note: + +1. `weed/storage/blockvol/csi/controller.go` + - `reuse as bounded adapter` +2. `weed/storage/blockvol/csi/node.go` + - `reuse as bounded adapter` +3. `weed/storage/blockvol/csi/volume_backend.go` + - `reuse as bounded bridge` +4. `weed/server/master_grpc_server_block.go` + - `reuse as bounded adapter` +5. `weed/storage/blockvol/csi/volume_manager.go` + - `reference only` for the accepted proof path +6. `weed/storage/blockvol/csi/export_test_helpers.go` + - test-only support surface + +Residual notes: + +1. `P2` closure does not imply CSI snapshot or expand readiness +2. `P2` closure does not imply NVMe publication/failover truth closure +3. `P3` remains necessary to close front-end publication/address/naming truth + +--- + +### P3 Technical Pack + +Date: 2026-04-02 +Goal: deliver bounded `NVMe` / `iSCSI` front-end publication rebinding on the chosen path so product-visible access metadata matches accepted backend/control truth without letting wrapper-local field plumbing become the semantic owner + +#### Layer 1: Semantic Core + +##### Problem statement + +`Phase 09` accepted execution truth on the chosen path. +`Phase 10` accepted bounded control-plane closure on that same path. +`P1` accepted snapshot product rebinding. +`P2` accepted bounded CSI controller/node lifecycle rebinding. + +What is still not accepted is whether front-end publication truth itself is coherent for the chosen path: + +1. are `iSCSI` publication fields coherent at create and lookup +2. are `NVMe` publication fields coherent at create and lookup +3. do heartbeat refresh and failover preserve or update those fields correctly +4. does fallback behavior remain explicit when `NVMe` is absent + +The first `P3` slice should close that gap for bounded front-end publication truth: + +1. `CreateBlockVolume` publication fields +2. `LookupBlockVolume` publication fields +3. heartbeat-driven publication refresh/reconstruction +4. failover publication truth update + +It should not turn into: + +1. full runtime initiator behavior proof +2. end-to-end transport performance proof +3. broad NVMe superiority/product positioning claims +4. broad cluster hardening or soak readiness + +##### System reality + +Current reality already includes these surfaces: + +1. master-facing publication surfaces: + - `weed/server/master_grpc_server_block.go` +2. registry publication truth carriers: + - `weed/server/master_block_registry.go` +3. volume-server publication source: + - `weed/server/volume_grpc_block.go` +4. block-service publication helpers: + - `weed/server/volume_server_block.go` +5. existing publication-focused test surfaces: + - `weed/server/qa_block_nvme_publication_test.go` + +The phase question is therefore: + +1. which fields are authoritative publication truth on the chosen path +2. how publication truth flows from volume server to registry to create/lookup responses +3. how failover and restart change or reconstruct that truth +4. how fallback is kept explicit when front-end capability differs by server + +##### High-level algorithm + +Use this rebinding model: + +1. volume server allocates or heartbeats with front-end publication fields +2. master registry stores or refreshes those fields as chosen-path truth +3. master create/lookup responses publish the same front-end truth +4. failover moves publication truth to the new primary and clears stale ownership +5. mixed-capability or fallback behavior remains explicit in published metadata + +##### Pseudo code + +```text +on create/lookup: + resolve chosen-path primary in master registry + return the current publication truth for that primary + include fallback publication fields where supported + +on heartbeat refresh or master restart: + rebuild or refresh publication fields from accepted volume-server truth + +on failover: + promote new primary + switch publication fields to the new primary's front-end truth + clear stale publication ownership from the old primary + +after operation: + create-visible, lookup-visible, and registry-visible publication truth match + fallback behavior is explicit + no runtime/performance closure is implied +``` + +##### State / contract + +`P3` must make these truths explicit: + +1. front-end publication truth is now the active product-surface rebinding target +2. registry publication fields may carry truth, but only as bounded carriers under accepted chosen-path semantics +3. volume-server allocation/heartbeat publication fields are bounded publication sources, not independent semantic owners +4. create-visible and lookup-visible publication truth must remain coherent through failover and restart reconstruction +5. absence of `NVMe` must produce explicit fallback behavior rather than false capability claims + +##### Reject shapes + +Reject before implementation if the slice: + +1. proves only field presence without create/lookup/failover coherence +2. lets stale publication truth survive failover or restart reconstruction +3. silently turns publication-field proof into runtime transport/performance claims +4. leaves fallback semantics implicit + +#### Layer 2: Execution Core + +##### Current gaps `P3` must close + +1. publication fields exist in code and tests, but have not yet been accepted as one bounded chosen-path closure package +2. `NVMe` and `iSCSI` publication truth still risks being treated as scattered field plumbing rather than one coherent product surface +3. failover/restart reconstruction proof exists directionally, but not yet as the active accepted slice +4. no-overclaim around full transport runtime/performance still needs explicit phase accounting + +##### Reuse / update instructions + +1. `weed/server/master_grpc_server_block.go` + - `update in place` + - this is the primary master-facing publication surface for create/lookup truth + - keep it as a bounded adapter over registry truth + - V1 reuse role: + - `reuse as bounded adapter` + +2. `weed/server/master_block_registry.go` + - `update in place` + - this is the bounded truth carrier for publication fields across heartbeat and failover + - do not let registry field caching become an unexamined semantic shortcut + - V1 reuse role: + - `reuse as bounded truth carrier` + +3. `weed/server/volume_grpc_block.go` + - `update in place` + - this is the primary publication source for allocate responses + - keep publication derivation explicit and canonical + - V1 reuse role: + - `reuse as publication source only` + +4. `weed/server/volume_server_block.go` + - `reference only` unless a real publication mismatch is exposed + - heartbeat publication helpers may be reused as bounded publication sources + - V1 reuse role: + - `reuse as publication source only` + +5. `weed/server/qa_block_nvme_publication_test.go` + - `update in place` + - prefer one bounded create/lookup/failover/restart publication package rather than scattered field tests + +6. `weed/storage/blockvol/csi/controller.go` + - `reference only` + - CSI remains a downstream consumer of the accepted publication truth in `P3` + +7. copy guidance + - prefer `update in place` + - no parallel publication path + - every reused V1/publication surface must be named explicitly in delivery notes + +##### Validation focus + +Required proofs: + +1. create/lookup proof + - create publishes coherent `iSCSI` and, where enabled, `NVMe` publication fields + - lookup returns the same chosen-path publication truth + +2. failover proof + - publication truth moves to the new primary after failover + - stale publication truth from the old primary does not survive + +3. heartbeat/restart reconstruction proof + - publication truth can be refreshed or rebuilt from accepted heartbeat data + +4. fallback/mixed-capability proof + - clusters with and without `NVMe` capability expose explicit coherent fallback behavior + +5. reuse-boundary proof + - delivery notes identify every publication surface touched and how it is bounded + +6. no-overclaim proof + - acceptance wording does not imply real initiator runtime, performance, or broad production readiness + +Reject if: + +1. evidence remains field-local and does not prove create/lookup/failover/restart coherence +2. publication truth is not checked across at least two user-visible surfaces +3. fallback behavior is implicit +4. slice claims transport runtime or performance closure without proof + +##### Suggested first cut + +1. freeze the bounded publication contract for create/lookup/failover/restart +2. identify the smallest proof chain through: + - volume-server publication source + - registry truth carrier + - master create/lookup responses +3. extend tests so `iSCSI` and `NVMe` fields are checked together where relevant +4. write explicit no-overclaim wording for runtime transport/performance deferral + +##### Assignment For `sw` + +1. Goal + - deliver bounded `P3` front-end publication rebinding on the chosen path + +2. Required outputs + - one bounded contract statement for publication/address/naming truth + - one focused create/lookup/failover/restart proof package + - one explicit fallback package for mixed-capability or no-`NVMe` behavior + - one explicit V1 reuse note listing: + - master publication surfaces + - registry truth carriers + - volume-server publication sources + - short reuse note: + - files updated in place + - files used as references only + - any copied code and why + +3. Hard rules + - do not reopen accepted `Phase 09`, `Phase 10`, `P1`, or `P2` semantics casually + - do not absorb full runtime transport proof, performance claims, or broad production hardening into the first publication slice + - keep all V1/publication reuse explicit and bounded + +##### Assignment For `tester` + +1. Goal + - validate that `P3` proves bounded publication rebinding rather than scattered field plumbing + +2. Validate + - create/lookup publication coherence + - failover publication update coherence + - heartbeat/restart reconstruction coherence + - explicit fallback behavior when `NVMe` is absent + - explicit V1/publication reuse boundaries in delivery notes + - no-overclaim around runtime transport/performance/product readiness + +3. Reject if + - evidence is field-local only + - stale publication truth can survive failover or restart + - fallback behavior is not explicit + - broader transport/runtime claims appear without proof + +#### Short judgment + +`P3` is acceptable when: + +1. create-visible and lookup-visible publication truth are proven coherent on the chosen path +2. failover and restart reconstruction update publication truth correctly +3. fallback behavior is explicit +4. all reused publication/V1 surfaces are named and bounded +5. broader runtime/performance readiness is still explicitly deferred + +--- + +### P3 Completion Record + +Date: 2026-04-02 +Status: accepted + +Accepted contract: + +1. `P3` proves bounded front-end publication rebinding for: + - create/lookup publication coherence + - failover publication switch + - heartbeat-driven publication reconstruction + - no-`NVMe` fallback behavior +2. accepted proof closes publication/address truth on the chosen path without claiming runtime transport closure +3. `P3` does not claim: + - full initiator/runtime transport proof + - performance claims + - broad production readiness + - any workflow closure beyond bounded publication truth + +Accepted evidence: + +1. `weed/server/qa_block_publication_test.go` + - `TestP11P3_CreateLookup_PublicationCoherence` + - `TestP11P3_Failover_PublicationSwitches` + - `TestP11P3_HeartbeatReconstruction` + - `TestP11P3_NoNVMe_Fallback` +2. accepted chain includes: + - volume-server allocation publication fields + - master registry publication truth + - `CreateBlockVolume` publication response + - `LookupBlockVolume` publication response + - failover-driven publication switch + - `UpdateFullHeartbeat()`-driven reconstruction +3. fallback proof explicitly checks `iSCSI` remains present while `NVMe` fields remain empty when disabled + +Reuse note: + +1. `weed/server/master_grpc_server_block.go` + - `reuse as bounded adapter` +2. `weed/server/master_block_registry.go` + - `reuse as bounded truth carrier` +3. `weed/server/volume_grpc_block.go` + - `reuse as publication source only` +4. `weed/server/volume_server_block.go` + - `reference only` +5. `weed/storage/blockvol/csi/controller.go` + - `reference only` as downstream consumer + +Residual notes: + +1. `P3` closure does not imply runtime initiator correctness or transport-performance leadership +2. `P3` closure does not close restore/clone workflow behavior +3. `P4` remains necessary for bounded restore workflow closure + +--- + +### P4 Technical Pack + +Date: 2026-04-02 +Goal: deliver bounded snapshot restore workflow closure on the chosen path so product-visible restore behavior is backed by accepted snapshot/execution/control truth without silently inflating scope to clone or broad workflow productization + +#### Layer 1: Semantic Core + +##### Problem statement + +`Phase 09` accepted snapshot execution truth. +`Phase 10` accepted bounded control-plane closure. +`P1` accepted snapshot create/list/delete product rebinding. +`P2` accepted CSI controller/node lifecycle rebinding. +`P3` accepted front-end publication truth rebinding. + +What is still not accepted is whether the next bounded snapshot workflow step, `restore`, is coherently rebound onto the same chosen path. + +Current system reality shows: + +1. real backend restore execution exists: + - `blockvol.RestoreSnapshot` +2. accepted product-facing snapshot create/list/delete surfaces already exist +3. a visible product-facing `clone` workflow surface is not yet evident in the current implementation + +Therefore the first `P4` slice should be: + +1. snapshot `restore` + +and should explicitly not turn into: + +1. snapshot `clone` +2. broad snapshot workflow productization +3. restarting accepted snapshot/backend/control semantics from scratch + +##### System reality + +Current reality already includes these surfaces: + +1. product-facing snapshot surfaces: + - `weed/server/master_grpc_server_block.go` + - `weed/server/volume_server_block.go` +2. backend restore execution: + - `weed/storage/blockvol/blockvol.go` + - `RestoreSnapshot` +3. accepted snapshot product-path test package: + - `weed/server/qa_block_snapshot_product_test.go` + +The phase question is therefore: + +1. what the product-visible restore entry surface is +2. how restore maps onto accepted backend restore execution +3. how post-restore visible truth is checked +4. how `clone` remains explicitly deferred + +##### High-level algorithm + +Use this rebinding model: + +1. product-visible restore request enters through the chosen restore entry surface +2. master resolves the authoritative volume and selected snapshot truth +3. volume server executes restore against accepted `blockvol.RestoreSnapshot` +4. post-restore visible state is checked against the selected snapshot truth +5. unsupported or invalid restore conditions fail closed rather than implying broader workflow readiness + +##### Pseudo code + +```text +on restore request: + validate volume + snapshot selection + resolve authoritative chosen-path volume entry + call bounded restore adapter surface + execute accepted blockvol restore + verify post-restore visible truth against selected snapshot + +after restore: + writes after the selected snapshot are lost as designed + visible snapshot/workflow state is explicit + no clone/full-workflow support is implied +``` + +##### State / contract + +`P4` must make these truths explicit: + +1. restore is now the active bounded workflow target +2. backend `RestoreSnapshot` remains execution reality, not product-truth ownership +3. product-visible restore must make destructive semantics explicit +4. post-restore visible truth must match the selected snapshot truth +5. `clone` remains deferred unless a real product-facing clone surface is separately accepted into scope + +##### Reject shapes + +Reject before implementation if the slice: + +1. proves only backend-local restore mechanics with no product-visible restore behavior +2. hides destructive semantics +3. does not assert post-restore visible truth +4. quietly absorbs clone or broad workflow product readiness + +#### Layer 2: Execution Core + +##### Current gaps `P4` must close + +1. restore execution exists in backend code, but has not yet been accepted as a bounded product/workflow slice +2. `P1` closed create/list/delete only; restore remains downstream workflow work +3. the product-facing restore entry surface and post-restore visibility contract are not yet frozen in phase accounting +4. clone is still better treated as explicitly deferred because no real product-facing clone surface is evident in the current code + +##### Reuse / update instructions + +1. `weed/server/master_grpc_server_block.go` + - `update in place` if this file becomes the product-facing restore entry surface + - keep it as a bounded adapter/resolution layer + - V1 reuse role: + - `reuse as bounded adapter` + +2. `weed/server/volume_server_block.go` + - `update in place` if this file becomes the volume-server restore adapter + - keep it as a bounded execution adapter over accepted backend restore truth + - V1 reuse role: + - `reuse as bounded adapter` + +3. `weed/server/*snapshot*test.go` + - `update in place` + - prefer one focused restore workflow proof package rather than scattering restore checks across backend tests + +4. `weed/storage/blockvol/blockvol.go` + - `reference only` unless the product-visible restore proof exposes a real restore-execution mismatch + - backend restore remains accepted execution reality + - V1 reuse role: + - `reuse as execution reality only` + +5. `weed/storage/blockvol/*restore*` and snapshot backend tests + - `reference only` + - use them to understand existing restore guarantees, not as product closure evidence + +6. clone-related work + - `reference only` + - do not create clone acceptance text unless a real product-facing clone surface is added to scope + +7. copy guidance + - prefer `update in place` + - no parallel restore path + - every reused restore-facing surface must be named explicitly in delivery notes + +##### Validation focus + +Required proofs: + +1. restore success proof + - product-visible restore succeeds on the chosen path + - post-restore visible truth matches the selected snapshot + +2. destructive-semantics proof + - data written after the selected snapshot is lost as designed + - this is stated explicitly in both tests and delivery wording + +3. fail-closed proof + - missing snapshot / missing volume / unsupported conditions do not imply false success + +4. post-restore coherence proof + - visible workflow state after restore is coherent with backend truth + +5. reuse-boundary proof + - delivery notes identify every restore-facing V1 surface and how it is bounded + +6. no-overclaim proof + - acceptance wording does not imply clone or broad snapshot workflow readiness + +Reject if: + +1. evidence remains backend-only +2. post-restore visible truth is not checked +3. destructive restore semantics are left implicit +4. clone or broader workflow claims appear without proof + +##### Suggested first cut + +1. freeze the bounded restore contract +2. identify the smallest product-visible restore chain through: + - product-facing restore entry + - volume-server restore adapter + - accepted `blockvol.RestoreSnapshot` +3. add or reshape tests so post-restore state is compared with selected snapshot truth +4. write explicit no-overclaim wording for clone deferral + +##### Assignment For `sw` + +1. Goal + - deliver bounded `P4` snapshot restore workflow closure on the chosen path + +2. Required outputs + - one bounded contract statement for restore + - one focused restore proof package + - one fail-closed package for missing snapshot/volume or unsupported conditions + - one explicit V1 reuse note listing: + - master-facing restore surfaces + - volume-server restore surfaces + - backend restore execution surfaces referenced only + - short reuse note: + - files updated in place + - files used as references only + - any copied code and why + +3. Hard rules + - do not reopen accepted `Phase 09`, `Phase 10`, `P1`, `P2`, or `P3` semantics casually + - do not absorb clone or broad workflow productization into the first restore slice + - keep all V1/restore reuse explicit and bounded + +##### Assignment For `tester` + +1. Goal + - validate that `P4` proves bounded product-visible restore closure rather than backend-only restore behavior + +2. Validate + - restore-visible truth coherence + - destructive restore semantics + - fail-closed behavior for missing snapshot/volume or unsupported conditions + - explicit V1 reuse boundaries in delivery notes + - no-overclaim around clone and broader workflow support + +3. Reject if + - evidence is backend-only + - post-restore visible truth is not checked + - destructive semantics are not explicit + - clone/broader workflow claims appear without proof + +#### Short judgment + +`P4` is acceptable when: + +1. product-visible restore is proven on the chosen path +2. post-restore visible truth matches selected snapshot truth +3. destructive restore semantics are explicit +4. all reused restore-facing V1 surfaces are named and bounded +5. clone remains explicitly deferred unless separately proven + +--- + +### P4 Completion Record + +Date: 2026-04-02 +Status: accepted + +Accepted contract: + +1. `P4` proves bounded snapshot restore workflow closure for: + - restore success + - destructive restore semantics + - post-restore visible truth + - fail-closed behavior for missing snapshot/volume +2. accepted proof routes restore through: + - master restore RPC + - real volume-server restore adapter + - accepted backend `blockvol.RestoreSnapshot` +3. `P4` does not claim: + - clone support + - broad snapshot workflow readiness + - reopening accepted `Phase 09` / `Phase 10` semantics + +Accepted evidence: + +1. `weed/server/qa_block_restore_test.go` + - `TestP11P4_RestoreSuccess` + - `TestP11P4_DestructiveSemantics` + - `TestP11P4_FailClosed_MissingSnapshot` + - `TestP11P4_FailClosed_MissingVolume` +2. accepted chain includes: + - `MasterServer.RestoreBlockSnapshot` + - real registry resolution + - `BlockService.RestoreBlockSnapshot` + - `blockvol.RestoreSnapshot` +3. restore success proof explicitly checks: + - `LBA 0 == 0x00` after restore + - `LBA 5 == 0x00` after restore +4. destructive-semantics proof explicitly checks: + - all snapshots removed after restore + +Reuse note: + +1. `weed/server/master_grpc_server_block.go` + - `reuse as bounded adapter` +2. `weed/server/volume_server_block.go` + - `reuse as bounded adapter` +3. `weed/storage/blockvol/blockvol.go` + - `reuse as execution reality only` +4. `clone`-related work + - `reference only` + - explicitly deferred + +Residual notes: + +1. `clone` remains outside accepted scope because no real product-facing clone surface is yet accepted into scope +2. restore closure completes the bounded workflow work needed inside `Phase 11` + +--- + +### Phase 11 Completion Record + +Date: 2026-04-02 +Status: accepted and closed + +Completion judgment: + +`Phase 11` is complete because it closed the selected product-surface rebinding work promised after accepted backend/control closure: + +1. `P1` closed bounded snapshot create/list/delete product rebinding +2. `P2` closed bounded `CSI` controller/node lifecycle rebinding +3. `P3` closed bounded `NVMe` / `iSCSI` publication/address truth rebinding +4. `P4` closed bounded snapshot restore workflow rebinding + +What `Phase 11` accomplished: + +1. product-facing surfaces are now rebound onto the accepted V2-backed chosen path in a bounded, no-overclaim way +2. accepted product closure was achieved without reopening accepted `Phase 09` execution or `Phase 10` control-plane truth +3. residual work has shifted from product rebinding to production hardening + +What remains explicitly outside `Phase 11` closure: + +1. `clone` +2. full runtime initiator proof +3. soak / restart / repeated disturbance hardening +4. operational diagnosis quality and production blockers accounting + +Next phase: + +1. start `Phase 12 P0` production-hardening planning diff --git a/sw-block/.private/phase/phase-11.md b/sw-block/.private/phase/phase-11.md new file mode 100644 index 000000000..578cecfd5 --- /dev/null +++ b/sw-block/.private/phase/phase-11.md @@ -0,0 +1,488 @@ +# Phase 11 + +Date: 2026-04-02 +Status: complete +Purpose: bind selected product-facing surfaces onto the accepted V2-backed chosen path without reopening accepted backend execution or control-plane closure + +## Why This Phase Exists + +`Phase 09` accepted production-grade execution closure on the chosen path. +`Phase 10` accepted bounded master-driven control-plane closure on that same path. + +What remains is no longer: + +1. whether the chosen backend path executes correctly +2. whether accepted control truth can reach the live volume-server path coherently + +It is now: + +1. whether selected product-facing surfaces can be rebound onto that accepted path without semantic drift +2. whether reuse of older V1-facing adapters reintroduces V1 recovery truth implicitly +3. whether the first product-facing surface can be proven in a bounded way before broader surface expansion + +## Phase Goal + +Move from accepted backend/control closure on one bounded chosen path to the first bounded product-surface rebinding proof. + +Execution note: + +1. treat `P0` as real planning work, not placeholder prose +2. use `phase-11-log.md` as the technical pack for: + - step breakdown + - hard indicators + - reject shapes + - assignment text for `sw` and `tester` + +## Scope + +### In scope + +1. one bounded first product-surface slice +2. explicit no-overclaim around what that first surface proves and does not prove +3. reuse of existing implementation only where V2 truth still owns placement, recovery, and correctness claims +4. focused integration tests and contract checks for the chosen first surface + +### Out of scope + +1. reopening accepted `Phase 09` execution semantics +2. reopening accepted `Phase 10` control-plane closure +3. broad multi-surface product completion in one slice +4. `RF>2`, new durability modes, or broad cluster hardening +5. full production readiness / soak / rollout gates + +## Phase 11 Items + +### P0: First Surface Selection + +Goal: + +- choose the first product-facing surface that gives real product completion movement without turning the phase into a multi-system rewrite + +Accepted decision: + +1. the first bounded slice is `snapshot product path` +2. `CSI` is deferred to a later `Phase 11` slice because it pulls controller/node lifecycle, staging/publish, and broader cluster contract surface +3. `NVMe` / `iSCSI` rebinding are also deferred because they are transport/front-end adapters whose useful proof should come after one simpler product surface is already closed + +Why this first: + +1. snapshot is closest to already accepted backend truth +2. it exercises a real product-facing contract without immediately absorbing node/attach orchestration +3. it keeps the first `Phase 11` slice bounded to metadata/visibility/restore-contract correctness rather than transport and lifecycle breadth + +Status: + +- accepted + +### P1: Snapshot Product-Path Rebinding + +Goal: + +- prove that the snapshot product path can be rebound onto the accepted V2-backed chosen path without semantic drift between snapshot-visible behavior and the accepted backend snapshot truth + +Execution steps: + +1. Step 1: contract freeze + - define exactly what the first slice claims: + - snapshot create + - snapshot list + - snapshot delete + - explicitly exclude clone/restore unless a later slice accepts them +2. Step 2: implementation binding + - bind product-visible snapshot operations onto the accepted backend snapshot path + - keep master/volume-server state and visible metadata coherent +3. Step 3: proof package + - prove create/list/delete on the chosen path + - prove fail-closed behavior for unsupported/invalid inputs + - prove no-overclaim around broader snapshot workflows + +Required scope: + +1. snapshot create/list/delete product-visible behavior on the chosen path +2. proof that snapshot metadata and visible snapshot set reflect the same accepted backend truth +3. proof that snapshot claims do not exceed the accepted V2 snapshot contract +4. explicit boundedness around restore/clone if they are not part of the first slice + +Must prove: + +1. snapshot creation on the product path maps to the accepted backend snapshot boundary rather than an implicit V1 truth +2. listing and deletion reflect the real volume-server/master state coherently +3. fail-closed behavior is preserved when snapshot prerequisites are missing or the volume is not eligible +4. the slice does not silently imply clone/restore/product workflow support that is not yet proven + +Reuse discipline: + +1. V1/master-facing snapshot RPC surface may be reused only as a product wrapper: + - `CreateBlockSnapshot` + - `DeleteBlockSnapshot` + - `ListBlockSnapshots` +2. V1/volume-server-facing snapshot surface may be reused only as the bounded execution adapter: + - `SnapshotBlockVol` + - `DeleteBlockSnapshot` + - `ListBlockSnapshots` +3. underlying `blockvol` snapshot implementation may be reused as execution reality, not as product truth ownership +4. every reused V1 surface must be called out explicitly in `phase-11-log.md` with one of: + - `update in place` + - `reference only` + - `reuse as bounded adapter` +5. no reused V1 surface may silently redefine snapshot semantics, placement truth, or product support claims + +Verification mechanism: + +1. focused integration tests for create/list/delete on the chosen path +2. contract checks that visible snapshot metadata matches the accepted backend snapshot truth +3. no-overclaim review on what user-visible snapshot behavior is actually supported after the slice + +Hard indicators: + +1. one accepted create proof: + - product-visible create succeeds on the chosen path + - created snapshot is observable through list/readback metadata +2. one accepted delete proof: + - deleted snapshot disappears from the visible snapshot set + - repeated delete is either idempotent-success or explicitly fail-closed as designed +3. one accepted list coherence proof: + - listed snapshot IDs/metadata match the real backend snapshot state +4. one accepted fail-closed proof: + - invalid volume / missing snapshot / unsupported preconditions do not imply false success +5. one accepted boundedness proof: + - docs/tests do not imply clone/restore/full snapshot workflow readiness unless separately proven +6. one accepted reuse-boundary proof: + - all V1 reuse surfaces touched by the slice are explicitly listed and their role is bounded + +Reject if: + +1. the slice proves only local helper behavior rather than product-visible snapshot behavior +2. visible snapshot metadata can drift from backend truth +3. the first slice quietly absorbs clone/restore or broader workflow work +4. the slice claims product readiness beyond create/list/delete on the chosen path +5. reuse of V1 surfaces is implicit or lets V1 semantics become the source of truth + +Status: + +- accepted + +Carry-forward from `P1`: + +1. bounded snapshot create/list/delete product rebinding is now accepted on the chosen path +2. `P1` does not claim restore/clone/full snapshot workflow readiness +3. `CSI` rebinding is now the next active `Phase 11` slice + +### Later candidate slices inside `Phase 11` + +1. `P2`: `CSI` rebinding after snapshot product-path closure +2. `P3`: `NVMe` / `iSCSI` front-end rebinding after one simpler product-visible surface is already accepted +3. `P4`: broader snapshot workflow closure (`restore` / `clone`) or other residual product workflow work only after earlier slices are bounded and proven + +### P2: CSI Rebinding + +Goal: + +- bind the accepted V2-backed chosen path to the `CSI` controller/node product surface without reintroducing V1 recovery truth + +Execution steps: + +1. Step 1: contract freeze + - define the first bounded `CSI` surface claims: + - `CreateVolume` + - `DeleteVolume` + - `ControllerPublishVolume` + - `NodeStageVolume` + - `NodePublishVolume` + - `NodeUnpublishVolume` + - `NodeUnstageVolume` + - explicitly exclude CSI snapshot, expand, and NVMe-specific transport work unless a later slice accepts them +2. Step 2: backend rebinding + - bind CSI controller operations to the accepted master-backed chosen-path volume surface + - bind CSI node operations to the accepted chosen-path access contract for remote attach/stage/publish +3. Step 3: proof package + - prove bounded create/publish/stage/use/delete lifecycle on the chosen path + - prove fail-closed behavior for unsupported or invalid cases + - prove no-overclaim around broader CSI/product workflow breadth + +Required scope: + +1. bounded CSI controller/node lifecycle on the chosen path +2. explicit separation between accepted backend/control truth and CSI orchestration wrappers +3. remote target publication/staging behavior for the chosen path +4. no-overclaim around snapshots via CSI, expand, NVMe transport preference, multi-node topology breadth, or broad K8s readiness + +Must prove: + +1. CSI controller create/delete/publish map to the accepted master-backed chosen-path truth rather than a local V1 shortcut +2. CSI node stage/publish/unstage/unpublish consume the same chosen-path access truth without redefining recovery semantics +3. product-visible CSI lifecycle behavior is coherent across controller and node surfaces +4. fail-closed behavior is preserved when required publish/volume context or target information is missing + +Reuse discipline: + +1. V1/CSI-facing controller and node RPC surfaces may be reused only as bounded product adapters: + - `controller.go` + - `node.go` + - `server.go` +2. `volume_backend.go` may be reused only as the bounded bridge between CSI and accepted master/local surfaces +3. `volume_manager.go` may be reused only as bounded local execution reality where the slice explicitly proves that local manager behavior does not become semantic owner +4. accepted master block RPC surfaces may be reused only as bounded control/product adapters underneath the CSI backend bridge: + - `CreateBlockVolume` + - `DeleteBlockVolume` + - `LookupBlockVolume` +5. every reused V1 surface must be called out explicitly in `phase-11-log.md` with one of: + - `update in place` + - `reference only` + - `reuse as bounded adapter` + - `reuse as bounded bridge` + - `reuse as execution reality only` +6. no reused V1 surface may silently redefine lifecycle semantics, placement truth, or product support claims + +Verification mechanism: + +1. focused CSI controller/node integration tests on the chosen path +2. contract checks that controller-visible and node-visible truth match accepted backend/control truth +3. no-overclaim review on what CSI behavior is actually supported after the slice + +Hard indicators: + +1. one accepted controller create/publish proof: + - CSI create returns coherent volume/publish context on the chosen path +2. one accepted node stage/publish proof: + - node consumes the published target info and stages/publishes coherently on the chosen path +3. one accepted unpublish/unstage/delete proof: + - teardown/deletion complete without leaving false-visible ownership +4. one accepted fail-closed proof: + - missing or partial transport/context information does not imply false success +5. one accepted reuse-boundary proof: + - all CSI/V1 reuse surfaces touched by the slice are explicitly listed and bounded +6. one accepted boundedness proof: + - docs/tests do not imply CSI snapshot, expand, NVMe transport preference, or broad K8s/product readiness unless separately proven + +Reject if: + +1. the slice proves only CSI wrapper-local behavior without chosen-path backend/control coherence +2. controller truth and node truth can drift from accepted master-backed volume truth +3. the first CSI slice quietly absorbs snapshot, expand, NVMe, or broad multi-node/K8s readiness work +4. reuse of V1 surfaces is implicit or lets V1 semantics become the source of truth + +Status: + +- accepted + +Carry-forward from `P2`: + +1. bounded CSI controller/node lifecycle rebinding is now accepted on the chosen path +2. accepted proof uses the real master-backed create/lookup/delete path plus `mgr=nil` node consumption of published target truth +3. `P2` does not claim CSI snapshot, CSI expand, NVMe preference/failover closure, or broad Kubernetes readiness + +### P3: NVMe / iSCSI Front-End Rebinding + +Goal: + +- bind transport/front-end publication surfaces onto the accepted V2-backed chosen path so the product-visible access path matches accepted backend/control truth + +Execution steps: + +1. Step 1: contract freeze + - define the first bounded front-end publication claims: + - create returns coherent front-end publication data + - lookup returns coherent front-end publication data + - heartbeat refresh preserves and updates publication truth + - failover switches publication truth to the new primary coherently + - explicitly exclude broad transport-performance claims, real initiator benchmarking, and broad cluster rollout readiness +2. Step 2: publication rebinding + - bind `iSCSI` and `NVMe` publication fields onto the accepted master-backed chosen-path truth + - keep registry-visible, lookup-visible, and CSI-visible publication truth coherent +3. Step 3: proof package + - prove bounded create/lookup/failover/restart publication truth on the chosen path + - prove fallback behavior is explicit where `NVMe` is absent + - prove no-overclaim around full transport runtime/performance closure + +Required scope: + +1. publication/address/naming truth for front-end adapters on the chosen path +2. bounded integration proof that master-visible and product-visible access metadata stay coherent +3. `NVMe` primary publication and `iSCSI` fallback publication where supported by the chosen path +4. explicit boundedness around real initiator behavior, transport performance, and broad cluster hardening + +Must prove: + +1. create/lookup publication fields map to accepted chosen-path truth rather than ad hoc wrapper-local construction +2. heartbeat refresh and failover preserve or update front-end publication truth coherently +3. `NVMe` and `iSCSI` publication fields do not drift between registry, lookup, and product-facing responses +4. mixed-capability or fallback behavior is explicit rather than silently overclaimed + +Reuse discipline: + +1. master-facing product/control publication surfaces may be reused only as bounded adapters: + - `CreateBlockVolume` + - `LookupBlockVolume` +2. registry publication fields may be reused only as bounded truth carriers, not independent semantic owners: + - `ISCSIAddr` + - `IQN` + - `NvmeAddr` + - `NQN` +3. volume-server allocation/publication surfaces may be reused only as bounded front-end publication sources: + - `AllocateBlockVolume` + - block heartbeat publication of `NvmeAddr` / `NQN` +4. existing `CSI` controller consumption of publication fields may be reused only as a bounded downstream consumer, not as the source of truth for `P3` +5. every reused V1 surface must be called out explicitly in `phase-11-log.md` with one of: + - `update in place` + - `reference only` + - `reuse as bounded adapter` + - `reuse as bounded truth carrier` + - `reuse as publication source only` +6. no reused V1 surface may silently redefine publication truth, failover truth, or supported transport claims + +Verification mechanism: + +1. focused integration tests for create/lookup publication truth on the chosen path +2. contract checks that registry-visible, lookup-visible, and consumer-visible publication fields match +3. failover/restart checks that front-end publication truth is reconstructed or updated coherently +4. no-overclaim review on what transport/front-end behavior is actually supported after the slice + +Hard indicators: + +1. one accepted create/lookup publication proof: + - create returns coherent front-end publication fields + - lookup returns the same chosen-path publication truth +2. one accepted failover publication proof: + - front-end publication fields move to the new primary coherently after failover +3. one accepted restart/heartbeat reconstruction proof: + - publication fields can be reconstructed or refreshed from accepted heartbeat truth +4. one accepted fallback proof: + - `iSCSI` fallback or mixed-capability behavior is explicit and coherent when `NVMe` is absent +5. one accepted reuse-boundary proof: + - all front-end publication surfaces touched by the slice are explicitly listed and bounded +6. one accepted boundedness proof: + - docs/tests do not imply real transport runtime, performance leadership, or broad production readiness unless separately proven + +Reject if: + +1. the slice proves only field plumbing without chosen-path publication coherence +2. publication truth can drift across create, lookup, heartbeat, or failover +3. the slice quietly absorbs full transport runtime or performance claims +4. reuse of V1/publication surfaces is implicit or lets wrappers become the truth owner + +Status: + +- accepted + +Carry-forward from `P3`: + +1. bounded front-end publication/address truth rebinding is now accepted on the chosen path +2. accepted proof closes create/lookup coherence, failover publication switch, heartbeat reconstruction, and no-`NVMe` fallback +3. `P3` does not claim full initiator/runtime transport proof, performance claims, or broad production readiness + +### P4: Broader Product Workflow Closure + +Goal: + +- close the remaining bounded snapshot product workflow gaps downstream of accepted `P1` / `P2` / `P3` without reopening earlier accepted truth + +Execution steps: + +1. Step 1: contract freeze + - define the first bounded `P4` workflow claim as snapshot `restore` + - explicitly defer `clone` unless and until a real product-facing clone surface exists and is accepted into scope +2. Step 2: workflow rebinding + - bind product-visible restore behavior onto the accepted snapshot and chosen-path execution truth + - keep restore-visible state coherent across master-visible and volume-server/backend-visible truth +3. Step 3: proof package + - prove bounded restore success, destructive semantics, and post-restore visible truth + - prove fail-closed behavior for missing snapshot or unsupported conditions + - prove no-overclaim around clone or broader workflow productization + +Required scope: + +1. bounded snapshot restore product workflow on the chosen path +2. explicit proof that restore uses accepted snapshot/backend truth rather than reopening new execution ownership +3. explicit post-restore visible truth checks +4. explicit boundedness around `clone` and any broader workflow work + +Must prove: + +1. product-visible restore maps to accepted backend restore execution truth on the chosen path +2. restore-visible outcome matches the selected snapshot truth after the operation completes +3. destructive restore semantics are explicit rather than hidden +4. fail-closed behavior is preserved for missing snapshot, missing volume, or unsupported preconditions + +Reuse discipline: + +1. accepted master-facing snapshot RPC surfaces may be reused only as bounded product adapters for restore if a restore entry surface exists +2. accepted volume-server-facing snapshot/restore surfaces may be reused only as bounded execution adapters +3. underlying `blockvol.RestoreSnapshot` may be reused only as execution reality, not as product-truth ownership +4. `clone` must stay explicitly deferred unless a real product-facing surface is brought into scope and written into `phase-11-log.md` +5. every reused V1 surface must be called out explicitly in `phase-11-log.md` with one of: + - `update in place` + - `reference only` + - `reuse as bounded adapter` + - `reuse as execution reality only` +6. no reused V1 surface may silently redefine restore semantics, workflow readiness, or clone claims + +Verification mechanism: + +1. focused restore integration tests on the chosen path +2. contract checks that post-restore visible truth matches selected snapshot truth +3. fail-closed checks for invalid or unsupported restore conditions +4. no-overclaim review on what restore/clone workflow behavior is actually supported after the slice + +Hard indicators: + +1. one accepted restore success proof: + - product-visible restore succeeds on the chosen path + - visible post-restore state matches the selected snapshot truth +2. one accepted destructive-semantics proof: + - writes after the snapshot are lost as designed and this is explicitly verified +3. one accepted fail-closed proof: + - missing snapshot / missing volume / unsupported conditions do not imply false success +4. one accepted post-restore coherence proof: + - list/readback/visible workflow state are coherent after restore +5. one accepted reuse-boundary proof: + - all restore-facing V1 surfaces touched by the slice are explicitly listed and bounded +6. one accepted boundedness proof: + - docs/tests do not imply clone or broad snapshot workflow readiness unless separately proven + +Reject if: + +1. the slice proves only backend-local restore mechanics without product-visible restore behavior +2. post-restore visible truth is not asserted +3. destructive semantics are left implicit +4. the slice quietly absorbs `clone` or broader workflow readiness work +5. reuse of V1 surfaces is implicit or lets V1 semantics become the source of truth + +Status: + +- accepted + +Carry-forward from `P4`: + +1. bounded snapshot restore workflow closure is now accepted on the chosen path +2. accepted proof closes restore success, destructive semantics, post-restore visible truth, and fail-closed behavior +3. `clone` remains explicitly deferred because no real product-facing clone surface is yet accepted into scope + +## Phase 11 Completion Judgment + +`Phase 11` is complete because: + +1. `P1` accepted bounded snapshot create/list/delete product rebinding +2. `P2` accepted bounded `CSI` controller/node lifecycle rebinding +3. `P3` accepted bounded `NVMe` / `iSCSI` publication/address truth rebinding +4. `P4` accepted bounded snapshot restore workflow closure +5. the chosen-path product surface rebinding goal is now closed without reopening accepted `Phase 09` / `Phase 10` semantics +6. remaining work is no longer product-surface rebinding inside `Phase 11`, but production hardening in `Phase 12` + +## Assignment For `sw` + +Current next tasks: + +1. `Phase 11` is closed +2. move next to `Phase 12 P0` production-hardening planning +3. do not reopen accepted `P1` / `P2` / `P3` / `P4` semantics casually during hardening planning +4. keep `clone` deferred unless separately re-scoped in a future phase + +## Assignment For `tester` + +Current next tasks: + +1. `Phase 11` is closed +2. validate `Phase 12 P0` as real planning work rather than placeholder prose +3. keep no-overclaim active around accepted `P1` / `P2` / `P3` / `P4` closure +4. treat `clone` or any other future workflow work as separate re-scoping work, not implicit `Phase 11` residue diff --git a/weed/server/qa_block_csi_lifecycle_test.go b/weed/server/qa_block_csi_lifecycle_test.go new file mode 100644 index 000000000..56882a438 --- /dev/null +++ b/weed/server/qa_block_csi_lifecycle_test.go @@ -0,0 +1,340 @@ +package weed_server + +import ( + "context" + "fmt" + "log" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" + bsi "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/csi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// ============================================================ +// Phase 11 P2: CSI lifecycle rebinding with REAL master backend +// +// These tests wire the CSI controller through a VolumeBackend that +// calls REAL MasterServer.CreateBlockVolume / LookupBlockVolume / +// DeleteBlockVolume — the same master logic that MasterVolumeClient +// calls over gRPC. The node runs with mgr=nil so publish_context +// consumption is the only viable staging path. +// +// Chain: +// CSI controller → masterServerBackend → MasterServer.CreateBlockVolume (real) +// → blockRegistry (real) → blockVSAllocate (real blockvol creation) +// → VolumeInfo with master-produced iscsiAddr/iqn +// → ControllerPublish → publish_context +// → CSI node (mgr=nil) → consumes publish_context → isLocal=false +// ============================================================ + +// masterServerBackend implements csi.VolumeBackend by calling real +// MasterServer methods. This is the SAME logic that MasterVolumeClient +// executes over gRPC, but called in-process for testability. +type masterServerBackend struct { + ms *MasterServer +} + +func (b *masterServerBackend) CreateVolume(ctx context.Context, name string, sizeBytes uint64) (*bsi.VolumeInfo, error) { + resp, err := b.ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: name, + SizeBytes: sizeBytes, + }) + if err != nil { + return nil, err + } + return &bsi.VolumeInfo{ + VolumeID: resp.VolumeId, + ISCSIAddr: resp.IscsiAddr, + IQN: resp.Iqn, + CapacityBytes: resp.CapacityBytes, + NvmeAddr: resp.NvmeAddr, + NQN: resp.Nqn, + }, nil +} + +func (b *masterServerBackend) DeleteVolume(ctx context.Context, name string) error { + _, err := b.ms.DeleteBlockVolume(ctx, &master_pb.DeleteBlockVolumeRequest{Name: name}) + return err +} + +func (b *masterServerBackend) LookupVolume(ctx context.Context, name string) (*bsi.VolumeInfo, error) { + resp, err := b.ms.LookupBlockVolume(ctx, &master_pb.LookupBlockVolumeRequest{Name: name}) + if err != nil { + return nil, err + } + return &bsi.VolumeInfo{ + VolumeID: name, + ISCSIAddr: resp.IscsiAddr, + IQN: resp.Iqn, + CapacityBytes: resp.CapacityBytes, + NvmeAddr: resp.NvmeAddr, + NQN: resp.Nqn, + }, nil +} + +func (b *masterServerBackend) CreateSnapshot(ctx context.Context, volumeID string, snapID uint32) (*bsi.SnapshotInfo, error) { + return nil, fmt.Errorf("not in P2 scope") +} +func (b *masterServerBackend) DeleteSnapshot(ctx context.Context, volumeID string, snapID uint32) error { + return fmt.Errorf("not in P2 scope") +} +func (b *masterServerBackend) ListSnapshots(ctx context.Context, volumeID string) ([]*bsi.SnapshotInfo, error) { + return nil, fmt.Errorf("not in P2 scope") +} +func (b *masterServerBackend) ExpandVolume(ctx context.Context, volumeID string, newSizeBytes uint64) (uint64, error) { + return 0, fmt.Errorf("not in P2 scope") +} + +func newCSILifecycleSetup(t *testing.T) (*bsi.ExportedControllerServer, *bsi.ExportedNodeServer, *MasterServer, string) { + t.Helper() + dir := t.TempDir() + + ms := &MasterServer{ + blockRegistry: NewBlockVolumeRegistry(), + blockAssignmentQueue: NewBlockAssignmentQueue(), + blockFailover: newBlockFailoverState(), + } + ms.blockRegistry.MarkBlockCapable("vs1:9333") + ms.blockRegistry.MarkBlockCapable("vs2:9333") + + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { + sanitized := strings.ReplaceAll(server, ":", "_") + serverDir := filepath.Join(dir, sanitized) + os.MkdirAll(serverDir, 0755) + volPath := filepath.Join(serverDir, fmt.Sprintf("%s.blk", name)) + vol, err := blockvol.CreateBlockVol(volPath, blockvol.CreateOptions{ + VolumeSize: 1 * 1024 * 1024, + BlockSize: 4096, + WALSize: 256 * 1024, + }) + if err != nil { + return nil, err + } + vol.Close() + // Derive a valid ip:port iSCSI address from the server identity. + // Server identity is "vs1:9333" — extract the host part for the iSCSI portal. + iscsiHost := server + if idx := strings.LastIndex(server, ":"); idx >= 0 { + iscsiHost = server[:idx] + } + return &blockAllocResult{ + Path: volPath, + IQN: fmt.Sprintf("iqn.2024.master:%s", name), + ISCSIAddr: iscsiHost + ":3260", + }, nil + } + ms.blockVSDelete = func(ctx context.Context, server string, name string) error { + // Real delete: remove the .blk file from the server's directory. + sanitized := strings.ReplaceAll(server, ":", "_") + volPath := filepath.Join(dir, sanitized, fmt.Sprintf("%s.blk", name)) + return os.Remove(volPath) + } + + backend := &masterServerBackend{ms: ms} + + ctrl := bsi.NewExportedControllerServer(backend) + node := bsi.NewExportedNodeServer("test-node-csi", nil, log.New(os.Stderr, "csi-node: ", log.LstdFlags)) + + return ctrl, node, ms, dir +} + +// --- 1. Master-produced publish_context --- + +func TestP11P2_CSI_MasterProduced_PublishContext(t *testing.T) { + ctrl, _, ms, _ := newCSILifecycleSetup(t) + ctx := context.Background() + + createResp, err := ctrl.CreateVolume(ctx, &csi.CreateVolumeRequest{ + Name: "csi-vol-1", + CapacityRange: &csi.CapacityRange{RequiredBytes: 1 << 20}, + VolumeCapabilities: []*csi.VolumeCapability{singleNodeWriter()}, + }) + if err != nil { + t.Fatalf("CreateVolume: %v", err) + } + + pubResp, err := ctrl.ControllerPublishVolume(ctx, &csi.ControllerPublishVolumeRequest{ + VolumeId: createResp.Volume.VolumeId, NodeId: "test-node-csi", + VolumeCapability: singleNodeWriter(), + }) + if err != nil { + t.Fatalf("Publish: %v", err) + } + + // Verify: publish_context carries master-produced target info. + entry, _ := ms.blockRegistry.Lookup("csi-vol-1") + // Derive expected iSCSI portal: host from VolumeServer + :3260. + vsHost := entry.VolumeServer + if idx := strings.LastIndex(vsHost, ":"); idx >= 0 { + vsHost = vsHost[:idx] + } + expectedISCSI := vsHost + ":3260" + expectedIQN := fmt.Sprintf("iqn.2024.master:%s", "csi-vol-1") + if pubResp.PublishContext["iscsiAddr"] != expectedISCSI { + t.Fatalf("iscsiAddr=%q, want %q (master-produced)", pubResp.PublishContext["iscsiAddr"], expectedISCSI) + } + if pubResp.PublishContext["iqn"] != expectedIQN { + t.Fatalf("iqn=%q, want %q (master-produced)", pubResp.PublishContext["iqn"], expectedIQN) + } + + t.Logf("P11P2 master-produced: iscsiAddr=%s iqn=%s (both asserted)", + pubResp.PublishContext["iscsiAddr"], pubResp.PublishContext["iqn"]) +} + +// --- 2. Full lifecycle: master backend + mgr=nil node --- + +func TestP11P2_CSI_FullLifecycle(t *testing.T) { + ctrl, node, _, testDir := newCSILifecycleSetup(t) + ctx := context.Background() + + // Create. + createResp, err := ctrl.CreateVolume(ctx, &csi.CreateVolumeRequest{ + Name: "csi-vol-2", CapacityRange: &csi.CapacityRange{RequiredBytes: 1 << 20}, + VolumeCapabilities: []*csi.VolumeCapability{singleNodeWriter()}, + }) + if err != nil { + t.Fatalf("Create: %v", err) + } + volID := createResp.Volume.VolumeId + + // Publish. + pubResp, err := ctrl.ControllerPublishVolume(ctx, &csi.ControllerPublishVolumeRequest{ + VolumeId: volID, NodeId: "test-node-csi", VolumeCapability: singleNodeWriter(), + }) + if err != nil { + t.Fatalf("Publish: %v", err) + } + + // Stage (mgr=nil → publish_context only path). + stagingDir := filepath.Join(t.TempDir(), "staging") + os.MkdirAll(stagingDir, 0755) + _, err = node.NodeStageVolume(ctx, &csi.NodeStageVolumeRequest{ + VolumeId: volID, StagingTargetPath: stagingDir, + PublishContext: pubResp.PublishContext, VolumeCapability: singleNodeWriter(), + }) + if err != nil { + t.Fatalf("Stage: %v", err) + } + + // Assert: isLocal=false and staged addr matches publish_context. + staged := node.GetStagedInfo(volID) + if staged == nil { + t.Fatal("not in staged map") + } + if staged.IsLocal { + t.Fatal("isLocal must be false (mgr=nil)") + } + if staged.ISCSIAddr != pubResp.PublishContext["iscsiAddr"] { + t.Fatalf("staged addr=%q != publish_context=%q", staged.ISCSIAddr, pubResp.PublishContext["iscsiAddr"]) + } + + // NodePublish. + targetDir := filepath.Join(t.TempDir(), "target") + os.MkdirAll(targetDir, 0755) + _, err = node.NodePublishVolume(ctx, &csi.NodePublishVolumeRequest{ + VolumeId: volID, StagingTargetPath: stagingDir, TargetPath: targetDir, + VolumeCapability: singleNodeWriter(), + }) + if err != nil { + t.Fatalf("NodePublish: %v", err) + } + + // Unpublish. + if _, err := node.NodeUnpublishVolume(ctx, &csi.NodeUnpublishVolumeRequest{VolumeId: volID, TargetPath: targetDir}); err != nil { + t.Fatalf("NodeUnpublish: %v", err) + } + // Unstage. + if _, err := node.NodeUnstageVolume(ctx, &csi.NodeUnstageVolumeRequest{VolumeId: volID, StagingTargetPath: stagingDir}); err != nil { + t.Fatalf("NodeUnstage: %v", err) + } + // Delete (real VS-side file removal via blockVSDelete). + if _, err := ctrl.DeleteVolume(ctx, &csi.DeleteVolumeRequest{VolumeId: volID}); err != nil { + t.Fatalf("DeleteVolume: %v", err) + } + + // Verify: .blk file is gone on disk (real file removal). + volFilePath := filepath.Join(testDir, "vs1_9333", "csi-vol-2.blk") + if _, err := os.Stat(volFilePath); !os.IsNotExist(err) { + t.Fatalf(".blk file should be gone after delete: %s (err=%v)", volFilePath, err) + } + + // Verify: publish fails (controller-level — volume unregistered). + _, err = ctrl.ControllerPublishVolume(ctx, &csi.ControllerPublishVolumeRequest{ + VolumeId: volID, NodeId: "test-node-csi", VolumeCapability: singleNodeWriter(), + }) + if err == nil { + t.Fatal("publish after delete should fail") + } + + t.Log("P11P2 lifecycle: master-created → publish → stage(isLocal=false) → pub → unpub → unstage → delete → gone") +} + +// --- 3. Fail-closed: missing context + mgr=nil --- + +func TestP11P2_CSI_FailClosed_NoContext(t *testing.T) { + ctrl, node, _, _ := newCSILifecycleSetup(t) + ctx := context.Background() + + createResp, err := ctrl.CreateVolume(ctx, &csi.CreateVolumeRequest{ + Name: "csi-vol-fc", CapacityRange: &csi.CapacityRange{RequiredBytes: 1 << 20}, + VolumeCapabilities: []*csi.VolumeCapability{singleNodeWriter()}, + }) + if err != nil { + t.Fatalf("setup CreateVolume: %v", err) + } + + stagingDir := filepath.Join(t.TempDir(), "staging") + os.MkdirAll(stagingDir, 0755) + + _, stageErr := node.NodeStageVolume(ctx, &csi.NodeStageVolumeRequest{ + VolumeId: createResp.Volume.VolumeId, StagingTargetPath: stagingDir, + VolumeCapability: singleNodeWriter(), + }) + if stageErr == nil { + t.Fatal("stage without context + mgr=nil should fail") + } + st, ok := status.FromError(stageErr) + if !ok || st.Code() != codes.FailedPrecondition { + t.Fatalf("expected FailedPrecondition, got %v", stageErr) + } + t.Logf("P11P2 fail-closed: %s — %s", st.Code(), st.Message()) +} + +// --- 4. Fail-closed: missing volume --- + +func TestP11P2_CSI_FailClosed_MissingVolume(t *testing.T) { + ctrl, _, _, _ := newCSILifecycleSetup(t) + ctx := context.Background() + + _, err := ctrl.ControllerPublishVolume(ctx, &csi.ControllerPublishVolumeRequest{ + VolumeId: "nonexistent", NodeId: "n", VolumeCapability: singleNodeWriter(), + }) + if err == nil { + t.Fatal("should fail") + } + st, _ := status.FromError(err) + if st.Code() != codes.NotFound { + t.Fatalf("expected NotFound, got %s", st.Code()) + } + + // Delete nonexistent: idempotent. + _, delErr := ctrl.DeleteVolume(ctx, &csi.DeleteVolumeRequest{VolumeId: "nonexistent"}) + if delErr != nil { + t.Fatalf("delete nonexistent should be idempotent: %v", delErr) + } + + t.Log("P11P2 fail-closed: NotFound on publish, idempotent on delete") +} + +func singleNodeWriter() *csi.VolumeCapability { + return &csi.VolumeCapability{ + AccessMode: &csi.VolumeCapability_AccessMode{Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER}, + } +} diff --git a/weed/server/qa_block_publication_test.go b/weed/server/qa_block_publication_test.go new file mode 100644 index 000000000..c65cebdcb --- /dev/null +++ b/weed/server/qa_block_publication_test.go @@ -0,0 +1,303 @@ +package weed_server + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" +) + +// ============================================================ +// Phase 11 P3: Front-end publication rebinding +// +// Proofs: +// 1. Create/Lookup coherence: iSCSI + NVMe fields match +// 2. Failover: publication truth switches to new primary +// 3. Heartbeat reconstruction: fields survive restart +// 4. No-NVMe fallback: explicit and coherent +// +// V1 reuse roles: +// master_grpc_server_block.go: reuse as bounded adapter +// master_block_registry.go: reuse as bounded truth carrier +// volume_server_block.go: reuse as publication source only +// ============================================================ + +func newPublicationMaster(t *testing.T, nvmeEnabled bool) *MasterServer { + t.Helper() + dir := t.TempDir() + + ms := &MasterServer{ + blockRegistry: NewBlockVolumeRegistry(), + blockAssignmentQueue: NewBlockAssignmentQueue(), + blockFailover: newBlockFailoverState(), + } + ms.blockRegistry.MarkBlockCapable("vs1:9333") + ms.blockRegistry.MarkBlockCapable("vs2:9333") + + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { + sanitized := strings.ReplaceAll(server, ":", "_") + serverDir := filepath.Join(dir, sanitized) + os.MkdirAll(serverDir, 0755) + volPath := filepath.Join(serverDir, fmt.Sprintf("%s.blk", name)) + vol, err := blockvol.CreateBlockVol(volPath, blockvol.CreateOptions{ + VolumeSize: 1 * 1024 * 1024, + BlockSize: 4096, + WALSize: 256 * 1024, + }) + if err != nil { + return nil, err + } + vol.Close() + + host := server + if idx := strings.LastIndex(server, ":"); idx >= 0 { + host = server[:idx] + } + + result := &blockAllocResult{ + Path: volPath, + IQN: fmt.Sprintf("iqn.2024.test:%s", name), + ISCSIAddr: host + ":3260", + ReplicaDataAddr: server + ":14260", + ReplicaCtrlAddr: server + ":14261", + RebuildListenAddr: server + ":15000", + } + if nvmeEnabled { + result.NvmeAddr = host + ":4420" + result.NQN = fmt.Sprintf("nqn.2024-01.com.seaweedfs:vol.%s", name) + } + return result, nil + } + ms.blockVSDelete = func(ctx context.Context, server string, name string) error { return nil } + + return ms +} + +// --- 1. Create/Lookup coherence --- + +func TestP11P3_CreateLookup_PublicationCoherence(t *testing.T) { + ms := newPublicationMaster(t, true) + ctx := context.Background() + + createResp, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "pub-vol-1", SizeBytes: 1 << 30, + }) + if err != nil { + t.Fatalf("Create: %v", err) + } + + // Create response has iSCSI + NVMe fields. + if createResp.IscsiAddr == "" || createResp.Iqn == "" { + t.Fatal("create missing iSCSI fields") + } + if createResp.NvmeAddr == "" || createResp.Nqn == "" { + t.Fatal("create missing NVMe fields") + } + + // Lookup must return the SAME publication fields. + lookupResp, err := ms.LookupBlockVolume(ctx, &master_pb.LookupBlockVolumeRequest{Name: "pub-vol-1"}) + if err != nil { + t.Fatalf("Lookup: %v", err) + } + + if lookupResp.IscsiAddr != createResp.IscsiAddr { + t.Fatalf("iSCSI addr mismatch: create=%q lookup=%q", createResp.IscsiAddr, lookupResp.IscsiAddr) + } + if lookupResp.Iqn != createResp.Iqn { + t.Fatalf("IQN mismatch: create=%q lookup=%q", createResp.Iqn, lookupResp.Iqn) + } + if lookupResp.NvmeAddr != createResp.NvmeAddr { + t.Fatalf("NVMe addr mismatch: create=%q lookup=%q", createResp.NvmeAddr, lookupResp.NvmeAddr) + } + if lookupResp.Nqn != createResp.Nqn { + t.Fatalf("NQN mismatch: create=%q lookup=%q", createResp.Nqn, lookupResp.Nqn) + } + + // Verify: registry truth matches. + entry, _ := ms.blockRegistry.Lookup("pub-vol-1") + if entry.ISCSIAddr != createResp.IscsiAddr { + t.Fatalf("registry iSCSI=%q != create=%q", entry.ISCSIAddr, createResp.IscsiAddr) + } + if entry.NvmeAddr != createResp.NvmeAddr { + t.Fatalf("registry NVMe=%q != create=%q", entry.NvmeAddr, createResp.NvmeAddr) + } + + t.Logf("P11P3 coherence: create=lookup=registry — iSCSI=%s NVMe=%s IQN=%s NQN=%s", + createResp.IscsiAddr, createResp.NvmeAddr, createResp.Iqn, createResp.Nqn) +} + +// --- 2. Failover: publication truth switches --- + +func TestP11P3_Failover_PublicationSwitches(t *testing.T) { + ms := newPublicationMaster(t, true) + ctx := context.Background() + + createResp, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "pub-vol-2", SizeBytes: 1 << 30, + }) + if err != nil { + t.Fatalf("Create: %v", err) + } + + oldPrimary := createResp.VolumeServer + oldISCSI := createResp.IscsiAddr + oldNVMe := createResp.NvmeAddr + + t.Logf("before failover: primary=%s iSCSI=%s NVMe=%s", oldPrimary, oldISCSI, oldNVMe) + + // Expire lease and failover. + ms.blockRegistry.UpdateEntry("pub-vol-2", func(e *BlockVolumeEntry) { + e.LastLeaseGrant = time.Now().Add(-1 * time.Minute) + }) + ms.failoverBlockVolumes(oldPrimary) + + // Lookup after failover: publication fields should reflect NEW primary. + lookupResp, err := ms.LookupBlockVolume(ctx, &master_pb.LookupBlockVolumeRequest{Name: "pub-vol-2"}) + if err != nil { + t.Fatalf("Lookup after failover: %v", err) + } + + entry, _ := ms.blockRegistry.Lookup("pub-vol-2") + if entry.VolumeServer == oldPrimary { + t.Fatalf("primary should have changed, still %s", oldPrimary) + } + + // New primary's publication fields should differ from old primary's. + if lookupResp.IscsiAddr == oldISCSI { + t.Fatalf("iSCSI addr should change after failover: still %q", lookupResp.IscsiAddr) + } + if lookupResp.NvmeAddr == oldNVMe { + t.Fatalf("NVMe addr should change after failover: still %q", lookupResp.NvmeAddr) + } + + // Registry and lookup must still agree. + if lookupResp.IscsiAddr != entry.ISCSIAddr { + t.Fatalf("post-failover: lookup iSCSI=%q != registry=%q", lookupResp.IscsiAddr, entry.ISCSIAddr) + } + if lookupResp.NvmeAddr != entry.NvmeAddr { + t.Fatalf("post-failover: lookup NVMe=%q != registry=%q", lookupResp.NvmeAddr, entry.NvmeAddr) + } + + t.Logf("P11P3 failover: old=%s→new=%s iSCSI=%s NVMe=%s (switched)", + oldPrimary, entry.VolumeServer, lookupResp.IscsiAddr, lookupResp.NvmeAddr) +} + +// --- 3. Heartbeat reconstruction --- + +func TestP11P3_HeartbeatReconstruction(t *testing.T) { + ms := newPublicationMaster(t, true) + ctx := context.Background() + + createResp, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "pub-vol-3", SizeBytes: 1 << 30, + }) + if err != nil { + t.Fatalf("Create: %v", err) + } + + entry, _ := ms.blockRegistry.Lookup("pub-vol-3") + originalNVMe := entry.NvmeAddr + originalNQN := entry.NQN + primaryServer := entry.VolumeServer + + // Clear NVMe fields to simulate master restart state loss. + ms.blockRegistry.UpdateEntry("pub-vol-3", func(e *BlockVolumeEntry) { + e.NvmeAddr = "" + e.NQN = "" + }) + + cleared, _ := ms.blockRegistry.Lookup("pub-vol-3") + if cleared.NvmeAddr != "" { + t.Fatal("NvmeAddr should be cleared") + } + + // Reconstruct via REAL heartbeat path: UpdateFullHeartbeat with proto info. + // This is the same code path as master_grpc_server.go:280. + heartbeatInfo := &master_pb.BlockVolumeInfoMessage{ + Path: entry.Path, + VolumeSize: entry.SizeBytes, + BlockSize: 4096, + Epoch: entry.Epoch, + Role: entry.Role, + NvmeAddr: originalNVMe, + Nqn: originalNQN, + } + + // Derive server-level NVMe addr (same as what the VS heartbeat carries). + host := primaryServer + if idx := strings.LastIndex(host, ":"); idx >= 0 { + host = host[:idx] + } + serverNvmeAddr := host + ":4420" + + ms.blockRegistry.UpdateFullHeartbeat( + primaryServer, + []*master_pb.BlockVolumeInfoMessage{heartbeatInfo}, + serverNvmeAddr, + ) + + // Verify: lookup reflects heartbeat-reconstructed NVMe truth. + lookupResp, err := ms.LookupBlockVolume(ctx, &master_pb.LookupBlockVolumeRequest{Name: "pub-vol-3"}) + if err != nil { + t.Fatalf("Lookup after reconstruction: %v", err) + } + if lookupResp.NvmeAddr != originalNVMe { + t.Fatalf("reconstructed NvmeAddr=%q, want %q", lookupResp.NvmeAddr, originalNVMe) + } + if lookupResp.Nqn != originalNQN { + t.Fatalf("reconstructed NQN=%q, want %q", lookupResp.Nqn, originalNQN) + } + + _ = createResp // used for initial creation + + t.Logf("P11P3 reconstruction: cleared → UpdateFullHeartbeat → lookup restored NVMe=%s NQN=%s", + lookupResp.NvmeAddr, lookupResp.Nqn) +} + +// --- 4. No-NVMe fallback --- + +func TestP11P3_NoNVMe_Fallback(t *testing.T) { + ms := newPublicationMaster(t, false) // NVMe disabled + ctx := context.Background() + + createResp, err := ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "pub-vol-4", SizeBytes: 1 << 30, + }) + if err != nil { + t.Fatalf("Create: %v", err) + } + + // iSCSI must be present. + if createResp.IscsiAddr == "" || createResp.Iqn == "" { + t.Fatal("iSCSI fields must be present even without NVMe") + } + + // NVMe must be explicitly empty (not fabricated). + if createResp.NvmeAddr != "" { + t.Fatalf("NvmeAddr=%q should be empty when NVMe disabled", createResp.NvmeAddr) + } + if createResp.Nqn != "" { + t.Fatalf("Nqn=%q should be empty when NVMe disabled", createResp.Nqn) + } + + // Lookup must also show empty NVMe. + lookupResp, _ := ms.LookupBlockVolume(ctx, &master_pb.LookupBlockVolumeRequest{Name: "pub-vol-4"}) + if lookupResp.NvmeAddr != "" || lookupResp.Nqn != "" { + t.Fatalf("lookup: NVMe should be empty, got addr=%q nqn=%q", lookupResp.NvmeAddr, lookupResp.Nqn) + } + + // Registry must also show empty NVMe. + entry, _ := ms.blockRegistry.Lookup("pub-vol-4") + if entry.NvmeAddr != "" || entry.NQN != "" { + t.Fatalf("registry: NVMe should be empty, got addr=%q nqn=%q", entry.NvmeAddr, entry.NQN) + } + + t.Logf("P11P3 no-NVMe: iSCSI=%s IQN=%s, NVMe explicitly empty across create/lookup/registry", + createResp.IscsiAddr, createResp.Iqn) +} diff --git a/weed/server/qa_block_restore_test.go b/weed/server/qa_block_restore_test.go new file mode 100644 index 000000000..6eb6457a0 --- /dev/null +++ b/weed/server/qa_block_restore_test.go @@ -0,0 +1,240 @@ +package weed_server + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + volume_server_pb "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" +) + +// ============================================================ +// Phase 11 P4: Snapshot restore workflow closure +// +// Proofs: +// 1. Restore success: product-visible restore reverts to snapshot +// 2. Destructive semantics: post-snapshot writes are lost +// 3. Post-restore coherence: list shows expected snapshot state +// 4. Fail-closed: missing snapshot, missing volume +// +// V1 reuse roles: +// master_grpc_server_block.go: reuse as bounded adapter (RestoreBlockSnapshot RPC) +// volume_server_block.go: reuse as bounded adapter (RestoreBlockSnapshot) +// blockvol.RestoreSnapshot: reuse as execution reality only +// ============================================================ + +func newRestoreMaster(t *testing.T) (*MasterServer, *storage.BlockVolumeStore, *BlockService) { + t.Helper() + dir := t.TempDir() + store := storage.NewBlockVolumeStore() + + ms := &MasterServer{ + blockRegistry: NewBlockVolumeRegistry(), + blockAssignmentQueue: NewBlockAssignmentQueue(), + blockFailover: newBlockFailoverState(), + } + ms.blockRegistry.MarkBlockCapable("vs1:9333") + ms.blockRegistry.MarkBlockCapable("vs2:9333") + + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { + sanitized := strings.ReplaceAll(server, ":", "_") + serverDir := filepath.Join(dir, sanitized) + os.MkdirAll(serverDir, 0755) + volPath := filepath.Join(serverDir, fmt.Sprintf("%s.blk", name)) + vol, err := blockvol.CreateBlockVol(volPath, blockvol.CreateOptions{ + VolumeSize: 1 * 1024 * 1024, + BlockSize: 4096, + WALSize: 256 * 1024, + }) + if err != nil { + return nil, err + } + vol.Close() + if _, err := store.AddBlockVolume(volPath, ""); err != nil { + return nil, err + } + host := server + if idx := strings.LastIndex(server, ":"); idx >= 0 { + host = server[:idx] + } + return &blockAllocResult{ + Path: volPath, + IQN: fmt.Sprintf("iqn.2024.test:%s", name), + ISCSIAddr: host + ":3260", + }, nil + } + ms.blockVSDelete = func(ctx context.Context, server string, name string) error { return nil } + + // Build a real BlockService so ALL callbacks go through the VS adapter. + // Set blockDir to the vs1 subdir so volumePath(name) resolves correctly. + bs := &BlockService{ + blockStore: store, + blockDir: filepath.Join(dir, strings.ReplaceAll("vs1:9333", ":", "_")), + listenAddr: "127.0.0.1:3260", + } + + // Wire snapshot callbacks through real BlockService adapters. + ms.blockVSSnapshot = func(ctx context.Context, server string, name string, snapID uint32) (int64, uint64, error) { + return bs.SnapshotBlockVol(name, snapID) + } + ms.blockVSDeleteSnap = func(ctx context.Context, server string, name string, snapID uint32) error { + return bs.DeleteBlockSnapshot(name, snapID) + } + ms.blockVSListSnaps = func(ctx context.Context, server string, name string) ([]*volume_server_pb.BlockSnapshotInfo, error) { + infos, volSize, err := bs.ListBlockSnapshots(name) + if err != nil { + return nil, err + } + var result []*volume_server_pb.BlockSnapshotInfo + for _, si := range infos { + result = append(result, &volume_server_pb.BlockSnapshotInfo{ + SnapshotId: si.ID, + CreatedAt: si.CreatedAt.Unix(), + VolumeSizeBytes: volSize, + }) + } + return result, nil + } + + // Wire restore callback through REAL BlockService.RestoreBlockSnapshot adapter. + ms.blockVSRestore = func(ctx context.Context, server string, name string, snapID uint32) error { + return bs.RestoreBlockSnapshot(name, snapID) + } + + t.Cleanup(func() { store.Close() }) + return ms, store, bs +} + +// --- 1. Restore success: reverts to snapshot --- + +func TestP11P4_RestoreSuccess(t *testing.T) { + ms, store, _ := newRestoreMaster(t) + ctx := context.Background() + + // Create volume. + ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "restore-vol-1", SizeBytes: 1 << 20, + }) + entry, _ := ms.blockRegistry.Lookup("restore-vol-1") + + // Write data, create snapshot. + store.WithVolume(entry.Path, func(vol *blockvol.BlockVol) error { + vol.WriteLBA(0, make([]byte, 4096)) + vol.WriteLBA(1, make([]byte, 4096)) + return nil + }) + ms.CreateBlockSnapshot(ctx, &master_pb.CreateBlockSnapshotRequest{ + VolumeName: "restore-vol-1", SnapshotId: 1, + }) + + // Write MORE data after snapshot. + store.WithVolume(entry.Path, func(vol *blockvol.BlockVol) error { + data := make([]byte, 4096) + for i := range data { + data[i] = 0xFF + } + vol.WriteLBA(0, data) // overwrite LBA 0 + vol.WriteLBA(5, data) // new LBA 5 + return nil + }) + + // Restore to snapshot 1. + _, err := ms.RestoreBlockSnapshot(ctx, &master_pb.RestoreBlockSnapshotRequest{ + VolumeName: "restore-vol-1", SnapshotId: 1, + }) + if err != nil { + t.Fatalf("Restore: %v", err) + } + + // Verify: LBA 0 should be exactly 0x00 (snapshot was taken after zero-fill write). + var lba0First, lba5First byte + store.WithVolume(entry.Path, func(vol *blockvol.BlockVol) error { + data0, _ := vol.ReadLBA(0, 4096) + lba0First = data0[0] + data5, _ := vol.ReadLBA(5, 4096) + lba5First = data5[0] + return nil + }) + if lba0First != 0x00 { + t.Fatalf("LBA 0 = 0x%02X, want 0x00 (reverted to snapshot state)", lba0First) + } + if lba5First != 0x00 { + t.Fatalf("LBA 5 = 0x%02X, want 0x00 (post-snapshot write should be lost)", lba5First) + } + + t.Logf("P11P4 restore: LBA 0 = 0x%02X (reverted), LBA 5 = 0x%02X (post-snap lost)", lba0First, lba5First) +} + +// --- 2. Destructive semantics: snapshots gone after restore --- + +func TestP11P4_DestructiveSemantics(t *testing.T) { + ms, _, _ := newRestoreMaster(t) + ctx := context.Background() + + ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "restore-vol-2", SizeBytes: 1 << 20, + }) + + // Create snapshots 1, 2, 3. + for i := uint32(1); i <= 3; i++ { + ms.CreateBlockSnapshot(ctx, &master_pb.CreateBlockSnapshotRequest{ + VolumeName: "restore-vol-2", SnapshotId: i, + }) + } + + // Restore to snapshot 1: all snapshots are removed by restore. + ms.RestoreBlockSnapshot(ctx, &master_pb.RestoreBlockSnapshotRequest{ + VolumeName: "restore-vol-2", SnapshotId: 1, + }) + + // List: should have 0 snapshots (restore removes all). + listResp, _ := ms.ListBlockSnapshots(ctx, &master_pb.ListBlockSnapshotsRequest{ + VolumeName: "restore-vol-2", + }) + if len(listResp.Snapshots) != 0 { + t.Fatalf("expected 0 snapshots after restore, got %d", len(listResp.Snapshots)) + } + + t.Log("P11P4 destructive: restore to snap 1 → all snapshots removed") +} + +// --- 3. Fail-closed: missing snapshot --- + +func TestP11P4_FailClosed_MissingSnapshot(t *testing.T) { + ms, _, _ := newRestoreMaster(t) + ctx := context.Background() + + ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: "restore-vol-3", SizeBytes: 1 << 20, + }) + + // Restore nonexistent snapshot. + _, err := ms.RestoreBlockSnapshot(ctx, &master_pb.RestoreBlockSnapshotRequest{ + VolumeName: "restore-vol-3", SnapshotId: 99, + }) + if err == nil { + t.Fatal("restore missing snapshot should fail") + } + t.Logf("P11P4 fail-closed: missing snapshot → %v", err) +} + +// --- 4. Fail-closed: missing volume --- + +func TestP11P4_FailClosed_MissingVolume(t *testing.T) { + ms, _, _ := newRestoreMaster(t) + ctx := context.Background() + + _, err := ms.RestoreBlockSnapshot(ctx, &master_pb.RestoreBlockSnapshotRequest{ + VolumeName: "nonexistent-vol", SnapshotId: 1, + }) + if err == nil { + t.Fatal("restore on missing volume should fail") + } + t.Logf("P11P4 fail-closed: missing volume → %v", err) +} diff --git a/weed/server/qa_block_snapshot_product_test.go b/weed/server/qa_block_snapshot_product_test.go new file mode 100644 index 000000000..3f5bc69fb --- /dev/null +++ b/weed/server/qa_block_snapshot_product_test.go @@ -0,0 +1,342 @@ +package weed_server + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage" + "github.com/seaweedfs/seaweedfs/weed/storage/blockvol" +) + +// ============================================================ +// Phase 11 P1: Snapshot product-path rebinding +// +// Proofs: +// 1. Create: product-visible create → backend snapshot → readback metadata +// 2. List: listed snapshots match backend truth after create and delete +// 3. Delete: snapshot removed from visible set, repeated delete idempotent +// 4. Fail-closed: nonexistent volume, missing snapshot +// 5. Reuse boundary: all V1 surfaces explicitly bounded +// +// V1 reuse roles: +// master_grpc_server_block.go: reuse as bounded adapter (RPC resolution) +// volume_server_block.go: reuse as bounded adapter (VS execution) +// blockvol/*: reuse as execution reality only +// ============================================================ + +// snapshotTestSetup creates a master + BlockService with a real volume, +// wiring the master's VS snapshot callbacks to the real BlockService. +type snapshotTestSetup struct { + ms *MasterServer + bs *BlockService + store *storage.BlockVolumeStore + dir string +} + +func newSnapshotTestSetup(t *testing.T) *snapshotTestSetup { + t.Helper() + dir := t.TempDir() + store := storage.NewBlockVolumeStore() + + bs := &BlockService{ + blockStore: store, + blockDir: dir, + listenAddr: "127.0.0.1:3260", + } + + ms := &MasterServer{ + blockRegistry: NewBlockVolumeRegistry(), + blockAssignmentQueue: NewBlockAssignmentQueue(), + blockFailover: newBlockFailoverState(), + } + ms.blockRegistry.MarkBlockCapable("vs1:9333") + ms.blockRegistry.MarkBlockCapable("vs2:9333") + + s := &snapshotTestSetup{ms: ms, bs: bs, store: store, dir: dir} + + // Wire master allocator to create real volumes. + ms.blockVSAllocate = func(ctx context.Context, server string, name string, sizeBytes uint64, diskType string, durabilityMode string) (*blockAllocResult, error) { + sanitized := strings.ReplaceAll(server, ":", "_") + serverDir := filepath.Join(dir, sanitized) + if err := os.MkdirAll(serverDir, 0755); err != nil { + return nil, err + } + volPath := filepath.Join(serverDir, fmt.Sprintf("%s.blk", name)) + vol, err := blockvol.CreateBlockVol(volPath, blockvol.CreateOptions{ + VolumeSize: 1 * 1024 * 1024, + BlockSize: 4096, + WALSize: 256 * 1024, + }) + if err != nil { + return nil, err + } + vol.Close() + if _, err := store.AddBlockVolume(volPath, ""); err != nil { + return nil, err + } + return &blockAllocResult{ + Path: volPath, + IQN: fmt.Sprintf("iqn.2024.test:%s", name), + ISCSIAddr: server + ":3260", + }, nil + } + ms.blockVSDelete = func(ctx context.Context, server string, name string) error { + return nil + } + + // Wire master snapshot callbacks through REAL BlockService adapter methods. + // This proves the VS adapter layer (SnapshotBlockVol, DeleteBlockSnapshot, + // ListBlockSnapshots) is part of the tested chain, not bypassed. + // + // Set blockDir to the vs1 subdir so BlockService.volumePath(name) resolves + // to the same paths registered in the store. + bs.blockDir = filepath.Join(dir, strings.ReplaceAll("vs1:9333", ":", "_")) + + ms.blockVSSnapshot = func(ctx context.Context, server string, name string, snapID uint32) (int64, uint64, error) { + return bs.SnapshotBlockVol(name, snapID) + } + ms.blockVSDeleteSnap = func(ctx context.Context, server string, name string, snapID uint32) error { + return bs.DeleteBlockSnapshot(name, snapID) + } + ms.blockVSListSnaps = func(ctx context.Context, server string, name string) ([]*volume_server_pb.BlockSnapshotInfo, error) { + infos, volSize, err := bs.ListBlockSnapshots(name) + if err != nil { + return nil, err + } + var result []*volume_server_pb.BlockSnapshotInfo + for _, si := range infos { + result = append(result, &volume_server_pb.BlockSnapshotInfo{ + SnapshotId: si.ID, + CreatedAt: si.CreatedAt.Unix(), + VolumeSizeBytes: volSize, + }) + } + return result, nil + } + + t.Cleanup(func() { store.Close() }) + return s +} + +func (s *snapshotTestSetup) createVolume(t *testing.T, name string) { + t.Helper() + ctx := context.Background() + _, err := s.ms.CreateBlockVolume(ctx, &master_pb.CreateBlockVolumeRequest{ + Name: name, + SizeBytes: 1 << 30, + }) + if err != nil { + t.Fatalf("CreateBlockVolume(%s): %v", name, err) + } +} + +// --- 1. Create: product-visible snapshot creation --- + +func TestP11P1_SnapshotCreate(t *testing.T) { + s := newSnapshotTestSetup(t) + s.createVolume(t, "snap-vol-1") + + ctx := context.Background() + + // Product-visible create through master RPC. + createResp, err := s.ms.CreateBlockSnapshot(ctx, &master_pb.CreateBlockSnapshotRequest{ + VolumeName: "snap-vol-1", + SnapshotId: 1, + }) + if err != nil { + t.Fatalf("CreateBlockSnapshot: %v", err) + } + if createResp.SnapshotId != 1 { + t.Fatalf("SnapshotId=%d, want 1", createResp.SnapshotId) + } + if createResp.CreatedAt == 0 { + t.Fatal("CreatedAt should be non-zero") + } + if createResp.SizeBytes == 0 { + t.Fatal("SizeBytes should be non-zero") + } + + // Verify: snapshot is visible through list. + listResp, err := s.ms.ListBlockSnapshots(ctx, &master_pb.ListBlockSnapshotsRequest{ + VolumeName: "snap-vol-1", + }) + if err != nil { + t.Fatalf("ListBlockSnapshots: %v", err) + } + if len(listResp.Snapshots) != 1 { + t.Fatalf("expected 1 snapshot, got %d", len(listResp.Snapshots)) + } + listed := listResp.Snapshots[0] + if listed.SnapshotId != 1 { + t.Fatalf("listed SnapshotId=%d, want 1", listed.SnapshotId) + } + // Metadata equality: create response and list response must agree. + if listed.CreatedAt != createResp.CreatedAt { + t.Fatalf("metadata mismatch: create.CreatedAt=%d != list.CreatedAt=%d", + createResp.CreatedAt, listed.CreatedAt) + } + if listed.VolumeSizeBytes != createResp.SizeBytes { + t.Fatalf("metadata mismatch: create.SizeBytes=%d != list.VolumeSizeBytes=%d", + createResp.SizeBytes, listed.VolumeSizeBytes) + } + + t.Logf("P11P1 create: master RPC → VS adapter → blockvol → snap(id=1, created=%d, size=%d) → list metadata matches", + createResp.CreatedAt, createResp.SizeBytes) +} + +// --- 2. List coherence: after create and delete --- + +func TestP11P1_SnapshotListCoherence(t *testing.T) { + s := newSnapshotTestSetup(t) + s.createVolume(t, "snap-vol-2") + + ctx := context.Background() + + // Create 3 snapshots. + for i := uint32(1); i <= 3; i++ { + _, err := s.ms.CreateBlockSnapshot(ctx, &master_pb.CreateBlockSnapshotRequest{ + VolumeName: "snap-vol-2", + SnapshotId: i, + }) + if err != nil { + t.Fatalf("create snap %d: %v", i, err) + } + } + + // List: should have 3. + listResp, err := s.ms.ListBlockSnapshots(ctx, &master_pb.ListBlockSnapshotsRequest{ + VolumeName: "snap-vol-2", + }) + if err != nil { + t.Fatalf("list: %v", err) + } + if len(listResp.Snapshots) != 3 { + t.Fatalf("expected 3 snapshots, got %d", len(listResp.Snapshots)) + } + + // Delete snapshot 2. + _, err = s.ms.DeleteBlockSnapshot(ctx, &master_pb.DeleteBlockSnapshotRequest{ + VolumeName: "snap-vol-2", + SnapshotId: 2, + }) + if err != nil { + t.Fatalf("delete snap 2: %v", err) + } + + // List: should have 2 (IDs 1 and 3). + listResp, err = s.ms.ListBlockSnapshots(ctx, &master_pb.ListBlockSnapshotsRequest{ + VolumeName: "snap-vol-2", + }) + if err != nil { + t.Fatalf("list after delete: %v", err) + } + if len(listResp.Snapshots) != 2 { + t.Fatalf("expected 2 snapshots after delete, got %d", len(listResp.Snapshots)) + } + + ids := map[uint32]bool{} + for _, snap := range listResp.Snapshots { + ids[snap.SnapshotId] = true + } + if !ids[1] || !ids[3] { + t.Fatalf("expected IDs {1,3}, got %v", ids) + } + if ids[2] { + t.Fatal("deleted snapshot 2 should not be in list") + } + + t.Log("P11P1 list coherence: create 3 → delete 2 → list shows {1,3}") +} + +// --- 3. Delete: remove + repeated delete idempotent --- + +func TestP11P1_SnapshotDelete(t *testing.T) { + s := newSnapshotTestSetup(t) + s.createVolume(t, "snap-vol-3") + + ctx := context.Background() + + // Create and delete. + s.ms.CreateBlockSnapshot(ctx, &master_pb.CreateBlockSnapshotRequest{ + VolumeName: "snap-vol-3", SnapshotId: 10, + }) + + _, err := s.ms.DeleteBlockSnapshot(ctx, &master_pb.DeleteBlockSnapshotRequest{ + VolumeName: "snap-vol-3", SnapshotId: 10, + }) + if err != nil { + t.Fatalf("delete: %v", err) + } + + // Repeated delete: should be idempotent (no error). + _, err = s.ms.DeleteBlockSnapshot(ctx, &master_pb.DeleteBlockSnapshotRequest{ + VolumeName: "snap-vol-3", SnapshotId: 10, + }) + if err != nil { + t.Fatalf("repeated delete should be idempotent, got: %v", err) + } + + // List: empty. + listResp, _ := s.ms.ListBlockSnapshots(ctx, &master_pb.ListBlockSnapshotsRequest{ + VolumeName: "snap-vol-3", + }) + if len(listResp.Snapshots) != 0 { + t.Fatalf("expected 0 snapshots, got %d", len(listResp.Snapshots)) + } + + t.Log("P11P1 delete: create → delete → repeated delete idempotent → list empty") +} + +// --- 4. Fail-closed: nonexistent volume, duplicate create --- + +func TestP11P1_SnapshotFailClosed(t *testing.T) { + s := newSnapshotTestSetup(t) + s.createVolume(t, "snap-vol-4") + + ctx := context.Background() + + // Create on nonexistent volume. + _, err := s.ms.CreateBlockSnapshot(ctx, &master_pb.CreateBlockSnapshotRequest{ + VolumeName: "nonexistent-vol", + SnapshotId: 1, + }) + if err == nil { + t.Fatal("create on nonexistent volume should fail") + } + + // List on nonexistent volume. + _, err = s.ms.ListBlockSnapshots(ctx, &master_pb.ListBlockSnapshotsRequest{ + VolumeName: "nonexistent-vol", + }) + if err == nil { + t.Fatal("list on nonexistent volume should fail") + } + + // Delete on nonexistent volume: idempotent (volume gone → snapshot gone). + _, err = s.ms.DeleteBlockSnapshot(ctx, &master_pb.DeleteBlockSnapshotRequest{ + VolumeName: "nonexistent-vol", + SnapshotId: 1, + }) + if err != nil { + t.Fatalf("delete on nonexistent volume should be idempotent, got: %v", err) + } + + // Duplicate create (same ID twice). + s.ms.CreateBlockSnapshot(ctx, &master_pb.CreateBlockSnapshotRequest{ + VolumeName: "snap-vol-4", SnapshotId: 99, + }) + _, err = s.ms.CreateBlockSnapshot(ctx, &master_pb.CreateBlockSnapshotRequest{ + VolumeName: "snap-vol-4", SnapshotId: 99, + }) + if err == nil { + t.Fatal("duplicate create should fail") + } + + t.Log("P11P1 fail-closed: nonexistent vol, duplicate create all handled correctly") +} diff --git a/weed/storage/blockvol/csi/export_test_helpers.go b/weed/storage/blockvol/csi/export_test_helpers.go new file mode 100644 index 000000000..4c8736e25 --- /dev/null +++ b/weed/storage/blockvol/csi/export_test_helpers.go @@ -0,0 +1,86 @@ +package csi + +import ( + "context" + "log" +) + +// ExportedControllerServer wraps controllerServer for cross-package testing. +type ExportedControllerServer struct { + *controllerServer +} + +// NewExportedControllerServer creates a controller server with the given backend. +func NewExportedControllerServer(backend VolumeBackend) *ExportedControllerServer { + return &ExportedControllerServer{&controllerServer{backend: backend}} +} + +// StagedInfo holds exported staged volume info for cross-package test assertions. +type StagedInfo struct { + IsLocal bool + ISCSIAddr string + IQN string + Transport string +} + +// ExportedNodeServer wraps nodeServer for cross-package testing. +type ExportedNodeServer struct { + *nodeServer +} + +// NewExportedNodeServer creates a node server with the given parameters. +// Pass mgr=nil to prove publish_context consumption (no local fallback). +// Uses NoopISCSI and NoopMount for cross-package test use. +func NewExportedNodeServer(nodeID string, mgr *VolumeManager, logger *log.Logger) *ExportedNodeServer { + return &ExportedNodeServer{&nodeServer{ + mgr: mgr, + nodeID: nodeID, + iscsiUtil: &noopISCSI{}, + mountUtil: &noopMount{}, + logger: logger, + staged: map[string]*stagedVolumeInfo{}, + }} +} + +// GetStagedInfo returns exported staging info for a volume. +func (e *ExportedNodeServer) GetStagedInfo(volumeID string) *StagedInfo { + e.stagedMu.Lock() + defer e.stagedMu.Unlock() + info, ok := e.staged[volumeID] + if !ok { + return nil + } + return &StagedInfo{ + IsLocal: info.isLocal, + ISCSIAddr: info.iscsiAddr, + IQN: info.iqn, + Transport: info.transport, + } +} + +// noopISCSI implements ISCSIUtil with no real iSCSI operations. +type noopISCSI struct{} + +func (n *noopISCSI) Discovery(ctx context.Context, portal string) error { return nil } +func (n *noopISCSI) Login(ctx context.Context, iqn, portal string) error { return nil } +func (n *noopISCSI) Logout(ctx context.Context, iqn string) error { return nil } +func (n *noopISCSI) GetDeviceByIQN(ctx context.Context, iqn string) (string, error) { + return "/dev/sda-noop", nil +} +func (n *noopISCSI) IsLoggedIn(ctx context.Context, iqn string) (bool, error) { return false, nil } +func (n *noopISCSI) RescanDevice(ctx context.Context, iqn string) error { return nil } + +// noopMount implements MountUtil with no real mount operations. +type noopMount struct{} + +func (n *noopMount) FormatAndMount(ctx context.Context, device, target, fsType string) error { return nil } +func (n *noopMount) Mount(ctx context.Context, source, target, fsType string, readOnly bool) error { + return nil +} +func (n *noopMount) BindMount(ctx context.Context, source, target string, readOnly bool) error { + return nil +} +func (n *noopMount) Unmount(ctx context.Context, target string) error { return nil } +func (n *noopMount) IsFormatted(ctx context.Context, device string) (bool, error) { return true, nil } +func (n *noopMount) IsMounted(ctx context.Context, target string) (bool, error) { return false, nil } +func (n *noopMount) ResizeFS(ctx context.Context, devicePath, mountPath, fsType string) error { return nil }