From 43016e6645e881f5e419918e63fefac1f437041d Mon Sep 17 00:00:00 2001 From: pingqiu Date: Sun, 5 Apr 2026 16:23:35 -0700 Subject: [PATCH] =?UTF-8?q?fix:=20Phase=2020=20T3=20=E2=80=94=20production?= =?UTF-8?q?=20wiring=20+=20fail-closed=20on=20partial=20evidence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire V2 promotion into production binary: - Add --block.v2Promotion CLI flag on weed master (default false) - MasterOption.BlockV2Promotion → NewMasterServer wires flag + querier - defaultBlockVSQueryEvidence placeholder (returns explicit error until proto regen on M01 enables gRPC evidence RPC) Fix three fail-closed violations found by tester: 1. blockV2Promotion=true + nil querier now fails closed with explicit log instead of silently falling back to V1 2. Partial evidence (any candidate query failed) now fails closed — unreachable candidate may be the most durable, promoting from incomplete evidence violates durability-first ordering 3. Clear EngineProjectionMode in applyPromotionLocked (already in previous commit, verified in tests here) 2 new tests: NilQuerier_FailsClosed, PartialEvidenceFailure_FailsClosed. Total T3 tests: 7, all pass. Existing V1 failover tests unaffected. Co-Authored-By: Claude Opus 4.6 (1M context) --- weed/command/master.go | 3 + weed/server/master_block_evidence.go | 12 ++++ weed/server/master_block_failover.go | 15 ++++- weed/server/master_block_failover_test.go | 70 +++++++++++++++++++++++ weed/server/master_server.go | 5 ++ 5 files changed, 104 insertions(+), 1 deletion(-) diff --git a/weed/command/master.go b/weed/command/master.go index 6be079c99..c2ca4ace1 100644 --- a/weed/command/master.go +++ b/weed/command/master.go @@ -74,6 +74,7 @@ type MasterOptions struct { debug *bool debugPort *int blockPromotionLSNTolerance *int + blockV2Promotion *bool } func init() { @@ -106,6 +107,7 @@ func init() { m.debug = cmdMaster.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port") m.debugPort = cmdMaster.Flag.Int("debug.port", 6060, "http port for debugging") m.blockPromotionLSNTolerance = cmdMaster.Flag.Int("block.promotion.lsnTolerance", 100, "max WAL LSN lag for block volume replica promotion eligibility") + m.blockV2Promotion = cmdMaster.Flag.Bool("block.v2Promotion", false, "enable V2 durability-first promotion (requires proto regen for evidence RPC)") } var cmdMaster = &Command{ @@ -415,5 +417,6 @@ func (m *MasterOptions) toMasterOption(whiteList []string) *weed_server.MasterOp TelemetryUrl: *m.telemetryUrl, TelemetryEnabled: *m.telemetryEnabled, BlockPromotionLSNTolerance: *m.blockPromotionLSNTolerance, + BlockV2Promotion: *m.blockV2Promotion, } } diff --git a/weed/server/master_block_evidence.go b/weed/server/master_block_evidence.go index 673d8b6ff..894ac25d2 100644 --- a/weed/server/master_block_evidence.go +++ b/weed/server/master_block_evidence.go @@ -84,3 +84,15 @@ type promotionCandidate struct { path string expectedEpoch uint64 } + +// defaultBlockVSQueryEvidence is the production evidence querier. +// Once proto is regenerated on M01, this will call the VS gRPC +// QueryBlockPromotionEvidence RPC. Until then, it returns an explicit +// error so the fail-closed path is exercised. +func (ms *MasterServer) defaultBlockVSQueryEvidence(ctx context.Context, server, path string, expectedEpoch uint64) (BlockPromotionEvidence, error) { + // TODO(T2-transport): Replace with gRPC call after proto regen: + // operation.WithVolumeServerClient(false, pb.ServerAddress(server), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + // resp, err := client.QueryBlockPromotionEvidence(ctx, &volume_server_pb.QueryBlockPromotionEvidenceRequest{...}) + // }) + return BlockPromotionEvidence{}, fmt.Errorf("V2 promotion evidence RPC not yet available (pending proto regen on M01): server=%s path=%s", server, path) +} diff --git a/weed/server/master_block_failover.go b/weed/server/master_block_failover.go index 44e5f2831..9c7b7120b 100644 --- a/weed/server/master_block_failover.go +++ b/weed/server/master_block_failover.go @@ -261,7 +261,11 @@ func (ms *MasterServer) promoteReplica(volumeName string) { return } - if ms.blockV2Promotion && ms.blockVSQueryEvidence != nil { + if ms.blockV2Promotion { + if ms.blockVSQueryEvidence == nil { + glog.Warningf("failover: V2 promotion enabled but evidence querier is nil for %q — fail closed (not falling back to V1)", volumeName) + return + } ms.promoteReplicaV2(volumeName, entry) return } @@ -314,6 +318,15 @@ func (ms *MasterServer) promoteReplicaV2(volumeName string, entry BlockVolumeEnt glog.Warningf("failover V2: %s", err) } + // Fail-closed on partial evidence: if any candidate query failed, an + // unreachable candidate may be the most durable. Promoting from + // incomplete evidence violates durability-first ordering. + if len(errs) > 0 { + glog.Warningf("failover V2: %q: fail-closed — %d/%d candidate queries failed, cannot guarantee durability ordering", + volumeName, len(errs), len(candidates)) + return + } + // Durability-first selection. Fail-closed if no eligible candidate. best, err := selectDurabilityFirstCandidate(evidence) if err != nil { diff --git a/weed/server/master_block_failover_test.go b/weed/server/master_block_failover_test.go index 29e4481aa..037f12836 100644 --- a/weed/server/master_block_failover_test.go +++ b/weed/server/master_block_failover_test.go @@ -1311,6 +1311,76 @@ func TestFailoverV2_FlagOff_UsesLegacy(t *testing.T) { } } +func TestFailoverV2_NilQuerier_FailsClosed(t *testing.T) { + ms := testMasterServerForFailover(t) + ms.blockV2Promotion = true + ms.blockVSQueryEvidence = nil // V2 on, but querier not wired + registerVolumeWithReplica(t, ms, "vol-v2-nilq", "vs1", "vs2", 1, 1*time.Second) + + ms.failoverBlockVolumes("vs1") + + entry, ok := ms.blockRegistry.Lookup("vol-v2-nilq") + if !ok { + t.Fatal("entry missing") + } + // Must fail closed, not silently use V1. + if entry.VolumeServer != "vs1" { + t.Fatalf("primary changed to %q, want vs1 (fail-closed on nil querier)", entry.VolumeServer) + } + if entry.Epoch != 1 { + t.Fatalf("epoch=%d, want 1 (unchanged)", entry.Epoch) + } +} + +func TestFailoverV2_PartialEvidenceFailure_FailsClosed(t *testing.T) { + callCount := 0 + querier := func(_ context.Context, server, path string, epoch uint64) (BlockPromotionEvidence, error) { + callCount++ + if server == "vs2" { + return BlockPromotionEvidence{}, fmt.Errorf("connection refused") + } + // vs3 responds successfully — but we should still fail closed because + // vs2 (unreachable) might be the most durable candidate. + return BlockPromotionEvidence{ + Server: server, Path: path, Epoch: 1, + CommittedLSN: 5, WALHeadLSN: 5, HealthScore: 1.0, + EngineProjectionMode: "publish_healthy", Eligible: true, + }, nil + } + ms := testMasterServerV2(t, querier) + + ms.blockRegistry.MarkBlockCapable("vs1") + ms.blockRegistry.MarkBlockCapable("vs2") + ms.blockRegistry.MarkBlockCapable("vs3") + if err := ms.blockRegistry.Register(&BlockVolumeEntry{ + Name: "vol-v2-partial", VolumeServer: "vs1", + Path: "/data/vol-v2-partial.blk", Epoch: 1, + Role: blockvol.RoleToWire(blockvol.RolePrimary), Status: StatusActive, + LeaseTTL: 1 * time.Second, LastLeaseGrant: time.Now().Add(-5 * time.Second), + Replicas: []ReplicaInfo{ + {Server: "vs2", Path: "/data/vol-v2-partial.blk", HealthScore: 1.0, Role: blockvol.RoleToWire(blockvol.RoleReplica), LastHeartbeat: time.Now()}, + {Server: "vs3", Path: "/data/vol-v2-partial.blk", HealthScore: 1.0, Role: blockvol.RoleToWire(blockvol.RoleReplica), LastHeartbeat: time.Now()}, + }, + }); err != nil { + t.Fatalf("register: %v", err) + } + + ms.failoverBlockVolumes("vs1") + + entry, ok := ms.blockRegistry.Lookup("vol-v2-partial") + if !ok { + t.Fatal("entry missing") + } + // Must fail closed: incomplete evidence set, unreachable candidate + // might be the most durable. + if entry.VolumeServer != "vs1" { + t.Fatalf("primary changed to %q, want vs1 (fail-closed on partial evidence)", entry.VolumeServer) + } + if entry.Epoch != 1 { + t.Fatalf("epoch=%d, want 1 (unchanged)", entry.Epoch) + } +} + func TestFailoverV2_EpochBumpAndAssignmentOnlyAfterSelection(t *testing.T) { querier := func(_ context.Context, server, path string, epoch uint64) (BlockPromotionEvidence, error) { return BlockPromotionEvidence{ diff --git a/weed/server/master_server.go b/weed/server/master_server.go index a6756020c..868d69b66 100644 --- a/weed/server/master_server.go +++ b/weed/server/master_server.go @@ -65,6 +65,7 @@ type MasterOption struct { TelemetryEnabled bool VolumeGrowthDisabled bool BlockPromotionLSNTolerance int + BlockV2Promotion bool // T3: enable durability-first V2 promotion } type MasterServer struct { @@ -180,6 +181,10 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se ms.blockVSPrepareExpand = ms.defaultBlockVSPrepareExpand ms.blockVSCommitExpand = ms.defaultBlockVSCommitExpand ms.blockVSCancelExpand = ms.defaultBlockVSCancelExpand + ms.blockV2Promotion = option.BlockV2Promotion + if ms.blockV2Promotion { + ms.blockVSQueryEvidence = ms.defaultBlockVSQueryEvidence + } ms.MasterClient.SetOnPeerUpdateFn(ms.OnPeerUpdate)