fix: Phase 20 T3 — production wiring + fail-closed on partial evidence

Wire V2 promotion into production binary:
- Add --block.v2Promotion CLI flag on weed master (default false)
- MasterOption.BlockV2Promotion → NewMasterServer wires flag + querier
- defaultBlockVSQueryEvidence placeholder (returns explicit error until
  proto regen on M01 enables gRPC evidence RPC)

Fix three fail-closed violations found by tester:
1. blockV2Promotion=true + nil querier now fails closed with explicit
   log instead of silently falling back to V1
2. Partial evidence (any candidate query failed) now fails closed —
   unreachable candidate may be the most durable, promoting from
   incomplete evidence violates durability-first ordering
3. Clear EngineProjectionMode in applyPromotionLocked (already in
   previous commit, verified in tests here)

2 new tests: NilQuerier_FailsClosed, PartialEvidenceFailure_FailsClosed.
Total T3 tests: 7, all pass. Existing V1 failover tests unaffected.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

weed/command/master.go

@@ -74,6 +74,7 @@ type MasterOptions struct {
 	debug                      *bool
 	debugPort                  *int
 	blockPromotionLSNTolerance *int
+	blockV2Promotion           *bool
 }
 
 func init() {
@@ -106,6 +107,7 @@ func init() {
 	m.debug = cmdMaster.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
 	m.debugPort = cmdMaster.Flag.Int("debug.port", 6060, "http port for debugging")
 	m.blockPromotionLSNTolerance = cmdMaster.Flag.Int("block.promotion.lsnTolerance", 100, "max WAL LSN lag for block volume replica promotion eligibility")
+	m.blockV2Promotion = cmdMaster.Flag.Bool("block.v2Promotion", false, "enable V2 durability-first promotion (requires proto regen for evidence RPC)")
 }
 
 var cmdMaster = &Command{
@@ -415,5 +417,6 @@ func (m *MasterOptions) toMasterOption(whiteList []string) *weed_server.MasterOp
 		TelemetryUrl:               *m.telemetryUrl,
 		TelemetryEnabled:           *m.telemetryEnabled,
 		BlockPromotionLSNTolerance: *m.blockPromotionLSNTolerance,
+		BlockV2Promotion:           *m.blockV2Promotion,
 	}
 }

weed/server/master_block_evidence.go

@@ -84,3 +84,15 @@ type promotionCandidate struct {
 	path          string
 	expectedEpoch uint64
 }
+
+// defaultBlockVSQueryEvidence is the production evidence querier.
+// Once proto is regenerated on M01, this will call the VS gRPC
+// QueryBlockPromotionEvidence RPC. Until then, it returns an explicit
+// error so the fail-closed path is exercised.
+func (ms *MasterServer) defaultBlockVSQueryEvidence(ctx context.Context, server, path string, expectedEpoch uint64) (BlockPromotionEvidence, error) {
+	// TODO(T2-transport): Replace with gRPC call after proto regen:
+	//   operation.WithVolumeServerClient(false, pb.ServerAddress(server), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
+	//     resp, err := client.QueryBlockPromotionEvidence(ctx, &volume_server_pb.QueryBlockPromotionEvidenceRequest{...})
+	//   })
+	return BlockPromotionEvidence{}, fmt.Errorf("V2 promotion evidence RPC not yet available (pending proto regen on M01): server=%s path=%s", server, path)
+}

weed/server/master_block_failover.go

@@ -261,7 +261,11 @@ func (ms *MasterServer) promoteReplica(volumeName string) {
 		return
 	}
 
-	if ms.blockV2Promotion && ms.blockVSQueryEvidence != nil {
+	if ms.blockV2Promotion {
+		if ms.blockVSQueryEvidence == nil {
+			glog.Warningf("failover: V2 promotion enabled but evidence querier is nil for %q — fail closed (not falling back to V1)", volumeName)
+			return
+		}
 		ms.promoteReplicaV2(volumeName, entry)
 		return
 	}
@@ -314,6 +318,15 @@ func (ms *MasterServer) promoteReplicaV2(volumeName string, entry BlockVolumeEnt
 		glog.Warningf("failover V2: %s", err)
 	}
 
+	// Fail-closed on partial evidence: if any candidate query failed, an
+	// unreachable candidate may be the most durable. Promoting from
+	// incomplete evidence violates durability-first ordering.
+	if len(errs) > 0 {
+		glog.Warningf("failover V2: %q: fail-closed — %d/%d candidate queries failed, cannot guarantee durability ordering",
+			volumeName, len(errs), len(candidates))
+		return
+	}
+
 	// Durability-first selection. Fail-closed if no eligible candidate.
 	best, err := selectDurabilityFirstCandidate(evidence)
 	if err != nil {

weed/server/master_block_failover_test.go

@@ -1311,6 +1311,76 @@ func TestFailoverV2_FlagOff_UsesLegacy(t *testing.T) {
 	}
 }
 
+func TestFailoverV2_NilQuerier_FailsClosed(t *testing.T) {
+	ms := testMasterServerForFailover(t)
+	ms.blockV2Promotion = true
+	ms.blockVSQueryEvidence = nil // V2 on, but querier not wired
+	registerVolumeWithReplica(t, ms, "vol-v2-nilq", "vs1", "vs2", 1, 1*time.Second)
+
+	ms.failoverBlockVolumes("vs1")
+
+	entry, ok := ms.blockRegistry.Lookup("vol-v2-nilq")
+	if !ok {
+		t.Fatal("entry missing")
+	}
+	// Must fail closed, not silently use V1.
+	if entry.VolumeServer != "vs1" {
+		t.Fatalf("primary changed to %q, want vs1 (fail-closed on nil querier)", entry.VolumeServer)
+	}
+	if entry.Epoch != 1 {
+		t.Fatalf("epoch=%d, want 1 (unchanged)", entry.Epoch)
+	}
+}
+
+func TestFailoverV2_PartialEvidenceFailure_FailsClosed(t *testing.T) {
+	callCount := 0
+	querier := func(_ context.Context, server, path string, epoch uint64) (BlockPromotionEvidence, error) {
+		callCount++
+		if server == "vs2" {
+			return BlockPromotionEvidence{}, fmt.Errorf("connection refused")
+		}
+		// vs3 responds successfully — but we should still fail closed because
+		// vs2 (unreachable) might be the most durable candidate.
+		return BlockPromotionEvidence{
+			Server: server, Path: path, Epoch: 1,
+			CommittedLSN: 5, WALHeadLSN: 5, HealthScore: 1.0,
+			EngineProjectionMode: "publish_healthy", Eligible: true,
+		}, nil
+	}
+	ms := testMasterServerV2(t, querier)
+
+	ms.blockRegistry.MarkBlockCapable("vs1")
+	ms.blockRegistry.MarkBlockCapable("vs2")
+	ms.blockRegistry.MarkBlockCapable("vs3")
+	if err := ms.blockRegistry.Register(&BlockVolumeEntry{
+		Name: "vol-v2-partial", VolumeServer: "vs1",
+		Path: "/data/vol-v2-partial.blk", Epoch: 1,
+		Role: blockvol.RoleToWire(blockvol.RolePrimary), Status: StatusActive,
+		LeaseTTL: 1 * time.Second, LastLeaseGrant: time.Now().Add(-5 * time.Second),
+		Replicas: []ReplicaInfo{
+			{Server: "vs2", Path: "/data/vol-v2-partial.blk", HealthScore: 1.0, Role: blockvol.RoleToWire(blockvol.RoleReplica), LastHeartbeat: time.Now()},
+			{Server: "vs3", Path: "/data/vol-v2-partial.blk", HealthScore: 1.0, Role: blockvol.RoleToWire(blockvol.RoleReplica), LastHeartbeat: time.Now()},
+		},
+	}); err != nil {
+		t.Fatalf("register: %v", err)
+	}
+
+	ms.failoverBlockVolumes("vs1")
+
+	entry, ok := ms.blockRegistry.Lookup("vol-v2-partial")
+	if !ok {
+		t.Fatal("entry missing")
+	}
+	// Must fail closed: incomplete evidence set, unreachable candidate
+	// might be the most durable.
+	if entry.VolumeServer != "vs1" {
+		t.Fatalf("primary changed to %q, want vs1 (fail-closed on partial evidence)", entry.VolumeServer)
+	}
+	if entry.Epoch != 1 {
+		t.Fatalf("epoch=%d, want 1 (unchanged)", entry.Epoch)
+	}
+}
+
 func TestFailoverV2_EpochBumpAndAssignmentOnlyAfterSelection(t *testing.T) {
 	querier := func(_ context.Context, server, path string, epoch uint64) (BlockPromotionEvidence, error) {
 		return BlockPromotionEvidence{

weed/server/master_server.go

@@ -65,6 +65,7 @@ type MasterOption struct {
 	TelemetryEnabled            bool
 	VolumeGrowthDisabled        bool
 	BlockPromotionLSNTolerance  int
+	BlockV2Promotion            bool // T3: enable durability-first V2 promotion
 }
 
 type MasterServer struct {
@@ -180,6 +181,10 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se
 	ms.blockVSPrepareExpand = ms.defaultBlockVSPrepareExpand
 	ms.blockVSCommitExpand = ms.defaultBlockVSCommitExpand
 	ms.blockVSCancelExpand = ms.defaultBlockVSCancelExpand
+	ms.blockV2Promotion = option.BlockV2Promotion
+	if ms.blockV2Promotion {
+		ms.blockVSQueryEvidence = ms.defaultBlockVSQueryEvidence
+	}
 
 	ms.MasterClient.SetOnPeerUpdateFn(ms.OnPeerUpdate)