From 46f72572c57ac82a763c196a1eb352ff569d2a65 Mon Sep 17 00:00:00 2001 From: pingqiu Date: Sun, 5 Apr 2026 18:22:49 -0700 Subject: [PATCH] =?UTF-8?q?fix:=20Phase=2020=20T4=20=E2=80=94=20real=20ser?= =?UTF-8?q?ving=20enforcement=20+=20wire=20propagation=20+=20runtime=20ung?= =?UTF-8?q?ate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix three tester findings on T4 activation gate: 1. Real serving enforcement: evaluateActivationGate now calls gateServing() → DisconnectVolume(iqn) on gate (terminates active iSCSI sessions, removes volume from target). ungateServing() → AddVolume(iqn, adapter) on clear (re-registers volume). This is actual serving enforcement, not just bookkeeping. 2. Wire propagation: add activation_gated (field 25) and activation_gate_reason (field 26) to proto BlockVolumeInfoMessage. Add generated Go fields + getters. Add proto conversion in InfoMessageToProto/InfoMessageFromProto. Gate state now rides the real VS→master heartbeat wire. 3. Runtime ungate: evaluateActivationGate() now also runs in applyCoreEvent() (the observation-driven path), not just applyCoreAssignmentEvent(). Recovery/catch-up completion that transitions the projection to publish_healthy/replica_ready now clears the gate and re-registers the volume automatically. ClearActivationGate() remains as an explicit override for edge cases but is no longer the primary ungate path. Co-Authored-By: Claude Opus 4.6 (1M context) --- weed/pb/master.proto | 2 + weed/pb/master_pb/master.pb.go | 16 ++++++ weed/server/volume_server_block.go | 57 ++++++++++++++++++- .../storage/blockvol/block_heartbeat_proto.go | 4 ++ 4 files changed, 77 insertions(+), 2 deletions(-) diff --git a/weed/pb/master.proto b/weed/pb/master.proto index 687db9ad1..3175b894c 100644 --- a/weed/pb/master.proto +++ b/weed/pb/master.proto @@ -519,6 +519,8 @@ message BlockVolumeInfoMessage { optional string volume_mode = 22; optional string volume_mode_reason = 23; optional string engine_projection_mode = 24; // V2: pure engine-derived local projection mode + optional bool activation_gated = 25; // T4: true if activation-gated from serving + optional string activation_gate_reason = 26; // T4: reason for activation gate } message BlockVolumeShortInfoMessage { diff --git a/weed/pb/master_pb/master.pb.go b/weed/pb/master_pb/master.pb.go index d322a2179..62028cdef 100644 --- a/weed/pb/master_pb/master.pb.go +++ b/weed/pb/master_pb/master.pb.go @@ -3925,6 +3925,8 @@ type BlockVolumeInfoMessage struct { VolumeMode *string `protobuf:"bytes,22,opt,name=volume_mode,json=volumeMode,proto3,oneof" json:"volume_mode,omitempty"` VolumeModeReason *string `protobuf:"bytes,23,opt,name=volume_mode_reason,json=volumeModeReason,proto3,oneof" json:"volume_mode_reason,omitempty"` EngineProjectionMode *string `protobuf:"bytes,24,opt,name=engine_projection_mode,json=engineProjectionMode,proto3,oneof" json:"engine_projection_mode,omitempty"` + ActivationGated *bool `protobuf:"varint,25,opt,name=activation_gated,json=activationGated,proto3,oneof" json:"activation_gated,omitempty"` + ActivationGateReason *string `protobuf:"bytes,26,opt,name=activation_gate_reason,json=activationGateReason,proto3,oneof" json:"activation_gate_reason,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -4127,6 +4129,20 @@ func (x *BlockVolumeInfoMessage) GetEngineProjectionMode() string { return "" } +func (x *BlockVolumeInfoMessage) GetActivationGated() bool { + if x != nil && x.ActivationGated != nil { + return *x.ActivationGated + } + return false +} + +func (x *BlockVolumeInfoMessage) GetActivationGateReason() string { + if x != nil && x.ActivationGateReason != nil { + return *x.ActivationGateReason + } + return "" +} + type BlockVolumeShortInfoMessage struct { state protoimpl.MessageState `protogen:"open.v1"` Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` diff --git a/weed/server/volume_server_block.go b/weed/server/volume_server_block.go index 37366964b..344bd21f6 100644 --- a/weed/server/volume_server_block.go +++ b/weed/server/volume_server_block.go @@ -581,26 +581,74 @@ func (bs *BlockService) applyCoreAssignmentEvent(a blockvol.BlockVolumeAssignmen // and gates or clears activation accordingly. This is the local enforcement // point for T4 — the promoted node decides locally whether reconstruction // quality allows serving. +// +// Enforcement: when gated, the volume is disconnected from the iSCSI target +// (active sessions terminated, volume removed). When ungated, the volume is +// re-registered with the iSCSI target. func (bs *BlockService) evaluateActivationGate(path string) { proj, ok := bs.CoreProjection(path) if !ok { return // no V2 core, no gate enforcement } bs.activationGateMu.Lock() - defer bs.activationGateMu.Unlock() + _, wasGated := bs.activationGated[path] switch proj.Mode.Name { case "publish_healthy", "replica_ready": delete(bs.activationGated, path) + bs.activationGateMu.Unlock() + // Ungate: re-register with iSCSI target if transitioning from gated. + if wasGated { + bs.ungateServing(path) + } default: reason := fmt.Sprintf("engine_projection_mode=%s", proj.Mode.Name) if proj.Mode.Reason != "" { reason += ": " + proj.Mode.Reason } bs.activationGated[path] = reason - glog.V(0).Infof("activation gated: %s — %s", path, reason) + bs.activationGateMu.Unlock() + // Gate: disconnect from iSCSI target if not already gated. + if !wasGated { + bs.gateServing(path, reason) + } } } +// gateServing disconnects a volume from the iSCSI target, terminating active +// sessions and preventing new connections. This is the actual serving +// enforcement for T4 — not just bookkeeping. +func (bs *BlockService) gateServing(path, reason string) { + glog.V(0).Infof("activation gated: %s — %s (disconnecting from iSCSI)", path, reason) + if bs.targetServer != nil { + name := volumeNameFromPath(path) + iqn := bs.iqnPrefix + blockvol.SanitizeIQN(name) + bs.targetServer.DisconnectVolume(iqn) + } +} + +// ungateServing re-registers a volume with the iSCSI target after the gate +// clears (projection reaches a serving-allowed state). +func (bs *BlockService) ungateServing(path string) { + glog.V(0).Infof("activation gate cleared: %s (re-registering with iSCSI)", path) + if bs.targetServer == nil { + return + } + vol, ok := bs.blockStore.GetBlockVolume(path) + if !ok || vol == nil { + return + } + name := volumeNameFromPath(path) + iqn := bs.iqnPrefix + blockvol.SanitizeIQN(name) + adapter := blockvol.NewBlockVolAdapter(vol) + bs.targetServer.AddVolume(iqn, adapter) +} + +// volumeNameFromPath extracts the volume name from a .blk file path. +func volumeNameFromPath(path string) string { + name := filepath.Base(path) + return strings.TrimSuffix(name, ".blk") +} + // IsActivationGated returns whether a volume is currently gated from serving // and the reason. Used by iSCSI/NVMe adapter and heartbeat surface. func (bs *BlockService) IsActivationGated(path string) (bool, string) { @@ -630,6 +678,11 @@ func (bs *BlockService) applyCoreEvent(ev engine.Event) { } result := bs.coreApplyAndLog(ev) bs.applyCoreCommands(result.Commands) + // T4: re-evaluate activation gate after every core event. This is the + // runtime recovery path — when recovery/catch-up completes and the + // projection transitions to a serving-allowed state, the gate clears + // and the volume is re-registered with the iSCSI target. + bs.evaluateActivationGate(ev.VolumeID()) } // coreApplyAndLog applies an event to the V2 core and logs the transition. diff --git a/weed/storage/blockvol/block_heartbeat_proto.go b/weed/storage/blockvol/block_heartbeat_proto.go index 046387ab8..d2ff8c6a7 100644 --- a/weed/storage/blockvol/block_heartbeat_proto.go +++ b/weed/storage/blockvol/block_heartbeat_proto.go @@ -31,6 +31,8 @@ func InfoMessageToProto(m BlockVolumeInfoMessage) *master_pb.BlockVolumeInfoMess VolumeMode: optionalStringPtr(m.VolumeMode), VolumeModeReason: optionalStringPtr(m.VolumeModeReason), EngineProjectionMode: optionalStringPtr(m.EngineProjectionMode), + ActivationGated: &m.ActivationGated, + ActivationGateReason: optionalStringPtr(m.ActivationGateReason), HealthScore: m.HealthScore, ScrubErrors: m.ScrubErrors, LastScrubTime: m.LastScrubTime, @@ -64,6 +66,8 @@ func InfoMessageFromProto(p *master_pb.BlockVolumeInfoMessage) BlockVolumeInfoMe VolumeMode: p.GetVolumeMode(), VolumeModeReason: p.GetVolumeModeReason(), EngineProjectionMode: p.GetEngineProjectionMode(), + ActivationGated: p.GetActivationGated(), + ActivationGateReason: p.GetActivationGateReason(), HealthScore: p.HealthScore, ScrubErrors: p.ScrubErrors, LastScrubTime: p.LastScrubTime,