mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-08-21 22:56:55 +00:00
feat: Phase 20 T4 — local activation gate on promoted primary
After assignment executes through V2 core, evaluateActivationGate() checks the resulting projection locally. If mode is degraded, needs_rebuild, bootstrap_pending, or allocated_only, the volume is gated from serving. Gate is enforced immediately after assignment, before the next heartbeat round-trip. Gate cleared only when projection reaches publish_healthy or replica_ready. IsActivationGated() provides the query surface for iSCSI/NVMe adapter enforcement. Heartbeat carries ActivationGated and ActivationGateReason fields so master can observe the gated state (report path, not enforcement path). activationGated map on BlockService tracks per-volume gate state. Initialized in constructor. Test helper updated to include it. 6 tests: degraded gates, needs_rebuild gates, healthy clears gate, gate enforced before heartbeat, recovery re-enables, assignment with degraded projection triggers gate. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
f825f08680
commit
a27569358b
@@ -0,0 +1,181 @@
|
||||
package weed_server
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
engine "github.com/seaweedfs/seaweedfs/sw-block/engine/replication"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
|
||||
)
|
||||
|
||||
func TestT4_DegradedProjection_GatesActivation(t *testing.T) {
|
||||
bs := newTestBlockServiceDirect(t)
|
||||
path := createTestVolDirect(t, bs, "gate-degraded")
|
||||
|
||||
// Inject degraded projection.
|
||||
bs.coreProjMu.Lock()
|
||||
bs.coreProj[path] = engine.PublicationProjection{
|
||||
Mode: engine.ModeView{Name: engine.ModeDegraded, Reason: "barrier_timeout"},
|
||||
}
|
||||
bs.coreProjMu.Unlock()
|
||||
|
||||
bs.evaluateActivationGate(path)
|
||||
|
||||
gated, reason := bs.IsActivationGated(path)
|
||||
if !gated {
|
||||
t.Fatal("expected activation gated for degraded projection")
|
||||
}
|
||||
if reason == "" {
|
||||
t.Fatal("expected non-empty gate reason")
|
||||
}
|
||||
}
|
||||
|
||||
func TestT4_NeedsRebuildProjection_GatesActivation(t *testing.T) {
|
||||
bs := newTestBlockServiceDirect(t)
|
||||
path := createTestVolDirect(t, bs, "gate-rebuild")
|
||||
|
||||
bs.coreProjMu.Lock()
|
||||
bs.coreProj[path] = engine.PublicationProjection{
|
||||
Mode: engine.ModeView{Name: engine.ModeNeedsRebuild, Reason: "gap_too_large"},
|
||||
}
|
||||
bs.coreProjMu.Unlock()
|
||||
|
||||
bs.evaluateActivationGate(path)
|
||||
|
||||
gated, reason := bs.IsActivationGated(path)
|
||||
if !gated {
|
||||
t.Fatal("expected activation gated for needs_rebuild projection")
|
||||
}
|
||||
if reason == "" {
|
||||
t.Fatal("expected non-empty gate reason")
|
||||
}
|
||||
}
|
||||
|
||||
func TestT4_HealthyProjection_ClearsGate(t *testing.T) {
|
||||
bs := newTestBlockServiceDirect(t)
|
||||
path := createTestVolDirect(t, bs, "gate-healthy")
|
||||
|
||||
// Start gated.
|
||||
bs.coreProjMu.Lock()
|
||||
bs.coreProj[path] = engine.PublicationProjection{
|
||||
Mode: engine.ModeView{Name: engine.ModeDegraded, Reason: "barrier_timeout"},
|
||||
}
|
||||
bs.coreProjMu.Unlock()
|
||||
bs.evaluateActivationGate(path)
|
||||
if gated, _ := bs.IsActivationGated(path); !gated {
|
||||
t.Fatal("expected gated initially")
|
||||
}
|
||||
|
||||
// Transition to healthy.
|
||||
bs.coreProjMu.Lock()
|
||||
bs.coreProj[path] = engine.PublicationProjection{
|
||||
Mode: engine.ModeView{Name: engine.ModePublishHealthy},
|
||||
}
|
||||
bs.coreProjMu.Unlock()
|
||||
bs.evaluateActivationGate(path)
|
||||
|
||||
if gated, _ := bs.IsActivationGated(path); gated {
|
||||
t.Fatal("expected gate cleared for publish_healthy projection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestT4_GateEnforcedBeforeHeartbeat(t *testing.T) {
|
||||
bs := newTestBlockServiceDirect(t)
|
||||
path := createTestVolDirect(t, bs, "gate-pre-hb")
|
||||
|
||||
// Inject degraded projection and evaluate gate.
|
||||
bs.coreProjMu.Lock()
|
||||
bs.coreProj[path] = engine.PublicationProjection{
|
||||
Mode: engine.ModeView{Name: engine.ModeDegraded, Reason: "test"},
|
||||
}
|
||||
bs.coreProjMu.Unlock()
|
||||
bs.evaluateActivationGate(path)
|
||||
|
||||
// Gate is set BEFORE any heartbeat. Verify the heartbeat carries
|
||||
// the gated state.
|
||||
msgs := bs.CollectBlockVolumeHeartbeat()
|
||||
var found *blockvol.BlockVolumeInfoMessage
|
||||
for i := range msgs {
|
||||
if msgs[i].Path == path {
|
||||
found = &msgs[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if found == nil {
|
||||
t.Fatal("heartbeat message not found for gated volume")
|
||||
}
|
||||
if !found.ActivationGated {
|
||||
t.Fatal("heartbeat should report ActivationGated=true")
|
||||
}
|
||||
if found.ActivationGateReason == "" {
|
||||
t.Fatal("heartbeat should report non-empty ActivationGateReason")
|
||||
}
|
||||
}
|
||||
|
||||
func TestT4_RecoveryFromGated_ReenablesServing(t *testing.T) {
|
||||
bs := newTestBlockServiceDirect(t)
|
||||
path := createTestVolDirect(t, bs, "gate-recovery")
|
||||
|
||||
// Start gated (needs_rebuild).
|
||||
bs.coreProjMu.Lock()
|
||||
bs.coreProj[path] = engine.PublicationProjection{
|
||||
Mode: engine.ModeView{Name: engine.ModeNeedsRebuild, Reason: "gap"},
|
||||
}
|
||||
bs.coreProjMu.Unlock()
|
||||
bs.evaluateActivationGate(path)
|
||||
if gated, _ := bs.IsActivationGated(path); !gated {
|
||||
t.Fatal("expected gated")
|
||||
}
|
||||
|
||||
// Simulate recovery: projection transitions to replica_ready.
|
||||
bs.coreProjMu.Lock()
|
||||
bs.coreProj[path] = engine.PublicationProjection{
|
||||
Mode: engine.ModeView{Name: engine.ModeReplicaReady},
|
||||
}
|
||||
bs.coreProjMu.Unlock()
|
||||
bs.evaluateActivationGate(path)
|
||||
if gated, _ := bs.IsActivationGated(path); gated {
|
||||
t.Fatal("expected gate cleared after recovery to replica_ready")
|
||||
}
|
||||
|
||||
// And then to publish_healthy.
|
||||
bs.coreProjMu.Lock()
|
||||
bs.coreProj[path] = engine.PublicationProjection{
|
||||
Mode: engine.ModeView{Name: engine.ModePublishHealthy},
|
||||
}
|
||||
bs.coreProjMu.Unlock()
|
||||
bs.evaluateActivationGate(path)
|
||||
if gated, _ := bs.IsActivationGated(path); gated {
|
||||
t.Fatal("expected gate cleared after recovery to publish_healthy")
|
||||
}
|
||||
}
|
||||
|
||||
func TestT4_ApplyCoreAssignment_GatesDegradedPrimary(t *testing.T) {
|
||||
bs := newTestBlockServiceDirect(t)
|
||||
path := createTestVolDirect(t, bs, "gate-assignment")
|
||||
|
||||
// Pre-inject degraded projection so that after assignment processing
|
||||
// the gate is evaluated.
|
||||
bs.coreProjMu.Lock()
|
||||
bs.coreProj[path] = engine.PublicationProjection{
|
||||
VolumeID: path,
|
||||
Mode: engine.ModeView{Name: engine.ModeDegraded, Reason: "incomplete_reconstruction"},
|
||||
}
|
||||
bs.coreProjMu.Unlock()
|
||||
|
||||
// Process primary assignment through the core path.
|
||||
bs.applyCoreAssignmentEvent(blockvol.BlockVolumeAssignment{
|
||||
Path: path,
|
||||
Epoch: 5,
|
||||
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
||||
LeaseTtlMs: 30000,
|
||||
})
|
||||
|
||||
// The gate should have been evaluated after assignment.
|
||||
gated, reason := bs.IsActivationGated(path)
|
||||
if !gated {
|
||||
t.Fatal("expected activation gated after primary assignment with degraded projection")
|
||||
}
|
||||
if reason == "" {
|
||||
t.Fatal("expected non-empty gate reason")
|
||||
}
|
||||
}
|
||||
@@ -84,6 +84,11 @@ type BlockService struct {
|
||||
coreExecMu sync.RWMutex
|
||||
coreExec map[string][]string
|
||||
|
||||
// T4: activation gate — promoted primaries that have not passed
|
||||
// reconstruction quality check are gated from serving.
|
||||
activationGateMu sync.RWMutex
|
||||
activationGated map[string]string // path → reason (non-empty = gated)
|
||||
|
||||
// P3: last-applied assignment per volume path for idempotence.
|
||||
lastAssignMu sync.RWMutex
|
||||
lastAssign map[string]lastAppliedAssignment
|
||||
@@ -234,6 +239,7 @@ func StartBlockService(listenAddr, blockDir, iqnPrefix, portalAddr string, nvmeC
|
||||
v2Core: engine.NewCoreEngine(),
|
||||
localServerID: listenAddr, // INTERIM: transport-shaped, see field doc
|
||||
coreProj: make(map[string]engine.PublicationProjection),
|
||||
activationGated: make(map[string]string),
|
||||
blockInventoryAuthoritative: false,
|
||||
}
|
||||
bs.v2Recovery = NewRecoveryManager(bs)
|
||||
@@ -560,7 +566,62 @@ func (bs *BlockService) applyCoreAssignmentEvent(a blockvol.BlockVolumeAssignmen
|
||||
return nil
|
||||
}
|
||||
result := bs.coreApplyAndLog(ev)
|
||||
return bs.applyCoreCommandsWithAssignment(result.Commands, &a)
|
||||
if err := bs.applyCoreCommandsWithAssignment(result.Commands, &a); err != nil {
|
||||
return err
|
||||
}
|
||||
// T4: After assignment execution, check the resulting V2 core projection
|
||||
// and gate activation locally if the mode indicates the node should not
|
||||
// serve. This is the enforcement point — it happens immediately after
|
||||
// assignment, before the next heartbeat round-trip.
|
||||
bs.evaluateActivationGate(a.Path)
|
||||
return nil
|
||||
}
|
||||
|
||||
// evaluateActivationGate checks the current V2 core projection for a volume
|
||||
// and gates or clears activation accordingly. This is the local enforcement
|
||||
// point for T4 — the promoted node decides locally whether reconstruction
|
||||
// quality allows serving.
|
||||
func (bs *BlockService) evaluateActivationGate(path string) {
|
||||
proj, ok := bs.CoreProjection(path)
|
||||
if !ok {
|
||||
return // no V2 core, no gate enforcement
|
||||
}
|
||||
bs.activationGateMu.Lock()
|
||||
defer bs.activationGateMu.Unlock()
|
||||
switch proj.Mode.Name {
|
||||
case "publish_healthy", "replica_ready":
|
||||
delete(bs.activationGated, path)
|
||||
default:
|
||||
reason := fmt.Sprintf("engine_projection_mode=%s", proj.Mode.Name)
|
||||
if proj.Mode.Reason != "" {
|
||||
reason += ": " + proj.Mode.Reason
|
||||
}
|
||||
bs.activationGated[path] = reason
|
||||
glog.V(0).Infof("activation gated: %s — %s", path, reason)
|
||||
}
|
||||
}
|
||||
|
||||
// IsActivationGated returns whether a volume is currently gated from serving
|
||||
// and the reason. Used by iSCSI/NVMe adapter and heartbeat surface.
|
||||
func (bs *BlockService) IsActivationGated(path string) (bool, string) {
|
||||
if bs == nil {
|
||||
return false, ""
|
||||
}
|
||||
bs.activationGateMu.RLock()
|
||||
defer bs.activationGateMu.RUnlock()
|
||||
reason, gated := bs.activationGated[path]
|
||||
return gated, reason
|
||||
}
|
||||
|
||||
// ClearActivationGate removes the activation gate for a volume. Called when
|
||||
// recovery completes and the projection reaches a serving-allowed state.
|
||||
func (bs *BlockService) ClearActivationGate(path string) {
|
||||
if bs == nil {
|
||||
return
|
||||
}
|
||||
bs.activationGateMu.Lock()
|
||||
defer bs.activationGateMu.Unlock()
|
||||
delete(bs.activationGated, path)
|
||||
}
|
||||
|
||||
func (bs *BlockService) applyCoreEvent(ev engine.Event) {
|
||||
@@ -1018,6 +1079,10 @@ func (bs *BlockService) CollectBlockVolumeHeartbeat() []blockvol.BlockVolumeInfo
|
||||
}
|
||||
msgs[i].ReplicaDegraded = bs.heartbeatReplicaDegraded(msgs[i].Path, msgs[i].ReplicaDegraded)
|
||||
msgs[i].EngineProjectionMode = bs.heartbeatEngineProjectionMode(msgs[i].Path)
|
||||
if gated, reason := bs.IsActivationGated(msgs[i].Path); gated {
|
||||
msgs[i].ActivationGated = true
|
||||
msgs[i].ActivationGateReason = reason
|
||||
}
|
||||
// NVMe publication: report nvme_addr and nqn if NVMe target is running.
|
||||
if bs.nvmeListenAddr != "" {
|
||||
msgs[i].NvmeAddr = bs.nvmeListenAddr
|
||||
|
||||
@@ -76,8 +76,9 @@ func newTestBlockServiceDirect(t *testing.T) *BlockService {
|
||||
iqnPrefix: "iqn.2024-01.com.seaweedfs:vol.",
|
||||
replStates: make(map[string]*volReplState),
|
||||
v2Core: engine.NewCoreEngine(),
|
||||
coreProj: make(map[string]engine.PublicationProjection),
|
||||
localServerID: "vs-test",
|
||||
coreProj: make(map[string]engine.PublicationProjection),
|
||||
activationGated: make(map[string]string),
|
||||
localServerID: "vs-test",
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -33,6 +33,8 @@ type BlockVolumeInfoMessage struct {
|
||||
NQN string // NVMe subsystem NQN, empty if NVMe disabled
|
||||
ReplicaShipperStates []ReplicaShipperStatus // CP13-7: per-replica state from primary's shipper group
|
||||
EngineProjectionMode string // T1: pure V2 engine-derived local projection mode (distinct from VolumeMode)
|
||||
ActivationGated bool // T4: true if this volume is gated from serving after promotion
|
||||
ActivationGateReason string // T4: reason for activation gate (empty if not gated)
|
||||
}
|
||||
|
||||
// BlockVolumeShortInfoMessage is used for delta heartbeats
|
||||
|
||||
Reference in New Issue
Block a user