mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-03 06:37:09 +00:00
pingqiuandClaude Opus 4.6
d2d57851b0
Rebuild session protocol implementation for v2-rebuild-mvp-session-protocol.md. New files: - rebuild_bitmap.go: RebuildBitmap — session-scoped dense bitset for WAL-applied LBA tracking. MarkApplied on local WAL write (not receive). ShouldApplyBase returns false for WAL-covered LBAs (WAL always wins). - rebuild_session.go: RebuildSession — replica-side two-line rebuild. WAL lane (ApplyWALEntry) + base lane (ApplyBaseBlock) with bitmap conflict resolution. TryComplete requires BOTH base_complete AND wal_applied_lsn >= target_lsn. Volume-level control surface: StartRebuildSession, ApplyRebuildSessionWALEntry/BaseBlock, MarkRebuildSessionBaseComplete, TryCompleteRebuildSession, CancelRebuildSession, ActiveRebuildSession. - rebuild_mvp_test.go: 4 correctness tests — base+WAL converge, WAL-applied never overwritten by base, bitmap set on applied not received, control surface start/supersede/complete. - rebuild_transport_test.go: 2 transport-level tests — two-line with real WAL shipping, live writes during base copy with bitmap conflict. Design docs: - v2-rebuild-mvp-session-protocol.md: MVP spec with message set, apply rules, completion/failure/crash rules, test matrix - v2-sync-recovery-protocol.md: full protocol context (keepup/catchup/ rebuild unified design, primary decision logic, two-line model) - v2-session-protocol-shape.md: protocol shape overview Protocol engine (reference, not production): - sw-block/protocol/: 7-event engine with ~300 lines, 13 tests 6 rebuild tests pass, all existing component tests pass. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
520 lines
16 KiB
Go
520 lines
16 KiB
Go
package replication
|
|
|
|
import "testing"
|
|
|
|
func TestPhase14_SyncAckObserved_AckUpdatesViewAndDurability(t *testing.T) {
|
|
core := NewCoreEngine()
|
|
|
|
core.ApplyEvent(AssignmentDelivered{
|
|
ID: "vol-sync-ack",
|
|
Epoch: 1,
|
|
Role: RolePrimary,
|
|
RecoveryTarget: SessionCatchUp,
|
|
Replicas: []ReplicaAssignment{
|
|
{ReplicaID: "replica-1", Endpoint: Endpoint{DataAddr: "10.0.0.50:9333", CtrlAddr: "10.0.0.50:9334", Version: 1}},
|
|
},
|
|
})
|
|
core.ApplyEvent(RoleApplied{ID: "vol-sync-ack"})
|
|
core.ApplyEvent(ShipperConfiguredObserved{ID: "vol-sync-ack"})
|
|
core.ApplyEvent(ShipperConnectedObserved{ID: "vol-sync-ack"})
|
|
|
|
result := core.ApplyEvent(SyncAckObserved{
|
|
ID: "vol-sync-ack",
|
|
ReplicaID: "replica-1",
|
|
AckKind: SyncAckQuorum,
|
|
TargetLSN: 120,
|
|
DurableLSN: 120,
|
|
AppliedLSN: 120,
|
|
})
|
|
|
|
if result.Projection.Sync.AckKind != SyncAckQuorum {
|
|
t.Fatalf("sync_ack_kind=%s", result.Projection.Sync.AckKind)
|
|
}
|
|
if result.Projection.Sync.Action != SyncActionKeepUp {
|
|
t.Fatalf("sync_action=%s", result.Projection.Sync.Action)
|
|
}
|
|
if result.Projection.Boundary.DurableLSN != 120 {
|
|
t.Fatalf("durable_lsn=%d", result.Projection.Boundary.DurableLSN)
|
|
}
|
|
if result.Projection.Mode.Name != ModePublishHealthy {
|
|
t.Fatalf("mode=%s", result.Projection.Mode.Name)
|
|
}
|
|
if !result.Projection.Publication.Healthy {
|
|
t.Fatal("ack should establish healthy publication on ready primary")
|
|
}
|
|
}
|
|
|
|
func TestPhase14_SyncAckObserved_TimedOutFactsStartCatchUp(t *testing.T) {
|
|
core := NewCoreEngine()
|
|
|
|
core.ApplyEvent(AssignmentDelivered{
|
|
ID: "vol-sync-catchup",
|
|
Epoch: 1,
|
|
Role: RolePrimary,
|
|
RecoveryTarget: SessionCatchUp,
|
|
Replicas: []ReplicaAssignment{
|
|
{ReplicaID: "replica-1", Endpoint: Endpoint{DataAddr: "10.0.0.51:9333", CtrlAddr: "10.0.0.51:9334", Version: 1}},
|
|
},
|
|
})
|
|
core.ApplyEvent(RoleApplied{ID: "vol-sync-catchup"})
|
|
core.ApplyEvent(ShipperConfiguredObserved{ID: "vol-sync-catchup"})
|
|
core.ApplyEvent(ShipperConnectedObserved{ID: "vol-sync-catchup"})
|
|
|
|
result := core.ApplyEvent(SyncAckObserved{
|
|
ID: "vol-sync-catchup",
|
|
ReplicaID: "replica-1",
|
|
AckKind: SyncAckTimedOut,
|
|
TargetLSN: 1000,
|
|
PrimaryTailLSN: 700,
|
|
DurableLSN: 500,
|
|
AppliedLSN: 900,
|
|
Reason: "gap_within_retention",
|
|
})
|
|
|
|
assertCommandNames(t, result.Commands, []string{
|
|
"start_catchup",
|
|
"publish_projection",
|
|
})
|
|
if result.State.Recovery.Phase != RecoveryCatchingUp {
|
|
t.Fatalf("recovery_phase=%s", result.State.Recovery.Phase)
|
|
}
|
|
if result.Projection.Recovery.TargetLSN != 1000 {
|
|
t.Fatalf("target_lsn=%d", result.Projection.Recovery.TargetLSN)
|
|
}
|
|
if result.Projection.Recovery.AchievedLSN != 900 {
|
|
t.Fatalf("achieved_lsn=%d", result.Projection.Recovery.AchievedLSN)
|
|
}
|
|
if result.Projection.Sync.AckKind != SyncAckTimedOut {
|
|
t.Fatalf("sync_ack_kind=%s", result.Projection.Sync.AckKind)
|
|
}
|
|
if result.Projection.Sync.Action != SyncActionCatchUp {
|
|
t.Fatalf("sync_action=%s", result.Projection.Sync.Action)
|
|
}
|
|
if result.Projection.Mode.Name != ModeBootstrapPending {
|
|
t.Fatalf("mode=%s", result.Projection.Mode.Name)
|
|
}
|
|
if result.Projection.Publication.Healthy {
|
|
t.Fatal("recoverable catch-up should not overclaim healthy publication")
|
|
}
|
|
}
|
|
|
|
func TestPhase14_SyncAckObserved_TimedOutStillPreservesCatchUpContract(t *testing.T) {
|
|
core := NewCoreEngine()
|
|
|
|
core.ApplyEvent(AssignmentDelivered{
|
|
ID: "vol-sync-timeout",
|
|
Epoch: 1,
|
|
Role: RolePrimary,
|
|
RecoveryTarget: SessionCatchUp,
|
|
Replicas: []ReplicaAssignment{
|
|
{ReplicaID: "replica-1", Endpoint: Endpoint{DataAddr: "10.0.0.52:9333", CtrlAddr: "10.0.0.52:9334", Version: 1}},
|
|
},
|
|
})
|
|
core.ApplyEvent(RoleApplied{ID: "vol-sync-timeout"})
|
|
core.ApplyEvent(ShipperConfiguredObserved{ID: "vol-sync-timeout"})
|
|
core.ApplyEvent(ShipperConnectedObserved{ID: "vol-sync-timeout"})
|
|
|
|
result := core.ApplyEvent(SyncAckObserved{
|
|
ID: "vol-sync-timeout",
|
|
ReplicaID: "replica-1",
|
|
AckKind: SyncAckTimedOut,
|
|
TargetLSN: 1000,
|
|
PrimaryTailLSN: 700,
|
|
DurableLSN: 500,
|
|
AppliedLSN: 880,
|
|
Reason: "deadline_exceeded",
|
|
})
|
|
|
|
assertCommandNames(t, result.Commands, []string{
|
|
"start_catchup",
|
|
"publish_projection",
|
|
})
|
|
if result.Projection.Mode.Name != ModeBootstrapPending {
|
|
t.Fatalf("mode=%s", result.Projection.Mode.Name)
|
|
}
|
|
if result.Projection.Recovery.Phase != RecoveryCatchingUp {
|
|
t.Fatalf("recovery_phase=%s", result.Projection.Recovery.Phase)
|
|
}
|
|
if result.Projection.Recovery.AchievedLSN != 880 {
|
|
t.Fatalf("achieved_lsn=%d", result.Projection.Recovery.AchievedLSN)
|
|
}
|
|
if result.Projection.Boundary.LastBarrierReason != "deadline_exceeded" {
|
|
t.Fatalf("last_barrier_reason=%q", result.Projection.Boundary.LastBarrierReason)
|
|
}
|
|
}
|
|
|
|
func TestPhase14_SyncAckObserved_TransportLostEscalatesTargetedReplica(t *testing.T) {
|
|
core := NewCoreEngine()
|
|
|
|
core.ApplyEvent(AssignmentDelivered{
|
|
ID: "vol-sync-rebuild",
|
|
Epoch: 1,
|
|
Role: RolePrimary,
|
|
RecoveryTarget: SessionCatchUp,
|
|
Replicas: []ReplicaAssignment{
|
|
{ReplicaID: "replica-1", Endpoint: Endpoint{DataAddr: "10.0.0.53:9333", CtrlAddr: "10.0.0.53:9334", Version: 1}},
|
|
{ReplicaID: "replica-2", Endpoint: Endpoint{DataAddr: "10.0.0.54:9333", CtrlAddr: "10.0.0.54:9334", Version: 1}},
|
|
},
|
|
})
|
|
|
|
result := core.ApplyEvent(SyncAckObserved{
|
|
ID: "vol-sync-rebuild",
|
|
ReplicaID: "replica-2",
|
|
AckKind: SyncAckTransportLost,
|
|
TargetLSN: 1200,
|
|
DurableLSN: 512,
|
|
AppliedLSN: 900,
|
|
Reason: "recoverability_lost",
|
|
})
|
|
|
|
assertCommandNames(t, result.Commands, []string{
|
|
"invalidate_session",
|
|
"publish_projection",
|
|
})
|
|
invalidate, ok := result.Commands[0].(InvalidateSessionCommand)
|
|
if !ok {
|
|
t.Fatalf("cmd0=%T", result.Commands[0])
|
|
}
|
|
if invalidate.ReplicaID != "replica-2" {
|
|
t.Fatalf("invalidate_replica=%q", invalidate.ReplicaID)
|
|
}
|
|
if result.Projection.Mode.Name != ModeNeedsRebuild {
|
|
t.Fatalf("mode=%s", result.Projection.Mode.Name)
|
|
}
|
|
if result.Projection.Recovery.Phase != RecoveryNeedsRebuild {
|
|
t.Fatalf("recovery_phase=%s", result.Projection.Recovery.Phase)
|
|
}
|
|
if result.Projection.Sync.Action != SyncActionRebuild {
|
|
t.Fatalf("sync_action=%s", result.Projection.Sync.Action)
|
|
}
|
|
}
|
|
|
|
func TestPhase14_SyncAckObserved_TracksPerReplicaSyncFacts(t *testing.T) {
|
|
core := NewCoreEngine()
|
|
|
|
core.ApplyEvent(AssignmentDelivered{
|
|
ID: "vol-sync-multi",
|
|
Epoch: 1,
|
|
Role: RolePrimary,
|
|
RecoveryTarget: SessionCatchUp,
|
|
Replicas: []ReplicaAssignment{
|
|
{ReplicaID: "replica-1", Endpoint: Endpoint{DataAddr: "10.0.0.55:9333", CtrlAddr: "10.0.0.55:9334", Version: 1}},
|
|
{ReplicaID: "replica-2", Endpoint: Endpoint{DataAddr: "10.0.0.56:9333", CtrlAddr: "10.0.0.56:9334", Version: 1}},
|
|
},
|
|
})
|
|
core.ApplyEvent(RoleApplied{ID: "vol-sync-multi"})
|
|
core.ApplyEvent(ShipperConfiguredObserved{ID: "vol-sync-multi"})
|
|
core.ApplyEvent(ShipperConnectedObserved{ID: "vol-sync-multi"})
|
|
|
|
core.ApplyEvent(SyncAckObserved{
|
|
ID: "vol-sync-multi",
|
|
ReplicaID: "replica-1",
|
|
AckKind: SyncAckTimedOut,
|
|
TargetLSN: 1000,
|
|
PrimaryTailLSN: 700,
|
|
DurableLSN: 400,
|
|
AppliedLSN: 850,
|
|
Reason: "deadline_exceeded",
|
|
})
|
|
result := core.ApplyEvent(SyncAckObserved{
|
|
ID: "vol-sync-multi",
|
|
ReplicaID: "replica-2",
|
|
AckKind: SyncAckQuorum,
|
|
TargetLSN: 1000,
|
|
DurableLSN: 1000,
|
|
AppliedLSN: 1000,
|
|
})
|
|
|
|
if len(result.Projection.ReplicaSync) != 2 {
|
|
t.Fatalf("replica_sync_len=%d", len(result.Projection.ReplicaSync))
|
|
}
|
|
if got := result.Projection.ReplicaSync["replica-1"].AckKind; got != SyncAckTimedOut {
|
|
t.Fatalf("replica1_ack_kind=%s", got)
|
|
}
|
|
if got := result.Projection.ReplicaSync["replica-1"].Action; got != SyncActionCatchUp {
|
|
t.Fatalf("replica1_action=%s", got)
|
|
}
|
|
if got := result.Projection.ReplicaSync["replica-2"].AckKind; got != SyncAckQuorum {
|
|
t.Fatalf("replica2_ack_kind=%s", got)
|
|
}
|
|
if got := result.Projection.ReplicaSync["replica-2"].Action; got != SyncActionKeepUp {
|
|
t.Fatalf("replica2_action=%s", got)
|
|
}
|
|
}
|
|
|
|
func TestPhase14_SyncAckObserved_ReplicaAckDoesNotClearOtherReplicaRebuild(t *testing.T) {
|
|
core := NewCoreEngine()
|
|
|
|
core.ApplyEvent(AssignmentDelivered{
|
|
ID: "vol-sync-rebuild-aggregate",
|
|
Epoch: 1,
|
|
Role: RolePrimary,
|
|
RecoveryTarget: SessionCatchUp,
|
|
Replicas: []ReplicaAssignment{
|
|
{ReplicaID: "replica-1", Endpoint: Endpoint{DataAddr: "10.0.0.61:9333", CtrlAddr: "10.0.0.61:9334", Version: 1}},
|
|
{ReplicaID: "replica-2", Endpoint: Endpoint{DataAddr: "10.0.0.62:9333", CtrlAddr: "10.0.0.62:9334", Version: 1}},
|
|
},
|
|
})
|
|
|
|
core.ApplyEvent(SyncAckObserved{
|
|
ID: "vol-sync-rebuild-aggregate",
|
|
ReplicaID: "replica-1",
|
|
AckKind: SyncAckTimedOut,
|
|
TargetLSN: 1000,
|
|
PrimaryTailLSN: 700,
|
|
DurableLSN: 500,
|
|
AppliedLSN: 600,
|
|
Reason: "gap_beyond_retention",
|
|
})
|
|
|
|
result := core.ApplyEvent(SyncAckObserved{
|
|
ID: "vol-sync-rebuild-aggregate",
|
|
ReplicaID: "replica-2",
|
|
AckKind: SyncAckQuorum,
|
|
TargetLSN: 1000,
|
|
DurableLSN: 1000,
|
|
AppliedLSN: 1000,
|
|
})
|
|
|
|
if result.Projection.Mode.Name != ModeNeedsRebuild {
|
|
t.Fatalf("mode=%s", result.Projection.Mode.Name)
|
|
}
|
|
if result.Projection.Recovery.Phase != RecoveryNeedsRebuild {
|
|
t.Fatalf("recovery_phase=%s", result.Projection.Recovery.Phase)
|
|
}
|
|
if got := result.Projection.ReplicaSync["replica-1"].Action; got != SyncActionRebuild {
|
|
t.Fatalf("replica1_action=%s", got)
|
|
}
|
|
if got := result.Projection.ReplicaSync["replica-2"].Action; got != SyncActionKeepUp {
|
|
t.Fatalf("replica2_action=%s", got)
|
|
}
|
|
}
|
|
|
|
func TestPhase14_SyncAckObserved_AssignmentChangeClearsReplicaSyncFacts(t *testing.T) {
|
|
core := NewCoreEngine()
|
|
|
|
core.ApplyEvent(AssignmentDelivered{
|
|
ID: "vol-sync-reset",
|
|
Epoch: 1,
|
|
Role: RolePrimary,
|
|
RecoveryTarget: SessionCatchUp,
|
|
Replicas: []ReplicaAssignment{
|
|
{ReplicaID: "replica-1", Endpoint: Endpoint{DataAddr: "10.0.0.57:9333", CtrlAddr: "10.0.0.57:9334", Version: 1}},
|
|
},
|
|
})
|
|
core.ApplyEvent(SyncAckObserved{
|
|
ID: "vol-sync-reset",
|
|
ReplicaID: "replica-1",
|
|
AckKind: SyncAckQuorum,
|
|
TargetLSN: 30,
|
|
DurableLSN: 30,
|
|
AppliedLSN: 30,
|
|
})
|
|
|
|
result := core.ApplyEvent(AssignmentDelivered{
|
|
ID: "vol-sync-reset",
|
|
Epoch: 2,
|
|
Role: RolePrimary,
|
|
RecoveryTarget: SessionCatchUp,
|
|
Replicas: []ReplicaAssignment{
|
|
{ReplicaID: "replica-1", Endpoint: Endpoint{DataAddr: "10.0.0.58:9333", CtrlAddr: "10.0.0.58:9334", Version: 2}},
|
|
},
|
|
})
|
|
|
|
if result.Projection.Sync.AckKind != SyncAckUnknown {
|
|
t.Fatalf("sync_ack_kind=%s", result.Projection.Sync.AckKind)
|
|
}
|
|
if len(result.Projection.ReplicaSync) != 0 {
|
|
t.Fatalf("replica_sync_len=%d", len(result.Projection.ReplicaSync))
|
|
}
|
|
}
|
|
|
|
func TestPhase14_SyncAckObserved_PrimaryTailFactsDriveCatchUp(t *testing.T) {
|
|
core := NewCoreEngine()
|
|
|
|
core.ApplyEvent(AssignmentDelivered{
|
|
ID: "vol-sync-facts-catchup",
|
|
Epoch: 1,
|
|
Role: RolePrimary,
|
|
RecoveryTarget: SessionCatchUp,
|
|
Replicas: []ReplicaAssignment{
|
|
{ReplicaID: "replica-1", Endpoint: Endpoint{DataAddr: "10.0.0.59:9333", CtrlAddr: "10.0.0.59:9334", Version: 1}},
|
|
},
|
|
})
|
|
core.ApplyEvent(RoleApplied{ID: "vol-sync-facts-catchup"})
|
|
core.ApplyEvent(ShipperConfiguredObserved{ID: "vol-sync-facts-catchup"})
|
|
core.ApplyEvent(ShipperConnectedObserved{ID: "vol-sync-facts-catchup"})
|
|
|
|
result := core.ApplyEvent(SyncAckObserved{
|
|
ID: "vol-sync-facts-catchup",
|
|
ReplicaID: "replica-1",
|
|
AckKind: SyncAckTimedOut,
|
|
TargetLSN: 1000,
|
|
PrimaryTailLSN: 700,
|
|
DurableLSN: 500,
|
|
AppliedLSN: 900,
|
|
Reason: "gap_within_retention",
|
|
})
|
|
|
|
assertCommandNames(t, result.Commands, []string{
|
|
"start_catchup",
|
|
"publish_projection",
|
|
})
|
|
if result.Projection.Sync.Action != SyncActionCatchUp {
|
|
t.Fatalf("sync_action=%s", result.Projection.Sync.Action)
|
|
}
|
|
if result.Projection.Recovery.Phase != RecoveryCatchingUp {
|
|
t.Fatalf("recovery_phase=%s", result.Projection.Recovery.Phase)
|
|
}
|
|
}
|
|
|
|
func TestPhase14_SyncAckObserved_PrimaryTailFactsDriveRebuild(t *testing.T) {
|
|
core := NewCoreEngine()
|
|
|
|
core.ApplyEvent(AssignmentDelivered{
|
|
ID: "vol-sync-facts-rebuild",
|
|
Epoch: 1,
|
|
Role: RolePrimary,
|
|
RecoveryTarget: SessionCatchUp,
|
|
Replicas: []ReplicaAssignment{
|
|
{ReplicaID: "replica-1", Endpoint: Endpoint{DataAddr: "10.0.0.60:9333", CtrlAddr: "10.0.0.60:9334", Version: 1}},
|
|
},
|
|
})
|
|
|
|
result := core.ApplyEvent(SyncAckObserved{
|
|
ID: "vol-sync-facts-rebuild",
|
|
ReplicaID: "replica-1",
|
|
AckKind: SyncAckTimedOut,
|
|
TargetLSN: 1000,
|
|
PrimaryTailLSN: 700,
|
|
DurableLSN: 500,
|
|
AppliedLSN: 600,
|
|
Reason: "gap_beyond_retention",
|
|
})
|
|
|
|
assertCommandNames(t, result.Commands, []string{
|
|
"invalidate_session",
|
|
"publish_projection",
|
|
})
|
|
if result.Projection.Sync.Action != SyncActionRebuild {
|
|
t.Fatalf("sync_action=%s", result.Projection.Sync.Action)
|
|
}
|
|
if result.Projection.Mode.Name != ModeNeedsRebuild {
|
|
t.Fatalf("mode=%s", result.Projection.Mode.Name)
|
|
}
|
|
}
|
|
|
|
func TestPhase14_SessionStarted_CatchUpUsesUnifiedPath(t *testing.T) {
|
|
core := NewCoreEngine()
|
|
|
|
core.ApplyEvent(AssignmentDelivered{
|
|
ID: "vol-session-catchup",
|
|
Epoch: 1,
|
|
Role: RolePrimary,
|
|
RecoveryTarget: SessionCatchUp,
|
|
Replicas: []ReplicaAssignment{
|
|
{ReplicaID: "replica-1", Endpoint: Endpoint{DataAddr: "10.0.0.63:9333", CtrlAddr: "10.0.0.63:9334", Version: 1}},
|
|
},
|
|
})
|
|
|
|
result := core.ApplyEvent(SessionStarted{
|
|
ID: "vol-session-catchup",
|
|
ReplicaID: "replica-1",
|
|
Kind: SessionCatchUp,
|
|
TargetLSN: 77,
|
|
})
|
|
|
|
assertCommandNames(t, result.Commands, []string{
|
|
"start_catchup",
|
|
"publish_projection",
|
|
})
|
|
if result.Projection.Recovery.Phase != RecoveryCatchingUp {
|
|
t.Fatalf("recovery_phase=%s", result.Projection.Recovery.Phase)
|
|
}
|
|
if result.Projection.Recovery.TargetLSN != 77 {
|
|
t.Fatalf("target_lsn=%d", result.Projection.Recovery.TargetLSN)
|
|
}
|
|
}
|
|
|
|
func TestPhase14_SessionCompleted_RebuildClearsNeedsRebuild(t *testing.T) {
|
|
core := NewCoreEngine()
|
|
|
|
core.ApplyEvent(AssignmentDelivered{
|
|
ID: "vol-session-rebuild",
|
|
Epoch: 1,
|
|
Role: RolePrimary,
|
|
RecoveryTarget: SessionRebuild,
|
|
Replicas: []ReplicaAssignment{
|
|
{ReplicaID: "replica-1", Endpoint: Endpoint{DataAddr: "10.0.0.64:9333", CtrlAddr: "10.0.0.64:9334", Version: 1}},
|
|
},
|
|
})
|
|
core.ApplyEvent(NeedsRebuildObserved{
|
|
ID: "vol-session-rebuild",
|
|
ReplicaID: "replica-1",
|
|
Reason: "gap_too_large",
|
|
})
|
|
core.ApplyEvent(SessionStarted{
|
|
ID: "vol-session-rebuild",
|
|
ReplicaID: "replica-1",
|
|
Kind: SessionRebuild,
|
|
TargetLSN: 120,
|
|
})
|
|
|
|
result := core.ApplyEvent(SessionCompleted{
|
|
ID: "vol-session-rebuild",
|
|
ReplicaID: "replica-1",
|
|
Kind: SessionRebuild,
|
|
AchievedLSN: 120,
|
|
FlushedLSN: 120,
|
|
CheckpointLSN: 120,
|
|
})
|
|
|
|
if result.Projection.Recovery.Phase != RecoveryIdle {
|
|
t.Fatalf("recovery_phase=%s", result.Projection.Recovery.Phase)
|
|
}
|
|
if result.Projection.Mode.Name == ModeNeedsRebuild {
|
|
t.Fatalf("mode=%s", result.Projection.Mode.Name)
|
|
}
|
|
if result.Projection.Boundary.DurableLSN != 120 {
|
|
t.Fatalf("durable_lsn=%d", result.Projection.Boundary.DurableLSN)
|
|
}
|
|
}
|
|
|
|
func TestPhase14_SessionFailed_CatchUpFallsBackToDegraded(t *testing.T) {
|
|
core := NewCoreEngine()
|
|
|
|
core.ApplyEvent(AssignmentDelivered{
|
|
ID: "vol-session-failed",
|
|
Epoch: 1,
|
|
Role: RolePrimary,
|
|
RecoveryTarget: SessionCatchUp,
|
|
Replicas: []ReplicaAssignment{
|
|
{ReplicaID: "replica-1", Endpoint: Endpoint{DataAddr: "10.0.0.65:9333", CtrlAddr: "10.0.0.65:9334", Version: 1}},
|
|
},
|
|
})
|
|
core.ApplyEvent(SessionStarted{
|
|
ID: "vol-session-failed",
|
|
ReplicaID: "replica-1",
|
|
Kind: SessionCatchUp,
|
|
TargetLSN: 80,
|
|
})
|
|
|
|
result := core.ApplyEvent(SessionFailed{
|
|
ID: "vol-session-failed",
|
|
ReplicaID: "replica-1",
|
|
Kind: SessionCatchUp,
|
|
Reason: "transport_lost",
|
|
})
|
|
|
|
assertCommandNames(t, result.Commands, []string{
|
|
"invalidate_session",
|
|
"publish_projection",
|
|
})
|
|
if result.Projection.Mode.Name != ModeDegraded {
|
|
t.Fatalf("mode=%s", result.Projection.Mode.Name)
|
|
}
|
|
if result.Projection.Boundary.LastBarrierReason != "transport_lost" {
|
|
t.Fatalf("last_barrier_reason=%q", result.Projection.Boundary.LastBarrierReason)
|
|
}
|
|
}
|