seaweedfs/weed/storage/blockvol/v2bridge/execution_chain_test.go

package v2bridge

import (
	"testing"

	bridge "github.com/seaweedfs/seaweedfs/sw-block/bridge/blockvol"
	engine "github.com/seaweedfs/seaweedfs/sw-block/engine/replication"
)

// ============================================================
// Phase 08 P2: Integrated execution chain — ONE CHAIN proofs
//
// Each test proves a complete outcome through:
//   assignment → engine plan → engine executor → v2bridge → blockvol
//   → progress → complete → release
//
// NOT split proofs. NOT manual sender calls for the execution path.
// ============================================================

func setupChainTest(t *testing.T) (*engine.RecoveryDriver, *bridge.ControlAdapter, *Reader, *Executor, *Pinner) {
	t.Helper()
	vol := createTestVol(t)
	t.Cleanup(func() { vol.Close() })

	reader := NewReader(vol)
	pinner := NewPinner(vol)
	executor := NewExecutor(vol, "")

	sa := bridge.NewStorageAdapter(
		&readerShim{reader},
		&pinnerShim{pinner},
	)

	ca := bridge.NewControlAdapter()
	driver := engine.NewRecoveryDriver(sa)

	return driver, ca, reader, executor, pinner
}

func makeIntent(ca *bridge.ControlAdapter, epoch uint64, role string) engine.AssignmentIntent {
	return ca.ToAssignmentIntent(
		bridge.MasterAssignment{VolumeName: "vol1", Epoch: epoch, Role: "primary"},
		[]bridge.MasterAssignment{
			{VolumeName: "vol1", ReplicaServerID: "vs2", Role: role,
				DataAddr: "10.0.0.2:9333", CtrlAddr: "10.0.0.2:9334"},
		},
	)
}

// --- ONE CHAIN: Catch-up closure ---

func TestP2_CatchUpClosure_OneChain(t *testing.T) {
	driver, ca, reader, executor, pinner := setupChainTest(t)
	vol := reader.vol

	// Phase 1: Write initial entries + flush → advances checkpoint.
	for i := 0; i < 5; i++ {
		vol.WriteLBA(uint64(i), makeBlock(byte('A'+i)))
	}
	vol.ForceFlush()

	// Phase 2: Write MORE entries AFTER flush → these are above checkpoint, in WAL.
	for i := 5; i < 10; i++ {
		vol.WriteLBA(uint64(i), makeBlock(byte('A'+i)))
	}

	state := reader.ReadState()
	t.Logf("catch-up: head=%d tail=%d committed=%d checkpoint=%d",
		state.WALHeadLSN, state.WALTailLSN, state.CommittedLSN, state.CheckpointLSN)

	// Precondition: CommittedLSN > TailLSN (catch-up window exists).
	if state.CommittedLSN <= state.WALTailLSN {
		t.Fatalf("no catch-up window: committed=%d tail=%d", state.CommittedLSN, state.WALTailLSN)
	}

	// Step 1: assignment.
	driver.Orchestrator.ProcessAssignment(makeIntent(ca, 1, "replica"))

	// Step 2: plan — replica WITHIN the catch-up window (between tail and committed).
	replicaLSN := state.WALTailLSN + 1 // just above tail, within window

	plan, err := driver.PlanRecovery("vol1/vs2", replicaLSN)
	if err != nil {
		t.Fatalf("plan: %v", err)
	}

	t.Logf("catch-up: replica=%d outcome=%s", replicaLSN, plan.Outcome)

	if plan.Outcome == engine.OutcomeCatchUp {
		// Step 3: engine executor drives catch-up — wired to real v2bridge I/O.
		exec := engine.NewCatchUpExecutor(driver, plan)
		exec.IO = executor // v2bridge.Executor implements CatchUpIO

		if err := exec.Execute(nil, 0); err != nil {
			t.Fatalf("catch-up executor: %v", err)
		}

		s := driver.Orchestrator.Registry.Sender("vol1/vs2")
		if s.State() != engine.StateInSync {
			t.Fatalf("catch-up: state=%s, want InSync", s.State())
		}

		// Verify pins released.
		if pinner.ActiveHoldCount() != 0 {
			t.Fatalf("catch-up: %d pins leaked", pinner.ActiveHoldCount())
		}

		t.Log("catch-up: ONE CHAIN proven: plan → CatchUpExecutor → complete → InSync → pins released")
	} else {
		t.Fatalf("catch-up: unexpected outcome=%s (replica=%d committed=%d tail=%d)",
			plan.Outcome, replicaLSN, state.CommittedLSN, state.WALTailLSN)
	}
}

// --- ONE CHAIN: Full-base rebuild closure ---

func TestP2_RebuildClosure_OneChain(t *testing.T) {
	driver, ca, reader, _, pinner := setupChainTest(t)
	vol := reader.vol

	// Write + flush → force rebuild condition.
	for i := 0; i < 20; i++ {
		vol.WriteLBA(uint64(i), makeBlock(byte('A'+i%26)))
	}
	vol.ForceFlush()

	state := reader.ReadState()
	t.Logf("rebuild: head=%d tail=%d committed=%d checkpoint=%d",
		state.WALHeadLSN, state.WALTailLSN, state.CommittedLSN, state.CheckpointLSN)

	if state.WALTailLSN == 0 {
		t.Fatal("rebuild: ForceFlush must advance tail")
	}

	// Step 1: catch-up fails → NeedsRebuild.
	driver.Orchestrator.ProcessAssignment(makeIntent(ca, 1, "replica"))
	plan, _ := driver.PlanRecovery("vol1/vs2", 0)
	if plan.Outcome != engine.OutcomeNeedsRebuild {
		t.Fatalf("rebuild: outcome=%s (expected NeedsRebuild)", plan.Outcome)
	}

	// Step 2: rebuild assignment.
	driver.Orchestrator.ProcessAssignment(makeIntent(ca, 1, "rebuilding"))

	// Step 3: plan rebuild from real storage.
	rebuildPlan, err := driver.PlanRebuild("vol1/vs2")
	if err != nil {
		t.Fatalf("rebuild plan: %v", err)
	}

	// Step 4: engine RebuildExecutor — test mode (IO=nil) for FSM proof.
	// Real snapshot_tail I/O is proven by TestP2_SnapshotTailRebuild_OneChain.
	exec := engine.NewRebuildExecutor(driver, rebuildPlan)
	if err := exec.Execute(); err != nil {
		t.Fatalf("rebuild executor: %v", err)
	}

	// Step 5: verify final state.
	s := driver.Orchestrator.Registry.Sender("vol1/vs2")
	if s.State() != engine.StateInSync {
		t.Fatalf("rebuild: state=%s, want InSync", s.State())
	}

	// Step 6: verify pins released by executor.
	if pinner.ActiveHoldCount() != 0 {
		t.Fatalf("rebuild: %d pins leaked", pinner.ActiveHoldCount())
	}

	t.Logf("rebuild: ONE CHAIN proven: plan(source=%s) → RebuildExecutor(IO=v2bridge) → complete → InSync → pins released",
		rebuildPlan.RebuildSource)
}

// --- Cleanup: cancel releases all resources ---

func TestP2_CancelDuringExecution_ReleasesAll(t *testing.T) {
	driver, ca, reader, _, pinner := setupChainTest(t)
	vol := reader.vol

	for i := 0; i < 5; i++ {
		vol.WriteLBA(uint64(i), makeBlock(byte('A'+i)))
	}

	driver.Orchestrator.ProcessAssignment(makeIntent(ca, 1, "replica"))
	plan, _ := driver.PlanRecovery("vol1/vs2", 0)

	// Cancel mid-plan.
	driver.CancelPlan(plan, "epoch_bump")

	// Session invalidated.
	s := driver.Orchestrator.Registry.Sender("vol1/vs2")
	if s.HasActiveSession() {
		t.Fatal("session should be invalidated")
	}

	// Pins released.
	if pinner.ActiveHoldCount() != 0 {
		t.Fatalf("cancel: %d pins leaked", pinner.ActiveHoldCount())
	}

	// Log shows cancellation.
	hasCancellation := false
	for _, e := range driver.Orchestrator.Log.EventsFor("vol1/vs2") {
		if e.Event == "plan_cancelled" {
			hasCancellation = true
		}
	}
	if !hasCancellation {
		t.Fatal("log must show plan_cancelled")
	}
}

// --- Observability: execution explains causality ---

func TestP2_Observability_ExecutionCausality(t *testing.T) {
	driver, ca, reader, _, _ := setupChainTest(t)
	vol := reader.vol

	for i := 0; i < 10; i++ {
		vol.WriteLBA(uint64(i), makeBlock(byte('A'+i)))
	}
	vol.ForceFlush()

	state := reader.ReadState()
	if state.WALTailLSN == 0 {
		t.Fatal("need post-flush state")
	}

	driver.Orchestrator.ProcessAssignment(makeIntent(ca, 1, "replica"))
	plan, _ := driver.PlanRecovery("vol1/vs2", 0)

	if plan.Outcome != engine.OutcomeNeedsRebuild {
		t.Fatalf("outcome=%s", plan.Outcome)
	}

	// Verify log explains why.
	events := driver.Orchestrator.Log.EventsFor("vol1/vs2")
	required := map[string]bool{
		"sender_added":    false,
		"session_created": false,
		"connected":       false,
		"escalated":       false,
	}
	for _, e := range events {
		if _, ok := required[e.Event]; ok {
			required[e.Event] = true
		}
	}
	for event, found := range required {
		if !found {
			t.Fatalf("observability: missing %s event", event)
		}
	}
}