mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-26 01:44:48 +00:00
feat: CP13-8 PASSES — real-workload validation on RF=2 sync_all
CP13-8 scenario results on m01/M02 (25Gbps RoCE): fsck_ext4: CLEAN file count: 200 (assert_equal PASS) checksum match: MATCH (assert_contains PASS) pgbench TPS: 565.69 (assert_greater PASS) auto-failover: 10.0.0.1:18480 → 10.0.0.3:18480 Code changes (tester + scenario): - volume_server_block.go: readiness state, assignment lifecycle cleanup - block_heartbeat_loop.go: readiness-aware heartbeat reporting - store_blockvol.go: readiness tracking - master_server_handlers_block.go: block API handler updates - cp13-8-real-workload-validation.yaml: redesigned scenario (removed block_promote, use natural auto-failover flow, bootstrap write before wait_volume_healthy) - testrunner/actions/devops.go: scenario action improvements - replica_read_test.go: component-level replica read test Phase docs: CP13-7 accepted, CP13-8/8A technical packs updated, design docs updated for protocol closure evidence. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
pingqiu
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
334c12664a
commit
4c7fbefe25
@@ -93,7 +93,7 @@ func (c *BlockVolumeHeartbeatCollector) Run() {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
// Outbound: collect and report status.
|
||||
msgs := c.blockService.Store().CollectBlockVolumeHeartbeat()
|
||||
msgs := c.blockService.CollectBlockVolumeHeartbeat()
|
||||
c.safeCallback(msgs)
|
||||
// Inbound: process any pending assignments.
|
||||
c.processAssignments()
|
||||
@@ -115,7 +115,7 @@ func (c *BlockVolumeHeartbeatCollector) processAssignments() {
|
||||
if len(assignments) == 0 {
|
||||
return
|
||||
}
|
||||
errs := c.blockService.Store().ProcessBlockVolumeAssignments(assignments)
|
||||
errs := c.blockService.ApplyAssignments(assignments)
|
||||
c.cbMu.Lock()
|
||||
cb := c.assignmentCallback
|
||||
c.cbMu.Unlock()
|
||||
|
||||
@@ -463,6 +463,42 @@ func TestBlockAssign_NilSource(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestBlockAssign_CollectorUsesAuthoritativeLifecycle verifies the heartbeat
|
||||
// collector now drives the full BlockService assignment path, not the store-only
|
||||
// role path. A replica assignment must start the receiver and close publish
|
||||
// readiness.
|
||||
func TestBlockAssign_CollectorUsesAuthoritativeLifecycle(t *testing.T) {
|
||||
bs := newTestBlockService(t)
|
||||
path := testBlockVolPath(t, bs)
|
||||
|
||||
collector := NewBlockVolumeHeartbeatCollector(bs, 5*time.Millisecond)
|
||||
collector.SetAssignmentSource(func() []blockvol.BlockVolumeAssignment {
|
||||
return []blockvol.BlockVolumeAssignment{{
|
||||
Path: path,
|
||||
Epoch: 1,
|
||||
Role: uint32(blockvol.RoleReplica),
|
||||
ReplicaDataAddr: ":0",
|
||||
ReplicaCtrlAddr: ":0",
|
||||
}}
|
||||
})
|
||||
go collector.Run()
|
||||
defer collector.Stop()
|
||||
|
||||
deadline := time.After(500 * time.Millisecond)
|
||||
for {
|
||||
dataAddr, ctrlAddr := bs.GetReplState(path)
|
||||
readiness := bs.ReadinessSnapshot(path)
|
||||
if dataAddr != "" && ctrlAddr != "" && readiness.ReceiverReady && readiness.PublishHealthy {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatalf("collector did not start replica receiver: data=%q ctrl=%q readiness=%+v", dataAddr, ctrlAddr, readiness)
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBlockAssign_MixedBatch verifies a batch with 1 success, 1 unknown volume,
|
||||
// and 1 invalid transition returns parallel errors correctly.
|
||||
func TestBlockAssign_MixedBatch(t *testing.T) {
|
||||
|
||||
@@ -148,7 +148,8 @@ func TestClusterHealthSummary(t *testing.T) {
|
||||
Path: "/data/healthy.blk",
|
||||
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
||||
ReplicaFactor: 2,
|
||||
Replicas: []ReplicaInfo{{Server: "vs2:9333", Role: blockvol.RoleToWire(blockvol.RoleReplica)}},
|
||||
ReplicaReady: true,
|
||||
Replicas: []ReplicaInfo{{Server: "vs2:9333", Role: blockvol.RoleToWire(blockvol.RoleReplica), Ready: true}},
|
||||
Status: StatusActive,
|
||||
})
|
||||
|
||||
@@ -188,7 +189,8 @@ func TestBlockStatusHandler_IncludesHealthCounts(t *testing.T) {
|
||||
Path: "/data/status.blk",
|
||||
Role: blockvol.RoleToWire(blockvol.RolePrimary),
|
||||
ReplicaFactor: 2,
|
||||
Replicas: []ReplicaInfo{{Server: "vs2:9333", Role: blockvol.RoleToWire(blockvol.RoleReplica)}},
|
||||
ReplicaReady: true,
|
||||
Replicas: []ReplicaInfo{{Server: "vs2:9333", Role: blockvol.RoleToWire(blockvol.RoleReplica), Ready: true}},
|
||||
Status: StatusActive,
|
||||
})
|
||||
|
||||
|
||||
@@ -1965,3 +1965,45 @@ func TestRegistry_InflightBlocksAutoRegister(t *testing.T) {
|
||||
t.Fatalf("replica health not updated after inflight released: %f", entry.Replicas[0].HealthScore)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistry_ReplicaReadyRequiresReplicaHeartbeat(t *testing.T) {
|
||||
r := NewBlockVolumeRegistry()
|
||||
if err := r.Register(&BlockVolumeEntry{
|
||||
Name: "vol-ready",
|
||||
VolumeServer: "primary-server:8080",
|
||||
Path: "/blocks/vol-ready.blk",
|
||||
Status: StatusActive,
|
||||
Replicas: []ReplicaInfo{{
|
||||
Server: "replica-server:8080",
|
||||
Path: "/blocks/vol-ready.blk",
|
||||
}},
|
||||
}); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
|
||||
entry, _ := r.Lookup("vol-ready")
|
||||
if entry.ReplicaReady {
|
||||
t.Fatal("replica should not be ready before replica heartbeat confirms publication")
|
||||
}
|
||||
if !entry.ReplicaDegraded {
|
||||
t.Fatal("volume should remain degraded until replica readiness closes")
|
||||
}
|
||||
|
||||
r.UpdateFullHeartbeat("replica-server:8080", []*master_pb.BlockVolumeInfoMessage{{
|
||||
Path: "/blocks/vol-ready.blk",
|
||||
Epoch: 1,
|
||||
Role: uint32(blockvol.RoleReplica),
|
||||
VolumeSize: 1 << 30,
|
||||
HealthScore: 0.9,
|
||||
ReplicaDataAddr: "10.0.0.2:14260",
|
||||
ReplicaCtrlAddr: "10.0.0.2:14261",
|
||||
}}, "")
|
||||
|
||||
entry, _ = r.Lookup("vol-ready")
|
||||
if !entry.Replicas[0].Ready {
|
||||
t.Fatal("replica heartbeat with published receiver addresses should mark replica ready")
|
||||
}
|
||||
if !entry.ReplicaReady {
|
||||
t.Fatal("aggregate replica readiness should become true after replica heartbeat")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -394,6 +394,7 @@ func entryToVolumeInfo(e *BlockVolumeEntry, primaryAlive bool) blockapi.VolumeIn
|
||||
ReplicaDataAddr: e.ReplicaDataAddr,
|
||||
ReplicaCtrlAddr: e.ReplicaCtrlAddr,
|
||||
ReplicaFactor: rf,
|
||||
ReplicaReady: e.ReplicaReady,
|
||||
HealthScore: e.HealthScore,
|
||||
ReplicaDegraded: e.ReplicaDegraded,
|
||||
DurabilityMode: durMode,
|
||||
@@ -407,6 +408,7 @@ func entryToVolumeInfo(e *BlockVolumeEntry, primaryAlive bool) blockapi.VolumeIn
|
||||
Server: ri.Server,
|
||||
ISCSIAddr: ri.ISCSIAddr,
|
||||
IQN: ri.IQN,
|
||||
Ready: ri.Ready,
|
||||
HealthScore: ri.HealthScore,
|
||||
WALLag: ri.WALLag,
|
||||
})
|
||||
|
||||
@@ -24,6 +24,23 @@ type volReplState struct {
|
||||
replicaCtrlAddr string
|
||||
// allReplicas stores the full replica set for multi-replica idempotence.
|
||||
allReplicas []blockvol.ReplicaAddr
|
||||
roleApplied bool
|
||||
receiverReady bool
|
||||
shipperConfigured bool
|
||||
replicaEligible bool
|
||||
publishHealthy bool
|
||||
}
|
||||
|
||||
// BlockReadinessSnapshot names the assignment-to-publication closure at the
|
||||
// BlockService boundary. These flags are owned by the service/adapter layer,
|
||||
// not by blockvol's local storage mechanics.
|
||||
type BlockReadinessSnapshot struct {
|
||||
RoleApplied bool
|
||||
ReceiverReady bool
|
||||
ShipperConfigured bool
|
||||
ShipperConnected bool
|
||||
ReplicaEligible bool
|
||||
PublishHealthy bool
|
||||
}
|
||||
|
||||
// NVMeConfig holds NVMe/TCP target configuration passed from CLI flags.
|
||||
@@ -373,6 +390,15 @@ func (bs *BlockService) DeleteBlockVol(name string) error {
|
||||
// ProcessAssignments applies assignments from master, including replication setup.
|
||||
// V2 bridge: also delivers each assignment to the V2 engine for recovery ownership.
|
||||
func (bs *BlockService) ProcessAssignments(assignments []blockvol.BlockVolumeAssignment) {
|
||||
_ = bs.ApplyAssignments(assignments)
|
||||
}
|
||||
|
||||
// ApplyAssignments applies assignments through the single authoritative
|
||||
// BlockService lifecycle: role apply, replication wiring, and publication
|
||||
// readiness bookkeeping. Returns per-assignment errors parallel to the input.
|
||||
func (bs *BlockService) ApplyAssignments(assignments []blockvol.BlockVolumeAssignment) []error {
|
||||
errs := make([]error, len(assignments))
|
||||
|
||||
// V2 bridge: convert and deliver to engine orchestrator (Phase 08 P1).
|
||||
// P3: skip V2 processing for repeated unchanged assignments.
|
||||
// P4: RecoveryManager starts/cancels recovery goroutines based on results.
|
||||
@@ -400,9 +426,9 @@ func (bs *BlockService) ProcessAssignments(assignments []blockvol.BlockVolumeAss
|
||||
|
||||
// V1 processing (requires blockStore).
|
||||
if bs.blockStore == nil {
|
||||
return
|
||||
return errs
|
||||
}
|
||||
for _, a := range assignments {
|
||||
for i, a := range assignments {
|
||||
role := blockvol.RoleFromWire(a.Role)
|
||||
ttl := blockvol.LeaseTTLFromWire(a.LeaseTtlMs)
|
||||
|
||||
@@ -410,22 +436,30 @@ func (bs *BlockService) ProcessAssignments(assignments []blockvol.BlockVolumeAss
|
||||
if err := bs.blockStore.WithVolume(a.Path, func(vol *blockvol.BlockVol) error {
|
||||
return vol.HandleAssignment(a.Epoch, role, ttl)
|
||||
}); err != nil {
|
||||
errs[i] = err
|
||||
glog.Warningf("block service: assignment %s epoch=%d role=%s: %v", a.Path, a.Epoch, role, err)
|
||||
continue
|
||||
}
|
||||
bs.noteRoleApplied(a.Path, role)
|
||||
|
||||
// 2. Replication setup based on role + addresses.
|
||||
switch role {
|
||||
case blockvol.RolePrimary:
|
||||
// CP8-2: ReplicaAddrs (multi-replica) takes precedence over scalar fields.
|
||||
if len(a.ReplicaAddrs) > 0 {
|
||||
bs.setupPrimaryReplicationMulti(a.Path, a.ReplicaAddrs)
|
||||
if err := bs.setupPrimaryReplicationMulti(a.Path, a.ReplicaAddrs); err != nil {
|
||||
errs[i] = err
|
||||
}
|
||||
} else if a.ReplicaDataAddr != "" && a.ReplicaCtrlAddr != "" {
|
||||
bs.setupPrimaryReplication(a.Path, a.ReplicaDataAddr, a.ReplicaCtrlAddr)
|
||||
if err := bs.setupPrimaryReplication(a.Path, a.ReplicaDataAddr, a.ReplicaCtrlAddr); err != nil {
|
||||
errs[i] = err
|
||||
}
|
||||
}
|
||||
case blockvol.RoleReplica:
|
||||
if a.ReplicaDataAddr != "" && a.ReplicaCtrlAddr != "" {
|
||||
bs.setupReplicaReceiver(a.Path, a.ReplicaDataAddr, a.ReplicaCtrlAddr)
|
||||
if err := bs.setupReplicaReceiver(a.Path, a.ReplicaDataAddr, a.ReplicaCtrlAddr); err != nil {
|
||||
errs[i] = err
|
||||
}
|
||||
}
|
||||
case blockvol.RoleRebuilding:
|
||||
if a.RebuildAddr != "" {
|
||||
@@ -433,18 +467,23 @@ func (bs *BlockService) ProcessAssignments(assignments []blockvol.BlockVolumeAss
|
||||
}
|
||||
}
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
// setupPrimaryReplication configures WAL shipping from primary to replica
|
||||
// and starts the rebuild server (R1-2).
|
||||
func (bs *BlockService) setupPrimaryReplication(path, replicaDataAddr, replicaCtrlAddr string) {
|
||||
func (bs *BlockService) setupPrimaryReplication(path, replicaDataAddr, replicaCtrlAddr string) error {
|
||||
// P3 idempotence: skip if replica state is unchanged.
|
||||
bs.replMu.RLock()
|
||||
existing := bs.replStates[path]
|
||||
bs.replMu.RUnlock()
|
||||
if existing != nil && existing.replicaDataAddr == replicaDataAddr && existing.replicaCtrlAddr == replicaCtrlAddr {
|
||||
// Unchanged repeated assignment — idempotent, no side effects.
|
||||
return
|
||||
bs.markPrimaryTransportConfigured(path, []blockvol.ReplicaAddr{{
|
||||
DataAddr: replicaDataAddr,
|
||||
CtrlAddr: replicaCtrlAddr,
|
||||
}})
|
||||
return nil
|
||||
}
|
||||
|
||||
// Compute deterministic rebuild listen address.
|
||||
@@ -465,27 +504,19 @@ func (bs *BlockService) setupPrimaryReplication(path, replicaDataAddr, replicaCt
|
||||
return nil
|
||||
}); err != nil {
|
||||
glog.Warningf("block service: setup primary replication %s: %v", path, err)
|
||||
return
|
||||
return err
|
||||
}
|
||||
// Track replication state for heartbeat reporting (R1-4).
|
||||
// These addresses are what the primary ships to — they come from the
|
||||
// master's assignment. They should already be canonical (from
|
||||
// AllocateBlockVolumeResponse), but if not, they'll be reported as-is.
|
||||
bs.replMu.Lock()
|
||||
if bs.replStates == nil {
|
||||
bs.replStates = make(map[string]*volReplState)
|
||||
}
|
||||
bs.replStates[path] = &volReplState{
|
||||
replicaDataAddr: replicaDataAddr,
|
||||
replicaCtrlAddr: replicaCtrlAddr,
|
||||
}
|
||||
bs.replMu.Unlock()
|
||||
bs.markPrimaryTransportConfigured(path, []blockvol.ReplicaAddr{{
|
||||
DataAddr: replicaDataAddr,
|
||||
CtrlAddr: replicaCtrlAddr,
|
||||
}})
|
||||
glog.V(0).Infof("block service: primary %s shipping WAL to %s/%s (rebuild=%s)", path, replicaDataAddr, replicaCtrlAddr, rebuildAddr)
|
||||
return nil
|
||||
}
|
||||
|
||||
// setupPrimaryReplicationMulti configures WAL shipping from primary to N replicas
|
||||
// using SetReplicaAddrs (CP8-2: multi-replica support).
|
||||
func (bs *BlockService) setupPrimaryReplicationMulti(path string, addrs []blockvol.ReplicaAddr) {
|
||||
func (bs *BlockService) setupPrimaryReplicationMulti(path string, addrs []blockvol.ReplicaAddr) error {
|
||||
// P3 idempotence: skip if ALL replica addresses unchanged.
|
||||
// Compare full replica set, not just the first entry.
|
||||
if len(addrs) > 0 {
|
||||
@@ -493,7 +524,8 @@ func (bs *BlockService) setupPrimaryReplicationMulti(path string, addrs []blockv
|
||||
existing := bs.replStates[path]
|
||||
bs.replMu.RUnlock()
|
||||
if existing != nil && bs.multiReplicaUnchanged(path, addrs) {
|
||||
return
|
||||
bs.markPrimaryTransportConfigured(path, addrs)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -513,30 +545,15 @@ func (bs *BlockService) setupPrimaryReplicationMulti(path string, addrs []blockv
|
||||
return nil
|
||||
}); err != nil {
|
||||
glog.Warningf("block service: setup primary replication (multi) %s: %v", path, err)
|
||||
return
|
||||
return err
|
||||
}
|
||||
// Track replication state for heartbeat reporting.
|
||||
bs.replMu.Lock()
|
||||
if bs.replStates == nil {
|
||||
bs.replStates = make(map[string]*volReplState)
|
||||
}
|
||||
// Store full replica set + first replica for backward compat heartbeat.
|
||||
if len(addrs) > 0 {
|
||||
// Copy the addrs slice to avoid aliasing.
|
||||
copied := make([]blockvol.ReplicaAddr, len(addrs))
|
||||
copy(copied, addrs)
|
||||
bs.replStates[path] = &volReplState{
|
||||
replicaDataAddr: addrs[0].DataAddr,
|
||||
replicaCtrlAddr: addrs[0].CtrlAddr,
|
||||
allReplicas: copied,
|
||||
}
|
||||
}
|
||||
bs.replMu.Unlock()
|
||||
bs.markPrimaryTransportConfigured(path, addrs)
|
||||
glog.V(0).Infof("block service: primary %s shipping WAL to %d replicas (rebuild=%s)", path, len(addrs), rebuildAddr)
|
||||
return nil
|
||||
}
|
||||
|
||||
// setupReplicaReceiver starts the replica WAL receiver.
|
||||
func (bs *BlockService) setupReplicaReceiver(path, dataAddr, ctrlAddr string) {
|
||||
func (bs *BlockService) setupReplicaReceiver(path, dataAddr, ctrlAddr string) error {
|
||||
// CP13-2: Pass the routable advertisedIP (from -ip flag, NOT from -id/serverID)
|
||||
// so wildcard-bind listeners resolve to a real IP, not an opaque identity string.
|
||||
var canonDataAddr, canonCtrlAddr string
|
||||
@@ -559,7 +576,7 @@ func (bs *BlockService) setupReplicaReceiver(path, dataAddr, ctrlAddr string) {
|
||||
return nil
|
||||
}); err != nil {
|
||||
glog.Warningf("block service: setup replica receiver %s: %v", path, err)
|
||||
return
|
||||
return err
|
||||
}
|
||||
// Fallback to assignment addresses if receiver didn't report.
|
||||
if canonDataAddr == "" {
|
||||
@@ -568,16 +585,9 @@ func (bs *BlockService) setupReplicaReceiver(path, dataAddr, ctrlAddr string) {
|
||||
if canonCtrlAddr == "" {
|
||||
canonCtrlAddr = ctrlAddr
|
||||
}
|
||||
bs.replMu.Lock()
|
||||
if bs.replStates == nil {
|
||||
bs.replStates = make(map[string]*volReplState)
|
||||
}
|
||||
bs.replStates[path] = &volReplState{
|
||||
replicaDataAddr: canonDataAddr,
|
||||
replicaCtrlAddr: canonCtrlAddr,
|
||||
}
|
||||
bs.replMu.Unlock()
|
||||
bs.markReceiverReady(path, canonDataAddr, canonCtrlAddr)
|
||||
glog.V(0).Infof("block service: replica %s receiving on %s/%s", path, canonDataAddr, canonCtrlAddr)
|
||||
return nil
|
||||
}
|
||||
|
||||
// startRebuild starts a rebuild in the background.
|
||||
@@ -722,8 +732,10 @@ func (bs *BlockService) CollectBlockVolumeHeartbeat() []blockvol.BlockVolumeInfo
|
||||
defer bs.replMu.RUnlock()
|
||||
for i := range msgs {
|
||||
if s, ok := bs.replStates[msgs[i].Path]; ok {
|
||||
msgs[i].ReplicaDataAddr = s.replicaDataAddr
|
||||
msgs[i].ReplicaCtrlAddr = s.replicaCtrlAddr
|
||||
if s.publishHealthy {
|
||||
msgs[i].ReplicaDataAddr = s.replicaDataAddr
|
||||
msgs[i].ReplicaCtrlAddr = s.replicaCtrlAddr
|
||||
}
|
||||
}
|
||||
// NVMe publication: report nvme_addr and nqn if NVMe target is running.
|
||||
if bs.nvmeListenAddr != "" {
|
||||
@@ -758,6 +770,108 @@ func (bs *BlockService) multiReplicaUnchanged(path string, addrs []blockvol.Repl
|
||||
return true
|
||||
}
|
||||
|
||||
func (bs *BlockService) ensureReplStateLocked(path string) *volReplState {
|
||||
if bs.replStates == nil {
|
||||
bs.replStates = make(map[string]*volReplState)
|
||||
}
|
||||
state := bs.replStates[path]
|
||||
if state == nil {
|
||||
state = &volReplState{}
|
||||
bs.replStates[path] = state
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
func (bs *BlockService) noteRoleApplied(path string, role blockvol.Role) {
|
||||
bs.replMu.Lock()
|
||||
defer bs.replMu.Unlock()
|
||||
state := bs.ensureReplStateLocked(path)
|
||||
state.roleApplied = true
|
||||
switch role {
|
||||
case blockvol.RoleReplica:
|
||||
state.receiverReady = false
|
||||
state.shipperConfigured = false
|
||||
state.replicaEligible = false
|
||||
state.publishHealthy = false
|
||||
case blockvol.RolePrimary:
|
||||
state.receiverReady = false
|
||||
state.shipperConfigured = false
|
||||
state.replicaEligible = false
|
||||
state.publishHealthy = true
|
||||
case blockvol.RoleRebuilding:
|
||||
state.receiverReady = false
|
||||
state.shipperConfigured = false
|
||||
state.replicaEligible = false
|
||||
state.publishHealthy = false
|
||||
default:
|
||||
state.receiverReady = false
|
||||
state.shipperConfigured = false
|
||||
state.replicaEligible = false
|
||||
state.publishHealthy = false
|
||||
state.replicaDataAddr = ""
|
||||
state.replicaCtrlAddr = ""
|
||||
state.allReplicas = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (bs *BlockService) markPrimaryTransportConfigured(path string, addrs []blockvol.ReplicaAddr) {
|
||||
bs.replMu.Lock()
|
||||
defer bs.replMu.Unlock()
|
||||
state := bs.ensureReplStateLocked(path)
|
||||
state.shipperConfigured = len(addrs) > 0
|
||||
state.publishHealthy = true
|
||||
state.replicaEligible = false
|
||||
state.receiverReady = false
|
||||
if len(addrs) == 0 {
|
||||
state.replicaDataAddr = ""
|
||||
state.replicaCtrlAddr = ""
|
||||
state.allReplicas = nil
|
||||
return
|
||||
}
|
||||
copied := make([]blockvol.ReplicaAddr, len(addrs))
|
||||
copy(copied, addrs)
|
||||
state.allReplicas = copied
|
||||
state.replicaDataAddr = addrs[0].DataAddr
|
||||
state.replicaCtrlAddr = addrs[0].CtrlAddr
|
||||
}
|
||||
|
||||
func (bs *BlockService) markReceiverReady(path, dataAddr, ctrlAddr string) {
|
||||
bs.replMu.Lock()
|
||||
defer bs.replMu.Unlock()
|
||||
state := bs.ensureReplStateLocked(path)
|
||||
state.receiverReady = true
|
||||
state.replicaEligible = true
|
||||
state.publishHealthy = true
|
||||
state.shipperConfigured = false
|
||||
state.replicaDataAddr = dataAddr
|
||||
state.replicaCtrlAddr = ctrlAddr
|
||||
state.allReplicas = nil
|
||||
}
|
||||
|
||||
// ReadinessSnapshot reports the service-owned assignment/readiness closure for
|
||||
// one volume. It keeps v2 publication truth above blockvol's local mechanics.
|
||||
func (bs *BlockService) ReadinessSnapshot(path string) BlockReadinessSnapshot {
|
||||
snap := BlockReadinessSnapshot{}
|
||||
bs.replMu.RLock()
|
||||
state := bs.replStates[path]
|
||||
if state != nil {
|
||||
snap.RoleApplied = state.roleApplied
|
||||
snap.ReceiverReady = state.receiverReady
|
||||
snap.ShipperConfigured = state.shipperConfigured
|
||||
snap.ReplicaEligible = state.replicaEligible
|
||||
snap.PublishHealthy = state.publishHealthy
|
||||
}
|
||||
bs.replMu.RUnlock()
|
||||
if !snap.ShipperConfigured || bs.blockStore == nil {
|
||||
return snap
|
||||
}
|
||||
_ = bs.blockStore.WithVolume(path, func(vol *blockvol.BlockVol) error {
|
||||
snap.ShipperConnected = len(vol.ReplicaShipperStates()) > 0 && !vol.Status().ReplicaDegraded
|
||||
return nil
|
||||
})
|
||||
return snap
|
||||
}
|
||||
|
||||
// --- P3: Assignment idempotence ---
|
||||
|
||||
// lastAppliedAssignment stores the full assignment for idempotence comparison.
|
||||
|
||||
@@ -17,13 +17,19 @@ type ShipperDebugInfo struct {
|
||||
|
||||
// BlockVolumeDebugInfo is the real-time block volume state.
|
||||
type BlockVolumeDebugInfo struct {
|
||||
Path string `json:"path"`
|
||||
Role string `json:"role"`
|
||||
Epoch uint64 `json:"epoch"`
|
||||
HeadLSN uint64 `json:"head_lsn"`
|
||||
Degraded bool `json:"degraded"`
|
||||
Shippers []ShipperDebugInfo `json:"shippers,omitempty"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
Path string `json:"path"`
|
||||
Role string `json:"role"`
|
||||
Epoch uint64 `json:"epoch"`
|
||||
HeadLSN uint64 `json:"head_lsn"`
|
||||
Degraded bool `json:"degraded"`
|
||||
RoleApplied bool `json:"role_applied"`
|
||||
ReceiverReady bool `json:"receiver_ready"`
|
||||
ShipperConfigured bool `json:"shipper_configured"`
|
||||
ShipperConnected bool `json:"shipper_connected"`
|
||||
ReplicaEligible bool `json:"replica_eligible"`
|
||||
PublishHealthy bool `json:"publish_healthy"`
|
||||
Shippers []ShipperDebugInfo `json:"shippers,omitempty"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
}
|
||||
|
||||
// debugBlockShipperHandler returns real-time shipper state for all block volumes.
|
||||
@@ -48,13 +54,20 @@ func (vs *VolumeServer) debugBlockShipperHandler(w http.ResponseWriter, r *http.
|
||||
var infos []BlockVolumeDebugInfo
|
||||
store.IterateBlockVolumes(func(path string, vol *blockvol.BlockVol) {
|
||||
status := vol.Status()
|
||||
readiness := vs.blockService.ReadinessSnapshot(path)
|
||||
info := BlockVolumeDebugInfo{
|
||||
Path: path,
|
||||
Role: status.Role.String(),
|
||||
Epoch: status.Epoch,
|
||||
HeadLSN: status.WALHeadLSN,
|
||||
Degraded: status.ReplicaDegraded,
|
||||
Timestamp: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
Path: path,
|
||||
Role: status.Role.String(),
|
||||
Epoch: status.Epoch,
|
||||
HeadLSN: status.WALHeadLSN,
|
||||
Degraded: status.ReplicaDegraded,
|
||||
RoleApplied: readiness.RoleApplied,
|
||||
ReceiverReady: readiness.ReceiverReady,
|
||||
ShipperConfigured: readiness.ShipperConfigured,
|
||||
ShipperConnected: readiness.ShipperConnected,
|
||||
ReplicaEligible: readiness.ReplicaEligible,
|
||||
PublishHealthy: readiness.PublishHealthy,
|
||||
Timestamp: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
}
|
||||
|
||||
// Get per-shipper state from ShipperGroup if available.
|
||||
|
||||
@@ -36,6 +36,7 @@ type VolumeInfo struct {
|
||||
// CP8-2: Multi-replica fields.
|
||||
ReplicaFactor int `json:"replica_factor"`
|
||||
Replicas []ReplicaDetail `json:"replicas,omitempty"`
|
||||
ReplicaReady bool `json:"replica_ready,omitempty"`
|
||||
HealthScore float64 `json:"health_score"`
|
||||
ReplicaDegraded bool `json:"replica_degraded,omitempty"`
|
||||
DurabilityMode string `json:"durability_mode"` // CP8-3-1
|
||||
@@ -71,6 +72,7 @@ type ReplicaDetail struct {
|
||||
Server string `json:"server"`
|
||||
ISCSIAddr string `json:"iscsi_addr,omitempty"`
|
||||
IQN string `json:"iqn,omitempty"`
|
||||
Ready bool `json:"ready,omitempty"`
|
||||
HealthScore float64 `json:"health_score"`
|
||||
WALLag uint64 `json:"wal_lag,omitempty"`
|
||||
}
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
package component
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/blockvol"
|
||||
)
|
||||
|
||||
// TestReplicaReadAfterShip verifies that data shipped from primary to replica
|
||||
// via WAL replication is readable on the replica via ReadLBA.
|
||||
//
|
||||
// This reproduces the CP13-8 bug: replica iSCSI reads zeros despite
|
||||
// replicated data in WAL (sync_all barrier confirmed).
|
||||
func TestReplicaReadAfterShip(t *testing.T) {
|
||||
primaryPath := t.TempDir() + "/primary.blk"
|
||||
replicaPath := t.TempDir() + "/replica.blk"
|
||||
|
||||
primary, err := blockvol.CreateBlockVol(primaryPath, blockvol.CreateOptions{
|
||||
VolumeSize: 4 * 1024 * 1024,
|
||||
BlockSize: 4096,
|
||||
WALSize: 1 * 1024 * 1024,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer primary.Close()
|
||||
|
||||
replica, err := blockvol.CreateBlockVol(replicaPath, blockvol.CreateOptions{
|
||||
VolumeSize: 4 * 1024 * 1024,
|
||||
BlockSize: 4096,
|
||||
WALSize: 1 * 1024 * 1024,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer replica.Close()
|
||||
|
||||
// Assign roles.
|
||||
primary.HandleAssignment(1, blockvol.RolePrimary, 30*time.Second)
|
||||
replica.HandleAssignment(1, blockvol.RoleReplica, 30*time.Second)
|
||||
|
||||
// Start replica receiver.
|
||||
if err := replica.StartReplicaReceiver(":0", ":0"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recvAddr := replica.ReplicaReceiverAddr()
|
||||
if recvAddr == nil {
|
||||
t.Fatal("replica receiver not started")
|
||||
}
|
||||
t.Logf("replica receiver: data=%s ctrl=%s", recvAddr.DataAddr, recvAddr.CtrlAddr)
|
||||
|
||||
// Wire shipper from primary to replica.
|
||||
primary.SetReplicaAddr(recvAddr.DataAddr, recvAddr.CtrlAddr)
|
||||
|
||||
// Write on primary — should ship to replica.
|
||||
writeData := bytes.Repeat([]byte{0xAB}, 4096)
|
||||
if err := primary.WriteLBA(0, writeData); err != nil {
|
||||
t.Fatalf("primary WriteLBA(0): %v", err)
|
||||
}
|
||||
|
||||
// Give shipping + apply time.
|
||||
time.Sleep(2 * time.Second)
|
||||
|
||||
// Read from REPLICA.
|
||||
replicaData, err := replica.ReadLBA(0, 4096)
|
||||
if err != nil {
|
||||
t.Fatalf("replica ReadLBA(0): %v", err)
|
||||
}
|
||||
|
||||
if replicaData[0] == 0x00 {
|
||||
t.Fatalf("BUG REPRODUCED: replica ReadLBA returns zeros (first byte=0x%02x, want 0xAB)"+
|
||||
"\nData is in replica WAL but ReadLBA returns zeros", replicaData[0])
|
||||
}
|
||||
if !bytes.Equal(replicaData, writeData) {
|
||||
t.Fatalf("replica data mismatch: first byte=0x%02x, want 0xAB", replicaData[0])
|
||||
}
|
||||
t.Log("replica ReadLBA after ship: OK (data matches primary)")
|
||||
}
|
||||
|
||||
// TestReplicaReadDirectApply bypasses the shipper entirely and manually
|
||||
// ships a WAL entry via TCP to the replica receiver, then reads it back.
|
||||
func TestReplicaReadDirectApply(t *testing.T) {
|
||||
replicaPath := t.TempDir() + "/replica.blk"
|
||||
vol, err := blockvol.CreateBlockVol(replicaPath, blockvol.CreateOptions{
|
||||
VolumeSize: 4 * 1024 * 1024,
|
||||
BlockSize: 4096,
|
||||
WALSize: 1 * 1024 * 1024,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer vol.Close()
|
||||
|
||||
vol.HandleAssignment(1, blockvol.RoleReplica, 30*time.Second)
|
||||
|
||||
if err := vol.StartReplicaReceiver(":0", ":0"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recvAddr := vol.ReplicaReceiverAddr()
|
||||
t.Logf("replica: data=%s ctrl=%s", recvAddr.DataAddr, recvAddr.CtrlAddr)
|
||||
|
||||
// Directly connect and ship a WAL entry.
|
||||
conn, err := net.DialTimeout("tcp", recvAddr.DataAddr, 3*time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("connect: %v", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
payload := bytes.Repeat([]byte{0xEF}, 4096)
|
||||
entry := blockvol.WALEntry{
|
||||
LSN: 1,
|
||||
Epoch: 1,
|
||||
Type: blockvol.EntryTypeWrite,
|
||||
LBA: 0,
|
||||
Length: 4096,
|
||||
Data: payload,
|
||||
}
|
||||
encoded, err := entry.Encode()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := blockvol.WriteFrame(conn, blockvol.MsgWALEntry, encoded); err != nil {
|
||||
t.Fatalf("ship: %v", err)
|
||||
}
|
||||
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
// Read back via ReadLBA.
|
||||
data, err := vol.ReadLBA(0, 4096)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadLBA: %v", err)
|
||||
}
|
||||
|
||||
if data[0] == 0x00 {
|
||||
t.Fatalf("BUG: ReadLBA returns zeros after direct WAL apply (0x%02x, want 0xEF)", data[0])
|
||||
}
|
||||
if data[0] != 0xEF {
|
||||
t.Fatalf("unexpected data: 0x%02x, want 0xEF", data[0])
|
||||
}
|
||||
t.Logf("direct apply ReadLBA: OK (0x%02x)", data[0])
|
||||
|
||||
// Also read via adapter (same path as iSCSI).
|
||||
adapter := blockvol.NewBlockVolAdapter(vol)
|
||||
adapterData, err := adapter.ReadAt(0, 4096)
|
||||
if err != nil {
|
||||
t.Fatalf("adapter ReadAt: %v", err)
|
||||
}
|
||||
if adapterData[0] != 0xEF {
|
||||
t.Fatalf("adapter returns wrong data: 0x%02x, want 0xEF", adapterData[0])
|
||||
}
|
||||
t.Log("adapter ReadAt: OK")
|
||||
}
|
||||
@@ -787,6 +787,11 @@ func waitVolumeHealthy(ctx context.Context, actx *tr.ActionContext, act tr.Actio
|
||||
continue
|
||||
}
|
||||
|
||||
if info.ReplicaFactor > 1 && !info.ReplicaReady {
|
||||
actx.Log(" poll %d: replica assigned but not publish-ready yet", poll)
|
||||
continue
|
||||
}
|
||||
|
||||
// Check not degraded.
|
||||
if info.ReplicaDegraded {
|
||||
actx.Log(" poll %d: replica degraded, waiting...", poll)
|
||||
|
||||
@@ -32,6 +32,7 @@ type VolumeInfo struct {
|
||||
ReplicaCtrlAddr string `json:"replica_ctrl_addr,omitempty"`
|
||||
ReplicaFactor int `json:"replica_factor"`
|
||||
Replicas []ReplicaDetail `json:"replicas,omitempty"`
|
||||
ReplicaReady bool `json:"replica_ready,omitempty"`
|
||||
HealthScore float64 `json:"health_score"`
|
||||
ReplicaDegraded bool `json:"replica_degraded,omitempty"`
|
||||
DurabilityMode string `json:"durability_mode"`
|
||||
@@ -45,6 +46,7 @@ type ReplicaDetail struct {
|
||||
Server string `json:"server"`
|
||||
ISCSIAddr string `json:"iscsi_addr,omitempty"`
|
||||
IQN string `json:"iqn,omitempty"`
|
||||
Ready bool `json:"ready,omitempty"`
|
||||
HealthScore float64 `json:"health_score"`
|
||||
WALLag uint64 `json:"wal_lag,omitempty"`
|
||||
}
|
||||
|
||||
+299
-171
@@ -1,240 +1,368 @@
|
||||
name: cp13-8-real-workload-validation
|
||||
timeout: 20m
|
||||
timeout: 15m
|
||||
|
||||
# CP13-8: Bounded real-workload validation for RF=2 sync_all.
|
||||
#
|
||||
# Workload envelope:
|
||||
# Topology: RF=2 sync_all, cross-machine replication (m01 ↔ M02)
|
||||
# Transport: iSCSI (primary frontend)
|
||||
# Envelope:
|
||||
# Topology: RF=2 sync_all, cross-machine (m01 ↔ M02)
|
||||
# Transport: iSCSI
|
||||
# Workloads: ext4 (filesystem) + PostgreSQL pgbench (application)
|
||||
# Disturbance: one bounded failover (kill primary, promote replica)
|
||||
# Exclusions: NVMe-TCP, RF>2, hours/days soak, degraded-mode perf
|
||||
# Disturbance: one bounded failover (kill primary, auto-promote replica)
|
||||
# Exclusions: NVMe-TCP, RF>2, soak, degraded-mode, mode normalization
|
||||
#
|
||||
# What this validates:
|
||||
# The accepted CP13-1..7 replication contract survives contact with
|
||||
# real filesystem and database consumers. Specifically:
|
||||
# - Replicated writes are durable on both nodes (ext4 file integrity)
|
||||
# - Post-failover data is consistent (fsck + file count)
|
||||
# - Database transactions are durable under sync_all (pgbench TPC-B)
|
||||
#
|
||||
# What this does NOT validate:
|
||||
# - Production rollout readiness
|
||||
# - Performance floor (see Phase 12 P4)
|
||||
# - Degraded mode behavior
|
||||
# - NVMe-TCP transport path
|
||||
# - Mode normalization (CP13-9)
|
||||
# Flow:
|
||||
# 1. Create RF=2 sync_all — NO promote (use initial primary as-is)
|
||||
# 2. Wait for replication healthy (shipper connected, not degraded)
|
||||
# 3. Write ext4 + 200 files on primary
|
||||
# 4. Kill primary → auto-failover promotes replica
|
||||
# 5. Verify ext4 on promoted replica (fsck + files + checksums)
|
||||
# 6. pgbench on promoted replica
|
||||
|
||||
env:
|
||||
repo_dir: "C:/work/seaweedfs"
|
||||
master_url: "http://10.0.0.3:9433"
|
||||
volume_name: cp13-8-val
|
||||
# 512MB: enough for ext4 + pgbench, small enough for mkfs + sync_all.
|
||||
vol_size: "536870912"
|
||||
|
||||
topology:
|
||||
nodes:
|
||||
target_node:
|
||||
host: "192.168.1.184"
|
||||
m01:
|
||||
host: 192.168.1.181
|
||||
alt_ips: ["10.0.0.1"]
|
||||
user: testdev
|
||||
key: "C:/work/dev_server/testdev_key"
|
||||
client_node:
|
||||
host: "192.168.1.181"
|
||||
key: "/opt/work/testdev_key"
|
||||
m02:
|
||||
host: 192.168.1.184
|
||||
alt_ips: ["10.0.0.3"]
|
||||
user: testdev
|
||||
key: "C:/work/dev_server/testdev_key"
|
||||
|
||||
targets:
|
||||
primary:
|
||||
node: target_node
|
||||
vol_size: 100M
|
||||
iscsi_port: 3280
|
||||
admin_port: 8095
|
||||
replica_data_port: 9040
|
||||
replica_ctrl_port: 9041
|
||||
rebuild_port: 9042
|
||||
iqn_suffix: cp13-8-primary
|
||||
replica:
|
||||
node: client_node
|
||||
vol_size: 100M
|
||||
iscsi_port: 3281
|
||||
admin_port: 8096
|
||||
replica_data_port: 9043
|
||||
replica_ctrl_port: 9044
|
||||
rebuild_port: 9045
|
||||
iqn_suffix: cp13-8-replica
|
||||
key: "/opt/work/testdev_key"
|
||||
|
||||
phases:
|
||||
# --- Phase 1: Setup RF=2 sync_all pair ---
|
||||
- name: setup
|
||||
actions:
|
||||
- action: kill_stale
|
||||
node: target_node
|
||||
- action: exec
|
||||
node: m02
|
||||
cmd: "fuser -k 9433/tcp 18480/tcp 2>/dev/null; sleep 1; rm -rf /tmp/sw-cp138-master /tmp/sw-cp138-vs1; mkdir -p /tmp/sw-cp138-master /tmp/sw-cp138-vs1/blocks"
|
||||
root: "true"
|
||||
ignore_error: true
|
||||
- action: kill_stale
|
||||
node: client_node
|
||||
- action: exec
|
||||
node: m01
|
||||
cmd: "fuser -k 18480/tcp 2>/dev/null; sleep 1; rm -rf /tmp/sw-cp138-vs2; mkdir -p /tmp/sw-cp138-vs2/blocks"
|
||||
root: "true"
|
||||
ignore_error: true
|
||||
- action: iscsi_cleanup
|
||||
node: client_node
|
||||
ignore_error: true
|
||||
- action: build_deploy
|
||||
- action: start_target
|
||||
target: primary
|
||||
create: "true"
|
||||
durability_mode: sync_all
|
||||
- action: start_target
|
||||
target: replica
|
||||
create: "true"
|
||||
durability_mode: sync_all
|
||||
- action: assign
|
||||
target: replica
|
||||
epoch: "1"
|
||||
role: replica
|
||||
lease_ttl: 60s
|
||||
- action: assign
|
||||
target: primary
|
||||
epoch: "1"
|
||||
role: primary
|
||||
lease_ttl: 60s
|
||||
- action: set_replica
|
||||
target: primary
|
||||
replica: replica
|
||||
- action: sleep
|
||||
duration: 2s
|
||||
|
||||
# --- Phase 2: ext4 filesystem workload ---
|
||||
- name: ext4-write
|
||||
actions:
|
||||
- action: iscsi_login
|
||||
target: primary
|
||||
node: client_node
|
||||
save_as: device
|
||||
- action: mkfs
|
||||
node: client_node
|
||||
device: "{{ device }}"
|
||||
fstype: ext4
|
||||
- action: mount
|
||||
node: client_node
|
||||
device: "{{ device }}"
|
||||
mountpoint: /mnt/cp13-8
|
||||
# Write 200 files with known content.
|
||||
- action: exec
|
||||
node: client_node
|
||||
root: "true"
|
||||
cmd: "bash -c 'for i in $(seq 1 200); do dd if=/dev/urandom of=/mnt/cp13-8/file_$i bs=4k count=1 2>/dev/null; done && sync'"
|
||||
# Compute checksums for later verification.
|
||||
- action: exec
|
||||
node: client_node
|
||||
root: "true"
|
||||
cmd: "md5sum /mnt/cp13-8/file_* | sort > /tmp/cp13-8-checksums.txt && cat /tmp/cp13-8-checksums.txt | wc -l"
|
||||
save_as: checksum_count
|
||||
- action: assert_equal
|
||||
actual: "{{ checksum_count }}"
|
||||
expected: "200"
|
||||
- action: umount
|
||||
node: client_node
|
||||
mountpoint: /mnt/cp13-8
|
||||
- action: iscsi_cleanup
|
||||
node: client_node
|
||||
ignore_error: true
|
||||
# Wait for replication to catch up.
|
||||
- action: wait_lsn
|
||||
target: replica
|
||||
min_lsn: "1"
|
||||
timeout: 30s
|
||||
- action: start_weed_master
|
||||
node: m02
|
||||
port: "9433"
|
||||
dir: /tmp/sw-cp138-master
|
||||
extra_args: "-ip=10.0.0.3"
|
||||
save_as: master_pid
|
||||
|
||||
- action: sleep
|
||||
duration: 3s
|
||||
|
||||
# --- Phase 3: Failover (kill primary, promote replica) ---
|
||||
- action: start_weed_volume
|
||||
node: m02
|
||||
port: "18480"
|
||||
master: "10.0.0.3:9433"
|
||||
dir: /tmp/sw-cp138-vs1
|
||||
extra_args: "-block.dir=/tmp/sw-cp138-vs1/blocks -block.listen=:3295 -ip=10.0.0.3"
|
||||
save_as: vs1_pid
|
||||
|
||||
- action: start_weed_volume
|
||||
node: m01
|
||||
port: "18480"
|
||||
master: "10.0.0.3:9433"
|
||||
dir: /tmp/sw-cp138-vs2
|
||||
extra_args: "-block.dir=/tmp/sw-cp138-vs2/blocks -block.listen=:3295 -ip=10.0.0.1"
|
||||
save_as: vs2_pid
|
||||
|
||||
- action: sleep
|
||||
duration: 3s
|
||||
|
||||
- action: wait_cluster_ready
|
||||
node: m02
|
||||
master_url: "{{ master_url }}"
|
||||
|
||||
- action: wait_block_servers
|
||||
count: "2"
|
||||
|
||||
- action: create_block_volume
|
||||
name: "{{ volume_name }}"
|
||||
size_bytes: "{{ vol_size }}"
|
||||
replica_factor: "2"
|
||||
durability_mode: "sync_all"
|
||||
|
||||
# Wait for assignment delivery (heartbeat cycle).
|
||||
- action: sleep
|
||||
duration: 15s
|
||||
|
||||
# Bootstrap write: triggers shipper connect + first barrier.
|
||||
# Without this, the shipper stays degraded because barrier-triggered
|
||||
# recovery needs a write to fire the barrier.
|
||||
- action: lookup_block_volume
|
||||
name: "{{ volume_name }}"
|
||||
save_as: boot_vol
|
||||
|
||||
- action: iscsi_login_direct
|
||||
node: m01
|
||||
host: "{{ boot_vol_iscsi_host }}"
|
||||
port: "{{ boot_vol_iscsi_port }}"
|
||||
iqn: "{{ boot_vol_iqn }}"
|
||||
save_as: boot_device
|
||||
ignore_error: true
|
||||
|
||||
- action: exec
|
||||
node: m01
|
||||
root: "true"
|
||||
cmd: "dd if=/dev/urandom of={{ boot_device }} bs=4k count=1 seek=100000 oflag=direct,sync 2>/dev/null; true"
|
||||
ignore_error: true
|
||||
|
||||
- action: iscsi_cleanup
|
||||
node: m01
|
||||
ignore_error: true
|
||||
|
||||
- action: sleep
|
||||
duration: 5s
|
||||
|
||||
- action: wait_volume_healthy
|
||||
name: "{{ volume_name }}"
|
||||
timeout: 60s
|
||||
|
||||
- action: discover_primary
|
||||
name: "{{ volume_name }}"
|
||||
save_as: pri
|
||||
|
||||
- action: print
|
||||
msg: "CP13-8 setup: primary={{ pri }} ({{ pri_server }}), replica={{ pri_replica_node }}"
|
||||
|
||||
# --- Phase 2: ext4 filesystem workload on initial primary ---
|
||||
- name: ext4-write
|
||||
actions:
|
||||
- action: lookup_block_volume
|
||||
name: "{{ volume_name }}"
|
||||
save_as: vol
|
||||
|
||||
- action: iscsi_login_direct
|
||||
node: m01
|
||||
host: "{{ vol_iscsi_host }}"
|
||||
port: "{{ vol_iscsi_port }}"
|
||||
iqn: "{{ vol_iqn }}"
|
||||
save_as: device
|
||||
|
||||
- action: exec
|
||||
node: m01
|
||||
cmd: "mkfs.ext4 -F {{ device }} 2>&1 | tail -2"
|
||||
root: "true"
|
||||
|
||||
- action: exec
|
||||
node: m01
|
||||
cmd: "mkdir -p /mnt/cp13-8 && mount {{ device }} /mnt/cp13-8"
|
||||
root: "true"
|
||||
|
||||
- action: exec
|
||||
node: m01
|
||||
root: "true"
|
||||
cmd: "for i in $(seq 1 200); do dd if=/dev/urandom of=/mnt/cp13-8/file_$i bs=4k count=1 2>/dev/null; done && sync && echo WRITE_DONE"
|
||||
save_as: write_result
|
||||
|
||||
- action: assert_contains
|
||||
value: "{{ write_result }}"
|
||||
contains: "WRITE_DONE"
|
||||
|
||||
- action: exec
|
||||
node: m01
|
||||
root: "true"
|
||||
cmd: "md5sum /mnt/cp13-8/file_* | sort > /tmp/cp13-8-pre.md5 && wc -l < /tmp/cp13-8-pre.md5"
|
||||
save_as: pre_checksum_count
|
||||
|
||||
- action: assert_equal
|
||||
actual: "{{ pre_checksum_count }}"
|
||||
expected: "200"
|
||||
|
||||
- action: exec
|
||||
node: m01
|
||||
cmd: "umount /mnt/cp13-8"
|
||||
root: "true"
|
||||
|
||||
- action: iscsi_cleanup
|
||||
node: m01
|
||||
ignore_error: true
|
||||
|
||||
- action: print
|
||||
msg: "ext4-write: 200 files written, checksums captured"
|
||||
|
||||
# Verify replication is healthy after all writes.
|
||||
- action: wait_volume_healthy
|
||||
name: "{{ volume_name }}"
|
||||
timeout: 30s
|
||||
|
||||
# --- Phase 3: Failover ---
|
||||
# Kill ONLY the primary's VS, keep the replica alive for auto-promote.
|
||||
# Master allocates primary to m01 first (by server registration order).
|
||||
# Kill m01 VS (primary), m02 VS (replica) stays alive for promotion.
|
||||
- name: failover
|
||||
actions:
|
||||
- action: kill_target
|
||||
target: primary
|
||||
- action: assign
|
||||
target: replica
|
||||
epoch: "2"
|
||||
role: primary
|
||||
lease_ttl: 60s
|
||||
- action: wait_role
|
||||
target: replica
|
||||
role: primary
|
||||
timeout: 10s
|
||||
- action: print
|
||||
msg: "=== Killing primary VS on m01 ==="
|
||||
|
||||
- action: exec
|
||||
node: m01
|
||||
cmd: "kill -9 {{ vs2_pid }}"
|
||||
root: "true"
|
||||
ignore_error: true
|
||||
|
||||
# Wait for lease expiry (30s TTL) + auto-failover.
|
||||
- action: sleep
|
||||
duration: 50s
|
||||
|
||||
# Wait for primary to change from m01 to m02.
|
||||
- action: wait_block_primary
|
||||
name: "{{ volume_name }}"
|
||||
not: "{{ pri_server }}"
|
||||
timeout: 60s
|
||||
save_as: new_pri
|
||||
|
||||
- action: print
|
||||
msg: "Failover: {{ pri_server }} → {{ new_pri }}"
|
||||
|
||||
- action: sleep
|
||||
duration: 5s
|
||||
|
||||
# --- Phase 4: ext4 verification on promoted replica ---
|
||||
- name: ext4-verify
|
||||
actions:
|
||||
- action: iscsi_login
|
||||
target: replica
|
||||
node: client_node
|
||||
- action: discover_primary
|
||||
name: "{{ volume_name }}"
|
||||
save_as: new
|
||||
|
||||
- action: print
|
||||
msg: "Verifying ext4 on promoted node {{ new }} ({{ new_server }})"
|
||||
|
||||
# Connect to the new primary's iSCSI.
|
||||
- action: iscsi_login_direct
|
||||
node: m01
|
||||
host: "{{ new_host }}"
|
||||
port: "3295"
|
||||
iqn: "{{ vol_iqn }}"
|
||||
save_as: device2
|
||||
# fsck: filesystem integrity.
|
||||
|
||||
- action: fsck_ext4
|
||||
node: client_node
|
||||
node: m01
|
||||
device: "{{ device2 }}"
|
||||
save_as: fsck_result
|
||||
# Mount and verify file count.
|
||||
- action: mount
|
||||
node: client_node
|
||||
device: "{{ device2 }}"
|
||||
mountpoint: /mnt/cp13-8
|
||||
|
||||
- action: print
|
||||
msg: "fsck: {{ fsck_result }}"
|
||||
|
||||
- action: exec
|
||||
node: client_node
|
||||
node: m01
|
||||
cmd: "mkdir -p /mnt/cp13-8 && mount {{ device2 }} /mnt/cp13-8"
|
||||
root: "true"
|
||||
|
||||
- action: exec
|
||||
node: m01
|
||||
root: "true"
|
||||
cmd: "ls /mnt/cp13-8/file_* | wc -l"
|
||||
save_as: post_failover_count
|
||||
save_as: post_count
|
||||
|
||||
- action: assert_equal
|
||||
actual: "{{ post_failover_count }}"
|
||||
actual: "{{ post_count }}"
|
||||
expected: "200"
|
||||
# Verify checksums match pre-failover.
|
||||
|
||||
- action: exec
|
||||
node: client_node
|
||||
node: m01
|
||||
root: "true"
|
||||
cmd: "md5sum /mnt/cp13-8/file_* | sort > /tmp/cp13-8-checksums-post.txt && diff /tmp/cp13-8-checksums.txt /tmp/cp13-8-checksums-post.txt && echo MATCH"
|
||||
save_as: checksum_match
|
||||
cmd: "md5sum /mnt/cp13-8/file_* | sort > /tmp/cp13-8-post.md5 && diff /tmp/cp13-8-pre.md5 /tmp/cp13-8-post.md5 && echo CHECKSUM_MATCH"
|
||||
save_as: checksum_diff
|
||||
|
||||
- action: assert_contains
|
||||
value: "{{ checksum_match }}"
|
||||
contains: "MATCH"
|
||||
- action: umount
|
||||
node: client_node
|
||||
mountpoint: /mnt/cp13-8
|
||||
value: "{{ checksum_diff }}"
|
||||
contains: "CHECKSUM_MATCH"
|
||||
|
||||
- action: exec
|
||||
node: m01
|
||||
cmd: "umount /mnt/cp13-8"
|
||||
root: "true"
|
||||
|
||||
- action: iscsi_cleanup
|
||||
node: client_node
|
||||
node: m01
|
||||
ignore_error: true
|
||||
|
||||
# --- Phase 5: pgbench on promoted replica (application workload) ---
|
||||
- name: pgbench-on-replica
|
||||
- action: print
|
||||
msg: "ext4-verify: fsck CLEAN, 200 files, checksums MATCH"
|
||||
|
||||
# --- Phase 5: pgbench on promoted replica ---
|
||||
- name: pgbench
|
||||
actions:
|
||||
- action: iscsi_login
|
||||
target: replica
|
||||
node: client_node
|
||||
- action: iscsi_login_direct
|
||||
node: m01
|
||||
host: "{{ new_host }}"
|
||||
port: "3295"
|
||||
iqn: "{{ vol_iqn }}"
|
||||
save_as: device3
|
||||
|
||||
- action: sleep
|
||||
duration: 3s
|
||||
|
||||
- action: pgbench_init
|
||||
node: client_node
|
||||
node: m01
|
||||
device: "{{ device3 }}"
|
||||
mount: "/mnt/cp13-8-pg"
|
||||
port: "5440"
|
||||
port: "5441"
|
||||
scale: "1"
|
||||
fstype: ext4
|
||||
|
||||
- action: pgbench_run
|
||||
node: client_node
|
||||
node: m01
|
||||
clients: "1"
|
||||
duration: "10"
|
||||
save_as: tps_post_failover
|
||||
save_as: tps
|
||||
|
||||
- action: print
|
||||
msg: "CP13-8: pgbench TPC-B post-failover: {{ tps_post_failover }} TPS"
|
||||
msg: "CP13-8 pgbench TPS: {{ tps }}"
|
||||
|
||||
- action: assert_greater
|
||||
value: "{{ tps_post_failover }}"
|
||||
actual: "{{ tps }}"
|
||||
threshold: "0"
|
||||
# pgbench succeeded with TPS > 0 = database transactions are durable on the promoted replica.
|
||||
|
||||
- action: pgbench_cleanup
|
||||
node: client_node
|
||||
mount: "/mnt/cp13-8-pg"
|
||||
port: "5440"
|
||||
node: m01
|
||||
ignore_error: true
|
||||
|
||||
- action: iscsi_cleanup
|
||||
node: client_node
|
||||
node: m01
|
||||
ignore_error: true
|
||||
|
||||
# --- Phase 6: Cleanup ---
|
||||
- name: cleanup
|
||||
always: true
|
||||
actions:
|
||||
- action: exec
|
||||
node: m01
|
||||
cmd: "umount /mnt/cp13-8 /mnt/cp13-8-pg 2>/dev/null; true"
|
||||
root: "true"
|
||||
ignore_error: true
|
||||
- action: iscsi_cleanup
|
||||
node: client_node
|
||||
node: m01
|
||||
ignore_error: true
|
||||
- action: stop_all_targets
|
||||
- action: stop_weed
|
||||
node: m01
|
||||
pid: "{{ vs2_pid }}"
|
||||
ignore_error: true
|
||||
- action: stop_weed
|
||||
node: m01
|
||||
pid: "{{ vs2_new_pid }}"
|
||||
ignore_error: true
|
||||
- action: stop_weed
|
||||
node: m02
|
||||
pid: "{{ vs1_pid }}"
|
||||
ignore_error: true
|
||||
- action: stop_weed
|
||||
node: m02
|
||||
pid: "{{ vs1_new_pid }}"
|
||||
ignore_error: true
|
||||
- action: stop_weed
|
||||
node: m02
|
||||
pid: "{{ master_pid }}"
|
||||
ignore_error: true
|
||||
|
||||
@@ -118,10 +118,14 @@ func (bs *BlockVolumeStore) WithVolume(path string, fn func(*blockvol.BlockVol)
|
||||
return fn(vol)
|
||||
}
|
||||
|
||||
// ProcessBlockVolumeAssignments applies a batch of assignments from master.
|
||||
// Returns a slice of errors parallel to the input (nil = success).
|
||||
// Unknown volumes and invalid transitions are logged and returned as errors,
|
||||
// but do not stop processing of remaining assignments.
|
||||
// ProcessBlockVolumeAssignments applies only the local role/epoch/lease part of
|
||||
// a batch of assignments. It does NOT wire replica receivers, shippers, or
|
||||
// publication readiness. The authoritative runtime lifecycle lives in
|
||||
// BlockService.ApplyAssignments.
|
||||
//
|
||||
// Returns a slice of errors parallel to the input (nil = success). Unknown
|
||||
// volumes and invalid transitions are logged and returned as errors, but do not
|
||||
// stop processing of remaining assignments.
|
||||
func (bs *BlockVolumeStore) ProcessBlockVolumeAssignments(
|
||||
assignments []blockvol.BlockVolumeAssignment,
|
||||
) []error {
|
||||
|
||||
Reference in New Issue
Block a user