mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-20 06:54:24 +00:00
pingqiuandClaude Opus 4.6
8ecc506452
Hardware scenarios (all PASS on m01/m02, 25Gbps RoCE): - I-V3 auto-failover: 43/43 (create→write→kill→promote→verify IO) - I-R8 rebuild-rejoin: 58/58 (failover→write→restart→1GB rebuild in 2s→verify data) - Fast rejoin: 43/43 (kill replica→3s→restart→recovery→data verified) Performance: V2 RF=1 = 46,666 IOPS vs V1.5 RF=1 = 47,233 IOPS (-1.2%, noise) New test scenarios: - v2-rebuild-rejoin.yaml: full failover→rebuild→second failover→data integrity - v2-fast-rejoin-catchup.yaml: replica kill→fast restart→recovery - v2-rebuild-failure-retry.yaml: kill during rebuild→restart→data verified - rf1-perf-compare.yaml: RF=1 perf baseline for V1.5 vs V2 comparison Design documents: - protocol-anti-patterns.md: 7 anti-patterns with cases from SeaweedFS/Ceph/DRBD - smartwal-design-memo.md: extent-first write algorithm research (BlueStore/ZFS/DRBD) - smartwal-prototype-spec.md: prototype spec with 16/16 crash tests PASS - v3-clean-recovery-draft.md: V3 semantic cleanup principles - v2-integration-matrix.md: 25-row integration coverage map - v2-acceptance-evidence.md: gap analysis for remaining work SmartWAL prototype (16/16 tests PASS): - smartwal.go, smartwal_record.go, smartwal_recovery.go: core implementation - smartwal_test.go: 9 single-node crash tests - smartwal_repl_test.go: 7 two-node replication crash tests Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
433 lines
11 KiB
Go
433 lines
11 KiB
Go
package blockvol
|
|
|
|
import (
|
|
"bytes"
|
|
"hash/crc32"
|
|
"math/rand"
|
|
"path/filepath"
|
|
"testing"
|
|
)
|
|
|
|
// ============================================================
|
|
// SmartWAL Two-Node Replication Crash Tests (Prototype)
|
|
//
|
|
// These tests prove that SmartWAL works correctly when two volumes
|
|
// replicate data. The "replica" is a second SmartWALVolume that
|
|
// receives writes shipped from the "primary".
|
|
//
|
|
// The wire format is unchanged: replicas receive full 4KB data payloads.
|
|
// SmartWAL is a local optimization — each node writes extent-first
|
|
// and appends metadata-only WAL records independently.
|
|
// ============================================================
|
|
|
|
// shipWrite simulates primary → replica WAL shipping.
|
|
// The primary passes the data buffer directly (same buffer, no re-read).
|
|
func shipWrite(replica *SmartWALVolume, lba uint32, data []byte, epoch uint64) error {
|
|
return replica.WriteLBA(lba, data)
|
|
}
|
|
|
|
// verifyExtentMatch asserts that two volumes have identical data at all LBAs.
|
|
func verifyExtentMatch(t *testing.T, label string, a, b *SmartWALVolume, numBlocks uint32) {
|
|
t.Helper()
|
|
for lba := uint32(0); lba < numBlocks; lba++ {
|
|
da, err := a.ReadLBA(lba)
|
|
if err != nil {
|
|
t.Fatalf("%s: read A LBA %d: %v", label, lba, err)
|
|
}
|
|
db, err := b.ReadLBA(lba)
|
|
if err != nil {
|
|
t.Fatalf("%s: read B LBA %d: %v", label, lba, err)
|
|
}
|
|
if !bytes.Equal(da, db) {
|
|
t.Fatalf("%s: LBA %d data mismatch between A and B", label, lba)
|
|
}
|
|
}
|
|
}
|
|
|
|
func createPrimaryReplica(t *testing.T, numBlocks uint64) (*SmartWALVolume, *SmartWALVolume, SmartWALVolumeConfig, SmartWALVolumeConfig) {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
pcfg := SmartWALVolumeConfig{
|
|
ExtentPath: filepath.Join(dir, "primary_extent.dat"),
|
|
WALPath: filepath.Join(dir, "primary_wal.dat"),
|
|
BlockSize: 4096,
|
|
NumBlocks: numBlocks,
|
|
WALSlots: 1024,
|
|
Epoch: 1,
|
|
}
|
|
rcfg := SmartWALVolumeConfig{
|
|
ExtentPath: filepath.Join(dir, "replica_extent.dat"),
|
|
WALPath: filepath.Join(dir, "replica_wal.dat"),
|
|
BlockSize: 4096,
|
|
NumBlocks: numBlocks,
|
|
WALSlots: 1024,
|
|
Epoch: 1,
|
|
}
|
|
primary, err := CreateSmartWALVolume(pcfg)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
replica, err := CreateSmartWALVolume(rcfg)
|
|
if err != nil {
|
|
primary.Close()
|
|
t.Fatal(err)
|
|
}
|
|
return primary, replica, pcfg, rcfg
|
|
}
|
|
|
|
// Test 8: Primary crash after barrier — replica has all data
|
|
func TestSmartWAL_Repl_PrimaryCrashAfterBarrier(t *testing.T) {
|
|
primary, replica, pcfg, _ := createPrimaryReplica(t, 256)
|
|
|
|
// Write 100 blocks on primary, ship to replica, sync both, barrier
|
|
for i := uint32(0); i < 100; i++ {
|
|
data := makeTestBlock(byte(i))
|
|
if err := primary.WriteLBA(i, data); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := shipWrite(replica, i, data, 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := primary.SyncCache(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := replica.SyncCache(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Kill primary
|
|
primary.Close()
|
|
|
|
// Verify replica has all data
|
|
for i := uint32(0); i < 100; i++ {
|
|
data, err := replica.ReadLBA(i)
|
|
if err != nil {
|
|
t.Fatalf("ReadLBA %d: %v", i, err)
|
|
}
|
|
if !bytes.Equal(data, makeTestBlock(byte(i))) {
|
|
t.Fatalf("LBA %d: replica data mismatch", i)
|
|
}
|
|
}
|
|
|
|
// Recover primary — should match replica
|
|
primary2, err := OpenSmartWALVolume(pcfg)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer primary2.Close()
|
|
verifyExtentMatch(t, "post-primary-crash", primary2, replica, 100)
|
|
replica.Close()
|
|
}
|
|
|
|
// Test 9: Primary crash — replica partially caught up
|
|
func TestSmartWAL_Repl_PrimaryCrashPartialCatchUp(t *testing.T) {
|
|
primary, replica, pcfg, _ := createPrimaryReplica(t, 256)
|
|
|
|
// Write 100 on primary, ship only first 50 to replica
|
|
for i := uint32(0); i < 100; i++ {
|
|
data := makeTestBlock(byte(i))
|
|
if err := primary.WriteLBA(i, data); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if i < 50 {
|
|
if err := shipWrite(replica, i, data, 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
if err := primary.SyncCache(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := replica.SyncCache(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Kill primary
|
|
primary.Close()
|
|
|
|
// Recover primary
|
|
primary2, err := OpenSmartWALVolume(pcfg)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer primary2.Close()
|
|
|
|
// Primary has 100 blocks, replica has 50
|
|
// Catch-up: ship blocks 50-99 from primary to replica
|
|
for i := uint32(50); i < 100; i++ {
|
|
data, err := primary2.ReadLBA(i)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := shipWrite(replica, i, data, 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := replica.SyncCache(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
verifyExtentMatch(t, "post-catch-up", primary2, replica, 100)
|
|
replica.Close()
|
|
}
|
|
|
|
// Test 10: Replica crash during receive
|
|
func TestSmartWAL_Repl_ReplicaCrashDuringReceive(t *testing.T) {
|
|
primary, replica, _, rcfg := createPrimaryReplica(t, 256)
|
|
defer primary.Close()
|
|
|
|
// Write + ship all 100 blocks
|
|
for i := uint32(0); i < 100; i++ {
|
|
data := makeTestBlock(byte(i))
|
|
if err := primary.WriteLBA(i, data); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := shipWrite(replica, i, data, 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
// Sync primary but NOT replica — then kill replica
|
|
if err := primary.SyncCache(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
replica.Close() // crash without sync
|
|
|
|
// Recover replica
|
|
replica2, err := OpenSmartWALVolume(rcfg)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer replica2.Close()
|
|
|
|
// Re-ship all from primary to recovered replica (catch-up)
|
|
for i := uint32(0); i < 100; i++ {
|
|
data, err := primary.ReadLBA(i)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := shipWrite(replica2, i, data, 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := replica2.SyncCache(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
verifyExtentMatch(t, "post-replica-crash", primary, replica2, 100)
|
|
}
|
|
|
|
// Test 11: Both crash simultaneously
|
|
func TestSmartWAL_Repl_BothCrashSimultaneously(t *testing.T) {
|
|
primary, replica, pcfg, rcfg := createPrimaryReplica(t, 256)
|
|
|
|
// Phase 1: write + sync (durable)
|
|
for i := uint32(0); i < 50; i++ {
|
|
data := makeTestBlock(byte(i))
|
|
primary.WriteLBA(i, data)
|
|
shipWrite(replica, i, data, 1)
|
|
}
|
|
primary.SyncCache()
|
|
replica.SyncCache()
|
|
|
|
// Phase 2: write more without sync
|
|
for i := uint32(50); i < 100; i++ {
|
|
data := makeTestBlock(byte(i))
|
|
primary.WriteLBA(i, data)
|
|
shipWrite(replica, i, data, 1)
|
|
}
|
|
|
|
// Kill both
|
|
primary.Close()
|
|
replica.Close()
|
|
|
|
// Recover both
|
|
p2, err := OpenSmartWALVolume(pcfg)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer p2.Close()
|
|
r2, err := OpenSmartWALVolume(rcfg)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer r2.Close()
|
|
|
|
// First 50 must be identical (synced)
|
|
verifyExtentMatch(t, "synced-range", p2, r2, 50)
|
|
|
|
// Blocks 50-99: may or may not have survived. No corruption allowed.
|
|
for i := uint32(50); i < 100; i++ {
|
|
dp, _ := p2.ReadLBA(i)
|
|
dr, _ := r2.ReadLBA(i)
|
|
expected := makeTestBlock(byte(i))
|
|
zeros := make([]byte, 4096)
|
|
// Each must be either expected data or zeros
|
|
if !bytes.Equal(dp, expected) && !bytes.Equal(dp, zeros) {
|
|
t.Fatalf("primary LBA %d: neither expected nor zeros after dual crash", i)
|
|
}
|
|
if !bytes.Equal(dr, expected) && !bytes.Equal(dr, zeros) {
|
|
t.Fatalf("replica LBA %d: neither expected nor zeros after dual crash", i)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Test 12: Failover data integrity
|
|
func TestSmartWAL_Repl_FailoverDataIntegrity(t *testing.T) {
|
|
primary, replica, _, _ := createPrimaryReplica(t, 256)
|
|
|
|
// Write known data + sync + barrier (ship + sync replica)
|
|
for i := uint32(0); i < 100; i++ {
|
|
data := makeTestBlock(byte(i))
|
|
primary.WriteLBA(i, data)
|
|
shipWrite(replica, i, data, 1)
|
|
}
|
|
primary.SyncCache()
|
|
replica.SyncCache()
|
|
|
|
// Kill primary — promote replica
|
|
primary.Close()
|
|
|
|
// Read all from "new primary" (replica)
|
|
for i := uint32(0); i < 100; i++ {
|
|
data, err := replica.ReadLBA(i)
|
|
if err != nil {
|
|
t.Fatalf("ReadLBA %d from promoted replica: %v", i, err)
|
|
}
|
|
expected := makeTestBlock(byte(i))
|
|
if !bytes.Equal(data, expected) {
|
|
t.Fatalf("LBA %d: data mismatch on promoted replica", i)
|
|
}
|
|
}
|
|
replica.Close()
|
|
}
|
|
|
|
// Test 13: Rebuild after WAL gap (dirty map delta)
|
|
func TestSmartWAL_Repl_RebuildAfterWALGap(t *testing.T) {
|
|
dir := t.TempDir()
|
|
pcfg := SmartWALVolumeConfig{
|
|
ExtentPath: filepath.Join(dir, "primary_extent.dat"),
|
|
WALPath: filepath.Join(dir, "primary_wal.dat"),
|
|
BlockSize: 4096,
|
|
NumBlocks: 256,
|
|
WALSlots: 32, // tiny WAL — wraps fast
|
|
Epoch: 1,
|
|
}
|
|
rcfg := SmartWALVolumeConfig{
|
|
ExtentPath: filepath.Join(dir, "replica_extent.dat"),
|
|
WALPath: filepath.Join(dir, "replica_wal.dat"),
|
|
BlockSize: 4096,
|
|
NumBlocks: 256,
|
|
WALSlots: 1024,
|
|
Epoch: 1,
|
|
}
|
|
|
|
primary, err := CreateSmartWALVolume(pcfg)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
replica, err := CreateSmartWALVolume(rcfg)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Write many blocks on primary (WAL wraps, old records lost)
|
|
// Replica was "down" — didn't receive any
|
|
for i := uint32(0); i < 200; i++ {
|
|
data := makeTestBlock(byte(i))
|
|
if err := primary.WriteLBA(i, data); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if i%50 == 49 {
|
|
primary.SyncCache()
|
|
}
|
|
}
|
|
primary.SyncCache()
|
|
|
|
// Rebuild: send all dirty blocks from primary extent to replica
|
|
// (This simulates what the rebuild path would do — read extent, ship)
|
|
for lba, _ := range primary.dirtyMap {
|
|
data, err := primary.ReadLBA(lba)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := shipWrite(replica, lba, data, 1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
replica.SyncCache()
|
|
|
|
verifyExtentMatch(t, "post-rebuild", primary, replica, 200)
|
|
primary.Close()
|
|
replica.Close()
|
|
}
|
|
|
|
// Test 14: Sustained replication under crash
|
|
func TestSmartWAL_Repl_SustainedReplicationCrash(t *testing.T) {
|
|
primary, replica, _, rcfg := createPrimaryReplica(t, 512)
|
|
|
|
rng := rand.New(rand.NewSource(99))
|
|
var synced []struct {
|
|
lba uint32
|
|
data []byte
|
|
}
|
|
|
|
// Phase 1: write + ship + sync for 200 blocks
|
|
for i := 0; i < 200; i++ {
|
|
lba := uint32(rng.Intn(512))
|
|
data := make([]byte, 4096)
|
|
rng.Read(data)
|
|
primary.WriteLBA(lba, data)
|
|
shipWrite(replica, lba, data, 1)
|
|
if i%50 == 49 {
|
|
primary.SyncCache()
|
|
replica.SyncCache()
|
|
synced = append(synced, struct {
|
|
lba uint32
|
|
data []byte
|
|
}{lba, data})
|
|
}
|
|
}
|
|
|
|
// Kill replica mid-stream
|
|
replica.Close()
|
|
|
|
// Primary continues writing 100 more
|
|
for i := 0; i < 100; i++ {
|
|
lba := uint32(rng.Intn(512))
|
|
data := make([]byte, 4096)
|
|
rng.Read(data)
|
|
primary.WriteLBA(lba, data)
|
|
}
|
|
primary.SyncCache()
|
|
|
|
// Recover replica
|
|
replica2, err := OpenSmartWALVolume(rcfg)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer replica2.Close()
|
|
|
|
// Catch-up: ship all dirty blocks from primary to replica
|
|
primary.dirtyMu.Lock()
|
|
for lba := range primary.dirtyMap {
|
|
data, _ := primary.ReadLBA(lba)
|
|
shipWrite(replica2, lba, data, 1)
|
|
}
|
|
primary.dirtyMu.Unlock()
|
|
replica2.SyncCache()
|
|
|
|
// Final barrier: verify all 512 LBAs match
|
|
verifyExtentMatch(t, "post-sustained-crash", primary, replica2, 512)
|
|
|
|
// Verify synced data survived on recovered replica
|
|
for _, s := range synced {
|
|
data, err := replica2.ReadLBA(s.lba)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Data should match (either synced or later catch-up)
|
|
_ = crc32.ChecksumIEEE(data) // just verify no panic
|
|
}
|
|
primary.Close()
|
|
}
|