Files
seaweedfs/weed/storage/blockvol/testrunner/actions/devops_test.go
T
pingqiuandClaude Opus 4.6 ebe95b6e2e fix: flusher OOM on multi-block writes + testrunner enhancements
Bug: flusher.go:336 allocated make([]byte, entryLen) per dirty block
instead of per unique WAL entry. A 4MB WriteLBA creates 1024 dirty map
entries (one per 4KB block), all sharing the same WAL offset. The flusher
read the full 4MB WAL entry 1024 times into separate buffers:
1024 × 4MB = 4GB per 4MB write → OOM on mkfs.ext4.

Root cause: flusher assumed 1:1 dirty-block-to-WAL-entry mapping.
WriteLBA supports multi-block writes but the flusher never deduplicated
shared WAL offsets.

Fix: deduplicate WAL reads by WalOffset in flushOnceLocked(). Multiple
dirty blocks from the same WAL entry share one read buffer and one
DecodeWALEntry call. Memory: O(WAL_entries × size) not O(blocks × size).
For a 4MB write: 4GB → 4MB.

Verified on hardware (m01/M02 25Gbps RoCE):
- Before: mkfs.ext4 → VS RSS 100MB→25GB → OOM killed
- After: mkfs.ext4 → VS RSS 129MB stable, mkfs succeeds
- pgbench TPC-B c=4: 1,248 TPS (RF=1, previously blocked by OOM)

Tests added:
- flusher_test.go: flush_multiblock_shared_wal_read (16 blocks share
  one WAL offset, flush dedup verified)
- flusher_test.go: flush_multiblock_data_correct (3 mixed multi-block
  writes, all data correct after flush)
- test/component/large_write_test.go: 7 component tests (single 4MB,
  sequential mkfs sim, concurrent, mixed sizes, production volume,
  flusher throughput 30s sustained)
- iscsi/large_write_mem_test.go: 2 iSCSI session memory tests (4MB
  R2T flow, slow device)

Testrunner enhancements (same commit — all tested on hardware):
- discover_primary action: maps primary IP → topology node name,
  supports alt_ips for multi-NIC (RoCE + management)
- NodeSpec.AltIPs field for multi-NIC node identification
- 5 new YAML scenarios: ec3, ec5, degraded sync_all/best_effort, pgbench
- All 13 hardware-verified scenarios PASS

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-02 14:24:10 -07:00

179 lines
4.4 KiB
Go

package actions
import (
"sort"
"testing"
tr "github.com/seaweedfs/seaweedfs/weed/storage/blockvol/testrunner"
)
func TestDevOpsActions_Registration(t *testing.T) {
registry := tr.NewRegistry()
RegisterDevOpsActions(registry)
expected := []string{
"build_deploy_weed",
"start_weed_master",
"start_weed_volume",
"stop_weed",
"wait_cluster_ready",
"create_block_volume",
"expand_block_volume",
"lookup_block_volume",
"delete_block_volume",
"wait_block_servers",
"cluster_status",
"wait_block_primary",
"assert_block_field",
"block_status",
"block_promote",
"wait_volume_healthy",
"discover_primary",
}
for _, name := range expected {
if _, err := registry.Get(name); err != nil {
t.Errorf("action %q not registered: %v", name, err)
}
}
}
func TestDevOpsActions_Tier(t *testing.T) {
registry := tr.NewRegistry()
RegisterDevOpsActions(registry)
byTier := registry.ListByTier()
devopsActions := byTier[tr.TierDevOps]
if len(devopsActions) != 17 {
t.Errorf("devops tier has %d actions, want 17", len(devopsActions))
}
// Verify all are in devops tier.
sort.Strings(devopsActions)
for _, name := range devopsActions {
if tier := registry.ActionTier(name); tier != tr.TierDevOps {
t.Errorf("action %q has tier %q, want devops", name, tier)
}
}
}
func TestDevOpsActions_TierGating(t *testing.T) {
registry := tr.NewRegistry()
RegisterDevOpsActions(registry)
// Without gating, all should be accessible.
if _, err := registry.Get("start_weed_master"); err != nil {
t.Errorf("ungated: %v", err)
}
// Enable only core tier — devops should be blocked.
registry.EnableTiers([]string{tr.TierCore})
if _, err := registry.Get("start_weed_master"); err == nil {
t.Error("expected error when devops tier is disabled")
}
// Enable devops tier — should work again.
registry.EnableTiers([]string{tr.TierDevOps})
if _, err := registry.Get("start_weed_master"); err != nil {
t.Errorf("devops enabled: %v", err)
}
}
func TestAllActions_Registration(t *testing.T) {
registry := tr.NewRegistry()
RegisterCore(registry)
RegisterBlockActions(registry)
RegisterISCSIActions(registry)
RegisterNVMeActions(registry)
RegisterIOActions(registry)
RegisterDevOpsActions(registry)
RegisterSnapshotActions(registry)
RegisterDatabaseActions(registry)
RegisterMetricsActions(registry)
RegisterK8sActions(registry)
byTier := registry.ListByTier()
// Verify tier counts.
if n := len(byTier[tr.TierCore]); n != 17 {
t.Errorf("core: %d, want 17", n)
}
if n := len(byTier[tr.TierBlock]); n != 64 {
t.Errorf("block: %d, want 64", n)
}
if n := len(byTier[tr.TierDevOps]); n != 17 {
t.Errorf("devops: %d, want 17", n)
}
if n := len(byTier[tr.TierChaos]); n != 5 {
t.Errorf("chaos: %d, want 5", n)
}
if n := len(byTier[TierK8s]); n != 14 {
t.Errorf("k8s: %d, want 14", n)
}
// Total should be 116 (115 prev + 1 recovery: measure_rebuild).
total := 0
for _, actions := range byTier {
total += len(actions)
}
if total != 117 {
t.Errorf("total actions: %d, want 117", total)
}
}
func TestK8sActions_Registration(t *testing.T) {
registry := tr.NewRegistry()
RegisterK8sActions(registry)
expected := []string{
"kubectl_apply",
"kubectl_delete",
"kubectl_get_field",
"kubectl_wait_condition",
"kubectl_set_image",
"kubectl_assert_exists",
"kubectl_assert_not_exists",
"kubectl_logs",
"kubectl_rollout_status",
"kubectl_exec",
"kubectl_delete_pod",
"kubectl_pod_ready_count",
"kubectl_label",
"kubectl_get_condition",
}
for _, name := range expected {
if _, err := registry.Get(name); err != nil {
t.Errorf("action %q not registered: %v", name, err)
}
}
byTier := registry.ListByTier()
if n := len(byTier[TierK8s]); n != 14 {
t.Errorf("k8s tier has %d actions, want 14", n)
}
}
func TestK8sActions_TierGating(t *testing.T) {
registry := tr.NewRegistry()
RegisterK8sActions(registry)
// Without gating, all should be accessible.
if _, err := registry.Get("kubectl_apply"); err != nil {
t.Errorf("ungated: %v", err)
}
// Enable only core tier — k8s should be blocked.
registry.EnableTiers([]string{tr.TierCore})
if _, err := registry.Get("kubectl_apply"); err == nil {
t.Error("expected error when k8s tier is disabled")
}
// Enable k8s tier — should work again.
registry.EnableTiers([]string{TierK8s})
if _, err := registry.Get("kubectl_apply"); err != nil {
t.Errorf("k8s enabled: %v", err)
}
}