Files
seaweedfs/weed/storage/blockvol/testrunner/console_test.go
T
Ping QiuandClaude Opus 4.6 da1b81d1c9 feat: CP8-3-1 durability modes + testrunner platform + 21 adversarial tests
Durability mode implementation (sync_all, sync_quorum, best_effort):
- DurabilityMode type with superblock persistence, parse/validate/string
- MakeDistributedSync mode-aware barrier enforcement in dist_group_commit
- blockerr sentinel package (ErrDurabilityBarrierFailed, ErrDurabilityQuorumLost)
- gRPC create path: mode validation, idempotent create consistency, partial cleanup
- F1: strict mode rejects partial replica provisioning with cleanup
- F3: empty heartbeat does not overwrite persisted strict mode
- F4: SCSI error mapping uses errors.Is sentinels (not string matching)
- Proto/wire/blockapi/CLI/UI plumbing for durability_mode field
- Observability dashboard: cluster health cards + per-volume columns

Testrunner platform (YAML-driven integration test framework):
- Engine, parser, registry, reporter (JUnit XML + HTML), metrics scraping
- 52 registered actions: block, iSCSI, I/O, fault injection, assertions
- Baseline regression framework with 7 hard-fail conditions
- 15 YAML scenarios (smoke, crash, HA, fault, consistency, snapshot)
- 49 unit tests for testrunner internals

QA adversarial suite (21 tests, all PASS):
- Idempotent create mode/RF mismatch detection
- Heartbeat mode downgrade prevention (F3)
- sync_all/sync_quorum partial replica enforcement (F1)
- Concurrent create race safety
- Failover/expand mode preservation
- Cleanup resilience when delete fails
- Master restart auto-register mode handling
- Superblock roundtrip all 3 modes
- Validate edge cases (mode×RF matrix)
- RequiredReplicas quorum math verification
- Sentinel error categorization

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-06 01:06:51 -08:00

333 lines
7.9 KiB
Go

package testrunner
import (
"context"
"encoding/json"
"fmt"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func TestConsole_ScenariosEndpoint(t *testing.T) {
dir := t.TempDir()
writeTestScenario(t, dir, "smoke.yaml", "Smoke Test", 2)
writeTestScenario(t, dir, "ha.yaml", "HA Test", 4)
console, baseURL := startTestConsole(t, dir)
_ = console
resp, err := http.Get(baseURL + "/api/scenarios")
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
t.Fatalf("status = %d", resp.StatusCode)
}
var scenarios []scenarioInfo
json.NewDecoder(resp.Body).Decode(&scenarios)
if len(scenarios) != 2 {
t.Fatalf("scenarios = %d, want 2", len(scenarios))
}
}
func TestConsole_StatusEndpoint_NoRun(t *testing.T) {
dir := t.TempDir()
_, baseURL := startTestConsole(t, dir)
resp, err := http.Get(baseURL + "/api/status")
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
var st statusResponse
json.NewDecoder(resp.Body).Decode(&st)
if st.Running {
t.Error("should not be running")
}
}
func TestConsole_RunEndpoint_MissingScenario(t *testing.T) {
dir := t.TempDir()
_, baseURL := startTestConsole(t, dir)
resp, err := http.Post(baseURL+"/api/run", "application/json",
strings.NewReader(`{"scenario":""}`))
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != 400 {
t.Errorf("status = %d, want 400", resp.StatusCode)
}
}
func TestConsole_RunEndpoint_InvalidScenario(t *testing.T) {
dir := t.TempDir()
_, baseURL := startTestConsole(t, dir)
resp, err := http.Post(baseURL+"/api/run", "application/json",
strings.NewReader(`{"scenario":"nonexistent.yaml"}`))
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != 400 {
t.Errorf("status = %d, want 400", resp.StatusCode)
}
}
func TestConsole_RunAndPollStatus(t *testing.T) {
dir := t.TempDir()
// Write a scenario with a registered action.
writeTestScenarioWithAction(t, dir, "test.yaml", "Test Run", "console_noop")
registry := NewRegistry()
registry.RegisterFunc("console_noop", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
return map[string]string{"value": "done"}, nil
})
console, baseURL := startTestConsoleWithRegistry(t, dir, registry)
_ = console
// Run the scenario.
resp, err := http.Post(baseURL+"/api/run", "application/json",
strings.NewReader(`{"scenario":"test.yaml"}`))
if err != nil {
t.Fatal(err)
}
resp.Body.Close()
if resp.StatusCode != 202 {
t.Fatalf("status = %d, want 202", resp.StatusCode)
}
// Poll until done.
deadline := time.After(5 * time.Second)
for {
select {
case <-deadline:
t.Fatal("timeout waiting for scenario to complete")
case <-time.After(100 * time.Millisecond):
}
statusResp, err := http.Get(baseURL + "/api/status")
if err != nil {
continue
}
var st statusResponse
json.NewDecoder(statusResp.Body).Decode(&st)
statusResp.Body.Close()
if !st.Running && st.Status != "" {
if st.Status != "done" {
t.Errorf("status = %s, want done", st.Status)
}
break
}
}
// Check result endpoint.
resultResp, err := http.Get(baseURL + "/api/result/test")
if err != nil {
t.Fatal(err)
}
defer resultResp.Body.Close()
if resultResp.StatusCode != 200 {
t.Fatalf("result status = %d", resultResp.StatusCode)
}
var result ScenarioResult
json.NewDecoder(resultResp.Body).Decode(&result)
if result.Status != StatusPass {
t.Errorf("result.Status = %s, want PASS", result.Status)
}
}
func TestConsole_ConflictOnDoubleRun(t *testing.T) {
dir := t.TempDir()
writeTestScenarioWithAction(t, dir, "slow.yaml", "Slow Test", "console_slow")
registry := NewRegistry()
registry.RegisterFunc("console_slow", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
time.Sleep(2 * time.Second)
return nil, nil
})
_, baseURL := startTestConsoleWithRegistry(t, dir, registry)
// Start first run.
resp1, _ := http.Post(baseURL+"/api/run", "application/json",
strings.NewReader(`{"scenario":"slow.yaml"}`))
resp1.Body.Close()
if resp1.StatusCode != 202 {
t.Fatalf("first run status = %d", resp1.StatusCode)
}
// Wait a moment for it to start.
time.Sleep(100 * time.Millisecond)
// Try second run — should get 409.
resp2, _ := http.Post(baseURL+"/api/run", "application/json",
strings.NewReader(`{"scenario":"slow.yaml"}`))
resp2.Body.Close()
if resp2.StatusCode != 409 {
t.Errorf("second run status = %d, want 409", resp2.StatusCode)
}
}
func TestConsole_AgentsEndpoint(t *testing.T) {
dir := t.TempDir()
_, baseURL := startTestConsole(t, dir)
resp, err := http.Get(baseURL + "/api/agents")
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
var agents []agentInfo
json.NewDecoder(resp.Body).Decode(&agents)
// No agents connected by default.
if len(agents) != 0 {
t.Errorf("agents = %d, want 0", len(agents))
}
}
func TestConsole_TiersEndpoint(t *testing.T) {
dir := t.TempDir()
registry := NewRegistry()
registry.RegisterFunc("a", TierCore, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
return nil, nil
})
registry.RegisterFunc("b", TierBlock, func(ctx context.Context, actx *ActionContext, act Action) (map[string]string, error) {
return nil, nil
})
_, baseURL := startTestConsoleWithRegistry(t, dir, registry)
resp, err := http.Get(baseURL + "/api/tiers")
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
var tiers map[string][]string
json.NewDecoder(resp.Body).Decode(&tiers)
if len(tiers["core"]) != 1 || tiers["core"][0] != "a" {
t.Errorf("core tier = %v", tiers["core"])
}
if len(tiers["block"]) != 1 || tiers["block"][0] != "b" {
t.Errorf("block tier = %v", tiers["block"])
}
}
func TestConsole_ReportNotFound(t *testing.T) {
dir := t.TempDir()
_, baseURL := startTestConsole(t, dir)
resp, err := http.Get(baseURL + "/api/report/nonexistent")
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != 404 {
t.Errorf("status = %d, want 404", resp.StatusCode)
}
}
func TestConsole_IndexPage(t *testing.T) {
dir := t.TempDir()
_, baseURL := startTestConsole(t, dir)
resp, err := http.Get(baseURL + "/")
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
t.Fatalf("status = %d", resp.StatusCode)
}
ct := resp.Header.Get("Content-Type")
if !strings.Contains(ct, "text/html") {
t.Errorf("content-type = %s, want text/html", ct)
}
}
// --- Helpers ---
func startTestConsole(t *testing.T, scenariosDir string) (*Console, string) {
t.Helper()
return startTestConsoleWithRegistry(t, scenariosDir, NewRegistry())
}
func startTestConsoleWithRegistry(t *testing.T, scenariosDir string, registry *Registry) (*Console, string) {
t.Helper()
console := NewConsole(ConsoleConfig{
Port: 0,
ScenarioDir: scenariosDir,
Registry: registry,
})
// Override to use ephemeral port — bind directly.
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
go console.Start(ctx)
// Wait for listener.
for i := 0; i < 50; i++ {
if console.ListenAddr() != "" {
break
}
time.Sleep(20 * time.Millisecond)
}
addr := console.ListenAddr()
if addr == "" {
t.Fatal("console failed to start")
}
return console, fmt.Sprintf("http://%s", addr)
}
func writeTestScenario(t *testing.T, dir, filename, name string, phases int) {
t.Helper()
content := fmt.Sprintf("name: %s\ntimeout: 30s\nphases:\n", name)
for i := 0; i < phases; i++ {
content += fmt.Sprintf(" - name: phase%d\n actions:\n - action: noop\n", i)
}
os.WriteFile(filepath.Join(dir, filename), []byte(content), 0644)
}
func writeTestScenarioWithAction(t *testing.T, dir, filename, name, action string) {
t.Helper()
content := fmt.Sprintf(`name: %s
timeout: 30s
phases:
- name: main
actions:
- action: %s
save_as: result
`, name, action)
os.WriteFile(filepath.Join(dir, filename), []byte(content), 0644)
}