seaweedfs/weed/storage/blockvol/testrunner/regression.go

package testrunner

import (
	"fmt"
	"math"
	"strings"
)

// RegressionResult holds the comparison of one metric against baseline.
type RegressionResult struct {
	Metric       string
	BaselineVal  float64
	CurrentVal   float64
	ThresholdPct float64 // positive = allowed increase, negative = allowed decrease
	Pass         bool
	Reason       string
}

// RegressionReport is the full regression comparison output.
type RegressionReport struct {
	Results       []RegressionResult
	HardFails     []HardFailResult
	OverallPass   bool
	BaselineGitSHA string
	CurrentGitSHA  string
}

// HardFailResult is the result of a hard fail condition check.
type HardFailResult struct {
	Condition string
	Pass      bool
	Detail    string
}

// RegressionThresholds defines the allowed deviation per metric.
var RegressionThresholds = map[string]float64{
	"p99_write_latency_seconds": 0.10,  // fail if >10% increase
	"p99_read_latency_seconds":  0.10,  // fail if >10% increase
	"write_iops":                -0.05, // fail if >5% decrease
	"read_iops":                 -0.05, // fail if >5% decrease
}

// CompareBaseline runs regression checks against a baseline.
func CompareBaseline(baseline, current map[string]float64) []RegressionResult {
	var results []RegressionResult
	for metric, threshold := range RegressionThresholds {
		bv, bOk := baseline[metric]
		cv, cOk := current[metric]
		if !bOk || !cOk {
			continue
		}
		if bv == 0 {
			results = append(results, RegressionResult{
				Metric: metric, BaselineVal: bv, CurrentVal: cv,
				ThresholdPct: threshold, Pass: true, Reason: "baseline is 0, skipped",
			})
			continue
		}

		r := RegressionResult{
			Metric: metric, BaselineVal: bv, CurrentVal: cv,
			ThresholdPct: threshold,
		}

		pctChange := (cv - bv) / math.Abs(bv)
		if threshold >= 0 {
			// Positive threshold: fail if increase exceeds threshold.
			r.Pass = pctChange <= threshold
			r.Reason = fmt.Sprintf("change=%.1f%% (limit=+%.1f%%)", pctChange*100, threshold*100)
		} else {
			// Negative threshold: fail if decrease exceeds |threshold|.
			r.Pass = pctChange >= threshold
			r.Reason = fmt.Sprintf("change=%.1f%% (limit=%.1f%%)", pctChange*100, threshold*100)
		}
		results = append(results, r)
	}
	return results
}

// HardFailChecks are immediate test failure conditions (no comparison needed).
// Each check takes the current metrics map and returns pass/fail.
var HardFailChecks = []struct {
	Name  string
	Check func(m map[string]float64) (bool, string)
}{
	{
		Name: "data_mismatch",
		Check: func(m map[string]float64) (bool, string) {
			if v, ok := m["data_mismatch_count"]; ok && v > 0 {
				return false, fmt.Sprintf("%.0f data mismatches detected", v)
			}
			return true, ""
		},
	},
	{
		Name: "promotion_panic",
		Check: func(m map[string]float64) (bool, string) {
			if v, ok := m["promotion_panic_count"]; ok && v > 0 {
				return false, fmt.Sprintf("%.0f promotion panics", v)
			}
			return true, ""
		},
	},
	{
		Name: "scrub_false_positive_burst",
		Check: func(m map[string]float64) (bool, string) {
			if v, ok := m["scrub_false_positives_per_pass"]; ok && v > 3 {
				return false, fmt.Sprintf("%.0f false positives in one scrub pass (limit=3)", v)
			}
			return true, ""
		},
	},
	{
		Name: "barrier_lag_lsn_unbounded",
		Check: func(m map[string]float64) (bool, string) {
			if v, ok := m["barrier_lag_lsn_max"]; ok && v > 1000 {
				return false, fmt.Sprintf("barrier_lag_lsn=%.0f (limit=1000)", v)
			}
			return true, ""
		},
	},
	{
		Name: "barrier_error_rate",
		Check: func(m map[string]float64) (bool, string) {
			total, tOk := m["barrier_requests_total"]
			failed, fOk := m["barrier_failures_total"]
			if tOk && fOk && total > 0 && (failed/total) > 0.05 {
				return false, fmt.Sprintf("barrier error rate=%.1f%% (limit=5%%)", (failed/total)*100)
			}
			return true, ""
		},
	},
	{
		Name: "health_zero_without_fault",
		Check: func(m map[string]float64) (bool, string) {
			health, hOk := m["health_score"]
			faultActive, fOk := m["fault_active"]
			if hOk && health == 0.0 && (!fOk || faultActive == 0) {
				return false, "health score dropped to 0.0 without injected fault"
			}
			return true, ""
		},
	},
	{
		Name: "wal_full_stall",
		Check: func(m map[string]float64) (bool, string) {
			if v, ok := m["wal_full_duration_seconds"]; ok && v > 10 {
				return false, fmt.Sprintf("WAL full for %.1fs (limit=10s)", v)
			}
			return true, ""
		},
	},
}

// RunHardFailChecks evaluates all hard fail conditions against current metrics.
func RunHardFailChecks(metrics map[string]float64) []HardFailResult {
	results := make([]HardFailResult, 0, len(HardFailChecks))
	for _, hf := range HardFailChecks {
		pass, detail := hf.Check(metrics)
		results = append(results, HardFailResult{
			Condition: hf.Name,
			Pass:      pass,
			Detail:    detail,
		})
	}
	return results
}

// FormatRegressionReport produces a human-readable report string.
func FormatRegressionReport(r *RegressionReport) string {
	var sb strings.Builder
	sb.WriteString("=== SLO Regression Report ===\n")
	sb.WriteString(fmt.Sprintf("Baseline: %s  Current: %s\n\n", r.BaselineGitSHA, r.CurrentGitSHA))

	sb.WriteString("--- Metric Comparisons ---\n")
	for _, res := range r.Results {
		status := "PASS"
		if !res.Pass {
			status = "FAIL"
		}
		sb.WriteString(fmt.Sprintf("  [%s] %s: baseline=%.4f current=%.4f %s\n",
			status, res.Metric, res.BaselineVal, res.CurrentVal, res.Reason))
	}

	sb.WriteString("\n--- Hard Fail Conditions ---\n")
	for _, hf := range r.HardFails {
		status := "PASS"
		if !hf.Pass {
			status = "FAIL"
		}
		detail := hf.Detail
		if detail == "" {
			detail = "ok"
		}
		sb.WriteString(fmt.Sprintf("  [%s] %s: %s\n", status, hf.Condition, detail))
	}

	sb.WriteString(fmt.Sprintf("\nOverall: %s\n", map[bool]string{true: "PASS", false: "FAIL"}[r.OverallPass]))
	return sb.String()
}