seaweedfs/weed/storage/blockvol/test/perf_test.go

//go:build integration

package test

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"
	"testing"
	"time"
)

func TestPerf(t *testing.T) {
	if *flagEnv != "remote" {
		t.Skip("perf tests require remote mode (-env remote)")
	}
	if !clientNode.HasCommand("fio") {
		t.Skip("fio required")
	}

	t.Run("GoBench", testPerfGoBench)
	t.Run("FioRandWrite", testPerfFioRandWrite)
	t.Run("FioRandRead", testPerfFioRandRead)
	t.Run("LatencyP99", testPerfLatencyP99)
}

func testPerfGoBench(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
	defer cancel()

	benchDir := "/opt/work/seaweedfs/weed/storage/blockvol"
	stdout, stderr, code, err := targetNode.Run(ctx,
		fmt.Sprintf("cd %s && go test -run=^$ -bench=. -benchmem -count=1 -timeout=5m ./...", benchDir))
	if err != nil || code != 0 {
		t.Fatalf("go bench: code=%d stderr=%s err=%v", code, stderr, err)
	}

	t.Log(stdout)
	artifacts.CollectPerf(t, "gobench", stdout)
}

func testPerfFioRandWrite(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
	defer cancel()

	tgt, iscsi, host := newTestTarget(t, "1G", "")
	dev := startAndLogin(t, ctx, tgt, iscsi, host)

	stdout, stderr, code, _ := clientNode.RunRoot(ctx,
		fmt.Sprintf("fio --name=randwrite --filename=%s --rw=randwrite "+
			"--bs=4k --size=500M --direct=1 --ioengine=libaio --iodepth=32 "+
			"--numjobs=4 --runtime=120 --time_based --group_reporting "+
			"--output-format=json", dev))
	if code != 0 {
		t.Fatalf("fio: code=%d stderr=%s", code, stderr)
	}

	iops := extractIOPS(stdout, "write")
	t.Logf("random write IOPS: %.0f", iops)
	if iops < 10000 {
		t.Fatalf("IOPS %.0f below threshold 10000", iops)
	}

	artifacts.CollectPerf(t, "fio-randwrite", stdout)
}

func testPerfFioRandRead(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
	defer cancel()

	tgt, iscsi, host := newTestTarget(t, "1G", "")
	dev := startAndLogin(t, ctx, tgt, iscsi, host)

	// Pre-fill
	clientNode.RunRoot(ctx,
		fmt.Sprintf("fio --name=prefill --filename=%s --rw=write --bs=1M "+
			"--size=500M --direct=1 --ioengine=libaio", dev))

	stdout, stderr, code, _ := clientNode.RunRoot(ctx,
		fmt.Sprintf("fio --name=randread --filename=%s --rw=randread "+
			"--bs=4k --size=500M --direct=1 --ioengine=libaio --iodepth=32 "+
			"--numjobs=4 --runtime=120 --time_based --group_reporting "+
			"--output-format=json", dev))
	if code != 0 {
		t.Fatalf("fio: code=%d stderr=%s", code, stderr)
	}

	iops := extractIOPS(stdout, "read")
	t.Logf("random read IOPS: %.0f", iops)
	if iops < 10000 {
		t.Fatalf("IOPS %.0f below threshold 10000", iops)
	}

	artifacts.CollectPerf(t, "fio-randread", stdout)
}

func testPerfLatencyP99(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
	defer cancel()

	tgt, iscsi, host := newTestTarget(t, "1G", "")
	dev := startAndLogin(t, ctx, tgt, iscsi, host)

	stdout, stderr, code, _ := clientNode.RunRoot(ctx,
		fmt.Sprintf("fio --name=latency --filename=%s --rw=randwrite "+
			"--bs=4k --size=500M --direct=1 --ioengine=libaio --iodepth=1 "+
			"--numjobs=1 --runtime=60 --time_based "+
			"--lat_percentiles=1 --output-format=json", dev))
	if code != 0 {
		t.Fatalf("fio: code=%d stderr=%s", code, stderr)
	}

	p99 := extractP99Latency(stdout) // nanoseconds (fio clat_ns)
	p99ms := p99 / 1_000_000       // ns -> ms
	t.Logf("P99 latency: %.2f ms", p99ms)
	if p99ms > 10 {
		t.Fatalf("P99 %.2fms exceeds 10ms threshold", p99ms)
	}

	artifacts.CollectPerf(t, "fio-latency", stdout)
}

// extractIOPS parses fio JSON output for IOPS.
func extractIOPS(fioJSON string, rw string) float64 {
	var result struct {
		Jobs []struct {
			Read  struct{ IOPS float64 `json:"iops"` } `json:"read"`
			Write struct{ IOPS float64 `json:"iops"` } `json:"write"`
		} `json:"jobs"`
	}
	if err := json.Unmarshal([]byte(fioJSON), &result); err != nil {
		return 0
	}
	if len(result.Jobs) == 0 {
		return 0
	}
	if rw == "read" {
		return result.Jobs[0].Read.IOPS
	}
	return result.Jobs[0].Write.IOPS
}

// extractP99Latency parses fio JSON output for P99 latency in microseconds.
func extractP99Latency(fioJSON string) float64 {
	// Look for clat_ns percentile 99.000000
	idx := strings.Index(fioJSON, "99.000000")
	if idx < 0 {
		return 0
	}
	// Find the value after the colon
	sub := fioJSON[idx:]
	colonIdx := strings.Index(sub, ":")
	if colonIdx < 0 {
		return 0
	}
	valStr := strings.TrimSpace(sub[colonIdx+1:])
	// Take until comma or closing bracket
	for i, c := range valStr {
		if c == ',' || c == '}' || c == ']' {
			valStr = valStr[:i]
			break
		}
	}
	var val float64
	fmt.Sscanf(strings.TrimSpace(valStr), "%f", &val)
	return val
}