seaweedfs/weed/storage/blockvol/test/stress_test.go

//go:build integration

package test

import (
	"context"
	"fmt"
	"strings"
	"testing"
	"time"
)

func TestStress(t *testing.T) {
	if !clientNode.HasCommand("fio") {
		t.Skip("fio required for stress tests")
	}
	t.Run("Fio5Min", testStressFio5Min)
	t.Run("WALPressure", testStressWALPressure)
	t.Run("SyncBatch", testStressSyncBatch)
	t.Run("TarExtract", testStressTarExtract)
	t.Run("Soak30Min", testStressSoak30Min)
	t.Run("MixedBlockSize", testStressMixedBlockSize)
}

func testStressFio5Min(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
	defer cancel()

	tgt, iscsi, host := newTestTarget(t, "200M", "")
	dev := startAndLogin(t, ctx, tgt, iscsi, host)

	runtime := 300
	if testing.Short() {
		runtime = 30
	}
	stdout, stderr, code, _ := clientNode.RunRoot(ctx,
		fmt.Sprintf("fio --name=stress5m --filename=%s --rw=randrw --verify=crc32 "+
			"--bs=4k --size=100M --randrepeat=1 --direct=1 --ioengine=libaio "+
			"--runtime=%d --time_based --output-format=json", dev, runtime))
	if code != 0 {
		t.Fatalf("fio: code=%d stderr=%s", code, stderr)
	}
	if strings.Contains(stdout, "\"verify_errors\"") && !strings.Contains(stdout, "\"verify_errors\" : 0") {
		t.Fatal("fio verify errors")
	}
	t.Log("5-minute fio randrw+verify passed")
}

func testStressWALPressure(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
	defer cancel()

	tgt, iscsi, host := newTestTarget(t, "100M", "4M") // small WAL
	dev := startAndLogin(t, ctx, tgt, iscsi, host)

	// Write more than WAL size to force WAL wrap
	stdout, stderr, code, _ := clientNode.RunRoot(ctx,
		fmt.Sprintf("fio --name=walpressure --filename=%s --rw=write --bs=64k "+
			"--size=50M --direct=1 --ioengine=libaio", dev))
	if code != 0 {
		t.Fatalf("fio: code=%d stderr=%s stdout=%s", code, stderr, stdout)
	}
	t.Log("WAL pressure test passed (4MB WAL, 50MB write)")
}

func testStressSyncBatch(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
	defer cancel()

	tgt, iscsi, host := newTestTarget(t, "100M", "")
	dev := startAndLogin(t, ctx, tgt, iscsi, host)

	stdout, stderr, code, _ := clientNode.RunRoot(ctx,
		fmt.Sprintf("fio --name=syncbatch --filename=%s --rw=randwrite --bs=4k "+
			"--size=50M --fdatasync=1 --numjobs=16 --direct=1 --ioengine=libaio "+
			"--runtime=60 --time_based --group_reporting --output-format=json", dev))
	if code != 0 {
		t.Fatalf("fio: code=%d stderr=%s", code, stderr)
	}

	// Extract IOPS from output
	if idx := strings.Index(stdout, "\"iops\""); idx >= 0 {
		end := idx + 30
		if end > len(stdout) {
			end = len(stdout)
		}
		t.Logf("sync batch IOPS: %s...", stdout[idx:end])
	}
	t.Log("sync batch test passed (16 jobs, fdatasync)")
}

func testStressTarExtract(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
	defer cancel()

	tgt, iscsi, host := newTestTarget(t, "200M", "")
	dev := startAndLogin(t, ctx, tgt, iscsi, host)
	mnt := "/tmp/blockvol-mnt"

	t.Cleanup(func() {
		cleanCtx, cleanCancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cleanCancel()
		clientNode.RunRoot(cleanCtx, fmt.Sprintf("umount -f %s 2>/dev/null", mnt))
	})

	// mkfs + mount
	clientNode.RunRoot(ctx, fmt.Sprintf("mkfs.ext4 -F %s", dev))
	clientNode.RunRoot(ctx, fmt.Sprintf("mkdir -p %s", mnt))
	_, _, code, _ := clientNode.RunRoot(ctx, fmt.Sprintf("mount %s %s", dev, mnt))
	if code != 0 {
		t.Fatalf("mount failed")
	}

	// Create a tarball with known content, extract, verify
	clientNode.RunRoot(ctx, fmt.Sprintf("mkdir -p %s/src", mnt))
	for i := 0; i < 100; i++ {
		clientNode.RunRoot(ctx, fmt.Sprintf("dd if=/dev/urandom of=%s/src/file%d bs=1k count=10 2>/dev/null", mnt, i))
	}

	// Tar and extract
	clientNode.RunRoot(ctx, fmt.Sprintf("cd %s && tar cf archive.tar src/", mnt))
	clientNode.RunRoot(ctx, fmt.Sprintf("mkdir -p %s/dst && cd %s/dst && tar xf %s/archive.tar", mnt, mnt, mnt))

	// Verify
	sum1, _, _, _ := clientNode.RunRoot(ctx, fmt.Sprintf("cd %s/src && find . -type f -exec md5sum {} \\; | sort", mnt))
	sum2, _, _, _ := clientNode.RunRoot(ctx, fmt.Sprintf("cd %s/dst/src && find . -type f -exec md5sum {} \\; | sort", mnt))
	if sum1 != sum2 {
		t.Fatalf("tar extract checksums differ")
	}
	t.Log("tar extract + verify passed (100 files)")
}

func testStressSoak30Min(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping 30-minute soak in short mode")
	}

	ctx, cancel := context.WithTimeout(context.Background(), 35*time.Minute)
	defer cancel()

	tgt, iscsi, host := newTestTarget(t, "200M", "")
	dev := startAndLogin(t, ctx, tgt, iscsi, host)

	soakTime := 1800
	if testing.Short() {
		soakTime = 60
	}
	stdout, stderr, code, _ := clientNode.RunRoot(ctx,
		fmt.Sprintf("fio --name=soak --filename=%s --rw=randrw --verify=crc32 "+
			"--bs=4k --size=100M --randrepeat=1 --direct=1 --ioengine=libaio "+
			"--runtime=%d --time_based --output-format=json", dev, soakTime))
	if code != 0 {
		t.Fatalf("fio: code=%d stderr=%s", code, stderr)
	}
	if strings.Contains(stdout, "\"verify_errors\"") && !strings.Contains(stdout, "\"verify_errors\" : 0") {
		t.Fatal("fio verify errors during 30-min soak")
	}
	t.Log("30-minute soak passed")
}

func testStressMixedBlockSize(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
	defer cancel()

	tgt, iscsi, host := newTestTarget(t, "200M", "")
	dev := startAndLogin(t, ctx, tgt, iscsi, host)

	sizes := []string{"4k", "64k", "1M"} // 512 below logical block size (4096)
	for _, bs := range sizes {
		t.Logf("testing bs=%s", bs)
		stdout, stderr, code, _ := clientNode.RunRoot(ctx,
			fmt.Sprintf("fio --name=mixed_%s --filename=%s --rw=randrw --verify=crc32 "+
				"--bs=%s --size=20M --randrepeat=1 --direct=1 --ioengine=libaio", bs, dev, bs))
		if code != 0 {
			t.Fatalf("fio bs=%s: code=%d stderr=%s", bs, code, stderr)
		}
		if strings.Contains(stdout, "\"verify_errors\"") && !strings.Contains(stdout, "\"verify_errors\" : 0") {
			t.Fatalf("fio verify errors at bs=%s", bs)
		}
	}
	t.Log("mixed block size test passed (512, 4k, 64k, 1M)")
}