seaweedfs/weed/storage/blockvol/testrunner/infra/fault.go

package infra

import (
	"context"
	"fmt"
	"strings"
	"time"
)

// InjectNetem adds a netem delay on the node's outbound traffic to targetIP.
// Returns the cleanup command string (for storage in vars).
func InjectNetem(ctx context.Context, node *Node, targetIP string, delayMs int) (cleanupCmd string, err error) {
	iface, _, code, err := node.RunRoot(ctx, fmt.Sprintf(
		"ip route get %s | head -1 | awk '{for(i=1;i<=NF;i++) if($i==\"dev\") print $(i+1)}'", targetIP))
	iface = strings.TrimSpace(iface)
	if err != nil || code != 0 || iface == "" {
		return "", fmt.Errorf("find interface for %s: iface=%q code=%d err=%v", targetIP, iface, code, err)
	}

	_, stderr, code, err := node.RunRoot(ctx, fmt.Sprintf(
		"tc qdisc add dev %s root netem delay %dms", iface, delayMs))
	if err != nil || code != 0 {
		return "", fmt.Errorf("tc qdisc add: code=%d stderr=%s err=%v", code, stderr, err)
	}

	cleanupCmd = fmt.Sprintf("tc qdisc del dev %s root 2>/dev/null || true", iface)
	return cleanupCmd, nil
}

// InjectIptablesDrop blocks outbound TCP traffic from node to targetIP on the given ports.
// Returns the cleanup command string.
func InjectIptablesDrop(ctx context.Context, node *Node, targetIP string, ports []int) (cleanupCmd string, err error) {
	for i, port := range ports {
		_, stderr, code, err := node.RunRoot(ctx, fmt.Sprintf(
			"iptables -A OUTPUT -d %s -p tcp --dport %d -j DROP", targetIP, port))
		if err != nil || code != 0 {
			// Rollback already-added rules
			for j := 0; j < i; j++ {
				node.RunRoot(ctx, fmt.Sprintf(
					"iptables -D OUTPUT -d %s -p tcp --dport %d -j DROP 2>/dev/null", targetIP, ports[j]))
			}
			return "", fmt.Errorf("iptables add port %d: code=%d stderr=%s err=%v", port, code, stderr, err)
		}
	}

	// Build cleanup command that removes all rules.
	// Use "|| true" so each removal succeeds even if the rule is already gone,
	// and ";" so all ports are attempted even if one fails.
	var cmds []string
	for _, port := range ports {
		cmds = append(cmds, fmt.Sprintf(
			"iptables -D OUTPUT -d %s -p tcp --dport %d -j DROP 2>/dev/null || true", targetIP, port))
	}
	cleanupCmd = strings.Join(cmds, " ; ")
	return cleanupCmd, nil
}

// FillDisk fills the filesystem at dir, leaving ~4MB free.
// Returns the cleanup command string.
func FillDisk(ctx context.Context, node *Node, dir string) (cleanupCmd string, err error) {
	stdout, _, code, err := node.RunRoot(ctx, fmt.Sprintf(
		"df -BM --output=avail %s | tail -1 | tr -d ' M'", dir))
	if err != nil || code != 0 {
		return "", fmt.Errorf("df: code=%d err=%v", code, err)
	}
	availMB := 0
	fmt.Sscanf(strings.TrimSpace(stdout), "%d", &availMB)
	if availMB < 8 {
		return "", fmt.Errorf("not enough space to fill: %dMB available", availMB)
	}
	fillMB := availMB - 4

	// Use fallocate (instant) instead of dd (linear time).
	_, stderr, code, err := node.RunRoot(ctx, fmt.Sprintf(
		"fallocate -l %dM %s/fillfile", fillMB, dir))
	if err != nil || code != 0 {
		// Fallback to dd if fallocate not available.
		_, stderr, code, err = node.RunRoot(ctx, fmt.Sprintf(
			"dd if=/dev/zero of=%s/fillfile bs=1M count=%d 2>/dev/null", dir, fillMB))
		if err != nil || code != 0 {
			stdout2, _, _, _ := node.RunRoot(ctx, fmt.Sprintf("test -f %s/fillfile && echo ok", dir))
			if !strings.Contains(stdout2, "ok") {
				return "", fmt.Errorf("fillDisk: code=%d stderr=%s err=%v", code, stderr, err)
			}
		}
	}

	cleanupCmd = fmt.Sprintf("rm -f %s/fillfile", dir)
	return cleanupCmd, nil
}

// CorruptWALRegion overwrites nBytes within the WAL section of the volume file.
func CorruptWALRegion(ctx context.Context, node *Node, volPath string, nBytes int) error {
	const walOffset = 4096 // SuperblockSize

	stdout, _, code, err := node.RunRoot(ctx, fmt.Sprintf("stat -c %%s %s", volPath))
	if err != nil || code != 0 {
		return fmt.Errorf("stat %s: code=%d err=%v", volPath, code, err)
	}
	fileSize := 0
	fmt.Sscanf(strings.TrimSpace(stdout), "%d", &fileSize)

	walEnd := walOffset + 64*1024*1024
	if walEnd > fileSize {
		walEnd = fileSize
	}
	walUsable := walEnd - walOffset
	if walUsable < nBytes*2 {
		return fmt.Errorf("WAL region too small: %d", walUsable)
	}
	seekPos := walOffset + walUsable/3

	_, stderr, code, err := node.RunRoot(ctx, fmt.Sprintf(
		"python3 -c \"import sys; sys.stdout.buffer.write(b'\\xff'*%d)\" | dd of=%s bs=1 seek=%d conv=notrunc 2>/dev/null",
		nBytes, volPath, seekPos))
	if err != nil || code != 0 {
		return fmt.Errorf("corrupt WAL region: code=%d stderr=%s err=%v", code, stderr, err)
	}
	return nil
}

// ClearFault executes a cleanup command stored in vars.
// Tolerates non-zero exit codes since cleanup commands are often
// idempotent (e.g. removing an already-removed iptables rule).
func ClearFault(ctx context.Context, node *Node, cleanupCmd string) error {
	if cleanupCmd == "" {
		return nil
	}
	cctx, cancel := context.WithTimeout(ctx, 10*time.Second)
	defer cancel()
	_, stderr, code, err := node.RunRoot(cctx, cleanupCmd)
	if err != nil {
		return fmt.Errorf("clear fault: code=%d stderr=%s err=%v", code, stderr, err)
	}
	// Non-zero exit is tolerated — cleanup commands use "|| true" but
	// legacy cleanup strings might not, and double-cleanup is harmless.
	return nil
}