seaweedfs/test/multi_master/cluster.go

package multi_master

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/test/testutil"
)

const (
	waitTimeout = 30 * time.Second
	waitTick    = 200 * time.Millisecond
)

// masterNode represents a single master process in the cluster.
type masterNode struct {
	port     int
	grpcPort int
	dataDir  string
	cmd      *exec.Cmd
	logFile  string
	stopped  bool
	// peersStr overrides the cluster-wide peer list for this node, so a test
	// can start a master that only knows about a subset of the cluster.
	peersStr string
	// raftBootstrap starts the master with -raftBootstrap, the way the helm
	// chart renders master.raftBootstrap on every master, every restart.
	raftBootstrap bool
}

// MasterCluster manages a 3-node master raft cluster for integration tests.
type MasterCluster struct {
	t          testing.TB
	weedBinary string
	baseDir    string
	logsDir    string
	keepLogs   bool

	nodes [3]*masterNode
	mu    sync.Mutex

	// peers string shared by all nodes, e.g. "127.0.0.1:9333,127.0.0.1:9334,127.0.0.1:9335"
	peersStr string

	// raftHashicorp starts the masters with -raftHashicorp
	raftHashicorp bool
}

// clusterStatus is the JSON returned by /cluster/status.
type clusterStatus struct {
	IsLeader bool     `json:"IsLeader"`
	Leader   string   `json:"Leader"`
	Peers    []string `json:"Peers"`
}

// StartMasterCluster boots a 3-node master raft cluster and waits for a leader.
func StartMasterCluster(t testing.TB) *MasterCluster {
	t.Helper()

	mc := NewMasterCluster(t, false)
	for i := range 3 {
		mc.StartNode(i)
	}

	if err := mc.WaitForLeader(waitTimeout); err != nil {
		mc.DumpLogs()
		mc.StopAll()
		t.Fatalf("cluster did not elect a leader: %v", err)
	}

	// Wait for TopologyId to be generated and propagated. This is async
	// after leader election, and we need it committed before tests can
	// reliably stop/restart nodes.
	if _, err := mc.WaitForTopologyId(waitTimeout); err != nil {
		mc.DumpLogs()
		mc.StopAll()
		t.Fatalf("TopologyId not generated: %v", err)
	}

	return mc
}

// NewMasterCluster allocates ports and data directories for a 3-node master
// cluster without starting anything, so a test can choose what each node comes
// up with.
func NewMasterCluster(t testing.TB, raftHashicorp bool) *MasterCluster {
	t.Helper()

	weedBinary, err := findOrBuildWeedBinary()
	if err != nil {
		t.Fatalf("resolve weed binary: %v", err)
	}

	keepLogs := os.Getenv("MULTI_MASTER_IT_KEEP_LOGS") == "1"
	baseDir, err := os.MkdirTemp("", "seaweedfs_multi_master_it_")
	if err != nil {
		t.Fatalf("create temp dir: %v", err)
	}
	logsDir := filepath.Join(baseDir, "logs")
	os.MkdirAll(logsDir, 0o755)

	// Allocate 3 mini-safe ports (each guarantees port+10000 is also free).
	httpPorts, err := testutil.AllocateMiniPorts(3)
	if err != nil {
		t.Fatalf("allocate ports: %v", err)
	}
	var nodes [3]*masterNode
	var peerParts []string
	for i, hp := range httpPorts {
		dataDir := filepath.Join(baseDir, fmt.Sprintf("m%d", i))
		os.MkdirAll(dataDir, 0o755)
		nodes[i] = &masterNode{
			port:     hp,
			grpcPort: hp + testutil.GrpcPortOffset,
			dataDir:  dataDir,
			logFile:  filepath.Join(logsDir, fmt.Sprintf("master%d.log", i)),
		}
		peerParts = append(peerParts, fmt.Sprintf("127.0.0.1:%d", hp))
	}

	mc := &MasterCluster{
		t:             t,
		weedBinary:    weedBinary,
		baseDir:       baseDir,
		logsDir:       logsDir,
		keepLogs:      keepLogs,
		nodes:         nodes,
		peersStr:      strings.Join(peerParts, ","),
		raftHashicorp: raftHashicorp,
	}

	t.Cleanup(func() {
		mc.StopAll()
	})
	return mc
}

// SetNodePeers narrows the peer list node i starts with, mirroring a
// StatefulSet whose replica count changed under a running master.
func (mc *MasterCluster) SetNodePeers(i int, peers string) {
	mc.mu.Lock()
	defer mc.mu.Unlock()
	mc.nodes[i].peersStr = peers
}

// SetRaftBootstrap makes node i start with -raftBootstrap.
func (mc *MasterCluster) SetRaftBootstrap(i int) {
	mc.mu.Lock()
	defer mc.mu.Unlock()
	mc.nodes[i].raftBootstrap = true
}

// StartNode starts the master process at the given index (0–2).
func (mc *MasterCluster) StartNode(i int) {
	mc.t.Helper()
	mc.mu.Lock()
	defer mc.mu.Unlock()

	n := mc.nodes[i]
	if n.cmd != nil && !n.stopped {
		return // already running
	}

	logFile, err := os.OpenFile(n.logFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
	if err != nil {
		mc.t.Fatalf("create log for node %d: %v", i, err)
	}

	peersStr := n.peersStr
	if peersStr == "" {
		peersStr = mc.peersStr
	}
	args := []string{
		"master",
		"-ip=127.0.0.1",
		"-port=" + strconv.Itoa(n.port),
		"-port.grpc=" + strconv.Itoa(n.grpcPort),
		"-mdir=" + n.dataDir,
		"-peers=" + peersStr,
		"-electionTimeout=3s",
		"-volumeSizeLimitMB=32",
		"-defaultReplication=000",
	}
	if mc.raftHashicorp {
		args = append(args, "-raftHashicorp")
	}
	if n.raftBootstrap {
		args = append(args, "-raftBootstrap")
	}

	n.cmd = exec.Command(mc.weedBinary, args...)
	n.cmd.Dir = mc.baseDir
	n.cmd.Stdout = logFile
	n.cmd.Stderr = logFile
	n.stopped = false
	if err := n.cmd.Start(); err != nil {
		mc.t.Fatalf("start node %d: %v", i, err)
	}
}

// StopNode gracefully stops the master at the given index.
func (mc *MasterCluster) StopNode(i int) {
	mc.mu.Lock()
	defer mc.mu.Unlock()
	mc.stopNodeLocked(i)
}

func (mc *MasterCluster) stopNodeLocked(i int) {
	n := mc.nodes[i]
	if n.cmd == nil || n.stopped {
		return
	}
	n.stopped = true
	_ = n.cmd.Process.Signal(os.Interrupt)
	done := make(chan error, 1)
	go func() { done <- n.cmd.Wait() }()
	select {
	case <-time.After(10 * time.Second):
		_ = n.cmd.Process.Kill()
		<-done
	case <-done:
	}
}

// StopAll stops all running master nodes.
func (mc *MasterCluster) StopAll() {
	mc.mu.Lock()
	defer mc.mu.Unlock()
	for i := range 3 {
		mc.stopNodeLocked(i)
	}
	if !mc.keepLogs && !mc.t.Failed() {
		os.RemoveAll(mc.baseDir)
	} else if mc.baseDir != "" {
		mc.t.Logf("multi-master logs kept at %s", mc.baseDir)
	}
}

// NodeURL returns the HTTP URL for node i.
func (mc *MasterCluster) NodeURL(i int) string {
	return fmt.Sprintf("http://127.0.0.1:%d", mc.nodes[i].port)
}

// NodeAddress returns "127.0.0.1:port" for node i.
func (mc *MasterCluster) NodeAddress(i int) string {
	return fmt.Sprintf("127.0.0.1:%d", mc.nodes[i].port)
}

// NodeGRPCAddress returns "127.0.0.1:grpcPort" for node i.
func (mc *MasterCluster) NodeGRPCAddress(i int) string {
	return fmt.Sprintf("127.0.0.1:%d", mc.nodes[i].grpcPort)
}

// IsNodeRunning returns true if the node at index i has a live process.
func (mc *MasterCluster) IsNodeRunning(i int) bool {
	mc.mu.Lock()
	defer mc.mu.Unlock()
	n := mc.nodes[i]
	return n.cmd != nil && !n.stopped
}

// GetClusterStatus fetches /cluster/status from node i.
func (mc *MasterCluster) GetClusterStatus(i int) (*clusterStatus, error) {
	client := &http.Client{Timeout: 2 * time.Second}
	resp, err := client.Get(mc.NodeURL(i) + "/cluster/status")
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, _ := io.ReadAll(resp.Body)
	var cs clusterStatus
	if err := json.Unmarshal(body, &cs); err != nil {
		return nil, fmt.Errorf("parse cluster/status: %w (body: %s)", err, string(body))
	}
	return &cs, nil
}

// GetTopologyId fetches the TopologyId from /dir/status on node i.
func (mc *MasterCluster) GetTopologyId(i int) (string, error) {
	client := &http.Client{Timeout: 2 * time.Second}
	resp, err := client.Get(mc.NodeURL(i) + "/dir/status")
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	body, _ := io.ReadAll(resp.Body)
	var raw map[string]any
	if err := json.Unmarshal(body, &raw); err != nil {
		return "", err
	}
	if id, ok := raw["TopologyId"].(string); ok {
		return id, nil
	}
	return "", nil
}

// FindLeader returns the index of the leader node and its address.
// Returns -1 if no leader is found.
func (mc *MasterCluster) FindLeader() (int, string) {
	for i := range 3 {
		if !mc.IsNodeRunning(i) {
			continue
		}
		cs, err := mc.GetClusterStatus(i)
		if err != nil {
			continue
		}
		if cs.IsLeader {
			return i, mc.NodeAddress(i)
		}
	}
	return -1, ""
}

// WaitForLeader polls until a leader is elected or timeout.
func (mc *MasterCluster) WaitForLeader(timeout time.Duration) error {
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		if idx, _ := mc.FindLeader(); idx >= 0 {
			return nil
		}
		time.Sleep(waitTick)
	}
	return fmt.Errorf("no leader elected within %v", timeout)
}

// WaitForNoLeader waits until no running master claims leadership. goraft only
// checks whether it still has a quorum on an election-timeout ticker, and needs
// its peers to go quiet for a full timeout first, so a master that has lost its
// quorum can keep claiming leadership for tens of seconds. It cannot commit
// anything in that window.
func (mc *MasterCluster) WaitForNoLeader(timeout time.Duration) error {
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		idx, _ := mc.FindLeader()
		if idx < 0 {
			return nil
		}
		time.Sleep(waitTick)
	}
	idx, addr := mc.FindLeader()
	return fmt.Errorf("master %d at %s still claims leadership after %v", idx, addr, timeout)
}

// WaitForNewLeader waits for a leader that is different from the given address.
func (mc *MasterCluster) WaitForNewLeader(oldLeaderAddr string, timeout time.Duration) (int, string, error) {
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		idx, addr := mc.FindLeader()
		if idx >= 0 && addr != oldLeaderAddr {
			return idx, addr, nil
		}
		time.Sleep(waitTick)
	}
	return -1, "", fmt.Errorf("no new leader (different from %s) within %v", oldLeaderAddr, timeout)
}

// WaitForTopologyId waits until the leader reports a non-empty TopologyId, and
// returns it. It is only readable once the leader has applied the raft entry
// carrying it, which lands after the election it won.
func (mc *MasterCluster) WaitForTopologyId(timeout time.Duration) (string, error) {
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		if idx, _ := mc.FindLeader(); idx >= 0 {
			if id, err := mc.GetTopologyId(idx); err == nil && id != "" {
				return id, nil
			}
		}
		time.Sleep(waitTick)
	}
	return "", fmt.Errorf("TopologyId not available within %v", timeout)
}

// WaitForNodeReady waits for node i to respond to HTTP.
func (mc *MasterCluster) WaitForNodeReady(i int, timeout time.Duration) error {
	client := &http.Client{Timeout: 1 * time.Second}
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		resp, err := client.Get(mc.NodeURL(i) + "/cluster/status")
		if err == nil {
			resp.Body.Close()
			return nil
		}
		time.Sleep(waitTick)
	}
	return fmt.Errorf("node %d not ready within %v", i, timeout)
}

// LogContains reports whether node i's log holds the given text.
func (mc *MasterCluster) LogContains(i int, text string) bool {
	b, err := os.ReadFile(mc.nodes[i].logFile)
	if err != nil {
		return false
	}
	return strings.Contains(string(b), text)
}

// topologyIdLine matches every line a master logs when it learns a TopologyId,
// whichever raft implementation applied it.
var topologyIdLine = regexp.MustCompile(`TopologyId[^:]*: ([0-9a-f-]{36})`)

// NodeTopologyIds returns the TopologyIds node i has logged. /dir/status is
// proxied to the leader, so a master's own view of the cluster identity is only
// visible in its log, and that is where a fork shows up.
func (mc *MasterCluster) NodeTopologyIds(i int) []string {
	b, err := os.ReadFile(mc.nodes[i].logFile)
	if err != nil {
		return nil
	}
	var ids []string
	for _, m := range topologyIdLine.FindAllStringSubmatch(string(b), -1) {
		ids = append(ids, m[1])
	}
	return ids
}

// WaitForNodeTopologyIds waits until every master has logged a TopologyId and
// returns what each one saw.
func (mc *MasterCluster) WaitForNodeTopologyIds(timeout time.Duration) ([3][]string, error) {
	var ids [3][]string
	deadline := time.Now().Add(timeout)
	for {
		missing := -1
		for i := range 3 {
			ids[i] = mc.NodeTopologyIds(i)
			if len(ids[i]) == 0 {
				missing = i
			}
		}
		if missing < 0 {
			return ids, nil
		}
		if !time.Now().Before(deadline) {
			return ids, fmt.Errorf("master %d logged no TopologyId within %v", missing, timeout)
		}
		time.Sleep(waitTick)
	}
}

// DumpLogs prints the tail of all master logs.
func (mc *MasterCluster) DumpLogs() {
	for i := range 3 {
		mc.t.Logf("=== master%d log tail ===\n%s", i, mc.tailLog(i))
	}
}

func (mc *MasterCluster) tailLog(i int) string {
	f, err := os.Open(mc.nodes[i].logFile)
	if err != nil {
		return "(no log)"
	}
	defer f.Close()
	scanner := bufio.NewScanner(f)
	lines := make([]string, 0, 50)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
		if len(lines) > 50 {
			lines = lines[1:]
		}
	}
	return strings.Join(lines, "\n")
}

func findOrBuildWeedBinary() (string, error) {
	if fromEnv := os.Getenv("WEED_BINARY"); fromEnv != "" {
		if isExecutableFile(fromEnv) {
			return fromEnv, nil
		}
		return "", fmt.Errorf("WEED_BINARY not executable: %s", fromEnv)
	}

	repoRoot := ""
	if _, file, _, ok := runtime.Caller(0); ok {
		repoRoot = filepath.Clean(filepath.Join(filepath.Dir(file), "..", ".."))
	}
	if repoRoot == "" {
		return "", fmt.Errorf("unable to detect repository root")
	}

	// Check if already built
	binDir := filepath.Join(os.TempDir(), "seaweedfs_multi_master_it_bin")
	os.MkdirAll(binDir, 0o755)
	binPath := filepath.Join(binDir, "weed")
	if isExecutableFile(binPath) {
		return binPath, nil
	}

	cmd := exec.Command("go", "build", "-o", binPath, ".")
	cmd.Dir = filepath.Join(repoRoot, "weed")
	var out bytes.Buffer
	cmd.Stdout = &out
	cmd.Stderr = &out
	if err := cmd.Run(); err != nil {
		return "", fmt.Errorf("build weed binary: %w\n%s", err, out.String())
	}
	return binPath, nil
}

func isExecutableFile(path string) bool {
	info, err := os.Stat(path)
	if err != nil || info.IsDir() {
		return false
	}
	return info.Mode().Perm()&0o111 != 0
}