package multi_master import ( "bufio" "bytes" "encoding/json" "fmt" "io" "net/http" "os" "os/exec" "path/filepath" "regexp" "runtime" "strconv" "strings" "sync" "testing" "time" "github.com/seaweedfs/seaweedfs/test/testutil" ) const ( waitTimeout = 30 * time.Second waitTick = 200 * time.Millisecond ) // masterNode represents a single master process in the cluster. type masterNode struct { port int grpcPort int dataDir string cmd *exec.Cmd logFile string stopped bool // peersStr overrides the cluster-wide peer list for this node, so a test // can start a master that only knows about a subset of the cluster. peersStr string // raftBootstrap starts the master with -raftBootstrap, the way the helm // chart renders master.raftBootstrap on every master, every restart. raftBootstrap bool } // MasterCluster manages a 3-node master raft cluster for integration tests. type MasterCluster struct { t testing.TB weedBinary string baseDir string logsDir string keepLogs bool nodes [3]*masterNode mu sync.Mutex // peers string shared by all nodes, e.g. "127.0.0.1:9333,127.0.0.1:9334,127.0.0.1:9335" peersStr string // raftHashicorp starts the masters with -raftHashicorp raftHashicorp bool } // clusterStatus is the JSON returned by /cluster/status. type clusterStatus struct { IsLeader bool `json:"IsLeader"` Leader string `json:"Leader"` Peers []string `json:"Peers"` } // StartMasterCluster boots a 3-node master raft cluster and waits for a leader. func StartMasterCluster(t testing.TB) *MasterCluster { t.Helper() mc := NewMasterCluster(t, false) for i := range 3 { mc.StartNode(i) } if err := mc.WaitForLeader(waitTimeout); err != nil { mc.DumpLogs() mc.StopAll() t.Fatalf("cluster did not elect a leader: %v", err) } // Wait for TopologyId to be generated and propagated. This is async // after leader election, and we need it committed before tests can // reliably stop/restart nodes. if _, err := mc.WaitForTopologyId(waitTimeout); err != nil { mc.DumpLogs() mc.StopAll() t.Fatalf("TopologyId not generated: %v", err) } return mc } // NewMasterCluster allocates ports and data directories for a 3-node master // cluster without starting anything, so a test can choose what each node comes // up with. func NewMasterCluster(t testing.TB, raftHashicorp bool) *MasterCluster { t.Helper() weedBinary, err := findOrBuildWeedBinary() if err != nil { t.Fatalf("resolve weed binary: %v", err) } keepLogs := os.Getenv("MULTI_MASTER_IT_KEEP_LOGS") == "1" baseDir, err := os.MkdirTemp("", "seaweedfs_multi_master_it_") if err != nil { t.Fatalf("create temp dir: %v", err) } logsDir := filepath.Join(baseDir, "logs") os.MkdirAll(logsDir, 0o755) // Allocate 3 mini-safe ports (each guarantees port+10000 is also free). httpPorts, err := testutil.AllocateMiniPorts(3) if err != nil { t.Fatalf("allocate ports: %v", err) } var nodes [3]*masterNode var peerParts []string for i, hp := range httpPorts { dataDir := filepath.Join(baseDir, fmt.Sprintf("m%d", i)) os.MkdirAll(dataDir, 0o755) nodes[i] = &masterNode{ port: hp, grpcPort: hp + testutil.GrpcPortOffset, dataDir: dataDir, logFile: filepath.Join(logsDir, fmt.Sprintf("master%d.log", i)), } peerParts = append(peerParts, fmt.Sprintf("127.0.0.1:%d", hp)) } mc := &MasterCluster{ t: t, weedBinary: weedBinary, baseDir: baseDir, logsDir: logsDir, keepLogs: keepLogs, nodes: nodes, peersStr: strings.Join(peerParts, ","), raftHashicorp: raftHashicorp, } t.Cleanup(func() { mc.StopAll() }) return mc } // SetNodePeers narrows the peer list node i starts with, mirroring a // StatefulSet whose replica count changed under a running master. func (mc *MasterCluster) SetNodePeers(i int, peers string) { mc.mu.Lock() defer mc.mu.Unlock() mc.nodes[i].peersStr = peers } // SetRaftBootstrap makes node i start with -raftBootstrap. func (mc *MasterCluster) SetRaftBootstrap(i int) { mc.mu.Lock() defer mc.mu.Unlock() mc.nodes[i].raftBootstrap = true } // StartNode starts the master process at the given index (0–2). func (mc *MasterCluster) StartNode(i int) { mc.t.Helper() mc.mu.Lock() defer mc.mu.Unlock() n := mc.nodes[i] if n.cmd != nil && !n.stopped { return // already running } logFile, err := os.OpenFile(n.logFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) if err != nil { mc.t.Fatalf("create log for node %d: %v", i, err) } peersStr := n.peersStr if peersStr == "" { peersStr = mc.peersStr } args := []string{ "master", "-ip=127.0.0.1", "-port=" + strconv.Itoa(n.port), "-port.grpc=" + strconv.Itoa(n.grpcPort), "-mdir=" + n.dataDir, "-peers=" + peersStr, "-electionTimeout=3s", "-volumeSizeLimitMB=32", "-defaultReplication=000", } if mc.raftHashicorp { args = append(args, "-raftHashicorp") } if n.raftBootstrap { args = append(args, "-raftBootstrap") } n.cmd = exec.Command(mc.weedBinary, args...) n.cmd.Dir = mc.baseDir n.cmd.Stdout = logFile n.cmd.Stderr = logFile n.stopped = false if err := n.cmd.Start(); err != nil { mc.t.Fatalf("start node %d: %v", i, err) } } // StopNode gracefully stops the master at the given index. func (mc *MasterCluster) StopNode(i int) { mc.mu.Lock() defer mc.mu.Unlock() mc.stopNodeLocked(i) } func (mc *MasterCluster) stopNodeLocked(i int) { n := mc.nodes[i] if n.cmd == nil || n.stopped { return } n.stopped = true _ = n.cmd.Process.Signal(os.Interrupt) done := make(chan error, 1) go func() { done <- n.cmd.Wait() }() select { case <-time.After(10 * time.Second): _ = n.cmd.Process.Kill() <-done case <-done: } } // StopAll stops all running master nodes. func (mc *MasterCluster) StopAll() { mc.mu.Lock() defer mc.mu.Unlock() for i := range 3 { mc.stopNodeLocked(i) } if !mc.keepLogs && !mc.t.Failed() { os.RemoveAll(mc.baseDir) } else if mc.baseDir != "" { mc.t.Logf("multi-master logs kept at %s", mc.baseDir) } } // NodeURL returns the HTTP URL for node i. func (mc *MasterCluster) NodeURL(i int) string { return fmt.Sprintf("http://127.0.0.1:%d", mc.nodes[i].port) } // NodeAddress returns "127.0.0.1:port" for node i. func (mc *MasterCluster) NodeAddress(i int) string { return fmt.Sprintf("127.0.0.1:%d", mc.nodes[i].port) } // NodeGRPCAddress returns "127.0.0.1:grpcPort" for node i. func (mc *MasterCluster) NodeGRPCAddress(i int) string { return fmt.Sprintf("127.0.0.1:%d", mc.nodes[i].grpcPort) } // IsNodeRunning returns true if the node at index i has a live process. func (mc *MasterCluster) IsNodeRunning(i int) bool { mc.mu.Lock() defer mc.mu.Unlock() n := mc.nodes[i] return n.cmd != nil && !n.stopped } // GetClusterStatus fetches /cluster/status from node i. func (mc *MasterCluster) GetClusterStatus(i int) (*clusterStatus, error) { client := &http.Client{Timeout: 2 * time.Second} resp, err := client.Get(mc.NodeURL(i) + "/cluster/status") if err != nil { return nil, err } defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) var cs clusterStatus if err := json.Unmarshal(body, &cs); err != nil { return nil, fmt.Errorf("parse cluster/status: %w (body: %s)", err, string(body)) } return &cs, nil } // GetTopologyId fetches the TopologyId from /dir/status on node i. func (mc *MasterCluster) GetTopologyId(i int) (string, error) { client := &http.Client{Timeout: 2 * time.Second} resp, err := client.Get(mc.NodeURL(i) + "/dir/status") if err != nil { return "", err } defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) var raw map[string]any if err := json.Unmarshal(body, &raw); err != nil { return "", err } if id, ok := raw["TopologyId"].(string); ok { return id, nil } return "", nil } // FindLeader returns the index of the leader node and its address. // Returns -1 if no leader is found. func (mc *MasterCluster) FindLeader() (int, string) { for i := range 3 { if !mc.IsNodeRunning(i) { continue } cs, err := mc.GetClusterStatus(i) if err != nil { continue } if cs.IsLeader { return i, mc.NodeAddress(i) } } return -1, "" } // WaitForLeader polls until a leader is elected or timeout. func (mc *MasterCluster) WaitForLeader(timeout time.Duration) error { deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { if idx, _ := mc.FindLeader(); idx >= 0 { return nil } time.Sleep(waitTick) } return fmt.Errorf("no leader elected within %v", timeout) } // WaitForNoLeader waits until no running master claims leadership. goraft only // checks whether it still has a quorum on an election-timeout ticker, and needs // its peers to go quiet for a full timeout first, so a master that has lost its // quorum can keep claiming leadership for tens of seconds. It cannot commit // anything in that window. func (mc *MasterCluster) WaitForNoLeader(timeout time.Duration) error { deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { idx, _ := mc.FindLeader() if idx < 0 { return nil } time.Sleep(waitTick) } idx, addr := mc.FindLeader() return fmt.Errorf("master %d at %s still claims leadership after %v", idx, addr, timeout) } // WaitForNewLeader waits for a leader that is different from the given address. func (mc *MasterCluster) WaitForNewLeader(oldLeaderAddr string, timeout time.Duration) (int, string, error) { deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { idx, addr := mc.FindLeader() if idx >= 0 && addr != oldLeaderAddr { return idx, addr, nil } time.Sleep(waitTick) } return -1, "", fmt.Errorf("no new leader (different from %s) within %v", oldLeaderAddr, timeout) } // WaitForTopologyId waits until the leader reports a non-empty TopologyId, and // returns it. It is only readable once the leader has applied the raft entry // carrying it, which lands after the election it won. func (mc *MasterCluster) WaitForTopologyId(timeout time.Duration) (string, error) { deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { if idx, _ := mc.FindLeader(); idx >= 0 { if id, err := mc.GetTopologyId(idx); err == nil && id != "" { return id, nil } } time.Sleep(waitTick) } return "", fmt.Errorf("TopologyId not available within %v", timeout) } // WaitForNodeReady waits for node i to respond to HTTP. func (mc *MasterCluster) WaitForNodeReady(i int, timeout time.Duration) error { client := &http.Client{Timeout: 1 * time.Second} deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { resp, err := client.Get(mc.NodeURL(i) + "/cluster/status") if err == nil { resp.Body.Close() return nil } time.Sleep(waitTick) } return fmt.Errorf("node %d not ready within %v", i, timeout) } // LogContains reports whether node i's log holds the given text. func (mc *MasterCluster) LogContains(i int, text string) bool { b, err := os.ReadFile(mc.nodes[i].logFile) if err != nil { return false } return strings.Contains(string(b), text) } // topologyIdLine matches every line a master logs when it learns a TopologyId, // whichever raft implementation applied it. var topologyIdLine = regexp.MustCompile(`TopologyId[^:]*: ([0-9a-f-]{36})`) // NodeTopologyIds returns the TopologyIds node i has logged. /dir/status is // proxied to the leader, so a master's own view of the cluster identity is only // visible in its log, and that is where a fork shows up. func (mc *MasterCluster) NodeTopologyIds(i int) []string { b, err := os.ReadFile(mc.nodes[i].logFile) if err != nil { return nil } var ids []string for _, m := range topologyIdLine.FindAllStringSubmatch(string(b), -1) { ids = append(ids, m[1]) } return ids } // WaitForNodeTopologyIds waits until every master has logged a TopologyId and // returns what each one saw. func (mc *MasterCluster) WaitForNodeTopologyIds(timeout time.Duration) ([3][]string, error) { var ids [3][]string deadline := time.Now().Add(timeout) for { missing := -1 for i := range 3 { ids[i] = mc.NodeTopologyIds(i) if len(ids[i]) == 0 { missing = i } } if missing < 0 { return ids, nil } if !time.Now().Before(deadline) { return ids, fmt.Errorf("master %d logged no TopologyId within %v", missing, timeout) } time.Sleep(waitTick) } } // DumpLogs prints the tail of all master logs. func (mc *MasterCluster) DumpLogs() { for i := range 3 { mc.t.Logf("=== master%d log tail ===\n%s", i, mc.tailLog(i)) } } func (mc *MasterCluster) tailLog(i int) string { f, err := os.Open(mc.nodes[i].logFile) if err != nil { return "(no log)" } defer f.Close() scanner := bufio.NewScanner(f) lines := make([]string, 0, 50) for scanner.Scan() { lines = append(lines, scanner.Text()) if len(lines) > 50 { lines = lines[1:] } } return strings.Join(lines, "\n") } func findOrBuildWeedBinary() (string, error) { if fromEnv := os.Getenv("WEED_BINARY"); fromEnv != "" { if isExecutableFile(fromEnv) { return fromEnv, nil } return "", fmt.Errorf("WEED_BINARY not executable: %s", fromEnv) } repoRoot := "" if _, file, _, ok := runtime.Caller(0); ok { repoRoot = filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..")) } if repoRoot == "" { return "", fmt.Errorf("unable to detect repository root") } // Check if already built binDir := filepath.Join(os.TempDir(), "seaweedfs_multi_master_it_bin") os.MkdirAll(binDir, 0o755) binPath := filepath.Join(binDir, "weed") if isExecutableFile(binPath) { return binPath, nil } cmd := exec.Command("go", "build", "-o", binPath, ".") cmd.Dir = filepath.Join(repoRoot, "weed") var out bytes.Buffer cmd.Stdout = &out cmd.Stderr = &out if err := cmd.Run(); err != nil { return "", fmt.Errorf("build weed binary: %w\n%s", err, out.String()) } return binPath, nil } func isExecutableFile(path string) bool { info, err := os.Stat(path) if err != nil || info.IsDir() { return false } return info.Mode().Perm()&0o111 != 0 }