seaweedfs/test/master_cold_start/cluster.go

// Package master_cold_start boots a fresh master plus empty volume servers
// and exercises the very first write against the cluster. The first assign
// triggers volume growth server-side; it must succeed within a single
// request, without client retries.
package master_cold_start

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/test/testutil"
)

const (
	waitTimeout = 30 * time.Second
	waitTick    = 200 * time.Millisecond
)

type process struct {
	name    string
	cmd     *exec.Cmd
	logFile string
}

// Cluster is one fresh master plus n empty volume servers.
type Cluster struct {
	t          testing.TB
	weedBinary string
	baseDir    string

	masterPort     int
	masterGrpcPort int
	procs          []*process
}

// StartCluster boots a master and volumeServerCount empty volume servers,
// then waits for all of them to register with the master.
func StartCluster(t testing.TB, volumeServerCount int) *Cluster {
	t.Helper()

	weedBinary, err := findOrBuildWeedBinary()
	if err != nil {
		t.Fatalf("resolve weed binary: %v", err)
	}

	baseDir, err := os.MkdirTemp("", "seaweedfs_master_cold_start_it_")
	if err != nil {
		t.Fatalf("create temp dir: %v", err)
	}
	logsDir := filepath.Join(baseDir, "logs")
	os.MkdirAll(logsDir, 0o755)

	ports, err := testutil.AllocateMiniPorts(1 + volumeServerCount)
	if err != nil {
		t.Fatalf("allocate ports: %v", err)
	}

	c := &Cluster{
		t:              t,
		weedBinary:     weedBinary,
		baseDir:        baseDir,
		masterPort:     ports[0],
		masterGrpcPort: ports[0] + testutil.GrpcPortOffset,
	}
	t.Cleanup(func() {
		c.StopAll()
		if t.Failed() {
			c.DumpLogs()
			t.Logf("cold-start logs kept at %s", baseDir)
		} else {
			os.RemoveAll(baseDir)
		}
	})

	masterDir := filepath.Join(baseDir, "master")
	os.MkdirAll(masterDir, 0o755)
	c.startProcess("master", filepath.Join(logsDir, "master.log"),
		"master",
		"-ip=127.0.0.1",
		"-port="+strconv.Itoa(c.masterPort),
		"-port.grpc="+strconv.Itoa(c.masterGrpcPort),
		"-mdir="+masterDir,
		"-volumeSizeLimitMB=32",
		"-defaultReplication=000",
	)
	if err := c.waitForMaster(waitTimeout); err != nil {
		t.Fatalf("master not ready: %v", err)
	}

	for i := 0; i < volumeServerCount; i++ {
		volDir := filepath.Join(baseDir, fmt.Sprintf("vol%d", i))
		os.MkdirAll(volDir, 0o755)
		name := fmt.Sprintf("volume%d", i)
		c.startProcess(name, filepath.Join(logsDir, name+".log"),
			"volume",
			"-ip=127.0.0.1",
			"-port="+strconv.Itoa(ports[1+i]),
			"-port.grpc="+strconv.Itoa(ports[1+i]+testutil.GrpcPortOffset),
			"-dir="+volDir,
			"-max=10",
			"-mserver=127.0.0.1:"+strconv.Itoa(c.masterPort),
		)
	}
	if err := c.waitForVolumeServers(volumeServerCount, waitTimeout); err != nil {
		t.Fatalf("volume servers not registered: %v", err)
	}
	return c
}

func (c *Cluster) startProcess(name, logFile string, args ...string) {
	c.t.Helper()
	f, err := os.OpenFile(logFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
	if err != nil {
		c.t.Fatalf("create log for %s: %v", name, err)
	}
	defer f.Close() // the child owns its own descriptor after Start
	cmd := exec.Command(c.weedBinary, args...)
	cmd.Dir = c.baseDir
	cmd.Stdout = f
	cmd.Stderr = f
	if err := cmd.Start(); err != nil {
		c.t.Fatalf("start %s: %v", name, err)
	}
	c.procs = append(c.procs, &process{name: name, cmd: cmd, logFile: logFile})
}

// StopAll interrupts every process, volume servers first so the master does
// not log reconnect noise.
func (c *Cluster) StopAll() {
	for i := len(c.procs) - 1; i >= 0; i-- {
		p := c.procs[i]
		if p.cmd.Process == nil {
			continue
		}
		_ = p.cmd.Process.Signal(os.Interrupt)
		done := make(chan error, 1)
		go func() { done <- p.cmd.Wait() }()
		select {
		case <-time.After(10 * time.Second):
			_ = p.cmd.Process.Kill()
			<-done
		case <-done:
		}
	}
	c.procs = nil
}

// MasterURL returns the master HTTP endpoint.
func (c *Cluster) MasterURL() string {
	return fmt.Sprintf("http://127.0.0.1:%d", c.masterPort)
}

// MasterAddress returns "127.0.0.1:port" for the master HTTP port.
func (c *Cluster) MasterAddress() string {
	return fmt.Sprintf("127.0.0.1:%d", c.masterPort)
}

func (c *Cluster) waitForMaster(timeout time.Duration) error {
	client := &http.Client{Timeout: 1 * time.Second}
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		resp, err := client.Get(c.MasterURL() + "/cluster/status")
		if err == nil {
			body, _ := io.ReadAll(resp.Body)
			resp.Body.Close()
			var cs struct {
				IsLeader bool `json:"IsLeader"`
			}
			if json.Unmarshal(body, &cs) == nil && cs.IsLeader {
				return nil
			}
		}
		time.Sleep(waitTick)
	}
	return fmt.Errorf("master did not become leader within %v", timeout)
}

// waitForVolumeServers polls /dir/status until want data nodes are registered
// with free volume slots.
func (c *Cluster) waitForVolumeServers(want int, timeout time.Duration) error {
	client := &http.Client{Timeout: 2 * time.Second}
	deadline := time.Now().Add(timeout)
	var last string
	for time.Now().Before(deadline) {
		resp, err := client.Get(c.MasterURL() + "/dir/status")
		if err == nil {
			body, _ := io.ReadAll(resp.Body)
			resp.Body.Close()
			var status struct {
				Topology struct {
					Max         int64 `json:"Max"`
					DataCenters []struct {
						Racks []struct {
							DataNodes []struct {
								Url string `json:"Url"`
							} `json:"DataNodes"`
						} `json:"Racks"`
					} `json:"DataCenters"`
				} `json:"Topology"`
			}
			if json.Unmarshal(body, &status) == nil {
				nodes := 0
				for _, dc := range status.Topology.DataCenters {
					for _, rack := range dc.Racks {
						nodes += len(rack.DataNodes)
					}
				}
				if nodes >= want && status.Topology.Max > 0 {
					return nil
				}
				last = fmt.Sprintf("%d/%d nodes, %d slots", nodes, want, status.Topology.Max)
			}
		}
		time.Sleep(waitTick)
	}
	return fmt.Errorf("volume servers not registered within %v (%s)", timeout, last)
}

// DumpLogs prints the tail of every process log.
func (c *Cluster) DumpLogs() {
	logsDir := filepath.Join(c.baseDir, "logs")
	entries, _ := os.ReadDir(logsDir)
	for _, e := range entries {
		c.t.Logf("=== %s tail ===\n%s", e.Name(), tailFile(filepath.Join(logsDir, e.Name())))
	}
}

func tailFile(path string) string {
	f, err := os.Open(path)
	if err != nil {
		return "(no log)"
	}
	defer f.Close()
	scanner := bufio.NewScanner(f)
	lines := make([]string, 0, 50)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
		if len(lines) > 50 {
			lines = lines[1:]
		}
	}
	return strings.Join(lines, "\n")
}

func findOrBuildWeedBinary() (string, error) {
	if fromEnv := os.Getenv("WEED_BINARY"); fromEnv != "" {
		if isExecutableFile(fromEnv) {
			return fromEnv, nil
		}
		return "", fmt.Errorf("WEED_BINARY not executable: %s", fromEnv)
	}

	repoRoot := ""
	if _, file, _, ok := runtime.Caller(0); ok {
		repoRoot = filepath.Clean(filepath.Join(filepath.Dir(file), "..", ".."))
	}
	if repoRoot == "" {
		return "", fmt.Errorf("unable to detect repository root")
	}

	binDir := filepath.Join(os.TempDir(), "seaweedfs_master_cold_start_it_bin")
	os.MkdirAll(binDir, 0o755)
	binPath := filepath.Join(binDir, "weed")
	if isExecutableFile(binPath) {
		return binPath, nil
	}

	cmd := exec.Command("go", "build", "-o", binPath, ".")
	cmd.Dir = filepath.Join(repoRoot, "weed")
	var out bytes.Buffer
	cmd.Stdout = &out
	cmd.Stderr = &out
	if err := cmd.Run(); err != nil {
		return "", fmt.Errorf("build weed binary: %w\n%s", err, out.String())
	}
	return binPath, nil
}

func isExecutableFile(path string) bool {
	info, err := os.Stat(path)
	if err != nil || info.IsDir() {
		return false
	}
	return info.Mode().Perm()&0o111 != 0
}