master: never re-seed a raft cluster over committed state under -raftBootstrap (#10883)

* master: never re-seed a raft cluster over committed state

-raftBootstrap deleted logs.dat, stable.dat and snapshots on every start and
then bootstrapped a fresh cluster. Since hashicorp raft only snapshots after
8192 log entries, the TopologyId lives in the log, not in a snapshot, so the
pre-wipe snapshot recovery found nothing and each restart minted a new cluster
identity. A master that came up while it could not reach its peers seeded a
rival cluster; when the two logs met, SetTopologyId's split-brain guard fatally
stopped every master holding the other id, and the master layer crash-looped
with no quorum.

Bootstrapping is genesis. Drop the wipe and the inline bootstrap. The first
master in -peers already mints a cluster once it has confirmed no peer has a
leader, so the flag has nothing left to do and is now ignored; keeping that one
master the sole bootstrap authority is what stops a partition from minting two
clusters, so the flag must not widen it either. A master with state rejoins its
peers, and one whose data dir was reset is admitted by the sitting leader
instead of forking again.

* test: cover -raftBootstrap restarts in the multi-master suite

Three masters start with -raftBootstrap, the way the helm chart renders it on
every master on every roll, and the cluster has to hold one TopologyId after
they all restart. /dir/status is proxied to the leader, so each master's own
view of the identity is read out of its log, which is where a fork shows up.
Before the fix the hashicorp case minted a new id on each restart.

test/multi_master/cluster.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"strconv"
 	"strings"
@@ -36,6 +37,9 @@ type masterNode struct {
 	// peersStr overrides the cluster-wide peer list for this node, so a test
 	// can start a master that only knows about a subset of the cluster.
 	peersStr string
+	// raftBootstrap starts the master with -raftBootstrap, the way the helm
+	// chart renders master.raftBootstrap on every master, every restart.
+	raftBootstrap bool
 }
 
 // MasterCluster manages a 3-node master raft cluster for integration tests.
@@ -153,6 +157,13 @@ func (mc *MasterCluster) SetNodePeers(i int, peers string) {
 	mc.nodes[i].peersStr = peers
 }
 
+// SetRaftBootstrap makes node i start with -raftBootstrap.
+func (mc *MasterCluster) SetRaftBootstrap(i int) {
+	mc.mu.Lock()
+	defer mc.mu.Unlock()
+	mc.nodes[i].raftBootstrap = true
+}
+
 // StartNode starts the master process at the given index (0–2).
 func (mc *MasterCluster) StartNode(i int) {
 	mc.t.Helper()
@@ -187,6 +198,9 @@ func (mc *MasterCluster) StartNode(i int) {
 	if mc.raftHashicorp {
 		args = append(args, "-raftHashicorp")
 	}
+	if n.raftBootstrap {
+		args = append(args, "-raftBootstrap")
+	}
 
 	n.cmd = exec.Command(mc.weedBinary, args...)
 	n.cmd.Dir = mc.baseDir
@@ -386,6 +400,57 @@ func (mc *MasterCluster) WaitForNodeReady(i int, timeout time.Duration) error {
 	return fmt.Errorf("node %d not ready within %v", i, timeout)
 }
 
+// LogContains reports whether node i's log holds the given text.
+func (mc *MasterCluster) LogContains(i int, text string) bool {
+	b, err := os.ReadFile(mc.nodes[i].logFile)
+	if err != nil {
+		return false
+	}
+	return strings.Contains(string(b), text)
+}
+
+// topologyIdLine matches every line a master logs when it learns a TopologyId,
+// whichever raft implementation applied it.
+var topologyIdLine = regexp.MustCompile(`TopologyId[^:]*: ([0-9a-f-]{36})`)
+
+// NodeTopologyIds returns the TopologyIds node i has logged. /dir/status is
+// proxied to the leader, so a master's own view of the cluster identity is only
+// visible in its log, and that is where a fork shows up.
+func (mc *MasterCluster) NodeTopologyIds(i int) []string {
+	b, err := os.ReadFile(mc.nodes[i].logFile)
+	if err != nil {
+		return nil
+	}
+	var ids []string
+	for _, m := range topologyIdLine.FindAllStringSubmatch(string(b), -1) {
+		ids = append(ids, m[1])
+	}
+	return ids
+}
+
+// WaitForNodeTopologyIds waits until every master has logged a TopologyId and
+// returns what each one saw.
+func (mc *MasterCluster) WaitForNodeTopologyIds(timeout time.Duration) ([3][]string, error) {
+	var ids [3][]string
+	deadline := time.Now().Add(timeout)
+	for {
+		missing := -1
+		for i := range 3 {
+			ids[i] = mc.NodeTopologyIds(i)
+			if len(ids[i]) == 0 {
+				missing = i
+			}
+		}
+		if missing < 0 {
+			return ids, nil
+		}
+		if !time.Now().Before(deadline) {
+			return ids, fmt.Errorf("master %d logged no TopologyId within %v", missing, timeout)
+		}
+		time.Sleep(waitTick)
+	}
+}
+
 // DumpLogs prints the tail of all master logs.
 func (mc *MasterCluster) DumpLogs() {
 	for i := range 3 {

test/multi_master/join_test.go

@@ -151,3 +151,62 @@ func peerCountExcludingSelf(peers []string, self string) int {
 	}
 	return count
 }
+
+// TestRaftBootstrapKeepsExistingCluster covers a master restarting under
+// -raftBootstrap, the way the helm chart renders it on every master on every
+// roll. Bootstrapping is genesis: seeding a second cluster over committed raft
+// state mints a rival TopologyId, and the split-brain guard then Fatals every
+// master that still holds the first one.
+func TestRaftBootstrapKeepsExistingCluster(t *testing.T) {
+	for _, impl := range raftImplementations {
+		t.Run(impl.name, func(t *testing.T) {
+			mc := NewMasterCluster(t, impl.raftHashicorp)
+			for i := range 3 {
+				mc.SetRaftBootstrap(i)
+				mc.StartNode(i)
+			}
+			before, err := mc.WaitForTopologyId(waitTimeout)
+			if err != nil {
+				mc.DumpLogs()
+				t.Fatalf("cluster did not mint a TopologyId: %v", err)
+			}
+
+			for i := range 3 {
+				mc.StopNode(i)
+			}
+			for i := range 3 {
+				mc.StartNode(i)
+			}
+
+			after, err := mc.WaitForTopologyId(waitTimeout)
+			if err != nil {
+				mc.DumpLogs()
+				t.Fatalf("cluster did not come back after a restart: %v", err)
+			}
+			if after != before {
+				mc.DumpLogs()
+				t.Fatalf("-raftBootstrap re-seeded the cluster: TopologyId %s became %s", before, after)
+			}
+
+			// The leader answers for the whole cluster, so a follower that
+			// forked is only visible in its own log.
+			seen, err := mc.WaitForNodeTopologyIds(waitTimeout)
+			if err != nil {
+				mc.DumpLogs()
+				t.Fatal(err)
+			}
+			for i, ids := range seen {
+				for _, id := range ids {
+					if id != before {
+						mc.DumpLogs()
+						t.Fatalf("master %d saw TopologyId %s, want %s", i, id, before)
+					}
+				}
+				if mc.LogContains(i, "Split-brain detected") {
+					mc.DumpLogs()
+					t.Fatalf("master %d hit the split-brain guard", i)
+				}
+			}
+		})
+	}
+}