seaweedfs/test/multi_master/join_test.go

package multi_master

import (
	"fmt"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/pb"
)

var raftImplementations = []struct {
	name          string
	raftHashicorp bool
}{
	{"goraft", false},
	{"hashicorp", true},
}

// TestFreshClusterFormsWithoutALeader covers the other half of the bootstrap
// decision: with no leader anywhere, three masters starting together still have
// to mint a cluster between them.
func TestFreshClusterFormsWithoutALeader(t *testing.T) {
	for _, impl := range raftImplementations {
		t.Run(impl.name, func(t *testing.T) {
			mc := NewMasterCluster(t, impl.raftHashicorp)
			for i := range 3 {
				mc.StartNode(i)
			}

			if _, err := waitForCommonLeader(mc, waitTimeout); err != nil {
				mc.DumpLogs()
				t.Fatalf("fresh cluster did not converge: %v", err)
			}
			for i := range 3 {
				if err := waitForPeerCount(mc, i, 2, waitTimeout); err != nil {
					mc.DumpLogs()
					t.Fatalf("master %d does not see the full cluster: %v", i, err)
				}
			}
		})
	}
}

// TestScaleUpOntoExistingLeader mirrors a Kubernetes master StatefulSet whose
// replica count goes back from one to three: master 0 keeps running as the
// leader of a single-peer cluster while two fresh masters come up pointing at
// all three. The newcomers start with an empty raft log, so neither raft
// implementation lets them campaign — the sitting leader has to admit them.
func TestScaleUpOntoExistingLeader(t *testing.T) {
	for _, impl := range raftImplementations {
		t.Run(impl.name, func(t *testing.T) {
			mc := NewMasterCluster(t, impl.raftHashicorp)

			// Master 0 is alone in its peer list, the way the operator renders
			// -peers when spec.master.replicas is 1.
			mc.SetNodePeers(0, mc.NodeAddress(0))
			mc.StartNode(0)
			if err := mc.WaitForLeader(waitTimeout); err != nil {
				mc.DumpLogs()
				t.Fatalf("single master did not become leader: %v", err)
			}

			// Scale up. These two carry the full peer list; master 0 still
			// runs with the old one and has never heard of them.
			mc.StartNode(1)
			mc.StartNode(2)

			leader, err := waitForCommonLeader(mc, waitTimeout)
			if err != nil {
				mc.DumpLogs()
				t.Fatalf("masters did not converge after scaling up: %v", err)
			}
			if leader != mc.NodeAddress(0) {
				t.Fatalf("leader moved to %s, want the sitting leader %s", leader, mc.NodeAddress(0))
			}

			if err := waitForPeerCount(mc, 0, 2, waitTimeout); err != nil {
				mc.DumpLogs()
				t.Fatalf("leader did not admit both new masters: %v", err)
			}
		})
	}
}

// waitForCommonLeader waits until every running master names the same leader,
// and returns it.
func waitForCommonLeader(mc *MasterCluster, timeout time.Duration) (string, error) {
	var lastErr error
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		leader, err := commonLeader(mc)
		if err == nil {
			return leader, nil
		}
		lastErr = err
		time.Sleep(waitTick)
	}
	return "", lastErr
}

func commonLeader(mc *MasterCluster) (string, error) {
	agreed := ""
	for i := range 3 {
		if !mc.IsNodeRunning(i) {
			continue
		}
		cs, err := mc.GetClusterStatus(i)
		if err != nil {
			return "", err
		}
		leader := pb.ServerAddress(cs.Leader).ToHttpAddress()
		if leader == "" {
			return "", fmt.Errorf("master %d has no leader", i)
		}
		if agreed == "" {
			agreed = leader
		} else if agreed != leader {
			return "", fmt.Errorf("masters disagree on the leader: %s and %s", agreed, leader)
		}
	}
	if agreed == "" {
		return "", fmt.Errorf("no master is running")
	}
	return agreed, nil
}

// waitForPeerCount waits until node i reports the given number of raft peers.
// The count excludes the node itself.
func waitForPeerCount(mc *MasterCluster, i, want int, timeout time.Duration) error {
	got := -1
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		cs, err := mc.GetClusterStatus(i)
		if err == nil {
			got = peerCountExcludingSelf(cs.Peers, mc.NodeAddress(i))
			if got == want {
				return nil
			}
		}
		time.Sleep(waitTick)
	}
	return fmt.Errorf("master %d reports %d peers, want %d", i, got, want)
}

func peerCountExcludingSelf(peers []string, self string) int {
	count := 0
	for _, peer := range peers {
		if pb.ServerAddress(peer).ToHttpAddress() != self {
			count++
		}
	}
	return count
}

// TestRaftBootstrapKeepsExistingCluster covers a master restarting under
// -raftBootstrap, the way the helm chart renders it on every master on every
// roll. Bootstrapping is genesis: seeding a second cluster over committed raft
// state mints a rival TopologyId, and the split-brain guard then Fatals every
// master that still holds the first one.
func TestRaftBootstrapKeepsExistingCluster(t *testing.T) {
	for _, impl := range raftImplementations {
		t.Run(impl.name, func(t *testing.T) {
			mc := NewMasterCluster(t, impl.raftHashicorp)
			for i := range 3 {
				mc.SetRaftBootstrap(i)
				mc.StartNode(i)
			}
			before, err := mc.WaitForTopologyId(waitTimeout)
			if err != nil {
				mc.DumpLogs()
				t.Fatalf("cluster did not mint a TopologyId: %v", err)
			}

			for i := range 3 {
				mc.StopNode(i)
			}
			for i := range 3 {
				mc.StartNode(i)
			}

			after, err := mc.WaitForTopologyId(waitTimeout)
			if err != nil {
				mc.DumpLogs()
				t.Fatalf("cluster did not come back after a restart: %v", err)
			}
			if after != before {
				mc.DumpLogs()
				t.Fatalf("-raftBootstrap re-seeded the cluster: TopologyId %s became %s", before, after)
			}

			// The leader answers for the whole cluster, so a follower that
			// forked is only visible in its own log.
			seen, err := mc.WaitForNodeTopologyIds(waitTimeout)
			if err != nil {
				mc.DumpLogs()
				t.Fatal(err)
			}
			for i, ids := range seen {
				for _, id := range ids {
					if id != before {
						mc.DumpLogs()
						t.Fatalf("master %d saw TopologyId %s, want %s", i, id, before)
					}
				}
				if mc.LogContains(i, "Split-brain detected") {
					mc.DumpLogs()
					t.Fatalf("master %d hit the split-brain guard", i)
				}
			}
		})
	}
}