From 301d83cc7a848084b6dd62310b59fe61b4be4fde Mon Sep 17 00:00:00 2001
From: Chris Lu <chrislusf@users.noreply.github.com>
Date: Fri, 21 Aug 2026 23:05:14 -0700
Subject: [PATCH] test: wait for the master to register the volume servers
 before failover tests run (#10871)

The failover harness treated an open volume server port as readiness, but the
master only learns of a volume server from its heartbeat. A lone master refuses
heartbeats until its bootstrap check elects it, and the servers back off and
retry, so registration lands seconds after the ports answer. Tests that started
writing in that window assigned against an empty topology, which fails with
"no free volumes left" and reaches the mount as ENOSPC.
---
 test/fuse_failover/framework_test.go       | 57 ++++++++++++++++++++++
 test/fuse_failover/volume_failover_test.go |  4 +-
 2 files changed, 60 insertions(+), 1 deletion(-)

diff --git a/test/fuse_failover/framework_test.go b/test/fuse_failover/framework_test.go
index 28c26a250..5ab1338d8 100644
--- a/test/fuse_failover/framework_test.go
+++ b/test/fuse_failover/framework_test.go
@@ -109,6 +109,7 @@ func startFailoverCluster(t testing.TB, numVolumes, numMounts int) *failoverClus
 		require.NoError(t, c.waitForMount(mp, 30*time.Second),
 			"mount %d not ready\n%s", i, c.tailLog(fmt.Sprintf("mount%d", i)))
 	}
+	require.NoError(t, c.WaitForVolumeServers(60*time.Second))
 	return c
 }
 
@@ -344,6 +345,62 @@ func (c *failoverCluster) WaitForHolders(vid uint32, count int, timeout time.Dur
 	}
 }
 
+// WaitForVolumeServers blocks until the master's topology lists every volume
+// server. A volume server accepts connections well before the master knows it
+// exists: a lone master only elects itself once its bootstrap check expires,
+// and heartbeats sent before that are refused and retried with backoff, so
+// registration lands seconds after the process is up. Until it does the
+// topology has no data node at all, an assign fails with "no free volumes
+// left", and the mount reports that as ENOSPC instead of retrying.
+func (c *failoverCluster) WaitForVolumeServers(timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	for {
+		registered := c.registeredVolumeServers()
+		var missing []string
+		for i := range c.volumePorts {
+			if address := c.VolumeServerAddress(i); !registered[address] {
+				missing = append(missing, address)
+			}
+		}
+		if len(missing) == 0 {
+			return nil
+		}
+		if time.Now().After(deadline) {
+			return fmt.Errorf("volume servers %v not registered with the master within %v\n%s",
+				missing, timeout, c.MasterGet("/dir/status?pretty=y"))
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+}
+
+// registeredVolumeServers is the set of volume server addresses the master
+// currently holds in its topology.
+func (c *failoverCluster) registeredVolumeServers() map[string]bool {
+	var status struct {
+		Topology struct {
+			DataCenters []struct {
+				Racks []struct {
+					DataNodes []struct {
+						Url string `json:"Url"`
+					} `json:"DataNodes"`
+				} `json:"Racks"`
+			} `json:"DataCenters"`
+		} `json:"Topology"`
+	}
+	if err := json.Unmarshal([]byte(c.MasterGet("/dir/status")), &status); err != nil {
+		return nil
+	}
+	registered := make(map[string]bool)
+	for _, dc := range status.Topology.DataCenters {
+		for _, rack := range dc.Racks {
+			for _, dn := range rack.DataNodes {
+				registered[dn.Url] = true
+			}
+		}
+	}
+	return registered
+}
+
 // volumeIndexOf maps a server address back to its index, or -1.
 func (c *failoverCluster) volumeIndexOf(address string) int {
 	for i := range c.volumePorts {
diff --git a/test/fuse_failover/volume_failover_test.go b/test/fuse_failover/volume_failover_test.go
index 60dc5b7cf..f5208e0ed 100644
--- a/test/fuse_failover/volume_failover_test.go
+++ b/test/fuse_failover/volume_failover_test.go
@@ -190,7 +190,9 @@ func TestReadWithVolumeServerDown(t *testing.T) {
 		t.Logf("read %s (held by volume%d) with volume%d down in %v", name, victim, victim, elapsed)
 
 		require.NoError(t, c.StartVolume(victim))
-		time.Sleep(3 * time.Second) // let the master see the heartbeat again
+		// The next iteration picks its victim from the master's placement view,
+		// so wait for the restarted server to be in it again.
+		require.NoError(t, c.WaitForVolumeServers(60*time.Second))
 	}
 }