master: cap the reported capacity at what the disks hold (#10960)

* master: cap the reported capacity at what the disks hold

Statistics reported max volume count times the volume size limit, which is
how many volumes the cluster is allowed to place, not how much space it has.
A cluster given far more slots than its disks can fill reported a capacity it
could never reach -- 65536 slots at 30GB read as 1.9PB on a 460GB disk -- and
the number never moved, since writing data changes neither the slot count nor
the size limit.

The volume servers already report each filesystem's total and free bytes in
their heartbeats, so bound the answer by what they say is left.

* mount: keep the last known sizes when filer statistics fails

A failed Statistics call returned before df's answer was filled in, so a
mount whose filer or master was briefly unreachable reported an empty
filesystem rather than the sizes it already had.

* master: drop the disk ceiling when a volume server does not report

A cluster part way through an upgrade has volume servers that predate the disk
bytes in the heartbeat. Summing only the ones that answered left the quiet
server's free space out of the total, and the server holding the room is
exactly the one that could make the cluster read as full.

Answer with the disks only when every one of them reported.

weed/mount/weedfs_stats.go

@@ -64,8 +64,8 @@ func (wfs *WFS) StatFs(cancel <-chan struct{}, in *fuse.InHeader, out *fuse.Stat
 			return nil
 		})
 		if err != nil {
+			// the last known sizes beat the empty filesystem a bare return reports
 			glog.V(0).Infof("filer Statistics: %v", err)
-			return fuse.OK
 		}
 	}
 

weed/server/master_grpc_server_statistics_test.go

@@ -122,3 +122,52 @@ func TestStatisticsReplicaCopyCount(t *testing.T) {
 		})
 	}
 }
+
+// reportDiskBytes has the first nodeCount nodes report the same filesystem
+// capacity, the way a volume server does in its heartbeat.
+func reportDiskBytes(ms *MasterServer, nodeCount int, totalBytes, freeBytes uint64) {
+	rack := ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1")
+	for _, node := range rack.Children()[:nodeCount] {
+		node.(*topology.DataNode).AdjustDiskUsageBytes(
+			map[string]uint64{"": totalBytes}, map[string]uint64{"": freeBytes})
+	}
+}
+
+// TestStatisticsPhysicalCapacity covers a cluster configured with more volume
+// slots than its disks hold. Slots promise 40MB here; what the disks can still
+// take is what gets reported.
+func TestStatisticsPhysicalCapacity(t *testing.T) {
+	ms := newStatisticsMaster(t)
+
+	slotCapacity := uint64(40 * 1024 * 1024)
+	statistics := func() *master_pb.StatisticsResponse {
+		t.Helper()
+		resp, err := ms.Statistics(context.Background(), &master_pb.StatisticsRequest{})
+		if err != nil {
+			t.Fatalf("Statistics: %v", err)
+		}
+		return resp
+	}
+
+	if got := statistics().TotalSize; got != slotCapacity {
+		t.Fatalf("disks that report nothing: got %d, want %d", got, slotCapacity)
+	}
+
+	// a volume server too old to report leaves the whole cluster on its slots,
+	// since the room it holds would otherwise go missing
+	reportDiskBytes(ms, 3, 8<<20, 1<<20)
+	if got := statistics().TotalSize; got != slotCapacity {
+		t.Errorf("one disk of four reporting nothing: got %d, want %d", got, slotCapacity)
+	}
+
+	reportDiskBytes(ms, 4, 8<<20, 1<<20)
+	resp := statistics()
+	if want := resp.UsedSize + (4 << 20); resp.TotalSize != want {
+		t.Errorf("four disks with 1MB free: got %d, want %d", resp.TotalSize, want)
+	}
+
+	reportDiskBytes(ms, 4, 1<<30, 1<<30)
+	if got := statistics().TotalSize; got != slotCapacity {
+		t.Errorf("disks roomier than the slots: got %d, want %d", got, slotCapacity)
+	}
+}

weed/server/master_grpc_server_volume.go

@@ -239,6 +239,13 @@ func (ms *MasterServer) Statistics(ctx context.Context, req *master_pb.Statistic
 	if req.Collection != "" {
 		clusterUsedSize = ms.Topo.CollectionVolumeStats("").UsedSize
 	}
+	// volume slots are provisioning, not capacity: a cluster configured with
+	// more of them than its disks hold would report space it can never take.
+	// What the disks still have free, on top of what the cluster already wrote,
+	// is the real ceiling.
+	if freeBytes, reported := ms.Topo.FreeBytes(); reported {
+		totalSize = min(totalSize, clusterUsedSize+freeBytes)
+	}
 	// and the free space holds that many copies fewer of whatever the caller writes
 	var freeSize uint64
 	if totalSize > clusterUsedSize {

weed/topology/disk.go

@@ -129,6 +129,26 @@ func (d *DiskUsages) GetMaxVolumeCount() (maxVolumeCount int64) {
 	return
 }
 
+// FreeBytes sums the space one volume server reports as still free on its
+// filesystems. reported is false as soon as a disk holding volume slots says
+// nothing -- a volume server older than the field looks that way -- since
+// leaving its space out would understate the room the server has.
+func (d *DiskUsages) FreeBytes() (freeBytes uint64, reported bool) {
+	d.RLock()
+	defer d.RUnlock()
+	for _, diskUsageCounts := range d.usages {
+		usage := diskUsageCounts.snapshot()
+		if usage.diskTotalBytes <= 0 {
+			if usage.maxVolumeCount > 0 {
+				return 0, false
+			}
+			continue
+		}
+		freeBytes += uint64(max(0, usage.diskFreeBytes))
+	}
+	return freeBytes, true
+}
+
 type DiskUsageCounts struct {
 	volumeCount       int64
 	remoteVolumeCount int64

weed/topology/topology.go

@@ -162,6 +162,25 @@ func (t *Topology) unregisterDataNodeAddress(addr pb.ServerAddress, dn *DataNode
 	}
 }
 
+// FreeBytes sums what every volume server reports as free on its filesystems.
+// reported is false unless all of them answered: the one that stayed quiet may
+// be the one holding the room, and a partial sum would read as a cluster with
+// none left.
+func (t *Topology) FreeBytes() (freeBytes uint64, reported bool) {
+	for _, dcNode := range t.Children() {
+		for _, rackNode := range dcNode.Children() {
+			for _, dataNode := range rackNode.Children() {
+				nodeFreeBytes, nodeReported := dataNode.GetDiskUsages().FreeBytes()
+				if !nodeReported {
+					return 0, false
+				}
+				freeBytes += nodeFreeBytes
+			}
+		}
+	}
+	return freeBytes, true
+}
+
 func (t *Topology) IsChildLocked() (bool, error) {
 	if t.IsLocked() {
 		return true, errors.New("topology is locked")