From a02c0024e582f523fb93fd4bd21681cea3bdf952 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 26 Aug 2026 00:12:56 -0700 Subject: [PATCH] master: cap the reported capacity at what the disks hold (#10960) * master: cap the reported capacity at what the disks hold Statistics reported max volume count times the volume size limit, which is how many volumes the cluster is allowed to place, not how much space it has. A cluster given far more slots than its disks can fill reported a capacity it could never reach -- 65536 slots at 30GB read as 1.9PB on a 460GB disk -- and the number never moved, since writing data changes neither the slot count nor the size limit. The volume servers already report each filesystem's total and free bytes in their heartbeats, so bound the answer by what they say is left. * mount: keep the last known sizes when filer statistics fails A failed Statistics call returned before df's answer was filled in, so a mount whose filer or master was briefly unreachable reported an empty filesystem rather than the sizes it already had. * master: drop the disk ceiling when a volume server does not report A cluster part way through an upgrade has volume servers that predate the disk bytes in the heartbeat. Summing only the ones that answered left the quiet server's free space out of the total, and the server holding the room is exactly the one that could make the cluster read as full. Answer with the disks only when every one of them reported. --- weed/mount/weedfs_stats.go | 2 +- .../master_grpc_server_statistics_test.go | 49 +++++++++++++++++++ weed/server/master_grpc_server_volume.go | 7 +++ weed/topology/disk.go | 20 ++++++++ weed/topology/topology.go | 19 +++++++ 5 files changed, 96 insertions(+), 1 deletion(-) diff --git a/weed/mount/weedfs_stats.go b/weed/mount/weedfs_stats.go index 152e0b1e8..701dc5619 100644 --- a/weed/mount/weedfs_stats.go +++ b/weed/mount/weedfs_stats.go @@ -64,8 +64,8 @@ func (wfs *WFS) StatFs(cancel <-chan struct{}, in *fuse.InHeader, out *fuse.Stat return nil }) if err != nil { + // the last known sizes beat the empty filesystem a bare return reports glog.V(0).Infof("filer Statistics: %v", err) - return fuse.OK } } diff --git a/weed/server/master_grpc_server_statistics_test.go b/weed/server/master_grpc_server_statistics_test.go index 6a8ba1daf..1e78ea959 100644 --- a/weed/server/master_grpc_server_statistics_test.go +++ b/weed/server/master_grpc_server_statistics_test.go @@ -122,3 +122,52 @@ func TestStatisticsReplicaCopyCount(t *testing.T) { }) } } + +// reportDiskBytes has the first nodeCount nodes report the same filesystem +// capacity, the way a volume server does in its heartbeat. +func reportDiskBytes(ms *MasterServer, nodeCount int, totalBytes, freeBytes uint64) { + rack := ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1") + for _, node := range rack.Children()[:nodeCount] { + node.(*topology.DataNode).AdjustDiskUsageBytes( + map[string]uint64{"": totalBytes}, map[string]uint64{"": freeBytes}) + } +} + +// TestStatisticsPhysicalCapacity covers a cluster configured with more volume +// slots than its disks hold. Slots promise 40MB here; what the disks can still +// take is what gets reported. +func TestStatisticsPhysicalCapacity(t *testing.T) { + ms := newStatisticsMaster(t) + + slotCapacity := uint64(40 * 1024 * 1024) + statistics := func() *master_pb.StatisticsResponse { + t.Helper() + resp, err := ms.Statistics(context.Background(), &master_pb.StatisticsRequest{}) + if err != nil { + t.Fatalf("Statistics: %v", err) + } + return resp + } + + if got := statistics().TotalSize; got != slotCapacity { + t.Fatalf("disks that report nothing: got %d, want %d", got, slotCapacity) + } + + // a volume server too old to report leaves the whole cluster on its slots, + // since the room it holds would otherwise go missing + reportDiskBytes(ms, 3, 8<<20, 1<<20) + if got := statistics().TotalSize; got != slotCapacity { + t.Errorf("one disk of four reporting nothing: got %d, want %d", got, slotCapacity) + } + + reportDiskBytes(ms, 4, 8<<20, 1<<20) + resp := statistics() + if want := resp.UsedSize + (4 << 20); resp.TotalSize != want { + t.Errorf("four disks with 1MB free: got %d, want %d", resp.TotalSize, want) + } + + reportDiskBytes(ms, 4, 1<<30, 1<<30) + if got := statistics().TotalSize; got != slotCapacity { + t.Errorf("disks roomier than the slots: got %d, want %d", got, slotCapacity) + } +} diff --git a/weed/server/master_grpc_server_volume.go b/weed/server/master_grpc_server_volume.go index 0f4d3f6a2..66aea2d24 100644 --- a/weed/server/master_grpc_server_volume.go +++ b/weed/server/master_grpc_server_volume.go @@ -239,6 +239,13 @@ func (ms *MasterServer) Statistics(ctx context.Context, req *master_pb.Statistic if req.Collection != "" { clusterUsedSize = ms.Topo.CollectionVolumeStats("").UsedSize } + // volume slots are provisioning, not capacity: a cluster configured with + // more of them than its disks hold would report space it can never take. + // What the disks still have free, on top of what the cluster already wrote, + // is the real ceiling. + if freeBytes, reported := ms.Topo.FreeBytes(); reported { + totalSize = min(totalSize, clusterUsedSize+freeBytes) + } // and the free space holds that many copies fewer of whatever the caller writes var freeSize uint64 if totalSize > clusterUsedSize { diff --git a/weed/topology/disk.go b/weed/topology/disk.go index 4ff5986e0..08ab99f63 100644 --- a/weed/topology/disk.go +++ b/weed/topology/disk.go @@ -129,6 +129,26 @@ func (d *DiskUsages) GetMaxVolumeCount() (maxVolumeCount int64) { return } +// FreeBytes sums the space one volume server reports as still free on its +// filesystems. reported is false as soon as a disk holding volume slots says +// nothing -- a volume server older than the field looks that way -- since +// leaving its space out would understate the room the server has. +func (d *DiskUsages) FreeBytes() (freeBytes uint64, reported bool) { + d.RLock() + defer d.RUnlock() + for _, diskUsageCounts := range d.usages { + usage := diskUsageCounts.snapshot() + if usage.diskTotalBytes <= 0 { + if usage.maxVolumeCount > 0 { + return 0, false + } + continue + } + freeBytes += uint64(max(0, usage.diskFreeBytes)) + } + return freeBytes, true +} + type DiskUsageCounts struct { volumeCount int64 remoteVolumeCount int64 diff --git a/weed/topology/topology.go b/weed/topology/topology.go index e8b12b357..276d7a092 100644 --- a/weed/topology/topology.go +++ b/weed/topology/topology.go @@ -162,6 +162,25 @@ func (t *Topology) unregisterDataNodeAddress(addr pb.ServerAddress, dn *DataNode } } +// FreeBytes sums what every volume server reports as free on its filesystems. +// reported is false unless all of them answered: the one that stayed quiet may +// be the one holding the room, and a partial sum would read as a cluster with +// none left. +func (t *Topology) FreeBytes() (freeBytes uint64, reported bool) { + for _, dcNode := range t.Children() { + for _, rackNode := range dcNode.Children() { + for _, dataNode := range rackNode.Children() { + nodeFreeBytes, nodeReported := dataNode.GetDiskUsages().FreeBytes() + if !nodeReported { + return 0, false + } + freeBytes += nodeFreeBytes + } + } + } + return freeBytes, true +} + func (t *Topology) IsChildLocked() (bool, error) { if t.IsLocked() { return true, errors.New("topology is locked")