diff --git a/weed/mount/weedfs_stats.go b/weed/mount/weedfs_stats.go index 152e0b1e8..701dc5619 100644 --- a/weed/mount/weedfs_stats.go +++ b/weed/mount/weedfs_stats.go @@ -64,8 +64,8 @@ func (wfs *WFS) StatFs(cancel <-chan struct{}, in *fuse.InHeader, out *fuse.Stat return nil }) if err != nil { + // the last known sizes beat the empty filesystem a bare return reports glog.V(0).Infof("filer Statistics: %v", err) - return fuse.OK } } diff --git a/weed/server/master_grpc_server_statistics_test.go b/weed/server/master_grpc_server_statistics_test.go index 6a8ba1daf..1e78ea959 100644 --- a/weed/server/master_grpc_server_statistics_test.go +++ b/weed/server/master_grpc_server_statistics_test.go @@ -122,3 +122,52 @@ func TestStatisticsReplicaCopyCount(t *testing.T) { }) } } + +// reportDiskBytes has the first nodeCount nodes report the same filesystem +// capacity, the way a volume server does in its heartbeat. +func reportDiskBytes(ms *MasterServer, nodeCount int, totalBytes, freeBytes uint64) { + rack := ms.Topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1") + for _, node := range rack.Children()[:nodeCount] { + node.(*topology.DataNode).AdjustDiskUsageBytes( + map[string]uint64{"": totalBytes}, map[string]uint64{"": freeBytes}) + } +} + +// TestStatisticsPhysicalCapacity covers a cluster configured with more volume +// slots than its disks hold. Slots promise 40MB here; what the disks can still +// take is what gets reported. +func TestStatisticsPhysicalCapacity(t *testing.T) { + ms := newStatisticsMaster(t) + + slotCapacity := uint64(40 * 1024 * 1024) + statistics := func() *master_pb.StatisticsResponse { + t.Helper() + resp, err := ms.Statistics(context.Background(), &master_pb.StatisticsRequest{}) + if err != nil { + t.Fatalf("Statistics: %v", err) + } + return resp + } + + if got := statistics().TotalSize; got != slotCapacity { + t.Fatalf("disks that report nothing: got %d, want %d", got, slotCapacity) + } + + // a volume server too old to report leaves the whole cluster on its slots, + // since the room it holds would otherwise go missing + reportDiskBytes(ms, 3, 8<<20, 1<<20) + if got := statistics().TotalSize; got != slotCapacity { + t.Errorf("one disk of four reporting nothing: got %d, want %d", got, slotCapacity) + } + + reportDiskBytes(ms, 4, 8<<20, 1<<20) + resp := statistics() + if want := resp.UsedSize + (4 << 20); resp.TotalSize != want { + t.Errorf("four disks with 1MB free: got %d, want %d", resp.TotalSize, want) + } + + reportDiskBytes(ms, 4, 1<<30, 1<<30) + if got := statistics().TotalSize; got != slotCapacity { + t.Errorf("disks roomier than the slots: got %d, want %d", got, slotCapacity) + } +} diff --git a/weed/server/master_grpc_server_volume.go b/weed/server/master_grpc_server_volume.go index 0f4d3f6a2..66aea2d24 100644 --- a/weed/server/master_grpc_server_volume.go +++ b/weed/server/master_grpc_server_volume.go @@ -239,6 +239,13 @@ func (ms *MasterServer) Statistics(ctx context.Context, req *master_pb.Statistic if req.Collection != "" { clusterUsedSize = ms.Topo.CollectionVolumeStats("").UsedSize } + // volume slots are provisioning, not capacity: a cluster configured with + // more of them than its disks hold would report space it can never take. + // What the disks still have free, on top of what the cluster already wrote, + // is the real ceiling. + if freeBytes, reported := ms.Topo.FreeBytes(); reported { + totalSize = min(totalSize, clusterUsedSize+freeBytes) + } // and the free space holds that many copies fewer of whatever the caller writes var freeSize uint64 if totalSize > clusterUsedSize { diff --git a/weed/topology/disk.go b/weed/topology/disk.go index 4ff5986e0..08ab99f63 100644 --- a/weed/topology/disk.go +++ b/weed/topology/disk.go @@ -129,6 +129,26 @@ func (d *DiskUsages) GetMaxVolumeCount() (maxVolumeCount int64) { return } +// FreeBytes sums the space one volume server reports as still free on its +// filesystems. reported is false as soon as a disk holding volume slots says +// nothing -- a volume server older than the field looks that way -- since +// leaving its space out would understate the room the server has. +func (d *DiskUsages) FreeBytes() (freeBytes uint64, reported bool) { + d.RLock() + defer d.RUnlock() + for _, diskUsageCounts := range d.usages { + usage := diskUsageCounts.snapshot() + if usage.diskTotalBytes <= 0 { + if usage.maxVolumeCount > 0 { + return 0, false + } + continue + } + freeBytes += uint64(max(0, usage.diskFreeBytes)) + } + return freeBytes, true +} + type DiskUsageCounts struct { volumeCount int64 remoteVolumeCount int64 diff --git a/weed/topology/topology.go b/weed/topology/topology.go index e8b12b357..276d7a092 100644 --- a/weed/topology/topology.go +++ b/weed/topology/topology.go @@ -162,6 +162,25 @@ func (t *Topology) unregisterDataNodeAddress(addr pb.ServerAddress, dn *DataNode } } +// FreeBytes sums what every volume server reports as free on its filesystems. +// reported is false unless all of them answered: the one that stayed quiet may +// be the one holding the room, and a partial sum would read as a cluster with +// none left. +func (t *Topology) FreeBytes() (freeBytes uint64, reported bool) { + for _, dcNode := range t.Children() { + for _, rackNode := range dcNode.Children() { + for _, dataNode := range rackNode.Children() { + nodeFreeBytes, nodeReported := dataNode.GetDiskUsages().FreeBytes() + if !nodeReported { + return 0, false + } + freeBytes += nodeFreeBytes + } + } + } + return freeBytes, true +} + func (t *Topology) IsChildLocked() (bool, error) { if t.IsLocked() { return true, errors.New("topology is locked")