master: statistics used size covers all collections and layouts (#10319)

StatFs on a mount reported cluster-wide total capacity but used size from
a single volume layout keyed by collection, replication, ttl, and disk
type. A mount without -collection therefore showed only the default
collection's usage, hiding data in named collections, and even a
collection-scoped mount missed volumes with a different replication,
ttl, or disk type.

Aggregate used size and file count across all layouts of the requested
collection, and across every collection when the collection is empty,
matching how Topology.Lookup treats an empty collection. Looking up
stats no longer creates a phantom collection as a side effect.

weed/server/master_grpc_server_volume.go

@@ -204,20 +204,9 @@ func (ms *MasterServer) Statistics(ctx context.Context, req *master_pb.Statistic
 		return nil, raft.NotLeaderError
 	}
 
-	if req.Replication == "" {
-		req.Replication = ms.option.DefaultReplicaPlacement
-	}
-	replicaPlacement, err := super_block.NewReplicaPlacementFromString(req.Replication)
-	if err != nil {
-		return nil, err
-	}
-	ttl, err := needle.ReadTTL(req.Ttl)
-	if err != nil {
-		return nil, err
-	}
-
-	volumeLayout := ms.Topo.GetVolumeLayout(req.Collection, replicaPlacement, ttl, types.ToDiskType(req.DiskType))
-	stats := volumeLayout.Stats()
+	// an empty collection means all collections, and a named collection covers
+	// all its layouts, so used size matches the topology-wide total size below
+	stats := ms.Topo.CollectionVolumeStats(req.Collection)
 	totalSize := ms.Topo.GetDiskUsages().GetMaxVolumeCount() * int64(ms.option.VolumeSizeLimitMB) * 1024 * 1024
 	resp := &master_pb.StatisticsResponse{
 		TotalSize: uint64(totalSize),

weed/topology/topology.go

@@ -431,6 +431,29 @@ func (t *Topology) GetVolumeLayout(collectionName string, rp *super_block.Replic
 	}).(*Collection).GetOrCreateVolumeLayout(rp, ttl, diskType)
 }
 
+// CollectionVolumeStats aggregates stats across all volume layouts of one
+// collection, or across every collection when collectionName is empty.
+func (t *Topology) CollectionVolumeStats(collectionName string) *VolumeLayoutStats {
+	ret := &VolumeLayoutStats{}
+	var collections []*Collection
+	if collectionName == "" {
+		for _, c := range t.collectionMap.Items() {
+			collections = append(collections, c.(*Collection))
+		}
+	} else if c, found := t.FindCollection(collectionName); found {
+		collections = append(collections, c)
+	}
+	for _, c := range collections {
+		for _, vl := range c.GetAllVolumeLayouts() {
+			stats := vl.Stats()
+			ret.TotalSize += stats.TotalSize
+			ret.UsedSize += stats.UsedSize
+			ret.FileCount += stats.FileCount
+		}
+	}
+	return ret
+}
+
 func (t *Topology) ListCollections(includeNormalVolumes, includeEcVolumes bool) (ret []string) {
 	found := make(map[string]bool)
 

weed/topology/topology_stats_test.go

@@ -0,0 +1,47 @@
+package topology
+
+import (
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/sequence"
+	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+)
+
+func TestCollectionVolumeStats(t *testing.T) {
+	topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
+
+	dc := topo.GetOrCreateDataCenter("dc1")
+	rack := dc.GetOrCreateRack("rack1")
+	maxVolumeCounts := map[string]uint32{"": 25, "ssd": 12}
+	dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts)
+
+	volumeMessages := []*master_pb.VolumeInformationMessage{
+		{Id: 1, Size: 1000, Collection: "", FileCount: 10, ReplicaPlacement: 0, Version: uint32(needle.GetCurrentVersion())},
+		{Id: 2, Size: 2000, Collection: "c1", FileCount: 20, ReplicaPlacement: 0, Version: uint32(needle.GetCurrentVersion())},
+		{Id: 3, Size: 3000, Collection: "c1", FileCount: 30, ReplicaPlacement: 0, Version: uint32(needle.GetCurrentVersion()), DiskType: "ssd"},
+		{Id: 4, Size: 4000, Collection: "c2", FileCount: 40, ReplicaPlacement: 1, Version: uint32(needle.GetCurrentVersion())},
+	}
+	topo.SyncDataNodeRegistration(volumeMessages, dn)
+
+	// VolumeLocationList.Stats only counts nodes connected for over a minute
+	dn.LastSeen = time.Now().Unix() - 61
+
+	allStats := topo.CollectionVolumeStats("")
+	assert(t, "all collections used size", int(allStats.UsedSize), 10000)
+	assert(t, "all collections file count", int(allStats.FileCount), 100)
+
+	c1Stats := topo.CollectionVolumeStats("c1")
+	assert(t, "c1 used size across disk types", int(c1Stats.UsedSize), 5000)
+	assert(t, "c1 file count", int(c1Stats.FileCount), 50)
+
+	c2Stats := topo.CollectionVolumeStats("c2")
+	assert(t, "c2 used size", int(c2Stats.UsedSize), 4000)
+
+	missingStats := topo.CollectionVolumeStats("no-such-collection")
+	assert(t, "missing collection used size", int(missingStats.UsedSize), 0)
+	if _, found := topo.FindCollection("no-such-collection"); found {
+		t.Errorf("stats query should not create a phantom collection")
+	}
+}