diff --git a/weed/server/master_grpc_server_volume.go b/weed/server/master_grpc_server_volume.go index 184a6c214..287706821 100644 --- a/weed/server/master_grpc_server_volume.go +++ b/weed/server/master_grpc_server_volume.go @@ -204,20 +204,9 @@ func (ms *MasterServer) Statistics(ctx context.Context, req *master_pb.Statistic return nil, raft.NotLeaderError } - if req.Replication == "" { - req.Replication = ms.option.DefaultReplicaPlacement - } - replicaPlacement, err := super_block.NewReplicaPlacementFromString(req.Replication) - if err != nil { - return nil, err - } - ttl, err := needle.ReadTTL(req.Ttl) - if err != nil { - return nil, err - } - - volumeLayout := ms.Topo.GetVolumeLayout(req.Collection, replicaPlacement, ttl, types.ToDiskType(req.DiskType)) - stats := volumeLayout.Stats() + // an empty collection means all collections, and a named collection covers + // all its layouts, so used size matches the topology-wide total size below + stats := ms.Topo.CollectionVolumeStats(req.Collection) totalSize := ms.Topo.GetDiskUsages().GetMaxVolumeCount() * int64(ms.option.VolumeSizeLimitMB) * 1024 * 1024 resp := &master_pb.StatisticsResponse{ TotalSize: uint64(totalSize), diff --git a/weed/topology/topology.go b/weed/topology/topology.go index ddba44d2d..4b2adb988 100644 --- a/weed/topology/topology.go +++ b/weed/topology/topology.go @@ -431,6 +431,29 @@ func (t *Topology) GetVolumeLayout(collectionName string, rp *super_block.Replic }).(*Collection).GetOrCreateVolumeLayout(rp, ttl, diskType) } +// CollectionVolumeStats aggregates stats across all volume layouts of one +// collection, or across every collection when collectionName is empty. +func (t *Topology) CollectionVolumeStats(collectionName string) *VolumeLayoutStats { + ret := &VolumeLayoutStats{} + var collections []*Collection + if collectionName == "" { + for _, c := range t.collectionMap.Items() { + collections = append(collections, c.(*Collection)) + } + } else if c, found := t.FindCollection(collectionName); found { + collections = append(collections, c) + } + for _, c := range collections { + for _, vl := range c.GetAllVolumeLayouts() { + stats := vl.Stats() + ret.TotalSize += stats.TotalSize + ret.UsedSize += stats.UsedSize + ret.FileCount += stats.FileCount + } + } + return ret +} + func (t *Topology) ListCollections(includeNormalVolumes, includeEcVolumes bool) (ret []string) { found := make(map[string]bool) diff --git a/weed/topology/topology_stats_test.go b/weed/topology/topology_stats_test.go new file mode 100644 index 000000000..2c17ab684 --- /dev/null +++ b/weed/topology/topology_stats_test.go @@ -0,0 +1,47 @@ +package topology + +import ( + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/sequence" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" +) + +func TestCollectionVolumeStats(t *testing.T) { + topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false) + + dc := topo.GetOrCreateDataCenter("dc1") + rack := dc.GetOrCreateRack("rack1") + maxVolumeCounts := map[string]uint32{"": 25, "ssd": 12} + dn := rack.GetOrCreateDataNode("127.0.0.1", 34534, 0, "127.0.0.1", "", maxVolumeCounts) + + volumeMessages := []*master_pb.VolumeInformationMessage{ + {Id: 1, Size: 1000, Collection: "", FileCount: 10, ReplicaPlacement: 0, Version: uint32(needle.GetCurrentVersion())}, + {Id: 2, Size: 2000, Collection: "c1", FileCount: 20, ReplicaPlacement: 0, Version: uint32(needle.GetCurrentVersion())}, + {Id: 3, Size: 3000, Collection: "c1", FileCount: 30, ReplicaPlacement: 0, Version: uint32(needle.GetCurrentVersion()), DiskType: "ssd"}, + {Id: 4, Size: 4000, Collection: "c2", FileCount: 40, ReplicaPlacement: 1, Version: uint32(needle.GetCurrentVersion())}, + } + topo.SyncDataNodeRegistration(volumeMessages, dn) + + // VolumeLocationList.Stats only counts nodes connected for over a minute + dn.LastSeen = time.Now().Unix() - 61 + + allStats := topo.CollectionVolumeStats("") + assert(t, "all collections used size", int(allStats.UsedSize), 10000) + assert(t, "all collections file count", int(allStats.FileCount), 100) + + c1Stats := topo.CollectionVolumeStats("c1") + assert(t, "c1 used size across disk types", int(c1Stats.UsedSize), 5000) + assert(t, "c1 file count", int(c1Stats.FileCount), 50) + + c2Stats := topo.CollectionVolumeStats("c2") + assert(t, "c2 used size", int(c2Stats.UsedSize), 4000) + + missingStats := topo.CollectionVolumeStats("no-such-collection") + assert(t, "missing collection used size", int(missingStats.UsedSize), 0) + if _, found := topo.FindCollection("no-such-collection"); found { + t.Errorf("stats query should not create a phantom collection") + } +}