diff --git a/weed/admin/dash/admin_server.go b/weed/admin/dash/admin_server.go index a339e4d39..ef7750348 100644 --- a/weed/admin/dash/admin_server.go +++ b/weed/admin/dash/admin_server.go @@ -23,6 +23,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb" "github.com/seaweedfs/seaweedfs/weed/pb/schema_pb" "github.com/seaweedfs/seaweedfs/weed/security" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" "github.com/seaweedfs/seaweedfs/weed/storage/super_block" "github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/wdclient" @@ -1698,6 +1699,17 @@ func collectCollectionStats(topologyInfo *master_pb.TopologyInfo) map[string]col } collectionMap[collection] = data } + for _, ecShardInfo := range diskInfo.EcShardInfos { + collection := ecShardInfo.Collection + if collection == "" { + collection = "default" + } + shards := erasure_coding.ShardsInfoFromVolumeEcShardInformationMessage(ecShardInfo) + data := collectionMap[collection] + data.PhysicalSize += int64(shards.TotalSize()) + data.LogicalSize += int64(shards.MinusParityShards().TotalSize()) + collectionMap[collection] = data + } } } } diff --git a/weed/admin/dash/collect_collection_stats_test.go b/weed/admin/dash/collect_collection_stats_test.go new file mode 100644 index 000000000..3cceab33c --- /dev/null +++ b/weed/admin/dash/collect_collection_stats_test.go @@ -0,0 +1,124 @@ +package dash + +import ( + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" +) + +// TestCollectCollectionStatsECUnevenShards verifies that EC shards spread +// across multiple disks are aggregated correctly, including the case where +// the last data shard is smaller than the rest. +func TestCollectCollectionStatsECUnevenShards(t *testing.T) { + // Standard 10+4 EC layout. Shards 0..9 are data, 10..13 are parity. + // Data shards 0..8 are 1000 bytes; data shard 9 is 500 bytes (uneven tail). + // Parity shards 10..13 are 1000 bytes each. + // + // Physical (raw) = 9*1000 + 500 + 4*1000 = 13500 + // Logical (data only) = 9*1000 + 500 = 9500 + + nodeA := &master_pb.DataNodeInfo{ + DiskInfos: map[string]*master_pb.DiskInfo{ + "disk1": { + EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ + { + Id: 42, + Collection: "bucket-a", + // Shards 0..6 held here. + EcIndexBits: (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3) | (1 << 4) | (1 << 5) | (1 << 6), + ShardSizes: []int64{1000, 1000, 1000, 1000, 1000, 1000, 1000}, + }, + }, + }, + }, + } + nodeB := &master_pb.DataNodeInfo{ + DiskInfos: map[string]*master_pb.DiskInfo{ + "disk1": { + EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ + { + Id: 42, + Collection: "bucket-a", + // Shards 7..13 held here. Shard 9 (data) is the short tail. + EcIndexBits: (1 << 7) | (1 << 8) | (1 << 9) | (1 << 10) | (1 << 11) | (1 << 12) | (1 << 13), + ShardSizes: []int64{1000, 1000, 500, 1000, 1000, 1000, 1000}, + }, + }, + }, + }, + } + + topo := &master_pb.TopologyInfo{ + DataCenterInfos: []*master_pb.DataCenterInfo{ + { + RackInfos: []*master_pb.RackInfo{ + { + DataNodeInfos: []*master_pb.DataNodeInfo{nodeA, nodeB}, + }, + }, + }, + }, + } + + stats := collectCollectionStats(topo) + + got, ok := stats["bucket-a"] + if !ok { + t.Fatalf("expected collection bucket-a in stats, got: %v", stats) + } + + const wantPhysical int64 = 13500 + const wantLogical int64 = 9500 + + if got.PhysicalSize != wantPhysical { + t.Errorf("PhysicalSize: got %d, want %d", got.PhysicalSize, wantPhysical) + } + if got.LogicalSize != wantLogical { + t.Errorf("LogicalSize: got %d, want %d", got.LogicalSize, wantLogical) + } +} + +// TestCollectCollectionStatsECEmptyCollection verifies that EC shards with +// an empty collection name are bucketed under "default". +func TestCollectCollectionStatsECEmptyCollection(t *testing.T) { + topo := &master_pb.TopologyInfo{ + DataCenterInfos: []*master_pb.DataCenterInfo{ + { + RackInfos: []*master_pb.RackInfo{ + { + DataNodeInfos: []*master_pb.DataNodeInfo{ + { + DiskInfos: map[string]*master_pb.DiskInfo{ + "disk1": { + EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ + { + Id: 1, + Collection: "", + EcIndexBits: (1 << 0) | (1 << 10), + ShardSizes: []int64{2000, 2000}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + stats := collectCollectionStats(topo) + + got, ok := stats["default"] + if !ok { + t.Fatalf("expected collection default in stats, got: %v", stats) + } + if got.PhysicalSize != 4000 { + t.Errorf("PhysicalSize: got %d, want 4000", got.PhysicalSize) + } + // Only shard 0 is a data shard; shard 10 is parity. + if got.LogicalSize != 2000 { + t.Errorf("LogicalSize: got %d, want 2000", got.LogicalSize) + } +}