fix(s3/shell): factor EC volumes into bucket size metrics and collection.list (#9182)

* fix(s3/shell): include EC volumes in bucket size metrics and collection.list

S3 bucket size metrics exported to Prometheus (and fed through
stats.UpdateBucketSizeMetrics) are computed by
collectCollectionInfoFromTopology, which only walked diskInfo.VolumeInfos.
As soon as a volume was encoded to EC it dropped out of every aggregate,
so Grafana showed bucket sizes shrinking while physical disk usage kept
climbing. The shell helper collectCollectionInfo — used by collection.list
and s3.bucket.quota.enforce — had the same gap, with the EC branch left as
a commented-out TODO.

Fold EC shards into both paths using the same approach the admin dashboard
already uses (PR #9093):

- PhysicalSize / Size sum across shard holders: EC shards are node-local
  (not replicas), so per-node TotalSize() and MinusParityShards().TotalSize()
  sum to the whole-volume physical and logical sizes respectively.
- FileCount is deduped via max across reporters (every shard holder reports
  the same .ecx count; a slow node with a not-yet-loaded .ecx reports 0 and
  must not pin the aggregate).
- DeleteCount is summed (each delete tombstones exactly one node's .ecj).
- VolumeCount increments once per unique EC volume id.

Adds regression tests covering pure-EC, mixed regular+EC, and the
slow-reporter FileCount dedupe case.

Refs #9086

* Address PR review feedback: EC size helpers, composite key, VolumeCount dedupe

- Add EcShardsTotalSize / EcShardsDataSize helpers in the erasure_coding
  package that walk the shard bitmap directly instead of materializing a
  ShardsInfo and copying it via MinusParityShards(). Keeps the
  DataShardsCount dependency encapsulated in one place and avoids the
  per-shard allocation/copy overhead in the metrics hot path.
- Switch shell collectCollectionInfo ecVolumes map to a composite
  {collection, volumeId} key, matching the bucket_size_metrics collector
  and defending against any cross-collection volume id aliasing.
- Dedupe VolumeCount in shell addToCollection by volume id so regular
  volumes aren't counted once per replica presence. Aligns the shell's
  collection.list output with the S3 metrics collector and the EC branch,
  all of which now report logical volume counts.
- Add unit tests for the new helpers and for the regular-volume
  VolumeCount dedupe.

* Parameterize EcShardsDataSize with dataShards for custom EC ratios

Add a dataShards parameter to EcShardsDataSize so forks with per-volume
ratio metadata (e.g. the enterprise data_shards field carried on an
extended VolumeEcShardInformationMessage) can pass the configured value
and get accurate logical sizes under custom EC policies like 6+3 or 16+6.
Passing 0 or a negative value falls back to the upstream DataShardsCount
default, which is correct for the fixed 10+4 layout — so OSS callers in
s3api and shell pass 0 and keep their current behavior.

Added table cases covering the custom 6+3 and 16+6 paths so the
parameterization is pinned by tests.

weed/s3api/bucket_size_metrics.go

@@ -13,6 +13,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"github.com/seaweedfs/seaweedfs/weed/stats"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
 )
 
 const (
@@ -197,13 +198,30 @@ func (s3a *S3ApiServer) listBucketNames(ctx context.Context) ([]string, error) {
 	return buckets, err
 }
 
+// ecVolumeAgg accumulates per-volume EC counts across the shard holders.
+// fileCount is volume-wide (every holder sees the same .ecx) so we take the
+// max across reporters to avoid a slow node with a not-yet-loaded .ecx
+// pinning the aggregate at 0. deleteCount is node-local to each .ecj
+// deletion journal, so it's summed across reporters.
+type ecVolumeAgg struct {
+	collection  string
+	fileCount   uint64
+	deleteCount uint64
+}
+
 // collectCollectionInfoFromTopology extracts collection info from topology.
 // Deduplicates by volume ID to correctly handle missing replicas.
 // Unlike dividing by copyCount (which would give wrong results if replicas are missing),
 // we track seen volume IDs and only count each volume once for logical size/count.
+// EC-encoded volumes are folded in via per-shard aggregation: every shard is
+// node-local (not a replica), so shard sizes are summed across nodes; the
+// per-volume file/delete counts carried on each shard message are deduped
+// via max/sum so the aggregate doesn't double-count or drop after a volume
+// is converted from regular to erasure coding.
 func collectCollectionInfoFromTopology(t *master_pb.TopologyInfo, collectionInfos map[string]*CollectionInfo) {
 	// Track which volumes we've already seen to deduplicate by volume ID
 	seenVolumes := make(map[volumeKey]bool)
+	ecVolumes := make(map[volumeKey]*ecVolumeAgg)
 
 	for _, dc := range t.DataCenterInfos {
 		for _, r := range dc.RackInfos {
@@ -235,8 +253,49 @@ func collectCollectionInfoFromTopology(t *master_pb.TopologyInfo, collectionInfo
 						cif.DeletedByteCount += float64(vi.DeletedByteCount)
 						cif.VolumeCount++
 					}
+
+					for _, esi := range diskInfo.EcShardInfos {
+						c := esi.Collection
+						cif, found := collectionInfos[c]
+						if !found {
+							cif = &CollectionInfo{}
+							collectionInfos[c] = cif
+						}
+
+						// EC shards are node-local (no replication), so both
+						// physical and logical shard sizes sum across nodes
+						// without any dedupe. Logical size excludes parity
+						// shards; physical size includes them. Upstream OSS
+						// uses the fixed 10+4 ratio (dataShards=0 → default);
+						// forks with per-volume ratio metadata can pass the
+						// configured value here.
+						cif.PhysicalSize += float64(erasure_coding.EcShardsTotalSize(esi))
+						cif.Size += float64(erasure_coding.EcShardsDataSize(esi, 0))
+
+						key := volumeKey{collection: c, volumeId: esi.Id}
+						agg, ok := ecVolumes[key]
+						if !ok {
+							agg = &ecVolumeAgg{collection: c}
+							ecVolumes[key] = agg
+							cif.VolumeCount++
+						}
+						if esi.FileCount > agg.fileCount {
+							agg.fileCount = esi.FileCount
+						}
+						agg.deleteCount += esi.DeleteCount
+					}
 				}
 			}
 		}
 	}
+
+	// Fold deduped EC file/delete counts into each collection's totals.
+	for _, agg := range ecVolumes {
+		cif := collectionInfos[agg.collection]
+		if cif == nil {
+			continue
+		}
+		cif.FileCount += float64(agg.fileCount)
+		cif.DeleteCount += float64(agg.deleteCount)
+	}
 }

weed/s3api/bucket_size_metrics_test.go

@@ -0,0 +1,211 @@
+package s3api
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+// TestCollectCollectionInfoFromTopologyEC verifies that EC-encoded volumes
+// contribute to per-collection logical/physical size, file count, and volume
+// count. Before this fix, encoding a volume to EC caused the bucket size
+// metrics exported to Prometheus to drop to zero for that volume.
+//
+// Layout: one 10+4 EC volume in collection "crm-docs-storage", 14 shards of
+// 1000 bytes each split across two nodes.
+//   - nodeA holds data shards 0..6 (7 * 1000 = 7000)
+//   - nodeB holds data shards 7..9 (3 * 1000 = 3000) and parity 10..13 (4 * 1000 = 4000)
+//
+// Expected:
+//   - PhysicalSize = 14 * 1000 = 14000
+//   - Size (logical, data shards) = 10 * 1000 = 10000
+//   - FileCount = 100 total - (2 + 3) local deletes = 95 is NOT what we check; the
+//     collector reports raw file_count and delete_count as separate gauges, so
+//     we assert FileCount = 100 (max across reporters) and DeleteCount = 5 (sum).
+//   - VolumeCount = 1 (one unique EC volume)
+func TestCollectCollectionInfoFromTopologyEC(t *testing.T) {
+	nodeA := &master_pb.DataNodeInfo{
+		DiskInfos: map[string]*master_pb.DiskInfo{
+			"disk1": {
+				EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+					{
+						Id:          42,
+						Collection:  "crm-docs-storage",
+						EcIndexBits: (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3) | (1 << 4) | (1 << 5) | (1 << 6),
+						ShardSizes:  []int64{1000, 1000, 1000, 1000, 1000, 1000, 1000},
+						FileCount:   100,
+						DeleteCount: 2,
+					},
+				},
+			},
+		},
+	}
+	nodeB := &master_pb.DataNodeInfo{
+		DiskInfos: map[string]*master_pb.DiskInfo{
+			"disk1": {
+				EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+					{
+						Id:          42,
+						Collection:  "crm-docs-storage",
+						EcIndexBits: (1 << 7) | (1 << 8) | (1 << 9) | (1 << 10) | (1 << 11) | (1 << 12) | (1 << 13),
+						ShardSizes:  []int64{1000, 1000, 1000, 1000, 1000, 1000, 1000},
+						FileCount:   100,
+						DeleteCount: 3,
+					},
+				},
+			},
+		},
+	}
+
+	topo := &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				RackInfos: []*master_pb.RackInfo{
+					{
+						DataNodeInfos: []*master_pb.DataNodeInfo{nodeA, nodeB},
+					},
+				},
+			},
+		},
+	}
+
+	got := make(map[string]*CollectionInfo)
+	collectCollectionInfoFromTopology(topo, got)
+
+	info, ok := got["crm-docs-storage"]
+	if !ok {
+		t.Fatalf("expected collection crm-docs-storage, got: %v", got)
+	}
+	if info.PhysicalSize != 14000 {
+		t.Errorf("PhysicalSize: got %.0f, want 14000", info.PhysicalSize)
+	}
+	if info.Size != 10000 {
+		t.Errorf("Size (logical): got %.0f, want 10000", info.Size)
+	}
+	if info.FileCount != 100 {
+		t.Errorf("FileCount: got %.0f, want 100 (max across reporters)", info.FileCount)
+	}
+	if info.DeleteCount != 5 {
+		t.Errorf("DeleteCount: got %.0f, want 5 (sum across reporters)", info.DeleteCount)
+	}
+	if info.VolumeCount != 1 {
+		t.Errorf("VolumeCount: got %d, want 1", info.VolumeCount)
+	}
+}
+
+// TestCollectCollectionInfoFromTopologyMixed verifies that regular and EC
+// volumes accumulate under the same collection without one clobbering the
+// other, which is the state during an in-progress EC conversion.
+func TestCollectCollectionInfoFromTopologyMixed(t *testing.T) {
+	node := &master_pb.DataNodeInfo{
+		DiskInfos: map[string]*master_pb.DiskInfo{
+			"disk1": {
+				VolumeInfos: []*master_pb.VolumeInformationMessage{
+					{
+						Id:               1,
+						Collection:       "bucket-mix",
+						Size:             5000,
+						FileCount:        50,
+						DeleteCount:      1,
+						DeletedByteCount: 100,
+					},
+				},
+				EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+					{
+						Id:          2,
+						Collection:  "bucket-mix",
+						EcIndexBits: (1 << 0) | (1 << 1) | (1 << 10), // 2 data + 1 parity
+						ShardSizes:  []int64{3000, 3000, 3000},
+						FileCount:   80,
+						DeleteCount: 4,
+					},
+				},
+			},
+		},
+	}
+
+	topo := &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				RackInfos: []*master_pb.RackInfo{
+					{
+						DataNodeInfos: []*master_pb.DataNodeInfo{node},
+					},
+				},
+			},
+		},
+	}
+
+	got := make(map[string]*CollectionInfo)
+	collectCollectionInfoFromTopology(topo, got)
+
+	info, ok := got["bucket-mix"]
+	if !ok {
+		t.Fatalf("expected collection bucket-mix, got: %v", got)
+	}
+	// Regular volume: 5000 physical + logical. EC shards: 9000 physical,
+	// 6000 logical (data shards 0 and 1).
+	if info.PhysicalSize != 5000+9000 {
+		t.Errorf("PhysicalSize: got %.0f, want 14000", info.PhysicalSize)
+	}
+	if info.Size != 5000+6000 {
+		t.Errorf("Size: got %.0f, want 11000", info.Size)
+	}
+	if info.FileCount != 50+80 {
+		t.Errorf("FileCount: got %.0f, want 130", info.FileCount)
+	}
+	if info.DeleteCount != 1+4 {
+		t.Errorf("DeleteCount: got %.0f, want 5", info.DeleteCount)
+	}
+	if info.VolumeCount != 2 {
+		t.Errorf("VolumeCount: got %d, want 2", info.VolumeCount)
+	}
+}
+
+// TestCollectCollectionInfoFromTopologyECFileCountMaxDedupe verifies that a
+// slow shard holder reporting file_count=0 (because it has not yet finished
+// loading .ecx) does not pin the per-volume FileCount at 0.
+func TestCollectCollectionInfoFromTopologyECFileCountMaxDedupe(t *testing.T) {
+	makeNode := func(bits uint32, sizes []int64, fileCount uint64) *master_pb.DataNodeInfo {
+		return &master_pb.DataNodeInfo{
+			DiskInfos: map[string]*master_pb.DiskInfo{
+				"disk1": {
+					EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+						{
+							Id:          11,
+							Collection:  "bucket-b",
+							EcIndexBits: bits,
+							ShardSizes:  sizes,
+							FileCount:   fileCount,
+						},
+					},
+				},
+			},
+		}
+	}
+
+	topo := &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				RackInfos: []*master_pb.RackInfo{
+					{
+						DataNodeInfos: []*master_pb.DataNodeInfo{
+							makeNode((1<<0)|(1<<1)|(1<<2)|(1<<3)|(1<<4)|(1<<5)|(1<<6), []int64{1, 1, 1, 1, 1, 1, 1}, 0),
+							makeNode((1<<7)|(1<<8)|(1<<9)|(1<<10)|(1<<11)|(1<<12)|(1<<13), []int64{1, 1, 1, 1, 1, 1, 1}, 6),
+						},
+					},
+				},
+			},
+		},
+	}
+
+	got := make(map[string]*CollectionInfo)
+	collectCollectionInfoFromTopology(topo, got)
+	info, ok := got["bucket-b"]
+	if !ok {
+		t.Fatalf("expected collection bucket-b, got: %v", got)
+	}
+	if info.FileCount != 6 {
+		t.Errorf("FileCount: got %.0f, want 6 (max across reporters)", info.FileCount)
+	}
+}

weed/shell/command_collection_list.go

@@ -6,6 +6,7 @@ import (
 	"io"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
 	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
 )
 
@@ -84,7 +85,21 @@ func ListCollectionNames(commandEnv *CommandEnv, includeNormalVolumes, includeEc
 	return
 }
 
-func addToCollection(collectionInfos map[string]*CollectionInfo, vif *master_pb.VolumeInformationMessage) {
+// volumeKey uniquely identifies a volume for per-collection dedupe. Volume
+// IDs are scoped to a collection, so we key by (collection, volumeId) to
+// avoid cross-collection aliasing if the same numeric ID is ever reused.
+type volumeKey struct {
+	collection string
+	volumeId   uint32
+}
+
+// addToCollection folds one replica of a regular volume into the collection
+// totals. Size/FileCount/DeleteCount/DeletedByteCount are divided by the
+// replication factor so that summing over all replicas yields the whole-
+// volume value. VolumeCount is deduped across replicas via seenVolumes so
+// it reports logical volumes (same semantics as the S3 bucket metrics
+// collector and the EC branch below), not shard/replica presences.
+func addToCollection(collectionInfos map[string]*CollectionInfo, seenVolumes map[volumeKey]bool, vif *master_pb.VolumeInformationMessage) {
 	c := vif.Collection
 	cif, found := collectionInfos[c]
 	if !found {
@@ -97,22 +112,71 @@ func addToCollection(collectionInfos map[string]*CollectionInfo, vif *master_pb.
 	cif.DeleteCount += float64(vif.DeleteCount) / copyCount
 	cif.FileCount += float64(vif.FileCount) / copyCount
 	cif.DeletedByteCount += float64(vif.DeletedByteCount) / copyCount
-	cif.VolumeCount++
+
+	key := volumeKey{collection: c, volumeId: vif.Id}
+	if !seenVolumes[key] {
+		seenVolumes[key] = true
+		cif.VolumeCount++
+	}
+}
+
+// ecCollectionAgg accumulates per-EC-volume counts across the shard holders.
+// fileCount is volume-wide (every holder reports the same .ecx count) so it
+// is deduped via max; deleteCount is node-local to each .ecj and summed.
+type ecCollectionAgg struct {
+	collection  string
+	fileCount   uint64
+	deleteCount uint64
 }
 
 func collectCollectionInfo(t *master_pb.TopologyInfo, collectionInfos map[string]*CollectionInfo) {
+	seenVolumes := make(map[volumeKey]bool)
+	ecVolumes := make(map[volumeKey]*ecCollectionAgg)
 	for _, dc := range t.DataCenterInfos {
 		for _, r := range dc.RackInfos {
 			for _, dn := range r.DataNodeInfos {
 				for _, diskInfo := range dn.DiskInfos {
 					for _, vi := range diskInfo.VolumeInfos {
-						addToCollection(collectionInfos, vi)
+						addToCollection(collectionInfos, seenVolumes, vi)
+					}
+					for _, esi := range diskInfo.EcShardInfos {
+						c := esi.Collection
+						cif, found := collectionInfos[c]
+						if !found {
+							cif = &CollectionInfo{}
+							collectionInfos[c] = cif
+						}
+
+						// EC shards are node-local, so data-shard sizes sum
+						// across nodes to give the logical volume size.
+						// Upstream OSS uses the fixed 10+4 ratio; forks with
+						// per-volume ratio metadata should pass the
+						// configured dataShards value here.
+						cif.Size += float64(erasure_coding.EcShardsDataSize(esi, 0))
+
+						key := volumeKey{collection: c, volumeId: esi.Id}
+						agg, ok := ecVolumes[key]
+						if !ok {
+							agg = &ecCollectionAgg{collection: c}
+							ecVolumes[key] = agg
+							cif.VolumeCount++
+						}
+						if esi.FileCount > agg.fileCount {
+							agg.fileCount = esi.FileCount
+						}
+						agg.deleteCount += esi.DeleteCount
 					}
-					//for _, ecShardInfo := range diskInfo.EcShardInfos {
-					//
-					//}
 				}
 			}
 		}
 	}
+
+	for _, agg := range ecVolumes {
+		cif := collectionInfos[agg.collection]
+		if cif == nil {
+			continue
+		}
+		cif.FileCount += float64(agg.fileCount)
+		cif.DeleteCount += float64(agg.deleteCount)
+	}
 }

weed/shell/command_collection_list_test.go

@@ -0,0 +1,206 @@
+package shell
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+// TestCollectCollectionInfoEC verifies that EC-encoded volumes contribute to
+// collection.list totals and s3.bucket.quota.enforce sums. Before this fix
+// the EC branch was a no-op, so encoded volumes silently dropped out of
+// collection size accounting.
+func TestCollectCollectionInfoEC(t *testing.T) {
+	// One 10+4 EC volume split across two nodes: 14 shards * 1000 bytes.
+	// Data shards 0..9 live on (nodeA: 0..6, nodeB: 7..9), parity 10..13
+	// on nodeB. Every shard holder reports file_count=100; node-local
+	// delete counts are 2 + 3 = 5.
+	nodeA := &master_pb.DataNodeInfo{
+		DiskInfos: map[string]*master_pb.DiskInfo{
+			"disk1": {
+				EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+					{
+						Id:          42,
+						Collection:  "bucket-a",
+						EcIndexBits: (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3) | (1 << 4) | (1 << 5) | (1 << 6),
+						ShardSizes:  []int64{1000, 1000, 1000, 1000, 1000, 1000, 1000},
+						FileCount:   100,
+						DeleteCount: 2,
+					},
+				},
+			},
+		},
+	}
+	nodeB := &master_pb.DataNodeInfo{
+		DiskInfos: map[string]*master_pb.DiskInfo{
+			"disk1": {
+				EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+					{
+						Id:          42,
+						Collection:  "bucket-a",
+						EcIndexBits: (1 << 7) | (1 << 8) | (1 << 9) | (1 << 10) | (1 << 11) | (1 << 12) | (1 << 13),
+						ShardSizes:  []int64{1000, 1000, 1000, 1000, 1000, 1000, 1000},
+						FileCount:   100,
+						DeleteCount: 3,
+					},
+				},
+			},
+		},
+	}
+
+	topo := &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				RackInfos: []*master_pb.RackInfo{
+					{
+						DataNodeInfos: []*master_pb.DataNodeInfo{nodeA, nodeB},
+					},
+				},
+			},
+		},
+	}
+
+	infos := make(map[string]*CollectionInfo)
+	collectCollectionInfo(topo, infos)
+
+	cif, ok := infos["bucket-a"]
+	if !ok {
+		t.Fatalf("expected collection bucket-a in infos, got: %v", infos)
+	}
+	// 10 data shards * 1000 bytes.
+	if cif.Size != 10000 {
+		t.Errorf("Size: got %.0f, want 10000", cif.Size)
+	}
+	if cif.FileCount != 100 {
+		t.Errorf("FileCount: got %.0f, want 100 (max across reporters)", cif.FileCount)
+	}
+	if cif.DeleteCount != 5 {
+		t.Errorf("DeleteCount: got %.0f, want 5 (sum across reporters)", cif.DeleteCount)
+	}
+	if cif.VolumeCount != 1 {
+		t.Errorf("VolumeCount: got %d, want 1", cif.VolumeCount)
+	}
+}
+
+// TestCollectCollectionInfoMixed verifies that a collection with both a
+// regular volume and an EC volume sums both branches without either clobbering
+// the other, matching the state mid-EC-conversion.
+func TestCollectCollectionInfoMixed(t *testing.T) {
+	node := &master_pb.DataNodeInfo{
+		DiskInfos: map[string]*master_pb.DiskInfo{
+			"disk1": {
+				VolumeInfos: []*master_pb.VolumeInformationMessage{
+					{
+						Id:          1,
+						Collection:  "bucket-mix",
+						Size:        5000,
+						FileCount:   50,
+						DeleteCount: 1,
+					},
+				},
+				EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{
+					{
+						Id:          2,
+						Collection:  "bucket-mix",
+						EcIndexBits: (1 << 0) | (1 << 1) | (1 << 10), // 2 data + 1 parity
+						ShardSizes:  []int64{3000, 3000, 3000},
+						FileCount:   80,
+						DeleteCount: 4,
+					},
+				},
+			},
+		},
+	}
+
+	topo := &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				RackInfos: []*master_pb.RackInfo{
+					{
+						DataNodeInfos: []*master_pb.DataNodeInfo{node},
+					},
+				},
+			},
+		},
+	}
+
+	infos := make(map[string]*CollectionInfo)
+	collectCollectionInfo(topo, infos)
+
+	cif, ok := infos["bucket-mix"]
+	if !ok {
+		t.Fatalf("expected collection bucket-mix in infos, got: %v", infos)
+	}
+	// Regular: 5000 (replicaCount=1). EC data shards: 6000.
+	if cif.Size != 5000+6000 {
+		t.Errorf("Size: got %.0f, want 11000", cif.Size)
+	}
+	if cif.FileCount != 50+80 {
+		t.Errorf("FileCount: got %.0f, want 130", cif.FileCount)
+	}
+	if cif.DeleteCount != 1+4 {
+		t.Errorf("DeleteCount: got %.0f, want 5", cif.DeleteCount)
+	}
+	if cif.VolumeCount != 2 {
+		t.Errorf("VolumeCount: got %d, want 2", cif.VolumeCount)
+	}
+}
+
+// TestCollectCollectionInfoRegularVolumeDedupesReplicas verifies that a
+// regular volume replicated across three nodes is counted once in
+// VolumeCount (logical, like the S3 metrics path and the EC branch) rather
+// than three times (one per replica presence).
+func TestCollectCollectionInfoRegularVolumeDedupesReplicas(t *testing.T) {
+	// 001 = one-copy replication placement byte; three copies of volume id=7
+	// on three distinct nodes. Per-replica Size/FileCount are divided by
+	// copyCount so summing yields the whole-volume totals.
+	makeNode := func() *master_pb.DataNodeInfo {
+		return &master_pb.DataNodeInfo{
+			DiskInfos: map[string]*master_pb.DiskInfo{
+				"disk1": {
+					VolumeInfos: []*master_pb.VolumeInformationMessage{
+						{
+							Id:               7,
+							Collection:       "bucket-rep",
+							Size:             3000,
+							FileCount:        30,
+							DeleteCount:      3,
+							DeletedByteCount: 300,
+							ReplicaPlacement: 002, // 0x02 = 3 total copies
+						},
+					},
+				},
+			},
+		}
+	}
+
+	topo := &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				RackInfos: []*master_pb.RackInfo{
+					{
+						DataNodeInfos: []*master_pb.DataNodeInfo{makeNode(), makeNode(), makeNode()},
+					},
+				},
+			},
+		},
+	}
+
+	infos := make(map[string]*CollectionInfo)
+	collectCollectionInfo(topo, infos)
+
+	cif, ok := infos["bucket-rep"]
+	if !ok {
+		t.Fatalf("expected collection bucket-rep in infos, got: %v", infos)
+	}
+	if cif.VolumeCount != 1 {
+		t.Errorf("VolumeCount: got %d, want 1 (deduped by volume id)", cif.VolumeCount)
+	}
+	// Per-replica values are Size/3, summed across 3 replicas gives 3000.
+	if cif.Size != 3000 {
+		t.Errorf("Size: got %.0f, want 3000", cif.Size)
+	}
+	if cif.FileCount != 30 {
+		t.Errorf("FileCount: got %.0f, want 30", cif.FileCount)
+	}
+}

weed/storage/erasure_coding/ec_shards_info.go

@@ -89,6 +89,51 @@ func GetShardCount(vi *master_pb.VolumeEcShardInformationMessage) int {
 	return ShardBits(vi.EcIndexBits).Count()
 }
 
+// EcShardsTotalSize returns the sum of all shard sizes (data + parity) in
+// the message. Walks vi.ShardSizes directly rather than materializing a
+// ShardsInfo, which is significantly cheaper for callers that only need the
+// aggregate size.
+func EcShardsTotalSize(vi *master_pb.VolumeEcShardInformationMessage) int64 {
+	if vi == nil {
+		return 0
+	}
+	var total int64
+	for _, s := range vi.ShardSizes {
+		total += s
+	}
+	return total
+}
+
+// EcShardsDataSize returns the sum of sizes for data shards only (parity
+// shards excluded). Data shards are those with id < dataShards; all higher
+// shard ids are treated as parity. Passing dataShards <= 0 falls back to
+// the upstream default of DataShardsCount (10), which is correct for the
+// fixed 10+4 layout. Forks with per-volume ratio metadata (e.g. the
+// data_shards field carried on an extended VolumeEcShardInformationMessage)
+// should pass the per-volume value so logical sizes remain accurate under
+// custom EC policies like 6+3 or 16+6.
+func EcShardsDataSize(vi *master_pb.VolumeEcShardInformationMessage, dataShards int) int64 {
+	if vi == nil {
+		return 0
+	}
+	if dataShards <= 0 {
+		dataShards = DataShardsCount
+	}
+	var total int64
+	var id ShardId
+	var j int
+	for bitmap := vi.EcIndexBits; bitmap != 0; bitmap >>= 1 {
+		if bitmap&1 != 0 {
+			if int(id) < dataShards && j < len(vi.ShardSizes) {
+				total += vi.ShardSizes[j]
+			}
+			j++
+		}
+		id++
+	}
+	return total
+}
+
 // Returns a string representation for a ShardsInfo.
 func (sp *ShardsInfo) String() string {
 	sp.mu.RLock()

weed/storage/erasure_coding/ec_shards_info_test.go

@@ -321,6 +321,128 @@ func TestShardsInfo_FromVolumeEcShardInformationMessage(t *testing.T) {
 	}
 }
 
+func TestEcShardsTotalSize(t *testing.T) {
+	tests := []struct {
+		name string
+		msg  *master_pb.VolumeEcShardInformationMessage
+		want int64
+	}{
+		{name: "nil message", msg: nil, want: 0},
+		{
+			name: "all shards present",
+			msg: &master_pb.VolumeEcShardInformationMessage{
+				EcIndexBits: (1 << 0) | (1 << 1) | (1 << 10),
+				ShardSizes:  []int64{1000, 2000, 3000},
+			},
+			want: 6000,
+		},
+		{
+			name: "only parity shards",
+			msg: &master_pb.VolumeEcShardInformationMessage{
+				EcIndexBits: (1 << 10) | (1 << 11),
+				ShardSizes:  []int64{500, 500},
+			},
+			want: 1000,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := EcShardsTotalSize(tt.msg); got != tt.want {
+				t.Errorf("EcShardsTotalSize() = %d, want %d", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestEcShardsDataSize(t *testing.T) {
+	tests := []struct {
+		name       string
+		msg        *master_pb.VolumeEcShardInformationMessage
+		dataShards int
+		want       int64
+	}{
+		{name: "nil message", msg: nil, dataShards: 0, want: 0},
+		{
+			name: "data shards only",
+			msg: &master_pb.VolumeEcShardInformationMessage{
+				EcIndexBits: (1 << 0) | (1 << 1) | (1 << 9),
+				ShardSizes:  []int64{1000, 2000, 500},
+			},
+			dataShards: 0, // default 10+4
+			want:       3500,
+		},
+		{
+			name: "mixed data and parity (default 10+4)",
+			msg: &master_pb.VolumeEcShardInformationMessage{
+				// shards 7, 8, 9 are data; 10..13 are parity.
+				EcIndexBits: (1 << 7) | (1 << 8) | (1 << 9) | (1 << 10) | (1 << 11) | (1 << 12) | (1 << 13),
+				ShardSizes:  []int64{1000, 1000, 500, 2000, 2000, 2000, 2000},
+			},
+			dataShards: 0,
+			want:       2500,
+		},
+		{
+			name: "only parity shards (excluded)",
+			msg: &master_pb.VolumeEcShardInformationMessage{
+				EcIndexBits: (1 << 10) | (1 << 11),
+				ShardSizes:  []int64{500, 500},
+			},
+			dataShards: 0,
+			want:       0,
+		},
+		{
+			name: "missing sizes tolerated",
+			msg: &master_pb.VolumeEcShardInformationMessage{
+				EcIndexBits: (1 << 0) | (1 << 3),
+				ShardSizes:  []int64{1000},
+			},
+			dataShards: 0,
+			want:       1000,
+		},
+		{
+			name: "custom 6+3 ratio: shards 0..5 are data, 6..8 are parity",
+			msg: &master_pb.VolumeEcShardInformationMessage{
+				// shards 0..8 all present. With dataShards=6, only 0..5
+				// are data (6*1000 = 6000); 6..8 are parity and excluded.
+				// Under the default 10+4, 0..8 would all count as data and
+				// return 9000 instead — this test pins the custom-ratio
+				// path.
+				EcIndexBits: (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3) | (1 << 4) | (1 << 5) | (1 << 6) | (1 << 7) | (1 << 8),
+				ShardSizes:  []int64{1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000},
+			},
+			dataShards: 6,
+			want:       6000,
+		},
+		{
+			name: "custom 16+6 ratio: shard id 12 is data, not parity",
+			msg: &master_pb.VolumeEcShardInformationMessage{
+				// Shards 10..12 present. Under default 10+4 these would
+				// all be parity (size 0). Under 16+6, all three are data.
+				EcIndexBits: (1 << 10) | (1 << 11) | (1 << 12),
+				ShardSizes:  []int64{2000, 2000, 2000},
+			},
+			dataShards: 16,
+			want:       6000,
+		},
+		{
+			name: "negative dataShards falls back to default",
+			msg: &master_pb.VolumeEcShardInformationMessage{
+				EcIndexBits: (1 << 0) | (1 << 1) | (1 << 10),
+				ShardSizes:  []int64{1000, 1000, 1000},
+			},
+			dataShards: -1,
+			want:       2000,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := EcShardsDataSize(tt.msg, tt.dataShards); got != tt.want {
+				t.Errorf("EcShardsDataSize(%d) = %d, want %d", tt.dataShards, got, tt.want)
+			}
+		})
+	}
+}
+
 func TestShardsInfo_String(t *testing.T) {
 	si := NewShardsInfo()
 	si.Set(ShardInfo{Id: 0, Size: 1024})