diff --git a/weed/s3api/bucket_size_metrics.go b/weed/s3api/bucket_size_metrics.go index 12a6afdd5..d6012275e 100644 --- a/weed/s3api/bucket_size_metrics.go +++ b/weed/s3api/bucket_size_metrics.go @@ -13,6 +13,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/stats" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" ) const ( @@ -197,13 +198,30 @@ func (s3a *S3ApiServer) listBucketNames(ctx context.Context) ([]string, error) { return buckets, err } +// ecVolumeAgg accumulates per-volume EC counts across the shard holders. +// fileCount is volume-wide (every holder sees the same .ecx) so we take the +// max across reporters to avoid a slow node with a not-yet-loaded .ecx +// pinning the aggregate at 0. deleteCount is node-local to each .ecj +// deletion journal, so it's summed across reporters. +type ecVolumeAgg struct { + collection string + fileCount uint64 + deleteCount uint64 +} + // collectCollectionInfoFromTopology extracts collection info from topology. // Deduplicates by volume ID to correctly handle missing replicas. // Unlike dividing by copyCount (which would give wrong results if replicas are missing), // we track seen volume IDs and only count each volume once for logical size/count. +// EC-encoded volumes are folded in via per-shard aggregation: every shard is +// node-local (not a replica), so shard sizes are summed across nodes; the +// per-volume file/delete counts carried on each shard message are deduped +// via max/sum so the aggregate doesn't double-count or drop after a volume +// is converted from regular to erasure coding. func collectCollectionInfoFromTopology(t *master_pb.TopologyInfo, collectionInfos map[string]*CollectionInfo) { // Track which volumes we've already seen to deduplicate by volume ID seenVolumes := make(map[volumeKey]bool) + ecVolumes := make(map[volumeKey]*ecVolumeAgg) for _, dc := range t.DataCenterInfos { for _, r := range dc.RackInfos { @@ -235,8 +253,49 @@ func collectCollectionInfoFromTopology(t *master_pb.TopologyInfo, collectionInfo cif.DeletedByteCount += float64(vi.DeletedByteCount) cif.VolumeCount++ } + + for _, esi := range diskInfo.EcShardInfos { + c := esi.Collection + cif, found := collectionInfos[c] + if !found { + cif = &CollectionInfo{} + collectionInfos[c] = cif + } + + // EC shards are node-local (no replication), so both + // physical and logical shard sizes sum across nodes + // without any dedupe. Logical size excludes parity + // shards; physical size includes them. Upstream OSS + // uses the fixed 10+4 ratio (dataShards=0 → default); + // forks with per-volume ratio metadata can pass the + // configured value here. + cif.PhysicalSize += float64(erasure_coding.EcShardsTotalSize(esi)) + cif.Size += float64(erasure_coding.EcShardsDataSize(esi, 0)) + + key := volumeKey{collection: c, volumeId: esi.Id} + agg, ok := ecVolumes[key] + if !ok { + agg = &ecVolumeAgg{collection: c} + ecVolumes[key] = agg + cif.VolumeCount++ + } + if esi.FileCount > agg.fileCount { + agg.fileCount = esi.FileCount + } + agg.deleteCount += esi.DeleteCount + } } } } } + + // Fold deduped EC file/delete counts into each collection's totals. + for _, agg := range ecVolumes { + cif := collectionInfos[agg.collection] + if cif == nil { + continue + } + cif.FileCount += float64(agg.fileCount) + cif.DeleteCount += float64(agg.deleteCount) + } } diff --git a/weed/s3api/bucket_size_metrics_test.go b/weed/s3api/bucket_size_metrics_test.go new file mode 100644 index 000000000..259d0b098 --- /dev/null +++ b/weed/s3api/bucket_size_metrics_test.go @@ -0,0 +1,211 @@ +package s3api + +import ( + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" +) + +// TestCollectCollectionInfoFromTopologyEC verifies that EC-encoded volumes +// contribute to per-collection logical/physical size, file count, and volume +// count. Before this fix, encoding a volume to EC caused the bucket size +// metrics exported to Prometheus to drop to zero for that volume. +// +// Layout: one 10+4 EC volume in collection "crm-docs-storage", 14 shards of +// 1000 bytes each split across two nodes. +// - nodeA holds data shards 0..6 (7 * 1000 = 7000) +// - nodeB holds data shards 7..9 (3 * 1000 = 3000) and parity 10..13 (4 * 1000 = 4000) +// +// Expected: +// - PhysicalSize = 14 * 1000 = 14000 +// - Size (logical, data shards) = 10 * 1000 = 10000 +// - FileCount = 100 total - (2 + 3) local deletes = 95 is NOT what we check; the +// collector reports raw file_count and delete_count as separate gauges, so +// we assert FileCount = 100 (max across reporters) and DeleteCount = 5 (sum). +// - VolumeCount = 1 (one unique EC volume) +func TestCollectCollectionInfoFromTopologyEC(t *testing.T) { + nodeA := &master_pb.DataNodeInfo{ + DiskInfos: map[string]*master_pb.DiskInfo{ + "disk1": { + EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ + { + Id: 42, + Collection: "crm-docs-storage", + EcIndexBits: (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3) | (1 << 4) | (1 << 5) | (1 << 6), + ShardSizes: []int64{1000, 1000, 1000, 1000, 1000, 1000, 1000}, + FileCount: 100, + DeleteCount: 2, + }, + }, + }, + }, + } + nodeB := &master_pb.DataNodeInfo{ + DiskInfos: map[string]*master_pb.DiskInfo{ + "disk1": { + EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ + { + Id: 42, + Collection: "crm-docs-storage", + EcIndexBits: (1 << 7) | (1 << 8) | (1 << 9) | (1 << 10) | (1 << 11) | (1 << 12) | (1 << 13), + ShardSizes: []int64{1000, 1000, 1000, 1000, 1000, 1000, 1000}, + FileCount: 100, + DeleteCount: 3, + }, + }, + }, + }, + } + + topo := &master_pb.TopologyInfo{ + DataCenterInfos: []*master_pb.DataCenterInfo{ + { + RackInfos: []*master_pb.RackInfo{ + { + DataNodeInfos: []*master_pb.DataNodeInfo{nodeA, nodeB}, + }, + }, + }, + }, + } + + got := make(map[string]*CollectionInfo) + collectCollectionInfoFromTopology(topo, got) + + info, ok := got["crm-docs-storage"] + if !ok { + t.Fatalf("expected collection crm-docs-storage, got: %v", got) + } + if info.PhysicalSize != 14000 { + t.Errorf("PhysicalSize: got %.0f, want 14000", info.PhysicalSize) + } + if info.Size != 10000 { + t.Errorf("Size (logical): got %.0f, want 10000", info.Size) + } + if info.FileCount != 100 { + t.Errorf("FileCount: got %.0f, want 100 (max across reporters)", info.FileCount) + } + if info.DeleteCount != 5 { + t.Errorf("DeleteCount: got %.0f, want 5 (sum across reporters)", info.DeleteCount) + } + if info.VolumeCount != 1 { + t.Errorf("VolumeCount: got %d, want 1", info.VolumeCount) + } +} + +// TestCollectCollectionInfoFromTopologyMixed verifies that regular and EC +// volumes accumulate under the same collection without one clobbering the +// other, which is the state during an in-progress EC conversion. +func TestCollectCollectionInfoFromTopologyMixed(t *testing.T) { + node := &master_pb.DataNodeInfo{ + DiskInfos: map[string]*master_pb.DiskInfo{ + "disk1": { + VolumeInfos: []*master_pb.VolumeInformationMessage{ + { + Id: 1, + Collection: "bucket-mix", + Size: 5000, + FileCount: 50, + DeleteCount: 1, + DeletedByteCount: 100, + }, + }, + EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ + { + Id: 2, + Collection: "bucket-mix", + EcIndexBits: (1 << 0) | (1 << 1) | (1 << 10), // 2 data + 1 parity + ShardSizes: []int64{3000, 3000, 3000}, + FileCount: 80, + DeleteCount: 4, + }, + }, + }, + }, + } + + topo := &master_pb.TopologyInfo{ + DataCenterInfos: []*master_pb.DataCenterInfo{ + { + RackInfos: []*master_pb.RackInfo{ + { + DataNodeInfos: []*master_pb.DataNodeInfo{node}, + }, + }, + }, + }, + } + + got := make(map[string]*CollectionInfo) + collectCollectionInfoFromTopology(topo, got) + + info, ok := got["bucket-mix"] + if !ok { + t.Fatalf("expected collection bucket-mix, got: %v", got) + } + // Regular volume: 5000 physical + logical. EC shards: 9000 physical, + // 6000 logical (data shards 0 and 1). + if info.PhysicalSize != 5000+9000 { + t.Errorf("PhysicalSize: got %.0f, want 14000", info.PhysicalSize) + } + if info.Size != 5000+6000 { + t.Errorf("Size: got %.0f, want 11000", info.Size) + } + if info.FileCount != 50+80 { + t.Errorf("FileCount: got %.0f, want 130", info.FileCount) + } + if info.DeleteCount != 1+4 { + t.Errorf("DeleteCount: got %.0f, want 5", info.DeleteCount) + } + if info.VolumeCount != 2 { + t.Errorf("VolumeCount: got %d, want 2", info.VolumeCount) + } +} + +// TestCollectCollectionInfoFromTopologyECFileCountMaxDedupe verifies that a +// slow shard holder reporting file_count=0 (because it has not yet finished +// loading .ecx) does not pin the per-volume FileCount at 0. +func TestCollectCollectionInfoFromTopologyECFileCountMaxDedupe(t *testing.T) { + makeNode := func(bits uint32, sizes []int64, fileCount uint64) *master_pb.DataNodeInfo { + return &master_pb.DataNodeInfo{ + DiskInfos: map[string]*master_pb.DiskInfo{ + "disk1": { + EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ + { + Id: 11, + Collection: "bucket-b", + EcIndexBits: bits, + ShardSizes: sizes, + FileCount: fileCount, + }, + }, + }, + }, + } + } + + topo := &master_pb.TopologyInfo{ + DataCenterInfos: []*master_pb.DataCenterInfo{ + { + RackInfos: []*master_pb.RackInfo{ + { + DataNodeInfos: []*master_pb.DataNodeInfo{ + makeNode((1<<0)|(1<<1)|(1<<2)|(1<<3)|(1<<4)|(1<<5)|(1<<6), []int64{1, 1, 1, 1, 1, 1, 1}, 0), + makeNode((1<<7)|(1<<8)|(1<<9)|(1<<10)|(1<<11)|(1<<12)|(1<<13), []int64{1, 1, 1, 1, 1, 1, 1}, 6), + }, + }, + }, + }, + }, + } + + got := make(map[string]*CollectionInfo) + collectCollectionInfoFromTopology(topo, got) + info, ok := got["bucket-b"] + if !ok { + t.Fatalf("expected collection bucket-b, got: %v", got) + } + if info.FileCount != 6 { + t.Errorf("FileCount: got %.0f, want 6 (max across reporters)", info.FileCount) + } +} diff --git a/weed/shell/command_collection_list.go b/weed/shell/command_collection_list.go index 582ea54ca..916b84431 100644 --- a/weed/shell/command_collection_list.go +++ b/weed/shell/command_collection_list.go @@ -6,6 +6,7 @@ import ( "io" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" "github.com/seaweedfs/seaweedfs/weed/storage/super_block" ) @@ -84,7 +85,21 @@ func ListCollectionNames(commandEnv *CommandEnv, includeNormalVolumes, includeEc return } -func addToCollection(collectionInfos map[string]*CollectionInfo, vif *master_pb.VolumeInformationMessage) { +// volumeKey uniquely identifies a volume for per-collection dedupe. Volume +// IDs are scoped to a collection, so we key by (collection, volumeId) to +// avoid cross-collection aliasing if the same numeric ID is ever reused. +type volumeKey struct { + collection string + volumeId uint32 +} + +// addToCollection folds one replica of a regular volume into the collection +// totals. Size/FileCount/DeleteCount/DeletedByteCount are divided by the +// replication factor so that summing over all replicas yields the whole- +// volume value. VolumeCount is deduped across replicas via seenVolumes so +// it reports logical volumes (same semantics as the S3 bucket metrics +// collector and the EC branch below), not shard/replica presences. +func addToCollection(collectionInfos map[string]*CollectionInfo, seenVolumes map[volumeKey]bool, vif *master_pb.VolumeInformationMessage) { c := vif.Collection cif, found := collectionInfos[c] if !found { @@ -97,22 +112,71 @@ func addToCollection(collectionInfos map[string]*CollectionInfo, vif *master_pb. cif.DeleteCount += float64(vif.DeleteCount) / copyCount cif.FileCount += float64(vif.FileCount) / copyCount cif.DeletedByteCount += float64(vif.DeletedByteCount) / copyCount - cif.VolumeCount++ + + key := volumeKey{collection: c, volumeId: vif.Id} + if !seenVolumes[key] { + seenVolumes[key] = true + cif.VolumeCount++ + } +} + +// ecCollectionAgg accumulates per-EC-volume counts across the shard holders. +// fileCount is volume-wide (every holder reports the same .ecx count) so it +// is deduped via max; deleteCount is node-local to each .ecj and summed. +type ecCollectionAgg struct { + collection string + fileCount uint64 + deleteCount uint64 } func collectCollectionInfo(t *master_pb.TopologyInfo, collectionInfos map[string]*CollectionInfo) { + seenVolumes := make(map[volumeKey]bool) + ecVolumes := make(map[volumeKey]*ecCollectionAgg) for _, dc := range t.DataCenterInfos { for _, r := range dc.RackInfos { for _, dn := range r.DataNodeInfos { for _, diskInfo := range dn.DiskInfos { for _, vi := range diskInfo.VolumeInfos { - addToCollection(collectionInfos, vi) + addToCollection(collectionInfos, seenVolumes, vi) + } + for _, esi := range diskInfo.EcShardInfos { + c := esi.Collection + cif, found := collectionInfos[c] + if !found { + cif = &CollectionInfo{} + collectionInfos[c] = cif + } + + // EC shards are node-local, so data-shard sizes sum + // across nodes to give the logical volume size. + // Upstream OSS uses the fixed 10+4 ratio; forks with + // per-volume ratio metadata should pass the + // configured dataShards value here. + cif.Size += float64(erasure_coding.EcShardsDataSize(esi, 0)) + + key := volumeKey{collection: c, volumeId: esi.Id} + agg, ok := ecVolumes[key] + if !ok { + agg = &ecCollectionAgg{collection: c} + ecVolumes[key] = agg + cif.VolumeCount++ + } + if esi.FileCount > agg.fileCount { + agg.fileCount = esi.FileCount + } + agg.deleteCount += esi.DeleteCount } - //for _, ecShardInfo := range diskInfo.EcShardInfos { - // - //} } } } } + + for _, agg := range ecVolumes { + cif := collectionInfos[agg.collection] + if cif == nil { + continue + } + cif.FileCount += float64(agg.fileCount) + cif.DeleteCount += float64(agg.deleteCount) + } } diff --git a/weed/shell/command_collection_list_test.go b/weed/shell/command_collection_list_test.go new file mode 100644 index 000000000..686834a0d --- /dev/null +++ b/weed/shell/command_collection_list_test.go @@ -0,0 +1,206 @@ +package shell + +import ( + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" +) + +// TestCollectCollectionInfoEC verifies that EC-encoded volumes contribute to +// collection.list totals and s3.bucket.quota.enforce sums. Before this fix +// the EC branch was a no-op, so encoded volumes silently dropped out of +// collection size accounting. +func TestCollectCollectionInfoEC(t *testing.T) { + // One 10+4 EC volume split across two nodes: 14 shards * 1000 bytes. + // Data shards 0..9 live on (nodeA: 0..6, nodeB: 7..9), parity 10..13 + // on nodeB. Every shard holder reports file_count=100; node-local + // delete counts are 2 + 3 = 5. + nodeA := &master_pb.DataNodeInfo{ + DiskInfos: map[string]*master_pb.DiskInfo{ + "disk1": { + EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ + { + Id: 42, + Collection: "bucket-a", + EcIndexBits: (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3) | (1 << 4) | (1 << 5) | (1 << 6), + ShardSizes: []int64{1000, 1000, 1000, 1000, 1000, 1000, 1000}, + FileCount: 100, + DeleteCount: 2, + }, + }, + }, + }, + } + nodeB := &master_pb.DataNodeInfo{ + DiskInfos: map[string]*master_pb.DiskInfo{ + "disk1": { + EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ + { + Id: 42, + Collection: "bucket-a", + EcIndexBits: (1 << 7) | (1 << 8) | (1 << 9) | (1 << 10) | (1 << 11) | (1 << 12) | (1 << 13), + ShardSizes: []int64{1000, 1000, 1000, 1000, 1000, 1000, 1000}, + FileCount: 100, + DeleteCount: 3, + }, + }, + }, + }, + } + + topo := &master_pb.TopologyInfo{ + DataCenterInfos: []*master_pb.DataCenterInfo{ + { + RackInfos: []*master_pb.RackInfo{ + { + DataNodeInfos: []*master_pb.DataNodeInfo{nodeA, nodeB}, + }, + }, + }, + }, + } + + infos := make(map[string]*CollectionInfo) + collectCollectionInfo(topo, infos) + + cif, ok := infos["bucket-a"] + if !ok { + t.Fatalf("expected collection bucket-a in infos, got: %v", infos) + } + // 10 data shards * 1000 bytes. + if cif.Size != 10000 { + t.Errorf("Size: got %.0f, want 10000", cif.Size) + } + if cif.FileCount != 100 { + t.Errorf("FileCount: got %.0f, want 100 (max across reporters)", cif.FileCount) + } + if cif.DeleteCount != 5 { + t.Errorf("DeleteCount: got %.0f, want 5 (sum across reporters)", cif.DeleteCount) + } + if cif.VolumeCount != 1 { + t.Errorf("VolumeCount: got %d, want 1", cif.VolumeCount) + } +} + +// TestCollectCollectionInfoMixed verifies that a collection with both a +// regular volume and an EC volume sums both branches without either clobbering +// the other, matching the state mid-EC-conversion. +func TestCollectCollectionInfoMixed(t *testing.T) { + node := &master_pb.DataNodeInfo{ + DiskInfos: map[string]*master_pb.DiskInfo{ + "disk1": { + VolumeInfos: []*master_pb.VolumeInformationMessage{ + { + Id: 1, + Collection: "bucket-mix", + Size: 5000, + FileCount: 50, + DeleteCount: 1, + }, + }, + EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ + { + Id: 2, + Collection: "bucket-mix", + EcIndexBits: (1 << 0) | (1 << 1) | (1 << 10), // 2 data + 1 parity + ShardSizes: []int64{3000, 3000, 3000}, + FileCount: 80, + DeleteCount: 4, + }, + }, + }, + }, + } + + topo := &master_pb.TopologyInfo{ + DataCenterInfos: []*master_pb.DataCenterInfo{ + { + RackInfos: []*master_pb.RackInfo{ + { + DataNodeInfos: []*master_pb.DataNodeInfo{node}, + }, + }, + }, + }, + } + + infos := make(map[string]*CollectionInfo) + collectCollectionInfo(topo, infos) + + cif, ok := infos["bucket-mix"] + if !ok { + t.Fatalf("expected collection bucket-mix in infos, got: %v", infos) + } + // Regular: 5000 (replicaCount=1). EC data shards: 6000. + if cif.Size != 5000+6000 { + t.Errorf("Size: got %.0f, want 11000", cif.Size) + } + if cif.FileCount != 50+80 { + t.Errorf("FileCount: got %.0f, want 130", cif.FileCount) + } + if cif.DeleteCount != 1+4 { + t.Errorf("DeleteCount: got %.0f, want 5", cif.DeleteCount) + } + if cif.VolumeCount != 2 { + t.Errorf("VolumeCount: got %d, want 2", cif.VolumeCount) + } +} + +// TestCollectCollectionInfoRegularVolumeDedupesReplicas verifies that a +// regular volume replicated across three nodes is counted once in +// VolumeCount (logical, like the S3 metrics path and the EC branch) rather +// than three times (one per replica presence). +func TestCollectCollectionInfoRegularVolumeDedupesReplicas(t *testing.T) { + // 001 = one-copy replication placement byte; three copies of volume id=7 + // on three distinct nodes. Per-replica Size/FileCount are divided by + // copyCount so summing yields the whole-volume totals. + makeNode := func() *master_pb.DataNodeInfo { + return &master_pb.DataNodeInfo{ + DiskInfos: map[string]*master_pb.DiskInfo{ + "disk1": { + VolumeInfos: []*master_pb.VolumeInformationMessage{ + { + Id: 7, + Collection: "bucket-rep", + Size: 3000, + FileCount: 30, + DeleteCount: 3, + DeletedByteCount: 300, + ReplicaPlacement: 002, // 0x02 = 3 total copies + }, + }, + }, + }, + } + } + + topo := &master_pb.TopologyInfo{ + DataCenterInfos: []*master_pb.DataCenterInfo{ + { + RackInfos: []*master_pb.RackInfo{ + { + DataNodeInfos: []*master_pb.DataNodeInfo{makeNode(), makeNode(), makeNode()}, + }, + }, + }, + }, + } + + infos := make(map[string]*CollectionInfo) + collectCollectionInfo(topo, infos) + + cif, ok := infos["bucket-rep"] + if !ok { + t.Fatalf("expected collection bucket-rep in infos, got: %v", infos) + } + if cif.VolumeCount != 1 { + t.Errorf("VolumeCount: got %d, want 1 (deduped by volume id)", cif.VolumeCount) + } + // Per-replica values are Size/3, summed across 3 replicas gives 3000. + if cif.Size != 3000 { + t.Errorf("Size: got %.0f, want 3000", cif.Size) + } + if cif.FileCount != 30 { + t.Errorf("FileCount: got %.0f, want 30", cif.FileCount) + } +} diff --git a/weed/storage/erasure_coding/ec_shards_info.go b/weed/storage/erasure_coding/ec_shards_info.go index 0d2ce5b63..04126230d 100644 --- a/weed/storage/erasure_coding/ec_shards_info.go +++ b/weed/storage/erasure_coding/ec_shards_info.go @@ -89,6 +89,51 @@ func GetShardCount(vi *master_pb.VolumeEcShardInformationMessage) int { return ShardBits(vi.EcIndexBits).Count() } +// EcShardsTotalSize returns the sum of all shard sizes (data + parity) in +// the message. Walks vi.ShardSizes directly rather than materializing a +// ShardsInfo, which is significantly cheaper for callers that only need the +// aggregate size. +func EcShardsTotalSize(vi *master_pb.VolumeEcShardInformationMessage) int64 { + if vi == nil { + return 0 + } + var total int64 + for _, s := range vi.ShardSizes { + total += s + } + return total +} + +// EcShardsDataSize returns the sum of sizes for data shards only (parity +// shards excluded). Data shards are those with id < dataShards; all higher +// shard ids are treated as parity. Passing dataShards <= 0 falls back to +// the upstream default of DataShardsCount (10), which is correct for the +// fixed 10+4 layout. Forks with per-volume ratio metadata (e.g. the +// data_shards field carried on an extended VolumeEcShardInformationMessage) +// should pass the per-volume value so logical sizes remain accurate under +// custom EC policies like 6+3 or 16+6. +func EcShardsDataSize(vi *master_pb.VolumeEcShardInformationMessage, dataShards int) int64 { + if vi == nil { + return 0 + } + if dataShards <= 0 { + dataShards = DataShardsCount + } + var total int64 + var id ShardId + var j int + for bitmap := vi.EcIndexBits; bitmap != 0; bitmap >>= 1 { + if bitmap&1 != 0 { + if int(id) < dataShards && j < len(vi.ShardSizes) { + total += vi.ShardSizes[j] + } + j++ + } + id++ + } + return total +} + // Returns a string representation for a ShardsInfo. func (sp *ShardsInfo) String() string { sp.mu.RLock() diff --git a/weed/storage/erasure_coding/ec_shards_info_test.go b/weed/storage/erasure_coding/ec_shards_info_test.go index f96fbbca6..5604447d4 100644 --- a/weed/storage/erasure_coding/ec_shards_info_test.go +++ b/weed/storage/erasure_coding/ec_shards_info_test.go @@ -321,6 +321,128 @@ func TestShardsInfo_FromVolumeEcShardInformationMessage(t *testing.T) { } } +func TestEcShardsTotalSize(t *testing.T) { + tests := []struct { + name string + msg *master_pb.VolumeEcShardInformationMessage + want int64 + }{ + {name: "nil message", msg: nil, want: 0}, + { + name: "all shards present", + msg: &master_pb.VolumeEcShardInformationMessage{ + EcIndexBits: (1 << 0) | (1 << 1) | (1 << 10), + ShardSizes: []int64{1000, 2000, 3000}, + }, + want: 6000, + }, + { + name: "only parity shards", + msg: &master_pb.VolumeEcShardInformationMessage{ + EcIndexBits: (1 << 10) | (1 << 11), + ShardSizes: []int64{500, 500}, + }, + want: 1000, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := EcShardsTotalSize(tt.msg); got != tt.want { + t.Errorf("EcShardsTotalSize() = %d, want %d", got, tt.want) + } + }) + } +} + +func TestEcShardsDataSize(t *testing.T) { + tests := []struct { + name string + msg *master_pb.VolumeEcShardInformationMessage + dataShards int + want int64 + }{ + {name: "nil message", msg: nil, dataShards: 0, want: 0}, + { + name: "data shards only", + msg: &master_pb.VolumeEcShardInformationMessage{ + EcIndexBits: (1 << 0) | (1 << 1) | (1 << 9), + ShardSizes: []int64{1000, 2000, 500}, + }, + dataShards: 0, // default 10+4 + want: 3500, + }, + { + name: "mixed data and parity (default 10+4)", + msg: &master_pb.VolumeEcShardInformationMessage{ + // shards 7, 8, 9 are data; 10..13 are parity. + EcIndexBits: (1 << 7) | (1 << 8) | (1 << 9) | (1 << 10) | (1 << 11) | (1 << 12) | (1 << 13), + ShardSizes: []int64{1000, 1000, 500, 2000, 2000, 2000, 2000}, + }, + dataShards: 0, + want: 2500, + }, + { + name: "only parity shards (excluded)", + msg: &master_pb.VolumeEcShardInformationMessage{ + EcIndexBits: (1 << 10) | (1 << 11), + ShardSizes: []int64{500, 500}, + }, + dataShards: 0, + want: 0, + }, + { + name: "missing sizes tolerated", + msg: &master_pb.VolumeEcShardInformationMessage{ + EcIndexBits: (1 << 0) | (1 << 3), + ShardSizes: []int64{1000}, + }, + dataShards: 0, + want: 1000, + }, + { + name: "custom 6+3 ratio: shards 0..5 are data, 6..8 are parity", + msg: &master_pb.VolumeEcShardInformationMessage{ + // shards 0..8 all present. With dataShards=6, only 0..5 + // are data (6*1000 = 6000); 6..8 are parity and excluded. + // Under the default 10+4, 0..8 would all count as data and + // return 9000 instead — this test pins the custom-ratio + // path. + EcIndexBits: (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3) | (1 << 4) | (1 << 5) | (1 << 6) | (1 << 7) | (1 << 8), + ShardSizes: []int64{1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000}, + }, + dataShards: 6, + want: 6000, + }, + { + name: "custom 16+6 ratio: shard id 12 is data, not parity", + msg: &master_pb.VolumeEcShardInformationMessage{ + // Shards 10..12 present. Under default 10+4 these would + // all be parity (size 0). Under 16+6, all three are data. + EcIndexBits: (1 << 10) | (1 << 11) | (1 << 12), + ShardSizes: []int64{2000, 2000, 2000}, + }, + dataShards: 16, + want: 6000, + }, + { + name: "negative dataShards falls back to default", + msg: &master_pb.VolumeEcShardInformationMessage{ + EcIndexBits: (1 << 0) | (1 << 1) | (1 << 10), + ShardSizes: []int64{1000, 1000, 1000}, + }, + dataShards: -1, + want: 2000, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := EcShardsDataSize(tt.msg, tt.dataShards); got != tt.want { + t.Errorf("EcShardsDataSize(%d) = %d, want %d", tt.dataShards, got, tt.want) + } + }) + } +} + func TestShardsInfo_String(t *testing.T) { si := NewShardsInfo() si.Set(ShardInfo{Id: 0, Size: 1024})