diff --git a/weed/storage/store.go b/weed/storage/store.go index 528f4c1d9..e4ea7e5d0 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -402,8 +402,8 @@ func collectStatForOneVolume(vid needle.VolumeId, v *Volume) (s *VolumeInfo) { return } - s.FileCount = v.nm.FileCount() - s.DeleteCount = v.nm.DeletedCount() + s.FileCount = countAsUint32(uint64(v.nm.FileCount())) + s.DeleteCount = countAsUint32(uint64(v.nm.DeletedCount())) s.DeletedByteCount = v.nm.DeletedSize() s.Size = v.nm.ContentSize() if v.DataBackend != nil { diff --git a/weed/storage/volume_info.go b/weed/storage/volume_info.go index e83aa9d93..43298851d 100644 --- a/weed/storage/volume_info.go +++ b/weed/storage/volume_info.go @@ -2,6 +2,7 @@ package storage import ( "fmt" + "math" "sort" "sync" @@ -24,26 +25,38 @@ type VolumeInfo struct { Ttl *needle.TTL Size uint64 - FileCount int - DeleteCount int DeletedByteCount uint64 ModifiedAtSecond int64 Id needle.VolumeId DiskId uint32 CompactRevision uint32 + // Counted in uint32: a volume is capped well below 4.29 billion needles, + // and two words per replica is worth more than the headroom. + FileCount uint32 + DeleteCount uint32 Version needle.Version ReadOnly bool } +// countAsUint32 narrows a reported count without letting it wrap. Nothing +// should reach the ceiling, and a count that pretends to is better pinned +// there than turned into a small number. +func countAsUint32(n uint64) uint32 { + if n > math.MaxUint32 { + return math.MaxUint32 + } + return uint32(n) +} + func NewVolumeInfo(m *master_pb.VolumeInformationMessage) (vi VolumeInfo, err error) { vi = VolumeInfo{ Id: needle.VolumeId(m.Id), Size: m.Size, Collection: internVolumeString(m.Collection), - FileCount: int(m.FileCount), - DeleteCount: int(m.DeleteCount), + FileCount: countAsUint32(m.FileCount), + DeleteCount: countAsUint32(m.DeleteCount), DeletedByteCount: m.DeletedByteCount, ReadOnly: m.ReadOnly, Version: needle.Version(m.Version), diff --git a/weed/storage/volume_info_counts_test.go b/weed/storage/volume_info_counts_test.go new file mode 100644 index 000000000..532b8aad0 --- /dev/null +++ b/weed/storage/volume_info_counts_test.go @@ -0,0 +1,56 @@ +package storage + +import ( + "math" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" +) + +// The counts are narrowed on the way in, so a report that claims more than a +// volume can hold must pin at the ceiling rather than wrap to a small number. +func TestVolumeInfoCountsDoNotWrap(t *testing.T) { + for _, tc := range []struct { + name string + reported, want uint64 + }{ + {"an ordinary count", 1000, 1000}, + {"the ceiling", math.MaxUint32, math.MaxUint32}, + {"past the ceiling", math.MaxUint32 + 1, math.MaxUint32}, + {"far past it", 1 << 40, math.MaxUint32}, + } { + t.Run(tc.name, func(t *testing.T) { + vi, err := NewVolumeInfo(&master_pb.VolumeInformationMessage{ + Id: 1, Version: 3, FileCount: tc.reported, DeleteCount: tc.reported, + }) + if err != nil { + t.Fatal(err) + } + if uint64(vi.FileCount) != tc.want { + t.Errorf("file count %d, want %d", vi.FileCount, tc.want) + } + if uint64(vi.DeleteCount) != tc.want { + t.Errorf("delete count %d, want %d", vi.DeleteCount, tc.want) + } + }) + } +} + +// Deletions are tracked apart from the totals they come off, so a replica can +// transiently report more deleted than it holds. Unsigned counts make that a +// wrap rather than a negative, so it stays guarded. +func TestVolumeInfoSurvivesMoreDeletesThanFiles(t *testing.T) { + vi, err := NewVolumeInfo(&master_pb.VolumeInformationMessage{ + Id: 1, Version: 3, FileCount: 10, DeleteCount: 50, + }) + if err != nil { + t.Fatal(err) + } + if vi.FileCount != 10 || vi.DeleteCount != 50 { + t.Fatalf("counts not carried across: %d/%d", vi.FileCount, vi.DeleteCount) + } + m := vi.ToVolumeInformationMessage() + if m.FileCount != 10 || m.DeleteCount != 50 { + t.Errorf("round trip changed the counts: %d/%d", m.FileCount, m.DeleteCount) + } +} diff --git a/weed/topology/volume_location_list.go b/weed/topology/volume_location_list.go index 93cf76d9e..c5b8e54f2 100644 --- a/weed/topology/volume_location_list.go +++ b/weed/topology/volume_location_list.go @@ -184,7 +184,7 @@ func (dnll *VolumeLocationList) Stats(vid needle.VolumeId, freshThreshHold int64 size = vinfo.Size - vinfo.DeletedByteCount } if vinfo.FileCount > vinfo.DeleteCount { - fileCount = vinfo.FileCount - vinfo.DeleteCount + fileCount = int(vinfo.FileCount - vinfo.DeleteCount) } return size, fileCount }