storage: count a volume's needles in uint32 (#10718)

FileCount and DeleteCount were int, so each cost a word on every replica the
master holds. A volume caps at 30GB on a 4-byte-offset build and 8TB on a
5-byte one, and neither holds 4.29 billion needles.

That takes VolumeInfo from 120 bytes to 112, which is its own size class rather
than rounding up into the 128 one, so a replica costs 135.7 bytes in the map
instead of 151.7 -- about 25MB across the 1.6M replicas in a cluster the size
of the one this came from.

Counts are narrowed where they are read rather than assigned across, so a
report claiming more than a volume can hold pins at the ceiling instead of
wrapping to a small number.

weed/storage/store.go

@@ -402,8 +402,8 @@ func collectStatForOneVolume(vid needle.VolumeId, v *Volume) (s *VolumeInfo) {
 		return
 	}
 
-	s.FileCount = v.nm.FileCount()
-	s.DeleteCount = v.nm.DeletedCount()
+	s.FileCount = countAsUint32(uint64(v.nm.FileCount()))
+	s.DeleteCount = countAsUint32(uint64(v.nm.DeletedCount()))
 	s.DeletedByteCount = v.nm.DeletedSize()
 	s.Size = v.nm.ContentSize()
 	if v.DataBackend != nil {

weed/storage/volume_info.go

@@ -2,6 +2,7 @@ package storage
 
 import (
 	"fmt"
+	"math"
 	"sort"
 	"sync"
 
@@ -24,26 +25,38 @@ type VolumeInfo struct {
 	Ttl              *needle.TTL
 
 	Size             uint64
-	FileCount        int
-	DeleteCount      int
 	DeletedByteCount uint64
 	ModifiedAtSecond int64
 
 	Id              needle.VolumeId
 	DiskId          uint32
 	CompactRevision uint32
+	// Counted in uint32: a volume is capped well below 4.29 billion needles,
+	// and two words per replica is worth more than the headroom.
+	FileCount   uint32
+	DeleteCount uint32
 
 	Version  needle.Version
 	ReadOnly bool
 }
 
+// countAsUint32 narrows a reported count without letting it wrap. Nothing
+// should reach the ceiling, and a count that pretends to is better pinned
+// there than turned into a small number.
+func countAsUint32(n uint64) uint32 {
+	if n > math.MaxUint32 {
+		return math.MaxUint32
+	}
+	return uint32(n)
+}
+
 func NewVolumeInfo(m *master_pb.VolumeInformationMessage) (vi VolumeInfo, err error) {
 	vi = VolumeInfo{
 		Id:                needle.VolumeId(m.Id),
 		Size:              m.Size,
 		Collection:        internVolumeString(m.Collection),
-		FileCount:         int(m.FileCount),
-		DeleteCount:       int(m.DeleteCount),
+		FileCount:         countAsUint32(m.FileCount),
+		DeleteCount:       countAsUint32(m.DeleteCount),
 		DeletedByteCount:  m.DeletedByteCount,
 		ReadOnly:          m.ReadOnly,
 		Version:           needle.Version(m.Version),

weed/storage/volume_info_counts_test.go

@@ -0,0 +1,56 @@
+package storage
+
+import (
+	"math"
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+)
+
+// The counts are narrowed on the way in, so a report that claims more than a
+// volume can hold must pin at the ceiling rather than wrap to a small number.
+func TestVolumeInfoCountsDoNotWrap(t *testing.T) {
+	for _, tc := range []struct {
+		name           string
+		reported, want uint64
+	}{
+		{"an ordinary count", 1000, 1000},
+		{"the ceiling", math.MaxUint32, math.MaxUint32},
+		{"past the ceiling", math.MaxUint32 + 1, math.MaxUint32},
+		{"far past it", 1 << 40, math.MaxUint32},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			vi, err := NewVolumeInfo(&master_pb.VolumeInformationMessage{
+				Id: 1, Version: 3, FileCount: tc.reported, DeleteCount: tc.reported,
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if uint64(vi.FileCount) != tc.want {
+				t.Errorf("file count %d, want %d", vi.FileCount, tc.want)
+			}
+			if uint64(vi.DeleteCount) != tc.want {
+				t.Errorf("delete count %d, want %d", vi.DeleteCount, tc.want)
+			}
+		})
+	}
+}
+
+// Deletions are tracked apart from the totals they come off, so a replica can
+// transiently report more deleted than it holds. Unsigned counts make that a
+// wrap rather than a negative, so it stays guarded.
+func TestVolumeInfoSurvivesMoreDeletesThanFiles(t *testing.T) {
+	vi, err := NewVolumeInfo(&master_pb.VolumeInformationMessage{
+		Id: 1, Version: 3, FileCount: 10, DeleteCount: 50,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if vi.FileCount != 10 || vi.DeleteCount != 50 {
+		t.Fatalf("counts not carried across: %d/%d", vi.FileCount, vi.DeleteCount)
+	}
+	m := vi.ToVolumeInformationMessage()
+	if m.FileCount != 10 || m.DeleteCount != 50 {
+		t.Errorf("round trip changed the counts: %d/%d", m.FileCount, m.DeleteCount)
+	}
+}

weed/topology/volume_location_list.go

@@ -184,7 +184,7 @@ func (dnll *VolumeLocationList) Stats(vid needle.VolumeId, freshThreshHold int64
 					size = vinfo.Size - vinfo.DeletedByteCount
 				}
 				if vinfo.FileCount > vinfo.DeleteCount {
-					fileCount = vinfo.FileCount - vinfo.DeleteCount
+					fileCount = int(vinfo.FileCount - vinfo.DeleteCount)
 				}
 				return size, fileCount
 			}