From b872d5e683ce59474881d5709ff78ca8a5a0ff37 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 30 Jun 2026 21:26:57 -0700 Subject: [PATCH] balance: extract the bytes-aware density metric to a shared package (#10174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * balance: extract the bytes-aware density metric to weed/topology/balancer The shell's volume.balance ranks servers by a bytes-aware density (used volume equivalents over free capacity). Move that math into the shared balancer package (VolumeDensity / DensityRatio / DensityNextRatio) so the maintenance worker can adopt the same metric next. Shell behavior is unchanged. * balance: rank a server with no free capacity as the fullest DensityRatio/DensityNextRatio divided by capacity, so a server past its slot limit (negative capacity) returned a negative ratio and sorted as the emptiest under ascending consumers — the opposite of reality. Treat any non-positive capacity (full, or overfull mid-run after receiving volumes) as the fullest (+Inf) so it is a move source, never a target. Covered by negative-capacity and ordering tests. --- weed/shell/command_volume_balance.go | 26 ++------- weed/topology/balancer/density.go | 52 +++++++++++++++++ weed/topology/balancer/density_test.go | 81 ++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 21 deletions(-) create mode 100644 weed/topology/balancer/density.go create mode 100644 weed/topology/balancer/density_test.go diff --git a/weed/shell/command_volume_balance.go b/weed/shell/command_volume_balance.go index dc3b2b37e..8c259645c 100644 --- a/weed/shell/command_volume_balance.go +++ b/weed/shell/command_volume_balance.go @@ -23,10 +23,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/storage/needle" ) -const ( - thresholdVolumeSize = 1.01 - countZeroSelectedVolumes = 0.5 -) +const thresholdVolumeSize = 1.01 func init() { Commands = append(Commands, &commandVolumeBalance{}) @@ -338,8 +335,7 @@ func capacityByMinVolumeDensity(diskType types.DiskType, volumeSizeLimitMb uint6 if volumeSizeLimitMb == 0 { volumeSizeLimitMb = util.VolumeSizeLimitGB * util.KiByte } - usedVolumeCount := volumeSizes / (volumeSizeLimitMb * util.MiByte) - return float64(diskInfo.MaxVolumeCount - int64(usedVolumeCount)), usedVolumeCount + return balancer.VolumeDensity(diskInfo.MaxVolumeCount, volumeSizes, volumeSizeLimitMb*util.MiByte) } } @@ -365,8 +361,7 @@ func capacityByActualDataUsage(diskType types.DiskType, volumeSizeLimitMb uint64 if volumeSizeLimitMb == 0 { volumeSizeLimitMb = util.VolumeSizeLimitGB * util.KiByte } - usedVolumeCount := volumeSizes / (volumeSizeLimitMb * util.MiByte) - return 1, usedVolumeCount + return 1, balancer.UsedVolumeEquivalents(volumeSizes, volumeSizeLimitMb*util.MiByte) } } @@ -399,22 +394,11 @@ func capacityByFreeVolumeCount(diskType types.DiskType) CapacityFunc { } func (n *Node) localVolumeDensityRatio(capacityFunc DensityFunc) float64 { - capacity, selectedVolumes := capacityFunc(n.info) - if capacity == 0 { - return 0 - } - if selectedVolumes == 0 { - return countZeroSelectedVolumes / capacity - } - return float64(selectedVolumes) / capacity + return balancer.DensityRatio(capacityFunc(n.info)) } func (n *Node) localVolumeDensityNextRatio(capacityFunc DensityFunc) float64 { - capacity, selectedVolumes := capacityFunc(n.info) - if capacity == 0 { - return 0 - } - return float64(selectedVolumes+1) / capacity + return balancer.DensityNextRatio(capacityFunc(n.info)) } func (n *Node) localVolumeRatio(capacityFunc CapacityFunc) float64 { diff --git a/weed/topology/balancer/density.go b/weed/topology/balancer/density.go new file mode 100644 index 000000000..7f75995eb --- /dev/null +++ b/weed/topology/balancer/density.go @@ -0,0 +1,52 @@ +package balancer + +import "math" + +// zeroVolumeDensityWeight is the ratio given to a server holding no data, so it +// still ranks below any loaded server (which has ratio >= 1) but is distinguished +// from a server with unknown capacity. +const zeroVolumeDensityWeight = 0.5 + +// UsedVolumeEquivalents converts the bytes a server holds into whole-volume +// equivalents (volumeBytes / volumeSizeLimit). Returns 0 when the limit is unset. +func UsedVolumeEquivalents(volumeBytes, volumeSizeLimitBytes uint64) uint64 { + if volumeSizeLimitBytes == 0 { + return 0 + } + return volumeBytes / volumeSizeLimitBytes +} + +// VolumeDensity is the bytes-aware capacity view balancing ranks servers by: +// used = volumeBytes / volumeSizeLimit, capacity = maxVolumeCount - used. Using +// real data (not just volume count) means an over-configured maxVolumeCount can't +// make a byte-full server look empty. capacity may be <= 0 for a server already +// past its configured slots. +func VolumeDensity(maxVolumeCount int64, volumeBytes, volumeSizeLimitBytes uint64) (capacity float64, usedVolumes uint64) { + usedVolumes = UsedVolumeEquivalents(volumeBytes, volumeSizeLimitBytes) + return float64(maxVolumeCount - int64(usedVolumes)), usedVolumes +} + +// DensityRatio is a server's load: usedVolumes / capacity, higher = fuller. An +// empty server gets a small positive ratio so it sorts below loaded servers. +// A server with no free capacity (capacity <= 0: full, or over its configured +// slots so capacity is negative) ranks as the fullest (+Inf), so ascending +// consumers treat it as a move source and never as the emptiest target. +func DensityRatio(capacity float64, usedVolumes uint64) float64 { + if capacity <= 0 { + return math.Inf(1) + } + if usedVolumes == 0 { + return zeroVolumeDensityWeight / capacity + } + return float64(usedVolumes) / capacity +} + +// DensityNextRatio is a server's load after one more volume lands on it: +// (usedVolumes+1) / capacity. Used to test whether a move would overshoot. A +// server with no free capacity (capacity <= 0) ranks as the fullest (+Inf). +func DensityNextRatio(capacity float64, usedVolumes uint64) float64 { + if capacity <= 0 { + return math.Inf(1) + } + return float64(usedVolumes+1) / capacity +} diff --git a/weed/topology/balancer/density_test.go b/weed/topology/balancer/density_test.go new file mode 100644 index 000000000..9f12e3dae --- /dev/null +++ b/weed/topology/balancer/density_test.go @@ -0,0 +1,81 @@ +package balancer + +import ( + "math" + "sort" + "testing" +) + +func TestVolumeDensity(t *testing.T) { + const gb = uint64(1) << 30 + // 20 full-ish volumes (~1900 GB) on a disk with a 30 GB limit and Max 1000. + cap, used := VolumeDensity(1000, 1900*gb, 30*gb) + if used != 63 { // 1900/30 = 63.33 -> 63 + t.Errorf("usedVolumes = %d, want 63", used) + } + if cap != float64(1000-63) { + t.Errorf("capacity = %v, want %v", cap, float64(1000-63)) + } + // Unset limit -> 0 used, capacity = maxVolumeCount. + if c, u := VolumeDensity(10, 1000*gb, 0); u != 0 || c != 10 { + t.Errorf("unset limit: got cap=%v used=%d, want 10/0", c, u) + } +} + +func TestDensityRatio(t *testing.T) { + // Empty server: small positive ratio (below any loaded server). + empty := DensityRatio(1000, 0) + loaded := DensityRatio(1000, 1) + if !(empty > 0 && empty < loaded) { + t.Errorf("empty ratio %v should be >0 and < loaded %v", empty, loaded) + } + // Loaded: used/capacity. + if got := DensityRatio(3, 7); got != 7.0/3.0 { + t.Errorf("DensityRatio(3,7) = %v, want %v", got, 7.0/3.0) + } + // No free capacity (full) or over the slot limit (negative capacity) -> fullest. + if got := DensityRatio(0, 5); !math.IsInf(got, 1) { + t.Errorf("DensityRatio(0,5) = %v, want +Inf", got) + } + if got := DensityRatio(-4, 20); !math.IsInf(got, 1) { + t.Errorf("DensityRatio(-4,20) = %v, want +Inf (overfull ranks fullest, not negative)", got) + } +} + +func TestDensityNextRatio(t *testing.T) { + if got := DensityNextRatio(3, 7); got != 8.0/3.0 { + t.Errorf("DensityNextRatio(3,7) = %v, want %v", got, 8.0/3.0) + } + if got := DensityNextRatio(0, 7); !math.IsInf(got, 1) { + t.Errorf("DensityNextRatio(0,7) = %v, want +Inf", got) + } + if got := DensityNextRatio(-1, 7); !math.IsInf(got, 1) { + t.Errorf("DensityNextRatio(-1,7) = %v, want +Inf", got) + } +} + +// An overfull server (negative capacity) must sort as the fullest, not the +// emptiest, when consumers rank ascending by DensityRatio. +func TestDensityRatio_OverfullSortsFullest(t *testing.T) { + // empty (cap 100, used 0), half (cap 5, used 5), overfull (cap -3, used 13). + type server struct { + name string + capacity float64 + usedVolumes uint64 + } + servers := []server{ + {"overfull", -3, 13}, + {"empty", 100, 0}, + {"half", 5, 5}, + } + sort.Slice(servers, func(i, j int) bool { + return DensityRatio(servers[i].capacity, servers[i].usedVolumes) < + DensityRatio(servers[j].capacity, servers[j].usedVolumes) + }) + if servers[0].name != "empty" { + t.Errorf("emptiest (ascending first) = %q, want empty", servers[0].name) + } + if servers[len(servers)-1].name != "overfull" { + t.Errorf("fullest (ascending last) = %q, want overfull", servers[len(servers)-1].name) + } +}