From c194924d13de12e2eb21e94f7987bb07fecc98b5 Mon Sep 17 00:00:00 2001
From: Chris Lu <chrislusf@users.noreply.github.com>
Date: Fri, 24 Jul 2026 01:43:48 -0700
Subject: [PATCH] telemetry: per-cluster size over time on the dashboard
 (#10417)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The dashboard charted one summed disk-usage line, so a step in the total
gave no hint which cluster moved. A new panel stacks each cluster's daily
size as its own band: the top of the stack is the fleet total, each band
is one cluster, and the clusters past the twentieth are summed into an
"other" band so the stack still adds up to the total.

The series is built from the per-cluster daily histories and served by
/api/cluster-sizes. Clusters report roughly once a day at no fixed hour,
so a day with no report carries the previous value forward — dropping it
to zero would sag the total every day as the clusters that have not
reported yet fall out from under it. A cluster that stops reporting past
the active window ends at its last sample instead of holding capacity
forever. Ranking is by the most recent day, tie-broken on cluster id so
the colors do not shuffle between refreshes.

Hover and click resolve to the band under the pointer: Chart.js's builtin
interaction modes match the nearest line, which on a stack of thin bands
is rarely the band being pointed at. Clicking one fills the per-cluster
history lookup below it.
---
 telemetry/README.md                     |   3 +
 telemetry/server/api/handlers.go        |  24 +++++
 telemetry/server/dashboard/dashboard.go | 129 ++++++++++++++++++++++++
 telemetry/server/main.go                |   1 +
 telemetry/server/storage/history.go     |   3 +
 telemetry/server/storage/prometheus.go  |   2 +-
 telemetry/server/storage/sizes.go       | 106 +++++++++++++++++++
 telemetry/server/storage/sizes_test.go  | 122 ++++++++++++++++++++++
 8 files changed, 389 insertions(+), 1 deletion(-)
 create mode 100644 telemetry/server/storage/sizes.go
 create mode 100644 telemetry/server/storage/sizes_test.go

diff --git a/telemetry/README.md b/telemetry/README.md
index 6e1db0aeb..f07f43b88 100644
--- a/telemetry/README.md
+++ b/telemetry/README.md
@@ -177,6 +177,9 @@ GET /api/metrics?days=30
 
 # Get one cluster's daily usage history (disk bytes, volumes, volume servers)
 GET /api/history?cluster_id=<uuid>&days=90
+
+# Get per-cluster disk usage over time, largest first, the rest summed as "other"
+GET /api/cluster-sizes?days=30&limit=20
 ```
 
 ### Monitoring
diff --git a/telemetry/server/api/handlers.go b/telemetry/server/api/handlers.go
index b3faf234d..a4c767ebc 100644
--- a/telemetry/server/api/handlers.go
+++ b/telemetry/server/api/handlers.go
@@ -150,6 +150,30 @@ func (h *Handler) GetMetrics(w http.ResponseWriter, r *http.Request) {
 	json.NewEncoder(w).Encode(metrics)
 }
 
+func (h *Handler) GetClusterSizes(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	days := 30 // default
+	if daysStr := r.URL.Query().Get("days"); daysStr != "" {
+		if d, err := strconv.Atoi(daysStr); err == nil && d > 0 && d <= 365 {
+			days = d
+		}
+	}
+
+	limit := 20 // default
+	if limitStr := r.URL.Query().Get("limit"); limitStr != "" {
+		if l, err := strconv.Atoi(limitStr); err == nil && l > 0 && l <= 1000 {
+			limit = l
+		}
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(h.storage.GetClusterSizeSeries(days, limit))
+}
+
 func (h *Handler) GetHistory(w http.ResponseWriter, r *http.Request) {
 	if r.Method != http.MethodGet {
 		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
diff --git a/telemetry/server/dashboard/dashboard.go b/telemetry/server/dashboard/dashboard.go
index 78301acc5..c977c4101 100644
--- a/telemetry/server/dashboard/dashboard.go
+++ b/telemetry/server/dashboard/dashboard.go
@@ -69,6 +69,10 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
             font-weight: bold;
             margin-bottom: 15px;
         }
+        .chart-subtitle {
+            color: #666;
+            margin: -10px 0 15px;
+        }
         .loading {
             text-align: center;
             padding: 40px;
@@ -159,6 +163,14 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
                 <canvas id="diskChart" width="400" height="200"></canvas>
             </div>
 
+            <div class="chart-container">
+                <div class="chart-title">Cluster Sizes Over Time</div>
+                <div class="chart-subtitle" id="clusterSizesTotal"></div>
+                <div style="position: relative; height: 420px;">
+                    <canvas id="clusterSizeChart"></canvas>
+                </div>
+            </div>
+
             <div class="chart-container">
                 <div class="chart-title">Per-Cluster History</div>
                 <div class="cluster-lookup">
@@ -177,6 +189,7 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
 
     <script>
         let charts = {};
+        let clusterSizeIds = [];
 
         async function loadDashboard() {
             try {
@@ -188,8 +201,13 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
                 const metricsResponse = await fetch('/api/metrics?days=30');
                 const metrics = await metricsResponse.json();
 
+                // Load per-cluster sizes over time
+                const sizesResponse = await fetch('/api/cluster-sizes?days=30&limit=20');
+                const sizes = await sizesResponse.json();
+
                 updateStats(stats);
                 updateCharts(stats, metrics);
+                updateClusterSizes(sizes);
                 
                 document.getElementById('loading').style.display = 'none';
                 document.getElementById('dashboard').style.display = 'block';
@@ -292,6 +310,117 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
             });
         }
 
+        // Decimal units, so the round numbers the chart picks for its ticks
+        // come out as round labels.
+        function formatBytes(bytes) {
+            const units = ['B', 'kB', 'MB', 'GB', 'TB', 'PB'];
+            let value = bytes || 0, unit = 0;
+            while (value >= 1000 && unit < units.length - 1) {
+                value /= 1000;
+                unit++;
+            }
+            return (unit === 0 ? value : value.toFixed(value >= 100 ? 0 : 1)) + ' ' + units[unit];
+        }
+
+        // One stacked band per cluster over time: the band is that cluster's
+        // size, the top of the stack is the fleet total. Clusters beyond the
+        // requested limit are summed into a trailing "other" band so the stack
+        // still adds up.
+        function updateClusterSizes(series) {
+            const clusters = series.clusters || [];
+            const dates = series.dates || [];
+            const count = series.cluster_count || 0;
+
+            document.getElementById('clusterSizesTotal').textContent =
+                formatBytes(series.total_disk) + ' across ' + count + ' cluster' + (count === 1 ? '' : 's') +
+                ' on ' + (dates[dates.length - 1] || 'no data');
+
+            clusterSizeIds = clusters.map(c => c.cluster_id);
+            const datasets = clusters.map((c, i) => {
+                // Evenly spaced hues keep neighbouring bands distinguishable.
+                const hue = Math.round(i * 360 / clusters.length);
+                return band(c.cluster_id.slice(0, 8), c.disk,
+                    'hsl(' + hue + ', 65%, 45%)', 'hsla(' + hue + ', 65%, 55%, 0.75)');
+            });
+            if (series.other) {
+                clusterSizeIds.push(null);
+                datasets.push(band('other (' + series.other.count + ')', series.other.disk,
+                    '#9E9E9E', 'rgba(158, 158, 158, 0.6)'));
+            }
+
+            const ctx = document.getElementById('clusterSizeChart').getContext('2d');
+            if (charts.clusterSizeChart) {
+                charts.clusterSizeChart.destroy();
+            }
+            charts.clusterSizeChart = new Chart(ctx, {
+                type: 'line',
+                data: { labels: dates, datasets: datasets },
+                options: {
+                    responsive: true,
+                    maintainAspectRatio: false,
+                    interaction: { mode: 'band', intersect: false },
+                    onClick: (event, elements) => {
+                        const id = elements.length && clusterSizeIds[elements[0].datasetIndex];
+                        if (id) {
+                            document.getElementById('clusterIdInput').value = id;
+                            loadClusterHistory();
+                        }
+                    },
+                    plugins: {
+                        legend: { position: 'bottom', labels: { boxWidth: 12, font: { size: 11 } } },
+                        tooltip: {
+                            callbacks: {
+                                label: item => {
+                                    const id = clusterSizeIds[item.datasetIndex];
+                                    return (id || item.dataset.label) + ': ' + formatBytes(item.raw);
+                                }
+                            }
+                        }
+                    },
+                    scales: {
+                        x: { ticks: { maxRotation: 0, autoSkip: true, maxTicksLimit: 12 } },
+                        y: {
+                            stacked: true,
+                            beginAtZero: true,
+                            ticks: { callback: value => formatBytes(value) }
+                        }
+                    }
+                }
+            });
+        }
+
+        // Hover and click resolve to the band the pointer is inside. Chart.js's
+        // built-in modes match the nearest line, which on a stack of thin bands
+        // is rarely the one under the pointer.
+        Chart.Interaction.modes.band = function(chart, event) {
+            const last = chart.data.labels.length - 1;
+            if (last < 0) return [];
+            const index = Math.min(Math.max(Math.round(chart.scales.x.getValueForPixel(event.x)), 0), last);
+            const value = chart.scales.y.getValueForPixel(event.y);
+            let stacked = 0;
+            for (let d = 0; d < chart.data.datasets.length; d++) {
+                stacked += chart.data.datasets[d].data[index] || 0;
+                if (value <= stacked) {
+                    return [{ element: chart.getDatasetMeta(d).data[index], datasetIndex: d, index: index }];
+                }
+            }
+            return [];
+        };
+
+        function band(label, data, borderColor, backgroundColor) {
+            return {
+                label: label,
+                data: data,
+                borderColor: borderColor,
+                backgroundColor: backgroundColor,
+                borderWidth: 1,
+                pointRadius: 0,
+                pointHitRadius: 8,
+                fill: true,
+                tension: 0.1
+            };
+        }
+
         async function loadClusterHistory() {
             const id = document.getElementById('clusterIdInput').value.trim();
             const errorDiv = document.getElementById('clusterHistoryError');
diff --git a/telemetry/server/main.go b/telemetry/server/main.go
index 05ebec8dc..0383c1d28 100644
--- a/telemetry/server/main.go
+++ b/telemetry/server/main.go
@@ -76,6 +76,7 @@ func main() {
 	mux.HandleFunc("/api/instances", corsMiddleware(logMiddleware(apiHandler.GetInstances)))
 	mux.HandleFunc("/api/metrics", corsMiddleware(logMiddleware(apiHandler.GetMetrics)))
 	mux.HandleFunc("/api/history", corsMiddleware(logMiddleware(apiHandler.GetHistory)))
+	mux.HandleFunc("/api/cluster-sizes", corsMiddleware(logMiddleware(apiHandler.GetClusterSizes)))
 
 	// Dashboard (optional)
 	if *enableDashboard {
diff --git a/telemetry/server/storage/history.go b/telemetry/server/storage/history.go
index eff68bc03..2ec6a1a93 100644
--- a/telemetry/server/storage/history.go
+++ b/telemetry/server/storage/history.go
@@ -10,6 +10,9 @@ import (
 // on before it counts as confirmed in the aggregated stats.
 const confirmDays = 2
 
+// activeDays is how recently a cluster must have reported to count as active.
+const activeDays = 7
+
 // HistorySample is one retained data point of a cluster's daily reports.
 // Tags are kept short because thousands of samples end up in the state file.
 type HistorySample struct {
diff --git a/telemetry/server/storage/prometheus.go b/telemetry/server/storage/prometheus.go
index 15e040f9d..10092e648 100644
--- a/telemetry/server/storage/prometheus.go
+++ b/telemetry/server/storage/prometheus.go
@@ -207,7 +207,7 @@ func (s *PrometheusStorage) GetMetrics(days int) (map[string]interface{}, error)
 
 func (s *PrometheusStorage) updateStats() {
 	now := time.Now()
-	last7Days := now.AddDate(0, 0, -7)
+	last7Days := now.AddDate(0, 0, -activeDays)
 	last30Days := now.AddDate(0, 0, -30)
 
 	totalInstances := 0
diff --git a/telemetry/server/storage/sizes.go b/telemetry/server/storage/sizes.go
new file mode 100644
index 000000000..aa2f7f287
--- /dev/null
+++ b/telemetry/server/storage/sizes.go
@@ -0,0 +1,106 @@
+package storage
+
+import (
+	"sort"
+	"time"
+)
+
+// ClusterSeries is one cluster's daily disk usage, aligned to the shared date
+// axis of the enclosing ClusterSizeSeries.
+type ClusterSeries struct {
+	ClusterId string   `json:"cluster_id"`
+	Disk      []uint64 `json:"disk"`
+}
+
+// OtherSeries is the clusters beyond the caller's limit, summed per day so a
+// stacked chart still adds up to the fleet total.
+type OtherSeries struct {
+	Count int      `json:"count"`
+	Disk  []uint64 `json:"disk"`
+}
+
+// ClusterSizeSeries is per-cluster disk usage over time: one value per cluster
+// per day, largest cluster first, ranked by their most recent day.
+type ClusterSizeSeries struct {
+	Dates        []string        `json:"dates"`
+	Clusters     []ClusterSeries `json:"clusters"`
+	Other        *OtherSeries    `json:"other,omitempty"`
+	ClusterCount int             `json:"cluster_count"`
+	TotalDisk    uint64          `json:"total_disk"` // across all clusters on the last day
+}
+
+// GetClusterSizeSeries returns the last `days` days of per-cluster disk usage.
+// Clusters beyond `limit` are folded into Other. Clusters report roughly once
+// a day at no fixed hour, so a day without a report carries the previous value
+// forward rather than dropping to zero; a cluster that stopped reporting
+// altogether ends at its last sample instead of holding capacity forever.
+func (s *PrometheusStorage) GetClusterSizeSeries(days, limit int) ClusterSizeSeries {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	now := time.Now().UTC()
+	today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
+	dayOf := make(map[string]int, days)
+	series := ClusterSizeSeries{Dates: make([]string, days)}
+	for i := range series.Dates {
+		series.Dates[i] = today.AddDate(0, 0, i-days+1).Format("2006-01-02")
+		dayOf[series.Dates[i]] = i
+	}
+	activeSince := now.AddDate(0, 0, -activeDays).Unix()
+
+	for id, history := range s.histories {
+		disk := make([]uint64, days)
+		reported := make([]bool, days)
+		first, last := -1, -1
+		for _, sample := range history {
+			i, ok := dayOf[time.Unix(sample.Ts, 0).UTC().Format("2006-01-02")]
+			if !ok {
+				continue
+			}
+			disk[i], reported[i] = sample.TotalDiskBytes, true
+			if first < 0 {
+				first = i
+			}
+			last = i
+			if sample.Ts >= activeSince {
+				last = days - 1 // still reporting, so hold its size to the right edge
+			}
+		}
+		if first < 0 {
+			continue
+		}
+		for i := first + 1; i <= last; i++ {
+			if !reported[i] {
+				disk[i] = disk[i-1]
+			}
+		}
+		series.Clusters = append(series.Clusters, ClusterSeries{ClusterId: id, Disk: disk})
+		series.TotalDisk += disk[days-1]
+	}
+	series.ClusterCount = len(series.Clusters)
+
+	// Rank by the latest day so the stack reads largest-first at its right
+	// edge, tie-breaking on id to keep the order stable across refreshes.
+	sort.Slice(series.Clusters, func(i, j int) bool {
+		a, b := series.Clusters[i], series.Clusters[j]
+		if a.Disk[days-1] != b.Disk[days-1] {
+			return a.Disk[days-1] > b.Disk[days-1]
+		}
+		return a.ClusterId < b.ClusterId
+	})
+
+	if limit > 0 && len(series.Clusters) > limit {
+		other := OtherSeries{
+			Count: len(series.Clusters) - limit,
+			Disk:  make([]uint64, days),
+		}
+		for _, c := range series.Clusters[limit:] {
+			for i, v := range c.Disk {
+				other.Disk[i] += v
+			}
+		}
+		series.Clusters = series.Clusters[:limit]
+		series.Other = &other
+	}
+	return series
+}
diff --git a/telemetry/server/storage/sizes_test.go b/telemetry/server/storage/sizes_test.go
new file mode 100644
index 000000000..54af4fd43
--- /dev/null
+++ b/telemetry/server/storage/sizes_test.go
@@ -0,0 +1,122 @@
+package storage
+
+import (
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/seaweedfs/seaweedfs/telemetry/proto"
+)
+
+// seedHistory gives a cluster one sample per listed day offset (0 is today).
+func seedHistory(s *PrometheusStorage, id string, disk uint64, dayOffsets ...int) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for _, offset := range dayOffsets {
+		s.histories[id] = append(s.histories[id], HistorySample{
+			Ts:             time.Now().AddDate(0, 0, offset).Unix(),
+			TotalDiskBytes: disk,
+		})
+	}
+}
+
+func TestClusterSizeSeries(t *testing.T) {
+	s := newPrometheusStorage(prometheus.NewRegistry())
+
+	// Reported every day of the window.
+	seedHistory(s, "daily", 300, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0)
+	// Reported two days ago and not since: still active, so its size is held
+	// to the right edge instead of dropping out of the stack.
+	seedHistory(s, "lagging", 200, -2)
+	// Stopped reporting past the active window: its own days only.
+	seedHistory(s, "gone", 900, -9, -8)
+
+	series := s.GetClusterSizeSeries(10, 0)
+	if len(series.Dates) != 10 {
+		t.Fatalf("dates = %v, want 10 days", series.Dates)
+	}
+	if series.ClusterCount != 3 {
+		t.Fatalf("cluster_count = %d, want 3", series.ClusterCount)
+	}
+
+	byId := map[string][]uint64{}
+	for _, c := range series.Clusters {
+		byId[c.ClusterId] = c.Disk
+	}
+	if got := byId["daily"]; !equal(got, []uint64{300, 300, 300, 300, 300, 300, 300, 300, 300, 300}) {
+		t.Errorf("daily = %v, want 300 every day", got)
+	}
+	if got := byId["lagging"]; !equal(got, []uint64{0, 0, 0, 0, 0, 0, 0, 200, 200, 200}) {
+		t.Errorf("lagging = %v, want its size carried to the right edge", got)
+	}
+	if got := byId["gone"]; !equal(got, []uint64{900, 900, 0, 0, 0, 0, 0, 0, 0, 0}) {
+		t.Errorf("gone = %v, want no capacity after its last report", got)
+	}
+
+	// The total is the last day's stack height: daily + lagging, not gone.
+	if series.TotalDisk != 500 {
+		t.Errorf("total_disk = %d, want 500", series.TotalDisk)
+	}
+	// Largest on the last day comes first so the stack reads top-down.
+	if series.Clusters[0].ClusterId != "daily" {
+		t.Errorf("order = %s first, want daily", series.Clusters[0].ClusterId)
+	}
+
+	// Clusters past the limit are summed into "other", per day.
+	series = s.GetClusterSizeSeries(10, 1)
+	if len(series.Clusters) != 1 || series.Clusters[0].ClusterId != "daily" {
+		t.Fatalf("limited clusters = %+v, want just daily", series.Clusters)
+	}
+	if series.Other == nil || series.Other.Count != 2 {
+		t.Fatalf("other = %+v, want 2 clusters", series.Other)
+	}
+	if !equal(series.Other.Disk, []uint64{900, 900, 0, 0, 0, 0, 0, 200, 200, 200}) {
+		t.Errorf("other = %v, want lagging+gone summed per day", series.Other.Disk)
+	}
+	if series.ClusterCount != 3 || series.TotalDisk != 500 {
+		t.Errorf("limit changed totals: count=%d disk=%d, want 3/500", series.ClusterCount, series.TotalDisk)
+	}
+}
+
+// A report that lands after an earlier one on the same day replaces it, so the
+// series shows one value per cluster per day.
+func TestClusterSizeSeriesUsesLatestDailySample(t *testing.T) {
+	s := newPrometheusStorage(prometheus.NewRegistry())
+
+	data := &proto.TelemetryData{
+		TopologyId:     "aaaaaaaa-0000-0000-0000-000000000001",
+		Version:        "4.40",
+		Os:             "linux/amd64",
+		TotalDiskBytes: 100,
+	}
+	if err := s.StoreTelemetry(data); err != nil {
+		t.Fatal(err)
+	}
+	data.TotalDiskBytes = 700
+	if err := s.StoreTelemetry(data); err != nil {
+		t.Fatal(err)
+	}
+
+	series := s.GetClusterSizeSeries(3, 0)
+	if len(series.Clusters) != 1 {
+		t.Fatalf("clusters = %+v, want 1", series.Clusters)
+	}
+	if got := series.Clusters[0].Disk; !equal(got, []uint64{0, 0, 700}) {
+		t.Errorf("disk = %v, want today's latest sample only", got)
+	}
+	if series.TotalDisk != 700 {
+		t.Errorf("total_disk = %d, want 700", series.TotalDisk)
+	}
+}
+
+func equal(a, b []uint64) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}