mirrors/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-08-16 12:16:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

telemetry: per-cluster size over time on the dashboard (#10417)

Browse Source 
The dashboard charted one summed disk-usage line, so a step in the total
gave no hint which cluster moved. A new panel stacks each cluster's daily
size as its own band: the top of the stack is the fleet total, each band
is one cluster, and the clusters past the twentieth are summed into an
"other" band so the stack still adds up to the total.

The series is built from the per-cluster daily histories and served by
/api/cluster-sizes. Clusters report roughly once a day at no fixed hour,
so a day with no report carries the previous value forward — dropping it
to zero would sag the total every day as the clusters that have not
reported yet fall out from under it. A cluster that stops reporting past
the active window ends at its last sample instead of holding capacity
forever. Ranking is by the most recent day, tie-broken on cluster id so
the colors do not shuffle between refreshes.

Hover and click resolve to the band under the pointer: Chart.js's builtin
interaction modes match the nearest line, which on a stack of thin bands
is rarely the band being pointed at. Clicking one fills the per-cluster
history lookup below it.
This commit is contained in:
Chris Lu
2026-07-24 01:43:48 -07:00
committed by GitHub
parent 6e6255b58e
commit c194924d13
8 changed files with 389 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
telemetry/README.md
+3
View File
@@ -177,6 +177,9 @@ GET /api/metrics?days=30
# Get one cluster's daily usage history (disk bytes, volumes, volume servers)
GET /api/history?cluster_id=<uuid>&days=90
# Get per-cluster disk usage over time, largest first, the rest summed as "other"
GET /api/cluster-sizes?days=30&limit=20
```
### Monitoring
telemetry/server/api/handlers.go
+24
View File
@@ -150,6 +150,30 @@ func (h *Handler) GetMetrics(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(metrics)
}
func (h *Handler) GetClusterSizes(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
days := 30 // default
if daysStr := r.URL.Query().Get("days"); daysStr != "" {
if d, err := strconv.Atoi(daysStr); err == nil && d > 0 && d <= 365 {
days = d
}
}
limit := 20 // default
if limitStr := r.URL.Query().Get("limit"); limitStr != "" {
if l, err := strconv.Atoi(limitStr); err == nil && l > 0 && l <= 1000 {
limit = l
}
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(h.storage.GetClusterSizeSeries(days, limit))
}
func (h *Handler) GetHistory(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
telemetry/server/dashboard/dashboard.go
+129
View File
@@ -69,6 +69,10 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
font-weight: bold;
margin-bottom: 15px;
}
.chart-subtitle {
color: #666;
margin: -10px 0 15px;
}
.loading {
text-align: center;
padding: 40px;
@@ -159,6 +163,14 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
<canvas id="diskChart" width="400" height="200"></canvas>
</div>
<div class="chart-container">
<div class="chart-title">Cluster Sizes Over Time</div>
<div class="chart-subtitle" id="clusterSizesTotal"></div>
<div style="position: relative; height: 420px;">
<canvas id="clusterSizeChart"></canvas>
</div>
</div>
<div class="chart-container">
<div class="chart-title">Per-Cluster History</div>
<div class="cluster-lookup">
@@ -177,6 +189,7 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
<script>
let charts = {};
let clusterSizeIds = [];
async function loadDashboard() {
try {
@@ -188,8 +201,13 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
const metricsResponse = await fetch('/api/metrics?days=30');
const metrics = await metricsResponse.json();
// Load per-cluster sizes over time
const sizesResponse = await fetch('/api/cluster-sizes?days=30&limit=20');
const sizes = await sizesResponse.json();
updateStats(stats);
updateCharts(stats, metrics);
updateClusterSizes(sizes);
document.getElementById('loading').style.display = 'none';
document.getElementById('dashboard').style.display = 'block';
@@ -292,6 +310,117 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
});
}
// Decimal units, so the round numbers the chart picks for its ticks
// come out as round labels.
function formatBytes(bytes) {
const units = ['B', 'kB', 'MB', 'GB', 'TB', 'PB'];
let value = bytes || 0, unit = 0;
while (value >= 1000 && unit < units.length - 1) {
value /= 1000;
unit++;
}
return (unit === 0 ? value : value.toFixed(value >= 100 ? 0 : 1)) + ' ' + units[unit];
}
// One stacked band per cluster over time: the band is that cluster's
// size, the top of the stack is the fleet total. Clusters beyond the
// requested limit are summed into a trailing "other" band so the stack
// still adds up.
function updateClusterSizes(series) {
const clusters = series.clusters || [];
const dates = series.dates || [];
const count = series.cluster_count || 0;
document.getElementById('clusterSizesTotal').textContent =
formatBytes(series.total_disk) + ' across ' + count + ' cluster' + (count === 1 ? '' : 's') +
' on ' + (dates[dates.length - 1] || 'no data');
clusterSizeIds = clusters.map(c => c.cluster_id);
const datasets = clusters.map((c, i) => {
// Evenly spaced hues keep neighbouring bands distinguishable.
const hue = Math.round(i * 360 / clusters.length);
return band(c.cluster_id.slice(0, 8), c.disk,
'hsl(' + hue + ', 65%, 45%)', 'hsla(' + hue + ', 65%, 55%, 0.75)');
});
if (series.other) {
clusterSizeIds.push(null);
datasets.push(band('other (' + series.other.count + ')', series.other.disk,
'#9E9E9E', 'rgba(158, 158, 158, 0.6)'));
}
const ctx = document.getElementById('clusterSizeChart').getContext('2d');
if (charts.clusterSizeChart) {
charts.clusterSizeChart.destroy();
}
charts.clusterSizeChart = new Chart(ctx, {
type: 'line',
data: { labels: dates, datasets: datasets },
options: {
responsive: true,
maintainAspectRatio: false,
interaction: { mode: 'band', intersect: false },
onClick: (event, elements) => {
const id = elements.length && clusterSizeIds[elements[0].datasetIndex];
if (id) {
document.getElementById('clusterIdInput').value = id;
loadClusterHistory();
}
},
plugins: {
legend: { position: 'bottom', labels: { boxWidth: 12, font: { size: 11 } } },
tooltip: {
callbacks: {
label: item => {
const id = clusterSizeIds[item.datasetIndex];
return (id || item.dataset.label) + ': ' + formatBytes(item.raw);
}
}
}
},
scales: {
x: { ticks: { maxRotation: 0, autoSkip: true, maxTicksLimit: 12 } },
y: {
stacked: true,
beginAtZero: true,
ticks: { callback: value => formatBytes(value) }
}
}
}
});
}
// Hover and click resolve to the band the pointer is inside. Chart.js's
// built-in modes match the nearest line, which on a stack of thin bands
// is rarely the one under the pointer.
Chart.Interaction.modes.band = function(chart, event) {
const last = chart.data.labels.length - 1;
if (last < 0) return [];
const index = Math.min(Math.max(Math.round(chart.scales.x.getValueForPixel(event.x)), 0), last);
const value = chart.scales.y.getValueForPixel(event.y);
let stacked = 0;
for (let d = 0; d < chart.data.datasets.length; d++) {
stacked += chart.data.datasets[d].data[index] || 0;
if (value <= stacked) {
return [{ element: chart.getDatasetMeta(d).data[index], datasetIndex: d, index: index }];
}
}
return [];
};
function band(label, data, borderColor, backgroundColor) {
return {
label: label,
data: data,
borderColor: borderColor,
backgroundColor: backgroundColor,
borderWidth: 1,
pointRadius: 0,
pointHitRadius: 8,
fill: true,
tension: 0.1
};
}
async function loadClusterHistory() {
const id = document.getElementById('clusterIdInput').value.trim();
const errorDiv = document.getElementById('clusterHistoryError');
telemetry/server/main.go
+1
View File
@@ -76,6 +76,7 @@ func main() {
mux.HandleFunc("/api/instances", corsMiddleware(logMiddleware(apiHandler.GetInstances)))
mux.HandleFunc("/api/metrics", corsMiddleware(logMiddleware(apiHandler.GetMetrics)))
mux.HandleFunc("/api/history", corsMiddleware(logMiddleware(apiHandler.GetHistory)))
mux.HandleFunc("/api/cluster-sizes", corsMiddleware(logMiddleware(apiHandler.GetClusterSizes)))
// Dashboard (optional)
if *enableDashboard {
telemetry/server/storage/history.go
+3
View File
@@ -10,6 +10,9 @@ import (
// on before it counts as confirmed in the aggregated stats.
const confirmDays = 2
// activeDays is how recently a cluster must have reported to count as active.
const activeDays = 7
// HistorySample is one retained data point of a cluster's daily reports.
// Tags are kept short because thousands of samples end up in the state file.
type HistorySample struct {
telemetry/server/storage/prometheus.go
+1 -1
View File
@@ -207,7 +207,7 @@ func (s *PrometheusStorage) GetMetrics(days int) (map[string]interface{}, error)
func (s *PrometheusStorage) updateStats() {
now := time.Now()
last7Days := now.AddDate(0, 0, -7)
last7Days := now.AddDate(0, 0, -activeDays)
last30Days := now.AddDate(0, 0, -30)
totalInstances := 0
telemetry/server/storage/sizes.go
+106
View File
@@ -0,0 +1,106 @@
package storage
import (
"sort"
"time"
)
// ClusterSeries is one cluster's daily disk usage, aligned to the shared date
// axis of the enclosing ClusterSizeSeries.
type ClusterSeries struct {
ClusterId string `json:"cluster_id"`
Disk []uint64 `json:"disk"`
}
// OtherSeries is the clusters beyond the caller's limit, summed per day so a
// stacked chart still adds up to the fleet total.
type OtherSeries struct {
Count int `json:"count"`
Disk []uint64 `json:"disk"`
}
// ClusterSizeSeries is per-cluster disk usage over time: one value per cluster
// per day, largest cluster first, ranked by their most recent day.
type ClusterSizeSeries struct {
Dates []string `json:"dates"`
Clusters []ClusterSeries `json:"clusters"`
Other *OtherSeries `json:"other,omitempty"`
ClusterCount int `json:"cluster_count"`
TotalDisk uint64 `json:"total_disk"` // across all clusters on the last day
}
// GetClusterSizeSeries returns the last `days` days of per-cluster disk usage.
// Clusters beyond `limit` are folded into Other. Clusters report roughly once
// a day at no fixed hour, so a day without a report carries the previous value
// forward rather than dropping to zero; a cluster that stopped reporting
// altogether ends at its last sample instead of holding capacity forever.
func (s *PrometheusStorage) GetClusterSizeSeries(days, limit int) ClusterSizeSeries {
s.mu.RLock()
defer s.mu.RUnlock()
now := time.Now().UTC()
today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
dayOf := make(map[string]int, days)
series := ClusterSizeSeries{Dates: make([]string, days)}
for i := range series.Dates {
series.Dates[i] = today.AddDate(0, 0, i-days+1).Format("2006-01-02")
dayOf[series.Dates[i]] = i
}
activeSince := now.AddDate(0, 0, -activeDays).Unix()
for id, history := range s.histories {
disk := make([]uint64, days)
reported := make([]bool, days)
first, last := -1, -1
for _, sample := range history {
i, ok := dayOf[time.Unix(sample.Ts, 0).UTC().Format("2006-01-02")]
if !ok {
continue
}
disk[i], reported[i] = sample.TotalDiskBytes, true
if first < 0 {
first = i
}
last = i
if sample.Ts >= activeSince {
last = days - 1 // still reporting, so hold its size to the right edge
}
}
if first < 0 {
continue
}
for i := first + 1; i <= last; i++ {
if !reported[i] {
disk[i] = disk[i-1]
}
}
series.Clusters = append(series.Clusters, ClusterSeries{ClusterId: id, Disk: disk})
series.TotalDisk += disk[days-1]
}
series.ClusterCount = len(series.Clusters)
// Rank by the latest day so the stack reads largest-first at its right
// edge, tie-breaking on id to keep the order stable across refreshes.
sort.Slice(series.Clusters, func(i, j int) bool {
a, b := series.Clusters[i], series.Clusters[j]
if a.Disk[days-1] != b.Disk[days-1] {
return a.Disk[days-1] > b.Disk[days-1]
}
return a.ClusterId < b.ClusterId
})
if limit > 0 && len(series.Clusters) > limit {
other := OtherSeries{
Count: len(series.Clusters) - limit,
Disk: make([]uint64, days),
}
for _, c := range series.Clusters[limit:] {
for i, v := range c.Disk {
other.Disk[i] += v
}
}
series.Clusters = series.Clusters[:limit]
series.Other = &other
}
return series
}
telemetry/server/storage/sizes_test.go
+122
View File
@@ -0,0 +1,122 @@
package storage
import (
"testing"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/seaweedfs/seaweedfs/telemetry/proto"
)
// seedHistory gives a cluster one sample per listed day offset (0 is today).
func seedHistory(s *PrometheusStorage, id string, disk uint64, dayOffsets ...int) {
s.mu.Lock()
defer s.mu.Unlock()
for _, offset := range dayOffsets {
s.histories[id] = append(s.histories[id], HistorySample{
Ts: time.Now().AddDate(0, 0, offset).Unix(),
TotalDiskBytes: disk,
})
}
}
func TestClusterSizeSeries(t *testing.T) {
s := newPrometheusStorage(prometheus.NewRegistry())
// Reported every day of the window.
seedHistory(s, "daily", 300, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0)
// Reported two days ago and not since: still active, so its size is held
// to the right edge instead of dropping out of the stack.
seedHistory(s, "lagging", 200, -2)
// Stopped reporting past the active window: its own days only.
seedHistory(s, "gone", 900, -9, -8)
series := s.GetClusterSizeSeries(10, 0)
if len(series.Dates) != 10 {
t.Fatalf("dates = %v, want 10 days", series.Dates)
}
if series.ClusterCount != 3 {
t.Fatalf("cluster_count = %d, want 3", series.ClusterCount)
}
byId := map[string][]uint64{}
for _, c := range series.Clusters {
byId[c.ClusterId] = c.Disk
}
if got := byId["daily"]; !equal(got, []uint64{300, 300, 300, 300, 300, 300, 300, 300, 300, 300}) {
t.Errorf("daily = %v, want 300 every day", got)
}
if got := byId["lagging"]; !equal(got, []uint64{0, 0, 0, 0, 0, 0, 0, 200, 200, 200}) {
t.Errorf("lagging = %v, want its size carried to the right edge", got)
}
if got := byId["gone"]; !equal(got, []uint64{900, 900, 0, 0, 0, 0, 0, 0, 0, 0}) {
t.Errorf("gone = %v, want no capacity after its last report", got)
}
// The total is the last day's stack height: daily + lagging, not gone.
if series.TotalDisk != 500 {
t.Errorf("total_disk = %d, want 500", series.TotalDisk)
}
// Largest on the last day comes first so the stack reads top-down.
if series.Clusters[0].ClusterId != "daily" {
t.Errorf("order = %s first, want daily", series.Clusters[0].ClusterId)
}
// Clusters past the limit are summed into "other", per day.
series = s.GetClusterSizeSeries(10, 1)
if len(series.Clusters) != 1 || series.Clusters[0].ClusterId != "daily" {
t.Fatalf("limited clusters = %+v, want just daily", series.Clusters)
}
if series.Other == nil || series.Other.Count != 2 {
t.Fatalf("other = %+v, want 2 clusters", series.Other)
}
if !equal(series.Other.Disk, []uint64{900, 900, 0, 0, 0, 0, 0, 200, 200, 200}) {
t.Errorf("other = %v, want lagging+gone summed per day", series.Other.Disk)
}
if series.ClusterCount != 3 || series.TotalDisk != 500 {
t.Errorf("limit changed totals: count=%d disk=%d, want 3/500", series.ClusterCount, series.TotalDisk)
}
}
// A report that lands after an earlier one on the same day replaces it, so the
// series shows one value per cluster per day.
func TestClusterSizeSeriesUsesLatestDailySample(t *testing.T) {
s := newPrometheusStorage(prometheus.NewRegistry())
data := &proto.TelemetryData{
TopologyId: "aaaaaaaa-0000-0000-0000-000000000001",
Version: "4.40",
Os: "linux/amd64",
TotalDiskBytes: 100,
}
if err := s.StoreTelemetry(data); err != nil {
t.Fatal(err)
}
data.TotalDiskBytes = 700
if err := s.StoreTelemetry(data); err != nil {
t.Fatal(err)
}
series := s.GetClusterSizeSeries(3, 0)
if len(series.Clusters) != 1 {
t.Fatalf("clusters = %+v, want 1", series.Clusters)
}
if got := series.Clusters[0].Disk; !equal(got, []uint64{0, 0, 700}) {
t.Errorf("disk = %v, want today's latest sample only", got)
}
if series.TotalDisk != 700 {
t.Errorf("total_disk = %d, want 700", series.TotalDisk)
}
}
func equal(a, b []uint64) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}