telemetry: version distribution over time (#10625)

* telemetry: keep the reported version in the daily history The version only ever lived on the instance record, which holds a cluster's latest report, so there was no way to ask what anything ran last Tuesday. Record it per sample, and let the daily axis carry strings as well as counts. State written before this has no version on its samples. The newest sample is the report the instance record itself came from, so fill that one in on load rather than starting a version series a day late. * telemetry: serve the fleet's version make-up over time /api/versions gives how many clusters ran each release per day, on the same axis and hold-forward rule as the cluster sizes. Releases are ordered by number rather than by size: the caller stacks them, and a stack whose order changes with the counts is unreadable over time. The tail past the limit is summed into "other" so the stack still adds up. Days with no version are dropped before the axis is built, so the series spans the days it knows a version for instead of climbing out of blanks. * telemetry: draw version distribution as a stacked growth chart The pie only ever showed today. Stacked over 30 days the height is the confirmed fleet and each band is a release, so one chart carries the growth and the rollouts at once. Newest release on the floor, so the band being read is anchored to the axis instead of riding on everything below it. Eight fixed hues instead of the evenly spaced ones the cluster stacks use: evenly spaced put a green and a cyan close enough to be hard to tell apart, which matters for a set you read rather than a wall of anonymous ids. Versions past the eighth fold into "other", and each band carries its own number so the chart reads without matching colours against the legend.
2026-08-16 04:06:44 +00:00 · 2026-08-07 12:08:04 -07:00
parent b46946ece5
commit a49cf11e16
11 changed files with 462 additions and 32 deletions
@@ -184,6 +184,10 @@ GET /api/history?cluster_id=<uuid>&days=90
 # Get per-cluster disk usage and volume servers over time, largest first,
 # the rest summed as "other"
 GET /api/cluster-sizes?days=30&limit=20
+
+# Get how many clusters ran each version over time, oldest version first,
+# the rest summed as "other"
+GET /api/versions?days=30&limit=8
 ```

 ### Monitoring
@@ -174,6 +174,30 @@ func (h *Handler) GetClusterSizes(w http.ResponseWriter, r *http.Request) {
 	json.NewEncoder(w).Encode(h.storage.GetClusterSizeSeries(days, limit))
 }

+func (h *Handler) GetVersions(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	days := 30 // default
+	if daysStr := r.URL.Query().Get("days"); daysStr != "" {
+		if d, err := strconv.Atoi(daysStr); err == nil && d > 0 && d <= 365 {
+			days = d
+		}
+	}
+
+	limit := 8 // default, the number of colours the version stack has
+	if limitStr := r.URL.Query().Get("limit"); limitStr != "" {
+		if l, err := strconv.Atoi(limitStr); err == nil && l > 0 && l <= 1000 {
+			limit = l
+		}
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(h.storage.GetVersionSeries(days, limit))
+}
+
 func (h *Handler) GetHistory(w http.ResponseWriter, r *http.Request) {
 	if r.Method != http.MethodGet {
 		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
@@ -143,7 +143,10 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {

            <div class="chart-container">
                <div class="chart-title">Version Distribution</div>
-                <canvas id="versionChart" width="400" height="200"></canvas>
+                <div class="chart-subtitle" id="versionTotal"></div>
+                <div style="position: relative; height: 420px;">
+                    <canvas id="versionChart"></canvas>
+                </div>
            </div>

            <div class="chart-container">
@@ -197,10 +200,15 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
                const sizesResponse = await fetch('/api/cluster-sizes?days=30&limit=20');
                const sizes = await sizesResponse.json();

+                // Load the fleet's version make-up over time
+                const versionsResponse = await fetch('/api/versions?days=30&limit=8');
+                const versions = await versionsResponse.json();
+
                updateStats(stats);
-                updateCharts(stats);
+                createPieChart('osChart', stats.os_distribution || {});
+                updateVersions(versions);
                updateClusterSizes(sizes);
-                
+
                document.getElementById('loading').style.display = 'none';
                document.getElementById('dashboard').style.display = 'block';
            } catch (error) {
@@ -217,12 +225,7 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
            document.getElementById('totalOS').textContent = Object.keys(stats.os_distribution || {}).length;
        }

-        function updateCharts(stats) {
-            createPieChart('versionChart', 'Version Distribution', stats.versions || {});
-            createPieChart('osChart', 'Operating System Distribution', stats.os_distribution || {});
-        }
-
-        function createPieChart(canvasId, title, data) {
+        function createPieChart(canvasId, data) {
            const ctx = document.getElementById(canvasId).getContext('2d');
            
            if (charts[canvasId]) {
@@ -298,6 +301,75 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
            return (unit === 0 ? value : value.toFixed(value >= 100 ? 0 : 1)) + ' ' + units[unit];
        }

+        // Fixed hues rather than the evenly spaced ones the cluster stacks use:
+        // a handful of versions is a set you read, and evenly spaced hues put
+        // pairs next to each other that colour-blind readers can't separate.
+        const versionColors = ['#2a78d6', '#eb6834', '#1baf7a', '#eda100',
+                               '#e87ba4', '#008300', '#4a3aa7', '#e34948'];
+
+        // One stacked band per version: the band is how many clusters ran that
+        // version that day, the top of the stack is the confirmed fleet, so the
+        // chart shows both how it grows and what it upgrades to.
+        function updateVersions(series) {
+            const versions = series.versions || [];
+            const dates = series.dates || [];
+            const total = series.total_clusters || 0;
+            document.getElementById('versionTotal').textContent =
+                total + ' cluster' + (total === 1 ? '' : 's') + ' on ' + (dates[dates.length - 1] || 'no data');
+
+            // Newest release on the floor, oldest on top: the current release
+            // is the band being read, and one anchored to the baseline reads
+            // straight off the axis instead of riding on everything below it.
+            // Colours follow the same order, so a release keeps its colour as
+            // older ones age out from the top.
+            const datasets = versions.slice().reverse().map((v, i) =>
+                band(v.version, v.clusters, '#ffffff', versionColors[i % versionColors.length], 2));
+            if (series.other) {
+                datasets.push(band('other (' + series.other.count + ' versions)', series.other.clusters,
+                    '#ffffff', '#9a9a94', 2));
+            }
+
+            stackedArea('versionChart', dates, datasets, value => value, { plugins: [bandLabels] });
+        }
+
+        // Writes each version into its own band, so the chart reads without
+        // matching colours against the legend. Bands too thin to hold the text
+        // keep it, and the halo carries it over whatever it crosses.
+        const bandLabels = {
+            id: 'bandLabels',
+            afterDatasetsDraw(chart) {
+                const ctx = chart.ctx;
+                ctx.save();
+                ctx.font = '600 12px -apple-system, BlinkMacSystemFont, sans-serif';
+                ctx.textAlign = 'center';
+                ctx.textBaseline = 'middle';
+                ctx.lineJoin = 'round';
+                chart.data.datasets.forEach((dataset, d) => {
+                    const meta = chart.getDatasetMeta(d);
+                    if (meta.hidden) return;
+                    // Label where the band is thickest, so the text has room.
+                    let at = -1, thickest = 0;
+                    dataset.data.forEach((value, i) => {
+                        if (value > thickest) {
+                            thickest = value;
+                            at = i;
+                        }
+                    });
+                    const point = at >= 0 && meta.data[at];
+                    if (!point) return;
+                    const below = d === 0 ? chart.scales.y.getPixelForValue(0)
+                                          : chart.getDatasetMeta(d - 1).data[at].y;
+                    if (below - point.y < 18) return;
+                    ctx.lineWidth = 3;
+                    ctx.strokeStyle = 'rgba(255, 255, 255, 0.85)';
+                    ctx.strokeText(dataset.label, point.x, (point.y + below) / 2);
+                    ctx.fillStyle = '#1a1a1a';
+                    ctx.fillText(dataset.label, point.x, (point.y + below) / 2);
+                });
+                ctx.restore();
+            }
+        };
+
        // Disk usage and volume servers both drawn as one stacked band per
        // cluster over time: the band is that cluster's share, the top of the
        // stack is the fleet total. Clusters beyond the requested limit are
@@ -337,6 +409,24 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
                    '#9E9E9E', 'rgba(158, 158, 158, 0.6)'));
            }

+            stackedArea(canvasId, dates, datasets, format, {
+                onClick: (event, elements) => {
+                    const id = elements.length && clusterSizeIds[elements[0].datasetIndex];
+                    if (id) {
+                        document.getElementById('clusterIdInput').value = id;
+                        loadClusterHistory();
+                    }
+                },
+                // The legend shows shortened ids; the tooltip has room for the
+                // full one to paste into the lookup box.
+                label: item => (clusterSizeIds[item.datasetIndex] || item.dataset.label) + ': ' + format(item.raw)
+            });
+        }
+
+        // Draws the bands as one stack, so their heights add up to the day's
+        // total. hooks.onClick, hooks.label and hooks.plugins are optional.
+        function stackedArea(canvasId, dates, datasets, format, hooks) {
+            hooks = hooks || {};
            const ctx = document.getElementById(canvasId).getContext('2d');
            if (charts[canvasId]) {
                charts[canvasId].destroy();
@@ -344,25 +434,17 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
            charts[canvasId] = new Chart(ctx, {
                type: 'line',
                data: { labels: dates, datasets: datasets },
+                plugins: hooks.plugins,
                options: {
                    responsive: true,
                    maintainAspectRatio: false,
                    interaction: { mode: 'band', intersect: false },
-                    onClick: (event, elements) => {
-                        const id = elements.length && clusterSizeIds[elements[0].datasetIndex];
-                        if (id) {
-                            document.getElementById('clusterIdInput').value = id;
-                            loadClusterHistory();
-                        }
-                    },
+                    onClick: hooks.onClick,
                    plugins: {
                        legend: { position: 'bottom', labels: { boxWidth: 12, font: { size: 11 } } },
                        tooltip: {
                            callbacks: {
-                                label: item => {
-                                    const id = clusterSizeIds[item.datasetIndex];
-                                    return (id || item.dataset.label) + ': ' + format(item.raw);
-                                }
+                                label: hooks.label || (item => item.dataset.label + ': ' + format(item.raw))
                            }
                        }
                    },
@@ -396,13 +478,13 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
            return [];
        };

-        function band(label, data, borderColor, backgroundColor) {
+        function band(label, data, borderColor, backgroundColor, borderWidth) {
            return {
                label: label,
                data: data,
                borderColor: borderColor,
                backgroundColor: backgroundColor,
-                borderWidth: 1,
+                borderWidth: borderWidth || 1,
                pointRadius: 0,
                pointHitRadius: 8,
                fill: true,
@@ -77,6 +77,7 @@ func main() {
 	mux.HandleFunc("/api/metrics", corsMiddleware(logMiddleware(apiHandler.GetMetrics)))
 	mux.HandleFunc("/api/history", corsMiddleware(logMiddleware(apiHandler.GetHistory)))
 	mux.HandleFunc("/api/cluster-sizes", corsMiddleware(logMiddleware(apiHandler.GetClusterSizes)))
+	mux.HandleFunc("/api/versions", corsMiddleware(logMiddleware(apiHandler.GetVersions)))

 	// Dashboard (optional)
 	if *enableDashboard {
@@ -20,6 +20,7 @@ type HistorySample struct {
 	TotalDiskBytes    uint64 `json:"disk"`
 	TotalVolumeCount  int32  `json:"volumes"`
 	VolumeServerCount int32  `json:"servers"`
+	Version           string `json:"ver,omitempty"` // empty in samples written before this was recorded
 }

 // appendHistory records the report as the cluster's sample for the day,
@@ -30,6 +31,7 @@ func (s *PrometheusStorage) appendHistory(data *proto.TelemetryData, receivedAt
 		TotalDiskBytes:    data.TotalDiskBytes,
 		TotalVolumeCount:  data.TotalVolumeCount,
 		VolumeServerCount: data.VolumeServerCount,
+		Version:           data.Version,
 	}
 	h := s.histories[data.TopologyId]
 	if n := len(h); n > 0 && sameUTCDay(h[n-1].Ts, sample.Ts) {
@@ -110,16 +112,17 @@ func utcDay(ts int64) time.Time {
 	return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC)
 }

-func diskBytes(s HistorySample) uint64   { return s.TotalDiskBytes }
-func serverCount(s HistorySample) uint64 { return uint64(s.VolumeServerCount) }
+func diskBytes(s HistorySample) uint64     { return s.TotalDiskBytes }
+func serverCount(s HistorySample) uint64   { return uint64(s.VolumeServerCount) }
+func sampleVersion(s HistorySample) string { return s.Version }

 // align lays one cluster's history onto the axis, picking `value` out of each
 // sample. Clusters report roughly once a day at no fixed hour, so a day without
 // a report carries the previous value forward rather than dropping to zero; a
 // cluster that stopped reporting altogether ends at its last sample instead of
 // holding capacity forever. Reports false when the cluster has nothing in range.
-func (d dailySeries) align(history []HistorySample, activeSince int64, value func(HistorySample) uint64) ([]uint64, bool) {
-	out := make([]uint64, len(d.dates))
+func align[T any](d dailySeries, history []HistorySample, activeSince int64, value func(HistorySample) T) ([]T, bool) {
+	out := make([]T, len(d.dates))
 	reported := make([]bool, len(d.dates))
 	first, last := -1, -1
 	for _, sample := range history {
@@ -44,9 +44,18 @@ func (s *PrometheusStorage) LoadState(path string) (int, error) {
 		loaded++
 	}
 	for id, h := range state.Histories {
-		if _, ok := s.instances[id]; ok && len(h) > 0 {
-			s.histories[id] = h
+		instance, ok := s.instances[id]
+		if !ok || len(h) == 0 {
+			continue
 		}
+		// State written before versions were recorded carries none on its
+		// samples. The newest sample is the report the instance record itself
+		// came from, so that day's version is known and the version series can
+		// start there rather than a day after the upgrade.
+		if newest := len(h) - 1; h[newest].Version == "" {
+			h[newest].Version = instance.TelemetryData.Version
+		}
+		s.histories[id] = h
 	}
 	s.updateStats()
 	return loaded, nil
@@ -72,3 +72,38 @@ func TestStateRoundTrip(t *testing.T) {
 		t.Error("dirty flag set after load")
 	}
 }
+
+// State written before versions were recorded still knows the version of the
+// report its newest sample came from: the instance record's.
+func TestLoadStateFillsNewestSampleVersion(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "telemetry-state.json")
+
+	s := newPrometheusStorage(prometheus.NewRegistry())
+	report := &proto.TelemetryData{TopologyId: "test-cluster-1", Version: "4.40", Os: "linux/amd64"}
+	if err := s.StoreTelemetry(report); err != nil {
+		t.Fatalf("store: %v", err)
+	}
+	yesterday := time.Now().AddDate(0, 0, -1).Unix()
+	s.histories[report.TopologyId] = []HistorySample{
+		{Ts: yesterday, TotalDiskBytes: 10},
+		{Ts: time.Now().Unix(), TotalDiskBytes: 20},
+	}
+	if err := s.SaveStateIfDirty(path); err != nil {
+		t.Fatalf("save: %v", err)
+	}
+
+	s = newPrometheusStorage(prometheus.NewRegistry())
+	if _, err := s.LoadState(path); err != nil {
+		t.Fatalf("load: %v", err)
+	}
+	loaded := s.histories[report.TopologyId]
+	if len(loaded) != 2 {
+		t.Fatalf("history = %+v, want 2 samples", loaded)
+	}
+	if loaded[1].Version != report.Version {
+		t.Errorf("newest sample version = %q, want %q", loaded[1].Version, report.Version)
+	}
+	if loaded[0].Version != "" {
+		t.Errorf("older sample version = %q, want it left unknown", loaded[0].Version)
+	}
+}
@@ -180,12 +180,12 @@ func (s *PrometheusStorage) GetMetrics(days int) (map[string]interface{}, error)
 	diskUsage := make([]uint64, len(axis.dates))
 	serverCounts := make([]int64, len(axis.dates))
 	for _, history := range histories {
-		if disk, ok := axis.align(history, activeSince, diskBytes); ok {
+		if disk, ok := align(axis, history, activeSince, diskBytes); ok {
 			for i, v := range disk {
 				diskUsage[i] += v
 			}
 		}
-		if servers, ok := axis.align(history, activeSince, serverCount); ok {
+		if servers, ok := align(axis, history, activeSince, serverCount); ok {
 			for i, v := range servers {
 				serverCounts[i] += int64(v)
 			}
@@ -48,11 +48,11 @@ func (s *PrometheusStorage) GetClusterSizeSeries(days, limit int) ClusterSizeSer
 	series := ClusterSizeSeries{Dates: axis.dates}

 	for id, history := range histories {
-		disk, ok := axis.align(history, activeSince, diskBytes)
+		disk, ok := align(axis, history, activeSince, diskBytes)
 		if !ok {
 			continue
 		}
-		servers, _ := axis.align(history, activeSince, serverCount)
+		servers, _ := align(axis, history, activeSince, serverCount)
 		series.Clusters = append(series.Clusters, ClusterSeries{ClusterId: id, Disk: disk, Servers: servers})
 		series.TotalDisk += disk[last]
 		series.TotalServers += servers[last]
@@ -0,0 +1,163 @@
+package storage
+
+import (
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// VersionCounts is how many clusters ran one version per day, aligned to the
+// shared date axis of the enclosing VersionSeries.
+type VersionCounts struct {
+	Version  string   `json:"version"`
+	Clusters []uint64 `json:"clusters"`
+}
+
+// OtherVersions is the versions beyond the caller's limit, summed per day so a
+// stacked chart still adds up to the cluster total.
+type OtherVersions struct {
+	Count    int      `json:"count"`
+	Clusters []uint64 `json:"clusters"`
+}
+
+// VersionSeries is the version make-up of the fleet over time: one cluster
+// count per version per day, oldest release first.
+type VersionSeries struct {
+	Dates         []string        `json:"dates"`
+	Versions      []VersionCounts `json:"versions"`
+	Other         *OtherVersions  `json:"other,omitempty"`
+	TotalClusters uint64          `json:"total_clusters"` // across all versions on the last day
+}
+
+// GetVersionSeries returns the last `days` days of per-version cluster counts
+// across confirmed clusters. Versions beyond `limit` are folded into Other,
+// keeping the ones with the most clusters on the last day.
+func (s *PrometheusStorage) GetVersionSeries(days, limit int) VersionSeries {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	histories := versionedHistories(s.seriesHistories())
+	axis := newDailySeries(days, histories)
+	activeSince := time.Now().UTC().AddDate(0, 0, -activeDays).Unix()
+	last := len(axis.dates) - 1
+	series := VersionSeries{Dates: axis.dates}
+
+	daily := make(map[string][]uint64)
+	for _, history := range histories {
+		versions, ok := align(axis, history, activeSince, sampleVersion)
+		if !ok {
+			continue
+		}
+		for i, v := range versions {
+			if v == "" {
+				continue // before the cluster's first day in the window
+			}
+			counts, ok := daily[v]
+			if !ok {
+				counts = make([]uint64, len(axis.dates))
+				daily[v] = counts
+			}
+			counts[i]++
+		}
+	}
+
+	for v, counts := range daily {
+		series.Versions = append(series.Versions, VersionCounts{Version: v, Clusters: counts})
+		series.TotalClusters += counts[last]
+	}
+	// Ordered by release rather than by size: the caller stacks them, and a
+	// stack whose order changes with the counts is unreadable over time.
+	sort.Slice(series.Versions, func(i, j int) bool {
+		return versionLess(series.Versions[i].Version, series.Versions[j].Version)
+	})
+
+	if limit > 0 && len(series.Versions) > limit {
+		// Old releases are a long tail of one-cluster bands; keep the versions
+		// most of the fleet is on now and sum the rest into one band.
+		ranked := append([]VersionCounts(nil), series.Versions...)
+		sort.Slice(ranked, func(i, j int) bool {
+			if ranked[i].Clusters[last] != ranked[j].Clusters[last] {
+				return ranked[i].Clusters[last] > ranked[j].Clusters[last]
+			}
+			return versionLess(ranked[j].Version, ranked[i].Version)
+		})
+		other := OtherVersions{Count: len(ranked) - limit, Clusters: make([]uint64, len(axis.dates))}
+		for _, v := range ranked[limit:] {
+			for i, c := range v.Clusters {
+				other.Clusters[i] += c
+			}
+		}
+		kept := make(map[string]bool, limit)
+		for _, v := range ranked[:limit] {
+			kept[v.Version] = true
+		}
+		versions := series.Versions[:0]
+		for _, v := range series.Versions {
+			if kept[v.Version] {
+				versions = append(versions, v)
+			}
+		}
+		series.Versions = versions
+		series.Other = &other
+	}
+	return series
+}
+
+// versionedHistories drops the samples recorded before the reported version was
+// kept in history, so the version series spans the days it actually knows a
+// version for instead of climbing out of a run of blank days.
+func versionedHistories(histories map[string][]HistorySample) map[string][]HistorySample {
+	out := make(map[string][]HistorySample, len(histories))
+	for id, history := range histories {
+		var kept []HistorySample
+		for _, sample := range history {
+			if sample.Version != "" {
+				kept = append(kept, sample)
+			}
+		}
+		if len(kept) > 0 {
+			out[id] = kept
+		}
+	}
+	return out
+}
+
+// versionLess orders release strings like "3.97" and "4.40" by their numeric
+// components rather than lexically, which would sort "10.02" before "9.99". A
+// build suffix sits next to the release it was built from. Anything that
+// doesn't parse, such as the "unknown" a client with no version compiled in
+// reports, sorts first.
+func versionLess(a, b string) bool {
+	an, aSuffix, aok := versionParts(a)
+	bn, bSuffix, bok := versionParts(b)
+	if aok != bok {
+		return !aok
+	}
+	if !aok {
+		return a < b
+	}
+	for i := 0; i < len(an) && i < len(bn); i++ {
+		if an[i] != bn[i] {
+			return an[i] < bn[i]
+		}
+	}
+	if len(an) != len(bn) {
+		return len(an) < len(bn)
+	}
+	return aSuffix < bSuffix
+}
+
+func versionParts(v string) ([]int, string, bool) {
+	number, suffix, _ := strings.Cut(v, "-")
+	fields := strings.Split(number, ".")
+	parts := make([]int, 0, len(fields))
+	for _, field := range fields {
+		n, err := strconv.Atoi(field)
+		if err != nil {
+			return nil, "", false
+		}
+		parts = append(parts, n)
+	}
+	return parts, suffix, len(parts) > 0
+}
@@ -0,0 +1,109 @@
+package storage
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+func TestVersionSeries(t *testing.T) {
+	s := newPrometheusStorage(prometheus.NewRegistry())
+
+	// Upgraded mid-window: its band leaves the old version for the new one.
+	seedSamples(s, "upgraded", HistorySample{Version: "4.39"}, -4, -3)
+	seedSamples(s, "upgraded", HistorySample{Version: "4.40"}, -2, -1, 0)
+	// Reported every day on the same version.
+	seedSamples(s, "steady", HistorySample{Version: "4.40"}, -4, -3, -2, -1, 0)
+	// One day of history only: unconfirmed, so it stays out of the stack.
+	seedSamples(s, "oneshot", HistorySample{Version: "4.40"}, -1)
+	// Confirmed, but its samples predate versions being recorded.
+	seedSamples(s, "versionless", HistorySample{}, -9, -8)
+
+	series := s.GetVersionSeries(10, 0)
+
+	// The versionless days are dropped before the axis is built, so the chart
+	// spans the days a version is known for instead of climbing out of blanks.
+	if len(series.Dates) != 5 {
+		t.Fatalf("dates = %v, want the 5 days with versions", series.Dates)
+	}
+	if len(series.Versions) != 2 ||
+		series.Versions[0].Version != "4.39" || series.Versions[1].Version != "4.40" {
+		t.Fatalf("versions = %+v, want 4.39 then 4.40", series.Versions)
+	}
+	if got := series.Versions[0].Clusters; !equal(got, []uint64{1, 1, 0, 0, 0}) {
+		t.Errorf("4.39 = %v, want the upgraded cluster's first two days", got)
+	}
+	if got := series.Versions[1].Clusters; !equal(got, []uint64{1, 1, 2, 2, 2}) {
+		t.Errorf("4.40 = %v, want steady plus upgraded from day 3", got)
+	}
+	if series.TotalClusters != 2 {
+		t.Errorf("total_clusters = %d, want 2", series.TotalClusters)
+	}
+	if series.Other != nil {
+		t.Errorf("other = %+v, want none without a limit", series.Other)
+	}
+}
+
+func TestVersionSeriesHoldsForwardAndLimits(t *testing.T) {
+	s := newPrometheusStorage(prometheus.NewRegistry())
+
+	// Stopped reporting past the active window: its own days only.
+	seedSamples(s, "gone", HistorySample{Version: "3.97"}, -9, -8)
+	// Reported every day of the window.
+	seedSamples(s, "daily", HistorySample{Version: "4.40"}, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0)
+	// Reported three days ago and not since: still active, so it holds its
+	// version to the right edge instead of dropping out of the stack.
+	seedSamples(s, "lagging", HistorySample{Version: "4.30"}, -3, -2)
+
+	series := s.GetVersionSeries(10, 0)
+	if len(series.Dates) != 10 {
+		t.Fatalf("dates = %v, want 10 days", series.Dates)
+	}
+	byVersion := map[string][]uint64{}
+	for _, v := range series.Versions {
+		byVersion[v.Version] = v.Clusters
+	}
+	if got := byVersion["3.97"]; !equal(got, []uint64{1, 1, 0, 0, 0, 0, 0, 0, 0, 0}) {
+		t.Errorf("3.97 = %v, want nothing after its last report", got)
+	}
+	if got := byVersion["4.30"]; !equal(got, []uint64{0, 0, 0, 0, 0, 0, 1, 1, 1, 1}) {
+		t.Errorf("4.30 = %v, want carried to the right edge", got)
+	}
+	if got := byVersion["4.40"]; !equal(got, []uint64{1, 1, 1, 1, 1, 1, 1, 1, 1, 1}) {
+		t.Errorf("4.40 = %v, want 1 every day", got)
+	}
+	if series.TotalClusters != 2 {
+		t.Errorf("total_clusters = %d, want the 2 clusters still reporting", series.TotalClusters)
+	}
+
+	// Past the limit, the versions the fewest clusters are on are summed into
+	// one band; ties on the last day keep the newer version.
+	series = s.GetVersionSeries(10, 1)
+	if len(series.Versions) != 1 || series.Versions[0].Version != "4.40" {
+		t.Fatalf("limited versions = %+v, want just 4.40", series.Versions)
+	}
+	if series.Other == nil || series.Other.Count != 2 {
+		t.Fatalf("other = %+v, want 2 versions", series.Other)
+	}
+	if !equal(series.Other.Clusters, []uint64{1, 1, 0, 0, 0, 0, 1, 1, 1, 1}) {
+		t.Errorf("other = %v, want 3.97+4.30 summed per day", series.Other.Clusters)
+	}
+	if series.TotalClusters != 2 {
+		t.Errorf("limit changed total_clusters: %d, want 2", series.TotalClusters)
+	}
+}
+
+func TestVersionLess(t *testing.T) {
+	// Ordered oldest to newest; every pair must compare in this order.
+	ordered := []string{"unknown", "3.97", "4.02", "4.30", "4.30-enterprise", "9.99", "10.02"}
+	for i := 0; i < len(ordered); i++ {
+		for j := i + 1; j < len(ordered); j++ {
+			if !versionLess(ordered[i], ordered[j]) {
+				t.Errorf("versionLess(%q, %q) = false, want true", ordered[i], ordered[j])
+			}
+			if versionLess(ordered[j], ordered[i]) {
+				t.Errorf("versionLess(%q, %q) = true, want false", ordered[j], ordered[i])
+			}
+		}
+	}
+}