From 1391a85a20b3f47740fff71bb36bc83f3ec15a7b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 14 Jun 2026 21:10:02 -0700 Subject: [PATCH] metrics: add per-bucket S3 panels and volume slot utilization to dashboard (#9969) * metrics: add per-bucket S3 panels and volume slot utilization to dashboard The S3 Gateway/Buckets rows sliced requests only by type, never by bucket, though SeaweedFS_s3_request_total and the request/TTFB histograms all carry a bucket label. Add a Bucket template variable and four per-bucket panels to the S3 Buckets row (request rate, response codes, request p95, TTFB p95), plus a Volume Slots Utilization panel (volumes/max_volumes) to the Volume Servers row. * metrics: scope bucket variable to cluster and guard slot utilization divide-by-zero Source the Bucket variable from SeaweedFS_s3_bucket_size_bytes (a gauge emitted for every bucket, including idle ones) scoped to the selected cluster, so the dropdown lists all buckets in the cluster rather than only those that received requests. Wrap the volume slot utilization denominator in clamp_min(..., 1) to avoid +Inf/NaN when max_volumes is briefly zero or absent. --- other/metrics/grafana_seaweedfs.json | 507 +++++++++++++++++++++++++++ 1 file changed, 507 insertions(+) diff --git a/other/metrics/grafana_seaweedfs.json b/other/metrics/grafana_seaweedfs.json index a8790d6c7..5c04a9992 100644 --- a/other/metrics/grafana_seaweedfs.json +++ b/other/metrics/grafana_seaweedfs.json @@ -3688,6 +3688,104 @@ } ], "pluginVersion": "10.3.1" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "Volume Slots Utilization", + "description": "Used volume slots as a percentage of max volumes, per volume server.", + "type": "timeseries", + "id": 205, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 94 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "100 * sum by (instance) (SeaweedFS_volumeServer_volumes{cluster=~\"$cluster\", type=\"volume\"}) / clamp_min(sum by (instance) (SeaweedFS_volumeServer_max_volumes{cluster=~\"$cluster\"}), 1)", + "range": true, + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "pluginVersion": "10.3.1" } ] }, @@ -6487,6 +6585,390 @@ } ], "pluginVersion": "10.3.1" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "S3 Requests/sec by Bucket", + "description": "Per-bucket S3 request rate.", + "type": "timeseries", + "id": 201, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 50 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "reqps", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (bucket) (rate(SeaweedFS_s3_request_total{cluster=~\"$cluster\", bucket=~\"$bucket\"}[$__rate_interval]))", + "range": true, + "refId": "A", + "legendFormat": "{{bucket}}" + } + ], + "pluginVersion": "10.3.1" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "S3 Response Codes by Bucket", + "description": "Per-bucket S3 responses by HTTP status code.", + "type": "timeseries", + "id": 202, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 50 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "reqps", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (bucket, code) (rate(SeaweedFS_s3_request_total{cluster=~\"$cluster\", bucket=~\"$bucket\"}[$__rate_interval]))", + "range": true, + "refId": "A", + "legendFormat": "{{bucket}} {{code}}" + } + ], + "pluginVersion": "10.3.1" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "S3 Request Duration p95 by Bucket", + "description": "Per-bucket p95 S3 request latency.", + "type": "timeseries", + "id": 203, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 58 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket{cluster=~\"$cluster\", bucket=~\"$bucket\"}[$__rate_interval])) by (le, bucket))", + "range": true, + "refId": "A", + "legendFormat": "{{bucket}}" + } + ], + "pluginVersion": "10.3.1" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "title": "S3 Time-to-First-Byte p95 by Bucket", + "description": "Per-bucket p95 S3 time to first byte.", + "type": "timeseries", + "id": 204, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 58 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "ms", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_time_to_first_byte_millisecond_bucket{cluster=~\"$cluster\", bucket=~\"$bucket\"}[$__rate_interval])) by (le, bucket))", + "range": true, + "refId": "A", + "legendFormat": "{{bucket}}" + } + ], + "pluginVersion": "10.3.1" } ] }, @@ -8628,6 +9110,31 @@ "skipUrlSync": false, "sort": 1, "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(SeaweedFS_s3_bucket_size_bytes{cluster=~\"$cluster\"}, bucket)", + "hide": 0, + "includeAll": true, + "allValue": ".*", + "label": "Bucket", + "multi": true, + "name": "bucket", + "options": [], + "refresh": 2, + "query": { + "qryType": 1, + "query": "label_values(SeaweedFS_s3_bucket_size_bytes{cluster=~\"$cluster\"}, bucket)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" } ] },