From 582b7268f5232007fdd350cf472e3b10321ddc70 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 11 Jun 2026 09:03:00 -0700 Subject: [PATCH] s3: export per-bucket quota and read-only state metrics (#9923) The quota enforcement loop already computes each bucket's configured quota and effective read-only flag every minute, but neither was visible to monitoring, so operators could not alert before a bucket flips read-only. Add two gauges next to the existing bucket size metrics: SeaweedFS_s3_bucket_quota_bytes configured quota; the series is only present while the quota is enabled, so size/quota utilization queries never divide by zero SeaweedFS_s3_bucket_read_only 1 when the bucket's location rule is read-only (over quota or manually locked), 0 otherwise Both are cleaned up with the other per-bucket gauges on bucket deletion and inactivity TTL. --- weed/s3api/bucket_size_metrics.go | 1 + weed/stats/metrics.go | 38 ++++++++++++++++++++++ weed/stats/metrics_bucket_quota_test.go | 42 +++++++++++++++++++++++++ 3 files changed, 81 insertions(+) create mode 100644 weed/stats/metrics_bucket_quota_test.go diff --git a/weed/s3api/bucket_size_metrics.go b/weed/s3api/bucket_size_metrics.go index b6881cb4d..7ab29c237 100644 --- a/weed/s3api/bucket_size_metrics.go +++ b/weed/s3api/bucket_size_metrics.go @@ -138,6 +138,7 @@ func (s3a *S3ApiServer) enforceBucketQuotas(ctx context.Context, buckets []*file } locPrefix := s3a.option.BucketsPath + "/" + bucket.Name + "/" readOnly, flipped := fc.ApplyBucketQuotaReadOnly(locPrefix, size, float64(bucket.Quota)) + stats.UpdateBucketQuotaMetrics(bucket.Name, float64(bucket.Quota), readOnly) if flipped { changed = true glog.V(0).Infof("bucket %s quota enforcement: readOnly=%v (size=%.0f quota=%d)", bucket.Name, readOnly, size, bucket.Quota) diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go index c75fae447..4c6451350 100644 --- a/weed/stats/metrics.go +++ b/weed/stats/metrics.go @@ -549,6 +549,22 @@ var ( Help: "Current number of objects in each S3 bucket (logical count, deduplicated across replicas).", }, []string{"bucket"}) + S3BucketQuotaBytesGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "s3", + Name: "bucket_quota_bytes", + Help: "Configured quota of each S3 bucket in bytes. Only present for buckets with an enabled quota.", + }, []string{"bucket"}) + + S3BucketReadOnlyGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: Namespace, + Subsystem: "s3", + Name: "bucket_read_only", + Help: "Whether each S3 bucket is read-only (1) or writable (0), e.g. after exceeding its quota.", + }, []string{"bucket"}) + UploadErrorCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: Namespace, @@ -805,6 +821,8 @@ func init() { Gather.MustRegister(S3BucketSizeBytesGauge) Gather.MustRegister(S3BucketPhysicalSizeBytesGauge) Gather.MustRegister(S3BucketObjectCountGauge) + Gather.MustRegister(S3BucketQuotaBytesGauge) + Gather.MustRegister(S3BucketReadOnlyGauge) Gather.MustRegister(S3LifecycleDispatchCounter) Gather.MustRegister(S3LifecycleScheduleDepthGauge) @@ -899,6 +917,8 @@ func DeleteBucketMetrics(bucket string) { c += S3BucketSizeBytesGauge.DeletePartialMatch(labels) c += S3BucketPhysicalSizeBytesGauge.DeletePartialMatch(labels) c += S3BucketObjectCountGauge.DeletePartialMatch(labels) + c += S3BucketQuotaBytesGauge.DeletePartialMatch(labels) + c += S3BucketReadOnlyGauge.DeletePartialMatch(labels) c += S3LifecycleDispatchCounter.DeletePartialMatch(labels) c += S3LifecycleBootstrapDispatchCounter.DeletePartialMatch(labels) c += S3LifecycleMetadataOnlyCounter.DeletePartialMatch(labels) @@ -946,6 +966,8 @@ func bucketMetricTTLControl() { c += S3BucketSizeBytesGauge.DeletePartialMatch(labels) c += S3BucketPhysicalSizeBytesGauge.DeletePartialMatch(labels) c += S3BucketObjectCountGauge.DeletePartialMatch(labels) + c += S3BucketQuotaBytesGauge.DeletePartialMatch(labels) + c += S3BucketReadOnlyGauge.DeletePartialMatch(labels) glog.V(0).Infof("delete inactive bucket metrics, %s: %d", bucket, c) } @@ -964,3 +986,19 @@ func UpdateBucketSizeMetrics(bucket string, logicalSize, physicalSize float64, o S3BucketObjectCountGauge.WithLabelValues(bucket).Set(objectCount) RecordBucketActiveTime(bucket) } + +// UpdateBucketQuotaMetrics updates the per-bucket quota gauges. A non-positive +// quota removes the quota series so utilization queries like +// bucket_size_bytes / bucket_quota_bytes only see enforced quotas. +func UpdateBucketQuotaMetrics(bucket string, quota float64, readOnly bool) { + if quota > 0 { + S3BucketQuotaBytesGauge.WithLabelValues(bucket).Set(quota) + } else { + S3BucketQuotaBytesGauge.DeleteLabelValues(bucket) + } + readOnlyValue := float64(0) + if readOnly { + readOnlyValue = 1 + } + S3BucketReadOnlyGauge.WithLabelValues(bucket).Set(readOnlyValue) +} diff --git a/weed/stats/metrics_bucket_quota_test.go b/weed/stats/metrics_bucket_quota_test.go new file mode 100644 index 000000000..1c73d2e3d --- /dev/null +++ b/weed/stats/metrics_bucket_quota_test.go @@ -0,0 +1,42 @@ +package stats + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestUpdateBucketQuotaMetrics(t *testing.T) { + bucket := "quota-test-bucket" + + UpdateBucketQuotaMetrics(bucket, 1024, true) + if got := testutil.ToFloat64(S3BucketQuotaBytesGauge.WithLabelValues(bucket)); got != 1024 { + t.Errorf("quota gauge: got %.0f, want 1024", got) + } + if got := testutil.ToFloat64(S3BucketReadOnlyGauge.WithLabelValues(bucket)); got != 1 { + t.Errorf("read-only gauge: got %.0f, want 1", got) + } + + UpdateBucketQuotaMetrics(bucket, 1024, false) + if got := testutil.ToFloat64(S3BucketReadOnlyGauge.WithLabelValues(bucket)); got != 0 { + t.Errorf("read-only gauge after clearing: got %.0f, want 0", got) + } + + // disabled quota (negative) removes the quota series but keeps reporting read-only state + UpdateBucketQuotaMetrics(bucket, -1024, true) + if n := testutil.CollectAndCount(S3BucketQuotaBytesGauge); n != 0 { + t.Errorf("quota series after disabling: got %d, want 0", n) + } + if got := testutil.ToFloat64(S3BucketReadOnlyGauge.WithLabelValues(bucket)); got != 1 { + t.Errorf("read-only gauge with disabled quota: got %.0f, want 1", got) + } + + UpdateBucketQuotaMetrics(bucket, 2048, true) + DeleteBucketMetrics(bucket) + if n := testutil.CollectAndCount(S3BucketQuotaBytesGauge); n != 0 { + t.Errorf("quota series after bucket deletion: got %d, want 0", n) + } + if n := testutil.CollectAndCount(S3BucketReadOnlyGauge); n != 0 { + t.Errorf("read-only series after bucket deletion: got %d, want 0", n) + } +}