s3: export per-bucket quota and read-only state metrics (#9923)

The quota enforcement loop already computes each bucket's configured
quota and effective read-only flag every minute, but neither was
visible to monitoring, so operators could not alert before a bucket
flips read-only.

Add two gauges next to the existing bucket size metrics:

  SeaweedFS_s3_bucket_quota_bytes  configured quota; the series is only
                                   present while the quota is enabled,
                                   so size/quota utilization queries
                                   never divide by zero
  SeaweedFS_s3_bucket_read_only    1 when the bucket's location rule is
                                   read-only (over quota or manually
                                   locked), 0 otherwise

Both are cleaned up with the other per-bucket gauges on bucket
deletion and inactivity TTL.

weed/s3api/bucket_size_metrics.go

@@ -138,6 +138,7 @@ func (s3a *S3ApiServer) enforceBucketQuotas(ctx context.Context, buckets []*file
 		}
 		locPrefix := s3a.option.BucketsPath + "/" + bucket.Name + "/"
 		readOnly, flipped := fc.ApplyBucketQuotaReadOnly(locPrefix, size, float64(bucket.Quota))
+		stats.UpdateBucketQuotaMetrics(bucket.Name, float64(bucket.Quota), readOnly)
 		if flipped {
 			changed = true
 			glog.V(0).Infof("bucket %s quota enforcement: readOnly=%v (size=%.0f quota=%d)", bucket.Name, readOnly, size, bucket.Quota)

weed/stats/metrics.go

@@ -549,6 +549,22 @@ var (
 			Help:      "Current number of objects in each S3 bucket (logical count, deduplicated across replicas).",
 		}, []string{"bucket"})
 
+	S3BucketQuotaBytesGauge = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: Namespace,
+			Subsystem: "s3",
+			Name:      "bucket_quota_bytes",
+			Help:      "Configured quota of each S3 bucket in bytes. Only present for buckets with an enabled quota.",
+		}, []string{"bucket"})
+
+	S3BucketReadOnlyGauge = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: Namespace,
+			Subsystem: "s3",
+			Name:      "bucket_read_only",
+			Help:      "Whether each S3 bucket is read-only (1) or writable (0), e.g. after exceeding its quota.",
+		}, []string{"bucket"})
+
 	UploadErrorCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: Namespace,
@@ -805,6 +821,8 @@ func init() {
 	Gather.MustRegister(S3BucketSizeBytesGauge)
 	Gather.MustRegister(S3BucketPhysicalSizeBytesGauge)
 	Gather.MustRegister(S3BucketObjectCountGauge)
+	Gather.MustRegister(S3BucketQuotaBytesGauge)
+	Gather.MustRegister(S3BucketReadOnlyGauge)
 
 	Gather.MustRegister(S3LifecycleDispatchCounter)
 	Gather.MustRegister(S3LifecycleScheduleDepthGauge)
@@ -899,6 +917,8 @@ func DeleteBucketMetrics(bucket string) {
 	c += S3BucketSizeBytesGauge.DeletePartialMatch(labels)
 	c += S3BucketPhysicalSizeBytesGauge.DeletePartialMatch(labels)
 	c += S3BucketObjectCountGauge.DeletePartialMatch(labels)
+	c += S3BucketQuotaBytesGauge.DeletePartialMatch(labels)
+	c += S3BucketReadOnlyGauge.DeletePartialMatch(labels)
 	c += S3LifecycleDispatchCounter.DeletePartialMatch(labels)
 	c += S3LifecycleBootstrapDispatchCounter.DeletePartialMatch(labels)
 	c += S3LifecycleMetadataOnlyCounter.DeletePartialMatch(labels)
@@ -946,6 +966,8 @@ func bucketMetricTTLControl() {
 			c += S3BucketSizeBytesGauge.DeletePartialMatch(labels)
 			c += S3BucketPhysicalSizeBytesGauge.DeletePartialMatch(labels)
 			c += S3BucketObjectCountGauge.DeletePartialMatch(labels)
+			c += S3BucketQuotaBytesGauge.DeletePartialMatch(labels)
+			c += S3BucketReadOnlyGauge.DeletePartialMatch(labels)
 			glog.V(0).Infof("delete inactive bucket metrics, %s: %d", bucket, c)
 		}
 
@@ -964,3 +986,19 @@ func UpdateBucketSizeMetrics(bucket string, logicalSize, physicalSize float64, o
 	S3BucketObjectCountGauge.WithLabelValues(bucket).Set(objectCount)
 	RecordBucketActiveTime(bucket)
 }
+
+// UpdateBucketQuotaMetrics updates the per-bucket quota gauges. A non-positive
+// quota removes the quota series so utilization queries like
+// bucket_size_bytes / bucket_quota_bytes only see enforced quotas.
+func UpdateBucketQuotaMetrics(bucket string, quota float64, readOnly bool) {
+	if quota > 0 {
+		S3BucketQuotaBytesGauge.WithLabelValues(bucket).Set(quota)
+	} else {
+		S3BucketQuotaBytesGauge.DeleteLabelValues(bucket)
+	}
+	readOnlyValue := float64(0)
+	if readOnly {
+		readOnlyValue = 1
+	}
+	S3BucketReadOnlyGauge.WithLabelValues(bucket).Set(readOnlyValue)
+}

weed/stats/metrics_bucket_quota_test.go

@@ -0,0 +1,42 @@
+package stats
+
+import (
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+func TestUpdateBucketQuotaMetrics(t *testing.T) {
+	bucket := "quota-test-bucket"
+
+	UpdateBucketQuotaMetrics(bucket, 1024, true)
+	if got := testutil.ToFloat64(S3BucketQuotaBytesGauge.WithLabelValues(bucket)); got != 1024 {
+		t.Errorf("quota gauge: got %.0f, want 1024", got)
+	}
+	if got := testutil.ToFloat64(S3BucketReadOnlyGauge.WithLabelValues(bucket)); got != 1 {
+		t.Errorf("read-only gauge: got %.0f, want 1", got)
+	}
+
+	UpdateBucketQuotaMetrics(bucket, 1024, false)
+	if got := testutil.ToFloat64(S3BucketReadOnlyGauge.WithLabelValues(bucket)); got != 0 {
+		t.Errorf("read-only gauge after clearing: got %.0f, want 0", got)
+	}
+
+	// disabled quota (negative) removes the quota series but keeps reporting read-only state
+	UpdateBucketQuotaMetrics(bucket, -1024, true)
+	if n := testutil.CollectAndCount(S3BucketQuotaBytesGauge); n != 0 {
+		t.Errorf("quota series after disabling: got %d, want 0", n)
+	}
+	if got := testutil.ToFloat64(S3BucketReadOnlyGauge.WithLabelValues(bucket)); got != 1 {
+		t.Errorf("read-only gauge with disabled quota: got %.0f, want 1", got)
+	}
+
+	UpdateBucketQuotaMetrics(bucket, 2048, true)
+	DeleteBucketMetrics(bucket)
+	if n := testutil.CollectAndCount(S3BucketQuotaBytesGauge); n != 0 {
+		t.Errorf("quota series after bucket deletion: got %d, want 0", n)
+	}
+	if n := testutil.CollectAndCount(S3BucketReadOnlyGauge); n != 0 {
+		t.Errorf("read-only series after bucket deletion: got %d, want 0", n)
+	}
+}