stats: remote-mount bucket cache hit/miss metrics (#10352)

Reads of remote-backed entries now record hit or miss in
SeaweedFS_remote_cache_read_total{source,bucket,result} on the filer HTTP
path and the S3 gateway, so cache effectiveness of mounted buckets can be
graphed. Inline-content entries count as hits since they are served
locally without chunks. The filer purges the per-bucket series when the
bucket directory is deleted, so a standalone filer does not accumulate
series across bucket delete/recreate churn.

weed/filer/filer_delete_entry.go

@@ -7,6 +7,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
@@ -63,6 +64,9 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
 	if isDeleteCollection {
 		collectionName := entry.Name()
 		f.DoDeleteCollection(collectionName)
+		// drop bucket-labeled series held by this process; the S3 gateway
+		// only cleans its own registry
+		stats.DeleteBucketMetrics(collectionName)
 	}
 
 	return nil

weed/filer/filer_rename_test.go

@@ -0,0 +1,31 @@
+package filer
+
+import (
+	"testing"
+
+	"github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+func TestDetectBucket(t *testing.T) {
+	tests := []struct {
+		name        string
+		fullPath    string
+		bucketsPath string
+		want        string
+	}{
+		{"object in bucket", "/buckets/mybucket/a/b.txt", "/buckets", "mybucket"},
+		{"bucket root", "/buckets/mybucket", "/buckets", "mybucket"},
+		{"not under buckets path", "/other/path/x", "/buckets", ""},
+		{"buckets path itself", "/buckets", "/buckets", ""},
+		{"custom buckets path", "/data/buckets/mybucket/x", "/data/buckets", "mybucket"},
+		{"nested object", "/buckets/b/deep/nested/key", "/buckets", "b"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			f := &Filer{DirBucketsPath: tt.bucketsPath}
+			if got := f.DetectBucket(util.FullPath(tt.fullPath)); got != tt.want {
+				t.Errorf("DetectBucket(%q) with buckets path %q = %q, want %q", tt.fullPath, tt.bucketsPath, got, tt.want)
+			}
+		})
+	}
+}

weed/s3api/s3api_object_handlers.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
 	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -1189,6 +1190,12 @@ func (s3a *S3ApiServer) createLookupFileIdFunction() func(context.Context, strin
 
 // streamFromVolumeServersWithSSE handles streaming with inline SSE decryption
 func (s3a *S3ApiServer) streamFromVolumeServersWithSSE(w http.ResponseWriter, r *http.Request, entry *filer_pb.Entry, sseType string, bucket, object, versionId string) error {
+	if entry.RemoteEntry != nil && entry.RemoteEntry.RemoteSize > 0 {
+		// inline content is served locally without chunks
+		hit := !entry.IsInRemoteOnly() || len(entry.Content) > 0
+		stats.RecordRemoteCacheRead(stats.RemoteCacheSourceS3, bucket, hit)
+	}
+
 	// If not encrypted, use fast path without decryption
 	if sseType == "" || sseType == "None" {
 		return s3a.streamFromVolumeServers(w, r, entry, sseType, bucket, object, versionId)

weed/server/filer_server_handlers_read.go

@@ -193,6 +193,12 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
+	if entry.Remote != nil && entry.Remote.RemoteSize > 0 {
+		// inline content is served locally without chunks
+		hit := !entry.IsInRemoteOnly() || len(entry.Content) > 0
+		stats.RecordRemoteCacheRead(stats.RemoteCacheSourceFiler, fs.filer.DetectBucket(entry.FullPath), hit)
+	}
+
 	ProcessRangeRequest(r, w, totalSize, mimeType, func(offset int64, size int64) (filer.DoStreamContent, error) {
 		if offset+size <= int64(len(entry.Content)) {
 			return func(writer io.Writer) error {

weed/stats/metrics.go

@@ -52,6 +52,7 @@ const (
 	subsystemS3           = "s3"
 	subsystemS3Lifecycle  = "s3_lifecycle"
 	subsystemAdmin        = "admin"
+	subsystemRemote       = "remote"
 )
 
 var bucketLastActiveTsNs map[string]int64 = map[string]int64{}
@@ -660,6 +661,14 @@ var (
 			Help:      "Whether each S3 bucket is read-only (1) or writable (0), e.g. after exceeding its quota.",
 		}, []string{"bucket"})
 
+	RemoteCacheReadCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: Namespace,
+			Subsystem: subsystemRemote,
+			Name:      "cache_read_total",
+			Help:      "Remote-mount object read attempts by source, bucket and cache result. A cold object retried before caching completes records a miss per attempt; paths outside the buckets folder use bucket \"_other\".",
+		}, []string{"source", "bucket", "result"})
+
 	UploadErrorCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: Namespace,
@@ -927,6 +936,8 @@ func init() {
 	Gather.MustRegister(S3BucketQuotaBytesGauge)
 	Gather.MustRegister(S3BucketReadOnlyGauge)
 
+	Gather.MustRegister(RemoteCacheReadCounter)
+
 	Gather.MustRegister(S3LifecycleDispatchCounter)
 	Gather.MustRegister(S3LifecycleScheduleDepthGauge)
 	Gather.MustRegister(S3LifecycleCursorMinTsNs)
@@ -1004,6 +1015,17 @@ func RecordBucketActiveTime(bucket string) {
 	bucketLastActiveLock.Unlock()
 }
 
+func RecordRemoteCacheRead(source, bucket string, hit bool) {
+	if bucket == "" {
+		bucket = "_other"
+	}
+	result := RemoteCacheResultMiss
+	if hit {
+		result = RemoteCacheResultHit
+	}
+	RemoteCacheReadCounter.WithLabelValues(source, bucket, result).Inc()
+}
+
 func DeleteBucketMetrics(bucket string) {
 	bucketLastActiveLock.Lock()
 	delete(bucketLastActiveTsNs, bucket)
@@ -1025,6 +1047,7 @@ func DeleteBucketMetrics(bucket string) {
 	c += S3LifecycleDispatchCounter.DeletePartialMatch(labels)
 	c += S3LifecycleBootstrapDispatchCounter.DeletePartialMatch(labels)
 	c += S3LifecycleMetadataOnlyCounter.DeletePartialMatch(labels)
+	c += RemoteCacheReadCounter.DeletePartialMatch(labels)
 
 	glog.V(0).Infof("delete bucket metrics, %s: %d", bucket, c)
 }

weed/stats/metrics_names.go

@@ -81,4 +81,10 @@ const (
 	FailureContextCancelled = "context_cancelled"
 	// FailureServerError is the failure reason label for generic server-side errors.
 	FailureServerError = "server_error"
+
+	// remote-mount cache read labels
+	RemoteCacheSourceFiler = "filer"
+	RemoteCacheSourceS3    = "s3"
+	RemoteCacheResultHit   = "hit"
+	RemoteCacheResultMiss  = "miss"
 )