From 6f14be1138302ae1306c4a26f858f6d0b59a5b89 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 16 Jul 2026 16:58:45 -0700 Subject: [PATCH] stats: remote-mount bucket cache hit/miss metrics (#10352) Reads of remote-backed entries now record hit or miss in SeaweedFS_remote_cache_read_total{source,bucket,result} on the filer HTTP path and the S3 gateway, so cache effectiveness of mounted buckets can be graphed. Inline-content entries count as hits since they are served locally without chunks. The filer purges the per-bucket series when the bucket directory is deleted, so a standalone filer does not accumulate series across bucket delete/recreate churn. --- weed/filer/filer_delete_entry.go | 4 +++ weed/filer/filer_rename_test.go | 31 +++++++++++++++++++++++ weed/s3api/s3api_object_handlers.go | 7 +++++ weed/server/filer_server_handlers_read.go | 6 +++++ weed/stats/metrics.go | 23 +++++++++++++++++ weed/stats/metrics_names.go | 6 +++++ 6 files changed, 77 insertions(+) create mode 100644 weed/filer/filer_rename_test.go diff --git a/weed/filer/filer_delete_entry.go b/weed/filer/filer_delete_entry.go index 8ede0826b..646caf5c9 100644 --- a/weed/filer/filer_delete_entry.go +++ b/weed/filer/filer_delete_entry.go @@ -7,6 +7,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/util" ) @@ -63,6 +64,9 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR if isDeleteCollection { collectionName := entry.Name() f.DoDeleteCollection(collectionName) + // drop bucket-labeled series held by this process; the S3 gateway + // only cleans its own registry + stats.DeleteBucketMetrics(collectionName) } return nil diff --git a/weed/filer/filer_rename_test.go b/weed/filer/filer_rename_test.go new file mode 100644 index 000000000..d7096963f --- /dev/null +++ b/weed/filer/filer_rename_test.go @@ -0,0 +1,31 @@ +package filer + +import ( + "testing" + + "github.com/seaweedfs/seaweedfs/weed/util" +) + +func TestDetectBucket(t *testing.T) { + tests := []struct { + name string + fullPath string + bucketsPath string + want string + }{ + {"object in bucket", "/buckets/mybucket/a/b.txt", "/buckets", "mybucket"}, + {"bucket root", "/buckets/mybucket", "/buckets", "mybucket"}, + {"not under buckets path", "/other/path/x", "/buckets", ""}, + {"buckets path itself", "/buckets", "/buckets", ""}, + {"custom buckets path", "/data/buckets/mybucket/x", "/data/buckets", "mybucket"}, + {"nested object", "/buckets/b/deep/nested/key", "/buckets", "b"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := &Filer{DirBucketsPath: tt.bucketsPath} + if got := f.DetectBucket(util.FullPath(tt.fullPath)); got != tt.want { + t.Errorf("DetectBucket(%q) with buckets path %q = %q, want %q", tt.fullPath, tt.bucketsPath, got, tt.want) + } + }) + } +} diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.go index 887a9f2a4..3c9427fa4 100644 --- a/weed/s3api/s3api_object_handlers.go +++ b/weed/s3api/s3api_object_handlers.go @@ -25,6 +25,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "github.com/seaweedfs/seaweedfs/weed/stats" util_http "github.com/seaweedfs/seaweedfs/weed/util/http" "github.com/seaweedfs/seaweedfs/weed/glog" @@ -1189,6 +1190,12 @@ func (s3a *S3ApiServer) createLookupFileIdFunction() func(context.Context, strin // streamFromVolumeServersWithSSE handles streaming with inline SSE decryption func (s3a *S3ApiServer) streamFromVolumeServersWithSSE(w http.ResponseWriter, r *http.Request, entry *filer_pb.Entry, sseType string, bucket, object, versionId string) error { + if entry.RemoteEntry != nil && entry.RemoteEntry.RemoteSize > 0 { + // inline content is served locally without chunks + hit := !entry.IsInRemoteOnly() || len(entry.Content) > 0 + stats.RecordRemoteCacheRead(stats.RemoteCacheSourceS3, bucket, hit) + } + // If not encrypted, use fast path without decryption if sseType == "" || sseType == "None" { return s3a.streamFromVolumeServers(w, r, entry, sseType, bucket, object, versionId) diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 62e65b0ef..48c7342a7 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -193,6 +193,12 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) return } + if entry.Remote != nil && entry.Remote.RemoteSize > 0 { + // inline content is served locally without chunks + hit := !entry.IsInRemoteOnly() || len(entry.Content) > 0 + stats.RecordRemoteCacheRead(stats.RemoteCacheSourceFiler, fs.filer.DetectBucket(entry.FullPath), hit) + } + ProcessRangeRequest(r, w, totalSize, mimeType, func(offset int64, size int64) (filer.DoStreamContent, error) { if offset+size <= int64(len(entry.Content)) { return func(writer io.Writer) error { diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go index 0db62c38e..37e75590f 100644 --- a/weed/stats/metrics.go +++ b/weed/stats/metrics.go @@ -52,6 +52,7 @@ const ( subsystemS3 = "s3" subsystemS3Lifecycle = "s3_lifecycle" subsystemAdmin = "admin" + subsystemRemote = "remote" ) var bucketLastActiveTsNs map[string]int64 = map[string]int64{} @@ -660,6 +661,14 @@ var ( Help: "Whether each S3 bucket is read-only (1) or writable (0), e.g. after exceeding its quota.", }, []string{"bucket"}) + RemoteCacheReadCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: subsystemRemote, + Name: "cache_read_total", + Help: "Remote-mount object read attempts by source, bucket and cache result. A cold object retried before caching completes records a miss per attempt; paths outside the buckets folder use bucket \"_other\".", + }, []string{"source", "bucket", "result"}) + UploadErrorCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: Namespace, @@ -927,6 +936,8 @@ func init() { Gather.MustRegister(S3BucketQuotaBytesGauge) Gather.MustRegister(S3BucketReadOnlyGauge) + Gather.MustRegister(RemoteCacheReadCounter) + Gather.MustRegister(S3LifecycleDispatchCounter) Gather.MustRegister(S3LifecycleScheduleDepthGauge) Gather.MustRegister(S3LifecycleCursorMinTsNs) @@ -1004,6 +1015,17 @@ func RecordBucketActiveTime(bucket string) { bucketLastActiveLock.Unlock() } +func RecordRemoteCacheRead(source, bucket string, hit bool) { + if bucket == "" { + bucket = "_other" + } + result := RemoteCacheResultMiss + if hit { + result = RemoteCacheResultHit + } + RemoteCacheReadCounter.WithLabelValues(source, bucket, result).Inc() +} + func DeleteBucketMetrics(bucket string) { bucketLastActiveLock.Lock() delete(bucketLastActiveTsNs, bucket) @@ -1025,6 +1047,7 @@ func DeleteBucketMetrics(bucket string) { c += S3LifecycleDispatchCounter.DeletePartialMatch(labels) c += S3LifecycleBootstrapDispatchCounter.DeletePartialMatch(labels) c += S3LifecycleMetadataOnlyCounter.DeletePartialMatch(labels) + c += RemoteCacheReadCounter.DeletePartialMatch(labels) glog.V(0).Infof("delete bucket metrics, %s: %d", bucket, c) } diff --git a/weed/stats/metrics_names.go b/weed/stats/metrics_names.go index c871ef5d2..956e8df8c 100644 --- a/weed/stats/metrics_names.go +++ b/weed/stats/metrics_names.go @@ -81,4 +81,10 @@ const ( FailureContextCancelled = "context_cancelled" // FailureServerError is the failure reason label for generic server-side errors. FailureServerError = "server_error" + + // remote-mount cache read labels + RemoteCacheSourceFiler = "filer" + RemoteCacheSourceS3 = "s3" + RemoteCacheResultHit = "hit" + RemoteCacheResultMiss = "miss" )