From a736ba1c21584d5b74cf41639656618bd25b7303 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 14 Jun 2026 21:43:03 -0700 Subject: [PATCH] filer: keep metadata-subscription send gauge fresh on idle heartbeat (#9966) * filer: keep metadata-subscription send gauge fresh on idle heartbeat last_send_timestamp_of_subscribe only advanced when a real matching metadata event was streamed to a subscriber. On a quiet path an idle but perfectly healthy subscriber therefore looked increasingly stale, and the dashboard panel rendered a large, misleading 'lag'. An idle heartbeat is a send too, so advance the gauge when one is emitted. Subscribers that opt into idle heartbeats (filer.sync) now report true freshness; the rest still show time since the last real event. Rename the dashboard panel 'Metadata Subscription Lag' -> 'Time Since Last Subscription Send' and clarify its description to match. * filer: guard nil option when advancing heartbeat gauge maybeSendIdleHeartbeat is unit-tested with a bare &FilerServer{} (nil option), so dereferencing fs.option.Host for the sourceFiler label panicked. Guard it: production always has option set; the test now gets an empty sourceFiler label instead of a nil-pointer panic. --- other/metrics/grafana_seaweedfs.json | 4 ++-- weed/server/filer_grpc_server_sub_meta.go | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/other/metrics/grafana_seaweedfs.json b/other/metrics/grafana_seaweedfs.json index 5c04a99922..7b15390a4d 100644 --- a/other/metrics/grafana_seaweedfs.json +++ b/other/metrics/grafana_seaweedfs.json @@ -5175,8 +5175,8 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "title": "Metadata Subscription Lag", - "description": "Seconds since the filer last sent a metadata-subscription event (per client/path); only present with active subscribers", + "title": "Time Since Last Subscription Send", + "description": "Seconds since the filer last sent to a metadata subscription (real event or idle heartbeat), per client/path. Low for healthy caught-up subscribers; clients that don't request idle heartbeats show time since the last real event.", "type": "timeseries", "id": 156, "gridPos": { diff --git a/weed/server/filer_grpc_server_sub_meta.go b/weed/server/filer_grpc_server_sub_meta.go index d1dffb3585..678f3629b4 100644 --- a/weed/server/filer_grpc_server_sub_meta.go +++ b/weed/server/filer_grpc_server_sub_meta.go @@ -564,6 +564,14 @@ func (fs *FilerServer) maybeSendIdleHeartbeat(req *filer_pb.SubscribeMetadataReq glog.V(0).Infof("=> idle heartbeat to %s: %v", req.ClientName, err) return lastHeartbeatNs } + // A heartbeat is a send too: advance the freshness gauge so an idle but + // healthy subscriber doesn't look stale. The gauge otherwise only moves on + // real matching events, which never arrive on a quiet path. + var sourceFiler string + if fs.option != nil { + sourceFiler = fs.option.Host.String() + } + stats.FilerServerLastSendTsOfSubscribeGauge.WithLabelValues(sourceFiler, req.ClientName, req.PathPrefix).Set(float64(now)) return now }