From fff243d4639f007d34b757879f283135e34c1979 Mon Sep 17 00:00:00 2001 From: Lisandro Pin Date: Wed, 22 Apr 2026 20:22:21 +0200 Subject: [PATCH] Export gRPC `file_{read,write}_failures` metrics on volume servers. (#9177) Allows to track overall R/W errors in real time through Prometheus. Will follow up with a PR for Seaweed's REST API. Co-authored-by: Lisandro Pin --- weed/server/volume_grpc_read_all.go | 2 + weed/server/volume_grpc_read_write.go | 88 +++++++++++++++++---------- weed/stats/metrics.go | 18 ++++++ 3 files changed, 75 insertions(+), 33 deletions(-) diff --git a/weed/server/volume_grpc_read_all.go b/weed/server/volume_grpc_read_all.go index 0b1aff2ff..190082a6f 100644 --- a/weed/server/volume_grpc_read_all.go +++ b/weed/server/volume_grpc_read_all.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/storage" "github.com/seaweedfs/seaweedfs/weed/storage/needle" ) @@ -12,6 +13,7 @@ func (vs *VolumeServer) ReadAllNeedles(req *volume_server_pb.ReadAllNeedlesReque for _, vid := range req.VolumeIds { if err := vs.streamReadOneVolume(needle.VolumeId(vid), stream); err != nil { + stats.VolumeServerFileReadFailures.Inc() return err } } diff --git a/weed/server/volume_grpc_read_write.go b/weed/server/volume_grpc_read_write.go index 1398075b1..7879090d6 100644 --- a/weed/server/volume_grpc_read_write.go +++ b/weed/server/volume_grpc_read_write.go @@ -5,20 +5,28 @@ import ( "fmt" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/types" ) func (vs *VolumeServer) ReadNeedleBlob(ctx context.Context, req *volume_server_pb.ReadNeedleBlobRequest) (resp *volume_server_pb.ReadNeedleBlobResponse, err error) { - resp = &volume_server_pb.ReadNeedleBlobResponse{} - v := vs.store.GetVolume(needle.VolumeId(req.VolumeId)) - if v == nil { - return nil, fmt.Errorf("not found volume id %d", req.VolumeId) - } + if err := func() error { + resp = &volume_server_pb.ReadNeedleBlobResponse{} + v := vs.store.GetVolume(needle.VolumeId(req.VolumeId)) + if v == nil { + return fmt.Errorf("not found volume id %d", req.VolumeId) + } - resp.NeedleBlob, err = v.ReadNeedleBlob(req.Offset, types.Size(req.Size)) - if err != nil { - return nil, fmt.Errorf("read needle blob offset %d size %d: %v", req.Offset, req.Size, err) + resp.NeedleBlob, err = v.ReadNeedleBlob(req.Offset, types.Size(req.Size)) + if err != nil { + return fmt.Errorf("read needle blob offset %d size %d: %v", req.Offset, req.Size, err) + } + + return nil + }(); err != nil { + stats.VolumeServerFileReadFailures.Inc() + return nil, err } return resp, nil @@ -28,29 +36,37 @@ func (vs *VolumeServer) ReadNeedleMeta(ctx context.Context, req *volume_server_p resp = &volume_server_pb.ReadNeedleMetaResponse{} volumeId := needle.VolumeId(req.VolumeId) - n := &needle.Needle{ - Id: types.NeedleId(req.NeedleId), - Flags: 0x08, - } - size := req.Size - offset := req.Offset + if err := func() error { + n := &needle.Needle{ + Id: types.NeedleId(req.NeedleId), + Flags: 0x08, + } + size := req.Size + offset := req.Offset - hasVolume := vs.store.HasVolume(volumeId) - if !hasVolume { - return nil, fmt.Errorf("not found volume id %d and read needle metadata at ec shards is not supported", req.VolumeId) - } - err = vs.store.ReadVolumeNeedleMetaAt(volumeId, n, offset, size) - if err != nil { + hasVolume := vs.store.HasVolume(volumeId) + if !hasVolume { + return fmt.Errorf("not found volume id %d and read needle metadata at ec shards is not supported", req.VolumeId) + } + err = vs.store.ReadVolumeNeedleMetaAt(volumeId, n, offset, size) + if err != nil { + return err + } + + resp.Cookie = uint32(n.Cookie) + resp.LastModified = n.LastModified + resp.Crc = n.Checksum.Value() + if n.HasTtl() { + resp.Ttl = n.Ttl.String() + } + resp.AppendAtNs = n.AppendAtNs + + return nil + }(); err != nil { + stats.VolumeServerFileReadFailures.Inc() return nil, err } - resp.Cookie = uint32(n.Cookie) - resp.LastModified = n.LastModified - resp.Crc = n.Checksum.Value() - if n.HasTtl() { - resp.Ttl = n.Ttl.String() - } - resp.AppendAtNs = n.AppendAtNs return resp, nil } @@ -61,13 +77,19 @@ func (vs *VolumeServer) WriteNeedleBlob(ctx context.Context, req *volume_server_ resp = &volume_server_pb.WriteNeedleBlobResponse{} - v := vs.store.GetVolume(needle.VolumeId(req.VolumeId)) - if v == nil { - return nil, fmt.Errorf("not found volume id %d", req.VolumeId) - } + if err := func() error { + v := vs.store.GetVolume(needle.VolumeId(req.VolumeId)) + if v == nil { + return fmt.Errorf("not found volume id %d", req.VolumeId) + } + if err = v.WriteNeedleBlob(types.NeedleId(req.NeedleId), req.NeedleBlob, types.Size(req.Size)); err != nil { + return fmt.Errorf("write blob needle %d size %d: %v", req.NeedleId, req.Size, err) + } - if err = v.WriteNeedleBlob(types.NeedleId(req.NeedleId), req.NeedleBlob, types.Size(req.Size)); err != nil { - return nil, fmt.Errorf("write blob needle %d size %d: %v", req.NeedleId, req.Size, err) + return nil + }(); err != nil { + stats.VolumeServerFileWriteFailures.Inc() + return nil, err } return resp, nil diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go index b2c056153..b7468b7f5 100644 --- a/weed/stats/metrics.go +++ b/weed/stats/metrics.go @@ -370,6 +370,22 @@ var ( Help: "Number of master server disconnections.", }, []string{"address"}) + VolumeServerFileReadFailures = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: "volumeServer", + Name: "file_read_failures", + Help: "Counter of overall failed file read requests from clients.", + }) + + VolumeServerFileWriteFailures = prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: Namespace, + Subsystem: "volumeServer", + Name: "file_write_failures", + Help: "Counter of overall failed file write requests from clients.", + }) + S3RequestCounter = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: Namespace, @@ -537,6 +553,8 @@ func init() { Gather.MustRegister(VolumeServerInFlightDownloadSize) Gather.MustRegister(VolumeServerInFlightUploadSize) Gather.MustRegister(VolumeServerMasterDisconnections) + Gather.MustRegister(VolumeServerFileReadFailures) + Gather.MustRegister(VolumeServerFileWriteFailures) Gather.MustRegister(S3RequestCounter) Gather.MustRegister(S3HandlerCounter)