mirrors/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-08-21 22:56:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Export gRPC file_{read,write}_failures metrics on volume servers. (#9177)

Browse Source 
Allows to track overall R/W errors in real time through Prometheus.
Will follow up with a PR for Seaweed's REST API.

Co-authored-by: Lisandro Pin <lisandro.pin@proton.ch>
This commit is contained in:
Lisandro Pin
2026-04-22 11:22:21 -07:00
committed by GitHub
co-authored by Lisandro Pin
parent cb882ced46
commit fff243d463
3 changed files with 75 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
weed/server/volume_grpc_read_all.go
+2
View File
@@ -4,6 +4,7 @@ import (
"fmt"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
)
@@ -12,6 +13,7 @@ func (vs *VolumeServer) ReadAllNeedles(req *volume_server_pb.ReadAllNeedlesReque
for _, vid := range req.VolumeIds {
if err := vs.streamReadOneVolume(needle.VolumeId(vid), stream); err != nil {
stats.VolumeServerFileReadFailures.Inc()
return err
}
}
weed/server/volume_grpc_read_write.go
+55 -33
View File
@@ -5,20 +5,28 @@ import (
"fmt"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
)
func (vs *VolumeServer) ReadNeedleBlob(ctx context.Context, req *volume_server_pb.ReadNeedleBlobRequest) (resp *volume_server_pb.ReadNeedleBlobResponse, err error) {
resp = &volume_server_pb.ReadNeedleBlobResponse{}
v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
if v == nil {
return nil, fmt.Errorf("not found volume id %d", req.VolumeId)
}
if err := func() error {
resp = &volume_server_pb.ReadNeedleBlobResponse{}
v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
if v == nil {
return fmt.Errorf("not found volume id %d", req.VolumeId)
}
resp.NeedleBlob, err = v.ReadNeedleBlob(req.Offset, types.Size(req.Size))
if err != nil {
return nil, fmt.Errorf("read needle blob offset %d size %d: %v", req.Offset, req.Size, err)
resp.NeedleBlob, err = v.ReadNeedleBlob(req.Offset, types.Size(req.Size))
if err != nil {
return fmt.Errorf("read needle blob offset %d size %d: %v", req.Offset, req.Size, err)
}
return nil
}(); err != nil {
stats.VolumeServerFileReadFailures.Inc()
return nil, err
}
return resp, nil
@@ -28,29 +36,37 @@ func (vs *VolumeServer) ReadNeedleMeta(ctx context.Context, req *volume_server_p
resp = &volume_server_pb.ReadNeedleMetaResponse{}
volumeId := needle.VolumeId(req.VolumeId)
n := &needle.Needle{
Id: types.NeedleId(req.NeedleId),
Flags: 0x08,
}
size := req.Size
offset := req.Offset
if err := func() error {
n := &needle.Needle{
Id: types.NeedleId(req.NeedleId),
Flags: 0x08,
}
size := req.Size
offset := req.Offset
hasVolume := vs.store.HasVolume(volumeId)
if !hasVolume {
return nil, fmt.Errorf("not found volume id %d and read needle metadata at ec shards is not supported", req.VolumeId)
}
err = vs.store.ReadVolumeNeedleMetaAt(volumeId, n, offset, size)
if err != nil {
hasVolume := vs.store.HasVolume(volumeId)
if !hasVolume {
return fmt.Errorf("not found volume id %d and read needle metadata at ec shards is not supported", req.VolumeId)
}
err = vs.store.ReadVolumeNeedleMetaAt(volumeId, n, offset, size)
if err != nil {
return err
}
resp.Cookie = uint32(n.Cookie)
resp.LastModified = n.LastModified
resp.Crc = n.Checksum.Value()
if n.HasTtl() {
resp.Ttl = n.Ttl.String()
}
resp.AppendAtNs = n.AppendAtNs
return nil
}(); err != nil {
stats.VolumeServerFileReadFailures.Inc()
return nil, err
}
resp.Cookie = uint32(n.Cookie)
resp.LastModified = n.LastModified
resp.Crc = n.Checksum.Value()
if n.HasTtl() {
resp.Ttl = n.Ttl.String()
}
resp.AppendAtNs = n.AppendAtNs
return resp, nil
}
@@ -61,13 +77,19 @@ func (vs *VolumeServer) WriteNeedleBlob(ctx context.Context, req *volume_server_
resp = &volume_server_pb.WriteNeedleBlobResponse{}
v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
if v == nil {
return nil, fmt.Errorf("not found volume id %d", req.VolumeId)
}
if err := func() error {
v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
if v == nil {
return fmt.Errorf("not found volume id %d", req.VolumeId)
}
if err = v.WriteNeedleBlob(types.NeedleId(req.NeedleId), req.NeedleBlob, types.Size(req.Size)); err != nil {
return fmt.Errorf("write blob needle %d size %d: %v", req.NeedleId, req.Size, err)
}
if err = v.WriteNeedleBlob(types.NeedleId(req.NeedleId), req.NeedleBlob, types.Size(req.Size)); err != nil {
return nil, fmt.Errorf("write blob needle %d size %d: %v", req.NeedleId, req.Size, err)
return nil
}(); err != nil {
stats.VolumeServerFileWriteFailures.Inc()
return nil, err
}
return resp, nil
weed/stats/metrics.go
+18
View File
@@ -370,6 +370,22 @@ var (
Help: "Number of master server disconnections.",
}, []string{"address"})
VolumeServerFileReadFailures = prometheus.NewCounter(
prometheus.CounterOpts{
Namespace: Namespace,
Subsystem: "volumeServer",
Name: "file_read_failures",
Help: "Counter of overall failed file read requests from clients.",
})
VolumeServerFileWriteFailures = prometheus.NewCounter(
prometheus.CounterOpts{
Namespace: Namespace,
Subsystem: "volumeServer",
Name: "file_write_failures",
Help: "Counter of overall failed file write requests from clients.",
})
S3RequestCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: Namespace,
@@ -537,6 +553,8 @@ func init() {
Gather.MustRegister(VolumeServerInFlightDownloadSize)
Gather.MustRegister(VolumeServerInFlightUploadSize)
Gather.MustRegister(VolumeServerMasterDisconnections)
Gather.MustRegister(VolumeServerFileReadFailures)
Gather.MustRegister(VolumeServerFileWriteFailures)
Gather.MustRegister(S3RequestCounter)
Gather.MustRegister(S3HandlerCounter)