Export master_disconnections metrics on volume servers. (#9104)

This allows to track connection issues and master failovers in real time via Prometheus metrics.

Co-authored-by: Lisandro Pin <lisandro.pin@proton.ch>

weed/server/volume_grpc_client_to_master.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/stats"
 
 	"google.golang.org/grpc"
 
@@ -84,6 +85,7 @@ func (vs *VolumeServer) heartbeat() {
 					time.Sleep(vs.pulsePeriod)
 				}
 
+				stats.VolumeServerMasterDisconnections.WithLabelValues(master.String()).Inc()
 				newLeader = ""
 				vs.store.MasterAddress = ""
 			} else {

weed/stats/metrics.go

@@ -362,6 +362,14 @@ var (
 			Help:      "In flight total upload size.",
 		})
 
+	VolumeServerMasterDisconnections = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: Namespace,
+			Subsystem: "volumeServer",
+			Name:      "master_disconnections",
+			Help:      "Number of master server disconnections.",
+		}, []string{"address"})
+
 	S3RequestCounter = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: Namespace,
@@ -528,6 +536,7 @@ func init() {
 	Gather.MustRegister(VolumeServerConcurrentUploadLimit)
 	Gather.MustRegister(VolumeServerInFlightDownloadSize)
 	Gather.MustRegister(VolumeServerInFlightUploadSize)
+	Gather.MustRegister(VolumeServerMasterDisconnections)
 
 	Gather.MustRegister(S3RequestCounter)
 	Gather.MustRegister(S3HandlerCounter)