s3, iam, volume, filer, master: add /healthz and /readyz health probes (#9738)

Adds standard Kubernetes liveness/readiness endpoints to all HTTP
servers that were missing them:

- S3:     adds /readyz (already had /healthz)
- IAM:    adds /healthz and /readyz (had none)
- Volume: adds /readyz (already had /healthz)
- Filer:  adds /readyz on default and readonly mux
- Master: adds /healthz and /readyz at root level
  (preserves existing /cluster/healthz)

All endpoints reuse existing health handlers or return 200 OK as a
minimal foundation. Future PRs can enhance /readyz with dependency
checks without breaking the contract.

Closes #9736

Co-authored-by: Mohamed Chorfa <mohamed.chorfa@thalesgroup.com>

weed/iamapi/iamapi_server.go

@@ -124,11 +124,23 @@ func (iama *IamApiServer) registerRouter(router *mux.Router) {
 
 	// apiRouter.Methods("GET").Path("/").HandlerFunc(track(s3a.iam.Auth(s3a.ListBucketsHandler, ACTION_ADMIN), "LIST"))
 	apiRouter.Methods(http.MethodPost).Path("/").HandlerFunc(iama.iam.Auth(iama.DoActions, ACTION_ADMIN))
-	//
+
+	// Health probes
+	apiRouter.Methods(http.MethodGet, http.MethodHead).Path("/healthz").HandlerFunc(iama.healthzHandler)
+	apiRouter.Methods(http.MethodGet, http.MethodHead).Path("/readyz").HandlerFunc(iama.readyzHandler)
+
 	// NotFound
 	apiRouter.NotFoundHandler = http.HandlerFunc(s3err.NotFoundHandler)
 }
 
+func (iama *IamApiServer) healthzHandler(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+}
+
+func (iama *IamApiServer) readyzHandler(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+}
+
 // Shutdown gracefully stops the IAM API server and releases resources.
 // It cancels the master client connection goroutine and closes gRPC connections.
 // This method is safe to call multiple times.

weed/s3api/s3api_server.go

@@ -698,9 +698,10 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
 	// plus REST-style endpoints for AWS CLI
 	s3a.registerS3TablesRoutes(apiRouter)
 
-	// Readiness Probe
+	// Health probes
 	apiRouter.Methods(http.MethodGet, http.MethodHead).Path("/status").HandlerFunc(s3a.StatusHandler)
 	apiRouter.Methods(http.MethodGet, http.MethodHead).Path("/healthz").HandlerFunc(s3a.StatusHandler)
+	apiRouter.Methods(http.MethodGet, http.MethodHead).Path("/readyz").HandlerFunc(s3a.StatusHandler)
 
 	// Object path pattern with (?s) flag to match newlines in object keys
 	const objectPath = "/{object:(?s).+}"

weed/server/filer_server.go

@@ -25,7 +25,6 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/util"
 
 	"github.com/seaweedfs/seaweedfs/weed/filer"
-	"github.com/seaweedfs/seaweedfs/weed/filer/posixlock"
 	_ "github.com/seaweedfs/seaweedfs/weed/filer/arangodb"
 	_ "github.com/seaweedfs/seaweedfs/weed/filer/cassandra"
 	_ "github.com/seaweedfs/seaweedfs/weed/filer/cassandra2"
@@ -39,6 +38,7 @@ import (
 	_ "github.com/seaweedfs/seaweedfs/weed/filer/mongodb"
 	_ "github.com/seaweedfs/seaweedfs/weed/filer/mysql"
 	_ "github.com/seaweedfs/seaweedfs/weed/filer/mysql2"
+	"github.com/seaweedfs/seaweedfs/weed/filer/posixlock"
 	_ "github.com/seaweedfs/seaweedfs/weed/filer/postgres"
 	_ "github.com/seaweedfs/seaweedfs/weed/filer/postgres2"
 	_ "github.com/seaweedfs/seaweedfs/weed/filer/redis"
@@ -245,6 +245,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
 	handleStaticResources(defaultMux)
 	if !option.DisableHttp {
 		defaultMux.HandleFunc("/healthz", requestIDMiddleware(fs.filerHealthzHandler))
+		defaultMux.HandleFunc("/readyz", requestIDMiddleware(fs.filerHealthzHandler))
 		// TUS resumable upload protocol handler
 		if option.TusBasePath != "" {
 			// Normalize TusPath to always have a leading slash and no trailing slash
@@ -268,6 +269,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
 	if defaultMux != readonlyMux {
 		handleStaticResources(readonlyMux)
 		readonlyMux.HandleFunc("/healthz", requestIDMiddleware(fs.filerHealthzHandler))
+		readonlyMux.HandleFunc("/readyz", requestIDMiddleware(fs.filerHealthzHandler))
 		readonlyMux.HandleFunc("/", fs.filerGuard.WhiteList(requestIDMiddleware(fs.readonlyFilerHandler)))
 	}
 

weed/server/master_server.go

@@ -169,6 +169,8 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se
 	ms.guard = security.NewGuard(append(ms.option.WhiteList, whiteList...), signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec)
 
 	handleStaticResources2(r)
+	r.HandleFunc("/healthz", requestIDMiddleware(ms.healthzHandler)).Methods(http.MethodGet, http.MethodHead)
+	r.HandleFunc("/readyz", requestIDMiddleware(ms.readyzHandler)).Methods(http.MethodGet, http.MethodHead)
 	r.HandleFunc("/", ms.proxyToLeader(requestIDMiddleware(ms.uiStatusHandler)))
 	r.HandleFunc("/ui/index.html", requestIDMiddleware(ms.uiStatusHandler))
 	if !ms.option.DisableHttp {
@@ -208,6 +210,31 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se
 	return ms
 }
 
+func (ms *MasterServer) healthzHandler(w http.ResponseWriter, r *http.Request) {
+	// Liveness: process is alive. Keep this fast and simple.
+	w.WriteHeader(http.StatusOK)
+}
+
+func (ms *MasterServer) readyzHandler(w http.ResponseWriter, r *http.Request) {
+	// Readiness: check we can serve traffic.
+	leader, err := ms.Topo.Leader()
+	if err != nil {
+		w.WriteHeader(http.StatusServiceUnavailable)
+		return
+	}
+	if ms.option.Master.Equals(leader) {
+		isLocked, err := ms.Topo.IsChildLocked()
+		if err != nil {
+			glog.Errorf("readyzHandler: %+v", err)
+		}
+		if isLocked {
+			w.WriteHeader(http.StatusLocked)
+			return
+		}
+	}
+	w.WriteHeader(http.StatusOK)
+}
+
 func (ms *MasterServer) SetRaftServer(raftServer *RaftServer) {
 	var raftServerName string
 

weed/server/volume_server.go

@@ -42,11 +42,11 @@ type VolumeServer struct {
 	currentMaster     pb.ServerAddress
 	currentMasterLock sync.RWMutex
 	pulsePeriod       time.Duration
-	dataCenter      string
-	rack            string
-	store           *storage.Store
-	guard           *security.Guard
-	grpcDialOption  grpc.DialOption
+	dataCenter        string
+	rack              string
+	store             *storage.Store
+	guard             *security.Guard
+	grpcDialOption    grpc.DialOption
 
 	needleMapKind                 storage.NeedleMapKind
 	ldbTimout                     int64
@@ -140,6 +140,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
 	handleStaticResources(adminMux)
 	adminMux.HandleFunc("/status", requestIDMiddleware(vs.statusHandler))
 	adminMux.HandleFunc("/healthz", requestIDMiddleware(vs.healthzHandler))
+	adminMux.HandleFunc("/readyz", requestIDMiddleware(vs.healthzHandler))
 	if signingKey == "" || enableUiAccess {
 		// only expose the volume server details for safe environments
 		adminMux.HandleFunc("/ui/index.html", requestIDMiddleware(vs.uiStatusHandler))