From 10c4ab3e33f8341e36b6e0960c6276b0d9affb72 Mon Sep 17 00:00:00 2001 From: Mohamed Chorfa Date: Fri, 29 May 2026 23:45:03 -0400 Subject: [PATCH] s3, iam, volume, filer, master: add /healthz and /readyz health probes (#9738) Adds standard Kubernetes liveness/readiness endpoints to all HTTP servers that were missing them: - S3: adds /readyz (already had /healthz) - IAM: adds /healthz and /readyz (had none) - Volume: adds /readyz (already had /healthz) - Filer: adds /readyz on default and readonly mux - Master: adds /healthz and /readyz at root level (preserves existing /cluster/healthz) All endpoints reuse existing health handlers or return 200 OK as a minimal foundation. Future PRs can enhance /readyz with dependency checks without breaking the contract. Closes #9736 Co-authored-by: Mohamed Chorfa --- weed/iamapi/iamapi_server.go | 14 +++++++++++++- weed/s3api/s3api_server.go | 3 ++- weed/server/filer_server.go | 4 +++- weed/server/master_server.go | 27 +++++++++++++++++++++++++++ weed/server/volume_server.go | 11 ++++++----- 5 files changed, 51 insertions(+), 8 deletions(-) diff --git a/weed/iamapi/iamapi_server.go b/weed/iamapi/iamapi_server.go index f4a229c7a..49138956a 100644 --- a/weed/iamapi/iamapi_server.go +++ b/weed/iamapi/iamapi_server.go @@ -124,11 +124,23 @@ func (iama *IamApiServer) registerRouter(router *mux.Router) { // apiRouter.Methods("GET").Path("/").HandlerFunc(track(s3a.iam.Auth(s3a.ListBucketsHandler, ACTION_ADMIN), "LIST")) apiRouter.Methods(http.MethodPost).Path("/").HandlerFunc(iama.iam.Auth(iama.DoActions, ACTION_ADMIN)) - // + + // Health probes + apiRouter.Methods(http.MethodGet, http.MethodHead).Path("/healthz").HandlerFunc(iama.healthzHandler) + apiRouter.Methods(http.MethodGet, http.MethodHead).Path("/readyz").HandlerFunc(iama.readyzHandler) + // NotFound apiRouter.NotFoundHandler = http.HandlerFunc(s3err.NotFoundHandler) } +func (iama *IamApiServer) healthzHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +} + +func (iama *IamApiServer) readyzHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +} + // Shutdown gracefully stops the IAM API server and releases resources. // It cancels the master client connection goroutine and closes gRPC connections. // This method is safe to call multiple times. diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index fa03ac7c7..2477557bf 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -698,9 +698,10 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { // plus REST-style endpoints for AWS CLI s3a.registerS3TablesRoutes(apiRouter) - // Readiness Probe + // Health probes apiRouter.Methods(http.MethodGet, http.MethodHead).Path("/status").HandlerFunc(s3a.StatusHandler) apiRouter.Methods(http.MethodGet, http.MethodHead).Path("/healthz").HandlerFunc(s3a.StatusHandler) + apiRouter.Methods(http.MethodGet, http.MethodHead).Path("/readyz").HandlerFunc(s3a.StatusHandler) // Object path pattern with (?s) flag to match newlines in object keys const objectPath = "/{object:(?s).+}" diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 54d9b0a92..173c771d3 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -25,7 +25,6 @@ import ( "github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/filer" - "github.com/seaweedfs/seaweedfs/weed/filer/posixlock" _ "github.com/seaweedfs/seaweedfs/weed/filer/arangodb" _ "github.com/seaweedfs/seaweedfs/weed/filer/cassandra" _ "github.com/seaweedfs/seaweedfs/weed/filer/cassandra2" @@ -39,6 +38,7 @@ import ( _ "github.com/seaweedfs/seaweedfs/weed/filer/mongodb" _ "github.com/seaweedfs/seaweedfs/weed/filer/mysql" _ "github.com/seaweedfs/seaweedfs/weed/filer/mysql2" + "github.com/seaweedfs/seaweedfs/weed/filer/posixlock" _ "github.com/seaweedfs/seaweedfs/weed/filer/postgres" _ "github.com/seaweedfs/seaweedfs/weed/filer/postgres2" _ "github.com/seaweedfs/seaweedfs/weed/filer/redis" @@ -245,6 +245,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) handleStaticResources(defaultMux) if !option.DisableHttp { defaultMux.HandleFunc("/healthz", requestIDMiddleware(fs.filerHealthzHandler)) + defaultMux.HandleFunc("/readyz", requestIDMiddleware(fs.filerHealthzHandler)) // TUS resumable upload protocol handler if option.TusBasePath != "" { // Normalize TusPath to always have a leading slash and no trailing slash @@ -268,6 +269,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) if defaultMux != readonlyMux { handleStaticResources(readonlyMux) readonlyMux.HandleFunc("/healthz", requestIDMiddleware(fs.filerHealthzHandler)) + readonlyMux.HandleFunc("/readyz", requestIDMiddleware(fs.filerHealthzHandler)) readonlyMux.HandleFunc("/", fs.filerGuard.WhiteList(requestIDMiddleware(fs.readonlyFilerHandler))) } diff --git a/weed/server/master_server.go b/weed/server/master_server.go index bc4eb96f3..14e4ed33e 100644 --- a/weed/server/master_server.go +++ b/weed/server/master_server.go @@ -169,6 +169,8 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se ms.guard = security.NewGuard(append(ms.option.WhiteList, whiteList...), signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec) handleStaticResources2(r) + r.HandleFunc("/healthz", requestIDMiddleware(ms.healthzHandler)).Methods(http.MethodGet, http.MethodHead) + r.HandleFunc("/readyz", requestIDMiddleware(ms.readyzHandler)).Methods(http.MethodGet, http.MethodHead) r.HandleFunc("/", ms.proxyToLeader(requestIDMiddleware(ms.uiStatusHandler))) r.HandleFunc("/ui/index.html", requestIDMiddleware(ms.uiStatusHandler)) if !ms.option.DisableHttp { @@ -208,6 +210,31 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se return ms } +func (ms *MasterServer) healthzHandler(w http.ResponseWriter, r *http.Request) { + // Liveness: process is alive. Keep this fast and simple. + w.WriteHeader(http.StatusOK) +} + +func (ms *MasterServer) readyzHandler(w http.ResponseWriter, r *http.Request) { + // Readiness: check we can serve traffic. + leader, err := ms.Topo.Leader() + if err != nil { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + if ms.option.Master.Equals(leader) { + isLocked, err := ms.Topo.IsChildLocked() + if err != nil { + glog.Errorf("readyzHandler: %+v", err) + } + if isLocked { + w.WriteHeader(http.StatusLocked) + return + } + } + w.WriteHeader(http.StatusOK) +} + func (ms *MasterServer) SetRaftServer(raftServer *RaftServer) { var raftServerName string diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go index d53d2919a..8fa56ac5d 100644 --- a/weed/server/volume_server.go +++ b/weed/server/volume_server.go @@ -42,11 +42,11 @@ type VolumeServer struct { currentMaster pb.ServerAddress currentMasterLock sync.RWMutex pulsePeriod time.Duration - dataCenter string - rack string - store *storage.Store - guard *security.Guard - grpcDialOption grpc.DialOption + dataCenter string + rack string + store *storage.Store + guard *security.Guard + grpcDialOption grpc.DialOption needleMapKind storage.NeedleMapKind ldbTimout int64 @@ -140,6 +140,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, handleStaticResources(adminMux) adminMux.HandleFunc("/status", requestIDMiddleware(vs.statusHandler)) adminMux.HandleFunc("/healthz", requestIDMiddleware(vs.healthzHandler)) + adminMux.HandleFunc("/readyz", requestIDMiddleware(vs.healthzHandler)) if signingKey == "" || enableUiAccess { // only expose the volume server details for safe environments adminMux.HandleFunc("/ui/index.html", requestIDMiddleware(vs.uiStatusHandler))