From 63ac260bd5d4be4db24b2ab5a890c364234ee4dc Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 1 Jul 2022 13:18:39 -0700 Subject: [PATCH] Simplify Prometheus metrics gather (#15210) --- cmd/metrics-v2.go | 94 ++++++++++++++++++++++++++++++++--------------- cmd/metrics.go | 65 +++++++++++++++++++++++--------- 2 files changed, 111 insertions(+), 48 deletions(-) diff --git a/cmd/metrics-v2.go b/cmd/metrics-v2.go index 0aa93e287..e3ec23845 100644 --- a/cmd/metrics-v2.go +++ b/cmd/metrics-v2.go @@ -31,8 +31,8 @@ import ( "github.com/minio/minio/internal/bucket/lifecycle" "github.com/minio/minio/internal/logger" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" "github.com/prometheus/procfs" ) @@ -2130,55 +2130,89 @@ func metricsServerHandler() http.Handler { registry := prometheus.NewRegistry() // Report all other metrics - err := registry.Register(clusterCollector) - if err != nil { - logger.CriticalIf(GlobalContext, err) - } + logger.CriticalIf(GlobalContext, registry.Register(clusterCollector)) // DefaultGatherers include golang metrics and process metrics. gatherers := prometheus.Gatherers{ registry, } - // Delegate http serving to Prometheus client library, which will call collector.Collect. - return promhttp.InstrumentMetricHandler( - registry, - promhttp.HandlerFor(gatherers, - promhttp.HandlerOpts{ - ErrorHandling: promhttp.ContinueOnError, - }), - ) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + tc, ok := r.Context().Value(contextTraceReqKey).(*traceCtxt) + if ok { + tc.funcName = "handler.MetricsCluster" + tc.responseRecorder.LogErrBody = true + } + + mfs, err := gatherers.Gather() + if err != nil { + if len(mfs) == 0 { + writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), err), r.URL) + return + } + } + + contentType := expfmt.Negotiate(r.Header) + w.Header().Set("Content-Type", string(contentType)) + + enc := expfmt.NewEncoder(w, contentType) + for _, mf := range mfs { + if err := enc.Encode(mf); err != nil { + logger.LogIf(r.Context(), err) + return + } + } + if closer, ok := enc.(expfmt.Closer); ok { + closer.Close() + } + }) } func metricsNodeHandler() http.Handler { registry := prometheus.NewRegistry() - err := registry.Register(nodeCollector) - if err != nil { - logger.CriticalIf(GlobalContext, err) - } - err = registry.Register(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{ + logger.CriticalIf(GlobalContext, registry.Register(nodeCollector)) + if err := registry.Register(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{ Namespace: minioNamespace, ReportErrors: true, - })) - if err != nil { + })); err != nil { logger.CriticalIf(GlobalContext, err) } - err = registry.Register(prometheus.NewGoCollector()) - if err != nil { + if err := registry.Register(prometheus.NewGoCollector()); err != nil { logger.CriticalIf(GlobalContext, err) } gatherers := prometheus.Gatherers{ registry, } - // Delegate http serving to Prometheus client library, which will call collector.Collect. - return promhttp.InstrumentMetricHandler( - registry, - promhttp.HandlerFor(gatherers, - promhttp.HandlerOpts{ - ErrorHandling: promhttp.ContinueOnError, - }), - ) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + tc, ok := r.Context().Value(contextTraceReqKey).(*traceCtxt) + if ok { + tc.funcName = "handler.MetricsNode" + tc.responseRecorder.LogErrBody = true + } + + mfs, err := gatherers.Gather() + if err != nil { + if len(mfs) == 0 { + writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), err), r.URL) + return + } + } + + contentType := expfmt.Negotiate(r.Header) + w.Header().Set("Content-Type", string(contentType)) + + enc := expfmt.NewEncoder(w, contentType) + for _, mf := range mfs { + if err := enc.Encode(mf); err != nil { + logger.LogIf(r.Context(), err) + return + } + } + if closer, ok := enc.(expfmt.Closer); ok { + closer.Close() + } + }) } func toSnake(camel string) (snake string) { diff --git a/cmd/metrics.go b/cmd/metrics.go index 80cd815e3..6e663ea9b 100644 --- a/cmd/metrics.go +++ b/cmd/metrics.go @@ -26,7 +26,7 @@ import ( "github.com/minio/minio/internal/logger" iampolicy "github.com/minio/pkg/iam/policy" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/common/expfmt" ) var ( @@ -648,35 +648,59 @@ func storageMetricsPrometheus(ch chan<- prometheus.Metric) { func metricsHandler() http.Handler { registry := prometheus.NewRegistry() - err := registry.Register(minioVersionInfo) - logger.LogIf(GlobalContext, err) + logger.CriticalIf(GlobalContext, registry.Register(minioVersionInfo)) - err = registry.Register(httpRequestsDuration) - logger.LogIf(GlobalContext, err) - - err = registry.Register(newMinioCollector()) - logger.LogIf(GlobalContext, err) + logger.CriticalIf(GlobalContext, registry.Register(newMinioCollector())) gatherers := prometheus.Gatherers{ prometheus.DefaultGatherer, registry, } - // Delegate http serving to Prometheus client library, which will call collector.Collect. - return promhttp.InstrumentMetricHandler( - registry, - promhttp.HandlerFor(gatherers, - promhttp.HandlerOpts{ - ErrorHandling: promhttp.ContinueOnError, - }), - ) + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + tc, ok := r.Context().Value(contextTraceReqKey).(*traceCtxt) + if ok { + tc.funcName = "handler.MetricsLegacy" + tc.responseRecorder.LogErrBody = true + } + + mfs, err := gatherers.Gather() + if err != nil { + if len(mfs) == 0 { + writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), err), r.URL) + return + } + } + + contentType := expfmt.Negotiate(r.Header) + w.Header().Set("Content-Type", string(contentType)) + + enc := expfmt.NewEncoder(w, contentType) + for _, mf := range mfs { + if err := enc.Encode(mf); err != nil { + logger.LogIf(r.Context(), err) + return + } + } + if closer, ok := enc.(expfmt.Closer); ok { + closer.Close() + } + }) } // AuthMiddleware checks if the bearer token is valid and authorized. func AuthMiddleware(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + tc, ok := r.Context().Value(contextTraceReqKey).(*traceCtxt) + claims, groups, owner, authErr := metricsRequestAuthenticate(r) if authErr != nil || !claims.VerifyIssuer("prometheus", true) { - w.WriteHeader(http.StatusForbidden) + if ok { + tc.funcName = "handler.MetricsAuth" + tc.responseRecorder.LogErrBody = true + } + + writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), errAuthentication), r.URL) return } // For authenticated users apply IAM policy. @@ -688,7 +712,12 @@ func AuthMiddleware(h http.Handler) http.Handler { IsOwner: owner, Claims: claims.Map(), }) { - w.WriteHeader(http.StatusForbidden) + if ok { + tc.funcName = "handler.MetricsAuth" + tc.responseRecorder.LogErrBody = true + } + + writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), errAuthentication), r.URL) return } h.ServeHTTP(w, r)