From 9a4d003ac7e07fc65be4333d980b415c6acd7005 Mon Sep 17 00:00:00 2001 From: Aditya Manthramurthy Date: Mon, 4 Mar 2024 10:05:56 -0800 Subject: [PATCH] Add common middleware to S3 API handlers (#19171) The middleware sets up tracing, throttling, gzipped responses and collecting API stats. Additionally, this change updates the names of handler functions in metric labels to be the same as the name derived from Go lang reflection on the handler name. The metric api labels are now stored in memory the same as the handler name - they will be camelcased, e.g. `GetObject` instead of `getobject`. For compatibility, we lowercase the metric api label values when emitting the metrics. --- cmd/admin-router.go | 27 +- cmd/api-router.go | 462 ++++++++++++------ cmd/api-utils.go | 17 + cmd/http-stats.go | 20 +- cmd/metrics-v2.go | 29 +- cmd/metrics-v2_test.go | 2 +- cmd/metrics.go | 2 +- cmd/tier.go | 2 +- docs/bucket/replication/delete-replication.sh | 2 + .../setup_2site_existing_replication.sh | 2 + .../replication/setup_3site_replication.sh | 6 +- docs/bucket/replication/sio-error.sh | 4 +- 12 files changed, 372 insertions(+), 203 deletions(-) diff --git a/cmd/admin-router.go b/cmd/admin-router.go index 8bc44704a..b6e0fb2e1 100644 --- a/cmd/admin-router.go +++ b/cmd/admin-router.go @@ -19,9 +19,6 @@ package cmd import ( "net/http" - "reflect" - "runtime" - "strings" "github.com/klauspost/compress/gzhttp" "github.com/klauspost/compress/gzip" @@ -69,14 +66,6 @@ func (h hFlag) Has(flag hFlag) bool { return h&flag != 0 } -func getHandlerName(f http.HandlerFunc) string { - name := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name() - name = strings.TrimPrefix(name, "github.com/minio/minio/cmd.adminAPIHandlers.") - name = strings.TrimSuffix(name, "Handler-fm") - name = strings.TrimSuffix(name, "-fm") - return name -} - // adminMiddleware performs some common admin handler functionality for all // handlers: // @@ -86,9 +75,13 @@ func getHandlerName(f http.HandlerFunc) string { // // - sets up call to send AuditLog // -// Note that, while this is a middleware function (i.e. it takes a handler -// function and returns one), due to flags being passed based on required -// conditions, it is done per-"handler function registration" in the router. +// While this is a middleware function (i.e. it takes a handler function and +// returns one), due to flags being passed based on required conditions, it is +// done per-"handler function registration" in the router. +// +// The passed in handler function must be a method of `adminAPIHandlers` for the +// name displayed in logs and trace to be accurate. The name is extracted via +// reflection. // // When no flags are passed, gzip compression, http tracing of headers and // checking of object layer availability are all enabled. Use flags to modify @@ -100,10 +93,8 @@ func adminMiddleware(f http.HandlerFunc, flags ...hFlag) http.HandlerFunc { handlerFlags |= flag } - // Get name of the handler using reflection. NOTE: The passed in handler - // function must be a method of `adminAPIHandlers` for this extraction to - // work as expected. - handlerName := getHandlerName(f) + // Get name of the handler using reflection. + handlerName := getHandlerName(f, "adminAPIHandlers") var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { // Update request context with `logger.ReqInfo`. diff --git a/cmd/api-router.go b/cmd/api-router.go index 085ac151a..6943ed423 100644 --- a/cmd/api-router.go +++ b/cmd/api-router.go @@ -18,14 +18,11 @@ package cmd import ( - "compress/gzip" "net" "net/http" - "github.com/klauspost/compress/gzhttp" consoleapi "github.com/minio/console/api" xhttp "github.com/minio/minio/internal/http" - "github.com/minio/minio/internal/logger" "github.com/minio/mux" "github.com/minio/pkg/v2/wildcard" "github.com/rs/cors" @@ -171,6 +168,87 @@ var rejectedBucketAPIs = []rejectedAPI{ }, } +// Set of s3 handler options as bit flags. +type s3HFlag uint8 + +const ( + // when provided, disables Gzip compression. + noGZS3HFlag = 1 << iota + + // when provided, enables only tracing of headers. Otherwise, both headers + // and body are traced. + traceHdrsS3HFlag + + // when provided, disables throttling via the `maxClients` middleware. + noThrottleS3HFlag +) + +func (h s3HFlag) has(flag s3HFlag) bool { + // Use bitwise-AND and check if the result is non-zero. + return h&flag != 0 +} + +// s3APIMiddleware - performs some common handler functionality for S3 API +// handlers. +// +// It is set per-"handler function registration" in the router to allow for +// behavior modification via flags. +// +// This middleware always calls `collectAPIStats` to collect API stats. +// +// The passed in handler function must be a method of `objectAPIHandlers` for +// the name displayed in logs and trace to be accurate. The name is extracted +// via reflection. +// +// When **no** flags are passed, the behavior is to trace both headers and body, +// gzip the response and throttle the handler via `maxClients`. Each of these +// can be disabled via the corresponding `s3HFlag`. +// +// CAUTION: for requests involving large req/resp bodies ensure to pass the +// `traceHdrsS3HFlag`, otherwise both headers and body will be traced, causing +// high memory usage! +func s3APIMiddleware(f http.HandlerFunc, flags ...s3HFlag) http.HandlerFunc { + // Collect all flags with bitwise-OR and assign operator + var handlerFlags s3HFlag + for _, flag := range flags { + handlerFlags |= flag + } + + // Get name of the handler using reflection. + handlerName := getHandlerName(f, "objectAPIHandlers") + + var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { + // Wrap the actual handler with the appropriate tracing middleware. + var tracedHandler http.HandlerFunc + if handlerFlags.has(traceHdrsS3HFlag) { + tracedHandler = httpTraceHdrs(f) + } else { + tracedHandler = httpTraceAll(f) + } + + // Skip wrapping with the gzip middleware if specified. + var gzippedHandler http.HandlerFunc = tracedHandler + if !handlerFlags.has(noGZS3HFlag) { + gzippedHandler = gzipHandler(gzippedHandler) + } + + // Skip wrapping with throttling middleware if specified. + var throttledHandler http.HandlerFunc = gzippedHandler + if !handlerFlags.has(noThrottleS3HFlag) { + throttledHandler = maxClients(throttledHandler) + } + + // Collect API stats using the API name got from reflection in + // `getHandlerName`. + statsCollectedHandler := collectAPIStats(handlerName, throttledHandler) + + // Call the final handler. + statsCollectedHandler(w, r) + } + + return handler +} + // registerAPIRouter - registers S3 compatible APIs. func registerAPIRouter(router *mux.Router) { // Initialize API. @@ -208,12 +286,6 @@ func registerAPIRouter(router *mux.Router) { } routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter()) - gz, err := gzhttp.NewWrapper(gzhttp.MinSize(1000), gzhttp.CompressionLevel(gzip.BestSpeed)) - if err != nil { - // Static params, so this is very unlikely. - logger.Fatal(err, "Unable to initialize server") - } - for _, router := range routers { // Register all rejected object APIs for _, r := range rejectedObjAPIs { @@ -225,240 +297,307 @@ func registerAPIRouter(router *mux.Router) { // Object operations // HeadObject - router.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc( - collectAPIStats("headobject", maxClients(gz(httpTraceAll(api.HeadObjectHandler))))) + router.Methods(http.MethodHead).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.HeadObjectHandler)) + // GetObjectAttributes - router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - collectAPIStats("getobjectattributes", maxClients(gz(httpTraceHdrs(api.GetObjectAttributesHandler))))).Queries("attributes", "") + router.Methods(http.MethodGet).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.GetObjectAttributesHandler, traceHdrsS3HFlag)). + Queries("attributes", "") + // CopyObjectPart router.Methods(http.MethodPut).Path("/{object:.+}"). HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?"). - HandlerFunc(collectAPIStats("copyobjectpart", maxClients(gz(httpTraceAll(api.CopyObjectPartHandler))))). + HandlerFunc(s3APIMiddleware(api.CopyObjectPartHandler)). Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}") // PutObjectPart - router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( - collectAPIStats("putobjectpart", maxClients(gz(httpTraceHdrs(api.PutObjectPartHandler))))).Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}") + router.Methods(http.MethodPut).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.PutObjectPartHandler, traceHdrsS3HFlag)). + Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}") // ListObjectParts - router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - collectAPIStats("listobjectparts", maxClients(gz(httpTraceAll(api.ListObjectPartsHandler))))).Queries("uploadId", "{uploadId:.*}") + router.Methods(http.MethodGet).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.ListObjectPartsHandler)). + Queries("uploadId", "{uploadId:.*}") // CompleteMultipartUpload - router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc( - collectAPIStats("completemultipartupload", maxClients(gz(httpTraceAll(api.CompleteMultipartUploadHandler))))).Queries("uploadId", "{uploadId:.*}") + router.Methods(http.MethodPost).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.CompleteMultipartUploadHandler)). + Queries("uploadId", "{uploadId:.*}") // NewMultipartUpload - router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc( - collectAPIStats("newmultipartupload", maxClients(gz(httpTraceAll(api.NewMultipartUploadHandler))))).Queries("uploads", "") + router.Methods(http.MethodPost).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.NewMultipartUploadHandler)). + Queries("uploads", "") // AbortMultipartUpload - router.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc( - collectAPIStats("abortmultipartupload", maxClients(gz(httpTraceAll(api.AbortMultipartUploadHandler))))).Queries("uploadId", "{uploadId:.*}") + router.Methods(http.MethodDelete).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.AbortMultipartUploadHandler)). + Queries("uploadId", "{uploadId:.*}") // GetObjectACL - this is a dummy call. - router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - collectAPIStats("getobjectacl", maxClients(gz(httpTraceHdrs(api.GetObjectACLHandler))))).Queries("acl", "") + router.Methods(http.MethodGet).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.GetObjectACLHandler, traceHdrsS3HFlag)). + Queries("acl", "") // PutObjectACL - this is a dummy call. - router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( - collectAPIStats("putobjectacl", maxClients(gz(httpTraceHdrs(api.PutObjectACLHandler))))).Queries("acl", "") + router.Methods(http.MethodPut).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.PutObjectACLHandler, traceHdrsS3HFlag)). + Queries("acl", "") // GetObjectTagging - router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - collectAPIStats("getobjecttagging", maxClients(gz(httpTraceHdrs(api.GetObjectTaggingHandler))))).Queries("tagging", "") + router.Methods(http.MethodGet).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.GetObjectTaggingHandler, traceHdrsS3HFlag)). + Queries("tagging", "") // PutObjectTagging - router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( - collectAPIStats("putobjecttagging", maxClients(gz(httpTraceHdrs(api.PutObjectTaggingHandler))))).Queries("tagging", "") + router.Methods(http.MethodPut).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.PutObjectTaggingHandler, traceHdrsS3HFlag)). + Queries("tagging", "") // DeleteObjectTagging - router.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc( - collectAPIStats("deleteobjecttagging", maxClients(gz(httpTraceHdrs(api.DeleteObjectTaggingHandler))))).Queries("tagging", "") + router.Methods(http.MethodDelete).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.DeleteObjectTaggingHandler, traceHdrsS3HFlag)). + Queries("tagging", "") // SelectObjectContent - router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc( - collectAPIStats("selectobjectcontent", maxClients(gz(httpTraceHdrs(api.SelectObjectContentHandler))))).Queries("select", "").Queries("select-type", "2") + router.Methods(http.MethodPost).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.SelectObjectContentHandler, traceHdrsS3HFlag)). + Queries("select", "").Queries("select-type", "2") // GetObjectRetention - router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - collectAPIStats("getobjectretention", maxClients(gz(httpTraceAll(api.GetObjectRetentionHandler))))).Queries("retention", "") + router.Methods(http.MethodGet).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.GetObjectRetentionHandler)). + Queries("retention", "") // GetObjectLegalHold - router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - collectAPIStats("getobjectlegalhold", maxClients(gz(httpTraceAll(api.GetObjectLegalHoldHandler))))).Queries("legal-hold", "") + router.Methods(http.MethodGet).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.GetObjectLegalHoldHandler)). + Queries("legal-hold", "") // GetObject with lambda ARNs - router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - collectAPIStats("getobjectlambda", maxClients(gz(httpTraceHdrs(api.GetObjectLambdaHandler))))).Queries("lambdaArn", "{lambdaArn:.*}") + router.Methods(http.MethodGet).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.GetObjectLambdaHandler, traceHdrsS3HFlag)). + Queries("lambdaArn", "{lambdaArn:.*}") // GetObject - router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc( - collectAPIStats("getobject", maxClients(gz(httpTraceHdrs(api.GetObjectHandler))))) + router.Methods(http.MethodGet).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.GetObjectHandler, traceHdrsS3HFlag)) // CopyObject - router.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc( - collectAPIStats("copyobject", maxClients(gz(httpTraceAll(api.CopyObjectHandler))))) + router.Methods(http.MethodPut).Path("/{object:.+}"). + HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?"). + HandlerFunc(s3APIMiddleware(api.CopyObjectHandler)) // PutObjectRetention - router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( - collectAPIStats("putobjectretention", maxClients(gz(httpTraceAll(api.PutObjectRetentionHandler))))).Queries("retention", "") + router.Methods(http.MethodPut).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.PutObjectRetentionHandler)). + Queries("retention", "") // PutObjectLegalHold - router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( - collectAPIStats("putobjectlegalhold", maxClients(gz(httpTraceAll(api.PutObjectLegalHoldHandler))))).Queries("legal-hold", "") + router.Methods(http.MethodPut).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.PutObjectLegalHoldHandler)). + Queries("legal-hold", "") // PutObject with auto-extract support for zip - router.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzSnowballExtract, "true").HandlerFunc( - collectAPIStats("putobjectextract", maxClients(gz(httpTraceHdrs(api.PutObjectExtractHandler))))) + router.Methods(http.MethodPut).Path("/{object:.+}"). + HeadersRegexp(xhttp.AmzSnowballExtract, "true"). + HandlerFunc(s3APIMiddleware(api.PutObjectExtractHandler, traceHdrsS3HFlag)) // PutObject - router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc( - collectAPIStats("putobject", maxClients(gz(httpTraceHdrs(api.PutObjectHandler))))) + router.Methods(http.MethodPut).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.PutObjectHandler, traceHdrsS3HFlag)) // DeleteObject - router.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc( - collectAPIStats("deleteobject", maxClients(gz(httpTraceAll(api.DeleteObjectHandler))))) + router.Methods(http.MethodDelete).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.DeleteObjectHandler)) // PostRestoreObject - router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc( - collectAPIStats("postrestoreobject", maxClients(gz(httpTraceAll(api.PostRestoreObjectHandler))))).Queries("restore", "") + router.Methods(http.MethodPost).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.PostRestoreObjectHandler)). + Queries("restore", "") // Bucket operations // GetBucketLocation - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketlocation", maxClients(gz(httpTraceAll(api.GetBucketLocationHandler))))).Queries("location", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketLocationHandler)). + Queries("location", "") // GetBucketPolicy - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketpolicy", maxClients(gz(httpTraceAll(api.GetBucketPolicyHandler))))).Queries("policy", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketPolicyHandler)). + Queries("policy", "") // GetBucketLifecycle - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketlifecycle", maxClients(gz(httpTraceAll(api.GetBucketLifecycleHandler))))).Queries("lifecycle", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketLifecycleHandler)). + Queries("lifecycle", "") // GetBucketEncryption - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketencryption", maxClients(gz(httpTraceAll(api.GetBucketEncryptionHandler))))).Queries("encryption", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketEncryptionHandler)). + Queries("encryption", "") // GetBucketObjectLockConfig - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketobjectlockconfig", maxClients(gz(httpTraceAll(api.GetBucketObjectLockConfigHandler))))).Queries("object-lock", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketObjectLockConfigHandler)). + Queries("object-lock", "") // GetBucketReplicationConfig - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketreplicationconfig", maxClients(gz(httpTraceAll(api.GetBucketReplicationConfigHandler))))).Queries("replication", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketReplicationConfigHandler)). + Queries("replication", "") // GetBucketVersioning - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketversioning", maxClients(gz(httpTraceAll(api.GetBucketVersioningHandler))))).Queries("versioning", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketVersioningHandler)). + Queries("versioning", "") // GetBucketNotification - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketnotification", maxClients(gz(httpTraceAll(api.GetBucketNotificationHandler))))).Queries("notification", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketNotificationHandler)). + Queries("notification", "") // ListenNotification - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("listennotification", gz(httpTraceAll(api.ListenNotificationHandler)))).Queries("events", "{events:.*}") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ListenNotificationHandler, noThrottleS3HFlag)). + Queries("events", "{events:.*}") // ResetBucketReplicationStatus - MinIO extension API - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("resetbucketreplicationstatus", maxClients(gz(httpTraceAll(api.ResetBucketReplicationStatusHandler))))).Queries("replication-reset-status", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ResetBucketReplicationStatusHandler)). + Queries("replication-reset-status", "") // Dummy Bucket Calls // GetBucketACL -- this is a dummy call. - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketacl", maxClients(gz(httpTraceAll(api.GetBucketACLHandler))))).Queries("acl", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketACLHandler)). + Queries("acl", "") // PutBucketACL -- this is a dummy call. - router.Methods(http.MethodPut).HandlerFunc( - collectAPIStats("putbucketacl", maxClients(gz(httpTraceAll(api.PutBucketACLHandler))))).Queries("acl", "") + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketACLHandler)). + Queries("acl", "") // GetBucketCors - this is a dummy call. - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketcors", maxClients(gz(httpTraceAll(api.GetBucketCorsHandler))))).Queries("cors", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketCorsHandler)). + Queries("cors", "") // GetBucketWebsiteHandler - this is a dummy call. - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketwebsite", maxClients(gz(httpTraceAll(api.GetBucketWebsiteHandler))))).Queries("website", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketWebsiteHandler)). + Queries("website", "") // GetBucketAccelerateHandler - this is a dummy call. - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketaccelerate", maxClients(gz(httpTraceAll(api.GetBucketAccelerateHandler))))).Queries("accelerate", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketAccelerateHandler)). + Queries("accelerate", "") // GetBucketRequestPaymentHandler - this is a dummy call. - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketrequestpayment", maxClients(gz(httpTraceAll(api.GetBucketRequestPaymentHandler))))).Queries("requestPayment", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketRequestPaymentHandler)). + Queries("requestPayment", "") // GetBucketLoggingHandler - this is a dummy call. - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketlogging", maxClients(gz(httpTraceAll(api.GetBucketLoggingHandler))))).Queries("logging", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketLoggingHandler)). + Queries("logging", "") // GetBucketTaggingHandler - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbuckettagging", maxClients(gz(httpTraceAll(api.GetBucketTaggingHandler))))).Queries("tagging", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketTaggingHandler)). + Queries("tagging", "") // DeleteBucketWebsiteHandler - router.Methods(http.MethodDelete).HandlerFunc( - collectAPIStats("deletebucketwebsite", maxClients(gz(httpTraceAll(api.DeleteBucketWebsiteHandler))))).Queries("website", "") + router.Methods(http.MethodDelete). + HandlerFunc(s3APIMiddleware(api.DeleteBucketWebsiteHandler)). + Queries("website", "") // DeleteBucketTaggingHandler - router.Methods(http.MethodDelete).HandlerFunc( - collectAPIStats("deletebuckettagging", maxClients(gz(httpTraceAll(api.DeleteBucketTaggingHandler))))).Queries("tagging", "") + router.Methods(http.MethodDelete). + HandlerFunc(s3APIMiddleware(api.DeleteBucketTaggingHandler)). + Queries("tagging", "") // ListMultipartUploads - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("listmultipartuploads", maxClients(gz(httpTraceAll(api.ListMultipartUploadsHandler))))).Queries("uploads", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ListMultipartUploadsHandler)). + Queries("uploads", "") // ListObjectsV2M - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("listobjectsv2M", maxClients(gz(httpTraceAll(api.ListObjectsV2MHandler))))).Queries("list-type", "2", "metadata", "true") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ListObjectsV2MHandler)). + Queries("list-type", "2", "metadata", "true") // ListObjectsV2 - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("listobjectsv2", maxClients(gz(httpTraceAll(api.ListObjectsV2Handler))))).Queries("list-type", "2") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ListObjectsV2Handler)). + Queries("list-type", "2") // ListObjectVersions - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("listobjectversionsM", maxClients(gz(httpTraceAll(api.ListObjectVersionsMHandler))))).Queries("versions", "", "metadata", "true") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ListObjectVersionsMHandler)). + Queries("versions", "", "metadata", "true") // ListObjectVersions - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("listobjectversions", maxClients(gz(httpTraceAll(api.ListObjectVersionsHandler))))).Queries("versions", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ListObjectVersionsHandler)). + Queries("versions", "") // GetBucketPolicyStatus - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketpolicystatus", maxClients(gz(httpTraceAll(api.GetBucketPolicyStatusHandler))))).Queries("policyStatus", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketPolicyStatusHandler)). + Queries("policyStatus", "") // PutBucketLifecycle - router.Methods(http.MethodPut).HandlerFunc( - collectAPIStats("putbucketlifecycle", maxClients(gz(httpTraceAll(api.PutBucketLifecycleHandler))))).Queries("lifecycle", "") + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketLifecycleHandler)). + Queries("lifecycle", "") // PutBucketReplicationConfig - router.Methods(http.MethodPut).HandlerFunc( - collectAPIStats("putbucketreplicationconfig", maxClients(gz(httpTraceAll(api.PutBucketReplicationConfigHandler))))).Queries("replication", "") + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketReplicationConfigHandler)). + Queries("replication", "") // PutBucketEncryption - router.Methods(http.MethodPut).HandlerFunc( - collectAPIStats("putbucketencryption", maxClients(gz(httpTraceAll(api.PutBucketEncryptionHandler))))).Queries("encryption", "") + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketEncryptionHandler)). + Queries("encryption", "") // PutBucketPolicy - router.Methods(http.MethodPut).HandlerFunc( - collectAPIStats("putbucketpolicy", maxClients(gz(httpTraceAll(api.PutBucketPolicyHandler))))).Queries("policy", "") + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketPolicyHandler)). + Queries("policy", "") // PutBucketObjectLockConfig - router.Methods(http.MethodPut).HandlerFunc( - collectAPIStats("putbucketobjectlockconfig", maxClients(gz(httpTraceAll(api.PutBucketObjectLockConfigHandler))))).Queries("object-lock", "") + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketObjectLockConfigHandler)). + Queries("object-lock", "") // PutBucketTaggingHandler - router.Methods(http.MethodPut).HandlerFunc( - collectAPIStats("putbuckettagging", maxClients(gz(httpTraceAll(api.PutBucketTaggingHandler))))).Queries("tagging", "") + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketTaggingHandler)). + Queries("tagging", "") // PutBucketVersioning - router.Methods(http.MethodPut).HandlerFunc( - collectAPIStats("putbucketversioning", maxClients(gz(httpTraceAll(api.PutBucketVersioningHandler))))).Queries("versioning", "") + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketVersioningHandler)). + Queries("versioning", "") // PutBucketNotification - router.Methods(http.MethodPut).HandlerFunc( - collectAPIStats("putbucketnotification", maxClients(gz(httpTraceAll(api.PutBucketNotificationHandler))))).Queries("notification", "") + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketNotificationHandler)). + Queries("notification", "") // ResetBucketReplicationStart - MinIO extension API - router.Methods(http.MethodPut).HandlerFunc( - collectAPIStats("resetbucketreplicationstart", maxClients(gz(httpTraceAll(api.ResetBucketReplicationStartHandler))))).Queries("replication-reset", "") + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.ResetBucketReplicationStartHandler)). + Queries("replication-reset", "") // PutBucket - router.Methods(http.MethodPut).HandlerFunc( - collectAPIStats("putbucket", maxClients(gz(httpTraceAll(api.PutBucketHandler))))) + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketHandler)) // HeadBucket - router.Methods(http.MethodHead).HandlerFunc( - collectAPIStats("headbucket", maxClients(gz(httpTraceAll(api.HeadBucketHandler))))) + router.Methods(http.MethodHead). + HandlerFunc(s3APIMiddleware(api.HeadBucketHandler)) // PostPolicy - router.Methods(http.MethodPost).MatcherFunc(func(r *http.Request, _ *mux.RouteMatch) bool { - return isRequestPostPolicySignatureV4(r) - }).HandlerFunc(collectAPIStats("postpolicybucket", maxClients(gz(httpTraceHdrs(api.PostPolicyBucketHandler))))) + router.Methods(http.MethodPost). + MatcherFunc(func(r *http.Request, _ *mux.RouteMatch) bool { + return isRequestPostPolicySignatureV4(r) + }). + HandlerFunc(s3APIMiddleware(api.PostPolicyBucketHandler, traceHdrsS3HFlag)) // DeleteMultipleObjects - router.Methods(http.MethodPost).HandlerFunc( - collectAPIStats("deletemultipleobjects", maxClients(gz(httpTraceAll(api.DeleteMultipleObjectsHandler))))).Queries("delete", "") + router.Methods(http.MethodPost). + HandlerFunc(s3APIMiddleware(api.DeleteMultipleObjectsHandler)). + Queries("delete", "") // DeleteBucketPolicy - router.Methods(http.MethodDelete).HandlerFunc( - collectAPIStats("deletebucketpolicy", maxClients(gz(httpTraceAll(api.DeleteBucketPolicyHandler))))).Queries("policy", "") + router.Methods(http.MethodDelete). + HandlerFunc(s3APIMiddleware(api.DeleteBucketPolicyHandler)). + Queries("policy", "") // DeleteBucketReplication - router.Methods(http.MethodDelete).HandlerFunc( - collectAPIStats("deletebucketreplicationconfig", maxClients(gz(httpTraceAll(api.DeleteBucketReplicationConfigHandler))))).Queries("replication", "") + router.Methods(http.MethodDelete). + HandlerFunc(s3APIMiddleware(api.DeleteBucketReplicationConfigHandler)). + Queries("replication", "") // DeleteBucketLifecycle - router.Methods(http.MethodDelete).HandlerFunc( - collectAPIStats("deletebucketlifecycle", maxClients(gz(httpTraceAll(api.DeleteBucketLifecycleHandler))))).Queries("lifecycle", "") + router.Methods(http.MethodDelete). + HandlerFunc(s3APIMiddleware(api.DeleteBucketLifecycleHandler)). + Queries("lifecycle", "") // DeleteBucketEncryption - router.Methods(http.MethodDelete).HandlerFunc( - collectAPIStats("deletebucketencryption", maxClients(gz(httpTraceAll(api.DeleteBucketEncryptionHandler))))).Queries("encryption", "") + router.Methods(http.MethodDelete). + HandlerFunc(s3APIMiddleware(api.DeleteBucketEncryptionHandler)). + Queries("encryption", "") // DeleteBucket - router.Methods(http.MethodDelete).HandlerFunc( - collectAPIStats("deletebucket", maxClients(gz(httpTraceAll(api.DeleteBucketHandler))))) + router.Methods(http.MethodDelete). + HandlerFunc(s3APIMiddleware(api.DeleteBucketHandler)) // MinIO extension API for replication. // - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketreplicationmetricsv2", maxClients(gz(httpTraceAll(api.GetBucketReplicationMetricsV2Handler))))).Queries("replication-metrics", "2") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketReplicationMetricsV2Handler)). + Queries("replication-metrics", "2") // deprecated handler - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("getbucketreplicationmetrics", maxClients(gz(httpTraceAll(api.GetBucketReplicationMetricsHandler))))).Queries("replication-metrics", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketReplicationMetricsHandler)). + Queries("replication-metrics", "") // ValidateBucketReplicationCreds - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("validatebucketreplicationcreds", maxClients(gz(httpTraceAll(api.ValidateBucketReplicationCredsHandler))))).Queries("replication-check", "") + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ValidateBucketReplicationCredsHandler)). + Queries("replication-check", "") // Register rejected bucket APIs for _, r := range rejectedBucketAPIs { @@ -468,24 +607,25 @@ func registerAPIRouter(router *mux.Router) { } // S3 ListObjectsV1 (Legacy) - router.Methods(http.MethodGet).HandlerFunc( - collectAPIStats("listobjectsv1", maxClients(gz(httpTraceAll(api.ListObjectsV1Handler))))) + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ListObjectsV1Handler)) } // Root operation // ListenNotification - apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc( - collectAPIStats("listennotification", gz(httpTraceAll(api.ListenNotificationHandler)))).Queries("events", "{events:.*}") + apiRouter.Methods(http.MethodGet).Path(SlashSeparator). + HandlerFunc(s3APIMiddleware(api.ListenNotificationHandler, noThrottleS3HFlag)). + Queries("events", "{events:.*}") // ListBuckets - apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc( - collectAPIStats("listbuckets", maxClients(gz(httpTraceAll(api.ListBucketsHandler))))) + apiRouter.Methods(http.MethodGet).Path(SlashSeparator). + HandlerFunc(s3APIMiddleware(api.ListBucketsHandler)) // S3 browser with signature v4 adds '//' for ListBuckets request, so rather // than failing with UnknownAPIRequest we simply handle it for now. - apiRouter.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator).HandlerFunc( - collectAPIStats("listbuckets", maxClients(gz(httpTraceAll(api.ListBucketsHandler))))) + apiRouter.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator). + HandlerFunc(s3APIMiddleware(api.ListBucketsHandler)) // If none of the routes match add default error handler routes apiRouter.NotFoundHandler = collectAPIStats("notfound", httpTraceAll(errorResponseHandler)) diff --git a/cmd/api-utils.go b/cmd/api-utils.go index 90f16071c..ab191f067 100644 --- a/cmd/api-utils.go +++ b/cmd/api-utils.go @@ -18,6 +18,10 @@ package cmd import ( + "fmt" + "net/http" + "reflect" + "runtime" "strings" ) @@ -100,3 +104,16 @@ func s3EncodeName(name, encodingType string) string { } return name } + +// getHandlerName returns the name of the handler function. It takes the type +// name as a string to clean up the name retrieved via reflection. This function +// only works correctly when the type is present in the cmd package. +func getHandlerName(f http.HandlerFunc, cmdType string) string { + name := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name() + + packageName := fmt.Sprintf("github.com/minio/minio/cmd.%s.", cmdType) + name = strings.TrimPrefix(name, packageName) + name = strings.TrimSuffix(name, "Handler-fm") + name = strings.TrimSuffix(name, "-fm") + return name +} diff --git a/cmd/http-stats.go b/cmd/http-stats.go index 08fb9ba52..1393636f6 100644 --- a/cmd/http-stats.go +++ b/cmd/http-stats.go @@ -19,6 +19,7 @@ package cmd import ( "net/http" + "strings" "sync" "sync/atomic" @@ -326,7 +327,7 @@ func (stats *HTTPAPIStats) Get(api string) int { } // Load returns the recorded stats. -func (stats *HTTPAPIStats) Load() map[string]int { +func (stats *HTTPAPIStats) Load(toLower bool) map[string]int { if stats == nil { return map[string]int{} } @@ -336,6 +337,9 @@ func (stats *HTTPAPIStats) Load() map[string]int { apiStats := make(map[string]int, len(stats.apiStats)) for k, v := range stats.apiStats { + if toLower { + k = strings.ToLower(k) + } apiStats[k] = v } return apiStats @@ -373,7 +377,7 @@ func (st *HTTPStats) incS3RequestsIncoming() { } // Converts http stats into struct to be sent back to the client. -func (st *HTTPStats) toServerHTTPStats() ServerHTTPStats { +func (st *HTTPStats) toServerHTTPStats(toLowerKeys bool) ServerHTTPStats { serverStats := ServerHTTPStats{} serverStats.S3RequestsIncoming = atomic.SwapUint64(&st.s3RequestsIncoming, 0) serverStats.S3RequestsInQueue = atomic.LoadInt32(&st.s3RequestsInQueue) @@ -382,22 +386,22 @@ func (st *HTTPStats) toServerHTTPStats() ServerHTTPStats { serverStats.TotalS3RejectedHeader = atomic.LoadUint64(&st.rejectedRequestsHeader) serverStats.TotalS3RejectedInvalid = atomic.LoadUint64(&st.rejectedRequestsInvalid) serverStats.CurrentS3Requests = ServerHTTPAPIStats{ - APIStats: st.currentS3Requests.Load(), + APIStats: st.currentS3Requests.Load(toLowerKeys), } serverStats.TotalS3Requests = ServerHTTPAPIStats{ - APIStats: st.totalS3Requests.Load(), + APIStats: st.totalS3Requests.Load(toLowerKeys), } serverStats.TotalS3Errors = ServerHTTPAPIStats{ - APIStats: st.totalS3Errors.Load(), + APIStats: st.totalS3Errors.Load(toLowerKeys), } serverStats.TotalS34xxErrors = ServerHTTPAPIStats{ - APIStats: st.totalS34xxErrors.Load(), + APIStats: st.totalS34xxErrors.Load(toLowerKeys), } serverStats.TotalS35xxErrors = ServerHTTPAPIStats{ - APIStats: st.totalS35xxErrors.Load(), + APIStats: st.totalS35xxErrors.Load(toLowerKeys), } serverStats.TotalS3Canceled = ServerHTTPAPIStats{ - APIStats: st.totalS3Canceled.Load(), + APIStats: st.totalS3Canceled.Load(toLowerKeys), } return serverStats } diff --git a/cmd/metrics-v2.go b/cmd/metrics-v2.go index c18ef497c..3ae7cc8ad 100644 --- a/cmd/metrics-v2.go +++ b/cmd/metrics-v2.go @@ -1827,7 +1827,10 @@ func getGoMetrics() *MetricsGroup { // getHistogramMetrics fetches histogram metrics and returns it in a []Metric // Note: Typically used in MetricGroup.RegisterRead -func getHistogramMetrics(hist *prometheus.HistogramVec, desc MetricDescription) []Metric { +// +// The last parameter is added for compatibility - if true it lowercases the +// `api` label values. +func getHistogramMetrics(hist *prometheus.HistogramVec, desc MetricDescription, toLowerAPILabels bool) []Metric { ch := make(chan prometheus.Metric) go func() { defer xioutil.SafeClose(ch) @@ -1851,7 +1854,11 @@ func getHistogramMetrics(hist *prometheus.HistogramVec, desc MetricDescription) for _, b := range h.Bucket { labels := make(map[string]string) for _, lp := range dtoMetric.GetLabel() { - labels[*lp.Name] = *lp.Value + if *lp.Name == "api" && toLowerAPILabels { + labels[*lp.Name] = strings.ToLower(*lp.Value) + } else { + labels[*lp.Name] = *lp.Value + } } labels["le"] = fmt.Sprintf("%.3f", *b.UpperBound) metric := Metric{ @@ -1881,7 +1888,8 @@ func getBucketTTFBMetric() *MetricsGroup { cacheInterval: 10 * time.Second, } mg.RegisterRead(func(ctx context.Context) []Metric { - return getHistogramMetrics(bucketHTTPRequestsDuration, getBucketTTFBDistributionMD()) + return getHistogramMetrics(bucketHTTPRequestsDuration, + getBucketTTFBDistributionMD(), true) }) return mg } @@ -1891,7 +1899,8 @@ func getS3TTFBMetric() *MetricsGroup { cacheInterval: 10 * time.Second, } mg.RegisterRead(func(ctx context.Context) []Metric { - return getHistogramMetrics(httpRequestsDuration, getS3TTFBDistributionMD()) + return getHistogramMetrics(httpRequestsDuration, + getS3TTFBDistributionMD(), true) }) return mg } @@ -2918,7 +2927,7 @@ func getHTTPMetrics(opts MetricsGroupOpts) *MetricsGroup { } mg.RegisterRead(func(ctx context.Context) (metrics []Metric) { if !mg.metricsGroupOpts.bucketOnly { - httpStats := globalHTTPStats.toServerHTTPStats() + httpStats := globalHTTPStats.toServerHTTPStats(true) metrics = make([]Metric, 0, 3+ len(httpStats.CurrentS3Requests.APIStats)+ len(httpStats.TotalS3Requests.APIStats)+ @@ -3014,7 +3023,7 @@ func getHTTPMetrics(opts MetricsGroupOpts) *MetricsGroup { } httpStats := globalBucketHTTPStats.load(bucket) - for k, v := range httpStats.currentS3Requests.Load() { + for k, v := range httpStats.currentS3Requests.Load(true) { metrics = append(metrics, Metric{ Description: getBucketS3RequestsInFlightMD(), Value: float64(v), @@ -3022,7 +3031,7 @@ func getHTTPMetrics(opts MetricsGroupOpts) *MetricsGroup { }) } - for k, v := range httpStats.totalS3Requests.Load() { + for k, v := range httpStats.totalS3Requests.Load(true) { metrics = append(metrics, Metric{ Description: getBucketS3RequestsTotalMD(), Value: float64(v), @@ -3030,7 +3039,7 @@ func getHTTPMetrics(opts MetricsGroupOpts) *MetricsGroup { }) } - for k, v := range httpStats.totalS3Canceled.Load() { + for k, v := range httpStats.totalS3Canceled.Load(true) { metrics = append(metrics, Metric{ Description: getBucketS3RequestsCanceledMD(), Value: float64(v), @@ -3038,7 +3047,7 @@ func getHTTPMetrics(opts MetricsGroupOpts) *MetricsGroup { }) } - for k, v := range httpStats.totalS34xxErrors.Load() { + for k, v := range httpStats.totalS34xxErrors.Load(true) { metrics = append(metrics, Metric{ Description: getBucketS3Requests4xxErrorsMD(), Value: float64(v), @@ -3046,7 +3055,7 @@ func getHTTPMetrics(opts MetricsGroupOpts) *MetricsGroup { }) } - for k, v := range httpStats.totalS35xxErrors.Load() { + for k, v := range httpStats.totalS35xxErrors.Load(true) { metrics = append(metrics, Metric{ Description: getBucketS3Requests5xxErrorsMD(), Value: float64(v), diff --git a/cmd/metrics-v2_test.go b/cmd/metrics-v2_test.go index c91c209fc..03f01e85f 100644 --- a/cmd/metrics-v2_test.go +++ b/cmd/metrics-v2_test.go @@ -80,7 +80,7 @@ func TestGetHistogramMetrics(t *testing.T) { } } - metrics := getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD()) + metrics := getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD(), false) // additional labels for +Inf for all histogram metrics if expPoints := len(labels) * (len(histBuckets) + 1); expPoints != len(metrics) { t.Fatalf("Expected %v data points but got %v", expPoints, len(metrics)) diff --git a/cmd/metrics.go b/cmd/metrics.go index d38fd27c7..a198593b4 100644 --- a/cmd/metrics.go +++ b/cmd/metrics.go @@ -190,7 +190,7 @@ func healingMetricsPrometheus(ch chan<- prometheus.Metric) { // collects http metrics for MinIO server in Prometheus specific format // and sends to given channel func httpMetricsPrometheus(ch chan<- prometheus.Metric) { - httpStats := globalHTTPStats.toServerHTTPStats() + httpStats := globalHTTPStats.toServerHTTPStats(true) for api, value := range httpStats.CurrentS3Requests.APIStats { ch <- prometheus.MustNewConstMetric( diff --git a/cmd/tier.go b/cmd/tier.go index 890dc7026..f785ce845 100644 --- a/cmd/tier.go +++ b/cmd/tier.go @@ -159,7 +159,7 @@ var ( ) func (t *tierMetrics) Report() []Metric { - metrics := getHistogramMetrics(t.histogram, tierTTLBMD) + metrics := getHistogramMetrics(t.histogram, tierTTLBMD, true) t.RLock() defer t.RUnlock() for tier, stat := range t.requestsCount { diff --git a/docs/bucket/replication/delete-replication.sh b/docs/bucket/replication/delete-replication.sh index b8c33c7ed..2eadf52f7 100755 --- a/docs/bucket/replication/delete-replication.sh +++ b/docs/bucket/replication/delete-replication.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +echo "Running $0" + if [ -n "$TEST_DEBUG" ]; then set -x fi diff --git a/docs/bucket/replication/setup_2site_existing_replication.sh b/docs/bucket/replication/setup_2site_existing_replication.sh index fe977e7c5..5a5e026c0 100755 --- a/docs/bucket/replication/setup_2site_existing_replication.sh +++ b/docs/bucket/replication/setup_2site_existing_replication.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +echo "Running $0" + set -x trap 'catch $LINENO' ERR diff --git a/docs/bucket/replication/setup_3site_replication.sh b/docs/bucket/replication/setup_3site_replication.sh index 193fc8be1..6aa45f582 100755 --- a/docs/bucket/replication/setup_3site_replication.sh +++ b/docs/bucket/replication/setup_3site_replication.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +echo "Running $0" + if [ -n "$TEST_DEBUG" ]; then set -x fi @@ -46,11 +48,11 @@ unset MINIO_KMS_KES_KEY_NAME go install -v ) -wget -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && +wget -q -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && chmod +x mc if [ ! -f mc.RELEASE.2021-03-12T03-36-59Z ]; then - wget -O mc.RELEASE.2021-03-12T03-36-59Z https://dl.minio.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2021-03-12T03-36-59Z && + wget -q -O mc.RELEASE.2021-03-12T03-36-59Z https://dl.minio.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2021-03-12T03-36-59Z && chmod +x mc.RELEASE.2021-03-12T03-36-59Z fi diff --git a/docs/bucket/replication/sio-error.sh b/docs/bucket/replication/sio-error.sh index e7106e93a..f7ebe323e 100755 --- a/docs/bucket/replication/sio-error.sh +++ b/docs/bucket/replication/sio-error.sh @@ -1,11 +1,13 @@ #!/bin/bash +echo "Running $0" + set -e set -x export CI=1 -make || exit -1 +make || exit 255 killall -9 minio || true