From b33090b6ac85df574fa8ebc7ff008af922898ac9 Mon Sep 17 00:00:00 2001 From: Ben McClelland Date: Thu, 5 Mar 2026 19:12:22 -0800 Subject: [PATCH] otel tracing wip --- cmd/versitygw/main.go | 25 + go.mod | 17 + go.sum | 35 + s3api/controllers/base.go | 17 + s3api/middlewares/acl-parser.go | 13 + s3api/middlewares/authentication.go | 23 + s3api/middlewares/checksum.go | 10 + s3api/middlewares/presign-auth.go | 20 + s3api/middlewares/public-bucket.go | 16 + s3api/middlewares/tracing.go | 116 ++ s3api/server.go | 14 + s3api/utils/context-keys.go | 1 + tracing/Makefile | 27 + tracing/README.md | 141 ++ tracing/docker-compose.yml | 52 + .../dashboards/provider.yaml | 7 + .../dashboards/versitygw.json | 1368 +++++++++++++++++ .../datasources/prometheus.yaml | 9 + .../datasources/tempo.yaml | 21 + tracing/prometheus.yml | 7 + tracing/tempo.yaml | 64 + tracing/tracing.go | 71 + 22 files changed, 2074 insertions(+) create mode 100644 s3api/middlewares/tracing.go create mode 100644 tracing/Makefile create mode 100644 tracing/README.md create mode 100644 tracing/docker-compose.yml create mode 100644 tracing/grafana-provisioning/dashboards/provider.yaml create mode 100644 tracing/grafana-provisioning/dashboards/versitygw.json create mode 100644 tracing/grafana-provisioning/datasources/prometheus.yaml create mode 100644 tracing/grafana-provisioning/datasources/tempo.yaml create mode 100644 tracing/prometheus.yml create mode 100644 tracing/tempo.yaml create mode 100644 tracing/tracing.go diff --git a/cmd/versitygw/main.go b/cmd/versitygw/main.go index 288b1edb..7e0efeb4 100644 --- a/cmd/versitygw/main.go +++ b/cmd/versitygw/main.go @@ -36,6 +36,7 @@ import ( "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3event" "github.com/versity/versitygw/s3log" + "github.com/versity/versitygw/tracing" "github.com/versity/versitygw/webui" ) @@ -95,6 +96,8 @@ var ( ipaUser, ipaPassword string ipaInsecure bool iamDebug bool + otelEndpoint string + otelServiceName string webuiPorts []string webuiCertFile, webuiKeyFile string webuiNoTLS bool @@ -727,6 +730,19 @@ func initFlags() []cli.Flag { EnvVars: []string{"VGW_IPA_INSECURE"}, Destination: &ipaInsecure, }, + &cli.StringFlag{ + Name: "otel-endpoint", + Usage: "OpenTelemetry collector endpoint URL for tracing (e.g. http://localhost:4318). Tracing is disabled when unset.", + EnvVars: []string{"VGW_OTEL_ENDPOINT", "OTEL_EXPORTER_OTLP_ENDPOINT"}, + Destination: &otelEndpoint, + }, + &cli.StringFlag{ + Name: "otel-service-name", + Usage: "Service name reported in OpenTelemetry traces", + EnvVars: []string{"VGW_OTEL_SERVICE_NAME", "OTEL_SERVICE_NAME"}, + Value: "versitygw", + Destination: &otelServiceName, + }, } } @@ -844,6 +860,15 @@ func runGateway(ctx context.Context, be backend.Backend) error { if disableACLs { opts = append(opts, s3api.WithDisableACL()) } + if otelEndpoint != "" { + shutdownTracer, err := tracing.InitTracer(ctx, otelServiceName, otelEndpoint) + if err != nil { + return fmt.Errorf("init otel tracer: %w", err) + } + defer shutdownTracer(context.Background()) //nolint:errcheck + opts = append(opts, s3api.WithTracing()) + } + if debug { debuglogger.SetDebugEnabled() } diff --git a/go.mod b/go.mod index afa11254..d1466e97 100644 --- a/go.mod +++ b/go.mod @@ -55,10 +55,15 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.8 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect @@ -79,8 +84,20 @@ require ( github.com/ryanuber/go-glob v1.0.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel v1.41.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.41.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.41.0 // indirect + go.opentelemetry.io/otel/metric v1.41.0 // indirect + go.opentelemetry.io/otel/sdk v1.41.0 // indirect + go.opentelemetry.io/otel/trace v1.41.0 // indirect + go.opentelemetry.io/proto/otlp v1.9.0 // indirect golang.org/x/crypto v0.48.0 // indirect golang.org/x/net v0.51.0 // indirect golang.org/x/text v0.34.0 // indirect golang.org/x/time v0.14.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect + google.golang.org/grpc v1.79.1 // indirect + google.golang.org/protobuf v1.36.11 // indirect ) diff --git a/go.sum b/go.sum index 9e684a26..a91cf99a 100644 --- a/go.sum +++ b/go.sum @@ -65,6 +65,10 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 h1:NITQpgo9A5NrDZ57uOWj+abvXSb8 github.com/aws/aws-sdk-go-v2/service/sts v1.41.7/go.mod h1:sks5UWBhEuWYDPdwlnRFn1w7xWdH29Jcpe+/PJQefEs= github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= @@ -78,6 +82,11 @@ github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 h1:BP4M0CvQ github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-ldap/ldap/v3 v3.4.12 h1:1b81mv7MagXZ7+1r7cLTWmyuTqVqdwbtJSjC0DAp9s4= github.com/go-ldap/ldap/v3 v3.4.12/go.mod h1:+SPAGcTtOfmGsCb3h1RFiq4xpp4N636G75OEace8lNo= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/gofiber/fiber/v2 v2.52.12 h1:0LdToKclcPOj8PktUdIKo9BUohjjwfnQl42Dhw8/WUw= github.com/gofiber/fiber/v2 v2.52.12/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= @@ -87,6 +96,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= @@ -196,6 +207,22 @@ github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBi github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.41.0 h1:ao6Oe+wSebTlQ1OEht7jlYTzQKE+pnx/iNywFvTbuuI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.41.0/go.mod h1:u3T6vz0gh/NVzgDgiwkgLxpsSF6PaPmo2il0apGJbls= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.41.0 h1:inYW9ZhgqiDqh6BioM7DVHHzEGVq76Db5897WLGZ5Go= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.41.0/go.mod h1:Izur+Wt8gClgMJqO/cZ8wdeeMryJ/xxiOVgFSSfpDTY= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= +go.opentelemetry.io/otel/sdk v1.41.0 h1:YPIEXKmiAwkGl3Gu1huk1aYWwtpRLeskpV+wPisxBp8= +go.opentelemetry.io/otel/sdk v1.41.0/go.mod h1:ahFdU0G5y8IxglBf0QBJXgSe7agzjE4GiTJ6HT9ud90= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= +go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= +go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -237,6 +264,14 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 h1:JLQynH/LBHfCTSbDWl+py8C+Rg/k1OVH3xfcaiANuF0= +google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:kSJwQxqmFXeo79zOmbrALdflXQeAYcUbgS7PbpMknCY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 h1:mWPCjDEyshlQYzBpMNHaEof6UX1PmHcaUODUywQ0uac= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.79.1 h1:zGhSi45ODB9/p3VAawt9a+O/MULLl9dpizzNNpq7flY= +google.golang.org/grpc v1.79.1/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 80e95188..9e03b960 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -30,6 +30,8 @@ import ( "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3event" "github.com/versity/versitygw/s3log" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/codes" ) type S3ApiController struct { @@ -129,6 +131,10 @@ func ProcessHandlers(controller Controller, s3action string, svc *Services, hand return ctx.Next() } + // Store the resolved S3 action name so the tracing middleware can + // attach it to the span after routing has completed. + utils.ContextKeyS3Action.Set(ctx, s3action) + for _, handler := range handlers { err := handler(ctx) if err != nil { @@ -179,10 +185,21 @@ func WrapMiddleware(handler fiber.Handler, logger s3log.AuditLogger, mm metrics. } } +const controllerTracerName = "github.com/versity/versitygw" + // ProcessController executes the given s3api controller and handles the metrics // access logs and s3 events func ProcessController(ctx *fiber.Ctx, controller Controller, s3action string, svc *Services) error { + parentCtx := ctx.UserContext() + backendCtx, backendSpan := otel.Tracer(controllerTracerName).Start(parentCtx, "backend."+s3action) + ctx.SetUserContext(backendCtx) response, err := controller(ctx) + if err != nil { + backendSpan.RecordError(err) + backendSpan.SetStatus(codes.Error, "") + } + backendSpan.End() + ctx.SetUserContext(parentCtx) // Set the response headers SetResponseHeaders(ctx, response.Headers) diff --git a/s3api/middlewares/acl-parser.go b/s3api/middlewares/acl-parser.go index e138a11f..118ed140 100644 --- a/s3api/middlewares/acl-parser.go +++ b/s3api/middlewares/acl-parser.go @@ -21,6 +21,9 @@ import ( "github.com/versity/versitygw/backend" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" ) // ParseAcl retreives the bucket acl and stores in the context locals @@ -28,8 +31,18 @@ import ( func ParseAcl(be backend.Backend) fiber.Handler { return func(ctx *fiber.Ctx) error { bucket := ctx.Params("bucket") + + parentCtx := ctx.UserContext() + sctx, span := otel.Tracer(tracerName).Start(parentCtx, "middleware.ParseAcl") + span.SetAttributes(attribute.String("s3.bucket", bucket)) + defer span.End() + ctx.SetUserContext(sctx) + defer ctx.SetUserContext(parentCtx) + data, err := be.GetBucketAcl(ctx.Context(), &s3.GetBucketAclInput{Bucket: &bucket}) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "") return err } diff --git a/s3api/middlewares/authentication.go b/s3api/middlewares/authentication.go index 91ed7797..beb188fc 100644 --- a/s3api/middlewares/authentication.go +++ b/s3api/middlewares/authentication.go @@ -25,6 +25,9 @@ import ( "github.com/versity/versitygw/auth" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" ) const ( @@ -50,6 +53,12 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string, return nil } + parentCtx := ctx.UserContext() + sctx, span := otel.Tracer(tracerName).Start(parentCtx, "middleware.VerifyV4Signature") + defer span.End() + ctx.SetUserContext(sctx) + defer ctx.SetUserContext(parentCtx) + // Check X-Amz-Date header date := ctx.Get("X-Amz-Date") if date == "" { @@ -84,14 +93,26 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string, utils.ContextKeyIsRoot.Set(ctx, authData.Access == root.Access) + _, iamSpan := otel.Tracer(tracerName).Start(sctx, "iam.GetUserAccount") account, err := acct.getAccount(authData.Access) + if err != nil { + iamSpan.RecordError(err) + iamSpan.SetStatus(codes.Error, "") + } + iamSpan.End() + if err == auth.ErrNoSuchUser { + span.SetStatus(codes.Error, "") return s3err.GetAPIError(s3err.ErrInvalidAccessKeyID) } if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "") return err } + span.SetAttributes(attribute.Bool("auth.is_root", authData.Access == root.Access)) + if date[:8] != authData.Date { return s3err.MalformedAuth.DateMismatch() } @@ -170,6 +191,8 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string, err = utils.CheckValidSignature(ctx, authData, account.Secret, hashPayload, tdate, contentLength, false) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "") return err } diff --git a/s3api/middlewares/checksum.go b/s3api/middlewares/checksum.go index 9ca12532..006dbb28 100644 --- a/s3api/middlewares/checksum.go +++ b/s3api/middlewares/checksum.go @@ -23,6 +23,8 @@ import ( "github.com/gofiber/fiber/v2" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/codes" ) // VerifyChecksums parses, validates, and calculates the @@ -33,6 +35,12 @@ import ( // the x-amz-checksum-* headers are explicitly processed by the backend. func VerifyChecksums(streamBody bool, requireBody bool, requireChecksum bool) fiber.Handler { return func(ctx *fiber.Ctx) error { + parentCtx := ctx.UserContext() + sctx, span := otel.Tracer(tracerName).Start(parentCtx, "middleware.VerifyChecksums") + defer span.End() + ctx.SetUserContext(sctx) + defer ctx.SetUserContext(parentCtx) + md5sum := ctx.Get("Content-Md5") if streamBody { @@ -103,6 +111,8 @@ func VerifyChecksums(streamBody bool, requireBody bool, requireChecksum bool) fi if rdr != nil { _, err = io.Copy(io.Discard, rdr) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "") return err } } diff --git a/s3api/middlewares/presign-auth.go b/s3api/middlewares/presign-auth.go index 6bebafdb..c275b320 100644 --- a/s3api/middlewares/presign-auth.go +++ b/s3api/middlewares/presign-auth.go @@ -22,6 +22,8 @@ import ( "github.com/versity/versitygw/auth" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/codes" ) func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, region string, streamBody bool) fiber.Handler { @@ -36,6 +38,12 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, region return nil } + parentCtx := ctx.UserContext() + sctx, span := otel.Tracer(tracerName).Start(parentCtx, "middleware.VerifyPresignedV4Signature") + defer span.End() + ctx.SetUserContext(sctx) + defer ctx.SetUserContext(parentCtx) + if ctx.Request().URI().QueryArgs().Has("X-Amz-Security-Token") { // OIDC Authorization with X-Amz-Security-Token is not supported return s3err.QueryAuthErrors.SecurityTokenNotSupported() @@ -52,11 +60,21 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, region utils.ContextKeyIsRoot.Set(ctx, authData.Access == root.Access) + _, iamSpan := otel.Tracer(tracerName).Start(sctx, "iam.GetUserAccount") account, err := acct.getAccount(authData.Access) + if err != nil { + iamSpan.RecordError(err) + iamSpan.SetStatus(codes.Error, "") + } + iamSpan.End() + if err == auth.ErrNoSuchUser { + span.SetStatus(codes.Error, "") return s3err.GetAPIError(s3err.ErrInvalidAccessKeyID) } if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "") return err } utils.ContextKeyAccount.Set(ctx, account) @@ -90,6 +108,8 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, region err = utils.CheckPresignedSignature(ctx, authData, account.Secret, streamBody) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "") return err } diff --git a/s3api/middlewares/public-bucket.go b/s3api/middlewares/public-bucket.go index ba7dce52..4f4851ce 100644 --- a/s3api/middlewares/public-bucket.go +++ b/s3api/middlewares/public-bucket.go @@ -26,6 +26,9 @@ import ( "github.com/versity/versitygw/metrics" "github.com/versity/versitygw/s3api/utils" "github.com/versity/versitygw/s3err" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" ) // AuthorizePublicBucketAccess checks if the bucket grants public @@ -37,6 +40,13 @@ func AuthorizePublicBucketAccess(be backend.Backend, s3action string, policyPerm return nil } + parentCtx := ctx.UserContext() + sctx, span := otel.Tracer(tracerName).Start(parentCtx, "middleware.AuthorizePublicBucketAccess") + span.SetAttributes(attribute.String("s3.action", s3action)) + defer span.End() + ctx.SetUserContext(sctx) + defer ctx.SetUserContext(parentCtx) + switch s3action { case metrics.ActionListAllMyBuckets: return s3err.GetAPIError(s3err.ErrAccessDenied) @@ -57,8 +67,14 @@ func AuthorizePublicBucketAccess(be backend.Backend, s3action string, policyPerm } bucket, object := parsePath(ctx.Path()) + span.SetAttributes( + attribute.String("s3.bucket", bucket), + attribute.String("s3.object", object), + ) err := auth.VerifyPublicAccess(ctx.Context(), be, policyPermission, permission, bucket, object) if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "") if s3action == metrics.ActionHeadBucket { // add the bucket region header for HeadBucket // if anonymous access is denied diff --git a/s3api/middlewares/tracing.go b/s3api/middlewares/tracing.go new file mode 100644 index 00000000..365da5c7 --- /dev/null +++ b/s3api/middlewares/tracing.go @@ -0,0 +1,116 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package middlewares + +import ( + "github.com/gofiber/fiber/v2" + "github.com/valyala/fasthttp" + "github.com/versity/versitygw/s3api/utils" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" + "go.opentelemetry.io/otel/trace" +) + +const tracerName = "github.com/versity/versitygw" + +// fasthttpCarrier adapts a *fasthttp.RequestHeader to the OTel TextMapCarrier +// interface so that W3C Trace Context headers can be extracted from incoming +// requests in the Fiber / fasthttp stack. +type fasthttpCarrier struct { + header *fasthttp.RequestHeader +} + +func (c fasthttpCarrier) Get(key string) string { + return string(c.header.Peek(key)) +} + +func (c fasthttpCarrier) Set(key, value string) { + c.header.Set(key, value) +} + +func (c fasthttpCarrier) Keys() []string { + keys := make([]string, 0, 8) + c.header.VisitAll(func(k, _ []byte) { + keys = append(keys, string(k)) + }) + return keys +} + +// OtelTracing returns a Fiber middleware that: +// 1. Extracts an incoming W3C Trace Context / Baggage from the request headers. +// 2. Starts a server-side span for the request. +// 3. Stores the span context in the Fiber user context so downstream handlers +// can create child spans via otel.Tracer(...).Start(c.UserContext(), ...). +// 4. After the handler chain returns, updates the span name to the matched +// route pattern (low-cardinality), records the HTTP status code, and sets +// the span status. +func OtelTracing() fiber.Handler { + tracer := otel.Tracer(tracerName) + propagator := otel.GetTextMapPropagator() + + return func(c *fiber.Ctx) error { + // Extract parent trace context from incoming HTTP headers. + parentCtx := propagator.Extract( + c.UserContext(), + fasthttpCarrier{&c.Request().Header}, + ) + + // Start a server span. Use method+path as initial name; it is + // replaced below with the low-cardinality route pattern once routing + // has resolved. + ctx, span := tracer.Start( + parentCtx, + c.Method()+" "+c.Path(), + trace.WithSpanKind(trace.SpanKindServer), + trace.WithAttributes( + semconv.HTTPRequestMethodKey.String(c.Method()), + semconv.URLPathKey.String(c.Path()), + semconv.ServerAddressKey.String(c.Hostname()), + ), + ) + defer span.End() + + // Make the span context available to handlers. + c.SetUserContext(ctx) + + err := c.Next() + + // Prefer the resolved S3 action name (e.g. "s3_ListAllMyBuckets") as + // the span name; fall back to the low-cardinality route pattern. + spanName := "" + if action, ok := c.Locals(string(utils.ContextKeyS3Action)).(string); ok && action != "" { + spanName = action + span.SetAttributes(attribute.String("s3.action", action)) + } else if r := c.Route(); r != nil && r.Path != "" { + spanName = c.Method() + " " + r.Path + } + if spanName != "" { + span.SetName(spanName) + } + + statusCode := c.Response().StatusCode() + span.SetAttributes(semconv.HTTPResponseStatusCodeKey.Int(statusCode)) + + if err != nil || statusCode >= 400 { + span.SetStatus(codes.Error, "") + } else { + span.SetStatus(codes.Ok, "") + } + + return err + } +} diff --git a/s3api/server.go b/s3api/server.go index 0e7728b0..d91394f5 100644 --- a/s3api/server.go +++ b/s3api/server.go @@ -51,6 +51,7 @@ type S3ApiServer struct { health string maxConnections int maxRequests int + tracingEnabled bool } func New( @@ -116,6 +117,12 @@ func New( }) } + // initialize OpenTelemetry tracing middleware (must be early so all + // subsequent handlers execute within the request span). + if server.tracingEnabled { + app.Use(middlewares.OtelTracing()) + } + // initialize total requests cap limiter middleware app.Use(middlewares.RateLimiter(server.maxRequests, mm, l)) @@ -144,6 +151,13 @@ func WithTLS(cs *utils.CertStorage) Option { return func(s *S3ApiServer) { s.CertStorage = cs } } +// WithTracing enables the OpenTelemetry request-tracing middleware. A tracer +// provider must already be configured globally (e.g. via tracing.InitTracer) +// before the first request arrives. +func WithTracing() Option { + return func(s *S3ApiServer) { s.tracingEnabled = true } +} + // WithAdminServer runs admin endpoints with the gateway in the same network func WithAdminServer() Option { return func(s *S3ApiServer) { s.Router.WithAdmSrv = true } diff --git a/s3api/utils/context-keys.go b/s3api/utils/context-keys.go index 76dfde0b..002b9519 100644 --- a/s3api/utils/context-keys.go +++ b/s3api/utils/context-keys.go @@ -37,6 +37,7 @@ const ( ContextKeySkip ContextKey = "__skip" ContextKeyStack ContextKey = "stack" ContextKeyBucketOwner ContextKey = "bucket-owner" + ContextKeyS3Action ContextKey = "s3-action" ) func (ck ContextKey) Values() []ContextKey { diff --git a/tracing/Makefile b/tracing/Makefile new file mode 100644 index 00000000..7823b5b4 --- /dev/null +++ b/tracing/Makefile @@ -0,0 +1,27 @@ +COMPOSE = docker compose -f $(dir $(abspath $(lastword $(MAKEFILE_LIST))))docker-compose.yml + +.PHONY: up down restart reload logs ps + +## Start all containers in the background +up: + $(COMPOSE) up -d + +## Stop and remove all containers +down: + $(COMPOSE) down + +## Restart a specific service (e.g. make restart svc=tempo) or all services +restart: + $(COMPOSE) restart $(svc) + +## Reload Grafana dashboard provisioning without restarting containers +reload: + curl -s --user admin:admin -X POST http://localhost:3000/api/admin/provisioning/dashboards/reload + +## Follow logs (optionally filtered: make logs svc=tempo) +logs: + $(COMPOSE) logs -f $(svc) + +## Show running container status +ps: + $(COMPOSE) ps diff --git a/tracing/README.md b/tracing/README.md new file mode 100644 index 00000000..d610ed8c --- /dev/null +++ b/tracing/README.md @@ -0,0 +1,141 @@ +# Distributed Tracing + +versitygw supports distributed tracing via [OpenTelemetry](https://opentelemetry.io/). +Every S3 API request produces a trace that flows through the middleware stack and into the backend, giving you end-to-end latency breakdowns, error attribution, and per-operation metrics. + +## How it works + +### Initialization (`tracing/tracing.go`) + +When the `--otel-endpoint` flag (or `VGW_OTEL_ENDPOINT` / `OTEL_EXPORTER_OTLP_ENDPOINT` env var) is set, `main` calls `tracing.InitTracer`, which: + +1. Creates an **OTLP HTTP exporter** pointed at the given endpoint (e.g. `http://localhost:4318`). +2. Builds an OTel `Resource` containing the service name (`versitygw` by default, overridable via `--otel-service-name` / `VGW_OTEL_SERVICE_NAME`), process info, and OS attributes. +3. Installs the `TracerProvider` and a W3C **Trace Context + Baggage** propagator as global OTel objects. +4. Returns a `Shutdown` function that is `defer`-ed in `main` to flush in-flight spans on exit. + +If the flag is not set, no tracer is installed and all OTel calls are no-ops — there is zero overhead. + +### Request span (`s3api/middlewares/tracing.go`) + +`OtelTracing()` is the first Fiber middleware registered on the server when tracing is enabled. For every incoming request it: + +1. **Extracts** any parent `traceparent` / `tracestate` / `baggage` headers from the request, enabling trace context propagation from upstream callers (e.g. an AWS SDK client that sets W3C headers). +2. **Starts a server span** scoped to the full request lifetime, initially named `METHOD /path`. +3. **Injects the span context** into the Fiber `UserContext` so all downstream code can access it via `c.UserContext()`. +4. After the handler chain returns: + - Renames the span to the resolved **S3 action** (e.g. `s3_ListObjectsV2`) and records it as the `s3.action` attribute. If no action could be resolved, falls back to the low-cardinality route pattern. + - Records `http.response.status_code`. + - Sets span status to **Error** for any HTTP ≥ 400 or handler error, otherwise **Ok**. + +### Middleware child spans + +Each middleware that does non-trivial work creates a **child span** parented to the request span: + +| Source file | Span name | +|---|---| +| `middlewares/authentication.go` | `middleware.VerifyV4Signature`, `iam.GetUserAccount` | +| `middlewares/presign-auth.go` | `middleware.VerifyPresignedV4Signature`, `iam.GetUserAccount` | +| `middlewares/acl-parser.go` | `middleware.ParseAcl` | +| `middlewares/public-bucket.go` | `middleware.AuthorizePublicBucketAccess` | +| `middlewares/checksum.go` | `middleware.VerifyChecksums` | + +Errors in any of these spans call `span.RecordError(err)` and set the span status to Error, so failures are clearly visible in the trace waterfall. + +### Backend span (`s3api/controllers/base.go`) + +`ProcessController` wraps every backend call with a `backend.` child span (e.g. `backend.s3_GetObject`). This lets you see exactly how much of the total request latency was spent inside the storage backend vs. the middleware stack. + +### Span hierarchy for a typical request + +``` +s3_PutObject (server span — OtelTracing middleware) +├── middleware.VerifyV4Signature (auth middleware) +│ └── iam.GetUserAccount +├── middleware.ParseAcl +├── middleware.VerifyChecksums +└── backend.s3_PutObject (storage backend) +``` + +## Local observability stack + +The `tracing/` directory ships a Docker Compose file that runs the full local stack: + +| Service | Port | Purpose | +|---|---|---| +| [Grafana Tempo](https://grafana.com/oss/tempo/) | 4317 (gRPC), 4318 (HTTP) | Receives OTLP spans from versitygw | +| [Prometheus](https://prometheus.io/) | 9090 | Receives spanmetrics remote-written from Tempo | +| [Grafana](https://grafana.com/) | 3000 | Dashboards and trace explorer | + +Tempo is configured (via `tempo.yaml`) to run the **spanmetrics** metrics generator, which derives RED metrics (rate, error rate, latency percentiles) from the incoming spans and remote-writes them to Prometheus. The pre-built Grafana dashboard (`grafana-provisioning/dashboards/versitygw.json`) visualises these metrics alongside raw trace data. + +### Starting the stack + +```sh +make -C tracing up +# or from the tracing/ directory: +make up +``` + +### Stopping the stack + +```sh +make -C tracing down +``` + +### Running versitygw with tracing enabled + +```sh +./versitygw --otel-endpoint http://localhost:4318 [other flags…] +``` + +Spans are exported over OTLP HTTP to Tempo. Open Grafana at (credentials: `admin` / `admin`). + +### Exploring traces + +1. In Grafana, go to **Explore** and select the **Tempo** datasource. +2. Use the **Search** tab to filter by service name `versitygw`, span name, or attributes such as `s3.action`. +3. Use **TraceQL** for programmatic queries, for example: + ``` + { span.s3.action = "s3_GetObject" && status = error } + ``` + +### Viewing metrics + +Open the **versitygw** dashboard from the Grafana home page. It shows: + +- Total requests, request rate, internal server error rate, and p50/p95/p99 latency (top-of-page stats) +- Per-operation request rate and error bars +- Authentication error rate (401/403) over time +- Full RED metrics table per operation with drill-through links into Tempo +- Middleware overhead breakdown — absolute latency and percentage of total request time for each middleware layer + +### Reloading the dashboard after edits + +```sh +make -C tracing reload +``` + +This calls the Grafana provisioning reload API without restarting any containers. + +## Adding new spans + +To instrument a new codepath, retrieve the span context from the Fiber `UserContext` and start a child span: + +```go +import ( + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/codes" +) + +ctx, span := otel.Tracer("github.com/versity/versitygw").Start(c.UserContext(), "my.operation") +defer span.End() + +if err := doWork(ctx); err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "") + return err +} +``` + +The new span automatically becomes a child of the current request span and appears in the Tempo waterfall view. diff --git a/tracing/docker-compose.yml b/tracing/docker-compose.yml new file mode 100644 index 00000000..23a0ce63 --- /dev/null +++ b/tracing/docker-compose.yml @@ -0,0 +1,52 @@ +# Minimal Grafana Tempo + Grafana stack for local trace viewing. +# +# Usage: +# docker compose -f tracing/docker-compose.yml up -d +# +# Then start versitygw with: +# --otel-endpoint http://localhost:4318 +# +# Open Grafana at http://localhost:3000 (admin / admin) +# Go to Explore → select "Tempo" datasource → run a TraceQL query or browse the +# "Search" tab to find traces by service name, span attributes, etc. + +services: + tempo: + image: grafana/tempo:2.6.1 + command: ["-config.file=/etc/tempo.yaml"] + volumes: + - ./tempo.yaml:/etc/tempo.yaml:ro + - tempo-data:/var/tempo + ports: + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP ← versitygw sends here + - "3200:3200" # Tempo query API (used by Grafana datasource) + + prometheus: + image: prom/prometheus:latest + command: + - --config.file=/etc/prometheus/prometheus.yml + - --web.enable-remote-write-receiver + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus + ports: + - "9090:9090" + + grafana: + image: grafana/grafana:latest + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_AUTH_DISABLE_LOGIN_FORM=false + volumes: + - ./grafana-provisioning:/etc/grafana/provisioning:ro + ports: + - "3000:3000" + depends_on: + - tempo + - prometheus + +volumes: + tempo-data: + prometheus-data: diff --git a/tracing/grafana-provisioning/dashboards/provider.yaml b/tracing/grafana-provisioning/dashboards/provider.yaml new file mode 100644 index 00000000..441baf77 --- /dev/null +++ b/tracing/grafana-provisioning/dashboards/provider.yaml @@ -0,0 +1,7 @@ +apiVersion: 1 + +providers: + - name: versitygw + type: file + options: + path: /etc/grafana/provisioning/dashboards diff --git a/tracing/grafana-provisioning/dashboards/versitygw.json b/tracing/grafana-provisioning/dashboards/versitygw.json new file mode 100644 index 00000000..a3735eab --- /dev/null +++ b/tracing/grafana-provisioning/dashboards/versitygw.json @@ -0,0 +1,1368 @@ +{ + "title": "VersityGW \u2013 S3 Gateway Traces", + "uid": "versitygw-traces", + "schemaVersion": 38, + "refresh": "10s", + "time": { + "from": "now-1h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "operation", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": "label_values(traces_spanmetrics_calls_total{service=\"versitygw\", span_name=~\"s3_.+\"}, span_name)", + "includeAll": true, + "allValue": ".+", + "multi": true, + "label": "Operation", + "refresh": 2, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + } + } + ] + }, + "panels": [ + { + "id": 10, + "type": "stat", + "title": "Total Requests", + "gridPos": { + "x": 0, + "y": 0, + "w": 4, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + } + }, + "options": { + "reduceOptions": { + "calcs": [ + "sum" + ] + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background" + }, + "targets": [ + { + "expr": "sum(increase(traces_spanmetrics_calls_total{service=\"versitygw\"}[$__range]))" + } + ] + }, + { + "id": 11, + "type": "stat", + "title": "Request Rate", + "gridPos": { + "x": 4, + "y": 0, + "w": 4, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + } + ] + } + } + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background" + }, + "targets": [ + { + "expr": "sum(rate(traces_spanmetrics_calls_total{service=\"versitygw\"}[1m]))" + } + ] + }, + { + "id": 12, + "type": "stat", + "title": "Internal Server Error Rate", + "gridPos": { + "x": 8, + "y": 0, + "w": 4, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "color": { + "mode": "thresholds" + } + } + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background" + }, + "targets": [ + { + "expr": "100 * sum(rate(traces_spanmetrics_calls_total{service=\"versitygw\", http_response_status_code=~\"5..\"}[1m])) / sum(rate(traces_spanmetrics_calls_total{service=\"versitygw\"}[1m]))" + } + ] + }, + { + "id": 13, + "type": "stat", + "title": "p50 Latency", + "gridPos": { + "x": 12, + "y": 0, + "w": 4, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 100 + }, + { + "color": "red", + "value": 500 + } + ] + } + } + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background" + }, + "targets": [ + { + "expr": "1000 * histogram_quantile(0.50, sum(rate(traces_spanmetrics_latency_bucket{service=\"versitygw\"}[1m])) by (le))" + } + ] + }, + { + "id": 14, + "type": "stat", + "title": "p95 Latency", + "gridPos": { + "x": 16, + "y": 0, + "w": 4, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 200 + }, + { + "color": "red", + "value": 1000 + } + ] + } + } + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background" + }, + "targets": [ + { + "expr": "1000 * histogram_quantile(0.95, sum(rate(traces_spanmetrics_latency_bucket{service=\"versitygw\"}[1m])) by (le))" + } + ] + }, + { + "id": 15, + "type": "stat", + "title": "p99 Latency", + "gridPos": { + "x": 20, + "y": 0, + "w": 4, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "color": { + "mode": "thresholds" + }, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 500 + }, + { + "color": "red", + "value": 2000 + } + ] + } + } + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background" + }, + "targets": [ + { + "expr": "1000 * histogram_quantile(0.99, sum(rate(traces_spanmetrics_latency_bucket{service=\"versitygw\"}[1m])) by (le))" + } + ] + }, + { + "id": 200, + "type": "bargauge", + "title": "Requests by API Operation (rate/s)", + "gridPos": { + "x": 0, + "y": 4, + "w": 12, + "h": 9 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + } + }, + "options": { + "orientation": "horizontal", + "displayMode": "basic", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "values": false + }, + "showUnfilled": true, + "maxVizHeight": 32, + "sizing": "manual", + "minVizHeight": 32, + "namePlacement": "top" + }, + "targets": [ + { + "refId": "A", + "expr": "sort_desc(sum by (span_name) (rate(traces_spanmetrics_calls_total{service=\"versitygw\",span_name=~\"s3_.+\"}[5m])))", + "legendFormat": "{{span_name}}", + "instant": true, + "range": false + } + ] + }, + { + "id": 201, + "type": "bargauge", + "title": "Errors by API Operation (rate/s)", + "gridPos": { + "x": 12, + "y": 4, + "w": 12, + "h": 9 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + } + } + }, + "options": { + "orientation": "horizontal", + "displayMode": "basic", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "values": false + }, + "showUnfilled": true, + "maxVizHeight": 32, + "sizing": "manual", + "minVizHeight": 32, + "namePlacement": "top" + }, + "targets": [ + { + "refId": "A", + "expr": "sort_desc(sum by (span_name) (rate(traces_spanmetrics_calls_total{service=\"versitygw\",span_name=~\"s3_.+\",status_code=\"STATUS_CODE_ERROR\"}[5m])))", + "legendFormat": "{{span_name}}", + "instant": true, + "range": false + } + ], + "transformations": [] + }, + { + "id": 202, + "type": "timeseries", + "title": "Authentication Error Rate (401 / 403)", + "description": "Rate of 401 Unauthorized and 403 Forbidden responses over time, broken down by operation. Spikes indicate authentication or authorization failures.", + "gridPos": { + "x": 0, + "y": 13, + "w": 24, + "h": 9 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "lineWidth": 2 + }, + "color": { + "mode": "palette-classic" + } + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } + }, + "targets": [ + { + "expr": "sum by (span_name) (rate(traces_spanmetrics_calls_total{service=\"versitygw\", http_response_status_code=~\"401|403\", span_name=~\"s3_.+\"}[1m]))", + "legendFormat": "{{span_name}}", + "refId": "A" + } + ] + }, + { + "id": 30, + "type": "table", + "title": "Operations \u2013 RED Metrics", + "gridPos": { + "x": 0, + "y": 22, + "w": 24, + "h": 9 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "options": { + "sortBy": [ + { + "displayName": "Rate (req/s)", + "desc": true + } + ], + "footer": { + "show": false + } + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "left" + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Rate (req/s)" + }, + "properties": [ + { + "id": "unit", + "value": "reqps" + }, + { + "id": "custom.width", + "value": 130 + }, + { + "id": "custom.displayMode", + "value": "color-background" + }, + { + "id": "color", + "value": { + "mode": "continuous-BlPu" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Error %" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + }, + { + "id": "custom.width", + "value": 100 + }, + { + "id": "custom.displayMode", + "value": "color-background" + }, + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "thresholds", + "value": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p50 (ms)" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + }, + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p95 (ms)" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + }, + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99 (ms)" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + }, + { + "id": "custom.width", + "value": 100 + }, + { + "id": "custom.displayMode", + "value": "color-background" + }, + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "thresholds", + "value": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 500 + }, + { + "color": "red", + "value": 2000 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Operation" + }, + "properties": [ + { + "id": "custom.width", + "value": 280 + }, + { + "id": "links", + "value": [ + { + "title": "View traces", + "url": "/explore?orgId=1&left=%7B%22datasource%22:%22tempo%22,%22queries%22:%5B%7B%22queryType%22:%22traceql%22,%22query%22:%22%7B%20span.s3.action%20%3D%20%5C%22${__data.fields.Operation}%5C%22%20%7D%22%7D%5D%7D" + } + ] + } + ] + } + ] + }, + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "renameByName": { + "span_name": "Operation", + "Value #A": "Rate (req/s)", + "Value #B": "Error %", + "Value #C": "p50 (ms)", + "Value #D": "p95 (ms)", + "Value #E": "p99 (ms)" + }, + "excludeByName": { + "Time": true + } + } + } + ], + "targets": [ + { + "expr": "sum by (span_name) (rate(traces_spanmetrics_calls_total{service=\"versitygw\", span_name=~\"s3_.+\", span_name=~\"$operation\"}[1m]))", + "instant": true, + "legendFormat": "", + "refId": "A", + "format": "table" + }, + { + "expr": "100 * sum by (span_name) (rate(traces_spanmetrics_calls_total{service=\"versitygw\", span_name=~\"s3_.+\", span_name=~\"$operation\", status_code=\"STATUS_CODE_ERROR\"}[1m])) / sum by (span_name) (rate(traces_spanmetrics_calls_total{service=\"versitygw\", span_name=~\"s3_.+\", span_name=~\"$operation\"}[1m]))", + "instant": true, + "legendFormat": "", + "refId": "B", + "format": "table" + }, + { + "expr": "1000 * histogram_quantile(0.50, sum by (span_name, le) (rate(traces_spanmetrics_latency_bucket{service=\"versitygw\", span_name=~\"s3_.+\", span_name=~\"$operation\"}[1m])))", + "instant": true, + "legendFormat": "", + "refId": "C", + "format": "table" + }, + { + "expr": "1000 * histogram_quantile(0.95, sum by (span_name, le) (rate(traces_spanmetrics_latency_bucket{service=\"versitygw\", span_name=~\"s3_.+\", span_name=~\"$operation\"}[1m])))", + "instant": true, + "legendFormat": "", + "refId": "D", + "format": "table" + }, + { + "expr": "1000 * histogram_quantile(0.99, sum by (span_name, le) (rate(traces_spanmetrics_latency_bucket{service=\"versitygw\", span_name=~\"s3_.+\", span_name=~\"$operation\"}[1m])))", + "instant": true, + "legendFormat": "", + "refId": "E", + "format": "table" + } + ] + }, + { + "id": 1, + "type": "timeseries", + "title": "Request Rate by Operation", + "gridPos": { + "x": 0, + "y": 31, + "w": 12, + "h": 9 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "lineWidth": 2 + } + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ], + "sortBy": "Mean", + "sortDesc": true + } + }, + "targets": [ + { + "expr": "sum by (span_name) (rate(traces_spanmetrics_calls_total{service=\"versitygw\", span_name=~\"s3_.+\", span_name=~\"$operation\"}[1m]))", + "legendFormat": "{{span_name}}" + } + ] + }, + { + "id": 2, + "type": "timeseries", + "title": "p99 Latency by Operation", + "gridPos": { + "x": 12, + "y": 31, + "w": 12, + "h": 9 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "lineWidth": 2 + } + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ], + "sortBy": "Mean", + "sortDesc": true + } + }, + "targets": [ + { + "expr": "1000 * histogram_quantile(0.99, sum by (span_name, le) (rate(traces_spanmetrics_latency_bucket{service=\"versitygw\", span_name=~\"s3_.+\", span_name=~\"$operation\"}[1m])))", + "legendFormat": "{{span_name}}" + } + ] + }, + { + "id": 50, + "type": "row", + "title": "Middleware Overhead", + "gridPos": { + "x": 0, + "y": 40, + "w": 24, + "h": 1 + }, + "collapsed": false + }, + { + "id": 40, + "type": "timeseries", + "title": "Middleware p50 Latency (ms)", + "description": "p50 latency of each middleware span over time. One line per layer \u2014 lets you spot which middleware is slowest or where latency is spiking.", + "gridPos": { + "x": 0, + "y": 41, + "w": 12, + "h": 9 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "lineWidth": 2 + } + } + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "mean", + "max" + ] + } + }, + "targets": [ + { + "expr": "1000 * histogram_quantile(0.50, sum by (span_name, le) (rate(traces_spanmetrics_latency_bucket{service=\"versitygw\", span_name=~\"middleware\\\\..+|iam\\\\..+\"}[5m])))", + "legendFormat": "{{span_name}}" + } + ] + }, + { + "id": 41, + "type": "bargauge", + "title": "Middleware Share of Total Request Time (p50 %)", + "description": "Each middleware span's p50 latency expressed as a percentage of the overall (non-middleware) request p50. Longer bars mean that layer is consuming more of the total request budget.", + "gridPos": { + "x": 12, + "y": 41, + "w": 12, + "h": 9 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 30 + } + ] + } + } + }, + "options": { + "orientation": "horizontal", + "displayMode": "gradient", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "expr": "100 * histogram_quantile(0.50, sum by (span_name, le) (rate(traces_spanmetrics_latency_bucket{service=\"versitygw\", span_name=~\"middleware\\\\..+|iam\\\\..+\"}[5m]))) / on() group_left() histogram_quantile(0.50, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"versitygw\", span_name!~\"middleware\\\\..+|iam\\\\..+\"}[5m])))", + "legendFormat": "{{span_name}}", + "instant": true, + "range": false + } + ] + }, + { + "id": 42, + "type": "table", + "title": "Middleware Overhead \u2013 p50 / p95 / p99 & % of Request Time", + "description": "Absolute latency and percentage share of total request time for each middleware and IAM sub-span. '% of Req p50/p95' uses the blended non-middleware percentile as the denominator.", + "gridPos": { + "x": 0, + "y": 50, + "w": 24, + "h": 9 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "options": { + "sortBy": [ + { + "displayName": "p50 (ms)", + "desc": true + } + ], + "footer": { + "show": false + } + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "left" + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Middleware" + }, + "properties": [ + { + "id": "custom.width", + "value": 300 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p50 (ms)" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + }, + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p95 (ms)" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + }, + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p99 (ms)" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + }, + { + "id": "custom.width", + "value": 100 + }, + { + "id": "custom.displayMode", + "value": "color-background" + }, + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "thresholds", + "value": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "red", + "value": 200 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "% of Req p50" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + }, + { + "id": "custom.width", + "value": 130 + }, + { + "id": "custom.displayMode", + "value": "color-background" + }, + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "thresholds", + "value": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 30 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "% of Req p95" + }, + "properties": [ + { + "id": "unit", + "value": "percent" + }, + { + "id": "custom.width", + "value": 130 + }, + { + "id": "custom.displayMode", + "value": "color-background" + }, + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "thresholds", + "value": { + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 30 + } + ] + } + } + ] + } + ] + }, + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "renameByName": { + "span_name": "Middleware", + "Value #A": "p50 (ms)", + "Value #B": "p95 (ms)", + "Value #C": "p99 (ms)", + "Value #D": "% of Req p50", + "Value #E": "% of Req p95" + }, + "excludeByName": { + "Time": true + } + } + } + ], + "targets": [ + { + "refId": "A", + "instant": true, + "format": "table", + "legendFormat": "", + "expr": "1000 * histogram_quantile(0.50, sum by (span_name, le) (rate(traces_spanmetrics_latency_bucket{service=\"versitygw\", span_name=~\"middleware\\\\..+|iam\\\\..+\"}[5m])))", + "range": false + }, + { + "refId": "B", + "instant": true, + "format": "table", + "legendFormat": "", + "expr": "1000 * histogram_quantile(0.95, sum by (span_name, le) (rate(traces_spanmetrics_latency_bucket{service=\"versitygw\", span_name=~\"middleware\\\\..+|iam\\\\..+\"}[5m])))", + "range": false + }, + { + "refId": "C", + "instant": true, + "format": "table", + "legendFormat": "", + "expr": "1000 * histogram_quantile(0.99, sum by (span_name, le) (rate(traces_spanmetrics_latency_bucket{service=\"versitygw\", span_name=~\"middleware\\\\..+|iam\\\\..+\"}[5m])))", + "range": false + }, + { + "refId": "D", + "instant": true, + "format": "table", + "legendFormat": "", + "expr": "100 * histogram_quantile(0.50, sum by (span_name, le) (rate(traces_spanmetrics_latency_bucket{service=\"versitygw\", span_name=~\"middleware\\\\..+|iam\\\\..+\"}[5m]))) / on() group_left() histogram_quantile(0.50, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"versitygw\", span_name!~\"middleware\\\\..+|iam\\\\..+\"}[5m])))", + "range": false + }, + { + "refId": "E", + "instant": true, + "format": "table", + "legendFormat": "", + "expr": "100 * histogram_quantile(0.95, sum by (span_name, le) (rate(traces_spanmetrics_latency_bucket{service=\"versitygw\", span_name=~\"middleware\\\\..+|iam\\\\..+\"}[5m]))) / on() group_left() histogram_quantile(0.95, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"versitygw\", span_name!~\"middleware\\\\..+|iam\\\\..+\"}[5m])))", + "range": false + } + ] + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 60, + "title": "Backend Action Overhead", + "type": "row" + }, + { + "id": 61, + "type": "timeseries", + "title": "Backend Action p50 Latency (ms)", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 60 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "editorMode": "code", + "expr": "(histogram_quantile(0.5, sum by(le, span_name) (rate(traces_spanmetrics_latency_bucket{service=\"versitygw\",span_name=~\"backend\\\\.s3_.+\"}[5m])))) * 1000", + "hide": false, + "instant": false, + "range": true, + "legendFormat": "{{span_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "lineWidth": 2, + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom" + } + } + }, + { + "id": 62, + "type": "bargauge", + "title": "Backend Share of Request Time (p50 %)", + "description": "For each action, what fraction of the root request span duration is spent in the backend call.", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 60 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "editorMode": "code", + "expr": "100 * histogram_quantile(0.5, sum by(le, span_name) (rate(traces_spanmetrics_latency_bucket{service=\"versitygw\",span_name=~\"backend\\\\.s3_.+\"}[5m]))) / on(span_name) group_left() label_replace(histogram_quantile(0.5, sum by(le, span_name) (rate(traces_spanmetrics_latency_bucket{service=\"versitygw\",span_name=~\"s3_.+\",span_name!~\"backend\\\\..+\"}[5m]))), \"span_name\", \"backend.$1\", \"span_name\", \"(s3_.+)\")", + "hide": false, + "instant": true, + "range": false, + "legendFormat": "{{span_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "values": false, + "fields": "" + }, + "orientation": "horizontal", + "displayMode": "gradient", + "valueMode": "color", + "showUnfilled": true + } + } + ] +} \ No newline at end of file diff --git a/tracing/grafana-provisioning/datasources/prometheus.yaml b/tracing/grafana-provisioning/datasources/prometheus.yaml new file mode 100644 index 00000000..0437aac9 --- /dev/null +++ b/tracing/grafana-provisioning/datasources/prometheus.yaml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + uid: prometheus + url: http://prometheus:9090 + access: proxy + isDefault: false diff --git a/tracing/grafana-provisioning/datasources/tempo.yaml b/tracing/grafana-provisioning/datasources/tempo.yaml new file mode 100644 index 00000000..e7da6845 --- /dev/null +++ b/tracing/grafana-provisioning/datasources/tempo.yaml @@ -0,0 +1,21 @@ +apiVersion: 1 + +datasources: + - name: Tempo + type: tempo + uid: tempo + url: http://tempo:3200 + access: proxy + isDefault: true + jsonData: + httpMethod: GET + serviceMap: + datasourceUid: tempo + search: + hide: false + nodeGraph: + enabled: true + traceQuery: + timeShiftEnabled: true + spanStartTimeShift: 1h + spanEndTimeShift: 1h diff --git a/tracing/prometheus.yml b/tracing/prometheus.yml new file mode 100644 index 00000000..443d91cf --- /dev/null +++ b/tracing/prometheus.yml @@ -0,0 +1,7 @@ +global: + scrape_interval: 15s + +scrape_configs: + - job_name: tempo + static_configs: + - targets: ['tempo:3200'] diff --git a/tracing/tempo.yaml b/tracing/tempo.yaml new file mode 100644 index 00000000..88c3f38b --- /dev/null +++ b/tracing/tempo.yaml @@ -0,0 +1,64 @@ +# Grafana Tempo configuration — single-binary / local-storage mode. +# Suitable for local development; not for production use. + +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + grpc: + endpoint: 0.0.0.0:4317 + +ingester: + trace_idle_period: 10s + max_block_bytes: 1_000_000 + max_block_duration: 5m + +storage: + trace: + backend: local + local: + path: /var/tempo/blocks + wal: + path: /var/tempo/wal + +metrics_generator: + registry: + external_labels: + source: tempo + storage: + path: /var/tempo/generator/wal + remote_write: + - url: http://prometheus:9090/api/v1/write + processor: + span_metrics: + dimensions: + - http.response.status_code + histogram_buckets: + - 0.000005 # 5µs + - 0.00001 # 10µs + - 0.000025 # 25µs + - 0.00005 # 50µs + - 0.0001 # 0.1ms + - 0.0002 + - 0.0005 + - 0.001 + - 0.002 + - 0.005 + - 0.01 + - 0.025 + - 0.05 + - 0.1 + - 0.25 + - 0.5 + - 1.0 + - 2.5 + - 5.0 + - 10.0 + +overrides: + metrics_generator_processors: [service-graphs, span-metrics] diff --git a/tracing/tracing.go b/tracing/tracing.go new file mode 100644 index 00000000..09722a75 --- /dev/null +++ b/tracing/tracing.go @@ -0,0 +1,71 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package tracing + +import ( + "context" + "fmt" + "strings" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" +) + +// InitTracer configures the global OpenTelemetry tracer provider with an OTLP +// HTTP exporter. endpointURL is a full URL such as "http://localhost:4318" or +// "https://otel-collector:4318"; if no scheme is present, "http://" is assumed. +// +// Returns a shutdown function that flushes and stops the provider. Call it +// before process exit (typically via defer in main/runGateway). +func InitTracer(ctx context.Context, serviceName, endpointURL string) (func(context.Context) error, error) { + if !strings.HasPrefix(endpointURL, "http://") && !strings.HasPrefix(endpointURL, "https://") { + endpointURL = "http://" + endpointURL + } + + exp, err := otlptracehttp.New(ctx, + otlptracehttp.WithEndpointURL(endpointURL), + ) + if err != nil { + return nil, fmt.Errorf("create OTLP trace exporter: %w", err) + } + + res, err := resource.New(ctx, + resource.WithAttributes(semconv.ServiceName(serviceName)), + resource.WithProcess(), + resource.WithOS(), + ) + if err != nil { + // resource.New can return a partially-populated resource alongside a + // non-fatal partial error; use it anyway. + fmt.Printf("WARNING: OTel resource detection: %v\n", err) + } + + tp := sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exp), + sdktrace.WithResource(res), + ) + + otel.SetTracerProvider(tp) + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + )) + + return tp.Shutdown, nil +}