otel tracing wip

2026-08-22 15:16:30 +00:00 · 2026-03-05 19:12:22 -08:00
parent 1595d2cf7f
commit b33090b6ac
22 changed files with 2074 additions and 0 deletions
@@ -36,6 +36,7 @@ import (
 	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3event"
 	"github.com/versity/versitygw/s3log"
+	"github.com/versity/versitygw/tracing"
 	"github.com/versity/versitygw/webui"
 )

@@ -95,6 +96,8 @@ var (
 	ipaUser, ipaPassword                   string
 	ipaInsecure                            bool
 	iamDebug                               bool
+	otelEndpoint                           string
+	otelServiceName                        string
 	webuiPorts                             []string
 	webuiCertFile, webuiKeyFile            string
 	webuiNoTLS                             bool
@@ -727,6 +730,19 @@ func initFlags() []cli.Flag {
 			EnvVars:     []string{"VGW_IPA_INSECURE"},
 			Destination: &ipaInsecure,
 		},
+		&cli.StringFlag{
+			Name:        "otel-endpoint",
+			Usage:       "OpenTelemetry collector endpoint URL for tracing (e.g. http://localhost:4318). Tracing is disabled when unset.",
+			EnvVars:     []string{"VGW_OTEL_ENDPOINT", "OTEL_EXPORTER_OTLP_ENDPOINT"},
+			Destination: &otelEndpoint,
+		},
+		&cli.StringFlag{
+			Name:        "otel-service-name",
+			Usage:       "Service name reported in OpenTelemetry traces",
+			EnvVars:     []string{"VGW_OTEL_SERVICE_NAME", "OTEL_SERVICE_NAME"},
+			Value:       "versitygw",
+			Destination: &otelServiceName,
+		},
 	}
 }

@@ -844,6 +860,15 @@ func runGateway(ctx context.Context, be backend.Backend) error {
 	if disableACLs {
 		opts = append(opts, s3api.WithDisableACL())
 	}
+	if otelEndpoint != "" {
+		shutdownTracer, err := tracing.InitTracer(ctx, otelServiceName, otelEndpoint)
+		if err != nil {
+			return fmt.Errorf("init otel tracer: %w", err)
+		}
+		defer shutdownTracer(context.Background()) //nolint:errcheck
+		opts = append(opts, s3api.WithTracing())
+	}
+
 	if debug {
 		debuglogger.SetDebugEnabled()
 	}
@@ -55,10 +55,15 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 // indirect
+	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
@@ -79,8 +84,20 @@ require (
 	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/otel v1.41.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.41.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.41.0 // indirect
+	go.opentelemetry.io/otel/metric v1.41.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.41.0 // indirect
+	go.opentelemetry.io/otel/trace v1.41.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
 	golang.org/x/crypto v0.48.0 // indirect
 	golang.org/x/net v0.51.0 // indirect
 	golang.org/x/text v0.34.0 // indirect
 	golang.org/x/time v0.14.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect
+	google.golang.org/grpc v1.79.1 // indirect
+	google.golang.org/protobuf v1.36.11 // indirect
 )
@@ -65,6 +65,10 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 h1:NITQpgo9A5NrDZ57uOWj+abvXSb8
 github.com/aws/aws-sdk-go-v2/service/sts v1.41.7/go.mod h1:sks5UWBhEuWYDPdwlnRFn1w7xWdH29Jcpe+/PJQefEs=
 github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
 github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
+github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM=
+github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
 github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
 github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
@@ -78,6 +82,11 @@ github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 h1:BP4M0CvQ
 github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-ldap/ldap/v3 v3.4.12 h1:1b81mv7MagXZ7+1r7cLTWmyuTqVqdwbtJSjC0DAp9s4=
 github.com/go-ldap/ldap/v3 v3.4.12/go.mod h1:+SPAGcTtOfmGsCb3h1RFiq4xpp4N636G75OEace8lNo=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/gofiber/fiber/v2 v2.52.12 h1:0LdToKclcPOj8PktUdIKo9BUohjjwfnQl42Dhw8/WUw=
 github.com/gofiber/fiber/v2 v2.52.12/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
 github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
@@ -87,6 +96,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
 github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k=
@@ -196,6 +207,22 @@ github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBi
 github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
 github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
+go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
+go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
+go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.41.0 h1:ao6Oe+wSebTlQ1OEht7jlYTzQKE+pnx/iNywFvTbuuI=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.41.0/go.mod h1:u3T6vz0gh/NVzgDgiwkgLxpsSF6PaPmo2il0apGJbls=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.41.0 h1:inYW9ZhgqiDqh6BioM7DVHHzEGVq76Db5897WLGZ5Go=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.41.0/go.mod h1:Izur+Wt8gClgMJqO/cZ8wdeeMryJ/xxiOVgFSSfpDTY=
+go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
+go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
+go.opentelemetry.io/otel/sdk v1.41.0 h1:YPIEXKmiAwkGl3Gu1huk1aYWwtpRLeskpV+wPisxBp8=
+go.opentelemetry.io/otel/sdk v1.41.0/go.mod h1:ahFdU0G5y8IxglBf0QBJXgSe7agzjE4GiTJ6HT9ud90=
+go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
+go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
+go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
+go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -237,6 +264,14 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 h1:JLQynH/LBHfCTSbDWl+py8C+Rg/k1OVH3xfcaiANuF0=
+google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:kSJwQxqmFXeo79zOmbrALdflXQeAYcUbgS7PbpMknCY=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 h1:mWPCjDEyshlQYzBpMNHaEof6UX1PmHcaUODUywQ0uac=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ=
+google.golang.org/grpc v1.79.1 h1:zGhSi45ODB9/p3VAawt9a+O/MULLl9dpizzNNpq7flY=
+google.golang.org/grpc v1.79.1/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
+google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
@@ -30,6 +30,8 @@ import (
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3event"
 	"github.com/versity/versitygw/s3log"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/codes"
 )

 type S3ApiController struct {
@@ -129,6 +131,10 @@ func ProcessHandlers(controller Controller, s3action string, svc *Services, hand
 			return ctx.Next()
 		}

+		// Store the resolved S3 action name so the tracing middleware can
+		// attach it to the span after routing has completed.
+		utils.ContextKeyS3Action.Set(ctx, s3action)
+
 		for _, handler := range handlers {
 			err := handler(ctx)
 			if err != nil {
@@ -179,10 +185,21 @@ func WrapMiddleware(handler fiber.Handler, logger s3log.AuditLogger, mm metrics.
 	}
 }

+const controllerTracerName = "github.com/versity/versitygw"
+
 // ProcessController executes the given s3api controller and handles the metrics
 // access logs and s3 events
 func ProcessController(ctx *fiber.Ctx, controller Controller, s3action string, svc *Services) error {
+	parentCtx := ctx.UserContext()
+	backendCtx, backendSpan := otel.Tracer(controllerTracerName).Start(parentCtx, "backend."+s3action)
+	ctx.SetUserContext(backendCtx)
 	response, err := controller(ctx)
+	if err != nil {
+		backendSpan.RecordError(err)
+		backendSpan.SetStatus(codes.Error, "")
+	}
+	backendSpan.End()
+	ctx.SetUserContext(parentCtx)

 	// Set the response headers
 	SetResponseHeaders(ctx, response.Headers)
@@ -21,6 +21,9 @@ import (
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
 )

 // ParseAcl retreives the bucket acl and stores in the context locals
@@ -28,8 +31,18 @@ import (
 func ParseAcl(be backend.Backend) fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
 		bucket := ctx.Params("bucket")
+
+		parentCtx := ctx.UserContext()
+		sctx, span := otel.Tracer(tracerName).Start(parentCtx, "middleware.ParseAcl")
+		span.SetAttributes(attribute.String("s3.bucket", bucket))
+		defer span.End()
+		ctx.SetUserContext(sctx)
+		defer ctx.SetUserContext(parentCtx)
+
 		data, err := be.GetBucketAcl(ctx.Context(), &s3.GetBucketAclInput{Bucket: &bucket})
 		if err != nil {
+			span.RecordError(err)
+			span.SetStatus(codes.Error, "")
 			return err
 		}

@@ -25,6 +25,9 @@ import (
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
 )

 const (
@@ -50,6 +53,12 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string,
 			return nil
 		}

+		parentCtx := ctx.UserContext()
+		sctx, span := otel.Tracer(tracerName).Start(parentCtx, "middleware.VerifyV4Signature")
+		defer span.End()
+		ctx.SetUserContext(sctx)
+		defer ctx.SetUserContext(parentCtx)
+
 		// Check X-Amz-Date header
 		date := ctx.Get("X-Amz-Date")
 		if date == "" {
@@ -84,14 +93,26 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string,

 		utils.ContextKeyIsRoot.Set(ctx, authData.Access == root.Access)

+		_, iamSpan := otel.Tracer(tracerName).Start(sctx, "iam.GetUserAccount")
 		account, err := acct.getAccount(authData.Access)
+		if err != nil {
+			iamSpan.RecordError(err)
+			iamSpan.SetStatus(codes.Error, "")
+		}
+		iamSpan.End()
+
 		if err == auth.ErrNoSuchUser {
+			span.SetStatus(codes.Error, "")
 			return s3err.GetAPIError(s3err.ErrInvalidAccessKeyID)
 		}
 		if err != nil {
+			span.RecordError(err)
+			span.SetStatus(codes.Error, "")
 			return err
 		}

+		span.SetAttributes(attribute.Bool("auth.is_root", authData.Access == root.Access))
+
 		if date[:8] != authData.Date {
 			return s3err.MalformedAuth.DateMismatch()
 		}
@@ -170,6 +191,8 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string,

 		err = utils.CheckValidSignature(ctx, authData, account.Secret, hashPayload, tdate, contentLength, false)
 		if err != nil {
+			span.RecordError(err)
+			span.SetStatus(codes.Error, "")
 			return err
 		}

@@ -23,6 +23,8 @@ import (
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/codes"
 )

 // VerifyChecksums parses, validates, and calculates the
@@ -33,6 +35,12 @@ import (
 // the x-amz-checksum-* headers are explicitly processed by the backend.
 func VerifyChecksums(streamBody bool, requireBody bool, requireChecksum bool) fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
+		parentCtx := ctx.UserContext()
+		sctx, span := otel.Tracer(tracerName).Start(parentCtx, "middleware.VerifyChecksums")
+		defer span.End()
+		ctx.SetUserContext(sctx)
+		defer ctx.SetUserContext(parentCtx)
+
 		md5sum := ctx.Get("Content-Md5")

 		if streamBody {
@@ -103,6 +111,8 @@ func VerifyChecksums(streamBody bool, requireBody bool, requireChecksum bool) fi
 		if rdr != nil {
 			_, err = io.Copy(io.Discard, rdr)
 			if err != nil {
+				span.RecordError(err)
+				span.SetStatus(codes.Error, "")
 				return err
 			}
 		}
@@ -22,6 +22,8 @@ import (
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/codes"
 )

 func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, region string, streamBody bool) fiber.Handler {
@@ -36,6 +38,12 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, region
 			return nil
 		}

+		parentCtx := ctx.UserContext()
+		sctx, span := otel.Tracer(tracerName).Start(parentCtx, "middleware.VerifyPresignedV4Signature")
+		defer span.End()
+		ctx.SetUserContext(sctx)
+		defer ctx.SetUserContext(parentCtx)
+
 		if ctx.Request().URI().QueryArgs().Has("X-Amz-Security-Token") {
 			// OIDC Authorization with X-Amz-Security-Token is not supported
 			return s3err.QueryAuthErrors.SecurityTokenNotSupported()
@@ -52,11 +60,21 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, region

 		utils.ContextKeyIsRoot.Set(ctx, authData.Access == root.Access)

+		_, iamSpan := otel.Tracer(tracerName).Start(sctx, "iam.GetUserAccount")
 		account, err := acct.getAccount(authData.Access)
+		if err != nil {
+			iamSpan.RecordError(err)
+			iamSpan.SetStatus(codes.Error, "")
+		}
+		iamSpan.End()
+
 		if err == auth.ErrNoSuchUser {
+			span.SetStatus(codes.Error, "")
 			return s3err.GetAPIError(s3err.ErrInvalidAccessKeyID)
 		}
 		if err != nil {
+			span.RecordError(err)
+			span.SetStatus(codes.Error, "")
 			return err
 		}
 		utils.ContextKeyAccount.Set(ctx, account)
@@ -90,6 +108,8 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, region

 		err = utils.CheckPresignedSignature(ctx, authData, account.Secret, streamBody)
 		if err != nil {
+			span.RecordError(err)
+			span.SetStatus(codes.Error, "")
 			return err
 		}

@@ -26,6 +26,9 @@ import (
 	"github.com/versity/versitygw/metrics"
 	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
 )

 // AuthorizePublicBucketAccess checks if the bucket grants public
@@ -37,6 +40,13 @@ func AuthorizePublicBucketAccess(be backend.Backend, s3action string, policyPerm
 			return nil
 		}

+		parentCtx := ctx.UserContext()
+		sctx, span := otel.Tracer(tracerName).Start(parentCtx, "middleware.AuthorizePublicBucketAccess")
+		span.SetAttributes(attribute.String("s3.action", s3action))
+		defer span.End()
+		ctx.SetUserContext(sctx)
+		defer ctx.SetUserContext(parentCtx)
+
 		switch s3action {
 		case metrics.ActionListAllMyBuckets:
 			return s3err.GetAPIError(s3err.ErrAccessDenied)
@@ -57,8 +67,14 @@ func AuthorizePublicBucketAccess(be backend.Backend, s3action string, policyPerm
 		}

 		bucket, object := parsePath(ctx.Path())
+		span.SetAttributes(
+			attribute.String("s3.bucket", bucket),
+			attribute.String("s3.object", object),
+		)
 		err := auth.VerifyPublicAccess(ctx.Context(), be, policyPermission, permission, bucket, object)
 		if err != nil {
+			span.RecordError(err)
+			span.SetStatus(codes.Error, "")
 			if s3action == metrics.ActionHeadBucket {
 				// add the bucket region header for HeadBucket
 				// if anonymous access is denied
@@ -0,0 +1,116 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"github.com/gofiber/fiber/v2"
+	"github.com/valyala/fasthttp"
+	"github.com/versity/versitygw/s3api/utils"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
+	"go.opentelemetry.io/otel/trace"
+)
+
+const tracerName = "github.com/versity/versitygw"
+
+// fasthttpCarrier adapts a *fasthttp.RequestHeader to the OTel TextMapCarrier
+// interface so that W3C Trace Context headers can be extracted from incoming
+// requests in the Fiber / fasthttp stack.
+type fasthttpCarrier struct {
+	header *fasthttp.RequestHeader
+}
+
+func (c fasthttpCarrier) Get(key string) string {
+	return string(c.header.Peek(key))
+}
+
+func (c fasthttpCarrier) Set(key, value string) {
+	c.header.Set(key, value)
+}
+
+func (c fasthttpCarrier) Keys() []string {
+	keys := make([]string, 0, 8)
+	c.header.VisitAll(func(k, _ []byte) {
+		keys = append(keys, string(k))
+	})
+	return keys
+}
+
+// OtelTracing returns a Fiber middleware that:
+//  1. Extracts an incoming W3C Trace Context / Baggage from the request headers.
+//  2. Starts a server-side span for the request.
+//  3. Stores the span context in the Fiber user context so downstream handlers
+//     can create child spans via otel.Tracer(...).Start(c.UserContext(), ...).
+//  4. After the handler chain returns, updates the span name to the matched
+//     route pattern (low-cardinality), records the HTTP status code, and sets
+//     the span status.
+func OtelTracing() fiber.Handler {
+	tracer := otel.Tracer(tracerName)
+	propagator := otel.GetTextMapPropagator()
+
+	return func(c *fiber.Ctx) error {
+		// Extract parent trace context from incoming HTTP headers.
+		parentCtx := propagator.Extract(
+			c.UserContext(),
+			fasthttpCarrier{&c.Request().Header},
+		)
+
+		// Start a server span.  Use method+path as initial name; it is
+		// replaced below with the low-cardinality route pattern once routing
+		// has resolved.
+		ctx, span := tracer.Start(
+			parentCtx,
+			c.Method()+" "+c.Path(),
+			trace.WithSpanKind(trace.SpanKindServer),
+			trace.WithAttributes(
+				semconv.HTTPRequestMethodKey.String(c.Method()),
+				semconv.URLPathKey.String(c.Path()),
+				semconv.ServerAddressKey.String(c.Hostname()),
+			),
+		)
+		defer span.End()
+
+		// Make the span context available to handlers.
+		c.SetUserContext(ctx)
+
+		err := c.Next()
+
+		// Prefer the resolved S3 action name (e.g. "s3_ListAllMyBuckets") as
+		// the span name; fall back to the low-cardinality route pattern.
+		spanName := ""
+		if action, ok := c.Locals(string(utils.ContextKeyS3Action)).(string); ok && action != "" {
+			spanName = action
+			span.SetAttributes(attribute.String("s3.action", action))
+		} else if r := c.Route(); r != nil && r.Path != "" {
+			spanName = c.Method() + " " + r.Path
+		}
+		if spanName != "" {
+			span.SetName(spanName)
+		}
+
+		statusCode := c.Response().StatusCode()
+		span.SetAttributes(semconv.HTTPResponseStatusCodeKey.Int(statusCode))
+
+		if err != nil || statusCode >= 400 {
+			span.SetStatus(codes.Error, "")
+		} else {
+			span.SetStatus(codes.Ok, "")
+		}
+
+		return err
+	}
+}
@@ -51,6 +51,7 @@ type S3ApiServer struct {
 	health         string
 	maxConnections int
 	maxRequests    int
+	tracingEnabled bool
 }

 func New(
@@ -116,6 +117,12 @@ func New(
 		})
 	}

+	// initialize OpenTelemetry tracing middleware (must be early so all
+	// subsequent handlers execute within the request span).
+	if server.tracingEnabled {
+		app.Use(middlewares.OtelTracing())
+	}
+
 	// initialize total requests cap limiter middleware
 	app.Use(middlewares.RateLimiter(server.maxRequests, mm, l))

@@ -144,6 +151,13 @@ func WithTLS(cs *utils.CertStorage) Option {
 	return func(s *S3ApiServer) { s.CertStorage = cs }
 }

+// WithTracing enables the OpenTelemetry request-tracing middleware.  A tracer
+// provider must already be configured globally (e.g. via tracing.InitTracer)
+// before the first request arrives.
+func WithTracing() Option {
+	return func(s *S3ApiServer) { s.tracingEnabled = true }
+}
+
 // WithAdminServer runs admin endpoints with the gateway in the same network
 func WithAdminServer() Option {
 	return func(s *S3ApiServer) { s.Router.WithAdmSrv = true }
@@ -37,6 +37,7 @@ const (
 	ContextKeySkip           ContextKey = "__skip"
 	ContextKeyStack          ContextKey = "stack"
 	ContextKeyBucketOwner    ContextKey = "bucket-owner"
+	ContextKeyS3Action       ContextKey = "s3-action"
 )

 func (ck ContextKey) Values() []ContextKey {
@@ -0,0 +1,27 @@
+COMPOSE = docker compose -f $(dir $(abspath $(lastword $(MAKEFILE_LIST))))docker-compose.yml
+
+.PHONY: up down restart reload logs ps
+
+## Start all containers in the background
+up:
+	$(COMPOSE) up -d
+
+## Stop and remove all containers
+down:
+	$(COMPOSE) down
+
+## Restart a specific service (e.g. make restart svc=tempo) or all services
+restart:
+	$(COMPOSE) restart $(svc)
+
+## Reload Grafana dashboard provisioning without restarting containers
+reload:
+	curl -s --user admin:admin -X POST http://localhost:3000/api/admin/provisioning/dashboards/reload
+
+## Follow logs (optionally filtered: make logs svc=tempo)
+logs:
+	$(COMPOSE) logs -f $(svc)
+
+## Show running container status
+ps:
+	$(COMPOSE) ps
@@ -0,0 +1,141 @@
+# Distributed Tracing
+
+versitygw supports distributed tracing via [OpenTelemetry](https://opentelemetry.io/).
+Every S3 API request produces a trace that flows through the middleware stack and into the backend, giving you end-to-end latency breakdowns, error attribution, and per-operation metrics.
+
+## How it works
+
+### Initialization (`tracing/tracing.go`)
+
+When the `--otel-endpoint` flag (or `VGW_OTEL_ENDPOINT` / `OTEL_EXPORTER_OTLP_ENDPOINT` env var) is set, `main` calls `tracing.InitTracer`, which:
+
+1. Creates an **OTLP HTTP exporter** pointed at the given endpoint (e.g. `http://localhost:4318`).
+2. Builds an OTel `Resource` containing the service name (`versitygw` by default, overridable via `--otel-service-name` / `VGW_OTEL_SERVICE_NAME`), process info, and OS attributes.
+3. Installs the `TracerProvider` and a W3C **Trace Context + Baggage** propagator as global OTel objects.
+4. Returns a `Shutdown` function that is `defer`-ed in `main` to flush in-flight spans on exit.
+
+If the flag is not set, no tracer is installed and all OTel calls are no-ops — there is zero overhead.
+
+### Request span (`s3api/middlewares/tracing.go`)
+
+`OtelTracing()` is the first Fiber middleware registered on the server when tracing is enabled. For every incoming request it:
+
+1. **Extracts** any parent `traceparent` / `tracestate` / `baggage` headers from the request, enabling trace context propagation from upstream callers (e.g. an AWS SDK client that sets W3C headers).
+2. **Starts a server span** scoped to the full request lifetime, initially named `METHOD /path`.
+3. **Injects the span context** into the Fiber `UserContext` so all downstream code can access it via `c.UserContext()`.
+4. After the handler chain returns:
+   - Renames the span to the resolved **S3 action** (e.g. `s3_ListObjectsV2`) and records it as the `s3.action` attribute. If no action could be resolved, falls back to the low-cardinality route pattern.
+   - Records `http.response.status_code`.
+   - Sets span status to **Error** for any HTTP ≥ 400 or handler error, otherwise **Ok**.
+
+### Middleware child spans
+
+Each middleware that does non-trivial work creates a **child span** parented to the request span:
+
+| Source file | Span name |
+|---|---|
+| `middlewares/authentication.go` | `middleware.VerifyV4Signature`, `iam.GetUserAccount` |
+| `middlewares/presign-auth.go` | `middleware.VerifyPresignedV4Signature`, `iam.GetUserAccount` |
+| `middlewares/acl-parser.go` | `middleware.ParseAcl` |
+| `middlewares/public-bucket.go` | `middleware.AuthorizePublicBucketAccess` |
+| `middlewares/checksum.go` | `middleware.VerifyChecksums` |
+
+Errors in any of these spans call `span.RecordError(err)` and set the span status to Error, so failures are clearly visible in the trace waterfall.
+
+### Backend span (`s3api/controllers/base.go`)
+
+`ProcessController` wraps every backend call with a `backend.<s3action>` child span (e.g. `backend.s3_GetObject`). This lets you see exactly how much of the total request latency was spent inside the storage backend vs. the middleware stack.
+
+### Span hierarchy for a typical request
+
+```
+s3_PutObject                              (server span — OtelTracing middleware)
+├── middleware.VerifyV4Signature           (auth middleware)
+│   └── iam.GetUserAccount
+├── middleware.ParseAcl
+├── middleware.VerifyChecksums
+└── backend.s3_PutObject                  (storage backend)
+```
+
+## Local observability stack
+
+The `tracing/` directory ships a Docker Compose file that runs the full local stack:
+
+| Service | Port | Purpose |
+|---|---|---|
+| [Grafana Tempo](https://grafana.com/oss/tempo/) | 4317 (gRPC), 4318 (HTTP) | Receives OTLP spans from versitygw |
+| [Prometheus](https://prometheus.io/) | 9090 | Receives spanmetrics remote-written from Tempo |
+| [Grafana](https://grafana.com/) | 3000 | Dashboards and trace explorer |
+
+Tempo is configured (via `tempo.yaml`) to run the **spanmetrics** metrics generator, which derives RED metrics (rate, error rate, latency percentiles) from the incoming spans and remote-writes them to Prometheus. The pre-built Grafana dashboard (`grafana-provisioning/dashboards/versitygw.json`) visualises these metrics alongside raw trace data.
+
+### Starting the stack
+
+```sh
+make -C tracing up
+# or from the tracing/ directory:
+make up
+```
+
+### Stopping the stack
+
+```sh
+make -C tracing down
+```
+
+### Running versitygw with tracing enabled
+
+```sh
+./versitygw --otel-endpoint http://localhost:4318 [other flags…]
+```
+
+Spans are exported over OTLP HTTP to Tempo. Open Grafana at <http://localhost:3000> (credentials: `admin` / `admin`).
+
+### Exploring traces
+
+1. In Grafana, go to **Explore** and select the **Tempo** datasource.
+2. Use the **Search** tab to filter by service name `versitygw`, span name, or attributes such as `s3.action`.
+3. Use **TraceQL** for programmatic queries, for example:
+   ```
+   { span.s3.action = "s3_GetObject" && status = error }
+   ```
+
+### Viewing metrics
+
+Open the **versitygw** dashboard from the Grafana home page. It shows:
+
+- Total requests, request rate, internal server error rate, and p50/p95/p99 latency (top-of-page stats)
+- Per-operation request rate and error bars
+- Authentication error rate (401/403) over time
+- Full RED metrics table per operation with drill-through links into Tempo
+- Middleware overhead breakdown — absolute latency and percentage of total request time for each middleware layer
+
+### Reloading the dashboard after edits
+
+```sh
+make -C tracing reload
+```
+
+This calls the Grafana provisioning reload API without restarting any containers.
+
+## Adding new spans
+
+To instrument a new codepath, retrieve the span context from the Fiber `UserContext` and start a child span:
+
+```go
+import (
+    "go.opentelemetry.io/otel"
+    "go.opentelemetry.io/otel/codes"
+)
+
+ctx, span := otel.Tracer("github.com/versity/versitygw").Start(c.UserContext(), "my.operation")
+defer span.End()
+
+if err := doWork(ctx); err != nil {
+    span.RecordError(err)
+    span.SetStatus(codes.Error, "")
+    return err
+}
+```
+
+The new span automatically becomes a child of the current request span and appears in the Tempo waterfall view.
@@ -0,0 +1,52 @@
+# Minimal Grafana Tempo + Grafana stack for local trace viewing.
+#
+# Usage:
+#   docker compose -f tracing/docker-compose.yml up -d
+#
+# Then start versitygw with:
+#   --otel-endpoint http://localhost:4318
+#
+# Open Grafana at http://localhost:3000  (admin / admin)
+# Go to Explore → select "Tempo" datasource → run a TraceQL query or browse the
+# "Search" tab to find traces by service name, span attributes, etc.
+
+services:
+  tempo:
+    image: grafana/tempo:2.6.1
+    command: ["-config.file=/etc/tempo.yaml"]
+    volumes:
+      - ./tempo.yaml:/etc/tempo.yaml:ro
+      - tempo-data:/var/tempo
+    ports:
+      - "4317:4317"   # OTLP gRPC
+      - "4318:4318"   # OTLP HTTP  ← versitygw sends here
+      - "3200:3200"   # Tempo query API (used by Grafana datasource)
+
+  prometheus:
+    image: prom/prometheus:latest
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --web.enable-remote-write-receiver
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-data:/prometheus
+    ports:
+      - "9090:9090"
+
+  grafana:
+    image: grafana/grafana:latest
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - GF_AUTH_DISABLE_LOGIN_FORM=false
+    volumes:
+      - ./grafana-provisioning:/etc/grafana/provisioning:ro
+    ports:
+      - "3000:3000"
+    depends_on:
+      - tempo
+      - prometheus
+
+volumes:
+  tempo-data:
+  prometheus-data:
@@ -0,0 +1,7 @@
+apiVersion: 1
+
+providers:
+  - name: versitygw
+    type: file
+    options:
+      path: /etc/grafana/provisioning/dashboards
@@ -0,0 +1,9 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    uid: prometheus
+    url: http://prometheus:9090
+    access: proxy
+    isDefault: false
@@ -0,0 +1,21 @@
+apiVersion: 1
+
+datasources:
+  - name: Tempo
+    type: tempo
+    uid: tempo
+    url: http://tempo:3200
+    access: proxy
+    isDefault: true
+    jsonData:
+      httpMethod: GET
+      serviceMap:
+        datasourceUid: tempo
+      search:
+        hide: false
+      nodeGraph:
+        enabled: true
+      traceQuery:
+        timeShiftEnabled: true
+        spanStartTimeShift: 1h
+        spanEndTimeShift: 1h
@@ -0,0 +1,7 @@
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: tempo
+    static_configs:
+      - targets: ['tempo:3200']
@@ -0,0 +1,64 @@
+# Grafana Tempo configuration — single-binary / local-storage mode.
+# Suitable for local development; not for production use.
+
+server:
+  http_listen_port: 3200
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        http:
+          endpoint: 0.0.0.0:4318
+        grpc:
+          endpoint: 0.0.0.0:4317
+
+ingester:
+  trace_idle_period: 10s
+  max_block_bytes: 1_000_000
+  max_block_duration: 5m
+
+storage:
+  trace:
+    backend: local
+    local:
+      path: /var/tempo/blocks
+    wal:
+      path: /var/tempo/wal
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+  storage:
+    path: /var/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+  processor:
+    span_metrics:
+      dimensions:
+        - http.response.status_code
+      histogram_buckets:
+        - 0.000005  # 5µs
+        - 0.00001   # 10µs
+        - 0.000025  # 25µs
+        - 0.00005   # 50µs
+        - 0.0001    # 0.1ms
+        - 0.0002
+        - 0.0005
+        - 0.001
+        - 0.002
+        - 0.005
+        - 0.01
+        - 0.025
+        - 0.05
+        - 0.1
+        - 0.25
+        - 0.5
+        - 1.0
+        - 2.5
+        - 5.0
+        - 10.0
+
+overrides:
+  metrics_generator_processors: [service-graphs, span-metrics]
@@ -0,0 +1,71 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package tracing
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
+)
+
+// InitTracer configures the global OpenTelemetry tracer provider with an OTLP
+// HTTP exporter. endpointURL is a full URL such as "http://localhost:4318" or
+// "https://otel-collector:4318"; if no scheme is present, "http://" is assumed.
+//
+// Returns a shutdown function that flushes and stops the provider. Call it
+// before process exit (typically via defer in main/runGateway).
+func InitTracer(ctx context.Context, serviceName, endpointURL string) (func(context.Context) error, error) {
+	if !strings.HasPrefix(endpointURL, "http://") && !strings.HasPrefix(endpointURL, "https://") {
+		endpointURL = "http://" + endpointURL
+	}
+
+	exp, err := otlptracehttp.New(ctx,
+		otlptracehttp.WithEndpointURL(endpointURL),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("create OTLP trace exporter: %w", err)
+	}
+
+	res, err := resource.New(ctx,
+		resource.WithAttributes(semconv.ServiceName(serviceName)),
+		resource.WithProcess(),
+		resource.WithOS(),
+	)
+	if err != nil {
+		// resource.New can return a partially-populated resource alongside a
+		// non-fatal partial error; use it anyway.
+		fmt.Printf("WARNING: OTel resource detection: %v\n", err)
+	}
+
+	tp := sdktrace.NewTracerProvider(
+		sdktrace.WithBatcher(exp),
+		sdktrace.WithResource(res),
+	)
+
+	otel.SetTracerProvider(tp)
+	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
+		propagation.TraceContext{},
+		propagation.Baggage{},
+	))
+
+	return tp.Shutdown, nil
+}