otel tracing wip

s3api/controllers/base.go

@@ -30,6 +30,8 @@ import (
 	"github.com/versity/versitygw/s3err"
 	"github.com/versity/versitygw/s3event"
 	"github.com/versity/versitygw/s3log"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/codes"
 )
 
 type S3ApiController struct {
@@ -129,6 +131,10 @@ func ProcessHandlers(controller Controller, s3action string, svc *Services, hand
 			return ctx.Next()
 		}
 
+		// Store the resolved S3 action name so the tracing middleware can
+		// attach it to the span after routing has completed.
+		utils.ContextKeyS3Action.Set(ctx, s3action)
+
 		for _, handler := range handlers {
 			err := handler(ctx)
 			if err != nil {
@@ -179,10 +185,21 @@ func WrapMiddleware(handler fiber.Handler, logger s3log.AuditLogger, mm metrics.
 	}
 }
 
+const controllerTracerName = "github.com/versity/versitygw"
+
 // ProcessController executes the given s3api controller and handles the metrics
 // access logs and s3 events
 func ProcessController(ctx *fiber.Ctx, controller Controller, s3action string, svc *Services) error {
+	parentCtx := ctx.UserContext()
+	backendCtx, backendSpan := otel.Tracer(controllerTracerName).Start(parentCtx, "backend."+s3action)
+	ctx.SetUserContext(backendCtx)
 	response, err := controller(ctx)
+	if err != nil {
+		backendSpan.RecordError(err)
+		backendSpan.SetStatus(codes.Error, "")
+	}
+	backendSpan.End()
+	ctx.SetUserContext(parentCtx)
 
 	// Set the response headers
 	SetResponseHeaders(ctx, response.Headers)

s3api/middlewares/acl-parser.go

@@ -21,6 +21,9 @@ import (
 	"github.com/versity/versitygw/backend"
 	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
 )
 
 // ParseAcl retreives the bucket acl and stores in the context locals
@@ -28,8 +31,18 @@ import (
 func ParseAcl(be backend.Backend) fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
 		bucket := ctx.Params("bucket")
+
+		parentCtx := ctx.UserContext()
+		sctx, span := otel.Tracer(tracerName).Start(parentCtx, "middleware.ParseAcl")
+		span.SetAttributes(attribute.String("s3.bucket", bucket))
+		defer span.End()
+		ctx.SetUserContext(sctx)
+		defer ctx.SetUserContext(parentCtx)
+
 		data, err := be.GetBucketAcl(ctx.Context(), &s3.GetBucketAclInput{Bucket: &bucket})
 		if err != nil {
+			span.RecordError(err)
+			span.SetStatus(codes.Error, "")
 			return err
 		}
 

s3api/middlewares/authentication.go

@@ -25,6 +25,9 @@ import (
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
 )
 
 const (
@@ -50,6 +53,12 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string,
 			return nil
 		}
 
+		parentCtx := ctx.UserContext()
+		sctx, span := otel.Tracer(tracerName).Start(parentCtx, "middleware.VerifyV4Signature")
+		defer span.End()
+		ctx.SetUserContext(sctx)
+		defer ctx.SetUserContext(parentCtx)
+
 		// Check X-Amz-Date header
 		date := ctx.Get("X-Amz-Date")
 		if date == "" {
@@ -84,14 +93,26 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string,
 
 		utils.ContextKeyIsRoot.Set(ctx, authData.Access == root.Access)
 
+		_, iamSpan := otel.Tracer(tracerName).Start(sctx, "iam.GetUserAccount")
 		account, err := acct.getAccount(authData.Access)
+		if err != nil {
+			iamSpan.RecordError(err)
+			iamSpan.SetStatus(codes.Error, "")
+		}
+		iamSpan.End()
+
 		if err == auth.ErrNoSuchUser {
+			span.SetStatus(codes.Error, "")
 			return s3err.GetAPIError(s3err.ErrInvalidAccessKeyID)
 		}
 		if err != nil {
+			span.RecordError(err)
+			span.SetStatus(codes.Error, "")
 			return err
 		}
 
+		span.SetAttributes(attribute.Bool("auth.is_root", authData.Access == root.Access))
+
 		if date[:8] != authData.Date {
 			return s3err.MalformedAuth.DateMismatch()
 		}
@@ -170,6 +191,8 @@ func VerifyV4Signature(root RootUserConfig, iam auth.IAMService, region string,
 
 		err = utils.CheckValidSignature(ctx, authData, account.Secret, hashPayload, tdate, contentLength, false)
 		if err != nil {
+			span.RecordError(err)
+			span.SetStatus(codes.Error, "")
 			return err
 		}
 

s3api/middlewares/checksum.go

@@ -23,6 +23,8 @@ import (
 	"github.com/gofiber/fiber/v2"
 	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/codes"
 )
 
 // VerifyChecksums parses, validates, and calculates the
@@ -33,6 +35,12 @@ import (
 // the x-amz-checksum-* headers are explicitly processed by the backend.
 func VerifyChecksums(streamBody bool, requireBody bool, requireChecksum bool) fiber.Handler {
 	return func(ctx *fiber.Ctx) error {
+		parentCtx := ctx.UserContext()
+		sctx, span := otel.Tracer(tracerName).Start(parentCtx, "middleware.VerifyChecksums")
+		defer span.End()
+		ctx.SetUserContext(sctx)
+		defer ctx.SetUserContext(parentCtx)
+
 		md5sum := ctx.Get("Content-Md5")
 
 		if streamBody {
@@ -103,6 +111,8 @@ func VerifyChecksums(streamBody bool, requireBody bool, requireChecksum bool) fi
 		if rdr != nil {
 			_, err = io.Copy(io.Discard, rdr)
 			if err != nil {
+				span.RecordError(err)
+				span.SetStatus(codes.Error, "")
 				return err
 			}
 		}

s3api/middlewares/presign-auth.go

@@ -22,6 +22,8 @@ import (
 	"github.com/versity/versitygw/auth"
 	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/codes"
 )
 
 func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, region string, streamBody bool) fiber.Handler {
@@ -36,6 +38,12 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, region
 			return nil
 		}
 
+		parentCtx := ctx.UserContext()
+		sctx, span := otel.Tracer(tracerName).Start(parentCtx, "middleware.VerifyPresignedV4Signature")
+		defer span.End()
+		ctx.SetUserContext(sctx)
+		defer ctx.SetUserContext(parentCtx)
+
 		if ctx.Request().URI().QueryArgs().Has("X-Amz-Security-Token") {
 			// OIDC Authorization with X-Amz-Security-Token is not supported
 			return s3err.QueryAuthErrors.SecurityTokenNotSupported()
@@ -52,11 +60,21 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, region
 
 		utils.ContextKeyIsRoot.Set(ctx, authData.Access == root.Access)
 
+		_, iamSpan := otel.Tracer(tracerName).Start(sctx, "iam.GetUserAccount")
 		account, err := acct.getAccount(authData.Access)
+		if err != nil {
+			iamSpan.RecordError(err)
+			iamSpan.SetStatus(codes.Error, "")
+		}
+		iamSpan.End()
+
 		if err == auth.ErrNoSuchUser {
+			span.SetStatus(codes.Error, "")
 			return s3err.GetAPIError(s3err.ErrInvalidAccessKeyID)
 		}
 		if err != nil {
+			span.RecordError(err)
+			span.SetStatus(codes.Error, "")
 			return err
 		}
 		utils.ContextKeyAccount.Set(ctx, account)
@@ -90,6 +108,8 @@ func VerifyPresignedV4Signature(root RootUserConfig, iam auth.IAMService, region
 
 		err = utils.CheckPresignedSignature(ctx, authData, account.Secret, streamBody)
 		if err != nil {
+			span.RecordError(err)
+			span.SetStatus(codes.Error, "")
 			return err
 		}
 

s3api/middlewares/public-bucket.go

@@ -26,6 +26,9 @@ import (
 	"github.com/versity/versitygw/metrics"
 	"github.com/versity/versitygw/s3api/utils"
 	"github.com/versity/versitygw/s3err"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
 )
 
 // AuthorizePublicBucketAccess checks if the bucket grants public
@@ -37,6 +40,13 @@ func AuthorizePublicBucketAccess(be backend.Backend, s3action string, policyPerm
 			return nil
 		}
 
+		parentCtx := ctx.UserContext()
+		sctx, span := otel.Tracer(tracerName).Start(parentCtx, "middleware.AuthorizePublicBucketAccess")
+		span.SetAttributes(attribute.String("s3.action", s3action))
+		defer span.End()
+		ctx.SetUserContext(sctx)
+		defer ctx.SetUserContext(parentCtx)
+
 		switch s3action {
 		case metrics.ActionListAllMyBuckets:
 			return s3err.GetAPIError(s3err.ErrAccessDenied)
@@ -57,8 +67,14 @@ func AuthorizePublicBucketAccess(be backend.Backend, s3action string, policyPerm
 		}
 
 		bucket, object := parsePath(ctx.Path())
+		span.SetAttributes(
+			attribute.String("s3.bucket", bucket),
+			attribute.String("s3.object", object),
+		)
 		err := auth.VerifyPublicAccess(ctx.Context(), be, policyPermission, permission, bucket, object)
 		if err != nil {
+			span.RecordError(err)
+			span.SetStatus(codes.Error, "")
 			if s3action == metrics.ActionHeadBucket {
 				// add the bucket region header for HeadBucket
 				// if anonymous access is denied

s3api/middlewares/tracing.go

@@ -0,0 +1,116 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package middlewares
+
+import (
+	"github.com/gofiber/fiber/v2"
+	"github.com/valyala/fasthttp"
+	"github.com/versity/versitygw/s3api/utils"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
+	"go.opentelemetry.io/otel/trace"
+)
+
+const tracerName = "github.com/versity/versitygw"
+
+// fasthttpCarrier adapts a *fasthttp.RequestHeader to the OTel TextMapCarrier
+// interface so that W3C Trace Context headers can be extracted from incoming
+// requests in the Fiber / fasthttp stack.
+type fasthttpCarrier struct {
+	header *fasthttp.RequestHeader
+}
+
+func (c fasthttpCarrier) Get(key string) string {
+	return string(c.header.Peek(key))
+}
+
+func (c fasthttpCarrier) Set(key, value string) {
+	c.header.Set(key, value)
+}
+
+func (c fasthttpCarrier) Keys() []string {
+	keys := make([]string, 0, 8)
+	c.header.VisitAll(func(k, _ []byte) {
+		keys = append(keys, string(k))
+	})
+	return keys
+}
+
+// OtelTracing returns a Fiber middleware that:
+//  1. Extracts an incoming W3C Trace Context / Baggage from the request headers.
+//  2. Starts a server-side span for the request.
+//  3. Stores the span context in the Fiber user context so downstream handlers
+//     can create child spans via otel.Tracer(...).Start(c.UserContext(), ...).
+//  4. After the handler chain returns, updates the span name to the matched
+//     route pattern (low-cardinality), records the HTTP status code, and sets
+//     the span status.
+func OtelTracing() fiber.Handler {
+	tracer := otel.Tracer(tracerName)
+	propagator := otel.GetTextMapPropagator()
+
+	return func(c *fiber.Ctx) error {
+		// Extract parent trace context from incoming HTTP headers.
+		parentCtx := propagator.Extract(
+			c.UserContext(),
+			fasthttpCarrier{&c.Request().Header},
+		)
+
+		// Start a server span.  Use method+path as initial name; it is
+		// replaced below with the low-cardinality route pattern once routing
+		// has resolved.
+		ctx, span := tracer.Start(
+			parentCtx,
+			c.Method()+" "+c.Path(),
+			trace.WithSpanKind(trace.SpanKindServer),
+			trace.WithAttributes(
+				semconv.HTTPRequestMethodKey.String(c.Method()),
+				semconv.URLPathKey.String(c.Path()),
+				semconv.ServerAddressKey.String(c.Hostname()),
+			),
+		)
+		defer span.End()
+
+		// Make the span context available to handlers.
+		c.SetUserContext(ctx)
+
+		err := c.Next()
+
+		// Prefer the resolved S3 action name (e.g. "s3_ListAllMyBuckets") as
+		// the span name; fall back to the low-cardinality route pattern.
+		spanName := ""
+		if action, ok := c.Locals(string(utils.ContextKeyS3Action)).(string); ok && action != "" {
+			spanName = action
+			span.SetAttributes(attribute.String("s3.action", action))
+		} else if r := c.Route(); r != nil && r.Path != "" {
+			spanName = c.Method() + " " + r.Path
+		}
+		if spanName != "" {
+			span.SetName(spanName)
+		}
+
+		statusCode := c.Response().StatusCode()
+		span.SetAttributes(semconv.HTTPResponseStatusCodeKey.Int(statusCode))
+
+		if err != nil || statusCode >= 400 {
+			span.SetStatus(codes.Error, "")
+		} else {
+			span.SetStatus(codes.Ok, "")
+		}
+
+		return err
+	}
+}

s3api/server.go

@@ -51,6 +51,7 @@ type S3ApiServer struct {
 	health         string
 	maxConnections int
 	maxRequests    int
+	tracingEnabled bool
 }
 
 func New(
@@ -116,6 +117,12 @@ func New(
 		})
 	}
 
+	// initialize OpenTelemetry tracing middleware (must be early so all
+	// subsequent handlers execute within the request span).
+	if server.tracingEnabled {
+		app.Use(middlewares.OtelTracing())
+	}
+
 	// initialize total requests cap limiter middleware
 	app.Use(middlewares.RateLimiter(server.maxRequests, mm, l))
 
@@ -144,6 +151,13 @@ func WithTLS(cs *utils.CertStorage) Option {
 	return func(s *S3ApiServer) { s.CertStorage = cs }
 }
 
+// WithTracing enables the OpenTelemetry request-tracing middleware.  A tracer
+// provider must already be configured globally (e.g. via tracing.InitTracer)
+// before the first request arrives.
+func WithTracing() Option {
+	return func(s *S3ApiServer) { s.tracingEnabled = true }
+}
+
 // WithAdminServer runs admin endpoints with the gateway in the same network
 func WithAdminServer() Option {
 	return func(s *S3ApiServer) { s.Router.WithAdmSrv = true }

s3api/utils/context-keys.go

@@ -37,6 +37,7 @@ const (
 	ContextKeySkip           ContextKey = "__skip"
 	ContextKeyStack          ContextKey = "stack"
 	ContextKeyBucketOwner    ContextKey = "bucket-owner"
+	ContextKeyS3Action       ContextKey = "s3-action"
 )
 
 func (ck ContextKey) Values() []ContextKey {