Unconditionally sample HTTP requests for tracing that take too long.

conf/config.example.toml

@@ -43,3 +43,6 @@ git-large-object-threshold = "1M"
 max-symlink-depth = 16
 update-timeout = "60s"
 max-heap-size-ratio = 0.5 # * RAM_size
+
+[observability]
+slow-response-threshold = "500ms"

src/config.go

@@ -35,13 +35,14 @@ func (t *Duration) MarshalText() ([]byte, error) {
 }
 
 type Config struct {
-	Insecure  bool             `toml:"-" env:"insecure"`
-	Features  []string         `toml:"features"`
-	LogFormat string           `toml:"log-format" default:"text"`
-	Server    ServerConfig     `toml:"server"`
-	Wildcard  []WildcardConfig `toml:"wildcard"`
-	Storage   StorageConfig    `toml:"storage"`
-	Limits    LimitsConfig     `toml:"limits"`
+	Insecure      bool                `toml:"-" env:"insecure"`
+	Features      []string            `toml:"features"`
+	LogFormat     string              `toml:"log-format" default:"text"`
+	Server        ServerConfig        `toml:"server"`
+	Wildcard      []WildcardConfig    `toml:"wildcard"`
+	Storage       StorageConfig       `toml:"storage"`
+	Limits        LimitsConfig        `toml:"limits"`
+	Observability ObservabilityConfig `toml:"observability"`
 }
 
 type ServerConfig struct {
@@ -107,6 +108,11 @@ type LimitsConfig struct {
 	AllowedRepositoryURLPrefixes []string `toml:"allowed-repository-url-prefixes"`
 }
 
+type ObservabilityConfig struct {
+	// Minimum duration for an HTTP request transaction to be unconditionally sampled.
+	SlowResponseThreshold Duration `toml:"slow-response-threshold" default:"500ms"`
+}
+
 func (config *Config) DebugJSON() string {
 	result, err := json.MarshalIndent(config, "", "  ")
 	if err != nil {

src/observe.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"log"
 	"log/slog"
+	"math/rand/v2"
 	"net/http"
 	"os"
 	"runtime/debug"
@@ -63,18 +64,32 @@ func InitObservability() {
 		options.Environment = environment
 		options.EnableLogs = enableLogs
 		options.EnableTracing = enableTracing
+		options.TracesSampleRate = 1
 		switch environment {
 		case "development", "staging":
-			options.TracesSampleRate = 1.0
-		case "production":
-			options.TracesSampler = func(ctx sentry.SamplingContext) float64 {
-				if method, ok := ctx.Span.Data["http.request.method"].(string); ok {
-					switch method {
-					case "PUT", "DELETE", "POST":
-						return 1.0
+		default:
+			options.BeforeSendTransaction = func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event {
+				sampleRate := 0.05
+				if trace, ok := event.Contexts["trace"]; ok {
+					if data, ok := trace["data"].(map[string]any); ok {
+						if method, ok := data["http.request.method"].(string); ok {
+							switch method {
+							case "PUT", "DELETE", "POST":
+								sampleRate = 1
+							default:
+								duration := event.Timestamp.Sub(event.StartTime)
+								threshold := time.Duration(config.Observability.SlowResponseThreshold)
+								if duration >= threshold {
+									sampleRate = 1
+								}
+							}
+						}
 					}
 				}
-				return 0.05
+				if rand.Float64() < sampleRate {
+					return event
+				}
+				return nil
 			}
 		}
 		if err := sentry.Init(options); err != nil {
@@ -99,9 +114,19 @@ func FiniObservability() {
 
 func ObserveHTTPHandler(handler http.Handler) http.Handler {
 	if hasSentry() {
-		handler = sentryhttp.New(sentryhttp.Options{
-			Repanic: true,
-		}).Handle(handler)
+		handler = func(next http.Handler) http.Handler {
+			next = sentryhttp.New(sentryhttp.Options{
+				Repanic: true,
+			}).Handle(handler)
+
+			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				// Prevent the Sentry SDK from continuing traces as we don't use this feature.
+				r.Header.Del(sentry.SentryTraceHeader)
+				r.Header.Del(sentry.SentryBaggageHeader)
+
+				next.ServeHTTP(w, r)
+			})
+		}(handler)
 	}
 
 	return handler

src/pages.go

@@ -465,17 +465,22 @@ func postPage(w http.ResponseWriter, r *http.Request) error {
 		return err
 	}
 
+	updateCtx := r.Context()
+	if isGitHub {
+		updateCtx = context.Background()
+	}
+
 	resultChan := make(chan UpdateResult, 1)
-	go func() {
+	go func(ctx context.Context) {
 		defer close(resultChan)
 
-		updateCtx, cancel := context.WithTimeout(context.Background(), time.Duration(config.Limits.UpdateTimeout))
+		ctx, cancel := context.WithTimeout(ctx, time.Duration(config.Limits.UpdateTimeout))
 		defer cancel()
 
-		result := UpdateFromRepository(updateCtx, webRoot, repoURL, auth.branch)
+		result := UpdateFromRepository(ctx, webRoot, repoURL, auth.branch)
 		resultChan <- result
 		reportSiteUpdate("webhook", &result)
-	}()
+	}(updateCtx)
 
 	var result UpdateResult
 	if isGitHub {

src/update.go

@@ -83,6 +83,9 @@ func UpdateFromRepository(
 	repoURL string,
 	branch string,
 ) UpdateResult {
+	span, ctx := ObserveFunction(ctx, "UpdateFromRepository", "repo.url", repoURL)
+	defer span.Finish()
+
 	log.Printf("update %s: %s %s\n", webRoot, repoURL, branch)
 
 	manifest, err := FetchRepository(ctx, repoURL, branch)