diff --git a/conf/config.example.toml b/conf/config.example.toml index 39db773..e4adab5 100644 --- a/conf/config.example.toml +++ b/conf/config.example.toml @@ -43,3 +43,6 @@ git-large-object-threshold = "1M" max-symlink-depth = 16 update-timeout = "60s" max-heap-size-ratio = 0.5 # * RAM_size + +[observability] +slow-response-threshold = "500ms" diff --git a/src/config.go b/src/config.go index 11bc6ec..86b6430 100644 --- a/src/config.go +++ b/src/config.go @@ -35,13 +35,14 @@ func (t *Duration) MarshalText() ([]byte, error) { } type Config struct { - Insecure bool `toml:"-" env:"insecure"` - Features []string `toml:"features"` - LogFormat string `toml:"log-format" default:"text"` - Server ServerConfig `toml:"server"` - Wildcard []WildcardConfig `toml:"wildcard"` - Storage StorageConfig `toml:"storage"` - Limits LimitsConfig `toml:"limits"` + Insecure bool `toml:"-" env:"insecure"` + Features []string `toml:"features"` + LogFormat string `toml:"log-format" default:"text"` + Server ServerConfig `toml:"server"` + Wildcard []WildcardConfig `toml:"wildcard"` + Storage StorageConfig `toml:"storage"` + Limits LimitsConfig `toml:"limits"` + Observability ObservabilityConfig `toml:"observability"` } type ServerConfig struct { @@ -107,6 +108,11 @@ type LimitsConfig struct { AllowedRepositoryURLPrefixes []string `toml:"allowed-repository-url-prefixes"` } +type ObservabilityConfig struct { + // Minimum duration for an HTTP request transaction to be unconditionally sampled. + SlowResponseThreshold Duration `toml:"slow-response-threshold" default:"500ms"` +} + func (config *Config) DebugJSON() string { result, err := json.MarshalIndent(config, "", " ") if err != nil { diff --git a/src/observe.go b/src/observe.go index c7700de..26ebea0 100644 --- a/src/observe.go +++ b/src/observe.go @@ -5,6 +5,7 @@ import ( "io" "log" "log/slog" + "math/rand/v2" "net/http" "os" "runtime/debug" @@ -63,18 +64,32 @@ func InitObservability() { options.Environment = environment options.EnableLogs = enableLogs options.EnableTracing = enableTracing + options.TracesSampleRate = 1 switch environment { case "development", "staging": - options.TracesSampleRate = 1.0 - case "production": - options.TracesSampler = func(ctx sentry.SamplingContext) float64 { - if method, ok := ctx.Span.Data["http.request.method"].(string); ok { - switch method { - case "PUT", "DELETE", "POST": - return 1.0 + default: + options.BeforeSendTransaction = func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event { + sampleRate := 0.05 + if trace, ok := event.Contexts["trace"]; ok { + if data, ok := trace["data"].(map[string]any); ok { + if method, ok := data["http.request.method"].(string); ok { + switch method { + case "PUT", "DELETE", "POST": + sampleRate = 1 + default: + duration := event.Timestamp.Sub(event.StartTime) + threshold := time.Duration(config.Observability.SlowResponseThreshold) + if duration >= threshold { + sampleRate = 1 + } + } + } } } - return 0.05 + if rand.Float64() < sampleRate { + return event + } + return nil } } if err := sentry.Init(options); err != nil { @@ -99,9 +114,19 @@ func FiniObservability() { func ObserveHTTPHandler(handler http.Handler) http.Handler { if hasSentry() { - handler = sentryhttp.New(sentryhttp.Options{ - Repanic: true, - }).Handle(handler) + handler = func(next http.Handler) http.Handler { + next = sentryhttp.New(sentryhttp.Options{ + Repanic: true, + }).Handle(handler) + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Prevent the Sentry SDK from continuing traces as we don't use this feature. + r.Header.Del(sentry.SentryTraceHeader) + r.Header.Del(sentry.SentryBaggageHeader) + + next.ServeHTTP(w, r) + }) + }(handler) } return handler diff --git a/src/pages.go b/src/pages.go index 86d89e4..344f5ec 100644 --- a/src/pages.go +++ b/src/pages.go @@ -465,17 +465,22 @@ func postPage(w http.ResponseWriter, r *http.Request) error { return err } + updateCtx := r.Context() + if isGitHub { + updateCtx = context.Background() + } + resultChan := make(chan UpdateResult, 1) - go func() { + go func(ctx context.Context) { defer close(resultChan) - updateCtx, cancel := context.WithTimeout(context.Background(), time.Duration(config.Limits.UpdateTimeout)) + ctx, cancel := context.WithTimeout(ctx, time.Duration(config.Limits.UpdateTimeout)) defer cancel() - result := UpdateFromRepository(updateCtx, webRoot, repoURL, auth.branch) + result := UpdateFromRepository(ctx, webRoot, repoURL, auth.branch) resultChan <- result reportSiteUpdate("webhook", &result) - }() + }(updateCtx) var result UpdateResult if isGitHub { diff --git a/src/update.go b/src/update.go index 240c982..da6fa36 100644 --- a/src/update.go +++ b/src/update.go @@ -83,6 +83,9 @@ func UpdateFromRepository( repoURL string, branch string, ) UpdateResult { + span, ctx := ObserveFunction(ctx, "UpdateFromRepository", "repo.url", repoURL) + defer span.Finish() + log.Printf("update %s: %s %s\n", webRoot, repoURL, branch) manifest, err := FetchRepository(ctx, repoURL, branch)