From eaf77565bc896d90c31f6bd1997e4dd8f6f60ee2 Mon Sep 17 00:00:00 2001 From: Catherine Date: Thu, 20 Nov 2025 04:04:57 +0000 Subject: [PATCH] Improve configuration reload and clarify scope. This commit also moves all of the globals into `main.go`. --- src/auth.go | 6 +-- src/backend.go | 6 +-- src/main.go | 103 ++++++++++++++++++++++++++++++++---------------- src/wildcard.go | 17 ++++---- 4 files changed, 81 insertions(+), 51 deletions(-) diff --git a/src/auth.go b/src/auth.go index 6689b21..48360f6 100644 --- a/src/auth.go +++ b/src/auth.go @@ -329,7 +329,7 @@ func AuthorizeMetadataRetrieval(r *http.Request) (*Authorization, error) { return auth, nil } - for _, pattern := range wildcardPatterns { + for _, pattern := range wildcards { auth, err = authorizeWildcardMatchHost(r, pattern) if err != nil && IsUnauthorized(err) { causes = append(causes, err) @@ -397,7 +397,7 @@ func AuthorizeUpdateFromRepository(r *http.Request) (*Authorization, error) { // Wildcard match is only available for webhooks, not the REST API. if r.Method == http.MethodPost { - for _, pattern := range wildcardPatterns { + for _, pattern := range wildcards { auth, err = authorizeWildcardMatchSite(r, pattern) if err != nil && IsUnauthorized(err) { causes = append(causes, err) @@ -592,7 +592,7 @@ func authorizeForgeWithToken(r *http.Request) (*Authorization, error) { } var errs []error - for _, pattern := range wildcardPatterns { + for _, pattern := range wildcards { if !pattern.Authorization { continue } diff --git a/src/backend.go b/src/backend.go index a9028b0..952f8c8 100644 --- a/src/backend.go +++ b/src/backend.go @@ -78,20 +78,16 @@ type Backend interface { CreateDomain(ctx context.Context, domain string) error } -var backend Backend - -func ConfigureBackend(config *StorageConfig) (err error) { +func CreateBackend(config *StorageConfig) (backend Backend, err error) { switch config.Type { case "fs": if backend, err = NewFSBackend(&config.FS); err != nil { err = fmt.Errorf("fs backend: %w", err) } - case "s3": if backend, err = NewS3Backend(context.Background(), &config.S3); err != nil { err = fmt.Errorf("s3 backend: %w", err) } - default: err = fmt.Errorf("unknown backend: %s", config.Type) } diff --git a/src/main.go b/src/main.go index 318a5c7..b33ac77 100644 --- a/src/main.go +++ b/src/main.go @@ -2,6 +2,7 @@ package git_pages import ( "context" + "errors" "flag" "fmt" "io" @@ -20,6 +21,47 @@ import ( ) var config *Config +var wildcards []*WildcardPattern +var backend Backend + +func configureFeatures() (err error) { + if len(config.Features) > 0 { + log.Println("features:", strings.Join(config.Features, ", ")) + } + return +} + +func configureMemLimit() (err error) { + // Avoid being OOM killed by not garbage collecting early enough. + memlimitBefore := datasize.ByteSize(debug.SetMemoryLimit(-1)) + automemlimit.SetGoMemLimitWithOpts( + automemlimit.WithLogger(slog.New(slog.DiscardHandler)), + automemlimit.WithProvider( + automemlimit.ApplyFallback( + automemlimit.FromCgroup, + automemlimit.FromSystem, + ), + ), + automemlimit.WithRatio(float64(config.Limits.MaxHeapSizeRatio)), + ) + memlimitAfter := datasize.ByteSize(debug.SetMemoryLimit(-1)) + if memlimitBefore == memlimitAfter { + log.Println("memlimit: now", memlimitBefore.HR()) + } else { + log.Println("memlimit: was", memlimitBefore.HR(), "now", memlimitAfter.HR()) + } + return +} + +func configureWildcards() (err error) { + newWildcards, err := TranslateWildcards(config.Wildcard) + if err != nil { + return err + } else { + wildcards = newWildcards + return nil + } +} func listen(name string, listen string) net.Listener { if listen == "-" { @@ -168,32 +210,17 @@ func Main() { InitObservability() defer FiniObservability() - if len(config.Features) > 0 { - log.Println("features:", strings.Join(config.Features, ", ")) - } - - // Avoid being OOM killed by not garbage collecting early enough. - memlimitBefore := datasize.ByteSize(debug.SetMemoryLimit(-1)) - automemlimit.SetGoMemLimitWithOpts( - automemlimit.WithLogger(slog.New(slog.DiscardHandler)), - automemlimit.WithProvider( - automemlimit.ApplyFallback( - automemlimit.FromCgroup, - automemlimit.FromSystem, - ), - ), - automemlimit.WithRatio(float64(config.Limits.MaxHeapSizeRatio)), - ) - memlimitAfter := datasize.ByteSize(debug.SetMemoryLimit(-1)) - if memlimitBefore == memlimitAfter { - log.Println("memlimit: now", memlimitBefore.HR()) - } else { - log.Println("memlimit: was", memlimitBefore.HR(), "now", memlimitAfter.HR()) + if err = errors.Join( + configureFeatures(), + configureMemLimit(), + configureWildcards(), + ); err != nil { + log.Fatalln(err) } switch { case *runMigration != "": - if err := ConfigureBackend(&config.Storage); err != nil { + if backend, err = CreateBackend(&config.Storage); err != nil { log.Fatalln(err) } @@ -202,7 +229,7 @@ func Main() { } case *getBlob != "": - if err := ConfigureBackend(&config.Storage); err != nil { + if backend, err = CreateBackend(&config.Storage); err != nil { log.Fatalln(err) } @@ -213,7 +240,7 @@ func Main() { io.Copy(fileOutputArg(), reader) case *getManifest != "": - if err := ConfigureBackend(&config.Storage); err != nil { + if backend, err = CreateBackend(&config.Storage); err != nil { log.Fatalln(err) } @@ -225,7 +252,7 @@ func Main() { fmt.Fprintln(fileOutputArg(), ManifestDebugJSON(manifest)) case *getArchive != "": - if err := ConfigureBackend(&config.Storage); err != nil { + if backend, err = CreateBackend(&config.Storage); err != nil { log.Fatalln(err) } @@ -238,7 +265,7 @@ func Main() { CollectTar(context.Background(), fileOutputArg(), manifest, manifestMtime) case *updateSite != "": - if err := ConfigureBackend(&config.Storage); err != nil { + if backend, err = CreateBackend(&config.Storage); err != nil { log.Fatalln(err) } @@ -307,12 +334,14 @@ func Main() { // at runtime. This is useful because it preserves S3 backend cache contents. Failed // configuration reloads will not crash the process; you may want to check the syntax // first with `git-pages -config ... -print-config` since there is no other feedback. + // + // Note that not all of the configuration is updated on reload. Listeners are kept as-is. + // The backend is not recreated (this is intentional as it allows preserving the cache). OnReload(func() { if newConfig, err := Configure(*configTomlPath); err != nil { log.Println("config:", err) - log.Println("config: reload failed") + log.Println("config: reload error") } else { - log.Println("config: reloaded") // From https://go.dev/ref/mem: // > A read r of a memory location x holding a value that is not larger than // > a machine word must observe some write w such that r does not happen before @@ -320,6 +349,17 @@ func Main() { // > before r. That is, each read must observe a value written by a preceding or // > concurrent write. config = newConfig + if err = errors.Join( + configureFeatures(), + configureMemLimit(), + configureWildcards(), + ); err != nil { + // At this point the configuration is in an in-between, corrupted state, so + // the only reasonable choice is to crash. + log.Fatalln("config: reload failure:", err) + } else { + log.Println("config: reloaded") + } } }) @@ -331,16 +371,11 @@ func Main() { caddyListener := listen("caddy", config.Server.Caddy) metricsListener := listen("metrics", config.Server.Metrics) - if err := ConfigureBackend(&config.Storage); err != nil { + if backend, err = CreateBackend(&config.Storage); err != nil { log.Fatalln(err) } - backend = NewObservedBackend(backend) - if err := ConfigureWildcards(config.Wildcard); err != nil { - log.Fatalln(err) - } - go serve(pagesListener, ObserveHTTPHandler(http.HandlerFunc(ServePages))) go serve(caddyListener, ObserveHTTPHandler(http.HandlerFunc(ServeCaddy))) go serve(metricsListener, promhttp.Handler()) diff --git a/src/wildcard.go b/src/wildcard.go index 2f19e0a..7a2f0d9 100644 --- a/src/wildcard.go +++ b/src/wildcard.go @@ -23,8 +23,6 @@ type WildcardPattern struct { Fallback http.Handler } -var wildcardPatterns []*WildcardPattern - func (pattern *WildcardPattern) GetHost() string { parts := []string{"*"} parts = append(parts, pattern.Domain...) @@ -95,7 +93,7 @@ func HandleWildcardFallback(w http.ResponseWriter, r *http.Request) (bool, error return false, err } - for _, pattern := range wildcardPatterns { + for _, pattern := range wildcards { if pattern.IsFallbackFor(host) { log.Printf("proxy: %s via %s", pattern.GetHost(), pattern.FallbackURL) pattern.Fallback.ServeHTTP(w, r) @@ -105,11 +103,12 @@ func HandleWildcardFallback(w http.ResponseWriter, r *http.Request) (bool, error return false, nil } -func ConfigureWildcards(configs []WildcardConfig) error { +func TranslateWildcards(configs []WildcardConfig) ([]*WildcardPattern, error) { + var wildcardPatterns []*WildcardPattern for _, config := range configs { cloneURLTemplate, err := fasttemplate.NewTemplate(config.CloneURL, "<", ">") if err != nil { - return fmt.Errorf("wildcard pattern: clone URL: %w", err) + return nil, fmt.Errorf("wildcard pattern: clone URL: %w", err) } var indexRepoTemplates []*fasttemplate.Template @@ -117,7 +116,7 @@ func ConfigureWildcards(configs []WildcardConfig) error { for _, indexRepo := range config.IndexRepos { indexRepoTemplate, err := fasttemplate.NewTemplate(indexRepo, "<", ">") if err != nil { - return fmt.Errorf("wildcard pattern: index repo: %w", err) + return nil, fmt.Errorf("wildcard pattern: index repo: %w", err) } indexRepoTemplates = append(indexRepoTemplates, indexRepoTemplate) } @@ -129,7 +128,7 @@ func ConfigureWildcards(configs []WildcardConfig) error { // is the same for all of them. authorization = true } else { - return fmt.Errorf( + return nil, fmt.Errorf( "wildcard pattern: unknown authorization mechanism: %s", config.Authorization, ) @@ -141,7 +140,7 @@ func ConfigureWildcards(configs []WildcardConfig) error { if config.FallbackProxyTo != "" { fallbackURL, err = url.Parse(config.FallbackProxyTo) if err != nil { - return fmt.Errorf("wildcard pattern: fallback URL: %w", err) + return nil, fmt.Errorf("wildcard pattern: fallback URL: %w", err) } fallback = &httputil.ReverseProxy{ @@ -168,5 +167,5 @@ func ConfigureWildcards(configs []WildcardConfig) error { Fallback: fallback, }) } - return nil + return wildcardPatterns, nil }