[security] Fix false positives on Caddy endpoint due to domain cache.

In commit bbdaae7280, a domain cache was
introduced to deal with misbehaving crawlers that forge `Host:` header
and may cause thousands of expensive S3 requests to be submitted.
This domain cache is implemented using a Bloom filter (which can
produce false positives but not false negatives) for S3 backend, and
using a function always returning true (which will be a false positive
in most cases) for the FS backend.

Both of these behaviors are unacceptable for the Caddy endpoint, but
the FS backend case much more so. If you use git-pages with Caddy you
should upgrade to a build that includes this commit as soon as possible
or Let's Encrypt may rate-limit or restrict your account when you get
unlucky with a crawler.

.forgejo/workflows/ci.yaml

@@ -12,12 +12,12 @@ jobs:
   check:
     runs-on: debian-trixie
     container:
-      image: docker.io/library/node:24-trixie-slim@sha256:4fc981bf8dfc5e36e15e0cb73c5761a14cabff0932dcad1cf26cd3c3425db5d4
+      image: docker.io/library/node:24-trixie-slim@sha256:28fd420825d8e922eab0fd91740c7cf88ddbdc8116a2b20a82049f0c946feb03
     steps:
       - name: Check out source code
         uses: https://code.forgejo.org/actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - name: Set up toolchain
-        uses: https://code.forgejo.org/actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
+        uses: https://code.forgejo.org/actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
         with:
           go-version: '>=1.25.6'
       - name: Install dependencies
@@ -42,18 +42,22 @@ jobs:
     needs: [check]
     runs-on: debian-trixie
     container:
-      image: docker.io/library/node:24-trixie-slim@sha256:4fc981bf8dfc5e36e15e0cb73c5761a14cabff0932dcad1cf26cd3c3425db5d4
+      image: docker.io/library/node:24-trixie-slim@sha256:28fd420825d8e922eab0fd91740c7cf88ddbdc8116a2b20a82049f0c946feb03
     steps:
-      - name: Check out source code
-        uses: https://code.forgejo.org/actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-      - name: Set up toolchain
-        uses: https://code.forgejo.org/actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
-        with:
-          go-version: '>=1.25.6'
       - name: Install dependencies
         run: |
           apt-get -y update
-          apt-get -y install ca-certificates
+          apt-get -y install ca-certificates git
+          # git needs to be installed for build information embedding to work
+      - name: Check out source code
+        uses: https://code.forgejo.org/actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+      - name: Set up toolchain
+        uses: https://code.forgejo.org/actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
+        with:
+          go-version: '>=1.25.6'
       - name: Build release assets
         # If you want more platforms to be represented, send a pull request.
         run: |
@@ -64,7 +68,7 @@ jobs:
           build linux   arm64
           build darwin  arm64
       - name: Create release
-        uses: https://code.forgejo.org/actions/forgejo-release@e7b60f9ae8d4bbf3ed4cc178e4656ce40eb67256 # v2.11.2
+        uses: https://code.forgejo.org/actions/forgejo-release@6a9510a9ea01b8b9b435933bf3c0fa45597ad530 # v2.11.3
         with:
           tag: ${{ startsWith(forge.event.ref, 'refs/tags/v') && forge.ref_name || 'latest' }}
           release-dir: assets
@@ -77,7 +81,7 @@ jobs:
     needs: [check]
     runs-on: debian-trixie
     container:
-      image: docker.io/library/node:24-trixie-slim@sha256:4fc981bf8dfc5e36e15e0cb73c5761a14cabff0932dcad1cf26cd3c3425db5d4
+      image: docker.io/library/node:24-trixie-slim@sha256:28fd420825d8e922eab0fd91740c7cf88ddbdc8116a2b20a82049f0c946feb03
     steps:
       - name: Install dependencies
         run: |

.gitignore

@@ -5,3 +5,4 @@
 /config*.toml*
 /git-pages
 /site
+/assets

Dockerfile

@@ -3,7 +3,7 @@ FROM docker.io/library/alpine:3 AS ca-certificates-builder
 RUN apk --no-cache add ca-certificates
 
 # Build supervisor.
-FROM docker.io/library/golang:1.26-alpine@sha256:2389ebfa5b7f43eeafbd6be0c3700cc46690ef842ad962f6c5bd6be49ed82039 AS supervisor-builder
+FROM docker.io/library/golang:1.26-alpine@sha256:f85330846cde1e57ca9ec309382da3b8e6ae3ab943d2739500e08c86393a21b1 AS supervisor-builder
 RUN apk --no-cache add git
 WORKDIR /build
 RUN git clone https://github.com/ochinchina/supervisord . && \
@@ -11,12 +11,12 @@ RUN git clone https://github.com/ochinchina/supervisord . && \
 RUN GOBIN=/usr/bin go install -ldflags "-s -w"
 
 # Build Caddy with S3 storage backend.
-FROM docker.io/library/caddy:2.11.2-builder@sha256:d4f984844fc3b867ac88fd814285a38eaaf5b3ecadb9ca1b3b0397182ef60cfe AS caddy-builder
+FROM docker.io/library/caddy:2.11.2-builder@sha256:10ed0251c5cd1dbb4db0b71ad43121147961a51adfec35febce2c93ea25c24f4 AS caddy-builder
 RUN xcaddy build ${CADDY_VERSION} \
     --with=github.com/ss098/certmagic-s3@v0.0.0-20250922022452-8af482af5f39
 
 # Build git-pages.
-FROM docker.io/library/golang:1.26-alpine@sha256:2389ebfa5b7f43eeafbd6be0c3700cc46690ef842ad962f6c5bd6be49ed82039 AS git-pages-builder
+FROM docker.io/library/golang:1.26-alpine@sha256:f85330846cde1e57ca9ec309382da3b8e6ae3ab943d2739500e08c86393a21b1 AS git-pages-builder
 RUN apk --no-cache add git
 WORKDIR /build
 COPY go.mod go.sum ./

README.md

@@ -1,6 +1,9 @@
 git-pages
 =========
 
+<a href="ircs://irc.libera.chat/#git-pages"><img alt="Discuss on IRC at #git-pages on libera.chat" src="https://img.shields.io/badge/irc-libera.chat-blue"></a>
+<a href="https://matrix.to/#/#git-pages:catircservices.org"><img alt="Discuss on Matrix at #git-pages:catircservices.org" src="https://img.shields.io/matrix/git-pages%3Acatircservices.org?server_fqdn=matrix.org&fetchMode=summary&label=matrix&color=blue"></a>
+
 _git-pages_ is a static site server for use with Git forges (i.e. a GitHub Pages replacement). It is written with efficiency in mind, scaling horizontally to any number of machines and serving sites up to multiple gigabytes in size, while being equally suitable for small single-user deployments.
 
 It is implemented in Go and has no other mandatory dependencies, although it is designed to be used together with the [Caddy server][caddy] for TLS termination. Site data may be stored on the filesystem or in an [Amazon S3](https://aws.amazon.com/s3/) compatible object store.
@@ -89,15 +92,18 @@ Features
 * All updates to site content are atomic (subject to consistency guarantees of the storage backend). That is, there is an instantaneous moment during an update before which the server will return the old content and after which it will return the new content.
 * Files with a certain name, when placed in the root of a site, have special functions:
     - [Netlify `_redirects`][_redirects] file can be used to specify HTTP redirect and rewrite rules. The _git-pages_ implementation currently does not support placeholders, query parameters, or conditions, and may differ from Netlify in other minor ways. If you find that a supported `_redirects` file feature does not work the same as on Netlify, please file an issue. (Note that _git-pages_ does not perform URL normalization; `/foo` and `/foo/` are *not* the same, unlike with Netlify.)
-    - [Netlify `_headers`][_headers] file can be used to specify custom HTTP response headers (if allowlisted by configuration). In particular, this is useful to enable [CORS requests][cors]. The _git-pages_ implementation may differ from Netlify in minor ways; if you find that a `_headers` file feature does not work the same as on Netlify, please file an issue.
+    - [Netlify `_headers`][_headers] file can be used to specify custom HTTP response headers (if allowlisted by configuration). In particular, this is useful to enable [cross-origin isolation (COOP/COEP)][isolation]. The _git-pages_ implementation may differ from Netlify in minor ways; if you find that a `_headers` file feature does not work the same as on Netlify, please file an issue.
+    - [Netlify `Basic-Auth:`][basic-auth] pseudo-header in the `_headers` file can be used to password-protect parts of a site, if enabled via the `[limits].allow-basic-auth` configuration option. **This is not a security feature: credentials are stored in cleartext and are accessible to anyone who can update the site. *Only* use it in low-stakes applications, e.g. preventing search engines from indexing parts of a site.** The authors of _git-pages_ shall not be held liable for any unauthorized information disclosures resulting from the use of this feature.
 * Incremental updates can be made using `PUT` or `PATCH` requests where the body contains an archive (both tar and zip are supported).
-    - Any archive entry that is a symlink to `/git/pages/<git-sha256>` is replaced with an existing manifest entry for the same site whose git blob hash matches `<git-sha256>`. If there is no existing manifest entry with the specified git hash, the update fails with a `422 Unprocessable Entity`.
+    - Any archive entry that is a symlink to `/git/blobs/<git-sha256>` is replaced with an existing manifest entry for the same site whose git blob hash matches `<git-sha256>`. If there is no existing manifest entry with the specified git hash, the update fails with a `422 Unprocessable Entity`.
     - For this error response only, if the negotiated content type is `application/vnd.git-pages.unresolved`, the response will contain the `<git-sha256>` of each unresolved reference, one per line.
 * Support for SHA-256 Git hashes is [limited by go-git][go-git-sha256]; once go-git implements the required features, _git-pages_ will automatically gain support for SHA-256 Git hashes. Note that shallow clones (used by _git-pages_ to conserve bandwidth if available) aren't supported yet in the Git protocol as of 2025.
+* Git LFS is not supported: it is a single-vendor specification/implementation with no stable Go API and a risk of misuse for reflected HTTP DoS attacks. A diagnostic is emitted for any files uploaded have the `filter=lfs` attribute set via `.gitattributes`.
 
 [_redirects]: https://docs.netlify.com/manage/routing/redirects/overview/
 [_headers]: https://docs.netlify.com/manage/routing/headers/
-[cors]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/CORS
+[basic-auth]: https://docs.netlify.com/manage/security/secure-access-to-sites/basic-authentication-with-custom-http-headers/
+[isolation]: https://web.dev/articles/cross-origin-isolation-guide
 [go-git-sha256]: https://github.com/go-git/go-git/issues/706
 [whiteout]: https://docs.kernel.org/filesystems/overlayfs.html#whiteouts-and-opaque-directories
 
@@ -110,21 +116,22 @@ DNS is the primary authorization method, using either TXT records or wildcard ma
 The authorization flow for content updates (`PUT`, `PATCH`, `DELETE`, `POST` requests) proceeds sequentially in the following order, with the first of multiple applicable rule taking precedence:
 
 1. **Development Mode:** If the environment variable `PAGES_INSECURE` is set to a truthful value like `1`, the request is authorized.
-2. **DNS Challenge:** If the method is `PUT`, `PATCH`, `DELETE`, `POST`, and a well-formed `Authorization:` header is provided containing a `<token>`, and a TXT record lookup at `_git-pages-challenge.<host>` returns a record whose concatenated value equals `SHA256("<host> <token>")`, the request is authorized.
+2. **DNS Challenge:** If the method is `PUT`, `PATCH`, `DELETE`, `POST`, and a well-formed `Authorization:` header is provided containing a `<token>`, and a TXT record lookup at `_git-pages-challenge.<host>` returns a record whose concatenated value equals `SHA256("<host> <token>")`, and (for `PUT` and `POST` requests) the requested branch is `pages`, the request is authorized.
     - **`Pages` scheme:** Request includes an `Authorization: Pages <token>` header.
     - **`Basic` scheme:** Request includes an `Authorization: Basic <basic>` header, where `<basic>` is equal to `Base64("Pages:<token>")`. (Useful for non-Forgejo forges.)
-3. **DNS Allowlist:** If the method is `PUT` or `POST`, and the request URL is `scheme://<user>.<host>/`, and a TXT record lookup at `_git-pages-repository.<host>` returns a set of well-formed absolute URLs, and (for `PUT` requests) the body contains a repository URL, and the requested clone URLs is contained in this set of URLs, the request is authorized.
+3. **DNS Allowlist:** If the method is `PUT` or `POST`, and the request URL is `scheme://<user>.<host>/`, and a TXT record lookup at `_git-pages-repository.<host>` returns a set of well-formed absolute URLs, and (for `PUT` requests) the body contains a repository URL or (for `POST` requests) the body contains a GitHub-style webhook payload, and the requested clone URLs is contained in this set of URLs, and the requested branch is `pages`, the request is authorized.
-4. **Wildcard Match (content):** If the method is `POST`, and a `[[wildcard]]` configuration section exists where the suffix of a hostname (compared label-wise) is equal to `[[wildcard]].domain`, and (for `PUT` requests) the body contains a repository URL, and the requested clone URL is a *matching* clone URL, the request is authorized.
+4. **Wildcard Match (content):** If the method is `POST`, and the body contains a GitHub-style webhook payload, and a `[[wildcard]]` configuration section exists such that `[[wildcard]].domain` is a suffix of the site hostname (compared label-wise), and the body contains a repository URL, and the requested clone URL is a *matching* clone URL, and the requested branch is a *matching* branch, the request is authorized.
-    - **Index repository:** If the request URL is `scheme://<user>.<host>/`, a *matching* clone URL is computed by templating `[[wildcard]].clone-url` with `<user>` and `<project>`, where `<project>` is computed by templating each element of `[[wildcard]].index-repos` with `<user>`, and `[[wildcard]]` is the section where the match occurred.
+    - **Index repository:** If the request URL is `scheme://<user>.<host>/`: a *matching* clone URL is computed by templating `[[wildcard]].clone-url` with `<user>` and `<project>`, where `<project>` is computed by templating `[[wildcard]].index-repo` with `<user>`, and `[[wildcard]]` is the section where the match occurred; and a *matching* branch is specified by `[[wildcard]].index-repo-branch`.
-    - **Project repository:** If the request URL is `scheme://<user>.<host>/<project>/`, a *matching* clone URL is computed by templating `[[wildcard]].clone-url` with `<user>` and `<project>`, and `[[wildcard]]` is the section where the match occurred.
+    - **Project repository:** If the request URL is `scheme://<user>.<host>/<project>/`: a *matching* clone URL is computed by templating `[[wildcard]].clone-url` with `<user>` and `<project>`, and `[[wildcard]]` is the section where the match occurred; and a *matching* branch is `pages`.
-5. **Forge Authorization:** If the method is `PUT` or `PATCH` or `DELETE`, and (unless the method is `DELETE`) the body contains an archive, and a `[[wildcard]]` configuration section exists where the suffix of a hostname (compared label-wise) is equal to `[[wildcard]].domain`, and `[[wildcard]].authorization` is non-empty, and the request includes a `Forge-Authorization:` header, and the header (when forwarded as `Authorization:`) grants push permissions to a repository at the *matching* clone URL (as defined above) as determined by an API call to the forge, the request is authorized. (This enables publishing a site for a private repository.)
+5. **Forge Authorization (wildcard):** If the method is `PUT` or `PATCH` or `DELETE`, and (unless the method is `DELETE`) the body contains an archive, and a `[[wildcard]]` configuration section exists such that `[[wildcard]].domain` is a suffix of the site hostname (compared label-wise), and `[[wildcard]].authorization` is defined, and the request includes a `Forge-Authorization:` header, and the header (when forwarded as `Authorization:`) grants push permissions to a repository at the *matching* clone URL (as defined above) as determined by an API call to the forge, the request is authorized.
-5. **Default Deny:** Otherwise, the request is not authorized.
+6. **Forge Authorization (DNS allowlist):** If the method is `PUT` or `PATCH` or `DELETE`, and (unless the method is `DELETE`) the body contains an archive, and the request URL is `scheme://<host>/`, and a TXT record lookup at `_git-pages-forge-allowlist.<host>` returns a set of well-formed absolute URLs, and the request includes a `Forge-Authorization:` header, and the header (when forwarded as `Authorization:`) grants push permissions to a repository at any of the URLs in the TXT records as determined by an API call to the forge, the request is authorized.
+7. **Default Deny:** Otherwise, the request is not authorized.
 
 The authorization flow for metadata retrieval (`GET` requests with site paths starting with `.git-pages/`) in the following order, with the first of multiple applicable rule taking precedence:
 
 1. **Development Mode:** Same as for content updates.
 2. **DNS Challenge:** Same as for content updates.
-3. **Wildcard Match (metadata):** If a `[[wildcard]]` configuration section exists where the suffix of a hostname (compared label-wise) is equal to `[[wildcard]].domain`, the request is authorized.
+3. **Wildcard Match (metadata):** If a `[[wildcard]]` configuration section exists where the suffix of a hostname (compared label-wise) is equal to `[[wildcard]].domain`, and the site never uses the `Basic-Auth:` pseudo-header, the request is authorized.
 4. **Default Deny:** Otherwise, the request is not authorized.
 
 
@@ -133,10 +140,6 @@ Observability
 
 _git-pages_ has robust observability features built in:
 * The metrics endpoint (bound to `:3002` by default) returns Go, pages server, and storage backend metrics in the [Prometheus](https://prometheus.io/) format.
-* Optional [Sentry](https://sentry.io/) integration allows greater visibility into the application. The `ENVIRONMENT` environment variable configures the deploy environment name (`development` by default).
-    * If `SENTRY_DSN` environment variable is set, panics are reported to Sentry.
-    * If `SENTRY_DSN` and `SENTRY_LOGS=1` environment variables are set, logs are uploaded to Sentry.
-    * If `SENTRY_DSN` and `SENTRY_TRACING=1` environment variables are set, traces are uploaded to Sentry.
 * Optional syslog integration allows transmitting application logs to a syslog daemon. When present, the `SYSLOG_ADDR` environment variable enables the integration, and the value is used to configure the syslog destination. The value must follow the format `family/address` and is usually one of the following:
     * a Unix datagram socket: `unixgram//dev/log`;
     * TLS over TCP: `tcp+tls/host:port`;

conf/config.default.toml

@@ -0,0 +1,37 @@
+# This is a configuration containing default values only. The `config.example.toml` file contains
+# a configuration more useful for demonstration purposes.
+
+log-format = 'text'
+
+[server]
+pages = 'tcp/localhost:3000'
+caddy = 'tcp/localhost:3001'
+metrics = 'tcp/localhost:3002'
+
+[storage]
+type = 'fs'
+
+[storage.fs]
+root = './data'
+
+[limits]
+max-site-size = '128MB'
+max-manifest-size = '1MB'
+max-inline-file-size = '256B'
+git-large-object-threshold = '1MB'
+max-symlink-depth = 16
+update-timeout = '1m0s'
+concurrent-uploads = 1024
+max-heap-size-ratio = 0.5
+forbidden-domains = []
+allowed-repository-url-prefixes = []
+allowed-custom-headers = ['X-Clacks-Overhead']
+allow-basic-auth = false
+
+[audit]
+node-id = 0
+collect = false
+include-ip = ''
+
+[observability]
+slow-response-threshold = '500ms'

conf/config.example.toml

@@ -1,5 +1,5 @@
-# Unless otherwise noted, every value in this file is the same
+# This is a configuration used for demonstration purposes. The `config.default.toml` file contains
-# as the intrinsic default value.
+# a configuration corresponding to default values only.
 
 log-format = "text"
 
@@ -48,10 +48,12 @@ max-inline-file-size = "256B"
 git-large-object-threshold = "1M"
 max-symlink-depth = 16
 update-timeout = "60s"
+concurrent-uploads = 1024
 max-heap-size-ratio = 0.5 # * RAM_size
 forbidden-domains = []
 allowed-repository-url-prefixes = []
 allowed-custom-headers = ["X-Clacks-Overhead"]
+allow-basic-auth = false
 
 [audit]
 node-id = 0

flake.nix

@@ -46,12 +46,18 @@
             ];
           };
 
-          buildInputs = with pkgs; [
+          buildInputs = pkgs.lib.optionals pkgs.stdenv.targetPlatform.isLinux (
-            pkgsStatic.musl
+            with pkgs;
-          ];
+            [
+              pkgsStatic.musl
+            ]
+          );
 
           ldflags = [
             "-linkmode external"
+            "-X main.versionOverride=${self.shortRev or self.dirtyShortRev}"
+          ]
+          ++ pkgs.lib.optionals pkgs.stdenv.targetPlatform.isLinux [
             "-extldflags -static"
             "-s -w"
           ];

go.mod

@@ -5,34 +5,33 @@ go 1.25.0
 require (
 	codeberg.org/git-pages/go-headers v1.1.1
 	codeberg.org/git-pages/go-slog-syslog v0.0.0-20251207093707-892f654e80b7
+	github.com/BurntSushi/toml v1.6.0
 	github.com/KimMachineGun/automemlimit v0.7.5
+	github.com/bits-and-blooms/bloom/v3 v3.7.1
 	github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500
 	github.com/creasty/defaults v1.8.0
-	github.com/dghubble/trie v0.1.0
+	github.com/fatih/color v1.19.0
-	github.com/fatih/color v1.18.0
+	github.com/go-git/go-billy/v6 v6.0.0-20260410103409-85b6241850b5
-	github.com/getsentry/sentry-go v0.43.0
+	github.com/go-git/go-git/v6 v6.0.0-alpha.2
-	github.com/getsentry/sentry-go/slog v0.43.0
-	github.com/go-git/go-billy/v6 v6.0.0-20260226131633-45bd0956d66f
-	github.com/go-git/go-git/v6 v6.0.0-20260305211659-2083cf940afa
 	github.com/jpillora/backoff v1.0.0
 	github.com/kankanreno/go-snowflake v1.2.0
-	github.com/klauspost/compress v1.18.4
+	github.com/klauspost/compress v1.18.5
 	github.com/maypok86/otter/v2 v2.3.0
-	github.com/minio/minio-go/v7 v7.0.99
+	github.com/minio/minio-go/v7 v7.0.100
-	github.com/pelletier/go-toml/v2 v2.2.4
 	github.com/pquerna/cachecontrol v0.2.0
 	github.com/prometheus/client_golang v1.23.2
-	github.com/samber/slog-multi v1.7.1
+	github.com/samber/slog-multi v1.8.0
 	github.com/tj/go-redirects v0.0.0-20200911105812-fd1ba1020b37
 	github.com/valyala/fasttemplate v1.2.2
-	golang.org/x/net v0.51.0
+	golang.org/x/net v0.53.0
 	google.golang.org/protobuf v1.36.11
 )
 
 require (
 	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/ProtonMail/go-crypto v1.3.0 // indirect
+	github.com/ProtonMail/go-crypto v1.4.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bits-and-blooms/bitset v1.24.2 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cloudflare/circl v1.6.3 // indirect
 	github.com/cyphar/filepath-securejoin v0.6.1 // indirect
@@ -41,13 +40,12 @@ require (
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/go-git/gcfg/v2 v2.0.2 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
-	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/kevinburke/ssh_config v1.5.0 // indirect
+	github.com/kevinburke/ssh_config v1.6.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
 	github.com/klauspost/crc32 v1.3.0 // indirect
 	github.com/leodido/go-syslog/v4 v4.3.0 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/minio/crc64nvme v1.1.1 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
@@ -61,8 +59,8 @@ require (
 	github.com/prometheus/common v0.66.1 // indirect
 	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/rs/xid v1.6.0 // indirect
-	github.com/samber/lo v1.52.0 // indirect
+	github.com/samber/lo v1.53.0 // indirect
-	github.com/samber/slog-common v0.20.0 // indirect
+	github.com/samber/slog-common v0.21.0 // indirect
 	github.com/sergi/go-diff v1.4.0 // indirect
 	github.com/stretchr/testify v1.11.1 // indirect
 	github.com/tinylib/msgp v1.6.1 // indirect
@@ -70,8 +68,9 @@ require (
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/crypto v0.48.0 // indirect
+	golang.org/x/crypto v0.50.0 // indirect
-	golang.org/x/sys v0.41.0 // indirect
+	golang.org/x/sync v0.20.0 // indirect
-	golang.org/x/text v0.34.0 // indirect
+	golang.org/x/sys v0.43.0 // indirect
+	golang.org/x/text v0.36.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -2,18 +2,24 @@ codeberg.org/git-pages/go-headers v1.1.1 h1:fpIBELKo66Z2k+gCeYl5mCEXVQ99Lmx1iup1
 codeberg.org/git-pages/go-headers v1.1.1/go.mod h1:N4gwH0U3YPwmuyxqH7xBA8j44fTPX+vOEP7ejJVBPts=
 codeberg.org/git-pages/go-slog-syslog v0.0.0-20251207093707-892f654e80b7 h1:+rkrAxhNZo/eKEcKOqVOsF6ohAPv5amz0JLburOeRjs=
 codeberg.org/git-pages/go-slog-syslog v0.0.0-20251207093707-892f654e80b7/go.mod h1:8NPSXbYcVb71qqNM5cIgn1/uQgMisLbu2dVD1BNxsUw=
+github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk=
+github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
 github.com/KimMachineGun/automemlimit v0.7.5 h1:RkbaC0MwhjL1ZuBKunGDjE/ggwAX43DwZrJqVwyveTk=
 github.com/KimMachineGun/automemlimit v0.7.5/go.mod h1:QZxpHaGOQoYvFhv/r4u3U0JTC2ZcOwbSr11UZF46UBM=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
-github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw=
+github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM=
-github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE=
+github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bits-and-blooms/bitset v1.24.2 h1:M7/NzVbsytmtfHbumG+K2bremQPMJuqv1JD3vOaFxp0=
+github.com/bits-and-blooms/bitset v1.24.2/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
+github.com/bits-and-blooms/bloom/v3 v3.7.1 h1:WXovk4TRKZttAMJfoQx6K2DM0zNIt8w+c67UqO+etV0=
+github.com/bits-and-blooms/bloom/v3 v3.7.1/go.mod h1:rZzYLLje2dfzXfAkJNxQQHsKurAyK55KUnL43Euk0hU=
 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 h1:6lhrsTEnloDPXyeZBvSYvQf8u86jbKehZPVDDlkgDl4=
 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
@@ -27,34 +33,24 @@ github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dghubble/trie v0.1.0 h1:kJnjBLFFElBwS60N4tkPvnLhnpcDxbBjIulgI8CpNGM=
-github.com/dghubble/trie v0.1.0/go.mod h1:sOmnzfBNH7H92ow2292dDFWNsVQuh/izuD7otCYb1ak=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
 github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
-github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
+github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w=
-github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
+github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE=
-github.com/getsentry/sentry-go v0.43.0 h1:XbXLpFicpo8HmBDaInk7dum18G9KSLcjZiyUKS+hLW4=
-github.com/getsentry/sentry-go v0.43.0/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
-github.com/getsentry/sentry-go/slog v0.43.0 h1:BYGiM4VFu4//S0vrTSf52MmZSmjhOikHIkBeZZw9P4Q=
-github.com/getsentry/sentry-go/slog v0.43.0/go.mod h1:EAq/2dhW43dV7fwy4OjTWSsvhZjTM9jjsck0kYt9MYE=
 github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
 github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
-github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
-github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-git/gcfg/v2 v2.0.2 h1:MY5SIIfTGGEMhdA7d7JePuVVxtKL7Hp+ApGDJAJ7dpo=
 github.com/go-git/gcfg/v2 v2.0.2/go.mod h1:/lv2NsxvhepuMrldsFilrgct6pxzpGdSRC13ydTLSLs=
-github.com/go-git/go-billy/v6 v6.0.0-20260226131633-45bd0956d66f h1:Uvbx7nITO3Sd1GdXarX0TbyYmOaSNIJP0mm4LocEyyA=
+github.com/go-git/go-billy/v6 v6.0.0-20260410103409-85b6241850b5 h1:r5Y4Hn9QwQj+u6vN0Ib1MipHkanYaG8Zj0kxsnv8Bu4=
-github.com/go-git/go-billy/v6 v6.0.0-20260226131633-45bd0956d66f/go.mod h1:ZW9JC5gionMP1kv5uiaOaV23q0FFmNrVOV8VW+y/acc=
+github.com/go-git/go-billy/v6 v6.0.0-20260410103409-85b6241850b5/go.mod h1:CdBVp7CXl9l3sOyNEog46cP1Pvx/hjCe9AD0mtaIUYU=
-github.com/go-git/go-git-fixtures/v5 v5.1.2-0.20260122163445-0622d7459a67 h1:3hutPZF+/FBjR/9MdsLJ7e1mlt9pwHgwxMW7CrbmWII=
+github.com/go-git/go-git-fixtures/v6 v6.0.0-20260405195209-b16dd39735e0 h1:XoTsdvaghuVfIr7HpNTmFDLu2nz3I2iGqyn6Uk6MkJc=
-github.com/go-git/go-git-fixtures/v5 v5.1.2-0.20260122163445-0622d7459a67/go.mod h1:xKt0pNHST9tYHvbiLxSY27CQWFwgIxBJuDrOE0JvbZw=
+github.com/go-git/go-git-fixtures/v6 v6.0.0-20260405195209-b16dd39735e0/go.mod h1:1Lr7/vYEYyl6Ir9Ku0tKrCIRreM5zovv0Jdx2MPSM4s=
-github.com/go-git/go-git/v6 v6.0.0-20260305211659-2083cf940afa h1:fIbZ264qSeJ+GRz+5nq6SFonkCanp/6CRXhYutq8GlE=
+github.com/go-git/go-git/v6 v6.0.0-alpha.2 h1:T3loNtDuAixNzXtlQxZhnYiYpaQ3CA4vn9RssAniEeI=
-github.com/go-git/go-git/v6 v6.0.0-20260305211659-2083cf940afa/go.mod h1:V/qoTD4qCYizR+fKFA9++d2APoE8Yheci7dXALaSeuI=
+github.com/go-git/go-git/v6 v6.0.0-alpha.2/go.mod h1:oCD3i19CTz7gBpeb11ZZqL91WzqbMq9avn5KpUYy/Ak=
 github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
 github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
-github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=
-github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@@ -63,10 +59,10 @@ github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2E
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/kankanreno/go-snowflake v1.2.0 h1:Zx2SctsH5pivIj9vyhwyDyQS23jcDJx4iT49Bjv81kk=
 github.com/kankanreno/go-snowflake v1.2.0/go.mod h1:6CZ+10PeVsFXKZUTYyJzPiRIjn1IXbInaWLCX/LDJ0g=
-github.com/kevinburke/ssh_config v1.5.0 h1:3cPZmE54xb5j3G5xQCjSvokqNwU2uW+3ry1+PRLSPpA=
+github.com/kevinburke/ssh_config v1.6.0 h1:J1FBfmuVosPHf5GRdltRLhPJtJpTlMdKTBjRgTaQBFY=
-github.com/kevinburke/ssh_config v1.5.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M=
+github.com/kevinburke/ssh_config v1.6.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M=
-github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
+github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
-github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
+github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
 github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
@@ -76,16 +72,14 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
-github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/leodido/go-syslog/v4 v4.3.0 h1:bbSpI/41bYK9iSdlYzcwvlxuLOE8yi4VTFmedtnghdA=
 github.com/leodido/go-syslog/v4 v4.3.0/go.mod h1:eJ8rUfDN5OS6dOkCOBYlg2a+hbAg6pJa99QXXgMrd98=
-github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
-github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
-github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/maypok86/otter/v2 v2.3.0 h1:8H8AVVFUSzJwIegKwv1uF5aGitTY+AIrtktg7OcLs8w=
@@ -94,18 +88,14 @@ github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI
 github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
-github.com/minio/minio-go/v7 v7.0.99 h1:2vH/byrwUkIpFQFOilvTfaUpvAX3fEFhEzO+DR3DlCE=
+github.com/minio/minio-go/v7 v7.0.100 h1:ShkWi8Tyj9RtU57OQB2HIXKz4bFgtVib0bbT1sbtLI8=
-github.com/minio/minio-go/v7 v7.0.99/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw=
+github.com/minio/minio-go/v7 v7.0.100/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
-github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
-github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
 github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
 github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
-github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
-github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pjbgf/sha1cd v0.5.0 h1:a+UkboSi1znleCDUNT3M5YxjOnN1fz2FhN48FlwCxs0=
 github.com/pjbgf/sha1cd v0.5.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -126,12 +116,12 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
 github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
-github.com/samber/lo v1.52.0 h1:Rvi+3BFHES3A8meP33VPAxiBZX/Aws5RxrschYGjomw=
+github.com/samber/lo v1.53.0 h1:t975lj2py4kJPQ6haz1QMgtId2gtmfktACxIXArw3HM=
-github.com/samber/lo v1.52.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0=
+github.com/samber/lo v1.53.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0=
-github.com/samber/slog-common v0.20.0 h1:WaLnm/aCvBJSk5nR5aXZTFBaV0B47A+AEaEOiZDeUnc=
+github.com/samber/slog-common v0.21.0 h1:Wo2hTly1Br5RjYqX/BTWJJeDnTE85oWk/7vqlpZuAUc=
-github.com/samber/slog-common v0.20.0/go.mod h1:+Ozat1jgnnE59UAlmNX1IF3IByHsODnnwf9jUcBZ+m8=
+github.com/samber/slog-common v0.21.0/go.mod h1:d/6OaSlzdkl9PFpfRLgn8FwY1OW6EFmPtBpsHX4MrU0=
-github.com/samber/slog-multi v1.7.1 h1:aCLXHRxgU+2v0PVlEOh7phynzM7CRo89ZgFtOwaqVEE=
+github.com/samber/slog-multi v1.8.0 h1:E05c1wnQ+8M58oQDBABlJ4TEIJWssNgtckso3zlaLlI=
-github.com/samber/slog-multi v1.7.1/go.mod h1:A4KQC99deqfkCDJcL/cO3kX6McX7FffQAx/8QHink+c=
+github.com/samber/slog-multi v1.8.0/go.mod h1:6+3j/ILxDvAcLD75YdQAm6iKWu6AmwlohLgQxL/2aiI=
 github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
 github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -145,6 +135,8 @@ github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
 github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
 github.com/tj/go-redirects v0.0.0-20200911105812-fd1ba1020b37 h1:K11tjwz8zTTSZkz4TUjfLN+y8uJWP38BbyPqZ2yB/Yk=
 github.com/tj/go-redirects v0.0.0-20200911105812-fd1ba1020b37/go.mod h1:E0E2H2gQA+uoi27VCSU+a/BULPtadQA78q3cpTjZbZw=
+github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg=
+github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
@@ -155,18 +147,19 @@ go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
 go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
 go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
-golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
+golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
-golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
+golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
-golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
+golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
-golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
+golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
+golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
-golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
+golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
-golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
+golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
-golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
+golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
-golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
+golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
 google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
 google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

gomod2nix.toml

@@ -7,6 +7,9 @@ schema = 3
   [mod."codeberg.org/git-pages/go-slog-syslog"]
     version = "v0.0.0-20251207093707-892f654e80b7"
     hash = "sha256-ye+DBIyxqTEOViYRrQPWyGJCaLmyKSDwH5btlqDPizM="
+  [mod."github.com/BurntSushi/toml"]
+    version = "v1.6.0"
+    hash = "sha256-ptdUJvuc21ixeLt+M5way/na3aCnCO4MYHWulWp8NEY="
   [mod."github.com/KimMachineGun/automemlimit"]
     version = "v0.7.5"
     hash = "sha256-lH/ip9j2hbYUc2W/XIYve/5TScQPZtEZe3hu76CY//k="
@@ -14,11 +17,17 @@ schema = 3
     version = "v0.6.2"
     hash = "sha256-tVNWDUMILZbJvarcl/E7tpSnkn7urqgSHa2Eaka5vSU="
   [mod."github.com/ProtonMail/go-crypto"]
-    version = "v1.3.0"
+    version = "v1.4.1"
-    hash = "sha256-TUG+C4MyeWglOmiwiW2/NUVurFHXLgEPRd3X9uQ1NGI="
+    hash = "sha256-6iGAFCjoNveY+ipbKqq2gt+RXpi2eQyPXAY01rxPcWc="
   [mod."github.com/beorn7/perks"]
     version = "v1.0.1"
     hash = "sha256-h75GUqfwJKngCJQVE5Ao5wnO3cfKD9lSIteoLp/3xJ4="
+  [mod."github.com/bits-and-blooms/bitset"]
+    version = "v1.24.2"
+    hash = "sha256-hT88EpdWmKnqdxApJhs/aIAptf33HmtSp2KXPI+Ym7o="
+  [mod."github.com/bits-and-blooms/bloom/v3"]
+    version = "v3.7.1"
+    hash = "sha256-KZduCu+k4+xqBcFRTfg8Yc/PEf5jfpjn0I1YoxfnVPo="
   [mod."github.com/c2h5oh/datasize"]
     version = "v0.0.0-20231215233829-aa82cc1e6500"
     hash = "sha256-8MqL7xCvE6fIjanz2jwkaLP1OE5kLu62TOcQx452DHQ="
@@ -37,9 +46,6 @@ schema = 3
   [mod."github.com/davecgh/go-spew"]
     version = "v1.1.1"
     hash = "sha256-nhzSUrE1fCkN0+RL04N4h8jWmRFPPPWbCuDc7Ss0akI="
-  [mod."github.com/dghubble/trie"]
-    version = "v0.1.0"
-    hash = "sha256-hVh7uYylpMCCSPcxl70hJTmzSwaA1MxBmJFBO5Xdncc="
   [mod."github.com/dustin/go-humanize"]
     version = "v1.0.1"
     hash = "sha256-yuvxYYngpfVkUg9yAmG99IUVmADTQA0tMbBXe0Fq0Mc="
@@ -47,29 +53,20 @@ schema = 3
     version = "v1.18.1"
     hash = "sha256-hGDKddjLj+5dn2woHtXKUdd49/3xdsqnhx7VEdCu1m4="
   [mod."github.com/fatih/color"]
-    version = "v1.18.0"
+    version = "v1.19.0"
-    hash = "sha256-pP5y72FSbi4j/BjyVq/XbAOFjzNjMxZt2R/lFFxGWvY="
+    hash = "sha256-YgMm1nid8yigNLG6aHfuMbsvMI1UYVf/Rkg44pp/NTU="
-  [mod."github.com/getsentry/sentry-go"]
-    version = "v0.43.0"
-    hash = "sha256-Wu1inIhjuAw6wKburwqIlNxC0I4akunHGh/8DOqo3xg="
-  [mod."github.com/getsentry/sentry-go/slog"]
-    version = "v0.43.0"
-    hash = "sha256-FJMx2E8anKtHknn867gCkYPjitZb9Okqp2uZ+dV7JqA="
   [mod."github.com/go-git/gcfg/v2"]
     version = "v2.0.2"
     hash = "sha256-icqMDeC/tEg/3979EuEN67Ml5KjdDA0R3QvR6iLLrSI="
   [mod."github.com/go-git/go-billy/v6"]
-    version = "v6.0.0-20260226131633-45bd0956d66f"
+    version = "v6.0.0-20260410103409-85b6241850b5"
-    hash = "sha256-s+dthtn+JewJ58R5VbvWaEoYLozDt5YpkHyXcN0xMvQ="
+    hash = "sha256-2qQeUjkswSqI9joCKhvMB1lvnKHL9INbAzy4UBveHsw="
   [mod."github.com/go-git/go-git/v6"]
-    version = "v6.0.0-20260305211659-2083cf940afa"
+    version = "v6.0.0-alpha.2"
-    hash = "sha256-aUUgVODQVanWVQ44tcrOKIvtzJPlZKFNPPvEdAxdPxw="
+    hash = "sha256-nUjRn1uIZKhIKqdNXfTirGtm07XCUKF2z3aat9O0dqM="
   [mod."github.com/go-ini/ini"]
     version = "v1.67.0"
     hash = "sha256-V10ahGNGT+NLRdKUyRg1dos5RxLBXBk1xutcnquc/+4="
-  [mod."github.com/golang/groupcache"]
-    version = "v0.0.0-20241129210726-2c02b8208cf8"
-    hash = "sha256-AdLZ3dJLe/yduoNvZiXugZxNfmwJjNQyQGsIdzYzH74="
   [mod."github.com/google/uuid"]
     version = "v1.6.0"
     hash = "sha256-VWl9sqUzdOuhW0KzQlv0gwwUQClYkmZwSydHG2sALYw="
@@ -80,11 +77,11 @@ schema = 3
     version = "v1.2.0"
     hash = "sha256-713xGEqjwaUGIu2EHII5sldWmcquFpxZmte/7R/O6LA="
   [mod."github.com/kevinburke/ssh_config"]
-    version = "v1.5.0"
+    version = "v1.6.0"
-    hash = "sha256-4SijlenzNuWb5CavWrky8qoQj+6fKCJgOiQANzN5TUE="
+    hash = "sha256-i/EYNJx0+HbAGFVoiKV4QF/zqb4fWewh+bpBKUkXDCc="
   [mod."github.com/klauspost/compress"]
-    version = "v1.18.4"
+    version = "v1.18.5"
-    hash = "sha256-swwNE6xKz4ZAOUHPWFlHYiqFeZLRZuuKYhLQ34aYnAU="
+    hash = "sha256-H9b5iFJf4XbEnkGQCjGQAJ3aYhVDiolKrDewTbhuzQo="
   [mod."github.com/klauspost/cpuid/v2"]
     version = "v2.3.0"
     hash = "sha256-50JhbQyT67BK38HIdJihPtjV7orYp96HknI2VP7A9Yc="
@@ -95,8 +92,8 @@ schema = 3
     version = "v4.3.0"
     hash = "sha256-fCJ2rgrrPR/Ey/PoAsJhd8Sl8mblAnnMAmBuoWFBTgg="
   [mod."github.com/mattn/go-colorable"]
-    version = "v0.1.13"
+    version = "v0.1.14"
-    hash = "sha256-qb3Qbo0CELGRIzvw7NVM1g/aayaz4Tguppk9MD2/OI8="
+    hash = "sha256-JC60PjKj7MvhZmUHTZ9p372FV72I9Mxvli3fivTbxuA="
   [mod."github.com/mattn/go-isatty"]
     version = "v0.0.20"
     hash = "sha256-qhw9hWtU5wnyFyuMbKx+7RB8ckQaFQ8D+8GKPkN3HHQ="
@@ -110,17 +107,14 @@ schema = 3
     version = "v1.1.2"
     hash = "sha256-vykcXvy2VBBAXnJott/XsGTT0gk2UL36JzZKfJ1KAUY="
   [mod."github.com/minio/minio-go/v7"]
-    version = "v7.0.99"
+    version = "v7.0.100"
-    hash = "sha256-Q2VISIvHDggBzidGWzgHbVUZrDCsSIGBPcWfMJcC39w="
+    hash = "sha256-MjWYoX4b+OOSOkjsitQQqcTbpQ7CYNghN9XCdrqgYaM="
   [mod."github.com/munnerz/goautoneg"]
     version = "v0.0.0-20191010083416-a7dc8b61c822"
     hash = "sha256-79URDDFenmGc9JZu+5AXHToMrtTREHb3BC84b/gym9Q="
   [mod."github.com/pbnjay/memory"]
     version = "v0.0.0-20210728143218-7b4eea64cf58"
     hash = "sha256-QI+F1oPLOOtwNp8+m45OOoSfYFs3QVjGzE0rFdpF/IA="
-  [mod."github.com/pelletier/go-toml/v2"]
-    version = "v2.2.4"
-    hash = "sha256-8qQIPldbsS5RO8v/FW/se3ZsAyvLzexiivzJCbGRg2Q="
   [mod."github.com/philhofer/fwd"]
     version = "v1.2.0"
     hash = "sha256-cGx2/0QQay46MYGZuamFmU0TzNaFyaO+J7Ddzlr/3dI="
@@ -152,14 +146,14 @@ schema = 3
     version = "v1.6.0"
     hash = "sha256-rJB7h3KuH1DPp5n4dY3MiGnV1Y96A10lf5OUl+MLkzU="
   [mod."github.com/samber/lo"]
-    version = "v1.52.0"
+    version = "v1.53.0"
-    hash = "sha256-xgMsPJv3rydHH10NZU8wz/DhK2VbbR8ymivOg1ChTp0="
+    hash = "sha256-RCf4Buf357TTWQnMPSWKrfdJ4L/RqOHNBD0g3+VpMw8="
   [mod."github.com/samber/slog-common"]
-    version = "v0.20.0"
+    version = "v0.21.0"
-    hash = "sha256-aWcvt9XNyKaolLhvthcXeFDl0t6uo7Vdo8WzCducf1E="
+    hash = "sha256-i9Nl4xRbk8qYM+0n48IQ6+vGZiS7xFe+GgyV3X9/Spc="
   [mod."github.com/samber/slog-multi"]
-    version = "v1.7.1"
+    version = "v1.8.0"
-    hash = "sha256-wHXt2lwFfjm1p7jnZi44SlHtjdk531BGz2O9pfiylxo="
+    hash = "sha256-KsFwNP9QMDr8golYoevpGtcqUuCrIT7zmGwR7/E6gzo="
   [mod."github.com/sergi/go-diff"]
     version = "v1.4.0"
     hash = "sha256-rs9NKpv/qcQEMRg7CmxGdP4HGuFdBxlpWf9LbA9wS4k="
@@ -188,17 +182,20 @@ schema = 3
     version = "v3.0.4"
     hash = "sha256-NkGFiDPoCxbr3LFsI6OCygjjkY0rdmg5ggvVVwpyDQ4="
   [mod."golang.org/x/crypto"]
-    version = "v0.48.0"
+    version = "v0.50.0"
-    hash = "sha256-uBIGGSGmWWklRxX6XTOqUECzz165UFY9Y99Ka3pLKAw="
+    hash = "sha256-vC1BJT7+3UBWLyEE5n3to0NKhMo6m2HGow2HiFgpQLo="
   [mod."golang.org/x/net"]
-    version = "v0.51.0"
+    version = "v0.53.0"
-    hash = "sha256-bLDpVRTPWM7IowHw1jdr9EPCRQNAVFsPwz69olySah4="
+    hash = "sha256-G9gKLmyaf6lIV429NKX+YlL6oUPJwlv+BrG6qGhzvmU="
+  [mod."golang.org/x/sync"]
+    version = "v0.20.0"
+    hash = "sha256-ybcjhCfK6lroUM0yswUvWooW8MOQZBXyiSqoxG6Uy0Y="
   [mod."golang.org/x/sys"]
-    version = "v0.41.0"
+    version = "v0.43.0"
-    hash = "sha256-owjs3/IzAKfFlIz1U1fiHSfl2+bTUhaXTyWEjL5SWHk="
+    hash = "sha256-aDQXqSTZES2l/132PBxhZN4ywldpPyfm7LByYCHzzwM="
   [mod."golang.org/x/text"]
-    version = "v0.34.0"
+    version = "v0.36.0"
-    hash = "sha256-wGKd1JkeiFROibvo2kkAuQ7JajSIfV4utGaoGbTQhQM="
+    hash = "sha256-/0t9C6Mc8kYjxweFB0us2lGKo8GovHhBiq5nlMOppC0="
   [mod."google.golang.org/protobuf"]
     version = "v1.36.11"
     hash = "sha256-7W+6jntfI/awWL3JP6yQedxqP5S9o3XvPgJ2XxxsIeE="

main.go

@@ -2,6 +2,27 @@
 
 package main
 
-import gitpages "codeberg.org/git-pages/git-pages/src"
+import (
+	"runtime/debug"
 
-func main() { gitpages.Main() }
+	git_pages "codeberg.org/git-pages/git-pages/src"
+)
+
+// By default the version information is retrieved from VCS. If not available during build,
+// override this variable using linker flags to change the displayed version.
+// Example: `-ldflags "-X main.versionOverride=v1.2.3"`
+var versionOverride = ""
+
+func extractVersion() string {
+	if versionOverride != "" {
+		return versionOverride
+	} else if buildInfo, ok := debug.ReadBuildInfo(); ok {
+		return buildInfo.Main.Version
+	} else {
+		panic("version information not available")
+	}
+}
+
+func main() {
+	git_pages.Main(extractVersion())
+}

src/audit.go

@@ -50,6 +50,8 @@ func GetPrincipal(ctx context.Context) *Principal {
 	return nil
 }
 
+var AuditSnowflakeStartTime = time.Date(2025, 12, 1, 0, 0, 0, 0, time.UTC)
+
 type AuditID int64
 
 func GenerateAuditID() AuditID {
@@ -74,6 +76,7 @@ func (id AuditID) String() string {
 
 func (id AuditID) CompareTime(when time.Time) int {
 	idMillis := int64(id) >> (snowflake.MachineIDLength + snowflake.SequenceLength)
+	idMillis += AuditSnowflakeStartTime.UnixMilli()
 	whenMillis := when.UTC().UnixNano() / 1e6
 	return cmp.Compare(idMillis, whenMillis)
 }
@@ -108,6 +111,9 @@ func (record *AuditRecord) DescribePrincipal() string {
 				record.Principal.GetForgeUser().GetHandle(),
 				record.Principal.GetForgeUser().GetId()))
 		}
+		if record.Principal.GetRepoUrl() != "" {
+			items = append(items, record.Principal.GetRepoUrl())
+		}
 		if record.Principal.GetCliAdmin() {
 			items = append(items, "<cli-admin>")
 		}
@@ -129,6 +135,14 @@ func (record *AuditRecord) DescribeResource() string {
 	return desc
 }
 
+func (record *AuditRecord) IsDetachable() bool {
+	return record.GetEvent() == AuditEvent_CommitManifest
+}
+
+func (record *AuditRecord) IsDetached() bool {
+	return record.IsDetachable() && record.Manifest == nil
+}
+
 type AuditRecordScope int
 
 const (

src/auth.go

@@ -78,16 +78,25 @@ func GetHost(r *http.Request) (string, error) {
 	return host, nil
 }
 
-func IsValidProjectName(name string) bool {
+func ValidateProjectName(name string) error {
-	return !strings.HasPrefix(name, ".") && !strings.Contains(name, "%")
+	if strings.HasPrefix(name, ".") {
+		return fmt.Errorf("must not start with %q", ".")
+	}
+
+	forbiddenChars := "%*"
+	if strings.ContainsAny(name, forbiddenChars) {
+		return fmt.Errorf("must not contain any of %q", forbiddenChars)
+	}
+
+	return nil
 }
 
 func GetProjectName(r *http.Request) (string, error) {
 	// path must be either `/` or `/foo/` (`/foo` is accepted as an alias)
 	path := strings.TrimPrefix(strings.TrimSuffix(r.URL.Path, "/"), "/")
-	if !IsValidProjectName(path) {
+	if err := ValidateProjectName(path); err != nil {
 		return "", AuthError{http.StatusBadRequest,
-			fmt.Sprintf("directory name %q is reserved", ".index")}
+			fmt.Sprintf("directory name: %v", err)}
 	} else if strings.Contains(path, "/") {
 		return "", AuthError{http.StatusBadRequest,
 			"directories nested too deep"}
@@ -110,6 +119,13 @@ type Authorization struct {
 	forgeUser *ForgeUser
 }
 
+func (auth *Authorization) ForgeRepoURL() string {
+	if auth.forgeUser != nil && len(auth.repoURLs) == 1 {
+		return auth.repoURLs[0]
+	}
+	return ""
+}
+
 func authorizeDNSChallenge(r *http.Request) (*Authorization, error) {
 	host, err := GetHost(r)
 	if err != nil {
@@ -179,7 +195,7 @@ func authorizeDNSChallenge(r *http.Request) (*Authorization, error) {
 	}, nil
 }
 
-func authorizeDNSAllowlist(r *http.Request) (*Authorization, error) {
+func authorizeDNSAllowlist(r *http.Request, scope string) (*Authorization, error) {
 	host, err := GetHost(r)
 	if err != nil {
 		return nil, err
@@ -190,7 +206,7 @@ func authorizeDNSAllowlist(r *http.Request) (*Authorization, error) {
 		return nil, err
 	}
 
-	allowlistHostname := fmt.Sprintf("_git-pages-repository.%s", host)
+	allowlistHostname := fmt.Sprintf("_%s.%s", scope, host)
 	records, err := net.LookupTXT(allowlistHostname)
 	if err != nil {
 		return nil, AuthError{http.StatusUnauthorized,
@@ -315,17 +331,16 @@ func authorizeCodebergPagesV2(r *http.Request) (*Authorization, error) {
 			if domainParts[0] == "page" && domainParts[1] == "codeberg" {
 				// map of domain names to allowed repository and branch:
 				//  * {username}.codeberg.page =>
-				//      https://codeberg.org/{username}/pages.git#main
+				//      https://codeberg.org/{username}/pages.git#pages
 				//  * {reponame}.{username}.codeberg.page =>
 				//      https://codeberg.org/{username}/{reponame}.git#pages
 				//  * {branch}.{reponame}.{username}.codeberg.page =>
 				//      https://codeberg.org/{username}/{reponame}.git#{branch}
 				username := domainParts[2]
 				reponame := "pages"
-				branch := "main"
+				branch := "pages"
 				if len(domainParts) >= 4 {
 					reponame = domainParts[3]
-					branch = "pages"
 				}
 				if len(domainParts) == 5 {
 					branch = domainParts[4]
@@ -347,7 +362,7 @@ func authorizeCodebergPagesV2(r *http.Request) (*Authorization, error) {
 }
 
 // Checks whether an operation that enables enumerating site contents is allowed.
-func AuthorizeMetadataRetrieval(r *http.Request) (*Authorization, error) {
+func AuthorizeMetadataRetrieval(r *http.Request, hasBasicAuth bool) (*Authorization, error) {
 	causes := []error{AuthError{http.StatusUnauthorized, "unauthorized"}}
 
 	auth := authorizeInsecure(r)
@@ -365,27 +380,32 @@ func AuthorizeMetadataRetrieval(r *http.Request) (*Authorization, error) {
 		return auth, nil
 	}
 
-	for _, pattern := range wildcards {
+	// Normally, sites that correspond to a forge via a wildcard match are considered completely
-		auth, err = authorizeWildcardMatchHost(r, pattern)
+	// public and safe to retrieve without authorization. However, this is no longer the case if
-		if err != nil && IsUnauthorized(err) {
+	// they have password-protected sections.
-			causes = append(causes, err)
+	if !hasBasicAuth {
-		} else if err != nil { // bad request
+		for _, pattern := range wildcards {
-			return nil, err
+			auth, err = authorizeWildcardMatchHost(r, pattern)
-		} else {
+			if err != nil && IsUnauthorized(err) {
-			logc.Printf(r.Context(), "auth: wildcard %s\n", pattern.GetHost())
+				causes = append(causes, err)
-			return auth, nil
+			} else if err != nil { // bad request
+				return nil, err
+			} else {
+				logc.Printf(r.Context(), "auth: wildcard %s\n", pattern.GetHost())
+				return auth, nil
+			}
 		}
-	}
 
-	if config.Feature("codeberg-pages-compat") {
+		if config.Feature("codeberg-pages-compat") {
-		auth, err = authorizeCodebergPagesV2(r)
+			auth, err = authorizeCodebergPagesV2(r)
-		if err != nil && IsUnauthorized(err) {
+			if err != nil && IsUnauthorized(err) {
-			causes = append(causes, err)
+				causes = append(causes, err)
-		} else if err != nil { // bad request
+			} else if err != nil { // bad request
-			return nil, err
+				return nil, err
-		} else {
+			} else {
-			logc.Printf(r.Context(), "auth: codeberg %s\n", r.Host)
+				logc.Printf(r.Context(), "auth: codeberg %s\n", r.Host)
-			return auth, nil
+				return auth, nil
+			}
 		}
 	}
 
@@ -417,7 +437,7 @@ func AuthorizeUpdateFromRepository(r *http.Request) (*Authorization, error) {
 
 	// DNS allowlist gives authority to update but not delete.
 	if r.Method == http.MethodPut || r.Method == http.MethodPost {
-		auth, err = authorizeDNSAllowlist(r)
+		auth, err = authorizeDNSAllowlist(r, "git-pages-repository")
 		if err != nil && IsUnauthorized(err) {
 			causes = append(causes, err)
 		} else if err != nil { // bad request
@@ -459,21 +479,23 @@ func AuthorizeUpdateFromRepository(r *http.Request) (*Authorization, error) {
 	return nil, joinErrors(causes...)
 }
 
-func checkAllowedURLPrefix(repoURL string) error {
+func checkAllowedURLPrefixes(repoURLs ...string) error {
 	if len(config.Limits.AllowedRepositoryURLPrefixes) > 0 {
-		allowedPrefix := false
+		for _, repoURL := range repoURLs {
-		repoURL = strings.ToLower(repoURL)
+			allowedPrefix := false
-		for _, allowedRepoURLPrefix := range config.Limits.AllowedRepositoryURLPrefixes {
+			repoURL = strings.ToLower(repoURL)
-			if strings.HasPrefix(repoURL, strings.ToLower(allowedRepoURLPrefix)) {
+			for _, allowedRepoURLPrefix := range config.Limits.AllowedRepositoryURLPrefixes {
-				allowedPrefix = true
+				if strings.HasPrefix(repoURL, strings.ToLower(allowedRepoURLPrefix)) {
-				break
+					allowedPrefix = true
+					break
+				}
 			}
-		}
+			if !allowedPrefix {
-		if !allowedPrefix {
+				return AuthError{
-			return AuthError{
+					http.StatusUnauthorized,
-				http.StatusUnauthorized,
+					fmt.Sprintf("clone URL %v not in prefix allowlist %v",
-				fmt.Sprintf("clone URL not in prefix allowlist %v",
+						repoURL, config.Limits.AllowedRepositoryURLPrefixes),
-					config.Limits.AllowedRepositoryURLPrefixes),
+				}
 			}
 		}
 	}
@@ -506,7 +528,7 @@ func AuthorizeRepository(repoURL string, auth *Authorization) error {
 		return nil // any
 	}
 
-	if err = checkAllowedURLPrefix(repoURL); err != nil {
+	if err = checkAllowedURLPrefixes(repoURL); err != nil {
 		return err
 	}
 
@@ -574,6 +596,11 @@ func checkGogsRepositoryPushPermission(baseURL *url.URL, authorization string) e
 			http.StatusNotFound,
 			fmt.Sprintf("no repository %s", ownerAndRepo),
 		}
+	} else if response.StatusCode == http.StatusUnauthorized {
+		return AuthError{
+			http.StatusUnauthorized,
+			fmt.Sprintf("no access to %s or invalid token", ownerAndRepo),
+		}
 	} else if response.StatusCode != http.StatusOK {
 		return AuthError{
 			http.StatusServiceUnavailable,
@@ -609,7 +636,7 @@ func checkGogsRepositoryPushPermission(baseURL *url.URL, authorization string) e
 }
 
 // Gogs, Gitea, and Forgejo all support the same API here.
-func fetchGogsAuthorizedUser(baseURL *url.URL, authorization string) (*ForgeUser, error) {
+func fetchGogsAuthorizedUser(baseURL *url.URL, forgeToken string) (*ForgeUser, error) {
 	request, err := http.NewRequest("GET", baseURL.ResolveReference(&url.URL{
 		Path: "/api/v1/user",
 	}).String(), nil)
@@ -617,7 +644,7 @@ func fetchGogsAuthorizedUser(baseURL *url.URL, authorization string) (*ForgeUser
 		panic(err) // misconfiguration
 	}
 	request.Header.Set("Accept", "application/json")
-	request.Header.Set("Authorization", authorization)
+	request.Header.Set("Authorization", forgeToken)
 
 	httpClient := http.Client{Timeout: 5 * time.Second}
 	response, err := httpClient.Do(request)
@@ -663,9 +690,34 @@ func fetchGogsAuthorizedUser(baseURL *url.URL, authorization string) (*ForgeUser
 	}, nil
 }
 
-func authorizeForgeWithToken(r *http.Request) (*Authorization, error) {
+// Check whether a forge token has access to a repository, and if it does, which user it
-	authorization := r.Header.Get("Forge-Authorization")
+// belongs to. Precondition: `repoURL` is well-formed.
-	if authorization == "" {
+func authorizeGogsUser(repoURL string, forgeToken string) (*Authorization, error) {
+	parsedRepoURL, err := url.Parse(repoURL)
+	if err != nil {
+		panic(err)
+	}
+
+	if err = checkGogsRepositoryPushPermission(parsedRepoURL, forgeToken); err != nil {
+		return nil, err
+	}
+
+	authorizedUser, err := fetchGogsAuthorizedUser(parsedRepoURL, forgeToken)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Authorization{
+		repoURLs:  []string{repoURL},
+		forgeUser: authorizedUser,
+	}, nil
+}
+
+// Validates a provided forge token against a repository URL constructed by mapping the host
+// and project name via the `[[wildcard]]` section of the configuration file.
+func authorizeForgeWildcard(r *http.Request) (*Authorization, error) {
+	forgeToken := r.Header.Get("Forge-Authorization")
+	if forgeToken == "" {
 		return nil, AuthError{http.StatusUnauthorized, "missing Forge-Authorization header"}
 	}
 
@@ -681,88 +733,61 @@ func authorizeForgeWithToken(r *http.Request) (*Authorization, error) {
 
 	var errs []error
 	for _, pattern := range wildcards {
-		if !pattern.Authorization {
+		if pattern.Authorization {
-			continue
+			if userName, found := pattern.Matches(host); found {
-		}
+				repoURL, branch := pattern.ApplyTemplate(userName, projectName)
-
+				auth, err := authorizeGogsUser(repoURL, forgeToken)
-		if userName, found := pattern.Matches(host); found {
+				if err != nil {
-			repoURL, branch := pattern.ApplyTemplate(userName, projectName)
+					errs = append(errs, err)
-			parsedRepoURL, err := url.Parse(repoURL)
+				} else {
-			if err != nil {
+					auth.branch = branch
-				panic(err) // misconfiguration
+					return auth, nil
-			}
+				}
-
+			}
-			if err = checkGogsRepositoryPushPermission(parsedRepoURL, authorization); err != nil {
-				errs = append(errs, err)
-				continue
-			}
-
-			authorizedUser, err := fetchGogsAuthorizedUser(parsedRepoURL, authorization)
-			if err != nil {
-				errs = append(errs, err)
-				continue
-			}
-
-			return &Authorization{
-				// This will actually be ignored by the callers of AuthorizeUpdateFromArchive and
-				// AuthorizeDeletion, but we return this information as it makes sense to do
-				// contextually here.
-				repoURLs: []string{repoURL},
-				branch:   branch,
-
-				forgeUser: authorizedUser,
-			}, nil
 		}
 	}
+	if len(errs) == 0 {
+		errs = append(errs, AuthError{http.StatusUnauthorized, "no matching wildcard domain"})
+	}
 
 	errs = append([]error{
-		AuthError{http.StatusUnauthorized, "not authorized by forge"},
+		AuthError{http.StatusUnauthorized, "not authorized by forge (wildcard)"},
 	}, errs...)
 	return nil, joinErrors(errs...)
 }
 
-func AuthorizeUpdateFromArchive(r *http.Request) (*Authorization, error) {
+// Validates a provided forge token against a repository URL extracted from the DNS allowlist
-	causes := []error{AuthError{http.StatusUnauthorized, "unauthorized"}}
+// records of the target domain specified in `_git-pages-forge-authorization.*`.
-
+func authorizeForgeDNSAllowlist(r *http.Request) (*Authorization, error) {
-	if err := CheckForbiddenDomain(r); err != nil {
+	forgeToken := r.Header.Get("Forge-Authorization")
-		return nil, err
+	if forgeToken == "" {
+		return nil, AuthError{http.StatusUnauthorized, "missing Forge-Authorization header"}
 	}
 
-	auth := authorizeInsecure(r)
+	var errs []error
-	if auth != nil {
+	if dnsAuth, err := authorizeDNSAllowlist(r, "git-pages-forge-allowlist"); err != nil {
-		return auth, nil
+		errs = append(errs, err)
-	}
+	} else if dnsAuth != nil {
-
+		// DNS allows uploads from some repositories, but we don't know yet if the forge token
-	// Token authorization allows updating a site on a wildcard domain from an archive.
+		// has a push permission to any of these repositories.
-	auth, err := authorizeForgeWithToken(r)
+		for _, repoURL := range dnsAuth.repoURLs {
-	if err != nil && IsUnauthorized(err) {
+			auth, err := authorizeGogsUser(repoURL, forgeToken)
-		causes = append(causes, err)
+			if err != nil {
-	} else if err != nil { // bad request
+				errs = append(errs, err)
-		return nil, err
+			} else {
-	} else {
+				// There is both DNS authorization and forge authorization.
-		logc.Printf(r.Context(), "auth: forge token: allow\n")
+				return auth, nil
-		return auth, nil
+			}
-	}
-
-	if len(config.Limits.AllowedRepositoryURLPrefixes) > 0 {
-		causes = append(causes, AuthError{http.StatusUnauthorized, "DNS challenge not allowed"})
-	} else {
-		// DNS challenge gives absolute authority.
-		auth, err = authorizeDNSChallenge(r)
-		if err != nil && IsUnauthorized(err) {
-			causes = append(causes, err)
-		} else if err != nil { // bad request
-			return nil, err
-		} else {
-			logc.Println(r.Context(), "auth: DNS challenge")
-			return auth, nil
 		}
 	}
 
-	return nil, joinErrors(causes...)
+	errs = append([]error{
+		AuthError{http.StatusUnauthorized, "not authorized by forge (DNS allowlist)"},
+	}, errs...)
+	return nil, joinErrors(errs...)
 }
 
-func AuthorizeDeletion(r *http.Request) (*Authorization, error) {
+func authorizeDNSChallengeOrForgeWithToken(r *http.Request) (*Authorization, error) {
 	causes := []error{AuthError{http.StatusUnauthorized, "unauthorized"}}
 
 	if err := CheckForbiddenDomain(r); err != nil {
@@ -774,29 +799,70 @@ func AuthorizeDeletion(r *http.Request) (*Authorization, error) {
 		return auth, nil
 	}
 
+	// DNS challenge gives absolute authority.
 	auth, err := authorizeDNSChallenge(r)
 	if err != nil && IsUnauthorized(err) {
 		causes = append(causes, err)
 	} else if err != nil { // bad request
 		return nil, err
 	} else {
-		logc.Printf(r.Context(), "auth: DNS challenge: allow *\n")
+		logc.Println(r.Context(), "auth: DNS challenge: allow *")
 		return auth, nil
 	}
 
-	auth, err = authorizeForgeWithToken(r)
+	// Token authorization allows updating a site on a wildcard domain from an archive.
+	// This sub-method uses the `[[wildcard]]` configuration section to derive repository URL.
+	auth, err = authorizeForgeWildcard(r)
 	if err != nil && IsUnauthorized(err) {
 		causes = append(causes, err)
 	} else if err != nil { // bad request
 		return nil, err
 	} else {
-		logc.Printf(r.Context(), "auth: forge token: allow\n")
+		logc.Printf(r.Context(), "auth: forge (wildcard): allow\n")
+		return auth, nil
+	}
+
+	// Token authorization allows updating a site on a wildcard domain from an archive.
+	// This sub-method uses the DNS allowlist authorization mechanism to derive repository URL.
+	auth, err = authorizeForgeDNSAllowlist(r)
+	if err != nil && IsUnauthorized(err) {
+		causes = append(causes, err)
+	} else if err != nil { // bad request
+		return nil, err
+	} else {
+		logc.Printf(r.Context(), "auth: forge (DNS allowlist): allow\n")
 		return auth, nil
 	}
 
 	return nil, joinErrors(causes...)
 }
 
+func AuthorizeUpdateFromArchive(r *http.Request) (*Authorization, error) {
+	auth, err := authorizeDNSChallengeOrForgeWithToken(r)
+	if err != nil {
+		return nil, err
+	}
+
+	// If only uploads from specific repositories are allowed, then only forge authorization
+	// is acceptable, and the repository must match the configured limits.
+	if len(config.Limits.AllowedRepositoryURLPrefixes) > 0 {
+		if len(auth.repoURLs) == 0 {
+			logc.Println(r.Context(), "auth: DNS challenge: deny (limits)")
+			return nil, AuthError{http.StatusUnauthorized, "DNS challenge not allowed"}
+		}
+
+		if err = checkAllowedURLPrefixes(auth.repoURLs...); err != nil {
+			return nil, err
+		}
+	}
+
+	return auth, nil
+}
+
+func AuthorizeDeletion(r *http.Request) (*Authorization, error) {
+	return authorizeDNSChallengeOrForgeWithToken(r)
+}
+
 func CheckForbiddenDomain(r *http.Request) error {
 	host, err := GetHost(r)
 	if err != nil {

src/backend.go

@@ -124,9 +124,13 @@ type Backend interface {
 	// Delete a manifest.
 	DeleteManifest(ctx context.Context, name string, opts ModifyManifestOptions) error
 
-	// Iterate through all manifests. Whether manifests that are newly added during iteration
+	// Iterate through metadata of all manifests. Whether manifests that are newly added during
-	// will appear in the results is unspecified.
+	// iteration will appear in the results is unspecified.
-	EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error]
+	EnumerateManifests(ctx context.Context) iter.Seq2[*ManifestMetadata, error]
+
+	// Iterate through contents of all manifests. Same considerations apply as for
+	// `EnumerateManifests`.
+	GetAllManifests(ctx context.Context) iter.Seq2[tuple[*ManifestMetadata, *Manifest], error]
 
 	// Check whether a domain has any deployments.
 	CheckDomain(ctx context.Context, domain string) (found bool, err error)
@@ -134,21 +138,33 @@ type Backend interface {
 	// Create a domain. This allows us to start serving content for the domain.
 	CreateDomain(ctx context.Context, domain string) error
 
-	// Freeze  a domain. This allows a site to be administratively locked, e.g. if it
+	// Freeze a domain. This allows a site to be administratively locked, e.g. if it
 	// is discovered serving abusive content.
 	FreezeDomain(ctx context.Context, domain string) error
 
 	// Thaw a domain. This removes the previously placed administrative lock (if any).
 	UnfreezeDomain(ctx context.Context, domain string) error
 
+	// Check whether the set of domains we serve has changed since the time passed to this method.
+	HaveDomainsChanged(ctx context.Context, since time.Time) (changed bool, err error)
+
 	// Append a record to the audit log.
 	AppendAuditLog(ctx context.Context, id AuditID, record *AuditRecord) error
 
 	// Retrieve a single record from the audit log.
 	QueryAuditLog(ctx context.Context, id AuditID) (record *AuditRecord, err error)
 
-	// Retrieve records from the audit log by time range.
+	// Retrieve record IDs from the audit log by time range.
 	SearchAuditLog(ctx context.Context, opts SearchAuditLogOptions) iter.Seq2[AuditID, error]
+
+	// Retrieve audit record contents for given IDs.
+	GetAuditLogRecords(ctx context.Context, ids iter.Seq2[AuditID, error]) iter.Seq2[*AuditRecord, error]
+
+	// Detach an audit record from its blobs.
+	DetachAuditRecord(ctx context.Context, id AuditID) error
+
+	// Delete an audit record with a given ID.
+	ExpireAuditRecord(ctx context.Context, id AuditID) error
 }
 
 func CreateBackend(ctx context.Context, config *StorageConfig) (backend Backend, err error) {

src/backend_fs.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 )
 
 type FSBackend struct {
@@ -152,7 +153,13 @@ func (fs *FSBackend) PutBlob(ctx context.Context, name string, data []byte) erro
 	}
 
 	if err := fs.blobRoot.Chmod(tempPath, 0o444); err != nil {
-		return fmt.Errorf("chmod: %w", err)
+		if errors.Is(err, os.ErrPermission) {
+			// NFSv4 configured with ACLs doesn't have a working `chmod` even though it's a Unix
+			// system. This `chmod` call is done entirely for convenience (to help the system
+			// administrator avoid accidentally overwriting files), so just skip it.
+		} else {
+			return fmt.Errorf("chmod: %w", err)
+		}
 	}
 
 again:
@@ -396,12 +403,12 @@ func (fs *FSBackend) DeleteManifest(
 	}
 }
 
-func (fs *FSBackend) EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error] {
+func (fs *FSBackend) EnumerateManifests(ctx context.Context) iter.Seq2[*ManifestMetadata, error] {
-	return func(yield func(ManifestMetadata, error) bool) {
+	return func(yield func(*ManifestMetadata, error) bool) {
 		iofs.WalkDir(fs.siteRoot.FS(), ".",
 			func(path string, entry iofs.DirEntry, err error) error {
 				_, project, _ := strings.Cut(path, "/")
-				var metadata ManifestMetadata
+				var metadata *ManifestMetadata
 				if err != nil {
 					// report error
 				} else if entry.IsDir() {
@@ -414,9 +421,11 @@ func (fs *FSBackend) EnumerateManifests(ctx context.Context) iter.Seq2[ManifestM
 					// report error
 				} else {
 					// report blob
-					metadata.Name = path
+					metadata = &ManifestMetadata{
-					metadata.Size = info.Size()
+						Name:         path,
-					metadata.LastModified = info.ModTime()
+						Size:         info.Size(),
+						LastModified: info.ModTime(),
+					}
 					// not setting metadata.ETag since it is too costly
 				}
 				if !yield(metadata, err) {
@@ -427,6 +436,22 @@ func (fs *FSBackend) EnumerateManifests(ctx context.Context) iter.Seq2[ManifestM
 	}
 }
 
+func (fs *FSBackend) GetAllManifests(ctx context.Context) iter.Seq2[tuple[*ManifestMetadata, *Manifest], error] {
+	return func(yield func(tuple[*ManifestMetadata, *Manifest], error) bool) {
+		for metadata, err := range fs.EnumerateManifests(ctx) {
+			var item tuple[*ManifestMetadata, *Manifest]
+			if err == nil {
+				var manifest *Manifest
+				manifest, _, err = backend.GetManifest(ctx, metadata.Name, GetManifestOptions{})
+				item = tuple[*ManifestMetadata, *Manifest]{metadata, manifest}
+			}
+			if !yield(item, err) {
+				break
+			}
+		}
+	}
+}
+
 func (fs *FSBackend) CheckDomain(ctx context.Context, domain string) (bool, error) {
 	_, err := fs.siteRoot.Stat(domain)
 	if errors.Is(err, os.ErrNotExist) {
@@ -455,12 +480,20 @@ func (fs *FSBackend) UnfreezeDomain(ctx context.Context, domain string) error {
 	}
 }
 
+func (fs *FSBackend) HaveDomainsChanged(ctx context.Context, since time.Time) (bool, error) {
+	return true, nil // not implemented
+}
+
+func auditDetachedName(id AuditID) string {
+	return fmt.Sprintf("%s.detached", id)
+}
+
 func (fs *FSBackend) AppendAuditLog(ctx context.Context, id AuditID, record *AuditRecord) error {
 	if _, err := fs.auditRoot.Stat(id.String()); err == nil {
 		panic(fmt.Errorf("audit ID collision: %s", id))
 	}
 
-	return fs.auditRoot.WriteFile(id.String(), EncodeAuditRecord(record), 0o644)
+	return fs.auditRoot.WriteFile(id.String(), EncodeAuditRecord(record), 0o444)
 }
 
 func (fs *FSBackend) QueryAuditLog(ctx context.Context, id AuditID) (*AuditRecord, error) {
@@ -469,6 +502,11 @@ func (fs *FSBackend) QueryAuditLog(ctx context.Context, id AuditID) (*AuditRecor
 	} else if record, err := DecodeAuditRecord(data); err != nil {
 		return nil, fmt.Errorf("decode: %w", err)
 	} else {
+		if _, err := fs.auditRoot.Stat(auditDetachedName(id)); err == nil {
+			record.Manifest = nil
+		} else if !errors.Is(err, os.ErrNotExist) {
+			return nil, fmt.Errorf("stat detached marker: %w", err)
+		}
 		return record, nil
 	}
 }
@@ -485,6 +523,8 @@ func (fs *FSBackend) SearchAuditLog(
 				var id AuditID
 				if err != nil {
 					// report error
+				} else if strings.Contains(path, ".") {
+					return nil // skip
 				} else if id, err = ParseAuditID(path); err != nil {
 					// report error
 				} else if !opts.Since.IsZero() && id.CompareTime(opts.Since) < 0 {
@@ -500,3 +540,27 @@ func (fs *FSBackend) SearchAuditLog(
 			})
 	}
 }
+
+func (fs *FSBackend) GetAuditLogRecords(
+	ctx context.Context, ids iter.Seq2[AuditID, error],
+) iter.Seq2[*AuditRecord, error] {
+	return func(yield func(*AuditRecord, error) bool) {
+		for id, err := range ids {
+			var record *AuditRecord
+			if err == nil {
+				record, err = fs.QueryAuditLog(ctx, id)
+			}
+			if !yield(record, err) {
+				break
+			}
+		}
+	}
+}
+
+func (fs *FSBackend) DetachAuditRecord(ctx context.Context, id AuditID) error {
+	return fs.auditRoot.WriteFile(auditDetachedName(id), []byte{}, 0o644)
+}
+
+func (fs *FSBackend) ExpireAuditRecord(ctx context.Context, id AuditID) error {
+	return fs.auditRoot.Remove(id.String())
+}

src/backend_s3.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"path"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/c2h5oh/datasize"
@@ -178,6 +179,12 @@ func NewS3Backend(ctx context.Context, config *S3Config) (*S3Backend, error) {
 		if err != nil {
 			return nil, err
 		}
+
+		err = (&S3Backend{client: client, bucket: bucket}).
+			EnableFeature(ctx, FeatureCheckDomainMarker)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	initS3BackendMetrics()
@@ -583,7 +590,7 @@ func (s3 *S3Backend) CommitManifest(
 	data := EncodeManifest(manifest)
 	logc.Printf(ctx, "s3: commit manifest %x -> %s", sha256.Sum256(data), name)
 
-	_, domain, _ := strings.Cut(name, "/")
+	domain, _, _ := strings.Cut(name, "/")
 	if err := s3.checkDomainFrozen(ctx, domain); err != nil {
 		return err
 	}
@@ -625,7 +632,7 @@ func (s3 *S3Backend) DeleteManifest(
 ) error {
 	logc.Printf(ctx, "s3: delete manifest %s\n", name)
 
-	_, domain, _ := strings.Cut(name, "/")
+	domain, _, _ := strings.Cut(name, "/")
 	if err := s3.checkDomainFrozen(ctx, domain); err != nil {
 		return err
 	}
@@ -636,12 +643,15 @@ func (s3 *S3Backend) DeleteManifest(
 
 	err := s3.client.RemoveObject(ctx, s3.bucket, manifestObjectName(name),
 		minio.RemoveObjectOptions{})
+	if err != nil {
+		return err
+	}
 	s3.siteCache.Cache.Invalidate(name)
-	return err
+	return s3.bumpLastDomainUpdateTimestamp(ctx)
 }
 
-func (s3 *S3Backend) EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error] {
+func (s3 *S3Backend) EnumerateManifests(ctx context.Context) iter.Seq2[*ManifestMetadata, error] {
-	return func(yield func(ManifestMetadata, error) bool) {
+	return func(yield func(*ManifestMetadata, error) bool) {
 		logc.Print(ctx, "s3: enumerate manifests")
 
 		ctx, cancel := context.WithCancel(ctx)
@@ -652,7 +662,7 @@ func (s3 *S3Backend) EnumerateManifests(ctx context.Context) iter.Seq2[ManifestM
 			Prefix:    prefix,
 			Recursive: true,
 		}) {
-			var metadata ManifestMetadata
+			var metadata *ManifestMetadata
 			var err error
 			if err = object.Err; err == nil {
 				key := strings.TrimPrefix(object.Key, prefix)
@@ -662,10 +672,12 @@ func (s3 *S3Backend) EnumerateManifests(ctx context.Context) iter.Seq2[ManifestM
 				} else if project == "" || strings.HasPrefix(project, ".") && project != ".index" {
 					continue // internal; skip
 				} else {
-					metadata.Name = key
+					metadata = &ManifestMetadata{
-					metadata.Size = object.Size
+						Name:         key,
-					metadata.LastModified = object.LastModified
+						Size:         object.Size,
-					metadata.ETag = object.ETag
+						LastModified: object.LastModified,
+						ETag:         object.ETag,
+					}
 				}
 			}
 			if !yield(metadata, err) {
@@ -675,6 +687,48 @@ func (s3 *S3Backend) EnumerateManifests(ctx context.Context) iter.Seq2[ManifestM
 	}
 }
 
+// Limits the number of concurrent uploads, globally across the entire git-pages process.
+// Not currently configurable as there seems to be little need.
+var getAllManifestsSemaphore = make(chan struct{}, 64)
+
+func (s3 *S3Backend) GetAllManifests(ctx context.Context) iter.Seq2[tuple[*ManifestMetadata, *Manifest], error] {
+	type result struct {
+		metadata *ManifestMetadata
+		manifest *Manifest
+		err      error
+	}
+
+	resultsChan := make(chan result)
+	enumeratorCtx, cancel := context.WithCancel(ctx)
+	go func(ctx context.Context) {
+		wg := sync.WaitGroup{}
+		for metadata, err := range s3.EnumerateManifests(ctx) {
+			if err != nil {
+				resultsChan <- result{nil, nil, err}
+			} else {
+				getAllManifestsSemaphore <- struct{}{} // acquire
+				wg.Go(func() {
+					defer func() { <-getAllManifestsSemaphore }() // release
+					manifest, _, err := backend.GetManifest(ctx, metadata.Name, GetManifestOptions{})
+					resultsChan <- result{metadata, manifest, err}
+				})
+			}
+		}
+		wg.Wait()
+		close(resultsChan)
+	}(enumeratorCtx)
+
+	return func(yield func(tuple[*ManifestMetadata, *Manifest], error) bool) {
+		for result := range resultsChan {
+			item := tuple[*ManifestMetadata, *Manifest]{result.metadata, result.manifest}
+			if !yield(item, result.err) {
+				cancel()
+				break
+			}
+		}
+	}
+}
+
 func domainCheckObjectName(domain string) string {
 	return manifestObjectName(fmt.Sprintf("%s/.exists", domain))
 }
@@ -713,8 +767,19 @@ func (s3 *S3Backend) CheckDomain(ctx context.Context, domain string) (exists boo
 func (s3 *S3Backend) CreateDomain(ctx context.Context, domain string) error {
 	logc.Printf(ctx, "s3: create domain %s\n", domain)
 
-	_, err := s3.client.PutObject(ctx, s3.bucket, domainCheckObjectName(domain),
+	exists, err := s3.CheckDomain(ctx, domain)
+	if err != nil {
+		return err
+	}
+
+	_, err = s3.client.PutObject(ctx, s3.bucket, domainCheckObjectName(domain),
 		&bytes.Reader{}, 0, minio.PutObjectOptions{})
+	if err != nil {
+		return err
+	}
+	if !exists {
+		err = s3.bumpLastDomainUpdateTimestamp(ctx)
+	}
 	return err
 }
 
@@ -739,10 +804,33 @@ func (s3 *S3Backend) UnfreezeDomain(ctx context.Context, domain string) error {
 	}
 }
 
+const lastDomainUpdateObjectName = "meta/last-domain-update"
+
+func (s3 *S3Backend) HaveDomainsChanged(ctx context.Context, since time.Time) (bool, error) {
+	info, err := s3.client.StatObject(ctx, s3.bucket, lastDomainUpdateObjectName,
+		minio.GetObjectOptions{})
+	if err != nil {
+		return false, err
+	}
+
+	return info.LastModified.After(since), nil
+}
+
+func (s3 *S3Backend) bumpLastDomainUpdateTimestamp(ctx context.Context) error {
+	logc.Print(ctx, "s3: bumping last domain update timestamp")
+	_, err := s3.client.PutObject(ctx, s3.bucket, lastDomainUpdateObjectName,
+		&bytes.Reader{}, 0, minio.PutObjectOptions{})
+	return err
+}
+
 func auditObjectName(id AuditID) string {
 	return fmt.Sprintf("audit/%s", id)
 }
 
+func auditDetachedObjectName(id AuditID) string {
+	return fmt.Sprintf("audit/%s.detached", id)
+}
+
 func (s3 *S3Backend) AppendAuditLog(ctx context.Context, id AuditID, record *AuditRecord) error {
 	logc.Printf(ctx, "s3: append audit %s\n", id)
 
@@ -774,7 +862,20 @@ func (s3 *S3Backend) QueryAuditLog(ctx context.Context, id AuditID) (*AuditRecor
 		return nil, err
 	}
 
-	return DecodeAuditRecord(data)
+	record, err := DecodeAuditRecord(data)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = s3.client.StatObject(ctx, s3.bucket, auditDetachedObjectName(id),
+		minio.StatObjectOptions{})
+	if err == nil {
+		record.Manifest = nil
+	} else if errResp := minio.ToErrorResponse(err); err != nil && errResp.Code != "NoSuchKey" {
+		return nil, err
+	}
+
+	return record, nil
 }
 
 func (s3 *S3Backend) SearchAuditLog(
@@ -794,8 +895,14 @@ func (s3 *S3Backend) SearchAuditLog(
 			var err error
 			if object.Err != nil {
 				err = object.Err
-			} else {
+			} else if strings.Contains(object.Key, ".") {
-				id, err = ParseAuditID(strings.TrimPrefix(object.Key, prefix))
+				continue
+			} else if id, err = ParseAuditID(strings.TrimPrefix(object.Key, prefix)); err != nil {
+				// report error
+			} else if !opts.Since.IsZero() && id.CompareTime(opts.Since) < 0 {
+				continue
+			} else if !opts.Until.IsZero() && id.CompareTime(opts.Until) > 0 {
+				continue
 			}
 			if !yield(id, err) {
 				break
@@ -803,3 +910,55 @@ func (s3 *S3Backend) SearchAuditLog(
 		}
 	}
 }
+
+var getAuditLogRecordsSemaphore = make(chan struct{}, 64)
+
+func (s3 *S3Backend) GetAuditLogRecords(
+	ctx context.Context, ids iter.Seq2[AuditID, error],
+) iter.Seq2[*AuditRecord, error] {
+	return func(yield func(*AuditRecord, error) bool) {
+		resultsChan := make(chan tuple[*AuditRecord, error])
+		enumeratorCtx, cancel := context.WithCancel(ctx)
+		defer cancel()
+
+		go func(ctx context.Context) {
+			wg := sync.WaitGroup{}
+			for id, err := range ids {
+				if err != nil {
+					resultsChan <- tuple[*AuditRecord, error]{nil, err}
+				} else {
+					getAuditLogRecordsSemaphore <- struct{}{} // acquire
+					wg.Go(func() {
+						defer func() { <-getAuditLogRecordsSemaphore }() // release
+						record, err := s3.QueryAuditLog(ctx, id)
+						resultsChan <- tuple[*AuditRecord, error]{record, err}
+					})
+				}
+			}
+			wg.Wait()
+			close(resultsChan)
+		}(enumeratorCtx)
+
+		for result := range resultsChan {
+			record, err := result.Splat()
+			if !yield(record, err) {
+				break
+			}
+		}
+	}
+}
+
+func (s3 *S3Backend) DetachAuditRecord(ctx context.Context, id AuditID) error {
+	logc.Printf(ctx, "s3: detach audit record %s\n", id)
+
+	_, err := s3.client.PutObject(ctx, s3.bucket, auditDetachedObjectName(id),
+		&bytes.Reader{}, 0, minio.PutObjectOptions{})
+	return err
+}
+
+func (s3 *S3Backend) ExpireAuditRecord(ctx context.Context, id AuditID) error {
+	logc.Printf(ctx, "s3: expire audit record %s\n", id)
+
+	return s3.client.RemoveObject(ctx, s3.bucket, auditObjectName(id),
+		minio.RemoveObjectOptions{})
+}

src/caddy.go

@@ -26,7 +26,17 @@ func ServeCaddy(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	found, err := backend.CheckDomain(r.Context(), strings.ToLower(domain))
+	var err error
+	domain = strings.ToLower(domain)
+
+	// Run a cheap check as to whether we might be serving the domain.
+	var found = domainCache.CheckDomain(r.Context(), domain)
+
+	if found {
+		// Run an expensive check as to whether we are actually serving the domain.
+		found, err = backend.CheckDomain(r.Context(), domain)
+	}
+
 	if !found {
 		// If we don't serve the domain, but a fallback server does, then we should let our
 		// Caddy instance request a TLS certificate. Otherwise, we'll never have an opportunity

src/collect.go

@@ -84,7 +84,11 @@ func CollectTar(
 			header.Typeflag = tar.TypeSymlink
 			header.Mode = 0644
 			header.ModTime = metadata.LastModified
-			err = appendFile(&header, entry.GetData(), Transform_Identity)
+			header.Linkname = string(entry.GetData())
+			err = archive.WriteHeader(&header)
+			if err != nil {
+				return fmt.Errorf("tar: %w", err)
+			}
 
 		default:
 			panic(fmt.Errorf("CollectTar encountered invalid entry: %v, %v",

src/config.go

@@ -12,9 +12,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/BurntSushi/toml"
 	"github.com/c2h5oh/datasize"
 	"github.com/creasty/defaults"
-	"github.com/pelletier/go-toml/v2"
 )
 
 // For an unknown reason, the standard `time.Duration` type doesn't implement the standard
@@ -134,6 +134,8 @@ type LimitsConfig struct {
 	// Maximum time that an update operation (PUT or POST request) could take before being
 	// interrupted.
 	UpdateTimeout Duration `toml:"update-timeout" default:"60s"`
+	// Maximum number of concurrent blob uploads, globally across every update request.
+	ConcurrentUploads uint `toml:"concurrent-uploads" default:"1024"`
 	// Soft limit on Go heap size, expressed as a fraction of total available RAM.
 	MaxHeapSizeRatio float64 `toml:"max-heap-size-ratio" default:"0.5"`
 	// List of domains unconditionally forbidden for uploads.
@@ -144,6 +146,9 @@ type LimitsConfig struct {
 	// e.g. `Foo-Bar`. Setting this option permits including this custom header in `_headers`,
 	// unless it is fundamentally unsafe.
 	AllowedCustomHeaders []string `toml:"allowed-custom-headers" default:"[\"X-Clacks-Overhead\"]"`
+	// Whether to allow Netlify-style credentials specified in a `Basic-Auth:` pseudo-header.
+	// These credentials are plaintext.
+	AllowBasicAuth bool `toml:"allow-basic-auth" default:"false"`
 }
 
 type AuditConfig struct {
@@ -304,25 +309,43 @@ func PrintConfigEnvVars() {
 	})
 }
 
-func Configure(tomlPath string) (config *Config, err error) {
+func PrettyTomlKey(key toml.Key) string {
+	if len(key) == 1 {
+		return key.String()
+	} else {
+		// `toml.Key.String()` adds quotes if necessary.
+		return fmt.Sprintf("[%s].%s", key[:len(key)-1].String(), key[len(key)-1:].String())
+	}
+}
+
+func ReadConfigFile(config *Config, tomlPath string) (err error) {
+	if tomlPath != "" {
+		meta, err := toml.DecodeFile(tomlPath, config)
+		if err != nil {
+			return err
+		}
+
+		unknownKeys := []string{}
+		for _, key := range meta.Undecoded() {
+			unknownKeys = append(unknownKeys, PrettyTomlKey(key))
+		}
+		if len(unknownKeys) > 0 {
+			return fmt.Errorf("unknown keys: %s", strings.Join(unknownKeys, ", "))
+		}
+	}
+	return nil
+}
+
+func Configure(tomlPaths ...string) (config *Config, err error) {
 	// start with an all-default configuration
 	config = new(Config)
 	defaults.MustSet(config)
 
-	// inject values from `config.toml`
+	// inject values from each toml file
-	if tomlPath != "" {
+	for _, tomlPath := range tomlPaths {
-		var file *os.File
+		err := ReadConfigFile(config, tomlPath)
-		file, err = os.Open(tomlPath)
 		if err != nil {
-			return
+			return nil, err
-		}
-		defer file.Close()
-
-		decoder := toml.NewDecoder(file)
-		decoder.DisallowUnknownFields()
-		decoder.EnableUnmarshalerInterface()
-		if err = decoder.Decode(&config); err != nil {
-			return
 		}
 	}
 

src/domain_cache.go

@@ -0,0 +1,132 @@
+package git_pages
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/bits-and-blooms/bloom/v3"
+)
+
+type DomainCache interface {
+	// Check if we might be serving the domain.
+	CheckDomain(ctx context.Context, domain string) (found bool)
+
+	// Add the domain to the cache.
+	AddDomain(ctx context.Context, domain string)
+}
+
+func CreateDomainCache(ctx context.Context) (DomainCache, error) {
+	if !config.Feature("domain-existence-cache") {
+		return &dummyDomainCache{}, nil
+	}
+	return createBloomDomainCache(ctx)
+}
+
+type bloomDomainCache struct {
+	filter   *bloom.BloomFilter
+	filterMu sync.Mutex
+
+	accessCh    chan struct{}
+	refreshMu   sync.Mutex
+	lastRefresh time.Time
+	maxAge      time.Duration
+}
+
+func createBloomDomainCache(ctx context.Context) (DomainCache, error) {
+	cache := bloomDomainCache{
+		accessCh: make(chan struct{}),
+	}
+
+	switch config.Storage.Type {
+	case "fs":
+		// the FS backend has no cache
+	case "s3":
+		cache.maxAge = time.Duration(config.Storage.S3.SiteCache.MaxAge)
+	default:
+		panic(fmt.Errorf("unknown backend: %s", config.Storage.Type))
+	}
+
+	if err := cache.refresh(ctx); err != nil {
+		return nil, err
+	}
+
+	go cache.handleFilterUpdates(ctx)
+
+	return &cache, nil
+}
+
+func (c *bloomDomainCache) handleFilterUpdates(ctx context.Context) {
+	for range c.accessCh {
+		if time.Since(c.lastRefresh) > c.maxAge {
+			logc.Print(ctx, "domain cache: refreshing")
+			if err := c.refresh(ctx); err != nil {
+				logc.Printf(ctx, "domain cache: refresh error: %v", err)
+			}
+		}
+	}
+}
+
+func (c *bloomDomainCache) refresh(ctx context.Context) error {
+	c.refreshMu.Lock()
+	defer c.refreshMu.Unlock()
+
+	if changed, err := backend.HaveDomainsChanged(ctx, c.lastRefresh); err != nil {
+		return err
+	} else if !changed {
+		logc.Print(ctx, "domain cache: unchanged")
+		c.lastRefresh = time.Now()
+		return nil
+	}
+
+	// Create a 256 KiB Bloom filter that will fit ~150K entries with 0.1% false positive rate.
+	filter := bloom.New(256*1024, 10)
+	for metadata, err := range backend.EnumerateManifests(ctx) {
+		if err != nil {
+			return fmt.Errorf("enum manifests: %w", err)
+		}
+		domain, _, _ := strings.Cut(metadata.Name, "/")
+		filter.AddString(domain)
+	}
+
+	c.filterMu.Lock()
+	c.filter = filter
+	c.filterMu.Unlock()
+
+	logc.Printf(ctx, "domain cache: refreshed with approx. %d domains", filter.ApproximatedSize())
+	c.lastRefresh = time.Now()
+	return nil
+}
+
+func (c *bloomDomainCache) CheckDomain(ctx context.Context, domain string) (found bool) {
+	select {
+	case c.accessCh <- struct{}{}:
+	default:
+	}
+
+	c.filterMu.Lock()
+	found = c.filter.TestString(domain)
+	c.filterMu.Unlock()
+
+	logc.Printf(ctx, "domain cache: bloom filter returns %v for %q", found, domain)
+	return
+}
+
+func (c *bloomDomainCache) AddDomain(ctx context.Context, domain string) {
+	c.refreshMu.Lock()
+	defer c.refreshMu.Unlock()
+
+	c.filterMu.Lock()
+	c.filter.AddString(domain)
+	c.filterMu.Unlock()
+
+	logc.Printf(ctx, "domain cache: added %q", domain)
+}
+
+type dummyDomainCache struct{}
+
+func (d dummyDomainCache) CheckDomain(context.Context, string) bool { return true }
+
+func (d dummyDomainCache) AddDomain(context.Context, string) {}

src/extract.go

@@ -15,7 +15,6 @@ import (
 	"strings"
 
 	"github.com/c2h5oh/datasize"
-	"github.com/go-git/go-git/v6/plumbing"
 	"github.com/klauspost/compress/zstd"
 )
 
@@ -52,16 +51,6 @@ func ExtractZstd(
 	return next(ctx, boundArchiveStream(stream))
 }
 
-const BlobReferencePrefix = "/git/blobs/"
-
-type UnresolvedRefError struct {
-	missing []string
-}
-
-func (err UnresolvedRefError) Error() string {
-	return fmt.Sprintf("%d unresolved blob references", len(err.missing))
-}
-
 func normalizeArchiveMemberName(fileName string) string {
 	// Strip the leading slash and any extraneous path segments.
 	fileName = path.Clean(fileName)
@@ -72,21 +61,6 @@ func normalizeArchiveMemberName(fileName string) string {
 	return fileName
 }
 
-// Returns a map of git hash to entry. If `manifest` is nil, returns an empty map.
-func indexManifestByGitHash(manifest *Manifest) map[string]*Entry {
-	index := map[string]*Entry{}
-	for _, entry := range manifest.GetContents() {
-		if hash := entry.GetGitHash(); hash != "" {
-			if _, ok := plumbing.FromHex(hash); ok {
-				index[hash] = entry
-			} else {
-				panic(fmt.Errorf("index: malformed hash: %s", hash))
-			}
-		}
-	}
-	return index
-}
-
 func addSymlinkOrBlobReference(
 	manifest *Manifest, fileName string, target string,
 	index map[string]*Entry, missing *[]string,
@@ -110,9 +84,10 @@ func ExtractTar(ctx context.Context, reader io.Reader, oldManifest *Manifest) (*
 	var dataBytesRecycled int64
 	var dataBytesTransferred int64
 
-	index := indexManifestByGitHash(oldManifest)
+	index := IndexManifestByGitHash(oldManifest)
 	missing := []string{}
 	manifest := NewManifest()
+	hardLinks := map[string]*Entry{}
 	for {
 		header, err := archive.Next()
 		if err == io.EOF {
@@ -133,12 +108,27 @@ func ExtractTar(ctx context.Context, reader io.Reader, oldManifest *Manifest) (*
 			if err != nil {
 				return nil, fmt.Errorf("tar: %s: %w", fileName, err)
 			}
-			AddFile(manifest, fileName, fileData)
+			entry := AddFile(manifest, fileName, fileData)
+			hardLinks[header.Name] = entry
 			dataBytesTransferred += int64(len(fileData))
 		case tar.TypeSymlink:
 			entry := addSymlinkOrBlobReference(
 				manifest, fileName, header.Linkname, index, &missing)
-			dataBytesRecycled += entry.GetOriginalSize()
+			hardLinks[header.Name] = entry
+			switch {
+			case entry == nil:
+				// unresolved blob reference
+			case entry.GetType() != Type_Symlink:
+				dataBytesRecycled += entry.GetOriginalSize() // resolved blob reference
+			default:
+				dataBytesTransferred += int64(len(header.Linkname)) // actual symlink
+			}
+		case tar.TypeLink:
+			if entry, found := hardLinks[header.Linkname]; found {
+				manifest.Contents[fileName] = entry
+			} else {
+				AddProblem(manifest, fileName, "tar: invalid hardlink %q", header.Linkname)
+			}
 		case tar.TypeDir:
 			AddDirectory(manifest, fileName)
 		default:
@@ -202,7 +192,7 @@ func ExtractZip(ctx context.Context, reader io.Reader, oldManifest *Manifest) (*
 	var dataBytesRecycled int64
 	var dataBytesTransferred int64
 
-	index := indexManifestByGitHash(oldManifest)
+	index := IndexManifestByGitHash(oldManifest)
 	missing := []string{}
 	manifest := NewManifest()
 	for _, file := range archive.File {
@@ -224,7 +214,14 @@ func ExtractZip(ctx context.Context, reader io.Reader, oldManifest *Manifest) (*
 			if file.Mode()&os.ModeSymlink != 0 {
 				entry := addSymlinkOrBlobReference(
 					manifest, normalizedName, string(fileData), index, &missing)
-				dataBytesRecycled += entry.GetOriginalSize()
+				switch {
+				case entry == nil:
+					// unresolved blob reference
+				case entry.GetType() != Type_Symlink:
+					dataBytesRecycled += entry.GetOriginalSize() // resolved blob reference
+				default:
+					dataBytesTransferred += int64(len(fileData)) // actual symlink
+				}
 			} else {
 				AddFile(manifest, normalizedName, fileData)
 				dataBytesTransferred += int64(len(fileData))

src/fetch.go

@@ -9,12 +9,14 @@ import (
 	"net/url"
 	"os"
 	"slices"
+	"strings"
 
 	"github.com/c2h5oh/datasize"
 	"github.com/go-git/go-billy/v6/osfs"
 	"github.com/go-git/go-git/v6"
 	"github.com/go-git/go-git/v6/plumbing"
 	"github.com/go-git/go-git/v6/plumbing/cache"
+	"github.com/go-git/go-git/v6/plumbing/client"
 	"github.com/go-git/go-git/v6/plumbing/filemode"
 	"github.com/go-git/go-git/v6/plumbing/object"
 	"github.com/go-git/go-git/v6/plumbing/protocol/packp"
@@ -164,28 +166,18 @@ func FetchRepository(
 	// Third, if we still don't have data for some manifest entries, re-establish a git transport
 	// and request the missing blobs (only) from the server.
 	if len(blobsNeeded) > 0 {
-		client, err := transport.Get(parsedRepoURL.Scheme)
+		gitClient := client.New()
-		if err != nil {
+		request := &transport.Request{
-			return nil, fmt.Errorf("git transport: %w", err)
+			URL:     parsedRepoURL,
-		}
+			Command: transport.UploadPackService}
 
-		endpoint, err := transport.NewEndpoint(repoURL)
+		session, err := gitClient.Handshake(ctx, request)
-		if err != nil {
-			return nil, fmt.Errorf("git endpoint: %w", err)
-		}
-
-		session, err := client.NewSession(storer, endpoint, nil)
-		if err != nil {
-			return nil, fmt.Errorf("git session: %w", err)
-		}
-
-		connection, err := session.Handshake(ctx, transport.UploadPackService)
 		if err != nil {
 			return nil, fmt.Errorf("git connection: %w", err)
 		}
-		defer connection.Close()
+		defer session.Close()
 
-		if err := connection.Fetch(ctx, &transport.FetchRequest{
+		if err := session.Fetch(ctx, storer, &transport.FetchRequest{
 			Wants: slices.Collect(maps.Keys(blobsNeeded)),
 			Depth: 1,
 			// Git CLI behaves like this, even if the wants above are references to blobs.
@@ -209,6 +201,8 @@ func FetchRepository(
 		datasize.ByteSize(dataBytesTransferred).HR(),
 	)
 
+	warnAboutGitLFS(ctx, manifest)
+
 	return manifest, nil
 }
 
@@ -254,3 +248,18 @@ func readGitBlob(
 
 	return nil
 }
+
+func warnAboutGitLFS(ctx context.Context, manifest *Manifest) {
+	gitattributes := ReadGitAttributes(ctx, manifest)
+	for _, name := range slices.Sorted(maps.Keys(manifest.GetContents())) {
+		entry := manifest.GetContents()[name]
+		if !IsEntryRegularFile(entry) {
+			continue
+		}
+		parts := strings.Split(name, "/")
+		attrs, _ := gitattributes.Match(parts, nil)
+		if attr, ok := attrs["filter"]; ok && attr.Value() == "lfs" {
+			AddProblem(manifest, name, "git-pages does not support Git LFS; move this file into Git or use incremental uploads")
+		}
+	}
+}

src/garbage.go

@@ -5,30 +5,29 @@ import (
 	"fmt"
 
 	"github.com/c2h5oh/datasize"
-	"github.com/dghubble/trie"
 )
 
-func trieReduce(data trie.Trier) (items, total int64) {
-	data.Walk(func(key string, value any) error {
-		items += 1
-		total += *value.(*int64)
-		return nil
-	})
-	return
-}
-
 func TraceGarbage(ctx context.Context) error {
-	allBlobs := trie.NewRuneTrie()
+	allBlobs := map[string]int64{}
-	liveBlobs := trie.NewRuneTrie()
+	liveBlobs := map[string]int64{}
 
-	traceManifest := func(manifestName string, manifest *Manifest) error {
+	reduceBlobs := func(data map[string]int64) (items, total int64) {
+		for _, value := range data {
+			items += 1
+			total += value
+		}
+		return
+	}
+
+	traceManifest := func(manifestKind string, manifestName string, manifest *Manifest) error {
 		for _, entry := range manifest.GetContents() {
 			if entry.GetType() == Type_ExternalFile {
 				blobName := string(entry.Data)
-				if size := allBlobs.Get(blobName); size == nil {
+				if size, ok := allBlobs[blobName]; ok {
-					return fmt.Errorf("%s: dangling reference %s", manifestName, blobName)
+					liveBlobs[blobName] = size
 				} else {
-					liveBlobs.Put(blobName, size)
+					logc.Printf(ctx, "trace manifest: %s/%s: dangling reference %s",
+						manifestKind, manifestName, blobName)
 				}
 			}
 		}
@@ -36,47 +35,44 @@ func TraceGarbage(ctx context.Context) error {
 	}
 
 	// Enumerate all blobs.
+	logc.Printf(ctx, "trace: enumerating blobs")
 	for metadata, err := range backend.EnumerateBlobs(ctx) {
 		if err != nil {
 			return fmt.Errorf("trace blobs err: %w", err)
 		}
-		allBlobs.Put(metadata.Name, &metadata.Size)
+		allBlobs[metadata.Name] = metadata.Size
 	}
 
 	// Enumerate blobs live via site manifests.
-	for metadata, err := range backend.EnumerateManifests(ctx) {
+	logc.Printf(ctx, "trace: enumerating manifests")
+	for item, err := range backend.GetAllManifests(ctx) {
+		metadata, manifest := item.Splat()
 		if err != nil {
 			return fmt.Errorf("trace sites err: %w", err)
 		}
-		manifest, _, err := backend.GetManifest(ctx, metadata.Name, GetManifestOptions{})
+		err = traceManifest("site", metadata.Name, manifest)
-		if err != nil {
-			return fmt.Errorf("trace sites err: %w", err)
-		}
-		err = traceManifest(metadata.Name, manifest)
 		if err != nil {
 			return fmt.Errorf("trace sites err: %w", err)
 		}
 	}
 
 	// Enumerate blobs live via audit records.
-	for auditID, err := range backend.SearchAuditLog(ctx, SearchAuditLogOptions{}) {
+	logc.Printf(ctx, "trace: enumerating audit records")
+	auditIDs := backend.SearchAuditLog(ctx, SearchAuditLogOptions{})
+	for record, err := range backend.GetAuditLogRecords(ctx, auditIDs) {
 		if err != nil {
 			return fmt.Errorf("trace audit err: %w", err)
 		}
-		auditRecord, err := backend.QueryAuditLog(ctx, auditID)
+		if record.Manifest != nil {
-		if err != nil {
+			err = traceManifest("audit", record.GetAuditID().String(), record.Manifest)
-			return fmt.Errorf("trace audit err: %w", err)
-		}
-		if auditRecord.Manifest != nil {
-			err = traceManifest(auditID.String(), auditRecord.Manifest)
 			if err != nil {
 				return fmt.Errorf("trace audit err: %w", err)
 			}
 		}
 	}
 
-	allBlobsCount, allBlobsSize := trieReduce(allBlobs)
+	allBlobsCount, allBlobsSize := reduceBlobs(allBlobs)
-	liveBlobsCount, liveBlobsSize := trieReduce(liveBlobs)
+	liveBlobsCount, liveBlobsSize := reduceBlobs(liveBlobs)
 	logc.Printf(ctx, "trace all: %d blobs, %s",
 		allBlobsCount, datasize.ByteSize(allBlobsSize).HR())
 	logc.Printf(ctx, "trace live: %d blobs, %s",

src/gitattributes.go

@@ -0,0 +1,61 @@
+package git_pages
+
+import (
+	"bytes"
+	"cmp"
+	"context"
+	"slices"
+	"strings"
+
+	"github.com/go-git/go-git/v6/plumbing/format/gitattributes"
+)
+
+func ReadGitAttributes(ctx context.Context, manifest *Manifest) gitattributes.Matcher {
+	type entryPair struct {
+		parts []string
+		entry *Entry
+	}
+
+	// Collect all .gitattributes files.
+	var files []entryPair
+	for name, entry := range manifest.GetContents() {
+		switch entry.GetType() {
+		case Type_InlineFile, Type_ExternalFile:
+			parts := strings.Split(name, "/")
+			if parts[len(parts)-1] == ".gitattributes" {
+				files = append(files, entryPair{parts, entry})
+			}
+		}
+	}
+
+	// Sort the file list by depth, then by name.
+	slices.SortFunc(files, func(a entryPair, b entryPair) int {
+		return cmp.Or(
+			cmp.Compare(len(a.parts), len(b.parts)),
+			slices.Compare(a.parts, b.parts),
+		)
+	})
+
+	// Gather all .gitattributes rules, sorted by depth.
+	var rules []gitattributes.MatchAttribute
+	for _, pair := range files {
+		parts, entry := pair.parts, pair.entry
+		data, err := GetEntryContents(ctx, entry)
+		if err != nil {
+			continue
+		}
+		dirs := parts[:len(parts)-1]
+		isRoot := len(parts) == 1
+		newRules, err := gitattributes.ReadAttributes(bytes.NewReader(data), dirs, isRoot)
+		if err != nil {
+			AddProblem(manifest, strings.Join(parts, "/"), "parsing .gitattributes: %v", err)
+			continue
+		}
+		rules = append(rules, newRules...)
+	}
+
+	// gitattributes.Matcher applies rules in reverse.
+	slices.Reverse(rules)
+	matcher := gitattributes.NewMatcher(rules)
+	return matcher
+}

src/headers.go

@@ -1,6 +1,7 @@
 package git_pages
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"net/http"
@@ -14,6 +15,7 @@ import (
 )
 
 var ErrHeaderNotAllowed = errors.New("custom header not allowed")
+var ErrBasicAuthNotAllowed = errors.New("basic authorization not allowed")
 
 const HeadersFileName string = "_headers"
 
@@ -74,28 +76,40 @@ func validateHeaderRule(rule headers.Rule) error {
 		if slices.Contains(unsafeHeaders, header) {
 			return fmt.Errorf("rule sets header %q (fundamentally unsafe)", header)
 		}
-		if !slices.Contains(config.Limits.AllowedCustomHeaders, header) {
+		switch header {
-			return fmt.Errorf("rule sets header %q (not allowlisted)", header)
+		case "Basic-Auth":
-		}
+			if !config.Limits.AllowBasicAuth {
-		if !IsAllowedCustomHeader(header) { // make sure we don't desync
+				return fmt.Errorf("rule sets header %q (forbidden by policy)", header)
-			panic(errors.New("header check inconsistency"))
+			}
+		default:
+			if !slices.Contains(config.Limits.AllowedCustomHeaders, header) {
+				return fmt.Errorf("rule sets header %q (not allowlisted)", header)
+			}
+			if !IsAllowedCustomHeader(header) { // make sure we don't desync
+				panic(errors.New("header check inconsistency"))
+			}
 		}
 	}
 	return nil
 }
 
 // Parses redirects file and injects rules into the manifest.
-func ProcessHeadersFile(manifest *Manifest) error {
+func ProcessHeadersFile(ctx context.Context, manifest *Manifest) error {
 	headersEntry := manifest.Contents[HeadersFileName]
 	delete(manifest.Contents, HeadersFileName)
 	if headersEntry == nil {
 		return nil
-	} else if headersEntry.GetType() != Type_InlineFile {
-		return AddProblem(manifest, HeadersFileName,
-			"not a regular file")
 	}
 
-	rules, err := headers.ParseString(string(headersEntry.GetData()))
+	data, err := GetEntryContents(ctx, headersEntry)
+	if errors.Is(err, ErrNotRegularFile) {
+		return AddProblem(manifest, HeadersFileName,
+			"not a regular file")
+	} else if err != nil {
+		return err
+	}
+
+	rules, err := headers.ParseString(string(data))
 	if err != nil {
 		return AddProblem(manifest, HeadersFileName,
 			"syntax error: %s", err)
@@ -108,16 +122,52 @@ func ProcessHeadersFile(manifest *Manifest) error {
 			continue
 		}
 		headerMap := []*Header{}
+		credentials := []*BasicCredential{}
+		hasBasicAuth := false
 		for header, values := range rule.Headers {
-			headerMap = append(headerMap, &Header{
+			switch header {
-				Name:   proto.String(header),
+			case "Basic-Auth":
-				Values: values,
+				hasBasicAuth = true
-			})
+				for _, value := range values {
+					for _, usernamePassword := range strings.Split(value, " ") {
+						if usernamePassword == "" {
+							continue
+						}
+						if username, password, found := strings.Cut(usernamePassword, ":"); !found {
+							AddProblem(manifest, HeadersFileName,
+								"rule #%d %q: malformed Basic-Auth credential", index+1, rule.Path)
+							continue
+						} else {
+							credentials = append(credentials, &BasicCredential{
+								Username: proto.String(username),
+								Password: proto.String(password),
+							})
+						}
+					}
+				}
+			default:
+				headerMap = append(headerMap, &Header{
+					Name:   proto.String(header),
+					Values: values,
+				})
+			}
 		}
+		// Note that we may add an empty `headerMap` here even if only credentials are defined.
+		// This is intentional: in `_headers` files processing terminates at the first matching
+		// clause, and Netlify mixes Basic-Auth with all the other headers.
 		manifest.Headers = append(manifest.Headers, &HeaderRule{
 			Path:      proto.String(rule.Path),
 			HeaderMap: headerMap,
 		})
+		// We're using `hasBasicAuth` instead of `len(credentials) > 0` so that if a `_headers`
+		// file defines only malformed credentials, we still add a rule (that in effect always
+		// denies access).
+		if hasBasicAuth {
+			manifest.BasicAuth = append(manifest.BasicAuth, &BasicAuthRule{
+				Path:        proto.String(rule.Path),
+				Credentials: credentials,
+			})
+		}
 	}
 	return nil
 }
@@ -137,13 +187,14 @@ func CollectHeadersFile(manifest *Manifest) string {
 	return headers.Must(headers.UnparseString(headersRules))
 }
 
-func ApplyHeaderRules(manifest *Manifest, url *url.URL) (headers http.Header, err error) {
+func matchPathRules[
-	headers = http.Header{}
+	Rule interface{ GetPath() string },
+](rules []Rule, url *url.URL) (matched Rule) {
 	fromSegments := pathSegments(url.Path)
 next:
-	for _, rule := range manifest.Headers {
+	for _, rule := range rules {
 		// check if the rule matches url
-		ruleURL, _ := url.Parse(*rule.Path) // pre-validated in `validateHeaderRule`
+		ruleURL, _ := url.Parse(rule.GetPath()) // pre-validated in `validateHeaderRule`
 		ruleSegments := pathSegments(ruleURL.Path)
 		if ruleSegments[len(ruleSegments)-1] != "*" {
 			if len(ruleSegments) < len(fromSegments) {
@@ -161,8 +212,19 @@ next:
 				continue next
 			}
 		}
+		matched = rule
+		break
+	}
+	return
+}
+
+func ApplyHeaderRules(manifest *Manifest, url *url.URL) (
+	headers http.Header, err error,
+) {
+	headers = http.Header{}
+	if rule := matchPathRules(manifest.Headers, url); rule != nil {
 		// the rule has matched url, validate headers against up-to-date policy
-		for _, header := range rule.HeaderMap {
+		for _, header := range rule.GetHeaderMap() {
 			name := header.GetName()
 			if !IsAllowedCustomHeader(name) {
 				return nil, fmt.Errorf("%w: %s", ErrHeaderNotAllowed, name)
@@ -171,7 +233,30 @@ next:
 				headers.Add(name, value)
 			}
 		}
-		break
 	}
 	return
 }
+
+func ApplyBasicAuthRules(manifest *Manifest, url *url.URL, r *http.Request) (bool, error) {
+	if rule := matchPathRules(manifest.BasicAuth, url); rule == nil {
+		// no matches, authorized by default
+		return true, nil
+	} else {
+		// the rule has matched url, check that basic auth is allowed per up-to-date policy
+		if !config.Limits.AllowBasicAuth {
+			// basic auth configured in the past but not allowed any more
+			return false, ErrBasicAuthNotAllowed
+		}
+		if username, password, ok := r.BasicAuth(); ok {
+			// request has credentials, check them
+			for _, credential := range rule.GetCredentials() {
+				if credential.GetUsername() == username && credential.GetPassword() == password {
+					// authorized!
+					return true, nil
+				}
+			}
+		}
+		// request has no credentials, unauthorized
+		return false, nil
+	}
+}

src/histogram.go

@@ -0,0 +1,35 @@
+package git_pages
+
+import (
+	"context"
+	"fmt"
+	"maps"
+	"slices"
+	"strings"
+)
+
+type DomainStatistics struct {
+	Domain         string
+	OriginalSize   int64
+	CompressedSize int64
+	StoredSize     int64
+}
+
+func SizeHistogram(ctx context.Context) ([]*DomainStatistics, error) {
+	statisticsMap := map[string]*DomainStatistics{}
+	for item, err := range backend.GetAllManifests(ctx) {
+		metadata, manifest := item.Splat()
+		if err != nil {
+			return nil, fmt.Errorf("size histogram err: %w", err)
+		}
+		domain, _, _ := strings.Cut(metadata.Name, "/")
+		if _, found := statisticsMap[domain]; !found {
+			statisticsMap[domain] = &DomainStatistics{Domain: domain}
+		}
+		statistics := statisticsMap[domain]
+		statistics.OriginalSize += metadata.Size + manifest.GetOriginalSize()
+		statistics.CompressedSize += metadata.Size + manifest.GetCompressedSize()
+		statistics.StoredSize += metadata.Size + manifest.GetStoredSize()
+	}
+	return slices.Collect(maps.Values(statisticsMap)), nil
+}

src/main.go

@@ -1,6 +1,7 @@
 package git_pages
 
 import (
+	"cmp"
 	"context"
 	"crypto/tls"
 	"errors"
@@ -16,6 +17,8 @@ import (
 	"os"
 	"path"
 	"runtime/debug"
+	"slices"
+	"strconv"
 	"strings"
 	"time"
 
@@ -31,6 +34,7 @@ var config *Config
 var wildcards []*WildcardPattern
 var fallback http.Handler
 var backend Backend
+var domainCache DomainCache
 
 func configureFeatures(ctx context.Context) (err error) {
 	if len(config.Features) > 0 {
@@ -61,6 +65,12 @@ func configureMemLimit(ctx context.Context) (err error) {
 	return
 }
 
+// Can only be safely called during initial configuration.
+func configureConcurrency(_ context.Context) (err error) {
+	putBlobSemaphore = make(chan struct{}, config.Limits.ConcurrentUploads)
+	return
+}
+
 func configureWildcards(_ context.Context) (err error) {
 	newWildcards, err := TranslateWildcards(config.Wildcard)
 	if err != nil {
@@ -92,7 +102,7 @@ func configureFallback(_ context.Context) (err error) {
 
 // Thread-unsafe, must be called only during initial configuration.
 func configureAudit(_ context.Context) (err error) {
-	snowflake.SetStartTime(time.Date(2025, 12, 1, 0, 0, 0, 0, time.UTC))
+	snowflake.SetStartTime(AuditSnowflakeStartTime)
 	snowflake.SetMachineID(config.Audit.NodeID)
 	return
 }
@@ -119,6 +129,9 @@ func panicHandler(handler http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		defer func() {
 			if err := recover(); err != nil {
+				if err, ok := err.(error); ok && errors.Is(err, http.ErrAbortHandler) {
+					panic(http.ErrAbortHandler)
+				}
 				logc.Printf(r.Context(), "panic: %s %s %s: %s\n%s",
 					r.Method, r.Host, r.URL.Path, err, string(debug.Stack()))
 				http.Error(w,
@@ -172,26 +185,32 @@ func usage() {
 	fmt.Fprintf(os.Stderr, "(server) "+
 		"git-pages [-config <file>|-no-config]\n")
 	fmt.Fprintf(os.Stderr, "(info)   "+
-		"git-pages {-print-config-env-vars|-print-config}\n")
+		"git-pages {-version|-print-config-env-vars|-print-config}\n")
 	fmt.Fprintf(os.Stderr, "(debug)  "+
 		"git-pages {-list-blobs|-list-manifests}\n")
 	fmt.Fprintf(os.Stderr, "(debug)  "+
 		"git-pages {-get-blob|-get-manifest|-get-archive|-update-site} <ref> [file]\n")
 	fmt.Fprintf(os.Stderr, "(admin)  "+
-		"git-pages {-freeze-domain <domain>|-unfreeze-domain <domain>}\n")
+		"git-pages {-freeze-domain|-unfreeze-domain} <domain>\n")
 	fmt.Fprintf(os.Stderr, "(audit)  "+
-		"git-pages {-audit-log|-audit-read <id>|-audit-server <endpoint> <program> [args...]}\n")
+		"git-pages {-audit-log|-audit-read <id>|-audit-rollback <id>}\n")
+	fmt.Fprintf(os.Stderr, "(audit)  "+
+		"git-pages {-audit-expire <days>|-audit-detach <domain>/<project>}\n")
+	fmt.Fprintf(os.Stderr, "(audit)  "+
+		"git-pages  -audit-server <endpoint> <program> [args...]\n")
 	fmt.Fprintf(os.Stderr, "(maint)  "+
-		"git-pages {-run-migration <name>|-trace-garbage}\n")
+		"git-pages {-run-migration <name>|-trace-garbage|-size-histogram {original|stored}}\n")
 	flag.PrintDefaults()
 }
 
-func Main() {
+func Main(versionInfo string) {
 	ctx := context.Background()
 
 	flag.Usage = usage
 	configTomlPath := flag.String("config", "",
 		"load configuration from `filename` (default: 'config.toml')")
+	secretTomlPath := flag.String("secrets", "",
+		"load additional configuration values from `filename` (default: '$CREDENTIALS_DIRECTORY/secrets.toml' if it exists)")
 	noConfig := flag.Bool("no-config", false,
 		"run without configuration file (configure via environment variables)")
 	printConfigEnvVars := flag.Bool("print-config-env-vars", false,
@@ -220,14 +239,27 @@ func Main() {
 		"extract contents of audit record `id` to files '<id>-*'")
 	auditRollback := flag.String("audit-rollback", "",
 		"restore site from contents of audit record `id`")
+	auditExpire := flag.String("audit-expire", "",
+		"expire audit records older than `days` old")
+	auditDetach := flag.String("audit-detach", "",
+		"detach all blobs of audit records for a single `site` (or the entire domain with 'domain.tld/*')")
 	auditServer := flag.String("audit-server", "",
 		"listen for notifications on `endpoint` and spawn a process for each audit event")
 	runMigration := flag.String("run-migration", "",
 		"run a store `migration` (one of: create-domain-markers)")
+	sizeHistogram := flag.String("size-histogram", "",
+		"display histogram of `size-type` (original or stored) per domain")
 	traceGarbage := flag.Bool("trace-garbage", false,
 		"estimate total size of unreachable blobs")
+	version := flag.Bool("version", false,
+		"display version")
 	flag.Parse()
 
+	if *version {
+		fmt.Printf("git-pages %s\n", versionInfo)
+		os.Exit(0)
+	}
+
 	var cliOperations int
 	for _, selected := range []bool{
 		*listBlobs,
@@ -241,8 +273,11 @@ func Main() {
 		*auditLog,
 		*auditRead != "",
 		*auditRollback != "",
+		*auditExpire != "",
+		*auditDetach != "",
 		*auditServer != "",
 		*runMigration != "",
+		*sizeHistogram != "",
 		*traceGarbage,
 	} {
 		if selected {
@@ -252,8 +287,8 @@ func Main() {
 	if cliOperations > 1 {
 		logc.Fatalln(ctx, "-list-blobs, -list-manifests, -get-blob, -get-manifest, -get-archive, "+
 			"-update-site, -freeze-domain, -unfreeze-domain, -audit-log, -audit-read, "+
-			"-audit-rollback, -audit-server, -run-migration, and -trace-garbage are "+
+			"-audit-rollback, -audit-expire, -audit-detach, -audit-server, -run-migration, "+
-			"mutually exclusive")
+			"-size-histogram, and -trace-garbage are mutually exclusive")
 	}
 
 	if *configTomlPath != "" && *noConfig {
@@ -269,7 +304,19 @@ func Main() {
 	if *configTomlPath == "" && !*noConfig {
 		*configTomlPath = "config.toml"
 	}
-	if config, err = Configure(*configTomlPath); err != nil {
+
+	if *secretTomlPath == "" && !*noConfig {
+		// check for a second config file at $CREDENTIALS_DIRECTORY/secrets.toml, and use it
+		if systemdCredentialsDir := os.Getenv("CREDENTIALS_DIRECTORY"); systemdCredentialsDir != "" {
+			secretTomlTestPath := path.Join(systemdCredentialsDir, "secrets.toml")
+			_, err := os.Stat(secretTomlTestPath)
+			if !errors.Is(err, os.ErrNotExist) {
+				*secretTomlPath = secretTomlTestPath
+			}
+		}
+	}
+
+	if config, err = Configure(*configTomlPath, *secretTomlPath); err != nil {
 		logc.Fatalln(ctx, "config:", err)
 	}
 
@@ -284,6 +331,7 @@ func Main() {
 	if err = errors.Join(
 		configureFeatures(ctx),
 		configureMemLimit(ctx),
+		configureConcurrency(ctx),
 		configureWildcards(ctx),
 		configureFallback(ctx),
 		configureAudit(ctx),
@@ -296,6 +344,10 @@ func Main() {
 		if backend, err = CreateBackend(ctx, &config.Storage); err != nil {
 			logc.Fatalln(ctx, err)
 		}
+
+		if domainCache, err = CreateDomainCache(ctx); err != nil {
+			logc.Fatalln(ctx, err)
+		}
 	}
 
 	switch {
@@ -385,7 +437,7 @@ func Main() {
 			}
 
 			webRoot := webRootArg(*updateSite)
-			result = UpdateFromArchive(ctx, webRoot, contentType, file)
+			result = UpdateFromArchive(ctx, webRoot, "", contentType, file)
 		} else {
 			branch := "pages"
 			if sourceURL.Fragment != "" {
@@ -440,37 +492,33 @@ func Main() {
 		}
 
 	case *auditLog:
-		ch := make(chan *AuditRecord)
+		records := []*AuditRecord{}
-		ids := []AuditID{}
+		ids := backend.SearchAuditLog(ctx, SearchAuditLogOptions{})
-		for id, err := range backend.SearchAuditLog(ctx, SearchAuditLogOptions{}) {
+		for record, err := range backend.GetAuditLogRecords(ctx, ids) {
 			if err != nil {
 				logc.Fatalln(ctx, err)
 			}
-			go func() {
+			records = append(records, record)
-				if record, err := backend.QueryAuditLog(ctx, id); err != nil {
-					logc.Fatalln(ctx, err)
-				} else {
-					ch <- record
-				}
-			}()
-			ids = append(ids, id)
 		}
 
-		records := map[AuditID]*AuditRecord{}
+		slices.SortFunc(records, func(a, b *AuditRecord) int {
-		for len(records) < len(ids) {
+			return cmp.Compare(a.GetAuditID(), b.GetAuditID())
-			record := <-ch
+		})
-			records[record.GetAuditID()] = record
-		}
 
-		for _, id := range ids {
+		for _, record := range records {
-			record := records[id]
+			parts := []string{
-			fmt.Fprintf(color.Output, "%s %s %s %s %s\n",
 				record.GetAuditID().String(),
 				color.HiWhiteString(record.GetTimestamp().AsTime().UTC().Format(time.RFC3339)),
 				color.HiMagentaString(record.DescribePrincipal()),
 				color.HiGreenString(record.DescribeResource()),
-				record.GetEvent(),
+				fmt.Sprint(record.GetEvent()),
-			)
+			}
+			if record.IsDetached() {
+				parts = append(parts,
+					color.HiYellowString("(detached)"),
+				)
+			}
+			fmt.Fprintln(color.Output, strings.Join(parts, " "))
 		}
 
 	case *auditRead != "":
@@ -516,6 +564,45 @@ func Main() {
 			logc.Fatalln(ctx, err)
 		}
 
+	case *auditDetach != "":
+		domain, project, found := strings.Cut(*auditDetach, "/")
+		if !found || domain == "" || project == "" {
+			logc.Fatalln(ctx, "argument to -audit-detach must be in the form of "+
+				"'domain.tld/project' or 'domain.tld/*'")
+		}
+
+		if project != "*" && project != ".index" {
+			if err := ValidateProjectName(project); err != nil {
+				logc.Fatalf(ctx, "audit detach: project name: %v\n", err)
+			}
+		}
+
+		count := 0
+		ids := backend.SearchAuditLog(ctx, SearchAuditLogOptions{})
+		for record, err := range backend.GetAuditLogRecords(ctx, ids) {
+			if err != nil {
+				logc.Fatalln(ctx, err)
+			}
+			if record.GetDomain() == domain && (project == "*" || record.GetProject() == project) {
+				if !record.IsDetachable() {
+					continue
+				} else if !record.IsDetached() {
+					logc.Printf(ctx, "detaching audit record %s\n", record.GetAuditID())
+					err = backend.DetachAuditRecord(ctx, record.GetAuditID())
+					if err != nil {
+						logc.Fatalln(ctx, err)
+					}
+					count++
+				} else {
+					logc.Printf(ctx, "audit record %s already detached\n", record.GetAuditID())
+				}
+			}
+		}
+
+		if count == 0 {
+			logc.Printf(ctx, "no detachable audit records found for %s/%s", domain, project)
+		}
+
 	case *auditServer != "":
 		if flag.NArg() < 1 {
 			logc.Fatalln(ctx, "handler path not provided")
@@ -528,11 +615,81 @@ func Main() {
 
 		serve(ctx, listen(ctx, "audit", *auditServer), ObserveHTTPHandler(processor))
 
+	case *auditExpire != "":
+		days, err := strconv.ParseInt(*auditExpire, 10, 0)
+		if err != nil {
+			logc.Fatalln(ctx, err)
+		}
+
+		ids := backend.SearchAuditLog(ctx, SearchAuditLogOptions{
+			Until: time.Now().AddDate(0, 0, int(-days)),
+		})
+
+		count := 0
+		for id, err := range ids {
+			if err != nil {
+				logc.Fatalln(ctx, err)
+				continue
+			}
+
+			err = backend.ExpireAuditRecord(ctx, id)
+			if err != nil {
+				logc.Fatalln(ctx, err)
+			} else {
+				logc.Printf(ctx, "audit: expired record %s\n", id)
+				count += 1
+			}
+		}
+
+		logc.Printf(ctx, "audit: expired %d records\n", count)
+
 	case *runMigration != "":
 		if err = RunMigration(ctx, *runMigration); err != nil {
 			logc.Fatalln(ctx, err)
 		}
 
+	case *sizeHistogram != "":
+		extractSize := func(s *DomainStatistics) int64 { return 0 }
+		switch *sizeHistogram {
+		case "original":
+			// Displays a size histogram using the `manifest.OriginalSize`, which is useful to see
+			// which site is the closest to hitting the size limit (checked against apparent size).
+			// This apparent size does not have any direct relationship with used storage.
+			extractSize = func(s *DomainStatistics) int64 { return s.OriginalSize }
+		case "stored":
+			// Displays a size histogram using the `manifest.StoredSize`, which is useful to see
+			// which site consumes the most resources. The site is keeping at least this many
+			// bytes worth of blobs alive, but removing it may not free any space because
+			// deduplication is global.
+			extractSize = func(s *DomainStatistics) int64 { return s.StoredSize }
+		default:
+			logc.Fatalln(ctx, "unknown histogram type")
+		}
+
+		histogram, err := SizeHistogram(ctx)
+		if err != nil {
+			logc.Fatalln(ctx, err)
+		}
+		slices.SortFunc(histogram, func(a *DomainStatistics, b *DomainStatistics) int {
+			return cmp.Compare(extractSize(a), extractSize(b))
+		})
+
+		if len(histogram) > 0 {
+			fullScaleSize := max(extractSize(histogram[len(histogram)-1]), 1)
+			fullScaleWidth := int64(40)
+			for _, statistics := range histogram {
+				size := extractSize(statistics)
+				barWidth := size * fullScaleWidth / fullScaleSize
+				spaceWidth := fullScaleWidth - barWidth
+				bar := strings.Repeat("*", int(barWidth)) + strings.Repeat(" ", int(spaceWidth))
+				fmt.Fprintf(color.Output, "%s %s %s\n",
+					color.HiBlackString(fmt.Sprint("|", bar, "|")),
+					statistics.Domain,
+					color.HiGreenString(datasize.ByteSize(extractSize(statistics)).HR()),
+				)
+			}
+		}
+
 	case *traceGarbage:
 		if err = TraceGarbage(ctx); err != nil {
 			logc.Fatalln(ctx, err)
@@ -547,7 +704,7 @@ func Main() {
 		// Note that not all of the configuration is updated on reload. Listeners are kept as-is.
 		// The backend is not recreated (this is intentional as it allows preserving the cache).
 		OnReload(func() {
-			if newConfig, err := Configure(*configTomlPath); err != nil {
+			if newConfig, err := Configure(*configTomlPath, *secretTomlPath); err != nil {
 				logc.Println(ctx, "config: reload err:", err)
 			} else {
 				// From https://go.dev/ref/mem:
@@ -585,6 +742,10 @@ func Main() {
 		}
 		backend = NewObservedBackend(backend)
 
+		if domainCache, err = CreateDomainCache(ctx); err != nil {
+			logc.Fatalln(ctx, err)
+		}
+
 		middleware := chainHTTPMiddleware(
 			panicHandler,
 			remoteAddrMiddleware,

src/manifest.go

@@ -8,6 +8,7 @@ import (
 	"crypto/sha256"
 	"errors"
 	"fmt"
+	"io"
 	"mime"
 	"net/http"
 	"path"
@@ -144,6 +145,63 @@ func AddProblem(manifest *Manifest, pathName, format string, args ...any) error
 	return fmt.Errorf("%s: %s", pathName, cause)
 }
 
+// Returns a map of git hash to entry. If `manifest` is nil, returns an empty map.
+func IndexManifestByGitHash(manifest *Manifest) map[string]*Entry {
+	index := map[string]*Entry{}
+	for _, entry := range manifest.GetContents() {
+		if hash := entry.GetGitHash(); hash != "" {
+			if _, ok := plumbing.FromHex(hash); ok {
+				index[hash] = entry
+			} else {
+				panic(fmt.Errorf("index: malformed hash: %s", hash))
+			}
+		}
+	}
+	return index
+}
+
+func ManifestHasBasicAuth(manifest *Manifest) bool {
+	return len(manifest.GetBasicAuth()) > 0
+}
+
+func IsEntryRegularFile(entry *Entry) bool {
+	return entry.GetType() == Type_InlineFile ||
+		entry.GetType() == Type_ExternalFile
+}
+
+var ErrNotRegularFile = errors.New("not a regular file")
+
+func GetEntryContents(ctx context.Context, entry *Entry) (data []byte, err error) {
+	switch entry.GetType() {
+	case Type_InlineFile:
+		data = entry.GetData()
+	case Type_ExternalFile:
+		reader, _, err := backend.GetBlob(ctx, string(entry.GetData()))
+		if err != nil {
+			return nil, err
+		}
+		data, err = io.ReadAll(reader)
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, ErrNotRegularFile
+	}
+
+	switch entry.GetTransform() {
+	case Transform_Identity:
+	case Transform_Zstd:
+		data, err = zstdDecoder.DecodeAll(data, []byte{})
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, fmt.Errorf("unexpected transform")
+	}
+
+	return
+}
+
 // EnsureLeadingDirectories adds directory entries for any parent directories
 // that are implicitly referenced by files in the manifest but don't have
 // explicit directory entries. (This can be the case if an archive is created
@@ -275,7 +333,7 @@ func CompressFiles(ctx context.Context, manifest *Manifest) {
 // (Perhaps in the future they could be exposed at `.git-pages/status.txt`?)
 func PrepareManifest(ctx context.Context, manifest *Manifest) error {
 	// Parse Netlify-style `_redirects`.
-	if err := ProcessRedirectsFile(manifest); err != nil {
+	if err := ProcessRedirectsFile(ctx, manifest); err != nil {
 		logc.Printf(ctx, "redirects err: %s\n", err)
 	} else if len(manifest.Redirects) > 0 {
 		logc.Printf(ctx, "redirects ok: %d rules\n", len(manifest.Redirects))
@@ -285,7 +343,7 @@ func PrepareManifest(ctx context.Context, manifest *Manifest) error {
 	LintRedirects(manifest)
 
 	// Parse Netlify-style `_headers`.
-	if err := ProcessHeadersFile(manifest); err != nil {
+	if err := ProcessHeadersFile(ctx, manifest); err != nil {
 		logc.Printf(ctx, "headers err: %s\n", err)
 	} else if len(manifest.Headers) > 0 {
 		logc.Printf(ctx, "headers ok: %d rules\n", len(manifest.Headers))
@@ -303,6 +361,12 @@ func PrepareManifest(ctx context.Context, manifest *Manifest) error {
 var ErrSiteTooLarge = errors.New("site too large")
 var ErrManifestTooLarge = errors.New("manifest too large")
 
+// Limits the number of concurrent uploads, globally across the entire git-pages process.
+// As created, there is no limit, but reinitializing the semaphore with a bounded channel
+// limits the concurrency to the channel size. Note that the default *configuration* does
+// limit the number of uploads.
+var putBlobSemaphore = make(chan struct{})
+
 // Uploads inline file data over certain size to the storage backend. Returns a copy of
 // the manifest updated to refer to an external content-addressable store.
 func StoreManifest(
@@ -311,19 +375,11 @@ func StoreManifest(
 	span, ctx := ObserveFunction(ctx, "StoreManifest", "manifest.name", name)
 	defer span.Finish()
 
+	extManifest := &Manifest{}
+	proto.Merge(extManifest, manifest)
+
 	// Replace inline files over certain size with references to external data.
-	extManifest := Manifest{
+	extManifest.Contents = make(map[string]*Entry)
-		RepoUrl:        manifest.RepoUrl,
-		Branch:         manifest.Branch,
-		Commit:         manifest.Commit,
-		Contents:       make(map[string]*Entry),
-		Redirects:      manifest.Redirects,
-		Headers:        manifest.Headers,
-		Problems:       manifest.Problems,
-		OriginalSize:   manifest.OriginalSize,
-		CompressedSize: manifest.CompressedSize,
-		StoredSize:     proto.Int64(0),
-	}
 	for name, entry := range manifest.Contents {
 		cannotBeInlined := entry.GetType() == Type_InlineFile &&
 			entry.GetCompressedSize() > int64(config.Limits.MaxInlineFileSize.Bytes())
@@ -359,12 +415,13 @@ func StoreManifest(
 			config.Limits.MaxSiteSize.HR(),
 		)
 	}
+	extManifest.StoredSize = proto.Int64(0)
 	for _, blobSize := range blobSizes {
 		*extManifest.StoredSize += blobSize
 	}
 
 	// Upload the resulting manifest and the blob it references.
-	extManifestData := EncodeManifest(&extManifest)
+	extManifestData := EncodeManifest(extManifest)
 	if uint64(len(extManifestData)) > config.Limits.MaxManifestSize.Bytes() {
 		return nil, fmt.Errorf("%w: manifest size %s exceeds %s limit",
 			ErrManifestTooLarge,
@@ -373,7 +430,7 @@ func StoreManifest(
 		)
 	}
 
-	if err := backend.StageManifest(ctx, &extManifest); err != nil {
+	if err := backend.StageManifest(ctx, extManifest); err != nil {
 		return nil, fmt.Errorf("stage manifest: %w", err)
 	}
 
@@ -384,7 +441,9 @@ func StoreManifest(
 		// If the entry in the original manifest is already an external reference, there's no need
 		// to externalize it (and no way for us to do so, since the entry only contains the blob name).
 		if entry.GetType() == Type_ExternalFile && manifest.Contents[name].GetType() == Type_InlineFile {
+			putBlobSemaphore <- struct{}{} // acquire (and maybe block)
 			wg.Go(func() {
+				defer func() { <-putBlobSemaphore }() // release
 				err := backend.PutBlob(ctx, string(entry.Data), manifest.Contents[name].Data)
 				if err != nil {
 					ch <- fmt.Errorf("put blob %s: %w", name, err)
@@ -398,7 +457,7 @@ func StoreManifest(
 		return nil, err // currently ignores all but 1st error
 	}
 
-	if err := backend.CommitManifest(ctx, name, &extManifest, opts); err != nil {
+	if err := backend.CommitManifest(ctx, name, extManifest, opts); err != nil {
 		if errors.Is(err, ErrDomainFrozen) {
 			return nil, err
 		} else {
@@ -406,5 +465,5 @@ func StoreManifest(
 		}
 	}
 
-	return &extManifest, nil
+	return extManifest, nil
 }

src/observe.go

@@ -8,11 +8,9 @@ import (
 	"iter"
 	"log"
 	"log/slog"
-	"math/rand/v2"
 	"net/http"
 	"os"
 	"runtime/debug"
-	"strconv"
 	"sync"
 	"time"
 
@@ -22,10 +20,6 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
-
-	"github.com/getsentry/sentry-go"
-	sentryhttp "github.com/getsentry/sentry-go/http"
-	sentryslog "github.com/getsentry/sentry-go/slog"
 )
 
 var (
@@ -47,71 +41,9 @@ var (
 
 var syslogHandler syslog.Handler
 
-func hasSentry() bool {
-	return os.Getenv("SENTRY_DSN") != ""
-}
-
-func chainSentryMiddleware(
-	middleware ...func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event,
-) func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event {
-	return func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event {
-		for idx := 0; idx < len(middleware) && event != nil; idx++ {
-			event = middleware[idx](event, hint)
-		}
-		return event
-	}
-}
-
-// sensitiveHTTPHeaders extends the list of sensitive headers defined in the Sentry Go SDK with our
-// own application-specific header field names.
-var sensitiveHTTPHeaders = map[string]struct{}{
-	"Forge-Authorization": {},
-}
-
-// scrubSentryEvent removes sensitive HTTP header fields from the Sentry event.
-func scrubSentryEvent(event *sentry.Event, hint *sentry.EventHint) *sentry.Event {
-	if event.Request != nil && event.Request.Headers != nil {
-		for key := range event.Request.Headers {
-			if _, ok := sensitiveHTTPHeaders[key]; ok {
-				delete(event.Request.Headers, key)
-			}
-		}
-	}
-	return event
-}
-
-// sampleSentryEvent returns a function that discards a Sentry event according to the sample rate,
-// unless the associated HTTP request triggers a mutation or it took too long to produce a response,
-// in which case the event is never discarded.
-func sampleSentryEvent(sampleRate float64) func(*sentry.Event, *sentry.EventHint) *sentry.Event {
-	return func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event {
-		newSampleRate := sampleRate
-		if event.Request != nil {
-			switch event.Request.Method {
-			case "PUT", "POST", "DELETE":
-				newSampleRate = 1
-			}
-		}
-		duration := event.Timestamp.Sub(event.StartTime)
-		threshold := time.Duration(config.Observability.SlowResponseThreshold)
-		if duration >= threshold {
-			newSampleRate = 1
-		}
-		if rand.Float64() < newSampleRate {
-			return event
-		}
-		return nil
-	}
-}
-
 func InitObservability() {
 	debug.SetPanicOnFault(true)
 
-	environment := "development"
-	if value, ok := os.LookupEnv("ENVIRONMENT"); ok {
-		environment = value
-	}
-
 	logHandlers := []slog.Handler{}
 
 	switch config.LogFormat {
@@ -140,45 +72,6 @@ func InitObservability() {
 		logHandlers = append(logHandlers, syslogHandler)
 	}
 
-	if hasSentry() {
-		enableLogs := false
-		if value, err := strconv.ParseBool(os.Getenv("SENTRY_LOGS")); err == nil {
-			enableLogs = value
-		}
-
-		enableTracing := false
-		if value, err := strconv.ParseBool(os.Getenv("SENTRY_TRACING")); err == nil {
-			enableTracing = value
-		}
-
-		tracesSampleRate := 1.00
-		switch environment {
-		case "development", "staging":
-		default:
-			tracesSampleRate = 0.05
-		}
-
-		options := sentry.ClientOptions{}
-		options.Environment = environment
-		options.EnableLogs = enableLogs
-		options.EnableTracing = enableTracing
-		options.TracesSampleRate = 1 // use our own custom sampling logic
-		options.BeforeSend = scrubSentryEvent
-		options.BeforeSendTransaction = chainSentryMiddleware(
-			sampleSentryEvent(tracesSampleRate),
-			scrubSentryEvent,
-		)
-		if err := sentry.Init(options); err != nil {
-			log.Fatalf("sentry: %s\n", err)
-		}
-
-		if enableLogs {
-			logHandlers = append(logHandlers, sentryslog.Option{
-				AddSource: true,
-			}.NewSentryHandler(context.Background()))
-		}
-	}
-
 	slog.SetDefault(slog.New(slogmulti.Fanout(logHandlers...)))
 }
 
@@ -188,9 +81,6 @@ func FiniObservability() {
 	if syslogHandler != nil {
 		wg.Go(func() { syslogHandler.Flush(timeout) })
 	}
-	if hasSentry() {
-		wg.Go(func() { sentry.Flush(timeout) })
-	}
 	wg.Wait()
 }
 
@@ -200,10 +90,6 @@ func ObserveError(err error) {
 		// Timeout results in a different error.
 		return
 	}
-
-	if hasSentry() {
-		sentry.CaptureException(err)
-	}
 }
 
 type observedResponseWriter struct {
@@ -236,22 +122,6 @@ func (w *observedResponseWriter) WriteHeader(statusCode int) {
 }
 
 func ObserveHTTPHandler(handler http.Handler) http.Handler {
-	if hasSentry() {
-		handler = func(next http.Handler) http.Handler {
-			next = sentryhttp.New(sentryhttp.Options{
-				Repanic: true,
-			}).Handle(handler)
-
-			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				// Prevent the Sentry SDK from continuing traces as we don't use this feature.
-				r.Header.Del(sentry.SentryTraceHeader)
-				r.Header.Del(sentry.SentryBaggageHeader)
-
-				next.ServeHTTP(w, r)
-			})
-		}(handler)
-	}
-
 	handler = func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			ow := newObservedResponseWriter(w)
@@ -282,23 +152,12 @@ func ObserveFunction(
 	interface{ Finish() }, context.Context,
 ) {
 	switch {
-	case hasSentry():
-		span := sentry.StartSpan(ctx, "function")
-		span.Description = funcName
-		ObserveData(span.Context(), data...)
-		return span, span.Context()
 	default:
 		return noopSpan{}, ctx
 	}
 }
 
 func ObserveData(ctx context.Context, data ...any) {
-	if span := sentry.SpanFromContext(ctx); span != nil {
-		for i := 0; i < len(data); i += 2 {
-			name, value := data[i], data[i+1]
-			span.SetData(name.(string), value)
-		}
-	}
 }
 
 var (
@@ -435,8 +294,8 @@ func (backend *observedBackend) DeleteManifest(ctx context.Context, name string,
 	return
 }
 
-func (backend *observedBackend) EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error] {
+func (backend *observedBackend) EnumerateManifests(ctx context.Context) iter.Seq2[*ManifestMetadata, error] {
-	return func(yield func(ManifestMetadata, error) bool) {
+	return func(yield func(*ManifestMetadata, error) bool) {
 		span, ctx := ObserveFunction(ctx, "EnumerateManifests")
 		for metadata, err := range backend.inner.EnumerateManifests(ctx) {
 			if !yield(metadata, err) {
@@ -447,6 +306,18 @@ func (backend *observedBackend) EnumerateManifests(ctx context.Context) iter.Seq
 	}
 }
 
+func (backend *observedBackend) GetAllManifests(ctx context.Context) iter.Seq2[tuple[*ManifestMetadata, *Manifest], error] {
+	return func(yield func(tuple[*ManifestMetadata, *Manifest], error) bool) {
+		span, ctx := ObserveFunction(ctx, "GetAllManifests")
+		for item, err := range backend.inner.GetAllManifests(ctx) {
+			if !yield(item, err) {
+				break
+			}
+		}
+		span.Finish()
+	}
+}
+
 func (backend *observedBackend) CheckDomain(ctx context.Context, domain string) (found bool, err error) {
 	span, ctx := ObserveFunction(ctx, "CheckDomain", "domain.name", domain)
 	found, err = backend.inner.CheckDomain(ctx, domain)
@@ -475,6 +346,13 @@ func (backend *observedBackend) UnfreezeDomain(ctx context.Context, domain strin
 	return
 }
 
+func (backend *observedBackend) HaveDomainsChanged(ctx context.Context, since time.Time) (changed bool, err error) {
+	span, ctx := ObserveFunction(ctx, "HaveDomainsChanged", "since", since)
+	changed, err = backend.inner.HaveDomainsChanged(ctx, since)
+	span.Finish()
+	return
+}
+
 func (backend *observedBackend) AppendAuditLog(ctx context.Context, id AuditID, record *AuditRecord) (err error) {
 	span, ctx := ObserveFunction(ctx, "AppendAuditLog", "audit.id", id)
 	err = backend.inner.AppendAuditLog(ctx, id, record)
@@ -505,3 +383,31 @@ func (backend *observedBackend) SearchAuditLog(
 		span.Finish()
 	}
 }
+
+func (backend *observedBackend) GetAuditLogRecords(
+	ctx context.Context, ids iter.Seq2[AuditID, error],
+) iter.Seq2[*AuditRecord, error] {
+	return func(yield func(*AuditRecord, error) bool) {
+		span, ctx := ObserveFunction(ctx, "GetAuditLogRecords")
+		for item, err := range backend.inner.GetAuditLogRecords(ctx, ids) {
+			if !yield(item, err) {
+				break
+			}
+		}
+		span.Finish()
+	}
+}
+
+func (backend *observedBackend) DetachAuditRecord(ctx context.Context, id AuditID) (err error) {
+	span, ctx := ObserveFunction(ctx, "DetachAuditRecord", "audit.id", id)
+	err = backend.inner.DetachAuditRecord(ctx, id)
+	span.Finish()
+	return
+}
+
+func (backend *observedBackend) ExpireAuditRecord(ctx context.Context, id AuditID) (err error) {
+	span, ctx := ObserveFunction(ctx, "ExpireAuditRecord", "audit.id", id)
+	err = backend.inner.ExpireAuditRecord(ctx, id)
+	span.Finish()
+	return
+}

src/pages.go

@@ -65,8 +65,23 @@ func observeSiteUpdate(via string, result *UpdateResult) {
 	}
 }
 
+func copyForgeAuthToPrincipal(principal *Principal, auth *Authorization) {
+	if auth.forgeUser != nil {
+		principal.ForgeUser = auth.forgeUser
+	}
+
+	repoURL := auth.ForgeRepoURL()
+	if repoURL != "" {
+		principal.RepoUrl = &repoURL
+	}
+}
+
+func normalizeHost(host string) string {
+	return strings.ToLower(host)
+}
+
 func makeWebRoot(host string, projectName string) string {
-	return path.Join(strings.ToLower(host), projectName)
+	return path.Join(normalizeHost(host), projectName)
 }
 
 func getWebRoot(r *http.Request) (string, error) {
@@ -115,6 +130,13 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 		return err
 	}
 
+	host = normalizeHost(host)
+	if !domainCache.CheckDomain(r.Context(), host) {
+		w.WriteHeader(http.StatusNotFound)
+		fmt.Fprintf(w, "site not found\n")
+		return nil
+	}
+
 	type indexManifestResult struct {
 		manifest *Manifest
 		metadata ManifestMetadata
@@ -132,7 +154,7 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 	err = nil
 	sitePath = strings.TrimPrefix(r.URL.Path, "/")
 	if projectName, projectPath, hasProjectSlash := strings.Cut(sitePath, "/"); projectName != "" {
-		if IsValidProjectName(projectName) {
+		if ValidateProjectName(projectName) == nil {
 			var projectManifest *Manifest
 			var projectMetadata ManifestMetadata
 			projectManifest, projectMetadata, err = backend.GetManifest(
@@ -193,8 +215,8 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 
 		case metadataPath == "manifest.json":
 			// metadata requests require authorization to avoid making pushes from private
-			// repositories enumerable
+			// repositories enumerable or exposing basic-auth protected sections
-			_, err := AuthorizeMetadataRetrieval(r)
+			_, err := AuthorizeMetadataRetrieval(r, ManifestHasBasicAuth(manifest))
 			if err != nil {
 				return err
 			}
@@ -208,7 +230,7 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 
 		case metadataPath == "archive.tar":
 			// same as above
-			_, err := AuthorizeMetadataRetrieval(r)
+			_, err := AuthorizeMetadataRetrieval(r, ManifestHasBasicAuth(manifest))
 			if err != nil {
 				return err
 			}
@@ -244,6 +266,19 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 		}
 	}
 
+	// Apply basic-auth rules before checking existence of a path to avoid leaking the latter.
+	authorized, err := ApplyBasicAuthRules(manifest, &url.URL{Path: sitePath}, r)
+	if err != nil {
+		// See comment below for the error case under `ApplyHeaderRules`.
+		w.WriteHeader(http.StatusInternalServerError)
+		fmt.Fprintf(w, "%s\n", err)
+		return err
+	} else if !authorized {
+		w.Header().Set("WWW-Authenticate", `Basic charset="UTF-8"`)
+		w.WriteHeader(http.StatusUnauthorized)
+		return nil
+	}
+
 	entryPath := sitePath
 	entry := (*Entry)(nil)
 	appliedRedirect := false
@@ -499,19 +534,23 @@ func putPage(w http.ResponseWriter, r *http.Request) error {
 		result = UpdateFromRepository(ctx, webRoot, repoURL, branch)
 
 	default:
-		if auth, err := AuthorizeUpdateFromArchive(r); err != nil {
+		auth, err := AuthorizeUpdateFromArchive(r)
+		if err != nil {
 			return err
-		} else if auth.forgeUser != nil {
-			GetPrincipal(r.Context()).ForgeUser = auth.forgeUser
 		}
 
+		principal := GetPrincipal(r.Context())
+		copyForgeAuthToPrincipal(principal, auth)
+
+		repoURL := auth.ForgeRepoURL()
+
 		if checkDryRun(w, r) {
 			return nil
 		}
 
 		// request body contains archive
 		reader := http.MaxBytesReader(w, r.Body, int64(config.Limits.MaxSiteSize.Bytes()))
-		result = UpdateFromArchive(ctx, webRoot, contentType, reader)
+		result = UpdateFromArchive(ctx, webRoot, repoURL, contentType, reader)
 	}
 
 	return reportUpdateResult(w, r, result)
@@ -532,12 +571,14 @@ func patchPage(w http.ResponseWriter, r *http.Request) error {
 		return err
 	}
 
-	if auth, err := AuthorizeUpdateFromArchive(r); err != nil {
+	auth, err := AuthorizeUpdateFromArchive(r)
+	if err != nil {
 		return err
-	} else if auth.forgeUser != nil {
-		GetPrincipal(r.Context()).ForgeUser = auth.forgeUser
 	}
 
+	principal := GetPrincipal(r.Context())
+	copyForgeAuthToPrincipal(principal, auth)
+
 	if checkDryRun(w, r) {
 		return nil
 	}
@@ -664,12 +705,14 @@ func deletePage(w http.ResponseWriter, r *http.Request) error {
 		return err
 	}
 
-	if auth, err := AuthorizeDeletion(r); err != nil {
+	auth, err := AuthorizeDeletion(r)
+	if err != nil {
 		return err
-	} else if auth.forgeUser != nil {
-		GetPrincipal(r.Context()).ForgeUser = auth.forgeUser
 	}
 
+	principal := GetPrincipal(r.Context())
+	copyForgeAuthToPrincipal(principal, auth)
+
 	if checkDryRun(w, r) {
 		return nil
 	}
@@ -822,31 +865,23 @@ func ServePages(w http.ResponseWriter, r *http.Request) {
 	if config.Audit.IncludeIPs != "" {
 		GetPrincipal(r.Context()).IpAddress = proto.String(r.RemoteAddr)
 	}
-	// We want upstream health checks to be done as closely to the normal flow as possible;
+	switch r.Method {
-	// any intentional deviation is an opportunity to miss an issue that will affect our
+	case "PUT", "PATCH", "POST":
-	// visitors but not our health checks.
+		mediaType := r.Header.Get("Content-Type")
-	if r.Header.Get("Health-Check") == "" {
-		var mediaType string
-		switch r.Method {
-		case "HEAD", "GET":
-			mediaType = r.Header.Get("Accept")
-		default:
-			mediaType = r.Header.Get("Content-Type")
-		}
 		logc.Println(r.Context(), "pages:", r.Method, r.Host, r.URL, mediaType)
-		if region := os.Getenv("FLY_REGION"); region != "" {
+	default:
-			machine_id := os.Getenv("FLY_MACHINE_ID")
+		logc.Println(r.Context(), "pages:", r.Method, r.Host, r.URL)
-			w.Header().Add("Server", fmt.Sprintf("git-pages (fly.io; %s; %s)", region, machine_id))
+	}
-			ObserveData(r.Context(), "server.name", machine_id, "server.region", region)
+	if hostname, err := os.Hostname(); err == nil {
-		} else if hostname, err := os.Hostname(); err == nil {
+		if region := os.Getenv("PAGES_REGION"); region != "" {
-			if region := os.Getenv("PAGES_REGION"); region != "" {
+			w.Header().Add("Server", fmt.Sprintf("git-pages (%s; %s)", region, hostname))
-				w.Header().Add("Server", fmt.Sprintf("git-pages (%s; %s)", region, hostname))
+			ObserveData(r.Context(), "server.name", hostname, "server.region", region)
-				ObserveData(r.Context(), "server.name", hostname, "server.region", region)
+		} else {
-			} else {
+			w.Header().Add("Server", fmt.Sprintf("git-pages (%s)", hostname))
-				w.Header().Add("Server", fmt.Sprintf("git-pages (%s)", hostname))
+			ObserveData(r.Context(), "server.name", hostname)
-				ObserveData(r.Context(), "server.name", hostname)
-			}
 		}
+	} else {
+		w.Header().Add("Server", "git-pages")
 	}
 	allowedMethods := []string{"OPTIONS", "HEAD", "GET", "PUT", "PATCH", "DELETE", "POST"}
 	if r.Method == "OPTIONS" || !slices.Contains(allowedMethods, r.Method) {

src/patch.go

@@ -30,8 +30,12 @@ func ApplyTarPatch(manifest *Manifest, reader io.Reader, parents CreateParentsMo
 		children map[string]*Node
 	}
 
+	// Index the manifest for incremental update operations.
+	index := IndexManifestByGitHash(manifest)
+	missing := []string{}
+
 	// Extract the manifest contents (which is using a flat hash map) into a directory tree
-	// so that recursive delete operations have O(1) complexity. s
+	// so that recursive delete operations have O(1) complexity.
 	var root *Node
 	sortedNames := slices.Sorted(maps.Keys(manifest.GetContents()))
 	for _, name := range sortedNames {
@@ -107,8 +111,16 @@ func ApplyTarPatch(manifest *Manifest, reader io.Reader, parents CreateParentsMo
 				entry: NewManifestEntry(Type_InlineFile, fileData),
 			}
 		case tar.TypeSymlink:
-			node.children[fileName] = &Node{
+			if hash, found := strings.CutPrefix(header.Linkname, BlobReferencePrefix); found {
-				entry: NewManifestEntry(Type_Symlink, []byte(header.Linkname)),
+				if entry, found := index[hash]; found {
+					node.children[fileName] = &Node{entry: entry}
+				} else {
+					missing = append(missing, hash)
+				}
+			} else {
+				node.children[fileName] = &Node{
+					entry: NewManifestEntry(Type_Symlink, []byte(header.Linkname)),
+				}
 			}
 		case tar.TypeDir:
 			node.children[fileName] = &Node{
@@ -129,6 +141,10 @@ func ApplyTarPatch(manifest *Manifest, reader io.Reader, parents CreateParentsMo
 		}
 	}
 
+	if len(missing) > 0 {
+		return UnresolvedRefError{missing}
+	}
+
 	// Repopulate manifest contents with the updated directory tree.
 	var traverse func([]string, *Node)
 	traverse = func(segments []string, node *Node) {

src/redirects.go

@@ -1,6 +1,8 @@
 package git_pages
 
 import (
+	"context"
+	"errors"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -96,17 +98,22 @@ func validateRedirectRule(rule *redirects.Rule) error {
 }
 
 // Parses redirects file and injects rules into the manifest.
-func ProcessRedirectsFile(manifest *Manifest) error {
+func ProcessRedirectsFile(ctx context.Context, manifest *Manifest) error {
 	redirectsEntry := manifest.Contents[RedirectsFileName]
 	delete(manifest.Contents, RedirectsFileName)
 	if redirectsEntry == nil {
 		return nil
-	} else if redirectsEntry.GetType() != Type_InlineFile {
-		return AddProblem(manifest, RedirectsFileName,
-			"not a regular file")
 	}
 
-	rules, err := redirects.ParseString(string(redirectsEntry.GetData()))
+	data, err := GetEntryContents(ctx, redirectsEntry)
+	if errors.Is(err, ErrNotRegularFile) {
+		return AddProblem(manifest, RedirectsFileName,
+			"not a regular file")
+	} else if err != nil {
+		return err
+	}
+
+	rules, err := redirects.ParseString(string(data))
 	if err != nil {
 		return AddProblem(manifest, RedirectsFileName,
 			"syntax error: %s", err)

src/schema.pb.go

@@ -479,6 +479,110 @@ func (x *HeaderRule) GetHeaderMap() []*Header {
 	return nil
 }
 
+type BasicCredential struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Username      *string                `protobuf:"bytes,1,opt,name=username" json:"username,omitempty"`
+	Password      *string                `protobuf:"bytes,2,opt,name=password" json:"password,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *BasicCredential) Reset() {
+	*x = BasicCredential{}
+	mi := &file_schema_proto_msgTypes[4]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *BasicCredential) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*BasicCredential) ProtoMessage() {}
+
+func (x *BasicCredential) ProtoReflect() protoreflect.Message {
+	mi := &file_schema_proto_msgTypes[4]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use BasicCredential.ProtoReflect.Descriptor instead.
+func (*BasicCredential) Descriptor() ([]byte, []int) {
+	return file_schema_proto_rawDescGZIP(), []int{4}
+}
+
+func (x *BasicCredential) GetUsername() string {
+	if x != nil && x.Username != nil {
+		return *x.Username
+	}
+	return ""
+}
+
+func (x *BasicCredential) GetPassword() string {
+	if x != nil && x.Password != nil {
+		return *x.Password
+	}
+	return ""
+}
+
+type BasicAuthRule struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Path          *string                `protobuf:"bytes,1,opt,name=path" json:"path,omitempty"`
+	Credentials   []*BasicCredential     `protobuf:"bytes,2,rep,name=credentials" json:"credentials,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *BasicAuthRule) Reset() {
+	*x = BasicAuthRule{}
+	mi := &file_schema_proto_msgTypes[5]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *BasicAuthRule) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*BasicAuthRule) ProtoMessage() {}
+
+func (x *BasicAuthRule) ProtoReflect() protoreflect.Message {
+	mi := &file_schema_proto_msgTypes[5]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use BasicAuthRule.ProtoReflect.Descriptor instead.
+func (*BasicAuthRule) Descriptor() ([]byte, []int) {
+	return file_schema_proto_rawDescGZIP(), []int{5}
+}
+
+func (x *BasicAuthRule) GetPath() string {
+	if x != nil && x.Path != nil {
+		return *x.Path
+	}
+	return ""
+}
+
+func (x *BasicAuthRule) GetCredentials() []*BasicCredential {
+	if x != nil {
+		return x.Credentials
+	}
+	return nil
+}
+
 type Problem struct {
 	state         protoimpl.MessageState `protogen:"open.v1"`
 	Path          *string                `protobuf:"bytes,1,opt,name=path" json:"path,omitempty"`
@@ -489,7 +593,7 @@ type Problem struct {
 
 func (x *Problem) Reset() {
 	*x = Problem{}
-	mi := &file_schema_proto_msgTypes[4]
+	mi := &file_schema_proto_msgTypes[6]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -501,7 +605,7 @@ func (x *Problem) String() string {
 func (*Problem) ProtoMessage() {}
 
 func (x *Problem) ProtoReflect() protoreflect.Message {
-	mi := &file_schema_proto_msgTypes[4]
+	mi := &file_schema_proto_msgTypes[6]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -514,7 +618,7 @@ func (x *Problem) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use Problem.ProtoReflect.Descriptor instead.
 func (*Problem) Descriptor() ([]byte, []int) {
-	return file_schema_proto_rawDescGZIP(), []int{4}
+	return file_schema_proto_rawDescGZIP(), []int{6}
 }
 
 func (x *Problem) GetPath() string {
@@ -543,8 +647,9 @@ type Manifest struct {
 	CompressedSize *int64            `protobuf:"varint,5,opt,name=compressed_size,json=compressedSize" json:"compressed_size,omitempty"` // sum of each `entry.compressed_size`
 	StoredSize     *int64            `protobuf:"varint,8,opt,name=stored_size,json=storedSize" json:"stored_size,omitempty"`             // sum of deduplicated `entry.compressed_size` for external files only
 	// Netlify-style `_redirects` and `_headers` rules.
-	Redirects []*RedirectRule `protobuf:"bytes,6,rep,name=redirects" json:"redirects,omitempty"`
+	Redirects []*RedirectRule  `protobuf:"bytes,6,rep,name=redirects" json:"redirects,omitempty"`
-	Headers   []*HeaderRule   `protobuf:"bytes,9,rep,name=headers" json:"headers,omitempty"`
+	Headers   []*HeaderRule    `protobuf:"bytes,9,rep,name=headers" json:"headers,omitempty"`
+	BasicAuth []*BasicAuthRule `protobuf:"bytes,11,rep,name=basic_auth,json=basicAuth" json:"basic_auth,omitempty"`
 	// Diagnostics for non-fatal errors.
 	Problems      []*Problem `protobuf:"bytes,7,rep,name=problems" json:"problems,omitempty"`
 	unknownFields protoimpl.UnknownFields
@@ -553,7 +658,7 @@ type Manifest struct {
 
 func (x *Manifest) Reset() {
 	*x = Manifest{}
-	mi := &file_schema_proto_msgTypes[5]
+	mi := &file_schema_proto_msgTypes[7]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -565,7 +670,7 @@ func (x *Manifest) String() string {
 func (*Manifest) ProtoMessage() {}
 
 func (x *Manifest) ProtoReflect() protoreflect.Message {
-	mi := &file_schema_proto_msgTypes[5]
+	mi := &file_schema_proto_msgTypes[7]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -578,7 +683,7 @@ func (x *Manifest) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use Manifest.ProtoReflect.Descriptor instead.
 func (*Manifest) Descriptor() ([]byte, []int) {
-	return file_schema_proto_rawDescGZIP(), []int{5}
+	return file_schema_proto_rawDescGZIP(), []int{7}
 }
 
 func (x *Manifest) GetRepoUrl() string {
@@ -644,6 +749,13 @@ func (x *Manifest) GetHeaders() []*HeaderRule {
 	return nil
 }
 
+func (x *Manifest) GetBasicAuth() []*BasicAuthRule {
+	if x != nil {
+		return x.BasicAuth
+	}
+	return nil
+}
+
 func (x *Manifest) GetProblems() []*Problem {
 	if x != nil {
 		return x.Problems
@@ -669,7 +781,7 @@ type AuditRecord struct {
 
 func (x *AuditRecord) Reset() {
 	*x = AuditRecord{}
-	mi := &file_schema_proto_msgTypes[6]
+	mi := &file_schema_proto_msgTypes[8]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -681,7 +793,7 @@ func (x *AuditRecord) String() string {
 func (*AuditRecord) ProtoMessage() {}
 
 func (x *AuditRecord) ProtoReflect() protoreflect.Message {
-	mi := &file_schema_proto_msgTypes[6]
+	mi := &file_schema_proto_msgTypes[8]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -694,7 +806,7 @@ func (x *AuditRecord) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use AuditRecord.ProtoReflect.Descriptor instead.
 func (*AuditRecord) Descriptor() ([]byte, []int) {
-	return file_schema_proto_rawDescGZIP(), []int{6}
+	return file_schema_proto_rawDescGZIP(), []int{8}
 }
 
 func (x *AuditRecord) GetId() int64 {
@@ -751,13 +863,14 @@ type Principal struct {
 	IpAddress     *string                `protobuf:"bytes,1,opt,name=ip_address,json=ipAddress" json:"ip_address,omitempty"`
 	CliAdmin      *bool                  `protobuf:"varint,2,opt,name=cli_admin,json=cliAdmin" json:"cli_admin,omitempty"`
 	ForgeUser     *ForgeUser             `protobuf:"bytes,3,opt,name=forge_user,json=forgeUser" json:"forge_user,omitempty"`
+	RepoUrl       *string                `protobuf:"bytes,4,opt,name=repo_url,json=repoUrl" json:"repo_url,omitempty"`
 	unknownFields protoimpl.UnknownFields
 	sizeCache     protoimpl.SizeCache
 }
 
 func (x *Principal) Reset() {
 	*x = Principal{}
-	mi := &file_schema_proto_msgTypes[7]
+	mi := &file_schema_proto_msgTypes[9]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -769,7 +882,7 @@ func (x *Principal) String() string {
 func (*Principal) ProtoMessage() {}
 
 func (x *Principal) ProtoReflect() protoreflect.Message {
-	mi := &file_schema_proto_msgTypes[7]
+	mi := &file_schema_proto_msgTypes[9]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -782,7 +895,7 @@ func (x *Principal) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use Principal.ProtoReflect.Descriptor instead.
 func (*Principal) Descriptor() ([]byte, []int) {
-	return file_schema_proto_rawDescGZIP(), []int{7}
+	return file_schema_proto_rawDescGZIP(), []int{9}
 }
 
 func (x *Principal) GetIpAddress() string {
@@ -806,6 +919,13 @@ func (x *Principal) GetForgeUser() *ForgeUser {
 	return nil
 }
 
+func (x *Principal) GetRepoUrl() string {
+	if x != nil && x.RepoUrl != nil {
+		return *x.RepoUrl
+	}
+	return ""
+}
+
 type ForgeUser struct {
 	state         protoimpl.MessageState `protogen:"open.v1"`
 	Origin        *string                `protobuf:"bytes,1,opt,name=origin" json:"origin,omitempty"`
@@ -817,7 +937,7 @@ type ForgeUser struct {
 
 func (x *ForgeUser) Reset() {
 	*x = ForgeUser{}
-	mi := &file_schema_proto_msgTypes[8]
+	mi := &file_schema_proto_msgTypes[10]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -829,7 +949,7 @@ func (x *ForgeUser) String() string {
 func (*ForgeUser) ProtoMessage() {}
 
 func (x *ForgeUser) ProtoReflect() protoreflect.Message {
-	mi := &file_schema_proto_msgTypes[8]
+	mi := &file_schema_proto_msgTypes[10]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -842,7 +962,7 @@ func (x *ForgeUser) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use ForgeUser.ProtoReflect.Descriptor instead.
 func (*ForgeUser) Descriptor() ([]byte, []int) {
-	return file_schema_proto_rawDescGZIP(), []int{8}
+	return file_schema_proto_rawDescGZIP(), []int{10}
 }
 
 func (x *ForgeUser) GetOrigin() string {
@@ -892,10 +1012,16 @@ const file_schema_proto_rawDesc = "" +
 	"HeaderRule\x12\x12\n" +
 	"\x04path\x18\x01 \x01(\tR\x04path\x12&\n" +
 	"\n" +
-	"header_map\x18\x02 \x03(\v2\a.HeaderR\theaderMap\"3\n" +
+	"header_map\x18\x02 \x03(\v2\a.HeaderR\theaderMap\"I\n" +
+	"\x0fBasicCredential\x12\x1a\n" +
+	"\busername\x18\x01 \x01(\tR\busername\x12\x1a\n" +
+	"\bpassword\x18\x02 \x01(\tR\bpassword\"W\n" +
+	"\rBasicAuthRule\x12\x12\n" +
+	"\x04path\x18\x01 \x01(\tR\x04path\x122\n" +
+	"\vcredentials\x18\x02 \x03(\v2\x10.BasicCredentialR\vcredentials\"3\n" +
 	"\aProblem\x12\x12\n" +
 	"\x04path\x18\x01 \x01(\tR\x04path\x12\x14\n" +
-	"\x05cause\x18\x02 \x01(\tR\x05cause\"\xb8\x03\n" +
+	"\x05cause\x18\x02 \x01(\tR\x05cause\"\xe7\x03\n" +
 	"\bManifest\x12\x19\n" +
 	"\brepo_url\x18\x01 \x01(\tR\arepoUrl\x12\x16\n" +
 	"\x06branch\x18\x02 \x01(\tR\x06branch\x12\x16\n" +
@@ -907,7 +1033,9 @@ const file_schema_proto_rawDesc = "" +
 	"\vstored_size\x18\b \x01(\x03R\n" +
 	"storedSize\x12+\n" +
 	"\tredirects\x18\x06 \x03(\v2\r.RedirectRuleR\tredirects\x12%\n" +
-	"\aheaders\x18\t \x03(\v2\v.HeaderRuleR\aheaders\x12$\n" +
+	"\aheaders\x18\t \x03(\v2\v.HeaderRuleR\aheaders\x12-\n" +
+	"\n" +
+	"basic_auth\x18\v \x03(\v2\x0e.BasicAuthRuleR\tbasicAuth\x12$\n" +
 	"\bproblems\x18\a \x03(\v2\b.ProblemR\bproblems\x1aC\n" +
 	"\rContentsEntry\x12\x10\n" +
 	"\x03key\x18\x01 \x01(\tR\x03key\x12\x1c\n" +
@@ -921,14 +1049,15 @@ const file_schema_proto_rawDesc = "" +
 	"\x06domain\x18\n" +
 	" \x01(\tR\x06domain\x12\x18\n" +
 	"\aproject\x18\v \x01(\tR\aproject\x12%\n" +
-	"\bmanifest\x18\f \x01(\v2\t.ManifestR\bmanifest\"r\n" +
+	"\bmanifest\x18\f \x01(\v2\t.ManifestR\bmanifest\"\x8d\x01\n" +
 	"\tPrincipal\x12\x1d\n" +
 	"\n" +
 	"ip_address\x18\x01 \x01(\tR\tipAddress\x12\x1b\n" +
 	"\tcli_admin\x18\x02 \x01(\bR\bcliAdmin\x12)\n" +
 	"\n" +
 	"forge_user\x18\x03 \x01(\v2\n" +
-	".ForgeUserR\tforgeUser\"K\n" +
+	".ForgeUserR\tforgeUser\x12\x19\n" +
+	"\brepo_url\x18\x04 \x01(\tR\arepoUrl\"K\n" +
 	"\tForgeUser\x12\x16\n" +
 	"\x06origin\x18\x01 \x01(\tR\x06origin\x12\x0e\n" +
 	"\x02id\x18\x02 \x01(\x03R\x02id\x12\x16\n" +
@@ -964,7 +1093,7 @@ func file_schema_proto_rawDescGZIP() []byte {
 }
 
 var file_schema_proto_enumTypes = make([]protoimpl.EnumInfo, 3)
-var file_schema_proto_msgTypes = make([]protoimpl.MessageInfo, 10)
+var file_schema_proto_msgTypes = make([]protoimpl.MessageInfo, 12)
 var file_schema_proto_goTypes = []any{
 	(Type)(0),                     // 0: Type
 	(Transform)(0),                // 1: Transform
@@ -973,33 +1102,37 @@ var file_schema_proto_goTypes = []any{
 	(*RedirectRule)(nil),          // 4: RedirectRule
 	(*Header)(nil),                // 5: Header
 	(*HeaderRule)(nil),            // 6: HeaderRule
-	(*Problem)(nil),               // 7: Problem
+	(*BasicCredential)(nil),       // 7: BasicCredential
-	(*Manifest)(nil),              // 8: Manifest
+	(*BasicAuthRule)(nil),         // 8: BasicAuthRule
-	(*AuditRecord)(nil),           // 9: AuditRecord
+	(*Problem)(nil),               // 9: Problem
-	(*Principal)(nil),             // 10: Principal
+	(*Manifest)(nil),              // 10: Manifest
-	(*ForgeUser)(nil),             // 11: ForgeUser
+	(*AuditRecord)(nil),           // 11: AuditRecord
-	nil,                           // 12: Manifest.ContentsEntry
+	(*Principal)(nil),             // 12: Principal
-	(*timestamppb.Timestamp)(nil), // 13: google.protobuf.Timestamp
+	(*ForgeUser)(nil),             // 13: ForgeUser
+	nil,                           // 14: Manifest.ContentsEntry
+	(*timestamppb.Timestamp)(nil), // 15: google.protobuf.Timestamp
 }
 var file_schema_proto_depIdxs = []int32{
 	0,  // 0: Entry.type:type_name -> Type
 	1,  // 1: Entry.transform:type_name -> Transform
 	5,  // 2: HeaderRule.header_map:type_name -> Header
-	12, // 3: Manifest.contents:type_name -> Manifest.ContentsEntry
+	7,  // 3: BasicAuthRule.credentials:type_name -> BasicCredential
-	4,  // 4: Manifest.redirects:type_name -> RedirectRule
+	14, // 4: Manifest.contents:type_name -> Manifest.ContentsEntry
-	6,  // 5: Manifest.headers:type_name -> HeaderRule
+	4,  // 5: Manifest.redirects:type_name -> RedirectRule
-	7,  // 6: Manifest.problems:type_name -> Problem
+	6,  // 6: Manifest.headers:type_name -> HeaderRule
-	13, // 7: AuditRecord.timestamp:type_name -> google.protobuf.Timestamp
+	8,  // 7: Manifest.basic_auth:type_name -> BasicAuthRule
-	2,  // 8: AuditRecord.event:type_name -> AuditEvent
+	9,  // 8: Manifest.problems:type_name -> Problem
-	10, // 9: AuditRecord.principal:type_name -> Principal
+	15, // 9: AuditRecord.timestamp:type_name -> google.protobuf.Timestamp
-	8,  // 10: AuditRecord.manifest:type_name -> Manifest
+	2,  // 10: AuditRecord.event:type_name -> AuditEvent
-	11, // 11: Principal.forge_user:type_name -> ForgeUser
+	12, // 11: AuditRecord.principal:type_name -> Principal
-	3,  // 12: Manifest.ContentsEntry.value:type_name -> Entry
+	10, // 12: AuditRecord.manifest:type_name -> Manifest
-	13, // [13:13] is the sub-list for method output_type
+	13, // 13: Principal.forge_user:type_name -> ForgeUser
-	13, // [13:13] is the sub-list for method input_type
+	3,  // 14: Manifest.ContentsEntry.value:type_name -> Entry
-	13, // [13:13] is the sub-list for extension type_name
+	15, // [15:15] is the sub-list for method output_type
-	13, // [13:13] is the sub-list for extension extendee
+	15, // [15:15] is the sub-list for method input_type
-	0,  // [0:13] is the sub-list for field type_name
+	15, // [15:15] is the sub-list for extension type_name
+	15, // [15:15] is the sub-list for extension extendee
+	0,  // [0:15] is the sub-list for field type_name
 }
 
 func init() { file_schema_proto_init() }
@@ -1013,7 +1146,7 @@ func file_schema_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: unsafe.Slice(unsafe.StringData(file_schema_proto_rawDesc), len(file_schema_proto_rawDesc)),
 			NumEnums:      3,
-			NumMessages:   10,
+			NumMessages:   12,
 			NumExtensions: 0,
 			NumServices:   0,
 		},

src/schema.proto

@@ -76,6 +76,16 @@ message HeaderRule {
 	repeated Header header_map = 2;
 }
 
+message BasicCredential {
+	string username = 1;
+	string password = 2;
+}
+
+message BasicAuthRule {
+	string path = 1;
+	repeated BasicCredential credentials = 2;
+}
+
 message Problem {
 	string path = 1;
 	string cause = 2;
@@ -96,6 +106,7 @@ message Manifest {
 	// Netlify-style `_redirects` and `_headers` rules.
 	repeated RedirectRule redirects = 6;
 	repeated HeaderRule headers = 9;
+	repeated BasicAuthRule basic_auth = 11;
 
 	// Diagnostics for non-fatal errors.
 	repeated Problem problems = 7;
@@ -133,6 +144,7 @@ message Principal {
 	string ip_address = 1;
 	bool cli_admin = 2;
 	ForgeUser forge_user = 3;
+	string repo_url = 4;
 }
 
 message ForgeUser {

src/update.go

@@ -10,6 +10,16 @@ import (
 	"google.golang.org/protobuf/proto"
 )
 
+const BlobReferencePrefix = "/git/blobs/"
+
+type UnresolvedRefError struct {
+	missing []string
+}
+
+func (err UnresolvedRefError) Error() string {
+	return fmt.Sprintf("%d unresolved blob references", len(err.missing))
+}
+
 type UpdateOutcome int
 
 const (
@@ -49,6 +59,7 @@ func Update(
 		if err == nil {
 			domain, _, _ := strings.Cut(webRoot, "/")
 			err = backend.CreateDomain(ctx, domain)
+			domainCache.AddDomain(ctx, domain)
 		}
 		if err == nil {
 			if oldManifest == nil {
@@ -117,6 +128,7 @@ var errArchiveFormat = errors.New("unsupported archive format")
 func UpdateFromArchive(
 	ctx context.Context,
 	webRoot string,
+	repoURL string,
 	contentType string,
 	reader io.Reader,
 ) (result UpdateResult) {
@@ -151,6 +163,10 @@ func UpdateFromArchive(
 		logc.Printf(ctx, "update %s err: %s", webRoot, err)
 		result = UpdateResult{UpdateError, nil, err}
 	} else {
+		if repoURL != "" {
+			newManifest.RepoUrl = &repoURL
+		}
+
 		result = Update(ctx, webRoot, oldManifest, newManifest, ModifyManifestOptions{})
 	}
 

src/util.go

@@ -6,6 +6,15 @@ import (
 	"strings"
 )
 
+type tuple[A, B any] struct {
+	A A
+	B B
+}
+
+func (t tuple[A, B]) Splat() (A, B) {
+	return t.A, t.B
+}
+
 type BoundedReader struct {
 	inner io.Reader
 	fuel  int64