[breaking-change] Reorder -audit-log columns for readability.

In commit bbdaae7280, a domain cache was
introduced to deal with misbehaving crawlers that forge `Host:` header
and may cause thousands of expensive S3 requests to be submitted.
This domain cache is implemented using a Bloom filter (which can
produce false positives but not false negatives) for S3 backend, and
using a function always returning true (which will be a false positive
in most cases) for the FS backend.

Both of these behaviors are unacceptable for the Caddy endpoint, but
the FS backend case much more so. If you use git-pages with Caddy you
should upgrade to a build that includes this commit as soon as possible
or Let's Encrypt may rate-limit or restrict your account when you get
unlucky with a crawler.

.forgejo/workflows/ci.yaml

@@ -10,16 +10,16 @@ env:
 
 jobs:
   check:
-    runs-on: codeberg-small-lazy
+    runs-on: debian-trixie
     container:
-      image: docker.io/library/node:24-trixie-slim@sha256:fcdfd7bcd8f641c8c76a8950343c73912d68ba341e8dd1074e663b784d3e76f4
+      image: docker.io/library/node:24-trixie-slim@sha256:28fd420825d8e922eab0fd91740c7cf88ddbdc8116a2b20a82049f0c946feb03
     steps:
       - name: Check out source code
-        uses: https://code.forgejo.org/actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
+        uses: https://code.forgejo.org/actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - name: Set up toolchain
-        uses: https://code.forgejo.org/actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0
+        uses: https://code.forgejo.org/actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
         with:
-          go-version: '>=1.25.0'
+          go-version: '>=1.25.6'
       - name: Install dependencies
         run: |
           apt-get -y update
@@ -28,29 +28,36 @@ jobs:
       - name: Build service
         run: |
           go build
+      - name: Run tests
+        run: |
+          go test ./...
       - name: Run static analysis
         run: |
-          go vet
-          staticcheck
+          go vet ./...
+          staticcheck ./...
 
   release:
     # IMPORTANT: This workflow step will not work without the Releases unit enabled!
     if: ${{ forge.ref == 'refs/heads/main' || startsWith(forge.event.ref, 'refs/tags/v') }}
     needs: [check]
-    runs-on: codeberg-medium-lazy
+    runs-on: debian-trixie
     container:
-      image: docker.io/library/node:24-trixie-slim@sha256:fcdfd7bcd8f641c8c76a8950343c73912d68ba341e8dd1074e663b784d3e76f4
+      image: docker.io/library/node:24-trixie-slim@sha256:28fd420825d8e922eab0fd91740c7cf88ddbdc8116a2b20a82049f0c946feb03
     steps:
-      - name: Check out source code
-        uses: https://code.forgejo.org/actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
-      - name: Set up toolchain
-        uses: https://code.forgejo.org/actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0
-        with:
-          go-version: '>=1.25.0'
       - name: Install dependencies
         run: |
           apt-get -y update
-          apt-get -y install ca-certificates
+          apt-get -y install ca-certificates git
+          # git needs to be installed for build information embedding to work
+      - name: Check out source code
+        uses: https://code.forgejo.org/actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+      - name: Set up toolchain
+        uses: https://code.forgejo.org/actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
+        with:
+          go-version: '>=1.25.6'
       - name: Build release assets
         # If you want more platforms to be represented, send a pull request.
         run: |
@@ -61,7 +68,7 @@ jobs:
           build linux   arm64
           build darwin  arm64
       - name: Create release
-        uses: https://code.forgejo.org/actions/forgejo-release@fc0488c944626f9265d87fbc4dd6c08f78014c63 # v2.7.3
+        uses: https://code.forgejo.org/actions/forgejo-release@6a9510a9ea01b8b9b435933bf3c0fa45597ad530 # v2.11.3
         with:
           tag: ${{ startsWith(forge.event.ref, 'refs/tags/v') && forge.ref_name || 'latest' }}
           release-dir: assets
@@ -72,16 +79,16 @@ jobs:
   package:
     if: ${{ forge.ref == 'refs/heads/main' || startsWith(forge.event.ref, 'refs/tags/v') }}
     needs: [check]
-    runs-on: codeberg-medium-lazy
+    runs-on: debian-trixie
     container:
-      image: docker.io/library/node:24-trixie-slim@sha256:fcdfd7bcd8f641c8c76a8950343c73912d68ba341e8dd1074e663b784d3e76f4
+      image: docker.io/library/node:24-trixie-slim@sha256:28fd420825d8e922eab0fd91740c7cf88ddbdc8116a2b20a82049f0c946feb03
     steps:
       - name: Install dependencies
         run: |
           apt-get -y update
-          apt-get -y install buildah ca-certificates
+          apt-get -y install ca-certificates buildah qemu-user-binfmt
       - name: Check out source code
-        uses: https://code.forgejo.org/actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
+        uses: https://code.forgejo.org/actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - name: Authenticate with Docker
         run: |
           buildah login --authfile=/tmp/authfile-${FORGE}.json \

.gitignore

@@ -5,3 +5,4 @@
 /config*.toml*
 /git-pages
 /site
+/assets

Dockerfile

@@ -1,9 +1,9 @@
 # Install CA certificates.
-FROM docker.io/library/alpine:latest@sha256:4b7ce07002c69e8f3d704a9c5d6fd3053be500b7f1c69fc0d80990c2ad8dd412 AS ca-certificates-builder
+FROM docker.io/library/alpine:3 AS ca-certificates-builder
 RUN apk --no-cache add ca-certificates
 
 # Build supervisor.
-FROM docker.io/library/golang:1.25-alpine@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS supervisor-builder
+FROM docker.io/library/golang:1.26-alpine@sha256:f85330846cde1e57ca9ec309382da3b8e6ae3ab943d2739500e08c86393a21b1 AS supervisor-builder
 RUN apk --no-cache add git
 WORKDIR /build
 RUN git clone https://github.com/ochinchina/supervisord . && \
@@ -11,12 +11,12 @@ RUN git clone https://github.com/ochinchina/supervisord . && \
 RUN GOBIN=/usr/bin go install -ldflags "-s -w"
 
 # Build Caddy with S3 storage backend.
-FROM docker.io/library/caddy:2.10.2-builder@sha256:6e3ed727ce8695fc58e0a8de8e5d11888f6463c430ea5b40e0b5f679ab734868 AS caddy-builder
+FROM docker.io/library/caddy:2.11.2-builder@sha256:10ed0251c5cd1dbb4db0b71ad43121147961a51adfec35febce2c93ea25c24f4 AS caddy-builder
 RUN xcaddy build ${CADDY_VERSION} \
     --with=github.com/ss098/certmagic-s3@v0.0.0-20250922022452-8af482af5f39
 
 # Build git-pages.
-FROM docker.io/library/golang:1.25-alpine@sha256:d3f0cf7723f3429e3f9ed846243970b20a2de7bae6a5b66fc5914e228d831bbb AS git-pages-builder
+FROM docker.io/library/golang:1.26-alpine@sha256:f85330846cde1e57ca9ec309382da3b8e6ae3ab943d2739500e08c86393a21b1 AS git-pages-builder
 RUN apk --no-cache add git
 WORKDIR /build
 COPY go.mod go.sum ./
@@ -26,7 +26,7 @@ COPY src/ ./src/
 RUN go build -ldflags "-s -w" -o git-pages .
 
 # Compose git-pages and Caddy.
-FROM docker.io/library/busybox:1.37.0-musl@sha256:ef13e7482851632be3faf5bd1d28d4727c0810901d564b35416f309975a12a30
+FROM docker.io/library/busybox:1.37.0-musl@sha256:19b646668802469d968a05342a601e78da4322a414a7c09b1c9ee25165042138
 COPY --from=ca-certificates-builder /etc/ssl/cert.pem /etc/ssl/cert.pem
 COPY --from=supervisor-builder /usr/bin/supervisord /bin/supervisord
 COPY --from=caddy-builder /usr/bin/caddy /bin/caddy
@@ -36,7 +36,7 @@ WORKDIR /app
 RUN mkdir /app/data
 COPY conf/supervisord.conf /app/supervisord.conf
 COPY conf/Caddyfile /app/Caddyfile
-COPY conf/config.example.toml /app/config.toml
+COPY conf/config.docker.toml /app/config.toml
 
 # Caddy ports:
 EXPOSE 80/tcp 443/tcp 443/udp
@@ -46,8 +46,8 @@ EXPOSE 3000/tcp 3001/tcp 3002/tcp
 # While the default command is to run git-pages standalone, the intended configuration
 # is to use it with Caddy and store both site data and credentials to an S3-compatible
 # object store.
-#  * In a standalone configuration, the default, git-caddy listens on port 3000 (http).
-#  * In a combined configuration, supervisord launches both git-caddy and Caddy, and
+#  * In a standalone configuration, the default, git-pages listens on port 3000 (http).
+#  * In a combined configuration, supervisord launches both git-pages and Caddy, and
 #    Caddy listens on ports 80 (http) and 443 (https).
 CMD ["git-pages"]
 # CMD ["supervisord"]

README.md

@@ -1,6 +1,9 @@
 git-pages
 =========
 
+<a href="ircs://irc.libera.chat/#git-pages"><img alt="Discuss on IRC at #git-pages on libera.chat" src="https://img.shields.io/badge/irc-libera.chat-blue"></a>
+<a href="https://matrix.to/#/#git-pages:catircservices.org"><img alt="Discuss on Matrix at #git-pages:catircservices.org" src="https://img.shields.io/matrix/git-pages%3Acatircservices.org?server_fqdn=matrix.org&fetchMode=summary&label=matrix&color=blue"></a>
+
 _git-pages_ is a static site server for use with Git forges (i.e. a GitHub Pages replacement). It is written with efficiency in mind, scaling horizontally to any number of machines and serving sites up to multiple gigabytes in size, while being equally suitable for small single-user deployments.
 
 It is implemented in Go and has no other mandatory dependencies, although it is designed to be used together with the [Caddy server][caddy] for TLS termination. Site data may be stored on the filesystem or in an [Amazon S3](https://aws.amazon.com/s3/) compatible object store.
@@ -89,12 +92,18 @@ Features
 * All updates to site content are atomic (subject to consistency guarantees of the storage backend). That is, there is an instantaneous moment during an update before which the server will return the old content and after which it will return the new content.
 * Files with a certain name, when placed in the root of a site, have special functions:
     - [Netlify `_redirects`][_redirects] file can be used to specify HTTP redirect and rewrite rules. The _git-pages_ implementation currently does not support placeholders, query parameters, or conditions, and may differ from Netlify in other minor ways. If you find that a supported `_redirects` file feature does not work the same as on Netlify, please file an issue. (Note that _git-pages_ does not perform URL normalization; `/foo` and `/foo/` are *not* the same, unlike with Netlify.)
-    - [Netlify `_headers`][_headers] file can be used to specify custom HTTP response headers (if allowlisted by configuration). In particular, this is useful to enable [CORS requests][cors]. The _git-pages_ implementation may differ from Netlify in minor ways; if you find that a `_headers` file feature does not work the same as on Netlify, please file an issue.
+    - [Netlify `_headers`][_headers] file can be used to specify custom HTTP response headers (if allowlisted by configuration). In particular, this is useful to enable [cross-origin isolation (COOP/COEP)][isolation]. The _git-pages_ implementation may differ from Netlify in minor ways; if you find that a `_headers` file feature does not work the same as on Netlify, please file an issue.
+    - [Netlify `Basic-Auth:`][basic-auth] pseudo-header in the `_headers` file can be used to password-protect parts of a site, if enabled via the `[limits].allow-basic-auth` configuration option. **This is not a security feature: credentials are stored in cleartext and are accessible to anyone who can update the site. *Only* use it in low-stakes applications, e.g. preventing search engines from indexing parts of a site.** The authors of _git-pages_ shall not be held liable for any unauthorized information disclosures resulting from the use of this feature.
+* Incremental updates can be made using `PUT` or `PATCH` requests where the body contains an archive (both tar and zip are supported).
+    - Any archive entry that is a symlink to `/git/blobs/<git-sha256>` is replaced with an existing manifest entry for the same site whose git blob hash matches `<git-sha256>`. If there is no existing manifest entry with the specified git hash, the update fails with a `422 Unprocessable Entity`.
+    - For this error response only, if the negotiated content type is `application/vnd.git-pages.unresolved`, the response will contain the `<git-sha256>` of each unresolved reference, one per line.
 * Support for SHA-256 Git hashes is [limited by go-git][go-git-sha256]; once go-git implements the required features, _git-pages_ will automatically gain support for SHA-256 Git hashes. Note that shallow clones (used by _git-pages_ to conserve bandwidth if available) aren't supported yet in the Git protocol as of 2025.
+* Git LFS is not supported: it is a single-vendor specification/implementation with no stable Go API and a risk of misuse for reflected HTTP DoS attacks. A diagnostic is emitted for any files uploaded have the `filter=lfs` attribute set via `.gitattributes`.
 
 [_redirects]: https://docs.netlify.com/manage/routing/redirects/overview/
 [_headers]: https://docs.netlify.com/manage/routing/headers/
-[cors]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/CORS
+[basic-auth]: https://docs.netlify.com/manage/security/secure-access-to-sites/basic-authentication-with-custom-http-headers/
+[isolation]: https://web.dev/articles/cross-origin-isolation-guide
 [go-git-sha256]: https://github.com/go-git/go-git/issues/706
 [whiteout]: https://docs.kernel.org/filesystems/overlayfs.html#whiteouts-and-opaque-directories
 
@@ -107,21 +116,22 @@ DNS is the primary authorization method, using either TXT records or wildcard ma
 The authorization flow for content updates (`PUT`, `PATCH`, `DELETE`, `POST` requests) proceeds sequentially in the following order, with the first of multiple applicable rule taking precedence:
 
 1. **Development Mode:** If the environment variable `PAGES_INSECURE` is set to a truthful value like `1`, the request is authorized.
-2. **DNS Challenge:** If the method is `PUT`, `PATCH`, `DELETE`, `POST`, and a well-formed `Authorization:` header is provided containing a `<token>`, and a TXT record lookup at `_git-pages-challenge.<host>` returns a record whose concatenated value equals `SHA256("<host> <token>")`, the request is authorized.
+2. **DNS Challenge:** If the method is `PUT`, `PATCH`, `DELETE`, `POST`, and a well-formed `Authorization:` header is provided containing a `<token>`, and a TXT record lookup at `_git-pages-challenge.<host>` returns a record whose concatenated value equals `SHA256("<host> <token>")`, and (for `PUT` and `POST` requests) the requested branch is `pages`, the request is authorized.
     - **`Pages` scheme:** Request includes an `Authorization: Pages <token>` header.
     - **`Basic` scheme:** Request includes an `Authorization: Basic <basic>` header, where `<basic>` is equal to `Base64("Pages:<token>")`. (Useful for non-Forgejo forges.)
-3. **DNS Allowlist:** If the method is `PUT` or `POST`, and the request URL is `scheme://<user>.<host>/`, and a TXT record lookup at `_git-pages-repository.<host>` returns a set of well-formed absolute URLs, and (for `PUT` requests) the body contains a repository URL, and the requested clone URLs is contained in this set of URLs, the request is authorized.
-4. **Wildcard Match (content):** If the method is `POST`, and a `[[wildcard]]` configuration section exists where the suffix of a hostname (compared label-wise) is equal to `[[wildcard]].domain`, and (for `PUT` requests) the body contains a repository URL, and the requested clone URL is a *matching* clone URL, the request is authorized.
-    - **Index repository:** If the request URL is `scheme://<user>.<host>/`, a *matching* clone URL is computed by templating `[[wildcard]].clone-url` with `<user>` and `<project>`, where `<project>` is computed by templating each element of `[[wildcard]].index-repos` with `<user>`, and `[[wildcard]]` is the section where the match occurred.
-    - **Project repository:** If the request URL is `scheme://<user>.<host>/<project>/`, a *matching* clone URL is computed by templating `[[wildcard]].clone-url` with `<user>` and `<project>`, and `[[wildcard]]` is the section where the match occurred.
-5. **Forge Authorization:** If the method is `PUT` or `PATCH`, and the body contains an archive, and a `[[wildcard]]` configuration section exists where the suffix of a hostname (compared label-wise) is equal to `[[wildcard]].domain`, and `[[wildcard]].authorization` is non-empty, and the request includes a `Forge-Authorization:` header, and the header (when forwarded as `Authorization:`) grants push permissions to a repository at the *matching* clone URL (as defined above) as determined by an API call to the forge, the request is authorized. (This enables publishing a site for a private repository.)
-5. **Default Deny:** Otherwise, the request is not authorized.
+3. **DNS Allowlist:** If the method is `PUT` or `POST`, and the request URL is `scheme://<user>.<host>/`, and a TXT record lookup at `_git-pages-repository.<host>` returns a set of well-formed absolute URLs, and (for `PUT` requests) the body contains a repository URL or (for `POST` requests) the body contains a GitHub-style webhook payload, and the requested clone URLs is contained in this set of URLs, and the requested branch is `pages`, the request is authorized.
+4. **Wildcard Match (content):** If the method is `POST`, and the body contains a GitHub-style webhook payload, and a `[[wildcard]]` configuration section exists such that `[[wildcard]].domain` is a suffix of the site hostname (compared label-wise), and the body contains a repository URL, and the requested clone URL is a *matching* clone URL, and the requested branch is a *matching* branch, the request is authorized.
+    - **Index repository:** If the request URL is `scheme://<user>.<host>/`: a *matching* clone URL is computed by templating `[[wildcard]].clone-url` with `<user>` and `<project>`, where `<project>` is computed by templating `[[wildcard]].index-repo` with `<user>`, and `[[wildcard]]` is the section where the match occurred; and a *matching* branch is specified by `[[wildcard]].index-repo-branch`.
+    - **Project repository:** If the request URL is `scheme://<user>.<host>/<project>/`: a *matching* clone URL is computed by templating `[[wildcard]].clone-url` with `<user>` and `<project>`, and `[[wildcard]]` is the section where the match occurred; and a *matching* branch is `pages`.
+5. **Forge Authorization (wildcard):** If the method is `PUT` or `PATCH` or `DELETE`, and (unless the method is `DELETE`) the body contains an archive, and a `[[wildcard]]` configuration section exists such that `[[wildcard]].domain` is a suffix of the site hostname (compared label-wise), and `[[wildcard]].authorization` is defined, and the request includes a `Forge-Authorization:` header, and the header (when forwarded as `Authorization:`) grants push permissions to a repository at the *matching* clone URL (as defined above) as determined by an API call to the forge, the request is authorized.
+6. **Forge Authorization (DNS allowlist):** If the method is `PUT` or `PATCH` or `DELETE`, and (unless the method is `DELETE`) the body contains an archive, and the request URL is `scheme://<host>/`, and a TXT record lookup at `_git-pages-forge-allowlist.<host>` returns a set of well-formed absolute URLs, and the request includes a `Forge-Authorization:` header, and the header (when forwarded as `Authorization:`) grants push permissions to a repository at any of the URLs in the TXT records as determined by an API call to the forge, the request is authorized.
+7. **Default Deny:** Otherwise, the request is not authorized.
 
 The authorization flow for metadata retrieval (`GET` requests with site paths starting with `.git-pages/`) in the following order, with the first of multiple applicable rule taking precedence:
 
 1. **Development Mode:** Same as for content updates.
 2. **DNS Challenge:** Same as for content updates.
-3. **Wildcard Match (metadata):** If a `[[wildcard]]` configuration section exists where the suffix of a hostname (compared label-wise) is equal to `[[wildcard]].domain`, the request is authorized.
+3. **Wildcard Match (metadata):** If a `[[wildcard]]` configuration section exists where the suffix of a hostname (compared label-wise) is equal to `[[wildcard]].domain`, and the site never uses the `Basic-Auth:` pseudo-header, the request is authorized.
 4. **Default Deny:** Otherwise, the request is not authorized.
 
 
@@ -130,14 +140,11 @@ Observability
 
 _git-pages_ has robust observability features built in:
 * The metrics endpoint (bound to `:3002` by default) returns Go, pages server, and storage backend metrics in the [Prometheus](https://prometheus.io/) format.
-* Optional [Sentry](https://sentry.io/) integration allows greater visibility into the application. The `ENVIRONMENT` environment variable configures the deploy environment name (`development` by default).
-    * If `SENTRY_DSN` environment variable is set, panics are reported to Sentry.
-    * If `SENTRY_DSN` and `SENTRY_LOGS=1` environment variables are set, logs are uploaded to Sentry.
-    * If `SENTRY_DSN` and `SENTRY_TRACING=1` environment variables are set, traces are uploaded to Sentry.
-* Optional syslog integration allows transmitting application logs to a syslog daemon. When present, the `SYSLOG_ADDR` environment variable enables the integration, and the variable's value is used to configure the absolute path to a Unix socket (usually located at `/dev/log` on Unix systems) or a network address of one of the following formats:
-    * for TLS over TCP: `tcp+tls://host:port`;
-    * for plain TCP: `tcp://host:post`;
-    * for UDP: `udp://host:port`.
+* Optional syslog integration allows transmitting application logs to a syslog daemon. When present, the `SYSLOG_ADDR` environment variable enables the integration, and the value is used to configure the syslog destination. The value must follow the format `family/address` and is usually one of the following:
+    * a Unix datagram socket: `unixgram//dev/log`;
+    * TLS over TCP: `tcp+tls/host:port`;
+    * plain TCP: `tcp/host:post`;
+    * UDP: `udp/host:port`.
 
 
 Architecture (v2)

conf/Caddyfile

@@ -25,11 +25,5 @@ https://, http:// {
 		on_demand
 	}
 
-	# initial PUT/POST for a new domain has to happen over HTTP
-	@upgrade `method('GET') && protocol('http')`
-	redir @upgrade https://{host}{uri} 301
-
 	reverse_proxy http://localhost:3000
-	header Alt-Svc `h3=":443"; persist=1, h2=":443"; persist=1`
-	encode
 }

conf/config.default.toml

@@ -0,0 +1,37 @@
+# This is a configuration containing default values only. The `config.example.toml` file contains
+# a configuration more useful for demonstration purposes.
+
+log-format = 'text'
+
+[server]
+pages = 'tcp/localhost:3000'
+caddy = 'tcp/localhost:3001'
+metrics = 'tcp/localhost:3002'
+
+[storage]
+type = 'fs'
+
+[storage.fs]
+root = './data'
+
+[limits]
+max-site-size = '128MB'
+max-manifest-size = '1MB'
+max-inline-file-size = '256B'
+git-large-object-threshold = '1MB'
+max-symlink-depth = 16
+update-timeout = '1m0s'
+concurrent-uploads = 1024
+max-heap-size-ratio = 0.5
+forbidden-domains = []
+allowed-repository-url-prefixes = []
+allowed-custom-headers = ['X-Clacks-Overhead']
+allow-basic-auth = false
+
+[audit]
+node-id = 0
+collect = false
+include-ip = ''
+
+[observability]
+slow-response-threshold = '500ms'

conf/config.docker.toml

@@ -0,0 +1,4 @@
+[server]
+pages = "tcp/:3000"
+caddy = "tcp/:3001"
+metrics = "tcp/:3002"

conf/config.example.toml

@@ -1,19 +1,18 @@
-# Unless otherwise noted, every value in this file is the same
-# as the intrinsic default value.
+# This is a configuration used for demonstration purposes. The `config.default.toml` file contains
+# a configuration corresponding to default values only.
 
 log-format = "text"
-log-level = "info"
 
 [server]
 # Use "-" to disable the handler.
-pages = "tcp/:3000"
-caddy = "tcp/:3001"
-metrics = "tcp/:3002"
+pages = "tcp/localhost:3000"
+caddy = "tcp/localhost:3001"
+metrics = "tcp/localhost:3002"
 
 [[wildcard]] # non-default section
 domain = "codeberg.page"
 clone-url = "https://codeberg.org/<user>/<project>.git"
-index-repos = ["<user>.codeberg.page", "pages"]
+index-repo = "pages"
 index-repo-branch = "main"
 authorization = "forgejo"
 
@@ -49,15 +48,17 @@ max-inline-file-size = "256B"
 git-large-object-threshold = "1M"
 max-symlink-depth = 16
 update-timeout = "60s"
+concurrent-uploads = 1024
 max-heap-size-ratio = 0.5 # * RAM_size
 forbidden-domains = []
-# allowed-repository-url-prefixes = <nil>
+allowed-repository-url-prefixes = []
 allowed-custom-headers = ["X-Clacks-Overhead"]
+allow-basic-auth = false
 
 [audit]
 node-id = 0
 collect = false
-include-ip = false
+include-ip = ""
 notify-url = ""
 
 [observability]

flake.lock

@@ -18,6 +18,29 @@
         "type": "github"
       }
     },
+    "gomod2nix": {
+      "inputs": {
+        "flake-utils": [
+          "flake-utils"
+        ],
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1763982521,
+        "narHash": "sha256-ur4QIAHwgFc0vXiaxn5No/FuZicxBr2p0gmT54xZkUQ=",
+        "owner": "nix-community",
+        "repo": "gomod2nix",
+        "rev": "02e63a239d6eabd595db56852535992c898eba72",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "gomod2nix",
+        "type": "github"
+      }
+    },
     "nix-filter": {
       "locked": {
         "lastModified": 1757882181,
@@ -52,6 +75,7 @@
     "root": {
       "inputs": {
         "flake-utils": "flake-utils",
+        "gomod2nix": "gomod2nix",
         "nix-filter": "nix-filter",
         "nixpkgs": "nixpkgs"
       }

flake.nix

@@ -3,6 +3,12 @@
     nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
     flake-utils.url = "github:numtide/flake-utils";
     nix-filter.url = "github:numtide/nix-filter";
+
+    gomod2nix = {
+      url = "github:nix-community/gomod2nix";
+      inputs.nixpkgs.follows = "nixpkgs";
+      inputs.flake-utils.follows = "flake-utils";
+    };
   };
 
   outputs =
@@ -11,13 +17,20 @@
       nixpkgs,
       flake-utils,
       nix-filter,
-    }:
+      ...
+    }@inputs:
     flake-utils.lib.eachDefaultSystem (
       system:
       let
-        pkgs = nixpkgs.legacyPackages.${system};
+        pkgs = import nixpkgs {
+          inherit system;
 
-        git-pages = pkgs.buildGo125Module {
+          overlays = [
+            inputs.gomod2nix.overlays.default
+          ];
+        };
+
+        git-pages = pkgs.buildGoApplication {
           pname = "git-pages";
           version = "0";
 
@@ -33,17 +46,24 @@
             ];
           };
 
-          buildInputs = with pkgs; [
-            pkgsStatic.musl
-          ];
+          buildInputs = pkgs.lib.optionals pkgs.stdenv.targetPlatform.isLinux (
+            with pkgs;
+            [
+              pkgsStatic.musl
+            ]
+          );
 
           ldflags = [
             "-linkmode external"
+            "-X main.versionOverride=${self.shortRev or self.dirtyShortRev}"
+          ]
+          ++ pkgs.lib.optionals pkgs.stdenv.targetPlatform.isLinux [
             "-extldflags -static"
             "-s -w"
           ];
 
-          vendorHash = "sha256-40LyEXdJDpWPe9UvqM2siqXdpbae1ba7kN7FtySPpBc=";
+          go = pkgs.go_1_25;
+          modules = ./gomod2nix.toml;
         };
       in
       {
@@ -56,6 +76,7 @@
 
           packages = with pkgs; [
             caddy
+            gomod2nix
           ];
         };
 

go.mod

@@ -3,70 +3,74 @@ module codeberg.org/git-pages/git-pages
 go 1.25.0
 
 require (
-	codeberg.org/git-pages/go-headers v1.1.0
-	codeberg.org/git-pages/go-slog-syslog v0.0.0-20251122144254-06c45d430fb9
+	codeberg.org/git-pages/go-headers v1.1.1
+	codeberg.org/git-pages/go-slog-syslog v0.0.0-20251207093707-892f654e80b7
+	github.com/BurntSushi/toml v1.6.0
 	github.com/KimMachineGun/automemlimit v0.7.5
+	github.com/bits-and-blooms/bloom/v3 v3.7.1
 	github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500
 	github.com/creasty/defaults v1.8.0
-	github.com/fatih/color v1.18.0
-	github.com/getsentry/sentry-go v0.40.0
-	github.com/getsentry/sentry-go/slog v0.40.0
-	github.com/go-git/go-billy/v6 v6.0.0-20251126203821-7f9c95185ee0
-	github.com/go-git/go-git/v6 v6.0.0-20251128074608-48f817f57805
+	github.com/fatih/color v1.19.0
+	github.com/go-git/go-billy/v6 v6.0.0-20260410103409-85b6241850b5
+	github.com/go-git/go-git/v6 v6.0.0-alpha.2
 	github.com/jpillora/backoff v1.0.0
 	github.com/kankanreno/go-snowflake v1.2.0
-	github.com/klauspost/compress v1.18.1
-	github.com/maypok86/otter/v2 v2.2.1
-	github.com/minio/minio-go/v7 v7.0.97
-	github.com/pelletier/go-toml/v2 v2.2.4
+	github.com/klauspost/compress v1.18.5
+	github.com/maypok86/otter/v2 v2.3.0
+	github.com/minio/minio-go/v7 v7.0.100
 	github.com/pquerna/cachecontrol v0.2.0
 	github.com/prometheus/client_golang v1.23.2
-	github.com/samber/slog-multi v1.6.0
+	github.com/samber/slog-multi v1.8.0
 	github.com/tj/go-redirects v0.0.0-20200911105812-fd1ba1020b37
 	github.com/valyala/fasttemplate v1.2.2
-	google.golang.org/protobuf v1.36.10
+	golang.org/x/net v0.53.0
+	google.golang.org/protobuf v1.36.11
 )
 
 require (
 	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/ProtonMail/go-crypto v1.3.0 // indirect
+	github.com/ProtonMail/go-crypto v1.4.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bits-and-blooms/bitset v1.24.2 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/cloudflare/circl v1.6.1 // indirect
+	github.com/cloudflare/circl v1.6.3 // indirect
 	github.com/cyphar/filepath-securejoin v0.6.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/go-git/gcfg/v2 v2.0.2 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
-	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/kevinburke/ssh_config v1.4.0 // indirect
+	github.com/kevinburke/ssh_config v1.6.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
 	github.com/klauspost/crc32 v1.3.0 // indirect
 	github.com/leodido/go-syslog/v4 v4.3.0 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/minio/crc64nvme v1.1.0 // indirect
+	github.com/minio/crc64nvme v1.1.1 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
 	github.com/philhofer/fwd v1.2.0 // indirect
 	github.com/pjbgf/sha1cd v0.5.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.66.1 // indirect
 	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/rs/xid v1.6.0 // indirect
-	github.com/samber/lo v1.52.0 // indirect
-	github.com/samber/slog-common v0.19.0 // indirect
+	github.com/samber/lo v1.53.0 // indirect
+	github.com/samber/slog-common v0.21.0 // indirect
 	github.com/sergi/go-diff v1.4.0 // indirect
-	github.com/tinylib/msgp v1.3.0 // indirect
+	github.com/stretchr/testify v1.11.1 // indirect
+	github.com/tinylib/msgp v1.6.1 // indirect
 	github.com/tj/assert v0.0.3 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
-	golang.org/x/crypto v0.45.0 // indirect
-	golang.org/x/net v0.47.0 // indirect
-	golang.org/x/sys v0.38.0 // indirect
-	golang.org/x/text v0.31.0 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/crypto v0.50.0 // indirect
+	golang.org/x/sync v0.20.0 // indirect
+	golang.org/x/sys v0.43.0 // indirect
+	golang.org/x/text v0.36.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,25 +1,31 @@
-codeberg.org/git-pages/go-headers v1.1.0 h1:rk7/SOSsn+XuL7PUQZFYUaWKHEaj6K8mXmUV9rF2VxE=
-codeberg.org/git-pages/go-headers v1.1.0/go.mod h1:N4gwH0U3YPwmuyxqH7xBA8j44fTPX+vOEP7ejJVBPts=
-codeberg.org/git-pages/go-slog-syslog v0.0.0-20251122144254-06c45d430fb9 h1:xfPDg8ThBt3+t+C+pvM3bEH4ePUzP5t5kY2v19TqgKc=
-codeberg.org/git-pages/go-slog-syslog v0.0.0-20251122144254-06c45d430fb9/go.mod h1:8NPSXbYcVb71qqNM5cIgn1/uQgMisLbu2dVD1BNxsUw=
+codeberg.org/git-pages/go-headers v1.1.1 h1:fpIBELKo66Z2k+gCeYl5mCEXVQ99Lmx1iup1nbo2shE=
+codeberg.org/git-pages/go-headers v1.1.1/go.mod h1:N4gwH0U3YPwmuyxqH7xBA8j44fTPX+vOEP7ejJVBPts=
+codeberg.org/git-pages/go-slog-syslog v0.0.0-20251207093707-892f654e80b7 h1:+rkrAxhNZo/eKEcKOqVOsF6ohAPv5amz0JLburOeRjs=
+codeberg.org/git-pages/go-slog-syslog v0.0.0-20251207093707-892f654e80b7/go.mod h1:8NPSXbYcVb71qqNM5cIgn1/uQgMisLbu2dVD1BNxsUw=
+github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk=
+github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
 github.com/KimMachineGun/automemlimit v0.7.5 h1:RkbaC0MwhjL1ZuBKunGDjE/ggwAX43DwZrJqVwyveTk=
 github.com/KimMachineGun/automemlimit v0.7.5/go.mod h1:QZxpHaGOQoYvFhv/r4u3U0JTC2ZcOwbSr11UZF46UBM=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
-github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw=
-github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE=
+github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM=
+github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bits-and-blooms/bitset v1.24.2 h1:M7/NzVbsytmtfHbumG+K2bremQPMJuqv1JD3vOaFxp0=
+github.com/bits-and-blooms/bitset v1.24.2/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
+github.com/bits-and-blooms/bloom/v3 v3.7.1 h1:WXovk4TRKZttAMJfoQx6K2DM0zNIt8w+c67UqO+etV0=
+github.com/bits-and-blooms/bloom/v3 v3.7.1/go.mod h1:rZzYLLje2dfzXfAkJNxQQHsKurAyK55KUnL43Euk0hU=
 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 h1:6lhrsTEnloDPXyeZBvSYvQf8u86jbKehZPVDDlkgDl4=
 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0=
-github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
+github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8=
+github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4=
 github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk=
 github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM=
 github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE=
@@ -29,32 +35,22 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
-github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o=
-github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
 github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
-github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
-github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
-github.com/getsentry/sentry-go v0.40.0 h1:VTJMN9zbTvqDqPwheRVLcp0qcUcM+8eFivvGocAaSbo=
-github.com/getsentry/sentry-go v0.40.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s=
-github.com/getsentry/sentry-go/slog v0.40.0 h1:uR2EPL9w6uHw3XB983IAqzqM9mP+fjJpNY9kfob3/Z8=
-github.com/getsentry/sentry-go/slog v0.40.0/go.mod h1:ArRaP+0rsbnJGyvZwYDo/vDQT/YBbOQeOlO+DGW+F9s=
+github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w=
+github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE=
 github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
 github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
-github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
-github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-git/gcfg/v2 v2.0.2 h1:MY5SIIfTGGEMhdA7d7JePuVVxtKL7Hp+ApGDJAJ7dpo=
 github.com/go-git/gcfg/v2 v2.0.2/go.mod h1:/lv2NsxvhepuMrldsFilrgct6pxzpGdSRC13ydTLSLs=
-github.com/go-git/go-billy/v6 v6.0.0-20251126203821-7f9c95185ee0 h1:eY5aB2GXiVdgTueBcqsBt53WuJTRZAuCdIS/86Pcq5c=
-github.com/go-git/go-billy/v6 v6.0.0-20251126203821-7f9c95185ee0/go.mod h1:0NjwVNrwtVFZBReAp5OoGklGJIgJFEbVyHneAr4lc8k=
-github.com/go-git/go-git-fixtures/v5 v5.1.1 h1:OH8i1ojV9bWfr0ZfasfpgtUXQHQyVS8HXik/V1C099w=
-github.com/go-git/go-git-fixtures/v5 v5.1.1/go.mod h1:Altk43lx3b1ks+dVoAG2300o5WWUnktvfY3VI6bcaXU=
-github.com/go-git/go-git/v6 v6.0.0-20251128074608-48f817f57805 h1:jxQ3BzYeErNRvlI/4+0mpwqMzvB4g97U+ksfgvrUEbY=
-github.com/go-git/go-git/v6 v6.0.0-20251128074608-48f817f57805/go.mod h1:dIwT3uWK1ooHInyVnK2JS5VfQ3peVGYaw2QPqX7uFvs=
+github.com/go-git/go-billy/v6 v6.0.0-20260410103409-85b6241850b5 h1:r5Y4Hn9QwQj+u6vN0Ib1MipHkanYaG8Zj0kxsnv8Bu4=
+github.com/go-git/go-billy/v6 v6.0.0-20260410103409-85b6241850b5/go.mod h1:CdBVp7CXl9l3sOyNEog46cP1Pvx/hjCe9AD0mtaIUYU=
+github.com/go-git/go-git-fixtures/v6 v6.0.0-20260405195209-b16dd39735e0 h1:XoTsdvaghuVfIr7HpNTmFDLu2nz3I2iGqyn6Uk6MkJc=
+github.com/go-git/go-git-fixtures/v6 v6.0.0-20260405195209-b16dd39735e0/go.mod h1:1Lr7/vYEYyl6Ir9Ku0tKrCIRreM5zovv0Jdx2MPSM4s=
+github.com/go-git/go-git/v6 v6.0.0-alpha.2 h1:T3loNtDuAixNzXtlQxZhnYiYpaQ3CA4vn9RssAniEeI=
+github.com/go-git/go-git/v6 v6.0.0-alpha.2/go.mod h1:oCD3i19CTz7gBpeb11ZZqL91WzqbMq9avn5KpUYy/Ak=
 github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
 github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
-github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=
-github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@@ -63,10 +59,10 @@ github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2E
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/kankanreno/go-snowflake v1.2.0 h1:Zx2SctsH5pivIj9vyhwyDyQS23jcDJx4iT49Bjv81kk=
 github.com/kankanreno/go-snowflake v1.2.0/go.mod h1:6CZ+10PeVsFXKZUTYyJzPiRIjn1IXbInaWLCX/LDJ0g=
-github.com/kevinburke/ssh_config v1.4.0 h1:6xxtP5bZ2E4NF5tuQulISpTO2z8XbtH8cg1PWkxoFkQ=
-github.com/kevinburke/ssh_config v1.4.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M=
-github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
-github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
+github.com/kevinburke/ssh_config v1.6.0 h1:J1FBfmuVosPHf5GRdltRLhPJtJpTlMdKTBjRgTaQBFY=
+github.com/kevinburke/ssh_config v1.6.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M=
+github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
+github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
 github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
@@ -76,36 +72,30 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
-github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/leodido/go-syslog/v4 v4.3.0 h1:bbSpI/41bYK9iSdlYzcwvlxuLOE8yi4VTFmedtnghdA=
 github.com/leodido/go-syslog/v4 v4.3.0/go.mod h1:eJ8rUfDN5OS6dOkCOBYlg2a+hbAg6pJa99QXXgMrd98=
-github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
-github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
-github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
+github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/maypok86/otter/v2 v2.2.1 h1:hnGssisMFkdisYcvQ8L019zpYQcdtPse+g0ps2i7cfI=
-github.com/maypok86/otter/v2 v2.2.1/go.mod h1:1NKY9bY+kB5jwCXBJfE59u+zAwOt6C7ni1FTlFFMqVs=
-github.com/minio/crc64nvme v1.1.0 h1:e/tAguZ+4cw32D+IO/8GSf5UVr9y+3eJcxZI2WOO/7Q=
-github.com/minio/crc64nvme v1.1.0/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
+github.com/maypok86/otter/v2 v2.3.0 h1:8H8AVVFUSzJwIegKwv1uF5aGitTY+AIrtktg7OcLs8w=
+github.com/maypok86/otter/v2 v2.3.0/go.mod h1:XgIdlpmL6jYz882/CAx1E4C1ukfgDKSaw4mWq59+7l8=
+github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
+github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
-github.com/minio/minio-go/v7 v7.0.97 h1:lqhREPyfgHTB/ciX8k2r8k0D93WaFqxbJX36UZq5occ=
-github.com/minio/minio-go/v7 v7.0.97/go.mod h1:re5VXuo0pwEtoNLsNuSr0RrLfT/MBtohwdaSmPPSRSk=
+github.com/minio/minio-go/v7 v7.0.100 h1:ShkWi8Tyj9RtU57OQB2HIXKz4bFgtVib0bbT1sbtLI8=
+github.com/minio/minio-go/v7 v7.0.100/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
-github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
-github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
 github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
 github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
-github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
-github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pjbgf/sha1cd v0.5.0 h1:a+UkboSi1znleCDUNT3M5YxjOnN1fz2FhN48FlwCxs0=
 github.com/pjbgf/sha1cd v0.5.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -122,16 +112,16 @@ github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9Z
 github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
 github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
 github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
-github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
-github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
 github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
-github.com/samber/lo v1.52.0 h1:Rvi+3BFHES3A8meP33VPAxiBZX/Aws5RxrschYGjomw=
-github.com/samber/lo v1.52.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0=
-github.com/samber/slog-common v0.19.0 h1:fNcZb8B2uOLooeYwFpAlKjkQTUafdjfqKcwcC89G9YI=
-github.com/samber/slog-common v0.19.0/go.mod h1:dTz+YOU76aH007YUU0DffsXNsGFQRQllPQh9XyNoA3M=
-github.com/samber/slog-multi v1.6.0 h1:i1uBY+aaln6ljwdf7Nrt4Sys8Kk6htuYuXDHWJsHtZg=
-github.com/samber/slog-multi v1.6.0/go.mod h1:qTqzmKdPpT0h4PFsTN5rYRgLwom1v+fNGuIrl1Xnnts=
+github.com/samber/lo v1.53.0 h1:t975lj2py4kJPQ6haz1QMgtId2gtmfktACxIXArw3HM=
+github.com/samber/lo v1.53.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0=
+github.com/samber/slog-common v0.21.0 h1:Wo2hTly1Br5RjYqX/BTWJJeDnTE85oWk/7vqlpZuAUc=
+github.com/samber/slog-common v0.21.0/go.mod h1:d/6OaSlzdkl9PFpfRLgn8FwY1OW6EFmPtBpsHX4MrU0=
+github.com/samber/slog-multi v1.8.0 h1:E05c1wnQ+8M58oQDBABlJ4TEIJWssNgtckso3zlaLlI=
+github.com/samber/slog-multi v1.8.0/go.mod h1:6+3j/ILxDvAcLD75YdQAm6iKWu6AmwlohLgQxL/2aiI=
 github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
 github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -139,12 +129,14 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
-github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww=
-github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0=
+github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
+github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
 github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
 github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
 github.com/tj/go-redirects v0.0.0-20200911105812-fd1ba1020b37 h1:K11tjwz8zTTSZkz4TUjfLN+y8uJWP38BbyPqZ2yB/Yk=
 github.com/tj/go-redirects v0.0.0-20200911105812-fd1ba1020b37/go.mod h1:E0E2H2gQA+uoi27VCSU+a/BULPtadQA78q3cpTjZbZw=
+github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg=
+github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
@@ -153,20 +145,23 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
 go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
-golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
-golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
-golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
-golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
+golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
+golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
+golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
-golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
-golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
-golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
-golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
-google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
-google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
+golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
+golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
+golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
+golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
+google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

gomod2nix.toml

@@ -0,0 +1,204 @@
+schema = 3
+
+[mod]
+  [mod."codeberg.org/git-pages/go-headers"]
+    version = "v1.1.1"
+    hash = "sha256-qgL7l1FHXxcBWhBnBLEI0yENd6P+frvwlKxEAXLA3VY="
+  [mod."codeberg.org/git-pages/go-slog-syslog"]
+    version = "v0.0.0-20251207093707-892f654e80b7"
+    hash = "sha256-ye+DBIyxqTEOViYRrQPWyGJCaLmyKSDwH5btlqDPizM="
+  [mod."github.com/BurntSushi/toml"]
+    version = "v1.6.0"
+    hash = "sha256-ptdUJvuc21ixeLt+M5way/na3aCnCO4MYHWulWp8NEY="
+  [mod."github.com/KimMachineGun/automemlimit"]
+    version = "v0.7.5"
+    hash = "sha256-lH/ip9j2hbYUc2W/XIYve/5TScQPZtEZe3hu76CY//k="
+  [mod."github.com/Microsoft/go-winio"]
+    version = "v0.6.2"
+    hash = "sha256-tVNWDUMILZbJvarcl/E7tpSnkn7urqgSHa2Eaka5vSU="
+  [mod."github.com/ProtonMail/go-crypto"]
+    version = "v1.4.1"
+    hash = "sha256-6iGAFCjoNveY+ipbKqq2gt+RXpi2eQyPXAY01rxPcWc="
+  [mod."github.com/beorn7/perks"]
+    version = "v1.0.1"
+    hash = "sha256-h75GUqfwJKngCJQVE5Ao5wnO3cfKD9lSIteoLp/3xJ4="
+  [mod."github.com/bits-and-blooms/bitset"]
+    version = "v1.24.2"
+    hash = "sha256-hT88EpdWmKnqdxApJhs/aIAptf33HmtSp2KXPI+Ym7o="
+  [mod."github.com/bits-and-blooms/bloom/v3"]
+    version = "v3.7.1"
+    hash = "sha256-KZduCu+k4+xqBcFRTfg8Yc/PEf5jfpjn0I1YoxfnVPo="
+  [mod."github.com/c2h5oh/datasize"]
+    version = "v0.0.0-20231215233829-aa82cc1e6500"
+    hash = "sha256-8MqL7xCvE6fIjanz2jwkaLP1OE5kLu62TOcQx452DHQ="
+  [mod."github.com/cespare/xxhash/v2"]
+    version = "v2.3.0"
+    hash = "sha256-7hRlwSR+fos1kx4VZmJ/7snR7zHh8ZFKX+qqqqGcQpY="
+  [mod."github.com/cloudflare/circl"]
+    version = "v1.6.3"
+    hash = "sha256-XZm4EastgX67Dgm5BpOEW/PY4aLcHM/O8+Xbz26PuTY="
+  [mod."github.com/creasty/defaults"]
+    version = "v1.8.0"
+    hash = "sha256-I1LE1cfOhMS5JxB7+fWTKieefw2Gge1UhIZh+A6pa6s="
+  [mod."github.com/cyphar/filepath-securejoin"]
+    version = "v0.6.1"
+    hash = "sha256-obqip8c1c9mjXFznyXF8aDnpcMw7ttzv+e28anCa/v0="
+  [mod."github.com/davecgh/go-spew"]
+    version = "v1.1.1"
+    hash = "sha256-nhzSUrE1fCkN0+RL04N4h8jWmRFPPPWbCuDc7Ss0akI="
+  [mod."github.com/dustin/go-humanize"]
+    version = "v1.0.1"
+    hash = "sha256-yuvxYYngpfVkUg9yAmG99IUVmADTQA0tMbBXe0Fq0Mc="
+  [mod."github.com/emirpasic/gods"]
+    version = "v1.18.1"
+    hash = "sha256-hGDKddjLj+5dn2woHtXKUdd49/3xdsqnhx7VEdCu1m4="
+  [mod."github.com/fatih/color"]
+    version = "v1.19.0"
+    hash = "sha256-YgMm1nid8yigNLG6aHfuMbsvMI1UYVf/Rkg44pp/NTU="
+  [mod."github.com/go-git/gcfg/v2"]
+    version = "v2.0.2"
+    hash = "sha256-icqMDeC/tEg/3979EuEN67Ml5KjdDA0R3QvR6iLLrSI="
+  [mod."github.com/go-git/go-billy/v6"]
+    version = "v6.0.0-20260410103409-85b6241850b5"
+    hash = "sha256-2qQeUjkswSqI9joCKhvMB1lvnKHL9INbAzy4UBveHsw="
+  [mod."github.com/go-git/go-git/v6"]
+    version = "v6.0.0-alpha.2"
+    hash = "sha256-nUjRn1uIZKhIKqdNXfTirGtm07XCUKF2z3aat9O0dqM="
+  [mod."github.com/go-ini/ini"]
+    version = "v1.67.0"
+    hash = "sha256-V10ahGNGT+NLRdKUyRg1dos5RxLBXBk1xutcnquc/+4="
+  [mod."github.com/google/uuid"]
+    version = "v1.6.0"
+    hash = "sha256-VWl9sqUzdOuhW0KzQlv0gwwUQClYkmZwSydHG2sALYw="
+  [mod."github.com/jpillora/backoff"]
+    version = "v1.0.0"
+    hash = "sha256-uxHg68NN8hrwPCrPfLYYprZHf7dMyEoPoF46JFx0IHU="
+  [mod."github.com/kankanreno/go-snowflake"]
+    version = "v1.2.0"
+    hash = "sha256-713xGEqjwaUGIu2EHII5sldWmcquFpxZmte/7R/O6LA="
+  [mod."github.com/kevinburke/ssh_config"]
+    version = "v1.6.0"
+    hash = "sha256-i/EYNJx0+HbAGFVoiKV4QF/zqb4fWewh+bpBKUkXDCc="
+  [mod."github.com/klauspost/compress"]
+    version = "v1.18.5"
+    hash = "sha256-H9b5iFJf4XbEnkGQCjGQAJ3aYhVDiolKrDewTbhuzQo="
+  [mod."github.com/klauspost/cpuid/v2"]
+    version = "v2.3.0"
+    hash = "sha256-50JhbQyT67BK38HIdJihPtjV7orYp96HknI2VP7A9Yc="
+  [mod."github.com/klauspost/crc32"]
+    version = "v1.3.0"
+    hash = "sha256-RsS/MDJbVzVB+i74whqABgwZJWMw+AutF6HhJBVgbag="
+  [mod."github.com/leodido/go-syslog/v4"]
+    version = "v4.3.0"
+    hash = "sha256-fCJ2rgrrPR/Ey/PoAsJhd8Sl8mblAnnMAmBuoWFBTgg="
+  [mod."github.com/mattn/go-colorable"]
+    version = "v0.1.14"
+    hash = "sha256-JC60PjKj7MvhZmUHTZ9p372FV72I9Mxvli3fivTbxuA="
+  [mod."github.com/mattn/go-isatty"]
+    version = "v0.0.20"
+    hash = "sha256-qhw9hWtU5wnyFyuMbKx+7RB8ckQaFQ8D+8GKPkN3HHQ="
+  [mod."github.com/maypok86/otter/v2"]
+    version = "v2.3.0"
+    hash = "sha256-ELzmi/s2WqDeUmzSGnfx+ys2Hs28XHqF7vlEzyRotIA="
+  [mod."github.com/minio/crc64nvme"]
+    version = "v1.1.1"
+    hash = "sha256-RVVi/gWPBEQqcW4n+KIKxlA3uY5+77e2rhkVk8fFNUo="
+  [mod."github.com/minio/md5-simd"]
+    version = "v1.1.2"
+    hash = "sha256-vykcXvy2VBBAXnJott/XsGTT0gk2UL36JzZKfJ1KAUY="
+  [mod."github.com/minio/minio-go/v7"]
+    version = "v7.0.100"
+    hash = "sha256-MjWYoX4b+OOSOkjsitQQqcTbpQ7CYNghN9XCdrqgYaM="
+  [mod."github.com/munnerz/goautoneg"]
+    version = "v0.0.0-20191010083416-a7dc8b61c822"
+    hash = "sha256-79URDDFenmGc9JZu+5AXHToMrtTREHb3BC84b/gym9Q="
+  [mod."github.com/pbnjay/memory"]
+    version = "v0.0.0-20210728143218-7b4eea64cf58"
+    hash = "sha256-QI+F1oPLOOtwNp8+m45OOoSfYFs3QVjGzE0rFdpF/IA="
+  [mod."github.com/philhofer/fwd"]
+    version = "v1.2.0"
+    hash = "sha256-cGx2/0QQay46MYGZuamFmU0TzNaFyaO+J7Ddzlr/3dI="
+  [mod."github.com/pjbgf/sha1cd"]
+    version = "v0.5.0"
+    hash = "sha256-11XBkhdciQGsQ7jEMZ6PgphRKjruTSc7ZxfOwDuPCr8="
+  [mod."github.com/pkg/errors"]
+    version = "v0.9.1"
+    hash = "sha256-mNfQtcrQmu3sNg/7IwiieKWOgFQOVVe2yXgKBpe/wZw="
+  [mod."github.com/pmezard/go-difflib"]
+    version = "v1.0.0"
+    hash = "sha256-/FtmHnaGjdvEIKAJtrUfEhV7EVo5A/eYrtdnUkuxLDA="
+  [mod."github.com/pquerna/cachecontrol"]
+    version = "v0.2.0"
+    hash = "sha256-tuTERCFfwmqPepw/rs5cyv9fArCD30BqgjZqwMV+vzQ="
+  [mod."github.com/prometheus/client_golang"]
+    version = "v1.23.2"
+    hash = "sha256-3GD4fBFa1tJu8MS4TNP6r2re2eViUE+kWUaieIOQXCg="
+  [mod."github.com/prometheus/client_model"]
+    version = "v0.6.2"
+    hash = "sha256-q6Fh6v8iNJN9ypD47LjWmx66YITa3FyRjZMRsuRTFeQ="
+  [mod."github.com/prometheus/common"]
+    version = "v0.66.1"
+    hash = "sha256-bqHPaV9IV70itx63wqwgy2PtxMN0sn5ThVxDmiD7+Tk="
+  [mod."github.com/prometheus/procfs"]
+    version = "v0.16.1"
+    hash = "sha256-OBCvKlLW2obct35p0L9Q+1ZrxZjpTmbgHMP2rng9hpo="
+  [mod."github.com/rs/xid"]
+    version = "v1.6.0"
+    hash = "sha256-rJB7h3KuH1DPp5n4dY3MiGnV1Y96A10lf5OUl+MLkzU="
+  [mod."github.com/samber/lo"]
+    version = "v1.53.0"
+    hash = "sha256-RCf4Buf357TTWQnMPSWKrfdJ4L/RqOHNBD0g3+VpMw8="
+  [mod."github.com/samber/slog-common"]
+    version = "v0.21.0"
+    hash = "sha256-i9Nl4xRbk8qYM+0n48IQ6+vGZiS7xFe+GgyV3X9/Spc="
+  [mod."github.com/samber/slog-multi"]
+    version = "v1.8.0"
+    hash = "sha256-KsFwNP9QMDr8golYoevpGtcqUuCrIT7zmGwR7/E6gzo="
+  [mod."github.com/sergi/go-diff"]
+    version = "v1.4.0"
+    hash = "sha256-rs9NKpv/qcQEMRg7CmxGdP4HGuFdBxlpWf9LbA9wS4k="
+  [mod."github.com/stretchr/testify"]
+    version = "v1.11.1"
+    hash = "sha256-sWfjkuKJyDllDEtnM8sb/pdLzPQmUYWYtmeWz/5suUc="
+  [mod."github.com/tinylib/msgp"]
+    version = "v1.6.1"
+    hash = "sha256-R2LutHQFZ7HAqeyzHqzMeyAJHxcYc+n1x7ysyrXefmQ="
+  [mod."github.com/tj/assert"]
+    version = "v0.0.3"
+    hash = "sha256-4xhmZcHpUafabaXejE9ucVnGxG/txomvKzBg6cbkusg="
+  [mod."github.com/tj/go-redirects"]
+    version = "v0.0.0-20200911105812-fd1ba1020b37"
+    hash = "sha256-GpYpxdT4F7PkwGXLo7cYVcIRJrzd1sKHtFDH+bRb6Tk="
+  [mod."github.com/valyala/bytebufferpool"]
+    version = "v1.0.0"
+    hash = "sha256-I9FPZ3kCNRB+o0dpMwBnwZ35Fj9+ThvITn8a3Jr8mAY="
+  [mod."github.com/valyala/fasttemplate"]
+    version = "v1.2.2"
+    hash = "sha256-gp+lNXE8zjO+qJDM/YbS6V43HFsYP6PKn4ux1qa5lZ0="
+  [mod."go.yaml.in/yaml/v2"]
+    version = "v2.4.2"
+    hash = "sha256-oC8RWdf1zbMYCtmR0ATy/kCkhIwPR9UqFZSMOKLVF/A="
+  [mod."go.yaml.in/yaml/v3"]
+    version = "v3.0.4"
+    hash = "sha256-NkGFiDPoCxbr3LFsI6OCygjjkY0rdmg5ggvVVwpyDQ4="
+  [mod."golang.org/x/crypto"]
+    version = "v0.50.0"
+    hash = "sha256-vC1BJT7+3UBWLyEE5n3to0NKhMo6m2HGow2HiFgpQLo="
+  [mod."golang.org/x/net"]
+    version = "v0.53.0"
+    hash = "sha256-G9gKLmyaf6lIV429NKX+YlL6oUPJwlv+BrG6qGhzvmU="
+  [mod."golang.org/x/sync"]
+    version = "v0.20.0"
+    hash = "sha256-ybcjhCfK6lroUM0yswUvWooW8MOQZBXyiSqoxG6Uy0Y="
+  [mod."golang.org/x/sys"]
+    version = "v0.43.0"
+    hash = "sha256-aDQXqSTZES2l/132PBxhZN4ywldpPyfm7LByYCHzzwM="
+  [mod."golang.org/x/text"]
+    version = "v0.36.0"
+    hash = "sha256-/0t9C6Mc8kYjxweFB0us2lGKo8GovHhBiq5nlMOppC0="
+  [mod."google.golang.org/protobuf"]
+    version = "v1.36.11"
+    hash = "sha256-7W+6jntfI/awWL3JP6yQedxqP5S9o3XvPgJ2XxxsIeE="
+  [mod."gopkg.in/yaml.v3"]
+    version = "v3.0.1"
+    hash = "sha256-FqL9TKYJ0XkNwJFnq9j0VvJ5ZUU1RvH/52h/f5bkYAU="

main.go

@@ -2,6 +2,27 @@
 
 package main
 
-import gitpages "codeberg.org/git-pages/git-pages/src"
+import (
+	"runtime/debug"
 
-func main() { gitpages.Main() }
+	git_pages "codeberg.org/git-pages/git-pages/src"
+)
+
+// By default the version information is retrieved from VCS. If not available during build,
+// override this variable using linker flags to change the displayed version.
+// Example: `-ldflags "-X main.versionOverride=v1.2.3"`
+var versionOverride = ""
+
+func extractVersion() string {
+	if versionOverride != "" {
+		return versionOverride
+	} else if buildInfo, ok := debug.ReadBuildInfo(); ok {
+		return buildInfo.Main.Version
+	} else {
+		panic("version information not available")
+	}
+}
+
+func main() {
+	git_pages.Main(extractVersion())
+}

renovate.json

@@ -14,5 +14,7 @@
   "lockFileMaintenance": {
     "enabled": true,
     "automerge": false
-  }
+  },
+  "semanticCommits": "disabled",
+  "commitMessagePrefix": "[Renovate]"
 }

src/audit.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -49,6 +50,8 @@ func GetPrincipal(ctx context.Context) *Principal {
 	return nil
 }
 
+var AuditSnowflakeStartTime = time.Date(2025, 12, 1, 0, 0, 0, 0, time.UTC)
+
 type AuditID int64
 
 func GenerateAuditID() AuditID {
@@ -73,6 +76,7 @@ func (id AuditID) String() string {
 
 func (id AuditID) CompareTime(when time.Time) int {
 	idMillis := int64(id) >> (snowflake.MachineIDLength + snowflake.SequenceLength)
+	idMillis += AuditSnowflakeStartTime.UnixMilli()
 	whenMillis := when.UTC().UnixNano() / 1e6
 	return cmp.Compare(idMillis, whenMillis)
 }
@@ -101,12 +105,21 @@ func (record *AuditRecord) DescribePrincipal() string {
 		if record.Principal.GetIpAddress() != "" {
 			items = append(items, record.Principal.GetIpAddress())
 		}
+		if record.Principal.GetForgeUser() != nil {
+			items = append(items, fmt.Sprintf("%s/%s(%d)",
+				record.Principal.GetForgeUser().GetOrigin(),
+				record.Principal.GetForgeUser().GetHandle(),
+				record.Principal.GetForgeUser().GetId()))
+		}
+		if record.Principal.GetRepoUrl() != "" {
+			items = append(items, record.Principal.GetRepoUrl())
+		}
 		if record.Principal.GetCliAdmin() {
 			items = append(items, "<cli-admin>")
 		}
 	}
 	if len(items) > 0 {
-		return strings.Join(items, ";")
+		return strings.Join(items, ",")
 	} else {
 		return "<unknown>"
 	}
@@ -115,13 +128,21 @@ func (record *AuditRecord) DescribePrincipal() string {
 func (record *AuditRecord) DescribeResource() string {
 	desc := "<unknown>"
 	if record.Domain != nil && record.Project != nil {
-		desc = fmt.Sprintf("%s/%s", *record.Domain, *record.Project)
+		desc = path.Join(*record.Domain, *record.Project)
 	} else if record.Domain != nil {
 		desc = *record.Domain
 	}
 	return desc
 }
 
+func (record *AuditRecord) IsDetachable() bool {
+	return record.GetEvent() == AuditEvent_CommitManifest
+}
+
+func (record *AuditRecord) IsDetached() bool {
+	return record.IsDetachable() && record.Manifest == nil
+}
+
 type AuditRecordScope int
 
 const (
@@ -264,11 +285,7 @@ type auditedBackend struct {
 var _ Backend = (*auditedBackend)(nil)
 
 func NewAuditedBackend(backend Backend) Backend {
-	if config.Feature("audit") {
-		return &auditedBackend{backend}
-	} else {
-		return backend
-	}
+	return &auditedBackend{backend}
 }
 
 // This function does not retry appending audit records; as such, if it returns an error,
@@ -292,7 +309,7 @@ func (audited *auditedBackend) appendNewAuditRecord(ctx context.Context, record
 			if record.Project == nil {
 				subject = *record.Domain
 			} else {
-				subject = fmt.Sprintf("%s/%s", *record.Domain, *record.Project)
+				subject = path.Join(*record.Domain, *record.Project)
 			}
 			logc.Printf(ctx, "audit %s ok: %s %s\n", subject, id, record.Event.String())
 
@@ -377,17 +394,20 @@ func (audited *auditedBackend) DeleteManifest(
 	return audited.Backend.DeleteManifest(ctx, name, opts)
 }
 
-func (audited *auditedBackend) FreezeDomain(ctx context.Context, domain string, freeze bool) (err error) {
-	var event AuditEvent
-	if freeze {
-		event = AuditEvent_FreezeDomain
-	} else {
-		event = AuditEvent_UnfreezeDomain
-	}
+func (audited *auditedBackend) FreezeDomain(ctx context.Context, domain string) (err error) {
 	audited.appendNewAuditRecord(ctx, &AuditRecord{
-		Event:  event.Enum(),
+		Event:  AuditEvent_FreezeDomain.Enum(),
 		Domain: proto.String(domain),
 	})
 
-	return audited.Backend.FreezeDomain(ctx, domain, freeze)
+	return audited.Backend.FreezeDomain(ctx, domain)
+}
+
+func (audited *auditedBackend) UnfreezeDomain(ctx context.Context, domain string) (err error) {
+	audited.appendNewAuditRecord(ctx, &AuditRecord{
+		Event:  AuditEvent_UnfreezeDomain.Enum(),
+		Domain: proto.String(domain),
+	})
+
+	return audited.Backend.UnfreezeDomain(ctx, domain)
 }

src/auth.go

@@ -12,6 +12,8 @@ import (
 	"slices"
 	"strings"
 	"time"
+
+	"golang.org/x/net/idna"
 )
 
 type AuthError struct {
@@ -42,27 +44,59 @@ func authorizeInsecure(r *http.Request) *Authorization {
 	return nil
 }
 
+var idnaProfile = idna.New(idna.MapForLookup(), idna.BidiRule())
+
 func GetHost(r *http.Request) (string, error) {
-	// FIXME: handle IDNA
 	host, _, err := net.SplitHostPort(r.Host)
 	if err != nil {
-		// dirty but the go stdlib doesn't have a "split port if present" function
 		host = r.Host
 	}
-	if strings.HasPrefix(host, ".") {
+	// this also rejects invalid characters and labels
+	host, err = idnaProfile.ToASCII(host)
+	if err != nil {
+		if config.Feature("relaxed-idna") {
+			// unfortunately, the go IDNA library has some significant issues around its
+			// Unicode TR46 implementation: https://github.com/golang/go/issues/76804
+			// we would like to allow *just* the _ here, but adding `idna.StrictDomainName(false)`
+			// would also accept domains like `*.foo.bar` which should clearly be disallowed.
+			// as a workaround, accept a domain name if it is valid with all `_` characters
+			// replaced with an alphanumeric character (we use `a`); this allows e.g. `foo_bar.xxx`
+			// and `foo__bar.xxx`, as well as `_foo.xxx` and `foo_.xxx`. labels starting with
+			// an underscore are explicitly rejected below.
+			_, err = idnaProfile.ToASCII(strings.ReplaceAll(host, "_", "a"))
+		}
+		if err != nil {
+			return "", AuthError{http.StatusBadRequest,
+				fmt.Sprintf("malformed host name %q", host)}
+		}
+	}
+	if strings.HasPrefix(host, ".") || strings.HasPrefix(host, "_") {
 		return "", AuthError{http.StatusBadRequest,
-			fmt.Sprintf("host name %q is reserved", host)}
+			fmt.Sprintf("reserved host name %q", host)}
 	}
 	host = strings.TrimSuffix(host, ".")
 	return host, nil
 }
 
+func ValidateProjectName(name string) error {
+	if strings.HasPrefix(name, ".") {
+		return fmt.Errorf("must not start with %q", ".")
+	}
+
+	forbiddenChars := "%*"
+	if strings.ContainsAny(name, forbiddenChars) {
+		return fmt.Errorf("must not contain any of %q", forbiddenChars)
+	}
+
+	return nil
+}
+
 func GetProjectName(r *http.Request) (string, error) {
 	// path must be either `/` or `/foo/` (`/foo` is accepted as an alias)
 	path := strings.TrimPrefix(strings.TrimSuffix(r.URL.Path, "/"), "/")
-	if path == ".index" || strings.HasPrefix(path, ".index/") {
+	if err := ValidateProjectName(path); err != nil {
 		return "", AuthError{http.StatusBadRequest,
-			fmt.Sprintf("directory name %q is reserved", ".index")}
+			fmt.Sprintf("directory name: %v", err)}
 	} else if strings.Contains(path, "/") {
 		return "", AuthError{http.StatusBadRequest,
 			"directories nested too deep"}
@@ -81,6 +115,15 @@ type Authorization struct {
 	repoURLs []string
 	// Only the exact branch is allowed.
 	branch string
+	// The authorized forge user.
+	forgeUser *ForgeUser
+}
+
+func (auth *Authorization) ForgeRepoURL() string {
+	if auth.forgeUser != nil && len(auth.repoURLs) == 1 {
+		return auth.repoURLs[0]
+	}
+	return ""
 }
 
 func authorizeDNSChallenge(r *http.Request) (*Authorization, error) {
@@ -152,7 +195,7 @@ func authorizeDNSChallenge(r *http.Request) (*Authorization, error) {
 	}, nil
 }
 
-func authorizeDNSAllowlist(r *http.Request) (*Authorization, error) {
+func authorizeDNSAllowlist(r *http.Request, scope string) (*Authorization, error) {
 	host, err := GetHost(r)
 	if err != nil {
 		return nil, err
@@ -163,7 +206,7 @@ func authorizeDNSAllowlist(r *http.Request) (*Authorization, error) {
 		return nil, err
 	}
 
-	allowlistHostname := fmt.Sprintf("_git-pages-repository.%s", host)
+	allowlistHostname := fmt.Sprintf("_%s.%s", scope, host)
 	records, err := net.LookupTXT(allowlistHostname)
 	if err != nil {
 		return nil, AuthError{http.StatusUnauthorized,
@@ -240,8 +283,8 @@ func authorizeWildcardMatchSite(r *http.Request, pattern *WildcardPattern) (*Aut
 	}
 
 	if userName, found := pattern.Matches(host); found {
-		repoURLs, branch := pattern.ApplyTemplate(userName, projectName)
-		return &Authorization{repoURLs, branch}, nil
+		repoURL, branch := pattern.ApplyTemplate(userName, projectName)
+		return &Authorization{repoURLs: []string{repoURL}, branch: branch}, nil
 	} else {
 		return nil, AuthError{
 			http.StatusUnauthorized,
@@ -288,17 +331,16 @@ func authorizeCodebergPagesV2(r *http.Request) (*Authorization, error) {
 			if domainParts[0] == "page" && domainParts[1] == "codeberg" {
 				// map of domain names to allowed repository and branch:
 				//  * {username}.codeberg.page =>
-				//      https://codeberg.org/{username}/pages.git#main
+				//      https://codeberg.org/{username}/pages.git#pages
 				//  * {reponame}.{username}.codeberg.page =>
 				//      https://codeberg.org/{username}/{reponame}.git#pages
 				//  * {branch}.{reponame}.{username}.codeberg.page =>
 				//      https://codeberg.org/{username}/{reponame}.git#{branch}
 				username := domainParts[2]
 				reponame := "pages"
-				branch := "main"
+				branch := "pages"
 				if len(domainParts) >= 4 {
 					reponame = domainParts[3]
-					branch = "pages"
 				}
 				if len(domainParts) == 5 {
 					branch = domainParts[4]
@@ -320,7 +362,7 @@ func authorizeCodebergPagesV2(r *http.Request) (*Authorization, error) {
 }
 
 // Checks whether an operation that enables enumerating site contents is allowed.
-func AuthorizeMetadataRetrieval(r *http.Request) (*Authorization, error) {
+func AuthorizeMetadataRetrieval(r *http.Request, hasBasicAuth bool) (*Authorization, error) {
 	causes := []error{AuthError{http.StatusUnauthorized, "unauthorized"}}
 
 	auth := authorizeInsecure(r)
@@ -338,36 +380,38 @@ func AuthorizeMetadataRetrieval(r *http.Request) (*Authorization, error) {
 		return auth, nil
 	}
 
-	for _, pattern := range wildcards {
-		auth, err = authorizeWildcardMatchHost(r, pattern)
-		if err != nil && IsUnauthorized(err) {
-			causes = append(causes, err)
-		} else if err != nil { // bad request
-			return nil, err
-		} else {
-			logc.Printf(r.Context(), "auth: wildcard %s\n", pattern.GetHost())
-			return auth, nil
+	// Normally, sites that correspond to a forge via a wildcard match are considered completely
+	// public and safe to retrieve without authorization. However, this is no longer the case if
+	// they have password-protected sections.
+	if !hasBasicAuth {
+		for _, pattern := range wildcards {
+			auth, err = authorizeWildcardMatchHost(r, pattern)
+			if err != nil && IsUnauthorized(err) {
+				causes = append(causes, err)
+			} else if err != nil { // bad request
+				return nil, err
+			} else {
+				logc.Printf(r.Context(), "auth: wildcard %s\n", pattern.GetHost())
+				return auth, nil
+			}
 		}
-	}
 
-	if config.Feature("codeberg-pages-compat") {
-		auth, err = authorizeCodebergPagesV2(r)
-		if err != nil && IsUnauthorized(err) {
-			causes = append(causes, err)
-		} else if err != nil { // bad request
-			return nil, err
-		} else {
-			logc.Printf(r.Context(), "auth: codeberg %s\n", r.Host)
-			return auth, nil
+		if config.Feature("codeberg-pages-compat") {
+			auth, err = authorizeCodebergPagesV2(r)
+			if err != nil && IsUnauthorized(err) {
+				causes = append(causes, err)
+			} else if err != nil { // bad request
+				return nil, err
+			} else {
+				logc.Printf(r.Context(), "auth: codeberg %s\n", r.Host)
+				return auth, nil
+			}
 		}
 	}
 
 	return nil, joinErrors(causes...)
 }
 
-// Returns `repoURLs, err` where if `err == nil` then the request is authorized to clone from
-// any repository URL included in `repoURLs` (by case-insensitive comparison), or any URL at all
-// if `repoURLs == nil`.
 func AuthorizeUpdateFromRepository(r *http.Request) (*Authorization, error) {
 	causes := []error{AuthError{http.StatusUnauthorized, "unauthorized"}}
 
@@ -393,7 +437,7 @@ func AuthorizeUpdateFromRepository(r *http.Request) (*Authorization, error) {
 
 	// DNS allowlist gives authority to update but not delete.
 	if r.Method == http.MethodPut || r.Method == http.MethodPost {
-		auth, err = authorizeDNSAllowlist(r)
+		auth, err = authorizeDNSAllowlist(r, "git-pages-repository")
 		if err != nil && IsUnauthorized(err) {
 			causes = append(causes, err)
 		} else if err != nil { // bad request
@@ -435,21 +479,23 @@ func AuthorizeUpdateFromRepository(r *http.Request) (*Authorization, error) {
 	return nil, joinErrors(causes...)
 }
 
-func checkAllowedURLPrefix(repoURL string) error {
-	if config.Limits.AllowedRepositoryURLPrefixes != nil {
-		allowedPrefix := false
-		repoURL = strings.ToLower(repoURL)
-		for _, allowedRepoURLPrefix := range config.Limits.AllowedRepositoryURLPrefixes {
-			if strings.HasPrefix(repoURL, strings.ToLower(allowedRepoURLPrefix)) {
-				allowedPrefix = true
-				break
+func checkAllowedURLPrefixes(repoURLs ...string) error {
+	if len(config.Limits.AllowedRepositoryURLPrefixes) > 0 {
+		for _, repoURL := range repoURLs {
+			allowedPrefix := false
+			repoURL = strings.ToLower(repoURL)
+			for _, allowedRepoURLPrefix := range config.Limits.AllowedRepositoryURLPrefixes {
+				if strings.HasPrefix(repoURL, strings.ToLower(allowedRepoURLPrefix)) {
+					allowedPrefix = true
+					break
+				}
 			}
-		}
-		if !allowedPrefix {
-			return AuthError{
-				http.StatusUnauthorized,
-				fmt.Sprintf("clone URL not in prefix allowlist %v",
-					config.Limits.AllowedRepositoryURLPrefixes),
+			if !allowedPrefix {
+				return AuthError{
+					http.StatusUnauthorized,
+					fmt.Sprintf("clone URL %v not in prefix allowlist %v",
+						repoURL, config.Limits.AllowedRepositoryURLPrefixes),
+				}
 			}
 		}
 	}
@@ -482,7 +528,7 @@ func AuthorizeRepository(repoURL string, auth *Authorization) error {
 		return nil // any
 	}
 
-	if err = checkAllowedURLPrefix(repoURL); err != nil {
+	if err = checkAllowedURLPrefixes(repoURL); err != nil {
 		return err
 	}
 
@@ -550,6 +596,11 @@ func checkGogsRepositoryPushPermission(baseURL *url.URL, authorization string) e
 			http.StatusNotFound,
 			fmt.Sprintf("no repository %s", ownerAndRepo),
 		}
+	} else if response.StatusCode == http.StatusUnauthorized {
+		return AuthError{
+			http.StatusUnauthorized,
+			fmt.Sprintf("no access to %s or invalid token", ownerAndRepo),
+		}
 	} else if response.StatusCode != http.StatusOK {
 		return AuthError{
 			http.StatusServiceUnavailable,
@@ -584,9 +635,89 @@ func checkGogsRepositoryPushPermission(baseURL *url.URL, authorization string) e
 	return nil
 }
 
-func authorizeForgeWithToken(r *http.Request) (*Authorization, error) {
-	authorization := r.Header.Get("Forge-Authorization")
-	if authorization == "" {
+// Gogs, Gitea, and Forgejo all support the same API here.
+func fetchGogsAuthorizedUser(baseURL *url.URL, forgeToken string) (*ForgeUser, error) {
+	request, err := http.NewRequest("GET", baseURL.ResolveReference(&url.URL{
+		Path: "/api/v1/user",
+	}).String(), nil)
+	if err != nil {
+		panic(err) // misconfiguration
+	}
+	request.Header.Set("Accept", "application/json")
+	request.Header.Set("Authorization", forgeToken)
+
+	httpClient := http.Client{Timeout: 5 * time.Second}
+	response, err := httpClient.Do(request)
+	if err != nil {
+		return nil, AuthError{
+			http.StatusServiceUnavailable,
+			fmt.Sprintf("cannot fetch authorized forge user: %s", err),
+		}
+	}
+	defer response.Body.Close()
+
+	if response.StatusCode != http.StatusOK {
+		return nil, AuthError{
+			http.StatusServiceUnavailable,
+			fmt.Sprintf(
+				"cannot fetch authorized forge user: GET %s returned %s",
+				request.URL,
+				response.Status,
+			),
+		}
+	}
+	decoder := json.NewDecoder(response.Body)
+
+	var userInfo struct {
+		ID    int64
+		Login string
+	}
+	if err = decoder.Decode(&userInfo); err != nil {
+		return nil, errors.Join(AuthError{
+			http.StatusServiceUnavailable,
+			fmt.Sprintf(
+				"cannot fetch authorized forge user: GET %s returned malformed JSON",
+				request.URL,
+			),
+		}, err)
+	}
+
+	origin := request.URL.Hostname()
+	return &ForgeUser{
+		Origin: &origin,
+		Id:     &userInfo.ID,
+		Handle: &userInfo.Login,
+	}, nil
+}
+
+// Check whether a forge token has access to a repository, and if it does, which user it
+// belongs to. Precondition: `repoURL` is well-formed.
+func authorizeGogsUser(repoURL string, forgeToken string) (*Authorization, error) {
+	parsedRepoURL, err := url.Parse(repoURL)
+	if err != nil {
+		panic(err)
+	}
+
+	if err = checkGogsRepositoryPushPermission(parsedRepoURL, forgeToken); err != nil {
+		return nil, err
+	}
+
+	authorizedUser, err := fetchGogsAuthorizedUser(parsedRepoURL, forgeToken)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Authorization{
+		repoURLs:  []string{repoURL},
+		forgeUser: authorizedUser,
+	}, nil
+}
+
+// Validates a provided forge token against a repository URL constructed by mapping the host
+// and project name via the `[[wildcard]]` section of the configuration file.
+func authorizeForgeWildcard(r *http.Request) (*Authorization, error) {
+	forgeToken := r.Header.Get("Forge-Authorization")
+	if forgeToken == "" {
 		return nil, AuthError{http.StatusUnauthorized, "missing Forge-Authorization header"}
 	}
 
@@ -602,40 +733,61 @@ func authorizeForgeWithToken(r *http.Request) (*Authorization, error) {
 
 	var errs []error
 	for _, pattern := range wildcards {
-		if !pattern.Authorization {
-			continue
-		}
-
-		if userName, found := pattern.Matches(host); found {
-			repoURLs, branch := pattern.ApplyTemplate(userName, projectName)
-			for _, repoURL := range repoURLs {
-				parsedRepoURL, err := url.Parse(repoURL)
+		if pattern.Authorization {
+			if userName, found := pattern.Matches(host); found {
+				repoURL, branch := pattern.ApplyTemplate(userName, projectName)
+				auth, err := authorizeGogsUser(repoURL, forgeToken)
 				if err != nil {
-					panic(err) // misconfiguration
-				}
-
-				if err = checkGogsRepositoryPushPermission(parsedRepoURL, authorization); err != nil {
 					errs = append(errs, err)
-					continue
+				} else {
+					auth.branch = branch
+					return auth, nil
 				}
+			}
+		}
+	}
+	if len(errs) == 0 {
+		errs = append(errs, AuthError{http.StatusUnauthorized, "no matching wildcard domain"})
+	}
 
-				// This will actually be ignored by the caller of AuthorizeUpdateFromArchive,
-				// but we return this information as it makes sense to do contextually here.
-				return &Authorization{
-					[]string{repoURL},
-					branch,
-				}, nil
+	errs = append([]error{
+		AuthError{http.StatusUnauthorized, "not authorized by forge (wildcard)"},
+	}, errs...)
+	return nil, joinErrors(errs...)
+}
+
+// Validates a provided forge token against a repository URL extracted from the DNS allowlist
+// records of the target domain specified in `_git-pages-forge-authorization.*`.
+func authorizeForgeDNSAllowlist(r *http.Request) (*Authorization, error) {
+	forgeToken := r.Header.Get("Forge-Authorization")
+	if forgeToken == "" {
+		return nil, AuthError{http.StatusUnauthorized, "missing Forge-Authorization header"}
+	}
+
+	var errs []error
+	if dnsAuth, err := authorizeDNSAllowlist(r, "git-pages-forge-allowlist"); err != nil {
+		errs = append(errs, err)
+	} else if dnsAuth != nil {
+		// DNS allows uploads from some repositories, but we don't know yet if the forge token
+		// has a push permission to any of these repositories.
+		for _, repoURL := range dnsAuth.repoURLs {
+			auth, err := authorizeGogsUser(repoURL, forgeToken)
+			if err != nil {
+				errs = append(errs, err)
+			} else {
+				// There is both DNS authorization and forge authorization.
+				return auth, nil
 			}
 		}
 	}
 
 	errs = append([]error{
-		AuthError{http.StatusUnauthorized, "not authorized by forge"},
+		AuthError{http.StatusUnauthorized, "not authorized by forge (DNS allowlist)"},
 	}, errs...)
 	return nil, joinErrors(errs...)
 }
 
-func AuthorizeUpdateFromArchive(r *http.Request) (*Authorization, error) {
+func authorizeDNSChallengeOrForgeWithToken(r *http.Request) (*Authorization, error) {
 	causes := []error{AuthError{http.StatusUnauthorized, "unauthorized"}}
 
 	if err := CheckForbiddenDomain(r); err != nil {
@@ -647,35 +799,70 @@ func AuthorizeUpdateFromArchive(r *http.Request) (*Authorization, error) {
 		return auth, nil
 	}
 
-	// Token authorization allows updating a site on a wildcard domain from an archive.
-	auth, err := authorizeForgeWithToken(r)
+	// DNS challenge gives absolute authority.
+	auth, err := authorizeDNSChallenge(r)
 	if err != nil && IsUnauthorized(err) {
 		causes = append(causes, err)
 	} else if err != nil { // bad request
 		return nil, err
 	} else {
-		logc.Printf(r.Context(), "auth: forge token: allow\n")
+		logc.Println(r.Context(), "auth: DNS challenge: allow *")
 		return auth, nil
 	}
 
-	if config.Limits.AllowedRepositoryURLPrefixes != nil {
-		causes = append(causes, AuthError{http.StatusUnauthorized, "DNS challenge not allowed"})
+	// Token authorization allows updating a site on a wildcard domain from an archive.
+	// This sub-method uses the `[[wildcard]]` configuration section to derive repository URL.
+	auth, err = authorizeForgeWildcard(r)
+	if err != nil && IsUnauthorized(err) {
+		causes = append(causes, err)
+	} else if err != nil { // bad request
+		return nil, err
 	} else {
-		// DNS challenge gives absolute authority.
-		auth, err = authorizeDNSChallenge(r)
-		if err != nil && IsUnauthorized(err) {
-			causes = append(causes, err)
-		} else if err != nil { // bad request
-			return nil, err
-		} else {
-			logc.Println(r.Context(), "auth: DNS challenge")
-			return auth, nil
-		}
+		logc.Printf(r.Context(), "auth: forge (wildcard): allow\n")
+		return auth, nil
+	}
+
+	// Token authorization allows updating a site on a wildcard domain from an archive.
+	// This sub-method uses the DNS allowlist authorization mechanism to derive repository URL.
+	auth, err = authorizeForgeDNSAllowlist(r)
+	if err != nil && IsUnauthorized(err) {
+		causes = append(causes, err)
+	} else if err != nil { // bad request
+		return nil, err
+	} else {
+		logc.Printf(r.Context(), "auth: forge (DNS allowlist): allow\n")
+		return auth, nil
 	}
 
 	return nil, joinErrors(causes...)
 }
 
+func AuthorizeUpdateFromArchive(r *http.Request) (*Authorization, error) {
+	auth, err := authorizeDNSChallengeOrForgeWithToken(r)
+	if err != nil {
+		return nil, err
+	}
+
+	// If only uploads from specific repositories are allowed, then only forge authorization
+	// is acceptable, and the repository must match the configured limits.
+	if len(config.Limits.AllowedRepositoryURLPrefixes) > 0 {
+		if len(auth.repoURLs) == 0 {
+			logc.Println(r.Context(), "auth: DNS challenge: deny (limits)")
+			return nil, AuthError{http.StatusUnauthorized, "DNS challenge not allowed"}
+		}
+
+		if err = checkAllowedURLPrefixes(auth.repoURLs...); err != nil {
+			return nil, err
+		}
+	}
+
+	return auth, nil
+}
+
+func AuthorizeDeletion(r *http.Request) (*Authorization, error) {
+	return authorizeDNSChallengeOrForgeWithToken(r)
+}
+
 func CheckForbiddenDomain(r *http.Request) error {
 	host, err := GetHost(r)
 	if err != nil {

src/backend.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"io"
 	"iter"
-	"slices"
 	"strings"
 	"time"
 )
@@ -17,20 +16,29 @@ var ErrWriteConflict = errors.New("write conflict")
 var ErrDomainFrozen = errors.New("domain administratively frozen")
 
 func splitBlobName(name string) []string {
-	algo, hash, found := strings.Cut(name, "-")
-	if found {
-		return slices.Concat([]string{algo}, splitBlobName(hash))
+	if algo, hash, found := strings.Cut(name, "-"); found {
+		return []string{algo, hash[0:2], hash[2:4], hash[4:]}
 	} else {
-		return []string{name[0:2], name[2:4], name[4:]}
+		panic("malformed blob name")
 	}
 }
 
+func joinBlobName(parts []string) string {
+	return fmt.Sprintf("%s-%s", parts[0], strings.Join(parts[1:], ""))
+}
+
 type BackendFeature string
 
 const (
 	FeatureCheckDomainMarker BackendFeature = "check-domain-marker"
 )
 
+type BlobMetadata struct {
+	Name         string
+	Size         int64
+	LastModified time.Time
+}
+
 type GetManifestOptions struct {
 	// If true and the manifest is past the cache `MaxAge`, `GetManifest` blocks and returns
 	// a fresh object instead of revalidating in background and returning a stale object.
@@ -38,6 +46,8 @@ type GetManifestOptions struct {
 }
 
 type ManifestMetadata struct {
+	Name         string
+	Size         int64
 	LastModified time.Time
 	ETag         string
 }
@@ -77,7 +87,7 @@ type Backend interface {
 
 	// Retrieve a blob. Returns `reader, size, mtime, err`.
 	GetBlob(ctx context.Context, name string) (
-		reader io.ReadSeeker, size uint64, mtime time.Time, err error,
+		reader io.ReadSeeker, metadata BlobMetadata, err error,
 	)
 
 	// Store a blob. If a blob called `name` already exists, this function returns `nil` without
@@ -88,6 +98,10 @@ type Backend interface {
 	// Delete a blob. This is an unconditional operation that can break integrity of manifests.
 	DeleteBlob(ctx context.Context, name string) error
 
+	// Iterate through all blobs. Whether blobs that are newly added during iteration will appear
+	// in the results is unspecified.
+	EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error]
+
 	// Retrieve a manifest.
 	GetManifest(ctx context.Context, name string, opts GetManifestOptions) (
 		manifest *Manifest, metadata ManifestMetadata, err error,
@@ -110,8 +124,13 @@ type Backend interface {
 	// Delete a manifest.
 	DeleteManifest(ctx context.Context, name string, opts ModifyManifestOptions) error
 
-	// List all manifests.
-	ListManifests(ctx context.Context) (manifests []string, err error)
+	// Iterate through metadata of all manifests. Whether manifests that are newly added during
+	// iteration will appear in the results is unspecified.
+	EnumerateManifests(ctx context.Context) iter.Seq2[*ManifestMetadata, error]
+
+	// Iterate through contents of all manifests. Same considerations apply as for
+	// `EnumerateManifests`.
+	GetAllManifests(ctx context.Context) iter.Seq2[tuple[*ManifestMetadata, *Manifest], error]
 
 	// Check whether a domain has any deployments.
 	CheckDomain(ctx context.Context, domain string) (found bool, err error)
@@ -119,9 +138,15 @@ type Backend interface {
 	// Create a domain. This allows us to start serving content for the domain.
 	CreateDomain(ctx context.Context, domain string) error
 
-	// Freeze or thaw a domain. This allows a site to be administratively locked, e.g. if it
+	// Freeze a domain. This allows a site to be administratively locked, e.g. if it
 	// is discovered serving abusive content.
-	FreezeDomain(ctx context.Context, domain string, freeze bool) error
+	FreezeDomain(ctx context.Context, domain string) error
+
+	// Thaw a domain. This removes the previously placed administrative lock (if any).
+	UnfreezeDomain(ctx context.Context, domain string) error
+
+	// Check whether the set of domains we serve has changed since the time passed to this method.
+	HaveDomainsChanged(ctx context.Context, since time.Time) (changed bool, err error)
 
 	// Append a record to the audit log.
 	AppendAuditLog(ctx context.Context, id AuditID, record *AuditRecord) error
@@ -129,8 +154,17 @@ type Backend interface {
 	// Retrieve a single record from the audit log.
 	QueryAuditLog(ctx context.Context, id AuditID) (record *AuditRecord, err error)
 
-	// Retrieve records from the audit log by time range.
+	// Retrieve record IDs from the audit log by time range.
 	SearchAuditLog(ctx context.Context, opts SearchAuditLogOptions) iter.Seq2[AuditID, error]
+
+	// Retrieve audit record contents for given IDs.
+	GetAuditLogRecords(ctx context.Context, ids iter.Seq2[AuditID, error]) iter.Seq2[*AuditRecord, error]
+
+	// Detach an audit record from its blobs.
+	DetachAuditRecord(ctx context.Context, id AuditID) error
+
+	// Delete an audit record with a given ID.
+	ExpireAuditRecord(ctx context.Context, id AuditID) error
 }
 
 func CreateBackend(ctx context.Context, config *StorageConfig) (backend Backend, err error) {

src/backend_fs.go

@@ -118,7 +118,7 @@ func (fs *FSBackend) EnableFeature(ctx context.Context, feature BackendFeature)
 func (fs *FSBackend) GetBlob(
 	ctx context.Context, name string,
 ) (
-	reader io.ReadSeeker, size uint64, mtime time.Time, err error,
+	reader io.ReadSeeker, metadata BlobMetadata, err error,
 ) {
 	blobPath := filepath.Join(splitBlobName(name)...)
 	stat, err := fs.blobRoot.Stat(blobPath)
@@ -134,7 +134,7 @@ func (fs *FSBackend) GetBlob(
 		err = fmt.Errorf("open: %w", err)
 		return
 	}
-	return file, uint64(stat.Size()), stat.ModTime(), nil
+	return file, BlobMetadata{name, int64(stat.Size()), stat.ModTime()}, nil
 }
 
 func (fs *FSBackend) PutBlob(ctx context.Context, name string, data []byte) error {
@@ -153,7 +153,13 @@ func (fs *FSBackend) PutBlob(ctx context.Context, name string, data []byte) erro
 	}
 
 	if err := fs.blobRoot.Chmod(tempPath, 0o444); err != nil {
-		return fmt.Errorf("chmod: %w", err)
+		if errors.Is(err, os.ErrPermission) {
+			// NFSv4 configured with ACLs doesn't have a working `chmod` even though it's a Unix
+			// system. This `chmod` call is done entirely for convenience (to help the system
+			// administrator avoid accidentally overwriting files), so just skip it.
+		} else {
+			return fmt.Errorf("chmod: %w", err)
+		}
 	}
 
 again:
@@ -182,20 +188,30 @@ func (fs *FSBackend) DeleteBlob(ctx context.Context, name string) error {
 	return fs.blobRoot.Remove(blobPath)
 }
 
-func (fs *FSBackend) ListManifests(ctx context.Context) (manifests []string, err error) {
-	err = iofs.WalkDir(fs.siteRoot.FS(), ".",
-		func(path string, entry iofs.DirEntry, err error) error {
-			if strings.Count(path, "/") > 1 {
-				return iofs.SkipDir
-			}
-			_, project, _ := strings.Cut(path, "/")
-			if project == "" || strings.HasPrefix(project, ".") && project != ".index" {
+func (fs *FSBackend) EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error] {
+	return func(yield func(BlobMetadata, error) bool) {
+		iofs.WalkDir(fs.blobRoot.FS(), ".",
+			func(path string, entry iofs.DirEntry, err error) error {
+				var metadata BlobMetadata
+				if err != nil {
+					// report error
+				} else if entry.IsDir() {
+					// skip directory
+					return nil
+				} else if info, err := entry.Info(); err != nil {
+					// report error
+				} else {
+					// report blob
+					metadata.Name = joinBlobName(strings.Split(path, "/"))
+					metadata.Size = info.Size()
+					metadata.LastModified = info.ModTime()
+				}
+				if !yield(metadata, err) {
+					return iofs.SkipAll
+				}
 				return nil
-			}
-			manifests = append(manifests, path)
-			return nil
-		})
-	return
+			})
+	}
 }
 
 func (fs *FSBackend) GetManifest(
@@ -387,6 +403,55 @@ func (fs *FSBackend) DeleteManifest(
 	}
 }
 
+func (fs *FSBackend) EnumerateManifests(ctx context.Context) iter.Seq2[*ManifestMetadata, error] {
+	return func(yield func(*ManifestMetadata, error) bool) {
+		iofs.WalkDir(fs.siteRoot.FS(), ".",
+			func(path string, entry iofs.DirEntry, err error) error {
+				_, project, _ := strings.Cut(path, "/")
+				var metadata *ManifestMetadata
+				if err != nil {
+					// report error
+				} else if entry.IsDir() {
+					// skip directory
+					return nil
+				} else if project == "" || strings.HasPrefix(project, ".") && project != ".index" {
+					// skip internal
+					return nil
+				} else if info, err := entry.Info(); err != nil {
+					// report error
+				} else {
+					// report blob
+					metadata = &ManifestMetadata{
+						Name:         path,
+						Size:         info.Size(),
+						LastModified: info.ModTime(),
+					}
+					// not setting metadata.ETag since it is too costly
+				}
+				if !yield(metadata, err) {
+					return iofs.SkipAll
+				}
+				return nil
+			})
+	}
+}
+
+func (fs *FSBackend) GetAllManifests(ctx context.Context) iter.Seq2[tuple[*ManifestMetadata, *Manifest], error] {
+	return func(yield func(tuple[*ManifestMetadata, *Manifest], error) bool) {
+		for metadata, err := range fs.EnumerateManifests(ctx) {
+			var item tuple[*ManifestMetadata, *Manifest]
+			if err == nil {
+				var manifest *Manifest
+				manifest, _, err = backend.GetManifest(ctx, metadata.Name, GetManifestOptions{})
+				item = tuple[*ManifestMetadata, *Manifest]{metadata, manifest}
+			}
+			if !yield(item, err) {
+				break
+			}
+		}
+	}
+}
+
 func (fs *FSBackend) CheckDomain(ctx context.Context, domain string) (bool, error) {
 	_, err := fs.siteRoot.Stat(domain)
 	if errors.Is(err, os.ErrNotExist) {
@@ -402,25 +467,33 @@ func (fs *FSBackend) CreateDomain(ctx context.Context, domain string) error {
 	return nil // no-op
 }
 
-func (fs *FSBackend) FreezeDomain(ctx context.Context, domain string, freeze bool) error {
-	if freeze {
-		return fs.siteRoot.WriteFile(domainFrozenMarkerName(domain), []byte{}, 0o644)
+func (fs *FSBackend) FreezeDomain(ctx context.Context, domain string) error {
+	return fs.siteRoot.WriteFile(domainFrozenMarkerName(domain), []byte{}, 0o644)
+}
+
+func (fs *FSBackend) UnfreezeDomain(ctx context.Context, domain string) error {
+	err := fs.siteRoot.Remove(domainFrozenMarkerName(domain))
+	if errors.Is(err, os.ErrNotExist) {
+		return nil
 	} else {
-		err := fs.siteRoot.Remove(domainFrozenMarkerName(domain))
-		if errors.Is(err, os.ErrNotExist) {
-			return nil
-		} else {
-			return err
-		}
+		return err
 	}
 }
 
+func (fs *FSBackend) HaveDomainsChanged(ctx context.Context, since time.Time) (bool, error) {
+	return true, nil // not implemented
+}
+
+func auditDetachedName(id AuditID) string {
+	return fmt.Sprintf("%s.detached", id)
+}
+
 func (fs *FSBackend) AppendAuditLog(ctx context.Context, id AuditID, record *AuditRecord) error {
 	if _, err := fs.auditRoot.Stat(id.String()); err == nil {
 		panic(fmt.Errorf("audit ID collision: %s", id))
 	}
 
-	return fs.auditRoot.WriteFile(id.String(), EncodeAuditRecord(record), 0o644)
+	return fs.auditRoot.WriteFile(id.String(), EncodeAuditRecord(record), 0o444)
 }
 
 func (fs *FSBackend) QueryAuditLog(ctx context.Context, id AuditID) (*AuditRecord, error) {
@@ -429,6 +502,11 @@ func (fs *FSBackend) QueryAuditLog(ctx context.Context, id AuditID) (*AuditRecor
 	} else if record, err := DecodeAuditRecord(data); err != nil {
 		return nil, fmt.Errorf("decode: %w", err)
 	} else {
+		if _, err := fs.auditRoot.Stat(auditDetachedName(id)); err == nil {
+			record.Manifest = nil
+		} else if !errors.Is(err, os.ErrNotExist) {
+			return nil, fmt.Errorf("stat detached marker: %w", err)
+		}
 		return record, nil
 	}
 }
@@ -445,6 +523,8 @@ func (fs *FSBackend) SearchAuditLog(
 				var id AuditID
 				if err != nil {
 					// report error
+				} else if strings.Contains(path, ".") {
+					return nil // skip
 				} else if id, err = ParseAuditID(path); err != nil {
 					// report error
 				} else if !opts.Since.IsZero() && id.CompareTime(opts.Since) < 0 {
@@ -460,3 +540,27 @@ func (fs *FSBackend) SearchAuditLog(
 			})
 	}
 }
+
+func (fs *FSBackend) GetAuditLogRecords(
+	ctx context.Context, ids iter.Seq2[AuditID, error],
+) iter.Seq2[*AuditRecord, error] {
+	return func(yield func(*AuditRecord, error) bool) {
+		for id, err := range ids {
+			var record *AuditRecord
+			if err == nil {
+				record, err = fs.QueryAuditLog(ctx, id)
+			}
+			if !yield(record, err) {
+				break
+			}
+		}
+	}
+}
+
+func (fs *FSBackend) DetachAuditRecord(ctx context.Context, id AuditID) error {
+	return fs.auditRoot.WriteFile(auditDetachedName(id), []byte{}, 0o644)
+}
+
+func (fs *FSBackend) ExpireAuditRecord(ctx context.Context, id AuditID) error {
+	return fs.auditRoot.Remove(id.String())
+}

src/backend_s3.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"path"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/c2h5oh/datasize"
@@ -178,6 +179,12 @@ func NewS3Backend(ctx context.Context, config *S3Config) (*S3Backend, error) {
 		if err != nil {
 			return nil, err
 		}
+
+		err = (&S3Backend{client: client, bucket: bucket}).
+			EnableFeature(ctx, FeatureCheckDomainMarker)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	initS3BackendMetrics()
@@ -266,7 +273,7 @@ func (s3 *S3Backend) EnableFeature(ctx context.Context, feature BackendFeature)
 func (s3 *S3Backend) GetBlob(
 	ctx context.Context, name string,
 ) (
-	reader io.ReadSeeker, size uint64, mtime time.Time, err error,
+	reader io.ReadSeeker, metadata BlobMetadata, err error,
 ) {
 	loader := func(ctx context.Context, name string) (*CachedBlob, error) {
 		logc.Printf(ctx, "s3: get blob %s\n", name)
@@ -316,8 +323,9 @@ func (s3 *S3Backend) GetBlob(
 		}
 	} else {
 		reader = bytes.NewReader(cached.blob)
-		size = uint64(len(cached.blob))
-		mtime = cached.mtime
+		metadata.Name = name
+		metadata.Size = int64(len(cached.blob))
+		metadata.LastModified = cached.mtime
 	}
 	return
 }
@@ -357,6 +365,37 @@ func (s3 *S3Backend) DeleteBlob(ctx context.Context, name string) error {
 		minio.RemoveObjectOptions{})
 }
 
+func (s3 *S3Backend) EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error] {
+	return func(yield func(BlobMetadata, error) bool) {
+		logc.Print(ctx, "s3: enumerate blobs")
+
+		ctx, cancel := context.WithCancel(ctx)
+		defer cancel()
+
+		prefix := "blob/"
+		for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{
+			Prefix:    prefix,
+			Recursive: true,
+		}) {
+			var metadata BlobMetadata
+			var err error
+			if err = object.Err; err == nil {
+				key := strings.TrimPrefix(object.Key, prefix)
+				if strings.HasSuffix(key, "/") {
+					continue // directory; skip
+				} else {
+					metadata.Name = joinBlobName(strings.Split(key, "/"))
+					metadata.Size = object.Size
+					metadata.LastModified = object.LastModified
+				}
+			}
+			if !yield(metadata, err) {
+				break
+			}
+		}
+	}
+}
+
 func manifestObjectName(name string) string {
 	return fmt.Sprintf("site/%s", name)
 }
@@ -365,34 +404,6 @@ func stagedManifestObjectName(manifestData []byte) string {
 	return fmt.Sprintf("dirty/%x", sha256.Sum256(manifestData))
 }
 
-func (s3 *S3Backend) ListManifests(ctx context.Context) (manifests []string, err error) {
-	logc.Print(ctx, "s3: list manifests")
-
-	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
-
-	prefix := manifestObjectName("")
-	for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{
-		Prefix:    prefix,
-		Recursive: true,
-	}) {
-		if object.Err != nil {
-			return nil, object.Err
-		}
-		key := strings.TrimRight(strings.TrimPrefix(object.Key, prefix), "/")
-		if strings.Count(key, "/") > 1 {
-			continue
-		}
-		_, project, _ := strings.Cut(key, "/")
-		if project == "" || strings.HasPrefix(project, ".") && project != ".index" {
-			continue
-		}
-		manifests = append(manifests, key)
-	}
-
-	return
-}
-
 type s3ManifestLoader struct {
 	s3 *S3Backend
 }
@@ -579,7 +590,7 @@ func (s3 *S3Backend) CommitManifest(
 	data := EncodeManifest(manifest)
 	logc.Printf(ctx, "s3: commit manifest %x -> %s", sha256.Sum256(data), name)
 
-	_, domain, _ := strings.Cut(name, "/")
+	domain, _, _ := strings.Cut(name, "/")
 	if err := s3.checkDomainFrozen(ctx, domain); err != nil {
 		return err
 	}
@@ -621,7 +632,7 @@ func (s3 *S3Backend) DeleteManifest(
 ) error {
 	logc.Printf(ctx, "s3: delete manifest %s\n", name)
 
-	_, domain, _ := strings.Cut(name, "/")
+	domain, _, _ := strings.Cut(name, "/")
 	if err := s3.checkDomainFrozen(ctx, domain); err != nil {
 		return err
 	}
@@ -632,8 +643,90 @@ func (s3 *S3Backend) DeleteManifest(
 
 	err := s3.client.RemoveObject(ctx, s3.bucket, manifestObjectName(name),
 		minio.RemoveObjectOptions{})
+	if err != nil {
+		return err
+	}
 	s3.siteCache.Cache.Invalidate(name)
-	return err
+	return s3.bumpLastDomainUpdateTimestamp(ctx)
+}
+
+func (s3 *S3Backend) EnumerateManifests(ctx context.Context) iter.Seq2[*ManifestMetadata, error] {
+	return func(yield func(*ManifestMetadata, error) bool) {
+		logc.Print(ctx, "s3: enumerate manifests")
+
+		ctx, cancel := context.WithCancel(ctx)
+		defer cancel()
+
+		prefix := "site/"
+		for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{
+			Prefix:    prefix,
+			Recursive: true,
+		}) {
+			var metadata *ManifestMetadata
+			var err error
+			if err = object.Err; err == nil {
+				key := strings.TrimPrefix(object.Key, prefix)
+				_, project, _ := strings.Cut(key, "/")
+				if strings.HasSuffix(key, "/") {
+					continue // directory; skip
+				} else if project == "" || strings.HasPrefix(project, ".") && project != ".index" {
+					continue // internal; skip
+				} else {
+					metadata = &ManifestMetadata{
+						Name:         key,
+						Size:         object.Size,
+						LastModified: object.LastModified,
+						ETag:         object.ETag,
+					}
+				}
+			}
+			if !yield(metadata, err) {
+				break
+			}
+		}
+	}
+}
+
+// Limits the number of concurrent uploads, globally across the entire git-pages process.
+// Not currently configurable as there seems to be little need.
+var getAllManifestsSemaphore = make(chan struct{}, 64)
+
+func (s3 *S3Backend) GetAllManifests(ctx context.Context) iter.Seq2[tuple[*ManifestMetadata, *Manifest], error] {
+	type result struct {
+		metadata *ManifestMetadata
+		manifest *Manifest
+		err      error
+	}
+
+	resultsChan := make(chan result)
+	enumeratorCtx, cancel := context.WithCancel(ctx)
+	go func(ctx context.Context) {
+		wg := sync.WaitGroup{}
+		for metadata, err := range s3.EnumerateManifests(ctx) {
+			if err != nil {
+				resultsChan <- result{nil, nil, err}
+			} else {
+				getAllManifestsSemaphore <- struct{}{} // acquire
+				wg.Go(func() {
+					defer func() { <-getAllManifestsSemaphore }() // release
+					manifest, _, err := backend.GetManifest(ctx, metadata.Name, GetManifestOptions{})
+					resultsChan <- result{metadata, manifest, err}
+				})
+			}
+		}
+		wg.Wait()
+		close(resultsChan)
+	}(enumeratorCtx)
+
+	return func(yield func(tuple[*ManifestMetadata, *Manifest], error) bool) {
+		for result := range resultsChan {
+			item := tuple[*ManifestMetadata, *Manifest]{result.metadata, result.manifest}
+			if !yield(item, result.err) {
+				cancel()
+				break
+			}
+		}
+	}
 }
 
 func domainCheckObjectName(domain string) string {
@@ -674,35 +767,70 @@ func (s3 *S3Backend) CheckDomain(ctx context.Context, domain string) (exists boo
 func (s3 *S3Backend) CreateDomain(ctx context.Context, domain string) error {
 	logc.Printf(ctx, "s3: create domain %s\n", domain)
 
-	_, err := s3.client.PutObject(ctx, s3.bucket, domainCheckObjectName(domain),
+	exists, err := s3.CheckDomain(ctx, domain)
+	if err != nil {
+		return err
+	}
+
+	_, err = s3.client.PutObject(ctx, s3.bucket, domainCheckObjectName(domain),
+		&bytes.Reader{}, 0, minio.PutObjectOptions{})
+	if err != nil {
+		return err
+	}
+	if !exists {
+		err = s3.bumpLastDomainUpdateTimestamp(ctx)
+	}
+	return err
+}
+
+func (s3 *S3Backend) FreezeDomain(ctx context.Context, domain string) error {
+	logc.Printf(ctx, "s3: freeze domain %s\n", domain)
+
+	_, err := s3.client.PutObject(ctx, s3.bucket, domainFrozenObjectName(domain),
+		&bytes.Reader{}, 0, minio.PutObjectOptions{})
+	return err
+
+}
+
+func (s3 *S3Backend) UnfreezeDomain(ctx context.Context, domain string) error {
+	logc.Printf(ctx, "s3: unfreeze domain %s\n", domain)
+
+	err := s3.client.RemoveObject(ctx, s3.bucket, domainFrozenObjectName(domain),
+		minio.RemoveObjectOptions{})
+	if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" {
+		return nil
+	} else {
+		return err
+	}
+}
+
+const lastDomainUpdateObjectName = "meta/last-domain-update"
+
+func (s3 *S3Backend) HaveDomainsChanged(ctx context.Context, since time.Time) (bool, error) {
+	info, err := s3.client.StatObject(ctx, s3.bucket, lastDomainUpdateObjectName,
+		minio.GetObjectOptions{})
+	if err != nil {
+		return false, err
+	}
+
+	return info.LastModified.After(since), nil
+}
+
+func (s3 *S3Backend) bumpLastDomainUpdateTimestamp(ctx context.Context) error {
+	logc.Print(ctx, "s3: bumping last domain update timestamp")
+	_, err := s3.client.PutObject(ctx, s3.bucket, lastDomainUpdateObjectName,
 		&bytes.Reader{}, 0, minio.PutObjectOptions{})
 	return err
 }
 
-func (s3 *S3Backend) FreezeDomain(ctx context.Context, domain string, freeze bool) error {
-	if freeze {
-		logc.Printf(ctx, "s3: freeze domain %s\n", domain)
-
-		_, err := s3.client.PutObject(ctx, s3.bucket, domainFrozenObjectName(domain),
-			&bytes.Reader{}, 0, minio.PutObjectOptions{})
-		return err
-	} else {
-		logc.Printf(ctx, "s3: thaw domain %s\n", domain)
-
-		err := s3.client.RemoveObject(ctx, s3.bucket, domainFrozenObjectName(domain),
-			minio.RemoveObjectOptions{})
-		if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" {
-			return nil
-		} else {
-			return err
-		}
-	}
-}
-
 func auditObjectName(id AuditID) string {
 	return fmt.Sprintf("audit/%s", id)
 }
 
+func auditDetachedObjectName(id AuditID) string {
+	return fmt.Sprintf("audit/%s.detached", id)
+}
+
 func (s3 *S3Backend) AppendAuditLog(ctx context.Context, id AuditID, record *AuditRecord) error {
 	logc.Printf(ctx, "s3: append audit %s\n", id)
 
@@ -734,14 +862,27 @@ func (s3 *S3Backend) QueryAuditLog(ctx context.Context, id AuditID) (*AuditRecor
 		return nil, err
 	}
 
-	return DecodeAuditRecord(data)
+	record, err := DecodeAuditRecord(data)
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = s3.client.StatObject(ctx, s3.bucket, auditDetachedObjectName(id),
+		minio.StatObjectOptions{})
+	if err == nil {
+		record.Manifest = nil
+	} else if errResp := minio.ToErrorResponse(err); err != nil && errResp.Code != "NoSuchKey" {
+		return nil, err
+	}
+
+	return record, nil
 }
 
 func (s3 *S3Backend) SearchAuditLog(
 	ctx context.Context, opts SearchAuditLogOptions,
 ) iter.Seq2[AuditID, error] {
 	return func(yield func(AuditID, error) bool) {
-		logc.Printf(ctx, "s3: query audit\n")
+		logc.Printf(ctx, "s3: search audit\n")
 
 		ctx, cancel := context.WithCancel(ctx)
 		defer cancel()
@@ -754,8 +895,14 @@ func (s3 *S3Backend) SearchAuditLog(
 			var err error
 			if object.Err != nil {
 				err = object.Err
-			} else {
-				id, err = ParseAuditID(strings.TrimPrefix(object.Key, prefix))
+			} else if strings.Contains(object.Key, ".") {
+				continue
+			} else if id, err = ParseAuditID(strings.TrimPrefix(object.Key, prefix)); err != nil {
+				// report error
+			} else if !opts.Since.IsZero() && id.CompareTime(opts.Since) < 0 {
+				continue
+			} else if !opts.Until.IsZero() && id.CompareTime(opts.Until) > 0 {
+				continue
 			}
 			if !yield(id, err) {
 				break
@@ -763,3 +910,55 @@ func (s3 *S3Backend) SearchAuditLog(
 		}
 	}
 }
+
+var getAuditLogRecordsSemaphore = make(chan struct{}, 64)
+
+func (s3 *S3Backend) GetAuditLogRecords(
+	ctx context.Context, ids iter.Seq2[AuditID, error],
+) iter.Seq2[*AuditRecord, error] {
+	return func(yield func(*AuditRecord, error) bool) {
+		resultsChan := make(chan tuple[*AuditRecord, error])
+		enumeratorCtx, cancel := context.WithCancel(ctx)
+		defer cancel()
+
+		go func(ctx context.Context) {
+			wg := sync.WaitGroup{}
+			for id, err := range ids {
+				if err != nil {
+					resultsChan <- tuple[*AuditRecord, error]{nil, err}
+				} else {
+					getAuditLogRecordsSemaphore <- struct{}{} // acquire
+					wg.Go(func() {
+						defer func() { <-getAuditLogRecordsSemaphore }() // release
+						record, err := s3.QueryAuditLog(ctx, id)
+						resultsChan <- tuple[*AuditRecord, error]{record, err}
+					})
+				}
+			}
+			wg.Wait()
+			close(resultsChan)
+		}(enumeratorCtx)
+
+		for result := range resultsChan {
+			record, err := result.Splat()
+			if !yield(record, err) {
+				break
+			}
+		}
+	}
+}
+
+func (s3 *S3Backend) DetachAuditRecord(ctx context.Context, id AuditID) error {
+	logc.Printf(ctx, "s3: detach audit record %s\n", id)
+
+	_, err := s3.client.PutObject(ctx, s3.bucket, auditDetachedObjectName(id),
+		&bytes.Reader{}, 0, minio.PutObjectOptions{})
+	return err
+}
+
+func (s3 *S3Backend) ExpireAuditRecord(ctx context.Context, id AuditID) error {
+	logc.Printf(ctx, "s3: expire audit record %s\n", id)
+
+	return s3.client.RemoveObject(ctx, s3.bucket, auditObjectName(id),
+		minio.RemoveObjectOptions{})
+}

src/caddy.go

@@ -26,7 +26,17 @@ func ServeCaddy(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	found, err := backend.CheckDomain(r.Context(), strings.ToLower(domain))
+	var err error
+	domain = strings.ToLower(domain)
+
+	// Run a cheap check as to whether we might be serving the domain.
+	var found = domainCache.CheckDomain(r.Context(), domain)
+
+	if found {
+		// Run an expensive check as to whether we are actually serving the domain.
+		found, err = backend.CheckDomain(r.Context(), domain)
+	}
+
 	if !found {
 		// If we don't serve the domain, but a fallback server does, then we should let our
 		// Caddy instance request a TLS certificate. Otherwise, we'll never have an opportunity

src/collect.go

@@ -5,7 +5,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"time"
 )
 
 type Flusher interface {
@@ -66,23 +65,30 @@ func CollectTar(
 
 		case Type_ExternalFile:
 			var blobReader io.Reader
-			var blobMtime time.Time
+			var blobMetadata BlobMetadata
 			var blobData []byte
-			blobReader, _, blobMtime, err = backend.GetBlob(context, string(entry.Data))
+			blobReader, blobMetadata, err = backend.GetBlob(context, string(entry.Data))
+			if err != nil {
+				return
+			}
+			blobData, err = io.ReadAll(blobReader)
 			if err != nil {
 				return
 			}
-			blobData, _ = io.ReadAll(blobReader)
 			header.Typeflag = tar.TypeReg
 			header.Mode = 0644
-			header.ModTime = blobMtime
+			header.ModTime = blobMetadata.LastModified
 			err = appendFile(&header, blobData, entry.GetTransform())
 
 		case Type_Symlink:
 			header.Typeflag = tar.TypeSymlink
 			header.Mode = 0644
 			header.ModTime = metadata.LastModified
-			err = appendFile(&header, entry.GetData(), Transform_Identity)
+			header.Linkname = string(entry.GetData())
+			err = archive.WriteHeader(&header)
+			if err != nil {
+				return fmt.Errorf("tar: %w", err)
+			}
 
 		default:
 			panic(fmt.Errorf("CollectTar encountered invalid entry: %v, %v",

src/config.go

@@ -12,9 +12,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/BurntSushi/toml"
 	"github.com/c2h5oh/datasize"
 	"github.com/creasty/defaults"
-	"github.com/pelletier/go-toml/v2"
 )
 
 // For an unknown reason, the standard `time.Duration` type doesn't implement the standard
@@ -63,7 +63,6 @@ type Config struct {
 	Insecure      bool                `toml:"-" env:"insecure"`
 	Features      []string            `toml:"features"`
 	LogFormat     string              `toml:"log-format" default:"text"`
-	LogLevel      string              `toml:"log-level" default:"info"`
 	Server        ServerConfig        `toml:"server"`
 	Wildcard      []WildcardConfig    `toml:"wildcard"`
 	Fallback      FallbackConfig      `toml:"fallback"`
@@ -74,17 +73,17 @@ type Config struct {
 }
 
 type ServerConfig struct {
-	Pages   string `toml:"pages" default:"tcp/:3000"`
-	Caddy   string `toml:"caddy" default:"tcp/:3001"`
-	Metrics string `toml:"metrics" default:"tcp/:3002"`
+	Pages   string `toml:"pages" default:"tcp/localhost:3000"`
+	Caddy   string `toml:"caddy" default:"tcp/localhost:3001"`
+	Metrics string `toml:"metrics" default:"tcp/localhost:3002"`
 }
 
 type WildcardConfig struct {
-	Domain          string   `toml:"domain"`
-	CloneURL        string   `toml:"clone-url"` // URL template, not an exact URL
-	IndexRepos      []string `toml:"index-repos" default:"[]"`
-	IndexRepoBranch string   `toml:"index-repo-branch" default:"pages"`
-	Authorization   string   `toml:"authorization"`
+	Domain          string `toml:"domain"`
+	CloneURL        string `toml:"clone-url"` // URL template, not an exact URL
+	IndexRepo       string `toml:"index-repo" default:"pages"`
+	IndexRepoBranch string `toml:"index-repo-branch" default:"pages"`
+	Authorization   string `toml:"authorization"`
 }
 
 type FallbackConfig struct {
@@ -135,16 +134,21 @@ type LimitsConfig struct {
 	// Maximum time that an update operation (PUT or POST request) could take before being
 	// interrupted.
 	UpdateTimeout Duration `toml:"update-timeout" default:"60s"`
+	// Maximum number of concurrent blob uploads, globally across every update request.
+	ConcurrentUploads uint `toml:"concurrent-uploads" default:"1024"`
 	// Soft limit on Go heap size, expressed as a fraction of total available RAM.
 	MaxHeapSizeRatio float64 `toml:"max-heap-size-ratio" default:"0.5"`
 	// List of domains unconditionally forbidden for uploads.
 	ForbiddenDomains []string `toml:"forbidden-domains" default:"[]"`
 	// List of allowed repository URL prefixes. Setting this option prohibits uploading archives.
-	AllowedRepositoryURLPrefixes []string `toml:"allowed-repository-url-prefixes"`
+	AllowedRepositoryURLPrefixes []string `toml:"allowed-repository-url-prefixes" default:"[]"`
 	// List of allowed custom headers. Header name must be in the MIME canonical form,
 	// e.g. `Foo-Bar`. Setting this option permits including this custom header in `_headers`,
 	// unless it is fundamentally unsafe.
 	AllowedCustomHeaders []string `toml:"allowed-custom-headers" default:"[\"X-Clacks-Overhead\"]"`
+	// Whether to allow Netlify-style credentials specified in a `Basic-Auth:` pseudo-header.
+	// These credentials are plaintext.
+	AllowBasicAuth bool `toml:"allow-basic-auth" default:"false"`
 }
 
 type AuditConfig struct {
@@ -152,8 +156,11 @@ type AuditConfig struct {
 	NodeID int `toml:"node-id"`
 	// Whether audit reports should be stored whenever an audit event occurs.
 	Collect bool `toml:"collect"`
-	// Whether audit reports should include principal's IP address.
-	IncludeIPs bool `toml:"include-ip"`
+	// If not empty, includes the principal's IP address in audit reports, with the value specifying
+	// the source of the IP address. If the value is "X-Forwarded-For", the last item of the
+	// corresponding header field (assumed to be comma-separated) is used. If the value is
+	// "RemoteAddr", the connecting host's address is used. Any other value is disallowed.
+	IncludeIPs string `toml:"include-ip"`
 	// Endpoint to notify with a `GET /<notify-url>?<id>` whenever an audit event occurs.
 	NotifyURL *URL `toml:"notify-url"`
 }
@@ -163,8 +170,8 @@ type ObservabilityConfig struct {
 	SlowResponseThreshold Duration `toml:"slow-response-threshold" default:"500ms"`
 }
 
-func (config *Config) DebugJSON() string {
-	result, err := json.MarshalIndent(config, "", "  ")
+func (config *Config) TOML() string {
+	result, err := toml.Marshal(config)
 	if err != nil {
 		panic(err)
 	}
@@ -302,25 +309,43 @@ func PrintConfigEnvVars() {
 	})
 }
 
-func Configure(tomlPath string) (config *Config, err error) {
+func PrettyTomlKey(key toml.Key) string {
+	if len(key) == 1 {
+		return key.String()
+	} else {
+		// `toml.Key.String()` adds quotes if necessary.
+		return fmt.Sprintf("[%s].%s", key[:len(key)-1].String(), key[len(key)-1:].String())
+	}
+}
+
+func ReadConfigFile(config *Config, tomlPath string) (err error) {
+	if tomlPath != "" {
+		meta, err := toml.DecodeFile(tomlPath, config)
+		if err != nil {
+			return err
+		}
+
+		unknownKeys := []string{}
+		for _, key := range meta.Undecoded() {
+			unknownKeys = append(unknownKeys, PrettyTomlKey(key))
+		}
+		if len(unknownKeys) > 0 {
+			return fmt.Errorf("unknown keys: %s", strings.Join(unknownKeys, ", "))
+		}
+	}
+	return nil
+}
+
+func Configure(tomlPaths ...string) (config *Config, err error) {
 	// start with an all-default configuration
 	config = new(Config)
 	defaults.MustSet(config)
 
-	// inject values from `config.toml`
-	if tomlPath != "" {
-		var file *os.File
-		file, err = os.Open(tomlPath)
+	// inject values from each toml file
+	for _, tomlPath := range tomlPaths {
+		err := ReadConfigFile(config, tomlPath)
 		if err != nil {
-			return
-		}
-		defer file.Close()
-
-		decoder := toml.NewDecoder(file)
-		decoder.DisallowUnknownFields()
-		decoder.EnableUnmarshalerInterface()
-		if err = decoder.Decode(&config); err != nil {
-			return
+			return nil, err
 		}
 	}
 

src/domain_cache.go

@@ -0,0 +1,132 @@
+package git_pages
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/bits-and-blooms/bloom/v3"
+)
+
+type DomainCache interface {
+	// Check if we might be serving the domain.
+	CheckDomain(ctx context.Context, domain string) (found bool)
+
+	// Add the domain to the cache.
+	AddDomain(ctx context.Context, domain string)
+}
+
+func CreateDomainCache(ctx context.Context) (DomainCache, error) {
+	if !config.Feature("domain-existence-cache") {
+		return &dummyDomainCache{}, nil
+	}
+	return createBloomDomainCache(ctx)
+}
+
+type bloomDomainCache struct {
+	filter   *bloom.BloomFilter
+	filterMu sync.Mutex
+
+	accessCh    chan struct{}
+	refreshMu   sync.Mutex
+	lastRefresh time.Time
+	maxAge      time.Duration
+}
+
+func createBloomDomainCache(ctx context.Context) (DomainCache, error) {
+	cache := bloomDomainCache{
+		accessCh: make(chan struct{}),
+	}
+
+	switch config.Storage.Type {
+	case "fs":
+		// the FS backend has no cache
+	case "s3":
+		cache.maxAge = time.Duration(config.Storage.S3.SiteCache.MaxAge)
+	default:
+		panic(fmt.Errorf("unknown backend: %s", config.Storage.Type))
+	}
+
+	if err := cache.refresh(ctx); err != nil {
+		return nil, err
+	}
+
+	go cache.handleFilterUpdates(ctx)
+
+	return &cache, nil
+}
+
+func (c *bloomDomainCache) handleFilterUpdates(ctx context.Context) {
+	for range c.accessCh {
+		if time.Since(c.lastRefresh) > c.maxAge {
+			logc.Print(ctx, "domain cache: refreshing")
+			if err := c.refresh(ctx); err != nil {
+				logc.Printf(ctx, "domain cache: refresh error: %v", err)
+			}
+		}
+	}
+}
+
+func (c *bloomDomainCache) refresh(ctx context.Context) error {
+	c.refreshMu.Lock()
+	defer c.refreshMu.Unlock()
+
+	if changed, err := backend.HaveDomainsChanged(ctx, c.lastRefresh); err != nil {
+		return err
+	} else if !changed {
+		logc.Print(ctx, "domain cache: unchanged")
+		c.lastRefresh = time.Now()
+		return nil
+	}
+
+	// Create a 256 KiB Bloom filter that will fit ~150K entries with 0.1% false positive rate.
+	filter := bloom.New(256*1024, 10)
+	for metadata, err := range backend.EnumerateManifests(ctx) {
+		if err != nil {
+			return fmt.Errorf("enum manifests: %w", err)
+		}
+		domain, _, _ := strings.Cut(metadata.Name, "/")
+		filter.AddString(domain)
+	}
+
+	c.filterMu.Lock()
+	c.filter = filter
+	c.filterMu.Unlock()
+
+	logc.Printf(ctx, "domain cache: refreshed with approx. %d domains", filter.ApproximatedSize())
+	c.lastRefresh = time.Now()
+	return nil
+}
+
+func (c *bloomDomainCache) CheckDomain(ctx context.Context, domain string) (found bool) {
+	select {
+	case c.accessCh <- struct{}{}:
+	default:
+	}
+
+	c.filterMu.Lock()
+	found = c.filter.TestString(domain)
+	c.filterMu.Unlock()
+
+	logc.Printf(ctx, "domain cache: bloom filter returns %v for %q", found, domain)
+	return
+}
+
+func (c *bloomDomainCache) AddDomain(ctx context.Context, domain string) {
+	c.refreshMu.Lock()
+	defer c.refreshMu.Unlock()
+
+	c.filterMu.Lock()
+	c.filter.AddString(domain)
+	c.filterMu.Unlock()
+
+	logc.Printf(ctx, "domain cache: added %q", domain)
+}
+
+type dummyDomainCache struct{}
+
+func (d dummyDomainCache) CheckDomain(context.Context, string) bool { return true }
+
+func (d dummyDomainCache) AddDomain(context.Context, string) {}

src/extract.go

@@ -5,10 +5,13 @@ import (
 	"archive/zip"
 	"bytes"
 	"compress/gzip"
+	"context"
 	"errors"
 	"fmt"
 	"io"
+	"math"
 	"os"
+	"path"
 	"strings"
 
 	"github.com/c2h5oh/datasize"
@@ -22,30 +25,69 @@ func boundArchiveStream(reader io.Reader) io.Reader {
 		fmt.Errorf("%w: %s limit exceeded", ErrArchiveTooLarge, config.Limits.MaxSiteSize.HR()))
 }
 
-func ExtractGzip(reader io.Reader, next func(io.Reader) (*Manifest, error)) (*Manifest, error) {
+func ExtractGzip(
+	ctx context.Context, reader io.Reader,
+	next func(context.Context, io.Reader) (*Manifest, error),
+) (*Manifest, error) {
 	stream, err := gzip.NewReader(reader)
 	if err != nil {
 		return nil, err
 	}
 	defer stream.Close()
 
-	return next(boundArchiveStream(stream))
+	return next(ctx, boundArchiveStream(stream))
 }
 
-func ExtractZstd(reader io.Reader, next func(io.Reader) (*Manifest, error)) (*Manifest, error) {
+func ExtractZstd(
+	ctx context.Context, reader io.Reader,
+	next func(context.Context, io.Reader) (*Manifest, error),
+) (*Manifest, error) {
 	stream, err := zstd.NewReader(reader)
 	if err != nil {
 		return nil, err
 	}
 	defer stream.Close()
 
-	return next(boundArchiveStream(stream))
+	return next(ctx, boundArchiveStream(stream))
 }
 
-func ExtractTar(reader io.Reader) (*Manifest, error) {
+func normalizeArchiveMemberName(fileName string) string {
+	// Strip the leading slash and any extraneous path segments.
+	fileName = path.Clean(fileName)
+	fileName = strings.TrimPrefix(fileName, "/")
+	if fileName == "." {
+		fileName = ""
+	}
+	return fileName
+}
+
+func addSymlinkOrBlobReference(
+	manifest *Manifest, fileName string, target string,
+	index map[string]*Entry, missing *[]string,
+) *Entry {
+	if hash, found := strings.CutPrefix(target, BlobReferencePrefix); found {
+		if entry, found := index[hash]; found {
+			manifest.Contents[fileName] = entry
+			return entry
+		} else {
+			*missing = append(*missing, hash)
+			return nil
+		}
+	} else {
+		return AddSymlink(manifest, fileName, target)
+	}
+}
+
+func ExtractTar(ctx context.Context, reader io.Reader, oldManifest *Manifest) (*Manifest, error) {
 	archive := tar.NewReader(reader)
 
+	var dataBytesRecycled int64
+	var dataBytesTransferred int64
+
+	index := IndexManifestByGitHash(oldManifest)
+	missing := []string{}
 	manifest := NewManifest()
+	hardLinks := map[string]*Entry{}
 	for {
 		header, err := archive.Next()
 		if err == io.EOF {
@@ -54,15 +96,10 @@ func ExtractTar(reader io.Reader) (*Manifest, error) {
 			return nil, err
 		}
 
-		// For some reason, GNU tar includes any leading `.` path segments in archive filenames,
-		// unless there is a `..` path segment anywhere in the input filenames.
-		fileName := header.Name
-		for {
-			if strippedName, found := strings.CutPrefix(fileName, "./"); found {
-				fileName = strippedName
-			} else {
-				break
-			}
+		fileName := normalizeArchiveMemberName(header.Name)
+		if fileName == "" {
+			// This must be the root directory. It will be filled in by EnsureLeadingDirectories.
+			continue
 		}
 
 		switch header.Typeflag {
@@ -71,9 +108,27 @@ func ExtractTar(reader io.Reader) (*Manifest, error) {
 			if err != nil {
 				return nil, fmt.Errorf("tar: %s: %w", fileName, err)
 			}
-			AddFile(manifest, fileName, fileData)
+			entry := AddFile(manifest, fileName, fileData)
+			hardLinks[header.Name] = entry
+			dataBytesTransferred += int64(len(fileData))
 		case tar.TypeSymlink:
-			AddSymlink(manifest, fileName, header.Linkname)
+			entry := addSymlinkOrBlobReference(
+				manifest, fileName, header.Linkname, index, &missing)
+			hardLinks[header.Name] = entry
+			switch {
+			case entry == nil:
+				// unresolved blob reference
+			case entry.GetType() != Type_Symlink:
+				dataBytesRecycled += entry.GetOriginalSize() // resolved blob reference
+			default:
+				dataBytesTransferred += int64(len(header.Linkname)) // actual symlink
+			}
+		case tar.TypeLink:
+			if entry, found := hardLinks[header.Linkname]; found {
+				manifest.Contents[fileName] = entry
+			} else {
+				AddProblem(manifest, fileName, "tar: invalid hardlink %q", header.Linkname)
+			}
 		case tar.TypeDir:
 			AddDirectory(manifest, fileName)
 		default:
@@ -81,10 +136,27 @@ func ExtractTar(reader io.Reader) (*Manifest, error) {
 			continue
 		}
 	}
+
+	if len(missing) > 0 {
+		return nil, UnresolvedRefError{missing}
+	}
+
+	// Ensure parent directories exist for all entries.
+	EnsureLeadingDirectories(manifest)
+
+	logc.Printf(ctx,
+		"reuse: %s recycled, %s transferred\n",
+		datasize.ByteSize(dataBytesRecycled).HR(),
+		datasize.ByteSize(dataBytesTransferred).HR(),
+	)
+
 	return manifest, nil
 }
 
-func ExtractZip(reader io.Reader) (*Manifest, error) {
+// Used for zstd decompression inside zip files, it is recommended to share this.
+var zstdDecomp = zstd.ZipDecompressor()
+
+func ExtractZip(ctx context.Context, reader io.Reader, oldManifest *Manifest) (*Manifest, error) {
 	data, err := io.ReadAll(reader)
 	if err != nil {
 		return nil, err
@@ -95,9 +167,18 @@ func ExtractZip(reader io.Reader) (*Manifest, error) {
 		return nil, err
 	}
 
+	// Support zstd compression inside zip files.
+	archive.RegisterDecompressor(zstd.ZipMethodWinZip, zstdDecomp)
+	archive.RegisterDecompressor(zstd.ZipMethodPKWare, zstdDecomp)
+
 	// Detect and defuse zipbombs.
 	var totalSize uint64
 	for _, file := range archive.File {
+		if totalSize+file.UncompressedSize64 < totalSize {
+			// Would overflow
+			totalSize = math.MaxUint64
+			break
+		}
 		totalSize += file.UncompressedSize64
 	}
 	if totalSize > config.Limits.MaxSiteSize.Bytes() {
@@ -108,10 +189,16 @@ func ExtractZip(reader io.Reader) (*Manifest, error) {
 		)
 	}
 
+	var dataBytesRecycled int64
+	var dataBytesTransferred int64
+
+	index := IndexManifestByGitHash(oldManifest)
+	missing := []string{}
 	manifest := NewManifest()
 	for _, file := range archive.File {
+		normalizedName := normalizeArchiveMemberName(file.Name)
 		if strings.HasSuffix(file.Name, "/") {
-			AddDirectory(manifest, file.Name)
+			AddDirectory(manifest, normalizedName)
 		} else {
 			fileReader, err := file.Open()
 			if err != nil {
@@ -125,11 +212,35 @@ func ExtractZip(reader io.Reader) (*Manifest, error) {
 			}
 
 			if file.Mode()&os.ModeSymlink != 0 {
-				AddSymlink(manifest, file.Name, string(fileData))
+				entry := addSymlinkOrBlobReference(
+					manifest, normalizedName, string(fileData), index, &missing)
+				switch {
+				case entry == nil:
+					// unresolved blob reference
+				case entry.GetType() != Type_Symlink:
+					dataBytesRecycled += entry.GetOriginalSize() // resolved blob reference
+				default:
+					dataBytesTransferred += int64(len(fileData)) // actual symlink
+				}
 			} else {
-				AddFile(manifest, file.Name, fileData)
+				AddFile(manifest, normalizedName, fileData)
+				dataBytesTransferred += int64(len(fileData))
 			}
 		}
 	}
+
+	if len(missing) > 0 {
+		return nil, UnresolvedRefError{missing}
+	}
+
+	// Ensure parent directories exist for all entries.
+	EnsureLeadingDirectories(manifest)
+
+	logc.Printf(ctx,
+		"reuse: %s recycled, %s transferred\n",
+		datasize.ByteSize(dataBytesRecycled).HR(),
+		datasize.ByteSize(dataBytesTransferred).HR(),
+	)
+
 	return manifest, nil
 }

src/fetch.go

@@ -9,12 +9,14 @@ import (
 	"net/url"
 	"os"
 	"slices"
+	"strings"
 
 	"github.com/c2h5oh/datasize"
 	"github.com/go-git/go-billy/v6/osfs"
 	"github.com/go-git/go-git/v6"
 	"github.com/go-git/go-git/v6/plumbing"
 	"github.com/go-git/go-git/v6/plumbing/cache"
+	"github.com/go-git/go-git/v6/plumbing/client"
 	"github.com/go-git/go-git/v6/plumbing/filemode"
 	"github.com/go-git/go-git/v6/plumbing/object"
 	"github.com/go-git/go-git/v6/plumbing/protocol/packp"
@@ -23,6 +25,8 @@ import (
 	"google.golang.org/protobuf/proto"
 )
 
+var ErrRepositoryTooLarge = errors.New("repository too large")
+
 func FetchRepository(
 	ctx context.Context, repoURL string, branch string, oldManifest *Manifest,
 ) (
@@ -57,7 +61,7 @@ func FetchRepository(
 		repo, err = git.CloneContext(ctx, storer, nil, &git.CloneOptions{
 			Bare:          true,
 			URL:           repoURL,
-			ReferenceName: plumbing.ReferenceName(branch),
+			ReferenceName: plumbing.NewBranchReferenceName(branch),
 			SingleBranch:  true,
 			Depth:         1,
 			Tags:          git.NoTags,
@@ -131,9 +135,8 @@ func FetchRepository(
 	}
 
 	// Collect checkout statistics.
-	var dataBytesFromOldManifest int64
-	var dataBytesFromGitCheckout int64
-	var dataBytesFromGitTransport int64
+	var dataBytesRecycled int64
+	var dataBytesTransferred int64
 
 	// First, see if we can extract the blobs from the old manifest. This is the preferred option
 	// because it avoids both network transfers and recompression. Note that we do not request
@@ -143,7 +146,7 @@ func FetchRepository(
 			if manifestEntry, found := blobsNeeded[hash]; found {
 				manifestEntry.Reset()
 				proto.Merge(manifestEntry, oldManifestEntry)
-				dataBytesFromOldManifest += oldManifestEntry.GetOriginalSize()
+				dataBytesRecycled += oldManifestEntry.GetOriginalSize()
 				delete(blobsNeeded, hash)
 			}
 		}
@@ -153,37 +156,28 @@ func FetchRepository(
 	// This will only succeed if a `blob:none` filter isn't supported and we got a full
 	// clone despite asking for a partial clone.
 	for hash, manifestEntry := range blobsNeeded {
-		if err := readGitBlob(repo, hash, manifestEntry); err == nil {
-			dataBytesFromGitCheckout += manifestEntry.GetOriginalSize()
+		if err := readGitBlob(repo, hash, manifestEntry, &dataBytesTransferred); err == nil {
 			delete(blobsNeeded, hash)
+		} else if errors.Is(err, ErrRepositoryTooLarge) {
+			return nil, err
 		}
 	}
 
 	// Third, if we still don't have data for some manifest entries, re-establish a git transport
 	// and request the missing blobs (only) from the server.
 	if len(blobsNeeded) > 0 {
-		client, err := transport.Get(parsedRepoURL.Scheme)
-		if err != nil {
-			return nil, fmt.Errorf("git transport: %w", err)
-		}
+		gitClient := client.New()
+		request := &transport.Request{
+			URL:     parsedRepoURL,
+			Command: transport.UploadPackService}
 
-		endpoint, err := transport.NewEndpoint(repoURL)
-		if err != nil {
-			return nil, fmt.Errorf("git endpoint: %w", err)
-		}
-
-		session, err := client.NewSession(storer, endpoint, nil)
-		if err != nil {
-			return nil, fmt.Errorf("git session: %w", err)
-		}
-
-		connection, err := session.Handshake(ctx, transport.UploadPackService)
+		session, err := gitClient.Handshake(ctx, request)
 		if err != nil {
 			return nil, fmt.Errorf("git connection: %w", err)
 		}
-		defer connection.Close()
+		defer session.Close()
 
-		if err := connection.Fetch(ctx, &transport.FetchRequest{
+		if err := session.Fetch(ctx, storer, &transport.FetchRequest{
 			Wants: slices.Collect(maps.Keys(blobsNeeded)),
 			Depth: 1,
 			// Git CLI behaves like this, even if the wants above are references to blobs.
@@ -194,24 +188,27 @@ func FetchRepository(
 
 		// All remaining blobs should now be available.
 		for hash, manifestEntry := range blobsNeeded {
-			if err := readGitBlob(repo, hash, manifestEntry); err != nil {
+			if err := readGitBlob(repo, hash, manifestEntry, &dataBytesTransferred); err != nil {
 				return nil, err
 			}
-			dataBytesFromGitTransport += manifestEntry.GetOriginalSize()
 			delete(blobsNeeded, hash)
 		}
 	}
 
 	logc.Printf(ctx,
-		"fetch: %s reused, %s received\n",
-		datasize.ByteSize(dataBytesFromOldManifest).HR(),
-		datasize.ByteSize(dataBytesFromGitCheckout+dataBytesFromGitTransport).HR(),
+		"reuse: %s recycled, %s transferred\n",
+		datasize.ByteSize(dataBytesRecycled).HR(),
+		datasize.ByteSize(dataBytesTransferred).HR(),
 	)
 
+	warnAboutGitLFS(ctx, manifest)
+
 	return manifest, nil
 }
 
-func readGitBlob(repo *git.Repository, hash plumbing.Hash, entry *Entry) error {
+func readGitBlob(
+	repo *git.Repository, hash plumbing.Hash, entry *Entry, bytesTransferred *int64,
+) error {
 	blob, err := repo.BlobObject(hash)
 	if err != nil {
 		return fmt.Errorf("git blob %s: %w", hash, err)
@@ -240,5 +237,29 @@ func readGitBlob(repo *git.Repository, hash plumbing.Hash, entry *Entry) error {
 	entry.Transform = Transform_Identity.Enum()
 	entry.OriginalSize = proto.Int64(blob.Size)
 	entry.CompressedSize = proto.Int64(blob.Size)
+
+	*bytesTransferred += blob.Size
+	if uint64(*bytesTransferred) > config.Limits.MaxSiteSize.Bytes() {
+		return fmt.Errorf("%w: fetch exceeds %s limit",
+			ErrRepositoryTooLarge,
+			config.Limits.MaxSiteSize.HR(),
+		)
+	}
+
 	return nil
 }
+
+func warnAboutGitLFS(ctx context.Context, manifest *Manifest) {
+	gitattributes := ReadGitAttributes(ctx, manifest)
+	for _, name := range slices.Sorted(maps.Keys(manifest.GetContents())) {
+		entry := manifest.GetContents()[name]
+		if !IsEntryRegularFile(entry) {
+			continue
+		}
+		parts := strings.Split(name, "/")
+		attrs, _ := gitattributes.Match(parts, nil)
+		if attr, ok := attrs["filter"]; ok && attr.Value() == "lfs" {
+			AddProblem(manifest, name, "git-pages does not support Git LFS; move this file into Git or use incremental uploads")
+		}
+	}
+}

src/garbage.go

@@ -0,0 +1,84 @@
+package git_pages
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/c2h5oh/datasize"
+)
+
+func TraceGarbage(ctx context.Context) error {
+	allBlobs := map[string]int64{}
+	liveBlobs := map[string]int64{}
+
+	reduceBlobs := func(data map[string]int64) (items, total int64) {
+		for _, value := range data {
+			items += 1
+			total += value
+		}
+		return
+	}
+
+	traceManifest := func(manifestKind string, manifestName string, manifest *Manifest) error {
+		for _, entry := range manifest.GetContents() {
+			if entry.GetType() == Type_ExternalFile {
+				blobName := string(entry.Data)
+				if size, ok := allBlobs[blobName]; ok {
+					liveBlobs[blobName] = size
+				} else {
+					logc.Printf(ctx, "trace manifest: %s/%s: dangling reference %s",
+						manifestKind, manifestName, blobName)
+				}
+			}
+		}
+		return nil
+	}
+
+	// Enumerate all blobs.
+	logc.Printf(ctx, "trace: enumerating blobs")
+	for metadata, err := range backend.EnumerateBlobs(ctx) {
+		if err != nil {
+			return fmt.Errorf("trace blobs err: %w", err)
+		}
+		allBlobs[metadata.Name] = metadata.Size
+	}
+
+	// Enumerate blobs live via site manifests.
+	logc.Printf(ctx, "trace: enumerating manifests")
+	for item, err := range backend.GetAllManifests(ctx) {
+		metadata, manifest := item.Splat()
+		if err != nil {
+			return fmt.Errorf("trace sites err: %w", err)
+		}
+		err = traceManifest("site", metadata.Name, manifest)
+		if err != nil {
+			return fmt.Errorf("trace sites err: %w", err)
+		}
+	}
+
+	// Enumerate blobs live via audit records.
+	logc.Printf(ctx, "trace: enumerating audit records")
+	auditIDs := backend.SearchAuditLog(ctx, SearchAuditLogOptions{})
+	for record, err := range backend.GetAuditLogRecords(ctx, auditIDs) {
+		if err != nil {
+			return fmt.Errorf("trace audit err: %w", err)
+		}
+		if record.Manifest != nil {
+			err = traceManifest("audit", record.GetAuditID().String(), record.Manifest)
+			if err != nil {
+				return fmt.Errorf("trace audit err: %w", err)
+			}
+		}
+	}
+
+	allBlobsCount, allBlobsSize := reduceBlobs(allBlobs)
+	liveBlobsCount, liveBlobsSize := reduceBlobs(liveBlobs)
+	logc.Printf(ctx, "trace all: %d blobs, %s",
+		allBlobsCount, datasize.ByteSize(allBlobsSize).HR())
+	logc.Printf(ctx, "trace live: %d blobs, %s",
+		liveBlobsCount, datasize.ByteSize(liveBlobsSize).HR())
+	logc.Printf(ctx, "trace dead: %d blobs, %s",
+		allBlobsCount-liveBlobsCount, datasize.ByteSize(allBlobsSize-liveBlobsSize).HR())
+
+	return nil
+}

src/gitattributes.go

@@ -0,0 +1,61 @@
+package git_pages
+
+import (
+	"bytes"
+	"cmp"
+	"context"
+	"slices"
+	"strings"
+
+	"github.com/go-git/go-git/v6/plumbing/format/gitattributes"
+)
+
+func ReadGitAttributes(ctx context.Context, manifest *Manifest) gitattributes.Matcher {
+	type entryPair struct {
+		parts []string
+		entry *Entry
+	}
+
+	// Collect all .gitattributes files.
+	var files []entryPair
+	for name, entry := range manifest.GetContents() {
+		switch entry.GetType() {
+		case Type_InlineFile, Type_ExternalFile:
+			parts := strings.Split(name, "/")
+			if parts[len(parts)-1] == ".gitattributes" {
+				files = append(files, entryPair{parts, entry})
+			}
+		}
+	}
+
+	// Sort the file list by depth, then by name.
+	slices.SortFunc(files, func(a entryPair, b entryPair) int {
+		return cmp.Or(
+			cmp.Compare(len(a.parts), len(b.parts)),
+			slices.Compare(a.parts, b.parts),
+		)
+	})
+
+	// Gather all .gitattributes rules, sorted by depth.
+	var rules []gitattributes.MatchAttribute
+	for _, pair := range files {
+		parts, entry := pair.parts, pair.entry
+		data, err := GetEntryContents(ctx, entry)
+		if err != nil {
+			continue
+		}
+		dirs := parts[:len(parts)-1]
+		isRoot := len(parts) == 1
+		newRules, err := gitattributes.ReadAttributes(bytes.NewReader(data), dirs, isRoot)
+		if err != nil {
+			AddProblem(manifest, strings.Join(parts, "/"), "parsing .gitattributes: %v", err)
+			continue
+		}
+		rules = append(rules, newRules...)
+	}
+
+	// gitattributes.Matcher applies rules in reverse.
+	slices.Reverse(rules)
+	matcher := gitattributes.NewMatcher(rules)
+	return matcher
+}

src/headers.go

@@ -1,6 +1,7 @@
 package git_pages
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"net/http"
@@ -14,6 +15,7 @@ import (
 )
 
 var ErrHeaderNotAllowed = errors.New("custom header not allowed")
+var ErrBasicAuthNotAllowed = errors.New("basic authorization not allowed")
 
 const HeadersFileName string = "_headers"
 
@@ -74,28 +76,40 @@ func validateHeaderRule(rule headers.Rule) error {
 		if slices.Contains(unsafeHeaders, header) {
 			return fmt.Errorf("rule sets header %q (fundamentally unsafe)", header)
 		}
-		if !slices.Contains(config.Limits.AllowedCustomHeaders, header) {
-			return fmt.Errorf("rule sets header %q (not allowlisted)", header)
-		}
-		if !IsAllowedCustomHeader(header) { // make sure we don't desync
-			panic(errors.New("header check inconsistency"))
+		switch header {
+		case "Basic-Auth":
+			if !config.Limits.AllowBasicAuth {
+				return fmt.Errorf("rule sets header %q (forbidden by policy)", header)
+			}
+		default:
+			if !slices.Contains(config.Limits.AllowedCustomHeaders, header) {
+				return fmt.Errorf("rule sets header %q (not allowlisted)", header)
+			}
+			if !IsAllowedCustomHeader(header) { // make sure we don't desync
+				panic(errors.New("header check inconsistency"))
+			}
 		}
 	}
 	return nil
 }
 
 // Parses redirects file and injects rules into the manifest.
-func ProcessHeadersFile(manifest *Manifest) error {
+func ProcessHeadersFile(ctx context.Context, manifest *Manifest) error {
 	headersEntry := manifest.Contents[HeadersFileName]
 	delete(manifest.Contents, HeadersFileName)
 	if headersEntry == nil {
 		return nil
-	} else if headersEntry.GetType() != Type_InlineFile {
-		return AddProblem(manifest, HeadersFileName,
-			"not a regular file")
 	}
 
-	rules, err := headers.ParseString(string(headersEntry.GetData()))
+	data, err := GetEntryContents(ctx, headersEntry)
+	if errors.Is(err, ErrNotRegularFile) {
+		return AddProblem(manifest, HeadersFileName,
+			"not a regular file")
+	} else if err != nil {
+		return err
+	}
+
+	rules, err := headers.ParseString(string(data))
 	if err != nil {
 		return AddProblem(manifest, HeadersFileName,
 			"syntax error: %s", err)
@@ -108,16 +122,52 @@ func ProcessHeadersFile(manifest *Manifest) error {
 			continue
 		}
 		headerMap := []*Header{}
+		credentials := []*BasicCredential{}
+		hasBasicAuth := false
 		for header, values := range rule.Headers {
-			headerMap = append(headerMap, &Header{
-				Name:   proto.String(header),
-				Values: values,
-			})
+			switch header {
+			case "Basic-Auth":
+				hasBasicAuth = true
+				for _, value := range values {
+					for _, usernamePassword := range strings.Split(value, " ") {
+						if usernamePassword == "" {
+							continue
+						}
+						if username, password, found := strings.Cut(usernamePassword, ":"); !found {
+							AddProblem(manifest, HeadersFileName,
+								"rule #%d %q: malformed Basic-Auth credential", index+1, rule.Path)
+							continue
+						} else {
+							credentials = append(credentials, &BasicCredential{
+								Username: proto.String(username),
+								Password: proto.String(password),
+							})
+						}
+					}
+				}
+			default:
+				headerMap = append(headerMap, &Header{
+					Name:   proto.String(header),
+					Values: values,
+				})
+			}
 		}
+		// Note that we may add an empty `headerMap` here even if only credentials are defined.
+		// This is intentional: in `_headers` files processing terminates at the first matching
+		// clause, and Netlify mixes Basic-Auth with all the other headers.
 		manifest.Headers = append(manifest.Headers, &HeaderRule{
 			Path:      proto.String(rule.Path),
 			HeaderMap: headerMap,
 		})
+		// We're using `hasBasicAuth` instead of `len(credentials) > 0` so that if a `_headers`
+		// file defines only malformed credentials, we still add a rule (that in effect always
+		// denies access).
+		if hasBasicAuth {
+			manifest.BasicAuth = append(manifest.BasicAuth, &BasicAuthRule{
+				Path:        proto.String(rule.Path),
+				Credentials: credentials,
+			})
+		}
 	}
 	return nil
 }
@@ -137,13 +187,14 @@ func CollectHeadersFile(manifest *Manifest) string {
 	return headers.Must(headers.UnparseString(headersRules))
 }
 
-func ApplyHeaderRules(manifest *Manifest, url *url.URL) (headers http.Header, err error) {
-	headers = http.Header{}
+func matchPathRules[
+	Rule interface{ GetPath() string },
+](rules []Rule, url *url.URL) (matched Rule) {
 	fromSegments := pathSegments(url.Path)
 next:
-	for _, rule := range manifest.Headers {
+	for _, rule := range rules {
 		// check if the rule matches url
-		ruleURL, _ := url.Parse(*rule.Path) // pre-validated in `validateHeaderRule`
+		ruleURL, _ := url.Parse(rule.GetPath()) // pre-validated in `validateHeaderRule`
 		ruleSegments := pathSegments(ruleURL.Path)
 		if ruleSegments[len(ruleSegments)-1] != "*" {
 			if len(ruleSegments) < len(fromSegments) {
@@ -161,8 +212,19 @@ next:
 				continue next
 			}
 		}
+		matched = rule
+		break
+	}
+	return
+}
+
+func ApplyHeaderRules(manifest *Manifest, url *url.URL) (
+	headers http.Header, err error,
+) {
+	headers = http.Header{}
+	if rule := matchPathRules(manifest.Headers, url); rule != nil {
 		// the rule has matched url, validate headers against up-to-date policy
-		for _, header := range rule.HeaderMap {
+		for _, header := range rule.GetHeaderMap() {
 			name := header.GetName()
 			if !IsAllowedCustomHeader(name) {
 				return nil, fmt.Errorf("%w: %s", ErrHeaderNotAllowed, name)
@@ -171,7 +233,30 @@ next:
 				headers.Add(name, value)
 			}
 		}
-		break
 	}
 	return
 }
+
+func ApplyBasicAuthRules(manifest *Manifest, url *url.URL, r *http.Request) (bool, error) {
+	if rule := matchPathRules(manifest.BasicAuth, url); rule == nil {
+		// no matches, authorized by default
+		return true, nil
+	} else {
+		// the rule has matched url, check that basic auth is allowed per up-to-date policy
+		if !config.Limits.AllowBasicAuth {
+			// basic auth configured in the past but not allowed any more
+			return false, ErrBasicAuthNotAllowed
+		}
+		if username, password, ok := r.BasicAuth(); ok {
+			// request has credentials, check them
+			for _, credential := range rule.GetCredentials() {
+				if credential.GetUsername() == username && credential.GetPassword() == password {
+					// authorized!
+					return true, nil
+				}
+			}
+		}
+		// request has no credentials, unauthorized
+		return false, nil
+	}
+}

src/histogram.go

@@ -0,0 +1,35 @@
+package git_pages
+
+import (
+	"context"
+	"fmt"
+	"maps"
+	"slices"
+	"strings"
+)
+
+type DomainStatistics struct {
+	Domain         string
+	OriginalSize   int64
+	CompressedSize int64
+	StoredSize     int64
+}
+
+func SizeHistogram(ctx context.Context) ([]*DomainStatistics, error) {
+	statisticsMap := map[string]*DomainStatistics{}
+	for item, err := range backend.GetAllManifests(ctx) {
+		metadata, manifest := item.Splat()
+		if err != nil {
+			return nil, fmt.Errorf("size histogram err: %w", err)
+		}
+		domain, _, _ := strings.Cut(metadata.Name, "/")
+		if _, found := statisticsMap[domain]; !found {
+			statisticsMap[domain] = &DomainStatistics{Domain: domain}
+		}
+		statistics := statisticsMap[domain]
+		statistics.OriginalSize += metadata.Size + manifest.GetOriginalSize()
+		statistics.CompressedSize += metadata.Size + manifest.GetCompressedSize()
+		statistics.StoredSize += metadata.Size + manifest.GetStoredSize()
+	}
+	return slices.Collect(maps.Values(statisticsMap)), nil
+}

src/http.go

@@ -2,43 +2,96 @@ package git_pages
 
 import (
 	"cmp"
+	"fmt"
+	"net"
+	"net/http"
 	"regexp"
 	"slices"
 	"strconv"
 	"strings"
 )
 
-var httpAcceptEncodingRegexp = regexp.MustCompile(`` +
+var httpAcceptRegexp = regexp.MustCompile(`` +
 	// token optionally prefixed by whitespace
-	`^[ \t]*([a-zA-Z0-9$!#$%&'*+.^_\x60|~-]+)` +
+	`^[ \t]*([a-zA-Z0-9$!#$%&'*+./^_\x60|~-]+)` +
 	// quality value prefixed by a semicolon optionally surrounded by whitespace
 	`(?:[ \t]*;[ \t]*q=(0(?:\.[0-9]{1,3})?|1(?:\.0{1,3})?))?` +
 	// optional whitespace followed by comma or end of line
 	`[ \t]*(?:,|$)`,
 )
 
-type httpEncoding struct {
+type httpAcceptOffer struct {
 	code string
 	qval float64
 }
 
-type httpEncodings struct {
-	encodings []httpEncoding
-}
-
-func parseHTTPEncodings(headerValue string) (result httpEncodings) {
+func parseGenericAcceptHeader(headerValue string) (result []httpAcceptOffer) {
 	for headerValue != "" {
-		matches := httpAcceptEncodingRegexp.FindStringSubmatch(headerValue)
+		matches := httpAcceptRegexp.FindStringSubmatch(headerValue)
 		if matches == nil {
-			return httpEncodings{}
+			return
 		}
-		enc := httpEncoding{strings.ToLower(matches[1]), 1.0}
+		offer := httpAcceptOffer{strings.ToLower(matches[1]), 1.0}
 		if matches[2] != "" {
-			enc.qval, _ = strconv.ParseFloat(matches[2], 64)
+			offer.qval, _ = strconv.ParseFloat(matches[2], 64)
 		}
-		result.encodings = append(result.encodings, enc)
+		result = append(result, offer)
 		headerValue = headerValue[len(matches[0]):]
 	}
+	return
+}
+
+func preferredAcceptOffer(offers []httpAcceptOffer) string {
+	slices.SortStableFunc(offers, func(a, b httpAcceptOffer) int {
+		return -cmp.Compare(a.qval, b.qval)
+	})
+	for _, offer := range offers {
+		if offer.qval != 0 {
+			return offer.code
+		}
+	}
+	return ""
+}
+
+type HTTPContentTypes struct {
+	contentTypes []httpAcceptOffer
+}
+
+func ParseAcceptHeader(headerValue string) (result HTTPContentTypes) {
+	if headerValue == "" {
+		headerValue = "*/*"
+	}
+	result = HTTPContentTypes{parseGenericAcceptHeader(headerValue)}
+	return
+}
+
+func (e *HTTPContentTypes) Negotiate(offers ...string) string {
+	prefs := make(map[string]float64, len(offers))
+	for _, code := range offers {
+		prefs[code] = 0
+	}
+	for _, ctyp := range e.contentTypes {
+		if ctyp.code == "*/*" {
+			for code := range prefs {
+				prefs[code] = ctyp.qval
+			}
+		} else if _, ok := prefs[ctyp.code]; ok {
+			prefs[ctyp.code] = ctyp.qval
+		}
+	}
+	ctyps := make([]httpAcceptOffer, len(offers))
+	for idx, code := range offers {
+		ctyps[idx] = httpAcceptOffer{code, prefs[code]}
+	}
+	return preferredAcceptOffer(ctyps)
+}
+
+type HTTPEncodings struct {
+	encodings []httpAcceptOffer
+}
+
+func ParseAcceptEncodingHeader(headerValue string) (result HTTPEncodings) {
+	result = HTTPEncodings{parseGenericAcceptHeader(headerValue)}
 	if len(result.encodings) == 0 {
 		// RFC 9110 says (https://httpwg.org/specs/rfc9110.html#field.accept-encoding):
 		// "If no Accept-Encoding header field is in the request, any content
@@ -51,9 +104,9 @@ func parseHTTPEncodings(headerValue string) (result httpEncodings) {
 
 // Negotiate returns the most preferred encoding that is acceptable by the
 // client, or an empty string if no encodings are acceptable.
-func (e *httpEncodings) Negotiate(codes ...string) string {
-	prefs := make(map[string]float64, len(codes))
-	for _, code := range codes {
+func (e *HTTPEncodings) Negotiate(offers ...string) string {
+	prefs := make(map[string]float64, len(offers))
+	for _, code := range offers {
 		prefs[code] = 0
 	}
 	implicitIdentity := true
@@ -73,17 +126,52 @@ func (e *httpEncodings) Negotiate(codes ...string) string {
 	if _, ok := prefs["identity"]; ok && implicitIdentity {
 		prefs["identity"] = -1 // sort last
 	}
-	encs := make([]httpEncoding, len(codes))
-	for idx, code := range codes {
-		encs[idx] = httpEncoding{code, prefs[code]}
+	encs := make([]httpAcceptOffer, len(offers))
+	for idx, code := range offers {
+		encs[idx] = httpAcceptOffer{code, prefs[code]}
 	}
-	slices.SortStableFunc(encs, func(a, b httpEncoding) int {
-		return -cmp.Compare(a.qval, b.qval)
-	})
-	for _, enc := range encs {
-		if enc.qval != 0 {
-			return enc.code
-		}
-	}
-	return ""
+	return preferredAcceptOffer(encs)
+}
+
+func chainHTTPMiddleware(middleware ...func(http.Handler) http.Handler) func(http.Handler) http.Handler {
+	return func(handler http.Handler) http.Handler {
+		for idx := len(middleware) - 1; idx >= 0; idx-- {
+			handler = middleware[idx](handler)
+		}
+		return handler
+	}
+}
+
+func remoteAddrMiddleware(handler http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var readXForwardedFor bool
+		switch config.Audit.IncludeIPs {
+		case "X-Forwarded-For":
+			readXForwardedFor = true
+		case "RemoteAddr", "":
+			readXForwardedFor = false
+		default:
+			panic(fmt.Errorf("config.Audit.IncludeIPs is set to an unknown value (%q)",
+				config.Audit.IncludeIPs))
+		}
+
+		usingOriginalRemoteAddr := true
+		if readXForwardedFor {
+			forwardedFor := strings.Split(r.Header.Get("X-Forwarded-For"), ",")
+			if len(forwardedFor) > 0 {
+				remoteAddr := strings.TrimSpace(forwardedFor[len(forwardedFor)-1])
+				if remoteAddr != "" {
+					r.RemoteAddr = remoteAddr
+					usingOriginalRemoteAddr = false
+				}
+			}
+		}
+		if usingOriginalRemoteAddr {
+			if ipAddress, _, err := net.SplitHostPort(r.RemoteAddr); err == nil {
+				r.RemoteAddr = ipAddress
+			}
+		}
+
+		handler.ServeHTTP(w, r)
+	})
 }

src/main.go

@@ -1,6 +1,7 @@
 package git_pages
 
 import (
+	"cmp"
 	"context"
 	"crypto/tls"
 	"errors"
@@ -14,7 +15,10 @@ import (
 	"net/http/httputil"
 	"net/url"
 	"os"
+	"path"
 	"runtime/debug"
+	"slices"
+	"strconv"
 	"strings"
 	"time"
 
@@ -30,6 +34,7 @@ var config *Config
 var wildcards []*WildcardPattern
 var fallback http.Handler
 var backend Backend
+var domainCache DomainCache
 
 func configureFeatures(ctx context.Context) (err error) {
 	if len(config.Features) > 0 {
@@ -60,6 +65,12 @@ func configureMemLimit(ctx context.Context) (err error) {
 	return
 }
 
+// Can only be safely called during initial configuration.
+func configureConcurrency(_ context.Context) (err error) {
+	putBlobSemaphore = make(chan struct{}, config.Limits.ConcurrentUploads)
+	return
+}
+
 func configureWildcards(_ context.Context) (err error) {
 	newWildcards, err := TranslateWildcards(config.Wildcard)
 	if err != nil {
@@ -91,7 +102,7 @@ func configureFallback(_ context.Context) (err error) {
 
 // Thread-unsafe, must be called only during initial configuration.
 func configureAudit(_ context.Context) (err error) {
-	snowflake.SetStartTime(time.Date(2025, 12, 1, 0, 0, 0, 0, time.UTC))
+	snowflake.SetStartTime(AuditSnowflakeStartTime)
 	snowflake.SetMachineID(config.Audit.NodeID)
 	return
 }
@@ -118,6 +129,9 @@ func panicHandler(handler http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		defer func() {
 			if err := recover(); err != nil {
+				if err, ok := err.(error); ok && errors.Is(err, http.ErrAbortHandler) {
+					panic(http.ErrAbortHandler)
+				}
 				logc.Printf(r.Context(), "panic: %s %s %s: %s\n%s",
 					r.Method, r.Host, r.URL.Path, err, string(debug.Stack()))
 				http.Error(w,
@@ -132,8 +146,6 @@ func panicHandler(handler http.Handler) http.Handler {
 
 func serve(ctx context.Context, listener net.Listener, handler http.Handler) {
 	if listener != nil {
-		handler = panicHandler(handler)
-
 		server := http.Server{Handler: handler}
 		server.Protocols = new(http.Protocols)
 		server.Protocols.SetHTTP1(true)
@@ -172,31 +184,43 @@ func usage() {
 	fmt.Fprintf(os.Stderr, "Usage:\n")
 	fmt.Fprintf(os.Stderr, "(server) "+
 		"git-pages [-config <file>|-no-config]\n")
-	fmt.Fprintf(os.Stderr, "(admin)  "+
-		"git-pages {-run-migration <name>|-freeze-domain <domain>|-unfreeze-domain <domain>}\n")
-	fmt.Fprintf(os.Stderr, "(audit)  "+
-		"git-pages {-audit-log|-audit-read <id>|-audit-server <endpoint> <program> [args...]}\n")
 	fmt.Fprintf(os.Stderr, "(info)   "+
-		"git-pages {-print-config-env-vars|-print-config}\n")
-	fmt.Fprintf(os.Stderr, "(cli)    "+
+		"git-pages {-version|-print-config-env-vars|-print-config}\n")
+	fmt.Fprintf(os.Stderr, "(debug)  "+
+		"git-pages {-list-blobs|-list-manifests}\n")
+	fmt.Fprintf(os.Stderr, "(debug)  "+
 		"git-pages {-get-blob|-get-manifest|-get-archive|-update-site} <ref> [file]\n")
+	fmt.Fprintf(os.Stderr, "(admin)  "+
+		"git-pages {-freeze-domain|-unfreeze-domain} <domain>\n")
+	fmt.Fprintf(os.Stderr, "(audit)  "+
+		"git-pages {-audit-log|-audit-read <id>|-audit-rollback <id>}\n")
+	fmt.Fprintf(os.Stderr, "(audit)  "+
+		"git-pages {-audit-expire <days>|-audit-detach <domain>/<project>}\n")
+	fmt.Fprintf(os.Stderr, "(audit)  "+
+		"git-pages  -audit-server <endpoint> <program> [args...]\n")
+	fmt.Fprintf(os.Stderr, "(maint)  "+
+		"git-pages {-run-migration <name>|-trace-garbage|-size-histogram {original|stored}}\n")
 	flag.PrintDefaults()
 }
 
-func Main() {
+func Main(versionInfo string) {
 	ctx := context.Background()
 
 	flag.Usage = usage
+	configTomlPath := flag.String("config", "",
+		"load configuration from `filename` (default: 'config.toml')")
+	secretTomlPath := flag.String("secrets", "",
+		"load additional configuration values from `filename` (default: '$CREDENTIALS_DIRECTORY/secrets.toml' if it exists)")
+	noConfig := flag.Bool("no-config", false,
+		"run without configuration file (configure via environment variables)")
 	printConfigEnvVars := flag.Bool("print-config-env-vars", false,
 		"print every recognized configuration environment variable and exit")
 	printConfig := flag.Bool("print-config", false,
 		"print configuration as JSON and exit")
-	configTomlPath := flag.String("config", "",
-		"load configuration from `filename` (default: 'config.toml')")
-	noConfig := flag.Bool("no-config", false,
-		"run without configuration file (configure via environment variables)")
-	runMigration := flag.String("run-migration", "",
-		"run a store `migration` (one of: create-domain-markers)")
+	listBlobs := flag.Bool("list-blobs", false,
+		"enumerate every blob with its metadata")
+	listManifests := flag.Bool("list-manifests", false,
+		"enumerate every manifest with its metadata")
 	getBlob := flag.String("get-blob", "",
 		"write contents of `blob` ('sha256-xxxxxxx...xxx')")
 	getManifest := flag.String("get-manifest", "",
@@ -213,13 +237,33 @@ func Main() {
 		"display audit log")
 	auditRead := flag.String("audit-read", "",
 		"extract contents of audit record `id` to files '<id>-*'")
+	auditRollback := flag.String("audit-rollback", "",
+		"restore site from contents of audit record `id`")
+	auditExpire := flag.String("audit-expire", "",
+		"expire audit records older than `days` old")
+	auditDetach := flag.String("audit-detach", "",
+		"detach all blobs of audit records for a single `site` (or the entire domain with 'domain.tld/*')")
 	auditServer := flag.String("audit-server", "",
 		"listen for notifications on `endpoint` and spawn a process for each audit event")
+	runMigration := flag.String("run-migration", "",
+		"run a store `migration` (one of: create-domain-markers)")
+	sizeHistogram := flag.String("size-histogram", "",
+		"display histogram of `size-type` (original or stored) per domain")
+	traceGarbage := flag.Bool("trace-garbage", false,
+		"estimate total size of unreachable blobs")
+	version := flag.Bool("version", false,
+		"display version")
 	flag.Parse()
 
+	if *version {
+		fmt.Printf("git-pages %s\n", versionInfo)
+		os.Exit(0)
+	}
+
 	var cliOperations int
 	for _, selected := range []bool{
-		*runMigration != "",
+		*listBlobs,
+		*listManifests,
 		*getBlob != "",
 		*getManifest != "",
 		*getArchive != "",
@@ -228,15 +272,23 @@ func Main() {
 		*unfreezeDomain != "",
 		*auditLog,
 		*auditRead != "",
+		*auditRollback != "",
+		*auditExpire != "",
+		*auditDetach != "",
 		*auditServer != "",
+		*runMigration != "",
+		*sizeHistogram != "",
+		*traceGarbage,
 	} {
 		if selected {
 			cliOperations++
 		}
 	}
 	if cliOperations > 1 {
-		logc.Fatalln(ctx, "-get-blob, -get-manifest, -get-archive, -update-site, "+
-			"-freeze, -unfreeze, -audit-log, and -audit-read are mutually exclusive")
+		logc.Fatalln(ctx, "-list-blobs, -list-manifests, -get-blob, -get-manifest, -get-archive, "+
+			"-update-site, -freeze-domain, -unfreeze-domain, -audit-log, -audit-read, "+
+			"-audit-rollback, -audit-expire, -audit-detach, -audit-server, -run-migration, "+
+			"-size-histogram, and -trace-garbage are mutually exclusive")
 	}
 
 	if *configTomlPath != "" && *noConfig {
@@ -252,12 +304,24 @@ func Main() {
 	if *configTomlPath == "" && !*noConfig {
 		*configTomlPath = "config.toml"
 	}
-	if config, err = Configure(*configTomlPath); err != nil {
+
+	if *secretTomlPath == "" && !*noConfig {
+		// check for a second config file at $CREDENTIALS_DIRECTORY/secrets.toml, and use it
+		if systemdCredentialsDir := os.Getenv("CREDENTIALS_DIRECTORY"); systemdCredentialsDir != "" {
+			secretTomlTestPath := path.Join(systemdCredentialsDir, "secrets.toml")
+			_, err := os.Stat(secretTomlTestPath)
+			if !errors.Is(err, os.ErrNotExist) {
+				*secretTomlPath = secretTomlTestPath
+			}
+		}
+	}
+
+	if config, err = Configure(*configTomlPath, *secretTomlPath); err != nil {
 		logc.Fatalln(ctx, "config:", err)
 	}
 
 	if *printConfig {
-		fmt.Println(config.DebugJSON())
+		fmt.Println(config.TOML())
 		return
 	}
 
@@ -267,6 +331,7 @@ func Main() {
 	if err = errors.Join(
 		configureFeatures(ctx),
 		configureMemLimit(ctx),
+		configureConcurrency(ctx),
 		configureWildcards(ctx),
 		configureFallback(ctx),
 		configureAudit(ctx),
@@ -274,32 +339,50 @@ func Main() {
 		logc.Fatalln(ctx, err)
 	}
 
-	switch {
-	case *runMigration != "":
+	// The server has its own logic for creating the backend.
+	if cliOperations > 0 {
 		if backend, err = CreateBackend(ctx, &config.Storage); err != nil {
 			logc.Fatalln(ctx, err)
 		}
 
-		if err := RunMigration(ctx, *runMigration); err != nil {
+		if domainCache, err = CreateDomainCache(ctx); err != nil {
 			logc.Fatalln(ctx, err)
 		}
+	}
+
+	switch {
+	case *listBlobs:
+		for metadata, err := range backend.EnumerateBlobs(ctx) {
+			if err != nil {
+				logc.Fatalln(ctx, err)
+			}
+			fmt.Fprintf(color.Output, "%s %s %s\n",
+				metadata.Name,
+				color.HiWhiteString(metadata.LastModified.UTC().Format(time.RFC3339)),
+				color.HiGreenString(fmt.Sprint(metadata.Size)),
+			)
+		}
+
+	case *listManifests:
+		for metadata, err := range backend.EnumerateManifests(ctx) {
+			if err != nil {
+				logc.Fatalln(ctx, err)
+			}
+			fmt.Fprintf(color.Output, "%s %s %s\n",
+				metadata.Name,
+				color.HiWhiteString(metadata.LastModified.UTC().Format(time.RFC3339)),
+				color.HiGreenString(fmt.Sprint(metadata.Size)),
+			)
+		}
 
 	case *getBlob != "":
-		if backend, err = CreateBackend(ctx, &config.Storage); err != nil {
-			logc.Fatalln(ctx, err)
-		}
-
-		reader, _, _, err := backend.GetBlob(ctx, *getBlob)
+		reader, _, err := backend.GetBlob(ctx, *getBlob)
 		if err != nil {
 			logc.Fatalln(ctx, err)
 		}
 		io.Copy(fileOutputArg(), reader)
 
 	case *getManifest != "":
-		if backend, err = CreateBackend(ctx, &config.Storage); err != nil {
-			logc.Fatalln(ctx, err)
-		}
-
 		webRoot := webRootArg(*getManifest)
 		manifest, _, err := backend.GetManifest(ctx, webRoot, GetManifestOptions{})
 		if err != nil {
@@ -308,10 +391,6 @@ func Main() {
 		fmt.Fprintln(fileOutputArg(), string(ManifestJSON(manifest)))
 
 	case *getArchive != "":
-		if backend, err = CreateBackend(ctx, &config.Storage); err != nil {
-			logc.Fatalln(ctx, err)
-		}
-
 		webRoot := webRootArg(*getArchive)
 		manifest, metadata, err :=
 			backend.GetManifest(ctx, webRoot, GetManifestOptions{})
@@ -326,10 +405,6 @@ func Main() {
 		ctx = WithPrincipal(ctx)
 		GetPrincipal(ctx).CliAdmin = proto.Bool(true)
 
-		if backend, err = CreateBackend(ctx, &config.Storage); err != nil {
-			logc.Fatalln(ctx, err)
-		}
-
 		if flag.NArg() != 1 {
 			logc.Fatalln(ctx, "update source must be provided as the argument")
 		}
@@ -362,7 +437,7 @@ func Main() {
 			}
 
 			webRoot := webRootArg(*updateSite)
-			result = UpdateFromArchive(ctx, webRoot, contentType, file)
+			result = UpdateFromArchive(ctx, webRoot, "", contentType, file)
 		} else {
 			branch := "pages"
 			if sourceURL.Fragment != "" {
@@ -404,62 +479,49 @@ func Main() {
 			freeze = false
 		}
 
-		if backend, err = CreateBackend(ctx, &config.Storage); err != nil {
-			logc.Fatalln(ctx, err)
-		}
-
-		if err = backend.FreezeDomain(ctx, domain, freeze); err != nil {
-			logc.Fatalln(ctx, err)
-		}
 		if freeze {
+			if err = backend.FreezeDomain(ctx, domain); err != nil {
+				logc.Fatalln(ctx, err)
+			}
 			logc.Println(ctx, "frozen")
 		} else {
+			if err = backend.UnfreezeDomain(ctx, domain); err != nil {
+				logc.Fatalln(ctx, err)
+			}
 			logc.Println(ctx, "thawed")
 		}
 
 	case *auditLog:
-		if backend, err = CreateBackend(ctx, &config.Storage); err != nil {
-			logc.Fatalln(ctx, err)
-		}
-
-		ch := make(chan *AuditRecord)
-		ids := []AuditID{}
-		for id, err := range backend.SearchAuditLog(ctx, SearchAuditLogOptions{}) {
+		records := []*AuditRecord{}
+		ids := backend.SearchAuditLog(ctx, SearchAuditLogOptions{})
+		for record, err := range backend.GetAuditLogRecords(ctx, ids) {
 			if err != nil {
 				logc.Fatalln(ctx, err)
 			}
-			go func() {
-				if record, err := backend.QueryAuditLog(ctx, id); err != nil {
-					logc.Fatalln(ctx, err)
-				} else {
-					ch <- record
-				}
-			}()
-			ids = append(ids, id)
+			records = append(records, record)
 		}
 
-		records := map[AuditID]*AuditRecord{}
-		for len(records) < len(ids) {
-			record := <-ch
-			records[record.GetAuditID()] = record
-		}
+		slices.SortFunc(records, func(a, b *AuditRecord) int {
+			return cmp.Compare(a.GetAuditID(), b.GetAuditID())
+		})
 
-		for _, id := range ids {
-			record := records[id]
-			fmt.Fprintf(color.Output, "%s %s %s %s %s\n",
+		for _, record := range records {
+			parts := []string{
 				record.GetAuditID().String(),
 				color.HiWhiteString(record.GetTimestamp().AsTime().UTC().Format(time.RFC3339)),
-				color.HiMagentaString(record.DescribePrincipal()),
+				fmt.Sprint(record.GetEvent()),
 				color.HiGreenString(record.DescribeResource()),
-				record.GetEvent(),
-			)
+				color.HiMagentaString(record.DescribePrincipal()),
+			}
+			if record.IsDetached() {
+				parts = append(parts,
+					color.HiYellowString("(detached)"),
+				)
+			}
+			fmt.Fprintln(color.Output, strings.Join(parts, " "))
 		}
 
 	case *auditRead != "":
-		if backend, err = CreateBackend(ctx, &config.Storage); err != nil {
-			logc.Fatalln(ctx, err)
-		}
-
 		id, err := ParseAuditID(*auditRead)
 		if err != nil {
 			logc.Fatalln(ctx, err)
@@ -474,11 +536,74 @@ func Main() {
 			logc.Fatalln(ctx, err)
 		}
 
-	case *auditServer != "":
-		if backend, err = CreateBackend(ctx, &config.Storage); err != nil {
+	case *auditRollback != "":
+		ctx = WithPrincipal(ctx)
+		GetPrincipal(ctx).CliAdmin = proto.Bool(true)
+
+		id, err := ParseAuditID(*auditRollback)
+		if err != nil {
 			logc.Fatalln(ctx, err)
 		}
 
+		record, err := backend.QueryAuditLog(ctx, id)
+		if err != nil {
+			logc.Fatalln(ctx, err)
+		}
+
+		if record.GetManifest() == nil || record.GetDomain() == "" || record.GetProject() == "" {
+			logc.Fatalln(ctx, "no manifest in audit record")
+		}
+
+		webRoot := path.Join(record.GetDomain(), record.GetProject())
+		err = backend.StageManifest(ctx, record.GetManifest())
+		if err != nil {
+			logc.Fatalln(ctx, err)
+		}
+		err = backend.CommitManifest(ctx, webRoot, record.GetManifest(), ModifyManifestOptions{})
+		if err != nil {
+			logc.Fatalln(ctx, err)
+		}
+
+	case *auditDetach != "":
+		domain, project, found := strings.Cut(*auditDetach, "/")
+		if !found || domain == "" || project == "" {
+			logc.Fatalln(ctx, "argument to -audit-detach must be in the form of "+
+				"'domain.tld/project' or 'domain.tld/*'")
+		}
+
+		if project != "*" && project != ".index" {
+			if err := ValidateProjectName(project); err != nil {
+				logc.Fatalf(ctx, "audit detach: project name: %v\n", err)
+			}
+		}
+
+		count := 0
+		ids := backend.SearchAuditLog(ctx, SearchAuditLogOptions{})
+		for record, err := range backend.GetAuditLogRecords(ctx, ids) {
+			if err != nil {
+				logc.Fatalln(ctx, err)
+			}
+			if record.GetDomain() == domain && (project == "*" || record.GetProject() == project) {
+				if !record.IsDetachable() {
+					continue
+				} else if !record.IsDetached() {
+					logc.Printf(ctx, "detaching audit record %s\n", record.GetAuditID())
+					err = backend.DetachAuditRecord(ctx, record.GetAuditID())
+					if err != nil {
+						logc.Fatalln(ctx, err)
+					}
+					count++
+				} else {
+					logc.Printf(ctx, "audit record %s already detached\n", record.GetAuditID())
+				}
+			}
+		}
+
+		if count == 0 {
+			logc.Printf(ctx, "no detachable audit records found for %s/%s", domain, project)
+		}
+
+	case *auditServer != "":
 		if flag.NArg() < 1 {
 			logc.Fatalln(ctx, "handler path not provided")
 		}
@@ -490,6 +615,86 @@ func Main() {
 
 		serve(ctx, listen(ctx, "audit", *auditServer), ObserveHTTPHandler(processor))
 
+	case *auditExpire != "":
+		days, err := strconv.ParseInt(*auditExpire, 10, 0)
+		if err != nil {
+			logc.Fatalln(ctx, err)
+		}
+
+		ids := backend.SearchAuditLog(ctx, SearchAuditLogOptions{
+			Until: time.Now().AddDate(0, 0, int(-days)),
+		})
+
+		count := 0
+		for id, err := range ids {
+			if err != nil {
+				logc.Fatalln(ctx, err)
+				continue
+			}
+
+			err = backend.ExpireAuditRecord(ctx, id)
+			if err != nil {
+				logc.Fatalln(ctx, err)
+			} else {
+				logc.Printf(ctx, "audit: expired record %s\n", id)
+				count += 1
+			}
+		}
+
+		logc.Printf(ctx, "audit: expired %d records\n", count)
+
+	case *runMigration != "":
+		if err = RunMigration(ctx, *runMigration); err != nil {
+			logc.Fatalln(ctx, err)
+		}
+
+	case *sizeHistogram != "":
+		extractSize := func(s *DomainStatistics) int64 { return 0 }
+		switch *sizeHistogram {
+		case "original":
+			// Displays a size histogram using the `manifest.OriginalSize`, which is useful to see
+			// which site is the closest to hitting the size limit (checked against apparent size).
+			// This apparent size does not have any direct relationship with used storage.
+			extractSize = func(s *DomainStatistics) int64 { return s.OriginalSize }
+		case "stored":
+			// Displays a size histogram using the `manifest.StoredSize`, which is useful to see
+			// which site consumes the most resources. The site is keeping at least this many
+			// bytes worth of blobs alive, but removing it may not free any space because
+			// deduplication is global.
+			extractSize = func(s *DomainStatistics) int64 { return s.StoredSize }
+		default:
+			logc.Fatalln(ctx, "unknown histogram type")
+		}
+
+		histogram, err := SizeHistogram(ctx)
+		if err != nil {
+			logc.Fatalln(ctx, err)
+		}
+		slices.SortFunc(histogram, func(a *DomainStatistics, b *DomainStatistics) int {
+			return cmp.Compare(extractSize(a), extractSize(b))
+		})
+
+		if len(histogram) > 0 {
+			fullScaleSize := max(extractSize(histogram[len(histogram)-1]), 1)
+			fullScaleWidth := int64(40)
+			for _, statistics := range histogram {
+				size := extractSize(statistics)
+				barWidth := size * fullScaleWidth / fullScaleSize
+				spaceWidth := fullScaleWidth - barWidth
+				bar := strings.Repeat("*", int(barWidth)) + strings.Repeat(" ", int(spaceWidth))
+				fmt.Fprintf(color.Output, "%s %s %s\n",
+					color.HiBlackString(fmt.Sprint("|", bar, "|")),
+					statistics.Domain,
+					color.HiGreenString(datasize.ByteSize(extractSize(statistics)).HR()),
+				)
+			}
+		}
+
+	case *traceGarbage:
+		if err = TraceGarbage(ctx); err != nil {
+			logc.Fatalln(ctx, err)
+		}
+
 	default:
 		// Hook a signal (SIGHUP on *nix, nothing on Windows) for reloading the configuration
 		// at runtime. This is useful because it preserves S3 backend cache contents. Failed
@@ -499,7 +704,7 @@ func Main() {
 		// Note that not all of the configuration is updated on reload. Listeners are kept as-is.
 		// The backend is not recreated (this is intentional as it allows preserving the cache).
 		OnReload(func() {
-			if newConfig, err := Configure(*configTomlPath); err != nil {
+			if newConfig, err := Configure(*configTomlPath, *secretTomlPath); err != nil {
 				logc.Println(ctx, "config: reload err:", err)
 			} else {
 				// From https://go.dev/ref/mem:
@@ -537,8 +742,17 @@ func Main() {
 		}
 		backend = NewObservedBackend(backend)
 
-		go serve(ctx, pagesListener, ObserveHTTPHandler(http.HandlerFunc(ServePages)))
-		go serve(ctx, caddyListener, ObserveHTTPHandler(http.HandlerFunc(ServeCaddy)))
+		if domainCache, err = CreateDomainCache(ctx); err != nil {
+			logc.Fatalln(ctx, err)
+		}
+
+		middleware := chainHTTPMiddleware(
+			panicHandler,
+			remoteAddrMiddleware,
+			ObserveHTTPHandler,
+		)
+		go serve(ctx, pagesListener, middleware(http.HandlerFunc(ServePages)))
+		go serve(ctx, caddyListener, middleware(http.HandlerFunc(ServeCaddy)))
 		go serve(ctx, metricsListener, promhttp.Handler())
 
 		if config.Insecure {

src/manifest.go

@@ -8,6 +8,7 @@ import (
 	"crypto/sha256"
 	"errors"
 	"fmt"
+	"io"
 	"mime"
 	"net/http"
 	"path"
@@ -17,6 +18,8 @@ import (
 	"time"
 
 	"github.com/c2h5oh/datasize"
+	"github.com/go-git/go-git/v6/plumbing"
+	format "github.com/go-git/go-git/v6/plumbing/format/config"
 	"github.com/klauspost/compress/zstd"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
@@ -102,39 +105,125 @@ func NewManifestEntry(type_ Type, data []byte) *Entry {
 	return entry
 }
 
-func AddFile(manifest *Manifest, path string, data []byte) *Entry {
+func AddFile(manifest *Manifest, fileName string, data []byte) *Entry {
+	// Fill in `git_hash` even for files not originating from git using the SHA256 algorithm;
+	// we use this primarily for incremental archive uploads, but when support for git SHA256
+	// repositories is complete, archive uploads and git checkouts will have cross-support for
+	// incremental updates.
+	hasher := plumbing.NewHasher(format.SHA256, plumbing.BlobObject, int64(len(data)))
+	hasher.Write(data)
 	entry := NewManifestEntry(Type_InlineFile, data)
-	manifest.Contents[path] = entry
+	entry.GitHash = proto.String(hasher.Sum().String())
+	manifest.Contents[fileName] = entry
 	return entry
 }
 
-func AddSymlink(manifest *Manifest, path string, target string) *Entry {
-	entry := NewManifestEntry(Type_Symlink, []byte(target))
-	manifest.Contents[path] = entry
-	return entry
+func AddSymlink(manifest *Manifest, fileName string, target string) *Entry {
+	if path.IsAbs(target) {
+		AddProblem(manifest, fileName, "absolute symlink: %s", target)
+		return nil
+	} else {
+		entry := NewManifestEntry(Type_Symlink, []byte(target))
+		manifest.Contents[fileName] = entry
+		return entry
+	}
 }
 
-func AddDirectory(manifest *Manifest, path string) *Entry {
-	path = strings.TrimSuffix(path, "/")
+func AddDirectory(manifest *Manifest, dirName string) *Entry {
+	dirName = strings.TrimSuffix(dirName, "/")
 	entry := NewManifestEntry(Type_Directory, nil)
-	manifest.Contents[path] = entry
+	manifest.Contents[dirName] = entry
 	return entry
 }
 
-func AddProblem(manifest *Manifest, path, format string, args ...any) error {
+func AddProblem(manifest *Manifest, pathName, format string, args ...any) error {
 	cause := fmt.Sprintf(format, args...)
 	manifest.Problems = append(manifest.Problems, &Problem{
-		Path:  proto.String(path),
+		Path:  proto.String(pathName),
 		Cause: proto.String(cause),
 	})
-	return fmt.Errorf("%s: %s", path, cause)
+	return fmt.Errorf("%s: %s", pathName, cause)
+}
+
+// Returns a map of git hash to entry. If `manifest` is nil, returns an empty map.
+func IndexManifestByGitHash(manifest *Manifest) map[string]*Entry {
+	index := map[string]*Entry{}
+	for _, entry := range manifest.GetContents() {
+		if hash := entry.GetGitHash(); hash != "" {
+			if _, ok := plumbing.FromHex(hash); ok {
+				index[hash] = entry
+			} else {
+				panic(fmt.Errorf("index: malformed hash: %s", hash))
+			}
+		}
+	}
+	return index
+}
+
+func ManifestHasBasicAuth(manifest *Manifest) bool {
+	return len(manifest.GetBasicAuth()) > 0
+}
+
+func IsEntryRegularFile(entry *Entry) bool {
+	return entry.GetType() == Type_InlineFile ||
+		entry.GetType() == Type_ExternalFile
+}
+
+var ErrNotRegularFile = errors.New("not a regular file")
+
+func GetEntryContents(ctx context.Context, entry *Entry) (data []byte, err error) {
+	switch entry.GetType() {
+	case Type_InlineFile:
+		data = entry.GetData()
+	case Type_ExternalFile:
+		reader, _, err := backend.GetBlob(ctx, string(entry.GetData()))
+		if err != nil {
+			return nil, err
+		}
+		data, err = io.ReadAll(reader)
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, ErrNotRegularFile
+	}
+
+	switch entry.GetTransform() {
+	case Transform_Identity:
+	case Transform_Zstd:
+		data, err = zstdDecoder.DecodeAll(data, []byte{})
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, fmt.Errorf("unexpected transform")
+	}
+
+	return
+}
+
+// EnsureLeadingDirectories adds directory entries for any parent directories
+// that are implicitly referenced by files in the manifest but don't have
+// explicit directory entries. (This can be the case if an archive is created
+// via globs rather than including a whole directory.)
+func EnsureLeadingDirectories(manifest *Manifest) {
+	for name := range manifest.Contents {
+		for dir := path.Dir(name); dir != "." && dir != ""; dir = path.Dir(dir) {
+			if dir == "/" {
+				panic("malformed manifest (paths must not be rooted in /)")
+			}
+			if _, exists := manifest.Contents[dir]; !exists {
+				AddDirectory(manifest, dir)
+			}
+		}
+	}
 }
 
 func GetProblemReport(manifest *Manifest) []string {
 	var report []string
 	for _, problem := range manifest.Problems {
 		report = append(report,
-			fmt.Sprintf("%s: %s", problem.GetPath(), problem.GetCause()))
+			fmt.Sprintf("/%s: %s", problem.GetPath(), problem.GetCause()))
 	}
 	return report
 }
@@ -243,31 +332,41 @@ func CompressFiles(ctx context.Context, manifest *Manifest) {
 // At the moment, there isn't a good way to report errors except to log them on the terminal.
 // (Perhaps in the future they could be exposed at `.git-pages/status.txt`?)
 func PrepareManifest(ctx context.Context, manifest *Manifest) error {
-	// Parse Netlify-style `_redirects`
-	if err := ProcessRedirectsFile(manifest); err != nil {
+	// Parse Netlify-style `_redirects`.
+	if err := ProcessRedirectsFile(ctx, manifest); err != nil {
 		logc.Printf(ctx, "redirects err: %s\n", err)
 	} else if len(manifest.Redirects) > 0 {
 		logc.Printf(ctx, "redirects ok: %d rules\n", len(manifest.Redirects))
 	}
 
-	// Parse Netlify-style `_headers`
-	if err := ProcessHeadersFile(manifest); err != nil {
+	// Check if any redirects are unreachable.
+	LintRedirects(manifest)
+
+	// Parse Netlify-style `_headers`.
+	if err := ProcessHeadersFile(ctx, manifest); err != nil {
 		logc.Printf(ctx, "headers err: %s\n", err)
 	} else if len(manifest.Headers) > 0 {
 		logc.Printf(ctx, "headers ok: %d rules\n", len(manifest.Headers))
 	}
 
-	// Sniff content type like `http.ServeContent`
+	// Sniff content type like `http.ServeContent`.
 	DetectContentType(manifest)
 
-	// Opportunistically compress blobs (must be done last)
+	// Opportunistically compress blobs (must be done last).
 	CompressFiles(ctx, manifest)
 
 	return nil
 }
 
+var ErrSiteTooLarge = errors.New("site too large")
 var ErrManifestTooLarge = errors.New("manifest too large")
 
+// Limits the number of concurrent uploads, globally across the entire git-pages process.
+// As created, there is no limit, but reinitializing the semaphore with a bounded channel
+// limits the concurrency to the channel size. Note that the default *configuration* does
+// limit the number of uploads.
+var putBlobSemaphore = make(chan struct{})
+
 // Uploads inline file data over certain size to the storage backend. Returns a copy of
 // the manifest updated to refer to an external content-addressable store.
 func StoreManifest(
@@ -276,19 +375,11 @@ func StoreManifest(
 	span, ctx := ObserveFunction(ctx, "StoreManifest", "manifest.name", name)
 	defer span.Finish()
 
+	extManifest := &Manifest{}
+	proto.Merge(extManifest, manifest)
+
 	// Replace inline files over certain size with references to external data.
-	extManifest := Manifest{
-		RepoUrl:        manifest.RepoUrl,
-		Branch:         manifest.Branch,
-		Commit:         manifest.Commit,
-		Contents:       make(map[string]*Entry),
-		Redirects:      manifest.Redirects,
-		Headers:        manifest.Headers,
-		Problems:       manifest.Problems,
-		OriginalSize:   manifest.OriginalSize,
-		CompressedSize: manifest.CompressedSize,
-		StoredSize:     proto.Int64(0),
-	}
+	extManifest.Contents = make(map[string]*Entry)
 	for name, entry := range manifest.Contents {
 		cannotBeInlined := entry.GetType() == Type_InlineFile &&
 			entry.GetCompressedSize() > int64(config.Limits.MaxInlineFileSize.Bytes())
@@ -308,19 +399,29 @@ func StoreManifest(
 		}
 	}
 
-	// Compute the deduplicated storage size.
-	var blobSizes = make(map[string]int64)
-	for _, entry := range manifest.Contents {
+	// Compute the total and deduplicated storage size.
+	totalSize := int64(0)
+	blobSizes := map[string]int64{}
+	for _, entry := range extManifest.Contents {
+		totalSize += entry.GetOriginalSize()
 		if entry.GetType() == Type_ExternalFile {
 			blobSizes[string(entry.Data)] = entry.GetCompressedSize()
 		}
 	}
+	if uint64(totalSize) > config.Limits.MaxSiteSize.Bytes() {
+		return nil, fmt.Errorf("%w: contents size %s exceeds %s limit",
+			ErrSiteTooLarge,
+			datasize.ByteSize(totalSize).HR(),
+			config.Limits.MaxSiteSize.HR(),
+		)
+	}
+	extManifest.StoredSize = proto.Int64(0)
 	for _, blobSize := range blobSizes {
 		*extManifest.StoredSize += blobSize
 	}
 
 	// Upload the resulting manifest and the blob it references.
-	extManifestData := EncodeManifest(&extManifest)
+	extManifestData := EncodeManifest(extManifest)
 	if uint64(len(extManifestData)) > config.Limits.MaxManifestSize.Bytes() {
 		return nil, fmt.Errorf("%w: manifest size %s exceeds %s limit",
 			ErrManifestTooLarge,
@@ -329,15 +430,20 @@ func StoreManifest(
 		)
 	}
 
-	if err := backend.StageManifest(ctx, &extManifest); err != nil {
+	if err := backend.StageManifest(ctx, extManifest); err != nil {
 		return nil, fmt.Errorf("stage manifest: %w", err)
 	}
 
 	wg := sync.WaitGroup{}
 	ch := make(chan error, len(extManifest.Contents))
 	for name, entry := range extManifest.Contents {
-		if entry.GetType() == Type_ExternalFile {
+		// Upload external entries (those that were decided as ineligible for being stored inline).
+		// If the entry in the original manifest is already an external reference, there's no need
+		// to externalize it (and no way for us to do so, since the entry only contains the blob name).
+		if entry.GetType() == Type_ExternalFile && manifest.Contents[name].GetType() == Type_InlineFile {
+			putBlobSemaphore <- struct{}{} // acquire (and maybe block)
 			wg.Go(func() {
+				defer func() { <-putBlobSemaphore }() // release
 				err := backend.PutBlob(ctx, string(entry.Data), manifest.Contents[name].Data)
 				if err != nil {
 					ch <- fmt.Errorf("put blob %s: %w", name, err)
@@ -351,7 +457,7 @@ func StoreManifest(
 		return nil, err // currently ignores all but 1st error
 	}
 
-	if err := backend.CommitManifest(ctx, name, &extManifest, opts); err != nil {
+	if err := backend.CommitManifest(ctx, name, extManifest, opts); err != nil {
 		if errors.Is(err, ErrDomainFrozen) {
 			return nil, err
 		} else {
@@ -359,5 +465,5 @@ func StoreManifest(
 		}
 	}
 
-	return &extManifest, nil
+	return extManifest, nil
 }

src/migrate.go

@@ -22,12 +22,15 @@ func createDomainMarkers(ctx context.Context) error {
 		return nil
 	}
 
-	var manifests, domains []string
-	manifests, err := backend.ListManifests(ctx)
-	if err != nil {
-		return fmt.Errorf("list manifests: %w", err)
+	var manifests []string
+	for metadata, err := range backend.EnumerateManifests(ctx) {
+		if err != nil {
+			return fmt.Errorf("enum manifests: %w", err)
+		}
+		manifests = append(manifests, metadata.Name)
 	}
 	slices.Sort(manifests)
+	var domains []string
 	for _, manifest := range manifests {
 		domain, _, _ := strings.Cut(manifest, "/")
 		if len(domains) == 0 || domains[len(domains)-1] != domain {

src/observe.go

@@ -8,12 +8,9 @@ import (
 	"iter"
 	"log"
 	"log/slog"
-	"math/rand/v2"
 	"net/http"
 	"os"
 	"runtime/debug"
-	"strconv"
-	"strings"
 	"sync"
 	"time"
 
@@ -23,10 +20,6 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
-
-	"github.com/getsentry/sentry-go"
-	sentryhttp "github.com/getsentry/sentry-go/http"
-	sentryslog "github.com/getsentry/sentry-go/slog"
 )
 
 var (
@@ -48,43 +41,20 @@ var (
 
 var syslogHandler syslog.Handler
 
-func hasSentry() bool {
-	return os.Getenv("SENTRY_DSN") != ""
-}
-
 func InitObservability() {
 	debug.SetPanicOnFault(true)
 
-	environment := "development"
-	if value, ok := os.LookupEnv("ENVIRONMENT"); ok {
-		environment = value
-	}
-
 	logHandlers := []slog.Handler{}
 
-	logLevel := slog.LevelInfo
-	switch strings.ToLower(config.LogLevel) {
-	case "debug":
-		logLevel = slog.LevelDebug
-	case "info":
-		logLevel = slog.LevelInfo
-	case "warn":
-		logLevel = slog.LevelWarn
-	case "error":
-		logLevel = slog.LevelError
-	default:
-		log.Println("unknown log level", config.LogLevel)
-	}
-
 	switch config.LogFormat {
 	case "none":
 		// nothing to do
 	case "text":
 		logHandlers = append(logHandlers,
-			slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: logLevel}))
+			slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{}))
 	case "json":
 		logHandlers = append(logHandlers,
-			slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: logLevel}))
+			slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{}))
 	default:
 		log.Println("unknown log format", config.LogFormat)
 	}
@@ -102,86 +72,15 @@ func InitObservability() {
 		logHandlers = append(logHandlers, syslogHandler)
 	}
 
-	if hasSentry() {
-		enableLogs := false
-		if value, err := strconv.ParseBool(os.Getenv("SENTRY_LOGS")); err == nil {
-			enableLogs = value
-		}
-
-		enableTracing := false
-		if value, err := strconv.ParseBool(os.Getenv("SENTRY_TRACING")); err == nil {
-			enableTracing = value
-		}
-
-		options := sentry.ClientOptions{}
-		options.DisableTelemetryBuffer = !config.Feature("sentry-telemetry-buffer")
-		options.Environment = environment
-		options.EnableLogs = enableLogs
-		options.EnableTracing = enableTracing
-		options.TracesSampleRate = 1
-		switch environment {
-		case "development", "staging":
-		default:
-			options.BeforeSendTransaction = func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event {
-				sampleRate := 0.05
-				if trace, ok := event.Contexts["trace"]; ok {
-					if data, ok := trace["data"].(map[string]any); ok {
-						if method, ok := data["http.request.method"].(string); ok {
-							switch method {
-							case "PUT", "DELETE", "POST":
-								sampleRate = 1
-							default:
-								duration := event.Timestamp.Sub(event.StartTime)
-								threshold := time.Duration(config.Observability.SlowResponseThreshold)
-								if duration >= threshold {
-									sampleRate = 1
-								}
-							}
-						}
-					}
-				}
-				if rand.Float64() < sampleRate {
-					return event
-				}
-				return nil
-			}
-		}
-		if err := sentry.Init(options); err != nil {
-			log.Fatalf("sentry: %s\n", err)
-		}
-
-		if enableLogs {
-			logHandlers = append(logHandlers, sentryslog.Option{
-				AddSource: true,
-				LogLevel:  levelsFromMinimum(logLevel),
-			}.NewSentryHandler(context.Background()))
-		}
-	}
-
 	slog.SetDefault(slog.New(slogmulti.Fanout(logHandlers...)))
 }
 
-// From sentryslog, because for some reason they don't make it public.
-func levelsFromMinimum(minLevel slog.Level) []slog.Level {
-	allLevels := []slog.Level{slog.LevelDebug, slog.LevelInfo, slog.LevelWarn, slog.LevelError, sentryslog.LevelFatal}
-	var result []slog.Level
-	for _, level := range allLevels {
-		if level >= minLevel {
-			result = append(result, level)
-		}
-	}
-	return result
-}
-
 func FiniObservability() {
 	var wg sync.WaitGroup
 	timeout := 2 * time.Second
 	if syslogHandler != nil {
 		wg.Go(func() { syslogHandler.Flush(timeout) })
 	}
-	if hasSentry() {
-		wg.Go(func() { sentry.Flush(timeout) })
-	}
 	wg.Wait()
 }
 
@@ -191,10 +90,6 @@ func ObserveError(err error) {
 		// Timeout results in a different error.
 		return
 	}
-
-	if hasSentry() {
-		sentry.CaptureException(err)
-	}
 }
 
 type observedResponseWriter struct {
@@ -227,22 +122,6 @@ func (w *observedResponseWriter) WriteHeader(statusCode int) {
 }
 
 func ObserveHTTPHandler(handler http.Handler) http.Handler {
-	if hasSentry() {
-		handler = func(next http.Handler) http.Handler {
-			next = sentryhttp.New(sentryhttp.Options{
-				Repanic: true,
-			}).Handle(handler)
-
-			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-				// Prevent the Sentry SDK from continuing traces as we don't use this feature.
-				r.Header.Del(sentry.SentryTraceHeader)
-				r.Header.Del(sentry.SentryBaggageHeader)
-
-				next.ServeHTTP(w, r)
-			})
-		}(handler)
-	}
-
 	handler = func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			ow := newObservedResponseWriter(w)
@@ -273,23 +152,12 @@ func ObserveFunction(
 	interface{ Finish() }, context.Context,
 ) {
 	switch {
-	case hasSentry():
-		span := sentry.StartSpan(ctx, "function")
-		span.Description = funcName
-		ObserveData(span.Context(), data...)
-		return span, span.Context()
 	default:
 		return noopSpan{}, ctx
 	}
 }
 
 func ObserveData(ctx context.Context, data ...any) {
-	if span := sentry.SpanFromContext(ctx); span != nil {
-		for i := 0; i < len(data); i += 2 {
-			name, value := data[i], data[i+1]
-			span.SetData(name.(string), value)
-		}
-	}
 }
 
 var (
@@ -344,13 +212,13 @@ func (backend *observedBackend) EnableFeature(ctx context.Context, feature Backe
 func (backend *observedBackend) GetBlob(
 	ctx context.Context, name string,
 ) (
-	reader io.ReadSeeker, size uint64, mtime time.Time, err error,
+	reader io.ReadSeeker, metadata BlobMetadata, err error,
 ) {
 	span, ctx := ObserveFunction(ctx, "GetBlob", "blob.name", name)
-	if reader, size, mtime, err = backend.inner.GetBlob(ctx, name); err == nil {
-		ObserveData(ctx, "blob.size", size)
+	if reader, metadata, err = backend.inner.GetBlob(ctx, name); err == nil {
+		ObserveData(ctx, "blob.size", metadata.Size)
 		blobsRetrievedCount.Inc()
-		blobsRetrievedBytes.Add(float64(size))
+		blobsRetrievedBytes.Add(float64(metadata.Size))
 	}
 	span.Finish()
 	return
@@ -373,11 +241,16 @@ func (backend *observedBackend) DeleteBlob(ctx context.Context, name string) (er
 	return
 }
 
-func (backend *observedBackend) ListManifests(ctx context.Context) (manifests []string, err error) {
-	span, ctx := ObserveFunction(ctx, "ListManifests")
-	manifests, err = backend.inner.ListManifests(ctx)
-	span.Finish()
-	return
+func (backend *observedBackend) EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error] {
+	return func(yield func(BlobMetadata, error) bool) {
+		span, ctx := ObserveFunction(ctx, "EnumerateBlobs")
+		for metadata, err := range backend.inner.EnumerateBlobs(ctx) {
+			if !yield(metadata, err) {
+				break
+			}
+		}
+		span.Finish()
+	}
 }
 
 func (backend *observedBackend) GetManifest(
@@ -421,6 +294,30 @@ func (backend *observedBackend) DeleteManifest(ctx context.Context, name string,
 	return
 }
 
+func (backend *observedBackend) EnumerateManifests(ctx context.Context) iter.Seq2[*ManifestMetadata, error] {
+	return func(yield func(*ManifestMetadata, error) bool) {
+		span, ctx := ObserveFunction(ctx, "EnumerateManifests")
+		for metadata, err := range backend.inner.EnumerateManifests(ctx) {
+			if !yield(metadata, err) {
+				break
+			}
+		}
+		span.Finish()
+	}
+}
+
+func (backend *observedBackend) GetAllManifests(ctx context.Context) iter.Seq2[tuple[*ManifestMetadata, *Manifest], error] {
+	return func(yield func(tuple[*ManifestMetadata, *Manifest], error) bool) {
+		span, ctx := ObserveFunction(ctx, "GetAllManifests")
+		for item, err := range backend.inner.GetAllManifests(ctx) {
+			if !yield(item, err) {
+				break
+			}
+		}
+		span.Finish()
+	}
+}
+
 func (backend *observedBackend) CheckDomain(ctx context.Context, domain string) (found bool, err error) {
 	span, ctx := ObserveFunction(ctx, "CheckDomain", "domain.name", domain)
 	found, err = backend.inner.CheckDomain(ctx, domain)
@@ -435,9 +332,23 @@ func (backend *observedBackend) CreateDomain(ctx context.Context, domain string)
 	return
 }
 
-func (backend *observedBackend) FreezeDomain(ctx context.Context, domain string, freeze bool) (err error) {
-	span, ctx := ObserveFunction(ctx, "FreezeDomain", "domain.name", domain, "domain.frozen", freeze)
-	err = backend.inner.FreezeDomain(ctx, domain, freeze)
+func (backend *observedBackend) FreezeDomain(ctx context.Context, domain string) (err error) {
+	span, ctx := ObserveFunction(ctx, "FreezeDomain", "domain.name", domain)
+	err = backend.inner.FreezeDomain(ctx, domain)
+	span.Finish()
+	return
+}
+
+func (backend *observedBackend) UnfreezeDomain(ctx context.Context, domain string) (err error) {
+	span, ctx := ObserveFunction(ctx, "UnfreezeDomain", "domain.name", domain)
+	err = backend.inner.UnfreezeDomain(ctx, domain)
+	span.Finish()
+	return
+}
+
+func (backend *observedBackend) HaveDomainsChanged(ctx context.Context, since time.Time) (changed bool, err error) {
+	span, ctx := ObserveFunction(ctx, "HaveDomainsChanged", "since", since)
+	changed, err = backend.inner.HaveDomainsChanged(ctx, since)
 	span.Finish()
 	return
 }
@@ -472,3 +383,31 @@ func (backend *observedBackend) SearchAuditLog(
 		span.Finish()
 	}
 }
+
+func (backend *observedBackend) GetAuditLogRecords(
+	ctx context.Context, ids iter.Seq2[AuditID, error],
+) iter.Seq2[*AuditRecord, error] {
+	return func(yield func(*AuditRecord, error) bool) {
+		span, ctx := ObserveFunction(ctx, "GetAuditLogRecords")
+		for item, err := range backend.inner.GetAuditLogRecords(ctx, ids) {
+			if !yield(item, err) {
+				break
+			}
+		}
+		span.Finish()
+	}
+}
+
+func (backend *observedBackend) DetachAuditRecord(ctx context.Context, id AuditID) (err error) {
+	span, ctx := ObserveFunction(ctx, "DetachAuditRecord", "audit.id", id)
+	err = backend.inner.DetachAuditRecord(ctx, id)
+	span.Finish()
+	return
+}
+
+func (backend *observedBackend) ExpireAuditRecord(ctx context.Context, id AuditID) (err error) {
+	span, ctx := ObserveFunction(ctx, "ExpireAuditRecord", "audit.id", id)
+	err = backend.inner.ExpireAuditRecord(ctx, id)
+	span.Finish()
+	return
+}

src/pages.go

@@ -9,7 +9,6 @@ import (
 	"fmt"
 	"io"
 	"maps"
-	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -66,8 +65,23 @@ func observeSiteUpdate(via string, result *UpdateResult) {
 	}
 }
 
+func copyForgeAuthToPrincipal(principal *Principal, auth *Authorization) {
+	if auth.forgeUser != nil {
+		principal.ForgeUser = auth.forgeUser
+	}
+
+	repoURL := auth.ForgeRepoURL()
+	if repoURL != "" {
+		principal.RepoUrl = &repoURL
+	}
+}
+
+func normalizeHost(host string) string {
+	return strings.ToLower(host)
+}
+
 func makeWebRoot(host string, projectName string) string {
-	return fmt.Sprintf("%s/%s", strings.ToLower(host), projectName)
+	return path.Join(normalizeHost(host), projectName)
 }
 
 func getWebRoot(r *http.Request) (string, error) {
@@ -116,6 +130,13 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 		return err
 	}
 
+	host = normalizeHost(host)
+	if !domainCache.CheckDomain(r.Context(), host) {
+		w.WriteHeader(http.StatusNotFound)
+		fmt.Fprintf(w, "site not found\n")
+		return nil
+	}
+
 	type indexManifestResult struct {
 		manifest *Manifest
 		metadata ManifestMetadata
@@ -133,18 +154,20 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 	err = nil
 	sitePath = strings.TrimPrefix(r.URL.Path, "/")
 	if projectName, projectPath, hasProjectSlash := strings.Cut(sitePath, "/"); projectName != "" {
-		var projectManifest *Manifest
-		var projectMetadata ManifestMetadata
-		projectManifest, projectMetadata, err = backend.GetManifest(
-			r.Context(), makeWebRoot(host, projectName),
-			GetManifestOptions{BypassCache: bypassCache},
-		)
-		if err == nil {
-			if !hasProjectSlash {
-				writeRedirect(w, http.StatusFound, r.URL.Path+"/")
-				return nil
+		if ValidateProjectName(projectName) == nil {
+			var projectManifest *Manifest
+			var projectMetadata ManifestMetadata
+			projectManifest, projectMetadata, err = backend.GetManifest(
+				r.Context(), makeWebRoot(host, projectName),
+				GetManifestOptions{BypassCache: bypassCache},
+			)
+			if err == nil {
+				if !hasProjectSlash {
+					writeRedirect(w, http.StatusFound, r.URL.Path+"/")
+					return nil
+				}
+				sitePath, manifest, metadata = projectPath, projectManifest, projectMetadata
 			}
-			sitePath, manifest, metadata = projectPath, projectManifest, projectMetadata
 		}
 	}
 	if manifest == nil && (err == nil || errors.Is(err, ErrObjectNotFound)) {
@@ -192,8 +215,8 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 
 		case metadataPath == "manifest.json":
 			// metadata requests require authorization to avoid making pushes from private
-			// repositories enumerable
-			_, err := AuthorizeMetadataRetrieval(r)
+			// repositories enumerable or exposing basic-auth protected sections
+			_, err := AuthorizeMetadataRetrieval(r, ManifestHasBasicAuth(manifest))
 			if err != nil {
 				return err
 			}
@@ -207,14 +230,15 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 
 		case metadataPath == "archive.tar":
 			// same as above
-			_, err := AuthorizeMetadataRetrieval(r)
+			_, err := AuthorizeMetadataRetrieval(r, ManifestHasBasicAuth(manifest))
 			if err != nil {
 				return err
 			}
 
 			// we only offer `/.git-pages/archive.tar` and not the `.tar.gz`/`.tar.zst` variants
 			// because HTTP can already request compression using the `Content-Encoding` mechanism
-			acceptedEncodings := parseHTTPEncodings(r.Header.Get("Accept-Encoding"))
+			acceptedEncodings := ParseAcceptEncodingHeader(r.Header.Get("Accept-Encoding"))
+			w.Header().Add("Vary", "Accept-Encoding")
 			negotiated := acceptedEncodings.Negotiate("zstd", "gzip", "identity")
 			if negotiated != "" {
 				w.Header().Set("Content-Encoding", negotiated)
@@ -242,10 +266,23 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 		}
 	}
 
+	// Apply basic-auth rules before checking existence of a path to avoid leaking the latter.
+	authorized, err := ApplyBasicAuthRules(manifest, &url.URL{Path: sitePath}, r)
+	if err != nil {
+		// See comment below for the error case under `ApplyHeaderRules`.
+		w.WriteHeader(http.StatusInternalServerError)
+		fmt.Fprintf(w, "%s\n", err)
+		return err
+	} else if !authorized {
+		w.Header().Set("WWW-Authenticate", `Basic charset="UTF-8"`)
+		w.WriteHeader(http.StatusUnauthorized)
+		return nil
+	}
+
 	entryPath := sitePath
 	entry := (*Entry)(nil)
 	appliedRedirect := false
-	status := 200
+	status := http.StatusOK
 	reader := io.ReadSeeker(nil)
 	mtime := time.Time{}
 	for {
@@ -263,7 +300,7 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 				redirectKind = RedirectForce
 			}
 			originalURL := (&url.URL{Host: r.Host}).ResolveReference(r.URL)
-			redirectURL, redirectStatus := ApplyRedirectRules(manifest, originalURL, redirectKind)
+			_, redirectURL, redirectStatus := ApplyRedirectRules(manifest, originalURL, redirectKind)
 			if Is3xxHTTPStatus(redirectStatus) {
 				writeRedirect(w, redirectStatus, redirectURL.String())
 				return nil
@@ -277,7 +314,7 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 			}
 		}
 		if entry == nil || entry.GetType() == Type_InvalidEntry {
-			status = 404
+			status = http.StatusNotFound
 			if entryPath != notFoundPage {
 				entryPath = notFoundPage
 				continue
@@ -293,13 +330,15 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 				w.WriteHeader(http.StatusNotModified)
 				return nil
 			} else {
-				reader, _, mtime, err = backend.GetBlob(r.Context(), string(entry.Data))
+				var metadata BlobMetadata
+				reader, metadata, err = backend.GetBlob(r.Context(), string(entry.Data))
 				if err != nil {
 					ObserveError(err) // all storage errors must be reported
 					w.WriteHeader(http.StatusInternalServerError)
 					fmt.Fprintf(w, "internal server error: %s\n", err)
 					return err
 				}
+				mtime = metadata.LastModified
 				w.Header().Set("ETag", etag)
 			}
 		} else if entry.GetType() == Type_Directory {
@@ -322,11 +361,14 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 		defer closer.Close()
 	}
 
-	acceptedEncodings := parseHTTPEncodings(r.Header.Get("Accept-Encoding"))
+	var offeredEncodings []string
+	acceptedEncodings := ParseAcceptEncodingHeader(r.Header.Get("Accept-Encoding"))
+	w.Header().Add("Vary", "Accept-Encoding")
 	negotiatedEncoding := true
 	switch entry.GetTransform() {
 	case Transform_Identity:
-		switch acceptedEncodings.Negotiate("identity") {
+		offeredEncodings = []string{"identity"}
+		switch acceptedEncodings.Negotiate(offeredEncodings...) {
 		case "identity":
 			serveEncodingCount.
 				With(prometheus.Labels{"transform": "identity", "negotiated": "identity"}).
@@ -338,13 +380,13 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 				Inc()
 		}
 	case Transform_Zstd:
-		supported := []string{"zstd", "identity"}
+		offeredEncodings = []string{"zstd", "identity"}
 		if entry.ContentType == nil {
 			// If Content-Type is unset, `http.ServeContent` will try to sniff
 			// the file contents. That won't work if it's compressed.
-			supported = []string{"identity"}
+			offeredEncodings = []string{"identity"}
 		}
-		switch acceptedEncodings.Negotiate(supported...) {
+		switch acceptedEncodings.Negotiate(offeredEncodings...) {
 		case "zstd":
 			// Set Content-Length ourselves since `http.ServeContent` only sets
 			// it if Content-Encoding is unset or if it's a range request.
@@ -375,8 +417,9 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 		return fmt.Errorf("unexpected transform")
 	}
 	if !negotiatedEncoding {
+		w.Header().Set("Accept-Encoding", strings.Join(offeredEncodings, ", "))
 		w.WriteHeader(http.StatusNotAcceptable)
-		return fmt.Errorf("no supported content encodings (Accept-Encoding: %q)",
+		return fmt.Errorf("no supported content encodings (Accept-Encoding: %s)",
 			r.Header.Get("Accept-Encoding"))
 	}
 
@@ -411,13 +454,15 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 			io.Copy(w, reader)
 		}
 	} else {
-		// consider content fresh for 60 seconds (the same as the freshness interval of
-		// manifests in the S3 backend), and use stale content anyway as long as it's not
-		// older than a hour; while it is cheap to handle If-Modified-Since queries
-		// server-side, on the client `max-age=0, must-revalidate` causes every resource
-		// to block the page load every time
-		w.Header().Set("Cache-Control", "max-age=60, stale-while-revalidate=3600")
-		// see https://web.dev/articles/stale-while-revalidate for details
+		if _, hasCacheControl := w.Header()["Cache-Control"]; !hasCacheControl {
+			// consider content fresh for 60 seconds (the same as the freshness interval of
+			// manifests in the S3 backend), and use stale content anyway as long as it's not
+			// older than a hour; while it is cheap to handle If-Modified-Since queries
+			// server-side, on the client `max-age=0, must-revalidate` causes every resource
+			// to block the page load every time
+			w.Header().Set("Cache-Control", "max-age=60, stale-while-revalidate=3600")
+			// see https://web.dev/articles/stale-while-revalidate for details
+		}
 
 		// http.ServeContent handles conditional requests and range requests
 		http.ServeContent(w, r, entryPath, mtime, reader)
@@ -489,21 +534,26 @@ func putPage(w http.ResponseWriter, r *http.Request) error {
 		result = UpdateFromRepository(ctx, webRoot, repoURL, branch)
 
 	default:
-		_, err := AuthorizeUpdateFromArchive(r)
+		auth, err := AuthorizeUpdateFromArchive(r)
 		if err != nil {
 			return err
 		}
 
+		principal := GetPrincipal(r.Context())
+		copyForgeAuthToPrincipal(principal, auth)
+
+		repoURL := auth.ForgeRepoURL()
+
 		if checkDryRun(w, r) {
 			return nil
 		}
 
 		// request body contains archive
 		reader := http.MaxBytesReader(w, r.Body, int64(config.Limits.MaxSiteSize.Bytes()))
-		result = UpdateFromArchive(ctx, webRoot, contentType, reader)
+		result = UpdateFromArchive(ctx, webRoot, repoURL, contentType, reader)
 	}
 
-	return reportUpdateResult(w, result)
+	return reportUpdateResult(w, r, result)
 }
 
 func patchPage(w http.ResponseWriter, r *http.Request) error {
@@ -521,10 +571,14 @@ func patchPage(w http.ResponseWriter, r *http.Request) error {
 		return err
 	}
 
-	if _, err = AuthorizeUpdateFromArchive(r); err != nil {
+	auth, err := AuthorizeUpdateFromArchive(r)
+	if err != nil {
 		return err
 	}
 
+	principal := GetPrincipal(r.Context())
+	copyForgeAuthToPrincipal(principal, auth)
+
 	if checkDryRun(w, r) {
 		return nil
 	}
@@ -566,18 +620,43 @@ func patchPage(w http.ResponseWriter, r *http.Request) error {
 	contentType := getMediaType(r.Header.Get("Content-Type"))
 	reader := http.MaxBytesReader(w, r.Body, int64(config.Limits.MaxSiteSize.Bytes()))
 	result := PartialUpdateFromArchive(ctx, webRoot, contentType, reader, parents)
-	return reportUpdateResult(w, result)
+	return reportUpdateResult(w, r, result)
 }
 
-func reportUpdateResult(w http.ResponseWriter, result UpdateResult) error {
+func reportUpdateResult(w http.ResponseWriter, r *http.Request, result UpdateResult) error {
+	var unresolvedRefErr UnresolvedRefError
+	if result.outcome == UpdateError && errors.As(result.err, &unresolvedRefErr) {
+		offeredContentTypes := []string{"text/plain", "application/vnd.git-pages.unresolved"}
+		acceptedContentTypes := ParseAcceptHeader(r.Header.Get("Accept"))
+		switch acceptedContentTypes.Negotiate(offeredContentTypes...) {
+		default:
+			w.Header().Set("Accept", strings.Join(offeredContentTypes, ", "))
+			w.WriteHeader(http.StatusNotAcceptable)
+			return fmt.Errorf("no supported content types (Accept: %s)", r.Header.Get("Accept"))
+		case "application/vnd.git-pages.unresolved":
+			w.Header().Set("Content-Type", "application/vnd.git-pages.unresolved")
+			w.WriteHeader(http.StatusUnprocessableEntity)
+			for _, missingRef := range unresolvedRefErr.missing {
+				fmt.Fprintln(w, missingRef)
+			}
+			return nil
+		case "text/plain":
+			// handled below
+		}
+	}
+
 	switch result.outcome {
 	case UpdateError:
-		if errors.Is(result.err, ErrManifestTooLarge) {
-			w.WriteHeader(http.StatusRequestEntityTooLarge)
+		if errors.Is(result.err, ErrSiteTooLarge) {
+			w.WriteHeader(http.StatusUnprocessableEntity)
+		} else if errors.Is(result.err, ErrManifestTooLarge) {
+			w.WriteHeader(http.StatusUnprocessableEntity)
 		} else if errors.Is(result.err, errArchiveFormat) {
 			w.WriteHeader(http.StatusUnsupportedMediaType)
 		} else if errors.Is(result.err, ErrArchiveTooLarge) {
 			w.WriteHeader(http.StatusRequestEntityTooLarge)
+		} else if errors.Is(result.err, ErrRepositoryTooLarge) {
+			w.WriteHeader(http.StatusUnprocessableEntity)
 		} else if errors.Is(result.err, ErrMalformedPatch) {
 			w.WriteHeader(http.StatusUnprocessableEntity)
 		} else if errors.Is(result.err, ErrPreconditionFailed) {
@@ -586,6 +665,8 @@ func reportUpdateResult(w http.ResponseWriter, result UpdateResult) error {
 			w.WriteHeader(http.StatusConflict)
 		} else if errors.Is(result.err, ErrDomainFrozen) {
 			w.WriteHeader(http.StatusForbidden)
+		} else if errors.As(result.err, &unresolvedRefErr) {
+			w.WriteHeader(http.StatusUnprocessableEntity)
 		} else {
 			w.WriteHeader(http.StatusServiceUnavailable)
 		}
@@ -624,11 +705,14 @@ func deletePage(w http.ResponseWriter, r *http.Request) error {
 		return err
 	}
 
-	_, err = AuthorizeUpdateFromRepository(r)
+	auth, err := AuthorizeDeletion(r)
 	if err != nil {
 		return err
 	}
 
+	principal := GetPrincipal(r.Context())
+	copyForgeAuthToPrincipal(principal, auth)
+
 	if checkDryRun(w, r) {
 		return nil
 	}
@@ -644,7 +728,9 @@ func deletePage(w http.ResponseWriter, r *http.Request) error {
 }
 
 func postPage(w http.ResponseWriter, r *http.Request) error {
-	// Start a timer for the request timeout immediately.
+	// The HTTP requests for webhook delivery usually have a short timeout. We start the timer
+	// before doing any time-consuming work so that it's closely aligned to the client's timeout and
+	// we can respond before the webhook delivery is considered failed.
 	requestTimeout := 3 * time.Second
 	requestTimer := time.NewTimer(requestTimeout)
 
@@ -705,7 +791,7 @@ func postPage(w http.ResponseWriter, r *http.Request) error {
 		return err
 	}
 
-	if event.Ref != fmt.Sprintf("refs/heads/%s", auth.branch) {
+	if event.Ref != path.Join("refs", "heads", auth.branch) {
 		code := http.StatusUnauthorized
 		if strings.Contains(r.Header.Get("User-Agent"), "GitHub-Hookshot") {
 			// GitHub has no way to restrict branches for a webhook, and responding with 401
@@ -735,7 +821,7 @@ func postPage(w http.ResponseWriter, r *http.Request) error {
 		result := UpdateFromRepository(ctx, webRoot, repoURL, auth.branch)
 		resultChan <- result
 		observeSiteUpdate("webhook", &result)
-	}(context.Background())
+	}(context.WithoutCancel(r.Context()))
 
 	var result UpdateResult
 	select {
@@ -776,29 +862,26 @@ func postPage(w http.ResponseWriter, r *http.Request) error {
 
 func ServePages(w http.ResponseWriter, r *http.Request) {
 	r = r.WithContext(WithPrincipal(r.Context()))
-	if config.Audit.IncludeIPs {
-		if ipAddress, _, err := net.SplitHostPort(r.RemoteAddr); err == nil {
-			GetPrincipal(r.Context()).IpAddress = proto.String(ipAddress)
-		}
+	if config.Audit.IncludeIPs != "" {
+		GetPrincipal(r.Context()).IpAddress = proto.String(r.RemoteAddr)
 	}
-	// We want upstream health checks to be done as closely to the normal flow as possible;
-	// any intentional deviation is an opportunity to miss an issue that will affect our
-	// visitors but not our health checks.
-	if r.Header.Get("Health-Check") == "" {
-		logc.Println(r.Context(), "pages:", r.Method, r.Host, r.URL, r.Header.Get("Content-Type"))
-		if region := os.Getenv("FLY_REGION"); region != "" {
-			machine_id := os.Getenv("FLY_MACHINE_ID")
-			w.Header().Add("Server", fmt.Sprintf("git-pages (fly.io; %s; %s)", region, machine_id))
-			ObserveData(r.Context(), "server.name", machine_id, "server.region", region)
-		} else if hostname, err := os.Hostname(); err == nil {
-			if region := os.Getenv("PAGES_REGION"); region != "" {
-				w.Header().Add("Server", fmt.Sprintf("git-pages (%s; %s)", region, hostname))
-				ObserveData(r.Context(), "server.name", hostname, "server.region", region)
-			} else {
-				w.Header().Add("Server", fmt.Sprintf("git-pages (%s)", hostname))
-				ObserveData(r.Context(), "server.name", hostname)
-			}
+	switch r.Method {
+	case "PUT", "PATCH", "POST":
+		mediaType := r.Header.Get("Content-Type")
+		logc.Println(r.Context(), "pages:", r.Method, r.Host, r.URL, mediaType)
+	default:
+		logc.Println(r.Context(), "pages:", r.Method, r.Host, r.URL)
+	}
+	if hostname, err := os.Hostname(); err == nil {
+		if region := os.Getenv("PAGES_REGION"); region != "" {
+			w.Header().Add("Server", fmt.Sprintf("git-pages (%s; %s)", region, hostname))
+			ObserveData(r.Context(), "server.name", hostname, "server.region", region)
+		} else {
+			w.Header().Add("Server", fmt.Sprintf("git-pages (%s)", hostname))
+			ObserveData(r.Context(), "server.name", hostname)
 		}
+	} else {
+		w.Header().Add("Server", "git-pages")
 	}
 	allowedMethods := []string{"OPTIONS", "HEAD", "GET", "PUT", "PATCH", "DELETE", "POST"}
 	if r.Method == "OPTIONS" || !slices.Contains(allowedMethods, r.Method) {
@@ -807,18 +890,18 @@ func ServePages(w http.ResponseWriter, r *http.Request) {
 	err := error(nil)
 	switch r.Method {
 	// REST API
-	case http.MethodOptions:
+	case "OPTIONS":
 		// no preflight options
-	case http.MethodHead, http.MethodGet:
+	case "HEAD", "GET":
 		err = getPage(w, r)
-	case http.MethodPut:
+	case "PUT":
 		err = putPage(w, r)
-	case http.MethodPatch:
+	case "PATCH":
 		err = patchPage(w, r)
-	case http.MethodDelete:
+	case "DELETE":
 		err = deletePage(w, r)
 	// webhook API
-	case http.MethodPost:
+	case "POST":
 		err = postPage(w, r)
 	default:
 		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)

src/pages_test.go

@@ -0,0 +1,55 @@
+package git_pages
+
+import (
+	"net/http"
+	"strings"
+	"testing"
+)
+
+func checkHost(t *testing.T, host string, expectOk string, expectErr string) {
+	host, err := GetHost(&http.Request{Host: host})
+	if expectErr != "" {
+		if err == nil || !strings.HasPrefix(err.Error(), expectErr) {
+			t.Errorf("%s: expect err %s, got err %s", host, expectErr, err)
+		}
+	}
+	if expectOk != "" {
+		if err != nil {
+			t.Errorf("%s: expect ok %s, got err %s", host, expectOk, err)
+		} else if host != expectOk {
+			t.Errorf("%s: expect ok %s, got ok %s", host, expectOk, host)
+		}
+	}
+}
+
+func TestHelloName(t *testing.T) {
+	config = &Config{Features: []string{}}
+
+	checkHost(t, "foo.bar", "foo.bar", "")
+	checkHost(t, "foo-baz.bar", "foo-baz.bar", "")
+	checkHost(t, "foo--baz.bar", "foo--baz.bar", "")
+	checkHost(t, "foo.bar.", "foo.bar", "")
+	checkHost(t, ".foo.bar", "", "reserved host name")
+	checkHost(t, "..foo.bar", "", "reserved host name")
+
+	checkHost(t, "ß.bar", "xn--zca.bar", "")
+	checkHost(t, "xn--zca.bar", "xn--zca.bar", "")
+
+	checkHost(t, "foo-.bar", "", "malformed host name")
+	checkHost(t, "-foo.bar", "", "malformed host name")
+	checkHost(t, "foo_.bar", "", "malformed host name")
+	checkHost(t, "_foo.bar", "", "malformed host name")
+	checkHost(t, "foo_baz.bar", "", "malformed host name")
+	checkHost(t, "foo__baz.bar", "", "malformed host name")
+	checkHost(t, "*.foo.bar", "", "malformed host name")
+
+	config = &Config{Features: []string{"relaxed-idna"}}
+
+	checkHost(t, "foo-.bar", "", "malformed host name")
+	checkHost(t, "-foo.bar", "", "malformed host name")
+	checkHost(t, "foo_.bar", "foo_.bar", "")
+	checkHost(t, "_foo.bar", "", "reserved host name")
+	checkHost(t, "foo_baz.bar", "foo_baz.bar", "")
+	checkHost(t, "foo__baz.bar", "foo__baz.bar", "")
+	checkHost(t, "*.foo.bar", "", "malformed host name")
+}

src/patch.go

@@ -30,8 +30,12 @@ func ApplyTarPatch(manifest *Manifest, reader io.Reader, parents CreateParentsMo
 		children map[string]*Node
 	}
 
+	// Index the manifest for incremental update operations.
+	index := IndexManifestByGitHash(manifest)
+	missing := []string{}
+
 	// Extract the manifest contents (which is using a flat hash map) into a directory tree
-	// so that recursive delete operations have O(1) complexity. s
+	// so that recursive delete operations have O(1) complexity.
 	var root *Node
 	sortedNames := slices.Sorted(maps.Keys(manifest.GetContents()))
 	for _, name := range sortedNames {
@@ -48,9 +52,9 @@ func ApplyTarPatch(manifest *Manifest, reader io.Reader, parents CreateParentsMo
 			iter := root
 			for _, segment := range segments[:len(segments)-1] {
 				if iter.children == nil {
-					panic("malformed manifest")
+					panic("malformed manifest (not a directory)")
 				} else if _, exists := iter.children[segment]; !exists {
-					panic("malformed manifest")
+					panic("malformed manifest (node does not exist)")
 				} else {
 					iter = iter.children[segment]
 				}
@@ -70,7 +74,7 @@ func ApplyTarPatch(manifest *Manifest, reader io.Reader, parents CreateParentsMo
 			return err
 		}
 
-		segments := strings.Split(strings.TrimRight(header.Name, "/"), "/")
+		segments := strings.Split(normalizeArchiveMemberName(header.Name), "/")
 		fileName := segments[len(segments)-1]
 		node := root
 		for index, segment := range segments[:len(segments)-1] {
@@ -107,8 +111,16 @@ func ApplyTarPatch(manifest *Manifest, reader io.Reader, parents CreateParentsMo
 				entry: NewManifestEntry(Type_InlineFile, fileData),
 			}
 		case tar.TypeSymlink:
-			node.children[fileName] = &Node{
-				entry: NewManifestEntry(Type_Symlink, []byte(header.Linkname)),
+			if hash, found := strings.CutPrefix(header.Linkname, BlobReferencePrefix); found {
+				if entry, found := index[hash]; found {
+					node.children[fileName] = &Node{entry: entry}
+				} else {
+					missing = append(missing, hash)
+				}
+			} else {
+				node.children[fileName] = &Node{
+					entry: NewManifestEntry(Type_Symlink, []byte(header.Linkname)),
+				}
 			}
 		case tar.TypeDir:
 			node.children[fileName] = &Node{
@@ -129,6 +141,10 @@ func ApplyTarPatch(manifest *Manifest, reader io.Reader, parents CreateParentsMo
 		}
 	}
 
+	if len(missing) > 0 {
+		return UnresolvedRefError{missing}
+	}
+
 	// Repopulate manifest contents with the updated directory tree.
 	var traverse func([]string, *Node)
 	traverse = func(segments []string, node *Node) {

src/redirects.go

@@ -1,6 +1,8 @@
 package git_pages
 
 import (
+	"context"
+	"errors"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -13,7 +15,17 @@ import (
 
 const RedirectsFileName string = "_redirects"
 
-func unparseRule(rule redirects.Rule) string {
+// Converts our Protobuf representation to tj/go-redirects.
+func exportRedirectRule(rule *RedirectRule) *redirects.Rule {
+	return &redirects.Rule{
+		From:   rule.GetFrom(),
+		To:     rule.GetTo(),
+		Status: int(rule.GetStatus()),
+		Force:  rule.GetForce(),
+	}
+}
+
+func unparseRedirectRule(rule *redirects.Rule) string {
 	var statusPart string
 	if rule.Force {
 		statusPart = fmt.Sprintf("%d!", rule.Status)
@@ -49,7 +61,7 @@ func Is3xxHTTPStatus(status int) bool {
 	return status >= 300 && status <= 399
 }
 
-func validateRedirectRule(rule redirects.Rule) error {
+func validateRedirectRule(rule *redirects.Rule) error {
 	if len(rule.Params) > 0 {
 		return fmt.Errorf("rules with parameters are not supported")
 	}
@@ -86,26 +98,31 @@ func validateRedirectRule(rule redirects.Rule) error {
 }
 
 // Parses redirects file and injects rules into the manifest.
-func ProcessRedirectsFile(manifest *Manifest) error {
+func ProcessRedirectsFile(ctx context.Context, manifest *Manifest) error {
 	redirectsEntry := manifest.Contents[RedirectsFileName]
 	delete(manifest.Contents, RedirectsFileName)
 	if redirectsEntry == nil {
 		return nil
-	} else if redirectsEntry.GetType() != Type_InlineFile {
-		return AddProblem(manifest, RedirectsFileName,
-			"not a regular file")
 	}
 
-	rules, err := redirects.ParseString(string(redirectsEntry.GetData()))
+	data, err := GetEntryContents(ctx, redirectsEntry)
+	if errors.Is(err, ErrNotRegularFile) {
+		return AddProblem(manifest, RedirectsFileName,
+			"not a regular file")
+	} else if err != nil {
+		return err
+	}
+
+	rules, err := redirects.ParseString(string(data))
 	if err != nil {
 		return AddProblem(manifest, RedirectsFileName,
 			"syntax error: %s", err)
 	}
 
 	for index, rule := range rules {
-		if err := validateRedirectRule(rule); err != nil {
+		if err := validateRedirectRule(&rule); err != nil {
 			AddProblem(manifest, RedirectsFileName,
-				"rule #%d %q: %s", index+1, unparseRule(rule), err)
+				"rule #%d %q: %s", index+1, unparseRedirectRule(&rule), err)
 			continue
 		}
 		manifest.Redirects = append(manifest.Redirects, &RedirectRule{
@@ -121,12 +138,7 @@ func ProcessRedirectsFile(manifest *Manifest) error {
 func CollectRedirectsFile(manifest *Manifest) string {
 	var rules []string
 	for _, rule := range manifest.GetRedirects() {
-		rules = append(rules, unparseRule(redirects.Rule{
-			From:   rule.GetFrom(),
-			To:     rule.GetTo(),
-			Status: int(rule.GetStatus()),
-			Force:  rule.GetForce(),
-		})+"\n")
+		rules = append(rules, unparseRedirectRule(exportRedirectRule(rule))+"\n")
 	}
 	return strings.Join(rules, "")
 }
@@ -147,18 +159,22 @@ type RedirectKind int
 
 const (
 	RedirectAny RedirectKind = iota
+	RedirectNormal
 	RedirectForce
 )
 
 func ApplyRedirectRules(
 	manifest *Manifest, fromURL *url.URL, kind RedirectKind,
 ) (
-	toURL *url.URL, status int,
+	rule *RedirectRule, toURL *url.URL, status int,
 ) {
 	fromSegments := pathSegments(fromURL.Path)
 next:
-	for _, rule := range manifest.Redirects {
-		if kind == RedirectForce && !*rule.Force {
+	for _, rule = range manifest.Redirects {
+		switch {
+		case kind == RedirectNormal && *rule.Force:
+			continue
+		case kind == RedirectForce && !*rule.Force:
 			continue
 		}
 		// check if the rule matches fromURL
@@ -205,8 +221,43 @@ next:
 			RawQuery: fromURL.RawQuery,
 		}
 		status = int(*rule.Status)
-		break
+		return
 	}
 	// no redirect found
+	rule = nil
 	return
 }
+
+func redirectHasSplat(rule *RedirectRule) bool {
+	ruleFromURL, _ := url.Parse(*rule.From) // pre-validated in `validateRedirectRule`
+	ruleFromSegments := pathSegments(ruleFromURL.Path)
+	return slices.Contains(ruleFromSegments, "*")
+}
+
+func LintRedirects(manifest *Manifest) {
+	for name, entry := range manifest.GetContents() {
+		nameURL, err := url.Parse("/" + name)
+		if err != nil {
+			continue
+		}
+
+		// Check if the entry URL would trigger a non-forced redirect if the entry didn't exist.
+		// If the redirect matches exactly one URL (i.e. has no splat) then it will never be
+		// triggered and an issue is reported; if the rule has a splat, it will always be possible
+		// to trigger it, as it matches an infinite number of URLs.
+		rule, _, _ := ApplyRedirectRules(manifest, nameURL, RedirectNormal)
+		if rule != nil && !redirectHasSplat(rule) {
+			entryDesc := "file"
+			if entry.GetType() == Type_Directory {
+				entryDesc = "directory"
+			}
+			AddProblem(manifest, name,
+				"%s shadows redirect %q; remove the %s or use a %d! forced redirect instead",
+				entryDesc,
+				unparseRedirectRule(exportRedirectRule(rule)),
+				entryDesc,
+				rule.GetStatus(),
+			)
+		}
+	}
+}

src/schema.pb.go

@@ -479,6 +479,110 @@ func (x *HeaderRule) GetHeaderMap() []*Header {
 	return nil
 }
 
+type BasicCredential struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Username      *string                `protobuf:"bytes,1,opt,name=username" json:"username,omitempty"`
+	Password      *string                `protobuf:"bytes,2,opt,name=password" json:"password,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *BasicCredential) Reset() {
+	*x = BasicCredential{}
+	mi := &file_schema_proto_msgTypes[4]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *BasicCredential) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*BasicCredential) ProtoMessage() {}
+
+func (x *BasicCredential) ProtoReflect() protoreflect.Message {
+	mi := &file_schema_proto_msgTypes[4]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use BasicCredential.ProtoReflect.Descriptor instead.
+func (*BasicCredential) Descriptor() ([]byte, []int) {
+	return file_schema_proto_rawDescGZIP(), []int{4}
+}
+
+func (x *BasicCredential) GetUsername() string {
+	if x != nil && x.Username != nil {
+		return *x.Username
+	}
+	return ""
+}
+
+func (x *BasicCredential) GetPassword() string {
+	if x != nil && x.Password != nil {
+		return *x.Password
+	}
+	return ""
+}
+
+type BasicAuthRule struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Path          *string                `protobuf:"bytes,1,opt,name=path" json:"path,omitempty"`
+	Credentials   []*BasicCredential     `protobuf:"bytes,2,rep,name=credentials" json:"credentials,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *BasicAuthRule) Reset() {
+	*x = BasicAuthRule{}
+	mi := &file_schema_proto_msgTypes[5]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *BasicAuthRule) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*BasicAuthRule) ProtoMessage() {}
+
+func (x *BasicAuthRule) ProtoReflect() protoreflect.Message {
+	mi := &file_schema_proto_msgTypes[5]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use BasicAuthRule.ProtoReflect.Descriptor instead.
+func (*BasicAuthRule) Descriptor() ([]byte, []int) {
+	return file_schema_proto_rawDescGZIP(), []int{5}
+}
+
+func (x *BasicAuthRule) GetPath() string {
+	if x != nil && x.Path != nil {
+		return *x.Path
+	}
+	return ""
+}
+
+func (x *BasicAuthRule) GetCredentials() []*BasicCredential {
+	if x != nil {
+		return x.Credentials
+	}
+	return nil
+}
+
 type Problem struct {
 	state         protoimpl.MessageState `protogen:"open.v1"`
 	Path          *string                `protobuf:"bytes,1,opt,name=path" json:"path,omitempty"`
@@ -489,7 +593,7 @@ type Problem struct {
 
 func (x *Problem) Reset() {
 	*x = Problem{}
-	mi := &file_schema_proto_msgTypes[4]
+	mi := &file_schema_proto_msgTypes[6]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -501,7 +605,7 @@ func (x *Problem) String() string {
 func (*Problem) ProtoMessage() {}
 
 func (x *Problem) ProtoReflect() protoreflect.Message {
-	mi := &file_schema_proto_msgTypes[4]
+	mi := &file_schema_proto_msgTypes[6]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -514,7 +618,7 @@ func (x *Problem) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use Problem.ProtoReflect.Descriptor instead.
 func (*Problem) Descriptor() ([]byte, []int) {
-	return file_schema_proto_rawDescGZIP(), []int{4}
+	return file_schema_proto_rawDescGZIP(), []int{6}
 }
 
 func (x *Problem) GetPath() string {
@@ -543,8 +647,9 @@ type Manifest struct {
 	CompressedSize *int64            `protobuf:"varint,5,opt,name=compressed_size,json=compressedSize" json:"compressed_size,omitempty"` // sum of each `entry.compressed_size`
 	StoredSize     *int64            `protobuf:"varint,8,opt,name=stored_size,json=storedSize" json:"stored_size,omitempty"`             // sum of deduplicated `entry.compressed_size` for external files only
 	// Netlify-style `_redirects` and `_headers` rules.
-	Redirects []*RedirectRule `protobuf:"bytes,6,rep,name=redirects" json:"redirects,omitempty"`
-	Headers   []*HeaderRule   `protobuf:"bytes,9,rep,name=headers" json:"headers,omitempty"`
+	Redirects []*RedirectRule  `protobuf:"bytes,6,rep,name=redirects" json:"redirects,omitempty"`
+	Headers   []*HeaderRule    `protobuf:"bytes,9,rep,name=headers" json:"headers,omitempty"`
+	BasicAuth []*BasicAuthRule `protobuf:"bytes,11,rep,name=basic_auth,json=basicAuth" json:"basic_auth,omitempty"`
 	// Diagnostics for non-fatal errors.
 	Problems      []*Problem `protobuf:"bytes,7,rep,name=problems" json:"problems,omitempty"`
 	unknownFields protoimpl.UnknownFields
@@ -553,7 +658,7 @@ type Manifest struct {
 
 func (x *Manifest) Reset() {
 	*x = Manifest{}
-	mi := &file_schema_proto_msgTypes[5]
+	mi := &file_schema_proto_msgTypes[7]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -565,7 +670,7 @@ func (x *Manifest) String() string {
 func (*Manifest) ProtoMessage() {}
 
 func (x *Manifest) ProtoReflect() protoreflect.Message {
-	mi := &file_schema_proto_msgTypes[5]
+	mi := &file_schema_proto_msgTypes[7]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -578,7 +683,7 @@ func (x *Manifest) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use Manifest.ProtoReflect.Descriptor instead.
 func (*Manifest) Descriptor() ([]byte, []int) {
-	return file_schema_proto_rawDescGZIP(), []int{5}
+	return file_schema_proto_rawDescGZIP(), []int{7}
 }
 
 func (x *Manifest) GetRepoUrl() string {
@@ -644,6 +749,13 @@ func (x *Manifest) GetHeaders() []*HeaderRule {
 	return nil
 }
 
+func (x *Manifest) GetBasicAuth() []*BasicAuthRule {
+	if x != nil {
+		return x.BasicAuth
+	}
+	return nil
+}
+
 func (x *Manifest) GetProblems() []*Problem {
 	if x != nil {
 		return x.Problems
@@ -669,7 +781,7 @@ type AuditRecord struct {
 
 func (x *AuditRecord) Reset() {
 	*x = AuditRecord{}
-	mi := &file_schema_proto_msgTypes[6]
+	mi := &file_schema_proto_msgTypes[8]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -681,7 +793,7 @@ func (x *AuditRecord) String() string {
 func (*AuditRecord) ProtoMessage() {}
 
 func (x *AuditRecord) ProtoReflect() protoreflect.Message {
-	mi := &file_schema_proto_msgTypes[6]
+	mi := &file_schema_proto_msgTypes[8]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -694,7 +806,7 @@ func (x *AuditRecord) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use AuditRecord.ProtoReflect.Descriptor instead.
 func (*AuditRecord) Descriptor() ([]byte, []int) {
-	return file_schema_proto_rawDescGZIP(), []int{6}
+	return file_schema_proto_rawDescGZIP(), []int{8}
 }
 
 func (x *AuditRecord) GetId() int64 {
@@ -750,13 +862,15 @@ type Principal struct {
 	state         protoimpl.MessageState `protogen:"open.v1"`
 	IpAddress     *string                `protobuf:"bytes,1,opt,name=ip_address,json=ipAddress" json:"ip_address,omitempty"`
 	CliAdmin      *bool                  `protobuf:"varint,2,opt,name=cli_admin,json=cliAdmin" json:"cli_admin,omitempty"`
+	ForgeUser     *ForgeUser             `protobuf:"bytes,3,opt,name=forge_user,json=forgeUser" json:"forge_user,omitempty"`
+	RepoUrl       *string                `protobuf:"bytes,4,opt,name=repo_url,json=repoUrl" json:"repo_url,omitempty"`
 	unknownFields protoimpl.UnknownFields
 	sizeCache     protoimpl.SizeCache
 }
 
 func (x *Principal) Reset() {
 	*x = Principal{}
-	mi := &file_schema_proto_msgTypes[7]
+	mi := &file_schema_proto_msgTypes[9]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -768,7 +882,7 @@ func (x *Principal) String() string {
 func (*Principal) ProtoMessage() {}
 
 func (x *Principal) ProtoReflect() protoreflect.Message {
-	mi := &file_schema_proto_msgTypes[7]
+	mi := &file_schema_proto_msgTypes[9]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -781,7 +895,7 @@ func (x *Principal) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use Principal.ProtoReflect.Descriptor instead.
 func (*Principal) Descriptor() ([]byte, []int) {
-	return file_schema_proto_rawDescGZIP(), []int{7}
+	return file_schema_proto_rawDescGZIP(), []int{9}
 }
 
 func (x *Principal) GetIpAddress() string {
@@ -798,6 +912,80 @@ func (x *Principal) GetCliAdmin() bool {
 	return false
 }
 
+func (x *Principal) GetForgeUser() *ForgeUser {
+	if x != nil {
+		return x.ForgeUser
+	}
+	return nil
+}
+
+func (x *Principal) GetRepoUrl() string {
+	if x != nil && x.RepoUrl != nil {
+		return *x.RepoUrl
+	}
+	return ""
+}
+
+type ForgeUser struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Origin        *string                `protobuf:"bytes,1,opt,name=origin" json:"origin,omitempty"`
+	Id            *int64                 `protobuf:"varint,2,opt,name=id" json:"id,omitempty"`
+	Handle        *string                `protobuf:"bytes,3,opt,name=handle" json:"handle,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ForgeUser) Reset() {
+	*x = ForgeUser{}
+	mi := &file_schema_proto_msgTypes[10]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ForgeUser) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ForgeUser) ProtoMessage() {}
+
+func (x *ForgeUser) ProtoReflect() protoreflect.Message {
+	mi := &file_schema_proto_msgTypes[10]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ForgeUser.ProtoReflect.Descriptor instead.
+func (*ForgeUser) Descriptor() ([]byte, []int) {
+	return file_schema_proto_rawDescGZIP(), []int{10}
+}
+
+func (x *ForgeUser) GetOrigin() string {
+	if x != nil && x.Origin != nil {
+		return *x.Origin
+	}
+	return ""
+}
+
+func (x *ForgeUser) GetId() int64 {
+	if x != nil && x.Id != nil {
+		return *x.Id
+	}
+	return 0
+}
+
+func (x *ForgeUser) GetHandle() string {
+	if x != nil && x.Handle != nil {
+		return *x.Handle
+	}
+	return ""
+}
+
 var File_schema_proto protoreflect.FileDescriptor
 
 const file_schema_proto_rawDesc = "" +
@@ -824,10 +1012,16 @@ const file_schema_proto_rawDesc = "" +
 	"HeaderRule\x12\x12\n" +
 	"\x04path\x18\x01 \x01(\tR\x04path\x12&\n" +
 	"\n" +
-	"header_map\x18\x02 \x03(\v2\a.HeaderR\theaderMap\"3\n" +
+	"header_map\x18\x02 \x03(\v2\a.HeaderR\theaderMap\"I\n" +
+	"\x0fBasicCredential\x12\x1a\n" +
+	"\busername\x18\x01 \x01(\tR\busername\x12\x1a\n" +
+	"\bpassword\x18\x02 \x01(\tR\bpassword\"W\n" +
+	"\rBasicAuthRule\x12\x12\n" +
+	"\x04path\x18\x01 \x01(\tR\x04path\x122\n" +
+	"\vcredentials\x18\x02 \x03(\v2\x10.BasicCredentialR\vcredentials\"3\n" +
 	"\aProblem\x12\x12\n" +
 	"\x04path\x18\x01 \x01(\tR\x04path\x12\x14\n" +
-	"\x05cause\x18\x02 \x01(\tR\x05cause\"\xb8\x03\n" +
+	"\x05cause\x18\x02 \x01(\tR\x05cause\"\xe7\x03\n" +
 	"\bManifest\x12\x19\n" +
 	"\brepo_url\x18\x01 \x01(\tR\arepoUrl\x12\x16\n" +
 	"\x06branch\x18\x02 \x01(\tR\x06branch\x12\x16\n" +
@@ -839,7 +1033,9 @@ const file_schema_proto_rawDesc = "" +
 	"\vstored_size\x18\b \x01(\x03R\n" +
 	"storedSize\x12+\n" +
 	"\tredirects\x18\x06 \x03(\v2\r.RedirectRuleR\tredirects\x12%\n" +
-	"\aheaders\x18\t \x03(\v2\v.HeaderRuleR\aheaders\x12$\n" +
+	"\aheaders\x18\t \x03(\v2\v.HeaderRuleR\aheaders\x12-\n" +
+	"\n" +
+	"basic_auth\x18\v \x03(\v2\x0e.BasicAuthRuleR\tbasicAuth\x12$\n" +
 	"\bproblems\x18\a \x03(\v2\b.ProblemR\bproblems\x1aC\n" +
 	"\rContentsEntry\x12\x10\n" +
 	"\x03key\x18\x01 \x01(\tR\x03key\x12\x1c\n" +
@@ -853,11 +1049,19 @@ const file_schema_proto_rawDesc = "" +
 	"\x06domain\x18\n" +
 	" \x01(\tR\x06domain\x12\x18\n" +
 	"\aproject\x18\v \x01(\tR\aproject\x12%\n" +
-	"\bmanifest\x18\f \x01(\v2\t.ManifestR\bmanifest\"G\n" +
+	"\bmanifest\x18\f \x01(\v2\t.ManifestR\bmanifest\"\x8d\x01\n" +
 	"\tPrincipal\x12\x1d\n" +
 	"\n" +
 	"ip_address\x18\x01 \x01(\tR\tipAddress\x12\x1b\n" +
-	"\tcli_admin\x18\x02 \x01(\bR\bcliAdmin*V\n" +
+	"\tcli_admin\x18\x02 \x01(\bR\bcliAdmin\x12)\n" +
+	"\n" +
+	"forge_user\x18\x03 \x01(\v2\n" +
+	".ForgeUserR\tforgeUser\x12\x19\n" +
+	"\brepo_url\x18\x04 \x01(\tR\arepoUrl\"K\n" +
+	"\tForgeUser\x12\x16\n" +
+	"\x06origin\x18\x01 \x01(\tR\x06origin\x12\x0e\n" +
+	"\x02id\x18\x02 \x01(\x03R\x02id\x12\x16\n" +
+	"\x06handle\x18\x03 \x01(\tR\x06handle*V\n" +
 	"\x04Type\x12\x10\n" +
 	"\fInvalidEntry\x10\x00\x12\r\n" +
 	"\tDirectory\x10\x01\x12\x0e\n" +
@@ -889,7 +1093,7 @@ func file_schema_proto_rawDescGZIP() []byte {
 }
 
 var file_schema_proto_enumTypes = make([]protoimpl.EnumInfo, 3)
-var file_schema_proto_msgTypes = make([]protoimpl.MessageInfo, 9)
+var file_schema_proto_msgTypes = make([]protoimpl.MessageInfo, 12)
 var file_schema_proto_goTypes = []any{
 	(Type)(0),                     // 0: Type
 	(Transform)(0),                // 1: Transform
@@ -898,31 +1102,37 @@ var file_schema_proto_goTypes = []any{
 	(*RedirectRule)(nil),          // 4: RedirectRule
 	(*Header)(nil),                // 5: Header
 	(*HeaderRule)(nil),            // 6: HeaderRule
-	(*Problem)(nil),               // 7: Problem
-	(*Manifest)(nil),              // 8: Manifest
-	(*AuditRecord)(nil),           // 9: AuditRecord
-	(*Principal)(nil),             // 10: Principal
-	nil,                           // 11: Manifest.ContentsEntry
-	(*timestamppb.Timestamp)(nil), // 12: google.protobuf.Timestamp
+	(*BasicCredential)(nil),       // 7: BasicCredential
+	(*BasicAuthRule)(nil),         // 8: BasicAuthRule
+	(*Problem)(nil),               // 9: Problem
+	(*Manifest)(nil),              // 10: Manifest
+	(*AuditRecord)(nil),           // 11: AuditRecord
+	(*Principal)(nil),             // 12: Principal
+	(*ForgeUser)(nil),             // 13: ForgeUser
+	nil,                           // 14: Manifest.ContentsEntry
+	(*timestamppb.Timestamp)(nil), // 15: google.protobuf.Timestamp
 }
 var file_schema_proto_depIdxs = []int32{
 	0,  // 0: Entry.type:type_name -> Type
 	1,  // 1: Entry.transform:type_name -> Transform
 	5,  // 2: HeaderRule.header_map:type_name -> Header
-	11, // 3: Manifest.contents:type_name -> Manifest.ContentsEntry
-	4,  // 4: Manifest.redirects:type_name -> RedirectRule
-	6,  // 5: Manifest.headers:type_name -> HeaderRule
-	7,  // 6: Manifest.problems:type_name -> Problem
-	12, // 7: AuditRecord.timestamp:type_name -> google.protobuf.Timestamp
-	2,  // 8: AuditRecord.event:type_name -> AuditEvent
-	10, // 9: AuditRecord.principal:type_name -> Principal
-	8,  // 10: AuditRecord.manifest:type_name -> Manifest
-	3,  // 11: Manifest.ContentsEntry.value:type_name -> Entry
-	12, // [12:12] is the sub-list for method output_type
-	12, // [12:12] is the sub-list for method input_type
-	12, // [12:12] is the sub-list for extension type_name
-	12, // [12:12] is the sub-list for extension extendee
-	0,  // [0:12] is the sub-list for field type_name
+	7,  // 3: BasicAuthRule.credentials:type_name -> BasicCredential
+	14, // 4: Manifest.contents:type_name -> Manifest.ContentsEntry
+	4,  // 5: Manifest.redirects:type_name -> RedirectRule
+	6,  // 6: Manifest.headers:type_name -> HeaderRule
+	8,  // 7: Manifest.basic_auth:type_name -> BasicAuthRule
+	9,  // 8: Manifest.problems:type_name -> Problem
+	15, // 9: AuditRecord.timestamp:type_name -> google.protobuf.Timestamp
+	2,  // 10: AuditRecord.event:type_name -> AuditEvent
+	12, // 11: AuditRecord.principal:type_name -> Principal
+	10, // 12: AuditRecord.manifest:type_name -> Manifest
+	13, // 13: Principal.forge_user:type_name -> ForgeUser
+	3,  // 14: Manifest.ContentsEntry.value:type_name -> Entry
+	15, // [15:15] is the sub-list for method output_type
+	15, // [15:15] is the sub-list for method input_type
+	15, // [15:15] is the sub-list for extension type_name
+	15, // [15:15] is the sub-list for extension extendee
+	0,  // [0:15] is the sub-list for field type_name
 }
 
 func init() { file_schema_proto_init() }
@@ -936,7 +1146,7 @@ func file_schema_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: unsafe.Slice(unsafe.StringData(file_schema_proto_rawDesc), len(file_schema_proto_rawDesc)),
 			NumEnums:      3,
-			NumMessages:   9,
+			NumMessages:   12,
 			NumExtensions: 0,
 			NumServices:   0,
 		},

src/schema.proto

@@ -76,6 +76,16 @@ message HeaderRule {
 	repeated Header header_map = 2;
 }
 
+message BasicCredential {
+	string username = 1;
+	string password = 2;
+}
+
+message BasicAuthRule {
+	string path = 1;
+	repeated BasicCredential credentials = 2;
+}
+
 message Problem {
 	string path = 1;
 	string cause = 2;
@@ -96,6 +106,7 @@ message Manifest {
 	// Netlify-style `_redirects` and `_headers` rules.
 	repeated RedirectRule redirects = 6;
 	repeated HeaderRule headers = 9;
+	repeated BasicAuthRule basic_auth = 11;
 
 	// Diagnostics for non-fatal errors.
 	repeated Problem problems = 7;
@@ -132,4 +143,12 @@ message AuditRecord {
 message Principal {
 	string ip_address = 1;
 	bool cli_admin = 2;
+	ForgeUser forge_user = 3;
+	string repo_url = 4;
+}
+
+message ForgeUser {
+	string origin = 1;
+	int64 id = 2;
+	string handle = 3;
 }

src/signal.go

@@ -1,4 +1,6 @@
-//go:build unix
+// See https://pkg.go.dev/os/signal#hdr-Windows for a description of what this module
+// will do on Windows (tl;dr nothing calls the reload handler, the interrupt handler works
+// more or less how you'd expect).
 
 package git_pages
 
@@ -21,7 +23,7 @@ func OnReload(handler func()) {
 
 func WaitForInterrupt() {
 	sigint := make(chan os.Signal, 1)
-	signal.Notify(sigint, syscall.SIGINT)
+	signal.Notify(sigint, syscall.SIGINT, syscall.SIGTERM)
 	<-sigint
 	signal.Stop(sigint)
 }

src/signal_other.go

@@ -1,13 +0,0 @@
-//go:build !unix
-
-package git_pages
-
-func OnReload(handler func()) {
-	// not implemented
-}
-
-func WaitForInterrupt() {
-	for {
-		// Ctrl+C not supported
-	}
-}

src/update.go

@@ -10,6 +10,16 @@ import (
 	"google.golang.org/protobuf/proto"
 )
 
+const BlobReferencePrefix = "/git/blobs/"
+
+type UnresolvedRefError struct {
+	missing []string
+}
+
+func (err UnresolvedRefError) Error() string {
+	return fmt.Sprintf("%d unresolved blob references", len(err.missing))
+}
+
 type UpdateOutcome int
 
 const (
@@ -49,6 +59,7 @@ func Update(
 		if err == nil {
 			domain, _, _ := strings.Cut(webRoot, "/")
 			err = backend.CreateDomain(ctx, domain)
+			domainCache.AddDomain(ctx, domain)
 		}
 		if err == nil {
 			if oldManifest == nil {
@@ -117,28 +128,33 @@ var errArchiveFormat = errors.New("unsupported archive format")
 func UpdateFromArchive(
 	ctx context.Context,
 	webRoot string,
+	repoURL string,
 	contentType string,
 	reader io.Reader,
 ) (result UpdateResult) {
 	var err error
 
-	// Ignore errors; here the old manifest is used only to determine the update outcome.
+	// Ignore errors; worst case we have to re-fetch all of the blobs.
 	oldManifest, _, _ := backend.GetManifest(ctx, webRoot, GetManifestOptions{})
 
+	extractTar := func(ctx context.Context, reader io.Reader) (*Manifest, error) {
+		return ExtractTar(ctx, reader, oldManifest)
+	}
+
 	var newManifest *Manifest
 	switch contentType {
 	case "application/x-tar":
 		logc.Printf(ctx, "update %s: (tar)", webRoot)
-		newManifest, err = ExtractTar(reader) // yellow?
+		newManifest, err = extractTar(ctx, reader) // yellow?
 	case "application/x-tar+gzip":
 		logc.Printf(ctx, "update %s: (tar.gz)", webRoot)
-		newManifest, err = ExtractGzip(reader, ExtractTar) // definitely yellow.
+		newManifest, err = ExtractGzip(ctx, reader, extractTar) // definitely yellow.
 	case "application/x-tar+zstd":
 		logc.Printf(ctx, "update %s: (tar.zst)", webRoot)
-		newManifest, err = ExtractZstd(reader, ExtractTar)
+		newManifest, err = ExtractZstd(ctx, reader, extractTar)
 	case "application/zip":
 		logc.Printf(ctx, "update %s: (zip)", webRoot)
-		newManifest, err = ExtractZip(reader)
+		newManifest, err = ExtractZip(ctx, reader, oldManifest)
 	default:
 		err = errArchiveFormat
 	}
@@ -147,6 +163,10 @@ func UpdateFromArchive(
 		logc.Printf(ctx, "update %s err: %s", webRoot, err)
 		result = UpdateResult{UpdateError, nil, err}
 	} else {
+		if repoURL != "" {
+			newManifest.RepoUrl = &repoURL
+		}
+
 		result = Update(ctx, webRoot, oldManifest, newManifest, ModifyManifestOptions{})
 	}
 
@@ -173,11 +193,14 @@ func PartialUpdateFromArchive(
 		return UpdateResult{UpdateError, nil, err}
 	}
 
-	applyTarPatch := func(reader io.Reader) (*Manifest, error) {
+	applyTarPatch := func(ctx context.Context, reader io.Reader) (*Manifest, error) {
 		// Clone the manifest before starting to mutate it. `GetManifest` may return cached
 		// `*Manifest` objects, which should never be mutated.
 		newManifest := &Manifest{}
 		proto.Merge(newManifest, oldManifest)
+		newManifest.RepoUrl = nil
+		newManifest.Branch = nil
+		newManifest.Commit = nil
 		if err := ApplyTarPatch(newManifest, reader, parents); err != nil {
 			return nil, err
 		} else {
@@ -189,13 +212,13 @@ func PartialUpdateFromArchive(
 	switch contentType {
 	case "application/x-tar":
 		logc.Printf(ctx, "patch %s: (tar)", webRoot)
-		newManifest, err = applyTarPatch(reader)
+		newManifest, err = applyTarPatch(ctx, reader)
 	case "application/x-tar+gzip":
 		logc.Printf(ctx, "patch %s: (tar.gz)", webRoot)
-		newManifest, err = ExtractGzip(reader, applyTarPatch)
+		newManifest, err = ExtractGzip(ctx, reader, applyTarPatch)
 	case "application/x-tar+zstd":
 		logc.Printf(ctx, "patch %s: (tar.zst)", webRoot)
-		newManifest, err = ExtractZstd(reader, applyTarPatch)
+		newManifest, err = ExtractZstd(ctx, reader, applyTarPatch)
 	default:
 		err = errArchiveFormat
 	}
@@ -222,7 +245,12 @@ func PartialUpdateFromArchive(
 }
 
 func observeUpdateResult(result UpdateResult) {
-	if result.err != nil {
+	var unresolvedRefErr UnresolvedRefError
+	if errors.As(result.err, &unresolvedRefErr) {
+		// This error is an expected outcome of an incremental update's probe phase.
+	} else if errors.Is(result.err, ErrWriteConflict) {
+		// This error is an expected outcome of an incremental update losing a race.
+	} else if result.err != nil {
 		ObserveError(result.err)
 	}
 }

src/util.go

@@ -6,6 +6,15 @@ import (
 	"strings"
 )
 
+type tuple[A, B any] struct {
+	A A
+	B B
+}
+
+func (t tuple[A, B]) Splat() (A, B) {
+	return t.A, t.B
+}
+
 type BoundedReader struct {
 	inner io.Reader
 	fuel  int64

src/wildcard.go

@@ -11,7 +11,7 @@ import (
 type WildcardPattern struct {
 	Domain        []string
 	CloneURL      *fasttemplate.Template
-	IndexRepos    []*fasttemplate.Template
+	IndexRepo     *fasttemplate.Template
 	IndexBranch   string
 	Authorization bool
 }
@@ -49,27 +49,24 @@ func (pattern *WildcardPattern) Matches(host string) (string, bool) {
 	return subdomain, true
 }
 
-func (pattern *WildcardPattern) ApplyTemplate(userName string, projectName string) ([]string, string) {
-	var repoURLs []string
+func (pattern *WildcardPattern) ApplyTemplate(userName string, projectName string) (string, string) {
+	var repoURL string
 	var branch string
 	repoURLTemplate := pattern.CloneURL
 	if projectName == ".index" {
-		for _, indexRepoTemplate := range pattern.IndexRepos {
-			indexRepo := indexRepoTemplate.ExecuteString(map[string]any{"user": userName})
-			repoURLs = append(repoURLs, repoURLTemplate.ExecuteString(map[string]any{
-				"user":    userName,
-				"project": indexRepo,
-			}))
-		}
+		repoURL = repoURLTemplate.ExecuteString(map[string]any{
+			"user":    userName,
+			"project": pattern.IndexRepo.ExecuteString(map[string]any{"user": userName}),
+		})
 		branch = pattern.IndexBranch
 	} else {
-		repoURLs = append(repoURLs, repoURLTemplate.ExecuteString(map[string]any{
+		repoURL = repoURLTemplate.ExecuteString(map[string]any{
 			"user":    userName,
 			"project": projectName,
-		}))
+		})
 		branch = "pages"
 	}
-	return repoURLs, branch
+	return repoURL, branch
 }
 
 func TranslateWildcards(configs []WildcardConfig) ([]*WildcardPattern, error) {
@@ -80,14 +77,10 @@ func TranslateWildcards(configs []WildcardConfig) ([]*WildcardPattern, error) {
 			return nil, fmt.Errorf("wildcard pattern: clone URL: %w", err)
 		}
 
-		var indexRepoTemplates []*fasttemplate.Template
 		var indexRepoBranch string = config.IndexRepoBranch
-		for _, indexRepo := range config.IndexRepos {
-			indexRepoTemplate, err := fasttemplate.NewTemplate(indexRepo, "<", ">")
-			if err != nil {
-				return nil, fmt.Errorf("wildcard pattern: index repo: %w", err)
-			}
-			indexRepoTemplates = append(indexRepoTemplates, indexRepoTemplate)
+		indexRepoTemplate, err := fasttemplate.NewTemplate(config.IndexRepo, "<", ">")
+		if err != nil {
+			return nil, fmt.Errorf("wildcard pattern: index repo: %w", err)
 		}
 
 		authorization := false
@@ -107,7 +100,7 @@ func TranslateWildcards(configs []WildcardConfig) ([]*WildcardPattern, error) {
 		wildcardPatterns = append(wildcardPatterns, &WildcardPattern{
 			Domain:        strings.Split(config.Domain, "."),
 			CloneURL:      cloneURLTemplate,
-			IndexRepos:    indexRepoTemplates,
+			IndexRepo:     indexRepoTemplate,
 			IndexBranch:   indexRepoBranch,
 			Authorization: authorization,
 		})