Use right URL when fetching Forgejo user data for audit.

No observed issues on Grebedoc for a month, so it should be stable now.

.forgejo/workflows/ci.yaml

@@ -5,18 +5,21 @@ on:
   pull_request:
   workflow_dispatch:
 
+env:
+  FORGE: codeberg.org
+
 jobs:
   check:
-    runs-on: codeberg-small-lazy
+    runs-on: debian-trixie
     container:
-      image: docker.io/library/node:24-trixie-slim@sha256:45babd1b4ce0349fb12c4e24bf017b90b96d52806db32e001e3013f341bef0fe
+      image: docker.io/library/node:24-trixie-slim@sha256:4fc981bf8dfc5e36e15e0cb73c5761a14cabff0932dcad1cf26cd3c3425db5d4
     steps:
       - name: Check out source code
-        uses: https://code.forgejo.org/actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
+        uses: https://code.forgejo.org/actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - name: Set up toolchain
-        uses: https://code.forgejo.org/actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6
+        uses: https://code.forgejo.org/actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
         with:
-          go-version: '>=1.25.0'
+          go-version: '>=1.25.6'
       - name: Install dependencies
         run: |
           apt-get -y update
@@ -25,25 +28,28 @@ jobs:
       - name: Build service
         run: |
           go build
+      - name: Run tests
+        run: |
+          go test ./...
       - name: Run static analysis
         run: |
-          go vet
-          staticcheck
+          go vet ./...
+          staticcheck ./...
 
   release:
     # IMPORTANT: This workflow step will not work without the Releases unit enabled!
     if: ${{ forge.ref == 'refs/heads/main' || startsWith(forge.event.ref, 'refs/tags/v') }}
     needs: [check]
-    runs-on: codeberg-medium-lazy
+    runs-on: debian-trixie
     container:
-      image: docker.io/library/node:24-trixie-slim@sha256:ef4ca6d078dd18322059a1f051225f7bbfc2bb60c16cbb5d8a1ba2cc8964fe8a
+      image: docker.io/library/node:24-trixie-slim@sha256:4fc981bf8dfc5e36e15e0cb73c5761a14cabff0932dcad1cf26cd3c3425db5d4
     steps:
       - name: Check out source code
-        uses: https://code.forgejo.org/actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
+        uses: https://code.forgejo.org/actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
       - name: Set up toolchain
-        uses: https://code.forgejo.org/actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6
+        uses: https://code.forgejo.org/actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
         with:
-          go-version: '>=1.25.0'
+          go-version: '>=1.25.6'
       - name: Install dependencies
         run: |
           apt-get -y update
@@ -58,7 +64,7 @@ jobs:
           build linux   arm64
           build darwin  arm64
       - name: Create release
-        uses: https://code.forgejo.org/actions/forgejo-release@v2.7.3
+        uses: https://code.forgejo.org/actions/forgejo-release@e7b60f9ae8d4bbf3ed4cc178e4656ce40eb67256 # v2.11.2
         with:
           tag: ${{ startsWith(forge.event.ref, 'refs/tags/v') && forge.ref_name || 'latest' }}
           release-dir: assets
@@ -69,35 +75,36 @@ jobs:
   package:
     if: ${{ forge.ref == 'refs/heads/main' || startsWith(forge.event.ref, 'refs/tags/v') }}
     needs: [check]
-    runs-on: codeberg-medium-lazy
+    runs-on: debian-trixie
     container:
-      image: docker.io/library/node:24-trixie-slim@sha256:ef4ca6d078dd18322059a1f051225f7bbfc2bb60c16cbb5d8a1ba2cc8964fe8a
+      image: docker.io/library/node:24-trixie-slim@sha256:4fc981bf8dfc5e36e15e0cb73c5761a14cabff0932dcad1cf26cd3c3425db5d4
     steps:
       - name: Install dependencies
         run: |
           apt-get -y update
-          apt-get -y install buildah ca-certificates
+          apt-get -y install ca-certificates buildah qemu-user-binfmt
       - name: Check out source code
-        uses: https://code.forgejo.org/actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
+        uses: https://code.forgejo.org/actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - name: Authenticate with Docker
+        run: |
+          buildah login --authfile=/tmp/authfile-${FORGE}.json \
+            -u ${{ vars.PACKAGES_USER }} -p ${{ secrets.PACKAGES_TOKEN }} ${FORGE}
       - name: Build container
         run: |
           printf '[storage]\ndriver="vfs"\nrunroot="/run/containers/storage"\ngraphroot="/var/lib/containers/storage"\n' | tee /etc/containers/storage.conf
-          buildah build --arch=amd64 --tag=container:${VER}-amd64 .
-          buildah build --arch=arm64 --tag=container:${VER}-arm64 .
+          buildah build ${CACHE} --arch=amd64 --tag=container:${VER}-amd64
+          buildah build ${CACHE} --arch=arm64 --tag=container:${VER}-arm64
           buildah manifest create container:${VER} \
             container:${VER}-amd64 \
             container:${VER}-arm64
         env:
           BUILDAH_ISOLATION: chroot
           VER: ${{ startsWith(forge.event.ref, 'refs/tags/v') && forge.ref_name || 'latest' }}
+          CACHE: ${{ format('--authfile=/tmp/authfile-{0}.json --layers --cache-from {0}/{1}/cache --cache-to {0}/{1}/cache', env.FORGE, forge.repository) }}
       - if: ${{ forge.repository == 'git-pages/git-pages' }}
         name: Push container to Codeberg
         run: |
-          buildah login --authfile=/tmp/authfile-${FORGE}.json \
-            -u ${{ vars.PACKAGES_USER }} -p ${{ secrets.PACKAGES_TOKEN }} ${FORGE}
           buildah manifest push --authfile=/tmp/authfile-${FORGE}.json \
             --all container:${VER} "docker://${FORGE}/${{ forge.repository }}:${VER/v/}"
         env:
-          BUILDAH_ISOLATION: chroot
-          FORGE: codeberg.org
           VER: ${{ startsWith(forge.event.ref, 'refs/tags/v') && forge.ref_name || 'latest' }}

.gitignore

@@ -4,3 +4,4 @@
 /data
 /config*.toml*
 /git-pages
+/site

Dockerfile

@@ -1,35 +1,32 @@
 # Install CA certificates.
-FROM docker.io/library/alpine:latest@sha256:4b7ce07002c69e8f3d704a9c5d6fd3053be500b7f1c69fc0d80990c2ad8dd412 AS ca-certificates-builder
+FROM docker.io/library/alpine:3 AS ca-certificates-builder
 RUN apk --no-cache add ca-certificates
 
 # Build supervisor.
-FROM docker.io/library/golang:1.25-alpine@sha256:aee43c3ccbf24fdffb7295693b6e33b21e01baec1b2a55acc351fde345e9ec34 AS supervisor-builder
+FROM docker.io/library/golang:1.26-alpine@sha256:2389ebfa5b7f43eeafbd6be0c3700cc46690ef842ad962f6c5bd6be49ed82039 AS supervisor-builder
 RUN apk --no-cache add git
 WORKDIR /build
 RUN git clone https://github.com/ochinchina/supervisord . && \
     git checkout 16cb640325b3a4962b2ba17d68fb5c2b1e1b6b3c
-RUN GOBIN=/usr/bin go install -ldflags "-s -w" && \
-    go clean -cache -modcache
+RUN GOBIN=/usr/bin go install -ldflags "-s -w"
 
 # Build Caddy with S3 storage backend.
-FROM docker.io/library/caddy:2.10.2-builder@sha256:53f91ad7c5f1ab9a607953199b7c1e10920c570ae002aef913d68ed7464fb19f AS caddy-builder
+FROM docker.io/library/caddy:2.11.2-builder@sha256:d4f984844fc3b867ac88fd814285a38eaaf5b3ecadb9ca1b3b0397182ef60cfe AS caddy-builder
 RUN xcaddy build ${CADDY_VERSION} \
-    --with=github.com/ss098/certmagic-s3@v0.0.0-20250922022452-8af482af5f39 && \
-    go clean -cache -modcache
+    --with=github.com/ss098/certmagic-s3@v0.0.0-20250922022452-8af482af5f39
 
 # Build git-pages.
-FROM docker.io/library/golang:1.25-alpine@sha256:aee43c3ccbf24fdffb7295693b6e33b21e01baec1b2a55acc351fde345e9ec34 AS git-pages-builder
+FROM docker.io/library/golang:1.26-alpine@sha256:2389ebfa5b7f43eeafbd6be0c3700cc46690ef842ad962f6c5bd6be49ed82039 AS git-pages-builder
 RUN apk --no-cache add git
 WORKDIR /build
 COPY go.mod go.sum ./
 RUN go mod download
 COPY *.go ./
 COPY src/ ./src/
-RUN go build -ldflags "-s -w" -o git-pages . && \
-    go clean -cache -modcache
+RUN go build -ldflags "-s -w" -o git-pages .
 
 # Compose git-pages and Caddy.
-FROM docker.io/library/busybox:1.37.0-musl@sha256:ef13e7482851632be3faf5bd1d28d4727c0810901d564b35416f309975a12a30
+FROM docker.io/library/busybox:1.37.0-musl@sha256:19b646668802469d968a05342a601e78da4322a414a7c09b1c9ee25165042138
 COPY --from=ca-certificates-builder /etc/ssl/cert.pem /etc/ssl/cert.pem
 COPY --from=supervisor-builder /usr/bin/supervisord /bin/supervisord
 COPY --from=caddy-builder /usr/bin/caddy /bin/caddy
@@ -39,7 +36,7 @@ WORKDIR /app
 RUN mkdir /app/data
 COPY conf/supervisord.conf /app/supervisord.conf
 COPY conf/Caddyfile /app/Caddyfile
-COPY conf/config.example.toml /app/config.toml
+COPY conf/config.docker.toml /app/config.toml
 
 # Caddy ports:
 EXPOSE 80/tcp 443/tcp 443/udp
@@ -49,8 +46,8 @@ EXPOSE 3000/tcp 3001/tcp 3002/tcp
 # While the default command is to run git-pages standalone, the intended configuration
 # is to use it with Caddy and store both site data and credentials to an S3-compatible
 # object store.
-#  * In a standalone configuration, the default, git-caddy listens on port 3000 (http).
-#  * In a combined configuration, supervisord launches both git-caddy and Caddy, and
+#  * In a standalone configuration, the default, git-pages listens on port 3000 (http).
+#  * In a combined configuration, supervisord launches both git-pages and Caddy, and
 #    Caddy listens on ports 80 (http) and 443 (https).
 CMD ["git-pages"]
 # CMD ["supervisord"]

README.md

@@ -1,13 +1,20 @@
 git-pages
 =========
 
-_git-pages_ is a static site server for use with Git forges (i.e. a GitHub Pages replacement). It is written with efficiency in mind, scaling horizontally to any number of deployed sites and concurrent requests and serving sites up to hundreds of megabytes in size, while being equally suitable for single-user deployments.
+_git-pages_ is a static site server for use with Git forges (i.e. a GitHub Pages replacement). It is written with efficiency in mind, scaling horizontally to any number of machines and serving sites up to multiple gigabytes in size, while being equally suitable for small single-user deployments.
 
-It is implemented in Go and has no other mandatory dependencies, although it is designed to be used together with the [Caddy server][caddy] (for TLS termination) and an [Amazon S3](https://aws.amazon.com/s3/) compatible object store (for horizontal scalability of storage).
+It is implemented in Go and has no other mandatory dependencies, although it is designed to be used together with the [Caddy server][caddy] for TLS termination. Site data may be stored on the filesystem or in an [Amazon S3](https://aws.amazon.com/s3/) compatible object store.
 
-The included Docker container provides everything needed to deploy a Pages service, including zero-configuration on-demand provisioning of TLS certificates from [Let's Encrypt](https://letsencrypt.org/), and runs on any commodity cloud infrastructure. There is also a first-party deployment of _git-pages_ at [grebedoc.dev](https://grebedoc.dev).
+The included Docker container provides everything needed to deploy a Pages service, including zero-configuration on-demand provisioning of TLS certificates from [Let's Encrypt](https://letsencrypt.org/), and runs on any commodity cloud infrastructure.
+
+> [!TIP]
+> If you want to publish a site using _git-pages_ to an existing service like Codeberg Pages or [Grebedoc][grebedoc], consider using the [CLI tool][git-pages-cli] or the [Forgejo Action][git-pages-action].
 
 [caddy]: https://caddyserver.com/
+[git-pages-cli]: https://codeberg.org/git-pages/git-pages-cli
+[git-pages-action]: https://codeberg.org/git-pages/action
+[codeberg-pages]: https://codeberg.page
+[grebedoc]: https://grebedoc.dev
 
 
 Quickstart
@@ -32,8 +39,6 @@ b70644b523c4aaf4efd206a588087a1d406cb047
 
 The `pages` branch of the repository is now available at http://localhost:3000/!
 
-[git-pages-cli]: https://codeberg.org/git-pages/git-pages-cli
-
 
 Deployment
 ----------
@@ -63,21 +68,38 @@ Features
     - Site URLs that have a path starting with `.git-pages/...` are reserved for _git-pages_ itself.
         - The `.git-pages/health` URL returns `ok` with the `Last-Modified:` header set to the manifest modification time.
         - The `.git-pages/manifest.json` URL returns a [ProtoJSON](https://protobuf.dev/programming-guides/json/) representation of the deployed site manifest with the `Last-Modified:` header set to the manifest modification time. It enumerates site structure, redirect rules, and errors that were not severe enough to abort publishing. Note that **the manifest JSON format is not stable and will change without notice**.
-        - **With feature `archive-site`:** The `.git-pages/archive.tar` URL returns a tar archive of all site contents, including `_redirects` and `_headers` files (reconstructed from the manifest), with the `Last-Modified:` header set to the manifest modification time. Compression can be enabled using the `Accept-Encoding:` HTTP header (only).
+        - The `.git-pages/archive.tar` URL returns a tar archive of all site contents, including `_redirects` and `_headers` files (reconstructed from the manifest), with the `Last-Modified:` header set to the manifest modification time. Compression can be enabled using the `Accept-Encoding:` HTTP header (only).
 * In response to a `PUT` or `POST` request, the server updates a site with new content. The URL of the request must be the root URL of the site that is being published.
     - If the `PUT` method receives an `application/x-www-form-urlencoded` body, it contains a repository URL to be shallowly cloned. The `Branch` header contains the branch to be checked out; the `pages` branch is used if the header is absent.
     - If the `PUT` method receives an `application/x-tar`, `application/x-tar+gzip`, `application/x-tar+zstd`, or `application/zip` body, it contains an archive to be extracted.
     - The `POST` method requires an `application/json` body containing a Forgejo/Gitea/Gogs/GitHub webhook event payload. Requests where the `ref` key contains anything other than `refs/heads/pages` are ignored, and only the `pages` branch is used. The `repository.clone_url` key contains a repository URL to be shallowly cloned.
     - If the received contents is empty, performs the same action as `DELETE`.
+* In response to a `PATCH` request, the server partially updates a site with new content. The URL of the request must be the root URL of the site that is being published.
+    - The request must have a `application/x-tar`, `application/x-tar+gzip`, or `application/x-tar+zstd` body, whose contents is *merged* with the existing site contents as follows:
+        - A character device entry with major 0 and minor 0 is treated as a "whiteout marker" (following [unionfs][whiteout]): it causes any existing file or directory with the same name to be deleted.
+        - A directory entry replaces any existing file or directory with the same name (if any), recursively removing the old contents.
+        - A file or symlink entry replaces any existing file or directory with the same name (if any).
+        - If there is no `Create-Parents:` header or a `Create-Parents: no` header is present, the parent path of an entry must exist and refer to a directory.
+        - If a `Create-Parents: yes` header is present, any missing segments in the parent path of an entry will be created (like `mkdir -p`). Any existing segments refer to directories.
+    - The request must have a `Atomic: yes` or `Atomic: no` header. Not every backend configuration makes it possible to perform atomic compare-and-swap operations; on backends without atomic CAS support, `Atomic: yes` requests will fail, while `Atomic: no` requests will provide a best-effort approximation.
+    - If a `PATCH` request loses a race against another content update request, it may return `409 Conflict`. This is true regardless of the `Atomic:` header value. Whenever this happens, resubmit the request as-is.
+    - If the site has no contents after the update is applied, performs the same action as `DELETE`.
 * In response to a `DELETE` request, the server unpublishes a site. The URL of the request must be the root URL of the site that is being unpublished. Site data remains stored for an indeterminate period of time, but becomes completely inaccessible.
+* If a `Dry-Run: yes` header is provided with a `PUT`, `PATCH`, `DELETE`, or `POST` request, only the authorization checks are run; no destructive updates are made.
 * All updates to site content are atomic (subject to consistency guarantees of the storage backend). That is, there is an instantaneous moment during an update before which the server will return the old content and after which it will return the new content.
 * Files with a certain name, when placed in the root of a site, have special functions:
     - [Netlify `_redirects`][_redirects] file can be used to specify HTTP redirect and rewrite rules. The _git-pages_ implementation currently does not support placeholders, query parameters, or conditions, and may differ from Netlify in other minor ways. If you find that a supported `_redirects` file feature does not work the same as on Netlify, please file an issue. (Note that _git-pages_ does not perform URL normalization; `/foo` and `/foo/` are *not* the same, unlike with Netlify.)
     - [Netlify `_headers`][_headers] file can be used to specify custom HTTP response headers (if allowlisted by configuration). In particular, this is useful to enable [CORS requests][cors]. The _git-pages_ implementation may differ from Netlify in minor ways; if you find that a `_headers` file feature does not work the same as on Netlify, please file an issue.
+* Incremental updates can be made using `PUT` or `PATCH` requests where the body contains an archive (both tar and zip are supported).
+    - Any archive entry that is a symlink to `/git/pages/<git-sha256>` is replaced with an existing manifest entry for the same site whose git blob hash matches `<git-sha256>`. If there is no existing manifest entry with the specified git hash, the update fails with a `422 Unprocessable Entity`.
+    - For this error response only, if the negotiated content type is `application/vnd.git-pages.unresolved`, the response will contain the `<git-sha256>` of each unresolved reference, one per line.
+* Support for SHA-256 Git hashes is [limited by go-git][go-git-sha256]; once go-git implements the required features, _git-pages_ will automatically gain support for SHA-256 Git hashes. Note that shallow clones (used by _git-pages_ to conserve bandwidth if available) aren't supported yet in the Git protocol as of 2025.
 
 [_redirects]: https://docs.netlify.com/manage/routing/redirects/overview/
 [_headers]: https://docs.netlify.com/manage/routing/headers/
 [cors]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/CORS
+[go-git-sha256]: https://github.com/go-git/go-git/issues/706
+[whiteout]: https://docs.kernel.org/filesystems/overlayfs.html#whiteouts-and-opaque-directories
 
 
 Authorization
@@ -85,17 +107,17 @@ Authorization
 
 DNS is the primary authorization method, using either TXT records or wildcard matching. In certain cases, git forge authorization is used in addition to DNS.
 
-The authorization flow for content updates (`PUT`, `DELETE`, `POST` requests) proceeds sequentially in the following order, with the first of multiple applicable rule taking precedence:
+The authorization flow for content updates (`PUT`, `PATCH`, `DELETE`, `POST` requests) proceeds sequentially in the following order, with the first of multiple applicable rule taking precedence:
 
 1. **Development Mode:** If the environment variable `PAGES_INSECURE` is set to a truthful value like `1`, the request is authorized.
-2. **DNS Challenge:** If the method is `PUT`, `DELETE`, `POST`, and a well-formed `Authorization:` header is provided containing a `<token>`, and a TXT record lookup at `_git-pages-challenge.<host>` returns a record whose concatenated value equals `SHA256("<host> <token>")`, the request is authorized.
+2. **DNS Challenge:** If the method is `PUT`, `PATCH`, `DELETE`, `POST`, and a well-formed `Authorization:` header is provided containing a `<token>`, and a TXT record lookup at `_git-pages-challenge.<host>` returns a record whose concatenated value equals `SHA256("<host> <token>")`, the request is authorized.
     - **`Pages` scheme:** Request includes an `Authorization: Pages <token>` header.
     - **`Basic` scheme:** Request includes an `Authorization: Basic <basic>` header, where `<basic>` is equal to `Base64("Pages:<token>")`. (Useful for non-Forgejo forges.)
-3. **DNS Allowlist:** If the method is `PUT` or `POST`, and a TXT record lookup at `_git-pages-repository.<host>` returns a set of well-formed absolute URLs, and (for `PUT` requests) the body contains a repository URL, and the requested clone URLs is contained in this set of URLs, the request is authorized.
+3. **DNS Allowlist:** If the method is `PUT` or `POST`, and the request URL is `scheme://<user>.<host>/`, and a TXT record lookup at `_git-pages-repository.<host>` returns a set of well-formed absolute URLs, and (for `PUT` requests) the body contains a repository URL, and the requested clone URLs is contained in this set of URLs, the request is authorized.
 4. **Wildcard Match (content):** If the method is `POST`, and a `[[wildcard]]` configuration section exists where the suffix of a hostname (compared label-wise) is equal to `[[wildcard]].domain`, and (for `PUT` requests) the body contains a repository URL, and the requested clone URL is a *matching* clone URL, the request is authorized.
     - **Index repository:** If the request URL is `scheme://<user>.<host>/`, a *matching* clone URL is computed by templating `[[wildcard]].clone-url` with `<user>` and `<project>`, where `<project>` is computed by templating each element of `[[wildcard]].index-repos` with `<user>`, and `[[wildcard]]` is the section where the match occurred.
     - **Project repository:** If the request URL is `scheme://<user>.<host>/<project>/`, a *matching* clone URL is computed by templating `[[wildcard]].clone-url` with `<user>` and `<project>`, and `[[wildcard]]` is the section where the match occurred.
-5. **Forge Authorization:** If the method is `PUT`, and the body contains an archive, and a `[[wildcard]]` configuration section exists where the suffix of a hostname (compared label-wise) is equal to `[[wildcard]].domain`, and `[[wildcard]].authorization` is non-empty, and the request includes a `Forge-Authorization:` header, and the header (when forwarded as `Authorization:`) grants push permissions to a repository at the *matching* clone URL (as defined above) as determined by an API call to the forge, the request is authorized. (This enables publishing a site for a private repository.)
+5. **Forge Authorization:** If the method is `PUT` or `PATCH` or `DELETE`, and (unless the method is `DELETE`) the body contains an archive, and a `[[wildcard]]` configuration section exists where the suffix of a hostname (compared label-wise) is equal to `[[wildcard]].domain`, and `[[wildcard]].authorization` is non-empty, and the request includes a `Forge-Authorization:` header, and the header (when forwarded as `Authorization:`) grants push permissions to a repository at the *matching* clone URL (as defined above) as determined by an API call to the forge, the request is authorized. (This enables publishing a site for a private repository.)
 5. **Default Deny:** Otherwise, the request is not authorized.
 
 The authorization flow for metadata retrieval (`GET` requests with site paths starting with `.git-pages/`) in the following order, with the first of multiple applicable rule taking precedence:
@@ -115,6 +137,11 @@ _git-pages_ has robust observability features built in:
     * If `SENTRY_DSN` environment variable is set, panics are reported to Sentry.
     * If `SENTRY_DSN` and `SENTRY_LOGS=1` environment variables are set, logs are uploaded to Sentry.
     * If `SENTRY_DSN` and `SENTRY_TRACING=1` environment variables are set, traces are uploaded to Sentry.
+* Optional syslog integration allows transmitting application logs to a syslog daemon. When present, the `SYSLOG_ADDR` environment variable enables the integration, and the value is used to configure the syslog destination. The value must follow the format `family/address` and is usually one of the following:
+    * a Unix datagram socket: `unixgram//dev/log`;
+    * TLS over TCP: `tcp+tls/host:port`;
+    * plain TCP: `tcp/host:post`;
+    * UDP: `udp/host:port`.
 
 
 Architecture (v2)
@@ -160,4 +187,4 @@ The specific arrangement used is clearly not optimal; at a minimum it is likely
 License
 -------
 
-[0-clause BSD](LICENSE-0BSD.txt)
+[0-clause BSD](LICENSE.txt)

conf/Caddyfile

@@ -25,11 +25,5 @@ https://, http:// {
 		on_demand
 	}
 
-	# initial PUT/POST for a new domain has to happen over HTTP
-	@upgrade `method('GET') && protocol('http')`
-	redir @upgrade https://{host}{uri} 301
-
 	reverse_proxy http://localhost:3000
-	header Alt-Svc `h3=":443"; persist=1, h2=":443"; persist=1`
-	encode
 }

conf/config.docker.toml

@@ -0,0 +1,4 @@
+[server]
+pages = "tcp/:3000"
+caddy = "tcp/:3001"
+metrics = "tcp/:3002"

conf/config.example.toml

@@ -5,17 +5,20 @@ log-format = "text"
 
 [server]
 # Use "-" to disable the handler.
-pages = "tcp/:3000"
-caddy = "tcp/:3001"
-metrics = "tcp/:3002"
+pages = "tcp/localhost:3000"
+caddy = "tcp/localhost:3001"
+metrics = "tcp/localhost:3002"
 
 [[wildcard]] # non-default section
 domain = "codeberg.page"
 clone-url = "https://codeberg.org/<user>/<project>.git"
-index-repos = ["<user>.codeberg.page", "pages"]
+index-repo = "pages"
 index-repo-branch = "main"
 authorization = "forgejo"
-fallback-proxy-to = "https://codeberg.page"
+
+[fallback] # non-default section
+proxy-to = "https://codeberg.page"
+insecure = false
 
 [storage]
 type = "fs"
@@ -23,7 +26,7 @@ type = "fs"
 [storage.fs]
 root = "./data"
 
-[storage.s3] # non-default bucket configuration
+[storage.s3] # non-default section
 endpoint = "play.min.io"
 access-key-id = "Q3AM3UQ867SPQQA43P2F"
 secret-access-key = "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
@@ -47,8 +50,14 @@ max-symlink-depth = 16
 update-timeout = "60s"
 max-heap-size-ratio = 0.5 # * RAM_size
 forbidden-domains = []
-# allowed-repository-url-prefixes = <nil>
+allowed-repository-url-prefixes = []
 allowed-custom-headers = ["X-Clacks-Overhead"]
 
+[audit]
+node-id = 0
+collect = false
+include-ip = ""
+notify-url = ""
+
 [observability]
 slow-response-threshold = "500ms"

flake.lock

@@ -18,6 +18,29 @@
         "type": "github"
       }
     },
+    "gomod2nix": {
+      "inputs": {
+        "flake-utils": [
+          "flake-utils"
+        ],
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1763982521,
+        "narHash": "sha256-ur4QIAHwgFc0vXiaxn5No/FuZicxBr2p0gmT54xZkUQ=",
+        "owner": "nix-community",
+        "repo": "gomod2nix",
+        "rev": "02e63a239d6eabd595db56852535992c898eba72",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "gomod2nix",
+        "type": "github"
+      }
+    },
     "nix-filter": {
       "locked": {
         "lastModified": 1757882181,
@@ -52,6 +75,7 @@
     "root": {
       "inputs": {
         "flake-utils": "flake-utils",
+        "gomod2nix": "gomod2nix",
         "nix-filter": "nix-filter",
         "nixpkgs": "nixpkgs"
       }

flake.nix

@@ -3,6 +3,12 @@
     nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
     flake-utils.url = "github:numtide/flake-utils";
     nix-filter.url = "github:numtide/nix-filter";
+
+    gomod2nix = {
+      url = "github:nix-community/gomod2nix";
+      inputs.nixpkgs.follows = "nixpkgs";
+      inputs.flake-utils.follows = "flake-utils";
+    };
   };
 
   outputs =
@@ -11,13 +17,20 @@
       nixpkgs,
       flake-utils,
       nix-filter,
-    }:
+      ...
+    }@inputs:
     flake-utils.lib.eachDefaultSystem (
       system:
       let
-        pkgs = nixpkgs.legacyPackages.${system};
+        pkgs = import nixpkgs {
+          inherit system;
 
-        git-pages = pkgs.buildGo125Module {
+          overlays = [
+            inputs.gomod2nix.overlays.default
+          ];
+        };
+
+        git-pages = pkgs.buildGoApplication {
           pname = "git-pages";
           version = "0";
 
@@ -43,7 +56,8 @@
             "-s -w"
           ];
 
-          vendorHash = "sha256-oVXELOXbRTzzU8pUGNE4K552thlZXGAX7qpv6ETwz6o=";
+          go = pkgs.go_1_25;
+          modules = ./gomod2nix.toml;
         };
       in
       {
@@ -56,6 +70,7 @@
 
           packages = with pkgs; [
             caddy
+            gomod2nix
           ];
         };
 

go.mod

@@ -3,24 +3,30 @@ module codeberg.org/git-pages/git-pages
 go 1.25.0
 
 require (
-	codeberg.org/git-pages/go-headers v1.1.0
+	codeberg.org/git-pages/go-headers v1.1.1
+	codeberg.org/git-pages/go-slog-syslog v0.0.0-20251207093707-892f654e80b7
 	github.com/KimMachineGun/automemlimit v0.7.5
 	github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500
 	github.com/creasty/defaults v1.8.0
-	github.com/getsentry/sentry-go v0.36.2
-	github.com/getsentry/sentry-go/slog v0.36.2
-	github.com/go-git/go-billy/v6 v6.0.0-20251026101908-623011986e70
-	github.com/go-git/go-git/v6 v6.0.0-20251029213217-0bbfc0875edd
-	github.com/klauspost/compress v1.18.1
-	github.com/maypok86/otter/v2 v2.2.1
-	github.com/minio/minio-go/v7 v7.0.95
+	github.com/dghubble/trie v0.1.0
+	github.com/fatih/color v1.18.0
+	github.com/getsentry/sentry-go v0.43.0
+	github.com/getsentry/sentry-go/slog v0.43.0
+	github.com/go-git/go-billy/v6 v6.0.0-20260226131633-45bd0956d66f
+	github.com/go-git/go-git/v6 v6.0.0-20260305211659-2083cf940afa
+	github.com/jpillora/backoff v1.0.0
+	github.com/kankanreno/go-snowflake v1.2.0
+	github.com/klauspost/compress v1.18.4
+	github.com/maypok86/otter/v2 v2.3.0
+	github.com/minio/minio-go/v7 v7.0.99
 	github.com/pelletier/go-toml/v2 v2.2.4
 	github.com/pquerna/cachecontrol v0.2.0
 	github.com/prometheus/client_golang v1.23.2
-	github.com/samber/slog-multi v1.5.0
+	github.com/samber/slog-multi v1.7.1
 	github.com/tj/go-redirects v0.0.0-20200911105812-fd1ba1020b37
 	github.com/valyala/fasttemplate v1.2.2
-	google.golang.org/protobuf v1.36.10
+	golang.org/x/net v0.51.0
+	google.golang.org/protobuf v1.36.11
 )
 
 require (
@@ -28,37 +34,44 @@ require (
 	github.com/ProtonMail/go-crypto v1.3.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/cloudflare/circl v1.6.1 // indirect
-	github.com/cyphar/filepath-securejoin v0.5.0 // indirect
+	github.com/cloudflare/circl v1.6.3 // indirect
+	github.com/cyphar/filepath-securejoin v0.6.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/go-git/gcfg/v2 v2.0.2 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
-	github.com/goccy/go-json v0.10.5 // indirect
 	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/kevinburke/ssh_config v1.4.0 // indirect
+	github.com/kevinburke/ssh_config v1.5.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
-	github.com/minio/crc64nvme v1.0.2 // indirect
+	github.com/klauspost/crc32 v1.3.0 // indirect
+	github.com/leodido/go-syslog/v4 v4.3.0 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/minio/crc64nvme v1.1.1 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
 	github.com/philhofer/fwd v1.2.0 // indirect
 	github.com/pjbgf/sha1cd v0.5.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.66.1 // indirect
 	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/rs/xid v1.6.0 // indirect
-	github.com/samber/lo v1.51.0 // indirect
-	github.com/samber/slog-common v0.19.0 // indirect
+	github.com/samber/lo v1.52.0 // indirect
+	github.com/samber/slog-common v0.20.0 // indirect
 	github.com/sergi/go-diff v1.4.0 // indirect
-	github.com/tinylib/msgp v1.3.0 // indirect
+	github.com/stretchr/testify v1.11.1 // indirect
+	github.com/tinylib/msgp v1.6.1 // indirect
 	github.com/tj/assert v0.0.3 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
-	golang.org/x/crypto v0.43.0 // indirect
-	golang.org/x/net v0.46.0 // indirect
-	golang.org/x/sys v0.37.0 // indirect
-	golang.org/x/text v0.30.0 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/crypto v0.48.0 // indirect
+	golang.org/x/sys v0.41.0 // indirect
+	golang.org/x/text v0.34.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,7 +1,7 @@
-codeberg.org/git-pages/go-headers v1.0.0 h1:hvGU97hQdXaT5HwCpZJWQdg7akvtOBCSUNL4u2a5uTs=
-codeberg.org/git-pages/go-headers v1.0.0/go.mod h1:N4gwH0U3YPwmuyxqH7xBA8j44fTPX+vOEP7ejJVBPts=
-codeberg.org/git-pages/go-headers v1.1.0 h1:rk7/SOSsn+XuL7PUQZFYUaWKHEaj6K8mXmUV9rF2VxE=
-codeberg.org/git-pages/go-headers v1.1.0/go.mod h1:N4gwH0U3YPwmuyxqH7xBA8j44fTPX+vOEP7ejJVBPts=
+codeberg.org/git-pages/go-headers v1.1.1 h1:fpIBELKo66Z2k+gCeYl5mCEXVQ99Lmx1iup1nbo2shE=
+codeberg.org/git-pages/go-headers v1.1.1/go.mod h1:N4gwH0U3YPwmuyxqH7xBA8j44fTPX+vOEP7ejJVBPts=
+codeberg.org/git-pages/go-slog-syslog v0.0.0-20251207093707-892f654e80b7 h1:+rkrAxhNZo/eKEcKOqVOsF6ohAPv5amz0JLburOeRjs=
+codeberg.org/git-pages/go-slog-syslog v0.0.0-20251207093707-892f654e80b7/go.mod h1:8NPSXbYcVb71qqNM5cIgn1/uQgMisLbu2dVD1BNxsUw=
 github.com/KimMachineGun/automemlimit v0.7.5 h1:RkbaC0MwhjL1ZuBKunGDjE/ggwAX43DwZrJqVwyveTk=
 github.com/KimMachineGun/automemlimit v0.7.5/go.mod h1:QZxpHaGOQoYvFhv/r4u3U0JTC2ZcOwbSr11UZF46UBM=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
@@ -18,54 +18,60 @@ github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 h1:6lhrsTEnloDPXye
 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0=
-github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
+github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8=
+github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4=
 github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk=
 github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM=
-github.com/cyphar/filepath-securejoin v0.5.0 h1:hIAhkRBMQ8nIeuVwcAoymp7MY4oherZdAxD+m0u9zaw=
-github.com/cyphar/filepath-securejoin v0.5.0/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI=
+github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE=
+github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dghubble/trie v0.1.0 h1:kJnjBLFFElBwS60N4tkPvnLhnpcDxbBjIulgI8CpNGM=
+github.com/dghubble/trie v0.1.0/go.mod h1:sOmnzfBNH7H92ow2292dDFWNsVQuh/izuD7otCYb1ak=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
-github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o=
-github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE=
 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
 github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
-github.com/getsentry/sentry-go v0.36.2 h1:uhuxRPTrUy0dnSzTd0LrYXlBYygLkKY0hhlG5LXarzM=
-github.com/getsentry/sentry-go v0.36.2/go.mod h1:p5Im24mJBeruET8Q4bbcMfCQ+F+Iadc4L48tB1apo2c=
-github.com/getsentry/sentry-go/slog v0.36.2 h1:PM27JHFE3lsE8fgI/cOueEOtjiktnC3Za2o5oL9PbJQ=
-github.com/getsentry/sentry-go/slog v0.36.2/go.mod h1:aVFAxnpA3FEtZeSBhBFAnWOlqhiLjaaoOZ0bmBN9IHo=
+github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
+github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
+github.com/getsentry/sentry-go v0.43.0 h1:XbXLpFicpo8HmBDaInk7dum18G9KSLcjZiyUKS+hLW4=
+github.com/getsentry/sentry-go v0.43.0/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
+github.com/getsentry/sentry-go/slog v0.43.0 h1:BYGiM4VFu4//S0vrTSf52MmZSmjhOikHIkBeZZw9P4Q=
+github.com/getsentry/sentry-go/slog v0.43.0/go.mod h1:EAq/2dhW43dV7fwy4OjTWSsvhZjTM9jjsck0kYt9MYE=
 github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
 github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
 github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-git/gcfg/v2 v2.0.2 h1:MY5SIIfTGGEMhdA7d7JePuVVxtKL7Hp+ApGDJAJ7dpo=
 github.com/go-git/gcfg/v2 v2.0.2/go.mod h1:/lv2NsxvhepuMrldsFilrgct6pxzpGdSRC13ydTLSLs=
-github.com/go-git/go-billy/v6 v6.0.0-20251026101908-623011986e70 h1:TWpNrg9JPxp0q+KG0hoFGBulPIP/kMK1b0mDqjdEB/s=
-github.com/go-git/go-billy/v6 v6.0.0-20251026101908-623011986e70/go.mod h1:TpCYxdQ0tWZkrnAkd7yqK+z1C8RKcyjcaYAJNAcnUnM=
-github.com/go-git/go-git-fixtures/v5 v5.1.1 h1:OH8i1ojV9bWfr0ZfasfpgtUXQHQyVS8HXik/V1C099w=
-github.com/go-git/go-git-fixtures/v5 v5.1.1/go.mod h1:Altk43lx3b1ks+dVoAG2300o5WWUnktvfY3VI6bcaXU=
-github.com/go-git/go-git/v6 v6.0.0-20251029213217-0bbfc0875edd h1:pn6+tR4O8McyqEr2MbQwqcySovpG8jDd11F/jQ6aAfA=
-github.com/go-git/go-git/v6 v6.0.0-20251029213217-0bbfc0875edd/go.mod h1:z9pQiXCfyOZIs/8qa5zmozzbcsDPtGN91UD7+qeX3hk=
+github.com/go-git/go-billy/v6 v6.0.0-20260226131633-45bd0956d66f h1:Uvbx7nITO3Sd1GdXarX0TbyYmOaSNIJP0mm4LocEyyA=
+github.com/go-git/go-billy/v6 v6.0.0-20260226131633-45bd0956d66f/go.mod h1:ZW9JC5gionMP1kv5uiaOaV23q0FFmNrVOV8VW+y/acc=
+github.com/go-git/go-git-fixtures/v5 v5.1.2-0.20260122163445-0622d7459a67 h1:3hutPZF+/FBjR/9MdsLJ7e1mlt9pwHgwxMW7CrbmWII=
+github.com/go-git/go-git-fixtures/v5 v5.1.2-0.20260122163445-0622d7459a67/go.mod h1:xKt0pNHST9tYHvbiLxSY27CQWFwgIxBJuDrOE0JvbZw=
+github.com/go-git/go-git/v6 v6.0.0-20260305211659-2083cf940afa h1:fIbZ264qSeJ+GRz+5nq6SFonkCanp/6CRXhYutq8GlE=
+github.com/go-git/go-git/v6 v6.0.0-20260305211659-2083cf940afa/go.mod h1:V/qoTD4qCYizR+fKFA9++d2APoE8Yheci7dXALaSeuI=
 github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
 github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
-github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
-github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ=
 github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/kevinburke/ssh_config v1.4.0 h1:6xxtP5bZ2E4NF5tuQulISpTO2z8XbtH8cg1PWkxoFkQ=
-github.com/kevinburke/ssh_config v1.4.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M=
-github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
-github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
+github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
+github.com/kankanreno/go-snowflake v1.2.0 h1:Zx2SctsH5pivIj9vyhwyDyQS23jcDJx4iT49Bjv81kk=
+github.com/kankanreno/go-snowflake v1.2.0/go.mod h1:6CZ+10PeVsFXKZUTYyJzPiRIjn1IXbInaWLCX/LDJ0g=
+github.com/kevinburke/ssh_config v1.5.0 h1:3cPZmE54xb5j3G5xQCjSvokqNwU2uW+3ry1+PRLSPpA=
+github.com/kevinburke/ssh_config v1.5.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M=
+github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
+github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
 github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
+github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
+github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -75,14 +81,21 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/maypok86/otter/v2 v2.2.1 h1:hnGssisMFkdisYcvQ8L019zpYQcdtPse+g0ps2i7cfI=
-github.com/maypok86/otter/v2 v2.2.1/go.mod h1:1NKY9bY+kB5jwCXBJfE59u+zAwOt6C7ni1FTlFFMqVs=
-github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg=
-github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
+github.com/leodido/go-syslog/v4 v4.3.0 h1:bbSpI/41bYK9iSdlYzcwvlxuLOE8yi4VTFmedtnghdA=
+github.com/leodido/go-syslog/v4 v4.3.0/go.mod h1:eJ8rUfDN5OS6dOkCOBYlg2a+hbAg6pJa99QXXgMrd98=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/maypok86/otter/v2 v2.3.0 h1:8H8AVVFUSzJwIegKwv1uF5aGitTY+AIrtktg7OcLs8w=
+github.com/maypok86/otter/v2 v2.3.0/go.mod h1:XgIdlpmL6jYz882/CAx1E4C1ukfgDKSaw4mWq59+7l8=
+github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
+github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
-github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU=
-github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo=
+github.com/minio/minio-go/v7 v7.0.99 h1:2vH/byrwUkIpFQFOilvTfaUpvAX3fEFhEzO+DR3DlCE=
+github.com/minio/minio-go/v7 v7.0.99/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
@@ -109,16 +122,16 @@ github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9Z
 github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
 github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
 github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
-github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
-github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
 github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
-github.com/samber/lo v1.51.0 h1:kysRYLbHy/MB7kQZf5DSN50JHmMsNEdeY24VzJFu7wI=
-github.com/samber/lo v1.51.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0=
-github.com/samber/slog-common v0.19.0 h1:fNcZb8B2uOLooeYwFpAlKjkQTUafdjfqKcwcC89G9YI=
-github.com/samber/slog-common v0.19.0/go.mod h1:dTz+YOU76aH007YUU0DffsXNsGFQRQllPQh9XyNoA3M=
-github.com/samber/slog-multi v1.5.0 h1:UDRJdsdb0R5vFQFy3l26rpX3rL3FEPJTJ2yKVjoiT1I=
-github.com/samber/slog-multi v1.5.0/go.mod h1:im2Zi3mH/ivSY5XDj6LFcKToRIWPw1OcjSVSdXt+2d0=
+github.com/samber/lo v1.52.0 h1:Rvi+3BFHES3A8meP33VPAxiBZX/Aws5RxrschYGjomw=
+github.com/samber/lo v1.52.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0=
+github.com/samber/slog-common v0.20.0 h1:WaLnm/aCvBJSk5nR5aXZTFBaV0B47A+AEaEOiZDeUnc=
+github.com/samber/slog-common v0.20.0/go.mod h1:+Ozat1jgnnE59UAlmNX1IF3IByHsODnnwf9jUcBZ+m8=
+github.com/samber/slog-multi v1.7.1 h1:aCLXHRxgU+2v0PVlEOh7phynzM7CRo89ZgFtOwaqVEE=
+github.com/samber/slog-multi v1.7.1/go.mod h1:A4KQC99deqfkCDJcL/cO3kX6McX7FffQAx/8QHink+c=
 github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
 github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -126,8 +139,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
-github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww=
-github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0=
+github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
+github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
 github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
 github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
 github.com/tj/go-redirects v0.0.0-20200911105812-fd1ba1020b37 h1:K11tjwz8zTTSZkz4TUjfLN+y8uJWP38BbyPqZ2yB/Yk=
@@ -140,18 +153,22 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
 go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
-golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
-golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
-golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4=
-golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210=
-golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
-golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q=
-golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss=
-golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
-golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
-google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
-google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
+golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
+golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
+golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
+golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg=
+golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM=
+golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
+golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
+google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

gomod2nix.toml

@@ -0,0 +1,207 @@
+schema = 3
+
+[mod]
+  [mod."codeberg.org/git-pages/go-headers"]
+    version = "v1.1.1"
+    hash = "sha256-qgL7l1FHXxcBWhBnBLEI0yENd6P+frvwlKxEAXLA3VY="
+  [mod."codeberg.org/git-pages/go-slog-syslog"]
+    version = "v0.0.0-20251207093707-892f654e80b7"
+    hash = "sha256-ye+DBIyxqTEOViYRrQPWyGJCaLmyKSDwH5btlqDPizM="
+  [mod."github.com/KimMachineGun/automemlimit"]
+    version = "v0.7.5"
+    hash = "sha256-lH/ip9j2hbYUc2W/XIYve/5TScQPZtEZe3hu76CY//k="
+  [mod."github.com/Microsoft/go-winio"]
+    version = "v0.6.2"
+    hash = "sha256-tVNWDUMILZbJvarcl/E7tpSnkn7urqgSHa2Eaka5vSU="
+  [mod."github.com/ProtonMail/go-crypto"]
+    version = "v1.3.0"
+    hash = "sha256-TUG+C4MyeWglOmiwiW2/NUVurFHXLgEPRd3X9uQ1NGI="
+  [mod."github.com/beorn7/perks"]
+    version = "v1.0.1"
+    hash = "sha256-h75GUqfwJKngCJQVE5Ao5wnO3cfKD9lSIteoLp/3xJ4="
+  [mod."github.com/c2h5oh/datasize"]
+    version = "v0.0.0-20231215233829-aa82cc1e6500"
+    hash = "sha256-8MqL7xCvE6fIjanz2jwkaLP1OE5kLu62TOcQx452DHQ="
+  [mod."github.com/cespare/xxhash/v2"]
+    version = "v2.3.0"
+    hash = "sha256-7hRlwSR+fos1kx4VZmJ/7snR7zHh8ZFKX+qqqqGcQpY="
+  [mod."github.com/cloudflare/circl"]
+    version = "v1.6.3"
+    hash = "sha256-XZm4EastgX67Dgm5BpOEW/PY4aLcHM/O8+Xbz26PuTY="
+  [mod."github.com/creasty/defaults"]
+    version = "v1.8.0"
+    hash = "sha256-I1LE1cfOhMS5JxB7+fWTKieefw2Gge1UhIZh+A6pa6s="
+  [mod."github.com/cyphar/filepath-securejoin"]
+    version = "v0.6.1"
+    hash = "sha256-obqip8c1c9mjXFznyXF8aDnpcMw7ttzv+e28anCa/v0="
+  [mod."github.com/davecgh/go-spew"]
+    version = "v1.1.1"
+    hash = "sha256-nhzSUrE1fCkN0+RL04N4h8jWmRFPPPWbCuDc7Ss0akI="
+  [mod."github.com/dghubble/trie"]
+    version = "v0.1.0"
+    hash = "sha256-hVh7uYylpMCCSPcxl70hJTmzSwaA1MxBmJFBO5Xdncc="
+  [mod."github.com/dustin/go-humanize"]
+    version = "v1.0.1"
+    hash = "sha256-yuvxYYngpfVkUg9yAmG99IUVmADTQA0tMbBXe0Fq0Mc="
+  [mod."github.com/emirpasic/gods"]
+    version = "v1.18.1"
+    hash = "sha256-hGDKddjLj+5dn2woHtXKUdd49/3xdsqnhx7VEdCu1m4="
+  [mod."github.com/fatih/color"]
+    version = "v1.18.0"
+    hash = "sha256-pP5y72FSbi4j/BjyVq/XbAOFjzNjMxZt2R/lFFxGWvY="
+  [mod."github.com/getsentry/sentry-go"]
+    version = "v0.43.0"
+    hash = "sha256-Wu1inIhjuAw6wKburwqIlNxC0I4akunHGh/8DOqo3xg="
+  [mod."github.com/getsentry/sentry-go/slog"]
+    version = "v0.43.0"
+    hash = "sha256-FJMx2E8anKtHknn867gCkYPjitZb9Okqp2uZ+dV7JqA="
+  [mod."github.com/go-git/gcfg/v2"]
+    version = "v2.0.2"
+    hash = "sha256-icqMDeC/tEg/3979EuEN67Ml5KjdDA0R3QvR6iLLrSI="
+  [mod."github.com/go-git/go-billy/v6"]
+    version = "v6.0.0-20260226131633-45bd0956d66f"
+    hash = "sha256-s+dthtn+JewJ58R5VbvWaEoYLozDt5YpkHyXcN0xMvQ="
+  [mod."github.com/go-git/go-git/v6"]
+    version = "v6.0.0-20260305211659-2083cf940afa"
+    hash = "sha256-aUUgVODQVanWVQ44tcrOKIvtzJPlZKFNPPvEdAxdPxw="
+  [mod."github.com/go-ini/ini"]
+    version = "v1.67.0"
+    hash = "sha256-V10ahGNGT+NLRdKUyRg1dos5RxLBXBk1xutcnquc/+4="
+  [mod."github.com/golang/groupcache"]
+    version = "v0.0.0-20241129210726-2c02b8208cf8"
+    hash = "sha256-AdLZ3dJLe/yduoNvZiXugZxNfmwJjNQyQGsIdzYzH74="
+  [mod."github.com/google/uuid"]
+    version = "v1.6.0"
+    hash = "sha256-VWl9sqUzdOuhW0KzQlv0gwwUQClYkmZwSydHG2sALYw="
+  [mod."github.com/jpillora/backoff"]
+    version = "v1.0.0"
+    hash = "sha256-uxHg68NN8hrwPCrPfLYYprZHf7dMyEoPoF46JFx0IHU="
+  [mod."github.com/kankanreno/go-snowflake"]
+    version = "v1.2.0"
+    hash = "sha256-713xGEqjwaUGIu2EHII5sldWmcquFpxZmte/7R/O6LA="
+  [mod."github.com/kevinburke/ssh_config"]
+    version = "v1.5.0"
+    hash = "sha256-4SijlenzNuWb5CavWrky8qoQj+6fKCJgOiQANzN5TUE="
+  [mod."github.com/klauspost/compress"]
+    version = "v1.18.4"
+    hash = "sha256-swwNE6xKz4ZAOUHPWFlHYiqFeZLRZuuKYhLQ34aYnAU="
+  [mod."github.com/klauspost/cpuid/v2"]
+    version = "v2.3.0"
+    hash = "sha256-50JhbQyT67BK38HIdJihPtjV7orYp96HknI2VP7A9Yc="
+  [mod."github.com/klauspost/crc32"]
+    version = "v1.3.0"
+    hash = "sha256-RsS/MDJbVzVB+i74whqABgwZJWMw+AutF6HhJBVgbag="
+  [mod."github.com/leodido/go-syslog/v4"]
+    version = "v4.3.0"
+    hash = "sha256-fCJ2rgrrPR/Ey/PoAsJhd8Sl8mblAnnMAmBuoWFBTgg="
+  [mod."github.com/mattn/go-colorable"]
+    version = "v0.1.13"
+    hash = "sha256-qb3Qbo0CELGRIzvw7NVM1g/aayaz4Tguppk9MD2/OI8="
+  [mod."github.com/mattn/go-isatty"]
+    version = "v0.0.20"
+    hash = "sha256-qhw9hWtU5wnyFyuMbKx+7RB8ckQaFQ8D+8GKPkN3HHQ="
+  [mod."github.com/maypok86/otter/v2"]
+    version = "v2.3.0"
+    hash = "sha256-ELzmi/s2WqDeUmzSGnfx+ys2Hs28XHqF7vlEzyRotIA="
+  [mod."github.com/minio/crc64nvme"]
+    version = "v1.1.1"
+    hash = "sha256-RVVi/gWPBEQqcW4n+KIKxlA3uY5+77e2rhkVk8fFNUo="
+  [mod."github.com/minio/md5-simd"]
+    version = "v1.1.2"
+    hash = "sha256-vykcXvy2VBBAXnJott/XsGTT0gk2UL36JzZKfJ1KAUY="
+  [mod."github.com/minio/minio-go/v7"]
+    version = "v7.0.99"
+    hash = "sha256-Q2VISIvHDggBzidGWzgHbVUZrDCsSIGBPcWfMJcC39w="
+  [mod."github.com/munnerz/goautoneg"]
+    version = "v0.0.0-20191010083416-a7dc8b61c822"
+    hash = "sha256-79URDDFenmGc9JZu+5AXHToMrtTREHb3BC84b/gym9Q="
+  [mod."github.com/pbnjay/memory"]
+    version = "v0.0.0-20210728143218-7b4eea64cf58"
+    hash = "sha256-QI+F1oPLOOtwNp8+m45OOoSfYFs3QVjGzE0rFdpF/IA="
+  [mod."github.com/pelletier/go-toml/v2"]
+    version = "v2.2.4"
+    hash = "sha256-8qQIPldbsS5RO8v/FW/se3ZsAyvLzexiivzJCbGRg2Q="
+  [mod."github.com/philhofer/fwd"]
+    version = "v1.2.0"
+    hash = "sha256-cGx2/0QQay46MYGZuamFmU0TzNaFyaO+J7Ddzlr/3dI="
+  [mod."github.com/pjbgf/sha1cd"]
+    version = "v0.5.0"
+    hash = "sha256-11XBkhdciQGsQ7jEMZ6PgphRKjruTSc7ZxfOwDuPCr8="
+  [mod."github.com/pkg/errors"]
+    version = "v0.9.1"
+    hash = "sha256-mNfQtcrQmu3sNg/7IwiieKWOgFQOVVe2yXgKBpe/wZw="
+  [mod."github.com/pmezard/go-difflib"]
+    version = "v1.0.0"
+    hash = "sha256-/FtmHnaGjdvEIKAJtrUfEhV7EVo5A/eYrtdnUkuxLDA="
+  [mod."github.com/pquerna/cachecontrol"]
+    version = "v0.2.0"
+    hash = "sha256-tuTERCFfwmqPepw/rs5cyv9fArCD30BqgjZqwMV+vzQ="
+  [mod."github.com/prometheus/client_golang"]
+    version = "v1.23.2"
+    hash = "sha256-3GD4fBFa1tJu8MS4TNP6r2re2eViUE+kWUaieIOQXCg="
+  [mod."github.com/prometheus/client_model"]
+    version = "v0.6.2"
+    hash = "sha256-q6Fh6v8iNJN9ypD47LjWmx66YITa3FyRjZMRsuRTFeQ="
+  [mod."github.com/prometheus/common"]
+    version = "v0.66.1"
+    hash = "sha256-bqHPaV9IV70itx63wqwgy2PtxMN0sn5ThVxDmiD7+Tk="
+  [mod."github.com/prometheus/procfs"]
+    version = "v0.16.1"
+    hash = "sha256-OBCvKlLW2obct35p0L9Q+1ZrxZjpTmbgHMP2rng9hpo="
+  [mod."github.com/rs/xid"]
+    version = "v1.6.0"
+    hash = "sha256-rJB7h3KuH1DPp5n4dY3MiGnV1Y96A10lf5OUl+MLkzU="
+  [mod."github.com/samber/lo"]
+    version = "v1.52.0"
+    hash = "sha256-xgMsPJv3rydHH10NZU8wz/DhK2VbbR8ymivOg1ChTp0="
+  [mod."github.com/samber/slog-common"]
+    version = "v0.20.0"
+    hash = "sha256-aWcvt9XNyKaolLhvthcXeFDl0t6uo7Vdo8WzCducf1E="
+  [mod."github.com/samber/slog-multi"]
+    version = "v1.7.1"
+    hash = "sha256-wHXt2lwFfjm1p7jnZi44SlHtjdk531BGz2O9pfiylxo="
+  [mod."github.com/sergi/go-diff"]
+    version = "v1.4.0"
+    hash = "sha256-rs9NKpv/qcQEMRg7CmxGdP4HGuFdBxlpWf9LbA9wS4k="
+  [mod."github.com/stretchr/testify"]
+    version = "v1.11.1"
+    hash = "sha256-sWfjkuKJyDllDEtnM8sb/pdLzPQmUYWYtmeWz/5suUc="
+  [mod."github.com/tinylib/msgp"]
+    version = "v1.6.1"
+    hash = "sha256-R2LutHQFZ7HAqeyzHqzMeyAJHxcYc+n1x7ysyrXefmQ="
+  [mod."github.com/tj/assert"]
+    version = "v0.0.3"
+    hash = "sha256-4xhmZcHpUafabaXejE9ucVnGxG/txomvKzBg6cbkusg="
+  [mod."github.com/tj/go-redirects"]
+    version = "v0.0.0-20200911105812-fd1ba1020b37"
+    hash = "sha256-GpYpxdT4F7PkwGXLo7cYVcIRJrzd1sKHtFDH+bRb6Tk="
+  [mod."github.com/valyala/bytebufferpool"]
+    version = "v1.0.0"
+    hash = "sha256-I9FPZ3kCNRB+o0dpMwBnwZ35Fj9+ThvITn8a3Jr8mAY="
+  [mod."github.com/valyala/fasttemplate"]
+    version = "v1.2.2"
+    hash = "sha256-gp+lNXE8zjO+qJDM/YbS6V43HFsYP6PKn4ux1qa5lZ0="
+  [mod."go.yaml.in/yaml/v2"]
+    version = "v2.4.2"
+    hash = "sha256-oC8RWdf1zbMYCtmR0ATy/kCkhIwPR9UqFZSMOKLVF/A="
+  [mod."go.yaml.in/yaml/v3"]
+    version = "v3.0.4"
+    hash = "sha256-NkGFiDPoCxbr3LFsI6OCygjjkY0rdmg5ggvVVwpyDQ4="
+  [mod."golang.org/x/crypto"]
+    version = "v0.48.0"
+    hash = "sha256-uBIGGSGmWWklRxX6XTOqUECzz165UFY9Y99Ka3pLKAw="
+  [mod."golang.org/x/net"]
+    version = "v0.51.0"
+    hash = "sha256-bLDpVRTPWM7IowHw1jdr9EPCRQNAVFsPwz69olySah4="
+  [mod."golang.org/x/sys"]
+    version = "v0.41.0"
+    hash = "sha256-owjs3/IzAKfFlIz1U1fiHSfl2+bTUhaXTyWEjL5SWHk="
+  [mod."golang.org/x/text"]
+    version = "v0.34.0"
+    hash = "sha256-wGKd1JkeiFROibvo2kkAuQ7JajSIfV4utGaoGbTQhQM="
+  [mod."google.golang.org/protobuf"]
+    version = "v1.36.11"
+    hash = "sha256-7W+6jntfI/awWL3JP6yQedxqP5S9o3XvPgJ2XxxsIeE="
+  [mod."gopkg.in/yaml.v3"]
+    version = "v3.0.1"
+    hash = "sha256-FqL9TKYJ0XkNwJFnq9j0VvJ5ZUU1RvH/52h/f5bkYAU="

renovate.json

@@ -14,5 +14,7 @@
   "lockFileMaintenance": {
     "enabled": true,
     "automerge": false
-  }
+  },
+  "semanticCommits": "disabled",
+  "commitMessagePrefix": "[Renovate]"
 }

src/audit.go

@@ -0,0 +1,399 @@
+package git_pages
+
+import (
+	"cmp"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	exponential "github.com/jpillora/backoff"
+	"github.com/kankanreno/go-snowflake"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/proto"
+	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
+)
+
+var (
+	auditNotifyOkCount = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "git_pages_audit_notify_ok",
+		Help: "Count of successful audit notifications",
+	})
+	auditNotifyErrorCount = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "git_pages_audit_notify_error",
+		Help: "Count of failed audit notifications",
+	})
+)
+
+type principalKey struct{}
+
+var PrincipalKey = principalKey{}
+
+func WithPrincipal(ctx context.Context) context.Context {
+	principal := &Principal{}
+	return context.WithValue(ctx, PrincipalKey, principal)
+}
+
+func GetPrincipal(ctx context.Context) *Principal {
+	if principal, ok := ctx.Value(PrincipalKey).(*Principal); ok {
+		return principal
+	}
+	return nil
+}
+
+type AuditID int64
+
+func GenerateAuditID() AuditID {
+	inner, err := snowflake.NextID()
+	if err != nil {
+		panic(err)
+	}
+	return AuditID(inner)
+}
+
+func ParseAuditID(repr string) (AuditID, error) {
+	inner, err := strconv.ParseInt(repr, 16, 64)
+	if err != nil {
+		return AuditID(0), err
+	}
+	return AuditID(inner), nil
+}
+
+func (id AuditID) String() string {
+	return fmt.Sprintf("%016x", int64(id))
+}
+
+func (id AuditID) CompareTime(when time.Time) int {
+	idMillis := int64(id) >> (snowflake.MachineIDLength + snowflake.SequenceLength)
+	whenMillis := when.UTC().UnixNano() / 1e6
+	return cmp.Compare(idMillis, whenMillis)
+}
+
+func EncodeAuditRecord(record *AuditRecord) (data []byte) {
+	data, err := proto.MarshalOptions{Deterministic: true}.Marshal(record)
+	if err != nil {
+		panic(err)
+	}
+	return
+}
+
+func DecodeAuditRecord(data []byte) (record *AuditRecord, err error) {
+	record = &AuditRecord{}
+	err = proto.Unmarshal(data, record)
+	return
+}
+
+func (record *AuditRecord) GetAuditID() AuditID {
+	return AuditID(record.GetId())
+}
+
+func (record *AuditRecord) DescribePrincipal() string {
+	var items []string
+	if record.Principal != nil {
+		if record.Principal.GetIpAddress() != "" {
+			items = append(items, record.Principal.GetIpAddress())
+		}
+		if record.Principal.GetForgeUser() != nil {
+			items = append(items, fmt.Sprintf("%s/%s(%d)",
+				record.Principal.GetForgeUser().GetOrigin(),
+				record.Principal.GetForgeUser().GetHandle(),
+				record.Principal.GetForgeUser().GetId()))
+		}
+		if record.Principal.GetCliAdmin() {
+			items = append(items, "<cli-admin>")
+		}
+	}
+	if len(items) > 0 {
+		return strings.Join(items, ",")
+	} else {
+		return "<unknown>"
+	}
+}
+
+func (record *AuditRecord) DescribeResource() string {
+	desc := "<unknown>"
+	if record.Domain != nil && record.Project != nil {
+		desc = path.Join(*record.Domain, *record.Project)
+	} else if record.Domain != nil {
+		desc = *record.Domain
+	}
+	return desc
+}
+
+type AuditRecordScope int
+
+const (
+	AuditRecordComplete AuditRecordScope = iota
+	AuditRecordNoManifest
+)
+
+func AuditRecordJSON(record *AuditRecord, scope AuditRecordScope) []byte {
+	switch scope {
+	case AuditRecordComplete:
+		// as-is
+	case AuditRecordNoManifest:
+		// trim the manifest
+		newRecord := &AuditRecord{}
+		proto.Merge(newRecord, record)
+		newRecord.Manifest = nil
+		record = newRecord
+	}
+
+	json, err := protojson.MarshalOptions{
+		Multiline:         true,
+		EmitDefaultValues: true,
+	}.Marshal(record)
+	if err != nil {
+		panic(err)
+	}
+	return json
+}
+
+// This function receives `id` and `record` separately because the record itself may have its
+// ID missing or mismatched. While this is very unlikely, using the actual primary key as
+// the filename is more robust.
+func ExtractAuditRecord(ctx context.Context, id AuditID, record *AuditRecord, dest string) error {
+	const mode = 0o400 // readable by current user, not writable
+
+	err := os.WriteFile(filepath.Join(dest, fmt.Sprintf("%s-event.json", id)),
+		AuditRecordJSON(record, AuditRecordNoManifest), mode)
+	if err != nil {
+		return err
+	}
+
+	if record.Manifest != nil {
+		err = os.WriteFile(filepath.Join(dest, fmt.Sprintf("%s-manifest.json", id)),
+			ManifestJSON(record.Manifest), mode)
+		if err != nil {
+			return err
+		}
+
+		archive, err := os.OpenFile(filepath.Join(dest, fmt.Sprintf("%s-archive.tar", id)),
+			os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode)
+		if err != nil {
+			return err
+		}
+		defer archive.Close()
+
+		err = CollectTar(ctx, archive, record.Manifest, ManifestMetadata{})
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func AuditEventProcessor(command string, args []string) (http.Handler, error) {
+	var err error
+
+	// Resolve the command to an absolute path, as it will be run from a different current
+	// directory, which would break e.g. `git-pages -audit-server tcp/:3004 ./handler.sh`.
+	if command, err = exec.LookPath(command); err != nil {
+		return nil, err
+	}
+	if command, err = filepath.Abs(command); err != nil {
+		return nil, err
+	}
+
+	router := http.NewServeMux()
+	router.Handle("GET /", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Go will cancel the request context if the client drops the connection. We don't want
+		// that to interrupt processing. However, we also want the client (not the server) to
+		// handle retries, so instead of spawning a goroutine to process the event, we do this
+		// within the HTTP handler. If an error is returned, the notify goroutine in the worker
+		// will retry the HTTP request (with backoff) until it succeeds.
+		//
+		// This is a somewhat idiosyncratic design and it's not clear that this is the best
+		// possible approach (e.g. if the worker gets restarted and the event processing fails,
+		// it will not be retried), but it should do the job for now. It is expected that
+		// some form of observability is used to highlight event processor errors.
+		ctx := context.WithoutCancel(r.Context())
+
+		id, err := ParseAuditID(r.URL.RawQuery)
+		if err != nil {
+			logc.Printf(ctx, "audit process err: malformed query\n")
+			http.Error(w, "malformed query", http.StatusBadRequest)
+			return
+		} else {
+			logc.Printf(ctx, "audit process %s", id)
+		}
+
+		record, err := backend.QueryAuditLog(ctx, id)
+		if err != nil {
+			logc.Printf(ctx, "audit process err: missing record\n")
+			http.Error(w, "missing record", http.StatusNotFound)
+			return
+		}
+
+		args := append(args, id.String(), record.GetEvent().String())
+		cmd := exec.CommandContext(ctx, command, args...)
+		if cmd.Dir, err = os.MkdirTemp("", "auditRecord"); err != nil {
+			panic(fmt.Errorf("mkdtemp: %w", err))
+		}
+		defer os.RemoveAll(cmd.Dir)
+
+		if err = ExtractAuditRecord(ctx, id, record, cmd.Dir); err != nil {
+			logc.Printf(ctx, "audit process %s err: %s\n", id, err)
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		output, err := cmd.CombinedOutput()
+		if err != nil {
+			logc.Printf(ctx, "audit process %s err: %s; %s\n", id, err, string(output))
+			w.WriteHeader(http.StatusServiceUnavailable)
+			if len(output) == 0 {
+				fmt.Fprintln(w, err.Error())
+			}
+		} else {
+			logc.Printf(ctx, "audit process %s ok: %s\n", id, string(output))
+			w.WriteHeader(http.StatusOK)
+		}
+		w.Write(output)
+	}))
+	return router, nil
+}
+
+type auditedBackend struct {
+	Backend
+}
+
+var _ Backend = (*auditedBackend)(nil)
+
+func NewAuditedBackend(backend Backend) Backend {
+	return &auditedBackend{backend}
+}
+
+// This function does not retry appending audit records; as such, if it returns an error,
+// this error must interrupt whatever operation it was auditing. A corollary is that it is
+// possible that appending an audit record succeeds but the audited operation fails.
+// This is considered fine since the purpose of auditing is to record end user intent, not
+// to be a 100% accurate reflection of performed actions. When in doubt, the audit records
+// should be examined together with the application logs.
+func (audited *auditedBackend) appendNewAuditRecord(ctx context.Context, record *AuditRecord) (err error) {
+	if config.Audit.Collect {
+		id := GenerateAuditID()
+		record.Id = proto.Int64(int64(id))
+		record.Timestamp = timestamppb.Now()
+		record.Principal = GetPrincipal(ctx)
+
+		err = audited.Backend.AppendAuditLog(ctx, id, record)
+		if err != nil {
+			err = fmt.Errorf("audit: %w", err)
+		} else {
+			var subject string
+			if record.Project == nil {
+				subject = *record.Domain
+			} else {
+				subject = path.Join(*record.Domain, *record.Project)
+			}
+			logc.Printf(ctx, "audit %s ok: %s %s\n", subject, id, record.Event.String())
+
+			// Send a notification to the audit server, if configured, and try to make sure
+			// it is delivered by retrying with exponential backoff on errors.
+			notifyAudit(context.WithoutCancel(ctx), id)
+		}
+	}
+	return
+}
+
+func notifyAudit(ctx context.Context, id AuditID) {
+	if config.Audit.NotifyURL != nil {
+		notifyURL := config.Audit.NotifyURL.URL
+		notifyURL.RawQuery = id.String()
+
+		// See also the explanation in `AuditEventProcessor` above.
+		go func() {
+			backoff := exponential.Backoff{
+				Jitter: true,
+				Min:    time.Second * 1,
+				Max:    time.Second * 60,
+			}
+			for {
+				resp, err := http.Get(notifyURL.String())
+				var body []byte
+				if err == nil {
+					defer resp.Body.Close()
+					body, _ = io.ReadAll(resp.Body)
+				}
+				if err == nil && resp.StatusCode == http.StatusOK {
+					logc.Printf(ctx, "audit notify %s ok: %s\n", id, string(body))
+					auditNotifyOkCount.Inc()
+					break
+				} else {
+					sleepFor := backoff.Duration()
+					if err != nil {
+						logc.Printf(ctx, "audit notify %s err: %s (retry in %s)",
+							id, err, sleepFor)
+					} else {
+						logc.Printf(ctx, "audit notify %s fail: %s (retry in %s); %s",
+							id, resp.Status, sleepFor, string(body))
+					}
+					auditNotifyErrorCount.Inc()
+					time.Sleep(sleepFor)
+				}
+			}
+		}()
+	}
+}
+
+func (audited *auditedBackend) CommitManifest(
+	ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions,
+) (err error) {
+	domain, project, ok := strings.Cut(name, "/")
+	if !ok {
+		panic("malformed manifest name")
+	}
+	audited.appendNewAuditRecord(ctx, &AuditRecord{
+		Event:    AuditEvent_CommitManifest.Enum(),
+		Domain:   proto.String(domain),
+		Project:  proto.String(project),
+		Manifest: manifest,
+	})
+
+	return audited.Backend.CommitManifest(ctx, name, manifest, opts)
+}
+
+func (audited *auditedBackend) DeleteManifest(
+	ctx context.Context, name string, opts ModifyManifestOptions,
+) (err error) {
+	domain, project, ok := strings.Cut(name, "/")
+	if !ok {
+		panic("malformed manifest name")
+	}
+	audited.appendNewAuditRecord(ctx, &AuditRecord{
+		Event:   AuditEvent_DeleteManifest.Enum(),
+		Domain:  proto.String(domain),
+		Project: proto.String(project),
+	})
+
+	return audited.Backend.DeleteManifest(ctx, name, opts)
+}
+
+func (audited *auditedBackend) FreezeDomain(ctx context.Context, domain string) (err error) {
+	audited.appendNewAuditRecord(ctx, &AuditRecord{
+		Event:  AuditEvent_FreezeDomain.Enum(),
+		Domain: proto.String(domain),
+	})
+
+	return audited.Backend.FreezeDomain(ctx, domain)
+}
+
+func (audited *auditedBackend) UnfreezeDomain(ctx context.Context, domain string) (err error) {
+	audited.appendNewAuditRecord(ctx, &AuditRecord{
+		Event:  AuditEvent_UnfreezeDomain.Enum(),
+		Domain: proto.String(domain),
+	})
+
+	return audited.Backend.UnfreezeDomain(ctx, domain)
+}

src/auth.go

@@ -6,13 +6,14 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"log"
 	"net"
 	"net/http"
 	"net/url"
 	"slices"
 	"strings"
 	"time"
+
+	"golang.org/x/net/idna"
 )
 
 type AuthError struct {
@@ -32,9 +33,9 @@ func IsUnauthorized(err error) bool {
 	return false
 }
 
-func authorizeInsecure() *Authorization {
+func authorizeInsecure(r *http.Request) *Authorization {
 	if config.Insecure { // for testing only
-		log.Println("auth: INSECURE mode")
+		logc.Println(r.Context(), "auth: INSECURE mode")
 		return &Authorization{
 			repoURLs: nil,
 			branch:   "pages",
@@ -43,25 +44,48 @@ func authorizeInsecure() *Authorization {
 	return nil
 }
 
+var idnaProfile = idna.New(idna.MapForLookup(), idna.BidiRule())
+
 func GetHost(r *http.Request) (string, error) {
-	// FIXME: handle IDNA
 	host, _, err := net.SplitHostPort(r.Host)
 	if err != nil {
-		// dirty but the go stdlib doesn't have a "split port if present" function
 		host = r.Host
 	}
-	if strings.HasPrefix(host, ".") {
+	// this also rejects invalid characters and labels
+	host, err = idnaProfile.ToASCII(host)
+	if err != nil {
+		if config.Feature("relaxed-idna") {
+			// unfortunately, the go IDNA library has some significant issues around its
+			// Unicode TR46 implementation: https://github.com/golang/go/issues/76804
+			// we would like to allow *just* the _ here, but adding `idna.StrictDomainName(false)`
+			// would also accept domains like `*.foo.bar` which should clearly be disallowed.
+			// as a workaround, accept a domain name if it is valid with all `_` characters
+			// replaced with an alphanumeric character (we use `a`); this allows e.g. `foo_bar.xxx`
+			// and `foo__bar.xxx`, as well as `_foo.xxx` and `foo_.xxx`. labels starting with
+			// an underscore are explicitly rejected below.
+			_, err = idnaProfile.ToASCII(strings.ReplaceAll(host, "_", "a"))
+		}
+		if err != nil {
+			return "", AuthError{http.StatusBadRequest,
+				fmt.Sprintf("malformed host name %q", host)}
+		}
+	}
+	if strings.HasPrefix(host, ".") || strings.HasPrefix(host, "_") {
 		return "", AuthError{http.StatusBadRequest,
-			fmt.Sprintf("host name %q is reserved", host)}
+			fmt.Sprintf("reserved host name %q", host)}
 	}
 	host = strings.TrimSuffix(host, ".")
 	return host, nil
 }
 
+func IsValidProjectName(name string) bool {
+	return !strings.HasPrefix(name, ".") && !strings.Contains(name, "%")
+}
+
 func GetProjectName(r *http.Request) (string, error) {
 	// path must be either `/` or `/foo/` (`/foo` is accepted as an alias)
 	path := strings.TrimPrefix(strings.TrimSuffix(r.URL.Path, "/"), "/")
-	if path == ".index" || strings.HasPrefix(path, ".index/") {
+	if !IsValidProjectName(path) {
 		return "", AuthError{http.StatusBadRequest,
 			fmt.Sprintf("directory name %q is reserved", ".index")}
 	} else if strings.Contains(path, "/") {
@@ -82,6 +106,8 @@ type Authorization struct {
 	repoURLs []string
 	// Only the exact branch is allowed.
 	branch string
+	// The authorized forge user.
+	forgeUser *ForgeUser
 }
 
 func authorizeDNSChallenge(r *http.Request) (*Authorization, error) {
@@ -159,6 +185,11 @@ func authorizeDNSAllowlist(r *http.Request) (*Authorization, error) {
 		return nil, err
 	}
 
+	projectName, err := GetProjectName(r)
+	if err != nil {
+		return nil, err
+	}
+
 	allowlistHostname := fmt.Sprintf("_git-pages-repository.%s", host)
 	records, err := net.LookupTXT(allowlistHostname)
 	if err != nil {
@@ -166,6 +197,11 @@ func authorizeDNSAllowlist(r *http.Request) (*Authorization, error) {
 			fmt.Sprintf("failed to look up DNS repository allowlist: %s TXT", allowlistHostname)}
 	}
 
+	if projectName != ".index" {
+		return nil, AuthError{http.StatusUnauthorized,
+			"DNS repository allowlist only authorizes index site"}
+	}
+
 	var (
 		repoURLs []string
 		errs     []error
@@ -231,8 +267,8 @@ func authorizeWildcardMatchSite(r *http.Request, pattern *WildcardPattern) (*Aut
 	}
 
 	if userName, found := pattern.Matches(host); found {
-		repoURLs, branch := pattern.ApplyTemplate(userName, projectName)
-		return &Authorization{repoURLs, branch}, nil
+		repoURL, branch := pattern.ApplyTemplate(userName, projectName)
+		return &Authorization{repoURLs: []string{repoURL}, branch: branch}, nil
 	} else {
 		return nil, AuthError{
 			http.StatusUnauthorized,
@@ -266,7 +302,7 @@ func authorizeCodebergPagesV2(r *http.Request) (*Authorization, error) {
 	}
 
 	if len(dnsRecords) > 0 {
-		log.Printf("auth: %s TXT/CNAME: %q\n", host, dnsRecords)
+		logc.Printf(r.Context(), "auth: %s TXT/CNAME: %q\n", host, dnsRecords)
 	}
 
 	for _, dnsRecord := range dnsRecords {
@@ -314,7 +350,7 @@ func authorizeCodebergPagesV2(r *http.Request) (*Authorization, error) {
 func AuthorizeMetadataRetrieval(r *http.Request) (*Authorization, error) {
 	causes := []error{AuthError{http.StatusUnauthorized, "unauthorized"}}
 
-	auth := authorizeInsecure()
+	auth := authorizeInsecure(r)
 	if auth != nil {
 		return auth, nil
 	}
@@ -325,7 +361,7 @@ func AuthorizeMetadataRetrieval(r *http.Request) (*Authorization, error) {
 	} else if err != nil { // bad request
 		return nil, err
 	} else {
-		log.Println("auth: DNS challenge")
+		logc.Println(r.Context(), "auth: DNS challenge")
 		return auth, nil
 	}
 
@@ -336,7 +372,7 @@ func AuthorizeMetadataRetrieval(r *http.Request) (*Authorization, error) {
 		} else if err != nil { // bad request
 			return nil, err
 		} else {
-			log.Printf("auth: wildcard %s\n", pattern.GetHost())
+			logc.Printf(r.Context(), "auth: wildcard %s\n", pattern.GetHost())
 			return auth, nil
 		}
 	}
@@ -348,7 +384,7 @@ func AuthorizeMetadataRetrieval(r *http.Request) (*Authorization, error) {
 		} else if err != nil { // bad request
 			return nil, err
 		} else {
-			log.Printf("auth: codeberg %s\n", r.Host)
+			logc.Printf(r.Context(), "auth: codeberg %s\n", r.Host)
 			return auth, nil
 		}
 	}
@@ -356,9 +392,6 @@ func AuthorizeMetadataRetrieval(r *http.Request) (*Authorization, error) {
 	return nil, joinErrors(causes...)
 }
 
-// Returns `repoURLs, err` where if `err == nil` then the request is authorized to clone from
-// any repository URL included in `repoURLs` (by case-insensitive comparison), or any URL at all
-// if `repoURLs == nil`.
 func AuthorizeUpdateFromRepository(r *http.Request) (*Authorization, error) {
 	causes := []error{AuthError{http.StatusUnauthorized, "unauthorized"}}
 
@@ -366,7 +399,7 @@ func AuthorizeUpdateFromRepository(r *http.Request) (*Authorization, error) {
 		return nil, err
 	}
 
-	auth := authorizeInsecure()
+	auth := authorizeInsecure(r)
 	if auth != nil {
 		return auth, nil
 	}
@@ -378,7 +411,7 @@ func AuthorizeUpdateFromRepository(r *http.Request) (*Authorization, error) {
 	} else if err != nil { // bad request
 		return nil, err
 	} else {
-		log.Println("auth: DNS challenge: allow *")
+		logc.Println(r.Context(), "auth: DNS challenge: allow *")
 		return auth, nil
 	}
 
@@ -390,7 +423,7 @@ func AuthorizeUpdateFromRepository(r *http.Request) (*Authorization, error) {
 		} else if err != nil { // bad request
 			return nil, err
 		} else {
-			log.Printf("auth: DNS allowlist: allow %v\n", auth.repoURLs)
+			logc.Printf(r.Context(), "auth: DNS allowlist: allow %v\n", auth.repoURLs)
 			return auth, nil
 		}
 	}
@@ -404,7 +437,7 @@ func AuthorizeUpdateFromRepository(r *http.Request) (*Authorization, error) {
 			} else if err != nil { // bad request
 				return nil, err
 			} else {
-				log.Printf("auth: wildcard %s: allow %v\n", pattern.GetHost(), auth.repoURLs)
+				logc.Printf(r.Context(), "auth: wildcard %s: allow %v\n", pattern.GetHost(), auth.repoURLs)
 				return auth, nil
 			}
 		}
@@ -416,7 +449,7 @@ func AuthorizeUpdateFromRepository(r *http.Request) (*Authorization, error) {
 			} else if err != nil { // bad request
 				return nil, err
 			} else {
-				log.Printf("auth: codeberg %s: allow %v branch %s\n",
+				logc.Printf(r.Context(), "auth: codeberg %s: allow %v branch %s\n",
 					r.Host, auth.repoURLs, auth.branch)
 				return auth, nil
 			}
@@ -427,7 +460,7 @@ func AuthorizeUpdateFromRepository(r *http.Request) (*Authorization, error) {
 }
 
 func checkAllowedURLPrefix(repoURL string) error {
-	if config.Limits.AllowedRepositoryURLPrefixes != nil {
+	if len(config.Limits.AllowedRepositoryURLPrefixes) > 0 {
 		allowedPrefix := false
 		repoURL = strings.ToLower(repoURL)
 		for _, allowedRepoURLPrefix := range config.Limits.AllowedRepositoryURLPrefixes {
@@ -575,6 +608,61 @@ func checkGogsRepositoryPushPermission(baseURL *url.URL, authorization string) e
 	return nil
 }
 
+// Gogs, Gitea, and Forgejo all support the same API here.
+func fetchGogsAuthorizedUser(baseURL *url.URL, authorization string) (*ForgeUser, error) {
+	request, err := http.NewRequest("GET", baseURL.ResolveReference(&url.URL{
+		Path: "/api/v1/user",
+	}).String(), nil)
+	if err != nil {
+		panic(err) // misconfiguration
+	}
+	request.Header.Set("Accept", "application/json")
+	request.Header.Set("Authorization", authorization)
+
+	httpClient := http.Client{Timeout: 5 * time.Second}
+	response, err := httpClient.Do(request)
+	if err != nil {
+		return nil, AuthError{
+			http.StatusServiceUnavailable,
+			fmt.Sprintf("cannot fetch authorized forge user: %s", err),
+		}
+	}
+	defer response.Body.Close()
+
+	if response.StatusCode != http.StatusOK {
+		return nil, AuthError{
+			http.StatusServiceUnavailable,
+			fmt.Sprintf(
+				"cannot fetch authorized forge user: GET %s returned %s",
+				request.URL,
+				response.Status,
+			),
+		}
+	}
+	decoder := json.NewDecoder(response.Body)
+
+	var userInfo struct {
+		ID    int64
+		Login string
+	}
+	if err = decoder.Decode(&userInfo); err != nil {
+		return nil, errors.Join(AuthError{
+			http.StatusServiceUnavailable,
+			fmt.Sprintf(
+				"cannot fetch authorized forge user: GET %s returned malformed JSON",
+				request.URL,
+			),
+		}, err)
+	}
+
+	origin := request.URL.Hostname()
+	return &ForgeUser{
+		Origin: &origin,
+		Id:     &userInfo.ID,
+		Handle: &userInfo.Login,
+	}, nil
+}
+
 func authorizeForgeWithToken(r *http.Request) (*Authorization, error) {
 	authorization := r.Header.Get("Forge-Authorization")
 	if authorization == "" {
@@ -598,25 +686,32 @@ func authorizeForgeWithToken(r *http.Request) (*Authorization, error) {
 		}
 
 		if userName, found := pattern.Matches(host); found {
-			repoURLs, branch := pattern.ApplyTemplate(userName, projectName)
-			for _, repoURL := range repoURLs {
-				parsedRepoURL, err := url.Parse(repoURL)
-				if err != nil {
-					panic(err) // misconfiguration
-				}
-
-				if err = checkGogsRepositoryPushPermission(parsedRepoURL, authorization); err != nil {
-					errs = append(errs, err)
-					continue
-				}
-
-				// This will actually be ignored by the caller of AuthorizeUpdateFromArchive,
-				// but we return this information as it makes sense to do contextually here.
-				return &Authorization{
-					[]string{repoURL},
-					branch,
-				}, nil
+			repoURL, branch := pattern.ApplyTemplate(userName, projectName)
+			parsedRepoURL, err := url.Parse(repoURL)
+			if err != nil {
+				panic(err) // misconfiguration
 			}
+
+			if err = checkGogsRepositoryPushPermission(parsedRepoURL, authorization); err != nil {
+				errs = append(errs, err)
+				continue
+			}
+
+			authorizedUser, err := fetchGogsAuthorizedUser(parsedRepoURL, authorization)
+			if err != nil {
+				errs = append(errs, err)
+				continue
+			}
+
+			return &Authorization{
+				// This will actually be ignored by the callers of AuthorizeUpdateFromArchive and
+				// AuthorizeDeletion, but we return this information as it makes sense to do
+				// contextually here.
+				repoURLs: []string{repoURL},
+				branch:   branch,
+
+				forgeUser: authorizedUser,
+			}, nil
 		}
 	}
 
@@ -633,7 +728,7 @@ func AuthorizeUpdateFromArchive(r *http.Request) (*Authorization, error) {
 		return nil, err
 	}
 
-	auth := authorizeInsecure()
+	auth := authorizeInsecure(r)
 	if auth != nil {
 		return auth, nil
 	}
@@ -645,11 +740,11 @@ func AuthorizeUpdateFromArchive(r *http.Request) (*Authorization, error) {
 	} else if err != nil { // bad request
 		return nil, err
 	} else {
-		log.Printf("auth: forge token: allow\n")
+		logc.Printf(r.Context(), "auth: forge token: allow\n")
 		return auth, nil
 	}
 
-	if config.Limits.AllowedRepositoryURLPrefixes != nil {
+	if len(config.Limits.AllowedRepositoryURLPrefixes) > 0 {
 		causes = append(causes, AuthError{http.StatusUnauthorized, "DNS challenge not allowed"})
 	} else {
 		// DNS challenge gives absolute authority.
@@ -659,7 +754,7 @@ func AuthorizeUpdateFromArchive(r *http.Request) (*Authorization, error) {
 		} else if err != nil { // bad request
 			return nil, err
 		} else {
-			log.Println("auth: DNS challenge")
+			logc.Println(r.Context(), "auth: DNS challenge")
 			return auth, nil
 		}
 	}
@@ -667,6 +762,41 @@ func AuthorizeUpdateFromArchive(r *http.Request) (*Authorization, error) {
 	return nil, joinErrors(causes...)
 }
 
+func AuthorizeDeletion(r *http.Request) (*Authorization, error) {
+	causes := []error{AuthError{http.StatusUnauthorized, "unauthorized"}}
+
+	if err := CheckForbiddenDomain(r); err != nil {
+		return nil, err
+	}
+
+	auth := authorizeInsecure(r)
+	if auth != nil {
+		return auth, nil
+	}
+
+	auth, err := authorizeDNSChallenge(r)
+	if err != nil && IsUnauthorized(err) {
+		causes = append(causes, err)
+	} else if err != nil { // bad request
+		return nil, err
+	} else {
+		logc.Printf(r.Context(), "auth: DNS challenge: allow *\n")
+		return auth, nil
+	}
+
+	auth, err = authorizeForgeWithToken(r)
+	if err != nil && IsUnauthorized(err) {
+		causes = append(causes, err)
+	} else if err != nil { // bad request
+		return nil, err
+	} else {
+		logc.Printf(r.Context(), "auth: forge token: allow\n")
+		return auth, nil
+	}
+
+	return nil, joinErrors(causes...)
+}
+
 func CheckForbiddenDomain(r *http.Request) error {
 	host, err := GetHost(r)
 	if err != nil {

src/backend.go

@@ -5,32 +5,79 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"slices"
+	"iter"
 	"strings"
 	"time"
 )
 
 var ErrObjectNotFound = errors.New("not found")
+var ErrPreconditionFailed = errors.New("precondition failed")
+var ErrWriteConflict = errors.New("write conflict")
+var ErrDomainFrozen = errors.New("domain administratively frozen")
 
 func splitBlobName(name string) []string {
-	algo, hash, found := strings.Cut(name, "-")
-	if found {
-		return slices.Concat([]string{algo}, splitBlobName(hash))
+	if algo, hash, found := strings.Cut(name, "-"); found {
+		return []string{algo, hash[0:2], hash[2:4], hash[4:]}
 	} else {
-		return []string{name[0:2], name[2:4], name[4:]}
+		panic("malformed blob name")
 	}
 }
 
+func joinBlobName(parts []string) string {
+	return fmt.Sprintf("%s-%s", parts[0], strings.Join(parts[1:], ""))
+}
+
 type BackendFeature string
 
 const (
 	FeatureCheckDomainMarker BackendFeature = "check-domain-marker"
 )
 
+type BlobMetadata struct {
+	Name         string
+	Size         int64
+	LastModified time.Time
+}
+
 type GetManifestOptions struct {
+	// If true and the manifest is past the cache `MaxAge`, `GetManifest` blocks and returns
+	// a fresh object instead of revalidating in background and returning a stale object.
 	BypassCache bool
 }
 
+type ManifestMetadata struct {
+	Name         string
+	Size         int64
+	LastModified time.Time
+	ETag         string
+}
+
+type ModifyManifestOptions struct {
+	// If non-zero, the request will only succeed if the manifest hasn't been changed since
+	// the given time. Whether this is racy or not is can be determined via `HasAtomicCAS()`.
+	IfUnmodifiedSince time.Time
+	// If non-empty, the request will only succeed if the manifest hasn't changed from
+	// the state corresponding to the ETag. Whether this is racy or not is can be determined
+	// via `HasAtomicCAS()`.
+	IfMatch string
+}
+
+type SearchAuditLogOptions struct {
+	// Inclusive lower bound on returned audit records, per their Snowflake ID (which may differ
+	// slightly from the embedded timestamp). If zero, audit records are returned since beginning
+	// of time.
+	Since time.Time
+	// Inclusive upper bound on returned audit records, per their Snowflake ID (which may differ
+	// slightly from the embedded timestamp). If zero, audit records are returned until the end
+	// of time.
+	Until time.Time
+}
+
+type SearchAuditLogResult struct {
+	ID  AuditID
+	Err error
+}
+
 type Backend interface {
 	// Returns true if the feature has been enabled for this store, false otherwise.
 	HasFeature(ctx context.Context, feature BackendFeature) bool
@@ -40,7 +87,7 @@ type Backend interface {
 
 	// Retrieve a blob. Returns `reader, size, mtime, err`.
 	GetBlob(ctx context.Context, name string) (
-		reader io.ReadSeeker, size uint64, mtime time.Time, err error,
+		reader io.ReadSeeker, metadata BlobMetadata, err error,
 	)
 
 	// Store a blob. If a blob called `name` already exists, this function returns `nil` without
@@ -51,9 +98,13 @@ type Backend interface {
 	// Delete a blob. This is an unconditional operation that can break integrity of manifests.
 	DeleteBlob(ctx context.Context, name string) error
 
+	// Iterate through all blobs. Whether blobs that are newly added during iteration will appear
+	// in the results is unspecified.
+	EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error]
+
 	// Retrieve a manifest.
 	GetManifest(ctx context.Context, name string, opts GetManifestOptions) (
-		manifest *Manifest, mtime time.Time, err error,
+		manifest *Manifest, metadata ManifestMetadata, err error,
 	)
 
 	// Stage a manifest. This operation stores a new version of a manifest, locking any blobs
@@ -61,35 +112,58 @@ type Backend interface {
 	// effects.
 	StageManifest(ctx context.Context, manifest *Manifest) error
 
+	// Whether a compare-and-swap operation on a manifest is truly race-free, or only best-effort
+	// atomic with a small but non-zero window where two requests may race where the one committing
+	// first will have its update lost. (Plain swap operations are always guaranteed to be atomic.)
+	HasAtomicCAS(ctx context.Context) bool
+
 	// Commit a manifest. This is an atomic operation; `GetManifest` calls will return either
 	// the old version or the new version of the manifest, never anything else.
-	CommitManifest(ctx context.Context, name string, manifest *Manifest) error
+	CommitManifest(ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions) error
 
 	// Delete a manifest.
-	DeleteManifest(ctx context.Context, name string) error
+	DeleteManifest(ctx context.Context, name string, opts ModifyManifestOptions) error
 
-	// List all manifests.
-	ListManifests(ctx context.Context) (manifests []string, err error)
+	// Iterate through all manifests. Whether manifests that are newly added during iteration
+	// will appear in the results is unspecified.
+	EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error]
 
 	// Check whether a domain has any deployments.
 	CheckDomain(ctx context.Context, domain string) (found bool, err error)
 
-	// Creates a domain. This allows us to start serving content for the domain.
+	// Create a domain. This allows us to start serving content for the domain.
 	CreateDomain(ctx context.Context, domain string) error
+
+	// Freeze  a domain. This allows a site to be administratively locked, e.g. if it
+	// is discovered serving abusive content.
+	FreezeDomain(ctx context.Context, domain string) error
+
+	// Thaw a domain. This removes the previously placed administrative lock (if any).
+	UnfreezeDomain(ctx context.Context, domain string) error
+
+	// Append a record to the audit log.
+	AppendAuditLog(ctx context.Context, id AuditID, record *AuditRecord) error
+
+	// Retrieve a single record from the audit log.
+	QueryAuditLog(ctx context.Context, id AuditID) (record *AuditRecord, err error)
+
+	// Retrieve records from the audit log by time range.
+	SearchAuditLog(ctx context.Context, opts SearchAuditLogOptions) iter.Seq2[AuditID, error]
 }
 
-func CreateBackend(config *StorageConfig) (backend Backend, err error) {
+func CreateBackend(ctx context.Context, config *StorageConfig) (backend Backend, err error) {
 	switch config.Type {
 	case "fs":
-		if backend, err = NewFSBackend(&config.FS); err != nil {
+		if backend, err = NewFSBackend(ctx, &config.FS); err != nil {
 			err = fmt.Errorf("fs backend: %w", err)
 		}
 	case "s3":
-		if backend, err = NewS3Backend(context.Background(), &config.S3); err != nil {
+		if backend, err = NewS3Backend(ctx, &config.S3); err != nil {
 			err = fmt.Errorf("s3 backend: %w", err)
 		}
 	default:
 		err = fmt.Errorf("unknown backend: %s", config.Type)
 	}
+	backend = NewAuditedBackend(backend)
 	return
 }

src/backend_fs.go

@@ -6,16 +6,18 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/fs"
+	iofs "io/fs"
+	"iter"
 	"os"
 	"path/filepath"
 	"strings"
-	"time"
 )
 
 type FSBackend struct {
-	blobRoot *os.Root
-	siteRoot *os.Root
+	blobRoot     *os.Root
+	siteRoot     *os.Root
+	auditRoot    *os.Root
+	hasAtomicCAS bool
 }
 
 var _ Backend = (*FSBackend)(nil)
@@ -54,7 +56,21 @@ func createTempInRoot(root *os.Root, name string, data []byte) (string, error) {
 	return tempPath, nil
 }
 
-func NewFSBackend(config *FSConfig) (*FSBackend, error) {
+func checkAtomicCAS(root *os.Root) bool {
+	fileName := ".hasAtomicCAS"
+	file, err := root.Create(fileName)
+	if err != nil {
+		panic(err)
+	}
+	root.Remove(fileName)
+	defer file.Close()
+
+	flockErr := FileLock(file)
+	funlockErr := FileUnlock(file)
+	return (flockErr == nil && funlockErr == nil)
+}
+
+func NewFSBackend(ctx context.Context, config *FSConfig) (*FSBackend, error) {
 	blobRoot, err := maybeCreateOpenRoot(config.Root, "blob")
 	if err != nil {
 		return nil, fmt.Errorf("blob: %w", err)
@@ -63,7 +79,17 @@ func NewFSBackend(config *FSConfig) (*FSBackend, error) {
 	if err != nil {
 		return nil, fmt.Errorf("site: %w", err)
 	}
-	return &FSBackend{blobRoot, siteRoot}, nil
+	auditRoot, err := maybeCreateOpenRoot(config.Root, "audit")
+	if err != nil {
+		return nil, fmt.Errorf("audit: %w", err)
+	}
+	hasAtomicCAS := checkAtomicCAS(siteRoot)
+	if hasAtomicCAS {
+		logc.Println(ctx, "fs: has atomic CAS")
+	} else {
+		logc.Println(ctx, "fs: has best-effort CAS")
+	}
+	return &FSBackend{blobRoot, siteRoot, auditRoot, hasAtomicCAS}, nil
 }
 
 func (fs *FSBackend) Backend() Backend {
@@ -91,7 +117,7 @@ func (fs *FSBackend) EnableFeature(ctx context.Context, feature BackendFeature)
 func (fs *FSBackend) GetBlob(
 	ctx context.Context, name string,
 ) (
-	reader io.ReadSeeker, size uint64, mtime time.Time, err error,
+	reader io.ReadSeeker, metadata BlobMetadata, err error,
 ) {
 	blobPath := filepath.Join(splitBlobName(name)...)
 	stat, err := fs.blobRoot.Stat(blobPath)
@@ -107,13 +133,19 @@ func (fs *FSBackend) GetBlob(
 		err = fmt.Errorf("open: %w", err)
 		return
 	}
-	return file, uint64(stat.Size()), stat.ModTime(), nil
+	return file, BlobMetadata{name, int64(stat.Size()), stat.ModTime()}, nil
 }
 
 func (fs *FSBackend) PutBlob(ctx context.Context, name string, data []byte) error {
 	blobPath := filepath.Join(splitBlobName(name)...)
 	blobDir := filepath.Dir(blobPath)
 
+	if _, err := fs.blobRoot.Stat(blobPath); err == nil {
+		// Blob already exists. While on Linux it would be benign to write and replace a blob
+		// that already exists, on Windows this is liable to cause access errors.
+		return nil
+	}
+
 	tempPath, err := createTempInRoot(fs.blobRoot, name, data)
 	if err != nil {
 		return err
@@ -149,25 +181,36 @@ func (fs *FSBackend) DeleteBlob(ctx context.Context, name string) error {
 	return fs.blobRoot.Remove(blobPath)
 }
 
-func (b *FSBackend) ListManifests(ctx context.Context) (manifests []string, err error) {
-	err = fs.WalkDir(b.siteRoot.FS(), ".", func(path string, d fs.DirEntry, err error) error {
-		if strings.Count(path, "/") > 1 {
-			return fs.SkipDir
-		}
-		_, project, _ := strings.Cut(path, "/")
-		if project == "" || strings.HasPrefix(project, ".") && project != ".index" {
-			return nil
-		}
-		manifests = append(manifests, path)
-		return nil
-	})
-	return
+func (fs *FSBackend) EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error] {
+	return func(yield func(BlobMetadata, error) bool) {
+		iofs.WalkDir(fs.blobRoot.FS(), ".",
+			func(path string, entry iofs.DirEntry, err error) error {
+				var metadata BlobMetadata
+				if err != nil {
+					// report error
+				} else if entry.IsDir() {
+					// skip directory
+					return nil
+				} else if info, err := entry.Info(); err != nil {
+					// report error
+				} else {
+					// report blob
+					metadata.Name = joinBlobName(strings.Split(path, "/"))
+					metadata.Size = info.Size()
+					metadata.LastModified = info.ModTime()
+				}
+				if !yield(metadata, err) {
+					return iofs.SkipAll
+				}
+				return nil
+			})
+	}
 }
 
 func (fs *FSBackend) GetManifest(
 	ctx context.Context, name string, opts GetManifestOptions,
 ) (
-	manifest *Manifest, mtime time.Time, err error,
+	manifest *Manifest, metadata ManifestMetadata, err error,
 ) {
 	stat, err := fs.siteRoot.Stat(name)
 	if errors.Is(err, os.ErrNotExist) {
@@ -186,7 +229,10 @@ func (fs *FSBackend) GetManifest(
 	if err != nil {
 		return
 	}
-	return manifest, stat.ModTime(), nil
+	return manifest, ManifestMetadata{
+		LastModified: stat.ModTime(),
+		ETag:         fmt.Sprintf("%x", sha256.Sum256(data)),
+	}, nil
 }
 
 func stagedManifestName(manifestData []byte) string {
@@ -208,7 +254,102 @@ func (fs *FSBackend) StageManifest(ctx context.Context, manifest *Manifest) erro
 	return nil
 }
 
-func (fs *FSBackend) CommitManifest(ctx context.Context, name string, manifest *Manifest) error {
+func domainFrozenMarkerName(domain string) string {
+	return filepath.Join(domain, ".frozen")
+}
+
+func (fs *FSBackend) checkDomainFrozen(ctx context.Context, domain string) error {
+	if _, err := fs.siteRoot.Stat(domainFrozenMarkerName(domain)); err == nil {
+		return ErrDomainFrozen
+	} else if !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("stat: %w", err)
+	} else {
+		return nil
+	}
+}
+
+func (fs *FSBackend) HasAtomicCAS(ctx context.Context) bool {
+	// On a suitable filesystem, POSIX advisory locks can be used to implement atomic CAS.
+	// An implementation consists of two parts:
+	//   - Intra-process mutex set (one per manifest), to prevent races between goroutines;
+	//   - Inter-process POSIX advisory locks (one per manifest), to prevent races between
+	//     different git-pages instances.
+	return fs.hasAtomicCAS
+}
+
+type manifestLockGuard struct {
+	file *os.File
+}
+
+func lockManifest(fs *os.Root, name string) (*manifestLockGuard, error) {
+	file, err := fs.Open(name)
+	if errors.Is(err, os.ErrNotExist) {
+		return &manifestLockGuard{nil}, nil
+	} else if err != nil {
+		return nil, fmt.Errorf("open: %w", err)
+	}
+	if err := FileLock(file); err != nil {
+		file.Close()
+		return nil, fmt.Errorf("flock(LOCK_EX): %w", err)
+	}
+	return &manifestLockGuard{file}, nil
+}
+
+func (guard *manifestLockGuard) Unlock() {
+	if guard.file != nil {
+		FileUnlock(guard.file)
+		guard.file.Close()
+	}
+}
+
+func (fs *FSBackend) checkManifestPrecondition(
+	ctx context.Context, name string, opts ModifyManifestOptions,
+) error {
+	if !opts.IfUnmodifiedSince.IsZero() {
+		stat, err := fs.siteRoot.Stat(name)
+		if err != nil {
+			return fmt.Errorf("stat: %w", err)
+		}
+
+		if stat.ModTime().Compare(opts.IfUnmodifiedSince) > 0 {
+			return fmt.Errorf("%w: If-Unmodified-Since", ErrPreconditionFailed)
+		}
+	}
+
+	if opts.IfMatch != "" {
+		data, err := fs.siteRoot.ReadFile(name)
+		if err != nil {
+			return fmt.Errorf("read: %w", err)
+		}
+
+		if fmt.Sprintf("%x", sha256.Sum256(data)) != opts.IfMatch {
+			return fmt.Errorf("%w: If-Match", ErrPreconditionFailed)
+		}
+	}
+
+	return nil
+}
+
+func (fs *FSBackend) CommitManifest(
+	ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions,
+) error {
+	if fs.hasAtomicCAS {
+		if guard, err := lockManifest(fs.siteRoot, name); err != nil {
+			return err
+		} else {
+			defer guard.Unlock()
+		}
+	}
+
+	domain := filepath.Dir(name)
+	if err := fs.checkDomainFrozen(ctx, domain); err != nil {
+		return err
+	}
+
+	if err := fs.checkManifestPrecondition(ctx, name, opts); err != nil {
+		return err
+	}
+
 	manifestData := EncodeManifest(manifest)
 	manifestHashName := stagedManifestName(manifestData)
 
@@ -216,7 +357,7 @@ func (fs *FSBackend) CommitManifest(ctx context.Context, name string, manifest *
 		return fmt.Errorf("manifest not staged")
 	}
 
-	if err := fs.siteRoot.MkdirAll(filepath.Dir(name), 0o755); err != nil {
+	if err := fs.siteRoot.MkdirAll(domain, 0o755); err != nil {
 		return fmt.Errorf("mkdir: %w", err)
 	}
 
@@ -227,7 +368,26 @@ func (fs *FSBackend) CommitManifest(ctx context.Context, name string, manifest *
 	return nil
 }
 
-func (fs *FSBackend) DeleteManifest(ctx context.Context, name string) error {
+func (fs *FSBackend) DeleteManifest(
+	ctx context.Context, name string, opts ModifyManifestOptions,
+) error {
+	if fs.hasAtomicCAS {
+		if guard, err := lockManifest(fs.siteRoot, name); err != nil {
+			return err
+		} else {
+			defer guard.Unlock()
+		}
+	}
+
+	domain := filepath.Dir(name)
+	if err := fs.checkDomainFrozen(ctx, domain); err != nil {
+		return err
+	}
+
+	if err := fs.checkManifestPrecondition(ctx, name, opts); err != nil {
+		return err
+	}
+
 	err := fs.siteRoot.Remove(name)
 	if errors.Is(err, os.ErrNotExist) {
 		return nil
@@ -236,6 +396,37 @@ func (fs *FSBackend) DeleteManifest(ctx context.Context, name string) error {
 	}
 }
 
+func (fs *FSBackend) EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error] {
+	return func(yield func(ManifestMetadata, error) bool) {
+		iofs.WalkDir(fs.siteRoot.FS(), ".",
+			func(path string, entry iofs.DirEntry, err error) error {
+				_, project, _ := strings.Cut(path, "/")
+				var metadata ManifestMetadata
+				if err != nil {
+					// report error
+				} else if entry.IsDir() {
+					// skip directory
+					return nil
+				} else if project == "" || strings.HasPrefix(project, ".") && project != ".index" {
+					// skip internal
+					return nil
+				} else if info, err := entry.Info(); err != nil {
+					// report error
+				} else {
+					// report blob
+					metadata.Name = path
+					metadata.Size = info.Size()
+					metadata.LastModified = info.ModTime()
+					// not setting metadata.ETag since it is too costly
+				}
+				if !yield(metadata, err) {
+					return iofs.SkipAll
+				}
+				return nil
+			})
+	}
+}
+
 func (fs *FSBackend) CheckDomain(ctx context.Context, domain string) (bool, error) {
 	_, err := fs.siteRoot.Stat(domain)
 	if errors.Is(err, os.ErrNotExist) {
@@ -250,3 +441,62 @@ func (fs *FSBackend) CheckDomain(ctx context.Context, domain string) (bool, erro
 func (fs *FSBackend) CreateDomain(ctx context.Context, domain string) error {
 	return nil // no-op
 }
+
+func (fs *FSBackend) FreezeDomain(ctx context.Context, domain string) error {
+	return fs.siteRoot.WriteFile(domainFrozenMarkerName(domain), []byte{}, 0o644)
+}
+
+func (fs *FSBackend) UnfreezeDomain(ctx context.Context, domain string) error {
+	err := fs.siteRoot.Remove(domainFrozenMarkerName(domain))
+	if errors.Is(err, os.ErrNotExist) {
+		return nil
+	} else {
+		return err
+	}
+}
+
+func (fs *FSBackend) AppendAuditLog(ctx context.Context, id AuditID, record *AuditRecord) error {
+	if _, err := fs.auditRoot.Stat(id.String()); err == nil {
+		panic(fmt.Errorf("audit ID collision: %s", id))
+	}
+
+	return fs.auditRoot.WriteFile(id.String(), EncodeAuditRecord(record), 0o644)
+}
+
+func (fs *FSBackend) QueryAuditLog(ctx context.Context, id AuditID) (*AuditRecord, error) {
+	if data, err := fs.auditRoot.ReadFile(id.String()); err != nil {
+		return nil, fmt.Errorf("read: %w", err)
+	} else if record, err := DecodeAuditRecord(data); err != nil {
+		return nil, fmt.Errorf("decode: %w", err)
+	} else {
+		return record, nil
+	}
+}
+
+func (fs *FSBackend) SearchAuditLog(
+	ctx context.Context, opts SearchAuditLogOptions,
+) iter.Seq2[AuditID, error] {
+	return func(yield func(AuditID, error) bool) {
+		iofs.WalkDir(fs.auditRoot.FS(), ".",
+			func(path string, entry iofs.DirEntry, err error) error {
+				if path == "." {
+					return nil // skip
+				}
+				var id AuditID
+				if err != nil {
+					// report error
+				} else if id, err = ParseAuditID(path); err != nil {
+					// report error
+				} else if !opts.Since.IsZero() && id.CompareTime(opts.Since) < 0 {
+					return nil // skip
+				} else if !opts.Until.IsZero() && id.CompareTime(opts.Until) > 0 {
+					return nil // skip
+				}
+				if !yield(id, err) {
+					return iofs.SkipAll // break
+				} else {
+					return nil // continue
+				}
+			})
+	}
+}

src/backend_s3.go

@@ -6,7 +6,7 @@ import (
 	"crypto/sha256"
 	"fmt"
 	"io"
-	"log"
+	"iter"
 	"net/http"
 	"path"
 	"strings"
@@ -36,7 +36,7 @@ var (
 	manifestCacheEvictionsCount prometheus.Counter
 
 	s3GetObjectDurationSeconds *prometheus.HistogramVec
-	s3GetObjectErrorsCount     *prometheus.CounterVec
+	s3GetObjectResponseCount   *prometheus.CounterVec
 )
 
 func initS3BackendMetrics() {
@@ -96,10 +96,10 @@ func initS3BackendMetrics() {
 		NativeHistogramMaxBucketNumber:  100,
 		NativeHistogramMinResetDuration: 10 * time.Minute,
 	}, []string{"kind"})
-	s3GetObjectErrorsCount = promauto.NewCounterVec(prometheus.CounterOpts{
-		Name: "git_pages_s3_get_object_errors_count",
-		Help: "Count of s3:GetObject errors",
-	}, []string{"object_kind"})
+	s3GetObjectResponseCount = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "git_pages_s3_get_object_responses_count",
+		Help: "Count of s3:GetObject responses",
+	}, []string{"kind", "code"})
 }
 
 // Blobs can be safely cached indefinitely. They only need to be evicted to preserve memory.
@@ -117,8 +117,7 @@ func (c *CachedBlob) Weight() uint32 { return uint32(len(c.blob)) }
 type CachedManifest struct {
 	manifest *Manifest
 	weight   uint32
-	mtime    time.Time
-	etag     string
+	metadata ManifestMetadata
 	err      error
 }
 
@@ -144,10 +143,12 @@ func makeCacheOptions[K comparable, V any](
 		options.Weigher = weigher
 	}
 	if config.MaxStale != 0 {
-		options.RefreshCalculator = otter.RefreshWriting[K, V](time.Duration(config.MaxAge))
+		options.RefreshCalculator = otter.RefreshWriting[K, V](
+			time.Duration(config.MaxAge))
 	}
 	if config.MaxAge != 0 || config.MaxStale != 0 {
-		options.ExpiryCalculator = otter.ExpiryWriting[K, V](time.Duration(config.MaxAge + config.MaxStale))
+		options.ExpiryCalculator = otter.ExpiryWriting[K, V](
+			time.Duration(config.MaxAge + config.MaxStale))
 	}
 	return options
 }
@@ -170,7 +171,7 @@ func NewS3Backend(ctx context.Context, config *S3Config) (*S3Backend, error) {
 	if err != nil {
 		return nil, err
 	} else if !exists {
-		log.Printf("s3: create bucket %s\n", bucket)
+		logc.Printf(ctx, "s3: create bucket %s\n", bucket)
 
 		err = client.MakeBucket(ctx, bucket,
 			minio.MakeBucketOptions{Region: config.Region})
@@ -236,13 +237,13 @@ func (s3 *S3Backend) HasFeature(ctx context.Context, feature BackendFeature) boo
 			minio.StatObjectOptions{})
 		if err != nil {
 			if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" {
-				log.Printf("s3 feature %q: disabled", feature)
+				logc.Printf(ctx, "s3 feature %q: disabled", feature)
 				return false, nil
 			} else {
 				return false, err
 			}
 		}
-		log.Printf("s3 feature %q: enabled", feature)
+		logc.Printf(ctx, "s3 feature %q: enabled", feature)
 		return true, nil
 	}
 
@@ -250,7 +251,7 @@ func (s3 *S3Backend) HasFeature(ctx context.Context, feature BackendFeature) boo
 	if err != nil {
 		err = fmt.Errorf("getting s3 backend feature %q: %w", feature, err)
 		ObserveError(err)
-		log.Print(err)
+		logc.Println(ctx, err)
 		return false
 	}
 	return isOn
@@ -265,10 +266,10 @@ func (s3 *S3Backend) EnableFeature(ctx context.Context, feature BackendFeature)
 func (s3 *S3Backend) GetBlob(
 	ctx context.Context, name string,
 ) (
-	reader io.ReadSeeker, size uint64, mtime time.Time, err error,
+	reader io.ReadSeeker, metadata BlobMetadata, err error,
 ) {
 	loader := func(ctx context.Context, name string) (*CachedBlob, error) {
-		log.Printf("s3: get blob %s\n", name)
+		logc.Printf(ctx, "s3: get blob %s\n", name)
 
 		startTime := time.Now()
 
@@ -297,23 +298,33 @@ func (s3 *S3Backend) GetBlob(
 		return &CachedBlob{data, stat.LastModified}, nil
 	}
 
+	observer := func(ctx context.Context, name string) (*CachedBlob, error) {
+		cached, err := loader(ctx, name)
+		var code = "OK"
+		if resp, ok := err.(minio.ErrorResponse); ok {
+			code = resp.Code
+		}
+		s3GetObjectResponseCount.With(prometheus.Labels{"kind": "blob", "code": code}).Inc()
+		return cached, err
+	}
+
 	var cached *CachedBlob
-	cached, err = s3.blobCache.Get(ctx, name, otter.LoaderFunc[string, *CachedBlob](loader))
+	cached, err = s3.blobCache.Get(ctx, name, otter.LoaderFunc[string, *CachedBlob](observer))
 	if err != nil {
 		if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" {
-			s3GetObjectErrorsCount.With(prometheus.Labels{"object_kind": "blob"}).Inc()
 			err = fmt.Errorf("%w: %s", ErrObjectNotFound, errResp.Key)
 		}
 	} else {
 		reader = bytes.NewReader(cached.blob)
-		size = uint64(len(cached.blob))
-		mtime = cached.mtime
+		metadata.Name = name
+		metadata.Size = int64(len(cached.blob))
+		metadata.LastModified = cached.mtime
 	}
 	return
 }
 
 func (s3 *S3Backend) PutBlob(ctx context.Context, name string, data []byte) error {
-	log.Printf("s3: put blob %s (%s)\n", name, datasize.ByteSize(len(data)).HumanReadable())
+	logc.Printf(ctx, "s3: put blob %s (%s)\n", name, datasize.ByteSize(len(data)).HumanReadable())
 
 	_, err := s3.client.StatObject(ctx, s3.bucket, blobObjectName(name),
 		minio.GetObjectOptions{})
@@ -325,7 +336,7 @@ func (s3 *S3Backend) PutBlob(ctx context.Context, name string, data []byte) erro
 				return err
 			} else {
 				ObserveData(ctx, "blob.status", "created")
-				log.Printf("s3: put blob %s (created)\n", name)
+				logc.Printf(ctx, "s3: put blob %s (created)\n", name)
 				return nil
 			}
 		} else {
@@ -333,7 +344,7 @@ func (s3 *S3Backend) PutBlob(ctx context.Context, name string, data []byte) erro
 		}
 	} else {
 		ObserveData(ctx, "blob.status", "exists")
-		log.Printf("s3: put blob %s (exists)\n", name)
+		logc.Printf(ctx, "s3: put blob %s (exists)\n", name)
 		blobsDedupedCount.Inc()
 		blobsDedupedBytes.Add(float64(len(data)))
 		return nil
@@ -341,12 +352,43 @@ func (s3 *S3Backend) PutBlob(ctx context.Context, name string, data []byte) erro
 }
 
 func (s3 *S3Backend) DeleteBlob(ctx context.Context, name string) error {
-	log.Printf("s3: delete blob %s\n", name)
+	logc.Printf(ctx, "s3: delete blob %s\n", name)
 
 	return s3.client.RemoveObject(ctx, s3.bucket, blobObjectName(name),
 		minio.RemoveObjectOptions{})
 }
 
+func (s3 *S3Backend) EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error] {
+	return func(yield func(BlobMetadata, error) bool) {
+		logc.Print(ctx, "s3: enumerate blobs")
+
+		ctx, cancel := context.WithCancel(ctx)
+		defer cancel()
+
+		prefix := "blob/"
+		for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{
+			Prefix:    prefix,
+			Recursive: true,
+		}) {
+			var metadata BlobMetadata
+			var err error
+			if err = object.Err; err == nil {
+				key := strings.TrimPrefix(object.Key, prefix)
+				if strings.HasSuffix(key, "/") {
+					continue // directory; skip
+				} else {
+					metadata.Name = joinBlobName(strings.Split(key, "/"))
+					metadata.Size = object.Size
+					metadata.LastModified = object.LastModified
+				}
+			}
+			if !yield(metadata, err) {
+				break
+			}
+		}
+	}
+}
+
 func manifestObjectName(name string) string {
 	return fmt.Sprintf("site/%s", name)
 }
@@ -355,55 +397,37 @@ func stagedManifestObjectName(manifestData []byte) string {
 	return fmt.Sprintf("dirty/%x", sha256.Sum256(manifestData))
 }
 
-func (s3 *S3Backend) ListManifests(ctx context.Context) (manifests []string, err error) {
-	log.Print("s3: list manifests")
-
-	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
-
-	prefix := manifestObjectName("")
-	for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{
-		Prefix:    prefix,
-		Recursive: true,
-	}) {
-		if object.Err != nil {
-			return nil, object.Err
-		}
-		key := strings.TrimRight(strings.TrimPrefix(object.Key, prefix), "/")
-		if strings.Count(key, "/") > 1 {
-			continue
-		}
-		_, project, _ := strings.Cut(key, "/")
-		if project == "" || strings.HasPrefix(project, ".") && project != ".index" {
-			continue
-		}
-		manifests = append(manifests, key)
-	}
-
-	return
-}
-
 type s3ManifestLoader struct {
 	s3 *S3Backend
 }
 
-func (l s3ManifestLoader) Load(ctx context.Context, key string) (*CachedManifest, error) {
+func (l s3ManifestLoader) Load(
+	ctx context.Context, key string,
+) (
+	*CachedManifest, error,
+) {
 	return l.load(ctx, key, nil)
 }
 
-func (l s3ManifestLoader) Reload(ctx context.Context, key string, oldValue *CachedManifest) (*CachedManifest, error) {
+func (l s3ManifestLoader) Reload(
+	ctx context.Context, key string, oldValue *CachedManifest,
+) (
+	*CachedManifest, error,
+) {
 	return l.load(ctx, key, oldValue)
 }
 
-func (l s3ManifestLoader) load(ctx context.Context, name string, oldManifest *CachedManifest) (*CachedManifest, error) {
+func (l s3ManifestLoader) load(
+	ctx context.Context, name string, oldManifest *CachedManifest,
+) (
+	*CachedManifest, error,
+) {
+	logc.Printf(ctx, "s3: get manifest %s\n", name)
+
 	loader := func() (*CachedManifest, error) {
-		log.Printf("s3: get manifest %s\n", name)
-
-		startTime := time.Now()
-
 		opts := minio.GetObjectOptions{}
-		if oldManifest != nil && oldManifest.etag != "" {
-			opts.SetMatchETagExcept(oldManifest.etag)
+		if oldManifest != nil && oldManifest.metadata.ETag != "" {
+			opts.SetMatchETagExcept(oldManifest.metadata.ETag)
 		}
 		object, err := l.s3.client.GetObject(ctx, l.s3.bucket, manifestObjectName(name), opts)
 		// Note that many errors (e.g. NoSuchKey) will be reported only after this point.
@@ -427,20 +451,34 @@ func (l s3ManifestLoader) load(ctx context.Context, name string, oldManifest *Ca
 			return nil, err
 		}
 
-		s3GetObjectDurationSeconds.
-			With(prometheus.Labels{"kind": "manifest"}).
-			Observe(time.Since(startTime).Seconds())
-
-		return &CachedManifest{manifest, uint32(len(data)), stat.LastModified, stat.ETag, nil}, nil
+		metadata := ManifestMetadata{
+			LastModified: stat.LastModified,
+			ETag:         stat.ETag,
+		}
+		return &CachedManifest{manifest, uint32(len(data)), metadata, nil}, nil
 	}
 
-	var cached *CachedManifest
-	cached, err := loader()
+	observer := func() (*CachedManifest, error) {
+		cached, err := loader()
+		var code = "OK"
+		if resp, ok := err.(minio.ErrorResponse); ok {
+			code = resp.Code
+		}
+		s3GetObjectResponseCount.With(prometheus.Labels{"kind": "manifest", "code": code}).Inc()
+		return cached, err
+	}
+
+	startTime := time.Now()
+	cached, err := observer()
+	s3GetObjectDurationSeconds.
+		With(prometheus.Labels{"kind": "manifest"}).
+		Observe(time.Since(startTime).Seconds())
+
 	if err != nil {
-		if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" {
-			s3GetObjectErrorsCount.With(prometheus.Labels{"object_kind": "manifest"}).Inc()
+		errResp := minio.ToErrorResponse(err)
+		if errResp.Code == "NoSuchKey" {
 			err = fmt.Errorf("%w: %s", ErrObjectNotFound, errResp.Key)
-			return &CachedManifest{nil, 1, time.Time{}, "", err}, nil
+			return &CachedManifest{nil, 1, ManifestMetadata{}, err}, nil
 		} else if errResp.StatusCode == http.StatusNotModified && oldManifest != nil {
 			return oldManifest, nil
 		} else {
@@ -454,7 +492,7 @@ func (l s3ManifestLoader) load(ctx context.Context, name string, oldManifest *Ca
 func (s3 *S3Backend) GetManifest(
 	ctx context.Context, name string, opts GetManifestOptions,
 ) (
-	manifest *Manifest, mtime time.Time, err error,
+	manifest *Manifest, metadata ManifestMetadata, err error,
 ) {
 	if opts.BypassCache {
 		entry, found := s3.siteCache.Cache.GetEntry(name)
@@ -469,33 +507,112 @@ func (s3 *S3Backend) GetManifest(
 		return
 	} else {
 		// This could be `manifest, mtime, nil` or `nil, time.Time{}, ErrObjectNotFound`.
-		manifest, mtime, err = cached.manifest, cached.mtime, cached.err
+		manifest, metadata, err = cached.manifest, cached.metadata, cached.err
 		return
 	}
 }
 
 func (s3 *S3Backend) StageManifest(ctx context.Context, manifest *Manifest) error {
 	data := EncodeManifest(manifest)
-	log.Printf("s3: stage manifest %x\n", sha256.Sum256(data))
+	logc.Printf(ctx, "s3: stage manifest %x\n", sha256.Sum256(data))
 
 	_, err := s3.client.PutObject(ctx, s3.bucket, stagedManifestObjectName(data),
 		bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{})
 	return err
 }
 
-func (s3 *S3Backend) CommitManifest(ctx context.Context, name string, manifest *Manifest) error {
+func domainFrozenObjectName(domain string) string {
+	return manifestObjectName(fmt.Sprintf("%s/.frozen", domain))
+}
+
+func (s3 *S3Backend) checkDomainFrozen(ctx context.Context, domain string) error {
+	_, err := s3.client.StatObject(ctx, s3.bucket, domainFrozenObjectName(domain),
+		minio.GetObjectOptions{})
+	if err == nil {
+		return ErrDomainFrozen
+	} else if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" {
+		return nil
+	} else {
+		return err
+	}
+}
+
+func (s3 *S3Backend) HasAtomicCAS(ctx context.Context) bool {
+	// Support for `If-Unmodified-Since:` or `If-Match:` for PutObject requests is very spotty:
+	//   - AWS supports only `If-Match:`:
+	//     https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html
+	//   - Minio supports `If-Match:`:
+	//     https://blog.min.io/leading-the-way-minios-conditional-write-feature-for-modern-data-workloads/
+	//   - Tigris supports `If-Unmodified-Since:` and `If-Match:`, but only with `X-Tigris-Consistent: true`;
+	//     https://www.tigrisdata.com/docs/objects/conditionals/
+	//     Note that the `X-Tigris-Consistent: true` header must be present on *every* transaction
+	//     touching the object, not just on the CAS transactions.
+	//   - Wasabi does not support either one and docs seem to suggest that the headers are ignored;
+	//   - Garage does not support either one and source code suggests the headers are ignored.
+	// It seems that the only safe option is to not claim support for atomic CAS, and only do
+	// best-effort CAS implementation using HeadObject and PutObject/DeleteObject.
+	return false
+}
+
+func (s3 *S3Backend) checkManifestPrecondition(
+	ctx context.Context, name string, opts ModifyManifestOptions,
+) error {
+	if opts.IfUnmodifiedSince.IsZero() && opts.IfMatch == "" {
+		return nil
+	}
+
+	stat, err := s3.client.StatObject(ctx, s3.bucket, manifestObjectName(name),
+		minio.GetObjectOptions{})
+	if err != nil {
+		return err
+	}
+
+	if !opts.IfUnmodifiedSince.IsZero() && stat.LastModified.Compare(opts.IfUnmodifiedSince) > 0 {
+		return fmt.Errorf("%w: If-Unmodified-Since", ErrPreconditionFailed)
+	}
+	if opts.IfMatch != "" && stat.ETag != opts.IfMatch {
+		return fmt.Errorf("%w: If-Match", ErrPreconditionFailed)
+	}
+
+	return nil
+}
+
+func (s3 *S3Backend) CommitManifest(
+	ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions,
+) error {
 	data := EncodeManifest(manifest)
-	log.Printf("s3: commit manifest %x -> %s", sha256.Sum256(data), name)
+	logc.Printf(ctx, "s3: commit manifest %x -> %s", sha256.Sum256(data), name)
+
+	_, domain, _ := strings.Cut(name, "/")
+	if err := s3.checkDomainFrozen(ctx, domain); err != nil {
+		return err
+	}
+
+	if err := s3.checkManifestPrecondition(ctx, name, opts); err != nil {
+		return err
+	}
 
 	// Remove staged object unconditionally (whether commit succeeded or failed), since
 	// the upper layer has to retry the complete operation anyway.
+	putOptions := minio.PutObjectOptions{}
+	putOptions.Header().Add("X-Tigris-Consistent", "true")
+	if opts.IfMatch != "" {
+		// Not guaranteed to do anything (see `HasAtomicCAS`), but let's try anyway;
+		// this is a "belt and suspenders" approach, together with `checkManifestPrecondition`.
+		// It does reliably work on MinIO at least.
+		putOptions.SetMatchETag(opts.IfMatch)
+	}
 	_, putErr := s3.client.PutObject(ctx, s3.bucket, manifestObjectName(name),
-		bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{})
+		bytes.NewReader(data), int64(len(data)), putOptions)
 	removeErr := s3.client.RemoveObject(ctx, s3.bucket, stagedManifestObjectName(data),
 		minio.RemoveObjectOptions{})
 	s3.siteCache.Cache.Invalidate(name)
 	if putErr != nil {
-		return putErr
+		if errResp := minio.ToErrorResponse(putErr); errResp.Code == "PreconditionFailed" {
+			return ErrPreconditionFailed
+		} else {
+			return putErr
+		}
 	} else if removeErr != nil {
 		return removeErr
 	} else {
@@ -503,8 +620,19 @@ func (s3 *S3Backend) CommitManifest(ctx context.Context, name string, manifest *
 	}
 }
 
-func (s3 *S3Backend) DeleteManifest(ctx context.Context, name string) error {
-	log.Printf("s3: delete manifest %s\n", name)
+func (s3 *S3Backend) DeleteManifest(
+	ctx context.Context, name string, opts ModifyManifestOptions,
+) error {
+	logc.Printf(ctx, "s3: delete manifest %s\n", name)
+
+	_, domain, _ := strings.Cut(name, "/")
+	if err := s3.checkDomainFrozen(ctx, domain); err != nil {
+		return err
+	}
+
+	if err := s3.checkManifestPrecondition(ctx, name, opts); err != nil {
+		return err
+	}
 
 	err := s3.client.RemoveObject(ctx, s3.bucket, manifestObjectName(name),
 		minio.RemoveObjectOptions{})
@@ -512,12 +640,47 @@ func (s3 *S3Backend) DeleteManifest(ctx context.Context, name string) error {
 	return err
 }
 
+func (s3 *S3Backend) EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error] {
+	return func(yield func(ManifestMetadata, error) bool) {
+		logc.Print(ctx, "s3: enumerate manifests")
+
+		ctx, cancel := context.WithCancel(ctx)
+		defer cancel()
+
+		prefix := "site/"
+		for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{
+			Prefix:    prefix,
+			Recursive: true,
+		}) {
+			var metadata ManifestMetadata
+			var err error
+			if err = object.Err; err == nil {
+				key := strings.TrimPrefix(object.Key, prefix)
+				_, project, _ := strings.Cut(key, "/")
+				if strings.HasSuffix(key, "/") {
+					continue // directory; skip
+				} else if project == "" || strings.HasPrefix(project, ".") && project != ".index" {
+					continue // internal; skip
+				} else {
+					metadata.Name = key
+					metadata.Size = object.Size
+					metadata.LastModified = object.LastModified
+					metadata.ETag = object.ETag
+				}
+			}
+			if !yield(metadata, err) {
+				break
+			}
+		}
+	}
+}
+
 func domainCheckObjectName(domain string) string {
 	return manifestObjectName(fmt.Sprintf("%s/.exists", domain))
 }
 
 func (s3 *S3Backend) CheckDomain(ctx context.Context, domain string) (exists bool, err error) {
-	log.Printf("s3: check domain %s\n", domain)
+	logc.Printf(ctx, "s3: check domain %s\n", domain)
 
 	_, err = s3.client.StatObject(ctx, s3.bucket, domainCheckObjectName(domain),
 		minio.StatObjectOptions{})
@@ -548,9 +711,95 @@ func (s3 *S3Backend) CheckDomain(ctx context.Context, domain string) (exists boo
 }
 
 func (s3 *S3Backend) CreateDomain(ctx context.Context, domain string) error {
-	log.Printf("s3: create domain %s\n", domain)
+	logc.Printf(ctx, "s3: create domain %s\n", domain)
 
 	_, err := s3.client.PutObject(ctx, s3.bucket, domainCheckObjectName(domain),
 		&bytes.Reader{}, 0, minio.PutObjectOptions{})
 	return err
 }
+
+func (s3 *S3Backend) FreezeDomain(ctx context.Context, domain string) error {
+	logc.Printf(ctx, "s3: freeze domain %s\n", domain)
+
+	_, err := s3.client.PutObject(ctx, s3.bucket, domainFrozenObjectName(domain),
+		&bytes.Reader{}, 0, minio.PutObjectOptions{})
+	return err
+
+}
+
+func (s3 *S3Backend) UnfreezeDomain(ctx context.Context, domain string) error {
+	logc.Printf(ctx, "s3: unfreeze domain %s\n", domain)
+
+	err := s3.client.RemoveObject(ctx, s3.bucket, domainFrozenObjectName(domain),
+		minio.RemoveObjectOptions{})
+	if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" {
+		return nil
+	} else {
+		return err
+	}
+}
+
+func auditObjectName(id AuditID) string {
+	return fmt.Sprintf("audit/%s", id)
+}
+
+func (s3 *S3Backend) AppendAuditLog(ctx context.Context, id AuditID, record *AuditRecord) error {
+	logc.Printf(ctx, "s3: append audit %s\n", id)
+
+	name := auditObjectName(id)
+	data := EncodeAuditRecord(record)
+
+	options := minio.PutObjectOptions{}
+	options.SetMatchETagExcept("*") // may or may not be supported
+	_, err := s3.client.PutObject(ctx, s3.bucket, name,
+		bytes.NewReader(data), int64(len(data)), options)
+	if errResp := minio.ToErrorResponse(err); errResp.StatusCode == 412 {
+		panic(fmt.Errorf("audit ID collision: %s", name))
+	}
+	return err
+}
+
+func (s3 *S3Backend) QueryAuditLog(ctx context.Context, id AuditID) (*AuditRecord, error) {
+	logc.Printf(ctx, "s3: read audit %s\n", id)
+
+	object, err := s3.client.GetObject(ctx, s3.bucket, auditObjectName(id),
+		minio.GetObjectOptions{})
+	if err != nil {
+		return nil, err
+	}
+	defer object.Close()
+
+	data, err := io.ReadAll(object)
+	if err != nil {
+		return nil, err
+	}
+
+	return DecodeAuditRecord(data)
+}
+
+func (s3 *S3Backend) SearchAuditLog(
+	ctx context.Context, opts SearchAuditLogOptions,
+) iter.Seq2[AuditID, error] {
+	return func(yield func(AuditID, error) bool) {
+		logc.Printf(ctx, "s3: search audit\n")
+
+		ctx, cancel := context.WithCancel(ctx)
+		defer cancel()
+
+		prefix := "audit/"
+		for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{
+			Prefix: prefix,
+		}) {
+			var id AuditID
+			var err error
+			if object.Err != nil {
+				err = object.Err
+			} else {
+				id, err = ParseAuditID(strings.TrimPrefix(object.Key, prefix))
+			}
+			if !yield(id, err) {
+				break
+			}
+		}
+	}
+}

src/caddy.go

@@ -1,12 +1,11 @@
 package git_pages
 
 import (
+	"context"
 	"crypto/tls"
 	"fmt"
-	"log"
 	"net"
 	"net/http"
-	"net/url"
 	"strings"
 )
 
@@ -22,7 +21,7 @@ func ServeCaddy(w http.ResponseWriter, r *http.Request) {
 	// this isn't really what git-pages is designed for, and object store accesses can cost money.
 	// [^1]: https://letsencrypt.org/2025/07/01/issuing-our-first-ip-address-certificate
 	if ip := net.ParseIP(domain); ip != nil {
-		log.Println("caddy:", domain, 404, "(bare IP)")
+		logc.Println(r.Context(), "caddy:", domain, 404, "(bare IP)")
 		w.WriteHeader(http.StatusNotFound)
 		return
 	}
@@ -35,43 +34,47 @@ func ServeCaddy(w http.ResponseWriter, r *http.Request) {
 		// Pages v2, which would under some circumstances return certificates with subjectAltName
 		// not valid for the SNI. Go's TLS stack makes `tls.Dial` return an error for these,
 		// thankfully making it unnecessary to examine X.509 certificates manually here.)
-		for _, wildcardConfig := range config.Wildcard {
-			if wildcardConfig.FallbackProxyTo == "" {
-				continue
-			}
-			fallbackURL, err := url.Parse(wildcardConfig.FallbackProxyTo)
-			if err != nil {
-				continue
-			}
-			if fallbackURL.Scheme != "https" {
-				continue
-			}
-			connectHost := fallbackURL.Host
-			if fallbackURL.Port() != "" {
-				connectHost += ":" + fallbackURL.Port()
-			} else {
-				connectHost += ":443"
-			}
-			log.Printf("caddy: check TLS %s", fallbackURL)
-			connection, err := tls.Dial("tcp", connectHost, &tls.Config{ServerName: domain})
-			if err != nil {
-				continue
-			}
-			connection.Close()
-			found = true
-			break
+		found, err = tryDialWithSNI(r.Context(), domain)
+		if err != nil {
+			logc.Printf(r.Context(), "caddy err: check SNI: %s\n", err)
 		}
 	}
 
 	if found {
-		log.Println("caddy:", domain, 200)
+		logc.Println(r.Context(), "caddy:", domain, 200)
 		w.WriteHeader(http.StatusOK)
 	} else if err == nil {
-		log.Println("caddy:", domain, 404)
+		logc.Println(r.Context(), "caddy:", domain, 404)
 		w.WriteHeader(http.StatusNotFound)
 	} else {
-		log.Println("caddy:", domain, 500)
+		logc.Println(r.Context(), "caddy:", domain, 500)
 		w.WriteHeader(http.StatusInternalServerError)
 		fmt.Fprintln(w, err)
 	}
 }
+
+func tryDialWithSNI(ctx context.Context, domain string) (bool, error) {
+	if config.Fallback.ProxyTo == nil {
+		return false, nil
+	}
+
+	fallbackURL := config.Fallback.ProxyTo
+	if fallbackURL.Scheme != "https" {
+		return false, nil
+	}
+
+	connectHost := fallbackURL.Host
+	if fallbackURL.Port() != "" {
+		connectHost += ":" + fallbackURL.Port()
+	} else {
+		connectHost += ":443"
+	}
+
+	logc.Printf(ctx, "caddy: check TLS %s", fallbackURL)
+	connection, err := tls.Dial("tcp", connectHost, &tls.Config{ServerName: domain})
+	if err != nil {
+		return false, err
+	}
+	connection.Close()
+	return true, nil
+}

src/collect.go

@@ -5,7 +5,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"time"
 )
 
 type Flusher interface {
@@ -14,7 +13,7 @@ type Flusher interface {
 
 // Inverse of `ExtractTar`.
 func CollectTar(
-	context context.Context, writer io.Writer, manifest *Manifest, manifestMtime time.Time,
+	context context.Context, writer io.Writer, manifest *Manifest, metadata ManifestMetadata,
 ) (
 	err error,
 ) {
@@ -22,22 +21,25 @@ func CollectTar(
 
 	appendFile := func(header *tar.Header, data []byte, transform Transform) (err error) {
 		switch transform {
-		case Transform_None:
-		case Transform_Zstandard:
+		case Transform_Identity:
+		case Transform_Zstd:
 			data, err = zstdDecoder.DecodeAll(data, []byte{})
 			if err != nil {
-				return err
+				return fmt.Errorf("zstd: %s: %w", header.Name, err)
 			}
 		default:
-			return fmt.Errorf("unexpected transform")
+			return fmt.Errorf("%s: unexpected transform", header.Name)
 		}
 		header.Size = int64(len(data))
 
 		err = archive.WriteHeader(header)
 		if err != nil {
-			return
+			return fmt.Errorf("tar: %w", err)
 		}
 		_, err = archive.Write(data)
+		if err != nil {
+			return fmt.Errorf("tar: %w", err)
+		}
 		return
 	}
 
@@ -52,37 +54,41 @@ func CollectTar(
 		case Type_Directory:
 			header.Typeflag = tar.TypeDir
 			header.Mode = 0755
-			header.ModTime = manifestMtime
-			err = appendFile(&header, nil, Transform_None)
+			header.ModTime = metadata.LastModified
+			err = appendFile(&header, nil, Transform_Identity)
 
 		case Type_InlineFile:
 			header.Typeflag = tar.TypeReg
 			header.Mode = 0644
-			header.ModTime = manifestMtime
+			header.ModTime = metadata.LastModified
 			err = appendFile(&header, entry.GetData(), entry.GetTransform())
 
 		case Type_ExternalFile:
 			var blobReader io.Reader
-			var blobMtime time.Time
+			var blobMetadata BlobMetadata
 			var blobData []byte
-			blobReader, _, blobMtime, err = backend.GetBlob(context, string(entry.Data))
+			blobReader, blobMetadata, err = backend.GetBlob(context, string(entry.Data))
+			if err != nil {
+				return
+			}
+			blobData, err = io.ReadAll(blobReader)
 			if err != nil {
 				return
 			}
-			blobData, _ = io.ReadAll(blobReader)
 			header.Typeflag = tar.TypeReg
 			header.Mode = 0644
-			header.ModTime = blobMtime
+			header.ModTime = blobMetadata.LastModified
 			err = appendFile(&header, blobData, entry.GetTransform())
 
 		case Type_Symlink:
 			header.Typeflag = tar.TypeSymlink
 			header.Mode = 0644
-			header.ModTime = manifestMtime
-			err = appendFile(&header, entry.GetData(), Transform_None)
+			header.ModTime = metadata.LastModified
+			err = appendFile(&header, entry.GetData(), Transform_Identity)
 
 		default:
-			return fmt.Errorf("unexpected entry type")
+			panic(fmt.Errorf("CollectTar encountered invalid entry: %v, %v",
+				entry.GetType(), entry.GetTransform()))
 		}
 		if err != nil {
 			return err
@@ -94,8 +100,8 @@ func CollectTar(
 			Name:     RedirectsFileName,
 			Typeflag: tar.TypeReg,
 			Mode:     0644,
-			ModTime:  manifestMtime,
-		}, []byte(redirects), Transform_None)
+			ModTime:  metadata.LastModified,
+		}, []byte(redirects), Transform_Identity)
 		if err != nil {
 			return err
 		}
@@ -106,8 +112,8 @@ func CollectTar(
 			Name:     HeadersFileName,
 			Typeflag: tar.TypeReg,
 			Mode:     0644,
-			ModTime:  manifestMtime,
-		}, []byte(headers), Transform_None)
+			ModTime:  metadata.LastModified,
+		}, []byte(headers), Transform_Identity)
 		if err != nil {
 			return err
 		}
@@ -115,7 +121,7 @@ func CollectTar(
 
 	err = archive.Flush()
 	if err != nil {
-		return err
+		return fmt.Errorf("tar: %w", err)
 	}
 
 	flusher, ok := writer.(Flusher)

src/config.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"net/url"
 	"os"
 	"reflect"
 	"slices"
@@ -16,7 +17,7 @@ import (
 	"github.com/pelletier/go-toml/v2"
 )
 
-// For some reason, the standard `time.Duration` type doesn't implement the standard
+// For an unknown reason, the standard `time.Duration` type doesn't implement the standard
 // `encoding.{TextMarshaler,TextUnmarshaler}` interfaces.
 type Duration time.Duration
 
@@ -26,7 +27,9 @@ func (t Duration) String() string {
 
 func (t *Duration) UnmarshalText(data []byte) (err error) {
 	u, err := time.ParseDuration(string(data))
-	*t = Duration(u)
+	if err == nil {
+		*t = Duration(u)
+	}
 	return
 }
 
@@ -34,31 +37,58 @@ func (t *Duration) MarshalText() ([]byte, error) {
 	return []byte(t.String()), nil
 }
 
+// For a known but upsetting reason, the standard `url.URL` type doesn't implement the standard
+// `encoding.{TextMarshaler,TextUnmarshaler}` interfaces.
+type URL struct {
+	url.URL
+}
+
+func (t *URL) String() string {
+	return fmt.Sprint(&t.URL)
+}
+
+func (t *URL) UnmarshalText(data []byte) (err error) {
+	u, err := url.Parse(string(data))
+	if err == nil {
+		*t = URL{*u}
+	}
+	return
+}
+
+func (t *URL) MarshalText() ([]byte, error) {
+	return []byte(t.String()), nil
+}
+
 type Config struct {
 	Insecure      bool                `toml:"-" env:"insecure"`
 	Features      []string            `toml:"features"`
 	LogFormat     string              `toml:"log-format" default:"text"`
 	Server        ServerConfig        `toml:"server"`
 	Wildcard      []WildcardConfig    `toml:"wildcard"`
+	Fallback      FallbackConfig      `toml:"fallback"`
 	Storage       StorageConfig       `toml:"storage"`
 	Limits        LimitsConfig        `toml:"limits"`
+	Audit         AuditConfig         `toml:"audit"`
 	Observability ObservabilityConfig `toml:"observability"`
 }
 
 type ServerConfig struct {
-	Pages   string `toml:"pages" default:"tcp/:3000"`
-	Caddy   string `toml:"caddy" default:"tcp/:3001"`
-	Metrics string `toml:"metrics" default:"tcp/:3002"`
+	Pages   string `toml:"pages" default:"tcp/localhost:3000"`
+	Caddy   string `toml:"caddy" default:"tcp/localhost:3001"`
+	Metrics string `toml:"metrics" default:"tcp/localhost:3002"`
 }
 
 type WildcardConfig struct {
-	Domain           string   `toml:"domain"`
-	CloneURL         string   `toml:"clone-url"`
-	IndexRepos       []string `toml:"index-repos" default:"[]"`
-	IndexRepoBranch  string   `toml:"index-repo-branch" default:"pages"`
-	Authorization    string   `toml:"authorization"`
-	FallbackProxyTo  string   `toml:"fallback-proxy-to"`
-	FallbackInsecure bool     `toml:"fallback-insecure"`
+	Domain          string `toml:"domain"`
+	CloneURL        string `toml:"clone-url"` // URL template, not an exact URL
+	IndexRepo       string `toml:"index-repo" default:"pages"`
+	IndexRepoBranch string `toml:"index-repo-branch" default:"pages"`
+	Authorization   string `toml:"authorization"`
+}
+
+type FallbackConfig struct {
+	ProxyTo  *URL `toml:"proxy-to"`
+	Insecure bool `toml:"insecure"`
 }
 
 type CacheConfig struct {
@@ -109,20 +139,34 @@ type LimitsConfig struct {
 	// List of domains unconditionally forbidden for uploads.
 	ForbiddenDomains []string `toml:"forbidden-domains" default:"[]"`
 	// List of allowed repository URL prefixes. Setting this option prohibits uploading archives.
-	AllowedRepositoryURLPrefixes []string `toml:"allowed-repository-url-prefixes"`
+	AllowedRepositoryURLPrefixes []string `toml:"allowed-repository-url-prefixes" default:"[]"`
 	// List of allowed custom headers. Header name must be in the MIME canonical form,
 	// e.g. `Foo-Bar`. Setting this option permits including this custom header in `_headers`,
 	// unless it is fundamentally unsafe.
 	AllowedCustomHeaders []string `toml:"allowed-custom-headers" default:"[\"X-Clacks-Overhead\"]"`
 }
 
+type AuditConfig struct {
+	// Globally unique machine identifier (0 to 63 inclusive).
+	NodeID int `toml:"node-id"`
+	// Whether audit reports should be stored whenever an audit event occurs.
+	Collect bool `toml:"collect"`
+	// If not empty, includes the principal's IP address in audit reports, with the value specifying
+	// the source of the IP address. If the value is "X-Forwarded-For", the last item of the
+	// corresponding header field (assumed to be comma-separated) is used. If the value is
+	// "RemoteAddr", the connecting host's address is used. Any other value is disallowed.
+	IncludeIPs string `toml:"include-ip"`
+	// Endpoint to notify with a `GET /<notify-url>?<id>` whenever an audit event occurs.
+	NotifyURL *URL `toml:"notify-url"`
+}
+
 type ObservabilityConfig struct {
 	// Minimum duration for an HTTP request transaction to be unconditionally sampled.
 	SlowResponseThreshold Duration `toml:"slow-response-threshold" default:"500ms"`
 }
 
-func (config *Config) DebugJSON() string {
-	result, err := json.MarshalIndent(config, "", "  ")
+func (config *Config) TOML() string {
+	result, err := toml.Marshal(config)
 	if err != nil {
 		panic(err)
 	}
@@ -192,6 +236,11 @@ func setConfigValue(reflValue reflect.Value, repr string) (err error) {
 		if valueCast, err = strconv.ParseBool(repr); err == nil {
 			reflValue.SetBool(valueCast)
 		}
+	case int:
+		var parsed int64
+		if parsed, err = strconv.ParseInt(repr, 10, strconv.IntSize); err == nil {
+			reflValue.SetInt(parsed)
+		}
 	case uint:
 		var parsed uint64
 		if parsed, err = strconv.ParseUint(repr, 10, strconv.IntSize); err == nil {
@@ -205,15 +254,20 @@ func setConfigValue(reflValue reflect.Value, repr string) (err error) {
 		if valueCast, err = datasize.ParseString(repr); err == nil {
 			reflValue.Set(reflect.ValueOf(valueCast))
 		}
-	case time.Duration:
-		if valueCast, err = time.ParseDuration(repr); err == nil {
-			reflValue.Set(reflect.ValueOf(valueCast))
-		}
 	case Duration:
 		var parsed time.Duration
 		if parsed, err = time.ParseDuration(repr); err == nil {
 			reflValue.Set(reflect.ValueOf(Duration(parsed)))
 		}
+	case *URL:
+		if repr == "" {
+			reflValue.Set(reflect.ValueOf(nil))
+		} else {
+			var parsed *url.URL
+			if parsed, err = url.Parse(repr); err == nil {
+				reflValue.Set(reflect.ValueOf(&URL{*parsed}))
+			}
+		}
 	case []WildcardConfig:
 		var parsed []*WildcardConfig
 		decoder := json.NewDecoder(bytes.NewReader([]byte(repr)))

src/extract.go

@@ -5,32 +5,114 @@ import (
 	"archive/zip"
 	"bytes"
 	"compress/gzip"
+	"context"
 	"errors"
 	"fmt"
 	"io"
+	"math"
 	"os"
+	"path"
 	"strings"
 
 	"github.com/c2h5oh/datasize"
+	"github.com/go-git/go-git/v6/plumbing"
 	"github.com/klauspost/compress/zstd"
-	"google.golang.org/protobuf/proto"
 )
 
 var ErrArchiveTooLarge = errors.New("archive too large")
 
-func ExtractTar(reader io.Reader) (*Manifest, error) {
-	// If the tar stream is itself compressed, both the outer and the inner bounds checks
-	// are load-bearing.
-	boundedReader := ReadAtMost(reader, int64(config.Limits.MaxSiteSize.Bytes()),
+func boundArchiveStream(reader io.Reader) io.Reader {
+	return ReadAtMost(reader, int64(config.Limits.MaxSiteSize.Bytes()),
 		fmt.Errorf("%w: %s limit exceeded", ErrArchiveTooLarge, config.Limits.MaxSiteSize.HR()))
+}
 
-	archive := tar.NewReader(boundedReader)
-
-	manifest := Manifest{
-		Contents: map[string]*Entry{
-			"": {Type: Type_Directory.Enum()},
-		},
+func ExtractGzip(
+	ctx context.Context, reader io.Reader,
+	next func(context.Context, io.Reader) (*Manifest, error),
+) (*Manifest, error) {
+	stream, err := gzip.NewReader(reader)
+	if err != nil {
+		return nil, err
 	}
+	defer stream.Close()
+
+	return next(ctx, boundArchiveStream(stream))
+}
+
+func ExtractZstd(
+	ctx context.Context, reader io.Reader,
+	next func(context.Context, io.Reader) (*Manifest, error),
+) (*Manifest, error) {
+	stream, err := zstd.NewReader(reader)
+	if err != nil {
+		return nil, err
+	}
+	defer stream.Close()
+
+	return next(ctx, boundArchiveStream(stream))
+}
+
+const BlobReferencePrefix = "/git/blobs/"
+
+type UnresolvedRefError struct {
+	missing []string
+}
+
+func (err UnresolvedRefError) Error() string {
+	return fmt.Sprintf("%d unresolved blob references", len(err.missing))
+}
+
+func normalizeArchiveMemberName(fileName string) string {
+	// Strip the leading slash and any extraneous path segments.
+	fileName = path.Clean(fileName)
+	fileName = strings.TrimPrefix(fileName, "/")
+	if fileName == "." {
+		fileName = ""
+	}
+	return fileName
+}
+
+// Returns a map of git hash to entry. If `manifest` is nil, returns an empty map.
+func indexManifestByGitHash(manifest *Manifest) map[string]*Entry {
+	index := map[string]*Entry{}
+	for _, entry := range manifest.GetContents() {
+		if hash := entry.GetGitHash(); hash != "" {
+			if _, ok := plumbing.FromHex(hash); ok {
+				index[hash] = entry
+			} else {
+				panic(fmt.Errorf("index: malformed hash: %s", hash))
+			}
+		}
+	}
+	return index
+}
+
+func addSymlinkOrBlobReference(
+	manifest *Manifest, fileName string, target string,
+	index map[string]*Entry, missing *[]string,
+) *Entry {
+	if hash, found := strings.CutPrefix(target, BlobReferencePrefix); found {
+		if entry, found := index[hash]; found {
+			manifest.Contents[fileName] = entry
+			return entry
+		} else {
+			*missing = append(*missing, hash)
+			return nil
+		}
+	} else {
+		return AddSymlink(manifest, fileName, target)
+	}
+}
+
+func ExtractTar(ctx context.Context, reader io.Reader, oldManifest *Manifest) (*Manifest, error) {
+	archive := tar.NewReader(reader)
+
+	var dataBytesRecycled int64
+	var dataBytesTransferred int64
+
+	index := indexManifestByGitHash(oldManifest)
+	missing := []string{}
+	manifest := NewManifest()
 	for {
 		header, err := archive.Next()
 		if err == io.EOF {
@@ -39,70 +121,52 @@ func ExtractTar(reader io.Reader) (*Manifest, error) {
 			return nil, err
 		}
 
-		// For some reason, GNU tar includes any leading `.` path segments in archive filenames,
-		// unless there is a `..` path segment anywhere in the input filenames.
-		fileName := header.Name
-		for {
-			if strippedName, found := strings.CutPrefix(fileName, "./"); found {
-				fileName = strippedName
-			} else {
-				break
-			}
+		fileName := normalizeArchiveMemberName(header.Name)
+		if fileName == "" {
+			// This must be the root directory. It will be filled in by EnsureLeadingDirectories.
+			continue
 		}
 
-		manifestEntry := Entry{}
 		switch header.Typeflag {
 		case tar.TypeReg:
 			fileData, err := io.ReadAll(archive)
 			if err != nil {
 				return nil, fmt.Errorf("tar: %s: %w", fileName, err)
 			}
-
-			manifestEntry.Type = Type_InlineFile.Enum()
-			manifestEntry.Size = proto.Int64(header.Size)
-			manifestEntry.Data = fileData
-
+			AddFile(manifest, fileName, fileData)
+			dataBytesTransferred += int64(len(fileData))
 		case tar.TypeSymlink:
-			manifestEntry.Type = Type_Symlink.Enum()
-			manifestEntry.Size = proto.Int64(header.Size)
-			manifestEntry.Data = []byte(header.Linkname)
-
+			entry := addSymlinkOrBlobReference(
+				manifest, fileName, header.Linkname, index, &missing)
+			dataBytesRecycled += entry.GetOriginalSize()
 		case tar.TypeDir:
-			manifestEntry.Type = Type_Directory.Enum()
-			fileName = strings.TrimSuffix(fileName, "/")
-
+			AddDirectory(manifest, fileName)
 		default:
-			AddProblem(&manifest, fileName, "unsupported type '%c'", header.Typeflag)
+			AddProblem(manifest, fileName, "tar: unsupported type '%c'", header.Typeflag)
 			continue
 		}
-		manifest.Contents[fileName] = &manifestEntry
 	}
-	return &manifest, nil
+
+	if len(missing) > 0 {
+		return nil, UnresolvedRefError{missing}
+	}
+
+	// Ensure parent directories exist for all entries.
+	EnsureLeadingDirectories(manifest)
+
+	logc.Printf(ctx,
+		"reuse: %s recycled, %s transferred\n",
+		datasize.ByteSize(dataBytesRecycled).HR(),
+		datasize.ByteSize(dataBytesTransferred).HR(),
+	)
+
+	return manifest, nil
 }
 
-func ExtractTarGzip(reader io.Reader) (*Manifest, error) {
-	stream, err := gzip.NewReader(reader)
-	if err != nil {
-		return nil, err
-	}
-	defer stream.Close()
+// Used for zstd decompression inside zip files, it is recommended to share this.
+var zstdDecomp = zstd.ZipDecompressor()
 
-	// stream length is limited in `ExtractTar`
-	return ExtractTar(stream)
-}
-
-func ExtractTarZstd(reader io.Reader) (*Manifest, error) {
-	stream, err := zstd.NewReader(reader)
-	if err != nil {
-		return nil, err
-	}
-	defer stream.Close()
-
-	// stream length is limited in `ExtractTar`
-	return ExtractTar(stream)
-}
-
-func ExtractZip(reader io.Reader) (*Manifest, error) {
+func ExtractZip(ctx context.Context, reader io.Reader, oldManifest *Manifest) (*Manifest, error) {
 	data, err := io.ReadAll(reader)
 	if err != nil {
 		return nil, err
@@ -113,9 +177,18 @@ func ExtractZip(reader io.Reader) (*Manifest, error) {
 		return nil, err
 	}
 
+	// Support zstd compression inside zip files.
+	archive.RegisterDecompressor(zstd.ZipMethodWinZip, zstdDecomp)
+	archive.RegisterDecompressor(zstd.ZipMethodPKWare, zstdDecomp)
+
 	// Detect and defuse zipbombs.
 	var totalSize uint64
 	for _, file := range archive.File {
+		if totalSize+file.UncompressedSize64 < totalSize {
+			// Would overflow
+			totalSize = math.MaxUint64
+			break
+		}
 		totalSize += file.UncompressedSize64
 	}
 	if totalSize > config.Limits.MaxSiteSize.Bytes() {
@@ -126,14 +199,17 @@ func ExtractZip(reader io.Reader) (*Manifest, error) {
 		)
 	}
 
-	manifest := Manifest{
-		Contents: map[string]*Entry{
-			"": {Type: Type_Directory.Enum()},
-		},
-	}
+	var dataBytesRecycled int64
+	var dataBytesTransferred int64
+
+	index := indexManifestByGitHash(oldManifest)
+	missing := []string{}
+	manifest := NewManifest()
 	for _, file := range archive.File {
-		manifestEntry := Entry{}
-		if !strings.HasSuffix(file.Name, "/") {
+		normalizedName := normalizeArchiveMemberName(file.Name)
+		if strings.HasSuffix(file.Name, "/") {
+			AddDirectory(manifest, normalizedName)
+		} else {
 			fileReader, err := file.Open()
 			if err != nil {
 				return nil, err
@@ -146,16 +222,28 @@ func ExtractZip(reader io.Reader) (*Manifest, error) {
 			}
 
 			if file.Mode()&os.ModeSymlink != 0 {
-				manifestEntry.Type = Type_Symlink.Enum()
+				entry := addSymlinkOrBlobReference(
+					manifest, normalizedName, string(fileData), index, &missing)
+				dataBytesRecycled += entry.GetOriginalSize()
 			} else {
-				manifestEntry.Type = Type_InlineFile.Enum()
+				AddFile(manifest, normalizedName, fileData)
+				dataBytesTransferred += int64(len(fileData))
 			}
-			manifestEntry.Size = proto.Int64(int64(file.UncompressedSize64))
-			manifestEntry.Data = fileData
-		} else {
-			manifestEntry.Type = Type_Directory.Enum()
 		}
-		manifest.Contents[strings.TrimSuffix(file.Name, "/")] = &manifestEntry
 	}
-	return &manifest, nil
+
+	if len(missing) > 0 {
+		return nil, UnresolvedRefError{missing}
+	}
+
+	// Ensure parent directories exist for all entries.
+	EnsureLeadingDirectories(manifest)
+
+	logc.Printf(ctx,
+		"reuse: %s recycled, %s transferred\n",
+		datasize.ByteSize(dataBytesRecycled).HR(),
+		datasize.ByteSize(dataBytesTransferred).HR(),
+	)
+
+	return manifest, nil
 }

src/fetch.go

@@ -2,45 +2,77 @@ package git_pages
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
+	"maps"
+	"net/url"
 	"os"
+	"slices"
 
+	"github.com/c2h5oh/datasize"
 	"github.com/go-git/go-billy/v6/osfs"
 	"github.com/go-git/go-git/v6"
 	"github.com/go-git/go-git/v6/plumbing"
 	"github.com/go-git/go-git/v6/plumbing/cache"
 	"github.com/go-git/go-git/v6/plumbing/filemode"
 	"github.com/go-git/go-git/v6/plumbing/object"
+	"github.com/go-git/go-git/v6/plumbing/protocol/packp"
+	"github.com/go-git/go-git/v6/plumbing/transport"
 	"github.com/go-git/go-git/v6/storage/filesystem"
 	"google.golang.org/protobuf/proto"
 )
 
-func FetchRepository(ctx context.Context, repoURL string, branch string) (*Manifest, error) {
+var ErrRepositoryTooLarge = errors.New("repository too large")
+
+func FetchRepository(
+	ctx context.Context, repoURL string, branch string, oldManifest *Manifest,
+) (
+	*Manifest, error,
+) {
 	span, ctx := ObserveFunction(ctx, "FetchRepository",
 		"git.repository", repoURL, "git.branch", branch)
 	defer span.Finish()
 
-	baseDir, err := os.MkdirTemp("", "fetchRepo")
+	parsedRepoURL, err := url.Parse(repoURL)
 	if err != nil {
-		return nil, fmt.Errorf("mkdtemp: %w", err)
+		return nil, fmt.Errorf("URL parse: %w", err)
 	}
-	defer os.RemoveAll(baseDir)
 
-	fs := osfs.New(baseDir, osfs.WithBoundOS())
-	cache := cache.NewObjectLRUDefault()
-	storer := filesystem.NewStorageWithOptions(fs, cache, filesystem.Options{
-		ExclusiveAccess:      true,
-		LargeObjectThreshold: int64(config.Limits.GitLargeObjectThreshold.Bytes()),
-	})
-	repo, err := git.CloneContext(ctx, storer, nil, &git.CloneOptions{
-		Bare:          true,
-		URL:           repoURL,
-		ReferenceName: plumbing.ReferenceName(branch),
-		SingleBranch:  true,
-		Depth:         1,
-		Tags:          git.NoTags,
-	})
+	var repo *git.Repository
+	var storer *filesystem.Storage
+	for _, filter := range []packp.Filter{packp.FilterBlobNone(), packp.Filter("")} {
+		var tempDir string
+		if tempDir, err = os.MkdirTemp("", "fetchRepo"); err != nil {
+			return nil, fmt.Errorf("mkdtemp: %w", err)
+		}
+		defer os.RemoveAll(tempDir)
+
+		storer = filesystem.NewStorageWithOptions(
+			osfs.New(tempDir, osfs.WithBoundOS()),
+			cache.NewObjectLRUDefault(),
+			filesystem.Options{
+				ExclusiveAccess:      true,
+				LargeObjectThreshold: int64(config.Limits.GitLargeObjectThreshold.Bytes()),
+			},
+		)
+		repo, err = git.CloneContext(ctx, storer, nil, &git.CloneOptions{
+			Bare:          true,
+			URL:           repoURL,
+			ReferenceName: plumbing.NewBranchReferenceName(branch),
+			SingleBranch:  true,
+			Depth:         1,
+			Tags:          git.NoTags,
+			Filter:        filter,
+		})
+		if err != nil {
+			logc.Printf(ctx, "clone err: %s %s filter=%q\n", repoURL, branch, filter)
+			continue
+		} else {
+			logc.Printf(ctx, "clone ok: %s %s filter=%q\n", repoURL, branch, filter)
+			break
+		}
+	}
 	if err != nil {
 		return nil, fmt.Errorf("git clone: %w", err)
 	}
@@ -63,14 +95,13 @@ func FetchRepository(ctx context.Context, repoURL string, branch string) (*Manif
 	walker := object.NewTreeWalker(tree, true, make(map[plumbing.Hash]bool))
 	defer walker.Close()
 
-	manifest := Manifest{
-		RepoUrl: proto.String(repoURL),
-		Branch:  proto.String(branch),
-		Commit:  proto.String(ref.Hash().String()),
-		Contents: map[string]*Entry{
-			"": {Type: Type_Directory.Enum()},
-		},
-	}
+	// Create a manifest for the tree object corresponding to `branch`, but do not populate it
+	// with data yet; instead, record all the blobs we'll need.
+	manifest := NewManifest()
+	manifest.RepoUrl = proto.String(repoURL)
+	manifest.Branch = proto.String(branch)
+	manifest.Commit = proto.String(ref.Hash().String())
+	blobsNeeded := map[plumbing.Hash]*Entry{}
 	for {
 		name, entry, err := walker.Next()
 		if err == io.EOF {
@@ -78,39 +109,148 @@ func FetchRepository(ctx context.Context, repoURL string, branch string) (*Manif
 		} else if err != nil {
 			return nil, fmt.Errorf("git walker: %w", err)
 		} else {
-			manifestEntry := Entry{}
-			if entry.Mode.IsFile() {
-				blob, err := repo.BlobObject(entry.Hash)
-				if err != nil {
-					return nil, fmt.Errorf("git blob %s: %w", name, err)
-				}
-
-				reader, err := blob.Reader()
-				if err != nil {
-					return nil, fmt.Errorf("git blob open: %w", err)
-				}
-				defer reader.Close()
-
-				data, err := io.ReadAll(reader)
-				if err != nil {
-					return nil, fmt.Errorf("git blob read: %w", err)
-				}
-
+			manifestEntry := &Entry{}
+			if existingManifestEntry, found := blobsNeeded[entry.Hash]; found {
+				// If the same blob is present twice, we only need to fetch it once (and both
+				// instances will alias the same `Entry` structure in the manifest).
+				manifestEntry = existingManifestEntry
+			} else if entry.Mode.IsFile() {
+				blobsNeeded[entry.Hash] = manifestEntry
 				if entry.Mode == filemode.Symlink {
 					manifestEntry.Type = Type_Symlink.Enum()
 				} else {
 					manifestEntry.Type = Type_InlineFile.Enum()
 				}
-				manifestEntry.Size = proto.Int64(blob.Size)
-				manifestEntry.Data = data
+				manifestEntry.GitHash = proto.String(entry.Hash.String())
 			} else if entry.Mode == filemode.Dir {
 				manifestEntry.Type = Type_Directory.Enum()
 			} else {
-				AddProblem(&manifest, name, "unsupported mode %#o", entry.Mode)
+				AddProblem(manifest, name, "unsupported mode %#o", entry.Mode)
 				continue
 			}
-			manifest.Contents[name] = &manifestEntry
+			manifest.Contents[name] = manifestEntry
 		}
 	}
-	return &manifest, nil
+
+	// Collect checkout statistics.
+	var dataBytesRecycled int64
+	var dataBytesTransferred int64
+
+	// First, see if we can extract the blobs from the old manifest. This is the preferred option
+	// because it avoids both network transfers and recompression. Note that we do not request
+	// blobs from the backend under any circumstances to avoid creating a blob existence oracle.
+	for _, oldManifestEntry := range oldManifest.GetContents() {
+		if hash, ok := plumbing.FromHex(oldManifestEntry.GetGitHash()); ok {
+			if manifestEntry, found := blobsNeeded[hash]; found {
+				manifestEntry.Reset()
+				proto.Merge(manifestEntry, oldManifestEntry)
+				dataBytesRecycled += oldManifestEntry.GetOriginalSize()
+				delete(blobsNeeded, hash)
+			}
+		}
+	}
+
+	// Second, fill the manifest entries with data from the git checkout we just made.
+	// This will only succeed if a `blob:none` filter isn't supported and we got a full
+	// clone despite asking for a partial clone.
+	for hash, manifestEntry := range blobsNeeded {
+		if err := readGitBlob(repo, hash, manifestEntry, &dataBytesTransferred); err == nil {
+			delete(blobsNeeded, hash)
+		} else if errors.Is(err, ErrRepositoryTooLarge) {
+			return nil, err
+		}
+	}
+
+	// Third, if we still don't have data for some manifest entries, re-establish a git transport
+	// and request the missing blobs (only) from the server.
+	if len(blobsNeeded) > 0 {
+		client, err := transport.Get(parsedRepoURL.Scheme)
+		if err != nil {
+			return nil, fmt.Errorf("git transport: %w", err)
+		}
+
+		endpoint, err := transport.NewEndpoint(repoURL)
+		if err != nil {
+			return nil, fmt.Errorf("git endpoint: %w", err)
+		}
+
+		session, err := client.NewSession(storer, endpoint, nil)
+		if err != nil {
+			return nil, fmt.Errorf("git session: %w", err)
+		}
+
+		connection, err := session.Handshake(ctx, transport.UploadPackService)
+		if err != nil {
+			return nil, fmt.Errorf("git connection: %w", err)
+		}
+		defer connection.Close()
+
+		if err := connection.Fetch(ctx, &transport.FetchRequest{
+			Wants: slices.Collect(maps.Keys(blobsNeeded)),
+			Depth: 1,
+			// Git CLI behaves like this, even if the wants above are references to blobs.
+			Filter: "blob:none",
+		}); err != nil && !errors.Is(err, transport.ErrNoChange) {
+			return nil, fmt.Errorf("git blob fetch request: %w", err)
+		}
+
+		// All remaining blobs should now be available.
+		for hash, manifestEntry := range blobsNeeded {
+			if err := readGitBlob(repo, hash, manifestEntry, &dataBytesTransferred); err != nil {
+				return nil, err
+			}
+			delete(blobsNeeded, hash)
+		}
+	}
+
+	logc.Printf(ctx,
+		"reuse: %s recycled, %s transferred\n",
+		datasize.ByteSize(dataBytesRecycled).HR(),
+		datasize.ByteSize(dataBytesTransferred).HR(),
+	)
+
+	return manifest, nil
+}
+
+func readGitBlob(
+	repo *git.Repository, hash plumbing.Hash, entry *Entry, bytesTransferred *int64,
+) error {
+	blob, err := repo.BlobObject(hash)
+	if err != nil {
+		return fmt.Errorf("git blob %s: %w", hash, err)
+	}
+
+	reader, err := blob.Reader()
+	if err != nil {
+		return fmt.Errorf("git blob open: %w", err)
+	}
+	defer reader.Close()
+
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		return fmt.Errorf("git blob read: %w", err)
+	}
+
+	switch entry.GetType() {
+	case Type_InlineFile, Type_Symlink:
+		// okay
+	default:
+		panic(fmt.Errorf("readGitBlob encountered invalid entry: %v, %v",
+			entry.GetType(), entry.GetTransform()))
+	}
+
+	entry.Data = data
+	entry.Transform = Transform_Identity.Enum()
+	entry.OriginalSize = proto.Int64(blob.Size)
+	entry.CompressedSize = proto.Int64(blob.Size)
+
+	*bytesTransferred += blob.Size
+	if uint64(*bytesTransferred) > config.Limits.MaxSiteSize.Bytes() {
+		return fmt.Errorf("%w: fetch exceeds %s limit",
+			ErrRepositoryTooLarge,
+			config.Limits.MaxSiteSize.HR(),
+		)
+	}
+
+	return nil
 }

src/flock_other.go

@@ -0,0 +1,16 @@
+//go:build !unix
+
+package git_pages
+
+import (
+	"fmt"
+	"os"
+)
+
+func FileLock(file *os.File) error {
+	return fmt.Errorf("unimplemented")
+}
+
+func FileUnlock(file *os.File) error {
+	return fmt.Errorf("unimplemented")
+}

src/flock_posix.go

@@ -0,0 +1,16 @@
+//go:build unix
+
+package git_pages
+
+import (
+	"os"
+	"syscall"
+)
+
+func FileLock(file *os.File) error {
+	return syscall.Flock(int(file.Fd()), syscall.LOCK_EX)
+}
+
+func FileUnlock(file *os.File) error {
+	return syscall.Flock(int(file.Fd()), syscall.LOCK_UN)
+}

src/garbage.go

@@ -0,0 +1,88 @@
+package git_pages
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/c2h5oh/datasize"
+	"github.com/dghubble/trie"
+)
+
+func trieReduce(data trie.Trier) (items, total int64) {
+	data.Walk(func(key string, value any) error {
+		items += 1
+		total += *value.(*int64)
+		return nil
+	})
+	return
+}
+
+func TraceGarbage(ctx context.Context) error {
+	allBlobs := trie.NewRuneTrie()
+	liveBlobs := trie.NewRuneTrie()
+
+	traceManifest := func(manifestName string, manifest *Manifest) error {
+		for _, entry := range manifest.GetContents() {
+			if entry.GetType() == Type_ExternalFile {
+				blobName := string(entry.Data)
+				if size := allBlobs.Get(blobName); size == nil {
+					return fmt.Errorf("%s: dangling reference %s", manifestName, blobName)
+				} else {
+					liveBlobs.Put(blobName, size)
+				}
+			}
+		}
+		return nil
+	}
+
+	// Enumerate all blobs.
+	for metadata, err := range backend.EnumerateBlobs(ctx) {
+		if err != nil {
+			return fmt.Errorf("trace blobs err: %w", err)
+		}
+		allBlobs.Put(metadata.Name, &metadata.Size)
+	}
+
+	// Enumerate blobs live via site manifests.
+	for metadata, err := range backend.EnumerateManifests(ctx) {
+		if err != nil {
+			return fmt.Errorf("trace sites err: %w", err)
+		}
+		manifest, _, err := backend.GetManifest(ctx, metadata.Name, GetManifestOptions{})
+		if err != nil {
+			return fmt.Errorf("trace sites err: %w", err)
+		}
+		err = traceManifest(metadata.Name, manifest)
+		if err != nil {
+			return fmt.Errorf("trace sites err: %w", err)
+		}
+	}
+
+	// Enumerate blobs live via audit records.
+	for auditID, err := range backend.SearchAuditLog(ctx, SearchAuditLogOptions{}) {
+		if err != nil {
+			return fmt.Errorf("trace audit err: %w", err)
+		}
+		auditRecord, err := backend.QueryAuditLog(ctx, auditID)
+		if err != nil {
+			return fmt.Errorf("trace audit err: %w", err)
+		}
+		if auditRecord.Manifest != nil {
+			err = traceManifest(auditID.String(), auditRecord.Manifest)
+			if err != nil {
+				return fmt.Errorf("trace audit err: %w", err)
+			}
+		}
+	}
+
+	allBlobsCount, allBlobsSize := trieReduce(allBlobs)
+	liveBlobsCount, liveBlobsSize := trieReduce(liveBlobs)
+	logc.Printf(ctx, "trace all: %d blobs, %s",
+		allBlobsCount, datasize.ByteSize(allBlobsSize).HR())
+	logc.Printf(ctx, "trace live: %d blobs, %s",
+		liveBlobsCount, datasize.ByteSize(liveBlobsSize).HR())
+	logc.Printf(ctx, "trace dead: %d blobs, %s",
+		allBlobsCount-liveBlobsCount, datasize.ByteSize(allBlobsSize-liveBlobsSize).HR())
+
+	return nil
+}

src/http.go

@@ -2,43 +2,96 @@ package git_pages
 
 import (
 	"cmp"
+	"fmt"
+	"net"
+	"net/http"
 	"regexp"
 	"slices"
 	"strconv"
 	"strings"
 )
 
-var httpAcceptEncodingRegexp = regexp.MustCompile(`` +
+var httpAcceptRegexp = regexp.MustCompile(`` +
 	// token optionally prefixed by whitespace
-	`^[ \t]*([a-zA-Z0-9$!#$%&'*+.^_\x60|~-]+)` +
+	`^[ \t]*([a-zA-Z0-9$!#$%&'*+./^_\x60|~-]+)` +
 	// quality value prefixed by a semicolon optionally surrounded by whitespace
 	`(?:[ \t]*;[ \t]*q=(0(?:\.[0-9]{1,3})?|1(?:\.0{1,3})?))?` +
 	// optional whitespace followed by comma or end of line
 	`[ \t]*(?:,|$)`,
 )
 
-type httpEncoding struct {
+type httpAcceptOffer struct {
 	code string
 	qval float64
 }
 
-type httpEncodings struct {
-	encodings []httpEncoding
-}
-
-func parseHTTPEncodings(headerValue string) (result httpEncodings) {
+func parseGenericAcceptHeader(headerValue string) (result []httpAcceptOffer) {
 	for headerValue != "" {
-		matches := httpAcceptEncodingRegexp.FindStringSubmatch(headerValue)
+		matches := httpAcceptRegexp.FindStringSubmatch(headerValue)
 		if matches == nil {
-			return httpEncodings{}
+			return
 		}
-		enc := httpEncoding{strings.ToLower(matches[1]), 1.0}
+		offer := httpAcceptOffer{strings.ToLower(matches[1]), 1.0}
 		if matches[2] != "" {
-			enc.qval, _ = strconv.ParseFloat(matches[2], 64)
+			offer.qval, _ = strconv.ParseFloat(matches[2], 64)
 		}
-		result.encodings = append(result.encodings, enc)
+		result = append(result, offer)
 		headerValue = headerValue[len(matches[0]):]
 	}
+	return
+}
+
+func preferredAcceptOffer(offers []httpAcceptOffer) string {
+	slices.SortStableFunc(offers, func(a, b httpAcceptOffer) int {
+		return -cmp.Compare(a.qval, b.qval)
+	})
+	for _, offer := range offers {
+		if offer.qval != 0 {
+			return offer.code
+		}
+	}
+	return ""
+}
+
+type HTTPContentTypes struct {
+	contentTypes []httpAcceptOffer
+}
+
+func ParseAcceptHeader(headerValue string) (result HTTPContentTypes) {
+	if headerValue == "" {
+		headerValue = "*/*"
+	}
+	result = HTTPContentTypes{parseGenericAcceptHeader(headerValue)}
+	return
+}
+
+func (e *HTTPContentTypes) Negotiate(offers ...string) string {
+	prefs := make(map[string]float64, len(offers))
+	for _, code := range offers {
+		prefs[code] = 0
+	}
+	for _, ctyp := range e.contentTypes {
+		if ctyp.code == "*/*" {
+			for code := range prefs {
+				prefs[code] = ctyp.qval
+			}
+		} else if _, ok := prefs[ctyp.code]; ok {
+			prefs[ctyp.code] = ctyp.qval
+		}
+	}
+	ctyps := make([]httpAcceptOffer, len(offers))
+	for idx, code := range offers {
+		ctyps[idx] = httpAcceptOffer{code, prefs[code]}
+	}
+	return preferredAcceptOffer(ctyps)
+}
+
+type HTTPEncodings struct {
+	encodings []httpAcceptOffer
+}
+
+func ParseAcceptEncodingHeader(headerValue string) (result HTTPEncodings) {
+	result = HTTPEncodings{parseGenericAcceptHeader(headerValue)}
 	if len(result.encodings) == 0 {
 		// RFC 9110 says (https://httpwg.org/specs/rfc9110.html#field.accept-encoding):
 		// "If no Accept-Encoding header field is in the request, any content
@@ -51,9 +104,9 @@ func parseHTTPEncodings(headerValue string) (result httpEncodings) {
 
 // Negotiate returns the most preferred encoding that is acceptable by the
 // client, or an empty string if no encodings are acceptable.
-func (e *httpEncodings) Negotiate(codes ...string) string {
-	prefs := make(map[string]float64, len(codes))
-	for _, code := range codes {
+func (e *HTTPEncodings) Negotiate(offers ...string) string {
+	prefs := make(map[string]float64, len(offers))
+	for _, code := range offers {
 		prefs[code] = 0
 	}
 	implicitIdentity := true
@@ -73,17 +126,52 @@ func (e *httpEncodings) Negotiate(codes ...string) string {
 	if _, ok := prefs["identity"]; ok && implicitIdentity {
 		prefs["identity"] = -1 // sort last
 	}
-	encs := make([]httpEncoding, len(codes))
-	for idx, code := range codes {
-		encs[idx] = httpEncoding{code, prefs[code]}
+	encs := make([]httpAcceptOffer, len(offers))
+	for idx, code := range offers {
+		encs[idx] = httpAcceptOffer{code, prefs[code]}
 	}
-	slices.SortStableFunc(encs, func(a, b httpEncoding) int {
-		return -cmp.Compare(a.qval, b.qval)
-	})
-	for _, enc := range encs {
-		if enc.qval != 0 {
-			return enc.code
-		}
-	}
-	return ""
+	return preferredAcceptOffer(encs)
+}
+
+func chainHTTPMiddleware(middleware ...func(http.Handler) http.Handler) func(http.Handler) http.Handler {
+	return func(handler http.Handler) http.Handler {
+		for idx := len(middleware) - 1; idx >= 0; idx-- {
+			handler = middleware[idx](handler)
+		}
+		return handler
+	}
+}
+
+func remoteAddrMiddleware(handler http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var readXForwardedFor bool
+		switch config.Audit.IncludeIPs {
+		case "X-Forwarded-For":
+			readXForwardedFor = true
+		case "RemoteAddr", "":
+			readXForwardedFor = false
+		default:
+			panic(fmt.Errorf("config.Audit.IncludeIPs is set to an unknown value (%q)",
+				config.Audit.IncludeIPs))
+		}
+
+		usingOriginalRemoteAddr := true
+		if readXForwardedFor {
+			forwardedFor := strings.Split(r.Header.Get("X-Forwarded-For"), ",")
+			if len(forwardedFor) > 0 {
+				remoteAddr := strings.TrimSpace(forwardedFor[len(forwardedFor)-1])
+				if remoteAddr != "" {
+					r.RemoteAddr = remoteAddr
+					usingOriginalRemoteAddr = false
+				}
+			}
+		}
+		if usingOriginalRemoteAddr {
+			if ipAddress, _, err := net.SplitHostPort(r.RemoteAddr); err == nil {
+				r.RemoteAddr = ipAddress
+			}
+		}
+
+		handler.ServeHTTP(w, r)
+	})
 }

src/log.go

@@ -0,0 +1,54 @@
+package git_pages
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+	"runtime"
+	"strings"
+	"time"
+)
+
+var logc slogWithCtx
+
+type slogWithCtx struct{}
+
+func (l slogWithCtx) log(ctx context.Context, level slog.Level, msg string) {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+	logger := slog.Default()
+	if !logger.Enabled(ctx, level) {
+		return
+	}
+
+	var pcs [1]uintptr
+	// skip [runtime.Callers, this method, method calling this method]
+	runtime.Callers(3, pcs[:])
+
+	record := slog.NewRecord(time.Now(), level, strings.TrimRight(msg, "\n"), pcs[0])
+	logger.Handler().Handle(ctx, record)
+}
+
+func (l slogWithCtx) Print(ctx context.Context, v ...any) {
+	l.log(ctx, slog.LevelInfo, fmt.Sprint(v...))
+}
+
+func (l slogWithCtx) Printf(ctx context.Context, format string, v ...any) {
+	l.log(ctx, slog.LevelInfo, fmt.Sprintf(format, v...))
+}
+
+func (l slogWithCtx) Println(ctx context.Context, v ...any) {
+	l.log(ctx, slog.LevelInfo, fmt.Sprintln(v...))
+}
+
+func (l slogWithCtx) Fatalf(ctx context.Context, format string, v ...any) {
+	l.log(ctx, slog.LevelError, fmt.Sprintf(format, v...))
+	os.Exit(1)
+}
+
+func (l slogWithCtx) Fatalln(ctx context.Context, v ...any) {
+	l.log(ctx, slog.LevelError, fmt.Sprintln(v...))
+	os.Exit(1)
+}

src/main.go

@@ -2,6 +2,7 @@ package git_pages
 
 import (
 	"context"
+	"crypto/tls"
 	"errors"
 	"flag"
 	"fmt"
@@ -10,28 +11,35 @@ import (
 	"log/slog"
 	"net"
 	"net/http"
+	"net/http/httputil"
 	"net/url"
 	"os"
+	"path"
 	"runtime/debug"
 	"strings"
+	"time"
 
 	automemlimit "github.com/KimMachineGun/automemlimit/memlimit"
 	"github.com/c2h5oh/datasize"
+	"github.com/fatih/color"
+	"github.com/kankanreno/go-snowflake"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"google.golang.org/protobuf/proto"
 )
 
 var config *Config
 var wildcards []*WildcardPattern
+var fallback http.Handler
 var backend Backend
 
-func configureFeatures() (err error) {
+func configureFeatures(ctx context.Context) (err error) {
 	if len(config.Features) > 0 {
-		log.Println("features:", strings.Join(config.Features, ", "))
+		logc.Println(ctx, "features:", strings.Join(config.Features, ", "))
 	}
 	return
 }
 
-func configureMemLimit() (err error) {
+func configureMemLimit(ctx context.Context) (err error) {
 	// Avoid being OOM killed by not garbage collecting early enough.
 	memlimitBefore := datasize.ByteSize(debug.SetMemoryLimit(-1))
 	automemlimit.SetGoMemLimitWithOpts(
@@ -46,14 +54,14 @@ func configureMemLimit() (err error) {
 	)
 	memlimitAfter := datasize.ByteSize(debug.SetMemoryLimit(-1))
 	if memlimitBefore == memlimitAfter {
-		log.Println("memlimit: now", memlimitBefore.HR())
+		logc.Println(ctx, "memlimit: now", memlimitBefore.HR())
 	} else {
-		log.Println("memlimit: was", memlimitBefore.HR(), "now", memlimitAfter.HR())
+		logc.Println(ctx, "memlimit: was", memlimitBefore.HR(), "now", memlimitAfter.HR())
 	}
 	return
 }
 
-func configureWildcards() (err error) {
+func configureWildcards(_ context.Context) (err error) {
 	newWildcards, err := TranslateWildcards(config.Wildcard)
 	if err != nil {
 		return err
@@ -63,19 +71,45 @@ func configureWildcards() (err error) {
 	}
 }
 
-func listen(name string, listen string) net.Listener {
+func configureFallback(_ context.Context) (err error) {
+	if config.Fallback.ProxyTo != nil {
+		fallbackURL := &config.Fallback.ProxyTo.URL
+		fallback = &httputil.ReverseProxy{
+			Rewrite: func(r *httputil.ProxyRequest) {
+				r.SetURL(fallbackURL)
+				r.Out.Host = r.In.Host
+				r.Out.Header["X-Forwarded-For"] = r.In.Header["X-Forwarded-For"]
+			},
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{
+					InsecureSkipVerify: config.Fallback.Insecure,
+				},
+			},
+		}
+	}
+	return
+}
+
+// Thread-unsafe, must be called only during initial configuration.
+func configureAudit(_ context.Context) (err error) {
+	snowflake.SetStartTime(time.Date(2025, 12, 1, 0, 0, 0, 0, time.UTC))
+	snowflake.SetMachineID(config.Audit.NodeID)
+	return
+}
+
+func listen(ctx context.Context, name string, listen string) net.Listener {
 	if listen == "-" {
 		return nil
 	}
 
 	protocol, address, ok := strings.Cut(listen, "/")
 	if !ok {
-		log.Fatalf("%s: %s: malformed endpoint", name, listen)
+		logc.Fatalf(ctx, "%s: %s: malformed endpoint", name, listen)
 	}
 
 	listener, err := net.Listen(protocol, address)
 	if err != nil {
-		log.Fatalf("%s: %s\n", name, err)
+		logc.Fatalf(ctx, "%s: %s\n", name, err)
 	}
 
 	return listener
@@ -85,7 +119,7 @@ func panicHandler(handler http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		defer func() {
 			if err := recover(); err != nil {
-				log.Printf("panic: %s %s %s: %s\n%s",
+				logc.Printf(r.Context(), "panic: %s %s %s: %s\n%s",
 					r.Method, r.Host, r.URL.Path, err, string(debug.Stack()))
 				http.Error(w,
 					fmt.Sprintf("internal server error: %s", err),
@@ -97,17 +131,13 @@ func panicHandler(handler http.Handler) http.Handler {
 	})
 }
 
-func serve(listener net.Listener, handler http.Handler) {
+func serve(ctx context.Context, listener net.Listener, handler http.Handler) {
 	if listener != nil {
-		handler = panicHandler(handler)
-
 		server := http.Server{Handler: handler}
 		server.Protocols = new(http.Protocols)
 		server.Protocols.SetHTTP1(true)
-		if config.Feature("serve-h2c") {
-			server.Protocols.SetUnencryptedHTTP2(true)
-		}
-		log.Fatalln(server.Serve(listener))
+		server.Protocols.SetUnencryptedHTTP2(true)
+		logc.Fatalln(ctx, server.Serve(listener))
 	}
 }
 
@@ -118,7 +148,8 @@ func webRootArg(arg string) string {
 	case 1:
 		return arg
 	default:
-		log.Fatalf("webroot argument must be either 'domain.tld' or 'domain.tld/dir")
+		logc.Fatalln(context.Background(),
+			"webroot argument must be either 'domain.tld' or 'domain.tld/dir")
 		return ""
 	}
 }
@@ -130,7 +161,7 @@ func fileOutputArg() (writer io.WriteCloser) {
 	} else {
 		writer, err = os.Create(flag.Arg(0))
 		if err != nil {
-			log.Fatalln(err)
+			logc.Fatalln(context.Background(), err)
 		}
 	}
 	return
@@ -140,27 +171,37 @@ func usage() {
 	fmt.Fprintf(os.Stderr, "Usage:\n")
 	fmt.Fprintf(os.Stderr, "(server) "+
 		"git-pages [-config <file>|-no-config]\n")
-	fmt.Fprintf(os.Stderr, "(admin)  "+
-		"git-pages {-run-migration <name>}\n")
 	fmt.Fprintf(os.Stderr, "(info)   "+
 		"git-pages {-print-config-env-vars|-print-config}\n")
-	fmt.Fprintf(os.Stderr, "(cli)    "+
+	fmt.Fprintf(os.Stderr, "(debug)  "+
+		"git-pages {-list-blobs|-list-manifests}\n")
+	fmt.Fprintf(os.Stderr, "(debug)  "+
 		"git-pages {-get-blob|-get-manifest|-get-archive|-update-site} <ref> [file]\n")
+	fmt.Fprintf(os.Stderr, "(admin)  "+
+		"git-pages {-freeze-domain <domain>|-unfreeze-domain <domain>}\n")
+	fmt.Fprintf(os.Stderr, "(audit)  "+
+		"git-pages {-audit-log|-audit-read <id>|-audit-server <endpoint> <program> [args...]}\n")
+	fmt.Fprintf(os.Stderr, "(maint)  "+
+		"git-pages {-run-migration <name>|-trace-garbage}\n")
 	flag.PrintDefaults()
 }
 
 func Main() {
+	ctx := context.Background()
+
 	flag.Usage = usage
-	printConfigEnvVars := flag.Bool("print-config-env-vars", false,
-		"print every recognized configuration environment variable and exit")
-	printConfig := flag.Bool("print-config", false,
-		"print configuration as JSON and exit")
 	configTomlPath := flag.String("config", "",
 		"load configuration from `filename` (default: 'config.toml')")
 	noConfig := flag.Bool("no-config", false,
 		"run without configuration file (configure via environment variables)")
-	runMigration := flag.String("run-migration", "",
-		"run a store `migration` (one of: create-domain-markers)")
+	printConfigEnvVars := flag.Bool("print-config-env-vars", false,
+		"print every recognized configuration environment variable and exit")
+	printConfig := flag.Bool("print-config", false,
+		"print configuration as JSON and exit")
+	listBlobs := flag.Bool("list-blobs", false,
+		"enumerate every blob with its metadata")
+	listManifests := flag.Bool("list-manifests", false,
+		"enumerate every manifest with its metadata")
 	getBlob := flag.String("get-blob", "",
 		"write contents of `blob` ('sha256-xxxxxxx...xxx')")
 	getManifest := flag.String("get-manifest", "",
@@ -169,24 +210,54 @@ func Main() {
 		"write archive for `site` (either 'domain.tld' or 'domain.tld/dir') in tar format")
 	updateSite := flag.String("update-site", "",
 		"update `site` (either 'domain.tld' or 'domain.tld/dir') from archive or repository URL")
+	freezeDomain := flag.String("freeze-domain", "",
+		"prevent any site uploads to a given `domain`")
+	unfreezeDomain := flag.String("unfreeze-domain", "",
+		"allow site uploads to a `domain` again after it has been frozen")
+	auditLog := flag.Bool("audit-log", false,
+		"display audit log")
+	auditRead := flag.String("audit-read", "",
+		"extract contents of audit record `id` to files '<id>-*'")
+	auditRollback := flag.String("audit-rollback", "",
+		"restore site from contents of audit record `id`")
+	auditServer := flag.String("audit-server", "",
+		"listen for notifications on `endpoint` and spawn a process for each audit event")
+	runMigration := flag.String("run-migration", "",
+		"run a store `migration` (one of: create-domain-markers)")
+	traceGarbage := flag.Bool("trace-garbage", false,
+		"estimate total size of unreachable blobs")
 	flag.Parse()
 
 	var cliOperations int
-	if *getBlob != "" {
-		cliOperations += 1
-	}
-	if *getManifest != "" {
-		cliOperations += 1
-	}
-	if *getArchive != "" {
-		cliOperations += 1
+	for _, selected := range []bool{
+		*listBlobs,
+		*listManifests,
+		*getBlob != "",
+		*getManifest != "",
+		*getArchive != "",
+		*updateSite != "",
+		*freezeDomain != "",
+		*unfreezeDomain != "",
+		*auditLog,
+		*auditRead != "",
+		*auditRollback != "",
+		*auditServer != "",
+		*runMigration != "",
+		*traceGarbage,
+	} {
+		if selected {
+			cliOperations++
+		}
 	}
 	if cliOperations > 1 {
-		log.Fatalln("-get-blob, -get-manifest, and -get-archive are mutually exclusive")
+		logc.Fatalln(ctx, "-list-blobs, -list-manifests, -get-blob, -get-manifest, -get-archive, "+
+			"-update-site, -freeze-domain, -unfreeze-domain, -audit-log, -audit-read, "+
+			"-audit-rollback, -audit-server, -run-migration, and -trace-garbage are "+
+			"mutually exclusive")
 	}
 
 	if *configTomlPath != "" && *noConfig {
-		log.Fatalln("-no-config and -config are mutually exclusive")
+		logc.Fatalln(ctx, "-no-config and -config are mutually exclusive")
 	}
 
 	if *printConfigEnvVars {
@@ -199,11 +270,11 @@ func Main() {
 		*configTomlPath = "config.toml"
 	}
 	if config, err = Configure(*configTomlPath); err != nil {
-		log.Fatalln("config:", err)
+		logc.Fatalln(ctx, "config:", err)
 	}
 
 	if *printConfig {
-		fmt.Println(config.DebugJSON())
+		fmt.Println(config.TOML())
 		return
 	}
 
@@ -211,78 +282,91 @@ func Main() {
 	defer FiniObservability()
 
 	if err = errors.Join(
-		configureFeatures(),
-		configureMemLimit(),
-		configureWildcards(),
+		configureFeatures(ctx),
+		configureMemLimit(ctx),
+		configureWildcards(ctx),
+		configureFallback(ctx),
+		configureAudit(ctx),
 	); err != nil {
-		log.Fatalln(err)
+		logc.Fatalln(ctx, err)
+	}
+
+	// The server has its own logic for creating the backend.
+	if cliOperations > 0 {
+		if backend, err = CreateBackend(ctx, &config.Storage); err != nil {
+			logc.Fatalln(ctx, err)
+		}
 	}
 
 	switch {
-	case *runMigration != "":
-		if backend, err = CreateBackend(&config.Storage); err != nil {
-			log.Fatalln(err)
+	case *listBlobs:
+		for metadata, err := range backend.EnumerateBlobs(ctx) {
+			if err != nil {
+				logc.Fatalln(ctx, err)
+			}
+			fmt.Fprintf(color.Output, "%s %s %s\n",
+				metadata.Name,
+				color.HiWhiteString(metadata.LastModified.UTC().Format(time.RFC3339)),
+				color.HiGreenString(fmt.Sprint(metadata.Size)),
+			)
 		}
 
-		if err := RunMigration(context.Background(), *runMigration); err != nil {
-			log.Fatalln(err)
+	case *listManifests:
+		for metadata, err := range backend.EnumerateManifests(ctx) {
+			if err != nil {
+				logc.Fatalln(ctx, err)
+			}
+			fmt.Fprintf(color.Output, "%s %s %s\n",
+				metadata.Name,
+				color.HiWhiteString(metadata.LastModified.UTC().Format(time.RFC3339)),
+				color.HiGreenString(fmt.Sprint(metadata.Size)),
+			)
 		}
 
 	case *getBlob != "":
-		if backend, err = CreateBackend(&config.Storage); err != nil {
-			log.Fatalln(err)
-		}
-
-		reader, _, _, err := backend.GetBlob(context.Background(), *getBlob)
+		reader, _, err := backend.GetBlob(ctx, *getBlob)
 		if err != nil {
-			log.Fatalln(err)
+			logc.Fatalln(ctx, err)
 		}
 		io.Copy(fileOutputArg(), reader)
 
 	case *getManifest != "":
-		if backend, err = CreateBackend(&config.Storage); err != nil {
-			log.Fatalln(err)
-		}
-
 		webRoot := webRootArg(*getManifest)
-		manifest, _, err := backend.GetManifest(context.Background(), webRoot, GetManifestOptions{})
+		manifest, _, err := backend.GetManifest(ctx, webRoot, GetManifestOptions{})
 		if err != nil {
-			log.Fatalln(err)
+			logc.Fatalln(ctx, err)
 		}
-		fmt.Fprintln(fileOutputArg(), ManifestDebugJSON(manifest))
+		fmt.Fprintln(fileOutputArg(), string(ManifestJSON(manifest)))
 
 	case *getArchive != "":
-		if backend, err = CreateBackend(&config.Storage); err != nil {
-			log.Fatalln(err)
-		}
-
 		webRoot := webRootArg(*getArchive)
-		manifest, manifestMtime, err :=
-			backend.GetManifest(context.Background(), webRoot, GetManifestOptions{})
+		manifest, metadata, err :=
+			backend.GetManifest(ctx, webRoot, GetManifestOptions{})
 		if err != nil {
-			log.Fatalln(err)
+			logc.Fatalln(ctx, err)
+		}
+		if err = CollectTar(ctx, fileOutputArg(), manifest, metadata); err != nil {
+			logc.Fatalln(ctx, err)
 		}
-		CollectTar(context.Background(), fileOutputArg(), manifest, manifestMtime)
 
 	case *updateSite != "":
-		if backend, err = CreateBackend(&config.Storage); err != nil {
-			log.Fatalln(err)
-		}
+		ctx = WithPrincipal(ctx)
+		GetPrincipal(ctx).CliAdmin = proto.Bool(true)
 
 		if flag.NArg() != 1 {
-			log.Fatalln("update source must be provided as the argument")
+			logc.Fatalln(ctx, "update source must be provided as the argument")
 		}
 
 		sourceURL, err := url.Parse(flag.Arg(0))
 		if err != nil {
-			log.Fatalln(err)
+			logc.Fatalln(ctx, err)
 		}
 
 		var result UpdateResult
 		if sourceURL.Scheme == "" {
 			file, err := os.Open(sourceURL.Path)
 			if err != nil {
-				log.Fatalln(err)
+				logc.Fatalln(ctx, err)
 			}
 			defer file.Close()
 
@@ -301,7 +385,7 @@ func Main() {
 			}
 
 			webRoot := webRootArg(*updateSite)
-			result = UpdateFromArchive(context.Background(), webRoot, contentType, file)
+			result = UpdateFromArchive(ctx, webRoot, contentType, file)
 		} else {
 			branch := "pages"
 			if sourceURL.Fragment != "" {
@@ -309,24 +393,149 @@ func Main() {
 			}
 
 			webRoot := webRootArg(*updateSite)
-			result = UpdateFromRepository(context.Background(), webRoot, sourceURL.String(), branch)
+			result = UpdateFromRepository(ctx, webRoot, sourceURL.String(), branch)
 		}
 
 		switch result.outcome {
 		case UpdateError:
-			log.Printf("error: %s\n", result.err)
+			logc.Printf(ctx, "error: %s\n", result.err)
 			os.Exit(2)
 		case UpdateTimeout:
-			log.Println("timeout")
+			logc.Println(ctx, "timeout")
 			os.Exit(1)
 		case UpdateCreated:
-			log.Println("created")
+			logc.Println(ctx, "created")
 		case UpdateReplaced:
-			log.Println("replaced")
+			logc.Println(ctx, "replaced")
 		case UpdateDeleted:
-			log.Println("deleted")
+			logc.Println(ctx, "deleted")
 		case UpdateNoChange:
-			log.Println("no-change")
+			logc.Println(ctx, "no-change")
+		}
+
+	case *freezeDomain != "" || *unfreezeDomain != "":
+		ctx = WithPrincipal(ctx)
+		GetPrincipal(ctx).CliAdmin = proto.Bool(true)
+
+		var domain string
+		var freeze bool
+		if *freezeDomain != "" {
+			domain = *freezeDomain
+			freeze = true
+		} else {
+			domain = *unfreezeDomain
+			freeze = false
+		}
+
+		if freeze {
+			if err = backend.FreezeDomain(ctx, domain); err != nil {
+				logc.Fatalln(ctx, err)
+			}
+			logc.Println(ctx, "frozen")
+		} else {
+			if err = backend.UnfreezeDomain(ctx, domain); err != nil {
+				logc.Fatalln(ctx, err)
+			}
+			logc.Println(ctx, "thawed")
+		}
+
+	case *auditLog:
+		ch := make(chan *AuditRecord)
+		ids := []AuditID{}
+		for id, err := range backend.SearchAuditLog(ctx, SearchAuditLogOptions{}) {
+			if err != nil {
+				logc.Fatalln(ctx, err)
+			}
+			go func() {
+				if record, err := backend.QueryAuditLog(ctx, id); err != nil {
+					logc.Fatalln(ctx, err)
+				} else {
+					ch <- record
+				}
+			}()
+			ids = append(ids, id)
+		}
+
+		records := map[AuditID]*AuditRecord{}
+		for len(records) < len(ids) {
+			record := <-ch
+			records[record.GetAuditID()] = record
+		}
+
+		for _, id := range ids {
+			record := records[id]
+			fmt.Fprintf(color.Output, "%s %s %s %s %s\n",
+				record.GetAuditID().String(),
+				color.HiWhiteString(record.GetTimestamp().AsTime().UTC().Format(time.RFC3339)),
+				color.HiMagentaString(record.DescribePrincipal()),
+				color.HiGreenString(record.DescribeResource()),
+				record.GetEvent(),
+			)
+		}
+
+	case *auditRead != "":
+		id, err := ParseAuditID(*auditRead)
+		if err != nil {
+			logc.Fatalln(ctx, err)
+		}
+
+		record, err := backend.QueryAuditLog(ctx, id)
+		if err != nil {
+			logc.Fatalln(ctx, err)
+		}
+
+		if err = ExtractAuditRecord(ctx, id, record, "."); err != nil {
+			logc.Fatalln(ctx, err)
+		}
+
+	case *auditRollback != "":
+		ctx = WithPrincipal(ctx)
+		GetPrincipal(ctx).CliAdmin = proto.Bool(true)
+
+		id, err := ParseAuditID(*auditRollback)
+		if err != nil {
+			logc.Fatalln(ctx, err)
+		}
+
+		record, err := backend.QueryAuditLog(ctx, id)
+		if err != nil {
+			logc.Fatalln(ctx, err)
+		}
+
+		if record.GetManifest() == nil || record.GetDomain() == "" || record.GetProject() == "" {
+			logc.Fatalln(ctx, "no manifest in audit record")
+		}
+
+		webRoot := path.Join(record.GetDomain(), record.GetProject())
+		err = backend.StageManifest(ctx, record.GetManifest())
+		if err != nil {
+			logc.Fatalln(ctx, err)
+		}
+		err = backend.CommitManifest(ctx, webRoot, record.GetManifest(), ModifyManifestOptions{})
+		if err != nil {
+			logc.Fatalln(ctx, err)
+		}
+
+	case *auditServer != "":
+		if flag.NArg() < 1 {
+			logc.Fatalln(ctx, "handler path not provided")
+		}
+
+		processor, err := AuditEventProcessor(flag.Arg(0), flag.Args()[1:])
+		if err != nil {
+			logc.Fatalln(ctx, err)
+		}
+
+		serve(ctx, listen(ctx, "audit", *auditServer), ObserveHTTPHandler(processor))
+
+	case *runMigration != "":
+		if err = RunMigration(ctx, *runMigration); err != nil {
+			logc.Fatalln(ctx, err)
+		}
+
+	case *traceGarbage:
+		if err = TraceGarbage(ctx); err != nil {
+			logc.Fatalln(ctx, err)
 		}
 
 	default:
@@ -339,7 +548,7 @@ func Main() {
 		// The backend is not recreated (this is intentional as it allows preserving the cache).
 		OnReload(func() {
 			if newConfig, err := Configure(*configTomlPath); err != nil {
-				log.Println("config: reload err:", err)
+				logc.Println(ctx, "config: reload err:", err)
 			} else {
 				// From https://go.dev/ref/mem:
 				// > A read r of a memory location x holding a value that is not larger than
@@ -349,15 +558,16 @@ func Main() {
 				// > concurrent write.
 				config = newConfig
 				if err = errors.Join(
-					configureFeatures(),
-					configureMemLimit(),
-					configureWildcards(),
+					configureFeatures(ctx),
+					configureMemLimit(ctx),
+					configureWildcards(ctx),
+					configureFallback(ctx),
 				); err != nil {
 					// At this point the configuration is in an in-between, corrupted state, so
 					// the only reasonable choice is to crash.
-					log.Fatalln("config: reload fail:", err)
+					logc.Fatalln(ctx, "config: reload fail:", err)
 				} else {
-					log.Println("config: reload ok")
+					logc.Println(ctx, "config: reload ok")
 				}
 			}
 		})
@@ -366,24 +576,31 @@ func Main() {
 		// spends some time initializing (which the S3 backend does) a proxy like Caddy can race
 		// with git-pages on startup and return errors for requests that would have been served
 		// just 0.5s later.
-		pagesListener := listen("pages", config.Server.Pages)
-		caddyListener := listen("caddy", config.Server.Caddy)
-		metricsListener := listen("metrics", config.Server.Metrics)
+		pagesListener := listen(ctx, "pages", config.Server.Pages)
+		caddyListener := listen(ctx, "caddy", config.Server.Caddy)
+		metricsListener := listen(ctx, "metrics", config.Server.Metrics)
 
-		if backend, err = CreateBackend(&config.Storage); err != nil {
-			log.Fatalln(err)
+		if backend, err = CreateBackend(ctx, &config.Storage); err != nil {
+			logc.Fatalln(ctx, err)
 		}
 		backend = NewObservedBackend(backend)
 
-		go serve(pagesListener, ObserveHTTPHandler(http.HandlerFunc(ServePages)))
-		go serve(caddyListener, ObserveHTTPHandler(http.HandlerFunc(ServeCaddy)))
-		go serve(metricsListener, promhttp.Handler())
+		middleware := chainHTTPMiddleware(
+			panicHandler,
+			remoteAddrMiddleware,
+			ObserveHTTPHandler,
+		)
+		go serve(ctx, pagesListener, middleware(http.HandlerFunc(ServePages)))
+		go serve(ctx, caddyListener, middleware(http.HandlerFunc(ServeCaddy)))
+		go serve(ctx, metricsListener, promhttp.Handler())
 
 		if config.Insecure {
-			log.Println("serve: ready (INSECURE)")
+			logc.Println(ctx, "serve: ready (INSECURE)")
 		} else {
-			log.Println("serve: ready")
+			logc.Println(ctx, "serve: ready")
 		}
-		select {}
+
+		WaitForInterrupt()
+		logc.Println(ctx, "serve: exiting")
 	}
 }

src/manifest.go

@@ -8,7 +8,6 @@ import (
 	"crypto/sha256"
 	"errors"
 	"fmt"
-	"log"
 	"mime"
 	"net/http"
 	"path"
@@ -18,6 +17,8 @@ import (
 	"time"
 
 	"github.com/c2h5oh/datasize"
+	"github.com/go-git/go-git/v6/plumbing"
+	format "github.com/go-git/go-git/v6/plumbing/format/config"
 	"github.com/klauspost/compress/zstd"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
@@ -37,6 +38,14 @@ var (
 	})
 )
 
+func NewManifest() *Manifest {
+	return &Manifest{
+		Contents: map[string]*Entry{
+			"": {Type: Type_Directory.Enum()},
+		},
+	}
+}
+
 func IsManifestEmpty(manifest *Manifest) bool {
 	if len(manifest.Contents) > 1 {
 		return false
@@ -69,47 +78,107 @@ func CompareManifest(left *Manifest, right *Manifest) bool {
 	return true
 }
 
-func EncodeManifest(manifest *Manifest) []byte {
-	result, err := proto.MarshalOptions{Deterministic: true}.Marshal(manifest)
+func EncodeManifest(manifest *Manifest) (data []byte) {
+	data, err := proto.MarshalOptions{Deterministic: true}.Marshal(manifest)
 	if err != nil {
 		panic(err)
 	}
-	return result
+	return
 }
 
-func DecodeManifest(data []byte) (*Manifest, error) {
-	manifest := Manifest{}
-	err := proto.Unmarshal(data, &manifest)
-	return &manifest, err
+func DecodeManifest(data []byte) (manifest *Manifest, err error) {
+	manifest = &Manifest{}
+	err = proto.Unmarshal(data, manifest)
+	return
 }
 
-func AddProblem(manifest *Manifest, path, format string, args ...any) error {
+func NewManifestEntry(type_ Type, data []byte) *Entry {
+	entry := &Entry{}
+	entry.Type = type_.Enum()
+	if data != nil {
+		entry.Data = data
+		entry.Transform = Transform_Identity.Enum()
+		entry.OriginalSize = proto.Int64(int64(len(data)))
+		entry.CompressedSize = proto.Int64(int64(len(data)))
+	}
+	return entry
+}
+
+func AddFile(manifest *Manifest, fileName string, data []byte) *Entry {
+	// Fill in `git_hash` even for files not originating from git using the SHA256 algorithm;
+	// we use this primarily for incremental archive uploads, but when support for git SHA256
+	// repositories is complete, archive uploads and git checkouts will have cross-support for
+	// incremental updates.
+	hasher := plumbing.NewHasher(format.SHA256, plumbing.BlobObject, int64(len(data)))
+	hasher.Write(data)
+	entry := NewManifestEntry(Type_InlineFile, data)
+	entry.GitHash = proto.String(hasher.Sum().String())
+	manifest.Contents[fileName] = entry
+	return entry
+}
+
+func AddSymlink(manifest *Manifest, fileName string, target string) *Entry {
+	if path.IsAbs(target) {
+		AddProblem(manifest, fileName, "absolute symlink: %s", target)
+		return nil
+	} else {
+		entry := NewManifestEntry(Type_Symlink, []byte(target))
+		manifest.Contents[fileName] = entry
+		return entry
+	}
+}
+
+func AddDirectory(manifest *Manifest, dirName string) *Entry {
+	dirName = strings.TrimSuffix(dirName, "/")
+	entry := NewManifestEntry(Type_Directory, nil)
+	manifest.Contents[dirName] = entry
+	return entry
+}
+
+func AddProblem(manifest *Manifest, pathName, format string, args ...any) error {
 	cause := fmt.Sprintf(format, args...)
 	manifest.Problems = append(manifest.Problems, &Problem{
-		Path:  proto.String(path),
+		Path:  proto.String(pathName),
 		Cause: proto.String(cause),
 	})
-	return fmt.Errorf("%s: %s", path, cause)
+	return fmt.Errorf("%s: %s", pathName, cause)
+}
+
+// EnsureLeadingDirectories adds directory entries for any parent directories
+// that are implicitly referenced by files in the manifest but don't have
+// explicit directory entries. (This can be the case if an archive is created
+// via globs rather than including a whole directory.)
+func EnsureLeadingDirectories(manifest *Manifest) {
+	for name := range manifest.Contents {
+		for dir := path.Dir(name); dir != "." && dir != ""; dir = path.Dir(dir) {
+			if dir == "/" {
+				panic("malformed manifest (paths must not be rooted in /)")
+			}
+			if _, exists := manifest.Contents[dir]; !exists {
+				AddDirectory(manifest, dir)
+			}
+		}
+	}
 }
 
 func GetProblemReport(manifest *Manifest) []string {
 	var report []string
 	for _, problem := range manifest.Problems {
 		report = append(report,
-			fmt.Sprintf("%s: %s", problem.GetPath(), problem.GetCause()))
+			fmt.Sprintf("/%s: %s", problem.GetPath(), problem.GetCause()))
 	}
 	return report
 }
 
-func ManifestDebugJSON(manifest *Manifest) string {
-	result, err := protojson.MarshalOptions{
+func ManifestJSON(manifest *Manifest) []byte {
+	json, err := protojson.MarshalOptions{
 		Multiline:         true,
 		EmitDefaultValues: true,
 	}.Marshal(manifest)
 	if err != nil {
 		panic(err)
 	}
-	return string(result)
+	return json
 }
 
 var ErrSymlinkLoop = errors.New("symbolic link loop")
@@ -145,20 +214,20 @@ func DetectContentType(manifest *Manifest) {
 	for path, entry := range manifest.Contents {
 		if entry.GetType() == Type_Directory || entry.GetType() == Type_Symlink {
 			// no Content-Type
-		} else if entry.GetType() == Type_InlineFile && entry.GetTransform() == Transform_None {
+		} else if entry.GetType() == Type_InlineFile && entry.GetTransform() == Transform_Identity {
 			contentType := mime.TypeByExtension(filepath.Ext(path))
 			if contentType == "" {
-				contentType = http.DetectContentType(entry.Data[:512])
+				contentType = http.DetectContentType(entry.Data[:min(512, len(entry.Data))])
 			}
 			entry.ContentType = proto.String(contentType)
-		} else {
+		} else if entry.GetContentType() == "" {
 			panic(fmt.Errorf("DetectContentType encountered invalid entry: %v, %v",
 				entry.GetType(), entry.GetTransform()))
 		}
 	}
 }
 
-// The `clauspost/compress/zstd` package recommends reusing a compressor to avoid repeated
+// The `klauspost/compress/zstd` package recommends reusing a compressor to avoid repeated
 // allocations of internal buffers.
 var zstdEncoder, _ = zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedBetterCompression))
 
@@ -167,29 +236,31 @@ func CompressFiles(ctx context.Context, manifest *Manifest) {
 	span, _ := ObserveFunction(ctx, "CompressFiles")
 	defer span.Finish()
 
-	var originalSize, compressedSize int64
+	var originalSize int64
+	var compressedSize int64
 	for _, entry := range manifest.Contents {
-		if entry.GetType() == Type_InlineFile && entry.GetTransform() == Transform_None {
-			mtype := getMediaType(entry.GetContentType())
-			if strings.HasPrefix(mtype, "video/") || strings.HasPrefix(mtype, "audio/") {
+		if entry.GetType() == Type_InlineFile && entry.GetTransform() == Transform_Identity {
+			mediaType := getMediaType(entry.GetContentType())
+			if strings.HasPrefix(mediaType, "video/") || strings.HasPrefix(mediaType, "audio/") {
 				continue
 			}
-			originalSize += entry.GetSize()
-			compressedData := zstdEncoder.EncodeAll(entry.GetData(), make([]byte, 0, entry.GetSize()))
-			if len(compressedData) < int(*entry.Size) {
+			compressedData := zstdEncoder.EncodeAll(entry.GetData(),
+				make([]byte, 0, entry.GetOriginalSize()))
+			if int64(len(compressedData)) < entry.GetOriginalSize() {
 				entry.Data = compressedData
-				entry.Size = proto.Int64(int64(len(entry.Data)))
-				entry.Transform = Transform_Zstandard.Enum()
+				entry.Transform = Transform_Zstd.Enum()
+				entry.CompressedSize = proto.Int64(int64(len(entry.Data)))
 			}
-			compressedSize += entry.GetSize()
 		}
+		originalSize += entry.GetOriginalSize()
+		compressedSize += entry.GetCompressedSize()
 	}
 	manifest.OriginalSize = proto.Int64(originalSize)
 	manifest.CompressedSize = proto.Int64(compressedSize)
 
 	if originalSize != 0 {
 		spaceSaving := (float64(originalSize) - float64(compressedSize)) / float64(originalSize)
-		log.Printf("compress: saved %.2f percent (%s to %s)",
+		logc.Printf(ctx, "compress: saved %.2f percent (%s to %s)",
 			spaceSaving*100.0,
 			datasize.ByteSize(originalSize).HR(),
 			datasize.ByteSize(compressedSize).HR(),
@@ -203,34 +274,40 @@ func CompressFiles(ctx context.Context, manifest *Manifest) {
 // At the moment, there isn't a good way to report errors except to log them on the terminal.
 // (Perhaps in the future they could be exposed at `.git-pages/status.txt`?)
 func PrepareManifest(ctx context.Context, manifest *Manifest) error {
-	// Parse Netlify-style `_redirects`
+	// Parse Netlify-style `_redirects`.
 	if err := ProcessRedirectsFile(manifest); err != nil {
-		log.Printf("redirects err: %s\n", err)
+		logc.Printf(ctx, "redirects err: %s\n", err)
 	} else if len(manifest.Redirects) > 0 {
-		log.Printf("redirects ok: %d rules\n", len(manifest.Redirects))
+		logc.Printf(ctx, "redirects ok: %d rules\n", len(manifest.Redirects))
 	}
 
-	// Parse Netlify-style `_headers`
+	// Check if any redirects are unreachable.
+	LintRedirects(manifest)
+
+	// Parse Netlify-style `_headers`.
 	if err := ProcessHeadersFile(manifest); err != nil {
-		log.Printf("headers err: %s\n", err)
+		logc.Printf(ctx, "headers err: %s\n", err)
 	} else if len(manifest.Headers) > 0 {
-		log.Printf("headers ok: %d rules\n", len(manifest.Headers))
+		logc.Printf(ctx, "headers ok: %d rules\n", len(manifest.Headers))
 	}
 
-	// Sniff content type like `http.ServeContent`
+	// Sniff content type like `http.ServeContent`.
 	DetectContentType(manifest)
 
-	// Opportunistically compress blobs (must be done last)
+	// Opportunistically compress blobs (must be done last).
 	CompressFiles(ctx, manifest)
 
 	return nil
 }
 
+var ErrSiteTooLarge = errors.New("site too large")
 var ErrManifestTooLarge = errors.New("manifest too large")
 
 // Uploads inline file data over certain size to the storage backend. Returns a copy of
 // the manifest updated to refer to an external content-addressable store.
-func StoreManifest(ctx context.Context, name string, manifest *Manifest) (*Manifest, error) {
+func StoreManifest(
+	ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions,
+) (*Manifest, error) {
 	span, ctx := ObserveFunction(ctx, "StoreManifest", "manifest.name", name)
 	defer span.Finish()
 
@@ -247,27 +324,43 @@ func StoreManifest(ctx context.Context, name string, manifest *Manifest) (*Manif
 		CompressedSize: manifest.CompressedSize,
 		StoredSize:     proto.Int64(0),
 	}
-	extObjectSizes := make(map[string]int64)
 	for name, entry := range manifest.Contents {
 		cannotBeInlined := entry.GetType() == Type_InlineFile &&
-			entry.GetSize() > int64(config.Limits.MaxInlineFileSize.Bytes())
+			entry.GetCompressedSize() > int64(config.Limits.MaxInlineFileSize.Bytes())
 		if cannotBeInlined {
 			dataHash := sha256.Sum256(entry.Data)
 			extManifest.Contents[name] = &Entry{
-				Type:        Type_ExternalFile.Enum(),
-				Size:        entry.Size,
-				Data:        fmt.Appendf(nil, "sha256-%x", dataHash),
-				Transform:   entry.Transform,
-				ContentType: entry.ContentType,
+				Type:           Type_ExternalFile.Enum(),
+				OriginalSize:   entry.OriginalSize,
+				CompressedSize: entry.CompressedSize,
+				Data:           fmt.Appendf(nil, "sha256-%x", dataHash),
+				Transform:      entry.Transform,
+				ContentType:    entry.ContentType,
+				GitHash:        entry.GitHash,
 			}
-			extObjectSizes[string(dataHash[:])] = entry.GetSize()
 		} else {
 			extManifest.Contents[name] = entry
 		}
 	}
-	// `extObjectMap` stores size once per object, deduplicating it
-	for _, storedSize := range extObjectSizes {
-		*extManifest.StoredSize += storedSize
+
+	// Compute the total and deduplicated storage size.
+	totalSize := int64(0)
+	blobSizes := map[string]int64{}
+	for _, entry := range extManifest.Contents {
+		totalSize += entry.GetOriginalSize()
+		if entry.GetType() == Type_ExternalFile {
+			blobSizes[string(entry.Data)] = entry.GetCompressedSize()
+		}
+	}
+	if uint64(totalSize) > config.Limits.MaxSiteSize.Bytes() {
+		return nil, fmt.Errorf("%w: contents size %s exceeds %s limit",
+			ErrSiteTooLarge,
+			datasize.ByteSize(totalSize).HR(),
+			config.Limits.MaxSiteSize.HR(),
+		)
+	}
+	for _, blobSize := range blobSizes {
+		*extManifest.StoredSize += blobSize
 	}
 
 	// Upload the resulting manifest and the blob it references.
@@ -287,7 +380,10 @@ func StoreManifest(ctx context.Context, name string, manifest *Manifest) (*Manif
 	wg := sync.WaitGroup{}
 	ch := make(chan error, len(extManifest.Contents))
 	for name, entry := range extManifest.Contents {
-		if entry.GetType() == Type_ExternalFile {
+		// Upload external entries (those that were decided as ineligible for being stored inline).
+		// If the entry in the original manifest is already an external reference, there's no need
+		// to externalize it (and no way for us to do so, since the entry only contains the blob name).
+		if entry.GetType() == Type_ExternalFile && manifest.Contents[name].GetType() == Type_InlineFile {
 			wg.Go(func() {
 				err := backend.PutBlob(ctx, string(entry.Data), manifest.Contents[name].Data)
 				if err != nil {
@@ -302,8 +398,12 @@ func StoreManifest(ctx context.Context, name string, manifest *Manifest) (*Manif
 		return nil, err // currently ignores all but 1st error
 	}
 
-	if err := backend.CommitManifest(ctx, name, &extManifest); err != nil {
-		return nil, fmt.Errorf("commit manifest: %w", err)
+	if err := backend.CommitManifest(ctx, name, &extManifest, opts); err != nil {
+		if errors.Is(err, ErrDomainFrozen) {
+			return nil, err
+		} else {
+			return nil, fmt.Errorf("commit manifest: %w", err)
+		}
 	}
 
 	return &extManifest, nil

src/migrate.go

@@ -3,7 +3,6 @@ package git_pages
 import (
 	"context"
 	"fmt"
-	"log"
 	"slices"
 	"strings"
 )
@@ -19,16 +18,19 @@ func RunMigration(ctx context.Context, name string) error {
 
 func createDomainMarkers(ctx context.Context) error {
 	if backend.HasFeature(ctx, FeatureCheckDomainMarker) {
-		log.Print("store already has domain markers")
+		logc.Print(ctx, "store already has domain markers")
 		return nil
 	}
 
-	var manifests, domains []string
-	manifests, err := backend.ListManifests(ctx)
-	if err != nil {
-		return fmt.Errorf("list manifests: %w", err)
+	var manifests []string
+	for metadata, err := range backend.EnumerateManifests(ctx) {
+		if err != nil {
+			return fmt.Errorf("enum manifests: %w", err)
+		}
+		manifests = append(manifests, metadata.Name)
 	}
 	slices.Sort(manifests)
+	var domains []string
 	for _, manifest := range manifests {
 		domain, _, _ := strings.Cut(manifest, "/")
 		if len(domains) == 0 || domains[len(domains)-1] != domain {
@@ -36,7 +38,7 @@ func createDomainMarkers(ctx context.Context) error {
 		}
 	}
 	for idx, domain := range domains {
-		log.Printf("(%d / %d) creating domain %s", idx+1, len(domains), domain)
+		logc.Printf(ctx, "(%d / %d) creating domain %s", idx+1, len(domains), domain)
 		if err := backend.CreateDomain(ctx, domain); err != nil {
 			return fmt.Errorf("creating domain %s: %w", domain, err)
 		}
@@ -44,6 +46,6 @@ func createDomainMarkers(ctx context.Context) error {
 	if err := backend.EnableFeature(ctx, FeatureCheckDomainMarker); err != nil {
 		return err
 	}
-	log.Printf("created markers for %d domains", len(domains))
+	logc.Printf(ctx, "created markers for %d domains", len(domains))
 	return nil
 }

src/observe.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"iter"
 	"log"
 	"log/slog"
 	"math/rand/v2"
@@ -12,10 +13,13 @@ import (
 	"os"
 	"runtime/debug"
 	"strconv"
+	"sync"
 	"time"
 
 	slogmulti "github.com/samber/slog-multi"
 
+	syslog "codeberg.org/git-pages/go-slog-syslog"
+
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 
@@ -41,10 +45,65 @@ var (
 	}, []string{"method"})
 )
 
+var syslogHandler syslog.Handler
+
 func hasSentry() bool {
 	return os.Getenv("SENTRY_DSN") != ""
 }
 
+func chainSentryMiddleware(
+	middleware ...func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event,
+) func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event {
+	return func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event {
+		for idx := 0; idx < len(middleware) && event != nil; idx++ {
+			event = middleware[idx](event, hint)
+		}
+		return event
+	}
+}
+
+// sensitiveHTTPHeaders extends the list of sensitive headers defined in the Sentry Go SDK with our
+// own application-specific header field names.
+var sensitiveHTTPHeaders = map[string]struct{}{
+	"Forge-Authorization": {},
+}
+
+// scrubSentryEvent removes sensitive HTTP header fields from the Sentry event.
+func scrubSentryEvent(event *sentry.Event, hint *sentry.EventHint) *sentry.Event {
+	if event.Request != nil && event.Request.Headers != nil {
+		for key := range event.Request.Headers {
+			if _, ok := sensitiveHTTPHeaders[key]; ok {
+				delete(event.Request.Headers, key)
+			}
+		}
+	}
+	return event
+}
+
+// sampleSentryEvent returns a function that discards a Sentry event according to the sample rate,
+// unless the associated HTTP request triggers a mutation or it took too long to produce a response,
+// in which case the event is never discarded.
+func sampleSentryEvent(sampleRate float64) func(*sentry.Event, *sentry.EventHint) *sentry.Event {
+	return func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event {
+		newSampleRate := sampleRate
+		if event.Request != nil {
+			switch event.Request.Method {
+			case "PUT", "POST", "DELETE":
+				newSampleRate = 1
+			}
+		}
+		duration := event.Timestamp.Sub(event.StartTime)
+		threshold := time.Duration(config.Observability.SlowResponseThreshold)
+		if duration >= threshold {
+			newSampleRate = 1
+		}
+		if rand.Float64() < newSampleRate {
+			return event
+		}
+		return nil
+	}
+}
+
 func InitObservability() {
 	debug.SetPanicOnFault(true)
 
@@ -68,6 +127,19 @@ func InitObservability() {
 		log.Println("unknown log format", config.LogFormat)
 	}
 
+	if syslogAddr := os.Getenv("SYSLOG_ADDR"); syslogAddr != "" {
+		var err error
+		syslogHandler, err = syslog.NewHandler(&syslog.HandlerOptions{
+			Address:          syslogAddr,
+			AppName:          "git-pages",
+			StructuredDataID: "git-pages",
+		})
+		if err != nil {
+			log.Fatalf("syslog: %v", err)
+		}
+		logHandlers = append(logHandlers, syslogHandler)
+	}
+
 	if hasSentry() {
 		enableLogs := false
 		if value, err := strconv.ParseBool(os.Getenv("SENTRY_LOGS")); err == nil {
@@ -79,38 +151,23 @@ func InitObservability() {
 			enableTracing = value
 		}
 
+		tracesSampleRate := 1.00
+		switch environment {
+		case "development", "staging":
+		default:
+			tracesSampleRate = 0.05
+		}
+
 		options := sentry.ClientOptions{}
 		options.Environment = environment
 		options.EnableLogs = enableLogs
 		options.EnableTracing = enableTracing
-		options.TracesSampleRate = 1
-		switch environment {
-		case "development", "staging":
-		default:
-			options.BeforeSendTransaction = func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event {
-				sampleRate := 0.05
-				if trace, ok := event.Contexts["trace"]; ok {
-					if data, ok := trace["data"].(map[string]any); ok {
-						if method, ok := data["http.request.method"].(string); ok {
-							switch method {
-							case "PUT", "DELETE", "POST":
-								sampleRate = 1
-							default:
-								duration := event.Timestamp.Sub(event.StartTime)
-								threshold := time.Duration(config.Observability.SlowResponseThreshold)
-								if duration >= threshold {
-									sampleRate = 1
-								}
-							}
-						}
-					}
-				}
-				if rand.Float64() < sampleRate {
-					return event
-				}
-				return nil
-			}
-		}
+		options.TracesSampleRate = 1 // use our own custom sampling logic
+		options.BeforeSend = scrubSentryEvent
+		options.BeforeSendTransaction = chainSentryMiddleware(
+			sampleSentryEvent(tracesSampleRate),
+			scrubSentryEvent,
+		)
 		if err := sentry.Init(options); err != nil {
 			log.Fatalf("sentry: %s\n", err)
 		}
@@ -126,9 +183,15 @@ func InitObservability() {
 }
 
 func FiniObservability() {
-	if hasSentry() {
-		sentry.Flush(2 * time.Second)
+	var wg sync.WaitGroup
+	timeout := 2 * time.Second
+	if syslogHandler != nil {
+		wg.Go(func() { syslogHandler.Flush(timeout) })
 	}
+	if hasSentry() {
+		wg.Go(func() { sentry.Flush(timeout) })
+	}
+	wg.Wait()
 }
 
 func ObserveError(err error) {
@@ -290,13 +353,13 @@ func (backend *observedBackend) EnableFeature(ctx context.Context, feature Backe
 func (backend *observedBackend) GetBlob(
 	ctx context.Context, name string,
 ) (
-	reader io.ReadSeeker, size uint64, mtime time.Time, err error,
+	reader io.ReadSeeker, metadata BlobMetadata, err error,
 ) {
 	span, ctx := ObserveFunction(ctx, "GetBlob", "blob.name", name)
-	if reader, size, mtime, err = backend.inner.GetBlob(ctx, name); err == nil {
-		ObserveData(ctx, "blob.size", size)
+	if reader, metadata, err = backend.inner.GetBlob(ctx, name); err == nil {
+		ObserveData(ctx, "blob.size", metadata.Size)
 		blobsRetrievedCount.Inc()
-		blobsRetrievedBytes.Add(float64(size))
+		blobsRetrievedBytes.Add(float64(metadata.Size))
 	}
 	span.Finish()
 	return
@@ -319,23 +382,28 @@ func (backend *observedBackend) DeleteBlob(ctx context.Context, name string) (er
 	return
 }
 
-func (backend *observedBackend) ListManifests(ctx context.Context) (manifests []string, err error) {
-	span, ctx := ObserveFunction(ctx, "ListManifests")
-	manifests, err = backend.inner.ListManifests(ctx)
-	span.Finish()
-	return
+func (backend *observedBackend) EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error] {
+	return func(yield func(BlobMetadata, error) bool) {
+		span, ctx := ObserveFunction(ctx, "EnumerateBlobs")
+		for metadata, err := range backend.inner.EnumerateBlobs(ctx) {
+			if !yield(metadata, err) {
+				break
+			}
+		}
+		span.Finish()
+	}
 }
 
 func (backend *observedBackend) GetManifest(
 	ctx context.Context, name string, opts GetManifestOptions,
 ) (
-	manifest *Manifest, mtime time.Time, err error,
+	manifest *Manifest, metadata ManifestMetadata, err error,
 ) {
 	span, ctx := ObserveFunction(ctx, "GetManifest",
 		"manifest.name", name,
 		"manifest.bypass_cache", opts.BypassCache,
 	)
-	if manifest, mtime, err = backend.inner.GetManifest(ctx, name, opts); err == nil {
+	if manifest, metadata, err = backend.inner.GetManifest(ctx, name, opts); err == nil {
 		manifestsRetrievedCount.Inc()
 	}
 	span.Finish()
@@ -349,30 +417,91 @@ func (backend *observedBackend) StageManifest(ctx context.Context, manifest *Man
 	return
 }
 
-func (backend *observedBackend) CommitManifest(ctx context.Context, name string, manifest *Manifest) (err error) {
+func (backend *observedBackend) HasAtomicCAS(ctx context.Context) bool {
+	return backend.inner.HasAtomicCAS(ctx)
+}
+
+func (backend *observedBackend) CommitManifest(ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions) (err error) {
 	span, ctx := ObserveFunction(ctx, "CommitManifest", "manifest.name", name)
-	err = backend.inner.CommitManifest(ctx, name, manifest)
+	err = backend.inner.CommitManifest(ctx, name, manifest, opts)
 	span.Finish()
 	return
 }
 
-func (backend *observedBackend) DeleteManifest(ctx context.Context, name string) (err error) {
+func (backend *observedBackend) DeleteManifest(ctx context.Context, name string, opts ModifyManifestOptions) (err error) {
 	span, ctx := ObserveFunction(ctx, "DeleteManifest", "manifest.name", name)
-	err = backend.inner.DeleteManifest(ctx, name)
+	err = backend.inner.DeleteManifest(ctx, name, opts)
 	span.Finish()
 	return
 }
 
+func (backend *observedBackend) EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error] {
+	return func(yield func(ManifestMetadata, error) bool) {
+		span, ctx := ObserveFunction(ctx, "EnumerateManifests")
+		for metadata, err := range backend.inner.EnumerateManifests(ctx) {
+			if !yield(metadata, err) {
+				break
+			}
+		}
+		span.Finish()
+	}
+}
+
 func (backend *observedBackend) CheckDomain(ctx context.Context, domain string) (found bool, err error) {
-	span, ctx := ObserveFunction(ctx, "CheckDomain", "manifest.domain", domain)
+	span, ctx := ObserveFunction(ctx, "CheckDomain", "domain.name", domain)
 	found, err = backend.inner.CheckDomain(ctx, domain)
 	span.Finish()
 	return
 }
 
 func (backend *observedBackend) CreateDomain(ctx context.Context, domain string) (err error) {
-	span, ctx := ObserveFunction(ctx, "CreateDomain", "manifest.domain", domain)
+	span, ctx := ObserveFunction(ctx, "CreateDomain", "domain.name", domain)
 	err = backend.inner.CreateDomain(ctx, domain)
 	span.Finish()
 	return
 }
+
+func (backend *observedBackend) FreezeDomain(ctx context.Context, domain string) (err error) {
+	span, ctx := ObserveFunction(ctx, "FreezeDomain", "domain.name", domain)
+	err = backend.inner.FreezeDomain(ctx, domain)
+	span.Finish()
+	return
+}
+
+func (backend *observedBackend) UnfreezeDomain(ctx context.Context, domain string) (err error) {
+	span, ctx := ObserveFunction(ctx, "UnfreezeDomain", "domain.name", domain)
+	err = backend.inner.UnfreezeDomain(ctx, domain)
+	span.Finish()
+	return
+}
+
+func (backend *observedBackend) AppendAuditLog(ctx context.Context, id AuditID, record *AuditRecord) (err error) {
+	span, ctx := ObserveFunction(ctx, "AppendAuditLog", "audit.id", id)
+	err = backend.inner.AppendAuditLog(ctx, id, record)
+	span.Finish()
+	return
+}
+
+func (backend *observedBackend) QueryAuditLog(ctx context.Context, id AuditID) (record *AuditRecord, err error) {
+	span, ctx := ObserveFunction(ctx, "QueryAuditLog", "audit.id", id)
+	record, err = backend.inner.QueryAuditLog(ctx, id)
+	span.Finish()
+	return
+}
+
+func (backend *observedBackend) SearchAuditLog(
+	ctx context.Context, opts SearchAuditLogOptions,
+) iter.Seq2[AuditID, error] {
+	return func(yield func(AuditID, error) bool) {
+		span, ctx := ObserveFunction(ctx, "SearchAuditLog",
+			"audit.search.since", opts.Since,
+			"audit.search.until", opts.Until,
+		)
+		for id, err := range backend.inner.SearchAuditLog(ctx, opts) {
+			if !yield(id, err) {
+				break
+			}
+		}
+		span.Finish()
+	}
+}

src/pages.go

@@ -8,12 +8,12 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"log"
 	"maps"
 	"net/http"
 	"net/url"
 	"os"
 	"path"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -22,11 +22,17 @@ import (
 	"github.com/pquerna/cachecontrol/cacheobject"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
+	"google.golang.org/protobuf/proto"
 )
 
 const notFoundPage = "404.html"
 
 var (
+	serveEncodingCount = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "git_pages_serve_encoding_count",
+		Help: "Count of blob transform vs negotiated encoding",
+	}, []string{"transform", "negotiated"})
+
 	siteUpdatesCount = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "git_pages_site_updates",
 		Help: "Count of site updates in total",
@@ -41,9 +47,8 @@ var (
 	}, []string{"cause"})
 )
 
-func reportSiteUpdate(via string, result *UpdateResult) {
+func observeSiteUpdate(via string, result *UpdateResult) {
 	siteUpdatesCount.With(prometheus.Labels{"via": via}).Inc()
-
 	switch result.outcome {
 	case UpdateError:
 		siteUpdateErrorCount.With(prometheus.Labels{"cause": "other"}).Inc()
@@ -61,7 +66,21 @@ func reportSiteUpdate(via string, result *UpdateResult) {
 }
 
 func makeWebRoot(host string, projectName string) string {
-	return fmt.Sprintf("%s/%s", strings.ToLower(host), projectName)
+	return path.Join(strings.ToLower(host), projectName)
+}
+
+func getWebRoot(r *http.Request) (string, error) {
+	host, err := GetHost(r)
+	if err != nil {
+		return "", err
+	}
+
+	projectName, err := GetProjectName(r)
+	if err != nil {
+		return "", err
+	}
+
+	return makeWebRoot(host, projectName), nil
 }
 
 func writeRedirect(w http.ResponseWriter, code int, path string) {
@@ -78,7 +97,7 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 	var err error
 	var sitePath string
 	var manifest *Manifest
-	var manifestMtime time.Time
+	var metadata ManifestMetadata
 
 	cacheControl, err := cacheobject.ParseRequestCacheControl(r.Header.Get("Cache-Control"))
 	if err != nil {
@@ -97,42 +116,46 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 	}
 
 	type indexManifestResult struct {
-		manifest      *Manifest
-		manifestMtime time.Time
-		err           error
+		manifest *Manifest
+		metadata ManifestMetadata
+		err      error
 	}
 	indexManifestCh := make(chan indexManifestResult, 1)
 	go func() {
-		manifest, mtime, err := backend.GetManifest(
+		manifest, metadata, err := backend.GetManifest(
 			r.Context(), makeWebRoot(host, ".index"),
 			GetManifestOptions{BypassCache: bypassCache},
 		)
-		indexManifestCh <- (indexManifestResult{manifest, mtime, err})
+		indexManifestCh <- (indexManifestResult{manifest, metadata, err})
 	}()
 
 	err = nil
 	sitePath = strings.TrimPrefix(r.URL.Path, "/")
 	if projectName, projectPath, hasProjectSlash := strings.Cut(sitePath, "/"); projectName != "" {
-		var projectManifest *Manifest
-		var projectManifestMtime time.Time
-		projectManifest, projectManifestMtime, err = backend.GetManifest(
-			r.Context(), makeWebRoot(host, projectName),
-			GetManifestOptions{BypassCache: bypassCache},
-		)
-		if err == nil {
-			if !hasProjectSlash {
-				writeRedirect(w, http.StatusFound, r.URL.Path+"/")
-				return nil
+		if IsValidProjectName(projectName) {
+			var projectManifest *Manifest
+			var projectMetadata ManifestMetadata
+			projectManifest, projectMetadata, err = backend.GetManifest(
+				r.Context(), makeWebRoot(host, projectName),
+				GetManifestOptions{BypassCache: bypassCache},
+			)
+			if err == nil {
+				if !hasProjectSlash {
+					writeRedirect(w, http.StatusFound, r.URL.Path+"/")
+					return nil
+				}
+				sitePath, manifest, metadata = projectPath, projectManifest, projectMetadata
 			}
-			sitePath, manifest, manifestMtime = projectPath, projectManifest, projectManifestMtime
 		}
 	}
 	if manifest == nil && (err == nil || errors.Is(err, ErrObjectNotFound)) {
 		result := <-indexManifestCh
-		manifest, manifestMtime, err = result.manifest, result.manifestMtime, result.err
+		manifest, metadata, err = result.manifest, result.metadata, result.err
 		if manifest == nil && errors.Is(err, ErrObjectNotFound) {
-			if found, fallbackErr := HandleWildcardFallback(w, r); found {
-				return fallbackErr
+			if fallback != nil {
+				logc.Printf(r.Context(), "fallback: %s via %s", host, config.Fallback.ProxyTo)
+				fallback.ServeHTTP(w, r)
+				return nil
 			} else {
 				w.WriteHeader(http.StatusNotFound)
 				fmt.Fprintf(w, "site not found\n")
@@ -159,10 +182,11 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 		return nil
 	}
 	if metadataPath, found := strings.CutPrefix(sitePath, ".git-pages/"); found {
-		lastModified := manifestMtime.UTC().Format(http.TimeFormat)
+		lastModified := metadata.LastModified.UTC().Format(http.TimeFormat)
 		switch {
 		case metadataPath == "health":
 			w.Header().Add("Last-Modified", lastModified)
+			w.Header().Add("ETag", fmt.Sprintf("\"%s\"", metadata.ETag))
 			w.WriteHeader(http.StatusOK)
 			fmt.Fprintf(w, "ok\n")
 			return nil
@@ -177,11 +201,12 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 
 			w.Header().Add("Content-Type", "application/json; charset=utf-8")
 			w.Header().Add("Last-Modified", lastModified)
+			w.Header().Add("ETag", fmt.Sprintf("\"%s-manifest\"", metadata.ETag))
 			w.WriteHeader(http.StatusOK)
-			w.Write([]byte(ManifestDebugJSON(manifest)))
+			w.Write(ManifestJSON(manifest))
 			return nil
 
-		case metadataPath == "archive.tar" && config.Feature("archive-site"):
+		case metadataPath == "archive.tar":
 			// same as above
 			_, err := AuthorizeMetadataRetrieval(r)
 			if err != nil {
@@ -190,13 +215,15 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 
 			// we only offer `/.git-pages/archive.tar` and not the `.tar.gz`/`.tar.zst` variants
 			// because HTTP can already request compression using the `Content-Encoding` mechanism
-			acceptedEncodings := parseHTTPEncodings(r.Header.Get("Accept-Encoding"))
+			acceptedEncodings := ParseAcceptEncodingHeader(r.Header.Get("Accept-Encoding"))
+			w.Header().Add("Vary", "Accept-Encoding")
 			negotiated := acceptedEncodings.Negotiate("zstd", "gzip", "identity")
 			if negotiated != "" {
 				w.Header().Set("Content-Encoding", negotiated)
 			}
 			w.Header().Add("Content-Type", "application/x-tar")
 			w.Header().Add("Last-Modified", lastModified)
+			w.Header().Add("ETag", fmt.Sprintf("\"%s-archive\"", metadata.ETag))
 			w.Header().Add("Transfer-Encoding", "chunked")
 			w.WriteHeader(http.StatusOK)
 			var iow io.Writer
@@ -208,7 +235,7 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 			case "zstd":
 				iow, _ = zstd.NewWriter(w)
 			}
-			return CollectTar(r.Context(), iow, manifest, manifestMtime)
+			return CollectTar(r.Context(), iow, manifest, metadata)
 
 		default:
 			w.WriteHeader(http.StatusNotFound)
@@ -220,7 +247,7 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 	entryPath := sitePath
 	entry := (*Entry)(nil)
 	appliedRedirect := false
-	status := 200
+	status := http.StatusOK
 	reader := io.ReadSeeker(nil)
 	mtime := time.Time{}
 	for {
@@ -234,11 +261,11 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 		entry = manifest.Contents[entryPath]
 		if !appliedRedirect {
 			redirectKind := RedirectAny
-			if entry != nil && entry.GetType() != Type_Invalid {
+			if entry != nil && entry.GetType() != Type_InvalidEntry {
 				redirectKind = RedirectForce
 			}
 			originalURL := (&url.URL{Host: r.Host}).ResolveReference(r.URL)
-			redirectURL, redirectStatus := ApplyRedirectRules(manifest, originalURL, redirectKind)
+			_, redirectURL, redirectStatus := ApplyRedirectRules(manifest, originalURL, redirectKind)
 			if Is3xxHTTPStatus(redirectStatus) {
 				writeRedirect(w, redirectStatus, redirectURL.String())
 				return nil
@@ -251,8 +278,8 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 				continue
 			}
 		}
-		if entry == nil || entry.GetType() == Type_Invalid {
-			status = 404
+		if entry == nil || entry.GetType() == Type_InvalidEntry {
+			status = http.StatusNotFound
 			if entryPath != notFoundPage {
 				entryPath = notFoundPage
 				continue
@@ -268,13 +295,15 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 				w.WriteHeader(http.StatusNotModified)
 				return nil
 			} else {
-				reader, _, mtime, err = backend.GetBlob(r.Context(), string(entry.Data))
+				var metadata BlobMetadata
+				reader, metadata, err = backend.GetBlob(r.Context(), string(entry.Data))
 				if err != nil {
 					ObserveError(err) // all storage errors must be reported
 					w.WriteHeader(http.StatusInternalServerError)
 					fmt.Fprintf(w, "internal server error: %s\n", err)
 					return err
 				}
+				mtime = metadata.LastModified
 				w.Header().Set("ETag", etag)
 			}
 		} else if entry.GetType() == Type_Directory {
@@ -297,26 +326,40 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 		defer closer.Close()
 	}
 
-	acceptedEncodings := parseHTTPEncodings(r.Header.Get("Accept-Encoding"))
+	var offeredEncodings []string
+	acceptedEncodings := ParseAcceptEncodingHeader(r.Header.Get("Accept-Encoding"))
+	w.Header().Add("Vary", "Accept-Encoding")
 	negotiatedEncoding := true
 	switch entry.GetTransform() {
-	case Transform_None:
-		if acceptedEncodings.Negotiate("identity") != "identity" {
+	case Transform_Identity:
+		offeredEncodings = []string{"identity"}
+		switch acceptedEncodings.Negotiate(offeredEncodings...) {
+		case "identity":
+			serveEncodingCount.
+				With(prometheus.Labels{"transform": "identity", "negotiated": "identity"}).
+				Inc()
+		default:
 			negotiatedEncoding = false
+			serveEncodingCount.
+				With(prometheus.Labels{"transform": "identity", "negotiated": "failure"}).
+				Inc()
 		}
-	case Transform_Zstandard:
-		supported := []string{"zstd", "identity"}
+	case Transform_Zstd:
+		offeredEncodings = []string{"zstd", "identity"}
 		if entry.ContentType == nil {
 			// If Content-Type is unset, `http.ServeContent` will try to sniff
 			// the file contents. That won't work if it's compressed.
-			supported = []string{"identity"}
+			offeredEncodings = []string{"identity"}
 		}
-		switch acceptedEncodings.Negotiate(supported...) {
+		switch acceptedEncodings.Negotiate(offeredEncodings...) {
 		case "zstd":
 			// Set Content-Length ourselves since `http.ServeContent` only sets
 			// it if Content-Encoding is unset or if it's a range request.
-			w.Header().Set("Content-Length", strconv.FormatInt(*entry.Size, 10))
+			w.Header().Set("Content-Length", strconv.FormatInt(entry.GetCompressedSize(), 10))
 			w.Header().Set("Content-Encoding", "zstd")
+			serveEncodingCount.
+				With(prometheus.Labels{"transform": "zstd", "negotiated": "zstd"}).
+				Inc()
 		case "identity":
 			compressedData, _ := io.ReadAll(reader)
 			decompressedData, err := zstdDecoder.DecodeAll(compressedData, []byte{})
@@ -326,15 +369,22 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 				return err
 			}
 			reader = bytes.NewReader(decompressedData)
+			serveEncodingCount.
+				With(prometheus.Labels{"transform": "zstd", "negotiated": "identity"}).
+				Inc()
 		default:
 			negotiatedEncoding = false
+			serveEncodingCount.
+				With(prometheus.Labels{"transform": "zstd", "negotiated": "failure"}).
+				Inc()
 		}
 	default:
 		return fmt.Errorf("unexpected transform")
 	}
 	if !negotiatedEncoding {
+		w.Header().Set("Accept-Encoding", strings.Join(offeredEncodings, ", "))
 		w.WriteHeader(http.StatusNotAcceptable)
-		return fmt.Errorf("no supported content encodings (accept-encoding: %q)",
+		return fmt.Errorf("no supported content encodings (Accept-Encoding: %s)",
 			r.Header.Get("Accept-Encoding"))
 	}
 
@@ -369,13 +419,15 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 			io.Copy(w, reader)
 		}
 	} else {
-		// consider content fresh for 60 seconds (the same as the freshness interval of
-		// manifests in the S3 backend), and use stale content anyway as long as it's not
-		// older than a hour; while it is cheap to handle If-Modified-Since queries
-		// server-side, on the client `max-age=0, must-revalidate` causes every resource
-		// to block the page load every time
-		w.Header().Set("Cache-Control", "max-age=60, stale-while-revalidate=3600")
-		// see https://web.dev/articles/stale-while-revalidate for details
+		if _, hasCacheControl := w.Header()["Cache-Control"]; !hasCacheControl {
+			// consider content fresh for 60 seconds (the same as the freshness interval of
+			// manifests in the S3 backend), and use stale content anyway as long as it's not
+			// older than a hour; while it is cheap to handle If-Modified-Since queries
+			// server-side, on the client `max-age=0, must-revalidate` causes every resource
+			// to block the page load every time
+			w.Header().Set("Cache-Control", "max-age=60, stale-while-revalidate=3600")
+			// see https://web.dev/articles/stale-while-revalidate for details
+		}
 
 		// http.ServeContent handles conditional requests and range requests
 		http.ServeContent(w, r, entryPath, mtime, reader)
@@ -383,27 +435,39 @@ func getPage(w http.ResponseWriter, r *http.Request) error {
 	return nil
 }
 
+func checkDryRun(w http.ResponseWriter, r *http.Request) bool {
+	// "Dry run" requests are used to non-destructively check if the request would have
+	// successfully been authorized.
+	if r.Header.Get("Dry-Run") != "" {
+		fmt.Fprintln(w, "dry-run ok")
+		return true
+	}
+	return false
+}
+
 func putPage(w http.ResponseWriter, r *http.Request) error {
 	var result UpdateResult
 
-	host, err := GetHost(r)
+	for _, header := range []string{
+		"If-Modified-Since", "If-Unmodified-Since", "If-Match", "If-None-Match",
+	} {
+		if r.Header.Get(header) != "" {
+			http.Error(w, fmt.Sprintf("unsupported precondition %s", header), http.StatusBadRequest)
+			return nil
+		}
+	}
+
+	webRoot, err := getWebRoot(r)
 	if err != nil {
 		return err
 	}
 
-	projectName, err := GetProjectName(r)
-	if err != nil {
-		return err
-	}
-
-	webRoot := makeWebRoot(host, projectName)
-
-	updateCtx, cancel := context.WithTimeout(r.Context(), time.Duration(config.Limits.UpdateTimeout))
+	ctx, cancel := context.WithTimeout(r.Context(), time.Duration(config.Limits.UpdateTimeout))
 	defer cancel()
 
 	contentType := getMediaType(r.Header.Get("Content-Type"))
-
-	if contentType == "application/x-www-form-urlencoded" {
+	switch contentType {
+	case "application/x-www-form-urlencoded":
 		auth, err := AuthorizeUpdateFromRepository(r)
 		if err != nil {
 			return err
@@ -428,26 +492,140 @@ func putPage(w http.ResponseWriter, r *http.Request) error {
 			return err
 		}
 
-		result = UpdateFromRepository(updateCtx, webRoot, repoURL, branch)
-	} else {
-		_, err := AuthorizeUpdateFromArchive(r)
-		if err != nil {
+		if checkDryRun(w, r) {
+			return nil
+		}
+
+		result = UpdateFromRepository(ctx, webRoot, repoURL, branch)
+
+	default:
+		if auth, err := AuthorizeUpdateFromArchive(r); err != nil {
 			return err
+		} else if auth.forgeUser != nil {
+			GetPrincipal(r.Context()).ForgeUser = auth.forgeUser
+		}
+
+		if checkDryRun(w, r) {
+			return nil
 		}
 
 		// request body contains archive
 		reader := http.MaxBytesReader(w, r.Body, int64(config.Limits.MaxSiteSize.Bytes()))
-		result = UpdateFromArchive(updateCtx, webRoot, contentType, reader)
+		result = UpdateFromArchive(ctx, webRoot, contentType, reader)
+	}
+
+	return reportUpdateResult(w, r, result)
+}
+
+func patchPage(w http.ResponseWriter, r *http.Request) error {
+	for _, header := range []string{
+		"If-Modified-Since", "If-Unmodified-Since", "If-Match", "If-None-Match",
+	} {
+		if r.Header.Get(header) != "" {
+			http.Error(w, fmt.Sprintf("unsupported precondition %s", header), http.StatusBadRequest)
+			return nil
+		}
+	}
+
+	webRoot, err := getWebRoot(r)
+	if err != nil {
+		return err
+	}
+
+	if auth, err := AuthorizeUpdateFromArchive(r); err != nil {
+		return err
+	} else if auth.forgeUser != nil {
+		GetPrincipal(r.Context()).ForgeUser = auth.forgeUser
+	}
+
+	if checkDryRun(w, r) {
+		return nil
+	}
+
+	// Providing atomic compare-and-swap operations might be difficult or impossible depending
+	// on the backend in use and its configuration, but for applications where a mostly-atomic
+	// compare-and-swap operation is good enough (e.g. generating page previews) we don't want
+	// to prevent the use of partial updates.
+	wantAtomicCAS := r.Header.Get("Atomic")
+	hasAtomicCAS := backend.HasAtomicCAS(r.Context())
+	switch {
+	case wantAtomicCAS == "yes" && hasAtomicCAS || wantAtomicCAS == "no":
+		// all good
+	case wantAtomicCAS == "yes":
+		http.Error(w, "atomic partial updates unsupported", http.StatusPreconditionFailed)
+		return nil
+	case wantAtomicCAS == "":
+		http.Error(w, "must provide \"Atomic: yes|no\" header", http.StatusPreconditionRequired)
+		return nil
+	default:
+		http.Error(w, "malformed Atomic: header", http.StatusBadRequest)
+		return nil
+	}
+
+	var parents CreateParentsMode
+	switch r.Header.Get("Create-Parents") {
+	case "", "no":
+		parents = RequireParents
+	case "yes":
+		parents = CreateParents
+	default:
+		http.Error(w, "malformed Create-Parents: header", http.StatusBadRequest)
+		return nil
+	}
+
+	ctx, cancel := context.WithTimeout(r.Context(), time.Duration(config.Limits.UpdateTimeout))
+	defer cancel()
+
+	contentType := getMediaType(r.Header.Get("Content-Type"))
+	reader := http.MaxBytesReader(w, r.Body, int64(config.Limits.MaxSiteSize.Bytes()))
+	result := PartialUpdateFromArchive(ctx, webRoot, contentType, reader, parents)
+	return reportUpdateResult(w, r, result)
+}
+
+func reportUpdateResult(w http.ResponseWriter, r *http.Request, result UpdateResult) error {
+	var unresolvedRefErr UnresolvedRefError
+	if result.outcome == UpdateError && errors.As(result.err, &unresolvedRefErr) {
+		offeredContentTypes := []string{"text/plain", "application/vnd.git-pages.unresolved"}
+		acceptedContentTypes := ParseAcceptHeader(r.Header.Get("Accept"))
+		switch acceptedContentTypes.Negotiate(offeredContentTypes...) {
+		default:
+			w.Header().Set("Accept", strings.Join(offeredContentTypes, ", "))
+			w.WriteHeader(http.StatusNotAcceptable)
+			return fmt.Errorf("no supported content types (Accept: %s)", r.Header.Get("Accept"))
+		case "application/vnd.git-pages.unresolved":
+			w.Header().Set("Content-Type", "application/vnd.git-pages.unresolved")
+			w.WriteHeader(http.StatusUnprocessableEntity)
+			for _, missingRef := range unresolvedRefErr.missing {
+				fmt.Fprintln(w, missingRef)
+			}
+			return nil
+		case "text/plain":
+			// handled below
+		}
 	}
 
 	switch result.outcome {
 	case UpdateError:
-		if errors.Is(result.err, ErrManifestTooLarge) {
-			w.WriteHeader(http.StatusRequestEntityTooLarge)
+		if errors.Is(result.err, ErrSiteTooLarge) {
+			w.WriteHeader(http.StatusUnprocessableEntity)
+		} else if errors.Is(result.err, ErrManifestTooLarge) {
+			w.WriteHeader(http.StatusUnprocessableEntity)
 		} else if errors.Is(result.err, errArchiveFormat) {
 			w.WriteHeader(http.StatusUnsupportedMediaType)
 		} else if errors.Is(result.err, ErrArchiveTooLarge) {
 			w.WriteHeader(http.StatusRequestEntityTooLarge)
+		} else if errors.Is(result.err, ErrRepositoryTooLarge) {
+			w.WriteHeader(http.StatusUnprocessableEntity)
+		} else if errors.Is(result.err, ErrMalformedPatch) {
+			w.WriteHeader(http.StatusUnprocessableEntity)
+		} else if errors.Is(result.err, ErrPreconditionFailed) {
+			w.WriteHeader(http.StatusPreconditionFailed)
+		} else if errors.Is(result.err, ErrWriteConflict) {
+			w.WriteHeader(http.StatusConflict)
+		} else if errors.Is(result.err, ErrDomainFrozen) {
+			w.WriteHeader(http.StatusForbidden)
+		} else if errors.As(result.err, &unresolvedRefErr) {
+			w.WriteHeader(http.StatusUnprocessableEntity)
 		} else {
 			w.WriteHeader(http.StatusServiceUnavailable)
 		}
@@ -476,61 +654,53 @@ func putPage(w http.ResponseWriter, r *http.Request) error {
 	} else {
 		fmt.Fprintln(w, "internal error")
 	}
-	reportSiteUpdate("rest", &result)
+	observeSiteUpdate("rest", &result)
 	return nil
 }
 
 func deletePage(w http.ResponseWriter, r *http.Request) error {
-	_, err := AuthorizeUpdateFromRepository(r)
+	webRoot, err := getWebRoot(r)
 	if err != nil {
 		return err
 	}
 
-	host, err := GetHost(r)
-	if err != nil {
+	if auth, err := AuthorizeDeletion(r); err != nil {
 		return err
+	} else if auth.forgeUser != nil {
+		GetPrincipal(r.Context()).ForgeUser = auth.forgeUser
 	}
 
-	projectName, err := GetProjectName(r)
-	if err != nil {
-		return err
+	if checkDryRun(w, r) {
+		return nil
 	}
 
-	err = backend.DeleteManifest(r.Context(), makeWebRoot(host, projectName))
-	if err != nil {
+	if err = backend.DeleteManifest(r.Context(), webRoot, ModifyManifestOptions{}); err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
+		fmt.Fprintln(w, err)
 	} else {
 		w.Header().Add("Update-Result", "deleted")
 		w.WriteHeader(http.StatusOK)
 	}
-	if err != nil {
-		fmt.Fprintln(w, err)
-	}
 	return err
 }
 
 func postPage(w http.ResponseWriter, r *http.Request) error {
-	// Start a timer for the request timeout immediately.
+	// The HTTP requests for webhook delivery usually have a short timeout. We start the timer
+	// before doing any time-consuming work so that it's closely aligned to the client's timeout and
+	// we can respond before the webhook delivery is considered failed.
 	requestTimeout := 3 * time.Second
 	requestTimer := time.NewTimer(requestTimeout)
 
+	webRoot, err := getWebRoot(r)
+	if err != nil {
+		return err
+	}
+
 	auth, err := AuthorizeUpdateFromRepository(r)
 	if err != nil {
 		return err
 	}
 
-	host, err := GetHost(r)
-	if err != nil {
-		return err
-	}
-
-	projectName, err := GetProjectName(r)
-	if err != nil {
-		return err
-	}
-
-	webRoot := makeWebRoot(host, projectName)
-
 	eventName := ""
 	for _, header := range []string{
 		"X-Forgejo-Event",
@@ -578,7 +748,7 @@ func postPage(w http.ResponseWriter, r *http.Request) error {
 		return err
 	}
 
-	if event.Ref != fmt.Sprintf("refs/heads/%s", auth.branch) {
+	if event.Ref != path.Join("refs", "heads", auth.branch) {
 		code := http.StatusUnauthorized
 		if strings.Contains(r.Header.Get("User-Agent"), "GitHub-Hookshot") {
 			// GitHub has no way to restrict branches for a webhook, and responding with 401
@@ -596,6 +766,10 @@ func postPage(w http.ResponseWriter, r *http.Request) error {
 		return err
 	}
 
+	if checkDryRun(w, r) {
+		return nil
+	}
+
 	resultChan := make(chan UpdateResult)
 	go func(ctx context.Context) {
 		ctx, cancel := context.WithTimeout(ctx, time.Duration(config.Limits.UpdateTimeout))
@@ -603,8 +777,8 @@ func postPage(w http.ResponseWriter, r *http.Request) error {
 
 		result := UpdateFromRepository(ctx, webRoot, repoURL, auth.branch)
 		resultChan <- result
-		reportSiteUpdate("webhook", &result)
-	}(context.Background())
+		observeSiteUpdate("webhook", &result)
+	}(context.WithoutCancel(r.Context()))
 
 	var result UpdateResult
 	select {
@@ -623,16 +797,12 @@ func postPage(w http.ResponseWriter, r *http.Request) error {
 		w.WriteHeader(http.StatusGatewayTimeout)
 		fmt.Fprintln(w, "update timeout")
 	case UpdateNoChange:
-		w.WriteHeader(http.StatusOK)
 		fmt.Fprintln(w, "unchanged")
 	case UpdateCreated:
-		w.WriteHeader(http.StatusOK)
 		fmt.Fprintln(w, "created")
 	case UpdateReplaced:
-		w.WriteHeader(http.StatusOK)
 		fmt.Fprintln(w, "replaced")
 	case UpdateDeleted:
-		w.WriteHeader(http.StatusOK)
 		fmt.Fprintln(w, "deleted")
 	}
 	if result.manifest != nil {
@@ -648,11 +818,22 @@ func postPage(w http.ResponseWriter, r *http.Request) error {
 }
 
 func ServePages(w http.ResponseWriter, r *http.Request) {
+	r = r.WithContext(WithPrincipal(r.Context()))
+	if config.Audit.IncludeIPs != "" {
+		GetPrincipal(r.Context()).IpAddress = proto.String(r.RemoteAddr)
+	}
 	// We want upstream health checks to be done as closely to the normal flow as possible;
 	// any intentional deviation is an opportunity to miss an issue that will affect our
 	// visitors but not our health checks.
 	if r.Header.Get("Health-Check") == "" {
-		log.Println("pages:", r.Method, r.Host, r.URL, r.Header.Get("Content-Type"))
+		var mediaType string
+		switch r.Method {
+		case "HEAD", "GET":
+			mediaType = r.Header.Get("Accept")
+		default:
+			mediaType = r.Header.Get("Content-Type")
+		}
+		logc.Println(r.Context(), "pages:", r.Method, r.Host, r.URL, mediaType)
 		if region := os.Getenv("FLY_REGION"); region != "" {
 			machine_id := os.Getenv("FLY_MACHINE_ID")
 			w.Header().Add("Server", fmt.Sprintf("git-pages (fly.io; %s; %s)", region, machine_id))
@@ -667,20 +848,27 @@ func ServePages(w http.ResponseWriter, r *http.Request) {
 			}
 		}
 	}
+	allowedMethods := []string{"OPTIONS", "HEAD", "GET", "PUT", "PATCH", "DELETE", "POST"}
+	if r.Method == "OPTIONS" || !slices.Contains(allowedMethods, r.Method) {
+		w.Header().Add("Allow", strings.Join(allowedMethods, ", "))
+	}
 	err := error(nil)
 	switch r.Method {
 	// REST API
-	case http.MethodHead, http.MethodGet:
+	case "OPTIONS":
+		// no preflight options
+	case "HEAD", "GET":
 		err = getPage(w, r)
-	case http.MethodPut:
+	case "PUT":
 		err = putPage(w, r)
-	case http.MethodDelete:
+	case "PATCH":
+		err = patchPage(w, r)
+	case "DELETE":
 		err = deletePage(w, r)
 	// webhook API
-	case http.MethodPost:
+	case "POST":
 		err = postPage(w, r)
 	default:
-		w.Header().Add("Allow", "HEAD, GET, PUT, DELETE, POST")
 		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
 		err = fmt.Errorf("method %s not allowed", r.Method)
 	}
@@ -695,6 +883,6 @@ func ServePages(w http.ResponseWriter, r *http.Request) {
 			http.Error(w, message, http.StatusRequestEntityTooLarge)
 			err = errors.New(message)
 		}
-		log.Println("pages err:", err)
+		logc.Println(r.Context(), "pages err:", err)
 	}
 }

src/pages_test.go

@@ -0,0 +1,55 @@
+package git_pages
+
+import (
+	"net/http"
+	"strings"
+	"testing"
+)
+
+func checkHost(t *testing.T, host string, expectOk string, expectErr string) {
+	host, err := GetHost(&http.Request{Host: host})
+	if expectErr != "" {
+		if err == nil || !strings.HasPrefix(err.Error(), expectErr) {
+			t.Errorf("%s: expect err %s, got err %s", host, expectErr, err)
+		}
+	}
+	if expectOk != "" {
+		if err != nil {
+			t.Errorf("%s: expect ok %s, got err %s", host, expectOk, err)
+		} else if host != expectOk {
+			t.Errorf("%s: expect ok %s, got ok %s", host, expectOk, host)
+		}
+	}
+}
+
+func TestHelloName(t *testing.T) {
+	config = &Config{Features: []string{}}
+
+	checkHost(t, "foo.bar", "foo.bar", "")
+	checkHost(t, "foo-baz.bar", "foo-baz.bar", "")
+	checkHost(t, "foo--baz.bar", "foo--baz.bar", "")
+	checkHost(t, "foo.bar.", "foo.bar", "")
+	checkHost(t, ".foo.bar", "", "reserved host name")
+	checkHost(t, "..foo.bar", "", "reserved host name")
+
+	checkHost(t, "ß.bar", "xn--zca.bar", "")
+	checkHost(t, "xn--zca.bar", "xn--zca.bar", "")
+
+	checkHost(t, "foo-.bar", "", "malformed host name")
+	checkHost(t, "-foo.bar", "", "malformed host name")
+	checkHost(t, "foo_.bar", "", "malformed host name")
+	checkHost(t, "_foo.bar", "", "malformed host name")
+	checkHost(t, "foo_baz.bar", "", "malformed host name")
+	checkHost(t, "foo__baz.bar", "", "malformed host name")
+	checkHost(t, "*.foo.bar", "", "malformed host name")
+
+	config = &Config{Features: []string{"relaxed-idna"}}
+
+	checkHost(t, "foo-.bar", "", "malformed host name")
+	checkHost(t, "-foo.bar", "", "malformed host name")
+	checkHost(t, "foo_.bar", "foo_.bar", "")
+	checkHost(t, "_foo.bar", "", "reserved host name")
+	checkHost(t, "foo_baz.bar", "foo_baz.bar", "")
+	checkHost(t, "foo__baz.bar", "foo__baz.bar", "")
+	checkHost(t, "*.foo.bar", "", "malformed host name")
+}

src/patch.go

@@ -0,0 +1,142 @@
+package git_pages
+
+import (
+	"archive/tar"
+	"errors"
+	"fmt"
+	"io"
+	"maps"
+	"slices"
+	"strings"
+)
+
+var ErrMalformedPatch = errors.New("malformed patch")
+
+type CreateParentsMode int
+
+const (
+	RequireParents CreateParentsMode = iota
+	CreateParents
+)
+
+// Mutates `manifest` according to a tar stream and the following rules:
+//   - A character device with major 0 and minor 0 is a "whiteout marker".  When placed
+//     at a given path, this path and its entire subtree (if any) are removed from the manifest.
+//   - When a directory is placed at a given path, this path and its entire subtree (if any) are
+//     removed from the manifest and replaced with the contents of the directory.
+func ApplyTarPatch(manifest *Manifest, reader io.Reader, parents CreateParentsMode) error {
+	type Node struct {
+		entry    *Entry
+		children map[string]*Node
+	}
+
+	// Extract the manifest contents (which is using a flat hash map) into a directory tree
+	// so that recursive delete operations have O(1) complexity. s
+	var root *Node
+	sortedNames := slices.Sorted(maps.Keys(manifest.GetContents()))
+	for _, name := range sortedNames {
+		entry := manifest.Contents[name]
+		node := &Node{entry: entry}
+		if entry.GetType() == Type_Directory {
+			node.children = map[string]*Node{}
+		}
+		if name == "" {
+			root = node
+		} else {
+			segments := strings.Split(name, "/")
+			fileName := segments[len(segments)-1]
+			iter := root
+			for _, segment := range segments[:len(segments)-1] {
+				if iter.children == nil {
+					panic("malformed manifest (not a directory)")
+				} else if _, exists := iter.children[segment]; !exists {
+					panic("malformed manifest (node does not exist)")
+				} else {
+					iter = iter.children[segment]
+				}
+			}
+			iter.children[fileName] = node
+		}
+	}
+	manifest.Contents = map[string]*Entry{}
+
+	// Process the archive as a patch operation.
+	archive := tar.NewReader(reader)
+	for {
+		header, err := archive.Next()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return err
+		}
+
+		segments := strings.Split(normalizeArchiveMemberName(header.Name), "/")
+		fileName := segments[len(segments)-1]
+		node := root
+		for index, segment := range segments[:len(segments)-1] {
+			if node.children == nil {
+				dirName := strings.Join(segments[:index], "/")
+				return fmt.Errorf("%w: %s: not a directory", ErrMalformedPatch, dirName)
+			}
+			if _, exists := node.children[segment]; !exists {
+				switch parents {
+				case RequireParents:
+					nodeName := strings.Join(segments[:index+1], "/")
+					return fmt.Errorf("%w: %s: path not found", ErrMalformedPatch, nodeName)
+				case CreateParents:
+					node.children[segment] = &Node{
+						entry:    NewManifestEntry(Type_Directory, nil),
+						children: map[string]*Node{},
+					}
+				}
+			}
+			node = node.children[segment]
+		}
+		if node.children == nil {
+			dirName := strings.Join(segments[:len(segments)-1], "/")
+			return fmt.Errorf("%w: %s: not a directory", ErrMalformedPatch, dirName)
+		}
+
+		switch header.Typeflag {
+		case tar.TypeReg:
+			fileData, err := io.ReadAll(archive)
+			if err != nil {
+				return fmt.Errorf("tar: %s: %w", header.Name, err)
+			}
+			node.children[fileName] = &Node{
+				entry: NewManifestEntry(Type_InlineFile, fileData),
+			}
+		case tar.TypeSymlink:
+			node.children[fileName] = &Node{
+				entry: NewManifestEntry(Type_Symlink, []byte(header.Linkname)),
+			}
+		case tar.TypeDir:
+			node.children[fileName] = &Node{
+				entry:    NewManifestEntry(Type_Directory, nil),
+				children: map[string]*Node{},
+			}
+		case tar.TypeChar:
+			if header.Devmajor == 0 && header.Devminor == 0 {
+				delete(node.children, fileName)
+			} else {
+				AddProblem(manifest, header.Name,
+					"tar: unsupported chardev %d,%d", header.Devmajor, header.Devminor)
+			}
+		default:
+			AddProblem(manifest, header.Name,
+				"tar: unsupported type '%c'", header.Typeflag)
+			continue
+		}
+	}
+
+	// Repopulate manifest contents with the updated directory tree.
+	var traverse func([]string, *Node)
+	traverse = func(segments []string, node *Node) {
+		manifest.Contents[strings.Join(segments, "/")] = node.entry
+		for fileName, childNode := range node.children {
+			traverse(append(segments, fileName), childNode)
+		}
+	}
+	traverse([]string{}, root)
+	return nil
+}

src/redirects.go

@@ -13,7 +13,17 @@ import (
 
 const RedirectsFileName string = "_redirects"
 
-func unparseRule(rule redirects.Rule) string {
+// Converts our Protobuf representation to tj/go-redirects.
+func exportRedirectRule(rule *RedirectRule) *redirects.Rule {
+	return &redirects.Rule{
+		From:   rule.GetFrom(),
+		To:     rule.GetTo(),
+		Status: int(rule.GetStatus()),
+		Force:  rule.GetForce(),
+	}
+}
+
+func unparseRedirectRule(rule *redirects.Rule) string {
 	var statusPart string
 	if rule.Force {
 		statusPart = fmt.Sprintf("%d!", rule.Status)
@@ -49,7 +59,7 @@ func Is3xxHTTPStatus(status int) bool {
 	return status >= 300 && status <= 399
 }
 
-func validateRedirectRule(rule redirects.Rule) error {
+func validateRedirectRule(rule *redirects.Rule) error {
 	if len(rule.Params) > 0 {
 		return fmt.Errorf("rules with parameters are not supported")
 	}
@@ -103,9 +113,9 @@ func ProcessRedirectsFile(manifest *Manifest) error {
 	}
 
 	for index, rule := range rules {
-		if err := validateRedirectRule(rule); err != nil {
+		if err := validateRedirectRule(&rule); err != nil {
 			AddProblem(manifest, RedirectsFileName,
-				"rule #%d %q: %s", index+1, unparseRule(rule), err)
+				"rule #%d %q: %s", index+1, unparseRedirectRule(&rule), err)
 			continue
 		}
 		manifest.Redirects = append(manifest.Redirects, &RedirectRule{
@@ -121,12 +131,7 @@ func ProcessRedirectsFile(manifest *Manifest) error {
 func CollectRedirectsFile(manifest *Manifest) string {
 	var rules []string
 	for _, rule := range manifest.GetRedirects() {
-		rules = append(rules, unparseRule(redirects.Rule{
-			From:   rule.GetFrom(),
-			To:     rule.GetTo(),
-			Status: int(rule.GetStatus()),
-			Force:  rule.GetForce(),
-		})+"\n")
+		rules = append(rules, unparseRedirectRule(exportRedirectRule(rule))+"\n")
 	}
 	return strings.Join(rules, "")
 }
@@ -147,18 +152,22 @@ type RedirectKind int
 
 const (
 	RedirectAny RedirectKind = iota
+	RedirectNormal
 	RedirectForce
 )
 
 func ApplyRedirectRules(
 	manifest *Manifest, fromURL *url.URL, kind RedirectKind,
 ) (
-	toURL *url.URL, status int,
+	rule *RedirectRule, toURL *url.URL, status int,
 ) {
 	fromSegments := pathSegments(fromURL.Path)
 next:
-	for _, rule := range manifest.Redirects {
-		if kind == RedirectForce && !*rule.Force {
+	for _, rule = range manifest.Redirects {
+		switch {
+		case kind == RedirectNormal && *rule.Force:
+			continue
+		case kind == RedirectForce && !*rule.Force:
 			continue
 		}
 		// check if the rule matches fromURL
@@ -205,8 +214,43 @@ next:
 			RawQuery: fromURL.RawQuery,
 		}
 		status = int(*rule.Status)
-		break
+		return
 	}
 	// no redirect found
+	rule = nil
 	return
 }
+
+func redirectHasSplat(rule *RedirectRule) bool {
+	ruleFromURL, _ := url.Parse(*rule.From) // pre-validated in `validateRedirectRule`
+	ruleFromSegments := pathSegments(ruleFromURL.Path)
+	return slices.Contains(ruleFromSegments, "*")
+}
+
+func LintRedirects(manifest *Manifest) {
+	for name, entry := range manifest.GetContents() {
+		nameURL, err := url.Parse("/" + name)
+		if err != nil {
+			continue
+		}
+
+		// Check if the entry URL would trigger a non-forced redirect if the entry didn't exist.
+		// If the redirect matches exactly one URL (i.e. has no splat) then it will never be
+		// triggered and an issue is reported; if the rule has a splat, it will always be possible
+		// to trigger it, as it matches an infinite number of URLs.
+		rule, _, _ := ApplyRedirectRules(manifest, nameURL, RedirectNormal)
+		if rule != nil && !redirectHasSplat(rule) {
+			entryDesc := "file"
+			if entry.GetType() == Type_Directory {
+				entryDesc = "directory"
+			}
+			AddProblem(manifest, name,
+				"%s shadows redirect %q; remove the %s or use a %d! forced redirect instead",
+				entryDesc,
+				unparseRedirectRule(exportRedirectRule(rule)),
+				entryDesc,
+				rule.GetStatus(),
+			)
+		}
+	}
+}

src/schema.pb.go

@@ -9,6 +9,7 @@ package git_pages
 import (
 	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	timestamppb "google.golang.org/protobuf/types/known/timestamppb"
 	reflect "reflect"
 	sync "sync"
 	unsafe "unsafe"
@@ -25,7 +26,7 @@ type Type int32
 
 const (
 	// Invalid entry.
-	Type_Invalid Type = 0
+	Type_InvalidEntry Type = 0
 	// Directory.
 	Type_Directory Type = 1
 	// Inline file. `Blob.Data` contains file contents.
@@ -39,14 +40,14 @@ const (
 // Enum value maps for Type.
 var (
 	Type_name = map[int32]string{
-		0: "Invalid",
+		0: "InvalidEntry",
 		1: "Directory",
 		2: "InlineFile",
 		3: "ExternalFile",
 		4: "Symlink",
 	}
 	Type_value = map[string]int32{
-		"Invalid":      0,
+		"InvalidEntry": 0,
 		"Directory":    1,
 		"InlineFile":   2,
 		"ExternalFile": 3,
@@ -81,24 +82,25 @@ func (Type) EnumDescriptor() ([]byte, []int) {
 	return file_schema_proto_rawDescGZIP(), []int{0}
 }
 
+// Transformation names should match HTTP `Accept-Encoding:` header.
 type Transform int32
 
 const (
 	// No transformation.
-	Transform_None Transform = 0
+	Transform_Identity Transform = 0
 	// Zstandard compression.
-	Transform_Zstandard Transform = 1
+	Transform_Zstd Transform = 1
 )
 
 // Enum value maps for Transform.
 var (
 	Transform_name = map[int32]string{
-		0: "None",
-		1: "Zstandard",
+		0: "Identity",
+		1: "Zstd",
 	}
 	Transform_value = map[string]int32{
-		"None":      0,
-		"Zstandard": 1,
+		"Identity": 0,
+		"Zstd":     1,
 	}
 )
 
@@ -129,12 +131,77 @@ func (Transform) EnumDescriptor() ([]byte, []int) {
 	return file_schema_proto_rawDescGZIP(), []int{1}
 }
 
+type AuditEvent int32
+
+const (
+	// Invalid event.
+	AuditEvent_InvalidEvent AuditEvent = 0
+	// A manifest was committed (a site was created or updated).
+	AuditEvent_CommitManifest AuditEvent = 1
+	// A manifest was deleted (a site was deleted).
+	AuditEvent_DeleteManifest AuditEvent = 2
+	// A domain was frozen.
+	AuditEvent_FreezeDomain AuditEvent = 3
+	// A domain was thawed.
+	AuditEvent_UnfreezeDomain AuditEvent = 4
+)
+
+// Enum value maps for AuditEvent.
+var (
+	AuditEvent_name = map[int32]string{
+		0: "InvalidEvent",
+		1: "CommitManifest",
+		2: "DeleteManifest",
+		3: "FreezeDomain",
+		4: "UnfreezeDomain",
+	}
+	AuditEvent_value = map[string]int32{
+		"InvalidEvent":   0,
+		"CommitManifest": 1,
+		"DeleteManifest": 2,
+		"FreezeDomain":   3,
+		"UnfreezeDomain": 4,
+	}
+)
+
+func (x AuditEvent) Enum() *AuditEvent {
+	p := new(AuditEvent)
+	*p = x
+	return p
+}
+
+func (x AuditEvent) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (AuditEvent) Descriptor() protoreflect.EnumDescriptor {
+	return file_schema_proto_enumTypes[2].Descriptor()
+}
+
+func (AuditEvent) Type() protoreflect.EnumType {
+	return &file_schema_proto_enumTypes[2]
+}
+
+func (x AuditEvent) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use AuditEvent.Descriptor instead.
+func (AuditEvent) EnumDescriptor() ([]byte, []int) {
+	return file_schema_proto_rawDescGZIP(), []int{2}
+}
+
 type Entry struct {
 	state protoimpl.MessageState `protogen:"open.v1"`
 	Type  *Type                  `protobuf:"varint,1,opt,name=type,enum=Type" json:"type,omitempty"`
 	// Only present for `type == InlineFile` and `type == ExternalFile`.
-	// For transformed entries, refers to the post-transformation (compressed) size.
-	Size *int64 `protobuf:"varint,2,opt,name=size" json:"size,omitempty"`
+	// For transformed entries, refers to the pre-transformation (decompressed) size; otherwise
+	// equal to `compressed_size`.
+	OriginalSize *int64 `protobuf:"varint,7,opt,name=original_size,json=originalSize" json:"original_size,omitempty"`
+	// Only present for `type == InlineFile` and `type == ExternalFile`.
+	// For transformed entries, refers to the post-transformation (compressed) size; otherwise
+	// equal to `original_size`.
+	CompressedSize *int64 `protobuf:"varint,2,opt,name=compressed_size,json=compressedSize" json:"compressed_size,omitempty"`
 	// Meaning depends on `type`:
 	//   - If `type == InlineFile`, contains file data.
 	//   - If `type == ExternalFile`, contains blob name (an otherwise unspecified
@@ -147,7 +214,13 @@ type Entry struct {
 	Transform *Transform `protobuf:"varint,4,opt,name=transform,enum=Transform" json:"transform,omitempty"`
 	// Only present for `type == InlineFile` and `type == ExternalFile`.
 	// Currently, optional (not present on certain legacy manifests).
-	ContentType   *string `protobuf:"bytes,5,opt,name=content_type,json=contentType" json:"content_type,omitempty"`
+	ContentType *string `protobuf:"bytes,5,opt,name=content_type,json=contentType" json:"content_type,omitempty"`
+	// May be present for `type == InlineFile` and `type == ExternalFile`.
+	// Used to reduce the amount of work being done during git checkouts.
+	// The type of hash used is determined by the length:
+	//   - 40 bytes: SHA1DC (as hex)
+	//   - 64 bytes: SHA256 (as hex)
+	GitHash       *string `protobuf:"bytes,6,opt,name=git_hash,json=gitHash" json:"git_hash,omitempty"`
 	unknownFields protoimpl.UnknownFields
 	sizeCache     protoimpl.SizeCache
 }
@@ -186,12 +259,19 @@ func (x *Entry) GetType() Type {
 	if x != nil && x.Type != nil {
 		return *x.Type
 	}
-	return Type_Invalid
+	return Type_InvalidEntry
 }
 
-func (x *Entry) GetSize() int64 {
-	if x != nil && x.Size != nil {
-		return *x.Size
+func (x *Entry) GetOriginalSize() int64 {
+	if x != nil && x.OriginalSize != nil {
+		return *x.OriginalSize
+	}
+	return 0
+}
+
+func (x *Entry) GetCompressedSize() int64 {
+	if x != nil && x.CompressedSize != nil {
+		return *x.CompressedSize
 	}
 	return 0
 }
@@ -207,7 +287,7 @@ func (x *Entry) GetTransform() Transform {
 	if x != nil && x.Transform != nil {
 		return *x.Transform
 	}
-	return Transform_None
+	return Transform_Identity
 }
 
 func (x *Entry) GetContentType() string {
@@ -217,6 +297,13 @@ func (x *Entry) GetContentType() string {
 	return ""
 }
 
+func (x *Entry) GetGitHash() string {
+	if x != nil && x.GitHash != nil {
+		return *x.GitHash
+	}
+	return ""
+}
+
 // See https://docs.netlify.com/manage/routing/redirects/overview/ for details.
 // Only a subset of the Netlify specification is representable here.
 type RedirectRule struct {
@@ -446,19 +533,19 @@ func (x *Problem) GetCause() string {
 
 type Manifest struct {
 	state protoimpl.MessageState `protogen:"open.v1"`
-	// Source metadata
+	// Source metadata.
 	RepoUrl *string `protobuf:"bytes,1,opt,name=repo_url,json=repoUrl" json:"repo_url,omitempty"`
 	Branch  *string `protobuf:"bytes,2,opt,name=branch" json:"branch,omitempty"`
 	Commit  *string `protobuf:"bytes,3,opt,name=commit" json:"commit,omitempty"`
-	// Contents
+	// Site contents.
 	Contents       map[string]*Entry `protobuf:"bytes,4,rep,name=contents" json:"contents,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
-	OriginalSize   *int64            `protobuf:"varint,10,opt,name=original_size,json=originalSize" json:"original_size,omitempty"`      // total size of entries before compression
-	CompressedSize *int64            `protobuf:"varint,5,opt,name=compressed_size,json=compressedSize" json:"compressed_size,omitempty"` // simple sum of each `entry.size`
-	StoredSize     *int64            `protobuf:"varint,8,opt,name=stored_size,json=storedSize" json:"stored_size,omitempty"`             // total size of (deduplicated) external objects
-	// Netlify-style `_redirects` and `_headers`
+	OriginalSize   *int64            `protobuf:"varint,10,opt,name=original_size,json=originalSize" json:"original_size,omitempty"`      // sum of each `entry.original_size`
+	CompressedSize *int64            `protobuf:"varint,5,opt,name=compressed_size,json=compressedSize" json:"compressed_size,omitempty"` // sum of each `entry.compressed_size`
+	StoredSize     *int64            `protobuf:"varint,8,opt,name=stored_size,json=storedSize" json:"stored_size,omitempty"`             // sum of deduplicated `entry.compressed_size` for external files only
+	// Netlify-style `_redirects` and `_headers` rules.
 	Redirects []*RedirectRule `protobuf:"bytes,6,rep,name=redirects" json:"redirects,omitempty"`
 	Headers   []*HeaderRule   `protobuf:"bytes,9,rep,name=headers" json:"headers,omitempty"`
-	// Diagnostics for non-fatal errors
+	// Diagnostics for non-fatal errors.
 	Problems      []*Problem `protobuf:"bytes,7,rep,name=problems" json:"problems,omitempty"`
 	unknownFields protoimpl.UnknownFields
 	sizeCache     protoimpl.SizeCache
@@ -564,18 +651,235 @@ func (x *Manifest) GetProblems() []*Problem {
 	return nil
 }
 
+type AuditRecord struct {
+	state protoimpl.MessageState `protogen:"open.v1"`
+	// Audit event metadata.
+	Id        *int64                 `protobuf:"varint,1,opt,name=id" json:"id,omitempty"`
+	Timestamp *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=timestamp" json:"timestamp,omitempty"`
+	Event     *AuditEvent            `protobuf:"varint,3,opt,name=event,enum=AuditEvent" json:"event,omitempty"`
+	Principal *Principal             `protobuf:"bytes,4,opt,name=principal" json:"principal,omitempty"`
+	// Affected resource.
+	Domain  *string `protobuf:"bytes,10,opt,name=domain" json:"domain,omitempty"`
+	Project *string `protobuf:"bytes,11,opt,name=project" json:"project,omitempty"` // only for `*Manifest` events
+	// Snapshot of site manifest.
+	Manifest      *Manifest `protobuf:"bytes,12,opt,name=manifest" json:"manifest,omitempty"` // only for `*Manifest` events
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *AuditRecord) Reset() {
+	*x = AuditRecord{}
+	mi := &file_schema_proto_msgTypes[6]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *AuditRecord) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*AuditRecord) ProtoMessage() {}
+
+func (x *AuditRecord) ProtoReflect() protoreflect.Message {
+	mi := &file_schema_proto_msgTypes[6]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use AuditRecord.ProtoReflect.Descriptor instead.
+func (*AuditRecord) Descriptor() ([]byte, []int) {
+	return file_schema_proto_rawDescGZIP(), []int{6}
+}
+
+func (x *AuditRecord) GetId() int64 {
+	if x != nil && x.Id != nil {
+		return *x.Id
+	}
+	return 0
+}
+
+func (x *AuditRecord) GetTimestamp() *timestamppb.Timestamp {
+	if x != nil {
+		return x.Timestamp
+	}
+	return nil
+}
+
+func (x *AuditRecord) GetEvent() AuditEvent {
+	if x != nil && x.Event != nil {
+		return *x.Event
+	}
+	return AuditEvent_InvalidEvent
+}
+
+func (x *AuditRecord) GetPrincipal() *Principal {
+	if x != nil {
+		return x.Principal
+	}
+	return nil
+}
+
+func (x *AuditRecord) GetDomain() string {
+	if x != nil && x.Domain != nil {
+		return *x.Domain
+	}
+	return ""
+}
+
+func (x *AuditRecord) GetProject() string {
+	if x != nil && x.Project != nil {
+		return *x.Project
+	}
+	return ""
+}
+
+func (x *AuditRecord) GetManifest() *Manifest {
+	if x != nil {
+		return x.Manifest
+	}
+	return nil
+}
+
+type Principal struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	IpAddress     *string                `protobuf:"bytes,1,opt,name=ip_address,json=ipAddress" json:"ip_address,omitempty"`
+	CliAdmin      *bool                  `protobuf:"varint,2,opt,name=cli_admin,json=cliAdmin" json:"cli_admin,omitempty"`
+	ForgeUser     *ForgeUser             `protobuf:"bytes,3,opt,name=forge_user,json=forgeUser" json:"forge_user,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *Principal) Reset() {
+	*x = Principal{}
+	mi := &file_schema_proto_msgTypes[7]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *Principal) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Principal) ProtoMessage() {}
+
+func (x *Principal) ProtoReflect() protoreflect.Message {
+	mi := &file_schema_proto_msgTypes[7]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Principal.ProtoReflect.Descriptor instead.
+func (*Principal) Descriptor() ([]byte, []int) {
+	return file_schema_proto_rawDescGZIP(), []int{7}
+}
+
+func (x *Principal) GetIpAddress() string {
+	if x != nil && x.IpAddress != nil {
+		return *x.IpAddress
+	}
+	return ""
+}
+
+func (x *Principal) GetCliAdmin() bool {
+	if x != nil && x.CliAdmin != nil {
+		return *x.CliAdmin
+	}
+	return false
+}
+
+func (x *Principal) GetForgeUser() *ForgeUser {
+	if x != nil {
+		return x.ForgeUser
+	}
+	return nil
+}
+
+type ForgeUser struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Origin        *string                `protobuf:"bytes,1,opt,name=origin" json:"origin,omitempty"`
+	Id            *int64                 `protobuf:"varint,2,opt,name=id" json:"id,omitempty"`
+	Handle        *string                `protobuf:"bytes,3,opt,name=handle" json:"handle,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *ForgeUser) Reset() {
+	*x = ForgeUser{}
+	mi := &file_schema_proto_msgTypes[8]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *ForgeUser) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ForgeUser) ProtoMessage() {}
+
+func (x *ForgeUser) ProtoReflect() protoreflect.Message {
+	mi := &file_schema_proto_msgTypes[8]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ForgeUser.ProtoReflect.Descriptor instead.
+func (*ForgeUser) Descriptor() ([]byte, []int) {
+	return file_schema_proto_rawDescGZIP(), []int{8}
+}
+
+func (x *ForgeUser) GetOrigin() string {
+	if x != nil && x.Origin != nil {
+		return *x.Origin
+	}
+	return ""
+}
+
+func (x *ForgeUser) GetId() int64 {
+	if x != nil && x.Id != nil {
+		return *x.Id
+	}
+	return 0
+}
+
+func (x *ForgeUser) GetHandle() string {
+	if x != nil && x.Handle != nil {
+		return *x.Handle
+	}
+	return ""
+}
+
 var File_schema_proto protoreflect.FileDescriptor
 
 const file_schema_proto_rawDesc = "" +
 	"\n" +
-	"\fschema.proto\"\x97\x01\n" +
+	"\fschema.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\xec\x01\n" +
 	"\x05Entry\x12\x19\n" +
-	"\x04type\x18\x01 \x01(\x0e2\x05.TypeR\x04type\x12\x12\n" +
-	"\x04size\x18\x02 \x01(\x03R\x04size\x12\x12\n" +
+	"\x04type\x18\x01 \x01(\x0e2\x05.TypeR\x04type\x12#\n" +
+	"\roriginal_size\x18\a \x01(\x03R\foriginalSize\x12'\n" +
+	"\x0fcompressed_size\x18\x02 \x01(\x03R\x0ecompressedSize\x12\x12\n" +
 	"\x04data\x18\x03 \x01(\fR\x04data\x12(\n" +
 	"\ttransform\x18\x04 \x01(\x0e2\n" +
 	".TransformR\ttransform\x12!\n" +
-	"\fcontent_type\x18\x05 \x01(\tR\vcontentType\"`\n" +
+	"\fcontent_type\x18\x05 \x01(\tR\vcontentType\x12\x19\n" +
+	"\bgit_hash\x18\x06 \x01(\tR\agitHash\"`\n" +
 	"\fRedirectRule\x12\x12\n" +
 	"\x04from\x18\x01 \x01(\tR\x04from\x12\x0e\n" +
 	"\x02to\x18\x02 \x01(\tR\x02to\x12\x16\n" +
@@ -607,17 +911,45 @@ const file_schema_proto_rawDesc = "" +
 	"\bproblems\x18\a \x03(\v2\b.ProblemR\bproblems\x1aC\n" +
 	"\rContentsEntry\x12\x10\n" +
 	"\x03key\x18\x01 \x01(\tR\x03key\x12\x1c\n" +
-	"\x05value\x18\x02 \x01(\v2\x06.EntryR\x05value:\x028\x01*Q\n" +
-	"\x04Type\x12\v\n" +
-	"\aInvalid\x10\x00\x12\r\n" +
+	"\x05value\x18\x02 \x01(\v2\x06.EntryR\x05value:\x028\x01\"\xfd\x01\n" +
+	"\vAuditRecord\x12\x0e\n" +
+	"\x02id\x18\x01 \x01(\x03R\x02id\x128\n" +
+	"\ttimestamp\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\ttimestamp\x12!\n" +
+	"\x05event\x18\x03 \x01(\x0e2\v.AuditEventR\x05event\x12(\n" +
+	"\tprincipal\x18\x04 \x01(\v2\n" +
+	".PrincipalR\tprincipal\x12\x16\n" +
+	"\x06domain\x18\n" +
+	" \x01(\tR\x06domain\x12\x18\n" +
+	"\aproject\x18\v \x01(\tR\aproject\x12%\n" +
+	"\bmanifest\x18\f \x01(\v2\t.ManifestR\bmanifest\"r\n" +
+	"\tPrincipal\x12\x1d\n" +
+	"\n" +
+	"ip_address\x18\x01 \x01(\tR\tipAddress\x12\x1b\n" +
+	"\tcli_admin\x18\x02 \x01(\bR\bcliAdmin\x12)\n" +
+	"\n" +
+	"forge_user\x18\x03 \x01(\v2\n" +
+	".ForgeUserR\tforgeUser\"K\n" +
+	"\tForgeUser\x12\x16\n" +
+	"\x06origin\x18\x01 \x01(\tR\x06origin\x12\x0e\n" +
+	"\x02id\x18\x02 \x01(\x03R\x02id\x12\x16\n" +
+	"\x06handle\x18\x03 \x01(\tR\x06handle*V\n" +
+	"\x04Type\x12\x10\n" +
+	"\fInvalidEntry\x10\x00\x12\r\n" +
 	"\tDirectory\x10\x01\x12\x0e\n" +
 	"\n" +
 	"InlineFile\x10\x02\x12\x10\n" +
 	"\fExternalFile\x10\x03\x12\v\n" +
-	"\aSymlink\x10\x04*$\n" +
-	"\tTransform\x12\b\n" +
-	"\x04None\x10\x00\x12\r\n" +
-	"\tZstandard\x10\x01B,Z*codeberg.org/git-pages/git-pages/git_pagesb\beditionsp\xe8\a"
+	"\aSymlink\x10\x04*#\n" +
+	"\tTransform\x12\f\n" +
+	"\bIdentity\x10\x00\x12\b\n" +
+	"\x04Zstd\x10\x01*l\n" +
+	"\n" +
+	"AuditEvent\x12\x10\n" +
+	"\fInvalidEvent\x10\x00\x12\x12\n" +
+	"\x0eCommitManifest\x10\x01\x12\x12\n" +
+	"\x0eDeleteManifest\x10\x02\x12\x10\n" +
+	"\fFreezeDomain\x10\x03\x12\x12\n" +
+	"\x0eUnfreezeDomain\x10\x04B,Z*codeberg.org/git-pages/git-pages/git_pagesb\beditionsp\xe8\a"
 
 var (
 	file_schema_proto_rawDescOnce sync.Once
@@ -631,33 +963,43 @@ func file_schema_proto_rawDescGZIP() []byte {
 	return file_schema_proto_rawDescData
 }
 
-var file_schema_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
-var file_schema_proto_msgTypes = make([]protoimpl.MessageInfo, 7)
+var file_schema_proto_enumTypes = make([]protoimpl.EnumInfo, 3)
+var file_schema_proto_msgTypes = make([]protoimpl.MessageInfo, 10)
 var file_schema_proto_goTypes = []any{
-	(Type)(0),            // 0: Type
-	(Transform)(0),       // 1: Transform
-	(*Entry)(nil),        // 2: Entry
-	(*RedirectRule)(nil), // 3: RedirectRule
-	(*Header)(nil),       // 4: Header
-	(*HeaderRule)(nil),   // 5: HeaderRule
-	(*Problem)(nil),      // 6: Problem
-	(*Manifest)(nil),     // 7: Manifest
-	nil,                  // 8: Manifest.ContentsEntry
+	(Type)(0),                     // 0: Type
+	(Transform)(0),                // 1: Transform
+	(AuditEvent)(0),               // 2: AuditEvent
+	(*Entry)(nil),                 // 3: Entry
+	(*RedirectRule)(nil),          // 4: RedirectRule
+	(*Header)(nil),                // 5: Header
+	(*HeaderRule)(nil),            // 6: HeaderRule
+	(*Problem)(nil),               // 7: Problem
+	(*Manifest)(nil),              // 8: Manifest
+	(*AuditRecord)(nil),           // 9: AuditRecord
+	(*Principal)(nil),             // 10: Principal
+	(*ForgeUser)(nil),             // 11: ForgeUser
+	nil,                           // 12: Manifest.ContentsEntry
+	(*timestamppb.Timestamp)(nil), // 13: google.protobuf.Timestamp
 }
 var file_schema_proto_depIdxs = []int32{
-	0, // 0: Entry.type:type_name -> Type
-	1, // 1: Entry.transform:type_name -> Transform
-	4, // 2: HeaderRule.header_map:type_name -> Header
-	8, // 3: Manifest.contents:type_name -> Manifest.ContentsEntry
-	3, // 4: Manifest.redirects:type_name -> RedirectRule
-	5, // 5: Manifest.headers:type_name -> HeaderRule
-	6, // 6: Manifest.problems:type_name -> Problem
-	2, // 7: Manifest.ContentsEntry.value:type_name -> Entry
-	8, // [8:8] is the sub-list for method output_type
-	8, // [8:8] is the sub-list for method input_type
-	8, // [8:8] is the sub-list for extension type_name
-	8, // [8:8] is the sub-list for extension extendee
-	0, // [0:8] is the sub-list for field type_name
+	0,  // 0: Entry.type:type_name -> Type
+	1,  // 1: Entry.transform:type_name -> Transform
+	5,  // 2: HeaderRule.header_map:type_name -> Header
+	12, // 3: Manifest.contents:type_name -> Manifest.ContentsEntry
+	4,  // 4: Manifest.redirects:type_name -> RedirectRule
+	6,  // 5: Manifest.headers:type_name -> HeaderRule
+	7,  // 6: Manifest.problems:type_name -> Problem
+	13, // 7: AuditRecord.timestamp:type_name -> google.protobuf.Timestamp
+	2,  // 8: AuditRecord.event:type_name -> AuditEvent
+	10, // 9: AuditRecord.principal:type_name -> Principal
+	8,  // 10: AuditRecord.manifest:type_name -> Manifest
+	11, // 11: Principal.forge_user:type_name -> ForgeUser
+	3,  // 12: Manifest.ContentsEntry.value:type_name -> Entry
+	13, // [13:13] is the sub-list for method output_type
+	13, // [13:13] is the sub-list for method input_type
+	13, // [13:13] is the sub-list for extension type_name
+	13, // [13:13] is the sub-list for extension extendee
+	0,  // [0:13] is the sub-list for field type_name
 }
 
 func init() { file_schema_proto_init() }
@@ -670,8 +1012,8 @@ func file_schema_proto_init() {
 		File: protoimpl.DescBuilder{
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: unsafe.Slice(unsafe.StringData(file_schema_proto_rawDesc), len(file_schema_proto_rawDesc)),
-			NumEnums:      2,
-			NumMessages:   7,
+			NumEnums:      3,
+			NumMessages:   10,
 			NumExtensions: 0,
 			NumServices:   0,
 		},

src/schema.proto

@@ -2,9 +2,11 @@ edition = "2023";
 
 option go_package = "codeberg.org/git-pages/git-pages/git_pages";
 
+import "google/protobuf/timestamp.proto";
+
 enum Type {
 	// Invalid entry.
-	Invalid = 0;
+	InvalidEntry = 0;
 	// Directory.
 	Directory = 1;
 	// Inline file. `Blob.Data` contains file contents.
@@ -15,18 +17,24 @@ enum Type {
 	Symlink = 4;
 }
 
+// Transformation names should match HTTP `Accept-Encoding:` header.
 enum Transform {
 	// No transformation.
-	None = 0;
+	Identity = 0;
 	// Zstandard compression.
-	Zstandard = 1;
+	Zstd = 1;
 }
 
 message Entry {
 	Type type = 1;
 	// Only present for `type == InlineFile` and `type == ExternalFile`.
-	// For transformed entries, refers to the post-transformation (compressed) size.
-	int64 size = 2;
+	// For transformed entries, refers to the pre-transformation (decompressed) size; otherwise
+	// equal to `compressed_size`.
+	int64 original_size = 7;
+	// Only present for `type == InlineFile` and `type == ExternalFile`.
+	// For transformed entries, refers to the post-transformation (compressed) size; otherwise
+	// equal to `original_size`.
+	int64 compressed_size = 2;
 	// Meaning depends on `type`:
 	//  * If `type == InlineFile`, contains file data.
 	//  * If `type == ExternalFile`, contains blob name (an otherwise unspecified
@@ -40,6 +48,12 @@ message Entry {
 	// Only present for `type == InlineFile` and `type == ExternalFile`.
 	// Currently, optional (not present on certain legacy manifests).
 	string content_type = 5;
+	// May be present for `type == InlineFile` and `type == ExternalFile`.
+	// Used to reduce the amount of work being done during git checkouts.
+	// The type of hash used is determined by the length:
+	//   * 40 bytes: SHA1DC (as hex)
+	//   * 64 bytes: SHA256 (as hex)
+	string git_hash = 6;
 }
 
 // See https://docs.netlify.com/manage/routing/redirects/overview/ for details.
@@ -68,21 +82,61 @@ message Problem {
 }
 
 message Manifest {
-	// Source metadata
+	// Source metadata.
 	string repo_url = 1;
 	string branch = 2;
 	string commit = 3;
 
-	// Contents
+	// Site contents.
 	map<string, Entry> contents = 4;
-	int64 original_size = 10; // total size of entries before compression
-	int64 compressed_size = 5; // simple sum of each `entry.size`
-	int64 stored_size = 8; // total size of (deduplicated) external objects
+	int64 original_size = 10; // sum of each `entry.original_size`
+	int64 compressed_size = 5; // sum of each `entry.compressed_size`
+	int64 stored_size = 8; // sum of deduplicated `entry.compressed_size` for external files only
 
-	// Netlify-style `_redirects` and `_headers`
+	// Netlify-style `_redirects` and `_headers` rules.
 	repeated RedirectRule redirects = 6;
 	repeated HeaderRule headers = 9;
 
-	// Diagnostics for non-fatal errors
+	// Diagnostics for non-fatal errors.
 	repeated Problem problems = 7;
 }
+
+enum AuditEvent {
+	// Invalid event.
+	InvalidEvent = 0;
+	// A manifest was committed (a site was created or updated).
+	CommitManifest = 1;
+	// A manifest was deleted (a site was deleted).
+	DeleteManifest = 2;
+	// A domain was frozen.
+	FreezeDomain = 3;
+	// A domain was thawed.
+	UnfreezeDomain = 4;
+}
+
+message AuditRecord {
+	// Audit event metadata.
+	int64 id = 1;
+	google.protobuf.Timestamp timestamp = 2;
+	AuditEvent event = 3;
+	Principal principal = 4;
+
+	// Affected resource.
+	string domain = 10;
+	string project = 11; // only for `*Manifest` events
+
+	// Snapshot of site manifest.
+	Manifest manifest = 12;  // only for `*Manifest` events
+}
+
+message Principal {
+	string ip_address = 1;
+	bool cli_admin = 2;
+	ForgeUser forge_user = 3;
+}
+
+message ForgeUser {
+	string origin = 1;
+	int64 id = 2;
+	string handle = 3;
+}

src/signal.go

@@ -0,0 +1,29 @@
+// See https://pkg.go.dev/os/signal#hdr-Windows for a description of what this module
+// will do on Windows (tl;dr nothing calls the reload handler, the interrupt handler works
+// more or less how you'd expect).
+
+package git_pages
+
+import (
+	"os"
+	"os/signal"
+	"syscall"
+)
+
+func OnReload(handler func()) {
+	sighup := make(chan os.Signal, 1)
+	signal.Notify(sighup, syscall.SIGHUP)
+	go func() {
+		for {
+			<-sighup
+			handler()
+		}
+	}()
+}
+
+func WaitForInterrupt() {
+	sigint := make(chan os.Signal, 1)
+	signal.Notify(sigint, syscall.SIGINT, syscall.SIGTERM)
+	<-sigint
+	signal.Stop(sigint)
+}

src/signal_other.go

@@ -1,7 +0,0 @@
-//go:build !unix
-
-package git_pages
-
-func OnReload(handler func()) {
-	// not implemented
-}

src/signal_posix.go

@@ -1,20 +0,0 @@
-//go:build unix
-
-package git_pages
-
-import (
-	"os"
-	"os/signal"
-	"syscall"
-)
-
-func OnReload(handler func()) {
-	sighup := make(chan os.Signal, 1)
-	signal.Notify(sighup, syscall.SIGHUP)
-	go func() {
-		for {
-			<-sighup
-			handler()
-		}
-	}()
-}

src/update.go

@@ -5,8 +5,9 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"log"
 	"strings"
+
+	"google.golang.org/protobuf/proto"
 )
 
 type UpdateOutcome int
@@ -26,14 +27,16 @@ type UpdateResult struct {
 	err      error
 }
 
-func Update(ctx context.Context, webRoot string, manifest *Manifest) UpdateResult {
-	var oldManifest, newManifest *Manifest
+func Update(
+	ctx context.Context, webRoot string, oldManifest, newManifest *Manifest,
+	opts ModifyManifestOptions,
+) UpdateResult {
 	var err error
+	var storedManifest *Manifest
 
 	outcome := UpdateError
-	oldManifest, _, _ = backend.GetManifest(ctx, webRoot, GetManifestOptions{})
-	if IsManifestEmpty(manifest) {
-		newManifest, err = manifest, backend.DeleteManifest(ctx, webRoot)
+	if IsManifestEmpty(newManifest) {
+		storedManifest, err = newManifest, backend.DeleteManifest(ctx, webRoot, opts)
 		if err == nil {
 			if oldManifest == nil {
 				outcome = UpdateNoChange
@@ -41,8 +44,8 @@ func Update(ctx context.Context, webRoot string, manifest *Manifest) UpdateResul
 				outcome = UpdateDeleted
 			}
 		}
-	} else if err = PrepareManifest(ctx, manifest); err == nil {
-		newManifest, err = StoreManifest(ctx, webRoot, manifest)
+	} else if err = PrepareManifest(ctx, newManifest); err == nil {
+		storedManifest, err = StoreManifest(ctx, webRoot, newManifest, opts)
 		if err == nil {
 			domain, _, _ := strings.Cut(webRoot, "/")
 			err = backend.CreateDomain(ctx, domain)
@@ -50,7 +53,7 @@ func Update(ctx context.Context, webRoot string, manifest *Manifest) UpdateResul
 		if err == nil {
 			if oldManifest == nil {
 				outcome = UpdateCreated
-			} else if CompareManifest(oldManifest, newManifest) {
+			} else if CompareManifest(oldManifest, storedManifest) {
 				outcome = UpdateNoChange
 			} else {
 				outcome = UpdateReplaced
@@ -70,16 +73,16 @@ func Update(ctx context.Context, webRoot string, manifest *Manifest) UpdateResul
 		case UpdateNoChange:
 			status = "unchanged"
 		}
-		if newManifest.Commit != nil {
-			log.Printf("update %s ok: %s %s", webRoot, status, *newManifest.Commit)
+		if storedManifest.Commit != nil {
+			logc.Printf(ctx, "update %s ok: %s %s", webRoot, *storedManifest.Commit, status)
 		} else {
-			log.Printf("update %s ok: %s", webRoot, status)
+			logc.Printf(ctx, "update %s ok: %s", webRoot, status)
 		}
 	} else {
-		log.Printf("update %s err: %s", webRoot, err)
+		logc.Printf(ctx, "update %s err: %s", webRoot, err)
 	}
 
-	return UpdateResult{outcome, newManifest, err}
+	return UpdateResult{outcome, storedManifest, err}
 }
 
 func UpdateFromRepository(
@@ -91,15 +94,18 @@ func UpdateFromRepository(
 	span, ctx := ObserveFunction(ctx, "UpdateFromRepository", "repo.url", repoURL)
 	defer span.Finish()
 
-	log.Printf("update %s: %s %s\n", webRoot, repoURL, branch)
+	logc.Printf(ctx, "update %s: %s %s\n", webRoot, repoURL, branch)
 
-	manifest, err := FetchRepository(ctx, repoURL, branch)
+	// Ignore errors; worst case we have to re-fetch all of the blobs.
+	oldManifest, _, _ := backend.GetManifest(ctx, webRoot, GetManifestOptions{})
+
+	newManifest, err := FetchRepository(ctx, repoURL, branch, oldManifest)
 	if errors.Is(err, context.DeadlineExceeded) {
 		result = UpdateResult{UpdateTimeout, nil, fmt.Errorf("update timeout")}
 	} else if err != nil {
 		result = UpdateResult{UpdateError, nil, err}
 	} else {
-		result = Update(ctx, webRoot, manifest)
+		result = Update(ctx, webRoot, oldManifest, newManifest, ModifyManifestOptions{})
 	}
 
 	observeUpdateResult(result)
@@ -114,31 +120,108 @@ func UpdateFromArchive(
 	contentType string,
 	reader io.Reader,
 ) (result UpdateResult) {
-	var manifest *Manifest
 	var err error
 
+	// Ignore errors; worst case we have to re-fetch all of the blobs.
+	oldManifest, _, _ := backend.GetManifest(ctx, webRoot, GetManifestOptions{})
+
+	extractTar := func(ctx context.Context, reader io.Reader) (*Manifest, error) {
+		return ExtractTar(ctx, reader, oldManifest)
+	}
+
+	var newManifest *Manifest
 	switch contentType {
 	case "application/x-tar":
-		log.Printf("update %s: (tar)", webRoot)
-		manifest, err = ExtractTar(reader) // yellow?
+		logc.Printf(ctx, "update %s: (tar)", webRoot)
+		newManifest, err = extractTar(ctx, reader) // yellow?
 	case "application/x-tar+gzip":
-		log.Printf("update %s: (tar.gz)", webRoot)
-		manifest, err = ExtractTarGzip(reader) // definitely yellow.
+		logc.Printf(ctx, "update %s: (tar.gz)", webRoot)
+		newManifest, err = ExtractGzip(ctx, reader, extractTar) // definitely yellow.
 	case "application/x-tar+zstd":
-		log.Printf("update %s: (tar.zst)", webRoot)
-		manifest, err = ExtractTarZstd(reader)
+		logc.Printf(ctx, "update %s: (tar.zst)", webRoot)
+		newManifest, err = ExtractZstd(ctx, reader, extractTar)
 	case "application/zip":
-		log.Printf("update %s: (zip)", webRoot)
-		manifest, err = ExtractZip(reader)
+		logc.Printf(ctx, "update %s: (zip)", webRoot)
+		newManifest, err = ExtractZip(ctx, reader, oldManifest)
 	default:
 		err = errArchiveFormat
 	}
 
 	if err != nil {
-		log.Printf("update %s err: %s", webRoot, err)
+		logc.Printf(ctx, "update %s err: %s", webRoot, err)
 		result = UpdateResult{UpdateError, nil, err}
 	} else {
-		result = Update(ctx, webRoot, manifest)
+		result = Update(ctx, webRoot, oldManifest, newManifest, ModifyManifestOptions{})
+	}
+
+	observeUpdateResult(result)
+	return
+}
+
+func PartialUpdateFromArchive(
+	ctx context.Context,
+	webRoot string,
+	contentType string,
+	reader io.Reader,
+	parents CreateParentsMode,
+) (result UpdateResult) {
+	var err error
+
+	// Here the old manifest is used both as a substrate to which a patch is applied, as well
+	// as a "load linked" operation for a future "store conditional" update which, taken together,
+	// create an atomic compare-and-swap operation.
+	oldManifest, oldMetadata, err := backend.GetManifest(ctx, webRoot,
+		GetManifestOptions{BypassCache: true})
+	if err != nil {
+		logc.Printf(ctx, "patch %s err: %s", webRoot, err)
+		return UpdateResult{UpdateError, nil, err}
+	}
+
+	applyTarPatch := func(ctx context.Context, reader io.Reader) (*Manifest, error) {
+		// Clone the manifest before starting to mutate it. `GetManifest` may return cached
+		// `*Manifest` objects, which should never be mutated.
+		newManifest := &Manifest{}
+		proto.Merge(newManifest, oldManifest)
+		newManifest.RepoUrl = nil
+		newManifest.Branch = nil
+		newManifest.Commit = nil
+		if err := ApplyTarPatch(newManifest, reader, parents); err != nil {
+			return nil, err
+		} else {
+			return newManifest, nil
+		}
+	}
+
+	var newManifest *Manifest
+	switch contentType {
+	case "application/x-tar":
+		logc.Printf(ctx, "patch %s: (tar)", webRoot)
+		newManifest, err = applyTarPatch(ctx, reader)
+	case "application/x-tar+gzip":
+		logc.Printf(ctx, "patch %s: (tar.gz)", webRoot)
+		newManifest, err = ExtractGzip(ctx, reader, applyTarPatch)
+	case "application/x-tar+zstd":
+		logc.Printf(ctx, "patch %s: (tar.zst)", webRoot)
+		newManifest, err = ExtractZstd(ctx, reader, applyTarPatch)
+	default:
+		err = errArchiveFormat
+	}
+
+	if err != nil {
+		logc.Printf(ctx, "patch %s err: %s", webRoot, err)
+		result = UpdateResult{UpdateError, nil, err}
+	} else {
+		result = Update(ctx, webRoot, oldManifest, newManifest,
+			ModifyManifestOptions{
+				IfUnmodifiedSince: oldMetadata.LastModified,
+				IfMatch:           oldMetadata.ETag,
+			})
+		// The `If-Unmodified-Since` precondition is internally generated here, which means its
+		// failure shouldn't be surfaced as-is in the HTTP response. If we also accepted options
+		// from the client, then that precondition failure should surface in the response.
+		if errors.Is(result.err, ErrPreconditionFailed) {
+			result.err = ErrWriteConflict
+		}
 	}
 
 	observeUpdateResult(result)
@@ -146,7 +229,12 @@ func UpdateFromArchive(
 }
 
 func observeUpdateResult(result UpdateResult) {
-	if result.err != nil {
+	var unresolvedRefErr UnresolvedRefError
+	if errors.As(result.err, &unresolvedRefErr) {
+		// This error is an expected outcome of an incremental update's probe phase.
+	} else if errors.Is(result.err, ErrWriteConflict) {
+		// This error is an expected outcome of an incremental update losing a race.
+	} else if result.err != nil {
 		ObserveError(result.err)
 	}
 }

src/wildcard.go

@@ -1,12 +1,7 @@
 package git_pages
 
 import (
-	"crypto/tls"
 	"fmt"
-	"log"
-	"net/http"
-	"net/http/httputil"
-	"net/url"
 	"slices"
 	"strings"
 
@@ -16,11 +11,9 @@ import (
 type WildcardPattern struct {
 	Domain        []string
 	CloneURL      *fasttemplate.Template
-	IndexRepos    []*fasttemplate.Template
+	IndexRepo     *fasttemplate.Template
 	IndexBranch   string
 	Authorization bool
-	FallbackURL   *url.URL
-	Fallback      http.Handler
 }
 
 func (pattern *WildcardPattern) GetHost() string {
@@ -56,51 +49,24 @@ func (pattern *WildcardPattern) Matches(host string) (string, bool) {
 	return subdomain, true
 }
 
-func (pattern *WildcardPattern) ApplyTemplate(userName string, projectName string) ([]string, string) {
-	var repoURLs []string
+func (pattern *WildcardPattern) ApplyTemplate(userName string, projectName string) (string, string) {
+	var repoURL string
 	var branch string
 	repoURLTemplate := pattern.CloneURL
 	if projectName == ".index" {
-		for _, indexRepoTemplate := range pattern.IndexRepos {
-			indexRepo := indexRepoTemplate.ExecuteString(map[string]any{"user": userName})
-			repoURLs = append(repoURLs, repoURLTemplate.ExecuteString(map[string]any{
-				"user":    userName,
-				"project": indexRepo,
-			}))
-		}
+		repoURL = repoURLTemplate.ExecuteString(map[string]any{
+			"user":    userName,
+			"project": pattern.IndexRepo.ExecuteString(map[string]any{"user": userName}),
+		})
 		branch = pattern.IndexBranch
 	} else {
-		repoURLs = append(repoURLs, repoURLTemplate.ExecuteString(map[string]any{
+		repoURL = repoURLTemplate.ExecuteString(map[string]any{
 			"user":    userName,
 			"project": projectName,
-		}))
+		})
 		branch = "pages"
 	}
-	return repoURLs, branch
-}
-
-func (pattern *WildcardPattern) IsFallbackFor(host string) bool {
-	if pattern.Fallback == nil {
-		return false
-	}
-	_, found := pattern.Matches(host)
-	return found
-}
-
-func HandleWildcardFallback(w http.ResponseWriter, r *http.Request) (bool, error) {
-	host, err := GetHost(r)
-	if err != nil {
-		return false, err
-	}
-
-	for _, pattern := range wildcards {
-		if pattern.IsFallbackFor(host) {
-			log.Printf("proxy: %s via %s", pattern.GetHost(), pattern.FallbackURL)
-			pattern.Fallback.ServeHTTP(w, r)
-			return true, nil
-		}
-	}
-	return false, nil
+	return repoURL, branch
 }
 
 func TranslateWildcards(configs []WildcardConfig) ([]*WildcardPattern, error) {
@@ -111,14 +77,10 @@ func TranslateWildcards(configs []WildcardConfig) ([]*WildcardPattern, error) {
 			return nil, fmt.Errorf("wildcard pattern: clone URL: %w", err)
 		}
 
-		var indexRepoTemplates []*fasttemplate.Template
 		var indexRepoBranch string = config.IndexRepoBranch
-		for _, indexRepo := range config.IndexRepos {
-			indexRepoTemplate, err := fasttemplate.NewTemplate(indexRepo, "<", ">")
-			if err != nil {
-				return nil, fmt.Errorf("wildcard pattern: index repo: %w", err)
-			}
-			indexRepoTemplates = append(indexRepoTemplates, indexRepoTemplate)
+		indexRepoTemplate, err := fasttemplate.NewTemplate(config.IndexRepo, "<", ">")
+		if err != nil {
+			return nil, fmt.Errorf("wildcard pattern: index repo: %w", err)
 		}
 
 		authorization := false
@@ -135,36 +97,12 @@ func TranslateWildcards(configs []WildcardConfig) ([]*WildcardPattern, error) {
 			}
 		}
 
-		var fallbackURL *url.URL
-		var fallback http.Handler
-		if config.FallbackProxyTo != "" {
-			fallbackURL, err = url.Parse(config.FallbackProxyTo)
-			if err != nil {
-				return nil, fmt.Errorf("wildcard pattern: fallback URL: %w", err)
-			}
-
-			fallback = &httputil.ReverseProxy{
-				Rewrite: func(r *httputil.ProxyRequest) {
-					r.SetURL(fallbackURL)
-					r.Out.Host = r.In.Host
-					r.Out.Header["X-Forwarded-For"] = r.In.Header["X-Forwarded-For"]
-				},
-				Transport: &http.Transport{
-					TLSClientConfig: &tls.Config{
-						InsecureSkipVerify: config.FallbackInsecure,
-					},
-				},
-			}
-		}
-
 		wildcardPatterns = append(wildcardPatterns, &WildcardPattern{
 			Domain:        strings.Split(config.Domain, "."),
 			CloneURL:      cloneURLTemplate,
-			IndexRepos:    indexRepoTemplates,
+			IndexRepo:     indexRepoTemplate,
 			IndexBranch:   indexRepoBranch,
 			Authorization: authorization,
-			FallbackURL:   fallbackURL,
-			Fallback:      fallback,
 		})
 	}
 	return wildcardPatterns, nil

test/stresspatch/main.go

@@ -0,0 +1,112 @@
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"sync"
+	"time"
+)
+
+func makeInit() []byte {
+	writer := bytes.NewBuffer(nil)
+	archive := tar.NewWriter(writer)
+	archive.WriteHeader(&tar.Header{
+		Typeflag: tar.TypeReg,
+		Name:     "index.html",
+	})
+	archive.Write([]byte{})
+	archive.Flush()
+	return writer.Bytes()
+}
+
+func initSite() {
+	req, err := http.NewRequest(http.MethodPut, "http://localhost:3000",
+		bytes.NewReader(makeInit()))
+	if err != nil {
+		panic(err)
+	}
+
+	req.Header.Add("Content-Type", "application/x-tar")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		panic(err)
+	}
+	defer resp.Body.Close()
+}
+
+func makePatch(n int) []byte {
+	writer := bytes.NewBuffer(nil)
+	archive := tar.NewWriter(writer)
+	archive.WriteHeader(&tar.Header{
+		Typeflag: tar.TypeReg,
+		Name:     fmt.Sprintf("%d.txt", n),
+	})
+	archive.Write([]byte{})
+	archive.Flush()
+	return writer.Bytes()
+}
+
+func patchRequest(n int) int {
+	req, err := http.NewRequest(http.MethodPatch, "http://localhost:3000",
+		bytes.NewReader(makePatch(n)))
+	if err != nil {
+		panic(err)
+	}
+
+	req.Header.Add("Atomic", "no")
+	req.Header.Add("Content-Type", "application/x-tar")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		panic(err)
+	}
+	defer resp.Body.Close()
+
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Printf("%d: %s %q\n", n, resp.Status, string(data))
+	return resp.StatusCode
+}
+
+func concurrentWriter(wg *sync.WaitGroup, n int) {
+	for {
+		if patchRequest(n) == 200 {
+			break
+		}
+	}
+	wg.Done()
+}
+
+var count = flag.Int("count", 10, "request count")
+
+func main() {
+	flag.Parse()
+
+	initSite()
+	time.Sleep(time.Second)
+
+	wg := &sync.WaitGroup{}
+	for n := range *count {
+		wg.Add(1)
+		go concurrentWriter(wg, n)
+	}
+	wg.Wait()
+
+	success := 0
+	for n := range *count {
+		resp, err := http.Get(fmt.Sprintf("http://localhost:3000/%d.txt", n))
+		if err != nil {
+			panic(err)
+		}
+		if resp.StatusCode == 200 {
+			success++
+		}
+	}
+	fmt.Printf("written: %d of %d\n", success, *count)
+}