fix(tranquil-pds): firehose car carries inductive proof

Lewis: May this revision serve well! <lu5a@proton.me>
fix(validation): discord usernames may contain periods at start or end
2026-08-26 03:06:39 +00:00 · 2026-04-26 20:11:27 +03:00 · 2026-04-25 09:59:12 +03:00 · 2026-04-24 10:50:08 +03:00 · 2026-04-24 10:50:08 +03:00 · 2026-04-23 08:39:16 +03:00
161 changed files with 23065 additions and 3409 deletions
@@ -25,6 +25,28 @@ fail-fast = false
 test-threads = "num-cpus"
 slow-timeout = { period = "300s", terminate-after = 2 }

+[profile.gauntlet-pr]
+retries = 0
+fail-fast = true
+test-threads = "num-cpus"
+slow-timeout = { period = "60s", terminate-after = 5 }
+
+[[profile.gauntlet-pr.overrides]]
+filter = "binary(gauntlet_smoke)"
+slow-timeout = { period = "300s", terminate-after = 2 }
+
+[profile.gauntlet-nightly]
+retries = 0
+fail-fast = false
+test-threads = "num-cpus"
+slow-timeout = { period = "600s", terminate-after = 1 }
+
+[profile.gauntlet-soak]
+retries = 0
+fail-fast = false
+test-threads = 1
+slow-timeout = { period = "5m", terminate-after = 1000 }
+
 [test-groups]
 serial-env-tests = { max-threads = 1 }
 heavy-load-tests = { max-threads = 4 }
@@ -72,6 +94,14 @@ test-group = "io-heavy-sim"
 filter = "test(/test_scale_/) | test(/full_backup_and_restore/)"
 slow-timeout = { period = "120s", terminate-after = 4 }

+[[profile.default.overrides]]
+filter = "binary(gauntlet_smoke)"
+slow-timeout = { period = "300s", terminate-after = 8 }
+
+[[profile.default.overrides]]
+filter = "binary(compaction_restart) | binary(mst_refcount_integrity) | binary(gc_compaction_restart)"
+slow-timeout = { period = "120s", terminate-after = 4 }
+
 [[profile.ci.overrides]]
 filter = "test(/import_with_verification/) | test(/plc_migration/)"
 test-group = "serial-env-tests"
@@ -1,4 +1,4 @@
-/target
+target/
 .env
 .direnv
 result
@@ -1,6 +1,6 @@
 {
  "db_name": "PostgreSQL",
-  "query": "SELECT did, password_hash FROM users WHERE handle = $1 OR email = $1",
+  "query": "SELECT did, password_hash FROM users WHERE handle = $1 OR did = $1",
  "describe": {
    "columns": [
      {
@@ -24,5 +24,5 @@
      true
    ]
  },
-  "hash": "c4621f6a8a1ab78a6355b09fdfc2bf8999d276564e93015792ec07cb05e79038"
+  "hash": "053c971024b0d29a441c3597d760b3e21db2383442c3e6f09de4eb49ea437e7c"
 }
@@ -1,6 +1,6 @@
 {
  "db_name": "PostgreSQL",
-  "query": "\n            SELECT id, did, email, password_hash, password_required, two_factor_enabled,\n                   preferred_comms_channel as \"preferred_comms_channel!: CommsChannel\",\n                   deactivated_at, takedown_ref,\n                   email_verified, discord_verified, telegram_verified, signal_verified,\n                   account_type as \"account_type!: AccountType\"\n            FROM users\n            WHERE handle = $1 OR email = $1\n            ",
+  "query": "\n            SELECT id, did, email, password_hash, password_required, two_factor_enabled,\n                   preferred_comms_channel as \"preferred_comms_channel!: CommsChannel\",\n                   deactivated_at, takedown_ref,\n                   email_verified, discord_verified, telegram_verified, signal_verified,\n                   account_type as \"account_type!: AccountType\"\n            FROM users\n            WHERE handle = $1 OR did = $1\n            ",
  "describe": {
    "columns": [
      {
@@ -118,5 +118,5 @@
      false
    ]
  },
-  "hash": "7061e8763ef7d91ff152ed0124f99e1820172fd06916d225ca6c5137a507b8fa"
+  "hash": "060c285c93a05252aab7d474df0186e7b5083fafedc582b8eac9916983e8fc2d"
 }
@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM webauthn_challenges WHERE did = $1 AND challenge_type = 'discoverable'",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "3155ef4f35698a3fe6aa38d5d976fd51b7f6a0381c81c4907dad61d2f37992bd"
+}
@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT state_json FROM webauthn_challenges\n               WHERE did = $1 AND challenge_type = 'discoverable' AND expires_at > NOW()\n               ORDER BY created_at DESC LIMIT 1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "state_json",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "6969c478a0922bac4b79902313a0e28c94d6b8d6b16035474dd8f484e6171d60"
+}
@@ -1,6 +1,6 @@
 {
  "db_name": "PostgreSQL",
-  "query": "SELECT\n                u.id, u.did, u.handle, u.password_hash, u.email, u.deactivated_at, u.takedown_ref,\n                u.email_verified, u.discord_verified, u.telegram_verified, u.signal_verified,\n                u.allow_legacy_login, u.migrated_to_pds,\n                u.preferred_comms_channel as \"preferred_comms_channel: CommsChannel\",\n                k.key_bytes, k.encryption_version,\n                (SELECT verified FROM user_totp WHERE did = u.did) as totp_enabled,\n                COALESCE((SELECT (value_json)::boolean FROM account_preferences WHERE user_id = u.id AND name = 'email_auth_factor' ORDER BY created_at DESC LIMIT 1), false) as \"email_2fa_enabled!\"\n            FROM users u\n            JOIN user_keys k ON u.id = k.user_id\n            WHERE u.handle = $1 OR u.email = $1 OR u.did = $1",
+  "query": "SELECT\n                u.id, u.did, u.handle, u.password_hash, u.email, u.deactivated_at, u.takedown_ref,\n                u.email_verified, u.discord_verified, u.telegram_verified, u.signal_verified,\n                u.allow_legacy_login, u.migrated_to_pds,\n                u.preferred_comms_channel as \"preferred_comms_channel: CommsChannel\",\n                k.key_bytes, k.encryption_version,\n                (SELECT verified FROM user_totp WHERE did = u.did) as totp_enabled,\n                COALESCE((SELECT (value_json)::boolean FROM account_preferences WHERE user_id = u.id AND name = 'email_auth_factor' ORDER BY created_at DESC LIMIT 1), false) as \"email_2fa_enabled!\"\n            FROM users u\n            JOIN user_keys k ON u.id = k.user_id\n            WHERE u.handle = $1 OR u.did = $1",
  "describe": {
    "columns": [
      {
@@ -132,5 +132,5 @@
      null
    ]
  },
-  "hash": "a960b981a146a0e422ef53601dfc31e29cf777aa194227c48c6ebc6905ea3249"
+  "hash": "aafc2a7e51200ca1e7071c63c13698bf34ef8b66758ca9ebab4ea706ffb62914"
 }
@@ -0,0 +1,18 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO webauthn_challenges (id, did, challenge, challenge_type, state_json, expires_at)\n               VALUES ($1, $2, $3, 'discoverable', $4, $5)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text",
+        "Bytea",
+        "Text",
+        "Timestamptz"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "c6e3388fc39983f1787917606ba3194c72322d2d1ec54402c262194791a2b06a"
+}
@@ -0,0 +1,21 @@
+when:
+  - event: ["push", "manual"]
+    branch: ["main"]
+
+engine: nixery
+
+dependencies:
+  nixpkgs:
+    - nix
+    - cachix
+    - gnugrep
+
+steps:
+  - name: Authenticate
+    command: |
+      cachix authtoken "$CACHIX_AUTH_TOKEN"
+
+  - name: Build and push aarch64
+    command: |
+      SUBS="--option extra-substituters https://tranquil.cachix.org --option extra-trusted-public-keys tranquil.cachix.org-1:PoO+mGL6a6LcJiPakMDHN4E218/ei/7v2sxeDtNkSRg="
+      nix-store -qR --include-outputs $(nix-store -qd $(nix build .#packages.x86_64-linux.tranquil-pds-aarch64 $SUBS --print-out-paths --no-link)) | grep -v '\.drv$' | cachix push "$CACHIX_CACHE_NAME"
@@ -0,0 +1,28 @@
+when:
+  - event: ["push", "manual"]
+    branch: ["main"]
+
+engine: nixery
+
+dependencies:
+  nixpkgs:
+    - nix
+    - cachix
+    - gnugrep
+
+steps:
+  - name: Authenticate
+    command: |
+      cachix authtoken "$CACHIX_AUTH_TOKEN"
+
+  - name: Build and push x86_64
+    command: |
+      SUBS="--option extra-substituters https://tranquil.cachix.org --option extra-trusted-public-keys tranquil.cachix.org-1:PoO+mGL6a6LcJiPakMDHN4E218/ei/7v2sxeDtNkSRg="
+      nix-store -qR --include-outputs $(nix-store -qd $(nix build .#packages.x86_64-linux.tranquil-pds $SUBS --print-out-paths --no-link)) | grep -v '\.drv$' | cachix push "$CACHIX_CACHE_NAME"
+      nix-store -qR --include-outputs $(nix-store -qd $(nix build .#packages.x86_64-linux.tranquil-frontend $SUBS --print-out-paths --no-link)) | grep -v '\.drv$' | cachix push "$CACHIX_CACHE_NAME"
+
+  - name: Build and push devShell
+    command: |
+      SUBS="--option extra-substituters https://tranquil.cachix.org --option extra-trusted-public-keys tranquil.cachix.org-1:PoO+mGL6a6LcJiPakMDHN4E218/ei/7v2sxeDtNkSRg="
+      nix develop $SUBS --profile dev-profile -c true
+      cachix push "$CACHIX_CACHE_NAME" dev-profile
@@ -1558,7 +1558,7 @@ checksum = "06b4f5ec222421e22bb0a8cbaa36b1d2b50fd45cdd30c915ded34108da78b29f"
 dependencies = [
 "confique-macro",
 "serde",
- "toml",
+ "toml 0.9.12+spec-1.1.0",
 ]

 [[package]]
@@ -5050,7 +5050,7 @@ version = "3.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f"
 dependencies = [
- "toml_edit",
+ "toml_edit 0.25.5+spec-1.1.0",
 ]

 [[package]]
@@ -6201,6 +6201,15 @@ dependencies = [
 "syn 2.0.117",
 ]

+[[package]]
+name = "serde_spanned"
+version = "0.6.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "serde_spanned"
 version = "1.0.4"
@@ -7144,6 +7153,18 @@ dependencies = [
 "tokio",
 ]

+[[package]]
+name = "toml"
+version = "0.8.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"
+dependencies = [
+ "serde",
+ "serde_spanned 0.6.9",
+ "toml_datetime 0.6.11",
+ "toml_edit 0.22.27",
+]
+
 [[package]]
 name = "toml"
 version = "0.9.12+spec-1.1.0"
@@ -7152,13 +7173,22 @@ checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863"
 dependencies = [
 "indexmap 2.13.0",
 "serde_core",
- "serde_spanned",
+ "serde_spanned 1.0.4",
 "toml_datetime 0.7.5+spec-1.1.0",
 "toml_parser",
 "toml_writer",
 "winnow 0.7.15",
 ]

+[[package]]
+name = "toml_datetime"
+version = "0.6.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "toml_datetime"
 version = "0.7.5+spec-1.1.0"
@@ -7177,6 +7207,20 @@ dependencies = [
 "serde_core",
 ]

+[[package]]
+name = "toml_edit"
+version = "0.22.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
+dependencies = [
+ "indexmap 2.13.0",
+ "serde",
+ "serde_spanned 0.6.9",
+ "toml_datetime 0.6.11",
+ "toml_write",
+ "winnow 0.7.15",
+]
+
 [[package]]
 name = "toml_edit"
 version = "0.25.5+spec-1.1.0"
@@ -7198,6 +7242,12 @@ dependencies = [
 "winnow 1.0.0",
 ]

+[[package]]
+name = "toml_write"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
+
 [[package]]
 name = "toml_writer"
 version = "1.0.7+spec-1.1.0"
@@ -7405,7 +7455,7 @@ dependencies = [

 [[package]]
 name = "tranquil-api"
-version = "0.5.0"
+version = "0.5.6"
 dependencies = [
 "anyhow",
 "axum",
@@ -7456,7 +7506,7 @@ dependencies = [

 [[package]]
 name = "tranquil-auth"
-version = "0.5.0"
+version = "0.5.6"
 dependencies = [
 "anyhow",
 "base32",
@@ -7479,7 +7529,7 @@ dependencies = [

 [[package]]
 name = "tranquil-cache"
-version = "0.5.0"
+version = "0.5.6"
 dependencies = [
 "async-trait",
 "base64 0.22.1",
@@ -7493,7 +7543,7 @@ dependencies = [

 [[package]]
 name = "tranquil-comms"
-version = "0.5.0"
+version = "0.5.6"
 dependencies = [
 "async-trait",
 "base64 0.22.1",
@@ -7511,7 +7561,7 @@ dependencies = [

 [[package]]
 name = "tranquil-config"
-version = "0.5.0"
+version = "0.5.6"
 dependencies = [
 "confique",
 "serde",
@@ -7519,7 +7569,7 @@ dependencies = [

 [[package]]
 name = "tranquil-crypto"
-version = "0.5.0"
+version = "0.5.6"
 dependencies = [
 "aes-gcm",
 "base64 0.22.1",
@@ -7535,7 +7585,7 @@ dependencies = [

 [[package]]
 name = "tranquil-db"
-version = "0.5.0"
+version = "0.5.6"
 dependencies = [
 "async-trait",
 "chrono",
@@ -7552,7 +7602,7 @@ dependencies = [

 [[package]]
 name = "tranquil-db-traits"
-version = "0.5.0"
+version = "0.5.6"
 dependencies = [
 "async-trait",
 "base64 0.22.1",
@@ -7568,7 +7618,7 @@ dependencies = [

 [[package]]
 name = "tranquil-infra"
-version = "0.5.0"
+version = "0.5.6"
 dependencies = [
 "async-trait",
 "bytes",
@@ -7579,9 +7629,10 @@ dependencies = [

 [[package]]
 name = "tranquil-lexicon"
-version = "0.5.0"
+version = "0.5.6"
 dependencies = [
 "chrono",
+ "futures",
 "hickory-resolver",
 "parking_lot",
 "reqwest",
@@ -7597,7 +7648,7 @@ dependencies = [

 [[package]]
 name = "tranquil-oauth"
-version = "0.5.0"
+version = "0.5.6"
 dependencies = [
 "anyhow",
 "axum",
@@ -7620,7 +7671,7 @@ dependencies = [

 [[package]]
 name = "tranquil-oauth-server"
-version = "0.5.0"
+version = "0.5.6"
 dependencies = [
 "axum",
 "base64 0.22.1",
@@ -7653,7 +7704,7 @@ dependencies = [

 [[package]]
 name = "tranquil-pds"
-version = "0.5.0"
+version = "0.5.6"
 dependencies = [
 "aes-gcm",
 "anyhow",
@@ -7738,13 +7789,14 @@ dependencies = [
 "urlencoding",
 "uuid",
 "webauthn-rs",
+ "webauthn-rs-proto",
 "wiremock",
 "zip",
 ]

 [[package]]
 name = "tranquil-repo"
-version = "0.5.0"
+version = "0.5.6"
 dependencies = [
 "bytes",
 "cid",
@@ -7756,7 +7808,7 @@ dependencies = [

 [[package]]
 name = "tranquil-ripple"
-version = "0.5.0"
+version = "0.5.6"
 dependencies = [
 "async-trait",
 "backon",
@@ -7781,7 +7833,7 @@ dependencies = [

 [[package]]
 name = "tranquil-scopes"
-version = "0.5.0"
+version = "0.5.6"
 dependencies = [
 "axum",
 "futures",
@@ -7797,7 +7849,7 @@ dependencies = [

 [[package]]
 name = "tranquil-server"
-version = "0.5.0"
+version = "0.5.6"
 dependencies = [
 "axum",
 "clap",
@@ -7818,7 +7870,7 @@ dependencies = [

 [[package]]
 name = "tranquil-signal"
-version = "0.5.0"
+version = "0.5.6"
 dependencies = [
 "async-trait",
 "chrono",
@@ -7841,7 +7893,7 @@ dependencies = [

 [[package]]
 name = "tranquil-storage"
-version = "0.5.0"
+version = "0.5.6"
 dependencies = [
 "async-trait",
 "aws-config",
@@ -7858,12 +7910,13 @@ dependencies = [

 [[package]]
 name = "tranquil-store"
-version = "0.5.0"
+version = "0.5.6"
 dependencies = [
 "async-trait",
 "bytes",
 "chrono",
 "cid",
+ "clap",
 "dashmap",
 "fjall",
 "flume 0.11.1",
@@ -7871,6 +7924,7 @@ dependencies = [
 "jacquard-common",
 "jacquard-repo",
 "k256",
+ "libc",
 "lsm-tree",
 "memmap2",
 "multihash",
@@ -7890,6 +7944,7 @@ dependencies = [
 "thiserror 2.0.18",
 "tikv-jemallocator",
 "tokio",
+ "toml 0.8.23",
 "tracing",
 "tracing-subscriber",
 "tranquil-db",
@@ -7904,7 +7959,7 @@ dependencies = [

 [[package]]
 name = "tranquil-sync"
-version = "0.5.0"
+version = "0.5.6"
 dependencies = [
 "anyhow",
 "axum",
@@ -7926,7 +7981,7 @@ dependencies = [

 [[package]]
 name = "tranquil-types"
-version = "0.5.0"
+version = "0.5.6"
 dependencies = [
 "chrono",
 "cid",
@@ -8851,6 +8906,9 @@ name = "winnow"
 version = "0.7.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
+dependencies = [
+ "memchr",
+]

 [[package]]
 name = "winnow"
@@ -26,7 +26,7 @@ members = [
 ]

 [workspace.package]
-version = "0.5.0"
+version = "0.5.6"
 edition = "2024"
 license = "AGPL-3.0-or-later"

@@ -126,7 +126,7 @@ tracing = "0.1"
 tracing-subscriber = "0.3"
 urlencoding = "2.1"
 uuid = { version = "1.19", features = ["v4", "v5", "v7", "fast-rng", "serde"] }
-webauthn-rs = { version = "0.5", features = ["danger-allow-state-serialisation", "danger-user-presence-only-security-keys"] }
+webauthn-rs = { version = "0.5", features = ["danger-allow-state-serialisation", "danger-user-presence-only-security-keys", "conditional-ui"] }
 webauthn-rs-proto = "0.5"
 zip = { version = "7.0", default-features = false, features = ["deflate"] }

@@ -1,7 +1,10 @@
-FROM denoland/deno:alpine AS frontend
+FROM node:24-alpine AS frontend
+RUN corepack enable && corepack prepare pnpm@latest --activate
 WORKDIR /app
+COPY frontend/package.json frontend/pnpm-lock.yaml ./
+RUN pnpm install --frozen-lockfile
 COPY frontend/ ./
-RUN deno task build
+RUN pnpm build

 FROM rust:1.92-alpine AS builder
 RUN apk add --no-cache ca-certificates musl-dev pkgconfig openssl-dev openssl-libs-static mold clang protoc
@@ -2,17 +2,25 @@

 A Personal Data Server for the AT Protocol.

-Bluesky runs on a federated protocol called AT Protocol. Your account lives on a PDS, a server that stores your posts, profile, follows, and cryptographic keys. Bluesky hosts one for you at bsky.social, but you can run your own. Self-hosting means you control your data; you're not dependent on any company's servers, and your account + data is actually yours.
+"A what for the what?" -> glad you asked: Bluesky, Tangled, and a bunch of other web applications use a federated protocol called AT Protocol (atproto). Your account lives on a PDS, a server that stores your posts, profile, follows, cryptographic keys, et cetera. The beauty is that a PDS is the *only* place your data lives permanently - so you can navigate any atproto app knowing that your data is yours and not getting locked behind any one app's walls.

-This particular PDS thrives under harsh conditions. It is a dandelion growing through the cracks in the sidewalk concrete.
+We came together to make this PDS to enable and empower our users to better host their data on this shared protocol. All of our decisions as a project are guided by their usefulness to the community: PDS hosters and end-users both. 

-It has full compatibility with Bluesky's reference PDS.
+Comparatively: Bluesky the company created a "reference PDS" that we can self-host quite easily, and that's great, but Bluesky has an incentive to make software for themselvess first & foremost, then secondly their software can be useful for us self-hosters. In contrast, Tranquil is not from a company, and will never be.

 ## What's different about Tranquil PDS

-It is a superset of the reference PDS, including: passkeys and 2FA (WebAuthn/FIDO2, TOTP, backup codes, trusted devices), SSO login and signup, did:web support (PDS-hosted subdomains or bring-your-own), multi-channel communication (email, discord, telegram, signal) for verification and alerts, granular OAuth scopes with a consent UI showing human-readable descriptions, app passwords with granular permissions (read-only, post-only, or custom scopes), account delegation (letting others manage an account with configurable permission levels), and a built-in web UI for account management, repo browsing, and admin.
+It is a superset of the reference PDS, including:
+- passkeys and 2FA: WebAuthn/FIDO2, TOTP, backup codes, trusted devices
+- SSO login and signup
+- did:web support: PDS-hosted subdomains or bring-your-own
+- multi-channel communication: you can be notified via email, discord, telegram, and signal for verification and alerts
+- granular OAuth scopes with a consent UI that allows unchecking specific scopes
+- app passwords with the same granular permission scope system as OAuth
+- account delegation: letting others manage an account with configurable permission levels
+- a built-in web UI for account management, repo browsing, and admin

-The PDS itself is a single binary with no nodeJS runtime. However, at time of writing, Tranquil requires postgres running separately. Blobs are stored on the local filesystem by default (S3 optional). Valkey is also optional (as an alternative to the built-in cache).
+Unlike the ref PDS, Tranquil itself is compiled to a single binary with no nodeJS runtime. However, at time of writing, Tranquil requires postgres running separately.

 ## Quick Start

@@ -38,6 +46,8 @@ just test
 just lint
 ```

+Nix users can enter a devshell with `nix develop`, or `direnv allow` to auto-enter via the bundled `.envrc`. Pre-built artifacts (including the devshell) are available from our [binary cache](docs/install-nix.md#binary-cache).
+
 ## Production Deployment

 ### Quick Deploy (Docker/Podman Compose)
@@ -51,20 +61,44 @@ podman-compose -f docker-compose.prod.yaml up -d

 ### Installation Guides

+- [Nix](docs/install-nix.md)
 - [Debian](docs/install-debian.md)
 - [Containers](docs/install-containers.md)
 - [Kubernetes](docs/install-kubernetes.md)

-## Maintainers to ping
+## Community

- [@lewis.moe](https://bsky.app/profile/did:plc:3fwecdnvtcscjnrx2p4n7alz)
- [@nel.pet](https://bsky.app/profile/did:plc:h5wsnqetncv6lu2weom35lg2)
+### "Let's connect!" or whatever linkedin-types say

-## Thanks
+We currently don't have a shared space to chat and organize Tranquil things, but we're very interested in changing that in the near future. What do you suggest? Anything but a discord server.

-This project is very grateful to [@nonbinary.computer](https://bsky.app/profile/did:plc:yfvwmnlztr4dwkb7hwz55r2g), [@juli.ee](https://bsky.app/profile/did:plc:7vimlesenouvuaqvle42yhvo), [@mary.my.id](https://bsky.app/profile/did:plc:ia76kvnndjutgedggx2ibrem), and [@baileytownsend.dev](https://bsky.app/profile/did:plc:rnpkyqnmsw4ipey6eotbdnnf) for their help and their code to lean on.
+### Core team
+
+- [@oyster.cafe](https://tangled.org/did:plc:3fwecdnvtcscjnrx2p4n7alz)
+- [@nel.pet](https://tangled.org/did:plc:h5wsnqetncv6lu2weom35lg2)
+
+### Amazing contributers
+
+- [@isabelroses.com](https://tangled.org/did:plc:qxichs7jsycphrsmbujwqbfb)
+- [@quilling.dev](https://tangled.org/did:plc:jrtgsidnmxaen4offglr5lsh)
+- [@koi.rip](https://tangled.org/did:plc:b26ewgkrnx3yvsp2cdao3ntu)
+- [@bas.sh](https://tangled.org/did:plc:c52wep6lj4sfbsqiz3yvb55h)
+- [@nekomimi.pet](https://tangled.org/did:plc:ttdrpj45ibqunmfhdsb4zdwq)
+- [@islacant.win](https://tangled.org/did:plc:aut6evcs6d6ngaunqgfhdzzu)
+- [@a.starrysky.fyi](https://tangled.org/did:plc:uuyqs6y3pwtbteet4swt5i5y)
+- [@sans-self.org](https://tangled.org/did:plc:wydyrngmxbcsqdvhmd7whmye)
+- [@tachyonism.tngl.sh](https://tangled.org/did:plc:w6qiwij62bmdugsd3gemhpy2)
+- Could be your name here too!
+
+### Tranquil PDS instances in the wild!
+
+- [Tranquil Farm](https://tranquil.farm)
+- Your instance here!! Don't be a stranger.
+
+### Special thanks
+
+This project is very grateful to [@nonbinary.computer](https://tangled.org/did:plc:yfvwmnlztr4dwkb7hwz55r2g), [@juliet.paris](https://tangled.org/did:plc:hs3aly5l26pozymy4b6hz7ae), [@mary.my.id](https://tangled.org/did:plc:ia76kvnndjutgedggx2ibrem), [@baileytownsend.dev](https://tangled.org/did:plc:rnpkyqnmsw4ipey6eotbdnnf), and [@ptr.pet](https://tangled.org/did:plc:dfl62fgb7wtjj3fcbb72naae) for their help and their code to lean on.

 ## License

 AGPL-3.0-or-later. Documentation is CC BY-SA 4.0. See [LICENSE](LICENSE) for details.
-
@@ -12,7 +12,6 @@ use tranquil_pds::auth::{Auth, NotTakendown, Permissive};
 use tranquil_pds::state::AppState;

 const APP_BSKY_NAMESPACE: &str = "app.bsky";
-const MAX_PREFERENCES_COUNT: usize = 100;
 const MAX_PREFERENCE_SIZE: usize = 10_000;
 const PERSONAL_DETAILS_PREF: &str = "app.bsky.actor.defs#personalDetailsPref";
 const DECLARED_AGE_PREF: &str = "app.bsky.actor.defs#declaredAgePref";
@@ -92,6 +91,8 @@ pub async fn put_preferences(
    auth: Auth<NotTakendown>,
    Json(input): Json<PutPreferencesInput>,
 ) -> Response {
+    let max_preferences_count: usize = tranquil_config::get().server.max_preferences_count;
+
    let has_full_access = auth.permissions().has_full_access();
    let user_id: uuid::Uuid = match state.repos.user.get_id_by_did(&auth.did).await {
        Ok(Some(id)) => id,
@@ -99,11 +100,11 @@ pub async fn put_preferences(
            return ApiError::InternalError(Some("User not found".into())).into_response();
        }
    };
-    if input.preferences.len() > MAX_PREFERENCES_COUNT {
+    if input.preferences.len() > max_preferences_count {
        return ApiError::InvalidRequest(format!(
            "Too many preferences: {} exceeds limit of {}",
            input.preferences.len(),
-            MAX_PREFERENCES_COUNT
+            max_preferences_count
        ))
        .into_response();
    }
@@ -76,7 +76,7 @@ pub async fn link_signal_device(
        let result = tokio::select! {
            biased;
            _ = shutdown.cancelled() => {
-                tracing::info!("Signal linking aborted due to server shutdown");
+                tracing::info!("server shutting down, aborting signal linking");
                return;
            }
            r = link_result.completion => r,
@@ -84,10 +84,10 @@ pub async fn link_signal_device(
        match result {
            Ok(Ok(client)) => {
                if slot_for_task.complete_link(generation, client).await {
-                    tracing::info!("Signal device linked successfully");
+                    tracing::info!("signal device linked");
                } else {
                    tracing::warn!(
-                        "Signal link completed but generation mismatch or already linked; discarding"
+                        "discarding completed signal link, generation mismatch or already linked"
                    );
                }
            }
@@ -231,10 +231,19 @@ pub async fn verify_credential(
    app_passwords
        .into_iter()
        .find(|app| bcrypt::verify(password, &app.password_hash).unwrap_or(false))
-        .map(|app| CredentialMatch::AppPassword {
-            name: app.name,
-            scopes: app.scopes,
-            controller_did: app.created_by_controller_did,
+        .map(|app| {
+            let scopes = app.scopes.unwrap_or_else(|| {
+                if app.privilege.is_privileged() {
+                    "transition:generic transition:chat.bsky".to_string()
+                } else {
+                    "transition:generic".to_string()
+                }
+            });
+            CredentialMatch::AppPassword {
+                name: app.name,
+                scopes: Some(scopes),
+                controller_did: app.created_by_controller_did,
+            }
        })
 }

@@ -282,7 +282,7 @@ async fn process_messaging_channel_update(
                "Invalid Telegram username. Must be 5-32 characters, alphanumeric or underscore".into(),
            ),
            CommsChannel::Signal => ApiError::InvalidRequest(
-                "Invalid Signal username. Must be 3-32 characters followed by .XX (e.g. username.01)".into(),
+                "Invalid Signal username. Must be a 3-32 character nickname, a dot, then a 2-20 digit discriminator".into(),
            ),
            CommsChannel::Email => ApiError::InvalidEmail,
        });
@@ -132,7 +132,14 @@ pub async fn create_app_password(
        };
        (scope_result, Some(controller.clone()))
    } else {
-        (input.scopes.clone(), None)
+        let scopes = match input.scopes {
+            Some(ref s) => s.clone(),
+            None => match input.privileged {
+                Some(false) => "transition:generic".to_string(),
+                _ => "transition:generic transition:chat.bsky".to_string(),
+            },
+        };
+        (Some(scopes), None)
    };

    let password = generate_app_password();
@@ -401,7 +401,7 @@ pub async fn create_passkey_account(
                    refresh_expires_at: refresh_expires,
                    login_type: tranquil_db_traits::LoginType::Modern,
                    mfa_verified: false,
-                    scope: Some("transition:generic".to_string()),
+                    scope: Some("transition:generic transition:chat.bsky".to_string()),
                    controller_did: None,
                    app_password_name: None,
                };
@@ -163,6 +163,7 @@ pub async fn create_session(
    let email_2fa_enabled = row.email_2fa_enabled;
    let is_legacy_login = has_totp || email_2fa_enabled;
    let twofa_ctx = tranquil_pds::auth::legacy_2fa::Legacy2faContext {
+        is_app_password: app_password_name.is_some(),
        email_2fa_enabled,
        has_totp,
        allow_legacy_login: row.allow_legacy_login,
@@ -299,7 +299,7 @@ impl DiscordSender {
            "type": 1,
            "options": [{
                "name": "handle",
-                "description": "Your PDS handle (e.g. alice.example.com)",
+                "description": "Your PDS handle",
                "type": 3,
                "required": false
            }]
@@ -65,6 +65,9 @@ pub fn ensure_test_defaults() {
            if env::var("ENABLE_PDS_HOSTED_DID_WEB").is_err() {
                env::set_var("ENABLE_PDS_HOSTED_DID_WEB", "true");
            }
+            if env::var("TRANQUIL_LEXICON_OFFLINE").is_err() {
+                env::set_var("TRANQUIL_LEXICON_OFFLINE", "1");
+            }
        }
        TranquilConfig::builder()
            .env()
@@ -108,6 +111,9 @@ pub struct TranquilConfig {
    #[config(nested)]
    pub storage: StorageConfig,

+    #[config(nested)]
+    pub tranquil_store: TranquilStoreConfig,
+
    #[config(nested)]
    pub cache: CacheConfig,

@@ -143,9 +149,6 @@ pub struct TranquilConfig {

    #[config(nested)]
    pub scheduled: ScheduledConfig,
-
-    #[config(nested)]
-    pub tranquil_store: TranquilStoreConfig,
 }

 impl TranquilConfig {
@@ -394,7 +397,7 @@ impl TranquilConfig {

 #[derive(Debug, Config)]
 pub struct ServerConfig {
-    /// Public hostname of the PDS (e.g. `pds.example.com`).
+    /// Public hostname of the PDS, such as `pds.example.com`.
    #[config(env = "PDS_HOSTNAME")]
    pub hostname: String,

@@ -451,6 +454,10 @@ pub struct ServerConfig {
    /// Maximum allowed blob size in bytes (default 10 GiB).
    #[config(env = "MAX_BLOB_SIZE", default = 10_737_418_240u64)]
    pub max_blob_size: u64,
+
+    /// Maximum allowed number of preferences
+    #[config(env = "MAX_PREFERENCES_COUNT", default = 1000)]
+    pub max_preferences_count: usize,
 }

 impl ServerConfig {
@@ -459,8 +466,7 @@ impl ServerConfig {
        format!("https://{}", self.hostname)
    }

-    /// Hostname without port suffix (e.g. `pds.example.com` from
-    /// `pds.example.com:443`).
+    /// Hostname without port suffix. Returns `pds.example.com` from `pds.example.com:443`.
    pub fn hostname_without_port(&self) -> &str {
        self.hostname.split(':').next().unwrap_or(&self.hostname)
    }
@@ -656,10 +662,12 @@ pub struct StorageConfig {
    #[config(env = "S3_BUCKET")]
    pub s3_bucket: Option<String>,

-    /// Custom S3 endpoint URL (for MinIO, R2, etc.).
+    /// Custom S3 endpoint URL.
    #[config(env = "S3_ENDPOINT")]
    pub s3_endpoint: Option<String>,

+    /// Repository backend: `postgres` by default, or `tranquil-store`, our embedded db.
+    /// tranquil-store is EXPERIMENTAL!!!! RISK OF TOTAL DATA LOSS.
    #[config(env = "REPO_BACKEND", default = "postgres")]
    pub repo_backend: String,
 }
@@ -674,7 +682,7 @@ impl StorageConfig {

 #[derive(Debug, Config)]
 pub struct CacheConfig {
-    /// Cache backend: `ripple` (default, built-in gossip) or `valkey`.
+    /// Cache backend: `ripple` by default, or `valkey`.
    #[config(env = "CACHE_BACKEND", default = "ripple")]
    pub backend: String,

@@ -720,6 +728,10 @@ pub struct FirehoseConfig {
    #[config(env = "FIREHOSE_MAX_LAG", default = 5000)]
    pub max_lag: u64,

+    /// Maximum concurrent full-repo exports, eg. getRepo without `since`.
+    #[config(env = "MAX_CONCURRENT_REPO_EXPORTS", default = 4)]
+    pub max_concurrent_repo_exports: usize,
+
    /// List of relay / crawler notification URLs.
    #[config(env = "CRAWLERS", parse_env = split_comma_list)]
    pub crawlers: Option<Vec<String>>,
@@ -770,6 +782,8 @@ pub struct TelegramConfig {

 #[derive(Debug, Config)]
 pub struct SignalConfig {
+    /// Protocol state is stored in postgres' signal_* tables.
+    /// Link a device via the admin API before enabling.
    #[config(env = "SIGNAL_ENABLED", default = false)]
    pub enabled: bool,
 }
@@ -1120,27 +1134,26 @@ pub struct ScheduledConfig {

    /// Maximum age of events retained in the eventlog before pruning.
    /// Per the atproto firehose spec, the relay backfill window only needs
-    /// to cover "hours or days". Default: 7 days.
+    /// to cover "hours or days".
    #[config(env = "EVENT_RETENTION_MAX_AGE_SECS", default = 604800)]
    pub event_retention_max_age_secs: u64,

    /// Interval in seconds between event retention prune passes.
-    /// Set to 0 to disable. Default: hourly.
+    /// Set to 0 to disable.
    #[config(env = "EVENT_RETENTION_INTERVAL_SECS", default = 3600)]
    pub event_retention_interval_secs: u64,
 }

 #[derive(Debug, Config)]
 pub struct TranquilStoreConfig {
-    /// Directory for tranquil-store data (metastore, eventlog).
+    /// Directory for tranquil-store data: the metastore, eventlog, and blockstore.
    #[config(
        env = "TRANQUIL_STORE_DATA_DIR",
        default = "/var/lib/tranquil-pds/store"
    )]
    pub data_dir: String,

-    /// Fjall block cache size in megabytes. Defaults to 20% of system RAM
-    /// when unset.
+    /// Fjall block cache size in megabytes. Defaults to 20% of system RAM when unset.
    #[config(env = "TRANQUIL_STORE_MEMORY_BUDGET_MB")]
    pub memory_budget_mb: Option<u64>,

@@ -1148,9 +1161,9 @@ pub struct TranquilStoreConfig {
    #[config(env = "TRANQUIL_STORE_HANDLER_THREADS")]
    pub handler_threads: Option<usize>,

-    /// Maximum total bytes of pending (unsynced) eventlog payloads. Appenders
-    /// block once this budget is exhausted until in-flight events drain via
-    /// fsync. Set to 0 to disable backpressure (unbounded). Default: 1 GiB.
+    /// Maximum total bytes of pending (unsynced) eventlog payloads. Appenders block
+    /// once this budget is exhausted until in-flight events drain via fsync. Set to
+    /// 0 to disable backpressure. Default: 1 GiB.
    #[config(
        env = "TRANQUIL_STORE_EVENTLOG_PENDING_BYTES_BUDGET",
        default = 1_073_741_824
@@ -144,12 +144,12 @@ pub trait UserRepository: Send + Sync {

    async fn get_by_email(&self, email: &str) -> Result<Option<UserForVerification>, DbError>;

-    async fn get_login_check_by_handle_or_email(
+    async fn get_login_check_by_identifier(
        &self,
        identifier: &str,
    ) -> Result<Option<UserLoginCheck>, DbError>;

-    async fn get_login_info_by_handle_or_email(
+    async fn get_login_info_by_identifier(
        &self,
        identifier: &str,
    ) -> Result<Option<UserLoginInfo>, DbError>;
@@ -358,6 +358,19 @@ pub trait UserRepository: Send + Sync {
        challenge_type: WebauthnChallengeType,
    ) -> Result<(), DbError>;

+    async fn save_discoverable_challenge(
+        &self,
+        request_key: &str,
+        state_json: &str,
+    ) -> Result<Uuid, DbError>;
+
+    async fn load_discoverable_challenge(
+        &self,
+        request_key: &str,
+    ) -> Result<Option<String>, DbError>;
+
+    async fn delete_discoverable_challenge(&self, request_key: &str) -> Result<(), DbError>;
+
    async fn get_totp_record(&self, did: &Did) -> Result<Option<TotpRecord>, DbError>;

    async fn get_totp_record_state(&self, did: &Did) -> Result<Option<TotpRecordState>, DbError>;
@@ -1102,6 +1102,59 @@ impl UserRepository for PostgresUserRepository {
        Ok(())
    }

+    async fn save_discoverable_challenge(
+        &self,
+        request_key: &str,
+        state_json: &str,
+    ) -> Result<Uuid, DbError> {
+        let id = Uuid::new_v4();
+        let challenge = id.as_bytes().to_vec();
+        let expires_at = chrono::Utc::now() + chrono::Duration::minutes(5);
+        sqlx::query!(
+            r#"INSERT INTO webauthn_challenges (id, did, challenge, challenge_type, state_json, expires_at)
+               VALUES ($1, $2, $3, 'discoverable', $4, $5)"#,
+            id,
+            request_key,
+            challenge,
+            state_json,
+            expires_at,
+        )
+        .execute(&self.pool)
+        .await
+        .map_err(map_sqlx_error)?;
+
+        Ok(id)
+    }
+
+    async fn load_discoverable_challenge(
+        &self,
+        request_key: &str,
+    ) -> Result<Option<String>, DbError> {
+        let row = sqlx::query_scalar!(
+            r#"SELECT state_json FROM webauthn_challenges
+               WHERE did = $1 AND challenge_type = 'discoverable' AND expires_at > NOW()
+               ORDER BY created_at DESC LIMIT 1"#,
+            request_key,
+        )
+        .fetch_optional(&self.pool)
+        .await
+        .map_err(map_sqlx_error)?;
+
+        Ok(row)
+    }
+
+    async fn delete_discoverable_challenge(&self, request_key: &str) -> Result<(), DbError> {
+        sqlx::query!(
+            "DELETE FROM webauthn_challenges WHERE did = $1 AND challenge_type = 'discoverable'",
+            request_key,
+        )
+        .execute(&self.pool)
+        .await
+        .map_err(map_sqlx_error)?;
+
+        Ok(())
+    }
+
    async fn get_totp_record(&self, did: &Did) -> Result<Option<TotpRecord>, DbError> {
        let row = sqlx::query!(
            "SELECT secret_encrypted, encryption_version, verified FROM user_totp WHERE did = $1",
@@ -1330,12 +1383,12 @@ impl UserRepository for PostgresUserRepository {
        Ok(())
    }

-    async fn get_login_check_by_handle_or_email(
+    async fn get_login_check_by_identifier(
        &self,
        identifier: &str,
    ) -> Result<Option<UserLoginCheck>, DbError> {
        sqlx::query!(
-            "SELECT did, password_hash FROM users WHERE handle = $1 OR email = $1",
+            "SELECT did, password_hash FROM users WHERE handle = $1 OR did = $1",
            identifier
        )
        .fetch_optional(&self.pool)
@@ -1349,7 +1402,7 @@ impl UserRepository for PostgresUserRepository {
        })
    }

-    async fn get_login_info_by_handle_or_email(
+    async fn get_login_info_by_identifier(
        &self,
        identifier: &str,
    ) -> Result<Option<UserLoginInfo>, DbError> {
@@ -1361,7 +1414,7 @@ impl UserRepository for PostgresUserRepository {
                   email_verified, discord_verified, telegram_verified, signal_verified,
                   account_type as "account_type!: AccountType"
            FROM users
-            WHERE handle = $1 OR email = $1
+            WHERE handle = $1 OR did = $1
            "#,
            identifier
        )
@@ -1524,7 +1577,7 @@ impl UserRepository for PostgresUserRepository {
                COALESCE((SELECT (value_json)::boolean FROM account_preferences WHERE user_id = u.id AND name = 'email_auth_factor' ORDER BY created_at DESC LIMIT 1), false) as "email_2fa_enabled!"
            FROM users u
            JOIN user_keys k ON u.id = k.user_id
-            WHERE u.handle = $1 OR u.email = $1 OR u.did = $1"#,
+            WHERE u.handle = $1 OR u.did = $1"#,
            identifier
        )
        .fetch_optional(&self.pool)
@@ -24,3 +24,4 @@ urlencoding = { workspace = true, optional = true }
 [dev-dependencies]
 wiremock = { workspace = true }
 tokio = { workspace = true }
+futures = { workspace = true }
@@ -5,46 +5,102 @@ use std::collections::{HashMap, VecDeque};
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::time::{Duration, Instant};
+use tokio::sync::Notify;

 const NEGATIVE_CACHE_TTL: Duration = Duration::from_secs(24 * 60 * 60);
+const POSITIVE_CACHE_TTL: Duration = Duration::from_secs(24 * 60 * 60);
+const REFRESH_FAILURE_BACKOFF: Duration = Duration::from_secs(60);
 const MAX_DYNAMIC_SCHEMAS: usize = 1024;

 struct NegativeEntry {
    expires_at: Instant,
 }

+struct PositiveEntry {
+    doc: Arc<LexiconDoc>,
+    expires_at: Instant,
+}
+
+pub(crate) enum CacheEntry {
+    Fresh(Arc<LexiconDoc>),
+    Stale(Arc<LexiconDoc>),
+}
+
+impl CacheEntry {
+    #[cfg(test)]
+    fn is_fresh(&self) -> bool {
+        matches!(self, Self::Fresh(_))
+    }
+}
+
 struct SchemaStore {
-    schemas: HashMap<String, Arc<LexiconDoc>>,
+    schemas: HashMap<String, PositiveEntry>,
    insertion_order: VecDeque<String>,
 }

 pub struct DynamicRegistry {
    store: RwLock<SchemaStore>,
    negative_cache: RwLock<HashMap<String, NegativeEntry>>,
+    in_flight: RwLock<HashMap<String, Arc<Notify>>>,
    network_disabled: AtomicBool,
 }

+struct InFlightGuard<'a> {
+    registry: &'a DynamicRegistry,
+    nsid: String,
+}
+
+impl Drop for InFlightGuard<'_> {
+    fn drop(&mut self) {
+        let notify = self.registry.in_flight.write().remove(&self.nsid);
+        if let Some(n) = notify {
+            n.notify_waiters();
+        }
+    }
+}
+
 impl DynamicRegistry {
    pub fn new() -> Self {
-        let network_disabled =
-            std::env::var("TRANQUIL_LEXICON_OFFLINE").is_ok_and(|v| v == "1" || v == "true");
        Self {
            store: RwLock::new(SchemaStore {
                schemas: HashMap::new(),
                insertion_order: VecDeque::new(),
            }),
            negative_cache: RwLock::new(HashMap::new()),
-            network_disabled: AtomicBool::new(network_disabled),
+            in_flight: RwLock::new(HashMap::new()),
+            network_disabled: AtomicBool::new(false),
        }
    }

-    #[allow(dead_code)]
+    pub fn from_env() -> Self {
+        let registry = Self::new();
+        let disabled =
+            std::env::var("TRANQUIL_LEXICON_OFFLINE").is_ok_and(|v| v == "1" || v == "true");
+        registry.set_network_disabled(disabled);
+        registry
+    }
+
    pub fn set_network_disabled(&self, disabled: bool) {
        self.network_disabled.store(disabled, Ordering::Relaxed);
    }

-    pub fn get(&self, nsid: &str) -> Option<Arc<LexiconDoc>> {
-        self.store.read().schemas.get(nsid).cloned()
+    pub fn get_cached(&self, nsid: &str) -> Option<Arc<LexiconDoc>> {
+        self.store
+            .read()
+            .schemas
+            .get(nsid)
+            .map(|e| Arc::clone(&e.doc))
+    }
+
+    pub(crate) fn get_entry(&self, nsid: &str) -> Option<CacheEntry> {
+        let now = Instant::now();
+        self.store.read().schemas.get(nsid).map(|e| {
+            if e.expires_at > now {
+                CacheEntry::Fresh(Arc::clone(&e.doc))
+            } else {
+                CacheEntry::Stale(Arc::clone(&e.doc))
+            }
+        })
    }

    pub fn is_negative_cached(&self, nsid: &str) -> bool {
@@ -56,7 +112,7 @@ impl DynamicRegistry {

    fn insert_negative(&self, nsid: &str) {
        let mut cache = self.negative_cache.write();
-        if cache.len() > MAX_DYNAMIC_SCHEMAS {
+        if cache.len() >= MAX_DYNAMIC_SCHEMAS {
            let now = Instant::now();
            cache.retain(|_, entry| entry.expires_at > now);
        }
@@ -87,29 +143,95 @@ impl DynamicRegistry {
            });
        }

-        if store
-            .schemas
-            .insert(nsid.clone(), Arc::clone(&arc))
-            .is_some()
-        {
+        let entry = PositiveEntry {
+            doc: Arc::clone(&arc),
+            expires_at: Instant::now() + POSITIVE_CACHE_TTL,
+        };
+        if store.schemas.insert(nsid.clone(), entry).is_some() {
            store.insertion_order.retain(|k| k != &nsid);
        }
        store.insertion_order.push_back(nsid.clone());
+        drop(store);

        self.negative_cache.write().remove(&arc.id);

        arc
    }

+    fn bump_expiry(&self, nsid: &str, duration: Duration) {
+        let mut store = self.store.write();
+        if let Some(entry) = store.schemas.get_mut(nsid) {
+            entry.expires_at = Instant::now() + duration;
+        }
+    }
+
    pub async fn resolve_and_cache(&self, nsid: &str) -> Result<Arc<LexiconDoc>, ResolveError> {
-        if let Some(doc) = self.get(nsid) {
-            return Ok(doc);
+        self.resolve_and_cache_with(nsid, |n| async move { resolve_lexicon(&n).await })
+            .await
+    }
+
+    async fn resolve_and_cache_with<F, Fut>(
+        &self,
+        nsid: &str,
+        resolver: F,
+    ) -> Result<Arc<LexiconDoc>, ResolveError>
+    where
+        F: FnOnce(String) -> Fut,
+        Fut: std::future::Future<Output = Result<LexiconDoc, ResolveError>>,
+    {
+        match self.get_entry(nsid) {
+            Some(CacheEntry::Fresh(doc)) => Ok(doc),
+            Some(CacheEntry::Stale(stale)) => self.refresh_stale(nsid, stale, resolver).await,
+            None => self.resolve_fresh(nsid, resolver).await,
+        }
+    }
+
+    async fn refresh_stale<F, Fut>(
+        &self,
+        nsid: &str,
+        stale: Arc<LexiconDoc>,
+        resolver: F,
+    ) -> Result<Arc<LexiconDoc>, ResolveError>
+    where
+        F: FnOnce(String) -> Fut,
+        Fut: std::future::Future<Output = Result<LexiconDoc, ResolveError>>,
+    {
+        if self.network_disabled.load(Ordering::Relaxed) {
+            return Ok(stale);
        }

+        match self.acquire_leadership(nsid) {
+            Some(_guard) => match resolver(nsid.to_string()).await {
+                Ok(doc) => Ok(self.insert_schema(doc)),
+                Err(e) => {
+                    self.bump_expiry(nsid, REFRESH_FAILURE_BACKOFF);
+                    tracing::warn!(
+                        nsid = nsid,
+                        error = %e,
+                        "lexicon refresh failed, serving stale cached entry"
+                    );
+                    Ok(stale)
+                }
+            },
+            None => {
+                self.wait_for_leader(nsid).await;
+                Ok(self.get_cached(nsid).unwrap_or(stale))
+            }
+        }
+    }
+
+    async fn resolve_fresh<F, Fut>(
+        &self,
+        nsid: &str,
+        resolver: F,
+    ) -> Result<Arc<LexiconDoc>, ResolveError>
+    where
+        F: FnOnce(String) -> Fut,
+        Fut: std::future::Future<Output = Result<LexiconDoc, ResolveError>>,
+    {
        if self.network_disabled.load(Ordering::Relaxed) {
            return Err(ResolveError::NetworkDisabled);
        }
-
        if self.is_negative_cached(nsid) {
            return Err(ResolveError::NegativelyCached {
                nsid: nsid.to_string(),
@@ -117,19 +239,73 @@ impl DynamicRegistry {
            });
        }

-        match resolve_lexicon(nsid).await {
-            Ok(doc) => Ok(self.insert_schema(doc)),
-            Err(e) => {
-                tracing::debug!(nsid = nsid, error = %e, "caching negative resolution result");
-                self.insert_negative(nsid);
-                Err(e)
+        match self.acquire_leadership(nsid) {
+            Some(_guard) => match resolver(nsid.to_string()).await {
+                Ok(doc) => Ok(self.insert_schema(doc)),
+                Err(e) => {
+                    self.insert_negative(nsid);
+                    tracing::debug!(nsid = nsid, error = %e, "caching negative resolution result");
+                    Err(e)
+                }
+            },
+            None => {
+                self.wait_for_leader(nsid).await;
+                match self.get_cached(nsid) {
+                    Some(doc) => Ok(doc),
+                    None if self.is_negative_cached(nsid) => Err(ResolveError::NegativelyCached {
+                        nsid: nsid.to_string(),
+                        ttl_secs: NEGATIVE_CACHE_TTL.as_secs(),
+                    }),
+                    None => Err(ResolveError::LeaderAborted {
+                        nsid: nsid.to_string(),
+                    }),
+                }
            }
        }
    }

+    fn acquire_leadership(&self, nsid: &str) -> Option<InFlightGuard<'_>> {
+        let mut map = self.in_flight.write();
+        if map.contains_key(nsid) {
+            None
+        } else {
+            map.insert(nsid.to_string(), Arc::new(Notify::new()));
+            Some(InFlightGuard {
+                registry: self,
+                nsid: nsid.to_string(),
+            })
+        }
+    }
+
+    async fn wait_for_leader(&self, nsid: &str) {
+        let notify = {
+            let map = self.in_flight.read();
+            match map.get(nsid) {
+                Some(n) => Arc::clone(n),
+                None => return,
+            }
+        };
+        let notified = notify.notified();
+        tokio::pin!(notified);
+        notified.as_mut().enable();
+        let still_active = self.in_flight.read().contains_key(nsid);
+        if !still_active {
+            return;
+        }
+        notified.as_mut().await;
+    }
+
    pub fn schema_count(&self) -> usize {
        self.store.read().schemas.len()
    }
+
+    #[cfg(test)]
+    fn expire_now(&self, nsid: &str) {
+        let mut store = self.store.write();
+        if let Some(entry) = store.schemas.get_mut(nsid) {
+            entry.expires_at = Instant::now();
+        }
+    }
 }

 impl Default for DynamicRegistry {
@@ -171,7 +347,7 @@ mod tests {
    #[test]
    fn test_empty_lookup() {
        let registry = DynamicRegistry::new();
-        assert!(registry.get("com.example.nonexistent").is_none());
+        assert!(registry.get_cached("com.example.nonexistent").is_none());
        assert_eq!(registry.schema_count(), 0);
    }

@@ -188,9 +364,12 @@ mod tests {
        assert_eq!(arc.id, "com.example.test");
        assert_eq!(registry.schema_count(), 1);

-        let retrieved = registry.get("com.example.test");
+        let retrieved = registry.get_cached("com.example.test");
        assert!(retrieved.is_some());
        assert_eq!(retrieved.unwrap().id, "com.example.test");
+
+        let entry = registry.get_entry("com.example.test").unwrap();
+        assert!(entry.is_fresh(), "freshly inserted entry must be fresh");
    }

    #[test]
@@ -210,6 +389,218 @@ mod tests {
        assert!(!registry.is_negative_cached("com.example.test"));
    }

+    #[test]
+    fn test_positive_entry_reports_stale_after_ttl() {
+        let registry = DynamicRegistry::new();
+        let doc = LexiconDoc {
+            lexicon: 1,
+            id: "pet.nel.stale".to_string(),
+            defs: HashMap::new(),
+        };
+        registry.insert_schema(doc);
+
+        assert!(registry.get_entry("pet.nel.stale").unwrap().is_fresh());
+
+        registry.expire_now("pet.nel.stale");
+
+        assert!(
+            !registry.get_entry("pet.nel.stale").unwrap().is_fresh(),
+            "entry past expiry must be reported stale"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_stale_served_on_resolve_failure() {
+        let registry = DynamicRegistry::new();
+        let doc = LexiconDoc {
+            lexicon: 1,
+            id: "pet.nel.flaky".to_string(),
+            defs: HashMap::new(),
+        };
+        registry.insert_schema(doc);
+        registry.expire_now("pet.nel.flaky");
+
+        let result = registry
+            .resolve_and_cache_with("pet.nel.flaky", |n| async move {
+                Err::<LexiconDoc, _>(ResolveError::DnsLookup {
+                    domain: n,
+                    reason: "simulated failure".to_string(),
+                })
+            })
+            .await;
+
+        let served = result.expect("stale entry must be served when refresh fails");
+        assert_eq!(served.id, "pet.nel.flaky");
+        assert!(
+            registry.get_entry("pet.nel.flaky").unwrap().is_fresh(),
+            "failed refresh must bump expiry so subsequent lookups skip the resolver"
+        );
+        assert!(
+            !registry.is_negative_cached("pet.nel.flaky"),
+            "stale refresh failure must not poison negative cache"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_fresh_hit_skips_resolver() {
+        let registry = DynamicRegistry::new();
+        let doc = LexiconDoc {
+            lexicon: 1,
+            id: "pet.nel.fresh".to_string(),
+            defs: HashMap::new(),
+        };
+        registry.insert_schema(doc);
+
+        let result = registry
+            .resolve_and_cache_with("pet.nel.fresh", |_| async move {
+                panic!("resolver must not run on fresh hit")
+            })
+            .await;
+
+        assert!(result.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_stale_served_when_network_disabled() {
+        let registry = DynamicRegistry::new();
+        let doc = LexiconDoc {
+            lexicon: 1,
+            id: "pet.nel.offline".to_string(),
+            defs: HashMap::new(),
+        };
+        registry.insert_schema(doc);
+        registry.expire_now("pet.nel.offline");
+        registry.set_network_disabled(true);
+
+        let result = registry
+            .resolve_and_cache_with("pet.nel.offline", |_| async move {
+                panic!("resolver must not run when network disabled")
+            })
+            .await;
+
+        assert!(result.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_successful_refresh_updates_cached_at() {
+        let registry = DynamicRegistry::new();
+        let doc = LexiconDoc {
+            lexicon: 1,
+            id: "pet.nel.refresh".to_string(),
+            defs: HashMap::new(),
+        };
+        registry.insert_schema(doc);
+        registry.expire_now("pet.nel.refresh");
+
+        assert!(!registry.get_entry("pet.nel.refresh").unwrap().is_fresh());
+
+        let refreshed = registry
+            .resolve_and_cache_with("pet.nel.refresh", |n| async move {
+                Ok(LexiconDoc {
+                    lexicon: 1,
+                    id: n,
+                    defs: HashMap::new(),
+                })
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(refreshed.id, "pet.nel.refresh");
+        assert!(
+            registry.get_entry("pet.nel.refresh").unwrap().is_fresh(),
+            "refresh must restore freshness"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_single_flight_dedups_concurrent_resolves() {
+        use std::sync::atomic::AtomicUsize;
+        let registry = Arc::new(DynamicRegistry::new());
+        let calls = Arc::new(AtomicUsize::new(0));
+
+        let tasks: Vec<_> = (0..16)
+            .map(|_| {
+                let registry = Arc::clone(&registry);
+                let calls = Arc::clone(&calls);
+                tokio::spawn(async move {
+                    registry
+                        .resolve_and_cache_with("pet.nel.herd", |n| {
+                            let calls = Arc::clone(&calls);
+                            async move {
+                                calls.fetch_add(1, Ordering::SeqCst);
+                                tokio::time::sleep(Duration::from_millis(50)).await;
+                                Ok(LexiconDoc {
+                                    lexicon: 1,
+                                    id: n,
+                                    defs: HashMap::new(),
+                                })
+                            }
+                        })
+                        .await
+                })
+            })
+            .collect();
+
+        let results = futures_collect(tasks).await;
+        results
+            .iter()
+            .for_each(|r| assert!(r.is_ok(), "all single-flight callers must succeed"));
+        assert_eq!(
+            calls.load(Ordering::SeqCst),
+            1,
+            "single-flight must coalesce concurrent resolves"
+        );
+        assert_eq!(registry.schema_count(), 1);
+    }
+
+    #[tokio::test]
+    async fn test_single_flight_followers_observe_leader_failure() {
+        use std::sync::atomic::AtomicUsize;
+        let registry = Arc::new(DynamicRegistry::new());
+        let calls = Arc::new(AtomicUsize::new(0));
+
+        let tasks: Vec<_> = (0..8)
+            .map(|_| {
+                let registry = Arc::clone(&registry);
+                let calls = Arc::clone(&calls);
+                tokio::spawn(async move {
+                    registry
+                        .resolve_and_cache_with("pet.nel.failHerd", |n| {
+                            let calls = Arc::clone(&calls);
+                            async move {
+                                calls.fetch_add(1, Ordering::SeqCst);
+                                tokio::time::sleep(Duration::from_millis(50)).await;
+                                Err::<LexiconDoc, _>(ResolveError::DnsLookup {
+                                    domain: n,
+                                    reason: "simulated".to_string(),
+                                })
+                            }
+                        })
+                        .await
+                })
+            })
+            .collect();
+
+        let results = futures_collect(tasks).await;
+        results
+            .iter()
+            .for_each(|r| assert!(r.is_err(), "all followers must observe leader failure"));
+        assert_eq!(
+            calls.load(Ordering::SeqCst),
+            1,
+            "single-flight must coalesce failing resolves too"
+        );
+        assert!(registry.is_negative_cached("pet.nel.failHerd"));
+    }
+
+    async fn futures_collect<T>(handles: Vec<tokio::task::JoinHandle<T>>) -> Vec<T> {
+        futures::future::join_all(handles)
+            .await
+            .into_iter()
+            .map(|r| r.expect("task panicked"))
+            .collect()
+    }
+
    #[test]
    fn test_eviction_is_fifo() {
        let registry = DynamicRegistry::new();
@@ -217,7 +608,7 @@ mod tests {
        (0..MAX_DYNAMIC_SCHEMAS).for_each(|i| {
            let doc = LexiconDoc {
                lexicon: 1,
-                id: format!("com.example.schema{}", i),
+                id: format!("pet.nel.schema{}", i),
                defs: HashMap::new(),
            };
            registry.insert_schema(doc);
@@ -226,23 +617,23 @@ mod tests {

        let trigger = LexiconDoc {
            lexicon: 1,
-            id: "com.example.trigger".to_string(),
+            id: "pet.nel.trigger".to_string(),
            defs: HashMap::new(),
        };
        registry.insert_schema(trigger);

        assert!(
-            registry.get("com.example.schema0").is_none(),
+            registry.get_cached("pet.nel.schema0").is_none(),
            "oldest entry should be evicted"
        );
        assert!(
-            registry.get("com.example.trigger").is_some(),
+            registry.get_cached("pet.nel.trigger").is_some(),
            "newly inserted entry should exist"
        );
        let evict_count = MAX_DYNAMIC_SCHEMAS / 4;
        assert!(
            registry
-                .get(&format!("com.example.schema{}", evict_count))
+                .get_cached(&format!("pet.nel.schema{}", evict_count))
                .is_some(),
            "entry after eviction window should survive"
        );
@@ -253,7 +644,7 @@ mod tests {
        let registry = DynamicRegistry::new();
        let doc = LexiconDoc {
            lexicon: 1,
-            id: "com.example.tracked".to_string(),
+            id: "pet.nel.tracked".to_string(),
            defs: HashMap::new(),
        };
        let arc = registry.insert_schema(doc);
@@ -265,7 +656,7 @@ mod tests {
        (0..MAX_DYNAMIC_SCHEMAS).for_each(|i| {
            registry.insert_schema(LexiconDoc {
                lexicon: 1,
-                id: format!("com.example.filler{}", i),
+                id: format!("pet.nel.filler{}", i),
                defs: HashMap::new(),
            });
        });
@@ -25,7 +25,7 @@ impl LexiconRegistry {
        Self {
            schemas: HashMap::new(),
            #[cfg(feature = "resolve")]
-            dynamic: crate::dynamic::DynamicRegistry::new(),
+            dynamic: crate::dynamic::DynamicRegistry::from_env(),
        }
    }

@@ -43,7 +43,7 @@ impl LexiconRegistry {
        self.schemas.get(nsid).cloned().or_else(|| {
            #[cfg(feature = "resolve")]
            {
-                self.dynamic.get(nsid)
+                self.dynamic.get_cached(nsid)
            }
            #[cfg(not(feature = "resolve"))]
            {
@@ -70,6 +70,8 @@ pub enum ResolveError {
    NegativelyCached { nsid: String, ttl_secs: u64 },
    #[error("network resolution disabled")]
    NetworkDisabled,
+    #[error("leader task for {nsid} aborted before completion")]
+    LeaderAborted { nsid: String },
 }

 pub fn nsid_to_authority(nsid: &str) -> Result<String, ResolveError> {
@@ -322,7 +322,9 @@ fn validate_blob_ref(

    if let Some(ref accept) = lex_blob.accept {
        let mime_type = obj.get("mimeType").and_then(|v| v.as_str()).unwrap_or("");
-        let matched = accept.iter().any(|pattern| mime_type_matches_accept_pattern(mime_type, pattern));
+        let matched = accept
+            .iter()
+            .any(|pattern| mime_type_matches_accept_pattern(mime_type, pattern));
        if !mime_type.is_empty() && !matched {
            return Err(LexValidationError::field(
                path,
@@ -108,7 +108,7 @@ pub async fn authorize_get(
        match state
            .repos
            .user
-            .get_login_check_by_handle_or_email(normalized.as_str())
+            .get_login_check_by_identifier(normalized.as_str())
            .await
        {
            Ok(Some(user)) => {
@@ -401,7 +401,7 @@ pub async fn authorize_post(
    let user = match state
        .repos
        .user
-        .get_login_info_by_handle_or_email(normalized_username.as_str())
+        .get_login_info_by_identifier(normalized_username.as_str())
        .await
    {
        Ok(Some(u)) => u,
@@ -410,7 +410,7 @@ pub async fn authorize_post(
                &form.password,
                "$2b$12$LQv3c1yqBWVHxkd0LHAkCOYz6TtxMQJqhN8/X4.VTtYw1ZzQKZqmK",
            );
-            return show_login_error("Invalid handle/email or password.", json_response);
+            return show_login_error("Invalid identifier or password.", json_response);
        }
        Err(_) => return show_login_error("An error occurred. Please try again.", json_response),
    };
@@ -486,7 +486,7 @@ pub async fn authorize_post(
        None => false,
    };
    if !password_valid {
-        return show_login_error("Invalid handle/email or password.", json_response);
+        return show_login_error("Invalid identifier or password.", json_response);
    }
    let is_verified = user.channel_verification.has_any_verified();
    if !is_verified {
@@ -22,7 +22,7 @@ pub async fn check_user_has_passkeys(
    let user = state
        .repos
        .user
-        .get_login_check_by_handle_or_email(bare_identifier.as_str())
+        .get_login_check_by_identifier(bare_identifier.as_str())
        .await;

    let has_passkeys = match user {
@@ -55,7 +55,7 @@ pub async fn check_user_security_status(
    let user = state
        .repos
        .user
-        .get_login_check_by_handle_or_email(normalized_identifier.as_str())
+        .get_login_check_by_identifier(normalized_identifier.as_str())
        .await;

    let (has_passkeys, has_totp, has_password, is_delegated, did): (
@@ -99,7 +99,7 @@ pub async fn check_user_security_status(
 #[derive(Debug, Deserialize)]
 pub struct PasskeyStartInput {
    pub request_uri: String,
-    pub identifier: String,
+    pub identifier: Option<String>,
    pub delegated_did: Option<String>,
 }

@@ -160,14 +160,88 @@ pub async fn passkey_start(
            .into_response();
    }

+    match form.identifier.filter(|s| !s.trim().is_empty()) {
+        Some(identifier) => {
+            passkey_start_named(
+                state,
+                identifier,
+                form.delegated_did,
+                request_data,
+                passkey_start_request_id,
+            )
+            .await
+        }
+        None => passkey_start_discoverable(state, passkey_start_request_id).await,
+    }
+}
+
+async fn passkey_start_discoverable(state: AppState, request_id: RequestId) -> Response {
+    let (rcr, auth_state) = match state.webauthn_config.start_discoverable_authentication() {
+        Ok(result) => result,
+        Err(e) => {
+            tracing::error!(error = %e, "Failed to start discoverable passkey authentication");
+            return (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                Json(serde_json::json!({
+                    "error": "server_error",
+                    "error_description": "Failed to start authentication."
+                })),
+            )
+                .into_response();
+        }
+    };
+
+    let state_json = match serde_json::to_string(&auth_state) {
+        Ok(j) => j,
+        Err(e) => {
+            tracing::error!(error = %e, "Failed to serialize authentication state");
+            return (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                Json(serde_json::json!({
+                    "error": "server_error",
+                    "error_description": "An error occurred."
+                })),
+            )
+                .into_response();
+        }
+    };
+
+    if let Err(e) = state
+        .repos
+        .user
+        .save_discoverable_challenge(request_id.as_str(), &state_json)
+        .await
+    {
+        tracing::error!(error = %e, "Failed to save discoverable authentication state");
+        return (
+            StatusCode::INTERNAL_SERVER_ERROR,
+            Json(serde_json::json!({
+                "error": "server_error",
+                "error_description": "An error occurred."
+            })),
+        )
+            .into_response();
+    }
+
+    let options = serde_json::to_value(&rcr).unwrap_or(serde_json::json!({}));
+    Json(PasskeyStartResponse { options }).into_response()
+}
+
+async fn passkey_start_named(
+    state: AppState,
+    identifier: String,
+    delegated_did: Option<String>,
+    request_data: tranquil_pds::oauth::RequestData,
+    passkey_start_request_id: RequestId,
+) -> Response {
    let hostname_for_handles = tranquil_config::get().server.hostname_without_port();
    let normalized_username =
-        NormalizedLoginIdentifier::normalize(&form.identifier, hostname_for_handles);
+        NormalizedLoginIdentifier::normalize(&identifier, hostname_for_handles);

    let user = match state
        .repos
        .user
-        .get_login_info_by_handle_or_email(normalized_username.as_str())
+        .get_login_info_by_identifier(normalized_username.as_str())
        .await
    {
        Ok(Some(u)) => u,
@@ -325,7 +399,7 @@ pub async fn passkey_start(
            .into_response();
    }

-    let delegation_from_param = match &form.delegated_did {
+    let delegation_from_param = match &delegated_did {
        Some(delegated_did_str) => match delegated_did_str.parse::<tranquil_types::Did>() {
            Ok(delegated_did) if delegated_did != user.did => {
                match state
@@ -471,85 +545,6 @@ pub async fn passkey_finish(
            .into_response();
    }

-    let did_str = match request_data.did {
-        Some(d) => d,
-        None => {
-            return (
-                StatusCode::BAD_REQUEST,
-                Json(serde_json::json!({
-                    "error": "invalid_request",
-                    "error_description": "No passkey authentication in progress."
-                })),
-            )
-                .into_response();
-        }
-    };
-    let did: tranquil_types::Did = match did_str.parse() {
-        Ok(d) => d,
-        Err(_) => {
-            return (
-                StatusCode::BAD_REQUEST,
-                Json(serde_json::json!({
-                    "error": "invalid_request",
-                    "error_description": "Invalid DID format."
-                })),
-            )
-                .into_response();
-        }
-    };
-
-    let controller_did: Option<tranquil_types::Did> = request_data
-        .controller_did
-        .as_ref()
-        .and_then(|s| s.parse().ok());
-    let passkey_owner_did = controller_did.as_ref().unwrap_or(&did);
-
-    let auth_state_json = match state
-        .repos
-        .user
-        .load_webauthn_challenge(passkey_owner_did, WebauthnChallengeType::Authentication)
-        .await
-    {
-        Ok(Some(s)) => s,
-        Ok(None) => {
-            return (
-                StatusCode::BAD_REQUEST,
-                Json(serde_json::json!({
-                    "error": "invalid_request",
-                    "error_description": "No passkey authentication in progress or challenge expired."
-                })),
-            )
-                .into_response();
-        }
-        Err(e) => {
-            tracing::error!(error = %e, "Failed to load authentication state");
-            return (
-                StatusCode::INTERNAL_SERVER_ERROR,
-                Json(serde_json::json!({
-                    "error": "server_error",
-                    "error_description": "An error occurred."
-                })),
-            )
-                .into_response();
-        }
-    };
-
-    let auth_state: webauthn_rs::prelude::SecurityKeyAuthentication =
-        match serde_json::from_str(&auth_state_json) {
-            Ok(s) => s,
-            Err(e) => {
-                tracing::error!(error = %e, "Failed to deserialize authentication state");
-                return (
-                    StatusCode::INTERNAL_SERVER_ERROR,
-                    Json(serde_json::json!({
-                        "error": "server_error",
-                        "error_description": "An error occurred."
-                    })),
-                )
-                    .into_response();
-            }
-        };
-
    let credential: webauthn_rs::prelude::PublicKeyCredential =
        match serde_json::from_value(form.credential) {
            Ok(c) => c,
@@ -566,33 +561,32 @@ pub async fn passkey_finish(
            }
        };

-    let auth_result = match state
-        .webauthn_config
-        .finish_authentication(&credential, &auth_state)
-    {
-        Ok(r) => r,
-        Err(e) => {
-            tracing::warn!(error = %e, did = %did, "Failed to verify passkey authentication");
-            return (
-                StatusCode::FORBIDDEN,
-                Json(serde_json::json!({
-                    "error": "access_denied",
-                    "error_description": "Passkey verification failed."
-                })),
-            )
-                .into_response();
+    let (did, auth_result) = match request_data.did.clone() {
+        Some(did) => match passkey_finish_named(&state, did, &request_data, &credential).await {
+            Ok(result) => result,
+            Err(response) => return response,
+        },
+        None => {
+            let result =
+                match passkey_finish_discoverable(&state, &credential, &passkey_finish_request_id)
+                    .await
+                {
+                    Ok(result) => result,
+                    Err(response) => return response,
+                };
+            if state
+                .repos
+                .oauth
+                .set_authorization_did(&passkey_finish_request_id, &result.0, None)
+                .await
+                .is_err()
+            {
+                return OAuthError::ServerError("An error occurred.".into()).into_response();
+            }
+            result
        }
    };

-    if let Err(e) = state
-        .repos
-        .user
-        .delete_webauthn_challenge(passkey_owner_did, WebauthnChallengeType::Authentication)
-        .await
-    {
-        tracing::warn!(error = %e, "Failed to delete authentication state");
-    }
-
    if auth_result.needs_update() {
        let cred_id_bytes = auth_result.cred_id().as_slice();
        match state
@@ -691,6 +685,187 @@ pub async fn passkey_finish(
    .into_response()
 }

+async fn passkey_finish_named(
+    state: &AppState,
+    did: tranquil_types::Did,
+    request_data: &tranquil_pds::oauth::RequestData,
+    credential: &webauthn_rs::prelude::PublicKeyCredential,
+) -> Result<
+    (
+        tranquil_types::Did,
+        webauthn_rs::prelude::AuthenticationResult,
+    ),
+    Response,
+> {
+    let passkey_owner_did = request_data.controller_did.as_ref().unwrap_or(&did);
+
+    let auth_state_json = state
+        .repos
+        .user
+        .load_webauthn_challenge(passkey_owner_did, WebauthnChallengeType::Authentication)
+        .await
+        .map_err(|e| {
+            tracing::error!(error = %e, "Failed to load authentication state");
+            (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                Json(serde_json::json!({"error": "server_error", "error_description": "An error occurred."})),
+            ).into_response()
+        })?
+        .ok_or_else(|| {
+            (
+                StatusCode::BAD_REQUEST,
+                Json(serde_json::json!({
+                    "error": "invalid_request",
+                    "error_description": "No passkey authentication in progress or challenge expired."
+                })),
+            ).into_response()
+        })?;
+
+    let auth_state: webauthn_rs::prelude::SecurityKeyAuthentication =
+        serde_json::from_str(&auth_state_json).map_err(|e| {
+            tracing::error!(error = %e, "Failed to deserialize authentication state");
+            (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                Json(serde_json::json!({"error": "server_error", "error_description": "An error occurred."})),
+            ).into_response()
+        })?;
+
+    let auth_result = state
+        .webauthn_config
+        .finish_authentication(credential, &auth_state)
+        .map_err(|e| {
+            tracing::warn!(error = %e, did = %did, "Failed to verify passkey authentication");
+            (
+                StatusCode::FORBIDDEN,
+                Json(serde_json::json!({
+                    "error": "access_denied",
+                    "error_description": "Passkey verification failed."
+                })),
+            )
+                .into_response()
+        })?;
+
+    let _ = state
+        .repos
+        .user
+        .delete_webauthn_challenge(passkey_owner_did, WebauthnChallengeType::Authentication)
+        .await;
+
+    Ok((did, auth_result))
+}
+
+async fn passkey_finish_discoverable(
+    state: &AppState,
+    credential: &webauthn_rs::prelude::PublicKeyCredential,
+    request_id: &RequestId,
+) -> Result<
+    (
+        tranquil_types::Did,
+        webauthn_rs::prelude::AuthenticationResult,
+    ),
+    Response,
+> {
+    let auth_state_json = state
+        .repos
+        .user
+        .load_discoverable_challenge(request_id.as_str())
+        .await
+        .map_err(|e| {
+            tracing::error!(error = %e, "Failed to load discoverable authentication state");
+            (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                Json(serde_json::json!({"error": "server_error", "error_description": "An error occurred."})),
+            ).into_response()
+        })?
+        .ok_or_else(|| {
+            (
+                StatusCode::BAD_REQUEST,
+                Json(serde_json::json!({
+                    "error": "invalid_request",
+                    "error_description": "No passkey authentication in progress or challenge expired."
+                })),
+            ).into_response()
+        })?;
+
+    let auth_state: webauthn_rs::prelude::DiscoverableAuthentication =
+        serde_json::from_str(&auth_state_json).map_err(|e| {
+            tracing::error!(error = %e, "Failed to deserialize discoverable authentication state");
+            (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                Json(serde_json::json!({"error": "server_error", "error_description": "An error occurred."})),
+            ).into_response()
+        })?;
+
+    let (_user_uuid, cred_id) = state
+        .webauthn_config
+        .identify_discoverable_authentication(credential)
+        .map_err(|e| {
+            tracing::warn!(error = %e, "Failed to identify discoverable credential");
+            (
+                StatusCode::FORBIDDEN,
+                Json(serde_json::json!({
+                    "error": "access_denied",
+                    "error_description": "Passkey verification failed."
+                })),
+            )
+                .into_response()
+        })?;
+
+    let stored_passkey = state
+        .repos
+        .user
+        .get_passkey_by_credential_id(cred_id)
+        .await
+        .map_err(|e| {
+            tracing::error!(error = %e, "Failed to look up passkey by credential ID");
+            (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                Json(serde_json::json!({"error": "server_error", "error_description": "An error occurred."})),
+            ).into_response()
+        })?
+        .ok_or_else(|| {
+            tracing::warn!("Discoverable credential not found in database");
+            (
+                StatusCode::FORBIDDEN,
+                Json(serde_json::json!({
+                    "error": "access_denied",
+                    "error_description": "Passkey not recognized."
+                })),
+            ).into_response()
+        })?;
+
+    let discoverable_key: webauthn_rs::prelude::DiscoverableKey =
+        serde_json::from_slice(&stored_passkey.public_key).map_err(|e| {
+            tracing::error!(error = %e, "Failed to deserialize stored passkey as DiscoverableKey");
+            (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                Json(serde_json::json!({"error": "server_error", "error_description": "An error occurred."})),
+            ).into_response()
+        })?;
+
+    let auth_result = state
+        .webauthn_config
+        .finish_discoverable_authentication(credential, auth_state, &[discoverable_key])
+        .map_err(|e| {
+            tracing::warn!(error = %e, did = %stored_passkey.did, "Failed to verify discoverable passkey authentication");
+            (
+                StatusCode::FORBIDDEN,
+                Json(serde_json::json!({
+                    "error": "access_denied",
+                    "error_description": "Passkey verification failed."
+                })),
+            ).into_response()
+        })?;
+
+    let _ = state
+        .repos
+        .user
+        .delete_discoverable_challenge(request_id.as_str())
+        .await;
+
+    Ok((stored_passkey.did, auth_result))
+}
+
 #[derive(Debug, Deserialize)]
 pub struct AuthorizePasskeyQuery {
    pub request_uri: String,
@@ -138,21 +138,29 @@ pub async fn pushed_authorization_request(
 }

 fn determine_client_auth(request: &ParRequest) -> Result<ClientAuth, OAuthError> {
-    if let (Some(assertion), Some(assertion_type)) =
-        (&request.client_assertion, &request.client_assertion_type)
-    {
+    let assertion = request
+        .client_assertion
+        .as_deref()
+        .filter(|s| !s.is_empty());
+    let assertion_type = request
+        .client_assertion_type
+        .as_deref()
+        .filter(|s| !s.is_empty());
+    let secret = request.client_secret.as_deref().filter(|s| !s.is_empty());
+
+    if let (Some(assertion), Some(assertion_type)) = (assertion, assertion_type) {
        if assertion_type != "urn:ietf:params:oauth:client-assertion-type:jwt-bearer" {
            return Err(OAuthError::InvalidRequest(
                "Unsupported client_assertion_type".to_string(),
            ));
        }
        return Ok(ClientAuth::PrivateKeyJwt {
-            client_assertion: assertion.clone(),
+            client_assertion: assertion.to_string(),
        });
    }
-    if let Some(secret) = &request.client_secret {
+    if let Some(secret) = secret {
        return Ok(ClientAuth::SecretPost {
-            client_secret: secret.clone(),
+            client_secret: secret.to_string(),
        });
    }
    Ok(ClientAuth::None)
@@ -138,13 +138,17 @@ impl TokenRequest {
            }
        };

-        let client_auth = match (self.client_assertion, self.client_assertion_type) {
+        let assertion = self.client_assertion.filter(|s| !s.is_empty());
+        let assertion_type = self.client_assertion_type.filter(|s| !s.is_empty());
+        let client_secret = self.client_secret.filter(|s| !s.is_empty());
+
+        let client_auth = match (assertion, assertion_type) {
            (Some(assertion), Some(assertion_type)) => RequestClientAuth::PrivateKeyJwt {
                client_id: self.client_id,
                assertion,
                assertion_type,
            },
-            _ => match self.client_secret {
+            _ => match client_secret {
                Some(secret) => RequestClientAuth::SecretPost {
                    client_id: self.client_id,
                    client_secret: secret,
@@ -1339,7 +1339,7 @@ pub async fn complete_registration(
                refresh_expires_at: refresh_meta.expires_at,
                login_type: tranquil_db_traits::LoginType::Modern,
                mfa_verified: false,
-                scope: Some("transition:generic".to_string()),
+                scope: Some("transition:generic transition:chat.bsky".to_string()),
                controller_did: None,
                app_password_name: None,
            };
@@ -78,6 +78,7 @@ tracing = { workspace = true }
 urlencoding = { workspace = true }
 uuid = { workspace = true }
 webauthn-rs = { workspace = true }
+webauthn-rs-proto = { workspace = true }
 zip = { workspace = true }
 aws-config = { workspace = true, optional = true }
 aws-sdk-s3 = { workspace = true, optional = true }
@@ -287,8 +287,6 @@ pub fn is_valid_discord_username(username: &str) -> bool {
        && username
            .chars()
            .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_' || c == '.')
-        && !username.starts_with('.')
-        && !username.ends_with('.')
        && !username.contains("..")
 }

@@ -493,8 +491,6 @@ mod tests {
        assert!(!is_valid_discord_username("Alice"));
        assert!(!is_valid_discord_username("ALICE"));
        assert!(!is_valid_discord_username("user-name"));
-        assert!(!is_valid_discord_username(".username"));
-        assert!(!is_valid_discord_username("username."));
        assert!(!is_valid_discord_username("user..name"));
        assert!(!is_valid_discord_username("user name"));
        assert!(!is_valid_discord_username(
@@ -156,6 +156,7 @@ pub enum Legacy2faOutcome {
 }

 pub struct Legacy2faContext {
+    pub is_app_password: bool,
    pub email_2fa_enabled: bool,
    pub has_totp: bool,
    pub allow_legacy_login: bool,
@@ -163,7 +164,7 @@ pub struct Legacy2faContext {

 impl Legacy2faContext {
    pub fn requires_2fa(&self) -> bool {
-        self.email_2fa_enabled || self.has_totp
+        !self.is_app_password && (self.email_2fa_enabled || self.has_totp)
    }

    pub fn is_blocked(&self) -> bool {
@@ -418,6 +419,7 @@ mod tests {
        let cache = MockCache::new();
        let did = Did::new("did:plc:test".to_string()).unwrap();
        let ctx = Legacy2faContext {
+            is_app_password: false,
            email_2fa_enabled: false,
            has_totp: false,
            allow_legacy_login: true,
@@ -427,11 +429,27 @@ mod tests {
        assert!(matches!(outcome, Legacy2faOutcome::NotRequired));
    }

+    #[tokio::test]
+    async fn test_process_flow_not_required_because_app_password() {
+        let cache = MockCache::new();
+        let did = Did::new("did:plc:test".to_string()).unwrap();
+        let ctx = Legacy2faContext {
+            is_app_password: true,
+            email_2fa_enabled: false,
+            has_totp: true,
+            allow_legacy_login: true,
+        };
+
+        let outcome = process_legacy_2fa(&cache, &did, &ctx, None).await.unwrap();
+        assert!(matches!(outcome, Legacy2faOutcome::NotRequired));
+    }
+
    #[tokio::test]
    async fn test_process_flow_blocked() {
        let cache = MockCache::new();
        let did = Did::new("did:plc:test".to_string()).unwrap();
        let ctx = Legacy2faContext {
+            is_app_password: false,
            email_2fa_enabled: false,
            has_totp: true,
            allow_legacy_login: false,
@@ -446,6 +464,7 @@ mod tests {
        let cache = MockCache::new();
        let did = Did::new("did:plc:test".to_string()).unwrap();
        let ctx = Legacy2faContext {
+            is_app_password: false,
            email_2fa_enabled: false,
            has_totp: true,
            allow_legacy_login: true,
@@ -460,6 +479,7 @@ mod tests {
        let cache = MockCache::new();
        let did = Did::new("did:plc:test2".to_string()).unwrap();
        let ctx = Legacy2faContext {
+            is_app_password: false,
            email_2fa_enabled: true,
            has_totp: false,
            allow_legacy_login: false,
@@ -474,6 +494,7 @@ mod tests {
        let cache = MockCache::new();
        let did = Did::new("did:plc:test".to_string()).unwrap();
        let ctx = Legacy2faContext {
+            is_app_password: false,
            email_2fa_enabled: true,
            has_totp: false,
            allow_legacy_login: false,
@@ -1,5 +1,8 @@
 use uuid::Uuid;
 use webauthn_rs::prelude::*;
+use webauthn_rs_proto::{
+    AuthenticatorSelectionCriteria, ResidentKeyRequirement, UserVerificationPolicy,
+};

 #[derive(Debug, thiserror::Error)]
 pub enum WebauthnError {
@@ -57,6 +60,15 @@ impl WebAuthnConfig {
                None,
                None,
            )
+            .map(|(mut ccr, state)| {
+                let sel = ccr
+                    .public_key
+                    .authenticator_selection
+                    .get_or_insert_with(AuthenticatorSelectionCriteria::default);
+                sel.resident_key = Some(ResidentKeyRequirement::Required);
+                sel.require_resident_key = true;
+                (ccr, state)
+            })
            .map_err(|e| WebauthnError::RegistrationFailed(e.to_string()))
    }

@@ -88,4 +100,49 @@ impl WebAuthnConfig {
            .finish_securitykey_authentication(auth, state)
            .map_err(|e| WebauthnError::AuthenticationFailed(e.to_string()))
    }
+
+    pub fn start_discoverable_authentication(
+        &self,
+    ) -> Result<(RequestChallengeResponse, DiscoverableAuthentication), WebauthnError> {
+        let (mut rcr, state) = self
+            .webauthn
+            .start_discoverable_authentication()
+            .map_err(|e| WebauthnError::AuthenticationFailed(e.to_string()))?;
+
+        rcr.mediation = None;
+        rcr.public_key.user_verification = UserVerificationPolicy::Discouraged_DO_NOT_USE;
+
+        let mut state_json = serde_json::to_value(&state)
+            .map_err(|e| WebauthnError::AuthenticationFailed(e.to_string()))?;
+        let ast = state_json
+            .get_mut("ast")
+            .ok_or_else(|| WebauthnError::AuthenticationFailed(
+                "webauthn-rs DiscoverableAuthentication missing 'ast' field, library version incompatible".into(),
+            ))?;
+        ast["policy"] = serde_json::json!("discouraged");
+        let patched: DiscoverableAuthentication = serde_json::from_value(state_json)
+            .map_err(|e| WebauthnError::AuthenticationFailed(e.to_string()))?;
+
+        Ok((rcr, patched))
+    }
+
+    pub fn identify_discoverable_authentication<'a>(
+        &self,
+        credential: &'a PublicKeyCredential,
+    ) -> Result<(Uuid, &'a [u8]), WebauthnError> {
+        self.webauthn
+            .identify_discoverable_authentication(credential)
+            .map_err(|e| WebauthnError::AuthenticationFailed(e.to_string()))
+    }
+
+    pub fn finish_discoverable_authentication(
+        &self,
+        credential: &PublicKeyCredential,
+        state: DiscoverableAuthentication,
+        creds: &[DiscoverableKey],
+    ) -> Result<AuthenticationResult, WebauthnError> {
+        self.webauthn
+            .finish_discoverable_authentication(credential, state, creds)
+            .map_err(|e| WebauthnError::AuthenticationFailed(e.to_string()))
+    }
 }
@@ -3,7 +3,7 @@ use std::sync::Arc;
 use std::time::Duration;

 use chrono::Utc;
-use tokio::time::interval;
+
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, info, warn};
 use tranquil_comms::{
@@ -75,17 +75,28 @@ impl CommsService {
            );
        }
        info!(
-            poll_interval_secs = self.poll_interval.as_secs(),
+            poll_interval_ms = self.poll_interval.as_millis() as u64,
            batch_size = self.batch_size,
            channels = ?self.senders.keys().collect::<Vec<_>>(),
            "Starting comms service"
        );
-        let mut ticker = interval(self.poll_interval);
+        let base = self.poll_interval;
+        let max_backoff = Duration::from_secs(30);
+        let mut current_delay = base;
        loop {
            tokio::select! {
-                _ = ticker.tick() => {
-                    if let Err(e) = self.process_batch().await {
-                        error!(error = %e, "Failed to process comms batch");
+                _ = tokio::time::sleep(current_delay) => {
+                    match self.process_batch().await {
+                        Ok(had_work) => {
+                            current_delay = match had_work {
+                                true => base,
+                                false => max_backoff.min(current_delay.saturating_mul(2)),
+                            };
+                        }
+                        Err(e) => {
+                            error!(error = %e, "Failed to process comms batch");
+                            current_delay = max_backoff.min(current_delay.saturating_mul(2));
+                        }
                    }
                }
                _ = shutdown.cancelled() => {
@@ -96,14 +107,14 @@ impl CommsService {
        }
    }

-    async fn process_batch(&self) -> Result<(), tranquil_db_traits::DbError> {
+    async fn process_batch(&self) -> Result<bool, tranquil_db_traits::DbError> {
        let items = self.fetch_pending().await?;
        if items.is_empty() {
-            return Ok(());
+            return Ok(false);
        }
        debug!(count = items.len(), "Processing comms batch");
        futures::future::join_all(items.into_iter().map(|item| self.process_item(item))).await;
-        Ok(())
+        Ok(true)
    }

    async fn fetch_pending(&self) -> Result<Vec<QueuedComms>, tranquil_db_traits::DbError> {
@@ -6,18 +6,19 @@ use crate::types::{Did, Handle, Nsid, Rkey};
 use backon::{ExponentialBuilder, Retryable};
 use bytes::Bytes;
 use cid::Cid;
+use jacquard_common::smol_str::SmolStr;
 use jacquard_common::types::{integer::LimitedU32, string::Tid};
 use jacquard_repo::commit::Commit;
-use jacquard_repo::mst::Mst;
 use jacquard_repo::mst::util::compute_cid;
+use jacquard_repo::mst::{Mst, VerifiedWriteOp};
 use jacquard_repo::storage::BlockStore;
 use k256::ecdsa::SigningKey;
 use serde_json::{Value, json};
-use std::collections::BTreeSet;
+use std::collections::{BTreeMap, HashSet};
 use std::str::FromStr;
 use std::sync::Arc;
 use tokio::sync::OwnedMutexGuard;
-use tracing::error;
+use tracing::{error, warn};
 use tranquil_db_traits::SequenceNumber;
 use uuid::Uuid;

@@ -226,30 +227,6 @@ pub async fn begin_repo_write(
    Ok((ctx, mst))
 }

-pub async fn compute_obsolete_cids(
-    original_mst: &Mst<TrackingBlockStore>,
-    new_mst: &Mst<TrackingBlockStore>,
-    original_root_cid: CommitCid,
-) -> Result<Vec<Cid>, jacquard_repo::error::RepoError> {
-    let (old_nodes, new_nodes, old_leaves, new_leaves) = tokio::try_join!(
-        original_mst.collect_node_cids(),
-        new_mst.collect_node_cids(),
-        original_mst.leaves(),
-        new_mst.leaves(),
-    )?;
-    let old_nodes_set: BTreeSet<Cid> = old_nodes.into_iter().collect();
-    let new_nodes_set: BTreeSet<Cid> = new_nodes.into_iter().collect();
-    let old_leaf_set: BTreeSet<Cid> = old_leaves.iter().map(|(_, cid)| *cid).collect();
-    let new_leaf_set: BTreeSet<Cid> = new_leaves.iter().map(|(_, cid)| *cid).collect();
-    let removed_nodes = old_nodes_set.difference(&new_nodes_set).copied();
-    let removed_leaves = old_leaf_set.difference(&new_leaf_set).copied();
-    let obsolete: BTreeSet<Cid> = std::iter::once(original_root_cid.into_cid())
-        .chain(removed_nodes)
-        .chain(removed_leaves)
-        .collect();
-    Ok(obsolete.into_iter().collect())
-}
-
 pub async fn finalize_repo_write(
    state: &AppState,
    ctx: RepoWriteContext,
@@ -261,40 +238,111 @@ pub async fn finalize_repo_write(
        ApiError::InternalError(None)
    })?;

-    let block_bytes = ctx.tracking_store.take_written_blocks();
+    let written_bytes = ctx.tracking_store.take_written_blocks();
+    let new_tree_cids: Vec<Cid> = written_bytes.keys().copied().collect();

-    let storage_for_diff = Arc::new(ctx.tracking_store.clone());
-    let original_settled = Mst::load(storage_for_diff.clone(), ctx.prev_data_cid, None);
-    let new_settled = Mst::load(storage_for_diff, new_mst_root, None);
-    let (obsolete_cids, new_tree_cids) = tokio::try_join!(
-        async {
-            compute_obsolete_cids(
-                &original_settled,
-                &new_settled,
-                CommitCid::from(ctx.current_root_cid),
-            )
+    let storage_for_proof = Arc::new(ctx.tracking_store.clone());
+    let original_settled = Mst::load(storage_for_proof.clone(), ctx.prev_data_cid, None);
+    let new_settled = Mst::load(storage_for_proof.clone(), new_mst_root, None);
+
+    let mut inverse_trace = new_settled.clone();
+    let mut non_invertible: Vec<String> = Vec::new();
+    let mut invert_errors: Vec<String> = Vec::new();
+    for op in params.ops.iter() {
+        let (collection, rkey) = match op {
+            RecordOp::Create {
+                collection, rkey, ..
+            }
+            | RecordOp::Update {
+                collection, rkey, ..
+            }
+            | RecordOp::Delete {
+                collection, rkey, ..
+            } => (collection, rkey),
+        };
+        let key = SmolStr::new(format!("{}/{}", collection, rkey));
+        let verified = match op {
+            RecordOp::Create { cid, .. } => VerifiedWriteOp::Create {
+                key,
+                cid: *cid.as_cid(),
+            },
+            RecordOp::Update { cid, prev, .. } => VerifiedWriteOp::Update {
+                key,
+                cid: *cid.as_cid(),
+                prev: *prev.as_cid(),
+            },
+            RecordOp::Delete { prev, .. } => VerifiedWriteOp::Delete {
+                key,
+                prev: *prev.as_cid(),
+            },
+        };
+        match inverse_trace.invert_op(verified.clone()).await {
+            Ok(true) => {}
+            Ok(false) => non_invertible.push(format!("{:?}", verified)),
+            Err(e) => invert_errors.push(format!("{:?} -> {:?}", verified, e)),
+        }
+    }
+    if !non_invertible.is_empty() {
+        warn!(
+            user_id = %params.user_id,
+            count = non_invertible.len(),
+            ops = ?non_invertible,
+            "firehose proof walk: ops not invertible on new MST, consumer will reject frame"
+        );
+    }
+    if !invert_errors.is_empty() {
+        warn!(
+            user_id = %params.user_id,
+            count = invert_errors.len(),
+            failures = ?invert_errors,
+            "firehose proof walk: invert_op errored, cover blocks may be incomplete"
+        );
+    }
+
+    let read_cid_set: HashSet<Cid> = ctx.tracking_store.get_read_cids().into_iter().collect();
+    let missing_read_cids: Vec<Cid> = read_cid_set
+        .iter()
+        .copied()
+        .filter(|cid| !written_bytes.contains_key(cid))
+        .collect();
+    let mut relevant: BTreeMap<Cid, Bytes> = BTreeMap::new();
+    if !missing_read_cids.is_empty() {
+        let fetched = ctx
+            .tracking_store
+            .get_many(&missing_read_cids)
            .await
            .map_err(|e| {
-                error!("MST diff failed during finalize_repo_write: {}", e);
-                ApiError::InternalError(Some("MST diff failed".into()))
-            })
-        },
-        async {
-            let (nodes, leaves) =
-                tokio::try_join!(new_settled.collect_node_cids(), new_settled.leaves(),).map_err(
-                    |e| {
-                        error!("new tree walk failed: {}", e);
-                        ApiError::InternalError(None)
-                    },
-                )?;
-            Ok::<Vec<Cid>, ApiError>(
-                nodes
-                    .into_iter()
-                    .chain(leaves.iter().map(|(_, cid)| *cid))
-                    .collect(),
-            )
-        },
-    )?;
+                error!("fetch cover read bytes: {e}");
+                ApiError::InternalError(None)
+            })?;
+        for (cid, maybe) in missing_read_cids.into_iter().zip(fetched) {
+            if let Some(bytes) = maybe {
+                relevant.insert(cid, bytes);
+            }
+        }
+    }
+
+    let obsolete_cids = match original_settled.diff(&new_settled).await {
+        Ok(diff) => {
+            let mut obsolete: Vec<Cid> =
+                Vec::with_capacity(1 + diff.removed_mst_blocks.len() + diff.removed_cids.len());
+            obsolete.push(ctx.current_root_cid);
+            obsolete.extend(diff.removed_mst_blocks);
+            obsolete.extend(diff.removed_cids);
+            obsolete
+        }
+        Err(e) => {
+            error!(
+                "MST diff failed during finalize_repo_write: {e}. \
+                 Proceeding with commit CID only; leaked blocks \
+                 will be reclaimed by reachability GC."
+            );
+            vec![ctx.current_root_cid]
+        }
+    };
+
+    let mut block_bytes = written_bytes;
+    block_bytes.extend(relevant);

    let result = commit_and_log(
        state,
@@ -667,6 +667,8 @@ async fn delete_account_data(
    Ok(())
 }

+const CAR_BLOCK_BATCH_SIZE: usize = 500;
+
 pub async fn generate_repo_car(
    block_store: &AnyBlockStore,
    head_cid: &Cid,
@@ -683,21 +685,20 @@ pub async fn generate_repo_car(
        })
        .collect();

-    let car_bytes = encode_car_header(head_cid).context("Failed to encode CAR header")?;
+    let mut car_bytes = encode_car_header(head_cid).context("Failed to encode CAR header")?;

-    let blocks = block_store
-        .get_many(&block_cids)
-        .await
-        .context("Failed to fetch blocks")?;
+    for chunk in block_cids.chunks(CAR_BLOCK_BATCH_SIZE) {
+        let blocks = block_store
+            .get_many(chunk)
+            .await
+            .context("Failed to fetch blocks")?;

-    let car_bytes = block_cids
-        .iter()
-        .zip(blocks.iter())
-        .filter_map(|(cid, block_opt)| block_opt.as_ref().map(|block| (cid, block)))
-        .fold(car_bytes, |mut acc, (cid, block)| {
-            acc.extend(encode_car_block(cid, block));
-            acc
-        });
+        chunk
+            .iter()
+            .zip(blocks.iter())
+            .filter_map(|(cid, block_opt)| block_opt.as_ref().map(|block| (cid, block)))
+            .for_each(|(cid, block)| car_bytes.extend(encode_car_block(cid, block)));
+    }

    Ok(car_bytes)
 }
@@ -50,6 +50,7 @@ pub struct AppState {
    pub signal_sender: Option<Arc<tranquil_signal::SignalSlot>>,
    pub signal_store_provider: Option<Arc<dyn tranquil_signal::SignalStoreProvider>>,
    pub eventlog_segments_dir: Option<PathBuf>,
+    pub repo_export_semaphore: Arc<tokio::sync::Semaphore>,
 }

 #[derive(Debug, Clone, Copy)]
@@ -394,6 +395,9 @@ impl AppState {
            signal_sender: None,
            signal_store_provider,
            eventlog_segments_dir,
+            repo_export_semaphore: Arc::new(tokio::sync::Semaphore::new(
+                cfg.firehose.max_concurrent_repo_exports,
+            )),
        }
    }

@@ -99,9 +99,8 @@ async fn test_check_account_status_returns_correct_block_count() {
        after_delete_blocks
    );
    assert!(
-        after_delete_blocks >= initial_blocks,
-        "Block count after delete should be at least initial count (initial {}, now {})",
-        initial_blocks,
+        after_delete_blocks >= 2,
+        "Block count after delete should have at least commit + MST root (got {})",
        after_delete_blocks
    );
 }
@@ -0,0 +1,169 @@
+mod common;
+
+use chrono::Utc;
+use common::*;
+use reqwest::StatusCode;
+use serde_json::{Value, json};
+
+fn run_compaction(store: &tranquil_store::blockstore::TranquilBlockStore) {
+    let liveness = store.compaction_liveness(0).unwrap();
+    liveness
+        .iter()
+        .filter(|(_, info)| info.total_blocks > 0 && info.ratio() < 0.95)
+        .map(|(&fid, _)| fid)
+        .collect::<Vec<_>>()
+        .into_iter()
+        .for_each(|fid| match store.compact_file(fid, 0) {
+            Ok(_) => {}
+            Err(tranquil_store::blockstore::CompactionError::ActiveFileCannotBeCompacted) => {}
+            Err(e) => eprintln!("compaction: {e}"),
+        });
+}
+
+#[tokio::test]
+async fn mst_blocks_survive_full_store_reopen() {
+    if !is_store_backend() {
+        eprintln!("skipping: only meaningful with tranquil-store backend");
+        return;
+    }
+
+    let client = client();
+    let base = base_url().await;
+    let block_store = get_test_block_store().await;
+
+    let store = block_store
+        .as_tranquil_store()
+        .expect("expected tranquil-store backend");
+
+    let (jwt, did) = create_account_and_login(&client).await;
+
+    let mut posts = Vec::new();
+    for i in 0..30 {
+        let res = client
+            .post(format!("{base}/xrpc/com.atproto.repo.createRecord"))
+            .bearer_auth(&jwt)
+            .json(&json!({
+                "repo": did,
+                "collection": "app.bsky.feed.post",
+                "record": {
+                    "$type": "app.bsky.feed.post",
+                    "text": format!("compaction test post {i}"),
+                    "createdAt": Utc::now().to_rfc3339()
+                }
+            }))
+            .send()
+            .await
+            .unwrap();
+        assert_eq!(res.status(), StatusCode::OK);
+        let body: Value = res.json().await.unwrap();
+        posts.push((
+            body["uri"].as_str().unwrap().to_string(),
+            body["cid"].as_str().unwrap().to_string(),
+        ));
+    }
+
+    for (uri, cid) in &posts[..20] {
+        let res = client
+            .post(format!("{base}/xrpc/com.atproto.repo.createRecord"))
+            .bearer_auth(&jwt)
+            .json(&json!({
+                "repo": did,
+                "collection": "app.bsky.feed.like",
+                "record": {
+                    "$type": "app.bsky.feed.like",
+                    "subject": { "uri": uri, "cid": cid },
+                    "createdAt": Utc::now().to_rfc3339()
+                }
+            }))
+            .send()
+            .await
+            .unwrap();
+        assert_eq!(res.status(), StatusCode::OK, "like failed for {uri}");
+    }
+
+    let data_dir = store.data_dir().to_path_buf();
+    let index_dir = data_dir.parent().unwrap().join("index");
+
+    let store_clone = store.clone();
+    tokio::task::spawn_blocking(move || {
+        (0..40).for_each(|_| run_compaction(&store_clone));
+    })
+    .await
+    .unwrap();
+
+    let repo_root_str: String = get_test_repos()
+        .await
+        .repo
+        .get_repo_root_by_did(&tranquil_types::Did::new(did.clone()).unwrap())
+        .await
+        .expect("db error")
+        .expect("no repo root")
+        .to_string();
+
+    let head_cid = cid::Cid::try_from(repo_root_str.as_str()).expect("invalid cid");
+
+    let car_blocks = tranquil_pds::scheduled::collect_current_repo_blocks(block_store, &head_cid)
+        .await
+        .expect("collect blocks");
+
+    let block_count_before = car_blocks.len();
+
+    let max_file_size = store
+        .list_data_files()
+        .ok()
+        .map(|_| 4 * 1024 * 1024u64)
+        .unwrap_or(4 * 1024 * 1024);
+
+    let reopened_missing = tokio::task::spawn_blocking(move || {
+        let rt = tokio::runtime::Handle::current();
+        let _guard = rt.enter();
+
+        let config = tranquil_store::blockstore::BlockStoreConfig {
+            data_dir: data_dir.clone(),
+            index_dir,
+            max_file_size,
+            group_commit: tranquil_store::blockstore::GroupCommitConfig::default(),
+            shard_count: 1,
+        };
+        let fresh =
+            tranquil_store::blockstore::TranquilBlockStore::open(config).expect("reopen failed");
+
+        let missing: Vec<String> = car_blocks
+            .iter()
+            .filter_map(|cid_bytes| {
+                if cid_bytes.len() < 36 {
+                    return None;
+                }
+                let mut arr = [0u8; 36];
+                arr.copy_from_slice(&cid_bytes[..36]);
+                match fresh.get_block_sync(&arr) {
+                    Ok(Some(_)) => None,
+                    Ok(None) => Some(format!(
+                        "missing {}",
+                        cid::Cid::try_from(cid_bytes.as_slice())
+                            .map(|c| c.to_string())
+                            .unwrap_or_else(|_| hex::encode(cid_bytes))
+                    )),
+                    Err(e) => Some(format!("error: {e}")),
+                }
+            })
+            .collect();
+
+        drop(fresh);
+        missing
+    })
+    .await
+    .unwrap();
+
+    assert!(
+        reopened_missing.is_empty(),
+        "{} of {block_count_before} blocks missing after blockstore reopen:\n{}",
+        reopened_missing.len(),
+        reopened_missing
+            .iter()
+            .take(20)
+            .map(|s| s.as_str())
+            .collect::<Vec<_>>()
+            .join("\n"),
+    );
+}
@@ -597,3 +597,157 @@ async fn test_request_account_delete() {
        "Token should not be expired"
    );
 }
+
+async fn create_app_password_session(
+    client: &reqwest::Client,
+    did: &str,
+    main_jwt: &str,
+    name: &str,
+    body: Value,
+) -> (String, Value) {
+    let base = base_url().await;
+    let create_res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.server.createAppPassword",
+            base
+        ))
+        .bearer_auth(main_jwt)
+        .json(&body)
+        .send()
+        .await
+        .expect("Failed to create app password");
+    assert_eq!(create_res.status(), StatusCode::OK);
+    let app_pass: Value = create_res.json().await.unwrap();
+    let password = app_pass["password"].as_str().unwrap().to_string();
+    let scopes_response = app_pass.clone();
+    let login_res = client
+        .post(format!("{}/xrpc/com.atproto.server.createSession", base))
+        .json(&json!({ "identifier": did, "password": password }))
+        .send()
+        .await
+        .expect("Failed to login with app password");
+    assert_eq!(
+        login_res.status(),
+        StatusCode::OK,
+        "App password login for '{}' failed",
+        name
+    );
+    let session: Value = login_res.json().await.unwrap();
+    let jwt = session["accessJwt"].as_str().unwrap().to_string();
+    (jwt, scopes_response)
+}
+
+async fn try_chat_service_auth(client: &reqwest::Client, jwt: &str) -> StatusCode {
+    let base = base_url().await;
+    let res = client
+        .get(format!("{}/xrpc/com.atproto.server.getServiceAuth", base))
+        .bearer_auth(jwt)
+        .query(&[
+            ("aud", "did:web:api.bsky.app"),
+            ("lxm", "chat.bsky.convo.listConvos"),
+        ])
+        .send()
+        .await
+        .expect("Failed to call getServiceAuth");
+    res.status()
+}
+
+#[tokio::test]
+async fn test_app_password_non_privileged_blocks_chat() {
+    let client = client();
+    let (did, jwt) = setup_new_user("appscope-nonchat").await;
+    let (app_jwt, create_body) = create_app_password_session(
+        &client,
+        &did,
+        &jwt,
+        "non-privileged",
+        json!({ "name": "NoChatApp", "privileged": false }),
+    )
+    .await;
+    assert_eq!(
+        create_body["scopes"].as_str().unwrap(),
+        "transition:generic",
+        "Non-privileged app password should not have chat scope"
+    );
+    let status = try_chat_service_auth(&client, &app_jwt).await;
+    assert_eq!(
+        status,
+        StatusCode::FORBIDDEN,
+        "Non-privileged app password must not access chat methods"
+    );
+}
+
+#[tokio::test]
+async fn test_app_password_privileged_allows_chat() {
+    let client = client();
+    let (did, jwt) = setup_new_user("appscope-chat").await;
+    let (app_jwt, create_body) = create_app_password_session(
+        &client,
+        &did,
+        &jwt,
+        "privileged",
+        json!({ "name": "ChatApp", "privileged": true }),
+    )
+    .await;
+    assert_eq!(
+        create_body["scopes"].as_str().unwrap(),
+        "transition:generic transition:chat.bsky",
+        "Privileged app password should have chat scope"
+    );
+    let status = try_chat_service_auth(&client, &app_jwt).await;
+    assert_eq!(
+        status,
+        StatusCode::OK,
+        "Privileged app password should access chat methods"
+    );
+}
+
+#[tokio::test]
+async fn test_app_password_no_privileged_field_allows_chat() {
+    let client = client();
+    let (did, jwt) = setup_new_user("appscope-full").await;
+    let (app_jwt, create_body) = create_app_password_session(
+        &client,
+        &did,
+        &jwt,
+        "full-access",
+        json!({ "name": "FullApp" }),
+    )
+    .await;
+    assert_eq!(
+        create_body["scopes"].as_str().unwrap(),
+        "transition:generic transition:chat.bsky",
+        "App password without privileged field should default to full access"
+    );
+    let status = try_chat_service_auth(&client, &app_jwt).await;
+    assert_eq!(
+        status,
+        StatusCode::OK,
+        "Full-access app password should access chat methods"
+    );
+}
+
+#[tokio::test]
+async fn test_app_password_explicit_scopes_respected() {
+    let client = client();
+    let (did, jwt) = setup_new_user("appscope-explicit").await;
+    let (app_jwt, create_body) = create_app_password_session(
+        &client,
+        &did,
+        &jwt,
+        "explicit-scopes",
+        json!({ "name": "ScopedApp", "scopes": "transition:generic" }),
+    )
+    .await;
+    assert_eq!(
+        create_body["scopes"].as_str().unwrap(),
+        "transition:generic",
+        "Explicit scopes should be stored as-is"
+    );
+    let status = try_chat_service_auth(&client, &app_jwt).await;
+    assert_eq!(
+        status,
+        StatusCode::FORBIDDEN,
+        "App password with only transition:generic should not access chat"
+    );
+}
@@ -0,0 +1,583 @@
+use std::collections::BTreeSet;
+use std::sync::Arc;
+
+use cid::Cid;
+use jacquard_repo::mst::Mst;
+use jacquard_repo::storage::MemoryBlockStore;
+
+fn test_cid(n: u32) -> Cid {
+    let data = n.to_be_bytes();
+    let mut buf = [0u8; 32];
+    buf[..4].copy_from_slice(&data);
+    buf[4] = (n >> 8) as u8 ^ 0xAB;
+    buf[5] = (n & 0xFF) as u8 ^ 0xCD;
+    let mh = multihash::Multihash::wrap(0x12, &buf).unwrap();
+    Cid::new_v1(0x71, mh)
+}
+
+async fn compute_obsolete_full_walk<
+    S: jacquard_repo::storage::BlockStore + Sync + Send + 'static,
+>(
+    old: &Mst<S>,
+    new: &Mst<S>,
+) -> BTreeSet<Cid> {
+    let old_nodes = old.collect_node_cids().await.unwrap();
+    let new_nodes = new.collect_node_cids().await.unwrap();
+    let old_leaves = old.leaves().await.unwrap();
+    let new_leaves = new.leaves().await.unwrap();
+    let old_nodes_set: BTreeSet<Cid> = old_nodes.into_iter().collect();
+    let new_nodes_set: BTreeSet<Cid> = new_nodes.into_iter().collect();
+    let old_leaf_set: BTreeSet<Cid> = old_leaves.iter().map(|(_, cid)| *cid).collect();
+    let new_leaf_set: BTreeSet<Cid> = new_leaves.iter().map(|(_, cid)| *cid).collect();
+    old_nodes_set
+        .difference(&new_nodes_set)
+        .copied()
+        .chain(old_leaf_set.difference(&new_leaf_set).copied())
+        .collect()
+}
+
+fn compute_obsolete_from_diff(diff: &jacquard_repo::mst::diff::MstDiff) -> BTreeSet<Cid> {
+    diff.removed_mst_blocks
+        .iter()
+        .copied()
+        .chain(diff.removed_cids.iter().copied())
+        .collect()
+}
+
+async fn assert_equivalence(
+    old_records: &[(String, u32)],
+    new_records: &[(String, u32)],
+    scenario: &str,
+) {
+    let storage = Arc::new(MemoryBlockStore::new());
+
+    let mut old_tree = Mst::new(storage.clone());
+    for (key, val) in old_records {
+        old_tree = old_tree.add(key, test_cid(*val)).await.unwrap();
+    }
+    let old_root = old_tree.persist().await.unwrap();
+
+    let mut new_tree = Mst::new(storage.clone());
+    for (key, val) in new_records {
+        new_tree = new_tree.add(key, test_cid(*val)).await.unwrap();
+    }
+    let new_root = new_tree.persist().await.unwrap();
+
+    let old_settled = Mst::load(storage.clone(), old_root, None);
+    let new_settled = Mst::load(storage.clone(), new_root, None);
+
+    let full_walk_obsolete = compute_obsolete_full_walk(&old_settled, &new_settled).await;
+
+    let old_for_diff = Mst::load(storage.clone(), old_root, None);
+    let new_for_diff = Mst::load(storage, new_root, None);
+    let diff = old_for_diff.diff(&new_for_diff).await.unwrap();
+    let diff_obsolete = compute_obsolete_from_diff(&diff);
+
+    assert_eq!(
+        full_walk_obsolete,
+        diff_obsolete,
+        "MISMATCH in scenario: {scenario}\n  full_walk count: {}\n  diff count: {}\n  in full_walk but not diff: {:?}\n  in diff but not full_walk: {:?}",
+        full_walk_obsolete.len(),
+        diff_obsolete.len(),
+        full_walk_obsolete
+            .difference(&diff_obsolete)
+            .collect::<Vec<_>>(),
+        diff_obsolete
+            .difference(&full_walk_obsolete)
+            .collect::<Vec<_>>(),
+    );
+}
+
+fn make_key(collection: &str, i: u32) -> String {
+    format!("{collection}/{i:06}")
+}
+
+fn generate_records(collection: &str, range: std::ops::Range<u32>) -> Vec<(String, u32)> {
+    range.map(|i| (make_key(collection, i), i)).collect()
+}
+
+fn generate_multi_collection_records(
+    collections: &[&str],
+    per_collection: u32,
+) -> Vec<(String, u32)> {
+    collections
+        .iter()
+        .enumerate()
+        .flat_map(|(ci, coll)| {
+            let base = ci as u32 * per_collection;
+            (0..per_collection).map(move |i| (make_key(coll, i), base + i))
+        })
+        .collect()
+}
+
+fn apply_scattered_updates(
+    records: &[(String, u32)],
+    stride: usize,
+    cid_offset: u32,
+) -> Vec<(String, u32)> {
+    records
+        .iter()
+        .enumerate()
+        .map(|(idx, (key, val))| {
+            if idx % stride == 0 {
+                (key.clone(), val + cid_offset)
+            } else {
+                (key.clone(), *val)
+            }
+        })
+        .collect()
+}
+
+fn remove_every_nth(records: &[(String, u32)], n: usize) -> Vec<(String, u32)> {
+    records
+        .iter()
+        .enumerate()
+        .filter(|(idx, _)| idx % n != 0)
+        .map(|(_, r)| r.clone())
+        .collect()
+}
+
+fn remove_range(records: &[(String, u32)], start: usize, count: usize) -> Vec<(String, u32)> {
+    records
+        .iter()
+        .enumerate()
+        .filter(|(idx, _)| *idx < start || *idx >= start + count)
+        .map(|(_, r)| r.clone())
+        .collect()
+}
+
+fn keep_only_collection(records: &[(String, u32)], collection: &str) -> Vec<(String, u32)> {
+    records
+        .iter()
+        .filter(|(key, _)| key.starts_with(collection))
+        .cloned()
+        .collect()
+}
+
+fn append_records(
+    base: &[(String, u32)],
+    collection: &str,
+    range: std::ops::Range<u32>,
+    cid_base: u32,
+) -> Vec<(String, u32)> {
+    let mut result = base.to_vec();
+    result.extend(range.map(|i| (make_key(collection, i), cid_base + i)));
+    result.sort_by(|(a, _), (b, _)| a.cmp(b));
+    result
+}
+
+#[tokio::test]
+async fn massive_tree_single_create() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let new_rec = append_records(&old, "app.bsky.feed.post", 2000..2001, 2000);
+    assert_equivalence(&old, &new_rec, "2000 records + 1 create").await;
+}
+
+#[tokio::test]
+async fn massive_tree_single_delete() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let new_rec = remove_range(&old, 1000, 1);
+    assert_equivalence(&old, &new_rec, "2000 records - 1 delete from middle").await;
+}
+
+#[tokio::test]
+async fn massive_tree_single_update() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let new_rec: Vec<_> = old
+        .iter()
+        .map(|(k, v)| {
+            if k == "app.bsky.feed.post/001000" {
+                (k.clone(), v + 50000)
+            } else {
+                (k.clone(), *v)
+            }
+        })
+        .collect();
+    assert_equivalence(&old, &new_rec, "2000 records - 1 update in middle").await;
+}
+
+#[tokio::test]
+async fn massive_tree_scattered_updates_every_3rd() {
+    let old = generate_records("app.bsky.feed.post", 0..1500);
+    let new_rec = apply_scattered_updates(&old, 3, 10000);
+    assert_equivalence(&old, &new_rec, "1500 records - update every 3rd").await;
+}
+
+#[tokio::test]
+async fn massive_tree_scattered_updates_every_7th() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let new_rec = apply_scattered_updates(&old, 7, 20000);
+    assert_equivalence(&old, &new_rec, "2000 records - update every 7th").await;
+}
+
+#[tokio::test]
+async fn massive_tree_delete_every_2nd() {
+    let old = generate_records("app.bsky.feed.post", 0..1000);
+    let new_rec = remove_every_nth(&old, 2);
+    assert_equivalence(&old, &new_rec, "1000 records - delete every 2nd").await;
+}
+
+#[tokio::test]
+async fn massive_tree_delete_every_5th() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let new_rec = remove_every_nth(&old, 5);
+    assert_equivalence(&old, &new_rec, "2000 records - delete every 5th").await;
+}
+
+#[tokio::test]
+async fn massive_tree_delete_first_half() {
+    let old = generate_records("app.bsky.feed.post", 0..1500);
+    let new_rec = remove_range(&old, 0, 750);
+    assert_equivalence(&old, &new_rec, "1500 records - delete first 750").await;
+}
+
+#[tokio::test]
+async fn massive_tree_delete_last_half() {
+    let old = generate_records("app.bsky.feed.post", 0..1500);
+    let new_rec = remove_range(&old, 750, 750);
+    assert_equivalence(&old, &new_rec, "1500 records - delete last 750").await;
+}
+
+#[tokio::test]
+async fn massive_tree_delete_middle_chunk() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let new_rec = remove_range(&old, 800, 400);
+    assert_equivalence(&old, &new_rec, "2000 records - delete 400 from middle").await;
+}
+
+#[tokio::test]
+async fn empty_to_massive() {
+    let new_rec = generate_records("app.bsky.feed.post", 0..1500);
+    assert_equivalence(&[], &new_rec, "empty to 1500 records").await;
+}
+
+#[tokio::test]
+async fn massive_to_empty() {
+    let old = generate_records("app.bsky.feed.post", 0..1500);
+    assert_equivalence(&old, &[], "1500 records to empty").await;
+}
+
+#[tokio::test]
+async fn massive_complete_replacement() {
+    let old = generate_records("app.bsky.feed.post", 0..1000);
+    let new_rec = generate_records("app.bsky.feed.post", 1000..2000);
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "1000 records fully replaced with 1000 different",
+    )
+    .await;
+}
+
+#[tokio::test]
+async fn massive_no_change() {
+    let records = generate_records("app.bsky.feed.post", 0..1500);
+    assert_equivalence(&records, &records, "1500 records unchanged").await;
+}
+
+#[tokio::test]
+async fn multi_collection_5_collections_500_each() {
+    let collections = [
+        "app.bsky.feed.like",
+        "app.bsky.feed.post",
+        "app.bsky.feed.repost",
+        "app.bsky.graph.follow",
+        "app.bsky.graph.block",
+    ];
+    let old = generate_multi_collection_records(&collections, 500);
+    let new_rec = apply_scattered_updates(&old, 4, 30000);
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "5 collections x 500 records - update every 4th",
+    )
+    .await;
+}
+
+#[tokio::test]
+async fn multi_collection_wipe_one_collection() {
+    let collections = [
+        "app.bsky.feed.like",
+        "app.bsky.feed.post",
+        "app.bsky.feed.repost",
+        "app.bsky.graph.follow",
+    ];
+    let old = generate_multi_collection_records(&collections, 400);
+
+    let new_rec: Vec<_> = old
+        .iter()
+        .filter(|(key, _)| !key.starts_with("app.bsky.feed.repost"))
+        .cloned()
+        .collect();
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "4 collections x 400 - wipe repost collection",
+    )
+    .await;
+}
+
+#[tokio::test]
+async fn multi_collection_keep_only_one() {
+    let collections = [
+        "app.bsky.feed.like",
+        "app.bsky.feed.post",
+        "app.bsky.feed.repost",
+        "app.bsky.graph.follow",
+        "app.bsky.graph.block",
+    ];
+    let old = generate_multi_collection_records(&collections, 300);
+    let new_rec = keep_only_collection(&old, "app.bsky.feed.post");
+    assert_equivalence(&old, &new_rec, "5 collections x 300 - keep only posts").await;
+}
+
+#[tokio::test]
+async fn multi_collection_add_new_collection() {
+    let old_collections = ["app.bsky.feed.like", "app.bsky.feed.post"];
+    let old = generate_multi_collection_records(&old_collections, 500);
+    let new_rec = append_records(&old, "app.bsky.graph.follow", 0..500, 40000);
+    assert_equivalence(&old, &new_rec, "2 collections x 500 + add 500 follows").await;
+}
+
+#[tokio::test]
+async fn mixed_ops_massive_tree() {
+    let collections = [
+        "app.bsky.feed.like",
+        "app.bsky.feed.post",
+        "app.bsky.feed.repost",
+        "app.bsky.graph.follow",
+    ];
+    let old = generate_multi_collection_records(&collections, 400);
+
+    let mut new_rec: Vec<_> = old
+        .iter()
+        .filter(|(key, _)| !key.starts_with("app.bsky.feed.repost"))
+        .enumerate()
+        .map(|(idx, (key, val))| {
+            if key.starts_with("app.bsky.feed.like") && idx % 3 == 0 {
+                (key.clone(), val + 50000)
+            } else {
+                (key.clone(), *val)
+            }
+        })
+        .collect();
+
+    new_rec.extend((0..200u32).map(|i| (make_key("app.bsky.graph.block", i), 60000 + i)));
+    new_rec.sort_by(|(a, _), (b, _)| a.cmp(b));
+
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "4 collections x 400: wipe reposts, update every 3rd like, add 200 blocks",
+    )
+    .await;
+}
+
+#[tokio::test]
+async fn grow_tree_by_double() {
+    let old = generate_records("app.bsky.feed.post", 0..1000);
+    let new_rec = generate_records("app.bsky.feed.post", 0..2000);
+    assert_equivalence(&old, &new_rec, "grow from 1000 to 2000").await;
+}
+
+#[tokio::test]
+async fn shrink_tree_by_half() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let new_rec = generate_records("app.bsky.feed.post", 0..1000);
+    assert_equivalence(&old, &new_rec, "shrink from 2000 to 1000").await;
+}
+
+#[tokio::test]
+async fn interleaved_keys_disjoint_ranges() {
+    let old: Vec<_> = (0..1000u32)
+        .map(|i| (make_key("app.bsky.feed.post", i * 2), i))
+        .collect();
+    let new_rec: Vec<_> = (0..1000u32)
+        .map(|i| (make_key("app.bsky.feed.post", i * 2 + 1), i + 10000))
+        .collect();
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "1000 even-keyed records replaced by 1000 odd-keyed",
+    )
+    .await;
+}
+
+#[tokio::test]
+async fn sparse_keys_wide_gaps() {
+    let old: Vec<_> = (0..500u32)
+        .map(|i| (make_key("app.bsky.feed.post", i * 100), i))
+        .collect();
+    let new_rec: Vec<_> = (0..500u32)
+        .map(|i| {
+            if i % 10 == 0 {
+                (make_key("app.bsky.feed.post", i * 100), i + 70000)
+            } else {
+                (make_key("app.bsky.feed.post", i * 100), i)
+            }
+        })
+        .collect();
+    assert_equivalence(&old, &new_rec, "500 sparse keys - update every 10th").await;
+}
+
+#[tokio::test]
+async fn many_collections_few_records_each() {
+    let collections: Vec<String> = (0..50u32)
+        .map(|i| format!("com.example.lexicon{i:02}.record"))
+        .collect();
+    let old: Vec<_> = collections
+        .iter()
+        .enumerate()
+        .flat_map(|(ci, coll)| {
+            let base = ci as u32 * 20;
+            (0..20u32).map(move |i| (make_key(coll, i), base + i))
+        })
+        .collect();
+
+    let new_rec: Vec<_> = old
+        .iter()
+        .enumerate()
+        .filter_map(|(idx, (key, val))| {
+            if idx % 15 == 0 {
+                None
+            } else if idx % 7 == 0 {
+                Some((key.clone(), val + 80000))
+            } else {
+                Some((key.clone(), *val))
+            }
+        })
+        .collect();
+
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "50 collections x 20 records - delete every 15th, update every 7th",
+    )
+    .await;
+}
+
+#[tokio::test]
+async fn update_all_records() {
+    let old = generate_records("app.bsky.feed.post", 0..1000);
+    let new_rec: Vec<_> = old
+        .iter()
+        .map(|(key, val)| (key.clone(), val + 90000))
+        .collect();
+    assert_equivalence(&old, &new_rec, "1000 records - update every single one").await;
+}
+
+#[tokio::test]
+async fn delete_all_but_one() {
+    let old = generate_records("app.bsky.feed.post", 0..1500);
+    let new_rec = vec![old[750].clone()];
+    assert_equivalence(&old, &new_rec, "1500 records - delete all but middle one").await;
+}
+
+#[tokio::test]
+async fn one_to_massive() {
+    let old = vec![(make_key("app.bsky.feed.post", 500), 500u32)];
+    let new_rec = generate_records("app.bsky.feed.post", 0..1500);
+    assert_equivalence(&old, &new_rec, "1 record to 1500 records").await;
+}
+
+#[tokio::test]
+async fn delete_head_and_tail() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let new_rec: Vec<_> = old[200..1800].to_vec();
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "2000 records - delete first 200 and last 200",
+    )
+    .await;
+}
+
+#[tokio::test]
+async fn keep_head_and_tail_only() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let mut new_rec: Vec<_> = old[..100].to_vec();
+    new_rec.extend_from_slice(&old[1900..]);
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "2000 records - keep only first 100 and last 100",
+    )
+    .await;
+}
+
+#[tokio::test]
+async fn massive_tree_update_first_and_last() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let mut new_rec = old.clone();
+    new_rec[0].1 += 99000;
+    new_rec[1999].1 += 99000;
+    assert_equivalence(&old, &new_rec, "2000 records - update only first and last").await;
+}
+
+#[tokio::test]
+async fn overlapping_collection_swap() {
+    let old_collections = [
+        "app.bsky.feed.like",
+        "app.bsky.feed.post",
+        "app.bsky.feed.repost",
+    ];
+    let old = generate_multi_collection_records(&old_collections, 500);
+
+    let mut new_rec: Vec<_> = old
+        .iter()
+        .filter(|(key, _)| key.starts_with("app.bsky.feed.post"))
+        .cloned()
+        .collect();
+    new_rec.extend((0..500u32).map(|i| (make_key("app.bsky.graph.follow", i), 70000 + i)));
+    new_rec.extend((0..500u32).map(|i| (make_key("app.bsky.graph.block", i), 71000 + i)));
+    new_rec.sort_by(|(a, _), (b, _)| a.cmp(b));
+
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "swap 2 of 3 collections, keep 1 (posts), 500 each",
+    )
+    .await;
+}
+
+#[tokio::test]
+async fn swiss_cheese_deletions() {
+    let old = generate_records("app.bsky.feed.post", 0..1500);
+    let new_rec: Vec<_> = old
+        .iter()
+        .enumerate()
+        .filter(|(idx, _)| {
+            let bucket = idx / 50;
+            bucket % 3 != 0
+        })
+        .map(|(_, r)| r.clone())
+        .collect();
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "1500 records - delete every 3rd chunk of 50",
+    )
+    .await;
+}
+
+#[tokio::test]
+async fn mixed_ops_with_key_density_change() {
+    let old: Vec<_> = (0..1000u32)
+        .map(|i| (make_key("app.bsky.feed.post", i * 3), i))
+        .collect();
+
+    let mut new_rec: Vec<_> = old
+        .iter()
+        .filter(|(_, val)| val % 4 != 0)
+        .cloned()
+        .collect();
+    new_rec.extend((0..500u32).map(|i| (make_key("app.bsky.feed.post", i * 3 + 1), i + 100000)));
+    new_rec.sort_by(|(a, _), (b, _)| a.cmp(b));
+
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "1000 sparse records: delete every 4th, insert 500 in gaps",
+    )
+    .await;
+}
@@ -0,0 +1,640 @@
+mod common;
+mod mst_verify;
+
+use std::collections::BTreeMap;
+use std::str::FromStr;
+use std::sync::Arc;
+
+use cid::Cid;
+use common::*;
+use jacquard_common::smol_str::SmolStr;
+use jacquard_repo::commit::Commit;
+use jacquard_repo::mst::{Mst, VerifiedWriteOp};
+use jacquard_repo::storage::{BlockStore, MemoryBlockStore};
+use mst_verify::{extract_event_blocks, inline_to_store};
+use reqwest::StatusCode;
+use serde_json::{Value, json};
+use tranquil_db_traits::{RepoEventType, SequenceNumber, SequencedEvent};
+use tranquil_types::Did;
+
+async fn new_commit_data_cid(
+    storage: &Arc<MemoryBlockStore>,
+    commit_cid: &Cid,
+) -> Result<Cid, String> {
+    let commit_bytes = storage
+        .get(commit_cid)
+        .await
+        .map_err(|e| format!("get commit: {e:?}"))?
+        .ok_or_else(|| format!("CAR missing commit block {commit_cid}"))?;
+    let commit = Commit::from_cbor(&commit_bytes).map_err(|e| format!("parse commit: {e:?}"))?;
+    Ok(*commit.data())
+}
+
+fn ops_json(event: &SequencedEvent) -> Result<&Vec<Value>, String> {
+    event
+        .ops
+        .as_ref()
+        .and_then(|v| v.as_array())
+        .ok_or_else(|| "event.ops not an array".into())
+}
+
+fn parse_op_to_verified(op: &Value) -> Result<VerifiedWriteOp, String> {
+    let action = op["action"].as_str().ok_or("op.action missing")?;
+    let path = op["path"].as_str().ok_or("op.path missing")?;
+    let key = SmolStr::new(path);
+    match action {
+        "create" => {
+            let cid_str = op["cid"].as_str().ok_or("create missing cid")?;
+            let cid = Cid::from_str(cid_str).map_err(|e| format!("parse cid: {e:?}"))?;
+            Ok(VerifiedWriteOp::Create { key, cid })
+        }
+        "update" => {
+            let cid_str = op["cid"].as_str().ok_or("update missing cid")?;
+            let cid = Cid::from_str(cid_str).map_err(|e| format!("parse cid: {e:?}"))?;
+            let prev_str = op["prev"].as_str().ok_or("update missing prev")?;
+            let prev = Cid::from_str(prev_str).map_err(|e| format!("parse prev: {e:?}"))?;
+            Ok(VerifiedWriteOp::Update { key, cid, prev })
+        }
+        "delete" => {
+            let prev_str = op["prev"].as_str().ok_or("delete missing prev")?;
+            let prev = Cid::from_str(prev_str).map_err(|e| format!("parse prev: {e:?}"))?;
+            Ok(VerifiedWriteOp::Delete { key, prev })
+        }
+        other => Err(format!("unknown op action: {other}")),
+    }
+}
+
+async fn verify_inductive_forward(event: &SequencedEvent) -> Result<(Cid, Cid), String> {
+    let prev_data_cid = event
+        .prev_data_cid
+        .as_ref()
+        .and_then(|c| c.to_cid())
+        .ok_or_else(|| "event missing prev_data_cid".to_string())?;
+    let commit_cid = event
+        .commit_cid
+        .as_ref()
+        .and_then(|c| c.to_cid())
+        .ok_or_else(|| "event missing commit_cid".to_string())?;
+
+    let storage = inline_to_store(extract_event_blocks(event)?);
+    let expected_new_data = new_commit_data_cid(&storage, &commit_cid).await?;
+
+    let mut mst = Mst::load(storage.clone(), prev_data_cid, None);
+    for op_value in ops_json(event)? {
+        let action = op_value["action"].as_str().ok_or("op.action missing")?;
+        let path = op_value["path"].as_str().ok_or("op.path missing")?;
+        match action {
+            "create" | "update" => {
+                let cid = Cid::from_str(op_value["cid"].as_str().ok_or("op.cid missing")?)
+                    .map_err(|e| format!("parse op.cid: {e:?}"))?;
+                mst = mst
+                    .add(path, cid)
+                    .await
+                    .map_err(|e| format!("mst.add({path}): {e:?}"))?;
+            }
+            "delete" => {
+                mst = mst
+                    .delete(path)
+                    .await
+                    .map_err(|e| format!("mst.delete({path}): {e:?}"))?;
+            }
+            other => return Err(format!("unknown op action: {other}")),
+        }
+    }
+    let computed = mst
+        .persist()
+        .await
+        .map_err(|e| format!("mst.persist: {e:?}"))?;
+    Ok((expected_new_data, computed))
+}
+
+async fn verify_inductive_inverse(event: &SequencedEvent) -> Result<(Cid, Cid), String> {
+    let prev_data_cid = event
+        .prev_data_cid
+        .as_ref()
+        .and_then(|c| c.to_cid())
+        .ok_or_else(|| "event missing prev_data_cid".to_string())?;
+    let commit_cid = event
+        .commit_cid
+        .as_ref()
+        .and_then(|c| c.to_cid())
+        .ok_or_else(|| "event missing commit_cid".to_string())?;
+
+    let storage = inline_to_store(extract_event_blocks(event)?);
+    let new_data_cid = new_commit_data_cid(&storage, &commit_cid).await?;
+
+    let mut mst = Mst::load(storage.clone(), new_data_cid, None);
+    for op_value in ops_json(event)? {
+        let verified = parse_op_to_verified(op_value)?;
+        let inverted = mst
+            .invert_op(verified.clone())
+            .await
+            .map_err(|e| format!("invert_op({verified:?}): {e:?}"))?;
+        if !inverted {
+            return Err(format!("op not invertible: {verified:?}"));
+        }
+    }
+    let computed_prev = mst
+        .get_pointer()
+        .await
+        .map_err(|e| format!("get_pointer: {e:?}"))?;
+    Ok((prev_data_cid, computed_prev))
+}
+
+fn report_failures(total: usize, failures: &[String], mode: &str) {
+    assert!(
+        failures.is_empty(),
+        "{} of {total} {mode} commit events failed inductive verification:\n  - {}",
+        failures.len(),
+        failures.join("\n  - "),
+    );
+}
+
+async fn apply_writes_batch(client: &reqwest::Client, token: &str, did: &str, writes: Vec<Value>) {
+    let payload = json!({ "repo": did, "writes": writes });
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.applyWrites",
+            base_url().await
+        ))
+        .bearer_auth(token)
+        .json(&payload)
+        .send()
+        .await
+        .expect("applyWrites request failed");
+    assert_eq!(
+        res.status(),
+        StatusCode::OK,
+        "applyWrites failed: {:?}",
+        res.text().await
+    );
+}
+
+async fn create_record(client: &reqwest::Client, token: &str, did: &str, col: &str, rkey: &str) {
+    let now = chrono::Utc::now().to_rfc3339();
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.createRecord",
+            base_url().await
+        ))
+        .bearer_auth(token)
+        .json(&json!({
+            "repo": did,
+            "collection": col,
+            "rkey": rkey,
+            "record": {
+                "$type": col,
+                "text": format!("post {rkey}"),
+                "createdAt": now,
+            }
+        }))
+        .send()
+        .await
+        .expect("createRecord request failed");
+    assert_eq!(res.status(), StatusCode::OK, "createRecord failed");
+}
+
+async fn put_record(
+    client: &reqwest::Client,
+    token: &str,
+    did: &str,
+    col: &str,
+    rkey: &str,
+    text: &str,
+) {
+    let now = chrono::Utc::now().to_rfc3339();
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.putRecord",
+            base_url().await
+        ))
+        .bearer_auth(token)
+        .json(&json!({
+            "repo": did,
+            "collection": col,
+            "rkey": rkey,
+            "record": {
+                "$type": col,
+                "text": text,
+                "createdAt": now,
+            }
+        }))
+        .send()
+        .await
+        .expect("putRecord request failed");
+    assert_eq!(res.status(), StatusCode::OK, "putRecord failed");
+}
+
+async fn delete_record(client: &reqwest::Client, token: &str, did: &str, col: &str, rkey: &str) {
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.deleteRecord",
+            base_url().await
+        ))
+        .bearer_auth(token)
+        .json(&json!({ "repo": did, "collection": col, "rkey": rkey }))
+        .send()
+        .await
+        .expect("deleteRecord request failed");
+    assert_eq!(res.status(), StatusCode::OK, "deleteRecord failed");
+}
+
+const COLLECTION: &str = "app.bsky.feed.post";
+fn rkey_for(prefix: &str, i: usize) -> String {
+    format!("3k{prefix}{:08}", i)
+}
+
+async fn our_commit_events(did: &str) -> Vec<SequencedEvent> {
+    let repos = get_test_repos().await;
+    let typed_did = Did::new(did.to_string()).unwrap();
+    let events = repos
+        .repo
+        .get_events_since_seq(SequenceNumber::ZERO, None)
+        .await
+        .expect("get_events_since_seq");
+    events
+        .into_iter()
+        .filter(|e| e.did == typed_did && e.event_type == RepoEventType::Commit)
+        .collect()
+}
+
+#[tokio::test]
+async fn inductive_forward_verifies_delete_commits() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let now = chrono::Utc::now().to_rfc3339();
+    const N_CREATE: usize = 200;
+
+    let all_writes: Vec<Value> = (0..N_CREATE)
+        .map(|i| {
+            json!({
+                "$type": "com.atproto.repo.applyWrites#create",
+                "collection": COLLECTION,
+                "rkey": rkey_for("del", i),
+                "value": {
+                    "$type": COLLECTION,
+                    "text": format!("record {i}"),
+                    "createdAt": now,
+                }
+            })
+        })
+        .collect();
+    for chunk in all_writes.chunks(50) {
+        apply_writes_batch(&client, &token, &did, chunk.to_vec()).await;
+    }
+
+    let delete_indices: Vec<usize> = (10..N_CREATE).step_by(7).collect();
+    for i in &delete_indices {
+        delete_record(&client, &token, &did, COLLECTION, &rkey_for("del", *i)).await;
+    }
+
+    let our = our_commit_events(&did).await;
+    let delete_events: Vec<&SequencedEvent> = our
+        .iter()
+        .filter(|e| {
+            ops_json(e)
+                .map(|arr| arr.iter().any(|op| op["action"].as_str() == Some("delete")))
+                .unwrap_or(false)
+        })
+        .collect();
+    assert_eq!(delete_events.len(), delete_indices.len());
+
+    let mut failures = Vec::new();
+    for e in &delete_events {
+        match verify_inductive_forward(e).await {
+            Ok((exp, got)) if exp == got => {}
+            Ok((exp, got)) => failures.push(format!(
+                "seq={}: root mismatch exp={exp} got={got}",
+                e.seq.as_i64()
+            )),
+            Err(msg) => failures.push(format!("seq={}: {msg}", e.seq.as_i64())),
+        }
+    }
+    report_failures(delete_events.len(), &failures, "delete forward");
+}
+
+#[tokio::test]
+async fn inductive_forward_verifies_create_commits() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+    for i in 0..60usize {
+        create_record(&client, &token, &did, COLLECTION, &rkey_for("cre", i)).await;
+    }
+
+    let our = our_commit_events(&did).await;
+    let create_events: Vec<&SequencedEvent> = our
+        .iter()
+        .filter(|e| {
+            ops_json(e)
+                .map(|arr| arr.iter().all(|op| op["action"].as_str() == Some("create")))
+                .unwrap_or(false)
+                && e.prev_data_cid.is_some()
+        })
+        .collect();
+    assert!(!create_events.is_empty());
+
+    let mut failures = Vec::new();
+    for e in &create_events {
+        match verify_inductive_forward(e).await {
+            Ok((exp, got)) if exp == got => {}
+            Ok((exp, got)) => failures.push(format!(
+                "seq={}: root mismatch exp={exp} got={got}",
+                e.seq.as_i64()
+            )),
+            Err(msg) => failures.push(format!("seq={}: {msg}", e.seq.as_i64())),
+        }
+    }
+    report_failures(create_events.len(), &failures, "create forward");
+}
+
+#[tokio::test]
+async fn inductive_forward_verifies_update_commits() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let now = chrono::Utc::now().to_rfc3339();
+    let creates: Vec<Value> = (0..80)
+        .map(|i| {
+            json!({
+                "$type": "com.atproto.repo.applyWrites#create",
+                "collection": COLLECTION,
+                "rkey": rkey_for("upd", i),
+                "value": {
+                    "$type": COLLECTION,
+                    "text": format!("original {i}"),
+                    "createdAt": now,
+                }
+            })
+        })
+        .collect();
+    for chunk in creates.chunks(40) {
+        apply_writes_batch(&client, &token, &did, chunk.to_vec()).await;
+    }
+
+    for i in (0..80).step_by(3) {
+        put_record(
+            &client,
+            &token,
+            &did,
+            COLLECTION,
+            &rkey_for("upd", i),
+            &format!("updated {i}"),
+        )
+        .await;
+    }
+
+    let our = our_commit_events(&did).await;
+    let update_events: Vec<&SequencedEvent> = our
+        .iter()
+        .filter(|e| {
+            ops_json(e)
+                .map(|arr| arr.iter().any(|op| op["action"].as_str() == Some("update")))
+                .unwrap_or(false)
+        })
+        .collect();
+    assert!(!update_events.is_empty());
+
+    let mut failures = Vec::new();
+    for e in &update_events {
+        match verify_inductive_forward(e).await {
+            Ok((exp, got)) if exp == got => {}
+            Ok((exp, got)) => failures.push(format!(
+                "seq={}: root mismatch exp={exp} got={got}",
+                e.seq.as_i64()
+            )),
+            Err(msg) => failures.push(format!("seq={}: {msg}", e.seq.as_i64())),
+        }
+    }
+    report_failures(update_events.len(), &failures, "update forward");
+}
+
+#[tokio::test]
+async fn inductive_forward_verifies_mixed_applywrites() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let now = chrono::Utc::now().to_rfc3339();
+    let seed: Vec<Value> = (0..120)
+        .map(|i| {
+            json!({
+                "$type": "com.atproto.repo.applyWrites#create",
+                "collection": COLLECTION,
+                "rkey": rkey_for("mix", i),
+                "value": {
+                    "$type": COLLECTION,
+                    "text": format!("seed {i}"),
+                    "createdAt": now,
+                }
+            })
+        })
+        .collect();
+    for chunk in seed.chunks(40) {
+        apply_writes_batch(&client, &token, &did, chunk.to_vec()).await;
+    }
+
+    let mixed: Vec<Value> = (0..40)
+        .flat_map(|i| {
+            vec![
+                json!({
+                    "$type": "com.atproto.repo.applyWrites#create",
+                    "collection": COLLECTION,
+                    "rkey": rkey_for("mxc", i),
+                    "value": {
+                        "$type": COLLECTION,
+                        "text": format!("new {i}"),
+                        "createdAt": now,
+                    }
+                }),
+                json!({
+                    "$type": "com.atproto.repo.applyWrites#update",
+                    "collection": COLLECTION,
+                    "rkey": rkey_for("mix", i),
+                    "value": {
+                        "$type": COLLECTION,
+                        "text": format!("updated-mix {i}"),
+                        "createdAt": now,
+                    }
+                }),
+                json!({
+                    "$type": "com.atproto.repo.applyWrites#delete",
+                    "collection": COLLECTION,
+                    "rkey": rkey_for("mix", i + 60),
+                }),
+            ]
+        })
+        .collect();
+    apply_writes_batch(&client, &token, &did, mixed).await;
+
+    let our = our_commit_events(&did).await;
+    let last = our
+        .iter()
+        .rfind(|e| e.prev_data_cid.is_some())
+        .expect("at least one non-genesis commit");
+
+    let actions: Vec<&str> = ops_json(last)
+        .unwrap()
+        .iter()
+        .filter_map(|op| op["action"].as_str())
+        .collect();
+    assert!(actions.contains(&"create"));
+    assert!(actions.contains(&"update"));
+    assert!(actions.contains(&"delete"));
+
+    let (exp, got) = verify_inductive_forward(last)
+        .await
+        .expect("mixed applyWrites forward verify");
+    assert_eq!(exp, got, "mixed applyWrites commit forward-verify mismatch");
+}
+
+#[tokio::test]
+async fn inductive_inverse_verifies_every_commit() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let now = chrono::Utc::now().to_rfc3339();
+    let seed: Vec<Value> = (0..100)
+        .map(|i| {
+            json!({
+                "$type": "com.atproto.repo.applyWrites#create",
+                "collection": COLLECTION,
+                "rkey": rkey_for("inv", i),
+                "value": {
+                    "$type": COLLECTION,
+                    "text": format!("seed {i}"),
+                    "createdAt": now,
+                }
+            })
+        })
+        .collect();
+    for chunk in seed.chunks(50) {
+        apply_writes_batch(&client, &token, &did, chunk.to_vec()).await;
+    }
+    for i in (0..100).step_by(5) {
+        put_record(
+            &client,
+            &token,
+            &did,
+            COLLECTION,
+            &rkey_for("inv", i),
+            &format!("upd {i}"),
+        )
+        .await;
+    }
+    for i in (2..100).step_by(11) {
+        delete_record(&client, &token, &did, COLLECTION, &rkey_for("inv", i)).await;
+    }
+
+    let our = our_commit_events(&did).await;
+    let non_genesis: Vec<&SequencedEvent> = our
+        .iter()
+        .filter(|e| e.prev_data_cid.is_some() && ops_json(e).is_ok())
+        .collect();
+    assert!(!non_genesis.is_empty());
+
+    let mut failures = Vec::new();
+    for e in &non_genesis {
+        match verify_inductive_inverse(e).await {
+            Ok((exp, got)) if exp == got => {}
+            Ok((exp, got)) => failures.push(format!(
+                "seq={}: inverse root mismatch exp={exp} got={got}",
+                e.seq.as_i64()
+            )),
+            Err(msg) => failures.push(format!("seq={}: {msg}", e.seq.as_i64())),
+        }
+    }
+    report_failures(non_genesis.len(), &failures, "any inverse");
+}
+
+#[tokio::test]
+async fn prev_cid_chain_walks_to_genesis() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+    for i in 0..8 {
+        create_record(&client, &token, &did, COLLECTION, &rkey_for("cha", i)).await;
+    }
+
+    let our = our_commit_events(&did).await;
+    assert!(our.len() >= 2);
+
+    let last = our.last().unwrap();
+    let mut current_prev: Option<Cid> = last.prev_cid.as_ref().and_then(|c| c.to_cid());
+    let head_commit_cid = last
+        .commit_cid
+        .as_ref()
+        .and_then(|c| c.to_cid())
+        .expect("head commit_cid");
+
+    let by_commit: BTreeMap<Cid, &SequencedEvent> = our
+        .iter()
+        .filter_map(|e| {
+            e.commit_cid
+                .as_ref()
+                .and_then(|c| c.to_cid())
+                .map(|c| (c, e))
+        })
+        .collect();
+
+    let mut visited = 1;
+    while let Some(prev) = current_prev {
+        let e = by_commit
+            .get(&prev)
+            .unwrap_or_else(|| panic!("prev commit {prev} missing from event list"));
+        visited += 1;
+        current_prev = e.prev_cid.as_ref().and_then(|c| c.to_cid());
+    }
+    assert!(
+        visited >= 2,
+        "chain too short: visited={visited}, head_commit={head_commit_cid}"
+    );
+    assert_eq!(
+        visited,
+        our.len(),
+        "chain did not reach genesis: walked {visited}, have {}",
+        our.len()
+    );
+}
+
+#[tokio::test]
+async fn record_bytes_present_in_car_for_creates() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let now = chrono::Utc::now().to_rfc3339();
+    let writes: Vec<Value> = (0..5)
+        .map(|i| {
+            json!({
+                "$type": "com.atproto.repo.applyWrites#create",
+                "collection": COLLECTION,
+                "rkey": rkey_for("rec", i),
+                "value": {
+                    "$type": COLLECTION,
+                    "text": format!("rec {i}"),
+                    "createdAt": now,
+                }
+            })
+        })
+        .collect();
+    apply_writes_batch(&client, &token, &did, writes).await;
+
+    let our = our_commit_events(&did).await;
+    let latest = our.iter().rfind(|e| e.prev_data_cid.is_some()).unwrap();
+
+    let inline = extract_event_blocks(latest).unwrap();
+    let have_cids: std::collections::HashSet<Cid> = inline
+        .iter()
+        .map(|b| Cid::read_bytes(b.cid_bytes.as_slice()).unwrap())
+        .collect();
+
+    for op in ops_json(latest).unwrap() {
+        if op["action"].as_str() == Some("create")
+            && let Some(cid_str) = op["cid"].as_str()
+        {
+            let cid = Cid::from_str(cid_str).unwrap();
+            assert!(
+                have_cids.contains(&cid),
+                "create op record CID {cid} not present in CAR inline blocks"
+            );
+        }
+    }
+}
@@ -0,0 +1,26 @@
+use std::collections::BTreeMap;
+use std::sync::Arc;
+
+use bytes::Bytes;
+use cid::Cid;
+use jacquard_repo::storage::MemoryBlockStore;
+use tranquil_db_traits::{EventBlockInline, EventBlocks, SequencedEvent};
+
+pub fn extract_event_blocks(event: &SequencedEvent) -> Result<&[EventBlockInline], String> {
+    match event.blocks.as_ref() {
+        Some(EventBlocks::Inline(v)) => Ok(v.as_slice()),
+        Some(EventBlocks::LegacyCids(_)) => Err("legacy cids, not inline".into()),
+        None => Err("event missing blocks".into()),
+    }
+}
+
+pub fn inline_to_store(inline: &[EventBlockInline]) -> Arc<MemoryBlockStore> {
+    let map: BTreeMap<Cid, Bytes> = inline
+        .iter()
+        .map(|b| {
+            let cid = Cid::read_bytes(b.cid_bytes.as_slice()).expect("valid cid bytes");
+            (cid, Bytes::from(b.data.clone()))
+        })
+        .collect();
+    Arc::new(MemoryBlockStore::new_from_blocks(map))
+}
@@ -187,6 +187,37 @@ async fn test_par_and_authorize() {
    );
 }

+#[tokio::test]
+async fn test_par_public_client_empty_assertion_fields() {
+    let url = base_url().await;
+    let client = client();
+    let redirect_uri = "https://nels.evil.oauth.pet/callback";
+    let mock_client = setup_mock_client_metadata(redirect_uri).await;
+    let client_id = mock_client.uri();
+    let (_, code_challenge) = generate_pkce();
+    let par_res = client
+        .post(format!("{}/oauth/par", url))
+        .form(&[
+            ("response_type", "code"),
+            ("client_id", &client_id),
+            ("redirect_uri", redirect_uri),
+            ("code_challenge", &code_challenge),
+            ("code_challenge_method", "S256"),
+            ("scope", "atproto"),
+            ("state", "test-state"),
+            ("client_assertion", ""),
+            ("client_assertion_type", ""),
+        ])
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(
+        par_res.status(),
+        StatusCode::CREATED,
+        "PAR with empty assertion fields from a public client should succeed"
+    );
+}
+
 #[tokio::test]
 async fn test_full_oauth_flow() {
    let url = base_url().await;
@@ -88,12 +88,17 @@ fn test_signal_username_validation() {
    assert!(is_valid_signal_username("bob_smith.99"));
    assert!(is_valid_signal_username("user123.42"));
    assert!(is_valid_signal_username("lu1.01"));
-    assert!(is_valid_signal_username("abc.00"));
    assert!(is_valid_signal_username("a_very_long_username_here.55"));
+    assert!(is_valid_signal_username("alice.123"));
+    assert!(is_valid_signal_username("alice.999999999"));
+    assert!(is_valid_signal_username("alice.18446744073709551615"));

    assert!(!is_valid_signal_username("alice"));
    assert!(!is_valid_signal_username("alice.1"));
    assert!(!is_valid_signal_username("alice.001"));
+    assert!(!is_valid_signal_username("abc.00"));
+    assert!(!is_valid_signal_username("alice.0"));
+    assert!(!is_valid_signal_username("alice.999999999999999999999"));
    assert!(!is_valid_signal_username(".01"));
    assert!(!is_valid_signal_username("ab.01"));
    assert!(!is_valid_signal_username(""));
@@ -0,0 +1,147 @@
+mod common;
+mod helpers;
+use common::*;
+use helpers::*;
+use reqwest::StatusCode;
+use std::sync::Once;
+
+static SET_SEMAPHORE: Once = Once::new();
+
+fn ensure_low_semaphore() {
+    SET_SEMAPHORE.call_once(|| unsafe {
+        std::env::set_var("MAX_CONCURRENT_REPO_EXPORTS", "1");
+    });
+}
+
+#[tokio::test]
+async fn test_get_repo_succeeds_with_many_records() {
+    ensure_low_semaphore();
+    let client = client();
+    let (did, jwt) = setup_new_user("sync-batched-car").await;
+
+    let create_futures = (0..20).map(|i| {
+        let client = &client;
+        let did = &did;
+        let jwt = &jwt;
+        async move {
+            create_post(client, did, jwt, &format!("Batch test post {}", i)).await;
+        }
+    });
+    futures::future::join_all(create_futures).await;
+
+    let res = client
+        .get(format!(
+            "{}/xrpc/com.atproto.sync.getRepo",
+            base_url().await
+        ))
+        .query(&[("did", did.as_str())])
+        .send()
+        .await
+        .expect("Failed to send getRepo request");
+
+    assert_eq!(res.status(), StatusCode::OK);
+    assert_eq!(
+        res.headers()
+            .get("content-type")
+            .and_then(|h| h.to_str().ok()),
+        Some("application/vnd.ipld.car")
+    );
+    let car_bytes = res.bytes().await.expect("Failed to read response body");
+    assert!(
+        car_bytes.len() > 200,
+        "CAR with 20 records should have substantial data, got {} bytes",
+        car_bytes.len()
+    );
+}
+
+#[tokio::test]
+async fn test_get_repo_semaphore_rejects_excess_concurrency() {
+    ensure_low_semaphore();
+    let client = client();
+    let (did, jwt) = setup_new_user("sync-semaphore").await;
+
+    for i in 0..50 {
+        create_post(&client, &did, &jwt, &format!("Padding post {}", i)).await;
+    }
+
+    let base = base_url().await;
+    let concurrent_requests = 10;
+
+    let request_futures = (0..concurrent_requests).map(|_| {
+        let client = client.clone();
+        let did = did.clone();
+        async move {
+            client
+                .get(format!("{}/xrpc/com.atproto.sync.getRepo", base))
+                .query(&[("did", did.as_str())])
+                .send()
+                .await
+                .expect("Failed to send request")
+                .status()
+        }
+    });
+
+    let statuses: Vec<StatusCode> = futures::future::join_all(request_futures).await;
+    let ok_count = statuses.iter().filter(|s| **s == StatusCode::OK).count();
+    let rejected_count = statuses
+        .iter()
+        .filter(|s| **s == StatusCode::SERVICE_UNAVAILABLE)
+        .count();
+
+    assert!(ok_count >= 1, "at least one request should succeed");
+    assert!(
+        rejected_count > 0,
+        "semaphore=1 with {} concurrent requests, expected some 503 rejections",
+        concurrent_requests
+    );
+    assert!(
+        ok_count + rejected_count == statuses.len(),
+        "expected only 200 or 503 responses: {:?}",
+        statuses
+    );
+}
+
+#[tokio::test]
+async fn test_get_repo_since_not_affected_by_semaphore() {
+    ensure_low_semaphore();
+    let client = client();
+    let (did, jwt) = setup_new_user("sync-since-no-sem").await;
+    create_post(&client, &did, &jwt, "First post").await;
+
+    let latest_res = client
+        .get(format!(
+            "{}/xrpc/com.atproto.sync.getLatestCommit",
+            base_url().await
+        ))
+        .query(&[("did", did.as_str())])
+        .send()
+        .await
+        .expect("Failed to get latest commit");
+    let body: serde_json::Value = latest_res.json().await.unwrap();
+    let rev = body["rev"].as_str().unwrap();
+
+    create_post(&client, &did, &jwt, "Second post").await;
+
+    let base = base_url().await;
+    let request_futures = (0..10).map(|_| {
+        let client = client.clone();
+        let did = did.clone();
+        let rev = rev.to_string();
+        async move {
+            client
+                .get(format!("{}/xrpc/com.atproto.sync.getRepo", base))
+                .query(&[("did", did.as_str()), ("since", rev.as_str())])
+                .send()
+                .await
+                .expect("Failed to send request")
+                .status()
+        }
+    });
+
+    let statuses: Vec<StatusCode> = futures::future::join_all(request_futures).await;
+    assert!(
+        statuses.iter().all(|s| *s == StatusCode::OK),
+        "getRepo with since should bypass semaphore, got: {:?}",
+        statuses
+    );
+}
@@ -157,11 +157,22 @@ impl ScopePermissions {
    }

    pub fn assert_rpc(&self, aud: &str, lxm: &str) -> Result<(), ScopeError> {
-        if self.has_transition_generic {
-            return Ok(());
+        if lxm.starts_with("chat.bsky.") {
+            if self.has_transition_chat {
+                return Ok(());
+            }
+            if self.has_transition_generic && !self.has_transition_chat {
+                return Err(ScopeError::InsufficientScope {
+                    required: "transition:chat.bsky".to_string(),
+                    message: format!(
+                        "Chat access requires transition:chat.bsky scope to call {}",
+                        lxm
+                    ),
+                });
+            }
        }

-        if lxm.starts_with("chat.bsky.") && self.has_transition_chat {
+        if self.has_transition_generic {
            return Ok(());
        }

@@ -347,6 +358,23 @@ mod tests {
        assert!(perms.allows_blob("image/png"));
    }

+    #[test]
+    fn test_transition_generic_without_chat_blocks_chat() {
+        let perms = ScopePermissions::from_scope_string(Some("transition:generic"));
+        assert!(perms.allows_rpc("did:web:api.bsky.app", "app.bsky.feed.getTimeline"));
+        assert!(!perms.allows_rpc("did:web:api.bsky.app", "chat.bsky.convo.listConvos"));
+        assert!(!perms.allows_rpc("did:web:api.bsky.app", "chat.bsky.convo.getMessages"));
+    }
+
+    #[test]
+    fn test_transition_generic_with_chat_allows_chat() {
+        let perms =
+            ScopePermissions::from_scope_string(Some("transition:generic transition:chat.bsky"));
+        assert!(perms.allows_rpc("did:web:api.bsky.app", "app.bsky.feed.getTimeline"));
+        assert!(perms.allows_rpc("did:web:api.bsky.app", "chat.bsky.convo.listConvos"));
+        assert!(perms.allows_rpc("did:web:api.bsky.app", "chat.bsky.convo.getMessages"));
+    }
+
    #[test]
    fn test_transition_chat_only_allows_chat() {
        let perms = ScopePermissions::from_scope_string(Some("transition:chat.bsky"));
@@ -33,15 +33,11 @@ impl SignalUsername {
    pub fn parse(username: &str) -> Result<Self, InvalidSignalUsername> {
        let reject = || Err(InvalidSignalUsername(username.to_string()));

-        if username.len() < 6 || username.len() > 35 {
-            return reject();
-        }
-
        let Some((base, discriminator)) = username.rsplit_once('.') else {
            return reject();
        };

-        if base.len() < 3 || base.len() > 32 {
+        if !matches!(base.len(), 3..=32) {
            return reject();
        }

@@ -53,7 +49,7 @@ impl SignalUsername {
            return reject();
        }

-        if discriminator.len() != 2 || !discriminator.chars().all(|c| c.is_ascii_digit()) {
+        if !is_valid_discriminator(discriminator) {
            return reject();
        }

@@ -65,6 +61,19 @@ impl SignalUsername {
    }
 }

+fn is_valid_discriminator(s: &str) -> bool {
+    if !s.chars().all(|c| c.is_ascii_digit()) {
+        return false;
+    }
+    if !matches!(s.len(), 2..=20) {
+        return false;
+    }
+    if s.len() > 2 && s.starts_with('0') {
+        return false;
+    }
+    s.parse::<u64>().is_ok_and(|n| n != 0)
+}
+
 impl fmt::Display for SignalUsername {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.write_str(&self.0)
@@ -113,11 +122,7 @@ pub struct MessageTooLong {

 impl fmt::Display for MessageTooLong {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(
-            f,
-            "message body too long: {} bytes (max {})",
-            self.len, self.max
-        )
+        write!(f, "message body is {} bytes, max {}", self.len, self.max)
    }
 }

@@ -368,13 +373,13 @@ impl SignalClient {
            let req = tokio::select! {
                biased;
                _ = shutdown.cancelled() => {
-                    tracing::info!("signal worker shutting down (cancellation)");
+                    tracing::info!("signal worker cancelled, shutting down");
                    break;
                }
                msg = rx.recv() => match msg {
                    Some(r) => r,
                    None => {
-                        tracing::info!("signal worker shutting down (channel closed)");
+                        tracing::info!("signal worker channel closed, shutting down");
                        break;
                    }
                },
@@ -1050,7 +1050,7 @@ impl ContentsStore for FjallSignalStore {
            .and_then(|v| match <[u8; 32]>::try_from(v.as_ref()) {
                Ok(arr) => Some(ProfileKey { bytes: arr }),
                Err(_) => {
-                    warn!(%uuid, len = v.len(), "corrupted profile key (expected 32 bytes)");
+                    warn!(%uuid, len = v.len(), "corrupted profile key, expected 32 bytes");
                    None
                }
            }))
@@ -1117,7 +1117,7 @@ impl ContentsStore for PgSignalStore {
            Some(r) => match <[u8; 32]>::try_from(r.key.as_slice()) {
                Ok(arr) => Some(ProfileKey { bytes: arr }),
                Err(_) => {
-                    warn!(%uuid, len = r.key.len(), "corrupted profile key (expected 32 bytes)");
+                    warn!(%uuid, len = r.key.len(), "corrupted profile key, expected 32 bytes");
                    None
                }
            },
@@ -14,7 +14,7 @@ parking_lot = { workspace = true }
 fjall = "3"
 lsm-tree = "3"
 flume = "0.11"
-tokio = { workspace = true, features = ["sync", "rt"] }
+tokio = { workspace = true, features = ["sync", "rt", "time"] }
 bytes = "1"
 memmap2 = "0.9"
 tracing = { workspace = true }
@@ -34,9 +34,21 @@ dashmap = "6"
 rayon = "1"
 smallvec = "1"
 uuid = { workspace = true }
+tempfile = { version = "3", optional = true }
+clap = { workspace = true, optional = true }
+toml = { version = "0.8", optional = true }
+tracing-subscriber = { workspace = true, features = ["env-filter"], optional = true }
+libc = "0.2"

 [features]
-test-harness = []
+test-harness = ["dep:tempfile"]
+gauntlet-cli = ["test-harness", "dep:clap", "dep:toml", "dep:tracing-subscriber"]
+gauntlet-jemalloc-prof = []
+
+[[bin]]
+name = "tranquil-gauntlet"
+path = "src/bin/tranquil_gauntlet.rs"
+required-features = ["gauntlet-cli"]

 [dev-dependencies]
 tranquil-store = { path = ".", features = ["test-harness"] }
@@ -50,7 +62,7 @@ tranquil-db = { workspace = true }
 sqlx = { workspace = true }
 k256 = { workspace = true }
 rand = { workspace = true }
-tikv-jemallocator = "0.6"
+tikv-jemallocator = { version = "0.6", features = ["profiling", "unprefixed_malloc_on_supported_platforms"] }
 tracing-subscriber = { workspace = true, features = ["env-filter"] }

 [[bench]]
@@ -100,9 +100,7 @@ impl<'a> DirectSeeder<'a> {
        }

        let loc = self.data_writer.append_block(cid, data).unwrap();
-        self.hint_writer
-            .append_hint(cid, loc.file_id, loc.offset, loc.length)
-            .unwrap();
+        self.hint_writer.append_hint(cid, &loc).unwrap();
        self.blocks_in_file += 1;

        if self.blocks_in_file.is_multiple_of(10_000) {
@@ -0,0 +1,4 @@
+target/
+artifacts/
+coverage/
+Cargo.lock
@@ -0,0 +1,52 @@
+[package]
+name = "tranquil-store-fuzz"
+version = "0.0.0"
+publish = false
+edition = "2024"
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+libfuzzer-sys = "0.4"
+arbitrary = { version = "1", features = ["derive"] }
+tranquil-store = { path = "..", features = ["test-harness"] }
+tempfile = "3"
+tokio = { version = "1", features = ["rt", "time", "macros", "sync"] }
+
+[[bin]]
+name = "decode_block_record"
+path = "fuzz_targets/decode_block_record.rs"
+test = false
+doc = false
+bench = false
+
+[[bin]]
+name = "decode_hint_record"
+path = "fuzz_targets/decode_hint_record.rs"
+test = false
+doc = false
+bench = false
+
+[[bin]]
+name = "segment_scan"
+path = "fuzz_targets/segment_scan.rs"
+test = false
+doc = false
+bench = false
+
+[[bin]]
+name = "metastore_key_codec"
+path = "fuzz_targets/metastore_key_codec.rs"
+test = false
+doc = false
+bench = false
+
+[[bin]]
+name = "gauntlet_micro"
+path = "fuzz_targets/gauntlet_micro.rs"
+test = false
+doc = false
+bench = false
+
+[workspace]
@@ -0,0 +1 @@
+
@@ -0,0 +1 @@
+BADX
@@ -0,0 +1 @@
+TQEV
@@ -0,0 +1,28 @@
+#![no_main]
+
+use std::path::Path;
+
+use libfuzzer_sys::fuzz_target;
+use tranquil_store::blockstore::{BlockOffset, decode_block_record};
+use tranquil_store::{FaultConfig, OpenOptions, SimulatedIO, StorageIO};
+
+fuzz_target!(|data: &[u8]| {
+    let sim = SimulatedIO::new(0, FaultConfig::none());
+    let opts = OpenOptions {
+        read: true,
+        write: true,
+        create: true,
+        truncate: false,
+    };
+    let fd = match sim.open(Path::new("/fuzz/block.tqb"), opts) {
+        Ok(fd) => fd,
+        Err(_) => return,
+    };
+    if !data.is_empty() {
+        let _ = sim.write_all_at(fd, 0, data);
+        let _ = sim.sync(fd);
+    }
+    let file_size = data.len() as u64;
+    let _ = decode_block_record(&sim, fd, BlockOffset::new(0), file_size);
+    let _ = sim.close(fd);
+});
@@ -0,0 +1,41 @@
+#![no_main]
+
+use std::path::Path;
+
+use libfuzzer_sys::fuzz_target;
+use tranquil_store::blockstore::{HintOffset, decode_hint_record};
+use tranquil_store::{FaultConfig, OpenOptions, SimulatedIO, StorageIO};
+
+fuzz_target!(|data: &[u8]| {
+    let sim = SimulatedIO::new(0, FaultConfig::none());
+    let opts = OpenOptions {
+        read: true,
+        write: true,
+        create: true,
+        truncate: false,
+    };
+    let fd = match sim.open(Path::new("/fuzz/hint.tqh"), opts) {
+        Ok(fd) => fd,
+        Err(_) => return,
+    };
+    if !data.is_empty() {
+        let _ = sim.write_all_at(fd, 0, data);
+        let _ = sim.sync(fd);
+    }
+    let file_size = data.len() as u64;
+    let cursor = std::cell::Cell::new(0u64);
+    std::iter::from_fn(|| {
+        if cursor.get() >= file_size {
+            return None;
+        }
+        match decode_hint_record(&sim, fd, HintOffset::new(cursor.get()), file_size) {
+            Ok(Some(_)) => {
+                cursor.set(cursor.get() + 64);
+                Some(())
+            }
+            _ => None,
+        }
+    })
+    .for_each(|()| {});
+    let _ = sim.close(fd);
+});
@@ -0,0 +1,109 @@
+#![no_main]
+
+use std::sync::OnceLock;
+
+use arbitrary::{Arbitrary, Unstructured};
+use libfuzzer_sys::fuzz_target;
+use tokio::runtime::Runtime;
+use tranquil_store::blockstore::GroupCommitConfig;
+use tranquil_store::gauntlet::{
+    CollectionName, DidSpaceSize, Gauntlet, GauntletConfig, InvariantSet, IoBackend, KeySpaceSize,
+    MaxFileSize, Op, OpCount, OpInterval, OpStream, OpWeights, RecordKey, RestartPolicy,
+    RetentionMaxSecs, RunLimits, Seed, ShardCount, SizeDistribution, StoreConfig, ValueBytes,
+    ValueSeed, WallMs, WorkloadModel, WriterConcurrency,
+};
+
+#[derive(Arbitrary, Debug)]
+enum FuzzOp {
+    Add { rkey: u8, value: u16 },
+    Delete { rkey: u8 },
+    Compact,
+    Checkpoint,
+    Read { rkey: u8 },
+    ReadBlock { value: u16 },
+}
+
+const COLLECTION: &str = "app.bsky.feed.post";
+const MAX_OPS: usize = 128;
+
+fn to_op(fuzz_op: FuzzOp) -> Op {
+    match fuzz_op {
+        FuzzOp::Add { rkey, value } => Op::AddRecord {
+            collection: CollectionName(COLLECTION.to_string()),
+            rkey: RecordKey(format!("k{rkey:03}")),
+            value_seed: ValueSeed(u32::from(value)),
+        },
+        FuzzOp::Delete { rkey } => Op::DeleteRecord {
+            collection: CollectionName(COLLECTION.to_string()),
+            rkey: RecordKey(format!("k{rkey:03}")),
+        },
+        FuzzOp::Compact => Op::Compact,
+        FuzzOp::Checkpoint => Op::Checkpoint,
+        FuzzOp::Read { rkey } => Op::ReadRecord {
+            collection: CollectionName(COLLECTION.to_string()),
+            rkey: RecordKey(format!("k{rkey:03}")),
+        },
+        FuzzOp::ReadBlock { value } => Op::ReadBlock {
+            value_seed: ValueSeed(u32::from(value)),
+        },
+    }
+}
+
+fn tiny_config() -> GauntletConfig {
+    GauntletConfig {
+        seed: Seed(0),
+        io: IoBackend::Real,
+        workload: WorkloadModel {
+            weights: OpWeights::default(),
+            size_distribution: SizeDistribution::Fixed(ValueBytes(64)),
+            collections: vec![CollectionName(COLLECTION.to_string())],
+            key_space: KeySpaceSize(256),
+            did_space: DidSpaceSize(8),
+            retention_max_secs: RetentionMaxSecs(3600),
+        },
+        op_count: OpCount(0),
+        invariants: InvariantSet::REFCOUNT_CONSERVATION
+            | InvariantSet::REACHABILITY
+            | InvariantSet::READ_AFTER_WRITE,
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(2_000)),
+        },
+        restart_policy: RestartPolicy::EveryNOps(OpInterval(32)),
+        store: StoreConfig {
+            max_file_size: MaxFileSize(4096),
+            group_commit: GroupCommitConfig::default(),
+            shard_count: ShardCount(1),
+        },
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn shared_runtime() -> &'static Runtime {
+    static RUNTIME: OnceLock<Runtime> = OnceLock::new();
+    RUNTIME.get_or_init(|| {
+        tokio::runtime::Builder::new_current_thread()
+            .enable_time()
+            .build()
+            .expect("build tokio runtime")
+    })
+}
+
+fuzz_target!(|data: &[u8]| {
+    if data.is_empty() {
+        return;
+    }
+    let mut u = Unstructured::new(data);
+    let ops: Vec<FuzzOp> = match Vec::<FuzzOp>::arbitrary(&mut u) {
+        Ok(ops) => ops.into_iter().take(MAX_OPS).collect(),
+        Err(_) => return,
+    };
+    if ops.is_empty() {
+        return;
+    }
+    let stream = OpStream::from_vec(ops.into_iter().map(to_op).collect());
+
+    let cfg = tiny_config();
+    let gauntlet = Gauntlet::new(cfg).expect("build gauntlet");
+    let _ = shared_runtime().block_on(gauntlet.run_with_ops(stream));
+});
@@ -0,0 +1,77 @@
+#![no_main]
+
+use arbitrary::Arbitrary;
+use libfuzzer_sys::fuzz_target;
+use tranquil_store::metastore::encoding::{KeyBuilder, KeyReader};
+
+#[derive(Arbitrary, Debug, PartialEq, Eq)]
+enum Field {
+    U64(u64),
+    I64(i64),
+    U32(u32),
+    U16(u16),
+    Bool(bool),
+    Bytes(Vec<u8>),
+    String(String),
+}
+
+fn append(builder: KeyBuilder, field: &Field) -> KeyBuilder {
+    match field {
+        Field::U64(v) => builder.u64(*v),
+        Field::I64(v) => builder.i64(*v),
+        Field::U32(v) => builder.u32(*v),
+        Field::U16(v) => builder.u16(*v),
+        Field::Bool(v) => builder.bool(*v),
+        Field::Bytes(v) => builder.bytes(v),
+        Field::String(v) => builder.string(v),
+    }
+}
+
+fn consume(reader: &mut KeyReader<'_>, field: &Field) -> bool {
+    match field {
+        Field::U64(v) => reader.u64() == Some(*v),
+        Field::I64(v) => reader.i64() == Some(*v),
+        Field::U32(v) => reader.u32() == Some(*v),
+        Field::U16(v) => reader.u16() == Some(*v),
+        Field::Bool(v) => reader.bool() == Some(*v),
+        Field::Bytes(v) => reader.bytes().as_deref() == Some(v.as_slice()),
+        Field::String(v) => reader.string().as_deref() == Some(v.as_str()),
+    }
+}
+
+#[derive(Arbitrary, Debug)]
+enum Mode {
+    Roundtrip(Vec<Field>),
+    Raw(Vec<u8>),
+}
+
+fuzz_target!(|mode: Mode| {
+    match mode {
+        Mode::Roundtrip(fields) => {
+            let encoded1 = fields.iter().fold(KeyBuilder::new(), append).build();
+
+            let mut reader = KeyReader::new(encoded1.as_slice());
+            let all_match = fields.iter().all(|f| consume(&mut reader, f));
+            assert!(all_match, "roundtrip decode failed");
+            assert!(reader.is_empty(), "trailing bytes after decode");
+
+            let encoded2 = fields.iter().fold(KeyBuilder::new(), append).build();
+            assert_eq!(
+                encoded1.as_slice(),
+                encoded2.as_slice(),
+                "encoding not deterministic",
+            );
+        }
+        Mode::Raw(data) => {
+            let mut reader = KeyReader::new(&data);
+            let _ = reader.u64();
+            let _ = reader.i64();
+            let _ = reader.u32();
+            let _ = reader.u16();
+            let _ = reader.bool();
+            let _ = reader.bytes();
+            let _ = reader.string();
+            let _ = reader.tag();
+        }
+    }
+});
@@ -0,0 +1,31 @@
+#![no_main]
+
+use std::path::Path;
+
+use libfuzzer_sys::fuzz_target;
+use tranquil_store::eventlog::SegmentReader;
+
+const FUZZ_MAX_PAYLOAD: u32 = 1 << 20;
+use tranquil_store::{FaultConfig, OpenOptions, SimulatedIO, StorageIO};
+
+fuzz_target!(|data: &[u8]| {
+    let sim = SimulatedIO::new(0, FaultConfig::none());
+    let opts = OpenOptions {
+        read: true,
+        write: true,
+        create: true,
+        truncate: false,
+    };
+    let fd = match sim.open(Path::new("/fuzz/segment.tqe"), opts) {
+        Ok(fd) => fd,
+        Err(_) => return,
+    };
+    if !data.is_empty() {
+        let _ = sim.write_all_at(fd, 0, data);
+        let _ = sim.sync(fd);
+    }
+    if let Ok(reader) = SegmentReader::open(&sim, fd, FUZZ_MAX_PAYLOAD) {
+        reader.for_each(|_result| {});
+    }
+    let _ = sim.close(fd);
+});
@@ -0,0 +1,47 @@
+use cid::Cid;
+use multihash::Multihash;
+use sha2::{Digest, Sha256};
+
+use super::data_file::CID_SIZE;
+
+pub const DAG_CBOR_CODEC: u64 = 0x71;
+pub const SHA2_256_CODE: u64 = 0x12;
+
+pub fn hash_to_cid(data: &[u8]) -> Cid {
+    let mut hasher = Sha256::new();
+    hasher.update(data);
+    let digest = hasher.finalize();
+    let mh = Multihash::wrap(SHA2_256_CODE, &digest)
+        .expect("SHA-256 digest is 32 bytes, well within multihash capacity");
+    Cid::new_v1(DAG_CBOR_CODEC, mh)
+}
+
+pub fn hash_to_cid_bytes(data: &[u8]) -> [u8; CID_SIZE] {
+    let raw = hash_to_cid(data).to_bytes();
+    raw.try_into()
+        .expect("CIDv1 + DAG-CBOR + SHA-256 always encodes to CID_SIZE bytes")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn hash_to_cid_bytes_is_deterministic() {
+        let a = hash_to_cid_bytes(b"hello");
+        let b = hash_to_cid_bytes(b"hello");
+        assert_eq!(a, b);
+    }
+
+    #[test]
+    fn hash_to_cid_bytes_diverges_on_single_byte_change() {
+        assert_ne!(hash_to_cid_bytes(b"abc"), hash_to_cid_bytes(b"abd"));
+    }
+
+    #[test]
+    fn hash_to_cid_and_bytes_agree() {
+        let cid = hash_to_cid(b"payload");
+        let raw: [u8; CID_SIZE] = cid.to_bytes().try_into().expect("36 bytes");
+        assert_eq!(raw, hash_to_cid_bytes(b"payload"));
+    }
+}
@@ -68,8 +68,8 @@ pub(super) fn compact_on_writer_thread<S: StorageIO>(
        return Err(CompactionError::ActiveFileCannotBeCompacted);
    }

-    let source_fd = manager.open_for_read(source_file_id)?;
-    let source_size = manager.io().file_size(source_fd)?;
+    let source_handle = manager.open_for_read(source_file_id)?;
+    let source_size = manager.io().file_size(source_handle.fd())?;

    let new_file_id = file_ids.allocate();

@@ -77,7 +77,7 @@ pub(super) fn compact_on_writer_thread<S: StorageIO>(
        manager,
        index,
        source_file_id,
-        source_fd,
+        source_handle.fd(),
        new_file_id,
        current_epoch,
        grace_period_ms,
@@ -93,8 +93,7 @@ pub(super) fn compact_on_writer_thread<S: StorageIO>(
            Err(e)
        }
        Ok((new_size, live_count, dead_count)) => {
-            let positions = hint_positions.snapshot();
-            if let Err(e) = index.write_checkpoint(epoch.current(), &positions) {
+            if let Err(e) = index.write_checkpoint(epoch.current(), hint_positions) {
                tracing::warn!(error = %e, "pre-delete checkpoint failed during compaction");
            }

@@ -103,6 +102,13 @@ pub(super) fn compact_on_writer_thread<S: StorageIO>(
                .io()
                .delete(&hint_file_path(manager.data_dir(), source_file_id))
                .ok();
+            if live_count == 0 {
+                manager.delete_data_file(new_file_id).ok();
+                manager
+                    .io()
+                    .delete(&hint_file_path(manager.data_dir(), new_file_id))
+                    .ok();
+            }
            manager.io().sync_dir(manager.data_dir())?;

            let reclaimed_bytes = source_size.saturating_sub(new_size);
@@ -142,8 +148,8 @@ fn stream_compact<S: StorageIO>(
    let mut reader = DataFileReader::open(manager.io(), source_fd)?;
    let now = crate::wall_clock_ms();

-    let new_fd = manager.open_for_append(new_file_id)?;
-    let mut writer = DataFileWriter::new(manager.io(), new_fd, new_file_id)?;
+    let new_handle = manager.open_for_append(new_file_id)?;
+    let mut writer = DataFileWriter::new(manager.io(), new_handle.fd(), new_file_id)?;

    let hint_path = hint_file_path(manager.data_dir(), new_file_id);
    let hint_fd = manager.io().open(&hint_path, OpenOptions::read_write())?;
@@ -154,7 +160,7 @@ fn stream_compact<S: StorageIO>(
    let mut live_count: u64 = 0;
    let mut dead_count: u64 = 0;

-    reader.try_for_each(|r| {
+    let scan_result = reader.try_for_each(|r| {
        let record = r?;
        match record {
            ReadBlockRecord::Valid {
@@ -162,7 +168,7 @@ fn stream_compact<S: StorageIO>(
            } => match index.get(&cid_bytes) {
                Some(e) if e.location.file_id == source_file_id && !e.refcount.is_zero() => {
                    let loc = writer.append_block(&cid_bytes, &data)?;
-                    hint_writer.append_relocate(&cid_bytes, loc.file_id, loc.offset, loc.length)?;
+                    hint_writer.append_relocate(&cid_bytes, &loc, e.refcount.raw())?;
                    relocations.push((cid_bytes, loc));
                    live_count = live_count.saturating_add(1);
                }
@@ -182,12 +188,7 @@ fn stream_compact<S: StorageIO>(
                        }
                        false => {
                            let loc = writer.append_block(&cid_bytes, &data)?;
-                            hint_writer.append_relocate(
-                                &cid_bytes,
-                                loc.file_id,
-                                loc.offset,
-                                loc.length,
-                            )?;
+                            hint_writer.append_relocate(&cid_bytes, &loc, e.refcount.raw())?;
                            relocations.push((cid_bytes, loc));
                            live_count = live_count.saturating_add(1);
                        }
@@ -198,11 +199,34 @@ fn stream_compact<S: StorageIO>(
            ReadBlockRecord::Corrupted { .. } | ReadBlockRecord::Truncated { .. } => {}
        }
        Ok::<_, CompactionError>(())
-    })?;
+    });

-    writer.sync()?;
-    hint_writer.sync()?;
-    manager.io().sync_dir(manager.data_dir())?;
+    let record_count =
+        u32::try_from((live_count as u128).saturating_add(dead_count as u128)).unwrap_or(u32::MAX);
+    let writer_position = writer.position();
+    let finalize_result = scan_result
+        .and_then(|()| writer.sync().map_err(CompactionError::from))
+        .and_then(|()| {
+            hint_writer
+                .append_commit_marker(
+                    current_epoch.raw(),
+                    record_count,
+                    new_file_id,
+                    writer_position,
+                )
+                .map_err(CompactionError::from)
+        })
+        .and_then(|()| hint_writer.sync().map_err(CompactionError::from))
+        .and_then(|()| {
+            manager
+                .io()
+                .sync_dir(manager.data_dir())
+                .map_err(CompactionError::from)
+        });
+
+    let _ = manager.io().close(hint_fd);
+
+    finalize_result?;

    let new_size = writer.position().raw();

@@ -11,7 +11,7 @@ use crate::fsync_order::PostBlockstoreHook;
 use super::BlocksSynced;
 use crate::io::{FileId, OpenOptions, StorageIO};

-use super::data_file::{CID_SIZE, DataFileWriter};
+use super::data_file::{CID_SIZE, DataFileWriter, ReadBlockRecord, decode_block_record};
 use super::hash_index::{BlockIndex, BlockIndexError, CheckpointPositions};
 use super::hint::{HintFileWriter, hint_file_path};
 use super::manager::DataFileManager;
@@ -106,6 +106,10 @@ pub enum CommitError {
    Io(Arc<io::Error>),
    Index(String),
    ChannelClosed,
+    VerifyFailed {
+        file_id: DataFileId,
+        offset: BlockOffset,
+    },
 }

 impl std::fmt::Display for CommitError {
@@ -114,6 +118,11 @@ impl std::fmt::Display for CommitError {
            Self::Io(e) => write!(f, "io: {}", e.as_ref()),
            Self::Index(e) => write!(f, "index: {e}"),
            Self::ChannelClosed => write!(f, "commit channel closed"),
+            Self::VerifyFailed { file_id, offset } => write!(
+                f,
+                "post-sync verify failed at {file_id}:{} (misdirected write or durable corruption)",
+                offset.raw()
+            ),
        }
    }
 }
@@ -122,7 +131,7 @@ impl std::error::Error for CommitError {
    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
        match self {
            Self::Io(e) => Some(e.as_ref()),
-            Self::Index(_) | Self::ChannelClosed => None,
+            Self::Index(_) | Self::ChannelClosed | Self::VerifyFailed { .. } => None,
        }
    }
 }
@@ -181,6 +190,7 @@ pub struct GroupCommitConfig {
    pub channel_capacity: usize,
    pub checkpoint_interval_ms: u64,
    pub checkpoint_write_threshold: u64,
+    pub verify_persisted_blocks: bool,
 }

 impl Default for GroupCommitConfig {
@@ -190,6 +200,7 @@ impl Default for GroupCommitConfig {
            channel_capacity: 4096,
            checkpoint_interval_ms: 60_000,
            checkpoint_write_threshold: 100_000,
+            verify_persisted_blocks: false,
        }
    }
 }
@@ -200,6 +211,7 @@ struct ShardContext {
    file_ids: Arc<FileIdAllocator>,
    active_files: Arc<ActiveFileSet>,
    hint_positions: Arc<ShardHintPositions>,
+    verify_persisted_blocks: bool,
 }

 struct ActiveState {
@@ -403,6 +415,7 @@ impl GroupCommitWriter {
                    file_ids: Arc::clone(&file_ids),
                    active_files: Arc::clone(&active_files),
                    hint_positions: Arc::clone(&hint_positions),
+                    verify_persisted_blocks: config.verify_persisted_blocks,
                };
                SingleShardWriter::spawn(
                    ctx,
@@ -540,7 +553,8 @@ fn initialize_active_state<S: StorageIO>(

    match cursor {
        Some(wc) => {
-            let fd = manager.open_for_append(wc.file_id)?;
+            let handle = manager.open_for_append(wc.file_id)?;
+            let fd = handle.fd();
            let file_size = manager.io().file_size(fd)?;

            if file_size < wc.offset.raw() {
@@ -550,6 +564,16 @@ fn initialize_active_state<S: StorageIO>(
                )));
            }

+            let header_end = super::data_file::BLOCK_HEADER_SIZE as u64;
+            let position = match file_size < header_end {
+                true => {
+                    let writer = DataFileWriter::new(manager.io(), fd, wc.file_id)?;
+                    writer.sync()?;
+                    writer.position()
+                }
+                false => BlockOffset::new(file_size),
+            };
+
            let hint_path = hint_file_path(data_dir, wc.file_id);
            let hint_fd = manager.io().open(&hint_path, OpenOptions::read_write())?;
            let hint_size = manager.io().file_size(hint_fd)?;
@@ -557,7 +581,7 @@ fn initialize_active_state<S: StorageIO>(
            Ok(ActiveState {
                file_id: wc.file_id,
                fd,
-                position: BlockOffset::new(file_size),
+                position,
                hint_fd,
                hint_position: HintOffset::new(hint_size),
            })
@@ -565,7 +589,8 @@ fn initialize_active_state<S: StorageIO>(
        None => {
            let file_id = file_ids.allocate();

-            let fd = manager.open_for_append(file_id)?;
+            let handle = manager.open_for_append(file_id)?;
+            let fd = handle.fd();
            let writer = DataFileWriter::new(manager.io(), fd, file_id)?;
            writer.sync()?;
            let position = writer.position();
@@ -813,8 +838,7 @@ fn maybe_checkpoint(
    if !elapsed && !threshold {
        return;
    }
-    let positions = hint_positions.snapshot();
-    match index.write_checkpoint(epoch.current(), &positions) {
+    match index.write_checkpoint(epoch.current(), hint_positions) {
        Ok(()) => {
            *last_checkpoint = std::time::Instant::now();
            *writes_since_checkpoint = 0;
@@ -831,8 +855,7 @@ fn shutdown_checkpoint(
    epoch: &EpochCounter,
    hint_positions: &ShardHintPositions,
 ) {
-    let positions = hint_positions.snapshot();
-    match index.write_checkpoint(epoch.current(), &positions) {
+    match index.write_checkpoint(epoch.current(), hint_positions) {
        Ok(()) => tracing::debug!("shutdown checkpoint written"),
        Err(e) => tracing::warn!(error = %e, "shutdown checkpoint failed"),
    }
@@ -924,8 +947,6 @@ fn commit_loop<S: StorageIO>(
            if let Ok((ref dedup, _)) = result {
                writes_since_checkpoint =
                    writes_since_checkpoint.saturating_add(dedup.len() as u64);
-                ctx.hint_positions
-                    .update(ctx.shard_id, state.file_id, state.hint_position);
            }

            dispatch_responses(drain.entries, result.map(|(dedup, _proof)| dedup));
@@ -1024,8 +1045,6 @@ fn drain_and_process_remaining<S: StorageIO>(

        if let Ok((ref _dedup, ref proof)) = result {
            run_post_sync_hook(post_sync_hook, proof);
-            ctx.hint_positions
-                .update(ctx.shard_id, state.file_id, state.hint_position);
        }

        dispatch_responses(entries, result.map(|(dedup, _proof)| dedup));
@@ -1057,9 +1076,124 @@ fn drain_and_process_remaining<S: StorageIO>(
    shutdown_checkpoint(index, epoch, &ctx.hint_positions);
 }

-struct RotationState {
+struct RotationState<S: StorageIO> {
    file_id: DataFileId,
+    handle: Arc<super::manager::CachedHandle<S>>,
+    hint_fd: FileId,
+}
+
+fn verify_persisted_blocks<S: StorageIO>(
+    manager: &DataFileManager<S>,
+    entries: &[([u8; CID_SIZE], BlockLocation)],
+) -> Result<(), CommitError> {
+    use std::collections::BTreeMap;
+    let by_file: BTreeMap<DataFileId, Vec<(&[u8; CID_SIZE], BlockLocation)>> =
+        entries.iter().fold(BTreeMap::new(), |mut acc, (cid, loc)| {
+            acc.entry(loc.file_id).or_default().push((cid, *loc));
+            acc
+        });
+
+    by_file.into_iter().try_for_each(|(file_id, locations)| {
+        let path = manager.data_file_path(file_id);
+        let fd = match manager.io().open(&path, OpenOptions::read_only_existing()) {
+            Ok(fd) => fd,
+            Err(_) => return Ok(()),
+        };
+        let file_size = match manager.io().file_size(fd) {
+            Ok(s) => s,
+            Err(_) => {
+                let _ = manager.io().close(fd);
+                return Ok(());
+            }
+        };
+        let result = locations.into_iter().try_for_each(|(expected_cid, loc)| {
+            verify_block_at(manager, fd, file_size, expected_cid, loc)
+        });
+        let _ = manager.io().close(fd);
+        result
+    })
+}
+
+#[derive(Debug)]
+enum VerifyOutcome {
+    NoFaultDetected,
+    Faulted,
+}
+
+fn verify_block_at<S: StorageIO>(
+    manager: &DataFileManager<S>,
    fd: FileId,
+    file_size: u64,
+    expected_cid: &[u8; CID_SIZE],
+    loc: BlockLocation,
+) -> Result<(), CommitError> {
+    let passed = (0..VERIFY_RETRY_ATTEMPTS).any(|_| {
+        matches!(
+            verify_once(manager, fd, file_size, expected_cid, loc),
+            VerifyOutcome::NoFaultDetected
+        )
+    });
+    match passed {
+        true => Ok(()),
+        false => Err(CommitError::VerifyFailed {
+            file_id: loc.file_id,
+            offset: loc.offset,
+        }),
+    }
+}
+
+fn verify_once<S: StorageIO>(
+    manager: &DataFileManager<S>,
+    fd: FileId,
+    file_size: u64,
+    expected_cid: &[u8; CID_SIZE],
+    loc: BlockLocation,
+) -> VerifyOutcome {
+    match decode_block_record(manager.io(), fd, loc.offset, file_size) {
+        Ok(Some(ReadBlockRecord::Valid { cid_bytes, .. })) if cid_bytes == *expected_cid => {
+            VerifyOutcome::NoFaultDetected
+        }
+        Ok(Some(ReadBlockRecord::Valid { .. })) => {
+            tracing::warn!(
+                file_id = %loc.file_id,
+                offset = loc.offset.raw(),
+                "verify: stored CID mismatch (misdirected write)"
+            );
+            VerifyOutcome::Faulted
+        }
+        Ok(Some(ReadBlockRecord::Corrupted { .. } | ReadBlockRecord::Truncated { .. }))
+        | Ok(None) => {
+            tracing::warn!(
+                file_id = %loc.file_id,
+                offset = loc.offset.raw(),
+                "verify: block undecodable at location"
+            );
+            VerifyOutcome::Faulted
+        }
+        Err(_) => VerifyOutcome::NoFaultDetected,
+    }
+}
+
+const VERIFY_RETRY_ATTEMPTS: u32 = 4;
+
+fn rollback_batch<S: StorageIO>(
+    manager: &DataFileManager<S>,
+    state: &ActiveState,
+    rotations: &[RotationState<S>],
+) {
+    let _ = manager.io().truncate(state.fd, state.position.raw());
+    let _ = manager.io().sync(state.fd);
+    let _ = manager
+        .io()
+        .truncate(state.hint_fd, state.hint_position.raw());
+    let _ = manager.io().sync(state.hint_fd);
+    rotations.iter().for_each(|rot| {
+        manager.rollback_rotation(rot.file_id);
+        let _ = manager.io().close(rot.hint_fd);
+        let _ = manager
+            .io()
+            .delete(&hint_file_path(manager.data_dir(), rot.file_id));
+    });
 }

 fn process_batch<S: StorageIO>(
@@ -1077,13 +1211,46 @@ fn process_batch<S: StorageIO>(
    let mut all_decrements: Vec<[u8; CID_SIZE]> = Vec::new();

    let mut current_hint_fd = state.hint_fd;
-    let mut rotation: Option<RotationState> = None;
+    let mut rotations: Vec<RotationState<S>> = Vec::new();

    let mut data_writer =
        DataFileWriter::resume(manager.io(), state.fd, state.file_id, state.position);
    let mut hint_writer =
        HintFileWriter::resume(manager.io(), current_hint_fd, state.hint_position);

+    if manager.should_rotate(data_writer.position()) {
+        data_writer.sync().map_err(CommitError::from)?;
+        hint_writer.sync().map_err(CommitError::from)?;
+
+        let next_id = ctx.file_ids.allocate();
+        let next_handle = manager.open_for_append(next_id)?;
+        let next_fd = next_handle.fd();
+
+        tracing::info!(
+            from = %data_writer.file_id(),
+            to = %next_id,
+            trigger = "batch_boundary",
+            "data file rotation"
+        );
+
+        data_writer = DataFileWriter::new(manager.io(), next_fd, next_id)?;
+
+        let new_hint_path = hint_file_path(manager.data_dir(), next_id);
+        let new_hint_fd = manager
+            .io()
+            .open(&new_hint_path, OpenOptions::read_write())?;
+
+        manager.io().sync_dir(manager.data_dir())?;
+
+        current_hint_fd = new_hint_fd;
+        hint_writer = HintFileWriter::new(manager.io(), new_hint_fd);
+        rotations.push(RotationState {
+            file_id: next_id,
+            handle: next_handle,
+            hint_fd: new_hint_fd,
+        });
+    }
+
    let mut block_bytes: u64 = 0;
    let mut block_count: u64 = 0;
    let mut dedup_hits: u64 = 0;
@@ -1102,47 +1269,27 @@ fn process_batch<S: StorageIO>(
            let location = match dedup.get(cid_bytes) {
                Some(&loc) => {
                    dedup_hits = dedup_hits.saturating_add(1);
+                    hint_writer.append_hint(cid_bytes, &loc)?;
                    loc
                }
-                None => {
-                    if manager.should_rotate(data_writer.position()) {
-                        data_writer.sync()?;
-                        hint_writer.sync()?;
-
-                        let next_id = ctx.file_ids.allocate();
-                        let next_fd = manager.open_for_append(next_id)?;
-
-                        tracing::info!(
-                            from = %data_writer.file_id(),
-                            to = %next_id,
-                            "data file rotation"
-                        );
-
-                        data_writer = DataFileWriter::new(manager.io(), next_fd, next_id)?;
-
-                        let new_hint_path = hint_file_path(manager.data_dir(), next_id);
-                        let new_hint_fd = manager
-                            .io()
-                            .open(&new_hint_path, OpenOptions::read_write())?;
-
-                        manager.io().sync_dir(manager.data_dir())?;
-
-                        current_hint_fd = new_hint_fd;
-                        hint_writer = HintFileWriter::new(manager.io(), new_hint_fd);
-                        rotation = Some(RotationState {
-                            file_id: next_id,
-                            fd: next_fd,
-                        });
+                None => match index.get(cid_bytes) {
+                    Some(existing) => {
+                        dedup_hits = dedup_hits.saturating_add(1);
+                        let loc = existing.location;
+                        hint_writer.append_hint(cid_bytes, &loc)?;
+                        dedup.insert(*cid_bytes, loc);
+                        loc
                    }
+                    None => {
+                        let loc = data_writer.append_block(cid_bytes, data)?;
+                        hint_writer.append_hint(cid_bytes, &loc)?;

-                    let loc = data_writer.append_block(cid_bytes, data)?;
-                    hint_writer.append_hint(cid_bytes, loc.file_id, loc.offset, loc.length)?;
-
-                    block_bytes = block_bytes.saturating_add(data.len() as u64);
-                    block_count = block_count.saturating_add(1);
-                    dedup.insert(*cid_bytes, loc);
-                    loc
-                }
+                        block_bytes = block_bytes.saturating_add(data.len() as u64);
+                        block_count = block_count.saturating_add(1);
+                        dedup.insert(*cid_bytes, loc);
+                        loc
+                    }
+                },
            };

            index_entries.push((*cid_bytes, location));
@@ -1157,9 +1304,7 @@ fn process_batch<S: StorageIO>(
    });

    if let Err(e) = write_result {
-        if let Some(rot) = rotation {
-            manager.rollback_rotation(rot.file_id, rot.fd);
-        }
+        rollback_batch(manager, state, &rotations);
        return Err(e);
    }

@@ -1168,18 +1313,53 @@ fn process_batch<S: StorageIO>(
    let current_epoch = epoch.current();
    let now = crate::wall_clock_ms();

+    let rollback_on_err = |e: CommitError| -> CommitError {
+        rollback_batch(manager, state, &rotations);
+        e
+    };
+
    all_decrements
        .iter()
-        .try_for_each(|cid| hint_writer.append_decrement(cid, current_epoch, now))?;
+        .try_for_each(|cid| hint_writer.append_decrement(cid, current_epoch, now))
+        .map_err(|e| rollback_on_err(CommitError::from(e)))?;

    let t = std::time::Instant::now();
-    data_writer.sync()?;
-    hint_writer.sync()?;
+    data_writer.sync().map_err(|e| rollback_on_err(e.into()))?;
+    if ctx.verify_persisted_blocks {
+        verify_persisted_blocks(manager, &index_entries).map_err(rollback_on_err)?;
+    }
+    let batch_record_count = u32::try_from(
+        block_count
+            .saturating_add(dedup_hits)
+            .saturating_add(all_decrements.len() as u64),
+    )
+    .unwrap_or(u32::MAX);
+    hint_writer
+        .append_commit_marker(
+            current_epoch.raw(),
+            batch_record_count,
+            data_writer.file_id(),
+            data_writer.position(),
+        )
+        .map_err(|e| rollback_on_err(CommitError::from(e)))?;
+    hint_writer.sync().map_err(|e| rollback_on_err(e.into()))?;
    let sync_nanos = t.elapsed().as_nanos() as u64;

-    if let Some(ref rot) = rotation {
-        manager.commit_rotation(rot.file_id, rot.fd);
-        ctx.active_files.register(ctx.shard_id, rot.file_id);
+    if !rotations.is_empty() {
+        let old_file_id = state.file_id;
+        let old_hint_fd = state.hint_fd;
+        let last_idx = rotations.len() - 1;
+        rotations.iter().enumerate().for_each(|(i, rot)| {
+            if i == last_idx {
+                manager.commit_rotation(rot.file_id, &rot.handle);
+                ctx.active_files.register(ctx.shard_id, rot.file_id);
+            } else {
+                let _ = manager.io().close(rot.hint_fd);
+                manager.evict_handle(rot.file_id);
+            }
+        });
+        manager.evict_handle(old_file_id);
+        let _ = manager.io().close(old_hint_fd);
    }

    state.file_id = data_writer.file_id();
@@ -1194,7 +1374,19 @@ fn process_batch<S: StorageIO>(
    };
    let t = std::time::Instant::now();
    index
-        .batch_put(&index_entries, &all_decrements, cursor, current_epoch, now)
+        .batch_put_and_advance_position(
+            &index_entries,
+            &all_decrements,
+            cursor,
+            current_epoch,
+            now,
+            super::hash_index::PositionUpdate {
+                hint_positions: &ctx.hint_positions,
+                shard_id: ctx.shard_id,
+                file_id: state.file_id,
+                offset: state.hint_position,
+            },
+        )
        .map_err(CommitError::from)?;
    let index_nanos = t.elapsed().as_nanos() as u64;

@@ -5,11 +5,19 @@ use std::path::{Path, PathBuf};
 use parking_lot::RwLock;

 use super::data_file::CID_SIZE;
+use super::group_commit::ShardHintPositions;
 use super::types::{
    BlockLength, BlockLocation, BlockOffset, CidBytes, CollectionResult, CommitEpoch, DataFileId,
-    HintOffset, IndexEntry, LivenessInfo, RefCount, WallClockMs, WriteCursor,
+    HintOffset, IndexEntry, LivenessInfo, RefCount, ShardId, WallClockMs, WriteCursor,
 };

+pub struct PositionUpdate<'a> {
+    pub hint_positions: &'a ShardHintPositions,
+    pub shard_id: ShardId,
+    pub file_id: DataFileId,
+    pub offset: HintOffset,
+}
+
 const EMPTY_CID: [u8; CID_SIZE] = [0u8; CID_SIZE];

 fn is_empty(cid: &[u8; CID_SIZE]) -> bool {
@@ -207,6 +215,10 @@ impl HashTable {
        self.get(cid).is_some()
    }

+    pub fn contains_live(&self, cid: &[u8; CID_SIZE]) -> bool {
+        self.get(cid).is_some_and(|s| !s.refcount.is_zero())
+    }
+
    pub fn insert(&mut self, new_slot: Slot) -> Result<Option<Slot>, CapacityExhausted> {
        if is_empty(&new_slot.cid) {
            tracing::error!("attempted to insert all-zero CID into hash table");
@@ -378,6 +390,7 @@ impl HashTable {
        &mut self,
        cid: &[u8; CID_SIZE],
        new_location: BlockLocation,
+        refcount: RefCount,
    ) -> Result<bool, CapacityExhausted> {
        if is_empty(cid) {
            return Ok(false);
@@ -394,7 +407,9 @@ impl HashTable {
            let slot_cid = self.slots[idx].cid;

            if is_empty(&slot_cid) {
-                self.slots[idx] = Slot::from_location(*cid, new_location);
+                let mut slot = Slot::from_location(*cid, new_location);
+                slot.refcount = refcount;
+                self.slots[idx] = slot;
                self.count += 1;
                return Ok(false);
            }
@@ -411,6 +426,7 @@ impl HashTable {
            let slot_dist = self.probe_distance(idx, slot_home);
            if slot_dist < dist {
                let mut displaced = Slot::from_location(*cid, new_location);
+                displaced.refcount = refcount;
                std::mem::swap(&mut self.slots[idx], &mut displaced);
                self.count += 1;
                self.relocate_displaced(displaced, idx, slot_dist);
@@ -571,7 +587,7 @@ impl HashTable {
        removals: &[CidBytes],
    ) {
        relocations.iter().for_each(|(cid, new_loc)| {
-            if let Err(e) = self.relocate(cid, *new_loc) {
+            if let Err(e) = self.relocate(cid, *new_loc, RefCount::one()) {
                tracing::error!(?e, "capacity exhausted during compaction relocation");
            }
        });
@@ -688,6 +704,7 @@ impl HashTable {
 const CHECKPOINT_MAGIC: [u8; 8] = *b"TQCKPT01";
 const CHECKPOINT_VERSION_V1: u32 = 1;
 const CHECKPOINT_VERSION_V2: u32 = 2;
+const CHECKPOINT_VERSION_V3: u32 = 3;
 const CHECKPOINT_HEADER_SIZE: usize = 128;
 const TRAILER_MAGIC: u64 = 0xDEAD_BEEF_CAFE_F00D;
 const SLOT_SIZE: usize = std::mem::size_of::<Slot>();
@@ -717,6 +734,7 @@ const H_CHECKPOINT_EPOCH: usize = 56;
 const H_HINT_FILE_ID: usize = 64;
 const H_HINT_OFFSET: usize = 72;
 const H_HEADER_CHECKSUM: usize = 80;
+const H_GENERATION: usize = 88;

 fn header_checksum(buf: &[u8; CHECKPOINT_HEADER_SIZE]) -> u64 {
    xxhash_rust::xxh3::xxh3_64(&buf[..H_HEADER_CHECKSUM])
@@ -729,10 +747,11 @@ fn serialize_header(
    cursor_offset: u64,
    checkpoint_epoch: u64,
    shard_count: u16,
+    generation: u64,
 ) -> [u8; CHECKPOINT_HEADER_SIZE] {
    let mut buf = [0u8; CHECKPOINT_HEADER_SIZE];
    buf[H_MAGIC..H_MAGIC + 8].copy_from_slice(&CHECKPOINT_MAGIC);
-    buf[H_VERSION..H_VERSION + 4].copy_from_slice(&CHECKPOINT_VERSION_V2.to_le_bytes());
+    buf[H_VERSION..H_VERSION + 4].copy_from_slice(&CHECKPOINT_VERSION_V3.to_le_bytes());
    buf[H_SHARD_COUNT..H_SHARD_COUNT + 2].copy_from_slice(&shard_count.to_le_bytes());
    buf[H_SLOT_COUNT..H_SLOT_COUNT + 8].copy_from_slice(&slot_count.to_le_bytes());
    buf[H_ENTRY_COUNT..H_ENTRY_COUNT + 8].copy_from_slice(&entry_count.to_le_bytes());
@@ -740,6 +759,7 @@ fn serialize_header(
    buf[H_CURSOR_OFFSET..H_CURSOR_OFFSET + 8].copy_from_slice(&cursor_offset.to_le_bytes());
    buf[H_CHECKPOINT_EPOCH..H_CHECKPOINT_EPOCH + 8]
        .copy_from_slice(&checkpoint_epoch.to_le_bytes());
+    buf[H_GENERATION..H_GENERATION + 8].copy_from_slice(&generation.to_le_bytes());
    let checksum = header_checksum(&buf);
    buf[H_HEADER_CHECKSUM..H_HEADER_CHECKSUM + 8].copy_from_slice(&checksum.to_le_bytes());
    buf
@@ -780,6 +800,7 @@ pub fn write_checkpoint(
    table: &HashTable,
    path: &Path,
    epoch: CommitEpoch,
+    generation: u64,
    positions: &CheckpointPositions,
 ) -> io::Result<()> {
    use std::io::Write;
@@ -801,6 +822,7 @@ pub fn write_checkpoint(
        cursor_offset,
        epoch.raw(),
        shard_count,
+        generation,
    );

    let slot_bytes = slots_as_bytes(&table.slots);
@@ -828,7 +850,7 @@ pub fn write_checkpoint(
    Ok(())
 }

-fn parse_checkpoint_header(data: &[u8]) -> io::Result<(usize, usize, u32, u64, u64, u16)> {
+fn parse_checkpoint_header(data: &[u8]) -> io::Result<(usize, usize, u32, u64, u64, u16, u64)> {
    if data.len() < CHECKPOINT_HEADER_SIZE + 16 {
        return Err(io::Error::new(
            io::ErrorKind::InvalidData,
@@ -847,7 +869,10 @@ fn parse_checkpoint_header(data: &[u8]) -> io::Result<(usize, usize, u32, u64, u
    }

    let version = u32::from_le_bytes(hdr[H_VERSION..H_VERSION + 4].try_into().unwrap());
-    if version != CHECKPOINT_VERSION_V1 && version != CHECKPOINT_VERSION_V2 {
+    if version != CHECKPOINT_VERSION_V1
+        && version != CHECKPOINT_VERSION_V2
+        && version != CHECKPOINT_VERSION_V3
+    {
        return Err(io::Error::new(
            io::ErrorKind::InvalidData,
            format!("checkpoint version {version} unsupported"),
@@ -905,12 +930,19 @@ fn parse_checkpoint_header(data: &[u8]) -> io::Result<(usize, usize, u32, u64, u
    );

    let shard_count = match version {
-        CHECKPOINT_VERSION_V2 => {
+        CHECKPOINT_VERSION_V2 | CHECKPOINT_VERSION_V3 => {
            u16::from_le_bytes(hdr[H_SHARD_COUNT..H_SHARD_COUNT + 2].try_into().unwrap())
        }
        _ => 0,
    };

+    let generation = match version {
+        CHECKPOINT_VERSION_V3 => {
+            u64::from_le_bytes(hdr[H_GENERATION..H_GENERATION + 8].try_into().unwrap())
+        }
+        _ => 0,
+    };
+
    Ok((
        slot_count,
        entry_count,
@@ -918,6 +950,7 @@ fn parse_checkpoint_header(data: &[u8]) -> io::Result<(usize, usize, u32, u64, u
        cursor_offset,
        checkpoint_epoch,
        shard_count,
+        generation,
    ))
 }

@@ -932,11 +965,20 @@ fn deserialize_shard_positions(data: &[u8], count: usize) -> Vec<(DataFileId, Hi
        .collect()
 }

-pub fn read_checkpoint(path: &Path) -> io::Result<(HashTable, CommitEpoch, CheckpointPositions)> {
+pub fn read_checkpoint(
+    path: &Path,
+) -> io::Result<(HashTable, CommitEpoch, CheckpointPositions, u64)> {
    let data = std::fs::read(path)?;

-    let (slot_count, entry_count, cursor_file_id, cursor_offset, checkpoint_epoch, shard_count) =
-        parse_checkpoint_header(&data)?;
+    let (
+        slot_count,
+        entry_count,
+        cursor_file_id,
+        cursor_offset,
+        checkpoint_epoch,
+        shard_count,
+        generation,
+    ) = parse_checkpoint_header(&data)?;

    let hdr: &[u8; CHECKPOINT_HEADER_SIZE] = data[..CHECKPOINT_HEADER_SIZE].try_into().unwrap();
    let version = u32::from_le_bytes(hdr[H_VERSION..H_VERSION + 4].try_into().unwrap());
@@ -956,7 +998,7 @@ pub fn read_checkpoint(path: &Path) -> io::Result<(HashTable, CommitEpoch, Check
    let shard_pos_region = &data[shard_pos_start..shard_pos_start + shard_pos_size];

    let data_checksum = match version {
-        CHECKPOINT_VERSION_V2 => {
+        CHECKPOINT_VERSION_V2 | CHECKPOINT_VERSION_V3 => {
            let mut hasher = xxhash_rust::xxh3::Xxh3::new();
            hasher.update(slot_region);
            hasher.update(shard_pos_region);
@@ -1019,7 +1061,7 @@ pub fn read_checkpoint(path: &Path) -> io::Result<(HashTable, CommitEpoch, Check
    let epoch = CommitEpoch::new(checkpoint_epoch);

    let positions = match version {
-        CHECKPOINT_VERSION_V2 if shard_count > 0 => CheckpointPositions(
+        CHECKPOINT_VERSION_V2 | CHECKPOINT_VERSION_V3 if shard_count > 0 => CheckpointPositions(
            deserialize_shard_positions(shard_pos_region, shard_count as usize),
        ),
        _ => {
@@ -1031,12 +1073,12 @@ pub fn read_checkpoint(path: &Path) -> io::Result<(HashTable, CommitEpoch, Check
        }
    };

-    Ok((table, epoch, positions))
+    Ok((table, epoch, positions, generation))
 }

 pub fn load_best_checkpoint(
    index_dir: &Path,
-) -> Option<(HashTable, CommitEpoch, CheckpointPositions)> {
+) -> Option<(HashTable, CommitEpoch, CheckpointPositions, u64)> {
    let path_a = index_dir.join("checkpoint_a.tqc");
    let path_b = index_dir.join("checkpoint_b.tqc");

@@ -1044,7 +1086,7 @@ pub fn load_best_checkpoint(
    let result_b = read_checkpoint(&path_b).ok();

    match (result_a, result_b) {
-        (Some(a), Some(b)) => match a.1.raw() >= b.1.raw() {
+        (Some(a), Some(b)) => match (a.3, a.1.raw()) >= (b.3, b.1.raw()) {
            true => Some(a),
            false => Some(b),
        },
@@ -1054,7 +1096,7 @@ pub fn load_best_checkpoint(
    }
 }

-fn read_checkpoint_epoch(path: &Path) -> Option<u64> {
+fn read_checkpoint_meta(path: &Path) -> Option<(u64, u64)> {
    let mut file = std::fs::File::open(path).ok()?;
    let mut buf = [0u8; CHECKPOINT_HEADER_SIZE];
    std::io::Read::read_exact(&mut file, &mut buf).ok()?;
@@ -1065,7 +1107,10 @@ fn read_checkpoint_epoch(path: &Path) -> Option<u64> {
    }

    let version = u32::from_le_bytes(buf[H_VERSION..H_VERSION + 4].try_into().ok()?);
-    if version != CHECKPOINT_VERSION_V1 && version != CHECKPOINT_VERSION_V2 {
+    if version != CHECKPOINT_VERSION_V1
+        && version != CHECKPOINT_VERSION_V2
+        && version != CHECKPOINT_VERSION_V3
+    {
        return None;
    }

@@ -1078,33 +1123,41 @@ fn read_checkpoint_epoch(path: &Path) -> Option<u64> {
        return None;
    }

-    Some(u64::from_le_bytes(
+    let epoch = u64::from_le_bytes(
        buf[H_CHECKPOINT_EPOCH..H_CHECKPOINT_EPOCH + 8]
            .try_into()
            .ok()?,
-    ))
+    );
+    let generation = match version {
+        CHECKPOINT_VERSION_V3 => {
+            u64::from_le_bytes(buf[H_GENERATION..H_GENERATION + 8].try_into().ok()?)
+        }
+        _ => 0,
+    };
+    Some((epoch, generation))
 }

 pub fn write_checkpoint_ab(
    table: &HashTable,
    index_dir: &Path,
    epoch: CommitEpoch,
+    generation: u64,
    positions: &CheckpointPositions,
 ) -> io::Result<()> {
    let path_a = index_dir.join("checkpoint_a.tqc");
    let path_b = index_dir.join("checkpoint_b.tqc");

-    let epoch_a = read_checkpoint_epoch(&path_a);
-    let epoch_b = read_checkpoint_epoch(&path_b);
+    let meta_a = read_checkpoint_meta(&path_a);
+    let meta_b = read_checkpoint_meta(&path_b);

-    let target_path = match (epoch_a, epoch_b) {
-        (Some(a), Some(b)) if a >= b => path_b,
+    let target_path = match (meta_a, meta_b) {
+        (Some(a), Some(b)) if (a.1, a.0) >= (b.1, b.0) => path_b,
        (Some(_), Some(_)) => path_a,
        (Some(_), None) => path_b,
        (None, _) => path_a,
    };

-    write_checkpoint(table, &target_path, epoch, positions)
+    write_checkpoint(table, &target_path, epoch, generation, positions)
 }

 #[derive(Debug)]
@@ -1130,6 +1183,7 @@ pub struct BlockIndex {
    checkpoint_lock: parking_lot::Mutex<()>,
    loaded_checkpoint_positions: Option<CheckpointPositions>,
    loaded_checkpoint_epoch: Option<CommitEpoch>,
+    next_generation: std::sync::atomic::AtomicU64,
 }

 impl BlockIndex {
@@ -1140,33 +1194,36 @@ impl BlockIndex {
            checkpoint_lock: parking_lot::Mutex::new(()),
            loaded_checkpoint_positions: None,
            loaded_checkpoint_epoch: None,
+            next_generation: std::sync::atomic::AtomicU64::new(1),
        }
    }

    pub fn open(index_dir: &Path) -> io::Result<Self> {
        std::fs::create_dir_all(index_dir)?;
-        let (table, checkpoint_positions, checkpoint_epoch) = match load_best_checkpoint(index_dir)
-        {
-            Some((table, epoch, positions)) => {
-                tracing::info!(
-                    blocks = table.len(),
-                    epoch = epoch.raw(),
-                    shard_positions = positions.0.len(),
-                    "loaded block index from checkpoint"
-                );
-                (table, Some(positions), Some(epoch))
-            }
-            None => {
-                tracing::info!("no valid checkpoint found, starting with empty index");
-                (HashTable::with_capacity(64), None, None)
-            }
-        };
+        let (table, checkpoint_positions, checkpoint_epoch, loaded_generation) =
+            match load_best_checkpoint(index_dir) {
+                Some((table, epoch, positions, gen_value)) => {
+                    tracing::info!(
+                        blocks = table.len(),
+                        epoch = epoch.raw(),
+                        shard_positions = positions.0.len(),
+                        generation = gen_value,
+                        "loaded block index from checkpoint"
+                    );
+                    (table, Some(positions), Some(epoch), gen_value)
+                }
+                None => {
+                    tracing::info!("no valid checkpoint found, starting with empty index");
+                    (HashTable::with_capacity(64), None, None, 0)
+                }
+            };
        Ok(Self {
            table: RwLock::new(table),
            index_dir: index_dir.to_path_buf(),
            checkpoint_lock: parking_lot::Mutex::new(()),
            loaded_checkpoint_positions: checkpoint_positions,
            loaded_checkpoint_epoch: checkpoint_epoch,
+            next_generation: std::sync::atomic::AtomicU64::new(loaded_generation + 1),
        })
    }

@@ -1183,7 +1240,16 @@ impl BlockIndex {
    }

    pub fn has(&self, cid: &[u8; CID_SIZE]) -> bool {
-        self.table.read().contains(cid)
+        self.table.read().contains_live(cid)
+    }
+
+    pub fn live_entries_snapshot(&self) -> Vec<([u8; CID_SIZE], RefCount)> {
+        self.table
+            .read()
+            .iter()
+            .filter(|s| !s.refcount.is_zero())
+            .map(|s| (s.cid, s.refcount))
+            .collect()
    }

    pub fn batch_put(
@@ -1193,6 +1259,37 @@ impl BlockIndex {
        cursor: WriteCursor,
        epoch: CommitEpoch,
        now: WallClockMs,
+    ) -> Result<(), BlockIndexError> {
+        self.batch_put_inner(entries, decrements, cursor, epoch, now, None)
+    }
+
+    pub fn batch_put_and_advance_position(
+        &self,
+        entries: &[([u8; CID_SIZE], BlockLocation)],
+        decrements: &[[u8; CID_SIZE]],
+        cursor: WriteCursor,
+        epoch: CommitEpoch,
+        now: WallClockMs,
+        position_update: PositionUpdate<'_>,
+    ) -> Result<(), BlockIndexError> {
+        self.batch_put_inner(
+            entries,
+            decrements,
+            cursor,
+            epoch,
+            now,
+            Some(position_update),
+        )
+    }
+
+    fn batch_put_inner(
+        &self,
+        entries: &[([u8; CID_SIZE], BlockLocation)],
+        decrements: &[[u8; CID_SIZE]],
+        cursor: WriteCursor,
+        epoch: CommitEpoch,
+        now: WallClockMs,
+        position_update: Option<PositionUpdate<'_>>,
    ) -> Result<(), BlockIndexError> {
        let mut table = self.table.write();

@@ -1213,6 +1310,12 @@ impl BlockIndex {
        });

        table.set_write_cursor(cursor);
+
+        if let Some(pos) = position_update {
+            pos.hint_positions
+                .update(pos.shard_id, pos.file_id, pos.offset);
+        }
+
        Ok(())
    }

@@ -1267,18 +1370,20 @@ impl BlockIndex {

    pub fn batch_relocate(
        &self,
-        relocations: &[(CidBytes, BlockLocation)],
+        relocations: &[(CidBytes, BlockLocation, u32)],
    ) -> Result<(), BlockIndexError> {
        if relocations.is_empty() {
            return Ok(());
        }
        let mut table = self.table.write();
-        relocations.iter().try_for_each(|(cid, location)| {
-            table
-                .relocate(cid, *location)
-                .map(|_| ())
-                .map_err(|_| BlockIndexError::CapacityExhausted)
-        })
+        relocations
+            .iter()
+            .try_for_each(|(cid, location, refcount)| {
+                table
+                    .relocate(cid, *location, RefCount::new(*refcount))
+                    .map(|_| ())
+                    .map_err(|_| BlockIndexError::CapacityExhausted)
+            })
    }

    pub fn batch_remove(&self, cids: &[CidBytes]) {
@@ -1412,13 +1517,30 @@ impl BlockIndex {
    }

    pub fn write_checkpoint(
+        &self,
+        epoch: CommitEpoch,
+        hint_positions: &ShardHintPositions,
+    ) -> io::Result<()> {
+        let _guard = self.checkpoint_lock.lock();
+        let generation = self
+            .next_generation
+            .fetch_add(1, std::sync::atomic::Ordering::AcqRel);
+        let table = self.table.read();
+        let positions = hint_positions.snapshot();
+        write_checkpoint_ab(&table, &self.index_dir, epoch, generation, &positions)
+    }
+
+    pub fn write_checkpoint_with_positions(
        &self,
        epoch: CommitEpoch,
        positions: &CheckpointPositions,
    ) -> io::Result<()> {
        let _guard = self.checkpoint_lock.lock();
+        let generation = self
+            .next_generation
+            .fetch_add(1, std::sync::atomic::Ordering::AcqRel);
        let table = self.table.read();
-        write_checkpoint_ab(&table, &self.index_dir, epoch, positions)
+        write_checkpoint_ab(&table, &self.index_dir, epoch, generation, positions)
    }

    pub fn index_dir(&self) -> &Path {
@@ -1499,20 +1621,24 @@ impl BlockIndex {
                        file_id,
                        offset,
                        length,
+                        refcount,
                    } => {
                        let loc = BlockLocation {
                            file_id,
                            offset,
                            length,
                        };
-                        table.relocate(&cid_bytes, loc).map_err(|_| {
-                            io::Error::other("hash table capacity exhausted during rebuild")
-                        })?;
+                        table
+                            .relocate(&cid_bytes, loc, RefCount::new(refcount))
+                            .map_err(|_| {
+                                io::Error::other("hash table capacity exhausted during rebuild")
+                            })?;
                    }
                    ReadHintRecord::Remove { cid_bytes } => {
                        let _ = table.remove(&cid_bytes);
                    }
-                    ReadHintRecord::UnknownVersion { .. }
+                    ReadHintRecord::CommitMarker { .. }
+                    | ReadHintRecord::UnknownVersion { .. }
                    | ReadHintRecord::UnknownType { .. }
                    | ReadHintRecord::Corrupted
                    | ReadHintRecord::Truncated => {}
@@ -1967,8 +2093,8 @@ mod tests {
        let epoch = CommitEpoch::new(42);
        let positions = CheckpointPositions::single(DataFileId::new(5), HintOffset::new(12345));

-        write_checkpoint(&table, &path, epoch, &positions).unwrap();
-        let (restored, restored_epoch, restored_pos) = read_checkpoint(&path).unwrap();
+        write_checkpoint(&table, &path, epoch, 7, &positions).unwrap();
+        let (restored, restored_epoch, restored_pos, _gen) = read_checkpoint(&path).unwrap();

        assert_eq!(restored.len(), 10);
        assert_eq!(restored_epoch.raw(), 42);
@@ -2000,14 +2126,14 @@ mod tests {
        table
            .insert_or_increment(&test_cid(1), test_loc(0, 0, 10))
            .unwrap();
-        write_checkpoint_ab(&table, dir.path(), CommitEpoch::new(1), &pos).unwrap();
+        write_checkpoint_ab(&table, dir.path(), CommitEpoch::new(1), 1, &pos).unwrap();

        table
            .insert_or_increment(&test_cid(2), test_loc(0, 100, 10))
            .unwrap();
-        write_checkpoint_ab(&table, dir.path(), CommitEpoch::new(2), &pos).unwrap();
+        write_checkpoint_ab(&table, dir.path(), CommitEpoch::new(2), 2, &pos).unwrap();

-        let (best, epoch, _) = load_best_checkpoint(dir.path()).unwrap();
+        let (best, epoch, _, _) = load_best_checkpoint(dir.path()).unwrap();
        assert_eq!(epoch.raw(), 2);
        assert_eq!(best.len(), 2);
    }
@@ -2021,16 +2147,16 @@ mod tests {
        table
            .insert_or_increment(&test_cid(1), test_loc(0, 0, 10))
            .unwrap();
-        write_checkpoint_ab(&table, dir.path(), CommitEpoch::new(1), &pos).unwrap();
+        write_checkpoint_ab(&table, dir.path(), CommitEpoch::new(1), 1, &pos).unwrap();

        table
            .insert_or_increment(&test_cid(2), test_loc(0, 100, 10))
            .unwrap();
-        write_checkpoint_ab(&table, dir.path(), CommitEpoch::new(2), &pos).unwrap();
+        write_checkpoint_ab(&table, dir.path(), CommitEpoch::new(2), 2, &pos).unwrap();

        std::fs::write(dir.path().join("checkpoint_b.tqc"), b"corrupt").unwrap();

-        let (best, epoch, _) = load_best_checkpoint(dir.path()).unwrap();
+        let (best, epoch, _, _) = load_best_checkpoint(dir.path()).unwrap();
        assert_eq!(epoch.raw(), 1);
        assert_eq!(best.len(), 1);
    }
@@ -22,6 +22,7 @@ const RECORD_TYPE_PUT: u8 = 0x01;
 const RECORD_TYPE_DECREMENT: u8 = 0x02;
 const RECORD_TYPE_RELOCATE: u8 = 0x03;
 const RECORD_TYPE_REMOVE: u8 = 0x04;
+const RECORD_TYPE_COMMIT_MARKER: u8 = 0x05;

 const HINT_FORMAT_VERSION: u8 = 1;

@@ -55,22 +56,24 @@ fn write_hint_record<S: StorageIO>(
    io.write_all_at(fd, write_offset.raw(), record)
 }

+fn encode_location_fields(record: &mut [u8; HINT_RECORD_SIZE], loc: &BlockLocation) {
+    record[FIELD_A_OFFSET..FIELD_A_OFFSET + 4].copy_from_slice(&loc.file_id.raw().to_le_bytes());
+    record[FIELD_A_OFFSET + 4..FIELD_A_OFFSET + 8].copy_from_slice(&loc.length.raw().to_le_bytes());
+    record[FIELD_B_OFFSET..FIELD_B_OFFSET + 8].copy_from_slice(&loc.offset.raw().to_le_bytes());
+}
+
 pub(crate) fn encode_hint_record<S: StorageIO>(
    io: &S,
    fd: FileId,
    write_offset: HintOffset,
    cid_bytes: &[u8; CID_SIZE],
-    file_id: DataFileId,
-    block_offset: BlockOffset,
-    length: BlockLength,
+    loc: &BlockLocation,
 ) -> io::Result<()> {
    let mut record = [0u8; HINT_RECORD_SIZE];
    record[TYPE_OFFSET] = RECORD_TYPE_PUT;
    record[VERSION_OFFSET] = HINT_FORMAT_VERSION;
    record[CID_OFFSET..CID_OFFSET + CID_SIZE].copy_from_slice(cid_bytes);
-    record[FIELD_A_OFFSET..FIELD_A_OFFSET + 4].copy_from_slice(&file_id.raw().to_le_bytes());
-    record[FIELD_A_OFFSET + 4..FIELD_A_OFFSET + 8].copy_from_slice(&length.raw().to_le_bytes());
-    record[FIELD_B_OFFSET..FIELD_B_OFFSET + 8].copy_from_slice(&block_offset.raw().to_le_bytes());
+    encode_location_fields(&mut record, loc);

    let checksum = hint_checksum(&record[..HINT_PAYLOAD_SIZE]);
    record[CHECKSUM_OFFSET..].copy_from_slice(&checksum.to_le_bytes());
@@ -78,22 +81,23 @@ pub(crate) fn encode_hint_record<S: StorageIO>(
    write_hint_record(io, fd, write_offset, &record)
 }

+const REFCOUNT_OFFSET: usize = 2;
+
 pub(crate) fn encode_relocate_record<S: StorageIO>(
    io: &S,
    fd: FileId,
    write_offset: HintOffset,
    cid_bytes: &[u8; CID_SIZE],
-    file_id: DataFileId,
-    block_offset: BlockOffset,
-    length: BlockLength,
+    loc: &BlockLocation,
+    refcount: u32,
 ) -> io::Result<()> {
    let mut record = [0u8; HINT_RECORD_SIZE];
    record[TYPE_OFFSET] = RECORD_TYPE_RELOCATE;
    record[VERSION_OFFSET] = HINT_FORMAT_VERSION;
+    let rc16 = u16::try_from(refcount).unwrap_or(u16::MAX);
+    record[REFCOUNT_OFFSET..REFCOUNT_OFFSET + 2].copy_from_slice(&rc16.to_le_bytes());
    record[CID_OFFSET..CID_OFFSET + CID_SIZE].copy_from_slice(cid_bytes);
-    record[FIELD_A_OFFSET..FIELD_A_OFFSET + 4].copy_from_slice(&file_id.raw().to_le_bytes());
-    record[FIELD_A_OFFSET + 4..FIELD_A_OFFSET + 8].copy_from_slice(&length.raw().to_le_bytes());
-    record[FIELD_B_OFFSET..FIELD_B_OFFSET + 8].copy_from_slice(&block_offset.raw().to_le_bytes());
+    encode_location_fields(&mut record, loc);

    let checksum = hint_checksum(&record[..HINT_PAYLOAD_SIZE]);
    record[CHECKSUM_OFFSET..].copy_from_slice(&checksum.to_le_bytes());
@@ -139,6 +143,36 @@ pub(crate) fn encode_decrement_record<S: StorageIO>(
    write_hint_record(io, fd, write_offset, &record)
 }

+const MARKER_DATA_OFFSET_POS: usize = CID_OFFSET;
+const MARKER_DATA_FILE_ID_POS: usize = CID_OFFSET + 8;
+const MARKER_RECORD_COUNT_POS: usize = FIELD_B_OFFSET;
+
+pub(crate) fn encode_commit_marker_record<S: StorageIO>(
+    io: &S,
+    fd: FileId,
+    write_offset: HintOffset,
+    batch_seq: u64,
+    record_count: u32,
+    data_file_id: DataFileId,
+    data_offset: BlockOffset,
+) -> io::Result<()> {
+    let mut record = [0u8; HINT_RECORD_SIZE];
+    record[TYPE_OFFSET] = RECORD_TYPE_COMMIT_MARKER;
+    record[VERSION_OFFSET] = HINT_FORMAT_VERSION;
+    record[MARKER_DATA_OFFSET_POS..MARKER_DATA_OFFSET_POS + 8]
+        .copy_from_slice(&data_offset.raw().to_le_bytes());
+    record[MARKER_DATA_FILE_ID_POS..MARKER_DATA_FILE_ID_POS + 4]
+        .copy_from_slice(&data_file_id.raw().to_le_bytes());
+    record[FIELD_A_OFFSET..FIELD_A_OFFSET + 8].copy_from_slice(&batch_seq.to_le_bytes());
+    record[MARKER_RECORD_COUNT_POS..MARKER_RECORD_COUNT_POS + 4]
+        .copy_from_slice(&record_count.to_le_bytes());
+
+    let checksum = hint_checksum(&record[..HINT_PAYLOAD_SIZE]);
+    record[CHECKSUM_OFFSET..].copy_from_slice(&checksum.to_le_bytes());
+
+    write_hint_record(io, fd, write_offset, &record)
+}
+
 #[must_use]
 #[derive(Debug)]
 pub enum ReadHintRecord {
@@ -158,10 +192,17 @@ pub enum ReadHintRecord {
        file_id: DataFileId,
        offset: BlockOffset,
        length: BlockLength,
+        refcount: u32,
    },
    Remove {
        cid_bytes: [u8; CID_SIZE],
    },
+    CommitMarker {
+        batch_seq: u64,
+        record_count: u32,
+        data_file_id: DataFileId,
+        data_offset: BlockOffset,
+    },
    UnknownVersion {
        version: u8,
    },
@@ -255,6 +296,15 @@ pub fn decode_hint_record<S: StorageIO>(
            }))
        }
        RECORD_TYPE_RELOCATE => {
+            let rc16 = u16::from_le_bytes(
+                record[REFCOUNT_OFFSET..REFCOUNT_OFFSET + 2]
+                    .try_into()
+                    .unwrap(),
+            );
+            let refcount = match rc16 {
+                0 => 1,
+                n => u32::from(n),
+            };
            let file_id = DataFileId::new(u32::from_le_bytes(
                record[FIELD_A_OFFSET..FIELD_A_OFFSET + 4]
                    .try_into()
@@ -278,9 +328,38 @@ pub fn decode_hint_record<S: StorageIO>(
                file_id,
                offset: block_offset,
                length: BlockLength::new(raw_length),
+                refcount,
            }))
        }
        RECORD_TYPE_REMOVE => Ok(Some(ReadHintRecord::Remove { cid_bytes })),
+        RECORD_TYPE_COMMIT_MARKER => {
+            let data_offset = BlockOffset::new(u64::from_le_bytes(
+                record[MARKER_DATA_OFFSET_POS..MARKER_DATA_OFFSET_POS + 8]
+                    .try_into()
+                    .unwrap(),
+            ));
+            let data_file_id = DataFileId::new(u32::from_le_bytes(
+                record[MARKER_DATA_FILE_ID_POS..MARKER_DATA_FILE_ID_POS + 4]
+                    .try_into()
+                    .unwrap(),
+            ));
+            let batch_seq = u64::from_le_bytes(
+                record[FIELD_A_OFFSET..FIELD_A_OFFSET + 8]
+                    .try_into()
+                    .unwrap(),
+            );
+            let record_count = u32::from_le_bytes(
+                record[MARKER_RECORD_COUNT_POS..MARKER_RECORD_COUNT_POS + 4]
+                    .try_into()
+                    .unwrap(),
+            );
+            Ok(Some(ReadHintRecord::CommitMarker {
+                batch_seq,
+                record_count,
+                data_file_id,
+                data_offset,
+            }))
+        }
        other => Ok(Some(ReadHintRecord::UnknownType { record_type: other })),
    }
 }
@@ -307,19 +386,9 @@ impl<'a, S: StorageIO> HintFileWriter<'a, S> {
    pub fn append_hint(
        &mut self,
        cid_bytes: &[u8; CID_SIZE],
-        file_id: DataFileId,
-        offset: BlockOffset,
-        length: BlockLength,
+        loc: &BlockLocation,
    ) -> io::Result<()> {
-        encode_hint_record(
-            self.io,
-            self.fd,
-            self.position,
-            cid_bytes,
-            file_id,
-            offset,
-            length,
-        )?;
+        encode_hint_record(self.io, self.fd, self.position, cid_bytes, loc)?;
        self.position = self.position.advance(HINT_RECORD_SIZE as u64);
        Ok(())
    }
@@ -338,19 +407,10 @@ impl<'a, S: StorageIO> HintFileWriter<'a, S> {
    pub fn append_relocate(
        &mut self,
        cid_bytes: &[u8; CID_SIZE],
-        file_id: DataFileId,
-        offset: BlockOffset,
-        length: BlockLength,
+        loc: &BlockLocation,
+        refcount: u32,
    ) -> io::Result<()> {
-        encode_relocate_record(
-            self.io,
-            self.fd,
-            self.position,
-            cid_bytes,
-            file_id,
-            offset,
-            length,
-        )?;
+        encode_relocate_record(self.io, self.fd, self.position, cid_bytes, loc, refcount)?;
        self.position = self.position.advance(HINT_RECORD_SIZE as u64);
        Ok(())
    }
@@ -361,6 +421,26 @@ impl<'a, S: StorageIO> HintFileWriter<'a, S> {
        Ok(())
    }

+    pub fn append_commit_marker(
+        &mut self,
+        batch_seq: u64,
+        record_count: u32,
+        data_file_id: DataFileId,
+        data_offset: BlockOffset,
+    ) -> io::Result<()> {
+        encode_commit_marker_record(
+            self.io,
+            self.fd,
+            self.position,
+            batch_seq,
+            record_count,
+            data_file_id,
+            data_offset,
+        )?;
+        self.position = self.position.advance(HINT_RECORD_SIZE as u64);
+        Ok(())
+    }
+
    pub fn sync(&self) -> io::Result<()> {
        self.io.sync(self.fd)
    }
@@ -421,12 +501,13 @@ impl<S: StorageIO> Iterator for HintFileReader<'_, S> {
                    | ReadHintRecord::Decrement { .. }
                    | ReadHintRecord::Relocate { .. }
                    | ReadHintRecord::Remove { .. }
-                    | ReadHintRecord::UnknownType { .. } => {
+                    | ReadHintRecord::CommitMarker { .. }
+                    | ReadHintRecord::UnknownType { .. }
+                    | ReadHintRecord::UnknownVersion { .. }
+                    | ReadHintRecord::Corrupted => {
                        self.position = self.position.advance(HINT_RECORD_SIZE as u64);
                    }
-                    ReadHintRecord::UnknownVersion { .. }
-                    | ReadHintRecord::Corrupted
-                    | ReadHintRecord::Truncated => {
+                    ReadHintRecord::Truncated => {
                        self.position = HintOffset::new(self.file_size);
                    }
                }
@@ -534,6 +615,7 @@ fn scan_single_hint_file<S: StorageIO>(
                ReadHintRecord::Decrement { .. }
                | ReadHintRecord::Relocate { .. }
                | ReadHintRecord::Remove { .. }
+                | ReadHintRecord::CommitMarker { .. }
                | ReadHintRecord::UnknownVersion { .. }
                | ReadHintRecord::UnknownType { .. }
                | ReadHintRecord::Corrupted
@@ -547,7 +629,95 @@ fn scan_single_hint_file<S: StorageIO>(
    entries
 }

-const REPLAY_BATCH_SIZE: usize = 10_000;
+#[derive(Default)]
+struct PendingBatch {
+    puts: Vec<([u8; CID_SIZE], BlockLocation)>,
+    relocates: Vec<([u8; CID_SIZE], BlockLocation, u32)>,
+    removes: Vec<[u8; CID_SIZE]>,
+    decrements: Vec<([u8; CID_SIZE], CommitEpoch, WallClockMs)>,
+    file_cursors: HashMap<DataFileId, BlockOffset>,
+    max_cursor: Option<WriteCursor>,
+    record_count: u32,
+    boundary_lost: bool,
+}
+
+impl PendingBatch {
+    fn reset(&mut self) {
+        self.puts.clear();
+        self.relocates.clear();
+        self.removes.clear();
+        self.decrements.clear();
+        self.file_cursors.clear();
+        self.max_cursor = None;
+        self.record_count = 0;
+        self.boundary_lost = false;
+    }
+
+    fn note_record(&mut self) {
+        self.record_count = self.record_count.saturating_add(1);
+    }
+
+    fn track_cursor(&mut self, file_id: DataFileId, end: BlockOffset) {
+        let candidate = WriteCursor {
+            file_id,
+            offset: end,
+        };
+        self.max_cursor = Some(match self.max_cursor {
+            Some(c) => std::cmp::max_by_key(c, candidate, |w| (w.file_id, w.offset)),
+            None => candidate,
+        });
+        self.file_cursors
+            .entry(file_id)
+            .and_modify(|existing| {
+                if end > *existing {
+                    *existing = end;
+                }
+            })
+            .or_insert(end);
+    }
+}
+
+fn commit_pending_batch(
+    pending: &mut PendingBatch,
+    index: &super::hash_index::BlockIndex,
+    file_cursors: &mut HashMap<DataFileId, BlockOffset>,
+    max_cursor: &mut Option<WriteCursor>,
+    replayed: &mut u64,
+) -> Result<(), RebuildError> {
+    if !pending.puts.is_empty() {
+        index.batch_insert_buffered(&pending.puts)?;
+    }
+    if !pending.relocates.is_empty() {
+        index.batch_relocate(&pending.relocates)?;
+    }
+    if !pending.removes.is_empty() {
+        index.batch_remove(&pending.removes);
+    }
+    pending
+        .decrements
+        .iter()
+        .try_for_each(|(cid, epoch, ts)| index.batch_decrement(&[*cid], *epoch, *ts))?;
+
+    pending.file_cursors.iter().for_each(|(fid, end)| {
+        file_cursors
+            .entry(*fid)
+            .and_modify(|existing| {
+                if *end > *existing {
+                    *existing = *end;
+                }
+            })
+            .or_insert(*end);
+    });
+    if let Some(c) = pending.max_cursor {
+        *max_cursor = Some(match *max_cursor {
+            Some(m) => std::cmp::max_by_key(m, c, |w| (w.file_id, w.offset)),
+            None => c,
+        });
+    }
+    *replayed = replayed.saturating_add(u64::from(pending.record_count));
+    pending.reset();
+    Ok(())
+}

 pub fn replay_hints_into_block_index<S: StorageIO>(
    io: &S,
@@ -573,11 +743,7 @@ pub fn replay_hints_into_block_index<S: StorageIO>(
    let mut max_cursor: Option<WriteCursor> = None;
    let mut file_cursors: HashMap<DataFileId, BlockOffset> = HashMap::new();
    let mut replayed: u64 = 0;
-    let mut put_buffer: Vec<([u8; CID_SIZE], BlockLocation)> =
-        Vec::with_capacity(REPLAY_BATCH_SIZE);
-    let mut relocate_buffer: Vec<([u8; CID_SIZE], BlockLocation)> =
-        Vec::with_capacity(REPLAY_BATCH_SIZE);
-    let mut remove_buffer: Vec<[u8; CID_SIZE]> = Vec::with_capacity(REPLAY_BATCH_SIZE);
+    let mut pending = PendingBatch::default();

    hint_files
        .iter()
@@ -609,135 +775,97 @@ pub fn replay_hints_into_block_index<S: StorageIO>(
                            offset,
                            length,
                        };
-                        put_buffer.push((cid_bytes, loc));
-
                        let record_end =
                            offset.advance(BLOCK_RECORD_OVERHEAD as u64 + length.as_u64());
-                        let candidate = WriteCursor {
-                            file_id,
-                            offset: record_end,
-                        };
-                        max_cursor = Some(match max_cursor {
-                            Some(c) => {
-                                std::cmp::max_by_key(c, candidate, |w| (w.file_id, w.offset))
-                            }
-                            None => candidate,
-                        });
-                        file_cursors
-                            .entry(file_id)
-                            .and_modify(|existing| {
-                                if record_end > *existing {
-                                    *existing = record_end;
-                                }
-                            })
-                            .or_insert(record_end);
-
-                        replayed = replayed.saturating_add(1);
-                        if put_buffer.len() >= REPLAY_BATCH_SIZE {
-                            index.batch_insert_buffered(&put_buffer)?;
-                            put_buffer.clear();
-                        }
+                        pending.puts.push((cid_bytes, loc));
+                        pending.track_cursor(file_id, record_end);
+                        pending.note_record();
                    }
                    ReadHintRecord::Decrement {
                        cid_bytes,
                        epoch,
                        timestamp,
                    } => {
-                        if !put_buffer.is_empty() {
-                            index.batch_insert_buffered(&put_buffer)?;
-                            put_buffer.clear();
-                        }
-                        if !relocate_buffer.is_empty() {
-                            index.batch_relocate(&relocate_buffer)?;
-                            relocate_buffer.clear();
-                        }
-                        if !remove_buffer.is_empty() {
-                            index.batch_remove(&remove_buffer);
-                            remove_buffer.clear();
-                        }
-                        index.batch_decrement(&[cid_bytes], epoch, timestamp)?;
-                        replayed = replayed.saturating_add(1);
+                        pending.decrements.push((cid_bytes, epoch, timestamp));
+                        pending.note_record();
                    }
                    ReadHintRecord::Relocate {
                        cid_bytes,
                        file_id,
                        offset,
                        length,
+                        refcount,
                    } => {
                        let loc = BlockLocation {
                            file_id,
                            offset,
                            length,
                        };
-                        relocate_buffer.push((cid_bytes, loc));
-
                        let record_end =
                            offset.advance(BLOCK_RECORD_OVERHEAD as u64 + length.as_u64());
-                        file_cursors
-                            .entry(file_id)
-                            .and_modify(|existing| {
-                                if record_end > *existing {
-                                    *existing = record_end;
-                                }
-                            })
-                            .or_insert(record_end);
-
-                        replayed = replayed.saturating_add(1);
-                        if relocate_buffer.len() >= REPLAY_BATCH_SIZE {
-                            if !put_buffer.is_empty() {
-                                index.batch_insert_buffered(&put_buffer)?;
-                                put_buffer.clear();
-                            }
-                            index.batch_relocate(&relocate_buffer)?;
-                            relocate_buffer.clear();
-                        }
+                        pending.relocates.push((cid_bytes, loc, refcount));
+                        pending.track_cursor(file_id, record_end);
+                        pending.note_record();
                    }
                    ReadHintRecord::Remove { cid_bytes } => {
-                        remove_buffer.push(cid_bytes);
-                        replayed = replayed.saturating_add(1);
-                        if remove_buffer.len() >= REPLAY_BATCH_SIZE {
-                            if !put_buffer.is_empty() {
-                                index.batch_insert_buffered(&put_buffer)?;
-                                put_buffer.clear();
+                        pending.removes.push(cid_bytes);
+                        pending.note_record();
+                    }
+                    ReadHintRecord::CommitMarker {
+                        batch_seq,
+                        record_count,
+                        data_file_id,
+                        data_offset,
+                    } => {
+                        let accepts =
+                            !pending.boundary_lost && pending.record_count == record_count;
+                        match accepts {
+                            true => {
+                                pending.track_cursor(data_file_id, data_offset);
+                                commit_pending_batch(
+                                    &mut pending,
+                                    index,
+                                    &mut file_cursors,
+                                    &mut max_cursor,
+                                    &mut replayed,
+                                )?;
                            }
-                            if !relocate_buffer.is_empty() {
-                                index.batch_relocate(&relocate_buffer)?;
-                                relocate_buffer.clear();
+                            false => {
+                                tracing::warn!(
+                                    file_id = %fid,
+                                    batch_seq,
+                                    expected_count = record_count,
+                                    observed_count = pending.record_count,
+                                    boundary_lost = pending.boundary_lost,
+                                    "rolling back torn hint batch"
+                                );
+                                pending.reset();
                            }
-                            index.batch_remove(&remove_buffer);
-                            remove_buffer.clear();
                        }
                    }
-                    ReadHintRecord::Corrupted => {
-                        tracing::warn!(
-                            file_id = %fid,
-                            "corrupted hint record during replay, skipping"
-                        );
+                    ReadHintRecord::Corrupted
+                    | ReadHintRecord::UnknownVersion { .. }
+                    | ReadHintRecord::UnknownType { .. } => {
+                        pending.boundary_lost = true;
                    }
-                    ReadHintRecord::UnknownVersion { .. }
-                    | ReadHintRecord::UnknownType { .. }
-                    | ReadHintRecord::Truncated => {}
+                    ReadHintRecord::Truncated => {}
                }
                Ok::<_, RebuildError>(())
            })?;

-            if !put_buffer.is_empty() {
-                index.batch_insert_buffered(&put_buffer)?;
-                put_buffer.clear();
-            }
-            if !relocate_buffer.is_empty() {
-                index.batch_relocate(&relocate_buffer)?;
-                relocate_buffer.clear();
-            }
-            if !remove_buffer.is_empty() {
-                index.batch_remove(&remove_buffer);
-                remove_buffer.clear();
-            }
-
            let _ = io.close(fd);
            Ok(())
        })?;

+    if pending.record_count > 0 || pending.boundary_lost {
+        tracing::warn!(
+            record_count = pending.record_count,
+            boundary_lost = pending.boundary_lost,
+            "discarding unterminated hint batch at replay end"
+        );
+        pending.reset();
+    }
+
    if let Some(cursor) = max_cursor {
        index.set_write_cursor(cursor)?;
    }
@@ -843,7 +971,12 @@ mod tests {
        let offset = BlockOffset::new(1024);
        let length = BlockLength::new(256);

-        encode_hint_record(&sim, fd, HintOffset::new(0), &cid, file_id, offset, length).unwrap();
+        let loc = BlockLocation {
+            file_id,
+            offset,
+            length,
+        };
+        encode_hint_record(&sim, fd, HintOffset::new(0), &cid, &loc).unwrap();

        let file_size = sim.file_size(fd).unwrap();
        let record = decode_hint_record(&sim, fd, HintOffset::new(0), file_size)
@@ -901,16 +1034,12 @@ mod tests {
        (0u8..5).for_each(|i| {
            let cid = test_cid(i);
            let write_offset = HintOffset::new(i as u64 * HINT_RECORD_SIZE as u64);
-            encode_hint_record(
-                &sim,
-                fd,
-                write_offset,
-                &cid,
-                DataFileId::new(i as u32),
-                BlockOffset::new(i as u64 * 100),
-                BlockLength::new(50 + i as u32),
-            )
-            .unwrap();
+            let loc = BlockLocation {
+                file_id: DataFileId::new(i as u32),
+                offset: BlockOffset::new(i as u64 * 100),
+                length: BlockLength::new(50 + i as u32),
+            };
+            encode_hint_record(&sim, fd, write_offset, &cid, &loc).unwrap();
        });

        let file_size = sim.file_size(fd).unwrap();
@@ -952,16 +1081,12 @@ mod tests {
    fn detects_corrupted_hint() {
        let (sim, fd) = setup();
        let cid = test_cid(1);
-        encode_hint_record(
-            &sim,
-            fd,
-            HintOffset::new(0),
-            &cid,
-            DataFileId::new(0),
-            BlockOffset::new(0),
-            BlockLength::new(100),
-        )
-        .unwrap();
+        let loc = BlockLocation {
+            file_id: DataFileId::new(0),
+            offset: BlockOffset::new(0),
+            length: BlockLength::new(100),
+        };
+        encode_hint_record(&sim, fd, HintOffset::new(0), &cid, &loc).unwrap();

        sim.write_all_at(fd, 10, &[0xFF]).unwrap();

@@ -987,16 +1112,12 @@ mod tests {
    fn oversized_length_treated_as_corrupted() {
        let (sim, fd) = setup();
        let cid = test_cid(1);
-        encode_hint_record(
-            &sim,
-            fd,
-            HintOffset::new(0),
-            &cid,
-            DataFileId::new(0),
-            BlockOffset::new(0),
-            BlockLength::new(100),
-        )
-        .unwrap();
+        let loc = BlockLocation {
+            file_id: DataFileId::new(0),
+            offset: BlockOffset::new(0),
+            length: BlockLength::new(100),
+        };
+        encode_hint_record(&sim, fd, HintOffset::new(0), &cid, &loc).unwrap();

        let length_offset = FIELD_A_OFFSET as u64 + 4;
        let oversized = (MAX_BLOCK_SIZE + 1).to_le_bytes();
@@ -1021,14 +1142,12 @@ mod tests {
        let mut writer = HintFileWriter::new(&sim, fd);

        (0u8..5).for_each(|i| {
-            writer
-                .append_hint(
-                    &test_cid(i),
-                    DataFileId::new(0),
-                    BlockOffset::new(i as u64 * 100),
-                    BlockLength::new(50 + i as u32),
-                )
-                .unwrap();
+            let loc = BlockLocation {
+                file_id: DataFileId::new(0),
+                offset: BlockOffset::new(i as u64 * 100),
+                length: BlockLength::new(50 + i as u32),
+            };
+            writer.append_hint(&test_cid(i), &loc).unwrap();
        });

        assert_eq!(
@@ -1055,25 +1174,21 @@ mod tests {
    fn hint_writer_resume_continues_at_position() {
        let (sim, fd) = setup();
        let mut writer = HintFileWriter::new(&sim, fd);
-        writer
-            .append_hint(
-                &test_cid(0),
-                DataFileId::new(0),
-                BlockOffset::new(0),
-                BlockLength::new(100),
-            )
-            .unwrap();
+        let loc0 = BlockLocation {
+            file_id: DataFileId::new(0),
+            offset: BlockOffset::new(0),
+            length: BlockLength::new(100),
+        };
+        writer.append_hint(&test_cid(0), &loc0).unwrap();

        let pos = writer.position();
        let mut writer2 = HintFileWriter::resume(&sim, fd, pos);
-        writer2
-            .append_hint(
-                &test_cid(1),
-                DataFileId::new(0),
-                BlockOffset::new(100),
-                BlockLength::new(200),
-            )
-            .unwrap();
+        let loc1 = BlockLocation {
+            file_id: DataFileId::new(0),
+            offset: BlockOffset::new(100),
+            length: BlockLength::new(200),
+        };
+        writer2.append_hint(&test_cid(1), &loc1).unwrap();

        let reader = HintFileReader::open(&sim, fd).unwrap();
        let valid_count = reader
@@ -1096,14 +1211,12 @@ mod tests {
    fn hint_reader_stops_on_truncated() {
        let (sim, fd) = setup();
        let mut writer = HintFileWriter::new(&sim, fd);
-        writer
-            .append_hint(
-                &test_cid(0),
-                DataFileId::new(0),
-                BlockOffset::new(0),
-                BlockLength::new(100),
-            )
-            .unwrap();
+        let loc = BlockLocation {
+            file_id: DataFileId::new(0),
+            offset: BlockOffset::new(0),
+            length: BlockLength::new(100),
+        };
+        writer.append_hint(&test_cid(0), &loc).unwrap();

        sim.write_all_at(fd, writer.position().raw(), &[0u8; HINT_RECORD_SIZE - 1])
            .unwrap();
@@ -1116,19 +1229,17 @@ mod tests {
    }

    #[test]
-    fn hint_reader_stops_on_corrupted() {
+    fn hint_reader_reports_corrupted_and_continues() {
        let (sim, fd) = setup();
        let mut writer = HintFileWriter::new(&sim, fd);

        (0u8..3).for_each(|i| {
-            writer
-                .append_hint(
-                    &test_cid(i),
-                    DataFileId::new(0),
-                    BlockOffset::new(i as u64 * 100),
-                    BlockLength::new(50),
-                )
-                .unwrap();
+            let loc = BlockLocation {
+                file_id: DataFileId::new(0),
+                offset: BlockOffset::new(i as u64 * 100),
+                length: BlockLength::new(50),
+            };
+            writer.append_hint(&test_cid(i), &loc).unwrap();
        });

        sim.write_all_at(fd, HINT_RECORD_SIZE as u64 + 5, &[0xFF])
@@ -1136,9 +1247,48 @@ mod tests {

        let reader = HintFileReader::open(&sim, fd).unwrap();
        let records: Vec<_> = reader.map(|r| r.unwrap()).collect();
-        assert_eq!(records.len(), 2);
+        assert_eq!(records.len(), 3);
        assert!(matches!(records[0], ReadHintRecord::Put { .. }));
        assert!(matches!(records[1], ReadHintRecord::Corrupted));
+        assert!(matches!(records[2], ReadHintRecord::Put { .. }));
+    }
+
+    #[test]
+    fn commit_marker_round_trip() {
+        let (sim, fd) = setup();
+        let data_file_id = DataFileId::new(7);
+        let data_offset = BlockOffset::new(9_876);
+
+        encode_commit_marker_record(
+            &sim,
+            fd,
+            HintOffset::new(0),
+            42,
+            128,
+            data_file_id,
+            data_offset,
+        )
+        .unwrap();
+
+        let file_size = sim.file_size(fd).unwrap();
+        let record = decode_hint_record(&sim, fd, HintOffset::new(0), file_size)
+            .unwrap()
+            .unwrap();
+
+        match record {
+            ReadHintRecord::CommitMarker {
+                batch_seq,
+                record_count,
+                data_file_id: fid,
+                data_offset: off,
+            } => {
+                assert_eq!(batch_seq, 42);
+                assert_eq!(record_count, 128);
+                assert_eq!(fid, data_file_id);
+                assert_eq!(off, data_offset);
+            }
+            other => panic!("expected CommitMarker, got {other:?}"),
+        }
    }

    #[test]
@@ -1,6 +1,7 @@
 use std::collections::HashMap;
 use std::io;
 use std::path::{Path, PathBuf};
+use std::sync::Arc;

 use parking_lot::RwLock;

@@ -13,22 +14,39 @@ pub const DEFAULT_MAX_FILE_SIZE: u64 = 256 * 1024 * 1024;

 pub(crate) const DATA_FILE_EXTENSION: &str = "tqb";

-struct CachedHandle {
+pub struct CachedHandle<S: StorageIO> {
    fd: FileId,
+    io: Arc<S>,
    writable: bool,
 }

+impl<S: StorageIO> CachedHandle<S> {
+    pub fn fd(&self) -> FileId {
+        self.fd
+    }
+
+    pub fn is_writable(&self) -> bool {
+        self.writable
+    }
+}
+
+impl<S: StorageIO> Drop for CachedHandle<S> {
+    fn drop(&mut self) {
+        let _ = self.io.close(self.fd);
+    }
+}
+
 pub struct DataFileManager<S: StorageIO> {
-    io: S,
+    io: Arc<S>,
    data_dir: PathBuf,
    max_file_size: u64,
-    handles: RwLock<HashMap<DataFileId, CachedHandle>>,
+    handles: RwLock<HashMap<DataFileId, Arc<CachedHandle<S>>>>,
 }

 impl<S: StorageIO> DataFileManager<S> {
    pub fn new(io: S, data_dir: PathBuf, max_file_size: u64) -> Self {
        Self {
-            io,
+            io: Arc::new(io),
            data_dir,
            max_file_size,
            handles: RwLock::new(HashMap::new()),
@@ -40,7 +58,7 @@ impl<S: StorageIO> DataFileManager<S> {
    }

    pub fn io(&self) -> &S {
-        &self.io
+        self.io.as_ref()
    }

    pub fn data_dir(&self) -> &Path {
@@ -56,77 +74,81 @@ impl<S: StorageIO> DataFileManager<S> {
            .join(format!("{file_id}.{DATA_FILE_EXTENSION}"))
    }

-    pub fn open_for_append(&self, file_id: DataFileId) -> io::Result<FileId> {
+    pub fn open_for_append(&self, file_id: DataFileId) -> io::Result<Arc<CachedHandle<S>>> {
        {
            let cache = self.handles.read();
            if let Some(entry) = cache.get(&file_id)
                && entry.writable
            {
-                return Ok(entry.fd);
+                return Ok(Arc::clone(entry));
            }
        }
        let path = self.data_file_path(file_id);
        let fd = self.io.open(&path, OpenOptions::read_write())?;
        let mut cache = self.handles.write();
-        match cache.get(&file_id) {
+        match cache.get(&file_id).cloned() {
            Some(entry) if entry.writable => {
                let _ = self.io.close(fd);
-                Ok(entry.fd)
+                Ok(entry)
            }
-            Some(entry) => {
-                let old_fd = entry.fd;
-                cache.insert(file_id, CachedHandle { fd, writable: true });
-                let _ = self.io.close(old_fd);
-                Ok(fd)
-            }
-            None => {
-                cache.insert(file_id, CachedHandle { fd, writable: true });
-                Ok(fd)
+            _ => {
+                let handle = Arc::new(CachedHandle {
+                    fd,
+                    io: Arc::clone(&self.io),
+                    writable: true,
+                });
+                cache.insert(file_id, Arc::clone(&handle));
+                Ok(handle)
            }
        }
    }

-    pub fn open_for_read(&self, file_id: DataFileId) -> io::Result<FileId> {
+    pub fn open_for_read(&self, file_id: DataFileId) -> io::Result<Arc<CachedHandle<S>>> {
        if let Some(entry) = self.handles.read().get(&file_id) {
-            return Ok(entry.fd);
+            return Ok(Arc::clone(entry));
        }
        let path = self.data_file_path(file_id);
        let fd = self.io.open(&path, OpenOptions::read_only_existing())?;
        let mut cache = self.handles.write();
-        match cache.get(&file_id) {
+        match cache.get(&file_id).cloned() {
            Some(entry) => {
                let _ = self.io.close(fd);
-                Ok(entry.fd)
+                Ok(entry)
            }
            None => {
-                cache.insert(
-                    file_id,
-                    CachedHandle {
-                        fd,
-                        writable: false,
-                    },
-                );
-                Ok(fd)
+                let handle = Arc::new(CachedHandle {
+                    fd,
+                    io: Arc::clone(&self.io),
+                    writable: false,
+                });
+                cache.insert(file_id, Arc::clone(&handle));
+                Ok(handle)
            }
        }
    }

-    pub fn prepare_rotation(&self, current: DataFileId) -> io::Result<(DataFileId, FileId)> {
+    pub fn prepare_rotation(
+        &self,
+        current: DataFileId,
+    ) -> io::Result<(DataFileId, Arc<CachedHandle<S>>)> {
        let next = current.next();
        let path = self.data_file_path(next);
        let fd = self.io.open(&path, OpenOptions::read_write())?;
-        Ok((next, fd))
+        let handle = Arc::new(CachedHandle {
+            fd,
+            io: Arc::clone(&self.io),
+            writable: true,
+        });
+        Ok((next, handle))
    }

-    pub fn commit_rotation(&self, file_id: DataFileId, fd: FileId) {
-        self.handles
-            .write()
-            .insert(file_id, CachedHandle { fd, writable: true });
+    pub fn commit_rotation(&self, file_id: DataFileId, handle: &Arc<CachedHandle<S>>) {
+        self.handles.write().insert(file_id, Arc::clone(handle));
    }

-    pub fn rollback_rotation(&self, file_id: DataFileId, fd: FileId) {
-        let _ = self.io.close(fd);
+    pub fn rollback_rotation(&self, file_id: DataFileId) {
        self.handles.write().remove(&file_id);
+        let _ = self.io.delete(&self.data_file_path(file_id));
    }

    pub fn should_rotate(&self, position: BlockOffset) -> bool {
@@ -134,14 +156,11 @@ impl<S: StorageIO> DataFileManager<S> {
    }

    pub fn list_files(&self) -> io::Result<Vec<DataFileId>> {
-        list_files_by_extension(&self.io, &self.data_dir, DATA_FILE_EXTENSION)
+        list_files_by_extension(&*self.io, &self.data_dir, DATA_FILE_EXTENSION)
    }

    pub fn evict_handle(&self, file_id: DataFileId) {
-        let removed = self.handles.write().remove(&file_id);
-        if let Some(entry) = removed {
-            let _ = self.io.close(entry.fd);
-        }
+        self.handles.write().remove(&file_id);
    }

    pub fn delete_data_file(&self, file_id: DataFileId) -> io::Result<()> {
@@ -151,14 +170,6 @@ impl<S: StorageIO> DataFileManager<S> {
    }
 }

-impl<S: StorageIO> Drop for DataFileManager<S> {
-    fn drop(&mut self) {
-        self.handles.write().drain().for_each(|(_, entry)| {
-            let _ = self.io.close(entry.fd);
-        });
-    }
-}
-
 #[cfg(test)]
 mod tests {
    use super::*;
@@ -177,8 +188,8 @@ mod tests {
    #[test]
    fn open_for_append_creates_file() {
        let mgr = setup_manager(1024);
-        let fd = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        assert_eq!(mgr.io().file_size(fd).unwrap(), 0);
+        let handle = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        assert_eq!(mgr.io().file_size(handle.fd()).unwrap(), 0);
    }

    #[test]
@@ -190,45 +201,46 @@ mod tests {
    #[test]
    fn handle_cache_returns_same_fd() {
        let mgr = setup_manager(1024);
-        let fd1 = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        let fd2 = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        assert_eq!(fd1, fd2);
+        let h1 = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        let h2 = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        assert_eq!(h1.fd(), h2.fd());
    }

    #[test]
    fn open_for_read_uses_cache_from_append() {
        let mgr = setup_manager(1024);
-        let fd_write = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        let fd_read = mgr.open_for_read(DataFileId::new(0)).unwrap();
-        assert_eq!(fd_write, fd_read);
+        let h_write = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        let h_read = mgr.open_for_read(DataFileId::new(0)).unwrap();
+        assert_eq!(h_write.fd(), h_read.fd());
    }

    #[test]
    fn rotation_lifecycle_prepare_commit() {
        let mgr = setup_manager(1024);
-        let _fd0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        let (next_id, next_fd) = mgr.prepare_rotation(DataFileId::new(0)).unwrap();
+        let _h0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        let (next_id, next_handle) = mgr.prepare_rotation(DataFileId::new(0)).unwrap();
        assert_eq!(next_id, DataFileId::new(1));
-        assert_eq!(mgr.io().file_size(next_fd).unwrap(), 0);
+        assert_eq!(mgr.io().file_size(next_handle.fd()).unwrap(), 0);
        mgr.io().sync_dir(mgr.data_dir()).unwrap();
-        mgr.commit_rotation(next_id, next_fd);
-        assert_eq!(mgr.open_for_read(next_id).unwrap(), next_fd);
+        mgr.commit_rotation(next_id, &next_handle);
+        assert_eq!(mgr.open_for_read(next_id).unwrap().fd(), next_handle.fd());
    }

    #[test]
-    fn rotation_rollback_cleans_handle() {
+    fn rotation_rollback_cleans_handle_and_deletes_file() {
        let mgr = setup_manager(1024);
-        let _fd0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        let (next_id, next_fd) = mgr.prepare_rotation(DataFileId::new(0)).unwrap();
-        mgr.commit_rotation(next_id, next_fd);
+        let _h0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        let (next_id, next_handle) = mgr.prepare_rotation(DataFileId::new(0)).unwrap();
+        mgr.commit_rotation(next_id, &next_handle);

-        assert_eq!(mgr.open_for_read(next_id).unwrap(), next_fd);
-        mgr.rollback_rotation(next_id, next_fd);
+        assert_eq!(mgr.open_for_read(next_id).unwrap().fd(), next_handle.fd());
+        drop(next_handle);
+        mgr.rollback_rotation(next_id);

-        let reopened_fd = mgr.open_for_read(next_id).unwrap();
-        assert_ne!(
-            reopened_fd, next_fd,
-            "rollback should have closed the cached fd"
+        let reopen = mgr.open_for_read(next_id);
+        assert!(
+            reopen.is_err_and(|e| e.kind() == io::ErrorKind::NotFound),
+            "rollback_rotation must delete the data file so recovery cannot resurrect uncommitted bytes"
        );
    }

@@ -244,8 +256,8 @@ mod tests {
    #[test]
    fn list_files_finds_data_files() {
        let mgr = setup_manager(1024);
-        let _fd0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        let _fd3 = mgr.open_for_append(DataFileId::new(3)).unwrap();
+        let _h0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        let _h3 = mgr.open_for_append(DataFileId::new(3)).unwrap();

        let files = mgr.list_files().unwrap();
        assert_eq!(files, vec![DataFileId::new(0), DataFileId::new(3)]);
@@ -254,7 +266,7 @@ mod tests {
    #[test]
    fn list_files_ignores_non_data_files() {
        let mgr = setup_manager(1024);
-        let _fd0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        let _h0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
        mgr.io()
            .open(Path::new("/data/notes.txt"), OpenOptions::read_write())
            .unwrap();
@@ -279,32 +291,32 @@ mod tests {
    #[test]
    fn rotate_and_write_across_files() {
        let mgr = setup_manager(1024);
-        let fd0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        let mut writer0 = DataFileWriter::new(mgr.io(), fd0, DataFileId::new(0)).unwrap();
+        let h0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        let mut writer0 = DataFileWriter::new(mgr.io(), h0.fd(), DataFileId::new(0)).unwrap();
        let _ = writer0
            .append_block(&test_cid(1), b"first file data")
            .unwrap();
        writer0.sync().unwrap();

-        let (id1, fd1) = mgr.prepare_rotation(DataFileId::new(0)).unwrap();
+        let (id1, h1) = mgr.prepare_rotation(DataFileId::new(0)).unwrap();
        mgr.io().sync_dir(mgr.data_dir()).unwrap();
-        mgr.commit_rotation(id1, fd1);
-        let mut writer1 = DataFileWriter::new(mgr.io(), fd1, id1).unwrap();
+        mgr.commit_rotation(id1, &h1);
+        let mut writer1 = DataFileWriter::new(mgr.io(), h1.fd(), id1).unwrap();
        let _ = writer1
            .append_block(&test_cid(2), b"second file data")
            .unwrap();
        writer1.sync().unwrap();

-        let fd0_read = mgr.open_for_read(DataFileId::new(0)).unwrap();
-        let blocks0 = DataFileReader::open(mgr.io(), fd0_read)
+        let h0_read = mgr.open_for_read(DataFileId::new(0)).unwrap();
+        let blocks0 = DataFileReader::open(mgr.io(), h0_read.fd())
            .unwrap()
            .valid_blocks()
            .unwrap();
        assert_eq!(blocks0.len(), 1);
        assert_eq!(blocks0[0].2, b"first file data");

-        let fd1_read = mgr.open_for_read(id1).unwrap();
-        let blocks1 = DataFileReader::open(mgr.io(), fd1_read)
+        let h1_read = mgr.open_for_read(id1).unwrap();
+        let blocks1 = DataFileReader::open(mgr.io(), h1_read.fd())
            .unwrap()
            .valid_blocks()
            .unwrap();
@@ -315,11 +327,11 @@ mod tests {
    #[test]
    fn read_cache_hit_from_writable_entry() {
        let mgr = setup_manager(1024);
-        let fd_write = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        DataFileWriter::new(mgr.io(), fd_write, DataFileId::new(0)).unwrap();
+        let h_write = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        DataFileWriter::new(mgr.io(), h_write.fd(), DataFileId::new(0)).unwrap();

-        let fd_read = mgr.open_for_read(DataFileId::new(0)).unwrap();
-        assert_eq!(fd_write, fd_read);
+        let h_read = mgr.open_for_read(DataFileId::new(0)).unwrap();
+        assert_eq!(h_write.fd(), h_read.fd());
    }

    #[test]
@@ -338,15 +350,15 @@ mod tests {
        mgr.io().sync_dir(mgr.data_dir()).unwrap();
        mgr.io().close(raw_fd).unwrap();

-        let fd_read = mgr.open_for_read(DataFileId::new(0)).unwrap();
-        let _reader = DataFileReader::open(mgr.io(), fd_read).unwrap();
+        let h_read = mgr.open_for_read(DataFileId::new(0)).unwrap();
+        let _reader = DataFileReader::open(mgr.io(), h_read.fd()).unwrap();

-        let fd_append = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        assert_ne!(fd_read, fd_append);
+        let h_append = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        assert_ne!(h_read.fd(), h_append.fd());

        let mut writer = DataFileWriter::resume(
            mgr.io(),
-            fd_append,
+            h_append.fd(),
            DataFileId::new(0),
            BlockOffset::new(BLOCK_HEADER_SIZE as u64),
        );
@@ -355,7 +367,7 @@ mod tests {
            .unwrap();
        writer.sync().unwrap();

-        let blocks = DataFileReader::open(mgr.io(), fd_append)
+        let blocks = DataFileReader::open(mgr.io(), h_append.fd())
            .unwrap()
            .valid_blocks()
            .unwrap();
@@ -1,3 +1,4 @@
+mod cid_util;
 mod compaction;
 mod data_file;
 mod group_commit;
@@ -8,6 +9,7 @@ mod reader;
 mod store;
 mod types;

+pub use cid_util::{DAG_CBOR_CODEC, SHA2_256_CODE, hash_to_cid, hash_to_cid_bytes};
 pub use compaction::CompactionError;
 pub use data_file::{
    BLOCK_FORMAT_VERSION, BLOCK_HEADER_SIZE, BLOCK_MAGIC, BLOCK_RECORD_OVERHEAD, CID_SIZE,
@@ -22,7 +24,7 @@ pub use hint::{
    HINT_FILE_EXTENSION, HINT_RECORD_SIZE, HintFileReader, HintFileWriter, HintIndex,
    ReadHintRecord, RebuildError, decode_hint_record, hint_file_path, scan_hints_to_memory,
 };
-pub use manager::{DEFAULT_MAX_FILE_SIZE, DataFileManager};
+pub use manager::{CachedHandle, DEFAULT_MAX_FILE_SIZE, DataFileManager};
 pub use reader::{BlockStoreReader, ReadError};
 pub use store::QuiesceGuard;
 pub use store::{BlockStoreConfig, DEFAULT_SHARD_COUNT, TranquilBlockStore};
@@ -104,11 +104,11 @@ impl<S: StorageIO> BlockStoreReader<S> {
        });

        by_file.into_iter().try_for_each(|(file_id, mut entries)| {
-            let fd = self.manager.open_for_read(file_id)?;
-            let file_size = self.manager.io().file_size(fd)?;
+            let handle = self.manager.open_for_read(file_id)?;
+            let file_size = self.manager.io().file_size(handle.fd())?;
            entries.sort_by_key(|(_, loc)| loc.offset);
            entries.into_iter().try_for_each(|(orig_idx, loc)| {
-                let data = self.decode_and_validate(fd, file_size, loc)?;
+                let data = self.decode_and_validate(handle.fd(), file_size, loc)?;
                results[orig_idx] = Some(data);
                Ok::<_, ReadError>(())
            })
@@ -116,9 +116,9 @@ impl<S: StorageIO> BlockStoreReader<S> {
    }

    fn read_block_at(&self, location: BlockLocation) -> Result<Bytes, ReadError> {
-        let fd = self.manager.open_for_read(location.file_id)?;
-        let file_size = self.manager.io().file_size(fd)?;
-        self.decode_and_validate(fd, file_size, location)
+        let handle = self.manager.open_for_read(location.file_id)?;
+        let file_size = self.manager.io().file_size(handle.fd())?;
+        self.decode_and_validate(handle.fd(), file_size, location)
    }

    fn decode_and_validate(
@@ -127,26 +127,37 @@ impl<S: StorageIO> BlockStoreReader<S> {
        file_size: u64,
        location: BlockLocation,
    ) -> Result<Bytes, ReadError> {
-        match decode_block_record(self.manager.io(), fd, location.offset, file_size)? {
-            Some(ReadBlockRecord::Valid { data, .. })
-                if data.len() == location.length.raw() as usize =>
-            {
-                Ok(Bytes::from(data))
-            }
-            Some(ReadBlockRecord::Valid { .. }) => Err(ReadError::Corrupted {
-                file_id: location.file_id,
-                offset: location.offset,
-            }),
-            Some(ReadBlockRecord::Corrupted { offset } | ReadBlockRecord::Truncated { offset }) => {
-                Err(ReadError::Corrupted {
+        let attempt_once = || -> Result<Bytes, ReadError> {
+            match decode_block_record(self.manager.io(), fd, location.offset, file_size)? {
+                Some(ReadBlockRecord::Valid { data, .. })
+                    if data.len() == location.length.raw() as usize =>
+                {
+                    Ok(Bytes::from(data))
+                }
+                Some(ReadBlockRecord::Valid { .. }) => Err(ReadError::Corrupted {
+                    file_id: location.file_id,
+                    offset: location.offset,
+                }),
+                Some(
+                    ReadBlockRecord::Corrupted { offset } | ReadBlockRecord::Truncated { offset },
+                ) => Err(ReadError::Corrupted {
                    file_id: location.file_id,
                    offset,
-                })
+                }),
+                None => Err(ReadError::Corrupted {
+                    file_id: location.file_id,
+                    offset: location.offset,
+                }),
            }
-            None => Err(ReadError::Corrupted {
-                file_id: location.file_id,
-                offset: location.offset,
-            }),
-        }
+        };
+        (0..READ_RETRY_ATTEMPTS.saturating_sub(1))
+            .find_map(|_| match attempt_once() {
+                Ok(bytes) => Some(Ok(bytes)),
+                Err(ReadError::Corrupted { .. }) => None,
+                Err(e) => Some(Err(e)),
+            })
+            .unwrap_or_else(attempt_once)
    }
 }
+
+const READ_RETRY_ATTEMPTS: u32 = 4;
@@ -8,12 +8,11 @@ use cid::Cid;
 use jacquard_repo::error::RepoError;
 use jacquard_repo::repo::CommitData;
 use jacquard_repo::storage::BlockStore;
-use multihash::Multihash;
-use sha2::{Digest, Sha256};

 use crate::fsync_order::PostBlockstoreHook;
 use crate::io::{OpenOptions, RealIO, StorageIO};

+use super::cid_util::hash_to_cid;
 use super::compaction::CompactionError;
 use super::data_file::{BLOCK_RECORD_OVERHEAD, CID_SIZE, ReadBlockRecord};
 use super::group_commit::{CommitError, CommitRequest, GroupCommitConfig, GroupCommitWriter};
@@ -21,13 +20,10 @@ use super::hash_index::BlockIndex;
 use super::manager::DataFileManager;
 use super::reader::{BlockStoreReader, ReadError};
 use super::types::{
-    BlockLength, BlockLocation, BlockOffset, CollectionResult, CompactionResult, DataFileId,
-    EpochCounter, LivenessInfo, WallClockMs, WriteCursor,
+    BlockLocation, BlockOffset, CollectionResult, CompactionResult, DataFileId, EpochCounter,
+    LivenessInfo, WallClockMs,
 };

-const DAG_CBOR_CODEC: u64 = 0x71;
-const SHA2_256_CODE: u64 = 0x12;
-
 fn cid_to_bytes(cid: &Cid) -> Result<[u8; CID_SIZE], RepoError> {
    let raw = cid.to_bytes();
    let len = raw.len();
@@ -41,20 +37,6 @@ fn cid_to_bytes(cid: &Cid) -> Result<[u8; CID_SIZE], RepoError> {
    })
 }

-fn hash_and_cid(data: &[u8]) -> Result<Cid, RepoError> {
-    let mut hasher = Sha256::new();
-    hasher.update(data);
-    let hash = hasher.finalize();
-    let multihash = Multihash::wrap(SHA2_256_CODE, &hash).map_err(|e| {
-        RepoError::storage(io::Error::new(io::ErrorKind::InvalidData, e.to_string()))
-    })?;
-    Ok(Cid::new_v1(DAG_CBOR_CODEC, multihash))
-}
-
-fn block_index_err_to_repo(e: super::hash_index::BlockIndexError) -> RepoError {
-    RepoError::storage(io::Error::other(e.to_string()))
-}
-
 fn commit_error_to_repo(e: CommitError) -> RepoError {
    match e {
        CommitError::Io(io_err) => {
@@ -65,6 +47,10 @@ fn commit_error_to_repo(e: CommitError) -> RepoError {
            io::ErrorKind::BrokenPipe,
            "blockstore commit channel closed",
        )),
+        CommitError::VerifyFailed { file_id, offset } => RepoError::storage(io::Error::new(
+            io::ErrorKind::InvalidData,
+            format!("post-sync verify failed at {file_id}:{}", offset.raw()),
+        )),
    }
 }

@@ -123,15 +109,26 @@ impl Drop for QuiesceGuard {
    }
 }

-#[derive(Clone)]
-pub struct TranquilBlockStore {
+pub struct TranquilBlockStore<S: StorageIO + Send + Sync + 'static = RealIO> {
    writer: Arc<WriterHandle>,
-    reader: Arc<BlockStoreReader<RealIO>>,
+    reader: Arc<BlockStoreReader<S>>,
    index: Arc<BlockIndex>,
    epoch: EpochCounter,
    data_dir: PathBuf,
 }

+impl<S: StorageIO + Send + Sync + 'static> Clone for TranquilBlockStore<S> {
+    fn clone(&self) -> Self {
+        Self {
+            writer: Arc::clone(&self.writer),
+            reader: Arc::clone(&self.reader),
+            index: Arc::clone(&self.index),
+            epoch: self.epoch.clone(),
+            data_dir: self.data_dir.clone(),
+        }
+    }
+}
+
 struct WriterHandle {
    inner: parking_lot::Mutex<Option<GroupCommitWriter>>,
 }
@@ -153,7 +150,7 @@ impl Drop for WriterHandle {
    }
 }

-impl TranquilBlockStore {
+impl TranquilBlockStore<RealIO> {
    pub fn open(config: BlockStoreConfig) -> Result<Self, RepoError> {
        Self::open_with_hook(config, None)
    }
@@ -162,6 +159,26 @@ impl TranquilBlockStore {
        config: BlockStoreConfig,
        post_sync_hook: Option<Arc<dyn PostBlockstoreHook>>,
    ) -> Result<Self, RepoError> {
+        Self::open_with_io_hook(config, RealIO::new, post_sync_hook)
+    }
+}
+
+impl<S: StorageIO + Send + Sync + 'static> TranquilBlockStore<S> {
+    pub fn open_with_io<F>(config: BlockStoreConfig, make_io: F) -> Result<Self, RepoError>
+    where
+        F: Fn() -> S + Send + Sync + Clone + 'static,
+    {
+        Self::open_with_io_hook(config, make_io, None)
+    }
+
+    pub fn open_with_io_hook<F>(
+        config: BlockStoreConfig,
+        make_io: F,
+        post_sync_hook: Option<Arc<dyn PostBlockstoreHook>>,
+    ) -> Result<Self, RepoError>
+    where
+        F: Fn() -> S + Send + Sync + Clone + 'static,
+    {
        if config.data_dir == config.index_dir {
            return Err(RepoError::storage(io::Error::new(
                io::ErrorKind::InvalidInput,
@@ -173,7 +190,7 @@ impl TranquilBlockStore {

        let index = BlockIndex::open(&config.index_dir).map_err(RepoError::storage)?;

-        let io = RealIO::new();
+        let io = make_io();

        let (replayed, file_cursors) = super::hint::replay_hints_into_block_index(
            &io,
@@ -195,8 +212,13 @@ impl TranquilBlockStore {
        let max_file_size = config.max_file_size;
        let shard_count = config.shard_count;
        let data_dir_for_closure = data_dir.clone();
+        let make_io_for_manager = make_io.clone();
        let make_manager = move || {
-            DataFileManager::new(RealIO::new(), data_dir_for_closure.clone(), max_file_size)
+            DataFileManager::new(
+                make_io_for_manager(),
+                data_dir_for_closure.clone(),
+                max_file_size,
+            )
        };

        let checkpoint_epoch = index.loaded_checkpoint_epoch();
@@ -214,7 +236,7 @@ impl TranquilBlockStore {
        let epoch = writer.epoch().clone();

        let manager_for_reader = Arc::new(DataFileManager::new(
-            RealIO::new(),
+            make_io(),
            data_dir.clone(),
            max_file_size,
        ));
@@ -234,7 +256,7 @@ impl TranquilBlockStore {
        })
    }

-    fn recover_from_file_cursors<S: StorageIO>(
+    fn recover_from_file_cursors(
        io: &S,
        data_dir: &Path,
        index: &BlockIndex,
@@ -256,7 +278,7 @@ impl TranquilBlockStore {
        })
    }

-    fn replay_single_file<S: StorageIO>(
+    fn replay_single_file(
        io: &S,
        data_dir: &Path,
        index: &BlockIndex,
@@ -277,19 +299,28 @@ impl TranquilBlockStore {
            Err(e) => return Err(RepoError::storage(e)),
        };

-        let result = Self::scan_and_index(io, index, fd, file_id, start_offset);
+        let hint_path = super::hint::hint_file_path(data_dir, file_id);
+        let hint_exists = io
+            .open(&hint_path, OpenOptions::read_only_existing())
+            .map(|fd| {
+                let _ = io.close(fd);
+            })
+            .is_ok();
+
+        let result = Self::scan_and_index(io, index, fd, file_id, start_offset, hint_exists);

        let _ = io.close(fd);

        result
    }

-    fn scan_and_index<S: StorageIO>(
+    fn scan_and_index(
        io: &S,
        index: &BlockIndex,
        fd: crate::io::FileId,
        file_id: DataFileId,
        start_offset: BlockOffset,
+        hint_exists: bool,
    ) -> Result<(), RepoError> {
        let file_size = io.file_size(fd).map_err(RepoError::storage)?;

@@ -298,7 +329,7 @@ impl TranquilBlockStore {
        }

        let scan_pos = &mut { start_offset };
-        let (recovered_entries, last_valid_end) = std::iter::from_fn(|| {
+        let (scanned_entries, last_valid_end) = std::iter::from_fn(|| {
            match super::data_file::decode_block_record(io, fd, *scan_pos, file_size) {
                Err(e) => {
                    tracing::warn!(
@@ -319,9 +350,10 @@ impl TranquilBlockStore {
                        Ok(n) if n <= super::types::MAX_BLOCK_SIZE => n,
                        _ => return None,
                    };
-                    let length = BlockLength::new(raw_len);
+                    let length = super::types::BlockLength::new(raw_len);
                    let record_size = BLOCK_RECORD_OVERHEAD as u64 + u64::from(raw_len);
-                    *scan_pos = scan_pos.advance(record_size);
+                    let new_end = offset.advance(record_size);
+                    *scan_pos = new_end;
                    Some((
                        cid_bytes,
                        BlockLocation {
@@ -329,6 +361,7 @@ impl TranquilBlockStore {
                            offset,
                            length,
                        },
+                        new_end,
                    ))
                }
                Ok(Some(ReadBlockRecord::Corrupted { .. } | ReadBlockRecord::Truncated { .. })) => {
@@ -338,11 +371,8 @@ impl TranquilBlockStore {
        })
        .fold(
            (Vec::new(), start_offset),
-            |(mut entries, _), (cid_bytes, location)| {
-                let new_end = location
-                    .offset
-                    .advance(BLOCK_RECORD_OVERHEAD as u64 + location.length.as_u64());
-                entries.push((cid_bytes, location));
+            |(mut entries, _), (cid, loc, new_end)| {
+                entries.push((cid, loc));
                (entries, new_end)
            },
        );
@@ -352,28 +382,28 @@ impl TranquilBlockStore {
                file_id = %file_id,
                truncating_from = last_valid_end.raw(),
                file_size,
-                "truncating partial/corrupted tail"
+                scanned_count = scanned_entries.len(),
+                "truncating partial/unacked tail"
            );
            io.truncate(fd, last_valid_end.raw())
                .map_err(RepoError::storage)?;
            io.sync(fd).map_err(RepoError::storage)?;
        }

-        if !recovered_entries.is_empty() {
-            let new_cursor = WriteCursor {
+        if !scanned_entries.is_empty() {
+            tracing::info!(
+                file_id = %file_id,
+                scanned = scanned_entries.len(),
+                hint_exists,
+                "reindexing blocks past hint coverage"
+            );
+            let cursor = super::types::WriteCursor {
                file_id,
                offset: last_valid_end,
            };
-            let inserted = index
-                .batch_put_if_absent(&recovered_entries, new_cursor)
-                .map_err(block_index_err_to_repo)?;
-            tracing::info!(
-                file_id = %file_id,
-                scanned = recovered_entries.len(),
-                inserted,
-                new_cursor_offset = last_valid_end.raw(),
-                "recovery data file scan"
-            );
+            index
+                .batch_put_if_absent(&scanned_entries, cursor)
+                .map_err(|e| RepoError::storage(io::Error::other(e.to_string())))?;
        }

        Ok(())
@@ -594,7 +624,7 @@ impl TranquilBlockStore {
    }
 }

-impl BlockStore for TranquilBlockStore {
+impl<S: StorageIO + Send + Sync + 'static> BlockStore for TranquilBlockStore<S> {
    async fn get(&self, cid: &Cid) -> Result<Option<Bytes>, RepoError> {
        let cid_bytes = cid_to_bytes(cid)?;
        let reader = Arc::clone(&self.reader);
@@ -605,7 +635,7 @@ impl BlockStore for TranquilBlockStore {
    }

    async fn put(&self, data: &[u8]) -> Result<Cid, RepoError> {
-        let cid = hash_and_cid(data)?;
+        let cid = hash_to_cid(data);
        let cid_bytes = cid_to_bytes(&cid)?;
        self.send_put_blocks(vec![(cid_bytes, data.to_vec())])
            .await?;
@@ -666,7 +696,7 @@ impl BlockStore for TranquilBlockStore {
    }
 }

-impl TranquilBlockStore {
+impl<S: StorageIO + Send + Sync + 'static> TranquilBlockStore<S> {
    pub async fn decrement_refs(&self, cids: &[Cid]) -> Result<(), RepoError> {
        if cids.is_empty() {
            return Ok(());
@@ -1,7 +1,8 @@
 use std::collections::HashMap;
 use std::io;
 use std::path::{Path, PathBuf};
-use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};

 use parking_lot::RwLock;

@@ -25,17 +26,38 @@ pub fn parse_segment_id(path: &Path) -> Option<SegmentId> {
    (ext == SEGMENT_FILE_EXTENSION).then(|| stem.parse::<u32>().ok().map(SegmentId::new))?
 }

-struct CachedSegmentHandle {
+pub struct CachedSegmentHandle<S: StorageIO> {
    fd: FileId,
-    sealed: bool,
+    io: Arc<S>,
+    sealed: AtomicBool,
    writable: bool,
 }

+impl<S: StorageIO> CachedSegmentHandle<S> {
+    pub fn fd(&self) -> FileId {
+        self.fd
+    }
+
+    pub fn is_sealed(&self) -> bool {
+        self.sealed.load(Ordering::Acquire)
+    }
+
+    pub fn is_writable(&self) -> bool {
+        self.writable
+    }
+}
+
+impl<S: StorageIO> Drop for CachedSegmentHandle<S> {
+    fn drop(&mut self) {
+        let _ = self.io.close(self.fd);
+    }
+}
+
 pub struct SegmentManager<S: StorageIO> {
-    io: S,
+    io: Arc<S>,
    segments_dir: PathBuf,
    max_segment_size: u64,
-    handles: RwLock<HashMap<SegmentId, CachedSegmentHandle>>,
+    handles: RwLock<HashMap<SegmentId, Arc<CachedSegmentHandle<S>>>>,
    retention_epoch: AtomicU64,
 }

@@ -51,7 +73,7 @@ impl<S: StorageIO> SegmentManager<S> {
        );
        io.mkdir(&segments_dir)?;
        Ok(Self {
-            io,
+            io: Arc::new(io),
            segments_dir,
            max_segment_size,
            handles: RwLock::new(HashMap::new()),
@@ -60,7 +82,7 @@ impl<S: StorageIO> SegmentManager<S> {
    }

    pub fn io(&self) -> &S {
-        &self.io
+        self.io.as_ref()
    }

    pub fn segments_dir(&self) -> &Path {
@@ -92,52 +114,51 @@ impl<S: StorageIO> SegmentManager<S> {
        Ok(ids)
    }

-    pub fn open_for_read(&self, id: SegmentId) -> io::Result<FileId> {
+    pub fn open_for_read(&self, id: SegmentId) -> io::Result<Arc<CachedSegmentHandle<S>>> {
        if let Some(entry) = self.handles.read().get(&id) {
-            return Ok(entry.fd);
+            return Ok(Arc::clone(entry));
        }
        let path = self.segment_path(id);
        let fd = self.io.open(&path, OpenOptions::read_only_existing())?;
        let mut cache = self.handles.write();
-        match cache.get(&id) {
+        match cache.get(&id).cloned() {
            Some(entry) => {
                let _ = self.io.close(fd);
-                Ok(entry.fd)
+                Ok(entry)
            }
            None => {
-                cache.insert(
-                    id,
-                    CachedSegmentHandle {
-                        fd,
-                        sealed: false,
-                        writable: false,
-                    },
-                );
-                Ok(fd)
+                let handle = Arc::new(CachedSegmentHandle {
+                    fd,
+                    io: Arc::clone(&self.io),
+                    sealed: AtomicBool::new(false),
+                    writable: false,
+                });
+                cache.insert(id, Arc::clone(&handle));
+                Ok(handle)
            }
        }
    }

-    pub fn open_for_append(&self, id: SegmentId) -> io::Result<FileId> {
+    pub fn open_for_append(&self, id: SegmentId) -> io::Result<Arc<CachedSegmentHandle<S>>> {
        {
            let cache = self.handles.read();
            if let Some(entry) = cache.get(&id) {
-                if entry.sealed {
+                if entry.is_sealed() {
                    return Err(io::Error::new(
                        io::ErrorKind::InvalidInput,
                        format!("cannot append to sealed segment {id}"),
                    ));
                }
                if entry.writable {
-                    return Ok(entry.fd);
+                    return Ok(Arc::clone(entry));
                }
            }
        }
        let path = self.segment_path(id);
        let fd = self.io.open(&path, OpenOptions::read_write())?;
        let mut cache = self.handles.write();
-        match cache.get(&id) {
-            Some(entry) if entry.sealed => {
+        match cache.get(&id).cloned() {
+            Some(entry) if entry.is_sealed() => {
                let _ = self.io.close(fd);
                Err(io::Error::new(
                    io::ErrorKind::InvalidInput,
@@ -146,31 +167,17 @@ impl<S: StorageIO> SegmentManager<S> {
            }
            Some(entry) if entry.writable => {
                let _ = self.io.close(fd);
-                Ok(entry.fd)
+                Ok(entry)
            }
-            Some(entry) => {
-                let old_fd = entry.fd;
-                cache.insert(
-                    id,
-                    CachedSegmentHandle {
-                        fd,
-                        sealed: false,
-                        writable: true,
-                    },
-                );
-                let _ = self.io.close(old_fd);
-                Ok(fd)
-            }
-            None => {
-                cache.insert(
-                    id,
-                    CachedSegmentHandle {
-                        fd,
-                        sealed: false,
-                        writable: true,
-                    },
-                );
-                Ok(fd)
+            _ => {
+                let handle = Arc::new(CachedSegmentHandle {
+                    fd,
+                    io: Arc::clone(&self.io),
+                    sealed: AtomicBool::new(false),
+                    writable: true,
+                });
+                cache.insert(id, Arc::clone(&handle));
+                Ok(handle)
            }
        }
    }
@@ -179,37 +186,39 @@ impl<S: StorageIO> SegmentManager<S> {
        position.raw() >= self.max_segment_size
    }

-    pub fn prepare_rotation(&self, current_id: SegmentId) -> io::Result<(SegmentId, FileId)> {
+    pub fn prepare_rotation(
+        &self,
+        current_id: SegmentId,
+    ) -> io::Result<(SegmentId, Arc<CachedSegmentHandle<S>>)> {
        let next = current_id.next();
        let path = self.segment_path(next);
        let fd = self.io.open(&path, OpenOptions::read_write())?;
        self.io.truncate(fd, 0)?;
        self.io.sync_dir(&self.segments_dir)?;
-        Ok((next, fd))
+        let handle = Arc::new(CachedSegmentHandle {
+            fd,
+            io: Arc::clone(&self.io),
+            sealed: AtomicBool::new(false),
+            writable: true,
+        });
+        Ok((next, handle))
    }

-    pub fn commit_rotation(&self, new_id: SegmentId, fd: FileId) {
-        self.handles.write().insert(
-            new_id,
-            CachedSegmentHandle {
-                fd,
-                sealed: false,
-                writable: true,
-            },
-        );
+    pub fn commit_rotation(&self, new_id: SegmentId, handle: &Arc<CachedSegmentHandle<S>>) {
+        self.handles.write().insert(new_id, Arc::clone(handle));
    }

    pub fn seal_segment(&self, id: SegmentId, index: &SegmentIndex) -> io::Result<()> {
        let path = self.index_path(id);
-        index.save(&self.io, &path)?;
-        let mut cache = self.handles.write();
-        let entry = cache.get_mut(&id).ok_or_else(|| {
+        index.save(self.io.as_ref(), &path)?;
+        let cache = self.handles.read();
+        let entry = cache.get(&id).ok_or_else(|| {
            io::Error::new(
                io::ErrorKind::InvalidInput,
                format!("seal_segment: segment {id} not in handle cache"),
            )
        })?;
-        entry.sealed = true;
+        entry.sealed.store(true, Ordering::Release);
        Ok(())
    }

@@ -217,22 +226,16 @@ impl<S: StorageIO> SegmentManager<S> {
        self.handles
            .read()
            .get(&id)
-            .is_some_and(|entry| entry.sealed)
+            .is_some_and(|entry| entry.is_sealed())
    }

-    pub fn rollback_rotation(&self, new_id: SegmentId, fd: FileId) {
-        let _ = self.io.close(fd);
+    pub fn rollback_rotation(&self, new_id: SegmentId) {
        self.handles.write().remove(&new_id);
        let _ = self.io.delete(&self.segment_path(new_id));
    }

    pub fn delete_segment(&self, id: SegmentId) -> io::Result<()> {
-        {
-            let mut cache = self.handles.write();
-            if let Some(entry) = cache.remove(&id) {
-                let _ = self.io.close(entry.fd);
-            }
-        }
+        self.handles.write().remove(&id);
        [self.index_path(id), self.sidecar_path(id)]
            .iter()
            .try_for_each(|path| match self.io.delete(path) {
@@ -255,15 +258,7 @@ impl<S: StorageIO> SegmentManager<S> {
    }

    pub fn shutdown(&self) {
-        self.handles.write().drain().for_each(|(_, handle)| {
-            let _ = self.io.close(handle.fd);
-        });
-    }
-}
-
-impl<S: StorageIO> Drop for SegmentManager<S> {
-    fn drop(&mut self) {
-        self.shutdown();
+        self.handles.write().clear();
    }
 }

@@ -329,7 +324,7 @@ mod tests {
    #[test]
    fn open_for_append_creates_file() {
        let mgr = setup_manager(1024);
-        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
        assert_eq!(mgr.io().file_size(fd).unwrap(), 0);
    }

@@ -342,24 +337,24 @@ mod tests {
    #[test]
    fn handle_cache_returns_same_fd() {
        let mgr = setup_manager(1024);
-        let fd1 = mgr.open_for_append(SegmentId::new(1)).unwrap();
-        let fd2 = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd1 = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        let fd2 = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
        assert_eq!(fd1, fd2);
    }

    #[test]
    fn open_for_read_uses_cache_from_append() {
        let mgr = setup_manager(1024);
-        let fd_write = mgr.open_for_append(SegmentId::new(1)).unwrap();
-        let fd_read = mgr.open_for_read(SegmentId::new(1)).unwrap();
+        let fd_write = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        let fd_read = mgr.open_for_read(SegmentId::new(1)).unwrap().fd();
        assert_eq!(fd_write, fd_read);
    }

    #[test]
    fn list_segments_finds_segment_files() {
        let mgr = setup_manager(1024);
-        mgr.open_for_append(SegmentId::new(1)).unwrap();
-        mgr.open_for_append(SegmentId::new(3)).unwrap();
+        mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        mgr.open_for_append(SegmentId::new(3)).unwrap().fd();

        let segments = mgr.list_segments().unwrap();
        assert_eq!(segments, vec![SegmentId::new(1), SegmentId::new(3)]);
@@ -368,7 +363,7 @@ mod tests {
    #[test]
    fn list_segments_ignores_non_segment_files() {
        let mgr = setup_manager(1024);
-        mgr.open_for_append(SegmentId::new(1)).unwrap();
+        mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
        mgr.io()
            .open(Path::new("/segments/notes.txt"), OpenOptions::read_write())
            .unwrap();
@@ -380,7 +375,7 @@ mod tests {
    #[test]
    fn list_segments_ignores_index_files() {
        let mgr = setup_manager(1024);
-        mgr.open_for_append(SegmentId::new(1)).unwrap();
+        mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
        mgr.io()
            .open(
                Path::new("/segments/00000001.tqi"),
@@ -395,9 +390,9 @@ mod tests {
    #[test]
    fn list_segments_sorted_ascending() {
        let mgr = setup_manager(1024);
-        mgr.open_for_append(SegmentId::new(5)).unwrap();
-        mgr.open_for_append(SegmentId::new(1)).unwrap();
-        mgr.open_for_append(SegmentId::new(3)).unwrap();
+        mgr.open_for_append(SegmentId::new(5)).unwrap().fd();
+        mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        mgr.open_for_append(SegmentId::new(3)).unwrap().fd();

        let segments = mgr.list_segments().unwrap();
        assert_eq!(
@@ -418,23 +413,24 @@ mod tests {
    #[test]
    fn rotation_lifecycle_prepare_commit() {
        let mgr = setup_manager(1024);
-        let _fd0 = mgr.open_for_append(SegmentId::new(1)).unwrap();
-        let (next_id, next_fd) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
+        let _h0 = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        let (next_id, next_handle) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
        assert_eq!(next_id, SegmentId::new(2));
-        assert_eq!(mgr.io().file_size(next_fd).unwrap(), 0);
-        mgr.commit_rotation(next_id, next_fd);
-        assert_eq!(mgr.open_for_read(next_id).unwrap(), next_fd);
+        assert_eq!(mgr.io().file_size(next_handle.fd()).unwrap(), 0);
+        mgr.commit_rotation(next_id, &next_handle);
+        assert_eq!(mgr.open_for_read(next_id).unwrap().fd(), next_handle.fd());
    }

    #[test]
    fn rotation_rollback_cleans_up() {
        let mgr = setup_manager(1024);
-        let _fd0 = mgr.open_for_append(SegmentId::new(1)).unwrap();
-        let (next_id, next_fd) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
-        mgr.commit_rotation(next_id, next_fd);
+        let _h0 = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        let (next_id, next_handle) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
+        mgr.commit_rotation(next_id, &next_handle);

-        assert_eq!(mgr.open_for_read(next_id).unwrap(), next_fd);
-        mgr.rollback_rotation(next_id, next_fd);
+        assert_eq!(mgr.open_for_read(next_id).unwrap().fd(), next_handle.fd());
+        drop(next_handle);
+        mgr.rollback_rotation(next_id);

        let segments = mgr.list_segments().unwrap();
        assert_eq!(segments, vec![SegmentId::new(1)]);
@@ -443,7 +439,7 @@ mod tests {
    #[test]
    fn seal_segment_persists_index_and_marks_sealed() {
        let mgr = setup_manager(64 * 1024);
-        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
        let mut writer = SegmentWriter::new(
            mgr.io(),
            fd,
@@ -476,7 +472,7 @@ mod tests {
    #[test]
    fn delete_segment_removes_files_and_handle() {
        let mgr = setup_manager(64 * 1024);
-        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
        let mut writer = SegmentWriter::new(
            mgr.io(),
            fd,
@@ -507,9 +503,9 @@ mod tests {
        let mgr = setup_manager(1024);
        assert_eq!(mgr.oldest_segment().unwrap(), None);

-        mgr.open_for_append(SegmentId::new(3)).unwrap();
-        mgr.open_for_append(SegmentId::new(1)).unwrap();
-        mgr.open_for_append(SegmentId::new(5)).unwrap();
+        mgr.open_for_append(SegmentId::new(3)).unwrap().fd();
+        mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        mgr.open_for_append(SegmentId::new(5)).unwrap().fd();

        assert_eq!(mgr.oldest_segment().unwrap(), Some(SegmentId::new(1)));
    }
@@ -524,7 +520,7 @@ mod tests {
    fn rotate_and_write_across_segments() {
        let mgr = setup_manager(1024);

-        let fd1 = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd1 = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
        let mut writer1 = SegmentWriter::new(
            mgr.io(),
            fd1,
@@ -538,8 +534,9 @@ mod tests {
            .unwrap();
        writer1.sync(mgr.io()).unwrap();

-        let (id2, fd2) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
-        mgr.commit_rotation(id2, fd2);
+        let (id2, handle2) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
+        let fd2 = handle2.fd();
+        mgr.commit_rotation(id2, &handle2);

        let mut writer2 =
            SegmentWriter::new(mgr.io(), fd2, id2, EventSequence::new(2), MAX_EVENT_PAYLOAD)
@@ -549,7 +546,7 @@ mod tests {
            .unwrap();
        writer2.sync(mgr.io()).unwrap();

-        let fd1_read = mgr.open_for_read(SegmentId::new(1)).unwrap();
+        let fd1_read = mgr.open_for_read(SegmentId::new(1)).unwrap().fd();
        let events1 = crate::eventlog::SegmentReader::open(mgr.io(), fd1_read, MAX_EVENT_PAYLOAD)
            .unwrap()
            .valid_prefix()
@@ -557,7 +554,7 @@ mod tests {
        assert_eq!(events1.len(), 1);
        assert_eq!(events1[0].payload, b"first segment");

-        let fd2_read = mgr.open_for_read(id2).unwrap();
+        let fd2_read = mgr.open_for_read(id2).unwrap().fd();
        let events2 = crate::eventlog::SegmentReader::open(mgr.io(), fd2_read, MAX_EVENT_PAYLOAD)
            .unwrap()
            .valid_prefix()
@@ -569,7 +566,7 @@ mod tests {
    #[test]
    fn seal_then_append_errors() {
        let mgr = setup_manager(64 * 1024);
-        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
        SegmentWriter::new(
            mgr.io(),
            fd,
@@ -596,9 +593,9 @@ mod tests {
    #[test]
    fn multiple_deletions_increment_epoch() {
        let mgr = setup_manager(1024);
-        mgr.open_for_append(SegmentId::new(1)).unwrap();
-        mgr.open_for_append(SegmentId::new(2)).unwrap();
-        mgr.open_for_append(SegmentId::new(3)).unwrap();
+        mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        mgr.open_for_append(SegmentId::new(2)).unwrap().fd();
+        mgr.open_for_append(SegmentId::new(3)).unwrap().fd();

        assert_eq!(mgr.retention_epoch(), 0);
        mgr.delete_segment(SegmentId::new(1)).unwrap();
@@ -610,7 +607,7 @@ mod tests {
    #[test]
    fn open_for_read_does_not_infer_sealed_from_index_file() {
        let mgr = setup_manager(64 * 1024);
-        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
        let mut writer = SegmentWriter::new(
            mgr.io(),
            fd,
@@ -630,26 +627,26 @@ mod tests {

        mgr.handles.write().remove(&SegmentId::new(1));

-        let _read_fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
+        let _read_fd = mgr.open_for_read(SegmentId::new(1)).unwrap().fd();
        assert!(!mgr.is_sealed(SegmentId::new(1)));
    }

    #[test]
    fn open_for_read_unsealed_allows_append() {
        let mgr = setup_manager(1024);
-        let _fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let _fd = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();

        mgr.handles.write().remove(&SegmentId::new(1));

-        let _read_fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
+        let _read_fd = mgr.open_for_read(SegmentId::new(1)).unwrap().fd();
        assert!(!mgr.is_sealed(SegmentId::new(1)));
    }

    #[test]
    fn shutdown_clears_handles() {
        let mgr = setup_manager(1024);
-        mgr.open_for_append(SegmentId::new(1)).unwrap();
-        mgr.open_for_append(SegmentId::new(2)).unwrap();
+        mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        mgr.open_for_append(SegmentId::new(2)).unwrap().fd();

        mgr.shutdown();
        assert!(mgr.handles.read().is_empty());
@@ -665,7 +662,7 @@ mod tests {
    #[test]
    fn prepare_rotation_truncates_stale_file() {
        let mgr = setup_manager(1024);
-        let _fd0 = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let _fd0 = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();

        let stale_path = mgr.segment_path(SegmentId::new(2));
        let stale_fd = mgr
@@ -677,23 +674,23 @@ mod tests {
        assert_eq!(mgr.io().file_size(stale_fd).unwrap(), 4096);
        mgr.io().close(stale_fd).unwrap();

-        let (next_id, next_fd) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
+        let (next_id, next_handle) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
        assert_eq!(next_id, SegmentId::new(2));
-        assert_eq!(mgr.io().file_size(next_fd).unwrap(), 0);
+        assert_eq!(mgr.io().file_size(next_handle.fd()).unwrap(), 0);
    }

    #[test]
    fn open_for_append_upgrades_read_only_handle() {
        let mgr = setup_manager(1024);
-        let fd_append = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd_append = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();

        mgr.handles.write().remove(&SegmentId::new(1));

-        let fd_read = mgr.open_for_read(SegmentId::new(1)).unwrap();
+        let fd_read = mgr.open_for_read(SegmentId::new(1)).unwrap().fd();
        assert_ne!(fd_read, fd_append);
        assert!(!mgr.handles.read().get(&SegmentId::new(1)).unwrap().writable);

-        let fd_upgraded = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd_upgraded = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
        assert_ne!(fd_upgraded, fd_read);
        assert!(mgr.handles.read().get(&SegmentId::new(1)).unwrap().writable);
    }
@@ -411,8 +411,8 @@ impl<S: StorageIO + 'static> EventLog<S> {
    pub fn disk_usage(&self) -> io::Result<u64> {
        let segments = self.manager.list_segments()?;
        segments.iter().try_fold(0u64, |acc, &id| {
-            let fd = self.manager.open_for_read(id)?;
-            let size = self.manager.io().file_size(fd)?;
+            let handle = self.manager.open_for_read(id)?;
+            let size = self.manager.io().file_size(handle.fd())?;
            Ok(acc.saturating_add(size))
        })
    }
@@ -145,10 +145,10 @@ impl<S: StorageIO> EventLogReader<S> {
    }

    fn rebuild_index(&self, segment_id: SegmentId) -> io::Result<SegmentIndex> {
-        let fd = self.manager.open_for_read(segment_id)?;
+        let handle = self.manager.open_for_read(segment_id)?;
        let (idx, _) = rebuild_from_segment(
            self.manager.io(),
-            fd,
+            handle.fd(),
            DEFAULT_INDEX_INTERVAL,
            self.max_payload,
        )?;
@@ -243,8 +243,8 @@ impl<S: StorageIO> EventLogReader<S> {
            return Ok(Arc::clone(m));
        }

-        let fd = self.manager.open_for_read(segment_id)?;
-        let mapped = self.manager.io().mmap_file(fd)?;
+        let handle = self.manager.open_for_read(segment_id)?;
+        let mapped = self.manager.io().mmap_file(handle.fd())?;
        let arc = Arc::new(mapped);
        self.mmaps.write().insert(segment_id, Arc::clone(&arc));
        Ok(arc)
@@ -269,10 +269,10 @@ impl<S: StorageIO> EventLogReader<S> {
                predicate,
            )
        } else {
-            let fd = self.manager.open_for_read(segment_id)?;
-            let file_size = self.manager.io().file_size(fd)?;
+            let handle = self.manager.open_for_read(segment_id)?;
+            let file_size = self.manager.io().file_size(handle.fd())?;
            self.scan_direct(
-                fd,
+                handle.fd(),
                file_size,
                start_offset,
                start_seq,
@@ -513,8 +513,8 @@ impl<S: StorageIO> EventLogReader<S> {
    }

    fn rebuild_sidecar(&self, segment_id: SegmentId) -> io::Result<SidecarIndex> {
-        let fd = self.manager.open_for_read(segment_id)?;
-        let sidecar = build_sidecar_from_segment(self.manager.io(), fd, self.max_payload)?;
+        let handle = self.manager.open_for_read(segment_id)?;
+        let sidecar = build_sidecar_from_segment(self.manager.io(), handle.fd(), self.max_payload)?;
        let _ = sidecar.save(self.manager.io(), &self.manager.sidecar_path(segment_id));
        Ok(sidecar)
    }
@@ -552,7 +552,7 @@ fn decode_mmap_event(
            segment = %segment_id,
            offset = raw,
            file_size,
-            "decode offset past file size (corrupt index?)"
+            "decode offset past file size, index likely corrupt"
        );
        return Ok(MmapDecodeResult::Corrupted);
    }
@@ -66,9 +66,10 @@ impl<S: StorageIO> EventLogWriter<S> {
        index_interval: usize,
        max_payload: u32,
    ) -> io::Result<Self> {
-        let fd = manager.open_for_append(segment_id)?;
-        manager.io().truncate(fd, 0)?;
-        let writer = SegmentWriter::new(manager.io(), fd, segment_id, next_seq, max_payload)?;
+        let handle = manager.open_for_append(segment_id)?;
+        manager.io().truncate(handle.fd(), 0)?;
+        let writer =
+            SegmentWriter::new(manager.io(), handle.fd(), segment_id, next_seq, max_payload)?;
        writer.sync(manager.io())?;
        manager.io().sync_dir(manager.segments_dir())?;

@@ -93,7 +94,8 @@ impl<S: StorageIO> EventLogWriter<S> {
        index_interval: usize,
        max_payload: u32,
    ) -> io::Result<Self> {
-        let fd = manager.open_for_append(active_id)?;
+        let handle = manager.open_for_append(active_id)?;
+        let fd = handle.fd();

        let (index, last_seq_in_active) = match rebuild_from_segment(
            manager.io(),
@@ -305,14 +307,14 @@ impl<S: StorageIO> EventLogWriter<S> {

        match self.build_sidecar_for_segment(old_id) {
            Ok(()) => {}
-            Err(e) => warn!(segment = %old_id, error = %e, "sidecar build failed (non-fatal)"),
+            Err(e) => warn!(segment = %old_id, error = %e, "non-fatal sidecar build failure"),
        }

-        let (new_id, new_fd) = self.manager.prepare_rotation(old_id)?;
+        let (new_id, new_handle) = self.manager.prepare_rotation(old_id)?;

        match SegmentWriter::new::<S>(
            self.manager.io(),
-            new_fd,
+            new_handle.fd(),
            new_id,
            self.next_seq,
            self.max_payload,
@@ -322,11 +324,12 @@ impl<S: StorageIO> EventLogWriter<S> {
                self.active_index = SegmentIndex::new();
                self.event_count_in_segment = 0;
                self.last_event_offset = None;
-                self.manager.commit_rotation(new_id, new_fd);
+                self.manager.commit_rotation(new_id, &new_handle);
                Ok(Some(old_id))
            }
            Err(e) => {
-                self.manager.rollback_rotation(new_id, new_fd);
+                drop(new_handle);
+                self.manager.rollback_rotation(new_id);
                Err(e)
            }
        }
@@ -361,8 +364,8 @@ impl<S: StorageIO> EventLogWriter<S> {
    }

    fn build_sidecar_for_segment(&self, segment_id: SegmentId) -> io::Result<()> {
-        let fd = self.manager.open_for_read(segment_id)?;
-        let sidecar = build_sidecar_from_segment(self.manager.io(), fd, self.max_payload)?;
+        let handle = self.manager.open_for_read(segment_id)?;
+        let sidecar = build_sidecar_from_segment(self.manager.io(), handle.fd(), self.max_payload)?;
        let path = self.manager.sidecar_path(segment_id);
        sidecar.save(self.manager.io(), &path)
    }
@@ -397,9 +400,13 @@ fn find_last_seq_from_segments<S: StorageIO>(
            Ok(Some(idx)) => Ok(idx.last_seq()),
            Err(e) if e.kind() != io::ErrorKind::InvalidData => Err(e),
            _ => {
-                let fd = manager.open_for_read(seg_id)?;
-                let (_, last_seq) =
-                    rebuild_from_segment(manager.io(), fd, DEFAULT_INDEX_INTERVAL, max_payload)?;
+                let handle = manager.open_for_read(seg_id)?;
+                let (_, last_seq) = rebuild_from_segment(
+                    manager.io(),
+                    handle.fd(),
+                    DEFAULT_INDEX_INTERVAL,
+                    max_payload,
+                )?;
                Ok(last_seq)
            }
        }
@@ -551,7 +558,7 @@ mod tests {
        assert_eq!(writer.synced_seq(), EventSequence::new(5));
        assert_eq!(writer.active_segment_id(), SegmentId::new(1));

-        let fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
+        let fd = mgr.open_for_read(SegmentId::new(1)).unwrap().fd();
        let events = SegmentReader::open(mgr.io(), fd, MAX_EVENT_PAYLOAD)
            .unwrap()
            .valid_prefix()
@@ -796,7 +803,7 @@ mod tests {
                .unwrap();
        assert_eq!(writer.next_seq, EventSequence::new(3));

-        let fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
+        let fd = mgr.open_for_read(SegmentId::new(1)).unwrap().fd();
        let events = SegmentReader::open(mgr.io(), fd, MAX_EVENT_PAYLOAD)
            .unwrap()
            .valid_prefix()
@@ -1003,7 +1010,7 @@ mod tests {
        assert_eq!(seq, EventSequence::new(4));
        writer.sync().unwrap();

-        let fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
+        let fd = mgr.open_for_read(SegmentId::new(1)).unwrap().fd();
        let events = SegmentReader::open(mgr.io(), fd, MAX_EVENT_PAYLOAD)
            .unwrap()
            .valid_prefix()
@@ -0,0 +1,86 @@
+use std::cell::RefCell;
+use std::panic::{AssertUnwindSafe, catch_unwind};
+use std::time::{Duration, Instant};
+
+use rayon::prelude::*;
+use tokio::runtime::Runtime;
+
+use super::invariants::InvariantViolation;
+use super::op::{OpStream, Seed};
+use super::runner::{
+    Gauntlet, GauntletConfig, GauntletReport, OpErrorCount, OpsExecuted, RestartCount,
+};
+
+thread_local! {
+    static RUNTIME: RefCell<Option<Runtime>> = const { RefCell::new(None) };
+}
+
+fn with_runtime<R>(f: impl FnOnce(&Runtime) -> R) -> R {
+    RUNTIME.with(|cell| {
+        let mut slot = cell.borrow_mut();
+        if slot.is_none() {
+            *slot = Some(
+                tokio::runtime::Builder::new_current_thread()
+                    .enable_all()
+                    .build()
+                    .expect("build rt"),
+            );
+        }
+        f(slot.as_ref().expect("runtime present"))
+    })
+}
+
+pub fn run_many<F>(make_config: F, seeds: impl IntoIterator<Item = Seed>) -> Vec<GauntletReport>
+where
+    F: Fn(Seed) -> GauntletConfig + Sync + Send,
+{
+    run_many_timed(make_config, seeds)
+        .into_iter()
+        .map(|(r, _)| r)
+        .collect()
+}
+
+pub fn run_many_timed<F>(
+    make_config: F,
+    seeds: impl IntoIterator<Item = Seed>,
+) -> Vec<(GauntletReport, Duration)>
+where
+    F: Fn(Seed) -> GauntletConfig + Sync + Send,
+{
+    let seeds: Vec<Seed> = seeds.into_iter().collect();
+    seeds
+        .into_par_iter()
+        .map(|s| {
+            let start = Instant::now();
+            let outcome = catch_unwind(AssertUnwindSafe(|| {
+                let cfg = make_config(s);
+                let gauntlet = Gauntlet::new(cfg).expect("build gauntlet");
+                with_runtime(|rt| rt.block_on(gauntlet.run()))
+            }));
+            let report = outcome.unwrap_or_else(|payload| {
+                RUNTIME.with(|cell| cell.borrow_mut().take());
+                panic_report(s, payload)
+            });
+            (report, start.elapsed())
+        })
+        .collect()
+}
+
+fn panic_report(seed: Seed, payload: Box<dyn std::any::Any + Send>) -> GauntletReport {
+    let msg = payload
+        .downcast_ref::<&'static str>()
+        .map(|s| (*s).to_string())
+        .or_else(|| payload.downcast_ref::<String>().cloned())
+        .unwrap_or_else(|| "non-string panic payload".to_string());
+    GauntletReport {
+        seed,
+        ops_executed: OpsExecuted(0),
+        op_errors: OpErrorCount(0),
+        restarts: RestartCount(0),
+        violations: vec![InvariantViolation {
+            invariant: "FarmPanic",
+            detail: msg,
+        }],
+        ops: OpStream::empty(),
+    }
+}
@@ -0,0 +1,448 @@
+use std::env;
+use std::num::NonZeroU32;
+use std::path::{Path, PathBuf};
+use std::process::{Command, Output};
+use std::sync::atomic::{AtomicU64, Ordering};
+
+use tempfile::TempDir;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct UpIntervalSecs(pub NonZeroU32);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct DownIntervalSecs(pub NonZeroU32);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct BackingMegabytes(pub u32);
+
+const fn nz(n: u32) -> NonZeroU32 {
+    match NonZeroU32::new(n) {
+        Some(v) => v,
+        None => panic!("zero interval not permitted"),
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct FlakyConfig {
+    pub up_interval: UpIntervalSecs,
+    pub down_interval: DownIntervalSecs,
+    pub backing_mb: BackingMegabytes,
+}
+
+impl FlakyConfig {
+    pub const fn default_stress() -> Self {
+        Self {
+            up_interval: UpIntervalSecs(nz(8)),
+            down_interval: DownIntervalSecs(nz(2)),
+            backing_mb: BackingMegabytes(256),
+        }
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum FlakyError {
+    #[error("not running as root, EUID != 0")]
+    NotRoot,
+    #[error("tool missing: {0}")]
+    ToolMissing(&'static str),
+    #[error("kernel target dm-flakey unavailable: {0}")]
+    DmFlakeyMissing(String),
+    #[error("{tool} failed: status={status}, stderr={stderr}")]
+    CommandFailed {
+        tool: &'static str,
+        status: String,
+        stderr: String,
+    },
+    #[error("io: {0}")]
+    Io(#[from] std::io::Error),
+}
+
+impl FlakyError {
+    pub const fn is_env_absent(&self) -> bool {
+        matches!(
+            self,
+            Self::NotRoot | Self::ToolMissing(_) | Self::DmFlakeyMissing(_)
+        )
+    }
+}
+
+static MOUNT_COUNTER: AtomicU64 = AtomicU64::new(0);
+
+pub struct FlakyMount {
+    mount_point: TempDir,
+    mapper_name: String,
+    mapper_path: PathBuf,
+    loop_device: PathBuf,
+    backing_file: PathBuf,
+    _backing_tempdir: TempDir,
+}
+
+impl FlakyMount {
+    pub fn try_new(cfg: &FlakyConfig) -> Result<Self, FlakyError> {
+        ensure_root()?;
+        ["losetup", "dmsetup", "mkfs.ext4", "mount", "umount"]
+            .iter()
+            .copied()
+            .try_for_each(ensure_tool)?;
+        probe_dm_flakey()?;
+        let _ = reap_stale_mounts();
+
+        let backing_dir = TempDir::new()?;
+        let backing_file = backing_dir.path().join("backing.img");
+        allocate_backing(&backing_file, cfg.backing_mb)?;
+
+        let loop_device = attach_loop(&backing_file)?;
+        let sectors = sector_count(&loop_device)?;
+
+        if let Err(e) = mkfs_ext4(&loop_device) {
+            let _ = detach_loop(&loop_device);
+            return Err(e);
+        }
+
+        let mapper_name = format!(
+            "tranquil-flaky-{}-{}",
+            std::process::id(),
+            MOUNT_COUNTER.fetch_add(1, Ordering::Relaxed),
+        );
+        match dm_create(
+            &mapper_name,
+            &loop_device,
+            sectors,
+            cfg.up_interval,
+            cfg.down_interval,
+        ) {
+            Ok(()) => {}
+            Err(e) => {
+                let _ = detach_loop(&loop_device);
+                return Err(e);
+            }
+        }
+        let mapper_path = PathBuf::from(format!("/dev/mapper/{mapper_name}"));
+
+        let mount_point = TempDir::new()?;
+        if let Err(e) = mount_ext4(&mapper_path, mount_point.path()) {
+            let _ = dm_remove(&mapper_name);
+            let _ = detach_loop(&loop_device);
+            return Err(e);
+        }
+
+        Ok(Self {
+            mount_point,
+            mapper_name,
+            mapper_path,
+            loop_device,
+            backing_file,
+            _backing_tempdir: backing_dir,
+        })
+    }
+
+    pub fn path(&self) -> &Path {
+        self.mount_point.path()
+    }
+
+    pub fn mapper_name(&self) -> &str {
+        &self.mapper_name
+    }
+
+    pub fn mapper_path(&self) -> &Path {
+        &self.mapper_path
+    }
+
+    pub fn loop_device(&self) -> &Path {
+        &self.loop_device
+    }
+
+    pub fn backing_file(&self) -> &Path {
+        &self.backing_file
+    }
+}
+
+impl Drop for FlakyMount {
+    fn drop(&mut self) {
+        if let Err(e) = umount(self.mount_point.path()) {
+            tracing::warn!(
+                mount = %self.mount_point.path().display(),
+                error = %e,
+                "flaky umount failed, trying lazy unmount",
+            );
+            if let Err(e2) = umount_lazy(self.mount_point.path()) {
+                tracing::warn!(
+                    mount = %self.mount_point.path().display(),
+                    error = %e2,
+                    "flaky lazy unmount also failed, device may leak",
+                );
+            }
+        }
+        if let Err(e) = dm_remove(&self.mapper_name) {
+            tracing::warn!(
+                name = %self.mapper_name,
+                error = %e,
+                "flaky dm remove failed, mapper device may leak",
+            );
+        }
+        if let Err(e) = detach_loop(&self.loop_device) {
+            tracing::warn!(
+                device = %self.loop_device.display(),
+                error = %e,
+                "flaky loop detach failed, loop device may leak",
+            );
+        }
+    }
+}
+
+#[cfg(unix)]
+fn ensure_root() -> Result<(), FlakyError> {
+    if unsafe { libc::geteuid() } == 0 {
+        Ok(())
+    } else {
+        Err(FlakyError::NotRoot)
+    }
+}
+
+#[cfg(not(unix))]
+fn ensure_root() -> Result<(), FlakyError> {
+    Err(FlakyError::NotRoot)
+}
+
+fn ensure_tool(tool: &'static str) -> Result<(), FlakyError> {
+    match find_in_path(tool) {
+        Some(_) => Ok(()),
+        None => Err(FlakyError::ToolMissing(tool)),
+    }
+}
+
+fn find_in_path(tool: &str) -> Option<PathBuf> {
+    let path = env::var_os("PATH")?;
+    env::split_paths(&path).find_map(|dir| {
+        let candidate = dir.join(tool);
+        is_executable_file(&candidate).then_some(candidate)
+    })
+}
+
+#[cfg(unix)]
+fn is_executable_file(p: &Path) -> bool {
+    use std::os::unix::fs::PermissionsExt;
+    std::fs::metadata(p)
+        .map(|m| m.is_file() && (m.permissions().mode() & 0o111) != 0)
+        .unwrap_or(false)
+}
+
+#[cfg(not(unix))]
+fn is_executable_file(p: &Path) -> bool {
+    p.is_file()
+}
+
+fn probe_dm_flakey() -> Result<(), FlakyError> {
+    let out = Command::new("dmsetup").arg("targets").output()?;
+    if !out.status.success() {
+        return Err(FlakyError::DmFlakeyMissing(stringify_output(&out)));
+    }
+    let stdout = String::from_utf8_lossy(&out.stdout);
+    if !stdout.lines().any(|l| l.starts_with("flakey")) {
+        return Err(FlakyError::DmFlakeyMissing(stdout.into_owned()));
+    }
+    Ok(())
+}
+
+fn reap_stale_mounts() -> Result<(), FlakyError> {
+    let out = Command::new("dmsetup")
+        .arg("ls")
+        .arg("--target")
+        .arg("flakey")
+        .output()?;
+    if !out.status.success() {
+        return Ok(());
+    }
+    String::from_utf8_lossy(&out.stdout)
+        .lines()
+        .filter_map(parse_flaky_entry)
+        .filter(|(_, pid)| !pid_alive(*pid))
+        .for_each(|(name, _)| {
+            let loop_device = mapper_backing_loop(&name);
+            if let Err(e) = dm_remove(&name) {
+                tracing::warn!(name, error = %e, "reap: dm_remove stale mapper failed");
+                return;
+            }
+            if let Some(loop_dev) = loop_device
+                && let Err(e) = detach_loop(&loop_dev)
+            {
+                tracing::warn!(
+                    device = %loop_dev.display(),
+                    error = %e,
+                    "reap: detach_loop stale device failed",
+                );
+            }
+        });
+    Ok(())
+}
+
+fn parse_flaky_entry(line: &str) -> Option<(String, u32)> {
+    let name = line.split_whitespace().next()?;
+    let suffix = name.strip_prefix("tranquil-flaky-")?;
+    let pid_str = suffix.split('-').next()?;
+    let pid = pid_str.parse::<u32>().ok()?;
+    Some((name.to_string(), pid))
+}
+
+fn pid_alive(pid: u32) -> bool {
+    Path::new(&format!("/proc/{pid}")).exists()
+}
+
+fn mapper_backing_loop(name: &str) -> Option<PathBuf> {
+    let out = Command::new("dmsetup")
+        .arg("deps")
+        .arg("-o")
+        .arg("devname")
+        .arg(name)
+        .output()
+        .ok()?;
+    if !out.status.success() {
+        return None;
+    }
+    let text = String::from_utf8_lossy(&out.stdout);
+    let inner = text.split('(').nth(1)?;
+    let dev = inner.split(')').next()?.trim();
+    if dev.is_empty() {
+        None
+    } else {
+        Some(PathBuf::from(format!("/dev/{dev}")))
+    }
+}
+
+fn allocate_backing(path: &Path, size: BackingMegabytes) -> Result<(), FlakyError> {
+    let out = Command::new("truncate")
+        .arg("-s")
+        .arg(format!("{}M", size.0))
+        .arg(path)
+        .output()?;
+    check_status("truncate", &out)
+}
+
+fn attach_loop(backing: &Path) -> Result<PathBuf, FlakyError> {
+    let out = Command::new("losetup")
+        .arg("--find")
+        .arg("--show")
+        .arg(backing)
+        .output()?;
+    check_status("losetup", &out)?;
+    let raw = String::from_utf8_lossy(&out.stdout).trim().to_string();
+    if raw.is_empty() {
+        return Err(FlakyError::CommandFailed {
+            tool: "losetup",
+            status: "exit 0".to_string(),
+            stderr: "no device path on stdout".to_string(),
+        });
+    }
+    Ok(PathBuf::from(raw))
+}
+
+fn detach_loop(device: &Path) -> Result<(), FlakyError> {
+    let out = Command::new("losetup").arg("-d").arg(device).output()?;
+    check_status("losetup -d", &out)
+}
+
+fn sector_count(device: &Path) -> Result<u64, FlakyError> {
+    let out = Command::new("blockdev")
+        .arg("--getsz")
+        .arg(device)
+        .output()?;
+    check_status("blockdev", &out)?;
+    let raw = String::from_utf8_lossy(&out.stdout).trim().to_string();
+    raw.parse::<u64>().map_err(|_| FlakyError::CommandFailed {
+        tool: "blockdev",
+        status: "exit 0".to_string(),
+        stderr: format!("could not parse sector count: {raw:?}"),
+    })
+}
+
+fn dm_create(
+    name: &str,
+    loop_device: &Path,
+    sectors: u64,
+    up: UpIntervalSecs,
+    down: DownIntervalSecs,
+) -> Result<(), FlakyError> {
+    let table = format!(
+        "0 {sectors} flakey {} 0 {} {}",
+        loop_device.display(),
+        up.0.get(),
+        down.0.get(),
+    );
+    let mut child = Command::new("dmsetup")
+        .arg("create")
+        .arg(name)
+        .stdin(std::process::Stdio::piped())
+        .stdout(std::process::Stdio::piped())
+        .stderr(std::process::Stdio::piped())
+        .spawn()?;
+    if let Some(stdin) = child.stdin.as_mut() {
+        use std::io::Write;
+        stdin.write_all(table.as_bytes())?;
+    }
+    let out = child.wait_with_output()?;
+    check_status("dmsetup create", &out)
+}
+
+fn dm_remove(name: &str) -> Result<(), FlakyError> {
+    let out = Command::new("dmsetup")
+        .arg("remove")
+        .arg("--retry")
+        .arg(name)
+        .output()?;
+    check_status("dmsetup remove", &out)
+}
+
+fn mkfs_ext4(device: &Path) -> Result<(), FlakyError> {
+    let out = Command::new("mkfs.ext4")
+        .arg("-q")
+        .arg("-F")
+        .arg(device)
+        .output()?;
+    check_status("mkfs.ext4", &out)
+}
+
+fn mount_ext4(device: &Path, target: &Path) -> Result<(), FlakyError> {
+    let out = Command::new("mount")
+        .arg("-t")
+        .arg("ext4")
+        .arg(device)
+        .arg(target)
+        .output()?;
+    check_status("mount", &out)
+}
+
+fn umount(target: &Path) -> Result<(), FlakyError> {
+    let out = Command::new("umount").arg(target).output()?;
+    check_status("umount", &out)
+}
+
+fn umount_lazy(target: &Path) -> Result<(), FlakyError> {
+    let out = Command::new("umount").arg("-l").arg(target).output()?;
+    check_status("umount -l", &out)
+}
+
+fn check_status(tool: &'static str, out: &Output) -> Result<(), FlakyError> {
+    if out.status.success() {
+        Ok(())
+    } else {
+        Err(FlakyError::CommandFailed {
+            tool,
+            status: format!("{}", out.status),
+            stderr: String::from_utf8_lossy(&out.stderr).into_owned(),
+        })
+    }
+}
+
+fn stringify_output(out: &Output) -> String {
+    let mut s = String::new();
+    if !out.stdout.is_empty() {
+        s.push_str(&String::from_utf8_lossy(&out.stdout));
+    }
+    if !out.stderr.is_empty() {
+        if !s.is_empty() {
+            s.push('\n');
+        }
+        s.push_str(&String::from_utf8_lossy(&out.stderr));
+    }
+    s
+}
@@ -0,0 +1,783 @@
+use std::collections::{HashMap, HashSet};
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use cid::Cid;
+use jacquard_repo::mst::Mst;
+
+use super::oracle::{Oracle, hex_short, try_cid_to_fixed};
+use crate::blockstore::{
+    BLOCK_HEADER_SIZE, CidBytes, CompactionError, TranquilBlockStore, hash_to_cid_bytes,
+};
+use crate::eventlog::{EventSequence, SegmentId};
+use crate::io::{RealIO, StorageIO};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct InvariantSet(u32);
+
+impl InvariantSet {
+    pub const EMPTY: Self = Self(0);
+    pub const REFCOUNT_CONSERVATION: Self = Self(1 << 0);
+    pub const REACHABILITY: Self = Self(1 << 1);
+    pub const ACKED_WRITE_PERSISTENCE: Self = Self(1 << 2);
+    pub const READ_AFTER_WRITE: Self = Self(1 << 3);
+    pub const RESTART_IDEMPOTENT: Self = Self(1 << 4);
+    pub const COMPACTION_IDEMPOTENT: Self = Self(1 << 5);
+    pub const NO_ORPHAN_FILES: Self = Self(1 << 6);
+    pub const BYTE_BUDGET: Self = Self(1 << 7);
+    pub const MANIFEST_EQUALS_REALITY: Self = Self(1 << 8);
+    pub const CHECKSUM_COVERAGE: Self = Self(1 << 9);
+    pub const MONOTONIC_SEQ: Self = Self(1 << 10);
+    pub const FSYNC_ORDERING: Self = Self(1 << 11);
+    pub const TOMBSTONE_BOUND: Self = Self(1 << 12);
+
+    const ALL_KNOWN: u32 = Self::REFCOUNT_CONSERVATION.0
+        | Self::REACHABILITY.0
+        | Self::ACKED_WRITE_PERSISTENCE.0
+        | Self::READ_AFTER_WRITE.0
+        | Self::RESTART_IDEMPOTENT.0
+        | Self::COMPACTION_IDEMPOTENT.0
+        | Self::NO_ORPHAN_FILES.0
+        | Self::BYTE_BUDGET.0
+        | Self::MANIFEST_EQUALS_REALITY.0
+        | Self::CHECKSUM_COVERAGE.0
+        | Self::MONOTONIC_SEQ.0
+        | Self::FSYNC_ORDERING.0
+        | Self::TOMBSTONE_BOUND.0;
+
+    pub const fn contains(self, other: Self) -> bool {
+        (self.0 & other.0) == other.0
+    }
+
+    pub const fn union(self, other: Self) -> Self {
+        Self(self.0 | other.0)
+    }
+
+    pub const fn without(self, other: Self) -> Self {
+        Self(self.0 & !other.0)
+    }
+
+    pub const fn unknown_bits(self) -> u32 {
+        self.0 & !Self::ALL_KNOWN
+    }
+}
+
+impl std::ops::BitOr for InvariantSet {
+    type Output = Self;
+    fn bitor(self, rhs: Self) -> Self {
+        self.union(rhs)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct InvariantViolation {
+    pub invariant: &'static str,
+    pub detail: String,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct SnapshotEvent {
+    pub seq: EventSequence,
+    pub timestamp_us: u64,
+    pub event_type_raw: u8,
+    pub did_hash: u32,
+}
+
+#[derive(Debug, Clone)]
+pub struct EventLogSnapshot {
+    pub segments_dir: PathBuf,
+    pub max_segment_size: u64,
+    pub synced_seq: EventSequence,
+    pub segments: Vec<SegmentId>,
+    pub events: Vec<SnapshotEvent>,
+    pub segment_last_ts: Vec<(SegmentId, u64)>,
+}
+
+pub struct InvariantCtx<'a, S: StorageIO + Send + Sync + 'static = RealIO> {
+    pub store: &'a Arc<TranquilBlockStore<S>>,
+    pub oracle: &'a Oracle,
+    pub root: Option<Cid>,
+    pub eventlog: Option<&'a EventLogSnapshot>,
+}
+
+#[async_trait]
+pub trait Invariant<S: StorageIO + Send + Sync + 'static>: Send + Sync {
+    fn name(&self) -> &'static str;
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation>;
+}
+
+pub struct RefcountConservation;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for RefcountConservation {
+    fn name(&self) -> &'static str {
+        "RefcountConservation"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let live: Vec<(String, CidBytes)> = ctx.oracle.live_cids_labeled();
+        let live_set: HashSet<CidBytes> = live.iter().map(|(_, c)| *c).collect();
+        let index: HashMap<CidBytes, u32> = ctx
+            .store
+            .block_index()
+            .live_entries_snapshot()
+            .into_iter()
+            .map(|(c, r)| (c, r.raw()))
+            .collect();
+
+        let forward: Vec<String> = live
+            .iter()
+            .filter_map(|(label, cid)| match index.get(cid) {
+                Some(&r) if r >= 1 => None,
+                Some(&r) => Some(format!("{label}: refcount {r}")),
+                None => Some(format!("{label}: missing from index")),
+            })
+            .collect();
+
+        let inverse: Vec<String> = index
+            .iter()
+            .filter(|(cid, _)| !live_set.contains(*cid))
+            .map(|(cid, r)| format!("orphan cid {} refcount {}", hex_short(cid), r))
+            .collect();
+
+        let violations: Vec<String> = forward.into_iter().chain(inverse).collect();
+        if violations.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "RefcountConservation",
+                detail: violations.join("; "),
+            })
+        }
+    }
+}
+
+pub struct Reachability;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for Reachability {
+    fn name(&self) -> &'static str {
+        "Reachability"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let violations: Vec<String> = ctx
+            .oracle
+            .live_cids_labeled()
+            .into_iter()
+            .filter_map(|(label, fixed)| match ctx.store.get_block_sync(&fixed) {
+                Ok(Some(_)) => None,
+                Ok(None) => Some(format!("{label}: missing")),
+                Err(e) => Some(format!("{label}: read error {e}")),
+            })
+            .collect();
+
+        if violations.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "Reachability",
+                detail: violations.join("; "),
+            })
+        }
+    }
+}
+
+pub struct AckedWritePersistence;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for AckedWritePersistence {
+    fn name(&self) -> &'static str {
+        "AckedWritePersistence"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let Some(root) = ctx.root else {
+            if ctx.oracle.live_count() == 0 {
+                return Ok(());
+            }
+            return Err(InvariantViolation {
+                invariant: "AckedWritePersistence",
+                detail: format!(
+                    "oracle has {} live records but reopened store has no root",
+                    ctx.oracle.live_count()
+                ),
+            });
+        };
+        let mst = Mst::load(ctx.store.clone(), root, None);
+        let keys: Vec<String> = ctx
+            .oracle
+            .live_records()
+            .map(|(c, r, _)| format!("{}/{}", c.0, r.0))
+            .collect();
+
+        let mut missing: Vec<String> = Vec::new();
+        for key in &keys {
+            match mst.get(key).await {
+                Ok(Some(_)) => {}
+                Ok(None) => missing.push(format!("{key}: missing after reopen")),
+                Err(e) => missing.push(format!("{key}: mst.get error after reopen: {e}")),
+            }
+        }
+
+        if missing.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "AckedWritePersistence",
+                detail: missing.join("; "),
+            })
+        }
+    }
+}
+
+pub struct ReadAfterWrite;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for ReadAfterWrite {
+    fn name(&self) -> &'static str {
+        "ReadAfterWrite"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let Some(root) = ctx.root else {
+            return Ok(());
+        };
+        let mst = Mst::load(ctx.store.clone(), root, None);
+
+        let entries: Vec<(String, CidBytes)> = ctx
+            .oracle
+            .live_records()
+            .map(|(c, r, v)| (format!("{}/{}", c.0, r.0), *v))
+            .collect();
+
+        let mut violations: Vec<String> = Vec::new();
+        for (key, expected) in &entries {
+            match mst.get(key).await {
+                Ok(Some(cid)) => match try_cid_to_fixed(&cid) {
+                    Ok(actual) if actual == *expected => match ctx.store.get_block_sync(&actual) {
+                        Ok(Some(_)) => {}
+                        Ok(None) => violations.push(format!("{key}: block missing for cid")),
+                        Err(e) => violations.push(format!("{key}: block read error {e}")),
+                    },
+                    Ok(actual) => violations.push(format!(
+                        "{key}: MST cid {} != oracle cid {}",
+                        hex_short(&actual),
+                        hex_short(expected),
+                    )),
+                    Err(e) => {
+                        violations.push(format!("{key}: unexpected CID format from MST: {e}"))
+                    }
+                },
+                Ok(None) => violations.push(format!("{key}: MST returned None")),
+                Err(e) => violations.push(format!("{key}: mst.get error {e}")),
+            }
+        }
+
+        if violations.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "ReadAfterWrite",
+                detail: violations.join("; "),
+            })
+        }
+    }
+}
+
+pub struct CompactionIdempotent;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for CompactionIdempotent {
+    fn name(&self) -> &'static str {
+        "CompactionIdempotent"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let store_a = ctx.store.clone();
+        let first = tokio::task::spawn_blocking(move || compact_by_liveness(&store_a))
+            .await
+            .map_err(|e| InvariantViolation {
+                invariant: "CompactionIdempotent",
+                detail: format!("first compaction join: {e}"),
+            })?;
+        if let Err(e) = first {
+            return Err(InvariantViolation {
+                invariant: "CompactionIdempotent",
+                detail: format!("first compaction: {e}"),
+            });
+        }
+
+        let pre = snapshot(ctx.store);
+
+        let store_b = ctx.store.clone();
+        let second = tokio::task::spawn_blocking(move || compact_by_liveness(&store_b))
+            .await
+            .map_err(|e| InvariantViolation {
+                invariant: "CompactionIdempotent",
+                detail: format!("second compaction join: {e}"),
+            })?;
+        if let Err(e) = second {
+            return Err(InvariantViolation {
+                invariant: "CompactionIdempotent",
+                detail: format!("second compaction: {e}"),
+            });
+        }
+
+        let post = snapshot(ctx.store);
+
+        if pre == post {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "CompactionIdempotent",
+                detail: format!(
+                    "second compaction changed observable state: pre={} entries, post={} entries",
+                    pre.len(),
+                    post.len(),
+                ),
+            })
+        }
+    }
+}
+
+fn snapshot<S: StorageIO + Send + Sync + 'static>(
+    store: &Arc<TranquilBlockStore<S>>,
+) -> Vec<(CidBytes, u32)> {
+    let mut v: Vec<(CidBytes, u32)> = store
+        .block_index()
+        .live_entries_snapshot()
+        .into_iter()
+        .map(|(c, r)| (c, r.raw()))
+        .collect();
+    v.sort_unstable_by_key(|a| a.0);
+    v
+}
+
+const COMPACT_LIVENESS_CEILING: f64 = 0.99;
+
+fn compact_by_liveness<S: StorageIO + Send + Sync + 'static>(
+    store: &TranquilBlockStore<S>,
+) -> Result<(), String> {
+    let liveness = store
+        .compaction_liveness(0)
+        .map_err(|e| format!("compaction_liveness: {e}"))?;
+    let targets: Vec<_> = liveness
+        .iter()
+        .filter(|(_, info)| info.total_blocks > 0 && info.ratio() < COMPACT_LIVENESS_CEILING)
+        .map(|(&fid, _)| fid)
+        .collect();
+    targets
+        .into_iter()
+        .try_for_each(|fid| match store.compact_file(fid, 0) {
+            Ok(_) => Ok(()),
+            Err(CompactionError::ActiveFileCannotBeCompacted) => Ok(()),
+            Err(e) => Err(format!("{fid}: {e}")),
+        })
+}
+
+pub struct NoOrphanFiles;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for NoOrphanFiles {
+    fn name(&self) -> &'static str {
+        "NoOrphanFiles"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let store_c = ctx.store.clone();
+        let result = tokio::task::spawn_blocking(move || {
+            let disk = store_c.list_data_files().map_err(|e| e.to_string())?;
+            let liveness = store_c.compaction_liveness(0).map_err(|e| e.to_string())?;
+            let header = BLOCK_HEADER_SIZE as u64;
+            let orphans: Vec<String> = disk
+                .iter()
+                .filter(|fid| !liveness.contains_key(fid))
+                .filter_map(|fid| {
+                    let path = store_c.data_file_path(*fid);
+                    let size = std::fs::metadata(&path).map(|m| m.len()).unwrap_or(0);
+                    match size > header {
+                        true => Some(format!("{fid} ({size} B)")),
+                        false => None,
+                    }
+                })
+                .collect();
+            Ok::<_, String>(orphans)
+        })
+        .await
+        .map_err(|e| InvariantViolation {
+            invariant: "NoOrphanFiles",
+            detail: format!("join: {e}"),
+        })?;
+
+        let orphans = result.map_err(|e| InvariantViolation {
+            invariant: "NoOrphanFiles",
+            detail: e,
+        })?;
+
+        if orphans.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "NoOrphanFiles",
+                detail: format!("files on disk missing from index: {}", orphans.join(", ")),
+            })
+        }
+    }
+}
+
+pub struct ByteBudget {
+    pub overhead_factor: f64,
+    pub floor_bytes: u64,
+}
+
+impl Default for ByteBudget {
+    fn default() -> Self {
+        Self {
+            overhead_factor: 8.0,
+            floor_bytes: 1 << 20,
+        }
+    }
+}
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for ByteBudget {
+    fn name(&self) -> &'static str {
+        "ByteBudget"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let store = ctx.store.clone();
+        let factor = self.overhead_factor;
+        let floor = self.floor_bytes;
+        tokio::task::spawn_blocking(move || {
+            let liveness = store.compaction_liveness(0).map_err(|e| e.to_string())?;
+            let live: u64 = liveness.values().map(|i| i.live_bytes).sum();
+            let total: u64 = liveness.values().map(|i| i.total_bytes).sum();
+            let budget = (live as f64 * factor) as u64 + floor;
+            if total <= budget {
+                Ok(())
+            } else {
+                Err(format!(
+                    "total_bytes {total} exceeds budget {budget}: live_bytes {live}, factor {factor}, floor {floor}"
+                ))
+            }
+        })
+        .await
+        .map_err(|e| InvariantViolation {
+            invariant: "ByteBudget",
+            detail: format!("join: {e}"),
+        })?
+        .map_err(|e| InvariantViolation {
+            invariant: "ByteBudget",
+            detail: e,
+        })
+    }
+}
+
+pub struct ManifestEqualsReality;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for ManifestEqualsReality {
+    fn name(&self) -> &'static str {
+        "ManifestEqualsReality"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let store = ctx.store.clone();
+        tokio::task::spawn_blocking(move || {
+            let listed = store.list_data_files().map_err(|e| e.to_string())?;
+            let liveness = store.compaction_liveness(0).map_err(|e| e.to_string())?;
+            let header = BLOCK_HEADER_SIZE as u64;
+
+            let mut violations: Vec<String> = Vec::new();
+            listed.iter().for_each(|fid| {
+                let path = store.data_file_path(*fid);
+                match std::fs::metadata(&path) {
+                    Err(e) => violations.push(format!("{fid}: metadata {e}")),
+                    Ok(meta) => {
+                        let on_disk = meta.len();
+                        let content = on_disk.saturating_sub(header);
+                        match liveness.get(fid) {
+                            None if on_disk > header => violations.push(format!(
+                                "{fid}: listed on disk at {on_disk} B but not in index liveness"
+                            )),
+                            None => {}
+                            Some(info) if content < info.total_bytes => {
+                                violations.push(format!(
+                                    "{fid}: on-disk {on_disk} B (content {content}) < index total_bytes {}",
+                                    info.total_bytes
+                                ));
+                            }
+                            Some(info) if content > info.total_bytes => {
+                                violations.push(format!(
+                                    "{fid}: on-disk {on_disk} B (content {content}) > index total_bytes {}, {} B unaccounted",
+                                    info.total_bytes,
+                                    content - info.total_bytes
+                                ));
+                            }
+                            Some(_) => {}
+                        }
+                    }
+                }
+            });
+
+            let listed_set: std::collections::HashSet<_> = listed.into_iter().collect();
+            liveness.keys().for_each(|fid| {
+                if !listed_set.contains(fid) {
+                    violations.push(format!("{fid}: in index liveness but missing on disk"));
+                }
+            });
+
+            if violations.is_empty() {
+                Ok(())
+            } else {
+                Err(violations.join("; "))
+            }
+        })
+        .await
+        .map_err(|e| InvariantViolation {
+            invariant: "ManifestEqualsReality",
+            detail: format!("join: {e}"),
+        })?
+        .map_err(|e| InvariantViolation {
+            invariant: "ManifestEqualsReality",
+            detail: e,
+        })
+    }
+}
+
+pub struct ChecksumCoverage;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for ChecksumCoverage {
+    fn name(&self) -> &'static str {
+        "ChecksumCoverage"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let violations: Vec<String> = ctx
+            .oracle
+            .live_cids_labeled()
+            .into_iter()
+            .filter_map(|(label, expected)| match ctx.store.get_block_sync(&expected) {
+                Ok(Some(bytes)) => {
+                    let actual = hash_to_cid_bytes(&bytes);
+                    (actual != expected).then(|| {
+                        format!(
+                            "{label}: silent corruption, bytes hash to {} but store returned them under {}",
+                            hex_short(&actual),
+                            hex_short(&expected),
+                        )
+                    })
+                }
+                Ok(None) => Some(format!(
+                    "{label}: live CID {} missing from store",
+                    hex_short(&expected)
+                )),
+                Err(e) => Some(format!(
+                    "{label}: read error for live CID {}: {e}",
+                    hex_short(&expected)
+                )),
+            })
+            .collect();
+
+        if violations.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "ChecksumCoverage",
+                detail: violations.join("; "),
+            })
+        }
+    }
+}
+
+pub struct MonotonicSeq;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for MonotonicSeq {
+    fn name(&self) -> &'static str {
+        "MonotonicSeq"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let Some(el) = ctx.eventlog else {
+            return Ok(());
+        };
+        let mut violations: Vec<String> = Vec::new();
+        el.events
+            .iter()
+            .zip(el.events.iter().skip(1))
+            .for_each(|(prev, next)| match next.seq.raw() {
+                n if n == prev.seq.raw() + 1 => {}
+                n if n == prev.seq.raw() => violations.push(format!("duplicate seq {n}")),
+                n => violations.push(format!(
+                    "gap: seq {} followed by {n}, expected {}",
+                    prev.seq.raw(),
+                    prev.seq.raw() + 1
+                )),
+            });
+        if ctx.oracle.last_retention_cutoff_us().is_none()
+            && let Some(first) = el.events.first()
+            && first.seq.raw() != 1
+        {
+            violations.push(format!(
+                "first persisted seq is {}, expected 1",
+                first.seq.raw()
+            ));
+        }
+        let acked_max = ctx
+            .oracle
+            .synced_events()
+            .iter()
+            .map(|e| e.seq.raw())
+            .max()
+            .unwrap_or(0);
+        let disk_max = el.events.last().map(|e| e.seq.raw()).unwrap_or(0);
+        if disk_max < acked_max {
+            violations.push(format!(
+                "acked seq {acked_max} missing on disk, disk max {disk_max}"
+            ));
+        }
+        if violations.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "MonotonicSeq",
+                detail: violations.join("; "),
+            })
+        }
+    }
+}
+
+pub struct FsyncOrdering;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for FsyncOrdering {
+    fn name(&self) -> &'static str {
+        "FsyncOrdering"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let Some(el) = ctx.eventlog else {
+            return Ok(());
+        };
+        let mut violations: Vec<String> = Vec::new();
+
+        let acked_seqs: HashSet<u64> = ctx
+            .oracle
+            .synced_events()
+            .iter()
+            .map(|e| e.seq.raw())
+            .collect();
+        let disk_seqs: HashSet<u64> = el.events.iter().map(|e| e.seq.raw()).collect();
+        let missing: Vec<u64> = acked_seqs.difference(&disk_seqs).copied().collect();
+        if !missing.is_empty() {
+            let mut sorted = missing;
+            sorted.sort_unstable();
+            violations.push(format!(
+                "{} acked events lost on disk, lowest missing seq {}",
+                sorted.len(),
+                sorted[0]
+            ));
+        }
+
+        if let Some(last_synced) = ctx.oracle.last_synced_seq()
+            && el.synced_seq.raw() != 0
+            && el.synced_seq.raw() < last_synced.raw()
+        {
+            violations.push(format!(
+                "writer synced_seq {} below oracle last_synced_seq {}",
+                el.synced_seq.raw(),
+                last_synced.raw()
+            ));
+        }
+
+        if violations.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "FsyncOrdering",
+                detail: violations.join("; "),
+            })
+        }
+    }
+}
+
+pub struct TombstoneBound;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for TombstoneBound {
+    fn name(&self) -> &'static str {
+        "TombstoneBound"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let Some(el) = ctx.eventlog else {
+            return Ok(());
+        };
+        let Some(cutoff_us) = ctx.oracle.last_retention_cutoff_us() else {
+            return Ok(());
+        };
+
+        let active = el.segments.last().copied();
+
+        let stale: Vec<String> = el
+            .segment_last_ts
+            .iter()
+            .filter(|(id, last_ts)| Some(*id) != active && *last_ts < cutoff_us)
+            .map(|(id, last_ts)| format!("segment {id} last_ts {last_ts} < cutoff {cutoff_us}"))
+            .collect();
+
+        if stale.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "TombstoneBound",
+                detail: stale.join("; "),
+            })
+        }
+    }
+}
+
+pub fn invariants_for<S: StorageIO + Send + Sync + 'static>(
+    set: InvariantSet,
+) -> Vec<Box<dyn Invariant<S>>> {
+    let unknown = set.unknown_bits();
+    assert!(
+        unknown == 0,
+        "invariants_for: unknown InvariantSet bits 0x{unknown:x}; all bits must map to an impl"
+    );
+    let candidates: Vec<(InvariantSet, Box<dyn Invariant<S>>)> = vec![
+        (
+            InvariantSet::REFCOUNT_CONSERVATION,
+            Box::new(RefcountConservation),
+        ),
+        (InvariantSet::REACHABILITY, Box::new(Reachability)),
+        (
+            InvariantSet::ACKED_WRITE_PERSISTENCE,
+            Box::new(AckedWritePersistence),
+        ),
+        (InvariantSet::READ_AFTER_WRITE, Box::new(ReadAfterWrite)),
+        (
+            InvariantSet::COMPACTION_IDEMPOTENT,
+            Box::new(CompactionIdempotent),
+        ),
+        (InvariantSet::NO_ORPHAN_FILES, Box::new(NoOrphanFiles)),
+        (InvariantSet::BYTE_BUDGET, Box::new(ByteBudget::default())),
+        (
+            InvariantSet::MANIFEST_EQUALS_REALITY,
+            Box::new(ManifestEqualsReality),
+        ),
+        (InvariantSet::CHECKSUM_COVERAGE, Box::new(ChecksumCoverage)),
+        (InvariantSet::MONOTONIC_SEQ, Box::new(MonotonicSeq)),
+        (InvariantSet::FSYNC_ORDERING, Box::new(FsyncOrdering)),
+        (InvariantSet::TOMBSTONE_BOUND, Box::new(TombstoneBound)),
+    ];
+    candidates
+        .into_iter()
+        .filter_map(|(flag, inv)| set.contains(flag).then_some(inv))
+        .collect()
+}
@@ -0,0 +1,234 @@
+use std::num::NonZeroU64;
+use std::time::Duration;
+
+use serde::{Deserialize, Serialize};
+
+use super::metrics::{MetricName, MetricsSample};
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub struct LeakGateConfig {
+    pub warmup_ms: u64,
+    pub window_ms: NonZeroU64,
+    pub growth_limit_pct: f64,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct LeakGateBuildError(pub &'static str);
+
+impl std::fmt::Display for LeakGateBuildError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(self.0)
+    }
+}
+
+impl std::error::Error for LeakGateBuildError {}
+
+impl LeakGateConfig {
+    pub fn try_new(
+        warmup_ms: u64,
+        window_ms: u64,
+        growth_limit_pct: f64,
+    ) -> Result<Self, LeakGateBuildError> {
+        let window_ms = NonZeroU64::new(window_ms)
+            .ok_or(LeakGateBuildError("leak gate window_ms must be > 0"))?;
+        if !growth_limit_pct.is_finite() || growth_limit_pct < 0.0 {
+            return Err(LeakGateBuildError(
+                "leak gate growth_limit_pct must be finite and non-negative",
+            ));
+        }
+        Ok(Self {
+            warmup_ms,
+            window_ms,
+            growth_limit_pct,
+        })
+    }
+
+    pub fn standard() -> Self {
+        Self::try_new(60 * 60 * 1_000, 4 * 60 * 60 * 1_000, 5.0)
+            .expect("standard leak gate config is valid")
+    }
+
+    pub fn short_for_tests() -> Self {
+        Self::try_new(60_000, 4 * 60_000, 5.0).expect("short_for_tests leak gate config is valid")
+    }
+
+    pub fn warmup(&self) -> Duration {
+        Duration::from_millis(self.warmup_ms)
+    }
+
+    pub fn window(&self) -> Duration {
+        Duration::from_millis(self.window_ms.get())
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct LeakViolation {
+    pub metric: String,
+    pub start_ms: u64,
+    pub end_ms: u64,
+    pub start_value: u64,
+    pub end_value: u64,
+    pub growth_pct: f64,
+    pub limit_pct: f64,
+}
+
+pub fn evaluate(samples: &[MetricsSample], cfg: LeakGateConfig) -> Vec<LeakViolation> {
+    if samples.len() < 2 {
+        return Vec::new();
+    }
+    MetricName::ALL
+        .iter()
+        .flat_map(|&m| evaluate_metric(samples, m, cfg))
+        .collect()
+}
+
+fn evaluate_metric(
+    samples: &[MetricsSample],
+    metric: MetricName,
+    cfg: LeakGateConfig,
+) -> Option<LeakViolation> {
+    let post_warmup: Vec<&MetricsSample> = samples
+        .iter()
+        .filter(|s| s.elapsed_ms >= cfg.warmup_ms)
+        .collect();
+    if post_warmup.len() < 2 {
+        return None;
+    }
+
+    let min_delta = metric.min_absolute_delta();
+    let window = cfg.window_ms.get();
+    let mut worst: Option<LeakViolation> = None;
+    for (i, start) in post_warmup.iter().enumerate() {
+        let Some(start_v) = start.metric(metric) else {
+            continue;
+        };
+        if start_v == 0 {
+            continue;
+        }
+        let deadline = start.elapsed_ms.saturating_add(window);
+        for end in post_warmup.iter().skip(i + 1) {
+            if end.elapsed_ms > deadline {
+                break;
+            }
+            let Some(end_v) = end.metric(metric) else {
+                continue;
+            };
+            if end_v <= start_v {
+                continue;
+            }
+            let delta = end_v - start_v;
+            if delta < min_delta {
+                continue;
+            }
+            let growth = (delta as f64 / start_v as f64) * 100.0;
+            if growth <= cfg.growth_limit_pct {
+                continue;
+            }
+            let candidate = LeakViolation {
+                metric: metric.as_str().to_string(),
+                start_ms: start.elapsed_ms,
+                end_ms: end.elapsed_ms,
+                start_value: start_v,
+                end_value: end_v,
+                growth_pct: growth,
+                limit_pct: cfg.growth_limit_pct,
+            };
+            match &worst {
+                Some(w) if w.growth_pct >= candidate.growth_pct => {}
+                _ => worst = Some(candidate),
+            }
+        }
+    }
+    worst
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const GIB: u64 = 1024 * 1024 * 1024;
+
+    fn sample(elapsed_ms: u64, rss: u64) -> MetricsSample {
+        MetricsSample {
+            elapsed_ms,
+            rss_bytes: Some(rss),
+            fd_count: Some(10),
+            data_dir_bytes: 0,
+            index_dir_bytes: 0,
+            segments_dir_bytes: 0,
+            data_file_count: Some(0),
+            segment_count: Some(0),
+            block_index_entries: 0,
+            hint_file_bytes: 0,
+        }
+    }
+
+    #[test]
+    fn flat_metrics_no_violation() {
+        let cfg = LeakGateConfig::short_for_tests();
+        let series: Vec<MetricsSample> =
+            (0..20).map(|i| sample(60_000 + i * 60_000, GIB)).collect();
+        assert!(evaluate(&series, cfg).is_empty());
+    }
+
+    #[test]
+    fn growing_rss_flagged() {
+        let cfg = LeakGateConfig::short_for_tests();
+        let series: Vec<MetricsSample> = (0..20)
+            .map(|i| sample(60_000 + i * 60_000, GIB + i * 64 * 1024 * 1024))
+            .collect();
+        let v = evaluate(&series, cfg);
+        assert!(!v.is_empty());
+        assert_eq!(v[0].metric, "rss_bytes");
+        assert!(v[0].growth_pct > 5.0);
+    }
+
+    #[test]
+    fn warmup_samples_ignored() {
+        let cfg = LeakGateConfig::short_for_tests();
+        let mut series: Vec<MetricsSample> = Vec::new();
+        series.push(sample(10_000, 1));
+        series.push(sample(30_000, GIB));
+        (0..10).for_each(|i| {
+            series.push(sample(60_000 + i * 60_000, GIB));
+        });
+        assert!(evaluate(&series, cfg).is_empty());
+    }
+
+    #[test]
+    fn window_bound_honored() {
+        let cfg = LeakGateConfig::try_new(0, 2 * 60_000, 5.0).unwrap();
+        let series = vec![sample(0, GIB), sample(200_000, 2 * GIB)];
+        assert!(
+            evaluate(&series, cfg).is_empty(),
+            "200s gap exceeds 120s window, growth must not be flagged"
+        );
+    }
+
+    #[test]
+    fn small_absolute_delta_not_flagged() {
+        let cfg = LeakGateConfig::short_for_tests();
+        let series: Vec<MetricsSample> = (0..10)
+            .map(|i| sample(60_000 + i * 60_000, GIB + i * 1024))
+            .collect();
+        assert!(
+            evaluate(&series, cfg).is_empty(),
+            "kilobyte growth is below the RSS absolute-delta floor"
+        );
+    }
+
+    #[test]
+    fn missing_metric_samples_skipped() {
+        let cfg = LeakGateConfig::short_for_tests();
+        let mut series: Vec<MetricsSample> =
+            (0..10).map(|i| sample(60_000 + i * 60_000, GIB)).collect();
+        series[3].rss_bytes = None;
+        series[7].rss_bytes = None;
+        assert!(evaluate(&series, cfg).is_empty());
+    }
+
+    #[test]
+    fn zero_window_rejected_at_construction() {
+        assert!(LeakGateConfig::try_new(0, 0, 5.0).is_err());
+    }
+}
@@ -0,0 +1,247 @@
+use std::path::Path;
+use std::sync::Arc;
+use std::time::Duration;
+
+use serde::{Deserialize, Serialize};
+use tracing::warn;
+
+use super::runner::{EventLogState, Harness};
+use crate::blockstore::TranquilBlockStore;
+use crate::io::StorageIO;
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub struct MetricsSample {
+    pub elapsed_ms: u64,
+    pub rss_bytes: Option<u64>,
+    pub fd_count: Option<u64>,
+    pub data_dir_bytes: u64,
+    pub index_dir_bytes: u64,
+    pub segments_dir_bytes: u64,
+    pub data_file_count: Option<u64>,
+    pub segment_count: Option<u64>,
+    pub block_index_entries: u64,
+    pub hint_file_bytes: u64,
+}
+
+impl MetricsSample {
+    pub fn metric(&self, name: MetricName) -> Option<u64> {
+        match name {
+            MetricName::RssBytes => self.rss_bytes,
+            MetricName::FdCount => self.fd_count,
+            MetricName::DataDirBytes => Some(self.data_dir_bytes),
+            MetricName::IndexDirBytes => Some(self.index_dir_bytes),
+            MetricName::SegmentsDirBytes => Some(self.segments_dir_bytes),
+            MetricName::DataFileCount => self.data_file_count,
+            MetricName::SegmentCount => self.segment_count,
+            MetricName::BlockIndexEntries => Some(self.block_index_entries),
+            MetricName::HintFileBytes => Some(self.hint_file_bytes),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum MetricName {
+    RssBytes,
+    FdCount,
+    DataDirBytes,
+    IndexDirBytes,
+    SegmentsDirBytes,
+    DataFileCount,
+    SegmentCount,
+    BlockIndexEntries,
+    HintFileBytes,
+}
+
+impl MetricName {
+    pub const ALL: &'static [MetricName] = &[
+        Self::RssBytes,
+        Self::FdCount,
+        Self::DataDirBytes,
+        Self::IndexDirBytes,
+        Self::SegmentsDirBytes,
+        Self::DataFileCount,
+        Self::SegmentCount,
+        Self::BlockIndexEntries,
+        Self::HintFileBytes,
+    ];
+
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            Self::RssBytes => "rss_bytes",
+            Self::FdCount => "fd_count",
+            Self::DataDirBytes => "data_dir_bytes",
+            Self::IndexDirBytes => "index_dir_bytes",
+            Self::SegmentsDirBytes => "segments_dir_bytes",
+            Self::DataFileCount => "data_file_count",
+            Self::SegmentCount => "segment_count",
+            Self::BlockIndexEntries => "block_index_entries",
+            Self::HintFileBytes => "hint_file_bytes",
+        }
+    }
+
+    pub const fn min_absolute_delta(self) -> u64 {
+        match self {
+            Self::RssBytes => 16 * 1024 * 1024,
+            Self::FdCount => 16,
+            Self::DataDirBytes => 16 * 1024 * 1024,
+            Self::IndexDirBytes => 1024 * 1024,
+            Self::SegmentsDirBytes => 16 * 1024 * 1024,
+            Self::DataFileCount => 16,
+            Self::SegmentCount => 4,
+            Self::BlockIndexEntries => 1024,
+            Self::HintFileBytes => 1024 * 1024,
+        }
+    }
+}
+
+pub fn sample_harness<S: StorageIO + Send + Sync + 'static>(
+    harness: &Harness<S>,
+    elapsed: Duration,
+) -> MetricsSample {
+    MetricsSample {
+        elapsed_ms: u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX),
+        rss_bytes: read_rss(),
+        fd_count: count_open_fds(),
+        data_dir_bytes: dir_bytes(harness.store.data_dir()),
+        index_dir_bytes: dir_bytes(harness.store.block_index().index_dir()),
+        segments_dir_bytes: harness
+            .eventlog
+            .as_ref()
+            .map(|el| dir_bytes(&el.segments_dir))
+            .unwrap_or(0),
+        data_file_count: data_file_count(&harness.store),
+        segment_count: harness.eventlog.as_ref().and_then(segment_count),
+        block_index_entries: harness.store.block_index().approximate_block_count(),
+        hint_file_bytes: hint_bytes(harness.store.data_dir()),
+    }
+}
+
+fn data_file_count<S: StorageIO + Send + Sync + 'static>(
+    store: &Arc<TranquilBlockStore<S>>,
+) -> Option<u64> {
+    match store.list_data_files() {
+        Ok(v) => Some(v.len() as u64),
+        Err(e) => {
+            warn!(error = %e, "gauntlet metrics: list_data_files failed");
+            None
+        }
+    }
+}
+
+fn segment_count<S: StorageIO + Send + Sync + 'static>(el: &EventLogState<S>) -> Option<u64> {
+    match el.manager.list_segments() {
+        Ok(v) => Some(v.len() as u64),
+        Err(e) => {
+            warn!(error = %e, "gauntlet metrics: list_segments failed");
+            None
+        }
+    }
+}
+
+fn dir_bytes(path: &Path) -> u64 {
+    let Ok(entries) = std::fs::read_dir(path) else {
+        return 0;
+    };
+    entries
+        .filter_map(Result::ok)
+        .map(|entry| match entry.file_type() {
+            Ok(ft) if ft.is_dir() => dir_bytes(&entry.path()),
+            Ok(_) => entry.metadata().map(|m| m.len()).unwrap_or(0),
+            Err(_) => 0,
+        })
+        .sum()
+}
+
+fn hint_bytes(data_dir: &Path) -> u64 {
+    let Ok(entries) = std::fs::read_dir(data_dir) else {
+        return 0;
+    };
+    entries
+        .filter_map(Result::ok)
+        .filter(|entry| {
+            entry
+                .path()
+                .extension()
+                .and_then(|e| e.to_str())
+                .map(|e| e == "tqh")
+                .unwrap_or(false)
+        })
+        .map(|entry| entry.metadata().map(|m| m.len()).unwrap_or(0))
+        .sum()
+}
+
+#[cfg(target_os = "linux")]
+fn read_rss() -> Option<u64> {
+    let status = match std::fs::read_to_string("/proc/self/status") {
+        Ok(s) => s,
+        Err(e) => {
+            warn!(error = %e, "gauntlet metrics: read /proc/self/status failed");
+            return None;
+        }
+    };
+    let parsed = status.lines().find_map(|line| {
+        let rest = line.strip_prefix("VmRSS:")?;
+        let kb: u64 = rest.split_whitespace().next()?.parse().ok()?;
+        Some(kb * 1024)
+    });
+    if parsed.is_none() {
+        warn!("gauntlet metrics: VmRSS line missing from /proc/self/status");
+    }
+    parsed
+}
+
+#[cfg(not(target_os = "linux"))]
+fn read_rss() -> Option<u64> {
+    None
+}
+
+#[cfg(target_os = "linux")]
+fn count_open_fds() -> Option<u64> {
+    match std::fs::read_dir("/proc/self/fd") {
+        Ok(entries) => Some(entries.filter_map(Result::ok).count() as u64),
+        Err(e) => {
+            warn!(error = %e, "gauntlet metrics: read /proc/self/fd failed");
+            None
+        }
+    }
+}
+
+#[cfg(not(target_os = "linux"))]
+fn count_open_fds() -> Option<u64> {
+    None
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn metric_names_roundtrip_strings() {
+        MetricName::ALL.iter().for_each(|m| {
+            let s = m.as_str();
+            assert!(!s.is_empty());
+        });
+    }
+
+    #[test]
+    #[cfg_attr(not(target_os = "linux"), ignore = "linux /proc only")]
+    fn rss_reads_nonzero() {
+        let rss = read_rss().expect("rss");
+        assert!(rss > 0, "rss should be positive, got {rss}");
+    }
+
+    #[test]
+    #[cfg_attr(not(target_os = "linux"), ignore = "linux /proc only")]
+    fn fd_count_reads_nonzero() {
+        let fd = count_open_fds().expect("fd");
+        assert!(fd > 0);
+    }
+
+    #[test]
+    fn dir_bytes_sums_entries() {
+        let dir = tempfile::TempDir::new().unwrap();
+        std::fs::write(dir.path().join("a"), b"1234").unwrap();
+        std::fs::write(dir.path().join("b"), b"5678").unwrap();
+        assert_eq!(dir_bytes(dir.path()), 8);
+    }
+}
@@ -0,0 +1,45 @@
+pub mod farm;
+pub mod flaky;
+pub mod invariants;
+pub mod leak;
+pub mod metrics;
+pub mod op;
+pub mod oracle;
+pub mod overrides;
+pub mod regression;
+pub mod runner;
+pub mod scenarios;
+pub mod shrink;
+pub mod soak;
+pub mod workload;
+
+pub use flaky::{
+    BackingMegabytes, DownIntervalSecs, FlakyConfig, FlakyError, FlakyMount, UpIntervalSecs,
+};
+pub use invariants::{
+    EventLogSnapshot, Invariant, InvariantSet, InvariantViolation, SnapshotEvent, invariants_for,
+};
+pub use leak::{LeakGateBuildError, LeakGateConfig, LeakViolation, evaluate as evaluate_leak_gate};
+pub use metrics::{MetricName, MetricsSample, sample_harness};
+pub use op::{
+    CollectionName, DidSeed, EventKind, Op, OpStream, PayloadSeed, RecordKey, RetentionSecs, Seed,
+    ValueSeed,
+};
+pub use oracle::{EventExpectation, Oracle};
+pub use overrides::{ConfigOverrides, GroupCommitOverrides, StoreOverrides};
+pub use regression::{RegressionRecord, RegressionViolation, default_root as regression_root};
+pub use runner::{
+    EventLogConfig, Gauntlet, GauntletBuildError, GauntletConfig, GauntletReport, Harness,
+    IoBackend, MaxFileSize, MaxSegmentSize, OpErrorCount, OpIndex, OpInterval, OpsExecuted,
+    RestartCount, RestartPolicy, RunLimits, ShardCount, StoreConfig, WallMs, WriterConcurrency,
+};
+pub use scenarios::{Scenario, UnknownScenario, config_for};
+pub use shrink::{ShrinkOutcome, shrink_failure};
+pub use soak::{
+    DEFAULT_CHUNK_OPS, DEFAULT_SAMPLE_INTERVAL_MS, InvariantViolationRecord, SoakConfig, SoakError,
+    SoakEvent, SoakReport, run_soak,
+};
+pub use workload::{
+    ByteRange, DidSpaceSize, KeySpaceSize, OpCount, OpWeights, RetentionMaxSecs, SizeDistribution,
+    ValueBytes, WorkloadModel,
+};
@@ -0,0 +1,198 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct Seed(pub u64);
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct CollectionName(pub String);
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct RecordKey(pub String);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct ValueSeed(pub u32);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct DidSeed(pub u32);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct PayloadSeed(pub u32);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct RetentionSecs(pub u32);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub enum EventKind {
+    Commit,
+    Identity,
+    Account,
+    Sync,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum Op {
+    AddRecord {
+        collection: CollectionName,
+        rkey: RecordKey,
+        value_seed: ValueSeed,
+    },
+    DeleteRecord {
+        collection: CollectionName,
+        rkey: RecordKey,
+    },
+    Compact,
+    Checkpoint,
+    AppendEvent {
+        did_seed: DidSeed,
+        event_kind: EventKind,
+        payload_seed: PayloadSeed,
+    },
+    SyncEventLog,
+    RunRetention {
+        max_age_secs: RetentionSecs,
+    },
+    ReadRecord {
+        collection: CollectionName,
+        rkey: RecordKey,
+    },
+    ReadBlock {
+        value_seed: ValueSeed,
+    },
+}
+
+impl Op {
+    pub const fn is_read_only(&self) -> bool {
+        matches!(self, Op::ReadRecord { .. } | Op::ReadBlock { .. })
+    }
+}
+
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct OpStream {
+    ops: Vec<Op>,
+}
+
+impl OpStream {
+    pub fn from_vec(ops: Vec<Op>) -> Self {
+        Self { ops }
+    }
+
+    pub fn empty() -> Self {
+        Self { ops: Vec::new() }
+    }
+
+    pub fn as_slice(&self) -> &[Op] {
+        &self.ops
+    }
+
+    pub fn into_vec(self) -> Vec<Op> {
+        self.ops
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = &Op> {
+        self.ops.iter()
+    }
+
+    pub fn len(&self) -> usize {
+        self.ops.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.ops.is_empty()
+    }
+
+    pub fn shrink_candidates(&self) -> impl Iterator<Item = OpStream> + '_ {
+        let len = self.ops.len();
+        let chunk_sizes: Vec<usize> = std::iter::successors((len >= 2).then_some(len / 2), |&s| {
+            (s >= 2).then_some(s / 2)
+        })
+        .collect();
+
+        let chunk_candidates = chunk_sizes.into_iter().flat_map(move |chunk_size| {
+            let count = len.div_ceil(chunk_size);
+            (0..count).map(move |i| {
+                let start = i * chunk_size;
+                let end = (start + chunk_size).min(len);
+                let mut reduced = Vec::with_capacity(len - (end - start));
+                reduced.extend_from_slice(&self.ops[..start]);
+                reduced.extend_from_slice(&self.ops[end..]);
+                OpStream::from_vec(reduced)
+            })
+        });
+
+        let single_candidates = (0..len).map(move |i| {
+            let mut reduced = self.ops.clone();
+            reduced.remove(i);
+            OpStream::from_vec(reduced)
+        });
+
+        chunk_candidates.chain(single_candidates)
+    }
+
+    pub fn shrink_to_fixpoint(mut self, mut fails: impl FnMut(&OpStream) -> bool) -> OpStream {
+        loop {
+            let next = self.shrink_candidates().find(|c| !c.is_empty() && fails(c));
+            match next {
+                Some(smaller) => self = smaller,
+                None => return self,
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn stream(n: usize) -> OpStream {
+        OpStream::from_vec(
+            (0..n)
+                .map(|i| Op::AddRecord {
+                    collection: CollectionName("c".into()),
+                    rkey: RecordKey(format!("{i:04}")),
+                    value_seed: ValueSeed(i as u32),
+                })
+                .collect(),
+        )
+    }
+
+    fn contains_index(s: &OpStream, target: u32) -> bool {
+        s.iter()
+            .any(|op| matches!(op, Op::AddRecord { value_seed, .. } if value_seed.0 == target))
+    }
+
+    #[test]
+    fn shrink_candidates_nonempty_for_len_ge_2() {
+        let s = stream(8);
+        let count = s.shrink_candidates().count();
+        assert!(count > 0);
+    }
+
+    #[test]
+    fn shrink_candidates_empty_for_len_0() {
+        let s = OpStream::from_vec(Vec::new());
+        assert_eq!(s.shrink_candidates().count(), 0);
+    }
+
+    #[test]
+    fn shrink_candidates_includes_every_single_removal() {
+        let s = stream(5);
+        let singles: Vec<_> = s.shrink_candidates().filter(|c| c.len() == 4).collect();
+        assert!(
+            singles.len() >= 5,
+            "expected at least 5 size-4 candidates, got {}",
+            singles.len()
+        );
+    }
+
+    #[test]
+    fn shrink_to_fixpoint_converges_to_culprit() {
+        let s = stream(64);
+        let shrunk = s.shrink_to_fixpoint(|c| contains_index(c, 17));
+        assert!(contains_index(&shrunk, 17));
+        assert!(
+            shrunk.len() < 4,
+            "expected shrink to close on culprit, got {} ops",
+            shrunk.len()
+        );
+    }
+}
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
LewisandTangled	180de29984	fix(tranquil-pds): firehose car carries inductive proof Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-26 20:11:27 +03:00
max.andTangled	0455dc20bd	fix(validation): discord usernames may contain periods at start or end	2026-04-25 09:59:12 +03:00
Lewis	2770b9b14a	feat(tranquil-store): cargo-fuzz targets with asan+ubsan Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-24 10:50:08 +03:00
Lewis	d436597184	feat(tranquil-store): flaky-device scenario, jemalloc heap-prof Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-24 10:50:08 +03:00
Lewis	4cfca6d956	feat(tranquil-store): soak harness driving leak gate, signal tweaks Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-23 08:39:16 +03:00
Lewis	98b94fb170	feat(tranquil-store): leak gate and metrics sampling for gauntlet Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-22 21:00:17 +03:00
Lewis	4fe01cff72	feat(tranquil-store): sweep subcommand with axis override fan-out Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-22 17:23:15 +03:00
Lewis	00c9eb732f	fix(tranquil-store): arc-counted cache handles, reader-eviction race Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-21 22:04:24 +03:00
Lewis	6d2d3b4be4	fix(tranquil-store): commit-marker batch replay, batch-boundary rotation Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-21 14:55:15 +03:00
Lewis	9d81e58803	fix(tranquil-store): no more orphan data files, recover torn-hint tails, header-safe resume Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-21 09:59:37 +03:00
Lewis	2afd075496	fix(tranquil-store): atomic record commits, hint-as-truth recovery Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-20 21:00:54 +03:00
Hayden YoungandTangled	c74bf967cf	fix(auth): don't require 2fa for app passwords	2026-04-20 18:20:20 +03:00
Lewis	1285d5c675	fix(tranquil-store): bound writer fd usage across rotations Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-20 16:50:19 +03:00
Lewis	c30d73cd4d	test(tranquil-store): migrate some tests to gauntlet Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-19 23:50:27 +03:00
Lewis	0fab8f2eb9	feat(tranquil-store): tranquil-gauntlet CLI, config overrides, profiles Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-19 23:50:27 +03:00
Lewis	57336fa124	feat(tranquil-store/gauntlet): new invariants & scenarios Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-19 10:25:07 +03:00
Lewis	ace105899f	feat(tranquil-store/gauntlet): concurrent executor, eventlog, fault recovery Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-19 00:19:08 +03:00
Lewis	c80a525e0d	feat(tranquil-store/gauntlet): op surface, oracle, workload for eventlog & reads Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-18 10:36:30 +03:00
Lewis	7edb76507b	feat(tranquil-store/gauntlet): simulated io fault modes, shrinker, regression dump Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-18 10:36:30 +03:00
LewisandTangled	a20e4e05e6	fix(oauth): indigo client send empty assertions Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-17 15:18:26 +03:00
teqandTangled	3e7a199736	docs: nixos and binary cache	2026-04-17 01:32:30 +03:00
teqandTangled	7ed9c8ce59	feat(ci): supply cache by arg	2026-04-17 01:32:30 +03:00
teqandTangled	b0c3402a82	feat(ci): add cachix publishing workflows	2026-04-17 01:32:30 +03:00
LewisandTangled	b9574f3ec1	fix(lexicons): cache ttl Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-16 19:45:27 +03:00
isabelandTangled	baef2be844	feat: allow max prefs to be configured some people (me) have a large amount of prefs and moving from ref pds to tranquil ment i lost a few of these. so in this pr i bumped the max prefs to 1000 and made it configurable via the config	2026-04-16 19:45:17 +03:00
Lewis	1f21b85069	fix(example.toml): lewis forgot about confique Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-16 14:49:57 +03:00
Lewis	09d437b3e3	feat(tranquil-store): gauntlet persistence & restart invariants Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-15 22:28:31 +03:00
Lewis	7f2e83e92f	feat(tranquil-store): beginnings of the gauntlet test suite Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-15 20:29:34 +03:00
nelind	ec273fa814	feat(nix): cross compiled aarch64 package	2026-04-14 23:26:34 +02:00
Lewis	d51bfd59da	fix(tranquil-store): exclude 0 refcount blocks from has() Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-14 08:40:21 +03:00
LewisandTangled	8ccdd30cb3	fix(repo): use mst diff instead of full tree walk for obsolete blocks Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-13 17:43:07 +00:00
Lewis	7a67361993	fix(tranquil-store): checkpoint-hint race & missing dedup hints Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-13 11:10:17 +03:00
isabelandTangled	cdbbaaccdf	fix(nix/frontend): add nodejs	2026-04-12 23:08:39 +00:00
isabelandTangled	55d3b7f83d	fix(nix/module): don't import using self	2026-04-12 22:50:00 +00:00
Gavin MoganandTangled	f00b0231fb	fix(Dockerfile): duplicate named stage causes failure to build	2026-04-12 18:53:21 +00:00
LewisandTangled	1dfbd27cce	fix(postgres): semaphore on car endpoint & more efficient query Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-12 17:18:45 +00:00
isabelandTangled	bc5e0e0446	build(frontend): use pnpm deno is evil	2026-04-12 16:01:21 +00:00
Lewis	1c87ef5536	fix(tranquil-store): blockstore tweaks Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-12 17:35:59 +03:00
Lewis	255c7135f9	fix(auth): no bsky chat access when not specifically privileged to have it Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-12 15:56:17 +03:00
LewisandTangled	34f050a1f0	feat(oauth): discoverable passkey authentication Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-12 11:16:58 +00:00
LewisandTangled	52c22060f3	fix(tranquil-store): preserve refcount in hint relocate records Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-12 11:16:58 +00:00
Lewis	c250d51978	chore(readme): another small update Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-11 17:23:43 +03:00
Lewis	378cb7a7fb	chore(readme): update with contributers and new description of project Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-11 16:40:32 +03:00
Bas van den WollenbergandTangled	bf20b4a78f	fix: use correct postgres container path for quadlet as well	2026-04-11 07:45:23 +00:00