feat(tranquil-comms): prework for email

Lewis: May this revision serve well! <lu5a@proton.me>
fix(tranquil-store): durable-tail recovery + sync semantics
2026-08-25 10:46:11 +00:00 · 2026-05-02 22:28:59 +03:00 · 2026-04-30 11:11:28 +03:00 · 2026-04-29 15:35:42 +03:00 · 2026-04-28 23:06:36 +03:00 · 2026-04-28 22:05:03 +03:00
138 changed files with 22539 additions and 1987 deletions
@@ -25,6 +25,28 @@ fail-fast = false
 test-threads = "num-cpus"
 slow-timeout = { period = "300s", terminate-after = 2 }

+[profile.gauntlet-pr]
+retries = 0
+fail-fast = true
+test-threads = "num-cpus"
+slow-timeout = { period = "60s", terminate-after = 5 }
+
+[[profile.gauntlet-pr.overrides]]
+filter = "binary(gauntlet_smoke)"
+slow-timeout = { period = "300s", terminate-after = 2 }
+
+[profile.gauntlet-nightly]
+retries = 0
+fail-fast = false
+test-threads = "num-cpus"
+slow-timeout = { period = "600s", terminate-after = 1 }
+
+[profile.gauntlet-soak]
+retries = 0
+fail-fast = false
+test-threads = 1
+slow-timeout = { period = "5m", terminate-after = 1000 }
+
 [test-groups]
 serial-env-tests = { max-threads = 1 }
 heavy-load-tests = { max-threads = 4 }
@@ -46,6 +68,10 @@ test-group = "serial-env-tests"
 filter = "package(tranquil-signal)"
 test-group = "serial-env-tests"

+[[profile.default.overrides]]
+filter = "package(tranquil-config)"
+test-group = "serial-env-tests"
+
 [[profile.default.overrides]]
 filter = "binary(whole_story)"
 test-group = "heavy-load-tests"
@@ -72,6 +98,14 @@ test-group = "io-heavy-sim"
 filter = "test(/test_scale_/) | test(/full_backup_and_restore/)"
 slow-timeout = { period = "120s", terminate-after = 4 }

+[[profile.default.overrides]]
+filter = "binary(gauntlet_smoke)"
+slow-timeout = { period = "300s", terminate-after = 8 }
+
+[[profile.default.overrides]]
+filter = "binary(compaction_restart) | binary(mst_refcount_integrity) | binary(gc_compaction_restart)"
+slow-timeout = { period = "120s", terminate-after = 4 }
+
 [[profile.ci.overrides]]
 filter = "test(/import_with_verification/) | test(/plc_migration/)"
 test-group = "serial-env-tests"
@@ -88,6 +122,10 @@ test-group = "serial-env-tests"
 filter = "package(tranquil-signal)"
 test-group = "serial-env-tests"

+[[profile.ci.overrides]]
+filter = "package(tranquil-config)"
+test-group = "serial-env-tests"
+
 [[profile.ci.overrides]]
 filter = "binary(whole_story)"
 test-group = "heavy-load-tests"
@@ -1,4 +1,4 @@
-/target
+target/
 .env
 .direnv
 result
@@ -0,0 +1,21 @@
+when:
+  - event: ["push", "manual"]
+    branch: ["main"]
+
+engine: nixery
+
+dependencies:
+  nixpkgs:
+    - nix
+    - cachix
+    - gnugrep
+
+steps:
+  - name: Authenticate
+    command: |
+      cachix authtoken "$CACHIX_AUTH_TOKEN"
+
+  - name: Build and push aarch64
+    command: |
+      SUBS="--option extra-substituters https://tranquil.cachix.org --option extra-trusted-public-keys tranquil.cachix.org-1:PoO+mGL6a6LcJiPakMDHN4E218/ei/7v2sxeDtNkSRg="
+      nix-store -qR --include-outputs $(nix-store -qd $(nix build .#packages.x86_64-linux.tranquil-pds-aarch64 $SUBS --print-out-paths --no-link)) | grep -v '\.drv$' | cachix push "$CACHIX_CACHE_NAME"
@@ -0,0 +1,28 @@
+when:
+  - event: ["push", "manual"]
+    branch: ["main"]
+
+engine: nixery
+
+dependencies:
+  nixpkgs:
+    - nix
+    - cachix
+    - gnugrep
+
+steps:
+  - name: Authenticate
+    command: |
+      cachix authtoken "$CACHIX_AUTH_TOKEN"
+
+  - name: Build and push x86_64
+    command: |
+      SUBS="--option extra-substituters https://tranquil.cachix.org --option extra-trusted-public-keys tranquil.cachix.org-1:PoO+mGL6a6LcJiPakMDHN4E218/ei/7v2sxeDtNkSRg="
+      nix-store -qR --include-outputs $(nix-store -qd $(nix build .#packages.x86_64-linux.tranquil-pds $SUBS --print-out-paths --no-link)) | grep -v '\.drv$' | cachix push "$CACHIX_CACHE_NAME"
+      nix-store -qR --include-outputs $(nix-store -qd $(nix build .#packages.x86_64-linux.tranquil-frontend $SUBS --print-out-paths --no-link)) | grep -v '\.drv$' | cachix push "$CACHIX_CACHE_NAME"
+
+  - name: Build and push devShell
+    command: |
+      SUBS="--option extra-substituters https://tranquil.cachix.org --option extra-trusted-public-keys tranquil.cachix.org-1:PoO+mGL6a6LcJiPakMDHN4E218/ei/7v2sxeDtNkSRg="
+      nix develop $SUBS --profile dev-profile -c true
+      cachix push "$CACHIX_CACHE_NAME" dev-profile
@@ -1,24 +1,36 @@
 when:
-  - event: []
-    branch: []
+  - event: [ "manual" ]
+  - event: [ "push" ]
+    branch: [ "main" ]

 engine: nixery

 dependencies:
  nixpkgs:
-    - podman
+    - kaniko
+
+environment:
+  DOCKER_CONFIG: "/kaniko/.docker"

 steps:
-  - name: Create podman config
+  - name: Configure Kaniko
    command: |
-      mkdir -p ~/.config/containers
-      echo "unqualified-search-registries = [\"docker.io\"]" >> ~/.config/containers/registries.conf
+      mkdir -p /kaniko/.docker/
+      echo "{
+        \"auths\": {
+          \"https://atcr.io/v1\":{
+            \"auth\": \"$ATCR_CREDENTIALS\"
+          }
+        }
+      }" > /kaniko/.docker/config.json

  - name: Build image
    command: |
-      podman build . -t tranquil-pds:latest -t "tranquil-pds:$TANGLED_COMMIT_SHA"
-
-  - name: Publish image
-    command: |
-      podman push --creds "$ATCR_USERNAME:$ATCR_PASSWORD" tranquil-pds:latest "atcr.io/tranquil.farm/tranquil-pds:latest"
-      podman push --creds "$ATCR_USERNAME:$ATCR_PASSWORD" "tranquil-pds:$TANGLED_COMMIT_SHA" "atcr.io/tranquil.farm/tranquil-pds:$TANGLED_COMMIT_SHA"
+      executor \
+        --context=$(pwd) \
+        --ignore-path=$(pwd) \
+        --dockerfile=$(pwd)/Dockerfile \
+        --destination="atcr.io/tranquil.farm/tranquil-pds:latest" \
+        --destination="atcr.io/tranquil.farm/tranquil-pds:$TANGLED_COMMIT_SHA" \
+        --push-retry=3 \
+        --skip-push-permission-check
@@ -9,7 +9,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "087113bd50d9adce24850eed5d0476c7d199d532fce8fab5173650331e09033a"
 dependencies = [
 "abnf-core",
- "nom",
+ "nom 7.1.3",
 ]

 [[package]]
@@ -18,7 +18,7 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c44e09c43ae1c368fb91a03a566472d0087c26cf7e1b9e8e289c14ede681dd7d"
 dependencies = [
- "nom",
+ "nom 7.1.3",
 ]

 [[package]]
@@ -213,7 +213,7 @@ dependencies = [
 "asn1-rs-derive",
 "asn1-rs-impl",
 "displaydoc",
- "nom",
+ "nom 7.1.3",
 "num-traits",
 "rusticata-macros",
 "thiserror 1.0.69",
@@ -1558,7 +1558,7 @@ checksum = "06b4f5ec222421e22bb0a8cbaa36b1d2b50fd45cdd30c915ded34108da78b29f"
 dependencies = [
 "confique-macro",
 "serde",
- "toml",
+ "toml 0.9.12+spec-1.1.0",
 ]

 [[package]]
@@ -1972,7 +1972,7 @@ checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553"
 dependencies = [
 "asn1-rs",
 "displaydoc",
- "nom",
+ "nom 7.1.3",
 "num-bigint",
 "num-traits",
 "rusticata-macros",
@@ -2216,6 +2216,22 @@ dependencies = [
 "zeroize",
 ]

+[[package]]
+name = "email-encoding"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9298e6504d9b9e780ed3f7dfd43a61be8cd0e09eb07f7706a945b0072b6670b6"
+dependencies = [
+ "base64 0.22.1",
+ "memchr",
+]
+
+[[package]]
+name = "email_address"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449"
+
 [[package]]
 name = "embedded-io"
 version = "0.4.0"
@@ -3780,6 +3796,37 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"

+[[package]]
+name = "lettre"
+version = "0.11.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dabda5859ee7c06b995b9d1165aa52c39110e079ef609db97178d86aeb051fa7"
+dependencies = [
+ "async-trait",
+ "base64 0.22.1",
+ "ed25519-dalek",
+ "email-encoding",
+ "email_address",
+ "fastrand",
+ "futures-io",
+ "futures-util",
+ "httpdate",
+ "idna",
+ "mime",
+ "nom 8.0.0",
+ "percent-encoding",
+ "quoted_printable",
+ "rsa",
+ "rustls 0.23.37",
+ "sha2",
+ "socket2 0.6.3",
+ "tokio",
+ "tokio-rustls 0.26.4",
+ "tracing",
+ "url",
+ "webpki-roots 1.0.6",
+]
+
 [[package]]
 name = "libc"
 version = "0.2.183"
@@ -4409,6 +4456,15 @@ dependencies = [
 "minimal-lexical",
 ]

+[[package]]
+name = "nom"
+version = "8.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "nonzero_ext"
 version = "0.3.0"
@@ -4839,7 +4895,7 @@ checksum = "9114f9c1683dd09c5f4fa024c89fdad783eaae21d3d52dd23ddaaffa29ffb168"
 dependencies = [
 "either",
 "fnv",
- "nom",
+ "nom 7.1.3",
 "once_cell",
 "postcard",
 "quick-xml",
@@ -5050,7 +5106,7 @@ version = "3.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f"
 dependencies = [
- "toml_edit",
+ "toml_edit 0.25.5+spec-1.1.0",
 ]

 [[package]]
@@ -5427,6 +5483,12 @@ dependencies = [
 "proc-macro2",
 ]

+[[package]]
+name = "quoted_printable"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "478e0585659a122aa407eb7e3c0e1fa51b1d8a870038bd29f0cf4a8551eea972"
+
 [[package]]
 name = "r-efi"
 version = "5.3.0"
@@ -5833,7 +5895,7 @@ version = "4.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632"
 dependencies = [
- "nom",
+ "nom 7.1.3",
 ]

 [[package]]
@@ -6050,6 +6112,16 @@ dependencies = [
 "zeroize",
 ]

+[[package]]
+name = "secrecy"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e891af845473308773346dc847b2c23ee78fe442e0472ac50e22a18a93d3ae5a"
+dependencies = [
+ "serde",
+ "zeroize",
+]
+
 [[package]]
 name = "security-framework"
 version = "3.7.0"
@@ -6201,6 +6273,15 @@ dependencies = [
 "syn 2.0.117",
 ]

+[[package]]
+name = "serde_spanned"
+version = "0.6.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "serde_spanned"
 version = "1.0.4"
@@ -7144,6 +7225,18 @@ dependencies = [
 "tokio",
 ]

+[[package]]
+name = "toml"
+version = "0.8.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"
+dependencies = [
+ "serde",
+ "serde_spanned 0.6.9",
+ "toml_datetime 0.6.11",
+ "toml_edit 0.22.27",
+]
+
 [[package]]
 name = "toml"
 version = "0.9.12+spec-1.1.0"
@@ -7152,13 +7245,22 @@ checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863"
 dependencies = [
 "indexmap 2.13.0",
 "serde_core",
- "serde_spanned",
+ "serde_spanned 1.0.4",
 "toml_datetime 0.7.5+spec-1.1.0",
 "toml_parser",
 "toml_writer",
 "winnow 0.7.15",
 ]

+[[package]]
+name = "toml_datetime"
+version = "0.6.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "toml_datetime"
 version = "0.7.5+spec-1.1.0"
@@ -7177,6 +7279,20 @@ dependencies = [
 "serde_core",
 ]

+[[package]]
+name = "toml_edit"
+version = "0.22.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
+dependencies = [
+ "indexmap 2.13.0",
+ "serde",
+ "serde_spanned 0.6.9",
+ "toml_datetime 0.6.11",
+ "toml_write",
+ "winnow 0.7.15",
+]
+
 [[package]]
 name = "toml_edit"
 version = "0.25.5+spec-1.1.0"
@@ -7198,6 +7314,12 @@ dependencies = [
 "winnow 1.0.0",
 ]

+[[package]]
+name = "toml_write"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
+
 [[package]]
 name = "toml_writer"
 version = "1.0.7+spec-1.1.0"
@@ -7405,7 +7527,7 @@ dependencies = [

 [[package]]
 name = "tranquil-api"
-version = "0.5.3"
+version = "0.6.0"
 dependencies = [
 "anyhow",
 "axum",
@@ -7456,7 +7578,7 @@ dependencies = [

 [[package]]
 name = "tranquil-auth"
-version = "0.5.3"
+version = "0.6.0"
 dependencies = [
 "anyhow",
 "base32",
@@ -7479,7 +7601,7 @@ dependencies = [

 [[package]]
 name = "tranquil-cache"
-version = "0.5.3"
+version = "0.6.0"
 dependencies = [
 "async-trait",
 "base64 0.22.1",
@@ -7493,11 +7615,19 @@ dependencies = [

 [[package]]
 name = "tranquil-comms"
-version = "0.5.3"
+version = "0.6.0"
 dependencies = [
 "async-trait",
 "base64 0.22.1",
+ "chrono",
+ "ed25519-dalek",
+ "futures",
+ "hickory-resolver",
+ "lettre",
+ "rand 0.8.5",
 "reqwest",
+ "rsa",
+ "secrecy",
 "serde_json",
 "sqlx",
 "thiserror 2.0.18",
@@ -7511,7 +7641,7 @@ dependencies = [

 [[package]]
 name = "tranquil-config"
-version = "0.5.3"
+version = "0.6.0"
 dependencies = [
 "confique",
 "serde",
@@ -7519,7 +7649,7 @@ dependencies = [

 [[package]]
 name = "tranquil-crypto"
-version = "0.5.3"
+version = "0.6.0"
 dependencies = [
 "aes-gcm",
 "base64 0.22.1",
@@ -7535,7 +7665,7 @@ dependencies = [

 [[package]]
 name = "tranquil-db"
-version = "0.5.3"
+version = "0.6.0"
 dependencies = [
 "async-trait",
 "chrono",
@@ -7552,7 +7682,7 @@ dependencies = [

 [[package]]
 name = "tranquil-db-traits"
-version = "0.5.3"
+version = "0.6.0"
 dependencies = [
 "async-trait",
 "base64 0.22.1",
@@ -7568,7 +7698,7 @@ dependencies = [

 [[package]]
 name = "tranquil-infra"
-version = "0.5.3"
+version = "0.6.0"
 dependencies = [
 "async-trait",
 "bytes",
@@ -7579,9 +7709,10 @@ dependencies = [

 [[package]]
 name = "tranquil-lexicon"
-version = "0.5.3"
+version = "0.6.0"
 dependencies = [
 "chrono",
+ "futures",
 "hickory-resolver",
 "parking_lot",
 "reqwest",
@@ -7597,7 +7728,7 @@ dependencies = [

 [[package]]
 name = "tranquil-oauth"
-version = "0.5.3"
+version = "0.6.0"
 dependencies = [
 "anyhow",
 "axum",
@@ -7620,7 +7751,7 @@ dependencies = [

 [[package]]
 name = "tranquil-oauth-server"
-version = "0.5.3"
+version = "0.6.0"
 dependencies = [
 "axum",
 "base64 0.22.1",
@@ -7653,7 +7784,7 @@ dependencies = [

 [[package]]
 name = "tranquil-pds"
-version = "0.5.3"
+version = "0.6.0"
 dependencies = [
 "aes-gcm",
 "anyhow",
@@ -7745,7 +7876,7 @@ dependencies = [

 [[package]]
 name = "tranquil-repo"
-version = "0.5.3"
+version = "0.6.0"
 dependencies = [
 "bytes",
 "cid",
@@ -7757,7 +7888,7 @@ dependencies = [

 [[package]]
 name = "tranquil-ripple"
-version = "0.5.3"
+version = "0.6.0"
 dependencies = [
 "async-trait",
 "backon",
@@ -7782,7 +7913,7 @@ dependencies = [

 [[package]]
 name = "tranquil-scopes"
-version = "0.5.3"
+version = "0.6.0"
 dependencies = [
 "axum",
 "futures",
@@ -7798,7 +7929,7 @@ dependencies = [

 [[package]]
 name = "tranquil-server"
-version = "0.5.3"
+version = "0.6.0"
 dependencies = [
 "axum",
 "clap",
@@ -7819,7 +7950,7 @@ dependencies = [

 [[package]]
 name = "tranquil-signal"
-version = "0.5.3"
+version = "0.6.0"
 dependencies = [
 "async-trait",
 "chrono",
@@ -7842,7 +7973,7 @@ dependencies = [

 [[package]]
 name = "tranquil-storage"
-version = "0.5.3"
+version = "0.6.0"
 dependencies = [
 "async-trait",
 "aws-config",
@@ -7859,12 +7990,13 @@ dependencies = [

 [[package]]
 name = "tranquil-store"
-version = "0.5.3"
+version = "0.6.0"
 dependencies = [
 "async-trait",
 "bytes",
 "chrono",
 "cid",
+ "clap",
 "dashmap",
 "fjall",
 "flume 0.11.1",
@@ -7892,6 +8024,7 @@ dependencies = [
 "thiserror 2.0.18",
 "tikv-jemallocator",
 "tokio",
+ "toml 0.8.23",
 "tracing",
 "tracing-subscriber",
 "tranquil-db",
@@ -7906,7 +8039,7 @@ dependencies = [

 [[package]]
 name = "tranquil-sync"
-version = "0.5.3"
+version = "0.6.0"
 dependencies = [
 "anyhow",
 "axum",
@@ -7928,7 +8061,7 @@ dependencies = [

 [[package]]
 name = "tranquil-types"
-version = "0.5.3"
+version = "0.6.0"
 dependencies = [
 "chrono",
 "cid",
@@ -8443,7 +8576,7 @@ dependencies = [
 "base64urlsafedata",
 "der-parser",
 "hex",
- "nom",
+ "nom 7.1.3",
 "openssl",
 "openssl-sys",
 "rand 0.9.2",
@@ -8853,6 +8986,9 @@ name = "winnow"
 version = "0.7.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
+dependencies = [
+ "memchr",
+]

 [[package]]
 name = "winnow"
@@ -9012,7 +9148,7 @@ dependencies = [
 "data-encoding",
 "der-parser",
 "lazy_static",
- "nom",
+ "nom 7.1.3",
 "oid-registry",
 "rusticata-macros",
 "thiserror 1.0.69",
@@ -26,7 +26,7 @@ members = [
 ]

 [workspace.package]
-version = "0.5.3"
+version = "0.6.0"
 edition = "2024"
 license = "AGPL-3.0-or-later"

@@ -93,6 +93,7 @@ ipld-core = "0.4"
 iroh-car = "0.5"
 jacquard-common = { version = "0.9", features = ["crypto-k256"] }
 jacquard-repo = "0.9"
+lettre = { version = "0.11", default-features = false, features = ["builder", "smtp-transport", "tokio1", "tokio1-rustls-tls", "pool", "dkim", "tracing"] }
 jsonwebtoken = { version = "10.2", features = ["rust_crypto"] }
 k256 = { version = "0.13", features = ["ecdsa", "pem", "pkcs8"] }
 metrics = "0.24"
@@ -105,6 +106,8 @@ p384 = { version = "0.13", features = ["ecdsa"] }
 rand = "0.8"
 redis = { version = "1.0", features = ["tokio-comp", "connection-manager"] }
 regex = "1"
+rsa = "0.9"
+secrecy = { version = "0.10", features = ["serde"] }
 reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-webpki-roots", "http2", "charset", "macos-system-configuration"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_bytes = "0.11"
@@ -35,7 +35,7 @@ COPY crates/tranquil-oauth-server ./crates/tranquil-oauth-server
 COPY crates/tranquil-store ./crates/tranquil-store
 COPY crates/tranquil-signal ./crates/tranquil-signal
 COPY crates/tranquil-server ./crates/tranquil-server
-COPY migrations ./crates/tranquil-pds/migrations
+COPY migrations ./migrations
 RUN --mount=type=cache,target=/usr/local/cargo/registry \
    --mount=type=cache,target=/app/target \
    if [ "$SLIM" = "true" ]; then \
@@ -50,7 +50,6 @@ RUN apk add --no-cache msmtp ca-certificates \
    && ln -sf /usr/bin/msmtp /usr/sbin/sendmail
 COPY --from=builder /tmp/tranquil-pds /usr/local/bin/tranquil-pds
 COPY --from=frontend /app/dist /var/lib/tranquil-pds/frontend
-COPY migrations /app/migrations
 WORKDIR /app
 ENV SERVER_HOST=0.0.0.0
 ENV SERVER_PORT=3000
@@ -46,6 +46,8 @@ just test
 just lint
 ```

+Nix users can enter a devshell with `nix develop`, or `direnv allow` to auto-enter via the bundled `.envrc`. Pre-built artifacts (including the devshell) are available from our [binary cache](docs/install-nix.md#binary-cache).
+
 ## Production Deployment

 ### Quick Deploy (Docker/Podman Compose)
@@ -59,7 +61,7 @@ podman-compose -f docker-compose.prod.yaml up -d

 ### Installation Guides

- [Debian](docs/install-debian.md)
+- [Nix](docs/install-nix.md)
 - [Containers](docs/install-containers.md)
 - [Kubernetes](docs/install-kubernetes.md)

@@ -99,4 +101,3 @@ This project is very grateful to [@nonbinary.computer](https://tangled.org/did:p
 ## License

 AGPL-3.0-or-later. Documentation is CC BY-SA 4.0. See [LICENSE](LICENSE) for details.
-
@@ -12,7 +12,6 @@ use tranquil_pds::auth::{Auth, NotTakendown, Permissive};
 use tranquil_pds::state::AppState;

 const APP_BSKY_NAMESPACE: &str = "app.bsky";
-const MAX_PREFERENCES_COUNT: usize = 100;
 const MAX_PREFERENCE_SIZE: usize = 10_000;
 const PERSONAL_DETAILS_PREF: &str = "app.bsky.actor.defs#personalDetailsPref";
 const DECLARED_AGE_PREF: &str = "app.bsky.actor.defs#declaredAgePref";
@@ -92,6 +91,8 @@ pub async fn put_preferences(
    auth: Auth<NotTakendown>,
    Json(input): Json<PutPreferencesInput>,
 ) -> Response {
+    let max_preferences_count: usize = tranquil_config::get().server.max_preferences_count;
+
    let has_full_access = auth.permissions().has_full_access();
    let user_id: uuid::Uuid = match state.repos.user.get_id_by_did(&auth.did).await {
        Ok(Some(id)) => id,
@@ -99,11 +100,11 @@ pub async fn put_preferences(
            return ApiError::InternalError(Some("User not found".into())).into_response();
        }
    };
-    if input.preferences.len() > MAX_PREFERENCES_COUNT {
+    if input.preferences.len() > max_preferences_count {
        return ApiError::InvalidRequest(format!(
            "Too many preferences: {} exceeds limit of {}",
            input.preferences.len(),
-            MAX_PREFERENCES_COUNT
+            max_preferences_count
        ))
        .into_response();
    }
@@ -76,7 +76,7 @@ pub async fn link_signal_device(
        let result = tokio::select! {
            biased;
            _ = shutdown.cancelled() => {
-                tracing::info!("Signal linking aborted due to server shutdown");
+                tracing::info!("server shutting down, aborting signal linking");
                return;
            }
            r = link_result.completion => r,
@@ -84,10 +84,10 @@ pub async fn link_signal_device(
        match result {
            Ok(Ok(client)) => {
                if slot_for_task.complete_link(generation, client).await {
-                    tracing::info!("Signal device linked successfully");
+                    tracing::info!("signal device linked");
                } else {
                    tracing::warn!(
-                        "Signal link completed but generation mismatch or already linked; discarding"
+                        "discarding completed signal link, generation mismatch or already linked"
                    );
                }
            }
@@ -282,7 +282,7 @@ async fn process_messaging_channel_update(
                "Invalid Telegram username. Must be 5-32 characters, alphanumeric or underscore".into(),
            ),
            CommsChannel::Signal => ApiError::InvalidRequest(
-                "Invalid Signal username. Must be 3-32 characters followed by .XX (e.g. username.01)".into(),
+                "Invalid Signal username. Must be a 3-32 character nickname, a dot, then a 2-20 digit discriminator".into(),
            ),
            CommsChannel::Email => ApiError::InvalidEmail,
        });
@@ -27,7 +27,6 @@ struct WriteAccumulator {
    mst: Mst<TrackingBlockStore>,
    results: Vec<WriteResult>,
    ops: Vec<RecordOp>,
-    modified_keys: Vec<String>,
    all_blob_cids: Vec<String>,
    backlinks_to_add: Vec<Backlink>,
    backlinks_to_remove: Vec<AtUri>,
@@ -44,7 +43,6 @@ async fn process_single_write(
        mst,
        mut results,
        mut ops,
-        mut modified_keys,
        mut all_blob_cids,
        mut backlinks_to_add,
        mut backlinks_to_remove,
@@ -69,8 +67,19 @@ async fn process_single_write(
                    .await?,
                )
            };
-            all_blob_cids.extend(extract_blob_cids(value));
            let rkey = rkey.clone().unwrap_or_else(Rkey::generate);
+            let key = format!("{}/{}", collection, rkey);
+            if mst
+                .get(&key)
+                .await
+                .map_err(|e| ApiError::InternalError(Some(format!("Failed to read MST: {e}"))))?
+                .is_some()
+            {
+                return Err(ApiError::InvalidRequest(format!(
+                    "Record already exists at {key}"
+                )));
+            }
+            all_blob_cids.extend(extract_blob_cids(value));
            let record_ipld = tranquil_pds::util::json_to_ipld(value);
            let record_bytes = serde_ipld_dagcbor::to_vec(&record_ipld)
                .map_err(|_| ApiError::InvalidRecord("Failed to serialize record".into()))?;
@@ -78,8 +87,6 @@ async fn process_single_write(
                .put(&record_bytes)
                .await
                .map_err(|_| ApiError::InternalError(Some("Failed to store record".into())))?;
-            let key = format!("{}/{}", collection, rkey);
-            modified_keys.push(key.clone());
            let new_mst = mst
                .add(&key, record_cid)
                .await
@@ -100,7 +107,6 @@ async fn process_single_write(
                mst: new_mst,
                results,
                ops,
-                modified_keys,
                all_blob_cids,
                backlinks_to_add,
                backlinks_to_remove,
@@ -124,16 +130,7 @@ async fn process_single_write(
                    .await?,
                )
            };
-            all_blob_cids.extend(extract_blob_cids(value));
-            let record_ipld = tranquil_pds::util::json_to_ipld(value);
-            let record_bytes = serde_ipld_dagcbor::to_vec(&record_ipld)
-                .map_err(|_| ApiError::InvalidRecord("Failed to serialize record".into()))?;
-            let record_cid = tracking_store
-                .put(&record_bytes)
-                .await
-                .map_err(|_| ApiError::InternalError(Some("Failed to store record".into())))?;
            let key = format!("{}/{}", collection, rkey);
-            modified_keys.push(key.clone());
            let prev_record_cid = mst
                .get(&key)
                .await
@@ -143,6 +140,14 @@ async fn process_single_write(
                .ok_or_else(|| {
                    ApiError::InvalidRequest("Update target record does not exist".into())
                })?;
+            all_blob_cids.extend(extract_blob_cids(value));
+            let record_ipld = tranquil_pds::util::json_to_ipld(value);
+            let record_bytes = serde_ipld_dagcbor::to_vec(&record_ipld)
+                .map_err(|_| ApiError::InvalidRecord("Failed to serialize record".into()))?;
+            let record_cid = tracking_store
+                .put(&record_bytes)
+                .await
+                .map_err(|_| ApiError::InternalError(Some("Failed to store record".into())))?;
            let new_mst = mst
                .update(&key, record_cid)
                .await
@@ -165,7 +170,6 @@ async fn process_single_write(
                mst: new_mst,
                results,
                ops,
-                modified_keys,
                all_blob_cids,
                backlinks_to_add,
                backlinks_to_remove,
@@ -173,7 +177,6 @@ async fn process_single_write(
        }
        WriteOp::Delete { collection, rkey } => {
            let key = format!("{}/{}", collection, rkey);
-            modified_keys.push(key.clone());
            let prev_record_cid = mst
                .get(&key)
                .await
@@ -198,7 +201,6 @@ async fn process_single_write(
                mst: new_mst,
                results,
                ops,
-                modified_keys,
                all_blob_cids,
                backlinks_to_add,
                backlinks_to_remove,
@@ -219,7 +221,6 @@ async fn process_writes(
        mst: initial_mst,
        results: Vec::new(),
        ops: Vec::new(),
-        modified_keys: Vec::new(),
        all_blob_cids: Vec::new(),
        backlinks_to_add: Vec::new(),
        backlinks_to_remove: Vec::new(),
@@ -351,7 +352,6 @@ pub async fn apply_writes(
        mst: final_mst,
        results,
        ops,
-        modified_keys,
        all_blob_cids,
        backlinks_to_add,
        backlinks_to_remove,
@@ -407,7 +407,6 @@ pub async fn apply_writes(
            controller_did: controller_did.as_ref(),
            delegation_detail: write_summary,
            ops,
-            modified_keys: &modified_keys,
            blob_cids: &all_blob_cids,
            backlinks_to_add,
            backlinks_to_remove,
@@ -74,7 +74,6 @@ pub async fn delete_record(
        prev: RecordCid::from(prev_record_cid),
    };

-    let modified_keys = [key];
    let deleted_uri = AtUri::from_parts(&did, &input.collection, &input.rkey);

    let commit_result = finalize_repo_write(
@@ -93,7 +92,6 @@ pub async fn delete_record(
                })
            }),
            ops: vec![op],
-            modified_keys: &modified_keys,
            blob_cids: &[],
            backlinks_to_add: vec![],
            backlinks_to_remove: vec![deleted_uri],
@@ -179,6 +179,18 @@ pub async fn create_record(
        }
    }

+    let key = format!("{}/{}", input.collection, rkey);
+    if mst
+        .get(&key)
+        .await
+        .map_err(|e| ApiError::InternalError(Some(format!("Failed to read MST: {e}"))))?
+        .is_some()
+    {
+        return Err(ApiError::InvalidRequest(format!(
+            "Record already exists at {key}"
+        )));
+    }
+
    let record_ipld = tranquil_pds::util::json_to_ipld(&input.record);
    let record_bytes = serde_ipld_dagcbor::to_vec(&record_ipld)
        .map_err(|_| ApiError::InvalidRecord("Failed to serialize record".into()))?;
@@ -187,8 +199,6 @@ pub async fn create_record(
        .put(&record_bytes)
        .await
        .map_err(|_| ApiError::InternalError(Some("Failed to save record block".into())))?;
-
-    let key = format!("{}/{}", input.collection, rkey);
    mst = mst
        .add(&key, record_cid)
        .await
@@ -200,20 +210,6 @@ pub async fn create_record(
        cid: tranquil_pds::cid_types::RecordCid::from(record_cid),
    });

-    let modified_keys: Vec<String> = ops
-        .iter()
-        .map(|op| match op {
-            RecordOp::Create {
-                collection, rkey, ..
-            }
-            | RecordOp::Update {
-                collection, rkey, ..
-            }
-            | RecordOp::Delete {
-                collection, rkey, ..
-            } => format!("{}/{}", collection, rkey),
-        })
-        .collect();
    let blob_cids = extract_blob_cids(&input.record);

    let created_uri = AtUri::from_parts(&did, &input.collection, &rkey);
@@ -235,7 +231,6 @@ pub async fn create_record(
                })
            }),
            ops,
-            modified_keys: &modified_keys,
            blob_cids: &blob_cids,
            backlinks_to_add,
            backlinks_to_remove: conflict_uris_to_cleanup,
@@ -367,7 +362,6 @@ pub async fn put_record(
        }
    };

-    let modified_keys = [key];
    let blob_cids = extract_blob_cids(&input.record);
    let backlinks_to_add = extract_backlinks(&record_uri, &input.record);

@@ -387,7 +381,6 @@ pub async fn put_record(
                })
            }),
            ops: vec![op],
-            modified_keys: &modified_keys,
            blob_cids: &blob_cids,
            backlinks_to_add,
            backlinks_to_remove,
@@ -163,6 +163,7 @@ pub async fn create_session(
    let email_2fa_enabled = row.email_2fa_enabled;
    let is_legacy_login = has_totp || email_2fa_enabled;
    let twofa_ctx = tranquil_pds::auth::legacy_2fa::Legacy2faContext {
+        is_app_password: app_password_name.is_some(),
        email_2fa_enabled,
        has_totp,
        allow_legacy_login: row.allow_legacy_login,
@@ -10,7 +10,14 @@ tranquil-signal = { workspace = true }

 async-trait = { workspace = true }
 base64 = { workspace = true }
+ed25519-dalek = { workspace = true }
+futures = { workspace = true }
+hickory-resolver = { workspace = true }
+lettre = { workspace = true }
+rand = { workspace = true }
 reqwest = { workspace = true }
+rsa = { workspace = true }
+secrecy = { workspace = true }
 serde_json = { workspace = true }
 sqlx = { workspace = true }
 thiserror = { workspace = true }
@@ -18,3 +25,7 @@ tokio = { workspace = true }
 tracing = { workspace = true }
 tranquil-db-traits = { workspace = true }
 uuid = { workspace = true }
+
+[dev-dependencies]
+chrono = { workspace = true }
+tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time", "io-util", "net"] }
@@ -0,0 +1,291 @@
+use std::path::PathBuf;
+
+#[derive(Debug, thiserror::Error)]
+pub enum ParseError {
+    #[error("empty value")]
+    Empty,
+    #[error("invalid character {0:?}")]
+    InvalidChar(char),
+    #[error("zero {0}")]
+    Zero(&'static str),
+    #[error("invalid TLS mode {0:?}")]
+    InvalidTlsMode(String),
+}
+
+fn parse_token(raw: &str, lowercase: bool, strip_trailing_dot: bool) -> Result<String, ParseError> {
+    let mut s = raw.trim();
+    if strip_trailing_dot {
+        s = s.trim_end_matches('.');
+    }
+    match s {
+        "" => Err(ParseError::Empty),
+        _ if s.chars().any(char::is_whitespace) => Err(ParseError::InvalidChar(' ')),
+        _ => Ok(match lowercase {
+            true => s.to_lowercase(),
+            false => s.to_string(),
+        }),
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct SmtpHost(String);
+
+impl SmtpHost {
+    pub fn parse(raw: &str) -> Result<Self, ParseError> {
+        parse_token(raw, true, false).map(Self)
+    }
+
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct SmtpPort(u16);
+
+impl SmtpPort {
+    pub fn parse(raw: u16) -> Result<Self, ParseError> {
+        match raw {
+            0 => Err(ParseError::Zero("smtp port")),
+            n => Ok(Self(n)),
+        }
+    }
+
+    pub fn as_u16(self) -> u16 {
+        self.0
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct HeloName(String);
+
+impl HeloName {
+    pub fn parse(raw: &str) -> Result<Self, ParseError> {
+        parse_token(raw, false, false).map(Self)
+    }
+
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+
+    pub fn into_inner(self) -> String {
+        self.0
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct EmailDomain(String);
+
+impl EmailDomain {
+    pub fn parse(raw: &str) -> Result<Self, ParseError> {
+        parse_token(raw, true, true).map(Self)
+    }
+
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+
+    pub fn into_inner(self) -> String {
+        self.0
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct MxHost(String);
+
+impl MxHost {
+    pub fn parse(raw: &str) -> Result<Self, ParseError> {
+        parse_token(raw, true, true).map(Self)
+    }
+
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct MxPriority(u16);
+
+impl MxPriority {
+    pub fn new(value: u16) -> Self {
+        Self(value)
+    }
+
+    pub fn as_u16(self) -> u16 {
+        self.0
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct MxRecord {
+    pub priority: MxPriority,
+    pub host: MxHost,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct DkimSelector(String);
+
+impl DkimSelector {
+    pub fn parse(raw: &str) -> Result<Self, ParseError> {
+        let trimmed = raw.trim();
+        let valid = !trimmed.is_empty() && trimmed.split('.').all(valid_subdomain);
+        match valid {
+            true => Ok(Self(trimmed.to_string())),
+            false => Err(ParseError::InvalidChar('?')),
+        }
+    }
+
+    pub fn into_inner(self) -> String {
+        self.0
+    }
+}
+
+fn valid_subdomain(seg: &str) -> bool {
+    let starts_alnum = seg
+        .chars()
+        .next()
+        .is_some_and(|c| c.is_ascii_alphanumeric());
+    let ends_alnum = seg
+        .chars()
+        .next_back()
+        .is_some_and(|c| c.is_ascii_alphanumeric());
+    let body_ok = seg.chars().all(|c| c.is_ascii_alphanumeric() || c == '-');
+    starts_alnum && ends_alnum && body_ok
+}
+
+#[derive(Debug, Clone)]
+pub struct DkimKeyPath(PathBuf);
+
+impl DkimKeyPath {
+    pub fn parse(raw: &str) -> Result<Self, ParseError> {
+        let trimmed = raw.trim();
+        match trimmed.is_empty() {
+            true => Err(ParseError::Empty),
+            false => Ok(Self(PathBuf::from(trimmed))),
+        }
+    }
+
+    pub fn as_path(&self) -> &std::path::Path {
+        &self.0
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SmtpUsername(String);
+
+impl SmtpUsername {
+    pub fn parse(raw: &str) -> Result<Self, ParseError> {
+        match raw.is_empty() {
+            true => Err(ParseError::Empty),
+            false => Ok(Self(raw.to_string())),
+        }
+    }
+
+    pub fn into_inner(self) -> String {
+        self.0
+    }
+}
+
+#[derive(Clone)]
+pub struct SmtpPassword(secrecy::SecretString);
+
+impl SmtpPassword {
+    pub fn parse(raw: &str) -> Result<Self, ParseError> {
+        match raw.is_empty() {
+            true => Err(ParseError::Empty),
+            false => Ok(Self(secrecy::SecretString::from(raw.to_string()))),
+        }
+    }
+
+    pub fn expose(&self) -> &str {
+        use secrecy::ExposeSecret;
+        self.0.expose_secret()
+    }
+}
+
+impl std::fmt::Debug for SmtpPassword {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str("SmtpPassword(***)")
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TlsMode {
+    Implicit,
+    Starttls,
+    None,
+}
+
+impl TlsMode {
+    pub fn parse(raw: &str) -> Result<Self, ParseError> {
+        match raw.to_ascii_lowercase().as_str() {
+            "implicit" => Ok(Self::Implicit),
+            "starttls" => Ok(Self::Starttls),
+            "none" => Ok(Self::None),
+            other => Err(ParseError::InvalidTlsMode(other.to_string())),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn smtp_host_lowercases_and_trims() {
+        let h = SmtpHost::parse("  SMTP.NEL.PET  ").unwrap();
+        assert_eq!(h.as_str(), "smtp.nel.pet");
+    }
+
+    #[test]
+    fn smtp_host_rejects_whitespace() {
+        assert!(SmtpHost::parse("a b").is_err());
+    }
+
+    #[test]
+    fn smtp_host_rejects_empty() {
+        assert!(SmtpHost::parse("").is_err());
+        assert!(SmtpHost::parse("   ").is_err());
+    }
+
+    #[test]
+    fn smtp_port_rejects_zero() {
+        assert!(SmtpPort::parse(0).is_err());
+        assert_eq!(SmtpPort::parse(587).unwrap().as_u16(), 587);
+    }
+
+    #[test]
+    fn email_domain_strips_trailing_dot() {
+        assert_eq!(EmailDomain::parse("Nel.pet.").unwrap().as_str(), "nel.pet");
+    }
+
+    #[test]
+    fn dkim_selector_validates() {
+        assert!(DkimSelector::parse("default").is_ok());
+        assert!(DkimSelector::parse("s1.nel.pet").is_ok());
+        assert!(DkimSelector::parse("s2024-q1").is_ok());
+        assert!(DkimSelector::parse("mailo-2024.nel.pet").is_ok());
+        assert!(DkimSelector::parse("a-b").is_ok());
+        assert!(DkimSelector::parse("").is_err());
+        assert!(DkimSelector::parse("a..b").is_err());
+        assert!(DkimSelector::parse("-leading").is_err());
+        assert!(DkimSelector::parse("trailing-").is_err());
+        assert!(DkimSelector::parse("s_under").is_err());
+    }
+
+    #[test]
+    fn tls_mode_parses_known_modes() {
+        assert_eq!(TlsMode::parse("STARTTLS").unwrap(), TlsMode::Starttls);
+        assert_eq!(TlsMode::parse("implicit").unwrap(), TlsMode::Implicit);
+        assert_eq!(TlsMode::parse("none").unwrap(), TlsMode::None);
+        assert!(TlsMode::parse("garbage").is_err());
+    }
+
+    #[test]
+    fn smtp_password_redacts_in_debug() {
+        let p = SmtpPassword::parse("hunter2").unwrap();
+        let dbg = format!("{:?}", p);
+        assert_eq!(dbg, "SmtpPassword(***)");
+        assert!(!dbg.contains("hunter2"));
+    }
+}
@@ -299,7 +299,7 @@ impl DiscordSender {
            "type": 1,
            "options": [{
                "name": "handle",
-                "description": "Your PDS handle (e.g. alice.example.com)",
+                "description": "Your PDS handle",
                "type": 3,
                "required": false
            }]
@@ -5,6 +5,14 @@ use std::sync::OnceLock;

 static CONFIG: OnceLock<TranquilConfig> = OnceLock::new();

+const REMOVED_ENV_VARS: &[(&str, &str)] = &[(
+    "SENDMAIL_PATH",
+    "the sendmail-binary transport was replaced with native SMTP. \
+     Configure MAIL_SMARTHOST_HOST for relay delivery, or leave it unset to \
+     deliver directly via recipient MX records. See example.toml for the full \
+     MAIL_* surface.",
+)];
+
 /// Errors discovered during configuration validation.
 #[derive(Debug)]
 pub struct ConfigError {
@@ -65,6 +73,9 @@ pub fn ensure_test_defaults() {
            if env::var("ENABLE_PDS_HOSTED_DID_WEB").is_err() {
                env::set_var("ENABLE_PDS_HOSTED_DID_WEB", "true");
            }
+            if env::var("TRANQUIL_LEXICON_OFFLINE").is_err() {
+                env::set_var("TRANQUIL_LEXICON_OFFLINE", "1");
+            }
        }
        TranquilConfig::builder()
            .env()
@@ -80,8 +91,9 @@ pub fn ensure_test_defaults() {
 ///
 /// Precedence (highest to lowest):
 /// 1. Environment variables
-/// 2. TOML config file (if provided)
-/// 3. Built-in defaults
+/// 2. Toml config file passed as `config_path`, if provided
+/// 3. `/etc/tranquil-pds/config.toml` - hardcoded fallback, silently skipped if absent
+/// 4. Built-in defaults
 pub fn load(config_path: Option<&PathBuf>) -> Result<TranquilConfig, confique::Error> {
    let mut builder = TranquilConfig::builder().env();
    if let Some(path) = config_path {
@@ -108,6 +120,9 @@ pub struct TranquilConfig {
    #[config(nested)]
    pub storage: StorageConfig,

+    #[config(nested)]
+    pub tranquil_store: TranquilStoreConfig,
+
    #[config(nested)]
    pub cache: CacheConfig,

@@ -143,9 +158,6 @@ pub struct TranquilConfig {

    #[config(nested)]
    pub scheduled: ScheduledConfig,
-
-    #[config(nested)]
-    pub tranquil_store: TranquilStoreConfig,
 }

 impl TranquilConfig {
@@ -158,6 +170,14 @@ impl TranquilConfig {
    pub fn validate(&self, ignore_secrets: bool) -> Result<(), ConfigError> {
        let mut errors = Vec::new();

+        // -- removed config ---------------------------------------------------
+        errors.extend(
+            REMOVED_ENV_VARS
+                .iter()
+                .filter(|(var, _)| std::env::var_os(var).is_some())
+                .map(|(var, guidance)| format!("{var} is no longer supported: {guidance}")),
+        );
+
        // -- secrets ----------------------------------------------------------
        if !ignore_secrets && !self.secrets.allow_insecure && !cfg!(test) {
            if let Some(ref s) = self.secrets.jwt_secret {
@@ -206,6 +226,85 @@ impl TranquilConfig {
            }
        }

+        // -- email smarthost --------------------------------------------------
+        match self.email.smarthost.tls.to_ascii_lowercase().as_str() {
+            "implicit" | "starttls" => {}
+            "none" => {
+                if self.email.smarthost.password.is_some() {
+                    errors.push(
+                        "email.smarthost.tls = \"none\" with email.smarthost.password set \
+                         would transmit credentials in plaintext; use \"starttls\" or \"implicit\""
+                            .to_string(),
+                    );
+                }
+            }
+            other => errors.push(format!(
+                "email.smarthost.tls must be \"implicit\", \"starttls\", or \"none\", got \"{other}\""
+            )),
+        }
+
+        let smarthost_host_set = self
+            .email
+            .smarthost
+            .host
+            .as_deref()
+            .is_some_and(|h| !h.is_empty());
+        let username_set = self.email.smarthost.username.is_some();
+        let password_set = self.email.smarthost.password.is_some();
+        if !smarthost_host_set && (username_set || password_set) {
+            errors.push(
+                "email.smarthost.username or email.smarthost.password is set but \
+                 email.smarthost.host is empty; credentials would be silently ignored"
+                    .to_string(),
+            );
+        }
+        if smarthost_host_set && username_set != password_set {
+            errors.push(
+                "email.smarthost.username and email.smarthost.password must both be set or \
+                 both unset; otherwise authentication would silently degrade to anonymous"
+                    .to_string(),
+            );
+        }
+
+        if self.email.smarthost.command_timeout_secs == 0 {
+            errors.push("email.smarthost.command_timeout_secs must be at least 1".to_string());
+        }
+        if self.email.smarthost.total_timeout_secs == 0 {
+            errors.push("email.smarthost.total_timeout_secs must be at least 1".to_string());
+        }
+        if self.email.smarthost.pool_size == 0 {
+            errors.push("email.smarthost.pool_size must be at least 1".to_string());
+        }
+
+        if self.email.direct_mx.max_concurrent_sends == 0 {
+            errors.push("email.direct_mx.max_concurrent_sends must be at least 1".to_string());
+        }
+        if self.email.direct_mx.command_timeout_secs == 0 {
+            errors.push("email.direct_mx.command_timeout_secs must be at least 1".to_string());
+        }
+        if self.email.direct_mx.total_timeout_secs == 0 {
+            errors.push("email.direct_mx.total_timeout_secs must be at least 1".to_string());
+        }
+
+        let dkim_set = self.email.dkim.selector.is_some()
+            || self.email.dkim.domain.is_some()
+            || self.email.dkim.private_key_path.is_some();
+        if dkim_set {
+            if self.email.dkim.selector.is_none() {
+                errors
+                    .push("email.dkim.selector is required when any DKIM field is set".to_string());
+            }
+            if self.email.dkim.domain.is_none() {
+                errors.push("email.dkim.domain is required when any DKIM field is set".to_string());
+            }
+            if self.email.dkim.private_key_path.is_none() {
+                errors.push(
+                    "email.dkim.private_key_path is required when any DKIM field is set"
+                        .to_string(),
+                );
+            }
+        }
+
        // -- telegram ---------------------------------------------------------
        if self.telegram.bot_token.is_some() && self.telegram.webhook_secret.is_none() {
            errors.push(
@@ -394,7 +493,7 @@ impl TranquilConfig {

 #[derive(Debug, Config)]
 pub struct ServerConfig {
-    /// Public hostname of the PDS (e.g. `pds.example.com`).
+    /// Public hostname of the PDS, such as `pds.example.com`.
    #[config(env = "PDS_HOSTNAME")]
    pub hostname: String,

@@ -451,6 +550,10 @@ pub struct ServerConfig {
    /// Maximum allowed blob size in bytes (default 10 GiB).
    #[config(env = "MAX_BLOB_SIZE", default = 10_737_418_240u64)]
    pub max_blob_size: u64,
+
+    /// Maximum allowed number of preferences
+    #[config(env = "MAX_PREFERENCES_COUNT", default = 1000)]
+    pub max_preferences_count: usize,
 }

 impl ServerConfig {
@@ -459,8 +562,7 @@ impl ServerConfig {
        format!("https://{}", self.hostname)
    }

-    /// Hostname without port suffix (e.g. `pds.example.com` from
-    /// `pds.example.com:443`).
+    /// Hostname without port suffix. Returns `pds.example.com` from `pds.example.com:443`.
    pub fn hostname_without_port(&self) -> &str {
        self.hostname.split(':').next().unwrap_or(&self.hostname)
    }
@@ -656,10 +758,12 @@ pub struct StorageConfig {
    #[config(env = "S3_BUCKET")]
    pub s3_bucket: Option<String>,

-    /// Custom S3 endpoint URL (for MinIO, R2, etc.).
+    /// Custom S3 endpoint URL.
    #[config(env = "S3_ENDPOINT")]
    pub s3_endpoint: Option<String>,

+    /// Repository backend: `postgres` by default, or `tranquil-store`, our embedded db.
+    /// tranquil-store is EXPERIMENTAL!!!! RISK OF TOTAL DATA LOSS.
    #[config(env = "REPO_BACKEND", default = "postgres")]
    pub repo_backend: String,
 }
@@ -674,7 +778,7 @@ impl StorageConfig {

 #[derive(Debug, Config)]
 pub struct CacheConfig {
-    /// Cache backend: `ripple` (default, built-in gossip) or `valkey`.
+    /// Cache backend: `ripple` by default, or `valkey`.
    #[config(env = "CACHE_BACKEND", default = "ripple")]
    pub backend: String,

@@ -716,10 +820,6 @@ pub struct FirehoseConfig {
    #[config(env = "FIREHOSE_BACKFILL_HOURS", default = 72)]
    pub backfill_hours: i64,

-    /// Maximum number of lagged events before disconnecting a slow consumer.
-    #[config(env = "FIREHOSE_MAX_LAG", default = 5000)]
-    pub max_lag: u64,
-
    /// Maximum concurrent full-repo exports, eg. getRepo without `since`.
    #[config(env = "MAX_CONCURRENT_REPO_EXPORTS", default = 4)]
    pub max_concurrent_repo_exports: usize,
@@ -749,9 +849,98 @@ pub struct EmailConfig {
    #[config(env = "MAIL_FROM_NAME", default = "Tranquil PDS")]
    pub from_name: String,

-    /// Path to the `sendmail` binary.
-    #[config(env = "SENDMAIL_PATH", default = "/usr/sbin/sendmail")]
-    pub sendmail_path: String,
+    /// HELO/EHLO name announced to remote SMTP servers. Applies to both
+    /// smarthost and direct-MX modes. Defaults to the server hostname.
+    #[config(env = "MAIL_HELO_NAME")]
+    pub helo_name: Option<String>,
+
+    #[config(nested)]
+    pub smarthost: SmarthostConfig,
+
+    #[config(nested)]
+    pub direct_mx: DirectMxConfig,
+
+    #[config(nested)]
+    pub dkim: DkimConfig,
+}
+
+#[derive(Debug, Config)]
+pub struct SmarthostConfig {
+    /// SMTP relay host. When set, mail is delivered through this host
+    /// instead of resolving recipient MX records directly.
+    #[config(env = "MAIL_SMARTHOST_HOST")]
+    pub host: Option<String>,
+
+    /// SMTP relay port.
+    #[config(env = "MAIL_SMARTHOST_PORT", default = 587)]
+    pub port: u16,
+
+    /// SMTP authentication username.
+    #[config(env = "MAIL_SMARTHOST_USERNAME")]
+    pub username: Option<String>,
+
+    /// SMTP authentication password.
+    #[config(env = "MAIL_SMARTHOST_PASSWORD")]
+    pub password: Option<String>,
+
+    /// TLS mode. Valid values: "implicit", "starttls", "none". Setting "none"
+    /// alongside a password is rejected at startup to prevent transmitting
+    /// credentials in plaintext.
+    #[config(env = "MAIL_SMARTHOST_TLS", default = "starttls")]
+    pub tls: String,
+
+    /// Max size of the connection pool.
+    #[config(env = "MAIL_SMARTHOST_POOL_SIZE", default = 4)]
+    pub pool_size: u32,
+
+    /// Per-command SMTP timeout in seconds. Bounds the security handshake.
+    #[config(env = "MAIL_SMARTHOST_COMMAND_TIMEOUT_SECS", default = 30)]
+    pub command_timeout_secs: u64,
+
+    /// Total per-message timeout in seconds. Wraps the entire send so a
+    /// stuck relay cannot stall the comms queue.
+    #[config(env = "MAIL_SMARTHOST_TOTAL_TIMEOUT_SECS", default = 60)]
+    pub total_timeout_secs: u64,
+}
+
+#[derive(Debug, Config)]
+pub struct DirectMxConfig {
+    /// Per-command SMTP timeout in seconds.
+    #[config(env = "MAIL_COMMAND_TIMEOUT_SECS", default = 30)]
+    pub command_timeout_secs: u64,
+
+    /// Total per-message timeout across all MX attempts in seconds.
+    #[config(env = "MAIL_TOTAL_TIMEOUT_SECS", default = 60)]
+    pub total_timeout_secs: u64,
+
+    /// Max number of concurrent direct-MX sends. Limits the load placed
+    /// on any single recipient MX during a backlog drain.
+    #[config(env = "MAIL_MAX_CONCURRENT_SENDS", default = 8)]
+    pub max_concurrent_sends: usize,
+
+    /// Require STARTTLS on every MX hop. When false, TLS is
+    /// attempted opportunistically and the session falls back to plaintext
+    /// if the remote does not advertise STARTTLS. Set true to refuse
+    /// plaintext delivery, at the cost of failing sends to MX hosts that
+    /// do not support TLS.
+    #[config(env = "MAIL_REQUIRE_TLS", default = false)]
+    pub require_tls: bool,
+}
+
+#[derive(Debug, Config)]
+pub struct DkimConfig {
+    /// DKIM selector. When unset, outgoing mail is not signed.
+    #[config(env = "MAIL_DKIM_SELECTOR")]
+    pub selector: Option<String>,
+
+    /// DKIM signing domain.
+    #[config(env = "MAIL_DKIM_DOMAIN")]
+    pub domain: Option<String>,
+
+    /// Path to the DKIM private key in PEM format. Supports RSA and
+    /// Ed25519 keys.
+    #[config(env = "MAIL_DKIM_KEY_PATH")]
+    pub private_key_path: Option<String>,
 }

 #[derive(Debug, Config)]
@@ -774,6 +963,8 @@ pub struct TelegramConfig {

 #[derive(Debug, Config)]
 pub struct SignalConfig {
+    /// Protocol state is stored in postgres' signal_* tables.
+    /// Link a device via the admin API before enabling.
    #[config(env = "SIGNAL_ENABLED", default = false)]
    pub enabled: bool,
 }
@@ -1124,27 +1315,26 @@ pub struct ScheduledConfig {

    /// Maximum age of events retained in the eventlog before pruning.
    /// Per the atproto firehose spec, the relay backfill window only needs
-    /// to cover "hours or days". Default: 7 days.
+    /// to cover "hours or days".
    #[config(env = "EVENT_RETENTION_MAX_AGE_SECS", default = 604800)]
    pub event_retention_max_age_secs: u64,

    /// Interval in seconds between event retention prune passes.
-    /// Set to 0 to disable. Default: hourly.
+    /// Set to 0 to disable.
    #[config(env = "EVENT_RETENTION_INTERVAL_SECS", default = 3600)]
    pub event_retention_interval_secs: u64,
 }

 #[derive(Debug, Config)]
 pub struct TranquilStoreConfig {
-    /// Directory for tranquil-store data (metastore, eventlog).
+    /// Directory for tranquil-store data: the metastore, eventlog, and blockstore.
    #[config(
        env = "TRANQUIL_STORE_DATA_DIR",
        default = "/var/lib/tranquil-pds/store"
    )]
    pub data_dir: String,

-    /// Fjall block cache size in megabytes. Defaults to 20% of system RAM
-    /// when unset.
+    /// Fjall block cache size in megabytes. Defaults to 20% of system RAM when unset.
    #[config(env = "TRANQUIL_STORE_MEMORY_BUDGET_MB")]
    pub memory_budget_mb: Option<u64>,

@@ -1152,9 +1342,9 @@ pub struct TranquilStoreConfig {
    #[config(env = "TRANQUIL_STORE_HANDLER_THREADS")]
    pub handler_threads: Option<usize>,

-    /// Maximum total bytes of pending (unsynced) eventlog payloads. Appenders
-    /// block once this budget is exhausted until in-flight events drain via
-    /// fsync. Set to 0 to disable backpressure (unbounded). Default: 1 GiB.
+    /// Maximum total bytes of pending (unsynced) eventlog payloads. Appenders block
+    /// once this budget is exhausted until in-flight events drain via fsync. Set to
+    /// 0 to disable backpressure. Default: 1 GiB.
    #[config(
        env = "TRANQUIL_STORE_EVENTLOG_PENDING_BYTES_BUDGET",
        default = 1_073_741_824
@@ -1190,3 +1380,64 @@ pub struct TranquilStoreConfig {
 pub fn template() -> String {
    confique::toml::template::<TranquilConfig>(confique::toml::FormatOptions::default())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn seed_required_env() {
+        let required = [
+            ("PDS_HOSTNAME", "test.local"),
+            ("DATABASE_URL", "postgres://localhost/test"),
+            ("TRANQUIL_PDS_ALLOW_INSECURE_SECRETS", "1"),
+            ("INVITE_CODE_REQUIRED", "false"),
+            ("ENABLE_PDS_HOSTED_DID_WEB", "true"),
+            ("TRANQUIL_LEXICON_OFFLINE", "1"),
+        ];
+        required
+            .iter()
+            .filter(|(k, _)| std::env::var_os(k).is_none())
+            .for_each(|(k, v)| unsafe { std::env::set_var(k, v) });
+    }
+
+    #[test]
+    fn serial_validate_rejects_legacy_sendmail_path() {
+        seed_required_env();
+        unsafe { std::env::set_var("SENDMAIL_PATH", "/usr/sbin/sendmail") };
+        let config = TranquilConfig::builder()
+            .env()
+            .load()
+            .expect("load fresh config");
+        let result = config.validate(true);
+        unsafe { std::env::remove_var("SENDMAIL_PATH") };
+
+        let err = result.expect_err("validate must reject SENDMAIL_PATH");
+        let mentions_sendmail = err.errors.iter().any(|e| e.contains("SENDMAIL_PATH"));
+        assert!(
+            mentions_sendmail,
+            "errors did not mention SENDMAIL_PATH: {:?}",
+            err.errors
+        );
+    }
+
+    #[test]
+    fn serial_validate_passes_when_no_legacy_env_set() {
+        seed_required_env();
+        unsafe { std::env::remove_var("SENDMAIL_PATH") };
+        let config = TranquilConfig::builder()
+            .env()
+            .load()
+            .expect("load fresh config");
+        let result = config.validate(true);
+        let leaked_legacy = result
+            .as_ref()
+            .err()
+            .map(|e| e.errors.iter().any(|s| s.contains("SENDMAIL_PATH")))
+            .unwrap_or(false);
+        assert!(
+            !leaked_legacy,
+            "validate spuriously flagged SENDMAIL_PATH when unset: {:?}",
+            result
+        );
+    }
+}
@@ -7,7 +7,7 @@ use uuid::Uuid;

 use crate::DbError;

-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub enum BacklinkPath {
    Subject,
    SubjectUri,
@@ -24,3 +24,4 @@ urlencoding = { workspace = true, optional = true }
 [dev-dependencies]
 wiremock = { workspace = true }
 tokio = { workspace = true }
+futures = { workspace = true }
@@ -5,46 +5,102 @@ use std::collections::{HashMap, VecDeque};
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::time::{Duration, Instant};
+use tokio::sync::Notify;

 const NEGATIVE_CACHE_TTL: Duration = Duration::from_secs(24 * 60 * 60);
+const POSITIVE_CACHE_TTL: Duration = Duration::from_secs(24 * 60 * 60);
+const REFRESH_FAILURE_BACKOFF: Duration = Duration::from_secs(60);
 const MAX_DYNAMIC_SCHEMAS: usize = 1024;

 struct NegativeEntry {
    expires_at: Instant,
 }

+struct PositiveEntry {
+    doc: Arc<LexiconDoc>,
+    expires_at: Instant,
+}
+
+pub(crate) enum CacheEntry {
+    Fresh(Arc<LexiconDoc>),
+    Stale(Arc<LexiconDoc>),
+}
+
+impl CacheEntry {
+    #[cfg(test)]
+    fn is_fresh(&self) -> bool {
+        matches!(self, Self::Fresh(_))
+    }
+}
+
 struct SchemaStore {
-    schemas: HashMap<String, Arc<LexiconDoc>>,
+    schemas: HashMap<String, PositiveEntry>,
    insertion_order: VecDeque<String>,
 }

 pub struct DynamicRegistry {
    store: RwLock<SchemaStore>,
    negative_cache: RwLock<HashMap<String, NegativeEntry>>,
+    in_flight: RwLock<HashMap<String, Arc<Notify>>>,
    network_disabled: AtomicBool,
 }

+struct InFlightGuard<'a> {
+    registry: &'a DynamicRegistry,
+    nsid: String,
+}
+
+impl Drop for InFlightGuard<'_> {
+    fn drop(&mut self) {
+        let notify = self.registry.in_flight.write().remove(&self.nsid);
+        if let Some(n) = notify {
+            n.notify_waiters();
+        }
+    }
+}
+
 impl DynamicRegistry {
    pub fn new() -> Self {
-        let network_disabled =
-            std::env::var("TRANQUIL_LEXICON_OFFLINE").is_ok_and(|v| v == "1" || v == "true");
        Self {
            store: RwLock::new(SchemaStore {
                schemas: HashMap::new(),
                insertion_order: VecDeque::new(),
            }),
            negative_cache: RwLock::new(HashMap::new()),
-            network_disabled: AtomicBool::new(network_disabled),
+            in_flight: RwLock::new(HashMap::new()),
+            network_disabled: AtomicBool::new(false),
        }
    }

-    #[allow(dead_code)]
+    pub fn from_env() -> Self {
+        let registry = Self::new();
+        let disabled =
+            std::env::var("TRANQUIL_LEXICON_OFFLINE").is_ok_and(|v| v == "1" || v == "true");
+        registry.set_network_disabled(disabled);
+        registry
+    }
+
    pub fn set_network_disabled(&self, disabled: bool) {
        self.network_disabled.store(disabled, Ordering::Relaxed);
    }

-    pub fn get(&self, nsid: &str) -> Option<Arc<LexiconDoc>> {
-        self.store.read().schemas.get(nsid).cloned()
+    pub fn get_cached(&self, nsid: &str) -> Option<Arc<LexiconDoc>> {
+        self.store
+            .read()
+            .schemas
+            .get(nsid)
+            .map(|e| Arc::clone(&e.doc))
+    }
+
+    pub(crate) fn get_entry(&self, nsid: &str) -> Option<CacheEntry> {
+        let now = Instant::now();
+        self.store.read().schemas.get(nsid).map(|e| {
+            if e.expires_at > now {
+                CacheEntry::Fresh(Arc::clone(&e.doc))
+            } else {
+                CacheEntry::Stale(Arc::clone(&e.doc))
+            }
+        })
    }

    pub fn is_negative_cached(&self, nsid: &str) -> bool {
@@ -56,7 +112,7 @@ impl DynamicRegistry {

    fn insert_negative(&self, nsid: &str) {
        let mut cache = self.negative_cache.write();
-        if cache.len() > MAX_DYNAMIC_SCHEMAS {
+        if cache.len() >= MAX_DYNAMIC_SCHEMAS {
            let now = Instant::now();
            cache.retain(|_, entry| entry.expires_at > now);
        }
@@ -87,29 +143,95 @@ impl DynamicRegistry {
            });
        }

-        if store
-            .schemas
-            .insert(nsid.clone(), Arc::clone(&arc))
-            .is_some()
-        {
+        let entry = PositiveEntry {
+            doc: Arc::clone(&arc),
+            expires_at: Instant::now() + POSITIVE_CACHE_TTL,
+        };
+        if store.schemas.insert(nsid.clone(), entry).is_some() {
            store.insertion_order.retain(|k| k != &nsid);
        }
        store.insertion_order.push_back(nsid.clone());
+        drop(store);

        self.negative_cache.write().remove(&arc.id);

        arc
    }

+    fn bump_expiry(&self, nsid: &str, duration: Duration) {
+        let mut store = self.store.write();
+        if let Some(entry) = store.schemas.get_mut(nsid) {
+            entry.expires_at = Instant::now() + duration;
+        }
+    }
+
    pub async fn resolve_and_cache(&self, nsid: &str) -> Result<Arc<LexiconDoc>, ResolveError> {
-        if let Some(doc) = self.get(nsid) {
-            return Ok(doc);
+        self.resolve_and_cache_with(nsid, |n| async move { resolve_lexicon(&n).await })
+            .await
+    }
+
+    async fn resolve_and_cache_with<F, Fut>(
+        &self,
+        nsid: &str,
+        resolver: F,
+    ) -> Result<Arc<LexiconDoc>, ResolveError>
+    where
+        F: FnOnce(String) -> Fut,
+        Fut: std::future::Future<Output = Result<LexiconDoc, ResolveError>>,
+    {
+        match self.get_entry(nsid) {
+            Some(CacheEntry::Fresh(doc)) => Ok(doc),
+            Some(CacheEntry::Stale(stale)) => self.refresh_stale(nsid, stale, resolver).await,
+            None => self.resolve_fresh(nsid, resolver).await,
+        }
+    }
+
+    async fn refresh_stale<F, Fut>(
+        &self,
+        nsid: &str,
+        stale: Arc<LexiconDoc>,
+        resolver: F,
+    ) -> Result<Arc<LexiconDoc>, ResolveError>
+    where
+        F: FnOnce(String) -> Fut,
+        Fut: std::future::Future<Output = Result<LexiconDoc, ResolveError>>,
+    {
+        if self.network_disabled.load(Ordering::Relaxed) {
+            return Ok(stale);
        }

+        match self.acquire_leadership(nsid) {
+            Some(_guard) => match resolver(nsid.to_string()).await {
+                Ok(doc) => Ok(self.insert_schema(doc)),
+                Err(e) => {
+                    self.bump_expiry(nsid, REFRESH_FAILURE_BACKOFF);
+                    tracing::warn!(
+                        nsid = nsid,
+                        error = %e,
+                        "lexicon refresh failed, serving stale cached entry"
+                    );
+                    Ok(stale)
+                }
+            },
+            None => {
+                self.wait_for_leader(nsid).await;
+                Ok(self.get_cached(nsid).unwrap_or(stale))
+            }
+        }
+    }
+
+    async fn resolve_fresh<F, Fut>(
+        &self,
+        nsid: &str,
+        resolver: F,
+    ) -> Result<Arc<LexiconDoc>, ResolveError>
+    where
+        F: FnOnce(String) -> Fut,
+        Fut: std::future::Future<Output = Result<LexiconDoc, ResolveError>>,
+    {
        if self.network_disabled.load(Ordering::Relaxed) {
            return Err(ResolveError::NetworkDisabled);
        }
-
        if self.is_negative_cached(nsid) {
            return Err(ResolveError::NegativelyCached {
                nsid: nsid.to_string(),
@@ -117,19 +239,73 @@ impl DynamicRegistry {
            });
        }

-        match resolve_lexicon(nsid).await {
-            Ok(doc) => Ok(self.insert_schema(doc)),
-            Err(e) => {
-                tracing::debug!(nsid = nsid, error = %e, "caching negative resolution result");
-                self.insert_negative(nsid);
-                Err(e)
+        match self.acquire_leadership(nsid) {
+            Some(_guard) => match resolver(nsid.to_string()).await {
+                Ok(doc) => Ok(self.insert_schema(doc)),
+                Err(e) => {
+                    self.insert_negative(nsid);
+                    tracing::debug!(nsid = nsid, error = %e, "caching negative resolution result");
+                    Err(e)
+                }
+            },
+            None => {
+                self.wait_for_leader(nsid).await;
+                match self.get_cached(nsid) {
+                    Some(doc) => Ok(doc),
+                    None if self.is_negative_cached(nsid) => Err(ResolveError::NegativelyCached {
+                        nsid: nsid.to_string(),
+                        ttl_secs: NEGATIVE_CACHE_TTL.as_secs(),
+                    }),
+                    None => Err(ResolveError::LeaderAborted {
+                        nsid: nsid.to_string(),
+                    }),
+                }
            }
        }
    }

+    fn acquire_leadership(&self, nsid: &str) -> Option<InFlightGuard<'_>> {
+        let mut map = self.in_flight.write();
+        if map.contains_key(nsid) {
+            None
+        } else {
+            map.insert(nsid.to_string(), Arc::new(Notify::new()));
+            Some(InFlightGuard {
+                registry: self,
+                nsid: nsid.to_string(),
+            })
+        }
+    }
+
+    async fn wait_for_leader(&self, nsid: &str) {
+        let notify = {
+            let map = self.in_flight.read();
+            match map.get(nsid) {
+                Some(n) => Arc::clone(n),
+                None => return,
+            }
+        };
+        let notified = notify.notified();
+        tokio::pin!(notified);
+        notified.as_mut().enable();
+        let still_active = self.in_flight.read().contains_key(nsid);
+        if !still_active {
+            return;
+        }
+        notified.as_mut().await;
+    }
+
    pub fn schema_count(&self) -> usize {
        self.store.read().schemas.len()
    }
+
+    #[cfg(test)]
+    fn expire_now(&self, nsid: &str) {
+        let mut store = self.store.write();
+        if let Some(entry) = store.schemas.get_mut(nsid) {
+            entry.expires_at = Instant::now();
+        }
+    }
 }

 impl Default for DynamicRegistry {
@@ -171,7 +347,7 @@ mod tests {
    #[test]
    fn test_empty_lookup() {
        let registry = DynamicRegistry::new();
-        assert!(registry.get("com.example.nonexistent").is_none());
+        assert!(registry.get_cached("com.example.nonexistent").is_none());
        assert_eq!(registry.schema_count(), 0);
    }

@@ -188,9 +364,12 @@ mod tests {
        assert_eq!(arc.id, "com.example.test");
        assert_eq!(registry.schema_count(), 1);

-        let retrieved = registry.get("com.example.test");
+        let retrieved = registry.get_cached("com.example.test");
        assert!(retrieved.is_some());
        assert_eq!(retrieved.unwrap().id, "com.example.test");
+
+        let entry = registry.get_entry("com.example.test").unwrap();
+        assert!(entry.is_fresh(), "freshly inserted entry must be fresh");
    }

    #[test]
@@ -210,6 +389,218 @@ mod tests {
        assert!(!registry.is_negative_cached("com.example.test"));
    }

+    #[test]
+    fn test_positive_entry_reports_stale_after_ttl() {
+        let registry = DynamicRegistry::new();
+        let doc = LexiconDoc {
+            lexicon: 1,
+            id: "pet.nel.stale".to_string(),
+            defs: HashMap::new(),
+        };
+        registry.insert_schema(doc);
+
+        assert!(registry.get_entry("pet.nel.stale").unwrap().is_fresh());
+
+        registry.expire_now("pet.nel.stale");
+
+        assert!(
+            !registry.get_entry("pet.nel.stale").unwrap().is_fresh(),
+            "entry past expiry must be reported stale"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_stale_served_on_resolve_failure() {
+        let registry = DynamicRegistry::new();
+        let doc = LexiconDoc {
+            lexicon: 1,
+            id: "pet.nel.flaky".to_string(),
+            defs: HashMap::new(),
+        };
+        registry.insert_schema(doc);
+        registry.expire_now("pet.nel.flaky");
+
+        let result = registry
+            .resolve_and_cache_with("pet.nel.flaky", |n| async move {
+                Err::<LexiconDoc, _>(ResolveError::DnsLookup {
+                    domain: n,
+                    reason: "simulated failure".to_string(),
+                })
+            })
+            .await;
+
+        let served = result.expect("stale entry must be served when refresh fails");
+        assert_eq!(served.id, "pet.nel.flaky");
+        assert!(
+            registry.get_entry("pet.nel.flaky").unwrap().is_fresh(),
+            "failed refresh must bump expiry so subsequent lookups skip the resolver"
+        );
+        assert!(
+            !registry.is_negative_cached("pet.nel.flaky"),
+            "stale refresh failure must not poison negative cache"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_fresh_hit_skips_resolver() {
+        let registry = DynamicRegistry::new();
+        let doc = LexiconDoc {
+            lexicon: 1,
+            id: "pet.nel.fresh".to_string(),
+            defs: HashMap::new(),
+        };
+        registry.insert_schema(doc);
+
+        let result = registry
+            .resolve_and_cache_with("pet.nel.fresh", |_| async move {
+                panic!("resolver must not run on fresh hit")
+            })
+            .await;
+
+        assert!(result.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_stale_served_when_network_disabled() {
+        let registry = DynamicRegistry::new();
+        let doc = LexiconDoc {
+            lexicon: 1,
+            id: "pet.nel.offline".to_string(),
+            defs: HashMap::new(),
+        };
+        registry.insert_schema(doc);
+        registry.expire_now("pet.nel.offline");
+        registry.set_network_disabled(true);
+
+        let result = registry
+            .resolve_and_cache_with("pet.nel.offline", |_| async move {
+                panic!("resolver must not run when network disabled")
+            })
+            .await;
+
+        assert!(result.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_successful_refresh_updates_cached_at() {
+        let registry = DynamicRegistry::new();
+        let doc = LexiconDoc {
+            lexicon: 1,
+            id: "pet.nel.refresh".to_string(),
+            defs: HashMap::new(),
+        };
+        registry.insert_schema(doc);
+        registry.expire_now("pet.nel.refresh");
+
+        assert!(!registry.get_entry("pet.nel.refresh").unwrap().is_fresh());
+
+        let refreshed = registry
+            .resolve_and_cache_with("pet.nel.refresh", |n| async move {
+                Ok(LexiconDoc {
+                    lexicon: 1,
+                    id: n,
+                    defs: HashMap::new(),
+                })
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(refreshed.id, "pet.nel.refresh");
+        assert!(
+            registry.get_entry("pet.nel.refresh").unwrap().is_fresh(),
+            "refresh must restore freshness"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_single_flight_dedups_concurrent_resolves() {
+        use std::sync::atomic::AtomicUsize;
+        let registry = Arc::new(DynamicRegistry::new());
+        let calls = Arc::new(AtomicUsize::new(0));
+
+        let tasks: Vec<_> = (0..16)
+            .map(|_| {
+                let registry = Arc::clone(&registry);
+                let calls = Arc::clone(&calls);
+                tokio::spawn(async move {
+                    registry
+                        .resolve_and_cache_with("pet.nel.herd", |n| {
+                            let calls = Arc::clone(&calls);
+                            async move {
+                                calls.fetch_add(1, Ordering::SeqCst);
+                                tokio::time::sleep(Duration::from_millis(50)).await;
+                                Ok(LexiconDoc {
+                                    lexicon: 1,
+                                    id: n,
+                                    defs: HashMap::new(),
+                                })
+                            }
+                        })
+                        .await
+                })
+            })
+            .collect();
+
+        let results = futures_collect(tasks).await;
+        results
+            .iter()
+            .for_each(|r| assert!(r.is_ok(), "all single-flight callers must succeed"));
+        assert_eq!(
+            calls.load(Ordering::SeqCst),
+            1,
+            "single-flight must coalesce concurrent resolves"
+        );
+        assert_eq!(registry.schema_count(), 1);
+    }
+
+    #[tokio::test]
+    async fn test_single_flight_followers_observe_leader_failure() {
+        use std::sync::atomic::AtomicUsize;
+        let registry = Arc::new(DynamicRegistry::new());
+        let calls = Arc::new(AtomicUsize::new(0));
+
+        let tasks: Vec<_> = (0..8)
+            .map(|_| {
+                let registry = Arc::clone(&registry);
+                let calls = Arc::clone(&calls);
+                tokio::spawn(async move {
+                    registry
+                        .resolve_and_cache_with("pet.nel.failHerd", |n| {
+                            let calls = Arc::clone(&calls);
+                            async move {
+                                calls.fetch_add(1, Ordering::SeqCst);
+                                tokio::time::sleep(Duration::from_millis(50)).await;
+                                Err::<LexiconDoc, _>(ResolveError::DnsLookup {
+                                    domain: n,
+                                    reason: "simulated".to_string(),
+                                })
+                            }
+                        })
+                        .await
+                })
+            })
+            .collect();
+
+        let results = futures_collect(tasks).await;
+        results
+            .iter()
+            .for_each(|r| assert!(r.is_err(), "all followers must observe leader failure"));
+        assert_eq!(
+            calls.load(Ordering::SeqCst),
+            1,
+            "single-flight must coalesce failing resolves too"
+        );
+        assert!(registry.is_negative_cached("pet.nel.failHerd"));
+    }
+
+    async fn futures_collect<T>(handles: Vec<tokio::task::JoinHandle<T>>) -> Vec<T> {
+        futures::future::join_all(handles)
+            .await
+            .into_iter()
+            .map(|r| r.expect("task panicked"))
+            .collect()
+    }
+
    #[test]
    fn test_eviction_is_fifo() {
        let registry = DynamicRegistry::new();
@@ -217,7 +608,7 @@ mod tests {
        (0..MAX_DYNAMIC_SCHEMAS).for_each(|i| {
            let doc = LexiconDoc {
                lexicon: 1,
-                id: format!("com.example.schema{}", i),
+                id: format!("pet.nel.schema{}", i),
                defs: HashMap::new(),
            };
            registry.insert_schema(doc);
@@ -226,23 +617,23 @@ mod tests {

        let trigger = LexiconDoc {
            lexicon: 1,
-            id: "com.example.trigger".to_string(),
+            id: "pet.nel.trigger".to_string(),
            defs: HashMap::new(),
        };
        registry.insert_schema(trigger);

        assert!(
-            registry.get("com.example.schema0").is_none(),
+            registry.get_cached("pet.nel.schema0").is_none(),
            "oldest entry should be evicted"
        );
        assert!(
-            registry.get("com.example.trigger").is_some(),
+            registry.get_cached("pet.nel.trigger").is_some(),
            "newly inserted entry should exist"
        );
        let evict_count = MAX_DYNAMIC_SCHEMAS / 4;
        assert!(
            registry
-                .get(&format!("com.example.schema{}", evict_count))
+                .get_cached(&format!("pet.nel.schema{}", evict_count))
                .is_some(),
            "entry after eviction window should survive"
        );
@@ -253,7 +644,7 @@ mod tests {
        let registry = DynamicRegistry::new();
        let doc = LexiconDoc {
            lexicon: 1,
-            id: "com.example.tracked".to_string(),
+            id: "pet.nel.tracked".to_string(),
            defs: HashMap::new(),
        };
        let arc = registry.insert_schema(doc);
@@ -265,7 +656,7 @@ mod tests {
        (0..MAX_DYNAMIC_SCHEMAS).for_each(|i| {
            registry.insert_schema(LexiconDoc {
                lexicon: 1,
-                id: format!("com.example.filler{}", i),
+                id: format!("pet.nel.filler{}", i),
                defs: HashMap::new(),
            });
        });
@@ -25,7 +25,7 @@ impl LexiconRegistry {
        Self {
            schemas: HashMap::new(),
            #[cfg(feature = "resolve")]
-            dynamic: crate::dynamic::DynamicRegistry::new(),
+            dynamic: crate::dynamic::DynamicRegistry::from_env(),
        }
    }

@@ -43,7 +43,7 @@ impl LexiconRegistry {
        self.schemas.get(nsid).cloned().or_else(|| {
            #[cfg(feature = "resolve")]
            {
-                self.dynamic.get(nsid)
+                self.dynamic.get_cached(nsid)
            }
            #[cfg(not(feature = "resolve"))]
            {
@@ -70,6 +70,8 @@ pub enum ResolveError {
    NegativelyCached { nsid: String, ttl_secs: u64 },
    #[error("network resolution disabled")]
    NetworkDisabled,
+    #[error("leader task for {nsid} aborted before completion")]
+    LeaderAborted { nsid: String },
 }

 pub fn nsid_to_authority(nsid: &str) -> Result<String, ResolveError> {
@@ -175,10 +175,7 @@ pub async fn passkey_start(
    }
 }

-async fn passkey_start_discoverable(
-    state: AppState,
-    request_id: RequestId,
-) -> Response {
+async fn passkey_start_discoverable(state: AppState, request_id: RequestId) -> Response {
    let (rcr, auth_state) = match state.webauthn_config.start_discoverable_authentication() {
        Ok(result) => result,
        Err(e) => {
@@ -570,16 +567,13 @@ pub async fn passkey_finish(
            Err(response) => return response,
        },
        None => {
-            let result = match passkey_finish_discoverable(
-                &state,
-                &credential,
-                &passkey_finish_request_id,
-            )
-            .await
-            {
-                Ok(result) => result,
-                Err(response) => return response,
-            };
+            let result =
+                match passkey_finish_discoverable(&state, &credential, &passkey_finish_request_id)
+                    .await
+                {
+                    Ok(result) => result,
+                    Err(response) => return response,
+                };
            if state
                .repos
                .oauth
@@ -138,21 +138,29 @@ pub async fn pushed_authorization_request(
 }

 fn determine_client_auth(request: &ParRequest) -> Result<ClientAuth, OAuthError> {
-    if let (Some(assertion), Some(assertion_type)) =
-        (&request.client_assertion, &request.client_assertion_type)
-    {
+    let assertion = request
+        .client_assertion
+        .as_deref()
+        .filter(|s| !s.is_empty());
+    let assertion_type = request
+        .client_assertion_type
+        .as_deref()
+        .filter(|s| !s.is_empty());
+    let secret = request.client_secret.as_deref().filter(|s| !s.is_empty());
+
+    if let (Some(assertion), Some(assertion_type)) = (assertion, assertion_type) {
        if assertion_type != "urn:ietf:params:oauth:client-assertion-type:jwt-bearer" {
            return Err(OAuthError::InvalidRequest(
                "Unsupported client_assertion_type".to_string(),
            ));
        }
        return Ok(ClientAuth::PrivateKeyJwt {
-            client_assertion: assertion.clone(),
+            client_assertion: assertion.to_string(),
        });
    }
-    if let Some(secret) = &request.client_secret {
+    if let Some(secret) = secret {
        return Ok(ClientAuth::SecretPost {
-            client_secret: secret.clone(),
+            client_secret: secret.to_string(),
        });
    }
    Ok(ClientAuth::None)
@@ -138,13 +138,17 @@ impl TokenRequest {
            }
        };

-        let client_auth = match (self.client_assertion, self.client_assertion_type) {
+        let assertion = self.client_assertion.filter(|s| !s.is_empty());
+        let assertion_type = self.client_assertion_type.filter(|s| !s.is_empty());
+        let client_secret = self.client_secret.filter(|s| !s.is_empty());
+
+        let client_auth = match (assertion, assertion_type) {
            (Some(assertion), Some(assertion_type)) => RequestClientAuth::PrivateKeyJwt {
                client_id: self.client_id,
                assertion,
                assertion_type,
            },
-            _ => match self.client_secret {
+            _ => match client_secret {
                Some(secret) => RequestClientAuth::SecretPost {
                    client_id: self.client_id,
                    client_secret: secret,
@@ -287,8 +287,6 @@ pub fn is_valid_discord_username(username: &str) -> bool {
        && username
            .chars()
            .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_' || c == '.')
-        && !username.starts_with('.')
-        && !username.ends_with('.')
        && !username.contains("..")
 }

@@ -493,8 +491,6 @@ mod tests {
        assert!(!is_valid_discord_username("Alice"));
        assert!(!is_valid_discord_username("ALICE"));
        assert!(!is_valid_discord_username("user-name"));
-        assert!(!is_valid_discord_username(".username"));
-        assert!(!is_valid_discord_username("username."));
        assert!(!is_valid_discord_username("user..name"));
        assert!(!is_valid_discord_username("user name"));
        assert!(!is_valid_discord_username(
@@ -156,6 +156,7 @@ pub enum Legacy2faOutcome {
 }

 pub struct Legacy2faContext {
+    pub is_app_password: bool,
    pub email_2fa_enabled: bool,
    pub has_totp: bool,
    pub allow_legacy_login: bool,
@@ -163,7 +164,7 @@ pub struct Legacy2faContext {

 impl Legacy2faContext {
    pub fn requires_2fa(&self) -> bool {
-        self.email_2fa_enabled || self.has_totp
+        !self.is_app_password && (self.email_2fa_enabled || self.has_totp)
    }

    pub fn is_blocked(&self) -> bool {
@@ -418,6 +419,7 @@ mod tests {
        let cache = MockCache::new();
        let did = Did::new("did:plc:test".to_string()).unwrap();
        let ctx = Legacy2faContext {
+            is_app_password: false,
            email_2fa_enabled: false,
            has_totp: false,
            allow_legacy_login: true,
@@ -427,11 +429,27 @@ mod tests {
        assert!(matches!(outcome, Legacy2faOutcome::NotRequired));
    }

+    #[tokio::test]
+    async fn test_process_flow_not_required_because_app_password() {
+        let cache = MockCache::new();
+        let did = Did::new("did:plc:test".to_string()).unwrap();
+        let ctx = Legacy2faContext {
+            is_app_password: true,
+            email_2fa_enabled: false,
+            has_totp: true,
+            allow_legacy_login: true,
+        };
+
+        let outcome = process_legacy_2fa(&cache, &did, &ctx, None).await.unwrap();
+        assert!(matches!(outcome, Legacy2faOutcome::NotRequired));
+    }
+
    #[tokio::test]
    async fn test_process_flow_blocked() {
        let cache = MockCache::new();
        let did = Did::new("did:plc:test".to_string()).unwrap();
        let ctx = Legacy2faContext {
+            is_app_password: false,
            email_2fa_enabled: false,
            has_totp: true,
            allow_legacy_login: false,
@@ -446,6 +464,7 @@ mod tests {
        let cache = MockCache::new();
        let did = Did::new("did:plc:test".to_string()).unwrap();
        let ctx = Legacy2faContext {
+            is_app_password: false,
            email_2fa_enabled: false,
            has_totp: true,
            allow_legacy_login: true,
@@ -460,6 +479,7 @@ mod tests {
        let cache = MockCache::new();
        let did = Did::new("did:plc:test2".to_string()).unwrap();
        let ctx = Legacy2faContext {
+            is_app_password: false,
            email_2fa_enabled: true,
            has_totp: false,
            allow_legacy_login: false,
@@ -474,6 +494,7 @@ mod tests {
        let cache = MockCache::new();
        let did = Did::new("did:plc:test".to_string()).unwrap();
        let ctx = Legacy2faContext {
+            is_app_password: false,
            email_2fa_enabled: true,
            has_totp: false,
            allow_legacy_login: false,
@@ -6,17 +6,19 @@ use crate::types::{Did, Handle, Nsid, Rkey};
 use backon::{ExponentialBuilder, Retryable};
 use bytes::Bytes;
 use cid::Cid;
+use jacquard_common::smol_str::SmolStr;
 use jacquard_common::types::{integer::LimitedU32, string::Tid};
 use jacquard_repo::commit::Commit;
-use jacquard_repo::mst::Mst;
 use jacquard_repo::mst::util::compute_cid;
+use jacquard_repo::mst::{Mst, VerifiedWriteOp};
 use jacquard_repo::storage::BlockStore;
 use k256::ecdsa::SigningKey;
 use serde_json::{Value, json};
+use std::collections::{BTreeMap, HashMap, HashSet};
 use std::str::FromStr;
 use std::sync::Arc;
 use tokio::sync::OwnedMutexGuard;
-use tracing::error;
+use tracing::{error, warn};
 use tranquil_db_traits::SequenceNumber;
 use uuid::Uuid;

@@ -38,6 +40,7 @@ pub enum CommitError {
    MstOperationFailed(String),
    RecordSerializationFailed(String),
    InvalidCid(String),
+    RecordAlreadyExists(String),
 }

 impl std::fmt::Display for CommitError {
@@ -63,6 +66,7 @@ impl std::fmt::Display for CommitError {
                write!(f, "Failed to serialize record: {}", e)
            }
            Self::InvalidCid(e) => write!(f, "Invalid CID: {}", e),
+            Self::RecordAlreadyExists(key) => write!(f, "Record already exists at {}", key),
        }
    }
 }
@@ -77,6 +81,9 @@ impl From<CommitError> for ApiError {
            }
            CommitError::RepoNotFound => ApiError::RepoNotFound(None),
            CommitError::UserNotFound => ApiError::RepoNotFound(Some("User not found".into())),
+            CommitError::RecordAlreadyExists(key) => {
+                ApiError::InvalidRequest(format!("Record already exists at {key}"))
+            }
            other => {
                error!("Commit failed: {}", other);
                ApiError::InternalError(Some("Failed to commit changes".into()))
@@ -160,7 +167,6 @@ pub struct FinalizeParams<'a> {
    pub controller_did: Option<&'a Did>,
    pub delegation_detail: Option<serde_json::Value>,
    pub ops: Vec<RecordOp>,
-    pub modified_keys: &'a [String],
    pub blob_cids: &'a [String],
    pub backlinks_to_add: Vec<Backlink>,
    pub backlinks_to_remove: Vec<AtUri>,
@@ -236,19 +242,84 @@ pub async fn finalize_repo_write(
        ApiError::InternalError(None)
    })?;

-    let block_bytes = ctx.tracking_store.take_written_blocks();
+    let written_bytes = ctx.tracking_store.take_written_blocks();
+    let new_tree_cids: Vec<Cid> = written_bytes.keys().copied().collect();

-    let storage_for_diff = Arc::new(ctx.tracking_store.clone());
-    let original_settled = Mst::load(storage_for_diff.clone(), ctx.prev_data_cid, None);
-    let new_settled = Mst::load(storage_for_diff, new_mst_root, None);
+    let storage_for_proof = Arc::new(ctx.tracking_store.clone());
+    let original_settled = Mst::load(storage_for_proof.clone(), ctx.prev_data_cid, None);
+    let new_settled = Mst::load(storage_for_proof.clone(), new_mst_root, None);

-    let new_tree_cids: Vec<Cid> = block_bytes.keys().copied().collect();
+    let mut inverse_trace = new_settled.clone();
+    let mut non_invertible: Vec<String> = Vec::new();
+    let mut invert_errors: Vec<String> = Vec::new();
+    for op in params.ops.iter().rev() {
+        let (collection, rkey) = op.collection_rkey();
+        let key = SmolStr::new(format!("{}/{}", collection, rkey));
+        let verified = match op {
+            RecordOp::Create { cid, .. } => VerifiedWriteOp::Create {
+                key,
+                cid: *cid.as_cid(),
+            },
+            RecordOp::Update { cid, prev, .. } => VerifiedWriteOp::Update {
+                key,
+                cid: *cid.as_cid(),
+                prev: *prev.as_cid(),
+            },
+            RecordOp::Delete { prev, .. } => VerifiedWriteOp::Delete {
+                key,
+                prev: *prev.as_cid(),
+            },
+        };
+        match inverse_trace.invert_op(verified.clone()).await {
+            Ok(true) => {}
+            Ok(false) => non_invertible.push(format!("{:?}", verified)),
+            Err(e) => invert_errors.push(format!("{:?} -> {:?}", verified, e)),
+        }
+    }
+    if !non_invertible.is_empty() {
+        warn!(
+            user_id = %params.user_id,
+            count = non_invertible.len(),
+            ops = ?non_invertible,
+            "firehose proof walk: ops not invertible on new MST, consumer will reject frame"
+        );
+    }
+    if !invert_errors.is_empty() {
+        warn!(
+            user_id = %params.user_id,
+            count = invert_errors.len(),
+            failures = ?invert_errors,
+            "firehose proof walk: invert_op errored, cover blocks may be incomplete"
+        );
+    }
+
+    let read_cid_set: HashSet<Cid> = ctx.tracking_store.get_read_cids().into_iter().collect();
+    let missing_read_cids: Vec<Cid> = read_cid_set
+        .iter()
+        .copied()
+        .filter(|cid| !written_bytes.contains_key(cid))
+        .collect();
+    let mut relevant: BTreeMap<Cid, Bytes> = BTreeMap::new();
+    if !missing_read_cids.is_empty() {
+        let fetched = ctx
+            .tracking_store
+            .get_many(&missing_read_cids)
+            .await
+            .map_err(|e| {
+                error!("fetch cover read bytes: {e}");
+                ApiError::InternalError(None)
+            })?;
+        for (cid, maybe) in missing_read_cids.into_iter().zip(fetched) {
+            if let Some(bytes) = maybe {
+                relevant.insert(cid, bytes);
+            }
+        }
+    }

    let obsolete_cids = match original_settled.diff(&new_settled).await {
        Ok(diff) => {
-            let mut obsolete: Vec<Cid> = Vec::with_capacity(
-                1 + diff.removed_mst_blocks.len() + diff.removed_cids.len(),
-            );
+            let mut obsolete: Vec<Cid> =
+                Vec::with_capacity(1 + diff.removed_mst_blocks.len() + diff.removed_cids.len());
            obsolete.push(ctx.current_root_cid);
            obsolete.extend(diff.removed_mst_blocks);
            obsolete.extend(diff.removed_cids);
@@ -264,6 +335,9 @@ pub async fn finalize_repo_write(
        }
    };

+    let mut block_bytes = written_bytes;
+    block_bytes.extend(relevant);
+
    let result = commit_and_log(
        state,
        CommitParams {
@@ -347,6 +421,22 @@ pub enum RecordOp {
    },
 }

+impl RecordOp {
+    pub fn collection_rkey(&self) -> (&Nsid, &Rkey) {
+        match self {
+            Self::Create {
+                collection, rkey, ..
+            }
+            | Self::Update {
+                collection, rkey, ..
+            }
+            | Self::Delete {
+                collection, rkey, ..
+            } => (collection, rkey),
+        }
+    }
+}
+
 pub struct CommitResult {
    pub commit_cid: Cid,
    pub rev: String,
@@ -377,8 +467,6 @@ pub async fn commit_and_log(
        RecordUpsert, RepoEventType,
    };

-    let backlinks_to_add = params.backlinks_to_add;
-    let backlinks_to_remove = params.backlinks_to_remove;
    let CommitParams {
        did,
        user_id,
@@ -391,7 +479,8 @@ pub async fn commit_and_log(
        new_tree_cids,
        blobs,
        obsolete_cids,
-        ..
+        backlinks_to_add,
+        backlinks_to_remove,
    } = params;
    debug_assert_eq!(
        current_root_cid.is_some(),
@@ -437,39 +526,65 @@ pub async fn commit_and_log(

    let obsolete_bytes: Vec<Vec<u8>> = obsolete_cids.iter().map(|c| c.to_bytes()).collect();

-    let (record_upserts, record_deletes): (Vec<RecordUpsert>, Vec<RecordDelete>) = ops.iter().fold(
-        (Vec::new(), Vec::new()),
-        |(mut upserts, mut deletes), op| {
-            match op {
-                RecordOp::Create {
-                    collection,
-                    rkey,
-                    cid,
-                }
-                | RecordOp::Update {
-                    collection,
-                    rkey,
-                    cid,
-                    ..
-                } => {
-                    upserts.push(RecordUpsert {
-                        collection: collection.clone(),
-                        rkey: rkey.clone(),
-                        cid: crate::types::CidLink::from(cid.as_cid()),
-                    });
-                }
-                RecordOp::Delete {
-                    collection, rkey, ..
-                } => {
-                    deletes.push(RecordDelete {
-                        collection: collection.clone(),
-                        rkey: rkey.clone(),
-                    });
-                }
+    let final_ops: HashMap<(&Nsid, &Rkey), &RecordOp> = ops
+        .iter()
+        .map(|op| (op.collection_rkey(), op))
+        .collect();
+
+    let final_record_uris: HashSet<AtUri> = final_ops
+        .iter()
+        .filter(|(_, op)| !matches!(op, RecordOp::Delete { .. }))
+        .map(|((c, r), _)| AtUri::from_parts(did, c, r))
+        .collect();
+
+    let record_upserts: Vec<RecordUpsert> = final_ops
+        .values()
+        .filter_map(|op| match op {
+            RecordOp::Create {
+                collection,
+                rkey,
+                cid,
            }
-            (upserts, deletes)
-        },
-    );
+            | RecordOp::Update {
+                collection,
+                rkey,
+                cid,
+                ..
+            } => Some(RecordUpsert {
+                collection: collection.clone(),
+                rkey: rkey.clone(),
+                cid: crate::types::CidLink::from(cid.as_cid()),
+            }),
+            RecordOp::Delete { .. } => None,
+        })
+        .collect();
+
+    let record_deletes: Vec<RecordDelete> = final_ops
+        .values()
+        .filter_map(|op| match op {
+            RecordOp::Delete {
+                collection, rkey, ..
+            } => Some(RecordDelete {
+                collection: collection.clone(),
+                rkey: rkey.clone(),
+            }),
+            _ => None,
+        })
+        .collect();
+
+    let backlinks_to_add: Vec<Backlink> = backlinks_to_add
+        .into_iter()
+        .filter(|b| final_record_uris.contains(&b.uri))
+        .map(|b| ((b.uri.clone(), b.path), b))
+        .collect::<HashMap<_, _>>()
+        .into_values()
+        .collect();
+
+    let backlinks_to_remove: Vec<AtUri> = backlinks_to_remove
+        .into_iter()
+        .collect::<HashSet<_>>()
+        .into_iter()
+        .collect();

    let ops_json: Vec<serde_json::Value> = ops
        .iter()
@@ -604,6 +719,16 @@ pub async fn create_record_internal(
        .await
        .map_err(to_commit_err)?;

+    let key = format!("{}/{}", collection, rkey);
+    if mst
+        .get(&key)
+        .await
+        .map_err(|e| CommitError::MstOperationFailed(e.to_string()))?
+        .is_some()
+    {
+        return Err(CommitError::RecordAlreadyExists(key));
+    }
+
    let record_ipld = crate::util::json_to_ipld(record);
    let mut record_bytes = Vec::new();
    serde_ipld_dagcbor::to_writer(&mut record_bytes, &record_ipld)
@@ -613,8 +738,6 @@ pub async fn create_record_internal(
        .put(&record_bytes)
        .await
        .map_err(|e| CommitError::BlockStoreFailed(e.to_string()))?;
-
-    let key = format!("{}/{}", collection, rkey);
    let new_mst = mst
        .add(&key, record_cid)
        .await
@@ -625,7 +748,6 @@ pub async fn create_record_internal(
        rkey: rkey.clone(),
        cid: RecordCid::from(record_cid),
    };
-    let modified_keys = [key];
    let blob_cids = extract_blob_cids(record);
    let record_uri = AtUri::from_parts(did.as_str(), collection.as_str(), rkey.as_str());
    let backlinks = extract_backlinks(&record_uri, record);
@@ -640,7 +762,6 @@ pub async fn create_record_internal(
            controller_did: None,
            delegation_detail: None,
            ops: vec![op],
-            modified_keys: &modified_keys,
            blob_cids: &blob_cids,
            backlinks_to_add: backlinks,
            backlinks_to_remove: vec![],
@@ -523,13 +523,16 @@ fn wire_tranquil_store(
    let metastore =
        Metastore::open(&metastore_dir, metastore_config).expect("failed to open metastore");

-    let blockstore = TranquilBlockStore::open(BlockStoreConfig {
-        data_dir: blockstore_data_dir,
-        index_dir: blockstore_index_dir,
-        max_file_size: store_cfg.max_blockstore_file_size,
-        group_commit: Default::default(),
-        shard_count: tranquil_store::blockstore::DEFAULT_SHARD_COUNT,
-    })
+    let blockstore = TranquilBlockStore::open_with_retry(
+        BlockStoreConfig {
+            data_dir: blockstore_data_dir,
+            index_dir: blockstore_index_dir,
+            max_file_size: store_cfg.max_blockstore_file_size,
+            group_commit: Default::default(),
+            shard_count: tranquil_store::blockstore::DEFAULT_SHARD_COUNT,
+        },
+        tranquil_store::blockstore::OpenRetryPolicy::default(),
+    )
    .expect("failed to open blockstore");

    let event_log = EventLog::open(
@@ -191,16 +191,12 @@ async fn setup_with_external_infra() -> String {
 async fn setup_with_testcontainers() -> String {
    let temp_dir = std::env::temp_dir().join(format!("tranquil-pds-test-{}", uuid::Uuid::new_v4()));
    let blob_path = temp_dir.join("blobs");
-    let backup_path = temp_dir.join("backups");
    std::fs::create_dir_all(&blob_path).expect("Failed to create blob temp directory");
-    std::fs::create_dir_all(&backup_path).expect("Failed to create backup temp directory");
    TEST_TEMP_DIR.set(temp_dir).ok();
    let plc_url = setup_mock_plc_directory().await;
    unsafe {
        std::env::set_var("BLOB_STORAGE_BACKEND", "filesystem");
        std::env::set_var("BLOB_STORAGE_PATH", blob_path.to_str().unwrap());
-        std::env::set_var("BACKUP_STORAGE_BACKEND", "filesystem");
-        std::env::set_var("BACKUP_STORAGE_PATH", backup_path.to_str().unwrap());
        std::env::set_var("MAX_IMPORT_SIZE", "100000000");
        std::env::set_var("SKIP_IMPORT_VERIFICATION", "true");
        std::env::set_var("PLC_DIRECTORY_URL", &plc_url);
@@ -242,8 +238,6 @@ async fn setup_with_testcontainers() -> String {
    let plc_url = setup_mock_plc_directory().await;
    unsafe {
        std::env::set_var("BLOB_STORAGE_BACKEND", "s3");
-        std::env::set_var("BACKUP_STORAGE_BACKEND", "s3");
-        std::env::set_var("BACKUP_S3_BUCKET", "test-backups");
        std::env::set_var("S3_BUCKET", "test-bucket");
        std::env::set_var("AWS_ACCESS_KEY_ID", "minioadmin");
        std::env::set_var("AWS_SECRET_ACCESS_KEY", "minioadmin");
@@ -333,8 +327,6 @@ unsafe fn configure_external_storage_env() {
        if std::env::var("S3_ENDPOINT").is_ok() {
            let s3_endpoint = std::env::var("S3_ENDPOINT").unwrap();
            std::env::set_var("BLOB_STORAGE_BACKEND", "s3");
-            std::env::set_var("BACKUP_STORAGE_BACKEND", "s3");
-            std::env::set_var("BACKUP_S3_BUCKET", "test-backups");
            std::env::set_var(
                "S3_BUCKET",
                std::env::var("S3_BUCKET").unwrap_or_else(|_| "test-bucket".to_string()),
@@ -356,14 +348,10 @@ unsafe fn configure_external_storage_env() {
            let process_dir =
                std::env::temp_dir().join(format!("tranquil-pds-test-{}", std::process::id()));
            let blob_path = process_dir.join("blobs");
-            let backup_path = process_dir.join("backups");
            std::fs::create_dir_all(&blob_path).expect("Failed to create blob directory");
-            std::fs::create_dir_all(&backup_path).expect("Failed to create backup directory");
            TEST_TEMP_DIR.set(process_dir).ok();
            std::env::set_var("BLOB_STORAGE_BACKEND", "filesystem");
            std::env::set_var("BLOB_STORAGE_PATH", blob_path.to_str().unwrap());
-            std::env::set_var("BACKUP_STORAGE_BACKEND", "filesystem");
-            std::env::set_var("BACKUP_STORAGE_PATH", backup_path.to_str().unwrap());
        }
        std::env::set_var("MAX_IMPORT_SIZE", "100000000");
        std::env::set_var("SKIP_IMPORT_VERIFICATION", "true");
@@ -622,18 +610,14 @@ async fn setup_store_backend() -> String {
    let temp_dir =
        std::env::temp_dir().join(format!("tranquil-pds-store-{}", uuid::Uuid::new_v4()));
    let blob_path = temp_dir.join("blobs");
-    let backup_path = temp_dir.join("backups");
    let store_path = temp_dir.join("store");
    std::fs::create_dir_all(&blob_path).expect("failed to create blob temp directory");
-    std::fs::create_dir_all(&backup_path).expect("failed to create backup temp directory");
    std::fs::create_dir_all(&store_path).expect("failed to create store temp directory");
    TEST_TEMP_DIR.set(temp_dir).ok();
    let plc_url = setup_mock_plc_directory().await;
    unsafe {
        std::env::set_var("BLOB_STORAGE_BACKEND", "filesystem");
        std::env::set_var("BLOB_STORAGE_PATH", blob_path.to_str().unwrap());
-        std::env::set_var("BACKUP_STORAGE_BACKEND", "filesystem");
-        std::env::set_var("BACKUP_STORAGE_PATH", backup_path.to_str().unwrap());
        std::env::set_var("MAX_IMPORT_SIZE", "100000000");
        std::env::set_var("SKIP_IMPORT_VERIFICATION", "true");
        std::env::set_var("PLC_DIRECTORY_URL", &plc_url);
@@ -790,18 +774,14 @@ async fn setup_cluster_store_backend() -> Option<sqlx::PgPool> {
        uuid::Uuid::new_v4()
    ));
    let blob_path = temp_dir.join("blobs");
-    let backup_path = temp_dir.join("backups");
    let store_path = temp_dir.join("store");
    std::fs::create_dir_all(&blob_path).expect("failed to create blob temp directory");
-    std::fs::create_dir_all(&backup_path).expect("failed to create backup temp directory");
    std::fs::create_dir_all(&store_path).expect("failed to create store temp directory");
    TEST_TEMP_DIR.set(temp_dir).ok();
    let plc_url = setup_mock_plc_directory().await;
    unsafe {
        std::env::set_var("BLOB_STORAGE_BACKEND", "filesystem");
        std::env::set_var("BLOB_STORAGE_PATH", blob_path.to_str().unwrap());
-        std::env::set_var("BACKUP_STORAGE_BACKEND", "filesystem");
-        std::env::set_var("BACKUP_STORAGE_PATH", backup_path.to_str().unwrap());
        std::env::set_var("MAX_IMPORT_SIZE", "100000000");
        std::env::set_var("SKIP_IMPORT_VERIFICATION", "true");
        std::env::set_var("PLC_DIRECTORY_URL", &plc_url);
@@ -847,16 +827,12 @@ async fn setup_cluster_testcontainers() -> Option<sqlx::PgPool> {
    let temp_dir =
        std::env::temp_dir().join(format!("tranquil-pds-cluster-{}", uuid::Uuid::new_v4()));
    let blob_path = temp_dir.join("blobs");
-    let backup_path = temp_dir.join("backups");
    std::fs::create_dir_all(&blob_path).expect("Failed to create blob temp directory");
-    std::fs::create_dir_all(&backup_path).expect("Failed to create backup temp directory");
    TEST_TEMP_DIR.set(temp_dir).ok();
    let plc_url = setup_mock_plc_directory().await;
    unsafe {
        std::env::set_var("BLOB_STORAGE_BACKEND", "filesystem");
        std::env::set_var("BLOB_STORAGE_PATH", blob_path.to_str().unwrap());
-        std::env::set_var("BACKUP_STORAGE_BACKEND", "filesystem");
-        std::env::set_var("BACKUP_STORAGE_PATH", backup_path.to_str().unwrap());
        std::env::set_var("MAX_IMPORT_SIZE", "100000000");
        std::env::set_var("SKIP_IMPORT_VERIFICATION", "true");
        std::env::set_var("PLC_DIRECTORY_URL", &plc_url);
@@ -0,0 +1,169 @@
+mod common;
+
+use chrono::Utc;
+use common::*;
+use reqwest::StatusCode;
+use serde_json::{Value, json};
+
+fn run_compaction(store: &tranquil_store::blockstore::TranquilBlockStore) {
+    let liveness = store.compaction_liveness(0).unwrap();
+    liveness
+        .iter()
+        .filter(|(_, info)| info.total_blocks > 0 && info.ratio() < 0.95)
+        .map(|(&fid, _)| fid)
+        .collect::<Vec<_>>()
+        .into_iter()
+        .for_each(|fid| match store.compact_file(fid, 0) {
+            Ok(_) => {}
+            Err(tranquil_store::blockstore::CompactionError::ActiveFileCannotBeCompacted) => {}
+            Err(e) => eprintln!("compaction: {e}"),
+        });
+}
+
+#[tokio::test]
+async fn mst_blocks_survive_full_store_reopen() {
+    if !is_store_backend() {
+        eprintln!("skipping: only meaningful with tranquil-store backend");
+        return;
+    }
+
+    let client = client();
+    let base = base_url().await;
+    let block_store = get_test_block_store().await;
+
+    let store = block_store
+        .as_tranquil_store()
+        .expect("expected tranquil-store backend");
+
+    let (jwt, did) = create_account_and_login(&client).await;
+
+    let mut posts = Vec::new();
+    for i in 0..30 {
+        let res = client
+            .post(format!("{base}/xrpc/com.atproto.repo.createRecord"))
+            .bearer_auth(&jwt)
+            .json(&json!({
+                "repo": did,
+                "collection": "app.bsky.feed.post",
+                "record": {
+                    "$type": "app.bsky.feed.post",
+                    "text": format!("compaction test post {i}"),
+                    "createdAt": Utc::now().to_rfc3339()
+                }
+            }))
+            .send()
+            .await
+            .unwrap();
+        assert_eq!(res.status(), StatusCode::OK);
+        let body: Value = res.json().await.unwrap();
+        posts.push((
+            body["uri"].as_str().unwrap().to_string(),
+            body["cid"].as_str().unwrap().to_string(),
+        ));
+    }
+
+    for (uri, cid) in &posts[..20] {
+        let res = client
+            .post(format!("{base}/xrpc/com.atproto.repo.createRecord"))
+            .bearer_auth(&jwt)
+            .json(&json!({
+                "repo": did,
+                "collection": "app.bsky.feed.like",
+                "record": {
+                    "$type": "app.bsky.feed.like",
+                    "subject": { "uri": uri, "cid": cid },
+                    "createdAt": Utc::now().to_rfc3339()
+                }
+            }))
+            .send()
+            .await
+            .unwrap();
+        assert_eq!(res.status(), StatusCode::OK, "like failed for {uri}");
+    }
+
+    let data_dir = store.data_dir().to_path_buf();
+    let index_dir = data_dir.parent().unwrap().join("index");
+
+    let store_clone = store.clone();
+    tokio::task::spawn_blocking(move || {
+        (0..40).for_each(|_| run_compaction(&store_clone));
+    })
+    .await
+    .unwrap();
+
+    let repo_root_str: String = get_test_repos()
+        .await
+        .repo
+        .get_repo_root_by_did(&tranquil_types::Did::new(did.clone()).unwrap())
+        .await
+        .expect("db error")
+        .expect("no repo root")
+        .to_string();
+
+    let head_cid = cid::Cid::try_from(repo_root_str.as_str()).expect("invalid cid");
+
+    let car_blocks = tranquil_pds::scheduled::collect_current_repo_blocks(block_store, &head_cid)
+        .await
+        .expect("collect blocks");
+
+    let block_count_before = car_blocks.len();
+
+    let max_file_size = store
+        .list_data_files()
+        .ok()
+        .map(|_| 4 * 1024 * 1024u64)
+        .unwrap_or(4 * 1024 * 1024);
+
+    let reopened_missing = tokio::task::spawn_blocking(move || {
+        let rt = tokio::runtime::Handle::current();
+        let _guard = rt.enter();
+
+        let config = tranquil_store::blockstore::BlockStoreConfig {
+            data_dir: data_dir.clone(),
+            index_dir,
+            max_file_size,
+            group_commit: tranquil_store::blockstore::GroupCommitConfig::default(),
+            shard_count: 1,
+        };
+        let fresh =
+            tranquil_store::blockstore::TranquilBlockStore::open(config).expect("reopen failed");
+
+        let missing: Vec<String> = car_blocks
+            .iter()
+            .filter_map(|cid_bytes| {
+                if cid_bytes.len() < 36 {
+                    return None;
+                }
+                let mut arr = [0u8; 36];
+                arr.copy_from_slice(&cid_bytes[..36]);
+                match fresh.get_block_sync(&arr) {
+                    Ok(Some(_)) => None,
+                    Ok(None) => Some(format!(
+                        "missing {}",
+                        cid::Cid::try_from(cid_bytes.as_slice())
+                            .map(|c| c.to_string())
+                            .unwrap_or_else(|_| hex::encode(cid_bytes))
+                    )),
+                    Err(e) => Some(format!("error: {e}")),
+                }
+            })
+            .collect();
+
+        drop(fresh);
+        missing
+    })
+    .await
+    .unwrap();
+
+    assert!(
+        reopened_missing.is_empty(),
+        "{} of {block_count_before} blocks missing after blockstore reopen:\n{}",
+        reopened_missing.len(),
+        reopened_missing
+            .iter()
+            .take(20)
+            .map(|s| s.as_str())
+            .collect::<Vec<_>>()
+            .join("\n"),
+    );
+}
@@ -8,7 +8,6 @@ const HANDLE_DOMAIN: &str = "handles.test";

 fn set_handle_domain() {
    unsafe {
-        std::env::set_var("AVAILABLE_USER_DOMAINS", HANDLE_DOMAIN);
        std::env::set_var("PDS_USER_HANDLE_DOMAINS", HANDLE_DOMAIN);
    }
 }
@@ -456,7 +456,7 @@ async fn test_apply_writes_batch() {
        "writes": [
            { "$type": "com.atproto.repo.applyWrites#create", "collection": "app.bsky.feed.post", "rkey": "batch-post-1", "value": { "$type": "app.bsky.feed.post", "text": "First batch post", "createdAt": now } },
            { "$type": "com.atproto.repo.applyWrites#create", "collection": "app.bsky.feed.post", "rkey": "batch-post-2", "value": { "$type": "app.bsky.feed.post", "text": "Second batch post", "createdAt": now } },
-            { "$type": "com.atproto.repo.applyWrites#create", "collection": "app.bsky.actor.profile", "rkey": "self", "value": { "$type": "app.bsky.actor.profile", "displayName": "Batch User" } }
+            { "$type": "com.atproto.repo.applyWrites#update", "collection": "app.bsky.actor.profile", "rkey": "self", "value": { "$type": "app.bsky.actor.profile", "displayName": "Batch User" } }
        ]
    });
    let apply_res = client
@@ -626,7 +626,12 @@ async fn create_app_password_session(
        .send()
        .await
        .expect("Failed to login with app password");
-    assert_eq!(login_res.status(), StatusCode::OK, "App password login for '{}' failed", name);
+    assert_eq!(
+        login_res.status(),
+        StatusCode::OK,
+        "App password login for '{}' failed",
+        name
+    );
    let session: Value = login_res.json().await.unwrap();
    let jwt = session["accessJwt"].as_str().unwrap().to_string();
    (jwt, scopes_response)
@@ -635,10 +640,7 @@ async fn create_app_password_session(
 async fn try_chat_service_auth(client: &reqwest::Client, jwt: &str) -> StatusCode {
    let base = base_url().await;
    let res = client
-        .get(format!(
-            "{}/xrpc/com.atproto.server.getServiceAuth",
-            base
-        ))
+        .get(format!("{}/xrpc/com.atproto.server.getServiceAuth", base))
        .bearer_auth(jwt)
        .query(&[
            ("aud", "did:web:api.bsky.app"),
@@ -15,7 +15,9 @@ fn test_cid(n: u32) -> Cid {
    Cid::new_v1(0x71, mh)
 }

-async fn compute_obsolete_full_walk<S: jacquard_repo::storage::BlockStore + Sync + Send + 'static>(
+async fn compute_obsolete_full_walk<
+    S: jacquard_repo::storage::BlockStore + Sync + Send + 'static,
+>(
    old: &Mst<S>,
    new: &Mst<S>,
 ) -> BTreeSet<Cid> {
@@ -34,9 +36,7 @@ async fn compute_obsolete_full_walk<S: jacquard_repo::storage::BlockStore + Sync
        .collect()
 }

-fn compute_obsolete_from_diff(
-    diff: &jacquard_repo::mst::diff::MstDiff,
-) -> BTreeSet<Cid> {
+fn compute_obsolete_from_diff(diff: &jacquard_repo::mst::diff::MstDiff) -> BTreeSet<Cid> {
    diff.removed_mst_blocks
        .iter()
        .copied()
@@ -74,12 +74,17 @@ async fn assert_equivalence(
    let diff_obsolete = compute_obsolete_from_diff(&diff);

    assert_eq!(
-        full_walk_obsolete, diff_obsolete,
+        full_walk_obsolete,
+        diff_obsolete,
        "MISMATCH in scenario: {scenario}\n  full_walk count: {}\n  diff count: {}\n  in full_walk but not diff: {:?}\n  in diff but not full_walk: {:?}",
        full_walk_obsolete.len(),
        diff_obsolete.len(),
-        full_walk_obsolete.difference(&diff_obsolete).collect::<Vec<_>>(),
-        diff_obsolete.difference(&full_walk_obsolete).collect::<Vec<_>>(),
+        full_walk_obsolete
+            .difference(&diff_obsolete)
+            .collect::<Vec<_>>(),
+        diff_obsolete
+            .difference(&full_walk_obsolete)
+            .collect::<Vec<_>>(),
    );
 }

@@ -256,7 +261,12 @@ async fn massive_to_empty() {
 async fn massive_complete_replacement() {
    let old = generate_records("app.bsky.feed.post", 0..1000);
    let new_rec = generate_records("app.bsky.feed.post", 1000..2000);
-    assert_equivalence(&old, &new_rec, "1000 records fully replaced with 1000 different").await;
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "1000 records fully replaced with 1000 different",
+    )
+    .await;
 }

 #[tokio::test]
@@ -276,7 +286,12 @@ async fn multi_collection_5_collections_500_each() {
    ];
    let old = generate_multi_collection_records(&collections, 500);
    let new_rec = apply_scattered_updates(&old, 4, 30000);
-    assert_equivalence(&old, &new_rec, "5 collections x 500 records - update every 4th").await;
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "5 collections x 500 records - update every 4th",
+    )
+    .await;
 }

 #[tokio::test]
@@ -294,7 +309,12 @@ async fn multi_collection_wipe_one_collection() {
        .filter(|(key, _)| !key.starts_with("app.bsky.feed.repost"))
        .cloned()
        .collect();
-    assert_equivalence(&old, &new_rec, "4 collections x 400 - wipe repost collection").await;
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "4 collections x 400 - wipe repost collection",
+    )
+    .await;
 }

 #[tokio::test]
@@ -313,10 +333,7 @@ async fn multi_collection_keep_only_one() {

 #[tokio::test]
 async fn multi_collection_add_new_collection() {
-    let old_collections = [
-        "app.bsky.feed.like",
-        "app.bsky.feed.post",
-    ];
+    let old_collections = ["app.bsky.feed.like", "app.bsky.feed.post"];
    let old = generate_multi_collection_records(&old_collections, 500);
    let new_rec = append_records(&old, "app.bsky.graph.follow", 0..500, 40000);
    assert_equivalence(&old, &new_rec, "2 collections x 500 + add 500 follows").await;
@@ -378,7 +395,12 @@ async fn interleaved_keys_disjoint_ranges() {
    let new_rec: Vec<_> = (0..1000u32)
        .map(|i| (make_key("app.bsky.feed.post", i * 2 + 1), i + 10000))
        .collect();
-    assert_equivalence(&old, &new_rec, "1000 even-keyed records replaced by 1000 odd-keyed").await;
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "1000 even-keyed records replaced by 1000 odd-keyed",
+    )
+    .await;
 }

 #[tokio::test]
@@ -426,7 +448,12 @@ async fn many_collections_few_records_each() {
        })
        .collect();

-    assert_equivalence(&old, &new_rec, "50 collections x 20 records - delete every 15th, update every 7th").await;
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "50 collections x 20 records - delete every 15th, update every 7th",
+    )
+    .await;
 }

 #[tokio::test]
@@ -457,7 +484,12 @@ async fn one_to_massive() {
 async fn delete_head_and_tail() {
    let old = generate_records("app.bsky.feed.post", 0..2000);
    let new_rec: Vec<_> = old[200..1800].to_vec();
-    assert_equivalence(&old, &new_rec, "2000 records - delete first 200 and last 200").await;
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "2000 records - delete first 200 and last 200",
+    )
+    .await;
 }

 #[tokio::test]
@@ -465,7 +497,12 @@ async fn keep_head_and_tail_only() {
    let old = generate_records("app.bsky.feed.post", 0..2000);
    let mut new_rec: Vec<_> = old[..100].to_vec();
    new_rec.extend_from_slice(&old[1900..]);
-    assert_equivalence(&old, &new_rec, "2000 records - keep only first 100 and last 100").await;
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "2000 records - keep only first 100 and last 100",
+    )
+    .await;
 }

 #[tokio::test]
@@ -515,7 +552,12 @@ async fn swiss_cheese_deletions() {
        })
        .map(|(_, r)| r.clone())
        .collect();
-    assert_equivalence(&old, &new_rec, "1500 records - delete every 3rd chunk of 50").await;
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "1500 records - delete every 3rd chunk of 50",
+    )
+    .await;
 }

 #[tokio::test]
@@ -529,9 +571,7 @@ async fn mixed_ops_with_key_density_change() {
        .filter(|(_, val)| val % 4 != 0)
        .cloned()
        .collect();
-    new_rec.extend((0..500u32).map(|i| {
-        (make_key("app.bsky.feed.post", i * 3 + 1), i + 100000)
-    }));
+    new_rec.extend((0..500u32).map(|i| (make_key("app.bsky.feed.post", i * 3 + 1), i + 100000)));
    new_rec.sort_by(|(a, _), (b, _)| a.cmp(b));

    assert_equivalence(
@@ -0,0 +1,475 @@
+mod common;
+mod firehose;
+
+use std::collections::BTreeMap;
+use std::io::Cursor;
+use std::str::FromStr;
+use std::sync::Arc;
+use std::time::Duration;
+
+use bytes::Bytes;
+use cid::Cid;
+use common::*;
+use firehose::{FirehoseConsumer, ParsedCommitFrame};
+use iroh_car::CarReader;
+use jacquard_common::smol_str::SmolStr;
+use jacquard_repo::commit::Commit;
+use jacquard_repo::mst::{Mst, VerifiedWriteOp};
+use jacquard_repo::storage::{BlockStore, MemoryBlockStore};
+use reqwest::StatusCode;
+use serde_json::{Value, json};
+use tranquil_scopes::RepoAction;
+
+async fn car_to_blocks(car_bytes: &[u8]) -> BTreeMap<Cid, Bytes> {
+    let mut reader = CarReader::new(Cursor::new(car_bytes))
+        .await
+        .expect("parse CAR header");
+    let mut blocks = BTreeMap::new();
+    while let Ok(Some((cid, data))) = reader.next_block().await {
+        blocks.insert(cid, Bytes::from(data));
+    }
+    blocks
+}
+
+fn op_to_verified(op: &firehose::ParsedRepoOp) -> Result<VerifiedWriteOp, String> {
+    let key = SmolStr::new(&op.path);
+    match op.action {
+        RepoAction::Create => {
+            let cid = op.cid.ok_or("create op missing cid")?;
+            Ok(VerifiedWriteOp::Create { key, cid })
+        }
+        RepoAction::Update => {
+            let cid = op.cid.ok_or("update op missing cid")?;
+            let prev = op.prev.ok_or("update op missing prev")?;
+            Ok(VerifiedWriteOp::Update { key, cid, prev })
+        }
+        RepoAction::Delete => {
+            let prev = op.prev.ok_or("delete op missing prev")?;
+            Ok(VerifiedWriteOp::Delete { key, prev })
+        }
+    }
+}
+
+async fn verify_frame_forward(frame: &ParsedCommitFrame) -> Result<(), String> {
+    let prev_data = frame
+        .prev_data
+        .ok_or_else(|| "frame missing prev_data (v1.1 required)".to_string())?;
+
+    let blocks = car_to_blocks(&frame.blocks).await;
+    let storage = Arc::new(MemoryBlockStore::new_from_blocks(blocks));
+
+    let commit_bytes = storage
+        .get(&frame.commit)
+        .await
+        .map_err(|e| format!("get commit: {e:?}"))?
+        .ok_or_else(|| format!("CAR missing commit {}", frame.commit))?;
+    let commit = Commit::from_cbor(&commit_bytes).map_err(|e| format!("parse commit: {e:?}"))?;
+    let expected = *commit.data();
+
+    let mut mst = Mst::load(storage, prev_data, None);
+    for op in &frame.ops {
+        let path = &op.path;
+        match op.action {
+            RepoAction::Create | RepoAction::Update => {
+                let cid = op.cid.ok_or_else(|| format!("{path}: op missing cid"))?;
+                mst = mst
+                    .add(path, cid)
+                    .await
+                    .map_err(|e| format!("forward {path}: {e:?}"))?;
+            }
+            RepoAction::Delete => {
+                mst = mst
+                    .delete(path)
+                    .await
+                    .map_err(|e| format!("forward delete {path}: {e:?}"))?;
+            }
+        }
+    }
+    let computed = mst.persist().await.map_err(|e| format!("persist: {e:?}"))?;
+    if computed != expected {
+        return Err(format!(
+            "root mismatch expected={expected} computed={computed}"
+        ));
+    }
+    Ok(())
+}
+
+async fn verify_frame_inverse(frame: &ParsedCommitFrame) -> Result<(), String> {
+    let prev_data = frame
+        .prev_data
+        .ok_or_else(|| "frame missing prev_data (v1.1 required)".to_string())?;
+
+    let blocks = car_to_blocks(&frame.blocks).await;
+    let storage = Arc::new(MemoryBlockStore::new_from_blocks(blocks));
+
+    let commit_bytes = storage
+        .get(&frame.commit)
+        .await
+        .map_err(|e| format!("get commit: {e:?}"))?
+        .ok_or_else(|| format!("CAR missing commit {}", frame.commit))?;
+    let commit = Commit::from_cbor(&commit_bytes).map_err(|e| format!("parse commit: {e:?}"))?;
+    let new_data = *commit.data();
+
+    let mut mst = Mst::load(storage, new_data, None);
+    for op in &frame.ops {
+        let verified = op_to_verified(op)?;
+        let inverted = mst
+            .invert_op(verified.clone())
+            .await
+            .map_err(|e| format!("invert {verified:?}: {e:?}"))?;
+        if !inverted {
+            return Err(format!("op not invertible: {verified:?}"));
+        }
+    }
+    let computed_prev = mst
+        .get_pointer()
+        .await
+        .map_err(|e| format!("get_pointer: {e:?}"))?;
+    if computed_prev != prev_data {
+        return Err(format!(
+            "inverse root mismatch expected={prev_data} computed={computed_prev}"
+        ));
+    }
+    Ok(())
+}
+
+async fn create_record(client: &reqwest::Client, token: &str, did: &str, rkey: &str, text: &str) {
+    let now = chrono::Utc::now().to_rfc3339();
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.createRecord",
+            base_url().await
+        ))
+        .bearer_auth(token)
+        .json(&json!({
+            "repo": did,
+            "collection": "app.bsky.feed.post",
+            "rkey": rkey,
+            "record": {
+                "$type": "app.bsky.feed.post",
+                "text": text,
+                "createdAt": now,
+            }
+        }))
+        .send()
+        .await
+        .expect("createRecord");
+    assert_eq!(res.status(), StatusCode::OK);
+}
+
+async fn put_record(client: &reqwest::Client, token: &str, did: &str, rkey: &str, text: &str) {
+    let now = chrono::Utc::now().to_rfc3339();
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.putRecord",
+            base_url().await
+        ))
+        .bearer_auth(token)
+        .json(&json!({
+            "repo": did,
+            "collection": "app.bsky.feed.post",
+            "rkey": rkey,
+            "record": {
+                "$type": "app.bsky.feed.post",
+                "text": text,
+                "createdAt": now,
+            }
+        }))
+        .send()
+        .await
+        .expect("putRecord");
+    assert_eq!(res.status(), StatusCode::OK);
+}
+
+async fn delete_record(client: &reqwest::Client, token: &str, did: &str, rkey: &str) {
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.deleteRecord",
+            base_url().await
+        ))
+        .bearer_auth(token)
+        .json(&json!({
+            "repo": did,
+            "collection": "app.bsky.feed.post",
+            "rkey": rkey,
+        }))
+        .send()
+        .await
+        .expect("deleteRecord");
+    assert_eq!(res.status(), StatusCode::OK);
+}
+
+async fn apply_writes_batch(client: &reqwest::Client, token: &str, did: &str, writes: Vec<Value>) {
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.applyWrites",
+            base_url().await
+        ))
+        .bearer_auth(token)
+        .json(&json!({ "repo": did, "writes": writes }))
+        .send()
+        .await
+        .expect("applyWrites");
+    assert_eq!(res.status(), StatusCode::OK);
+}
+
+fn rkey_for(i: usize) -> String {
+    format!("3ke2e{:08}", i)
+}
+
+#[tokio::test]
+async fn websocket_firehose_frames_pass_inductive_forward_and_inverse() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let repos = get_test_repos().await;
+    let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
+    let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    let now = chrono::Utc::now().to_rfc3339();
+    let seed: Vec<Value> = (0..120)
+        .map(|i| {
+            json!({
+                "$type": "com.atproto.repo.applyWrites#create",
+                "collection": "app.bsky.feed.post",
+                "rkey": rkey_for(i),
+                "value": {
+                    "$type": "app.bsky.feed.post",
+                    "text": format!("e2e {i}"),
+                    "createdAt": now,
+                }
+            })
+        })
+        .collect();
+    for chunk in seed.chunks(40) {
+        apply_writes_batch(&client, &token, &did, chunk.to_vec()).await;
+    }
+    for i in (0..120).step_by(6) {
+        put_record(&client, &token, &did, &rkey_for(i), &format!("upd {i}")).await;
+    }
+    for i in (2..120).step_by(11) {
+        delete_record(&client, &token, &did, &rkey_for(i)).await;
+    }
+    create_record(&client, &token, &did, "3ke2efinal001", "final").await;
+
+    let target_commits = 3 + 20 + 11 + 1;
+    let frames = consumer
+        .wait_for_commits(&did, target_commits, Duration::from_secs(90))
+        .await;
+    assert!(
+        frames.len() >= target_commits,
+        "expected {} commit frames, got {}",
+        target_commits,
+        frames.len()
+    );
+
+    let mut forward_failures = Vec::new();
+    let mut inverse_failures = Vec::new();
+    for frame in &frames {
+        if frame.prev_data.is_none() {
+            continue;
+        }
+        if frame.ops.is_empty() {
+            continue;
+        }
+        if let Err(msg) = verify_frame_forward(frame).await {
+            forward_failures.push(format!("seq={}: {msg}", frame.seq));
+        }
+        if let Err(msg) = verify_frame_inverse(frame).await {
+            inverse_failures.push(format!("seq={}: {msg}", frame.seq));
+        }
+    }
+    assert!(
+        forward_failures.is_empty(),
+        "forward verification failures:\n  - {}",
+        forward_failures.join("\n  - ")
+    );
+    assert!(
+        inverse_failures.is_empty(),
+        "inverse verification failures:\n  - {}",
+        inverse_failures.join("\n  - ")
+    );
+}
+
+#[tokio::test]
+async fn websocket_firehose_car_root_matches_commit_cid() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let repos = get_test_repos().await;
+    let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
+    let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    for i in 0..4 {
+        create_record(&client, &token, &did, &rkey_for(i), "ck").await;
+    }
+
+    let frames = consumer
+        .wait_for_commits(&did, 4, Duration::from_secs(10))
+        .await;
+
+    for frame in &frames {
+        let mut reader = CarReader::new(Cursor::new(&frame.blocks))
+            .await
+            .expect("CAR header");
+        let roots = reader.header().roots();
+        assert_eq!(roots.len(), 1, "CAR must have exactly one root");
+        assert_eq!(
+            roots[0], frame.commit,
+            "CAR root must equal frame commit CID"
+        );
+        let mut found = false;
+        while let Ok(Some((cid, _))) = reader.next_block().await {
+            if cid == frame.commit {
+                found = true;
+            }
+        }
+        assert!(found, "CAR body must contain commit block");
+    }
+}
+
+#[tokio::test]
+async fn websocket_firehose_resumption_from_cursor_yields_valid_frames() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+    let repos = get_test_repos().await;
+
+    for i in 0..5 {
+        create_record(&client, &token, &did, &rkey_for(i), "pre").await;
+    }
+
+    let resume_cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
+
+    for i in 5..12 {
+        create_record(&client, &token, &did, &rkey_for(i), "post").await;
+    }
+
+    let consumer = FirehoseConsumer::connect_with_cursor(app_port(), resume_cursor).await;
+    let frames = consumer
+        .wait_for_commits(&did, 7, Duration::from_secs(20))
+        .await;
+    assert!(
+        frames.len() >= 7,
+        "expected 7+ frames after cursor resume, got {}",
+        frames.len()
+    );
+
+    for frame in &frames {
+        if frame.prev_data.is_none() || frame.ops.is_empty() {
+            continue;
+        }
+        verify_frame_forward(frame)
+            .await
+            .unwrap_or_else(|e| panic!("resumed frame seq={} invalid: {e}", frame.seq));
+    }
+}
+
+#[tokio::test]
+async fn websocket_firehose_ops_include_prev_field_for_update_delete() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+    let repos = get_test_repos().await;
+    let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
+    let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    create_record(&client, &token, &did, "3ke2eprev01", "v1").await;
+    put_record(&client, &token, &did, "3ke2eprev01", "v2").await;
+    delete_record(&client, &token, &did, "3ke2eprev01").await;
+
+    let frames = consumer
+        .wait_for_commits(&did, 3, Duration::from_secs(10))
+        .await;
+    assert!(frames.len() >= 3);
+
+    for frame in &frames {
+        for op in &frame.ops {
+            match op.action {
+                RepoAction::Create => {
+                    assert!(op.cid.is_some(), "create must have cid");
+                    assert!(op.prev.is_none(), "create must not have prev");
+                }
+                RepoAction::Update => {
+                    assert!(op.cid.is_some(), "update must have cid");
+                    assert!(
+                        op.prev.is_some(),
+                        "v1.1 update must carry prev CID (seq={})",
+                        frame.seq
+                    );
+                }
+                RepoAction::Delete => {
+                    assert!(op.cid.is_none(), "delete must have null cid");
+                    assert!(
+                        op.prev.is_some(),
+                        "v1.1 delete must carry prev CID (seq={})",
+                        frame.seq
+                    );
+                }
+            }
+        }
+    }
+}
+
+#[tokio::test]
+async fn websocket_firehose_rebuild_new_mst_from_car_matches_commit_data() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+    let repos = get_test_repos().await;
+    let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
+    let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    let now = chrono::Utc::now().to_rfc3339();
+    let writes: Vec<Value> = (0..30)
+        .map(|i| {
+            json!({
+                "$type": "com.atproto.repo.applyWrites#create",
+                "collection": "app.bsky.feed.post",
+                "rkey": rkey_for(i),
+                "value": {
+                    "$type": "app.bsky.feed.post",
+                    "text": format!("rb {i}"),
+                    "createdAt": now,
+                }
+            })
+        })
+        .collect();
+    apply_writes_batch(&client, &token, &did, writes).await;
+
+    let frames = consumer
+        .wait_for_commits(&did, 1, Duration::from_secs(10))
+        .await;
+    let last = frames.last().expect("frame");
+
+    let blocks = car_to_blocks(&last.blocks).await;
+    let storage = Arc::new(MemoryBlockStore::new_from_blocks(blocks));
+    let commit_bytes = storage
+        .get(&last.commit)
+        .await
+        .unwrap()
+        .expect("commit block");
+    let commit = Commit::from_cbor(&commit_bytes).unwrap();
+
+    let new_root_cid = *commit.data();
+    let mst = Mst::load(storage, new_root_cid, None);
+    let rehydrated_cid = mst.get_pointer().await.expect("rebuild mst");
+    assert_eq!(
+        rehydrated_cid, new_root_cid,
+        "MST loaded from CAR must yield same root as commit.data()"
+    );
+
+    for op in &last.ops {
+        if op.action == RepoAction::Create {
+            let expected_cid = op.cid.unwrap();
+            let got = mst
+                .get(&op.path)
+                .await
+                .expect("mst.get")
+                .unwrap_or_else(|| panic!("key {} missing from rebuilt tree", op.path));
+            assert_eq!(got, expected_cid, "record CID mismatch for {}", op.path);
+            let _ = Cid::from_str(&expected_cid.to_string()).unwrap();
+        }
+    }
+}
@@ -0,0 +1,697 @@
+mod common;
+mod mst_verify;
+
+use std::collections::BTreeMap;
+use std::str::FromStr;
+use std::sync::Arc;
+
+use cid::Cid;
+use common::*;
+use jacquard_common::smol_str::SmolStr;
+use jacquard_repo::commit::Commit;
+use jacquard_repo::mst::{Mst, VerifiedWriteOp};
+use jacquard_repo::storage::{BlockStore, MemoryBlockStore};
+use mst_verify::{extract_event_blocks, inline_to_store};
+use reqwest::StatusCode;
+use serde_json::{Value, json};
+use tranquil_db_traits::{RepoEventType, SequenceNumber, SequencedEvent};
+use tranquil_types::Did;
+
+async fn new_commit_data_cid(
+    storage: &Arc<MemoryBlockStore>,
+    commit_cid: &Cid,
+) -> Result<Cid, String> {
+    let commit_bytes = storage
+        .get(commit_cid)
+        .await
+        .map_err(|e| format!("get commit: {e:?}"))?
+        .ok_or_else(|| format!("CAR missing commit block {commit_cid}"))?;
+    let commit = Commit::from_cbor(&commit_bytes).map_err(|e| format!("parse commit: {e:?}"))?;
+    Ok(*commit.data())
+}
+
+fn ops_json(event: &SequencedEvent) -> Result<&Vec<Value>, String> {
+    event
+        .ops
+        .as_ref()
+        .and_then(|v| v.as_array())
+        .ok_or_else(|| "event.ops not an array".into())
+}
+
+fn parse_op_to_verified(op: &Value) -> Result<VerifiedWriteOp, String> {
+    let action = op["action"].as_str().ok_or("op.action missing")?;
+    let path = op["path"].as_str().ok_or("op.path missing")?;
+    let key = SmolStr::new(path);
+    match action {
+        "create" => {
+            let cid_str = op["cid"].as_str().ok_or("create missing cid")?;
+            let cid = Cid::from_str(cid_str).map_err(|e| format!("parse cid: {e:?}"))?;
+            Ok(VerifiedWriteOp::Create { key, cid })
+        }
+        "update" => {
+            let cid_str = op["cid"].as_str().ok_or("update missing cid")?;
+            let cid = Cid::from_str(cid_str).map_err(|e| format!("parse cid: {e:?}"))?;
+            let prev_str = op["prev"].as_str().ok_or("update missing prev")?;
+            let prev = Cid::from_str(prev_str).map_err(|e| format!("parse prev: {e:?}"))?;
+            Ok(VerifiedWriteOp::Update { key, cid, prev })
+        }
+        "delete" => {
+            let prev_str = op["prev"].as_str().ok_or("delete missing prev")?;
+            let prev = Cid::from_str(prev_str).map_err(|e| format!("parse prev: {e:?}"))?;
+            Ok(VerifiedWriteOp::Delete { key, prev })
+        }
+        other => Err(format!("unknown op action: {other}")),
+    }
+}
+
+async fn verify_inductive_forward(event: &SequencedEvent) -> Result<(Cid, Cid), String> {
+    let prev_data_cid = event
+        .prev_data_cid
+        .as_ref()
+        .and_then(|c| c.to_cid())
+        .ok_or_else(|| "event missing prev_data_cid".to_string())?;
+    let commit_cid = event
+        .commit_cid
+        .as_ref()
+        .and_then(|c| c.to_cid())
+        .ok_or_else(|| "event missing commit_cid".to_string())?;
+
+    let storage = inline_to_store(extract_event_blocks(event)?);
+    let expected_new_data = new_commit_data_cid(&storage, &commit_cid).await?;
+
+    let mut mst = Mst::load(storage.clone(), prev_data_cid, None);
+    for op_value in ops_json(event)? {
+        let action = op_value["action"].as_str().ok_or("op.action missing")?;
+        let path = op_value["path"].as_str().ok_or("op.path missing")?;
+        match action {
+            "create" | "update" => {
+                let cid = Cid::from_str(op_value["cid"].as_str().ok_or("op.cid missing")?)
+                    .map_err(|e| format!("parse op.cid: {e:?}"))?;
+                mst = mst
+                    .add(path, cid)
+                    .await
+                    .map_err(|e| format!("mst.add({path}): {e:?}"))?;
+            }
+            "delete" => {
+                mst = mst
+                    .delete(path)
+                    .await
+                    .map_err(|e| format!("mst.delete({path}): {e:?}"))?;
+            }
+            other => return Err(format!("unknown op action: {other}")),
+        }
+    }
+    let computed = mst
+        .persist()
+        .await
+        .map_err(|e| format!("mst.persist: {e:?}"))?;
+    Ok((expected_new_data, computed))
+}
+
+async fn verify_inductive_inverse(event: &SequencedEvent) -> Result<(Cid, Cid), String> {
+    let prev_data_cid = event
+        .prev_data_cid
+        .as_ref()
+        .and_then(|c| c.to_cid())
+        .ok_or_else(|| "event missing prev_data_cid".to_string())?;
+    let commit_cid = event
+        .commit_cid
+        .as_ref()
+        .and_then(|c| c.to_cid())
+        .ok_or_else(|| "event missing commit_cid".to_string())?;
+
+    let storage = inline_to_store(extract_event_blocks(event)?);
+    let new_data_cid = new_commit_data_cid(&storage, &commit_cid).await?;
+
+    let mut mst = Mst::load(storage.clone(), new_data_cid, None);
+    for op_value in ops_json(event)?.iter().rev() {
+        let verified = parse_op_to_verified(op_value)?;
+        let inverted = mst
+            .invert_op(verified.clone())
+            .await
+            .map_err(|e| format!("invert_op({verified:?}): {e:?}"))?;
+        if !inverted {
+            return Err(format!("op not invertible: {verified:?}"));
+        }
+    }
+    let computed_prev = mst
+        .get_pointer()
+        .await
+        .map_err(|e| format!("get_pointer: {e:?}"))?;
+    Ok((prev_data_cid, computed_prev))
+}
+
+fn report_failures(total: usize, failures: &[String], mode: &str) {
+    assert!(
+        failures.is_empty(),
+        "{} of {total} {mode} commit events failed inductive verification:\n  - {}",
+        failures.len(),
+        failures.join("\n  - "),
+    );
+}
+
+async fn apply_writes_batch(client: &reqwest::Client, token: &str, did: &str, writes: Vec<Value>) {
+    let payload = json!({ "repo": did, "writes": writes });
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.applyWrites",
+            base_url().await
+        ))
+        .bearer_auth(token)
+        .json(&payload)
+        .send()
+        .await
+        .expect("applyWrites request failed");
+    assert_eq!(
+        res.status(),
+        StatusCode::OK,
+        "applyWrites failed: {:?}",
+        res.text().await
+    );
+}
+
+async fn create_record(client: &reqwest::Client, token: &str, did: &str, col: &str, rkey: &str) {
+    let now = chrono::Utc::now().to_rfc3339();
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.createRecord",
+            base_url().await
+        ))
+        .bearer_auth(token)
+        .json(&json!({
+            "repo": did,
+            "collection": col,
+            "rkey": rkey,
+            "record": {
+                "$type": col,
+                "text": format!("post {rkey}"),
+                "createdAt": now,
+            }
+        }))
+        .send()
+        .await
+        .expect("createRecord request failed");
+    assert_eq!(res.status(), StatusCode::OK, "createRecord failed");
+}
+
+async fn put_record(
+    client: &reqwest::Client,
+    token: &str,
+    did: &str,
+    col: &str,
+    rkey: &str,
+    text: &str,
+) {
+    let now = chrono::Utc::now().to_rfc3339();
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.putRecord",
+            base_url().await
+        ))
+        .bearer_auth(token)
+        .json(&json!({
+            "repo": did,
+            "collection": col,
+            "rkey": rkey,
+            "record": {
+                "$type": col,
+                "text": text,
+                "createdAt": now,
+            }
+        }))
+        .send()
+        .await
+        .expect("putRecord request failed");
+    assert_eq!(res.status(), StatusCode::OK, "putRecord failed");
+}
+
+async fn delete_record(client: &reqwest::Client, token: &str, did: &str, col: &str, rkey: &str) {
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.deleteRecord",
+            base_url().await
+        ))
+        .bearer_auth(token)
+        .json(&json!({ "repo": did, "collection": col, "rkey": rkey }))
+        .send()
+        .await
+        .expect("deleteRecord request failed");
+    assert_eq!(res.status(), StatusCode::OK, "deleteRecord failed");
+}
+
+const COLLECTION: &str = "app.bsky.feed.post";
+fn rkey_for(prefix: &str, i: usize) -> String {
+    format!("3k{prefix}{:08}", i)
+}
+
+async fn our_commit_events(did: &str) -> Vec<SequencedEvent> {
+    let repos = get_test_repos().await;
+    let typed_did = Did::new(did.to_string()).unwrap();
+    let events = repos
+        .repo
+        .get_events_since_seq(SequenceNumber::ZERO, None)
+        .await
+        .expect("get_events_since_seq");
+    events
+        .into_iter()
+        .filter(|e| e.did == typed_did && e.event_type == RepoEventType::Commit)
+        .collect()
+}
+
+#[tokio::test]
+async fn inductive_forward_verifies_delete_commits() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let now = chrono::Utc::now().to_rfc3339();
+    const N_CREATE: usize = 200;
+
+    let all_writes: Vec<Value> = (0..N_CREATE)
+        .map(|i| {
+            json!({
+                "$type": "com.atproto.repo.applyWrites#create",
+                "collection": COLLECTION,
+                "rkey": rkey_for("del", i),
+                "value": {
+                    "$type": COLLECTION,
+                    "text": format!("record {i}"),
+                    "createdAt": now,
+                }
+            })
+        })
+        .collect();
+    for chunk in all_writes.chunks(50) {
+        apply_writes_batch(&client, &token, &did, chunk.to_vec()).await;
+    }
+
+    let delete_indices: Vec<usize> = (10..N_CREATE).step_by(7).collect();
+    for i in &delete_indices {
+        delete_record(&client, &token, &did, COLLECTION, &rkey_for("del", *i)).await;
+    }
+
+    let our = our_commit_events(&did).await;
+    let delete_events: Vec<&SequencedEvent> = our
+        .iter()
+        .filter(|e| {
+            ops_json(e)
+                .map(|arr| arr.iter().any(|op| op["action"].as_str() == Some("delete")))
+                .unwrap_or(false)
+        })
+        .collect();
+    assert_eq!(delete_events.len(), delete_indices.len());
+
+    let mut failures = Vec::new();
+    for e in &delete_events {
+        match verify_inductive_forward(e).await {
+            Ok((exp, got)) if exp == got => {}
+            Ok((exp, got)) => failures.push(format!(
+                "seq={}: root mismatch exp={exp} got={got}",
+                e.seq.as_i64()
+            )),
+            Err(msg) => failures.push(format!("seq={}: {msg}", e.seq.as_i64())),
+        }
+    }
+    report_failures(delete_events.len(), &failures, "delete forward");
+}
+
+#[tokio::test]
+async fn inductive_forward_verifies_create_commits() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+    for i in 0..60usize {
+        create_record(&client, &token, &did, COLLECTION, &rkey_for("cre", i)).await;
+    }
+
+    let our = our_commit_events(&did).await;
+    let create_events: Vec<&SequencedEvent> = our
+        .iter()
+        .filter(|e| {
+            ops_json(e)
+                .map(|arr| arr.iter().all(|op| op["action"].as_str() == Some("create")))
+                .unwrap_or(false)
+                && e.prev_data_cid.is_some()
+        })
+        .collect();
+    assert!(!create_events.is_empty());
+
+    let mut failures = Vec::new();
+    for e in &create_events {
+        match verify_inductive_forward(e).await {
+            Ok((exp, got)) if exp == got => {}
+            Ok((exp, got)) => failures.push(format!(
+                "seq={}: root mismatch exp={exp} got={got}",
+                e.seq.as_i64()
+            )),
+            Err(msg) => failures.push(format!("seq={}: {msg}", e.seq.as_i64())),
+        }
+    }
+    report_failures(create_events.len(), &failures, "create forward");
+}
+
+#[tokio::test]
+async fn inductive_forward_verifies_update_commits() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let now = chrono::Utc::now().to_rfc3339();
+    let creates: Vec<Value> = (0..80)
+        .map(|i| {
+            json!({
+                "$type": "com.atproto.repo.applyWrites#create",
+                "collection": COLLECTION,
+                "rkey": rkey_for("upd", i),
+                "value": {
+                    "$type": COLLECTION,
+                    "text": format!("original {i}"),
+                    "createdAt": now,
+                }
+            })
+        })
+        .collect();
+    for chunk in creates.chunks(40) {
+        apply_writes_batch(&client, &token, &did, chunk.to_vec()).await;
+    }
+
+    for i in (0..80).step_by(3) {
+        put_record(
+            &client,
+            &token,
+            &did,
+            COLLECTION,
+            &rkey_for("upd", i),
+            &format!("updated {i}"),
+        )
+        .await;
+    }
+
+    let our = our_commit_events(&did).await;
+    let update_events: Vec<&SequencedEvent> = our
+        .iter()
+        .filter(|e| {
+            ops_json(e)
+                .map(|arr| arr.iter().any(|op| op["action"].as_str() == Some("update")))
+                .unwrap_or(false)
+        })
+        .collect();
+    assert!(!update_events.is_empty());
+
+    let mut failures = Vec::new();
+    for e in &update_events {
+        match verify_inductive_forward(e).await {
+            Ok((exp, got)) if exp == got => {}
+            Ok((exp, got)) => failures.push(format!(
+                "seq={}: root mismatch exp={exp} got={got}",
+                e.seq.as_i64()
+            )),
+            Err(msg) => failures.push(format!("seq={}: {msg}", e.seq.as_i64())),
+        }
+    }
+    report_failures(update_events.len(), &failures, "update forward");
+}
+
+#[tokio::test]
+async fn inductive_forward_verifies_mixed_applywrites() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let now = chrono::Utc::now().to_rfc3339();
+    let seed: Vec<Value> = (0..120)
+        .map(|i| {
+            json!({
+                "$type": "com.atproto.repo.applyWrites#create",
+                "collection": COLLECTION,
+                "rkey": rkey_for("mix", i),
+                "value": {
+                    "$type": COLLECTION,
+                    "text": format!("seed {i}"),
+                    "createdAt": now,
+                }
+            })
+        })
+        .collect();
+    for chunk in seed.chunks(40) {
+        apply_writes_batch(&client, &token, &did, chunk.to_vec()).await;
+    }
+
+    let mixed: Vec<Value> = (0..40)
+        .flat_map(|i| {
+            vec![
+                json!({
+                    "$type": "com.atproto.repo.applyWrites#create",
+                    "collection": COLLECTION,
+                    "rkey": rkey_for("mxc", i),
+                    "value": {
+                        "$type": COLLECTION,
+                        "text": format!("new {i}"),
+                        "createdAt": now,
+                    }
+                }),
+                json!({
+                    "$type": "com.atproto.repo.applyWrites#update",
+                    "collection": COLLECTION,
+                    "rkey": rkey_for("mix", i),
+                    "value": {
+                        "$type": COLLECTION,
+                        "text": format!("updated-mix {i}"),
+                        "createdAt": now,
+                    }
+                }),
+                json!({
+                    "$type": "com.atproto.repo.applyWrites#delete",
+                    "collection": COLLECTION,
+                    "rkey": rkey_for("mix", i + 60),
+                }),
+            ]
+        })
+        .collect();
+    apply_writes_batch(&client, &token, &did, mixed).await;
+
+    let our = our_commit_events(&did).await;
+    let last = our
+        .iter()
+        .rfind(|e| e.prev_data_cid.is_some())
+        .expect("at least one non-genesis commit");
+
+    let actions: Vec<&str> = ops_json(last)
+        .unwrap()
+        .iter()
+        .filter_map(|op| op["action"].as_str())
+        .collect();
+    assert!(actions.contains(&"create"));
+    assert!(actions.contains(&"update"));
+    assert!(actions.contains(&"delete"));
+
+    let (exp, got) = verify_inductive_forward(last)
+        .await
+        .expect("mixed applyWrites forward verify");
+    assert_eq!(exp, got, "mixed applyWrites commit forward-verify mismatch");
+}
+
+#[tokio::test]
+async fn inductive_inverse_verifies_every_commit() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let now = chrono::Utc::now().to_rfc3339();
+    let seed: Vec<Value> = (0..100)
+        .map(|i| {
+            json!({
+                "$type": "com.atproto.repo.applyWrites#create",
+                "collection": COLLECTION,
+                "rkey": rkey_for("inv", i),
+                "value": {
+                    "$type": COLLECTION,
+                    "text": format!("seed {i}"),
+                    "createdAt": now,
+                }
+            })
+        })
+        .collect();
+    for chunk in seed.chunks(50) {
+        apply_writes_batch(&client, &token, &did, chunk.to_vec()).await;
+    }
+    for i in (0..100).step_by(5) {
+        put_record(
+            &client,
+            &token,
+            &did,
+            COLLECTION,
+            &rkey_for("inv", i),
+            &format!("upd {i}"),
+        )
+        .await;
+    }
+    for i in (2..100).step_by(11) {
+        delete_record(&client, &token, &did, COLLECTION, &rkey_for("inv", i)).await;
+    }
+
+    let our = our_commit_events(&did).await;
+    let non_genesis: Vec<&SequencedEvent> = our
+        .iter()
+        .filter(|e| e.prev_data_cid.is_some() && ops_json(e).is_ok())
+        .collect();
+    assert!(!non_genesis.is_empty());
+
+    let mut failures = Vec::new();
+    for e in &non_genesis {
+        match verify_inductive_inverse(e).await {
+            Ok((exp, got)) if exp == got => {}
+            Ok((exp, got)) => failures.push(format!(
+                "seq={}: inverse root mismatch exp={exp} got={got}",
+                e.seq.as_i64()
+            )),
+            Err(msg) => failures.push(format!("seq={}: {msg}", e.seq.as_i64())),
+        }
+    }
+    report_failures(non_genesis.len(), &failures, "any inverse");
+}
+
+#[tokio::test]
+async fn inductive_inverse_handles_same_rkey_in_batch() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let now = chrono::Utc::now().to_rfc3339();
+    let rkey = rkey_for("dup", 0);
+    create_record(&client, &token, &did, COLLECTION, &rkey).await;
+
+    let writes = vec![
+        json!({
+            "$type": "com.atproto.repo.applyWrites#update",
+            "collection": COLLECTION,
+            "rkey": rkey,
+            "value": {
+                "$type": COLLECTION,
+                "text": "v1",
+                "createdAt": now,
+            }
+        }),
+        json!({
+            "$type": "com.atproto.repo.applyWrites#update",
+            "collection": COLLECTION,
+            "rkey": rkey,
+            "value": {
+                "$type": COLLECTION,
+                "text": "v2",
+                "createdAt": now,
+            }
+        }),
+    ];
+    apply_writes_batch(&client, &token, &did, writes).await;
+
+    let our = our_commit_events(&did).await;
+    let dup_event = our
+        .iter()
+        .find(|e| {
+            ops_json(e)
+                .map(|arr| {
+                    arr.iter()
+                        .filter(|op| op["action"].as_str() == Some("update"))
+                        .count()
+                        == 2
+                })
+                .unwrap_or(false)
+        })
+        .expect("commit event with two same-rkey updates");
+
+    let (exp, got) = verify_inductive_inverse(dup_event)
+        .await
+        .expect("inverse verify should succeed for same-rkey batch");
+    assert_eq!(
+        exp, got,
+        "inverse root mismatch for same-rkey batch: exp={exp} got={got}"
+    );
+}
+
+#[tokio::test]
+async fn prev_cid_chain_walks_to_genesis() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+    for i in 0..8 {
+        create_record(&client, &token, &did, COLLECTION, &rkey_for("cha", i)).await;
+    }
+
+    let our = our_commit_events(&did).await;
+    assert!(our.len() >= 2);
+
+    let last = our.last().unwrap();
+    let mut current_prev: Option<Cid> = last.prev_cid.as_ref().and_then(|c| c.to_cid());
+    let head_commit_cid = last
+        .commit_cid
+        .as_ref()
+        .and_then(|c| c.to_cid())
+        .expect("head commit_cid");
+
+    let by_commit: BTreeMap<Cid, &SequencedEvent> = our
+        .iter()
+        .filter_map(|e| {
+            e.commit_cid
+                .as_ref()
+                .and_then(|c| c.to_cid())
+                .map(|c| (c, e))
+        })
+        .collect();
+
+    let mut visited = 1;
+    while let Some(prev) = current_prev {
+        let e = by_commit
+            .get(&prev)
+            .unwrap_or_else(|| panic!("prev commit {prev} missing from event list"));
+        visited += 1;
+        current_prev = e.prev_cid.as_ref().and_then(|c| c.to_cid());
+    }
+    assert!(
+        visited >= 2,
+        "chain too short: visited={visited}, head_commit={head_commit_cid}"
+    );
+    assert_eq!(
+        visited,
+        our.len(),
+        "chain did not reach genesis: walked {visited}, have {}",
+        our.len()
+    );
+}
+
+#[tokio::test]
+async fn record_bytes_present_in_car_for_creates() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let now = chrono::Utc::now().to_rfc3339();
+    let writes: Vec<Value> = (0..5)
+        .map(|i| {
+            json!({
+                "$type": "com.atproto.repo.applyWrites#create",
+                "collection": COLLECTION,
+                "rkey": rkey_for("rec", i),
+                "value": {
+                    "$type": COLLECTION,
+                    "text": format!("rec {i}"),
+                    "createdAt": now,
+                }
+            })
+        })
+        .collect();
+    apply_writes_batch(&client, &token, &did, writes).await;
+
+    let our = our_commit_events(&did).await;
+    let latest = our.iter().rfind(|e| e.prev_data_cid.is_some()).unwrap();
+
+    let inline = extract_event_blocks(latest).unwrap();
+    let have_cids: std::collections::HashSet<Cid> = inline
+        .iter()
+        .map(|b| Cid::read_bytes(b.cid_bytes.as_slice()).unwrap())
+        .collect();
+
+    for op in ops_json(latest).unwrap() {
+        if op["action"].as_str() == Some("create")
+            && let Some(cid_str) = op["cid"].as_str()
+        {
+            let cid = Cid::from_str(cid_str).unwrap();
+            assert!(
+                have_cids.contains(&cid),
+                "create op record CID {cid} not present in CAR inline blocks"
+            );
+        }
+    }
+}
@@ -0,0 +1,334 @@
+mod common;
+mod firehose;
+mod helpers;
+
+use std::collections::BTreeMap;
+use std::io::Cursor;
+use std::sync::Arc;
+use std::time::Duration;
+
+use bytes::Bytes;
+use cid::Cid;
+use common::*;
+use firehose::FirehoseConsumer;
+use helpers::build_car_with_signature;
+use iroh_car::CarReader;
+use jacquard_repo::commit::Commit;
+use jacquard_repo::mst::Mst;
+use jacquard_repo::storage::{BlockStore, MemoryBlockStore};
+use k256::ecdsa::SigningKey;
+use reqwest::StatusCode;
+use serde_json::{Value, json};
+use tranquil_db_traits::{EventBlocks, RepoEventType, SequenceNumber, SequencedEvent};
+use tranquil_scopes::RepoAction;
+use tranquil_types::Did;
+
+async fn car_to_blocks(car_bytes: &[u8]) -> (Vec<Cid>, BTreeMap<Cid, Bytes>) {
+    let mut reader = CarReader::new(Cursor::new(car_bytes))
+        .await
+        .expect("parse CAR");
+    let roots = reader.header().roots().to_vec();
+    let mut blocks = BTreeMap::new();
+    while let Ok(Some((cid, data))) = reader.next_block().await {
+        blocks.insert(cid, Bytes::from(data));
+    }
+    (roots, blocks)
+}
+
+async fn create_post(client: &reqwest::Client, token: &str, did: &str, rkey: &str, text: &str) {
+    let now = chrono::Utc::now().to_rfc3339();
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.createRecord",
+            base_url().await
+        ))
+        .bearer_auth(token)
+        .json(&json!({
+            "repo": did,
+            "collection": "app.bsky.feed.post",
+            "rkey": rkey,
+            "record": {
+                "$type": "app.bsky.feed.post",
+                "text": text,
+                "createdAt": now,
+            }
+        }))
+        .send()
+        .await
+        .expect("createRecord");
+    assert_eq!(res.status(), StatusCode::OK);
+}
+
+#[tokio::test]
+async fn getrepo_car_roundtrips_mst_structure_and_records() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let expected_records: Vec<(String, String)> = (0..20)
+        .map(|i| {
+            let rkey = format!("3krtp{:08}", i);
+            let text = format!("roundtrip record {i}");
+            (rkey, text)
+        })
+        .collect();
+    for (rkey, text) in &expected_records {
+        create_post(&client, &token, &did, rkey, text).await;
+    }
+
+    let res = client
+        .get(format!(
+            "{}/xrpc/com.atproto.sync.getRepo",
+            base_url().await
+        ))
+        .query(&[("did", did.as_str())])
+        .send()
+        .await
+        .expect("getRepo");
+    assert_eq!(res.status(), StatusCode::OK);
+    let car_bytes = res.bytes().await.unwrap();
+
+    let (roots, block_map) = car_to_blocks(&car_bytes).await;
+    assert_eq!(roots.len(), 1, "CAR must have exactly one root");
+    let commit_cid = roots[0];
+    let storage = Arc::new(MemoryBlockStore::new_from_blocks(block_map));
+
+    let commit_bytes = storage
+        .get(&commit_cid)
+        .await
+        .unwrap()
+        .expect("CAR contains commit block");
+    let commit = Commit::from_cbor(&commit_bytes).expect("parse commit");
+    let data_cid = *commit.data();
+
+    let mst = Mst::load(storage.clone(), data_cid, None);
+    let loaded_root = mst.get_pointer().await.expect("load root");
+    assert_eq!(loaded_root, data_cid, "loaded MST pointer == commit.data()");
+
+    for (rkey, _) in &expected_records {
+        let path = format!("app.bsky.feed.post/{rkey}");
+        let leaf = mst
+            .get(&path)
+            .await
+            .expect("mst.get")
+            .unwrap_or_else(|| panic!("record {path} missing from exported MST"));
+        let leaf_bytes = storage
+            .get(&leaf)
+            .await
+            .unwrap()
+            .unwrap_or_else(|| panic!("record block {leaf} missing from CAR"));
+        assert!(!leaf_bytes.is_empty(), "record bytes empty");
+    }
+}
+
+#[tokio::test]
+async fn concurrent_swap_commit_writes_serialize() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    create_post(&client, &token, &did, "3kswap00000001", "anchor").await;
+
+    let latest_res = client
+        .get(format!(
+            "{}/xrpc/com.atproto.sync.getLatestCommit",
+            base_url().await
+        ))
+        .query(&[("did", did.as_str())])
+        .send()
+        .await
+        .expect("getLatestCommit");
+    assert_eq!(latest_res.status(), StatusCode::OK);
+    let latest: Value = latest_res.json().await.unwrap();
+    let swap_cid = latest["cid"].as_str().unwrap().to_string();
+
+    let now = chrono::Utc::now().to_rfc3339();
+    let payload_a = json!({
+        "repo": did,
+        "collection": "app.bsky.feed.post",
+        "rkey": "3kswap00000002",
+        "record": {
+            "$type": "app.bsky.feed.post",
+            "text": "writer A",
+            "createdAt": now,
+        },
+        "swapCommit": swap_cid,
+    });
+    let payload_b = json!({
+        "repo": did,
+        "collection": "app.bsky.feed.post",
+        "rkey": "3kswap00000003",
+        "record": {
+            "$type": "app.bsky.feed.post",
+            "text": "writer B",
+            "createdAt": now,
+        },
+        "swapCommit": swap_cid,
+    });
+
+    let base = base_url().await;
+    let (res_a, res_b) = tokio::join!(
+        client
+            .post(format!("{base}/xrpc/com.atproto.repo.putRecord"))
+            .bearer_auth(&token)
+            .json(&payload_a)
+            .send(),
+        client
+            .post(format!("{base}/xrpc/com.atproto.repo.putRecord"))
+            .bearer_auth(&token)
+            .json(&payload_b)
+            .send(),
+    );
+    let status_a = res_a.expect("A send").status();
+    let status_b = res_b.expect("B send").status();
+
+    let ok_a = status_a == StatusCode::OK;
+    let ok_b = status_b == StatusCode::OK;
+    assert!(
+        ok_a ^ ok_b,
+        "exactly one swap_commit write must succeed: status_a={status_a}, status_b={status_b}"
+    );
+}
+
+#[tokio::test]
+async fn imported_repo_emits_commit_event_with_valid_car() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let signing_key = SigningKey::random(&mut rand::thread_rng());
+    let (car_bytes, _car_root_cid) = build_car_with_signature(&did, &signing_key);
+
+    let import_res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.importRepo",
+            base_url().await
+        ))
+        .bearer_auth(&token)
+        .header("Content-Type", "application/vnd.ipld.car")
+        .body(car_bytes)
+        .send()
+        .await
+        .expect("importRepo");
+    assert_eq!(
+        import_res.status(),
+        StatusCode::OK,
+        "import failed: {:?}",
+        import_res.text().await.unwrap_or_default()
+    );
+
+    let repos = get_test_repos().await;
+    let typed_did = Did::new(did.clone()).unwrap();
+    let events = repos
+        .repo
+        .get_events_since_seq(SequenceNumber::ZERO, None)
+        .await
+        .expect("events");
+    let our: Vec<&SequencedEvent> = events
+        .iter()
+        .filter(|e| e.did == typed_did && e.event_type == RepoEventType::Commit)
+        .collect();
+    let last = our.last().expect("at least one commit event after import");
+
+    let inline = match last.blocks.as_ref().expect("blocks present") {
+        EventBlocks::Inline(v) => v,
+        _ => panic!("expected inline blocks"),
+    };
+    assert!(
+        !inline.is_empty(),
+        "import event inline blocks must not be empty"
+    );
+
+    let have_commit = inline.iter().any(|b| {
+        let cid = Cid::read_bytes(b.cid_bytes.as_slice()).unwrap();
+        last.commit_cid
+            .as_ref()
+            .and_then(|c| c.to_cid())
+            .map(|commit_cid| cid == commit_cid)
+            .unwrap_or(false)
+    });
+    assert!(have_commit, "import event CAR must include commit block");
+}
+
+#[tokio::test]
+async fn firehose_commit_block_bytes_roundtrip_to_same_cid() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let repos = get_test_repos().await;
+    let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
+    let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    create_post(&client, &token, &did, "3krt001", "round-trip me").await;
+    let frames = consumer
+        .wait_for_commits(&did, 1, Duration::from_secs(10))
+        .await;
+    let frame = frames.last().expect("frame");
+
+    let (_, block_map) = car_to_blocks(&frame.blocks).await;
+    use sha2::{Digest, Sha256};
+    for (cid, bytes) in &block_map {
+        let mut hasher = Sha256::new();
+        hasher.update(bytes);
+        let hash = hasher.finalize();
+        let mh = multihash::Multihash::wrap(0x12, hash.as_slice()).expect("wrap");
+        let recomputed = Cid::new_v1(cid.codec(), mh);
+        assert_eq!(
+            recomputed, *cid,
+            "CAR block {cid} bytes do not hash back to same CID"
+        );
+    }
+}
+
+#[tokio::test]
+async fn firehose_commit_car_contains_new_record_bytes_for_every_create() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let repos = get_test_repos().await;
+    let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
+    let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    let now = chrono::Utc::now().to_rfc3339();
+    let writes: Vec<Value> = (0..8)
+        .map(|i| {
+            json!({
+                "$type": "com.atproto.repo.applyWrites#create",
+                "collection": "app.bsky.feed.post",
+                "rkey": format!("3krec{:08}", i),
+                "value": {
+                    "$type": "app.bsky.feed.post",
+                    "text": format!("rec {i}"),
+                    "createdAt": now,
+                }
+            })
+        })
+        .collect();
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.applyWrites",
+            base_url().await
+        ))
+        .bearer_auth(&token)
+        .json(&json!({ "repo": did, "writes": writes }))
+        .send()
+        .await
+        .expect("applyWrites");
+    assert_eq!(res.status(), StatusCode::OK);
+
+    let frames = consumer
+        .wait_for_commits(&did, 1, Duration::from_secs(10))
+        .await;
+    let frame = frames.last().expect("frame");
+
+    let (_, block_map) = car_to_blocks(&frame.blocks).await;
+    for op in &frame.ops {
+        if op.action == RepoAction::Create {
+            let cid = op.cid.expect("create cid");
+            assert!(
+                block_map.contains_key(&cid),
+                "record CID {cid} for path {} missing from CAR",
+                op.path
+            );
+        }
+    }
+}
@@ -0,0 +1,348 @@
+mod common;
+mod mst_verify;
+
+use std::collections::HashMap;
+use std::str::FromStr;
+
+use cid::Cid;
+use common::*;
+use jacquard_common::smol_str::SmolStr;
+use jacquard_repo::commit::Commit;
+use jacquard_repo::mst::{Mst, VerifiedWriteOp};
+use jacquard_repo::storage::BlockStore;
+use mst_verify::{extract_event_blocks, inline_to_store};
+use rand::rngs::StdRng;
+use rand::{Rng, SeedableRng};
+use reqwest::StatusCode;
+use serde_json::{Value, json};
+use tranquil_db_traits::{RepoEventType, SequenceNumber, SequencedEvent};
+use tranquil_types::Did;
+
+const COLLECTIONS: &[&str] = &[
+    "app.bsky.feed.post",
+    "app.bsky.feed.like",
+    "app.bsky.graph.follow",
+    "app.bsky.feed.repost",
+];
+
+#[derive(Copy, Clone, Debug)]
+enum FuzzOp {
+    Create,
+    Update,
+    Delete,
+}
+
+fn pick_op(rng: &mut StdRng, have_keys: bool) -> FuzzOp {
+    match (have_keys, rng.gen_range(0..10)) {
+        (false, _) => FuzzOp::Create,
+        (_, 0..=5) => FuzzOp::Create,
+        (_, 6..=7) => FuzzOp::Update,
+        _ => FuzzOp::Delete,
+    }
+}
+
+fn random_rkey(rng: &mut StdRng) -> String {
+    let tid_char_pool = b"234567abcdefghijklmnopqrstuvwxyz";
+    let mut out = Vec::with_capacity(13);
+    (0..13).for_each(|_| {
+        let c = tid_char_pool[rng.gen_range(0..tid_char_pool.len())];
+        out.push(c);
+    });
+    String::from_utf8(out).unwrap()
+}
+
+fn random_collection(rng: &mut StdRng) -> &'static str {
+    COLLECTIONS[rng.gen_range(0..COLLECTIONS.len())]
+}
+
+fn record_for_collection(col: &str, text: &str, now: &str) -> Value {
+    match col {
+        "app.bsky.feed.post" | "app.bsky.feed.repost" | "app.bsky.feed.like" => json!({
+            "$type": col,
+            "text": text,
+            "createdAt": now,
+        }),
+        _ => json!({
+            "$type": col,
+            "subject": format!("did:plc:synthetic{text}"),
+            "createdAt": now,
+        }),
+    }
+}
+
+async fn verify_commit_forward_and_inverse(event: &SequencedEvent) -> Result<(), String> {
+    let prev_data = event
+        .prev_data_cid
+        .as_ref()
+        .and_then(|c| c.to_cid())
+        .ok_or("no prev_data_cid")?;
+    let commit_cid = event
+        .commit_cid
+        .as_ref()
+        .and_then(|c| c.to_cid())
+        .ok_or("no commit_cid")?;
+    let inline = extract_event_blocks(event)?;
+    let ops = event
+        .ops
+        .as_ref()
+        .and_then(|v| v.as_array())
+        .ok_or("ops not array")?;
+
+    let storage = inline_to_store(inline);
+    let commit_bytes = storage
+        .get(&commit_cid)
+        .await
+        .map_err(|e| format!("get commit: {e:?}"))?
+        .ok_or("missing commit block")?;
+    let commit = Commit::from_cbor(&commit_bytes).map_err(|e| format!("parse commit: {e:?}"))?;
+    let new_data = *commit.data();
+
+    let mut forward = Mst::load(storage.clone(), prev_data, None);
+    for op in ops {
+        let action = op["action"].as_str().ok_or("op.action")?;
+        let path = op["path"].as_str().ok_or("op.path")?;
+        match action {
+            "create" | "update" => {
+                let cid = Cid::from_str(op["cid"].as_str().ok_or("op.cid")?)
+                    .map_err(|e| format!("{e:?}"))?;
+                forward = forward
+                    .add(path, cid)
+                    .await
+                    .map_err(|e| format!("fwd add {path}: {e:?}"))?;
+            }
+            "delete" => {
+                forward = forward
+                    .delete(path)
+                    .await
+                    .map_err(|e| format!("fwd delete {path}: {e:?}"))?;
+            }
+            other => return Err(format!("unknown action {other}")),
+        }
+    }
+    let got = forward
+        .persist()
+        .await
+        .map_err(|e| format!("persist: {e:?}"))?;
+    if got != new_data {
+        return Err(format!("forward root mismatch exp={new_data} got={got}"));
+    }
+
+    let mut inverse = Mst::load(storage, new_data, None);
+    for op in ops {
+        let action = op["action"].as_str().ok_or("op.action")?;
+        let path = op["path"].as_str().ok_or("op.path")?;
+        let key = SmolStr::new(path);
+        let verified = match action {
+            "create" => {
+                let cid = Cid::from_str(op["cid"].as_str().ok_or("op.cid")?)
+                    .map_err(|e| format!("{e:?}"))?;
+                VerifiedWriteOp::Create { key, cid }
+            }
+            "update" => {
+                let cid = Cid::from_str(op["cid"].as_str().ok_or("op.cid")?)
+                    .map_err(|e| format!("{e:?}"))?;
+                let prev = Cid::from_str(op["prev"].as_str().ok_or("op.prev")?)
+                    .map_err(|e| format!("{e:?}"))?;
+                VerifiedWriteOp::Update { key, cid, prev }
+            }
+            "delete" => {
+                let prev = Cid::from_str(op["prev"].as_str().ok_or("op.prev")?)
+                    .map_err(|e| format!("{e:?}"))?;
+                VerifiedWriteOp::Delete { key, prev }
+            }
+            other => return Err(format!("unknown action {other}")),
+        };
+        let inverted = inverse
+            .invert_op(verified.clone())
+            .await
+            .map_err(|e| format!("invert {verified:?}: {e:?}"))?;
+        if !inverted {
+            return Err(format!("op not invertible: {verified:?}"));
+        }
+    }
+    let got_prev = inverse
+        .get_pointer()
+        .await
+        .map_err(|e| format!("get_pointer: {e:?}"))?;
+    if got_prev != prev_data {
+        return Err(format!(
+            "inverse root mismatch exp={prev_data} got={got_prev}"
+        ));
+    }
+    Ok(())
+}
+
+async fn fuzz_run_with_seed(seed: u64, steps: usize) -> Vec<String> {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+    let mut rng = StdRng::seed_from_u64(seed);
+
+    let mut live_keys: HashMap<String, String> = HashMap::new();
+
+    for step in 0..steps {
+        let now = chrono::Utc::now().to_rfc3339();
+        let op = pick_op(&mut rng, !live_keys.is_empty());
+        match op {
+            FuzzOp::Create => {
+                let col = random_collection(&mut rng);
+                let rkey = random_rkey(&mut rng);
+                let path = format!("{col}/{rkey}");
+                if live_keys.contains_key(&path) {
+                    continue;
+                }
+                let record = record_for_collection(col, &format!("s{seed}-n{step}"), &now);
+                let res = client
+                    .post(format!(
+                        "{}/xrpc/com.atproto.repo.createRecord",
+                        base_url().await
+                    ))
+                    .bearer_auth(&token)
+                    .json(&json!({
+                        "repo": did,
+                        "collection": col,
+                        "rkey": rkey,
+                        "record": record,
+                    }))
+                    .send()
+                    .await
+                    .expect("createRecord");
+                if res.status() == StatusCode::OK {
+                    live_keys.insert(path, col.to_string());
+                }
+            }
+            FuzzOp::Update => {
+                let keys: Vec<&String> = live_keys.keys().collect();
+                if keys.is_empty() {
+                    continue;
+                }
+                let path = keys[rng.gen_range(0..keys.len())].clone();
+                let col = live_keys.get(&path).unwrap().clone();
+                let rkey = path.split('/').nth(1).unwrap().to_string();
+                let record = record_for_collection(&col, &format!("s{seed}-u{step}"), &now);
+                let res = client
+                    .post(format!(
+                        "{}/xrpc/com.atproto.repo.putRecord",
+                        base_url().await
+                    ))
+                    .bearer_auth(&token)
+                    .json(&json!({
+                        "repo": did,
+                        "collection": col,
+                        "rkey": rkey,
+                        "record": record,
+                    }))
+                    .send()
+                    .await
+                    .expect("putRecord");
+                assert_eq!(res.status(), StatusCode::OK, "putRecord failed");
+            }
+            FuzzOp::Delete => {
+                let keys: Vec<String> = live_keys.keys().cloned().collect();
+                if keys.is_empty() {
+                    continue;
+                }
+                let path = keys[rng.gen_range(0..keys.len())].clone();
+                let col = live_keys.get(&path).unwrap().clone();
+                let rkey = path.split('/').nth(1).unwrap().to_string();
+                let res = client
+                    .post(format!(
+                        "{}/xrpc/com.atproto.repo.deleteRecord",
+                        base_url().await
+                    ))
+                    .bearer_auth(&token)
+                    .json(&json!({
+                        "repo": did,
+                        "collection": col,
+                        "rkey": rkey,
+                    }))
+                    .send()
+                    .await
+                    .expect("deleteRecord");
+                if res.status() == StatusCode::OK {
+                    live_keys.remove(&path);
+                }
+            }
+        }
+    }
+
+    let repos = get_test_repos().await;
+    let typed_did = Did::new(did.clone()).unwrap();
+    let events = repos
+        .repo
+        .get_events_since_seq(SequenceNumber::ZERO, None)
+        .await
+        .expect("get_events_since_seq");
+
+    let our: Vec<SequencedEvent> = events
+        .into_iter()
+        .filter(|e| {
+            e.did == typed_did
+                && e.event_type == RepoEventType::Commit
+                && e.prev_data_cid.is_some()
+                && e.ops
+                    .as_ref()
+                    .and_then(|v| v.as_array())
+                    .is_some_and(|a| !a.is_empty())
+        })
+        .collect();
+
+    let mut failures = Vec::new();
+    for event in &our {
+        if let Err(msg) = verify_commit_forward_and_inverse(event).await {
+            failures.push(format!(
+                "seed={seed} seq={} ops={:?}: {msg}",
+                event.seq.as_i64(),
+                event
+                    .ops
+                    .as_ref()
+                    .and_then(|v| v.as_array())
+                    .map(|a| a.len())
+            ));
+        }
+    }
+    failures
+}
+
+#[tokio::test]
+async fn mst_property_fuzz_seed_1() {
+    let failures = fuzz_run_with_seed(1, 150).await;
+    assert!(
+        failures.is_empty(),
+        "fuzz seed=1 found {} invalid commits:\n  - {}",
+        failures.len(),
+        failures.join("\n  - ")
+    );
+}
+
+#[tokio::test]
+async fn mst_property_fuzz_seed_42() {
+    let failures = fuzz_run_with_seed(42, 150).await;
+    assert!(
+        failures.is_empty(),
+        "fuzz seed=42 found {} invalid commits:\n  - {}",
+        failures.len(),
+        failures.join("\n  - ")
+    );
+}
+
+#[tokio::test]
+async fn mst_property_fuzz_seed_9001() {
+    let failures = fuzz_run_with_seed(9001, 150).await;
+    assert!(
+        failures.is_empty(),
+        "fuzz seed=9001 found {} invalid commits:\n  - {}",
+        failures.len(),
+        failures.join("\n  - ")
+    );
+}
+
+#[tokio::test]
+async fn mst_property_fuzz_deep_tree_seed_7() {
+    let failures = fuzz_run_with_seed(7, 400).await;
+    assert!(
+        failures.is_empty(),
+        "fuzz deep seed=7 found {} invalid commits:\n  - {}",
+        failures.len(),
+        failures.join("\n  - ")
+    );
+}
@@ -0,0 +1,26 @@
+use std::collections::BTreeMap;
+use std::sync::Arc;
+
+use bytes::Bytes;
+use cid::Cid;
+use jacquard_repo::storage::MemoryBlockStore;
+use tranquil_db_traits::{EventBlockInline, EventBlocks, SequencedEvent};
+
+pub fn extract_event_blocks(event: &SequencedEvent) -> Result<&[EventBlockInline], String> {
+    match event.blocks.as_ref() {
+        Some(EventBlocks::Inline(v)) => Ok(v.as_slice()),
+        Some(EventBlocks::LegacyCids(_)) => Err("legacy cids, not inline".into()),
+        None => Err("event missing blocks".into()),
+    }
+}
+
+pub fn inline_to_store(inline: &[EventBlockInline]) -> Arc<MemoryBlockStore> {
+    let map: BTreeMap<Cid, Bytes> = inline
+        .iter()
+        .map(|b| {
+            let cid = Cid::read_bytes(b.cid_bytes.as_slice()).expect("valid cid bytes");
+            (cid, Bytes::from(b.data.clone()))
+        })
+        .collect();
+    Arc::new(MemoryBlockStore::new_from_blocks(map))
+}
@@ -187,6 +187,37 @@ async fn test_par_and_authorize() {
    );
 }

+#[tokio::test]
+async fn test_par_public_client_empty_assertion_fields() {
+    let url = base_url().await;
+    let client = client();
+    let redirect_uri = "https://nels.evil.oauth.pet/callback";
+    let mock_client = setup_mock_client_metadata(redirect_uri).await;
+    let client_id = mock_client.uri();
+    let (_, code_challenge) = generate_pkce();
+    let par_res = client
+        .post(format!("{}/oauth/par", url))
+        .form(&[
+            ("response_type", "code"),
+            ("client_id", &client_id),
+            ("redirect_uri", redirect_uri),
+            ("code_challenge", &code_challenge),
+            ("code_challenge_method", "S256"),
+            ("scope", "atproto"),
+            ("state", "test-state"),
+            ("client_assertion", ""),
+            ("client_assertion_type", ""),
+        ])
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(
+        par_res.status(),
+        StatusCode::CREATED,
+        "PAR with empty assertion fields from a public client should succeed"
+    );
+}
+
 #[tokio::test]
 async fn test_full_oauth_flow() {
    let url = base_url().await;
@@ -88,12 +88,17 @@ fn test_signal_username_validation() {
    assert!(is_valid_signal_username("bob_smith.99"));
    assert!(is_valid_signal_username("user123.42"));
    assert!(is_valid_signal_username("lu1.01"));
-    assert!(is_valid_signal_username("abc.00"));
    assert!(is_valid_signal_username("a_very_long_username_here.55"));
+    assert!(is_valid_signal_username("alice.123"));
+    assert!(is_valid_signal_username("alice.999999999"));
+    assert!(is_valid_signal_username("alice.18446744073709551615"));

    assert!(!is_valid_signal_username("alice"));
    assert!(!is_valid_signal_username("alice.1"));
    assert!(!is_valid_signal_username("alice.001"));
+    assert!(!is_valid_signal_username("abc.00"));
+    assert!(!is_valid_signal_username("alice.0"));
+    assert!(!is_valid_signal_username("alice.999999999999999999999"));
    assert!(!is_valid_signal_username(".01"));
    assert!(!is_valid_signal_username("ab.01"));
    assert!(!is_valid_signal_username(""));
@@ -164,7 +164,10 @@ impl ScopePermissions {
            if self.has_transition_generic && !self.has_transition_chat {
                return Err(ScopeError::InsufficientScope {
                    required: "transition:chat.bsky".to_string(),
-                    message: format!("Chat access requires transition:chat.bsky scope to call {}", lxm),
+                    message: format!(
+                        "Chat access requires transition:chat.bsky scope to call {}",
+                        lxm
+                    ),
                });
            }
        }
@@ -33,15 +33,11 @@ impl SignalUsername {
    pub fn parse(username: &str) -> Result<Self, InvalidSignalUsername> {
        let reject = || Err(InvalidSignalUsername(username.to_string()));

-        if username.len() < 6 || username.len() > 35 {
-            return reject();
-        }
-
        let Some((base, discriminator)) = username.rsplit_once('.') else {
            return reject();
        };

-        if base.len() < 3 || base.len() > 32 {
+        if !matches!(base.len(), 3..=32) {
            return reject();
        }

@@ -53,7 +49,7 @@ impl SignalUsername {
            return reject();
        }

-        if discriminator.len() != 2 || !discriminator.chars().all(|c| c.is_ascii_digit()) {
+        if !is_valid_discriminator(discriminator) {
            return reject();
        }

@@ -65,6 +61,19 @@ impl SignalUsername {
    }
 }

+fn is_valid_discriminator(s: &str) -> bool {
+    if !s.chars().all(|c| c.is_ascii_digit()) {
+        return false;
+    }
+    if !matches!(s.len(), 2..=20) {
+        return false;
+    }
+    if s.len() > 2 && s.starts_with('0') {
+        return false;
+    }
+    s.parse::<u64>().is_ok_and(|n| n != 0)
+}
+
 impl fmt::Display for SignalUsername {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.write_str(&self.0)
@@ -113,11 +122,7 @@ pub struct MessageTooLong {

 impl fmt::Display for MessageTooLong {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(
-            f,
-            "message body too long: {} bytes (max {})",
-            self.len, self.max
-        )
+        write!(f, "message body is {} bytes, max {}", self.len, self.max)
    }
 }

@@ -368,13 +373,13 @@ impl SignalClient {
            let req = tokio::select! {
                biased;
                _ = shutdown.cancelled() => {
-                    tracing::info!("signal worker shutting down (cancellation)");
+                    tracing::info!("signal worker cancelled, shutting down");
                    break;
                }
                msg = rx.recv() => match msg {
                    Some(r) => r,
                    None => {
-                        tracing::info!("signal worker shutting down (channel closed)");
+                        tracing::info!("signal worker channel closed, shutting down");
                        break;
                    }
                },
@@ -1050,7 +1050,7 @@ impl ContentsStore for FjallSignalStore {
            .and_then(|v| match <[u8; 32]>::try_from(v.as_ref()) {
                Ok(arr) => Some(ProfileKey { bytes: arr }),
                Err(_) => {
-                    warn!(%uuid, len = v.len(), "corrupted profile key (expected 32 bytes)");
+                    warn!(%uuid, len = v.len(), "corrupted profile key, expected 32 bytes");
                    None
                }
            }))
@@ -1117,7 +1117,7 @@ impl ContentsStore for PgSignalStore {
            Some(r) => match <[u8; 32]>::try_from(r.key.as_slice()) {
                Ok(arr) => Some(ProfileKey { bytes: arr }),
                Err(_) => {
-                    warn!(%uuid, len = r.key.len(), "corrupted profile key (expected 32 bytes)");
+                    warn!(%uuid, len = r.key.len(), "corrupted profile key, expected 32 bytes");
                    None
                }
            },
@@ -14,7 +14,7 @@ parking_lot = { workspace = true }
 fjall = "3"
 lsm-tree = "3"
 flume = "0.11"
-tokio = { workspace = true, features = ["sync", "rt"] }
+tokio = { workspace = true, features = ["sync", "rt", "time"] }
 bytes = "1"
 memmap2 = "0.9"
 tracing = { workspace = true }
@@ -34,9 +34,21 @@ dashmap = "6"
 rayon = "1"
 smallvec = "1"
 uuid = { workspace = true }
+tempfile = { version = "3", optional = true }
+clap = { workspace = true, optional = true }
+toml = { version = "0.8", optional = true }
+tracing-subscriber = { workspace = true, features = ["env-filter"], optional = true }
+libc = "0.2"

 [features]
-test-harness = []
+test-harness = ["dep:tempfile"]
+gauntlet-cli = ["test-harness", "dep:clap", "dep:toml", "dep:tracing-subscriber"]
+gauntlet-jemalloc-prof = []
+
+[[bin]]
+name = "tranquil-gauntlet"
+path = "src/bin/tranquil_gauntlet.rs"
+required-features = ["gauntlet-cli"]

 [dev-dependencies]
 tranquil-store = { path = ".", features = ["test-harness"] }
@@ -50,9 +62,8 @@ tranquil-db = { workspace = true }
 sqlx = { workspace = true }
 k256 = { workspace = true }
 rand = { workspace = true }
-tikv-jemallocator = "0.6"
+tikv-jemallocator = { version = "0.6", features = ["profiling", "unprefixed_malloc_on_supported_platforms"] }
 tracing-subscriber = { workspace = true, features = ["env-filter"] }
-libc = "0.2"

 [[bench]]
 name = "blockstore"
@@ -0,0 +1,4 @@
+target/
+artifacts/
+coverage/
+Cargo.lock
@@ -0,0 +1,52 @@
+[package]
+name = "tranquil-store-fuzz"
+version = "0.0.0"
+publish = false
+edition = "2024"
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+libfuzzer-sys = "0.4"
+arbitrary = { version = "1", features = ["derive"] }
+tranquil-store = { path = "..", features = ["test-harness"] }
+tempfile = "3"
+tokio = { version = "1", features = ["rt", "time", "macros", "sync"] }
+
+[[bin]]
+name = "decode_block_record"
+path = "fuzz_targets/decode_block_record.rs"
+test = false
+doc = false
+bench = false
+
+[[bin]]
+name = "decode_hint_record"
+path = "fuzz_targets/decode_hint_record.rs"
+test = false
+doc = false
+bench = false
+
+[[bin]]
+name = "segment_scan"
+path = "fuzz_targets/segment_scan.rs"
+test = false
+doc = false
+bench = false
+
+[[bin]]
+name = "metastore_key_codec"
+path = "fuzz_targets/metastore_key_codec.rs"
+test = false
+doc = false
+bench = false
+
+[[bin]]
+name = "gauntlet_micro"
+path = "fuzz_targets/gauntlet_micro.rs"
+test = false
+doc = false
+bench = false
+
+[workspace]
@@ -0,0 +1 @@
+
@@ -0,0 +1 @@
+BADX
@@ -0,0 +1 @@
+TQEV
@@ -0,0 +1,28 @@
+#![no_main]
+
+use std::path::Path;
+
+use libfuzzer_sys::fuzz_target;
+use tranquil_store::blockstore::{BlockOffset, decode_block_record};
+use tranquil_store::{FaultConfig, OpenOptions, SimulatedIO, StorageIO};
+
+fuzz_target!(|data: &[u8]| {
+    let sim = SimulatedIO::new(0, FaultConfig::none());
+    let opts = OpenOptions {
+        read: true,
+        write: true,
+        create: true,
+        truncate: false,
+    };
+    let fd = match sim.open(Path::new("/fuzz/block.tqb"), opts) {
+        Ok(fd) => fd,
+        Err(_) => return,
+    };
+    if !data.is_empty() {
+        let _ = sim.write_all_at(fd, 0, data);
+        let _ = sim.sync(fd);
+    }
+    let file_size = data.len() as u64;
+    let _ = decode_block_record(&sim, fd, BlockOffset::new(0), file_size);
+    let _ = sim.close(fd);
+});
@@ -0,0 +1,41 @@
+#![no_main]
+
+use std::path::Path;
+
+use libfuzzer_sys::fuzz_target;
+use tranquil_store::blockstore::{HintOffset, decode_hint_record};
+use tranquil_store::{FaultConfig, OpenOptions, SimulatedIO, StorageIO};
+
+fuzz_target!(|data: &[u8]| {
+    let sim = SimulatedIO::new(0, FaultConfig::none());
+    let opts = OpenOptions {
+        read: true,
+        write: true,
+        create: true,
+        truncate: false,
+    };
+    let fd = match sim.open(Path::new("/fuzz/hint.tqh"), opts) {
+        Ok(fd) => fd,
+        Err(_) => return,
+    };
+    if !data.is_empty() {
+        let _ = sim.write_all_at(fd, 0, data);
+        let _ = sim.sync(fd);
+    }
+    let file_size = data.len() as u64;
+    let cursor = std::cell::Cell::new(0u64);
+    std::iter::from_fn(|| {
+        if cursor.get() >= file_size {
+            return None;
+        }
+        match decode_hint_record(&sim, fd, HintOffset::new(cursor.get()), file_size) {
+            Ok(Some(_)) => {
+                cursor.set(cursor.get() + 64);
+                Some(())
+            }
+            _ => None,
+        }
+    })
+    .for_each(|()| {});
+    let _ = sim.close(fd);
+});
@@ -0,0 +1,109 @@
+#![no_main]
+
+use std::sync::OnceLock;
+
+use arbitrary::{Arbitrary, Unstructured};
+use libfuzzer_sys::fuzz_target;
+use tokio::runtime::Runtime;
+use tranquil_store::blockstore::GroupCommitConfig;
+use tranquil_store::gauntlet::{
+    CollectionName, DidSpaceSize, Gauntlet, GauntletConfig, InvariantSet, IoBackend, KeySpaceSize,
+    MaxFileSize, Op, OpCount, OpInterval, OpStream, OpWeights, RecordKey, RestartPolicy,
+    RetentionMaxSecs, RunLimits, Seed, ShardCount, SizeDistribution, StoreConfig, ValueBytes,
+    ValueSeed, WallMs, WorkloadModel, WriterConcurrency,
+};
+
+#[derive(Arbitrary, Debug)]
+enum FuzzOp {
+    Add { rkey: u8, value: u16 },
+    Delete { rkey: u8 },
+    Compact,
+    Checkpoint,
+    Read { rkey: u8 },
+    ReadBlock { value: u16 },
+}
+
+const COLLECTION: &str = "app.bsky.feed.post";
+const MAX_OPS: usize = 128;
+
+fn to_op(fuzz_op: FuzzOp) -> Op {
+    match fuzz_op {
+        FuzzOp::Add { rkey, value } => Op::AddRecord {
+            collection: CollectionName(COLLECTION.to_string()),
+            rkey: RecordKey(format!("k{rkey:03}")),
+            value_seed: ValueSeed(u32::from(value)),
+        },
+        FuzzOp::Delete { rkey } => Op::DeleteRecord {
+            collection: CollectionName(COLLECTION.to_string()),
+            rkey: RecordKey(format!("k{rkey:03}")),
+        },
+        FuzzOp::Compact => Op::Compact,
+        FuzzOp::Checkpoint => Op::Checkpoint,
+        FuzzOp::Read { rkey } => Op::ReadRecord {
+            collection: CollectionName(COLLECTION.to_string()),
+            rkey: RecordKey(format!("k{rkey:03}")),
+        },
+        FuzzOp::ReadBlock { value } => Op::ReadBlock {
+            value_seed: ValueSeed(u32::from(value)),
+        },
+    }
+}
+
+fn tiny_config() -> GauntletConfig {
+    GauntletConfig {
+        seed: Seed(0),
+        io: IoBackend::Real,
+        workload: WorkloadModel {
+            weights: OpWeights::default(),
+            size_distribution: SizeDistribution::Fixed(ValueBytes(64)),
+            collections: vec![CollectionName(COLLECTION.to_string())],
+            key_space: KeySpaceSize(256),
+            did_space: DidSpaceSize(8),
+            retention_max_secs: RetentionMaxSecs(3600),
+        },
+        op_count: OpCount(0),
+        invariants: InvariantSet::REFCOUNT_CONSERVATION
+            | InvariantSet::REACHABILITY
+            | InvariantSet::READ_AFTER_WRITE,
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(2_000)),
+        },
+        restart_policy: RestartPolicy::EveryNOps(OpInterval(32)),
+        store: StoreConfig {
+            max_file_size: MaxFileSize(4096),
+            group_commit: GroupCommitConfig::default(),
+            shard_count: ShardCount(1),
+        },
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn shared_runtime() -> &'static Runtime {
+    static RUNTIME: OnceLock<Runtime> = OnceLock::new();
+    RUNTIME.get_or_init(|| {
+        tokio::runtime::Builder::new_current_thread()
+            .enable_time()
+            .build()
+            .expect("build tokio runtime")
+    })
+}
+
+fuzz_target!(|data: &[u8]| {
+    if data.is_empty() {
+        return;
+    }
+    let mut u = Unstructured::new(data);
+    let ops: Vec<FuzzOp> = match Vec::<FuzzOp>::arbitrary(&mut u) {
+        Ok(ops) => ops.into_iter().take(MAX_OPS).collect(),
+        Err(_) => return,
+    };
+    if ops.is_empty() {
+        return;
+    }
+    let stream = OpStream::from_vec(ops.into_iter().map(to_op).collect());
+
+    let cfg = tiny_config();
+    let gauntlet = Gauntlet::new(cfg).expect("build gauntlet");
+    let _ = shared_runtime().block_on(gauntlet.run_with_ops(stream));
+});
@@ -0,0 +1,77 @@
+#![no_main]
+
+use arbitrary::Arbitrary;
+use libfuzzer_sys::fuzz_target;
+use tranquil_store::metastore::encoding::{KeyBuilder, KeyReader};
+
+#[derive(Arbitrary, Debug, PartialEq, Eq)]
+enum Field {
+    U64(u64),
+    I64(i64),
+    U32(u32),
+    U16(u16),
+    Bool(bool),
+    Bytes(Vec<u8>),
+    String(String),
+}
+
+fn append(builder: KeyBuilder, field: &Field) -> KeyBuilder {
+    match field {
+        Field::U64(v) => builder.u64(*v),
+        Field::I64(v) => builder.i64(*v),
+        Field::U32(v) => builder.u32(*v),
+        Field::U16(v) => builder.u16(*v),
+        Field::Bool(v) => builder.bool(*v),
+        Field::Bytes(v) => builder.bytes(v),
+        Field::String(v) => builder.string(v),
+    }
+}
+
+fn consume(reader: &mut KeyReader<'_>, field: &Field) -> bool {
+    match field {
+        Field::U64(v) => reader.u64() == Some(*v),
+        Field::I64(v) => reader.i64() == Some(*v),
+        Field::U32(v) => reader.u32() == Some(*v),
+        Field::U16(v) => reader.u16() == Some(*v),
+        Field::Bool(v) => reader.bool() == Some(*v),
+        Field::Bytes(v) => reader.bytes().as_deref() == Some(v.as_slice()),
+        Field::String(v) => reader.string().as_deref() == Some(v.as_str()),
+    }
+}
+
+#[derive(Arbitrary, Debug)]
+enum Mode {
+    Roundtrip(Vec<Field>),
+    Raw(Vec<u8>),
+}
+
+fuzz_target!(|mode: Mode| {
+    match mode {
+        Mode::Roundtrip(fields) => {
+            let encoded1 = fields.iter().fold(KeyBuilder::new(), append).build();
+
+            let mut reader = KeyReader::new(encoded1.as_slice());
+            let all_match = fields.iter().all(|f| consume(&mut reader, f));
+            assert!(all_match, "roundtrip decode failed");
+            assert!(reader.is_empty(), "trailing bytes after decode");
+
+            let encoded2 = fields.iter().fold(KeyBuilder::new(), append).build();
+            assert_eq!(
+                encoded1.as_slice(),
+                encoded2.as_slice(),
+                "encoding not deterministic",
+            );
+        }
+        Mode::Raw(data) => {
+            let mut reader = KeyReader::new(&data);
+            let _ = reader.u64();
+            let _ = reader.i64();
+            let _ = reader.u32();
+            let _ = reader.u16();
+            let _ = reader.bool();
+            let _ = reader.bytes();
+            let _ = reader.string();
+            let _ = reader.tag();
+        }
+    }
+});
@@ -0,0 +1,31 @@
+#![no_main]
+
+use std::path::Path;
+
+use libfuzzer_sys::fuzz_target;
+use tranquil_store::eventlog::SegmentReader;
+
+const FUZZ_MAX_PAYLOAD: u32 = 1 << 20;
+use tranquil_store::{FaultConfig, OpenOptions, SimulatedIO, StorageIO};
+
+fuzz_target!(|data: &[u8]| {
+    let sim = SimulatedIO::new(0, FaultConfig::none());
+    let opts = OpenOptions {
+        read: true,
+        write: true,
+        create: true,
+        truncate: false,
+    };
+    let fd = match sim.open(Path::new("/fuzz/segment.tqe"), opts) {
+        Ok(fd) => fd,
+        Err(_) => return,
+    };
+    if !data.is_empty() {
+        let _ = sim.write_all_at(fd, 0, data);
+        let _ = sim.sync(fd);
+    }
+    if let Ok(reader) = SegmentReader::open(&sim, fd, FUZZ_MAX_PAYLOAD) {
+        reader.for_each(|_result| {});
+    }
+    let _ = sim.close(fd);
+});
@@ -0,0 +1,47 @@
+use cid::Cid;
+use multihash::Multihash;
+use sha2::{Digest, Sha256};
+
+use super::data_file::CID_SIZE;
+
+pub const DAG_CBOR_CODEC: u64 = 0x71;
+pub const SHA2_256_CODE: u64 = 0x12;
+
+pub fn hash_to_cid(data: &[u8]) -> Cid {
+    let mut hasher = Sha256::new();
+    hasher.update(data);
+    let digest = hasher.finalize();
+    let mh = Multihash::wrap(SHA2_256_CODE, &digest)
+        .expect("SHA-256 digest is 32 bytes, well within multihash capacity");
+    Cid::new_v1(DAG_CBOR_CODEC, mh)
+}
+
+pub fn hash_to_cid_bytes(data: &[u8]) -> [u8; CID_SIZE] {
+    let raw = hash_to_cid(data).to_bytes();
+    raw.try_into()
+        .expect("CIDv1 + DAG-CBOR + SHA-256 always encodes to CID_SIZE bytes")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn hash_to_cid_bytes_is_deterministic() {
+        let a = hash_to_cid_bytes(b"hello");
+        let b = hash_to_cid_bytes(b"hello");
+        assert_eq!(a, b);
+    }
+
+    #[test]
+    fn hash_to_cid_bytes_diverges_on_single_byte_change() {
+        assert_ne!(hash_to_cid_bytes(b"abc"), hash_to_cid_bytes(b"abd"));
+    }
+
+    #[test]
+    fn hash_to_cid_and_bytes_agree() {
+        let cid = hash_to_cid(b"payload");
+        let raw: [u8; CID_SIZE] = cid.to_bytes().try_into().expect("36 bytes");
+        assert_eq!(raw, hash_to_cid_bytes(b"payload"));
+    }
+}
@@ -68,8 +68,8 @@ pub(super) fn compact_on_writer_thread<S: StorageIO>(
        return Err(CompactionError::ActiveFileCannotBeCompacted);
    }

-    let source_fd = manager.open_for_read(source_file_id)?;
-    let source_size = manager.io().file_size(source_fd)?;
+    let source_handle = manager.open_for_read(source_file_id)?;
+    let source_size = manager.io().file_size(source_handle.fd())?;

    let new_file_id = file_ids.allocate();

@@ -77,7 +77,7 @@ pub(super) fn compact_on_writer_thread<S: StorageIO>(
        manager,
        index,
        source_file_id,
-        source_fd,
+        source_handle.fd(),
        new_file_id,
        current_epoch,
        grace_period_ms,
@@ -102,6 +102,13 @@ pub(super) fn compact_on_writer_thread<S: StorageIO>(
                .io()
                .delete(&hint_file_path(manager.data_dir(), source_file_id))
                .ok();
+            if live_count == 0 {
+                manager.delete_data_file(new_file_id).ok();
+                manager
+                    .io()
+                    .delete(&hint_file_path(manager.data_dir(), new_file_id))
+                    .ok();
+            }
            manager.io().sync_dir(manager.data_dir())?;

            let reclaimed_bytes = source_size.saturating_sub(new_size);
@@ -141,8 +148,8 @@ fn stream_compact<S: StorageIO>(
    let mut reader = DataFileReader::open(manager.io(), source_fd)?;
    let now = crate::wall_clock_ms();

-    let new_fd = manager.open_for_append(new_file_id)?;
-    let mut writer = DataFileWriter::new(manager.io(), new_fd, new_file_id)?;
+    let new_handle = manager.open_for_append(new_file_id)?;
+    let mut writer = DataFileWriter::new(manager.io(), new_handle.fd(), new_file_id)?;

    let hint_path = hint_file_path(manager.data_dir(), new_file_id);
    let hint_fd = manager.io().open(&hint_path, OpenOptions::read_write())?;
@@ -153,7 +160,7 @@ fn stream_compact<S: StorageIO>(
    let mut live_count: u64 = 0;
    let mut dead_count: u64 = 0;

-    reader.try_for_each(|r| {
+    let scan_result = reader.try_for_each(|r| {
        let record = r?;
        match record {
            ReadBlockRecord::Valid {
@@ -192,11 +199,35 @@ fn stream_compact<S: StorageIO>(
            ReadBlockRecord::Corrupted { .. } | ReadBlockRecord::Truncated { .. } => {}
        }
        Ok::<_, CompactionError>(())
-    })?;
+    });

-    writer.sync()?;
-    hint_writer.sync()?;
-    manager.io().sync_dir(manager.data_dir())?;
+    let record_count =
+        u32::try_from((live_count as u128).saturating_add(dead_count as u128)).unwrap_or(u32::MAX);
+    let writer_position = writer.position();
+    let finalize_result = scan_result
+        .and_then(|()| writer.sync().map_err(CompactionError::from))
+        .and_then(|()| {
+            hint_writer
+                .append_commit_marker(
+                    current_epoch.raw(),
+                    record_count,
+                    new_file_id,
+                    writer_position,
+                )
+                .map_err(CompactionError::from)
+        })
+        .and_then(|()| hint_writer.sync().map_err(CompactionError::from))
+        .and_then(|()| {
+            manager
+                .io()
+                .sync_dir(manager.data_dir())
+                .map_err(CompactionError::from)
+        })
+        .and_then(|()| manager.io().barrier().map_err(CompactionError::from));
+
+    let _ = manager.io().close(hint_fd);
+
+    finalize_result?;

    let new_size = writer.position().raw();

@@ -11,7 +11,7 @@ use crate::fsync_order::PostBlockstoreHook;
 use super::BlocksSynced;
 use crate::io::{FileId, OpenOptions, StorageIO};

-use super::data_file::{CID_SIZE, DataFileWriter};
+use super::data_file::{CID_SIZE, DataFileWriter, ReadBlockRecord, decode_block_record};
 use super::hash_index::{BlockIndex, BlockIndexError, CheckpointPositions};
 use super::hint::{HintFileWriter, hint_file_path};
 use super::manager::DataFileManager;
@@ -106,6 +106,10 @@ pub enum CommitError {
    Io(Arc<io::Error>),
    Index(String),
    ChannelClosed,
+    VerifyFailed {
+        file_id: DataFileId,
+        offset: BlockOffset,
+    },
 }

 impl std::fmt::Display for CommitError {
@@ -114,6 +118,11 @@ impl std::fmt::Display for CommitError {
            Self::Io(e) => write!(f, "io: {}", e.as_ref()),
            Self::Index(e) => write!(f, "index: {e}"),
            Self::ChannelClosed => write!(f, "commit channel closed"),
+            Self::VerifyFailed { file_id, offset } => write!(
+                f,
+                "post-sync verify failed at {file_id}:{} (misdirected write or durable corruption)",
+                offset.raw()
+            ),
        }
    }
 }
@@ -122,7 +131,7 @@ impl std::error::Error for CommitError {
    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
        match self {
            Self::Io(e) => Some(e.as_ref()),
-            Self::Index(_) | Self::ChannelClosed => None,
+            Self::Index(_) | Self::ChannelClosed | Self::VerifyFailed { .. } => None,
        }
    }
 }
@@ -181,6 +190,7 @@ pub struct GroupCommitConfig {
    pub channel_capacity: usize,
    pub checkpoint_interval_ms: u64,
    pub checkpoint_write_threshold: u64,
+    pub verify_persisted_blocks: bool,
 }

 impl Default for GroupCommitConfig {
@@ -190,6 +200,7 @@ impl Default for GroupCommitConfig {
            channel_capacity: 4096,
            checkpoint_interval_ms: 60_000,
            checkpoint_write_threshold: 100_000,
+            verify_persisted_blocks: false,
        }
    }
 }
@@ -200,6 +211,7 @@ struct ShardContext {
    file_ids: Arc<FileIdAllocator>,
    active_files: Arc<ActiveFileSet>,
    hint_positions: Arc<ShardHintPositions>,
+    verify_persisted_blocks: bool,
 }

 struct ActiveState {
@@ -403,6 +415,7 @@ impl GroupCommitWriter {
                    file_ids: Arc::clone(&file_ids),
                    active_files: Arc::clone(&active_files),
                    hint_positions: Arc::clone(&hint_positions),
+                    verify_persisted_blocks: config.verify_persisted_blocks,
                };
                SingleShardWriter::spawn(
                    ctx,
@@ -540,7 +553,8 @@ fn initialize_active_state<S: StorageIO>(

    match cursor {
        Some(wc) => {
-            let fd = manager.open_for_append(wc.file_id)?;
+            let handle = manager.open_for_append(wc.file_id)?;
+            let fd = handle.fd();
            let file_size = manager.io().file_size(fd)?;

            if file_size < wc.offset.raw() {
@@ -550,6 +564,16 @@ fn initialize_active_state<S: StorageIO>(
                )));
            }

+            let header_end = super::data_file::BLOCK_HEADER_SIZE as u64;
+            let position = match file_size < header_end {
+                true => {
+                    let writer = DataFileWriter::new(manager.io(), fd, wc.file_id)?;
+                    writer.sync()?;
+                    writer.position()
+                }
+                false => BlockOffset::new(file_size),
+            };
+
            let hint_path = hint_file_path(data_dir, wc.file_id);
            let hint_fd = manager.io().open(&hint_path, OpenOptions::read_write())?;
            let hint_size = manager.io().file_size(hint_fd)?;
@@ -557,7 +581,7 @@ fn initialize_active_state<S: StorageIO>(
            Ok(ActiveState {
                file_id: wc.file_id,
                fd,
-                position: BlockOffset::new(file_size),
+                position,
                hint_fd,
                hint_position: HintOffset::new(hint_size),
            })
@@ -565,7 +589,8 @@ fn initialize_active_state<S: StorageIO>(
        None => {
            let file_id = file_ids.allocate();

-            let fd = manager.open_for_append(file_id)?;
+            let handle = manager.open_for_append(file_id)?;
+            let fd = handle.fd();
            let writer = DataFileWriter::new(manager.io(), fd, file_id)?;
            writer.sync()?;
            let position = writer.position();
@@ -1051,9 +1076,124 @@ fn drain_and_process_remaining<S: StorageIO>(
    shutdown_checkpoint(index, epoch, &ctx.hint_positions);
 }

-struct RotationState {
+struct RotationState<S: StorageIO> {
    file_id: DataFileId,
+    handle: Arc<super::manager::CachedHandle<S>>,
+    hint_fd: FileId,
+}
+
+fn verify_persisted_blocks<S: StorageIO>(
+    manager: &DataFileManager<S>,
+    entries: &[([u8; CID_SIZE], BlockLocation)],
+) -> Result<(), CommitError> {
+    use std::collections::BTreeMap;
+    let by_file: BTreeMap<DataFileId, Vec<(&[u8; CID_SIZE], BlockLocation)>> =
+        entries.iter().fold(BTreeMap::new(), |mut acc, (cid, loc)| {
+            acc.entry(loc.file_id).or_default().push((cid, *loc));
+            acc
+        });
+
+    by_file.into_iter().try_for_each(|(file_id, locations)| {
+        let path = manager.data_file_path(file_id);
+        let fd = match manager.io().open(&path, OpenOptions::read_only_existing()) {
+            Ok(fd) => fd,
+            Err(_) => return Ok(()),
+        };
+        let file_size = match manager.io().file_size(fd) {
+            Ok(s) => s,
+            Err(_) => {
+                let _ = manager.io().close(fd);
+                return Ok(());
+            }
+        };
+        let result = locations.into_iter().try_for_each(|(expected_cid, loc)| {
+            verify_block_at(manager, fd, file_size, expected_cid, loc)
+        });
+        let _ = manager.io().close(fd);
+        result
+    })
+}
+
+#[derive(Debug)]
+enum VerifyOutcome {
+    NoFaultDetected,
+    Faulted,
+}
+
+fn verify_block_at<S: StorageIO>(
+    manager: &DataFileManager<S>,
    fd: FileId,
+    file_size: u64,
+    expected_cid: &[u8; CID_SIZE],
+    loc: BlockLocation,
+) -> Result<(), CommitError> {
+    let passed = (0..VERIFY_RETRY_ATTEMPTS).any(|_| {
+        matches!(
+            verify_once(manager, fd, file_size, expected_cid, loc),
+            VerifyOutcome::NoFaultDetected
+        )
+    });
+    match passed {
+        true => Ok(()),
+        false => Err(CommitError::VerifyFailed {
+            file_id: loc.file_id,
+            offset: loc.offset,
+        }),
+    }
+}
+
+fn verify_once<S: StorageIO>(
+    manager: &DataFileManager<S>,
+    fd: FileId,
+    file_size: u64,
+    expected_cid: &[u8; CID_SIZE],
+    loc: BlockLocation,
+) -> VerifyOutcome {
+    match decode_block_record(manager.io(), fd, loc.offset, file_size) {
+        Ok(Some(ReadBlockRecord::Valid { cid_bytes, .. })) if cid_bytes == *expected_cid => {
+            VerifyOutcome::NoFaultDetected
+        }
+        Ok(Some(ReadBlockRecord::Valid { .. })) => {
+            tracing::warn!(
+                file_id = %loc.file_id,
+                offset = loc.offset.raw(),
+                "verify: stored CID mismatch (misdirected write)"
+            );
+            VerifyOutcome::Faulted
+        }
+        Ok(Some(ReadBlockRecord::Corrupted { .. } | ReadBlockRecord::Truncated { .. }))
+        | Ok(None) => {
+            tracing::warn!(
+                file_id = %loc.file_id,
+                offset = loc.offset.raw(),
+                "verify: block undecodable at location"
+            );
+            VerifyOutcome::Faulted
+        }
+        Err(_) => VerifyOutcome::NoFaultDetected,
+    }
+}
+
+const VERIFY_RETRY_ATTEMPTS: u32 = 4;
+
+fn rollback_batch<S: StorageIO>(
+    manager: &DataFileManager<S>,
+    state: &ActiveState,
+    rotations: &[RotationState<S>],
+) {
+    let _ = manager.io().truncate(state.fd, state.position.raw());
+    let _ = manager.io().sync(state.fd);
+    let _ = manager
+        .io()
+        .truncate(state.hint_fd, state.hint_position.raw());
+    let _ = manager.io().sync(state.hint_fd);
+    rotations.iter().for_each(|rot| {
+        manager.rollback_rotation(rot.file_id);
+        let _ = manager.io().close(rot.hint_fd);
+        let _ = manager
+            .io()
+            .delete(&hint_file_path(manager.data_dir(), rot.file_id));
+    });
 }

 fn process_batch<S: StorageIO>(
@@ -1071,13 +1211,46 @@ fn process_batch<S: StorageIO>(
    let mut all_decrements: Vec<[u8; CID_SIZE]> = Vec::new();

    let mut current_hint_fd = state.hint_fd;
-    let mut rotation: Option<RotationState> = None;
+    let mut rotations: Vec<RotationState<S>> = Vec::new();

    let mut data_writer =
        DataFileWriter::resume(manager.io(), state.fd, state.file_id, state.position);
    let mut hint_writer =
        HintFileWriter::resume(manager.io(), current_hint_fd, state.hint_position);

+    if manager.should_rotate(data_writer.position()) {
+        data_writer.sync().map_err(CommitError::from)?;
+        hint_writer.sync().map_err(CommitError::from)?;
+
+        let next_id = ctx.file_ids.allocate();
+        let next_handle = manager.open_for_append(next_id)?;
+        let next_fd = next_handle.fd();
+
+        tracing::info!(
+            from = %data_writer.file_id(),
+            to = %next_id,
+            trigger = "batch_boundary",
+            "data file rotation"
+        );
+
+        data_writer = DataFileWriter::new(manager.io(), next_fd, next_id)?;
+
+        let new_hint_path = hint_file_path(manager.data_dir(), next_id);
+        let new_hint_fd = manager
+            .io()
+            .open(&new_hint_path, OpenOptions::read_write())?;
+
+        manager.io().sync_dir(manager.data_dir())?;
+
+        current_hint_fd = new_hint_fd;
+        hint_writer = HintFileWriter::new(manager.io(), new_hint_fd);
+        rotations.push(RotationState {
+            file_id: next_id,
+            handle: next_handle,
+            hint_fd: new_hint_fd,
+        });
+    }
+
    let mut block_bytes: u64 = 0;
    let mut block_count: u64 = 0;
    let mut dedup_hits: u64 = 0;
@@ -1099,45 +1272,24 @@ fn process_batch<S: StorageIO>(
                    hint_writer.append_hint(cid_bytes, &loc)?;
                    loc
                }
-                None => {
-                    if manager.should_rotate(data_writer.position()) {
-                        data_writer.sync()?;
-                        hint_writer.sync()?;
-
-                        let next_id = ctx.file_ids.allocate();
-                        let next_fd = manager.open_for_append(next_id)?;
-
-                        tracing::info!(
-                            from = %data_writer.file_id(),
-                            to = %next_id,
-                            "data file rotation"
-                        );
-
-                        data_writer = DataFileWriter::new(manager.io(), next_fd, next_id)?;
-
-                        let new_hint_path = hint_file_path(manager.data_dir(), next_id);
-                        let new_hint_fd = manager
-                            .io()
-                            .open(&new_hint_path, OpenOptions::read_write())?;
-
-                        manager.io().sync_dir(manager.data_dir())?;
-
-                        current_hint_fd = new_hint_fd;
-                        hint_writer = HintFileWriter::new(manager.io(), new_hint_fd);
-                        rotation = Some(RotationState {
-                            file_id: next_id,
-                            fd: next_fd,
-                        });
+                None => match index.get(cid_bytes) {
+                    Some(existing) => {
+                        dedup_hits = dedup_hits.saturating_add(1);
+                        let loc = existing.location;
+                        hint_writer.append_hint(cid_bytes, &loc)?;
+                        dedup.insert(*cid_bytes, loc);
+                        loc
                    }
+                    None => {
+                        let loc = data_writer.append_block(cid_bytes, data)?;
+                        hint_writer.append_hint(cid_bytes, &loc)?;

-                    let loc = data_writer.append_block(cid_bytes, data)?;
-                    hint_writer.append_hint(cid_bytes, &loc)?;
-
-                    block_bytes = block_bytes.saturating_add(data.len() as u64);
-                    block_count = block_count.saturating_add(1);
-                    dedup.insert(*cid_bytes, loc);
-                    loc
-                }
+                        block_bytes = block_bytes.saturating_add(data.len() as u64);
+                        block_count = block_count.saturating_add(1);
+                        dedup.insert(*cid_bytes, loc);
+                        loc
+                    }
+                },
            };

            index_entries.push((*cid_bytes, location));
@@ -1152,9 +1304,7 @@ fn process_batch<S: StorageIO>(
    });

    if let Err(e) = write_result {
-        if let Some(rot) = rotation {
-            manager.rollback_rotation(rot.file_id, rot.fd);
-        }
+        rollback_batch(manager, state, &rotations);
        return Err(e);
    }

@@ -1163,18 +1313,57 @@ fn process_batch<S: StorageIO>(
    let current_epoch = epoch.current();
    let now = crate::wall_clock_ms();

+    let rollback_on_err = |e: CommitError| -> CommitError {
+        rollback_batch(manager, state, &rotations);
+        e
+    };
+
    all_decrements
        .iter()
-        .try_for_each(|cid| hint_writer.append_decrement(cid, current_epoch, now))?;
+        .try_for_each(|cid| hint_writer.append_decrement(cid, current_epoch, now))
+        .map_err(|e| rollback_on_err(CommitError::from(e)))?;

    let t = std::time::Instant::now();
-    data_writer.sync()?;
-    hint_writer.sync()?;
+    data_writer.sync().map_err(|e| rollback_on_err(e.into()))?;
+    if ctx.verify_persisted_blocks {
+        verify_persisted_blocks(manager, &index_entries).map_err(rollback_on_err)?;
+    }
+    let batch_record_count = u32::try_from(
+        block_count
+            .saturating_add(dedup_hits)
+            .saturating_add(all_decrements.len() as u64),
+    )
+    .unwrap_or(u32::MAX);
+    hint_writer
+        .append_commit_marker(
+            current_epoch.raw(),
+            batch_record_count,
+            data_writer.file_id(),
+            data_writer.position(),
+        )
+        .map_err(|e| rollback_on_err(CommitError::from(e)))?;
+    hint_writer.sync().map_err(|e| rollback_on_err(e.into()))?;
+    manager
+        .io()
+        .barrier()
+        .map_err(|e| rollback_on_err(e.into()))?;
    let sync_nanos = t.elapsed().as_nanos() as u64;

-    if let Some(ref rot) = rotation {
-        manager.commit_rotation(rot.file_id, rot.fd);
-        ctx.active_files.register(ctx.shard_id, rot.file_id);
+    if !rotations.is_empty() {
+        let old_file_id = state.file_id;
+        let old_hint_fd = state.hint_fd;
+        let last_idx = rotations.len() - 1;
+        rotations.iter().enumerate().for_each(|(i, rot)| {
+            if i == last_idx {
+                manager.commit_rotation(rot.file_id, &rot.handle);
+                ctx.active_files.register(ctx.shard_id, rot.file_id);
+            } else {
+                let _ = manager.io().close(rot.hint_fd);
+                manager.evict_handle(rot.file_id);
+            }
+        });
+        manager.evict_handle(old_file_id);
+        let _ = manager.io().close(old_hint_fd);
    }

    state.file_id = data_writer.file_id();
@@ -215,6 +215,10 @@ impl HashTable {
        self.get(cid).is_some()
    }

+    pub fn contains_live(&self, cid: &[u8; CID_SIZE]) -> bool {
+        self.get(cid).is_some_and(|s| !s.refcount.is_zero())
+    }
+
    pub fn insert(&mut self, new_slot: Slot) -> Result<Option<Slot>, CapacityExhausted> {
        if is_empty(&new_slot.cid) {
            tracing::error!("attempted to insert all-zero CID into hash table");
@@ -700,6 +704,7 @@ impl HashTable {
 const CHECKPOINT_MAGIC: [u8; 8] = *b"TQCKPT01";
 const CHECKPOINT_VERSION_V1: u32 = 1;
 const CHECKPOINT_VERSION_V2: u32 = 2;
+const CHECKPOINT_VERSION_V3: u32 = 3;
 const CHECKPOINT_HEADER_SIZE: usize = 128;
 const TRAILER_MAGIC: u64 = 0xDEAD_BEEF_CAFE_F00D;
 const SLOT_SIZE: usize = std::mem::size_of::<Slot>();
@@ -729,6 +734,7 @@ const H_CHECKPOINT_EPOCH: usize = 56;
 const H_HINT_FILE_ID: usize = 64;
 const H_HINT_OFFSET: usize = 72;
 const H_HEADER_CHECKSUM: usize = 80;
+const H_GENERATION: usize = 88;

 fn header_checksum(buf: &[u8; CHECKPOINT_HEADER_SIZE]) -> u64 {
    xxhash_rust::xxh3::xxh3_64(&buf[..H_HEADER_CHECKSUM])
@@ -741,10 +747,11 @@ fn serialize_header(
    cursor_offset: u64,
    checkpoint_epoch: u64,
    shard_count: u16,
+    generation: u64,
 ) -> [u8; CHECKPOINT_HEADER_SIZE] {
    let mut buf = [0u8; CHECKPOINT_HEADER_SIZE];
    buf[H_MAGIC..H_MAGIC + 8].copy_from_slice(&CHECKPOINT_MAGIC);
-    buf[H_VERSION..H_VERSION + 4].copy_from_slice(&CHECKPOINT_VERSION_V2.to_le_bytes());
+    buf[H_VERSION..H_VERSION + 4].copy_from_slice(&CHECKPOINT_VERSION_V3.to_le_bytes());
    buf[H_SHARD_COUNT..H_SHARD_COUNT + 2].copy_from_slice(&shard_count.to_le_bytes());
    buf[H_SLOT_COUNT..H_SLOT_COUNT + 8].copy_from_slice(&slot_count.to_le_bytes());
    buf[H_ENTRY_COUNT..H_ENTRY_COUNT + 8].copy_from_slice(&entry_count.to_le_bytes());
@@ -752,6 +759,7 @@ fn serialize_header(
    buf[H_CURSOR_OFFSET..H_CURSOR_OFFSET + 8].copy_from_slice(&cursor_offset.to_le_bytes());
    buf[H_CHECKPOINT_EPOCH..H_CHECKPOINT_EPOCH + 8]
        .copy_from_slice(&checkpoint_epoch.to_le_bytes());
+    buf[H_GENERATION..H_GENERATION + 8].copy_from_slice(&generation.to_le_bytes());
    let checksum = header_checksum(&buf);
    buf[H_HEADER_CHECKSUM..H_HEADER_CHECKSUM + 8].copy_from_slice(&checksum.to_le_bytes());
    buf
@@ -792,6 +800,7 @@ pub fn write_checkpoint(
    table: &HashTable,
    path: &Path,
    epoch: CommitEpoch,
+    generation: u64,
    positions: &CheckpointPositions,
 ) -> io::Result<()> {
    use std::io::Write;
@@ -813,6 +822,7 @@ pub fn write_checkpoint(
        cursor_offset,
        epoch.raw(),
        shard_count,
+        generation,
    );

    let slot_bytes = slots_as_bytes(&table.slots);
@@ -840,7 +850,7 @@ pub fn write_checkpoint(
    Ok(())
 }

-fn parse_checkpoint_header(data: &[u8]) -> io::Result<(usize, usize, u32, u64, u64, u16)> {
+fn parse_checkpoint_header(data: &[u8]) -> io::Result<(usize, usize, u32, u64, u64, u16, u64)> {
    if data.len() < CHECKPOINT_HEADER_SIZE + 16 {
        return Err(io::Error::new(
            io::ErrorKind::InvalidData,
@@ -859,7 +869,10 @@ fn parse_checkpoint_header(data: &[u8]) -> io::Result<(usize, usize, u32, u64, u
    }

    let version = u32::from_le_bytes(hdr[H_VERSION..H_VERSION + 4].try_into().unwrap());
-    if version != CHECKPOINT_VERSION_V1 && version != CHECKPOINT_VERSION_V2 {
+    if version != CHECKPOINT_VERSION_V1
+        && version != CHECKPOINT_VERSION_V2
+        && version != CHECKPOINT_VERSION_V3
+    {
        return Err(io::Error::new(
            io::ErrorKind::InvalidData,
            format!("checkpoint version {version} unsupported"),
@@ -917,12 +930,19 @@ fn parse_checkpoint_header(data: &[u8]) -> io::Result<(usize, usize, u32, u64, u
    );

    let shard_count = match version {
-        CHECKPOINT_VERSION_V2 => {
+        CHECKPOINT_VERSION_V2 | CHECKPOINT_VERSION_V3 => {
            u16::from_le_bytes(hdr[H_SHARD_COUNT..H_SHARD_COUNT + 2].try_into().unwrap())
        }
        _ => 0,
    };

+    let generation = match version {
+        CHECKPOINT_VERSION_V3 => {
+            u64::from_le_bytes(hdr[H_GENERATION..H_GENERATION + 8].try_into().unwrap())
+        }
+        _ => 0,
+    };
+
    Ok((
        slot_count,
        entry_count,
@@ -930,6 +950,7 @@ fn parse_checkpoint_header(data: &[u8]) -> io::Result<(usize, usize, u32, u64, u
        cursor_offset,
        checkpoint_epoch,
        shard_count,
+        generation,
    ))
 }

@@ -944,11 +965,20 @@ fn deserialize_shard_positions(data: &[u8], count: usize) -> Vec<(DataFileId, Hi
        .collect()
 }

-pub fn read_checkpoint(path: &Path) -> io::Result<(HashTable, CommitEpoch, CheckpointPositions)> {
+pub fn read_checkpoint(
+    path: &Path,
+) -> io::Result<(HashTable, CommitEpoch, CheckpointPositions, u64)> {
    let data = std::fs::read(path)?;

-    let (slot_count, entry_count, cursor_file_id, cursor_offset, checkpoint_epoch, shard_count) =
-        parse_checkpoint_header(&data)?;
+    let (
+        slot_count,
+        entry_count,
+        cursor_file_id,
+        cursor_offset,
+        checkpoint_epoch,
+        shard_count,
+        generation,
+    ) = parse_checkpoint_header(&data)?;

    let hdr: &[u8; CHECKPOINT_HEADER_SIZE] = data[..CHECKPOINT_HEADER_SIZE].try_into().unwrap();
    let version = u32::from_le_bytes(hdr[H_VERSION..H_VERSION + 4].try_into().unwrap());
@@ -968,7 +998,7 @@ pub fn read_checkpoint(path: &Path) -> io::Result<(HashTable, CommitEpoch, Check
    let shard_pos_region = &data[shard_pos_start..shard_pos_start + shard_pos_size];

    let data_checksum = match version {
-        CHECKPOINT_VERSION_V2 => {
+        CHECKPOINT_VERSION_V2 | CHECKPOINT_VERSION_V3 => {
            let mut hasher = xxhash_rust::xxh3::Xxh3::new();
            hasher.update(slot_region);
            hasher.update(shard_pos_region);
@@ -1031,7 +1061,7 @@ pub fn read_checkpoint(path: &Path) -> io::Result<(HashTable, CommitEpoch, Check
    let epoch = CommitEpoch::new(checkpoint_epoch);

    let positions = match version {
-        CHECKPOINT_VERSION_V2 if shard_count > 0 => CheckpointPositions(
+        CHECKPOINT_VERSION_V2 | CHECKPOINT_VERSION_V3 if shard_count > 0 => CheckpointPositions(
            deserialize_shard_positions(shard_pos_region, shard_count as usize),
        ),
        _ => {
@@ -1043,12 +1073,12 @@ pub fn read_checkpoint(path: &Path) -> io::Result<(HashTable, CommitEpoch, Check
        }
    };

-    Ok((table, epoch, positions))
+    Ok((table, epoch, positions, generation))
 }

 pub fn load_best_checkpoint(
    index_dir: &Path,
-) -> Option<(HashTable, CommitEpoch, CheckpointPositions)> {
+) -> Option<(HashTable, CommitEpoch, CheckpointPositions, u64)> {
    let path_a = index_dir.join("checkpoint_a.tqc");
    let path_b = index_dir.join("checkpoint_b.tqc");

@@ -1056,7 +1086,7 @@ pub fn load_best_checkpoint(
    let result_b = read_checkpoint(&path_b).ok();

    match (result_a, result_b) {
-        (Some(a), Some(b)) => match a.1.raw() >= b.1.raw() {
+        (Some(a), Some(b)) => match (a.3, a.1.raw()) >= (b.3, b.1.raw()) {
            true => Some(a),
            false => Some(b),
        },
@@ -1066,7 +1096,7 @@ pub fn load_best_checkpoint(
    }
 }

-fn read_checkpoint_epoch(path: &Path) -> Option<u64> {
+fn read_checkpoint_meta(path: &Path) -> Option<(u64, u64)> {
    let mut file = std::fs::File::open(path).ok()?;
    let mut buf = [0u8; CHECKPOINT_HEADER_SIZE];
    std::io::Read::read_exact(&mut file, &mut buf).ok()?;
@@ -1077,7 +1107,10 @@ fn read_checkpoint_epoch(path: &Path) -> Option<u64> {
    }

    let version = u32::from_le_bytes(buf[H_VERSION..H_VERSION + 4].try_into().ok()?);
-    if version != CHECKPOINT_VERSION_V1 && version != CHECKPOINT_VERSION_V2 {
+    if version != CHECKPOINT_VERSION_V1
+        && version != CHECKPOINT_VERSION_V2
+        && version != CHECKPOINT_VERSION_V3
+    {
        return None;
    }

@@ -1090,33 +1123,41 @@ fn read_checkpoint_epoch(path: &Path) -> Option<u64> {
        return None;
    }

-    Some(u64::from_le_bytes(
+    let epoch = u64::from_le_bytes(
        buf[H_CHECKPOINT_EPOCH..H_CHECKPOINT_EPOCH + 8]
            .try_into()
            .ok()?,
-    ))
+    );
+    let generation = match version {
+        CHECKPOINT_VERSION_V3 => {
+            u64::from_le_bytes(buf[H_GENERATION..H_GENERATION + 8].try_into().ok()?)
+        }
+        _ => 0,
+    };
+    Some((epoch, generation))
 }

 pub fn write_checkpoint_ab(
    table: &HashTable,
    index_dir: &Path,
    epoch: CommitEpoch,
+    generation: u64,
    positions: &CheckpointPositions,
 ) -> io::Result<()> {
    let path_a = index_dir.join("checkpoint_a.tqc");
    let path_b = index_dir.join("checkpoint_b.tqc");

-    let epoch_a = read_checkpoint_epoch(&path_a);
-    let epoch_b = read_checkpoint_epoch(&path_b);
+    let meta_a = read_checkpoint_meta(&path_a);
+    let meta_b = read_checkpoint_meta(&path_b);

-    let target_path = match (epoch_a, epoch_b) {
-        (Some(a), Some(b)) if a >= b => path_b,
+    let target_path = match (meta_a, meta_b) {
+        (Some(a), Some(b)) if (a.1, a.0) >= (b.1, b.0) => path_b,
        (Some(_), Some(_)) => path_a,
        (Some(_), None) => path_b,
        (None, _) => path_a,
    };

-    write_checkpoint(table, &target_path, epoch, positions)
+    write_checkpoint(table, &target_path, epoch, generation, positions)
 }

 #[derive(Debug)]
@@ -1142,6 +1183,7 @@ pub struct BlockIndex {
    checkpoint_lock: parking_lot::Mutex<()>,
    loaded_checkpoint_positions: Option<CheckpointPositions>,
    loaded_checkpoint_epoch: Option<CommitEpoch>,
+    next_generation: std::sync::atomic::AtomicU64,
 }

 impl BlockIndex {
@@ -1152,33 +1194,36 @@ impl BlockIndex {
            checkpoint_lock: parking_lot::Mutex::new(()),
            loaded_checkpoint_positions: None,
            loaded_checkpoint_epoch: None,
+            next_generation: std::sync::atomic::AtomicU64::new(1),
        }
    }

    pub fn open(index_dir: &Path) -> io::Result<Self> {
        std::fs::create_dir_all(index_dir)?;
-        let (table, checkpoint_positions, checkpoint_epoch) = match load_best_checkpoint(index_dir)
-        {
-            Some((table, epoch, positions)) => {
-                tracing::info!(
-                    blocks = table.len(),
-                    epoch = epoch.raw(),
-                    shard_positions = positions.0.len(),
-                    "loaded block index from checkpoint"
-                );
-                (table, Some(positions), Some(epoch))
-            }
-            None => {
-                tracing::info!("no valid checkpoint found, starting with empty index");
-                (HashTable::with_capacity(64), None, None)
-            }
-        };
+        let (table, checkpoint_positions, checkpoint_epoch, loaded_generation) =
+            match load_best_checkpoint(index_dir) {
+                Some((table, epoch, positions, gen_value)) => {
+                    tracing::info!(
+                        blocks = table.len(),
+                        epoch = epoch.raw(),
+                        shard_positions = positions.0.len(),
+                        generation = gen_value,
+                        "loaded block index from checkpoint"
+                    );
+                    (table, Some(positions), Some(epoch), gen_value)
+                }
+                None => {
+                    tracing::info!("no valid checkpoint found, starting with empty index");
+                    (HashTable::with_capacity(64), None, None, 0)
+                }
+            };
        Ok(Self {
            table: RwLock::new(table),
            index_dir: index_dir.to_path_buf(),
            checkpoint_lock: parking_lot::Mutex::new(()),
            loaded_checkpoint_positions: checkpoint_positions,
            loaded_checkpoint_epoch: checkpoint_epoch,
+            next_generation: std::sync::atomic::AtomicU64::new(loaded_generation + 1),
        })
    }

@@ -1195,7 +1240,16 @@ impl BlockIndex {
    }

    pub fn has(&self, cid: &[u8; CID_SIZE]) -> bool {
-        self.table.read().contains(cid)
+        self.table.read().contains_live(cid)
+    }
+
+    pub fn live_entries_snapshot(&self) -> Vec<([u8; CID_SIZE], RefCount)> {
+        self.table
+            .read()
+            .iter()
+            .filter(|s| !s.refcount.is_zero())
+            .map(|s| (s.cid, s.refcount))
+            .collect()
    }

    pub fn batch_put(
@@ -1218,7 +1272,14 @@ impl BlockIndex {
        now: WallClockMs,
        position_update: PositionUpdate<'_>,
    ) -> Result<(), BlockIndexError> {
-        self.batch_put_inner(entries, decrements, cursor, epoch, now, Some(position_update))
+        self.batch_put_inner(
+            entries,
+            decrements,
+            cursor,
+            epoch,
+            now,
+            Some(position_update),
+        )
    }

    fn batch_put_inner(
@@ -1461,9 +1522,12 @@ impl BlockIndex {
        hint_positions: &ShardHintPositions,
    ) -> io::Result<()> {
        let _guard = self.checkpoint_lock.lock();
+        let generation = self
+            .next_generation
+            .fetch_add(1, std::sync::atomic::Ordering::AcqRel);
        let table = self.table.read();
        let positions = hint_positions.snapshot();
-        write_checkpoint_ab(&table, &self.index_dir, epoch, &positions)
+        write_checkpoint_ab(&table, &self.index_dir, epoch, generation, &positions)
    }

    pub fn write_checkpoint_with_positions(
@@ -1472,8 +1536,11 @@ impl BlockIndex {
        positions: &CheckpointPositions,
    ) -> io::Result<()> {
        let _guard = self.checkpoint_lock.lock();
+        let generation = self
+            .next_generation
+            .fetch_add(1, std::sync::atomic::Ordering::AcqRel);
        let table = self.table.read();
-        write_checkpoint_ab(&table, &self.index_dir, epoch, positions)
+        write_checkpoint_ab(&table, &self.index_dir, epoch, generation, positions)
    }

    pub fn index_dir(&self) -> &Path {
@@ -1570,7 +1637,8 @@ impl BlockIndex {
                    ReadHintRecord::Remove { cid_bytes } => {
                        let _ = table.remove(&cid_bytes);
                    }
-                    ReadHintRecord::UnknownVersion { .. }
+                    ReadHintRecord::CommitMarker { .. }
+                    | ReadHintRecord::UnknownVersion { .. }
                    | ReadHintRecord::UnknownType { .. }
                    | ReadHintRecord::Corrupted
                    | ReadHintRecord::Truncated => {}
@@ -2025,8 +2093,8 @@ mod tests {
        let epoch = CommitEpoch::new(42);
        let positions = CheckpointPositions::single(DataFileId::new(5), HintOffset::new(12345));

-        write_checkpoint(&table, &path, epoch, &positions).unwrap();
-        let (restored, restored_epoch, restored_pos) = read_checkpoint(&path).unwrap();
+        write_checkpoint(&table, &path, epoch, 7, &positions).unwrap();
+        let (restored, restored_epoch, restored_pos, _gen) = read_checkpoint(&path).unwrap();

        assert_eq!(restored.len(), 10);
        assert_eq!(restored_epoch.raw(), 42);
@@ -2058,14 +2126,14 @@ mod tests {
        table
            .insert_or_increment(&test_cid(1), test_loc(0, 0, 10))
            .unwrap();
-        write_checkpoint_ab(&table, dir.path(), CommitEpoch::new(1), &pos).unwrap();
+        write_checkpoint_ab(&table, dir.path(), CommitEpoch::new(1), 1, &pos).unwrap();

        table
            .insert_or_increment(&test_cid(2), test_loc(0, 100, 10))
            .unwrap();
-        write_checkpoint_ab(&table, dir.path(), CommitEpoch::new(2), &pos).unwrap();
+        write_checkpoint_ab(&table, dir.path(), CommitEpoch::new(2), 2, &pos).unwrap();

-        let (best, epoch, _) = load_best_checkpoint(dir.path()).unwrap();
+        let (best, epoch, _, _) = load_best_checkpoint(dir.path()).unwrap();
        assert_eq!(epoch.raw(), 2);
        assert_eq!(best.len(), 2);
    }
@@ -2079,16 +2147,16 @@ mod tests {
        table
            .insert_or_increment(&test_cid(1), test_loc(0, 0, 10))
            .unwrap();
-        write_checkpoint_ab(&table, dir.path(), CommitEpoch::new(1), &pos).unwrap();
+        write_checkpoint_ab(&table, dir.path(), CommitEpoch::new(1), 1, &pos).unwrap();

        table
            .insert_or_increment(&test_cid(2), test_loc(0, 100, 10))
            .unwrap();
-        write_checkpoint_ab(&table, dir.path(), CommitEpoch::new(2), &pos).unwrap();
+        write_checkpoint_ab(&table, dir.path(), CommitEpoch::new(2), 2, &pos).unwrap();

        std::fs::write(dir.path().join("checkpoint_b.tqc"), b"corrupt").unwrap();

-        let (best, epoch, _) = load_best_checkpoint(dir.path()).unwrap();
+        let (best, epoch, _, _) = load_best_checkpoint(dir.path()).unwrap();
        assert_eq!(epoch.raw(), 1);
        assert_eq!(best.len(), 1);
    }
@@ -22,6 +22,7 @@ const RECORD_TYPE_PUT: u8 = 0x01;
 const RECORD_TYPE_DECREMENT: u8 = 0x02;
 const RECORD_TYPE_RELOCATE: u8 = 0x03;
 const RECORD_TYPE_REMOVE: u8 = 0x04;
+const RECORD_TYPE_COMMIT_MARKER: u8 = 0x05;

 const HINT_FORMAT_VERSION: u8 = 1;

@@ -57,10 +58,8 @@ fn write_hint_record<S: StorageIO>(

 fn encode_location_fields(record: &mut [u8; HINT_RECORD_SIZE], loc: &BlockLocation) {
    record[FIELD_A_OFFSET..FIELD_A_OFFSET + 4].copy_from_slice(&loc.file_id.raw().to_le_bytes());
-    record[FIELD_A_OFFSET + 4..FIELD_A_OFFSET + 8]
-        .copy_from_slice(&loc.length.raw().to_le_bytes());
-    record[FIELD_B_OFFSET..FIELD_B_OFFSET + 8]
-        .copy_from_slice(&loc.offset.raw().to_le_bytes());
+    record[FIELD_A_OFFSET + 4..FIELD_A_OFFSET + 8].copy_from_slice(&loc.length.raw().to_le_bytes());
+    record[FIELD_B_OFFSET..FIELD_B_OFFSET + 8].copy_from_slice(&loc.offset.raw().to_le_bytes());
 }

 pub(crate) fn encode_hint_record<S: StorageIO>(
@@ -144,6 +143,36 @@ pub(crate) fn encode_decrement_record<S: StorageIO>(
    write_hint_record(io, fd, write_offset, &record)
 }

+const MARKER_DATA_OFFSET_POS: usize = CID_OFFSET;
+const MARKER_DATA_FILE_ID_POS: usize = CID_OFFSET + 8;
+const MARKER_RECORD_COUNT_POS: usize = FIELD_B_OFFSET;
+
+pub(crate) fn encode_commit_marker_record<S: StorageIO>(
+    io: &S,
+    fd: FileId,
+    write_offset: HintOffset,
+    batch_seq: u64,
+    record_count: u32,
+    data_file_id: DataFileId,
+    data_offset: BlockOffset,
+) -> io::Result<()> {
+    let mut record = [0u8; HINT_RECORD_SIZE];
+    record[TYPE_OFFSET] = RECORD_TYPE_COMMIT_MARKER;
+    record[VERSION_OFFSET] = HINT_FORMAT_VERSION;
+    record[MARKER_DATA_OFFSET_POS..MARKER_DATA_OFFSET_POS + 8]
+        .copy_from_slice(&data_offset.raw().to_le_bytes());
+    record[MARKER_DATA_FILE_ID_POS..MARKER_DATA_FILE_ID_POS + 4]
+        .copy_from_slice(&data_file_id.raw().to_le_bytes());
+    record[FIELD_A_OFFSET..FIELD_A_OFFSET + 8].copy_from_slice(&batch_seq.to_le_bytes());
+    record[MARKER_RECORD_COUNT_POS..MARKER_RECORD_COUNT_POS + 4]
+        .copy_from_slice(&record_count.to_le_bytes());
+
+    let checksum = hint_checksum(&record[..HINT_PAYLOAD_SIZE]);
+    record[CHECKSUM_OFFSET..].copy_from_slice(&checksum.to_le_bytes());
+
+    write_hint_record(io, fd, write_offset, &record)
+}
+
 #[must_use]
 #[derive(Debug)]
 pub enum ReadHintRecord {
@@ -168,6 +197,12 @@ pub enum ReadHintRecord {
    Remove {
        cid_bytes: [u8; CID_SIZE],
    },
+    CommitMarker {
+        batch_seq: u64,
+        record_count: u32,
+        data_file_id: DataFileId,
+        data_offset: BlockOffset,
+    },
    UnknownVersion {
        version: u8,
    },
@@ -297,6 +332,34 @@ pub fn decode_hint_record<S: StorageIO>(
            }))
        }
        RECORD_TYPE_REMOVE => Ok(Some(ReadHintRecord::Remove { cid_bytes })),
+        RECORD_TYPE_COMMIT_MARKER => {
+            let data_offset = BlockOffset::new(u64::from_le_bytes(
+                record[MARKER_DATA_OFFSET_POS..MARKER_DATA_OFFSET_POS + 8]
+                    .try_into()
+                    .unwrap(),
+            ));
+            let data_file_id = DataFileId::new(u32::from_le_bytes(
+                record[MARKER_DATA_FILE_ID_POS..MARKER_DATA_FILE_ID_POS + 4]
+                    .try_into()
+                    .unwrap(),
+            ));
+            let batch_seq = u64::from_le_bytes(
+                record[FIELD_A_OFFSET..FIELD_A_OFFSET + 8]
+                    .try_into()
+                    .unwrap(),
+            );
+            let record_count = u32::from_le_bytes(
+                record[MARKER_RECORD_COUNT_POS..MARKER_RECORD_COUNT_POS + 4]
+                    .try_into()
+                    .unwrap(),
+            );
+            Ok(Some(ReadHintRecord::CommitMarker {
+                batch_seq,
+                record_count,
+                data_file_id,
+                data_offset,
+            }))
+        }
        other => Ok(Some(ReadHintRecord::UnknownType { record_type: other })),
    }
 }
@@ -358,6 +421,26 @@ impl<'a, S: StorageIO> HintFileWriter<'a, S> {
        Ok(())
    }

+    pub fn append_commit_marker(
+        &mut self,
+        batch_seq: u64,
+        record_count: u32,
+        data_file_id: DataFileId,
+        data_offset: BlockOffset,
+    ) -> io::Result<()> {
+        encode_commit_marker_record(
+            self.io,
+            self.fd,
+            self.position,
+            batch_seq,
+            record_count,
+            data_file_id,
+            data_offset,
+        )?;
+        self.position = self.position.advance(HINT_RECORD_SIZE as u64);
+        Ok(())
+    }
+
    pub fn sync(&self) -> io::Result<()> {
        self.io.sync(self.fd)
    }
@@ -418,12 +501,13 @@ impl<S: StorageIO> Iterator for HintFileReader<'_, S> {
                    | ReadHintRecord::Decrement { .. }
                    | ReadHintRecord::Relocate { .. }
                    | ReadHintRecord::Remove { .. }
-                    | ReadHintRecord::UnknownType { .. } => {
+                    | ReadHintRecord::CommitMarker { .. }
+                    | ReadHintRecord::UnknownType { .. }
+                    | ReadHintRecord::UnknownVersion { .. }
+                    | ReadHintRecord::Corrupted => {
                        self.position = self.position.advance(HINT_RECORD_SIZE as u64);
                    }
-                    ReadHintRecord::UnknownVersion { .. }
-                    | ReadHintRecord::Corrupted
-                    | ReadHintRecord::Truncated => {
+                    ReadHintRecord::Truncated => {
                        self.position = HintOffset::new(self.file_size);
                    }
                }
@@ -531,6 +615,7 @@ fn scan_single_hint_file<S: StorageIO>(
                ReadHintRecord::Decrement { .. }
                | ReadHintRecord::Relocate { .. }
                | ReadHintRecord::Remove { .. }
+                | ReadHintRecord::CommitMarker { .. }
                | ReadHintRecord::UnknownVersion { .. }
                | ReadHintRecord::UnknownType { .. }
                | ReadHintRecord::Corrupted
@@ -544,7 +629,95 @@ fn scan_single_hint_file<S: StorageIO>(
    entries
 }

-const REPLAY_BATCH_SIZE: usize = 10_000;
+#[derive(Default)]
+struct PendingBatch {
+    puts: Vec<([u8; CID_SIZE], BlockLocation)>,
+    relocates: Vec<([u8; CID_SIZE], BlockLocation, u32)>,
+    removes: Vec<[u8; CID_SIZE]>,
+    decrements: Vec<([u8; CID_SIZE], CommitEpoch, WallClockMs)>,
+    file_cursors: HashMap<DataFileId, BlockOffset>,
+    max_cursor: Option<WriteCursor>,
+    record_count: u32,
+    boundary_lost: bool,
+}
+
+impl PendingBatch {
+    fn reset(&mut self) {
+        self.puts.clear();
+        self.relocates.clear();
+        self.removes.clear();
+        self.decrements.clear();
+        self.file_cursors.clear();
+        self.max_cursor = None;
+        self.record_count = 0;
+        self.boundary_lost = false;
+    }
+
+    fn note_record(&mut self) {
+        self.record_count = self.record_count.saturating_add(1);
+    }
+
+    fn track_cursor(&mut self, file_id: DataFileId, end: BlockOffset) {
+        let candidate = WriteCursor {
+            file_id,
+            offset: end,
+        };
+        self.max_cursor = Some(match self.max_cursor {
+            Some(c) => std::cmp::max_by_key(c, candidate, |w| (w.file_id, w.offset)),
+            None => candidate,
+        });
+        self.file_cursors
+            .entry(file_id)
+            .and_modify(|existing| {
+                if end > *existing {
+                    *existing = end;
+                }
+            })
+            .or_insert(end);
+    }
+}
+
+fn commit_pending_batch(
+    pending: &mut PendingBatch,
+    index: &super::hash_index::BlockIndex,
+    file_cursors: &mut HashMap<DataFileId, BlockOffset>,
+    max_cursor: &mut Option<WriteCursor>,
+    replayed: &mut u64,
+) -> Result<(), RebuildError> {
+    if !pending.puts.is_empty() {
+        index.batch_insert_buffered(&pending.puts)?;
+    }
+    if !pending.relocates.is_empty() {
+        index.batch_relocate(&pending.relocates)?;
+    }
+    if !pending.removes.is_empty() {
+        index.batch_remove(&pending.removes);
+    }
+    pending
+        .decrements
+        .iter()
+        .try_for_each(|(cid, epoch, ts)| index.batch_decrement(&[*cid], *epoch, *ts))?;
+
+    pending.file_cursors.iter().for_each(|(fid, end)| {
+        file_cursors
+            .entry(*fid)
+            .and_modify(|existing| {
+                if *end > *existing {
+                    *existing = *end;
+                }
+            })
+            .or_insert(*end);
+    });
+    if let Some(c) = pending.max_cursor {
+        *max_cursor = Some(match *max_cursor {
+            Some(m) => std::cmp::max_by_key(m, c, |w| (w.file_id, w.offset)),
+            None => c,
+        });
+    }
+    *replayed = replayed.saturating_add(u64::from(pending.record_count));
+    pending.reset();
+    Ok(())
+}

 pub fn replay_hints_into_block_index<S: StorageIO>(
    io: &S,
@@ -570,11 +743,7 @@ pub fn replay_hints_into_block_index<S: StorageIO>(
    let mut max_cursor: Option<WriteCursor> = None;
    let mut file_cursors: HashMap<DataFileId, BlockOffset> = HashMap::new();
    let mut replayed: u64 = 0;
-    let mut put_buffer: Vec<([u8; CID_SIZE], BlockLocation)> =
-        Vec::with_capacity(REPLAY_BATCH_SIZE);
-    let mut relocate_buffer: Vec<([u8; CID_SIZE], BlockLocation, u32)> =
-        Vec::with_capacity(REPLAY_BATCH_SIZE);
-    let mut remove_buffer: Vec<[u8; CID_SIZE]> = Vec::with_capacity(REPLAY_BATCH_SIZE);
+    let mut pending = PendingBatch::default();

    hint_files
        .iter()
@@ -606,54 +775,19 @@ pub fn replay_hints_into_block_index<S: StorageIO>(
                            offset,
                            length,
                        };
-                        put_buffer.push((cid_bytes, loc));
-
                        let record_end =
                            offset.advance(BLOCK_RECORD_OVERHEAD as u64 + length.as_u64());
-                        let candidate = WriteCursor {
-                            file_id,
-                            offset: record_end,
-                        };
-                        max_cursor = Some(match max_cursor {
-                            Some(c) => {
-                                std::cmp::max_by_key(c, candidate, |w| (w.file_id, w.offset))
-                            }
-                            None => candidate,
-                        });
-                        file_cursors
-                            .entry(file_id)
-                            .and_modify(|existing| {
-                                if record_end > *existing {
-                                    *existing = record_end;
-                                }
-                            })
-                            .or_insert(record_end);
-
-                        replayed = replayed.saturating_add(1);
-                        if put_buffer.len() >= REPLAY_BATCH_SIZE {
-                            index.batch_insert_buffered(&put_buffer)?;
-                            put_buffer.clear();
-                        }
+                        pending.puts.push((cid_bytes, loc));
+                        pending.track_cursor(file_id, record_end);
+                        pending.note_record();
                    }
                    ReadHintRecord::Decrement {
                        cid_bytes,
                        epoch,
                        timestamp,
                    } => {
-                        if !put_buffer.is_empty() {
-                            index.batch_insert_buffered(&put_buffer)?;
-                            put_buffer.clear();
-                        }
-                        if !relocate_buffer.is_empty() {
-                            index.batch_relocate(&relocate_buffer)?;
-                            relocate_buffer.clear();
-                        }
-                        if !remove_buffer.is_empty() {
-                            index.batch_remove(&remove_buffer);
-                            remove_buffer.clear();
-                        }
-                        index.batch_decrement(&[cid_bytes], epoch, timestamp)?;
-                        replayed = replayed.saturating_add(1);
+                        pending.decrements.push((cid_bytes, epoch, timestamp));
+                        pending.note_record();
                    }
                    ReadHintRecord::Relocate {
                        cid_bytes,
@@ -667,75 +801,71 @@ pub fn replay_hints_into_block_index<S: StorageIO>(
                            offset,
                            length,
                        };
-                        relocate_buffer.push((cid_bytes, loc, refcount));
-
                        let record_end =
                            offset.advance(BLOCK_RECORD_OVERHEAD as u64 + length.as_u64());
-                        file_cursors
-                            .entry(file_id)
-                            .and_modify(|existing| {
-                                if record_end > *existing {
-                                    *existing = record_end;
-                                }
-                            })
-                            .or_insert(record_end);
-
-                        replayed = replayed.saturating_add(1);
-                        if relocate_buffer.len() >= REPLAY_BATCH_SIZE {
-                            if !put_buffer.is_empty() {
-                                index.batch_insert_buffered(&put_buffer)?;
-                                put_buffer.clear();
-                            }
-                            index.batch_relocate(&relocate_buffer)?;
-                            relocate_buffer.clear();
-                        }
+                        pending.relocates.push((cid_bytes, loc, refcount));
+                        pending.track_cursor(file_id, record_end);
+                        pending.note_record();
                    }
                    ReadHintRecord::Remove { cid_bytes } => {
-                        remove_buffer.push(cid_bytes);
-                        replayed = replayed.saturating_add(1);
-                        if remove_buffer.len() >= REPLAY_BATCH_SIZE {
-                            if !put_buffer.is_empty() {
-                                index.batch_insert_buffered(&put_buffer)?;
-                                put_buffer.clear();
+                        pending.removes.push(cid_bytes);
+                        pending.note_record();
+                    }
+                    ReadHintRecord::CommitMarker {
+                        batch_seq,
+                        record_count,
+                        data_file_id,
+                        data_offset,
+                    } => {
+                        let accepts =
+                            !pending.boundary_lost && pending.record_count == record_count;
+                        match accepts {
+                            true => {
+                                pending.track_cursor(data_file_id, data_offset);
+                                commit_pending_batch(
+                                    &mut pending,
+                                    index,
+                                    &mut file_cursors,
+                                    &mut max_cursor,
+                                    &mut replayed,
+                                )?;
                            }
-                            if !relocate_buffer.is_empty() {
-                                index.batch_relocate(&relocate_buffer)?;
-                                relocate_buffer.clear();
+                            false => {
+                                tracing::warn!(
+                                    file_id = %fid,
+                                    batch_seq,
+                                    expected_count = record_count,
+                                    observed_count = pending.record_count,
+                                    boundary_lost = pending.boundary_lost,
+                                    "rolling back torn hint batch"
+                                );
+                                pending.reset();
                            }
-                            index.batch_remove(&remove_buffer);
-                            remove_buffer.clear();
                        }
                    }
-                    ReadHintRecord::Corrupted => {
-                        tracing::warn!(
-                            file_id = %fid,
-                            "corrupted hint record during replay, skipping"
-                        );
+                    ReadHintRecord::Corrupted
+                    | ReadHintRecord::UnknownVersion { .. }
+                    | ReadHintRecord::UnknownType { .. } => {
+                        pending.boundary_lost = true;
                    }
-                    ReadHintRecord::UnknownVersion { .. }
-                    | ReadHintRecord::UnknownType { .. }
-                    | ReadHintRecord::Truncated => {}
+                    ReadHintRecord::Truncated => {}
                }
                Ok::<_, RebuildError>(())
            })?;

-            if !put_buffer.is_empty() {
-                index.batch_insert_buffered(&put_buffer)?;
-                put_buffer.clear();
-            }
-            if !relocate_buffer.is_empty() {
-                index.batch_relocate(&relocate_buffer)?;
-                relocate_buffer.clear();
-            }
-            if !remove_buffer.is_empty() {
-                index.batch_remove(&remove_buffer);
-                remove_buffer.clear();
-            }
-
            let _ = io.close(fd);
            Ok(())
        })?;

+    if pending.record_count > 0 || pending.boundary_lost {
+        tracing::warn!(
+            record_count = pending.record_count,
+            boundary_lost = pending.boundary_lost,
+            "discarding unterminated hint batch at replay end"
+        );
+        pending.reset();
+    }
+
    if let Some(cursor) = max_cursor {
        index.set_write_cursor(cursor)?;
    }
@@ -841,7 +971,11 @@ mod tests {
        let offset = BlockOffset::new(1024);
        let length = BlockLength::new(256);

-        let loc = BlockLocation { file_id, offset, length };
+        let loc = BlockLocation {
+            file_id,
+            offset,
+            length,
+        };
        encode_hint_record(&sim, fd, HintOffset::new(0), &cid, &loc).unwrap();

        let file_size = sim.file_size(fd).unwrap();
@@ -1095,7 +1229,7 @@ mod tests {
    }

    #[test]
-    fn hint_reader_stops_on_corrupted() {
+    fn hint_reader_reports_corrupted_and_continues() {
        let (sim, fd) = setup();
        let mut writer = HintFileWriter::new(&sim, fd);

@@ -1113,9 +1247,48 @@ mod tests {

        let reader = HintFileReader::open(&sim, fd).unwrap();
        let records: Vec<_> = reader.map(|r| r.unwrap()).collect();
-        assert_eq!(records.len(), 2);
+        assert_eq!(records.len(), 3);
        assert!(matches!(records[0], ReadHintRecord::Put { .. }));
        assert!(matches!(records[1], ReadHintRecord::Corrupted));
+        assert!(matches!(records[2], ReadHintRecord::Put { .. }));
+    }
+
+    #[test]
+    fn commit_marker_round_trip() {
+        let (sim, fd) = setup();
+        let data_file_id = DataFileId::new(7);
+        let data_offset = BlockOffset::new(9_876);
+
+        encode_commit_marker_record(
+            &sim,
+            fd,
+            HintOffset::new(0),
+            42,
+            128,
+            data_file_id,
+            data_offset,
+        )
+        .unwrap();
+
+        let file_size = sim.file_size(fd).unwrap();
+        let record = decode_hint_record(&sim, fd, HintOffset::new(0), file_size)
+            .unwrap()
+            .unwrap();
+
+        match record {
+            ReadHintRecord::CommitMarker {
+                batch_seq,
+                record_count,
+                data_file_id: fid,
+                data_offset: off,
+            } => {
+                assert_eq!(batch_seq, 42);
+                assert_eq!(record_count, 128);
+                assert_eq!(fid, data_file_id);
+                assert_eq!(off, data_offset);
+            }
+            other => panic!("expected CommitMarker, got {other:?}"),
+        }
    }

    #[test]
@@ -1,6 +1,7 @@
 use std::collections::HashMap;
 use std::io;
 use std::path::{Path, PathBuf};
+use std::sync::Arc;

 use parking_lot::RwLock;

@@ -13,22 +14,39 @@ pub const DEFAULT_MAX_FILE_SIZE: u64 = 256 * 1024 * 1024;

 pub(crate) const DATA_FILE_EXTENSION: &str = "tqb";

-struct CachedHandle {
+pub struct CachedHandle<S: StorageIO> {
    fd: FileId,
+    io: Arc<S>,
    writable: bool,
 }

+impl<S: StorageIO> CachedHandle<S> {
+    pub fn fd(&self) -> FileId {
+        self.fd
+    }
+
+    pub fn is_writable(&self) -> bool {
+        self.writable
+    }
+}
+
+impl<S: StorageIO> Drop for CachedHandle<S> {
+    fn drop(&mut self) {
+        let _ = self.io.close(self.fd);
+    }
+}
+
 pub struct DataFileManager<S: StorageIO> {
-    io: S,
+    io: Arc<S>,
    data_dir: PathBuf,
    max_file_size: u64,
-    handles: RwLock<HashMap<DataFileId, CachedHandle>>,
+    handles: RwLock<HashMap<DataFileId, Arc<CachedHandle<S>>>>,
 }

 impl<S: StorageIO> DataFileManager<S> {
    pub fn new(io: S, data_dir: PathBuf, max_file_size: u64) -> Self {
        Self {
-            io,
+            io: Arc::new(io),
            data_dir,
            max_file_size,
            handles: RwLock::new(HashMap::new()),
@@ -40,7 +58,7 @@ impl<S: StorageIO> DataFileManager<S> {
    }

    pub fn io(&self) -> &S {
-        &self.io
+        self.io.as_ref()
    }

    pub fn data_dir(&self) -> &Path {
@@ -56,77 +74,81 @@ impl<S: StorageIO> DataFileManager<S> {
            .join(format!("{file_id}.{DATA_FILE_EXTENSION}"))
    }

-    pub fn open_for_append(&self, file_id: DataFileId) -> io::Result<FileId> {
+    pub fn open_for_append(&self, file_id: DataFileId) -> io::Result<Arc<CachedHandle<S>>> {
        {
            let cache = self.handles.read();
            if let Some(entry) = cache.get(&file_id)
                && entry.writable
            {
-                return Ok(entry.fd);
+                return Ok(Arc::clone(entry));
            }
        }
        let path = self.data_file_path(file_id);
        let fd = self.io.open(&path, OpenOptions::read_write())?;
        let mut cache = self.handles.write();
-        match cache.get(&file_id) {
+        match cache.get(&file_id).cloned() {
            Some(entry) if entry.writable => {
                let _ = self.io.close(fd);
-                Ok(entry.fd)
+                Ok(entry)
            }
-            Some(entry) => {
-                let old_fd = entry.fd;
-                cache.insert(file_id, CachedHandle { fd, writable: true });
-                let _ = self.io.close(old_fd);
-                Ok(fd)
-            }
-            None => {
-                cache.insert(file_id, CachedHandle { fd, writable: true });
-                Ok(fd)
+            _ => {
+                let handle = Arc::new(CachedHandle {
+                    fd,
+                    io: Arc::clone(&self.io),
+                    writable: true,
+                });
+                cache.insert(file_id, Arc::clone(&handle));
+                Ok(handle)
            }
        }
    }

-    pub fn open_for_read(&self, file_id: DataFileId) -> io::Result<FileId> {
+    pub fn open_for_read(&self, file_id: DataFileId) -> io::Result<Arc<CachedHandle<S>>> {
        if let Some(entry) = self.handles.read().get(&file_id) {
-            return Ok(entry.fd);
+            return Ok(Arc::clone(entry));
        }
        let path = self.data_file_path(file_id);
        let fd = self.io.open(&path, OpenOptions::read_only_existing())?;
        let mut cache = self.handles.write();
-        match cache.get(&file_id) {
+        match cache.get(&file_id).cloned() {
            Some(entry) => {
                let _ = self.io.close(fd);
-                Ok(entry.fd)
+                Ok(entry)
            }
            None => {
-                cache.insert(
-                    file_id,
-                    CachedHandle {
-                        fd,
-                        writable: false,
-                    },
-                );
-                Ok(fd)
+                let handle = Arc::new(CachedHandle {
+                    fd,
+                    io: Arc::clone(&self.io),
+                    writable: false,
+                });
+                cache.insert(file_id, Arc::clone(&handle));
+                Ok(handle)
            }
        }
    }

-    pub fn prepare_rotation(&self, current: DataFileId) -> io::Result<(DataFileId, FileId)> {
+    pub fn prepare_rotation(
+        &self,
+        current: DataFileId,
+    ) -> io::Result<(DataFileId, Arc<CachedHandle<S>>)> {
        let next = current.next();
        let path = self.data_file_path(next);
        let fd = self.io.open(&path, OpenOptions::read_write())?;
-        Ok((next, fd))
+        let handle = Arc::new(CachedHandle {
+            fd,
+            io: Arc::clone(&self.io),
+            writable: true,
+        });
+        Ok((next, handle))
    }

-    pub fn commit_rotation(&self, file_id: DataFileId, fd: FileId) {
-        self.handles
-            .write()
-            .insert(file_id, CachedHandle { fd, writable: true });
+    pub fn commit_rotation(&self, file_id: DataFileId, handle: &Arc<CachedHandle<S>>) {
+        self.handles.write().insert(file_id, Arc::clone(handle));
    }

-    pub fn rollback_rotation(&self, file_id: DataFileId, fd: FileId) {
-        let _ = self.io.close(fd);
+    pub fn rollback_rotation(&self, file_id: DataFileId) {
        self.handles.write().remove(&file_id);
+        let _ = self.io.delete(&self.data_file_path(file_id));
    }

    pub fn should_rotate(&self, position: BlockOffset) -> bool {
@@ -134,14 +156,11 @@ impl<S: StorageIO> DataFileManager<S> {
    }

    pub fn list_files(&self) -> io::Result<Vec<DataFileId>> {
-        list_files_by_extension(&self.io, &self.data_dir, DATA_FILE_EXTENSION)
+        list_files_by_extension(&*self.io, &self.data_dir, DATA_FILE_EXTENSION)
    }

    pub fn evict_handle(&self, file_id: DataFileId) {
-        let removed = self.handles.write().remove(&file_id);
-        if let Some(entry) = removed {
-            let _ = self.io.close(entry.fd);
-        }
+        self.handles.write().remove(&file_id);
    }

    pub fn delete_data_file(&self, file_id: DataFileId) -> io::Result<()> {
@@ -151,14 +170,6 @@ impl<S: StorageIO> DataFileManager<S> {
    }
 }

-impl<S: StorageIO> Drop for DataFileManager<S> {
-    fn drop(&mut self) {
-        self.handles.write().drain().for_each(|(_, entry)| {
-            let _ = self.io.close(entry.fd);
-        });
-    }
-}
-
 #[cfg(test)]
 mod tests {
    use super::*;
@@ -177,8 +188,8 @@ mod tests {
    #[test]
    fn open_for_append_creates_file() {
        let mgr = setup_manager(1024);
-        let fd = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        assert_eq!(mgr.io().file_size(fd).unwrap(), 0);
+        let handle = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        assert_eq!(mgr.io().file_size(handle.fd()).unwrap(), 0);
    }

    #[test]
@@ -190,45 +201,46 @@ mod tests {
    #[test]
    fn handle_cache_returns_same_fd() {
        let mgr = setup_manager(1024);
-        let fd1 = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        let fd2 = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        assert_eq!(fd1, fd2);
+        let h1 = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        let h2 = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        assert_eq!(h1.fd(), h2.fd());
    }

    #[test]
    fn open_for_read_uses_cache_from_append() {
        let mgr = setup_manager(1024);
-        let fd_write = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        let fd_read = mgr.open_for_read(DataFileId::new(0)).unwrap();
-        assert_eq!(fd_write, fd_read);
+        let h_write = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        let h_read = mgr.open_for_read(DataFileId::new(0)).unwrap();
+        assert_eq!(h_write.fd(), h_read.fd());
    }

    #[test]
    fn rotation_lifecycle_prepare_commit() {
        let mgr = setup_manager(1024);
-        let _fd0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        let (next_id, next_fd) = mgr.prepare_rotation(DataFileId::new(0)).unwrap();
+        let _h0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        let (next_id, next_handle) = mgr.prepare_rotation(DataFileId::new(0)).unwrap();
        assert_eq!(next_id, DataFileId::new(1));
-        assert_eq!(mgr.io().file_size(next_fd).unwrap(), 0);
+        assert_eq!(mgr.io().file_size(next_handle.fd()).unwrap(), 0);
        mgr.io().sync_dir(mgr.data_dir()).unwrap();
-        mgr.commit_rotation(next_id, next_fd);
-        assert_eq!(mgr.open_for_read(next_id).unwrap(), next_fd);
+        mgr.commit_rotation(next_id, &next_handle);
+        assert_eq!(mgr.open_for_read(next_id).unwrap().fd(), next_handle.fd());
    }

    #[test]
-    fn rotation_rollback_cleans_handle() {
+    fn rotation_rollback_cleans_handle_and_deletes_file() {
        let mgr = setup_manager(1024);
-        let _fd0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        let (next_id, next_fd) = mgr.prepare_rotation(DataFileId::new(0)).unwrap();
-        mgr.commit_rotation(next_id, next_fd);
+        let _h0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        let (next_id, next_handle) = mgr.prepare_rotation(DataFileId::new(0)).unwrap();
+        mgr.commit_rotation(next_id, &next_handle);

-        assert_eq!(mgr.open_for_read(next_id).unwrap(), next_fd);
-        mgr.rollback_rotation(next_id, next_fd);
+        assert_eq!(mgr.open_for_read(next_id).unwrap().fd(), next_handle.fd());
+        drop(next_handle);
+        mgr.rollback_rotation(next_id);

-        let reopened_fd = mgr.open_for_read(next_id).unwrap();
-        assert_ne!(
-            reopened_fd, next_fd,
-            "rollback should have closed the cached fd"
+        let reopen = mgr.open_for_read(next_id);
+        assert!(
+            reopen.is_err_and(|e| e.kind() == io::ErrorKind::NotFound),
+            "rollback_rotation must delete the data file so recovery cannot resurrect uncommitted bytes"
        );
    }

@@ -244,8 +256,8 @@ mod tests {
    #[test]
    fn list_files_finds_data_files() {
        let mgr = setup_manager(1024);
-        let _fd0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        let _fd3 = mgr.open_for_append(DataFileId::new(3)).unwrap();
+        let _h0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        let _h3 = mgr.open_for_append(DataFileId::new(3)).unwrap();

        let files = mgr.list_files().unwrap();
        assert_eq!(files, vec![DataFileId::new(0), DataFileId::new(3)]);
@@ -254,7 +266,7 @@ mod tests {
    #[test]
    fn list_files_ignores_non_data_files() {
        let mgr = setup_manager(1024);
-        let _fd0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        let _h0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
        mgr.io()
            .open(Path::new("/data/notes.txt"), OpenOptions::read_write())
            .unwrap();
@@ -279,32 +291,32 @@ mod tests {
    #[test]
    fn rotate_and_write_across_files() {
        let mgr = setup_manager(1024);
-        let fd0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        let mut writer0 = DataFileWriter::new(mgr.io(), fd0, DataFileId::new(0)).unwrap();
+        let h0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        let mut writer0 = DataFileWriter::new(mgr.io(), h0.fd(), DataFileId::new(0)).unwrap();
        let _ = writer0
            .append_block(&test_cid(1), b"first file data")
            .unwrap();
        writer0.sync().unwrap();

-        let (id1, fd1) = mgr.prepare_rotation(DataFileId::new(0)).unwrap();
+        let (id1, h1) = mgr.prepare_rotation(DataFileId::new(0)).unwrap();
        mgr.io().sync_dir(mgr.data_dir()).unwrap();
-        mgr.commit_rotation(id1, fd1);
-        let mut writer1 = DataFileWriter::new(mgr.io(), fd1, id1).unwrap();
+        mgr.commit_rotation(id1, &h1);
+        let mut writer1 = DataFileWriter::new(mgr.io(), h1.fd(), id1).unwrap();
        let _ = writer1
            .append_block(&test_cid(2), b"second file data")
            .unwrap();
        writer1.sync().unwrap();

-        let fd0_read = mgr.open_for_read(DataFileId::new(0)).unwrap();
-        let blocks0 = DataFileReader::open(mgr.io(), fd0_read)
+        let h0_read = mgr.open_for_read(DataFileId::new(0)).unwrap();
+        let blocks0 = DataFileReader::open(mgr.io(), h0_read.fd())
            .unwrap()
            .valid_blocks()
            .unwrap();
        assert_eq!(blocks0.len(), 1);
        assert_eq!(blocks0[0].2, b"first file data");

-        let fd1_read = mgr.open_for_read(id1).unwrap();
-        let blocks1 = DataFileReader::open(mgr.io(), fd1_read)
+        let h1_read = mgr.open_for_read(id1).unwrap();
+        let blocks1 = DataFileReader::open(mgr.io(), h1_read.fd())
            .unwrap()
            .valid_blocks()
            .unwrap();
@@ -315,11 +327,11 @@ mod tests {
    #[test]
    fn read_cache_hit_from_writable_entry() {
        let mgr = setup_manager(1024);
-        let fd_write = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        DataFileWriter::new(mgr.io(), fd_write, DataFileId::new(0)).unwrap();
+        let h_write = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        DataFileWriter::new(mgr.io(), h_write.fd(), DataFileId::new(0)).unwrap();

-        let fd_read = mgr.open_for_read(DataFileId::new(0)).unwrap();
-        assert_eq!(fd_write, fd_read);
+        let h_read = mgr.open_for_read(DataFileId::new(0)).unwrap();
+        assert_eq!(h_write.fd(), h_read.fd());
    }

    #[test]
@@ -338,15 +350,15 @@ mod tests {
        mgr.io().sync_dir(mgr.data_dir()).unwrap();
        mgr.io().close(raw_fd).unwrap();

-        let fd_read = mgr.open_for_read(DataFileId::new(0)).unwrap();
-        let _reader = DataFileReader::open(mgr.io(), fd_read).unwrap();
+        let h_read = mgr.open_for_read(DataFileId::new(0)).unwrap();
+        let _reader = DataFileReader::open(mgr.io(), h_read.fd()).unwrap();

-        let fd_append = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        assert_ne!(fd_read, fd_append);
+        let h_append = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        assert_ne!(h_read.fd(), h_append.fd());

        let mut writer = DataFileWriter::resume(
            mgr.io(),
-            fd_append,
+            h_append.fd(),
            DataFileId::new(0),
            BlockOffset::new(BLOCK_HEADER_SIZE as u64),
        );
@@ -355,7 +367,7 @@ mod tests {
            .unwrap();
        writer.sync().unwrap();

-        let blocks = DataFileReader::open(mgr.io(), fd_append)
+        let blocks = DataFileReader::open(mgr.io(), h_append.fd())
            .unwrap()
            .valid_blocks()
            .unwrap();
@@ -1,3 +1,4 @@
+mod cid_util;
 mod compaction;
 mod data_file;
 mod group_commit;
@@ -8,6 +9,7 @@ mod reader;
 mod store;
 mod types;

+pub use cid_util::{DAG_CBOR_CODEC, SHA2_256_CODE, hash_to_cid, hash_to_cid_bytes};
 pub use compaction::CompactionError;
 pub use data_file::{
    BLOCK_FORMAT_VERSION, BLOCK_HEADER_SIZE, BLOCK_MAGIC, BLOCK_RECORD_OVERHEAD, CID_SIZE,
@@ -22,10 +24,10 @@ pub use hint::{
    HINT_FILE_EXTENSION, HINT_RECORD_SIZE, HintFileReader, HintFileWriter, HintIndex,
    ReadHintRecord, RebuildError, decode_hint_record, hint_file_path, scan_hints_to_memory,
 };
-pub use manager::{DEFAULT_MAX_FILE_SIZE, DataFileManager};
+pub use manager::{CachedHandle, DEFAULT_MAX_FILE_SIZE, DataFileManager};
 pub use reader::{BlockStoreReader, ReadError};
 pub use store::QuiesceGuard;
-pub use store::{BlockStoreConfig, DEFAULT_SHARD_COUNT, TranquilBlockStore};
+pub use store::{BlockStoreConfig, DEFAULT_SHARD_COUNT, OpenRetryPolicy, TranquilBlockStore};
 pub use types::{
    BlockLength, BlockLocation, BlockOffset, BlockstoreSnapshot, CidBytes, CollectionResult,
    CommitEpoch, CompactionResult, DataFileId, EpochCounter, HintOffset, IndexEntry, LivenessInfo,
@@ -104,11 +104,11 @@ impl<S: StorageIO> BlockStoreReader<S> {
        });

        by_file.into_iter().try_for_each(|(file_id, mut entries)| {
-            let fd = self.manager.open_for_read(file_id)?;
-            let file_size = self.manager.io().file_size(fd)?;
+            let handle = self.manager.open_for_read(file_id)?;
+            let file_size = self.manager.io().file_size(handle.fd())?;
            entries.sort_by_key(|(_, loc)| loc.offset);
            entries.into_iter().try_for_each(|(orig_idx, loc)| {
-                let data = self.decode_and_validate(fd, file_size, loc)?;
+                let data = self.decode_and_validate(handle.fd(), file_size, loc)?;
                results[orig_idx] = Some(data);
                Ok::<_, ReadError>(())
            })
@@ -116,9 +116,9 @@ impl<S: StorageIO> BlockStoreReader<S> {
    }

    fn read_block_at(&self, location: BlockLocation) -> Result<Bytes, ReadError> {
-        let fd = self.manager.open_for_read(location.file_id)?;
-        let file_size = self.manager.io().file_size(fd)?;
-        self.decode_and_validate(fd, file_size, location)
+        let handle = self.manager.open_for_read(location.file_id)?;
+        let file_size = self.manager.io().file_size(handle.fd())?;
+        self.decode_and_validate(handle.fd(), file_size, location)
    }

    fn decode_and_validate(
@@ -127,26 +127,37 @@ impl<S: StorageIO> BlockStoreReader<S> {
        file_size: u64,
        location: BlockLocation,
    ) -> Result<Bytes, ReadError> {
-        match decode_block_record(self.manager.io(), fd, location.offset, file_size)? {
-            Some(ReadBlockRecord::Valid { data, .. })
-                if data.len() == location.length.raw() as usize =>
-            {
-                Ok(Bytes::from(data))
-            }
-            Some(ReadBlockRecord::Valid { .. }) => Err(ReadError::Corrupted {
-                file_id: location.file_id,
-                offset: location.offset,
-            }),
-            Some(ReadBlockRecord::Corrupted { offset } | ReadBlockRecord::Truncated { offset }) => {
-                Err(ReadError::Corrupted {
+        let attempt_once = || -> Result<Bytes, ReadError> {
+            match decode_block_record(self.manager.io(), fd, location.offset, file_size)? {
+                Some(ReadBlockRecord::Valid { data, .. })
+                    if data.len() == location.length.raw() as usize =>
+                {
+                    Ok(Bytes::from(data))
+                }
+                Some(ReadBlockRecord::Valid { .. }) => Err(ReadError::Corrupted {
+                    file_id: location.file_id,
+                    offset: location.offset,
+                }),
+                Some(
+                    ReadBlockRecord::Corrupted { offset } | ReadBlockRecord::Truncated { offset },
+                ) => Err(ReadError::Corrupted {
                    file_id: location.file_id,
                    offset,
-                })
+                }),
+                None => Err(ReadError::Corrupted {
+                    file_id: location.file_id,
+                    offset: location.offset,
+                }),
            }
-            None => Err(ReadError::Corrupted {
-                file_id: location.file_id,
-                offset: location.offset,
-            }),
-        }
+        };
+        (0..READ_RETRY_ATTEMPTS.saturating_sub(1))
+            .find_map(|_| match attempt_once() {
+                Ok(bytes) => Some(Ok(bytes)),
+                Err(ReadError::Corrupted { .. }) => None,
+                Err(e) => Some(Err(e)),
+            })
+            .unwrap_or_else(attempt_once)
    }
 }
+
+const READ_RETRY_ATTEMPTS: u32 = 4;
@@ -1,19 +1,20 @@
 use std::collections::HashMap;
 use std::io;
+use std::num::NonZeroU8;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
+use std::time::Duration;

 use bytes::Bytes;
 use cid::Cid;
 use jacquard_repo::error::RepoError;
 use jacquard_repo::repo::CommitData;
 use jacquard_repo::storage::BlockStore;
-use multihash::Multihash;
-use sha2::{Digest, Sha256};

 use crate::fsync_order::PostBlockstoreHook;
 use crate::io::{OpenOptions, RealIO, StorageIO};

+use super::cid_util::hash_to_cid;
 use super::compaction::CompactionError;
 use super::data_file::{BLOCK_RECORD_OVERHEAD, CID_SIZE, ReadBlockRecord};
 use super::group_commit::{CommitError, CommitRequest, GroupCommitConfig, GroupCommitWriter};
@@ -21,13 +22,10 @@ use super::hash_index::BlockIndex;
 use super::manager::DataFileManager;
 use super::reader::{BlockStoreReader, ReadError};
 use super::types::{
-    BlockLength, BlockLocation, BlockOffset, CollectionResult, CompactionResult, DataFileId,
-    EpochCounter, LivenessInfo, WallClockMs, WriteCursor,
+    BlockLocation, BlockOffset, CollectionResult, CompactionResult, DataFileId, EpochCounter,
+    LivenessInfo, WallClockMs,
 };

-const DAG_CBOR_CODEC: u64 = 0x71;
-const SHA2_256_CODE: u64 = 0x12;
-
 fn cid_to_bytes(cid: &Cid) -> Result<[u8; CID_SIZE], RepoError> {
    let raw = cid.to_bytes();
    let len = raw.len();
@@ -41,20 +39,6 @@ fn cid_to_bytes(cid: &Cid) -> Result<[u8; CID_SIZE], RepoError> {
    })
 }

-fn hash_and_cid(data: &[u8]) -> Result<Cid, RepoError> {
-    let mut hasher = Sha256::new();
-    hasher.update(data);
-    let hash = hasher.finalize();
-    let multihash = Multihash::wrap(SHA2_256_CODE, &hash).map_err(|e| {
-        RepoError::storage(io::Error::new(io::ErrorKind::InvalidData, e.to_string()))
-    })?;
-    Ok(Cid::new_v1(DAG_CBOR_CODEC, multihash))
-}
-
-fn block_index_err_to_repo(e: super::hash_index::BlockIndexError) -> RepoError {
-    RepoError::storage(io::Error::other(e.to_string()))
-}
-
 fn commit_error_to_repo(e: CommitError) -> RepoError {
    match e {
        CommitError::Io(io_err) => {
@@ -65,6 +49,10 @@ fn commit_error_to_repo(e: CommitError) -> RepoError {
            io::ErrorKind::BrokenPipe,
            "blockstore commit channel closed",
        )),
+        CommitError::VerifyFailed { file_id, offset } => RepoError::storage(io::Error::new(
+            io::ErrorKind::InvalidData,
+            format!("post-sync verify failed at {file_id}:{}", offset.raw()),
+        )),
    }
 }

@@ -123,15 +111,26 @@ impl Drop for QuiesceGuard {
    }
 }

-#[derive(Clone)]
-pub struct TranquilBlockStore {
+pub struct TranquilBlockStore<S: StorageIO + Send + Sync + 'static = RealIO> {
    writer: Arc<WriterHandle>,
-    reader: Arc<BlockStoreReader<RealIO>>,
+    reader: Arc<BlockStoreReader<S>>,
    index: Arc<BlockIndex>,
    epoch: EpochCounter,
    data_dir: PathBuf,
 }

+impl<S: StorageIO + Send + Sync + 'static> Clone for TranquilBlockStore<S> {
+    fn clone(&self) -> Self {
+        Self {
+            writer: Arc::clone(&self.writer),
+            reader: Arc::clone(&self.reader),
+            index: Arc::clone(&self.index),
+            epoch: self.epoch.clone(),
+            data_dir: self.data_dir.clone(),
+        }
+    }
+}
+
 struct WriterHandle {
    inner: parking_lot::Mutex<Option<GroupCommitWriter>>,
 }
@@ -153,7 +152,25 @@ impl Drop for WriterHandle {
    }
 }

-impl TranquilBlockStore {
+#[derive(Clone, Copy, Debug)]
+pub struct OpenRetryPolicy {
+    pub max_attempts: NonZeroU8,
+    pub initial_backoff: Duration,
+    pub max_backoff: Duration,
+}
+
+impl Default for OpenRetryPolicy {
+    fn default() -> Self {
+        const DEFAULT_MAX_ATTEMPTS: NonZeroU8 = NonZeroU8::new(5).unwrap();
+        Self {
+            max_attempts: DEFAULT_MAX_ATTEMPTS,
+            initial_backoff: Duration::from_millis(100),
+            max_backoff: Duration::from_secs(2),
+        }
+    }
+}
+
+impl TranquilBlockStore<RealIO> {
    pub fn open(config: BlockStoreConfig) -> Result<Self, RepoError> {
        Self::open_with_hook(config, None)
    }
@@ -162,6 +179,70 @@ impl TranquilBlockStore {
        config: BlockStoreConfig,
        post_sync_hook: Option<Arc<dyn PostBlockstoreHook>>,
    ) -> Result<Self, RepoError> {
+        Self::open_with_io_hook(config, RealIO::new, post_sync_hook)
+    }
+
+    pub fn open_with_retry(
+        config: BlockStoreConfig,
+        policy: OpenRetryPolicy,
+    ) -> Result<Self, RepoError> {
+        retry_with_backoff(policy, &mut |_| Self::open(config.clone()))
+    }
+}
+
+fn retry_with_backoff<T, F>(policy: OpenRetryPolicy, op: &mut F) -> Result<T, RepoError>
+where
+    F: FnMut(u8) -> Result<T, RepoError>,
+{
+    retry_attempt(policy, op, 0, policy.initial_backoff)
+}
+
+fn retry_attempt<T, F>(
+    policy: OpenRetryPolicy,
+    op: &mut F,
+    attempt: u8,
+    backoff: Duration,
+) -> Result<T, RepoError>
+where
+    F: FnMut(u8) -> Result<T, RepoError>,
+{
+    match op(attempt) {
+        Ok(t) => Ok(t),
+        Err(e) if attempt + 1 >= policy.max_attempts.get() => Err(e),
+        Err(e) => {
+            tracing::warn!(
+                attempt,
+                error = %e,
+                backoff_ms = u64::try_from(backoff.as_millis()).unwrap_or(u64::MAX),
+                "blockstore open failed, retrying"
+            );
+            std::thread::sleep(backoff);
+            retry_attempt(
+                policy,
+                op,
+                attempt + 1,
+                (backoff * 2).min(policy.max_backoff),
+            )
+        }
+    }
+}
+
+impl<S: StorageIO + Send + Sync + 'static> TranquilBlockStore<S> {
+    pub fn open_with_io<F>(config: BlockStoreConfig, make_io: F) -> Result<Self, RepoError>
+    where
+        F: Fn() -> S + Send + Sync + Clone + 'static,
+    {
+        Self::open_with_io_hook(config, make_io, None)
+    }
+
+    pub fn open_with_io_hook<F>(
+        config: BlockStoreConfig,
+        make_io: F,
+        post_sync_hook: Option<Arc<dyn PostBlockstoreHook>>,
+    ) -> Result<Self, RepoError>
+    where
+        F: Fn() -> S + Send + Sync + Clone + 'static,
+    {
        if config.data_dir == config.index_dir {
            return Err(RepoError::storage(io::Error::new(
                io::ErrorKind::InvalidInput,
@@ -173,7 +254,7 @@ impl TranquilBlockStore {

        let index = BlockIndex::open(&config.index_dir).map_err(RepoError::storage)?;

-        let io = RealIO::new();
+        let io = make_io();

        let (replayed, file_cursors) = super::hint::replay_hints_into_block_index(
            &io,
@@ -195,8 +276,13 @@ impl TranquilBlockStore {
        let max_file_size = config.max_file_size;
        let shard_count = config.shard_count;
        let data_dir_for_closure = data_dir.clone();
+        let make_io_for_manager = make_io.clone();
        let make_manager = move || {
-            DataFileManager::new(RealIO::new(), data_dir_for_closure.clone(), max_file_size)
+            DataFileManager::new(
+                make_io_for_manager(),
+                data_dir_for_closure.clone(),
+                max_file_size,
+            )
        };

        let checkpoint_epoch = index.loaded_checkpoint_epoch();
@@ -214,7 +300,7 @@ impl TranquilBlockStore {
        let epoch = writer.epoch().clone();

        let manager_for_reader = Arc::new(DataFileManager::new(
-            RealIO::new(),
+            make_io(),
            data_dir.clone(),
            max_file_size,
        ));
@@ -234,7 +320,7 @@ impl TranquilBlockStore {
        })
    }

-    fn recover_from_file_cursors<S: StorageIO>(
+    fn recover_from_file_cursors(
        io: &S,
        data_dir: &Path,
        index: &BlockIndex,
@@ -256,7 +342,7 @@ impl TranquilBlockStore {
        })
    }

-    fn replay_single_file<S: StorageIO>(
+    fn replay_single_file(
        io: &S,
        data_dir: &Path,
        index: &BlockIndex,
@@ -277,19 +363,28 @@ impl TranquilBlockStore {
            Err(e) => return Err(RepoError::storage(e)),
        };

-        let result = Self::scan_and_index(io, index, fd, file_id, start_offset);
+        let hint_path = super::hint::hint_file_path(data_dir, file_id);
+        let hint_exists = io
+            .open(&hint_path, OpenOptions::read_only_existing())
+            .map(|fd| {
+                let _ = io.close(fd);
+            })
+            .is_ok();
+
+        let result = Self::scan_and_index(io, index, fd, file_id, start_offset, hint_exists);

        let _ = io.close(fd);

        result
    }

-    fn scan_and_index<S: StorageIO>(
+    fn scan_and_index(
        io: &S,
        index: &BlockIndex,
        fd: crate::io::FileId,
        file_id: DataFileId,
        start_offset: BlockOffset,
+        hint_exists: bool,
    ) -> Result<(), RepoError> {
        let file_size = io.file_size(fd).map_err(RepoError::storage)?;

@@ -298,17 +393,9 @@ impl TranquilBlockStore {
        }

        let scan_pos = &mut { start_offset };
-        let (recovered_entries, last_valid_end) = std::iter::from_fn(|| {
+        let (scanned_entries, last_valid_end) = std::iter::from_fn(|| {
            match super::data_file::decode_block_record(io, fd, *scan_pos, file_size) {
-                Err(e) => {
-                    tracing::warn!(
-                        file_id = %file_id,
-                        offset = scan_pos.raw(),
-                        error = %e,
-                        "IO error during recovery scan, stopping"
-                    );
-                    None
-                }
+                Err(e) => Some(Err(e)),
                Ok(None) => None,
                Ok(Some(ReadBlockRecord::Valid {
                    offset,
@@ -319,61 +406,70 @@ impl TranquilBlockStore {
                        Ok(n) if n <= super::types::MAX_BLOCK_SIZE => n,
                        _ => return None,
                    };
-                    let length = BlockLength::new(raw_len);
+                    let length = super::types::BlockLength::new(raw_len);
                    let record_size = BLOCK_RECORD_OVERHEAD as u64 + u64::from(raw_len);
-                    *scan_pos = scan_pos.advance(record_size);
-                    Some((
+                    let new_end = offset.advance(record_size);
+                    *scan_pos = new_end;
+                    Some(Ok((
                        cid_bytes,
                        BlockLocation {
                            file_id,
                            offset,
                            length,
                        },
-                    ))
+                        new_end,
+                    )))
                }
                Ok(Some(ReadBlockRecord::Corrupted { .. } | ReadBlockRecord::Truncated { .. })) => {
                    None
                }
            }
        })
-        .fold(
+        .try_fold(
            (Vec::new(), start_offset),
-            |(mut entries, _), (cid_bytes, location)| {
-                let new_end = location
-                    .offset
-                    .advance(BLOCK_RECORD_OVERHEAD as u64 + location.length.as_u64());
-                entries.push((cid_bytes, location));
-                (entries, new_end)
+            |(mut entries, _), item: io::Result<_>| {
+                let (cid, loc, new_end) = item?;
+                entries.push((cid, loc));
+                Ok::<_, io::Error>((entries, new_end))
            },
-        );
+        )
+        .map_err(|e| {
+            tracing::warn!(
+                file_id = %file_id,
+                offset = scan_pos.raw(),
+                error = %e,
+                "IO error during recovery scan, aborting to preserve durable tail"
+            );
+            RepoError::storage(e)
+        })?;

        if file_size > last_valid_end.raw() {
            tracing::info!(
                file_id = %file_id,
                truncating_from = last_valid_end.raw(),
                file_size,
-                "truncating partial/corrupted tail"
+                scanned_count = scanned_entries.len(),
+                "truncating partial/unacked tail"
            );
            io.truncate(fd, last_valid_end.raw())
                .map_err(RepoError::storage)?;
            io.sync(fd).map_err(RepoError::storage)?;
        }

-        if !recovered_entries.is_empty() {
-            let new_cursor = WriteCursor {
+        if !scanned_entries.is_empty() {
+            tracing::info!(
+                file_id = %file_id,
+                scanned = scanned_entries.len(),
+                hint_exists,
+                "reindexing blocks past hint coverage"
+            );
+            let cursor = super::types::WriteCursor {
                file_id,
                offset: last_valid_end,
            };
-            let inserted = index
-                .batch_put_if_absent(&recovered_entries, new_cursor)
-                .map_err(block_index_err_to_repo)?;
-            tracing::info!(
-                file_id = %file_id,
-                scanned = recovered_entries.len(),
-                inserted,
-                new_cursor_offset = last_valid_end.raw(),
-                "recovery data file scan"
-            );
+            index
+                .batch_put_if_absent(&scanned_entries, cursor)
+                .map_err(|e| RepoError::storage(io::Error::other(e.to_string())))?;
        }

        Ok(())
@@ -594,7 +690,7 @@ impl TranquilBlockStore {
    }
 }

-impl BlockStore for TranquilBlockStore {
+impl<S: StorageIO + Send + Sync + 'static> BlockStore for TranquilBlockStore<S> {
    async fn get(&self, cid: &Cid) -> Result<Option<Bytes>, RepoError> {
        let cid_bytes = cid_to_bytes(cid)?;
        let reader = Arc::clone(&self.reader);
@@ -605,7 +701,7 @@ impl BlockStore for TranquilBlockStore {
    }

    async fn put(&self, data: &[u8]) -> Result<Cid, RepoError> {
-        let cid = hash_and_cid(data)?;
+        let cid = hash_to_cid(data);
        let cid_bytes = cid_to_bytes(&cid)?;
        self.send_put_blocks(vec![(cid_bytes, data.to_vec())])
            .await?;
@@ -666,7 +762,7 @@ impl BlockStore for TranquilBlockStore {
    }
 }

-impl TranquilBlockStore {
+impl<S: StorageIO + Send + Sync + 'static> TranquilBlockStore<S> {
    pub async fn decrement_refs(&self, cids: &[Cid]) -> Result<(), RepoError> {
        if cids.is_empty() {
            return Ok(());
@@ -683,3 +779,213 @@ impl TranquilBlockStore {
        Ok(self.index.get(&cid_bytes).map(|entry| entry.refcount.raw()))
    }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use std::collections::HashMap;
+    use std::sync::Mutex;
+    use std::sync::atomic::{AtomicBool, Ordering};
+
+    use crate::blockstore::data_file::{
+        BLOCK_FORMAT_VERSION, BLOCK_HEADER_SIZE, BLOCK_MAGIC, encode_block_record,
+    };
+    use crate::blockstore::manager::DATA_FILE_EXTENSION;
+    use crate::io::FileId;
+
+    struct EioOnReadAtRange {
+        inner: RealIO,
+        target_path: PathBuf,
+        target_min: u64,
+        target_max: u64,
+        fired: AtomicBool,
+        fd_paths: Mutex<HashMap<FileId, PathBuf>>,
+    }
+
+    impl StorageIO for EioOnReadAtRange {
+        fn open(&self, path: &Path, opts: OpenOptions) -> io::Result<FileId> {
+            let fd = self.inner.open(path, opts)?;
+            self.fd_paths
+                .lock()
+                .unwrap()
+                .insert(fd, path.to_path_buf());
+            Ok(fd)
+        }
+
+        fn close(&self, fd: FileId) -> io::Result<()> {
+            self.fd_paths.lock().unwrap().remove(&fd);
+            self.inner.close(fd)
+        }
+
+        fn read_at(&self, fd: FileId, offset: u64, buf: &mut [u8]) -> io::Result<usize> {
+            let path_match = self.fd_paths.lock().unwrap().get(&fd).cloned();
+            let in_target_range = path_match.as_ref() == Some(&self.target_path)
+                && offset >= self.target_min
+                && offset <= self.target_max;
+            if in_target_range && !self.fired.swap(true, Ordering::SeqCst) {
+                return Err(io::Error::other("simulated EIO on read"));
+            }
+            self.inner.read_at(fd, offset, buf)
+        }
+
+        fn write_at(&self, fd: FileId, offset: u64, buf: &[u8]) -> io::Result<usize> {
+            self.inner.write_at(fd, offset, buf)
+        }
+
+        fn sync(&self, fd: FileId) -> io::Result<()> {
+            self.inner.sync(fd)
+        }
+
+        fn file_size(&self, fd: FileId) -> io::Result<u64> {
+            self.inner.file_size(fd)
+        }
+
+        fn truncate(&self, fd: FileId, size: u64) -> io::Result<()> {
+            self.inner.truncate(fd, size)
+        }
+
+        fn rename(&self, from: &Path, to: &Path) -> io::Result<()> {
+            self.inner.rename(from, to)
+        }
+
+        fn delete(&self, path: &Path) -> io::Result<()> {
+            self.inner.delete(path)
+        }
+
+        fn mkdir(&self, path: &Path) -> io::Result<()> {
+            self.inner.mkdir(path)
+        }
+
+        fn sync_dir(&self, path: &Path) -> io::Result<()> {
+            self.inner.sync_dir(path)
+        }
+
+        fn list_dir(&self, path: &Path) -> io::Result<Vec<PathBuf>> {
+            self.inner.list_dir(path)
+        }
+    }
+
+    #[test]
+    fn scan_and_index_does_not_truncate_acked_block_on_transient_eio() {
+        let tmp = tempfile::TempDir::new().unwrap();
+        let data_dir = tmp.path().join("data");
+        let index_dir = tmp.path().join("index");
+        std::fs::create_dir_all(&data_dir).unwrap();
+        std::fs::create_dir_all(&index_dir).unwrap();
+
+        let file_id = DataFileId::new(0);
+        let file_path = data_dir.join(format!("{file_id}.{DATA_FILE_EXTENSION}"));
+
+        let setup = RealIO::new();
+        let fd = setup.open(&file_path, OpenOptions::read_write()).unwrap();
+        let mut header = [0u8; BLOCK_HEADER_SIZE];
+        header[..4].copy_from_slice(&BLOCK_MAGIC);
+        header[4] = BLOCK_FORMAT_VERSION;
+        setup.write_all_at(fd, 0, &header).unwrap();
+
+        let cid_a = [0xAAu8; CID_SIZE];
+        let data_a = vec![1u8; 64];
+        let block_a_offset = BlockOffset::new(BLOCK_HEADER_SIZE as u64);
+        let len_a =
+            encode_block_record(&setup, fd, block_a_offset, &cid_a, &data_a).unwrap();
+
+        let block_b_offset_raw = BLOCK_HEADER_SIZE as u64 + len_a;
+        let block_b_offset = BlockOffset::new(block_b_offset_raw);
+        let cid_b = [0xBBu8; CID_SIZE];
+        let data_b = vec![2u8; 64];
+        let len_b = encode_block_record(&setup, fd, block_b_offset, &cid_b, &data_b).unwrap();
+
+        setup.sync(fd).unwrap();
+        setup.close(fd).unwrap();
+        drop(setup);
+
+        let total_size = block_b_offset_raw + len_b;
+        assert_eq!(std::fs::metadata(&file_path).unwrap().len(), total_size);
+
+        let wrapper = EioOnReadAtRange {
+            inner: RealIO::new(),
+            target_path: file_path.clone(),
+            target_min: block_b_offset_raw,
+            target_max: block_b_offset_raw + (BLOCK_RECORD_OVERHEAD as u64) - 1,
+            fired: AtomicBool::new(false),
+            fd_paths: Mutex::new(HashMap::new()),
+        };
+
+        let index = BlockIndex::open(&index_dir).unwrap();
+
+        let result = TranquilBlockStore::<EioOnReadAtRange>::replay_single_file(
+            &wrapper,
+            &data_dir,
+            &index,
+            file_id,
+            BlockOffset::new(BLOCK_HEADER_SIZE as u64),
+        );
+
+        assert!(
+            result.is_err(),
+            "replay must surface transient EIO instead of silently truncating"
+        );
+
+        let post_size = std::fs::metadata(&file_path).unwrap().len();
+        assert_eq!(
+            post_size, total_size,
+            "scan truncated durable acked block past EIO point: expected {total_size} bytes, got {post_size}"
+        );
+    }
+
+    fn instant_policy(max_attempts: u8) -> OpenRetryPolicy {
+        OpenRetryPolicy {
+            max_attempts: NonZeroU8::new(max_attempts).expect("max_attempts must be nonzero"),
+            initial_backoff: Duration::ZERO,
+            max_backoff: Duration::ZERO,
+        }
+    }
+
+    #[test]
+    fn retry_with_backoff_succeeds_on_first_attempt() {
+        let calls = std::sync::atomic::AtomicUsize::new(0);
+        let result = retry_with_backoff(instant_policy(5), &mut |_| {
+            calls.fetch_add(1, Ordering::Relaxed);
+            Ok::<u8, RepoError>(42)
+        });
+        assert_eq!(result.expect("ok"), 42);
+        assert_eq!(calls.load(Ordering::Relaxed), 1);
+    }
+
+    #[test]
+    fn retry_with_backoff_recovers_after_transient_failures() {
+        let calls = std::sync::atomic::AtomicUsize::new(0);
+        let result = retry_with_backoff(instant_policy(5), &mut |_| {
+            let n = calls.fetch_add(1, Ordering::Relaxed);
+            if n >= 2 {
+                Ok::<u8, RepoError>(7)
+            } else {
+                Err(RepoError::storage(io::Error::other("transient EIO")))
+            }
+        });
+        assert_eq!(result.expect("ok"), 7);
+        assert_eq!(calls.load(Ordering::Relaxed), 3);
+    }
+
+    #[test]
+    fn retry_with_backoff_gives_up_after_max_attempts() {
+        let calls = std::sync::atomic::AtomicUsize::new(0);
+        let result: Result<u8, RepoError> = retry_with_backoff(instant_policy(3), &mut |_| {
+            calls.fetch_add(1, Ordering::Relaxed);
+            Err(RepoError::storage(io::Error::other("permanent EIO")))
+        });
+        assert!(result.is_err(), "expected exhaustion error");
+        assert_eq!(calls.load(Ordering::Relaxed), 3);
+    }
+
+    #[test]
+    fn retry_with_backoff_passes_attempt_index_to_op() {
+        let observed = std::sync::Mutex::new(Vec::<u8>::new());
+        let _result: Result<(), RepoError> = retry_with_backoff(instant_policy(4), &mut |attempt| {
+            observed.lock().unwrap().push(attempt);
+            Err(RepoError::storage(io::Error::other("EIO")))
+        });
+        assert_eq!(*observed.lock().unwrap(), vec![0, 1, 2, 3]);
+    }
+}
@@ -1,7 +1,8 @@
 use std::collections::HashMap;
 use std::io;
 use std::path::{Path, PathBuf};
-use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};

 use parking_lot::RwLock;

@@ -25,17 +26,38 @@ pub fn parse_segment_id(path: &Path) -> Option<SegmentId> {
    (ext == SEGMENT_FILE_EXTENSION).then(|| stem.parse::<u32>().ok().map(SegmentId::new))?
 }

-struct CachedSegmentHandle {
+pub struct CachedSegmentHandle<S: StorageIO> {
    fd: FileId,
-    sealed: bool,
+    io: Arc<S>,
+    sealed: AtomicBool,
    writable: bool,
 }

+impl<S: StorageIO> CachedSegmentHandle<S> {
+    pub fn fd(&self) -> FileId {
+        self.fd
+    }
+
+    pub fn is_sealed(&self) -> bool {
+        self.sealed.load(Ordering::Acquire)
+    }
+
+    pub fn is_writable(&self) -> bool {
+        self.writable
+    }
+}
+
+impl<S: StorageIO> Drop for CachedSegmentHandle<S> {
+    fn drop(&mut self) {
+        let _ = self.io.close(self.fd);
+    }
+}
+
 pub struct SegmentManager<S: StorageIO> {
-    io: S,
+    io: Arc<S>,
    segments_dir: PathBuf,
    max_segment_size: u64,
-    handles: RwLock<HashMap<SegmentId, CachedSegmentHandle>>,
+    handles: RwLock<HashMap<SegmentId, Arc<CachedSegmentHandle<S>>>>,
    retention_epoch: AtomicU64,
 }

@@ -51,7 +73,7 @@ impl<S: StorageIO> SegmentManager<S> {
        );
        io.mkdir(&segments_dir)?;
        Ok(Self {
-            io,
+            io: Arc::new(io),
            segments_dir,
            max_segment_size,
            handles: RwLock::new(HashMap::new()),
@@ -60,7 +82,7 @@ impl<S: StorageIO> SegmentManager<S> {
    }

    pub fn io(&self) -> &S {
-        &self.io
+        self.io.as_ref()
    }

    pub fn segments_dir(&self) -> &Path {
@@ -92,52 +114,51 @@ impl<S: StorageIO> SegmentManager<S> {
        Ok(ids)
    }

-    pub fn open_for_read(&self, id: SegmentId) -> io::Result<FileId> {
+    pub fn open_for_read(&self, id: SegmentId) -> io::Result<Arc<CachedSegmentHandle<S>>> {
        if let Some(entry) = self.handles.read().get(&id) {
-            return Ok(entry.fd);
+            return Ok(Arc::clone(entry));
        }
        let path = self.segment_path(id);
        let fd = self.io.open(&path, OpenOptions::read_only_existing())?;
        let mut cache = self.handles.write();
-        match cache.get(&id) {
+        match cache.get(&id).cloned() {
            Some(entry) => {
                let _ = self.io.close(fd);
-                Ok(entry.fd)
+                Ok(entry)
            }
            None => {
-                cache.insert(
-                    id,
-                    CachedSegmentHandle {
-                        fd,
-                        sealed: false,
-                        writable: false,
-                    },
-                );
-                Ok(fd)
+                let handle = Arc::new(CachedSegmentHandle {
+                    fd,
+                    io: Arc::clone(&self.io),
+                    sealed: AtomicBool::new(false),
+                    writable: false,
+                });
+                cache.insert(id, Arc::clone(&handle));
+                Ok(handle)
            }
        }
    }

-    pub fn open_for_append(&self, id: SegmentId) -> io::Result<FileId> {
+    pub fn open_for_append(&self, id: SegmentId) -> io::Result<Arc<CachedSegmentHandle<S>>> {
        {
            let cache = self.handles.read();
            if let Some(entry) = cache.get(&id) {
-                if entry.sealed {
+                if entry.is_sealed() {
                    return Err(io::Error::new(
                        io::ErrorKind::InvalidInput,
                        format!("cannot append to sealed segment {id}"),
                    ));
                }
                if entry.writable {
-                    return Ok(entry.fd);
+                    return Ok(Arc::clone(entry));
                }
            }
        }
        let path = self.segment_path(id);
        let fd = self.io.open(&path, OpenOptions::read_write())?;
        let mut cache = self.handles.write();
-        match cache.get(&id) {
-            Some(entry) if entry.sealed => {
+        match cache.get(&id).cloned() {
+            Some(entry) if entry.is_sealed() => {
                let _ = self.io.close(fd);
                Err(io::Error::new(
                    io::ErrorKind::InvalidInput,
@@ -146,31 +167,17 @@ impl<S: StorageIO> SegmentManager<S> {
            }
            Some(entry) if entry.writable => {
                let _ = self.io.close(fd);
-                Ok(entry.fd)
+                Ok(entry)
            }
-            Some(entry) => {
-                let old_fd = entry.fd;
-                cache.insert(
-                    id,
-                    CachedSegmentHandle {
-                        fd,
-                        sealed: false,
-                        writable: true,
-                    },
-                );
-                let _ = self.io.close(old_fd);
-                Ok(fd)
-            }
-            None => {
-                cache.insert(
-                    id,
-                    CachedSegmentHandle {
-                        fd,
-                        sealed: false,
-                        writable: true,
-                    },
-                );
-                Ok(fd)
+            _ => {
+                let handle = Arc::new(CachedSegmentHandle {
+                    fd,
+                    io: Arc::clone(&self.io),
+                    sealed: AtomicBool::new(false),
+                    writable: true,
+                });
+                cache.insert(id, Arc::clone(&handle));
+                Ok(handle)
            }
        }
    }
@@ -179,37 +186,39 @@ impl<S: StorageIO> SegmentManager<S> {
        position.raw() >= self.max_segment_size
    }

-    pub fn prepare_rotation(&self, current_id: SegmentId) -> io::Result<(SegmentId, FileId)> {
+    pub fn prepare_rotation(
+        &self,
+        current_id: SegmentId,
+    ) -> io::Result<(SegmentId, Arc<CachedSegmentHandle<S>>)> {
        let next = current_id.next();
        let path = self.segment_path(next);
        let fd = self.io.open(&path, OpenOptions::read_write())?;
        self.io.truncate(fd, 0)?;
        self.io.sync_dir(&self.segments_dir)?;
-        Ok((next, fd))
+        let handle = Arc::new(CachedSegmentHandle {
+            fd,
+            io: Arc::clone(&self.io),
+            sealed: AtomicBool::new(false),
+            writable: true,
+        });
+        Ok((next, handle))
    }

-    pub fn commit_rotation(&self, new_id: SegmentId, fd: FileId) {
-        self.handles.write().insert(
-            new_id,
-            CachedSegmentHandle {
-                fd,
-                sealed: false,
-                writable: true,
-            },
-        );
+    pub fn commit_rotation(&self, new_id: SegmentId, handle: &Arc<CachedSegmentHandle<S>>) {
+        self.handles.write().insert(new_id, Arc::clone(handle));
    }

    pub fn seal_segment(&self, id: SegmentId, index: &SegmentIndex) -> io::Result<()> {
        let path = self.index_path(id);
-        index.save(&self.io, &path)?;
-        let mut cache = self.handles.write();
-        let entry = cache.get_mut(&id).ok_or_else(|| {
+        index.save(self.io.as_ref(), &path)?;
+        let cache = self.handles.read();
+        let entry = cache.get(&id).ok_or_else(|| {
            io::Error::new(
                io::ErrorKind::InvalidInput,
                format!("seal_segment: segment {id} not in handle cache"),
            )
        })?;
-        entry.sealed = true;
+        entry.sealed.store(true, Ordering::Release);
        Ok(())
    }

@@ -217,22 +226,16 @@ impl<S: StorageIO> SegmentManager<S> {
        self.handles
            .read()
            .get(&id)
-            .is_some_and(|entry| entry.sealed)
+            .is_some_and(|entry| entry.is_sealed())
    }

-    pub fn rollback_rotation(&self, new_id: SegmentId, fd: FileId) {
-        let _ = self.io.close(fd);
+    pub fn rollback_rotation(&self, new_id: SegmentId) {
        self.handles.write().remove(&new_id);
        let _ = self.io.delete(&self.segment_path(new_id));
    }

    pub fn delete_segment(&self, id: SegmentId) -> io::Result<()> {
-        {
-            let mut cache = self.handles.write();
-            if let Some(entry) = cache.remove(&id) {
-                let _ = self.io.close(entry.fd);
-            }
-        }
+        self.handles.write().remove(&id);
        [self.index_path(id), self.sidecar_path(id)]
            .iter()
            .try_for_each(|path| match self.io.delete(path) {
@@ -255,15 +258,7 @@ impl<S: StorageIO> SegmentManager<S> {
    }

    pub fn shutdown(&self) {
-        self.handles.write().drain().for_each(|(_, handle)| {
-            let _ = self.io.close(handle.fd);
-        });
-    }
-}
-
-impl<S: StorageIO> Drop for SegmentManager<S> {
-    fn drop(&mut self) {
-        self.shutdown();
+        self.handles.write().clear();
    }
 }

@@ -329,7 +324,7 @@ mod tests {
    #[test]
    fn open_for_append_creates_file() {
        let mgr = setup_manager(1024);
-        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
        assert_eq!(mgr.io().file_size(fd).unwrap(), 0);
    }

@@ -342,24 +337,24 @@ mod tests {
    #[test]
    fn handle_cache_returns_same_fd() {
        let mgr = setup_manager(1024);
-        let fd1 = mgr.open_for_append(SegmentId::new(1)).unwrap();
-        let fd2 = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd1 = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        let fd2 = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
        assert_eq!(fd1, fd2);
    }

    #[test]
    fn open_for_read_uses_cache_from_append() {
        let mgr = setup_manager(1024);
-        let fd_write = mgr.open_for_append(SegmentId::new(1)).unwrap();
-        let fd_read = mgr.open_for_read(SegmentId::new(1)).unwrap();
+        let fd_write = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        let fd_read = mgr.open_for_read(SegmentId::new(1)).unwrap().fd();
        assert_eq!(fd_write, fd_read);
    }

    #[test]
    fn list_segments_finds_segment_files() {
        let mgr = setup_manager(1024);
-        mgr.open_for_append(SegmentId::new(1)).unwrap();
-        mgr.open_for_append(SegmentId::new(3)).unwrap();
+        mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        mgr.open_for_append(SegmentId::new(3)).unwrap().fd();

        let segments = mgr.list_segments().unwrap();
        assert_eq!(segments, vec![SegmentId::new(1), SegmentId::new(3)]);
@@ -368,7 +363,7 @@ mod tests {
    #[test]
    fn list_segments_ignores_non_segment_files() {
        let mgr = setup_manager(1024);
-        mgr.open_for_append(SegmentId::new(1)).unwrap();
+        mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
        mgr.io()
            .open(Path::new("/segments/notes.txt"), OpenOptions::read_write())
            .unwrap();
@@ -380,7 +375,7 @@ mod tests {
    #[test]
    fn list_segments_ignores_index_files() {
        let mgr = setup_manager(1024);
-        mgr.open_for_append(SegmentId::new(1)).unwrap();
+        mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
        mgr.io()
            .open(
                Path::new("/segments/00000001.tqi"),
@@ -395,9 +390,9 @@ mod tests {
    #[test]
    fn list_segments_sorted_ascending() {
        let mgr = setup_manager(1024);
-        mgr.open_for_append(SegmentId::new(5)).unwrap();
-        mgr.open_for_append(SegmentId::new(1)).unwrap();
-        mgr.open_for_append(SegmentId::new(3)).unwrap();
+        mgr.open_for_append(SegmentId::new(5)).unwrap().fd();
+        mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        mgr.open_for_append(SegmentId::new(3)).unwrap().fd();

        let segments = mgr.list_segments().unwrap();
        assert_eq!(
@@ -418,23 +413,24 @@ mod tests {
    #[test]
    fn rotation_lifecycle_prepare_commit() {
        let mgr = setup_manager(1024);
-        let _fd0 = mgr.open_for_append(SegmentId::new(1)).unwrap();
-        let (next_id, next_fd) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
+        let _h0 = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        let (next_id, next_handle) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
        assert_eq!(next_id, SegmentId::new(2));
-        assert_eq!(mgr.io().file_size(next_fd).unwrap(), 0);
-        mgr.commit_rotation(next_id, next_fd);
-        assert_eq!(mgr.open_for_read(next_id).unwrap(), next_fd);
+        assert_eq!(mgr.io().file_size(next_handle.fd()).unwrap(), 0);
+        mgr.commit_rotation(next_id, &next_handle);
+        assert_eq!(mgr.open_for_read(next_id).unwrap().fd(), next_handle.fd());
    }

    #[test]
    fn rotation_rollback_cleans_up() {
        let mgr = setup_manager(1024);
-        let _fd0 = mgr.open_for_append(SegmentId::new(1)).unwrap();
-        let (next_id, next_fd) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
-        mgr.commit_rotation(next_id, next_fd);
+        let _h0 = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        let (next_id, next_handle) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
+        mgr.commit_rotation(next_id, &next_handle);

-        assert_eq!(mgr.open_for_read(next_id).unwrap(), next_fd);
-        mgr.rollback_rotation(next_id, next_fd);
+        assert_eq!(mgr.open_for_read(next_id).unwrap().fd(), next_handle.fd());
+        drop(next_handle);
+        mgr.rollback_rotation(next_id);

        let segments = mgr.list_segments().unwrap();
        assert_eq!(segments, vec![SegmentId::new(1)]);
@@ -443,7 +439,7 @@ mod tests {
    #[test]
    fn seal_segment_persists_index_and_marks_sealed() {
        let mgr = setup_manager(64 * 1024);
-        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
        let mut writer = SegmentWriter::new(
            mgr.io(),
            fd,
@@ -476,7 +472,7 @@ mod tests {
    #[test]
    fn delete_segment_removes_files_and_handle() {
        let mgr = setup_manager(64 * 1024);
-        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
        let mut writer = SegmentWriter::new(
            mgr.io(),
            fd,
@@ -507,9 +503,9 @@ mod tests {
        let mgr = setup_manager(1024);
        assert_eq!(mgr.oldest_segment().unwrap(), None);

-        mgr.open_for_append(SegmentId::new(3)).unwrap();
-        mgr.open_for_append(SegmentId::new(1)).unwrap();
-        mgr.open_for_append(SegmentId::new(5)).unwrap();
+        mgr.open_for_append(SegmentId::new(3)).unwrap().fd();
+        mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        mgr.open_for_append(SegmentId::new(5)).unwrap().fd();

        assert_eq!(mgr.oldest_segment().unwrap(), Some(SegmentId::new(1)));
    }
@@ -524,7 +520,7 @@ mod tests {
    fn rotate_and_write_across_segments() {
        let mgr = setup_manager(1024);

-        let fd1 = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd1 = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
        let mut writer1 = SegmentWriter::new(
            mgr.io(),
            fd1,
@@ -538,8 +534,9 @@ mod tests {
            .unwrap();
        writer1.sync(mgr.io()).unwrap();

-        let (id2, fd2) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
-        mgr.commit_rotation(id2, fd2);
+        let (id2, handle2) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
+        let fd2 = handle2.fd();
+        mgr.commit_rotation(id2, &handle2);

        let mut writer2 =
            SegmentWriter::new(mgr.io(), fd2, id2, EventSequence::new(2), MAX_EVENT_PAYLOAD)
@@ -549,7 +546,7 @@ mod tests {
            .unwrap();
        writer2.sync(mgr.io()).unwrap();

-        let fd1_read = mgr.open_for_read(SegmentId::new(1)).unwrap();
+        let fd1_read = mgr.open_for_read(SegmentId::new(1)).unwrap().fd();
        let events1 = crate::eventlog::SegmentReader::open(mgr.io(), fd1_read, MAX_EVENT_PAYLOAD)
            .unwrap()
            .valid_prefix()
@@ -557,7 +554,7 @@ mod tests {
        assert_eq!(events1.len(), 1);
        assert_eq!(events1[0].payload, b"first segment");

-        let fd2_read = mgr.open_for_read(id2).unwrap();
+        let fd2_read = mgr.open_for_read(id2).unwrap().fd();
        let events2 = crate::eventlog::SegmentReader::open(mgr.io(), fd2_read, MAX_EVENT_PAYLOAD)
            .unwrap()
            .valid_prefix()
@@ -569,7 +566,7 @@ mod tests {
    #[test]
    fn seal_then_append_errors() {
        let mgr = setup_manager(64 * 1024);
-        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
        SegmentWriter::new(
            mgr.io(),
            fd,
@@ -596,9 +593,9 @@ mod tests {
    #[test]
    fn multiple_deletions_increment_epoch() {
        let mgr = setup_manager(1024);
-        mgr.open_for_append(SegmentId::new(1)).unwrap();
-        mgr.open_for_append(SegmentId::new(2)).unwrap();
-        mgr.open_for_append(SegmentId::new(3)).unwrap();
+        mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        mgr.open_for_append(SegmentId::new(2)).unwrap().fd();
+        mgr.open_for_append(SegmentId::new(3)).unwrap().fd();

        assert_eq!(mgr.retention_epoch(), 0);
        mgr.delete_segment(SegmentId::new(1)).unwrap();
@@ -610,7 +607,7 @@ mod tests {
    #[test]
    fn open_for_read_does_not_infer_sealed_from_index_file() {
        let mgr = setup_manager(64 * 1024);
-        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
        let mut writer = SegmentWriter::new(
            mgr.io(),
            fd,
@@ -630,26 +627,26 @@ mod tests {

        mgr.handles.write().remove(&SegmentId::new(1));

-        let _read_fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
+        let _read_fd = mgr.open_for_read(SegmentId::new(1)).unwrap().fd();
        assert!(!mgr.is_sealed(SegmentId::new(1)));
    }

    #[test]
    fn open_for_read_unsealed_allows_append() {
        let mgr = setup_manager(1024);
-        let _fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let _fd = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();

        mgr.handles.write().remove(&SegmentId::new(1));

-        let _read_fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
+        let _read_fd = mgr.open_for_read(SegmentId::new(1)).unwrap().fd();
        assert!(!mgr.is_sealed(SegmentId::new(1)));
    }

    #[test]
    fn shutdown_clears_handles() {
        let mgr = setup_manager(1024);
-        mgr.open_for_append(SegmentId::new(1)).unwrap();
-        mgr.open_for_append(SegmentId::new(2)).unwrap();
+        mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        mgr.open_for_append(SegmentId::new(2)).unwrap().fd();

        mgr.shutdown();
        assert!(mgr.handles.read().is_empty());
@@ -665,7 +662,7 @@ mod tests {
    #[test]
    fn prepare_rotation_truncates_stale_file() {
        let mgr = setup_manager(1024);
-        let _fd0 = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let _fd0 = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();

        let stale_path = mgr.segment_path(SegmentId::new(2));
        let stale_fd = mgr
@@ -677,23 +674,23 @@ mod tests {
        assert_eq!(mgr.io().file_size(stale_fd).unwrap(), 4096);
        mgr.io().close(stale_fd).unwrap();

-        let (next_id, next_fd) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
+        let (next_id, next_handle) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
        assert_eq!(next_id, SegmentId::new(2));
-        assert_eq!(mgr.io().file_size(next_fd).unwrap(), 0);
+        assert_eq!(mgr.io().file_size(next_handle.fd()).unwrap(), 0);
    }

    #[test]
    fn open_for_append_upgrades_read_only_handle() {
        let mgr = setup_manager(1024);
-        let fd_append = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd_append = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();

        mgr.handles.write().remove(&SegmentId::new(1));

-        let fd_read = mgr.open_for_read(SegmentId::new(1)).unwrap();
+        let fd_read = mgr.open_for_read(SegmentId::new(1)).unwrap().fd();
        assert_ne!(fd_read, fd_append);
        assert!(!mgr.handles.read().get(&SegmentId::new(1)).unwrap().writable);

-        let fd_upgraded = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd_upgraded = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
        assert_ne!(fd_upgraded, fd_read);
        assert!(mgr.handles.read().get(&SegmentId::new(1)).unwrap().writable);
    }
@@ -411,8 +411,8 @@ impl<S: StorageIO + 'static> EventLog<S> {
    pub fn disk_usage(&self) -> io::Result<u64> {
        let segments = self.manager.list_segments()?;
        segments.iter().try_fold(0u64, |acc, &id| {
-            let fd = self.manager.open_for_read(id)?;
-            let size = self.manager.io().file_size(fd)?;
+            let handle = self.manager.open_for_read(id)?;
+            let size = self.manager.io().file_size(handle.fd())?;
            Ok(acc.saturating_add(size))
        })
    }
@@ -145,10 +145,10 @@ impl<S: StorageIO> EventLogReader<S> {
    }

    fn rebuild_index(&self, segment_id: SegmentId) -> io::Result<SegmentIndex> {
-        let fd = self.manager.open_for_read(segment_id)?;
+        let handle = self.manager.open_for_read(segment_id)?;
        let (idx, _) = rebuild_from_segment(
            self.manager.io(),
-            fd,
+            handle.fd(),
            DEFAULT_INDEX_INTERVAL,
            self.max_payload,
        )?;
@@ -243,8 +243,8 @@ impl<S: StorageIO> EventLogReader<S> {
            return Ok(Arc::clone(m));
        }

-        let fd = self.manager.open_for_read(segment_id)?;
-        let mapped = self.manager.io().mmap_file(fd)?;
+        let handle = self.manager.open_for_read(segment_id)?;
+        let mapped = self.manager.io().mmap_file(handle.fd())?;
        let arc = Arc::new(mapped);
        self.mmaps.write().insert(segment_id, Arc::clone(&arc));
        Ok(arc)
@@ -269,10 +269,10 @@ impl<S: StorageIO> EventLogReader<S> {
                predicate,
            )
        } else {
-            let fd = self.manager.open_for_read(segment_id)?;
-            let file_size = self.manager.io().file_size(fd)?;
+            let handle = self.manager.open_for_read(segment_id)?;
+            let file_size = self.manager.io().file_size(handle.fd())?;
            self.scan_direct(
-                fd,
+                handle.fd(),
                file_size,
                start_offset,
                start_seq,
@@ -513,8 +513,8 @@ impl<S: StorageIO> EventLogReader<S> {
    }

    fn rebuild_sidecar(&self, segment_id: SegmentId) -> io::Result<SidecarIndex> {
-        let fd = self.manager.open_for_read(segment_id)?;
-        let sidecar = build_sidecar_from_segment(self.manager.io(), fd, self.max_payload)?;
+        let handle = self.manager.open_for_read(segment_id)?;
+        let sidecar = build_sidecar_from_segment(self.manager.io(), handle.fd(), self.max_payload)?;
        let _ = sidecar.save(self.manager.io(), &self.manager.sidecar_path(segment_id));
        Ok(sidecar)
    }
@@ -552,7 +552,7 @@ fn decode_mmap_event(
            segment = %segment_id,
            offset = raw,
            file_size,
-            "decode offset past file size (corrupt index?)"
+            "decode offset past file size, index likely corrupt"
        );
        return Ok(MmapDecodeResult::Corrupted);
    }
@@ -3,16 +3,27 @@ use std::sync::Arc;

 use tracing::warn;

-use crate::io::StorageIO;
+use crate::io::{FileId, StorageIO};

 use super::manager::SegmentManager;
-use super::segment_file::{SEGMENT_HEADER_SIZE, SegmentWriter, ValidEvent};
+use super::segment_file::{
+    SEGMENT_HEADER_SIZE, SEGMENT_MAGIC, SegmentWriter, ValidEvent, ValidateEventRecord,
+    validate_event_record,
+};
 use super::segment_index::{DEFAULT_INDEX_INTERVAL, SegmentIndex, rebuild_from_segment};
 use super::sidecar::build_sidecar_from_segment;
 use super::types::{
    DidHash, EventSequence, EventTypeTag, SegmentId, SegmentOffset, TimestampMicros,
 };

+const VALIDATE_RETRY_ATTEMPTS: u32 = 32;
+
+#[derive(Debug, Clone)]
+struct PendingAppend {
+    event: ValidEvent,
+    offset: SegmentOffset,
+}
+
 #[derive(Debug)]
 pub struct SyncResult {
    pub synced_through: EventSequence,
@@ -31,7 +42,8 @@ pub struct EventLogWriter<S: StorageIO> {
    max_payload: u32,
    event_count_in_segment: usize,
    last_event_offset: Option<SegmentOffset>,
-    pending_events: Vec<ValidEvent>,
+    pending: Vec<PendingAppend>,
+    poisoned: bool,
 }

 impl<S: StorageIO> EventLogWriter<S> {
@@ -66,9 +78,10 @@ impl<S: StorageIO> EventLogWriter<S> {
        index_interval: usize,
        max_payload: u32,
    ) -> io::Result<Self> {
-        let fd = manager.open_for_append(segment_id)?;
-        manager.io().truncate(fd, 0)?;
-        let writer = SegmentWriter::new(manager.io(), fd, segment_id, next_seq, max_payload)?;
+        let handle = manager.open_for_append(segment_id)?;
+        manager.io().truncate(handle.fd(), 0)?;
+        let writer =
+            SegmentWriter::new(manager.io(), handle.fd(), segment_id, next_seq, max_payload)?;
        writer.sync(manager.io())?;
        manager.io().sync_dir(manager.segments_dir())?;

@@ -82,10 +95,25 @@ impl<S: StorageIO> EventLogWriter<S> {
            max_payload,
            event_count_in_segment: 0,
            last_event_offset: None,
-            pending_events: Vec::new(),
+            pending: Vec::new(),
+            poisoned: false,
        })
    }

+    fn truncate_and_init_fresh(
+        manager: Arc<SegmentManager<S>>,
+        fd: FileId,
+        active_id: SegmentId,
+        prev_segments: &[SegmentId],
+        index_interval: usize,
+        max_payload: u32,
+    ) -> io::Result<Self> {
+        manager.io().truncate(fd, 0)?;
+        let next_seq = find_last_seq_from_segments(&manager, prev_segments, max_payload)?
+            .map_or(EventSequence::new(1), |s| s.next());
+        Self::init_fresh(manager, active_id, next_seq, index_interval, max_payload)
+    }
+
    fn recover_active(
        manager: Arc<SegmentManager<S>>,
        segments: &[SegmentId],
@@ -93,7 +121,21 @@ impl<S: StorageIO> EventLogWriter<S> {
        index_interval: usize,
        max_payload: u32,
    ) -> io::Result<Self> {
-        let fd = manager.open_for_append(active_id)?;
+        let handle = manager.open_for_append(active_id)?;
+        let fd = handle.fd();
+
+        let prev_segments = &segments[..segments.len().saturating_sub(1)];
+
+        if highest_segment_has_torn_header(manager.io(), fd)? {
+            return Self::truncate_and_init_fresh(
+                Arc::clone(&manager),
+                fd,
+                active_id,
+                prev_segments,
+                index_interval,
+                max_payload,
+            );
+        }

        let (index, last_seq_in_active) = match rebuild_from_segment(
            manager.io(),
@@ -105,15 +147,11 @@ impl<S: StorageIO> EventLogWriter<S> {
            Err(rebuild_err) => {
                let file_size = manager.io().file_size(fd)?;
                if file_size <= SEGMENT_HEADER_SIZE as u64 {
-                    manager.io().truncate(fd, 0)?;
-                    let prev_segments = &segments[..segments.len().saturating_sub(1)];
-                    let next_seq =
-                        find_last_seq_from_segments(&manager, prev_segments, max_payload)?
-                            .map_or(EventSequence::new(1), |s| s.next());
-                    return Self::init_fresh(
+                    return Self::truncate_and_init_fresh(
                        Arc::clone(&manager),
+                        fd,
                        active_id,
-                        next_seq,
+                        prev_segments,
                        index_interval,
                        max_payload,
                    );
@@ -129,8 +167,6 @@ impl<S: StorageIO> EventLogWriter<S> {

        let position = SegmentOffset::new(manager.io().file_size(fd)?);

-        let prev_segments = &segments[..segments.len().saturating_sub(1)];
-
        let next_seq = match last_seq_in_active {
            Some(seq) => {
                if let Some(sealed_last) =
@@ -194,7 +230,8 @@ impl<S: StorageIO> EventLogWriter<S> {
            max_payload,
            event_count_in_segment,
            last_event_offset,
-            pending_events: Vec::new(),
+            pending: Vec::new(),
+            poisoned: false,
        })
    }

@@ -225,28 +262,20 @@ impl<S: StorageIO> EventLogWriter<S> {
            payload,
        };

-        let offset = self.active_writer.append_event(self.manager.io(), &event)?;
-
-        let should_index = self.event_count_in_segment == 0
-            || self
-                .event_count_in_segment
-                .is_multiple_of(self.index_interval);
-        if should_index {
-            self.active_index.record(seq, offset);
-        }
-
-        self.event_count_in_segment = self
-            .event_count_in_segment
-            .checked_add(1)
-            .expect("event_count_in_segment overflow");
-        self.last_event_offset = Some(offset);
-        self.next_seq = seq.next();
-        self.pending_events.push(event);
-
-        Ok(seq)
+        self.append_inner(event).map(|_| seq)
    }

    pub fn append_valid_event(&mut self, event: ValidEvent) -> io::Result<()> {
+        self.append_inner(event)
+    }
+
+    fn append_inner(&mut self, event: ValidEvent) -> io::Result<()> {
+        if self.poisoned {
+            return Err(io::Error::other(
+                "writer poisoned by partial-valid sync; reopen required",
+            ));
+        }
+
        let offset = self.active_writer.append_event(self.manager.io(), &event)?;

        let should_index = self.event_count_in_segment == 0
@@ -263,21 +292,52 @@ impl<S: StorageIO> EventLogWriter<S> {
            .expect("event_count_in_segment overflow");
        self.last_event_offset = Some(offset);
        self.next_seq = event.seq.next();
-        self.pending_events.push(event);
+        self.pending.push(PendingAppend { event, offset });

        Ok(())
    }

-    pub fn peek_pending_event(&self, seq: EventSequence) -> Option<&ValidEvent> {
-        self.pending_events.iter().find(|e| e.seq == seq)
-    }
-
    pub fn sync(&mut self) -> io::Result<SyncResult> {
-        if !self.pending_events.is_empty() {
-            self.active_writer.sync(self.manager.io())?;
+        if self.poisoned {
+            return Err(io::Error::other(
+                "writer poisoned by partial-valid sync; reopen required",
+            ));
        }

-        let flushed = std::mem::take(&mut self.pending_events);
+        if !self.pending.is_empty() {
+            self.active_writer.sync(self.manager.io())?;
+            self.manager.io().barrier()?;
+        }
+
+        let pending = std::mem::take(&mut self.pending);
+
+        let fd = self.active_writer.fd();
+        let file_size = self.manager.io().file_size(fd)?;
+
+        let valid_count = pending
+            .iter()
+            .take_while(|p| {
+                validate_with_retry(
+                    self.manager.io(),
+                    fd,
+                    p.offset,
+                    file_size,
+                    self.max_payload,
+                    p.event.seq,
+                )
+            })
+            .count();
+
+        if valid_count < pending.len() {
+            self.poisoned = true;
+        }
+
+        let flushed: Vec<ValidEvent> = pending
+            .into_iter()
+            .take(valid_count)
+            .map(|p| p.event)
+            .collect();
+
        self.synced_seq = flushed.last().map(|e| e.seq).unwrap_or(self.synced_seq);

        Ok(SyncResult {
@@ -288,12 +348,22 @@ impl<S: StorageIO> EventLogWriter<S> {
        })
    }

+    pub fn is_poisoned(&self) -> bool {
+        self.poisoned
+    }
+
    pub fn rotate_if_needed(&mut self) -> io::Result<Option<SegmentId>> {
+        if self.poisoned {
+            return Err(io::Error::other(
+                "writer poisoned by partial-valid sync; reopen required",
+            ));
+        }
+
        if !self.manager.should_rotate(self.active_writer.position()) {
            return Ok(None);
        }

-        if !self.pending_events.is_empty() {
+        if !self.pending.is_empty() {
            return Ok(None);
        }

@@ -305,14 +375,14 @@ impl<S: StorageIO> EventLogWriter<S> {

        match self.build_sidecar_for_segment(old_id) {
            Ok(()) => {}
-            Err(e) => warn!(segment = %old_id, error = %e, "sidecar build failed (non-fatal)"),
+            Err(e) => warn!(segment = %old_id, error = %e, "non-fatal sidecar build failure"),
        }

-        let (new_id, new_fd) = self.manager.prepare_rotation(old_id)?;
+        let (new_id, new_handle) = self.manager.prepare_rotation(old_id)?;

        match SegmentWriter::new::<S>(
            self.manager.io(),
-            new_fd,
+            new_handle.fd(),
            new_id,
            self.next_seq,
            self.max_payload,
@@ -322,11 +392,12 @@ impl<S: StorageIO> EventLogWriter<S> {
                self.active_index = SegmentIndex::new();
                self.event_count_in_segment = 0;
                self.last_event_offset = None;
-                self.manager.commit_rotation(new_id, new_fd);
+                self.manager.commit_rotation(new_id, &new_handle);
                Ok(Some(old_id))
            }
            Err(e) => {
-                self.manager.rollback_rotation(new_id, new_fd);
+                drop(new_handle);
+                self.manager.rollback_rotation(new_id);
                Err(e)
            }
        }
@@ -361,8 +432,8 @@ impl<S: StorageIO> EventLogWriter<S> {
    }

    fn build_sidecar_for_segment(&self, segment_id: SegmentId) -> io::Result<()> {
-        let fd = self.manager.open_for_read(segment_id)?;
-        let sidecar = build_sidecar_from_segment(self.manager.io(), fd, self.max_payload)?;
+        let handle = self.manager.open_for_read(segment_id)?;
+        let sidecar = build_sidecar_from_segment(self.manager.io(), handle.fd(), self.max_payload)?;
        let path = self.manager.sidecar_path(segment_id);
        sidecar.save(self.manager.io(), &path)
    }
@@ -383,6 +454,40 @@ impl<S: StorageIO> EventLogWriter<S> {
    }
 }

+fn validate_with_retry<S: StorageIO>(
+    io: &S,
+    fd: FileId,
+    offset: SegmentOffset,
+    file_size: u64,
+    max_payload: u32,
+    expected_seq: EventSequence,
+) -> bool {
+    (0..VALIDATE_RETRY_ATTEMPTS).any(|_| {
+        matches!(
+            validate_event_record(io, fd, offset, file_size, max_payload),
+            Ok(Some(ValidateEventRecord::Valid { seq, .. })) if seq == expected_seq
+        )
+    })
+}
+
+fn highest_segment_has_torn_header<S: StorageIO>(io: &S, fd: FileId) -> io::Result<bool> {
+    let file_size = io.file_size(fd)?;
+    if file_size < SEGMENT_HEADER_SIZE as u64 {
+        return Ok(true);
+    }
+    let outcomes: Vec<bool> = (0..VALIDATE_RETRY_ATTEMPTS)
+        .filter_map(|_| {
+            let mut header = [0u8; SEGMENT_MAGIC.len()];
+            io.read_exact_at(fd, 0, &mut header)
+                .ok()
+                .map(|()| header == SEGMENT_MAGIC)
+        })
+        .collect();
+    let saw_match = outcomes.iter().any(|&ok| ok);
+    let saw_mismatch = outcomes.iter().any(|&ok| !ok);
+    Ok(!saw_match && saw_mismatch)
+}
+
 fn find_last_seq_from_segments<S: StorageIO>(
    manager: &SegmentManager<S>,
    segments: &[SegmentId],
@@ -397,9 +502,13 @@ fn find_last_seq_from_segments<S: StorageIO>(
            Ok(Some(idx)) => Ok(idx.last_seq()),
            Err(e) if e.kind() != io::ErrorKind::InvalidData => Err(e),
            _ => {
-                let fd = manager.open_for_read(seg_id)?;
-                let (_, last_seq) =
-                    rebuild_from_segment(manager.io(), fd, DEFAULT_INDEX_INTERVAL, max_payload)?;
+                let handle = manager.open_for_read(seg_id)?;
+                let (_, last_seq) = rebuild_from_segment(
+                    manager.io(),
+                    handle.fd(),
+                    DEFAULT_INDEX_INTERVAL,
+                    max_payload,
+                )?;
                Ok(last_seq)
            }
        }
@@ -551,7 +660,7 @@ mod tests {
        assert_eq!(writer.synced_seq(), EventSequence::new(5));
        assert_eq!(writer.active_segment_id(), SegmentId::new(1));

-        let fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
+        let fd = mgr.open_for_read(SegmentId::new(1)).unwrap().fd();
        let events = SegmentReader::open(mgr.io(), fd, MAX_EVENT_PAYLOAD)
            .unwrap()
            .valid_prefix()
@@ -796,7 +905,7 @@ mod tests {
                .unwrap();
        assert_eq!(writer.next_seq, EventSequence::new(3));

-        let fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
+        let fd = mgr.open_for_read(SegmentId::new(1)).unwrap().fd();
        let events = SegmentReader::open(mgr.io(), fd, MAX_EVENT_PAYLOAD)
            .unwrap()
            .valid_prefix()
@@ -1003,7 +1112,7 @@ mod tests {
        assert_eq!(seq, EventSequence::new(4));
        writer.sync().unwrap();

-        let fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
+        let fd = mgr.open_for_read(SegmentId::new(1)).unwrap().fd();
        let events = SegmentReader::open(mgr.io(), fd, MAX_EVENT_PAYLOAD)
            .unwrap()
            .valid_prefix()
@@ -1087,4 +1196,62 @@ mod tests {

        assert!(writer.rotate_if_needed().unwrap().is_none());
    }
+
+    #[test]
+    fn sync_must_not_certify_durability_when_io_sync_silently_drops() {
+        use crate::sim::{FaultConfig, Probability};
+
+        let sim = Arc::new(SimulatedIO::new(
+            0,
+            FaultConfig {
+                sync_failure_probability: Probability::new(1.0),
+                ..FaultConfig::none()
+            },
+        ));
+        sim.set_pristine_mode(true);
+
+        let mgr = Arc::new(
+            SegmentManager::new(Arc::clone(&sim), PathBuf::from("/segments"), 64 * 1024).unwrap(),
+        );
+
+        let mut writer =
+            EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
+                .unwrap();
+
+        sim.set_pristine_mode(false);
+
+        writer
+            .append(
+                DidHash::from_did("did:plc:bug2"),
+                EventTypeTag::COMMIT,
+                b"bug2-payload".to_vec(),
+            )
+            .unwrap();
+
+        assert!(
+            writer.sync().is_err(),
+            "sync must surface dropped fsync as an error"
+        );
+        let claimed_synced = writer.synced_seq();
+        assert_eq!(
+            claimed_synced.raw(),
+            0,
+            "synced_seq must not advance past a failed sync"
+        );
+        drop(writer);
+
+        mgr.shutdown();
+        sim.crash();
+        sim.set_pristine_mode(true);
+
+        let reopened =
+            EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
+                .unwrap();
+        let actually_durable = reopened.current_seq();
+
+        assert!(
+            actually_durable >= claimed_synced,
+            "writer claimed sync through {claimed_synced} but post-crash recovery only reaches {actually_durable}"
+        );
+    }
 }
@@ -0,0 +1,169 @@
+use std::cell::RefCell;
+use std::panic::{AssertUnwindSafe, catch_unwind};
+use std::path::PathBuf;
+use std::time::{Duration, Instant};
+
+use rayon::prelude::*;
+use tokio::runtime::Runtime;
+
+use super::invariants::InvariantViolation;
+use super::op::{OpStream, Seed};
+use super::runner::{
+    Gauntlet, GauntletConfig, GauntletReport, OpErrorCount, OpsExecuted, RestartCount,
+};
+
+thread_local! {
+    static RUNTIME: RefCell<Option<Runtime>> = const { RefCell::new(None) };
+}
+
+fn with_runtime<R>(f: impl FnOnce(&Runtime) -> R) -> R {
+    RUNTIME.with(|cell| {
+        let mut slot = cell.borrow_mut();
+        if slot.is_none() {
+            *slot = Some(
+                tokio::runtime::Builder::new_current_thread()
+                    .enable_all()
+                    .build()
+                    .expect("build rt"),
+            );
+        }
+        f(slot.as_ref().expect("runtime present"))
+    })
+}
+
+pub fn run_many<F>(make_config: F, seeds: impl IntoIterator<Item = Seed>) -> Vec<GauntletReport>
+where
+    F: Fn(Seed) -> GauntletConfig + Sync + Send,
+{
+    run_many_timed(make_config, seeds)
+        .into_iter()
+        .map(|(r, _)| r)
+        .collect()
+}
+
+pub fn run_many_timed<F>(
+    make_config: F,
+    seeds: impl IntoIterator<Item = Seed>,
+) -> Vec<(GauntletReport, Duration)>
+where
+    F: Fn(Seed) -> GauntletConfig + Sync + Send,
+{
+    run_many_timed_with_scratch_roots(make_config, &[], seeds)
+}
+
+pub fn run_many_timed_with_scratch_roots<F>(
+    make_config: F,
+    scratch_roots: &[PathBuf],
+    seeds: impl IntoIterator<Item = Seed>,
+) -> Vec<(GauntletReport, Duration)>
+where
+    F: Fn(Seed) -> GauntletConfig + Sync + Send,
+{
+    let seeds: Vec<Seed> = seeds.into_iter().collect();
+    seeds
+        .into_par_iter()
+        .map(|s| {
+            let scratch = scratch_for_thread(scratch_roots, rayon::current_thread_index());
+            let start = Instant::now();
+            let outcome = catch_unwind(AssertUnwindSafe(|| {
+                let cfg = make_config(s);
+                let mut gauntlet = Gauntlet::new(cfg).expect("build gauntlet");
+                if let Some(root) = scratch {
+                    gauntlet = gauntlet.with_scratch_root(root);
+                }
+                with_runtime(|rt| rt.block_on(gauntlet.run()))
+            }));
+            let report = outcome.unwrap_or_else(|payload| {
+                RUNTIME.with(|cell| cell.borrow_mut().take());
+                panic_report(s, payload)
+            });
+            (report, start.elapsed())
+        })
+        .collect()
+}
+
+fn scratch_for_thread(roots: &[PathBuf], thread_idx: Option<usize>) -> Option<PathBuf> {
+    if roots.is_empty() {
+        None
+    } else {
+        Some(roots[thread_idx.unwrap_or(0) % roots.len()].clone())
+    }
+}
+
+fn panic_report(seed: Seed, payload: Box<dyn std::any::Any + Send>) -> GauntletReport {
+    let msg = payload
+        .downcast_ref::<&'static str>()
+        .map(|s| (*s).to_string())
+        .or_else(|| payload.downcast_ref::<String>().cloned())
+        .unwrap_or_else(|| "non-string panic payload".to_string());
+    GauntletReport {
+        seed,
+        ops_executed: OpsExecuted(0),
+        op_errors: OpErrorCount(0),
+        restarts: RestartCount(0),
+        violations: vec![InvariantViolation {
+            invariant: "FarmPanic",
+            detail: msg,
+        }],
+        ops: OpStream::empty(),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn scratch_for_thread_returns_none_when_roots_empty() {
+        assert!(scratch_for_thread(&[], Some(0)).is_none());
+        assert!(scratch_for_thread(&[], Some(7)).is_none());
+        assert!(scratch_for_thread(&[], None).is_none());
+    }
+
+    #[test]
+    fn scratch_for_thread_round_robins_across_roots() {
+        let roots = vec![
+            PathBuf::from("/scratch/a"),
+            PathBuf::from("/scratch/b"),
+            PathBuf::from("/scratch/c"),
+        ];
+        let assigned: Vec<PathBuf> = (0..7)
+            .map(|i| scratch_for_thread(&roots, Some(i)).expect("scratch path"))
+            .collect();
+        assert_eq!(
+            assigned,
+            vec![
+                PathBuf::from("/scratch/a"),
+                PathBuf::from("/scratch/b"),
+                PathBuf::from("/scratch/c"),
+                PathBuf::from("/scratch/a"),
+                PathBuf::from("/scratch/b"),
+                PathBuf::from("/scratch/c"),
+                PathBuf::from("/scratch/a"),
+            ]
+        );
+    }
+
+    #[test]
+    fn scratch_for_thread_with_single_root_returns_same_path() {
+        let roots = vec![PathBuf::from("/scratch/only")];
+        (0..5).for_each(|i| {
+            assert_eq!(
+                scratch_for_thread(&roots, Some(i)),
+                Some(PathBuf::from("/scratch/only"))
+            );
+        });
+    }
+
+    #[test]
+    fn scratch_for_thread_falls_back_to_root_zero_outside_pool() {
+        let roots = vec![
+            PathBuf::from("/scratch/a"),
+            PathBuf::from("/scratch/b"),
+        ];
+        assert_eq!(
+            scratch_for_thread(&roots, None),
+            Some(PathBuf::from("/scratch/a"))
+        );
+    }
+}
@@ -0,0 +1,450 @@
+use std::env;
+use std::num::NonZeroU32;
+use std::path::{Path, PathBuf};
+use std::process::{Command, Output};
+use std::sync::atomic::{AtomicU64, Ordering};
+
+use tempfile::TempDir;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct UpIntervalSecs(pub NonZeroU32);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct DownIntervalSecs(pub NonZeroU32);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct BackingMegabytes(pub u32);
+
+const fn nz(n: u32) -> NonZeroU32 {
+    match NonZeroU32::new(n) {
+        Some(v) => v,
+        None => panic!("zero interval not permitted"),
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct FlakyConfig {
+    pub up_interval: UpIntervalSecs,
+    pub down_interval: DownIntervalSecs,
+    pub backing_mb: BackingMegabytes,
+}
+
+impl FlakyConfig {
+    pub const fn default_stress() -> Self {
+        Self {
+            up_interval: UpIntervalSecs(nz(8)),
+            down_interval: DownIntervalSecs(nz(2)),
+            backing_mb: BackingMegabytes(256),
+        }
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum FlakyError {
+    #[error("not running as root, EUID != 0")]
+    NotRoot,
+    #[error("tool missing: {0}")]
+    ToolMissing(&'static str),
+    #[error("kernel target dm-flakey unavailable: {0}")]
+    DmFlakeyMissing(String),
+    #[error("{tool} failed: status={status}, stderr={stderr}")]
+    CommandFailed {
+        tool: &'static str,
+        status: String,
+        stderr: String,
+    },
+    #[error("io: {0}")]
+    Io(#[from] std::io::Error),
+}
+
+impl FlakyError {
+    pub const fn is_env_absent(&self) -> bool {
+        matches!(
+            self,
+            Self::NotRoot | Self::ToolMissing(_) | Self::DmFlakeyMissing(_)
+        )
+    }
+}
+
+static MOUNT_COUNTER: AtomicU64 = AtomicU64::new(0);
+
+pub struct FlakyMount {
+    mount_point: TempDir,
+    mapper_name: String,
+    mapper_path: PathBuf,
+    loop_device: PathBuf,
+    backing_file: PathBuf,
+    _backing_tempdir: TempDir,
+}
+
+impl FlakyMount {
+    pub fn try_new(cfg: &FlakyConfig) -> Result<Self, FlakyError> {
+        ensure_root()?;
+        ["losetup", "dmsetup", "mkfs.ext4", "mount", "umount"]
+            .iter()
+            .copied()
+            .try_for_each(ensure_tool)?;
+        probe_dm_flakey()?;
+        let _ = reap_stale_mounts();
+
+        let backing_dir = TempDir::new()?;
+        let backing_file = backing_dir.path().join("backing.img");
+        allocate_backing(&backing_file, cfg.backing_mb)?;
+
+        let loop_device = attach_loop(&backing_file)?;
+        let sectors = sector_count(&loop_device)?;
+
+        if let Err(e) = mkfs_ext4(&loop_device) {
+            let _ = detach_loop(&loop_device);
+            return Err(e);
+        }
+
+        let mapper_name = format!(
+            "tranquil-flaky-{}-{}",
+            std::process::id(),
+            MOUNT_COUNTER.fetch_add(1, Ordering::Relaxed),
+        );
+        match dm_create(
+            &mapper_name,
+            &loop_device,
+            sectors,
+            cfg.up_interval,
+            cfg.down_interval,
+        ) {
+            Ok(()) => {}
+            Err(e) => {
+                let _ = detach_loop(&loop_device);
+                return Err(e);
+            }
+        }
+        let mapper_path = PathBuf::from(format!("/dev/mapper/{mapper_name}"));
+
+        let mount_point = TempDir::new()?;
+        if let Err(e) = mount_ext4(&mapper_path, mount_point.path()) {
+            let _ = dm_remove(&mapper_name);
+            let _ = detach_loop(&loop_device);
+            return Err(e);
+        }
+
+        Ok(Self {
+            mount_point,
+            mapper_name,
+            mapper_path,
+            loop_device,
+            backing_file,
+            _backing_tempdir: backing_dir,
+        })
+    }
+
+    pub fn path(&self) -> &Path {
+        self.mount_point.path()
+    }
+
+    pub fn mapper_name(&self) -> &str {
+        &self.mapper_name
+    }
+
+    pub fn mapper_path(&self) -> &Path {
+        &self.mapper_path
+    }
+
+    pub fn loop_device(&self) -> &Path {
+        &self.loop_device
+    }
+
+    pub fn backing_file(&self) -> &Path {
+        &self.backing_file
+    }
+}
+
+impl Drop for FlakyMount {
+    fn drop(&mut self) {
+        if let Err(e) = umount(self.mount_point.path()) {
+            tracing::warn!(
+                mount = %self.mount_point.path().display(),
+                error = %e,
+                "flaky umount failed, trying lazy unmount",
+            );
+            if let Err(e2) = umount_lazy(self.mount_point.path()) {
+                tracing::warn!(
+                    mount = %self.mount_point.path().display(),
+                    error = %e2,
+                    "flaky lazy unmount also failed, device may leak",
+                );
+            }
+        }
+        if let Err(e) = dm_remove(&self.mapper_name) {
+            tracing::warn!(
+                name = %self.mapper_name,
+                error = %e,
+                "flaky dm remove failed, mapper device may leak",
+            );
+        }
+        if let Err(e) = detach_loop(&self.loop_device) {
+            tracing::warn!(
+                device = %self.loop_device.display(),
+                error = %e,
+                "flaky loop detach failed, loop device may leak",
+            );
+        }
+    }
+}
+
+#[cfg(unix)]
+fn ensure_root() -> Result<(), FlakyError> {
+    if unsafe { libc::geteuid() } == 0 {
+        Ok(())
+    } else {
+        Err(FlakyError::NotRoot)
+    }
+}
+
+#[cfg(not(unix))]
+fn ensure_root() -> Result<(), FlakyError> {
+    Err(FlakyError::NotRoot)
+}
+
+fn ensure_tool(tool: &'static str) -> Result<(), FlakyError> {
+    match find_in_path(tool) {
+        Some(_) => Ok(()),
+        None => Err(FlakyError::ToolMissing(tool)),
+    }
+}
+
+fn find_in_path(tool: &str) -> Option<PathBuf> {
+    let path = env::var_os("PATH")?;
+    env::split_paths(&path).find_map(|dir| {
+        let candidate = dir.join(tool);
+        is_executable_file(&candidate).then_some(candidate)
+    })
+}
+
+#[cfg(unix)]
+fn is_executable_file(p: &Path) -> bool {
+    use std::os::unix::fs::PermissionsExt;
+    std::fs::metadata(p)
+        .map(|m| m.is_file() && (m.permissions().mode() & 0o111) != 0)
+        .unwrap_or(false)
+}
+
+#[cfg(not(unix))]
+fn is_executable_file(p: &Path) -> bool {
+    p.is_file()
+}
+
+fn probe_dm_flakey() -> Result<(), FlakyError> {
+    let out = Command::new("dmsetup").arg("targets").output()?;
+    if !out.status.success() {
+        return Err(FlakyError::DmFlakeyMissing(stringify_output(&out)));
+    }
+    let stdout = String::from_utf8_lossy(&out.stdout);
+    if !stdout.lines().any(|l| l.starts_with("flakey")) {
+        return Err(FlakyError::DmFlakeyMissing(stdout.into_owned()));
+    }
+    Ok(())
+}
+
+fn reap_stale_mounts() -> Result<(), FlakyError> {
+    let out = Command::new("dmsetup")
+        .arg("ls")
+        .arg("--target")
+        .arg("flakey")
+        .output()?;
+    if !out.status.success() {
+        return Ok(());
+    }
+    String::from_utf8_lossy(&out.stdout)
+        .lines()
+        .filter_map(parse_flaky_entry)
+        .filter(|(_, pid)| !pid_alive(*pid))
+        .for_each(|(name, _)| {
+            let loop_device = mapper_backing_loop(&name);
+            if let Err(e) = dm_remove(&name) {
+                tracing::warn!(name, error = %e, "reap: dm_remove stale mapper failed");
+                return;
+            }
+            if let Some(loop_dev) = loop_device
+                && let Err(e) = detach_loop(&loop_dev)
+            {
+                tracing::warn!(
+                    device = %loop_dev.display(),
+                    error = %e,
+                    "reap: detach_loop stale device failed",
+                );
+            }
+        });
+    Ok(())
+}
+
+fn parse_flaky_entry(line: &str) -> Option<(String, u32)> {
+    let name = line.split_whitespace().next()?;
+    let suffix = name.strip_prefix("tranquil-flaky-")?;
+    let pid_str = suffix.split('-').next()?;
+    let pid = pid_str.parse::<u32>().ok()?;
+    Some((name.to_string(), pid))
+}
+
+fn pid_alive(pid: u32) -> bool {
+    Path::new(&format!("/proc/{pid}")).exists()
+}
+
+fn mapper_backing_loop(name: &str) -> Option<PathBuf> {
+    let out = Command::new("dmsetup")
+        .arg("deps")
+        .arg("-o")
+        .arg("devname")
+        .arg(name)
+        .output()
+        .ok()?;
+    if !out.status.success() {
+        return None;
+    }
+    let text = String::from_utf8_lossy(&out.stdout);
+    let inner = text.split('(').nth(1)?;
+    let dev = inner.split(')').next()?.trim();
+    if dev.is_empty() {
+        None
+    } else {
+        Some(PathBuf::from(format!("/dev/{dev}")))
+    }
+}
+
+fn allocate_backing(path: &Path, size: BackingMegabytes) -> Result<(), FlakyError> {
+    let out = Command::new("truncate")
+        .arg("-s")
+        .arg(format!("{}M", size.0))
+        .arg(path)
+        .output()?;
+    check_status("truncate", &out)
+}
+
+fn attach_loop(backing: &Path) -> Result<PathBuf, FlakyError> {
+    let out = Command::new("losetup")
+        .arg("--find")
+        .arg("--show")
+        .arg(backing)
+        .output()?;
+    check_status("losetup", &out)?;
+    let raw = String::from_utf8_lossy(&out.stdout).trim().to_string();
+    if raw.is_empty() {
+        return Err(FlakyError::CommandFailed {
+            tool: "losetup",
+            status: "exit 0".to_string(),
+            stderr: "no device path on stdout".to_string(),
+        });
+    }
+    Ok(PathBuf::from(raw))
+}
+
+fn detach_loop(device: &Path) -> Result<(), FlakyError> {
+    let out = Command::new("losetup").arg("-d").arg(device).output()?;
+    check_status("losetup -d", &out)
+}
+
+fn sector_count(device: &Path) -> Result<u64, FlakyError> {
+    let out = Command::new("blockdev")
+        .arg("--getsz")
+        .arg(device)
+        .output()?;
+    check_status("blockdev", &out)?;
+    let raw = String::from_utf8_lossy(&out.stdout).trim().to_string();
+    raw.parse::<u64>().map_err(|_| FlakyError::CommandFailed {
+        tool: "blockdev",
+        status: "exit 0".to_string(),
+        stderr: format!("could not parse sector count: {raw:?}"),
+    })
+}
+
+fn dm_create(
+    name: &str,
+    loop_device: &Path,
+    sectors: u64,
+    up: UpIntervalSecs,
+    down: DownIntervalSecs,
+) -> Result<(), FlakyError> {
+    let table = format!(
+        "0 {sectors} flakey {} 0 {} {}",
+        loop_device.display(),
+        up.0.get(),
+        down.0.get(),
+    );
+    let mut child = Command::new("dmsetup")
+        .arg("create")
+        .arg(name)
+        .stdin(std::process::Stdio::piped())
+        .stdout(std::process::Stdio::piped())
+        .stderr(std::process::Stdio::piped())
+        .spawn()?;
+    if let Some(stdin) = child.stdin.as_mut() {
+        use std::io::Write;
+        stdin.write_all(table.as_bytes())?;
+    }
+    let out = child.wait_with_output()?;
+    check_status("dmsetup create", &out)
+}
+
+fn dm_remove(name: &str) -> Result<(), FlakyError> {
+    let out = Command::new("dmsetup")
+        .arg("remove")
+        .arg("--retry")
+        .arg(name)
+        .output()?;
+    check_status("dmsetup remove", &out)
+}
+
+fn mkfs_ext4(device: &Path) -> Result<(), FlakyError> {
+    let out = Command::new("mkfs.ext4")
+        .arg("-q")
+        .arg("-F")
+        .arg(device)
+        .output()?;
+    check_status("mkfs.ext4", &out)
+}
+
+fn mount_ext4(device: &Path, target: &Path) -> Result<(), FlakyError> {
+    let out = Command::new("mount")
+        .arg("-t")
+        .arg("ext4")
+        .arg("-o")
+        .arg("errors=continue")
+        .arg(device)
+        .arg(target)
+        .output()?;
+    check_status("mount", &out)
+}
+
+fn umount(target: &Path) -> Result<(), FlakyError> {
+    let out = Command::new("umount").arg(target).output()?;
+    check_status("umount", &out)
+}
+
+fn umount_lazy(target: &Path) -> Result<(), FlakyError> {
+    let out = Command::new("umount").arg("-l").arg(target).output()?;
+    check_status("umount -l", &out)
+}
+
+fn check_status(tool: &'static str, out: &Output) -> Result<(), FlakyError> {
+    if out.status.success() {
+        Ok(())
+    } else {
+        Err(FlakyError::CommandFailed {
+            tool,
+            status: format!("{}", out.status),
+            stderr: String::from_utf8_lossy(&out.stderr).into_owned(),
+        })
+    }
+}
+
+fn stringify_output(out: &Output) -> String {
+    let mut s = String::new();
+    if !out.stdout.is_empty() {
+        s.push_str(&String::from_utf8_lossy(&out.stdout));
+    }
+    if !out.stderr.is_empty() {
+        if !s.is_empty() {
+            s.push('\n');
+        }
+        s.push_str(&String::from_utf8_lossy(&out.stderr));
+    }
+    s
+}
@@ -0,0 +1,783 @@
+use std::collections::{HashMap, HashSet};
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use cid::Cid;
+use jacquard_repo::mst::Mst;
+
+use super::oracle::{Oracle, hex_short, try_cid_to_fixed};
+use crate::blockstore::{
+    BLOCK_HEADER_SIZE, CidBytes, CompactionError, TranquilBlockStore, hash_to_cid_bytes,
+};
+use crate::eventlog::{EventSequence, SegmentId};
+use crate::io::{RealIO, StorageIO};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct InvariantSet(u32);
+
+impl InvariantSet {
+    pub const EMPTY: Self = Self(0);
+    pub const REFCOUNT_CONSERVATION: Self = Self(1 << 0);
+    pub const REACHABILITY: Self = Self(1 << 1);
+    pub const ACKED_WRITE_PERSISTENCE: Self = Self(1 << 2);
+    pub const READ_AFTER_WRITE: Self = Self(1 << 3);
+    pub const RESTART_IDEMPOTENT: Self = Self(1 << 4);
+    pub const COMPACTION_IDEMPOTENT: Self = Self(1 << 5);
+    pub const NO_ORPHAN_FILES: Self = Self(1 << 6);
+    pub const BYTE_BUDGET: Self = Self(1 << 7);
+    pub const MANIFEST_EQUALS_REALITY: Self = Self(1 << 8);
+    pub const CHECKSUM_COVERAGE: Self = Self(1 << 9);
+    pub const MONOTONIC_SEQ: Self = Self(1 << 10);
+    pub const FSYNC_ORDERING: Self = Self(1 << 11);
+    pub const TOMBSTONE_BOUND: Self = Self(1 << 12);
+
+    const ALL_KNOWN: u32 = Self::REFCOUNT_CONSERVATION.0
+        | Self::REACHABILITY.0
+        | Self::ACKED_WRITE_PERSISTENCE.0
+        | Self::READ_AFTER_WRITE.0
+        | Self::RESTART_IDEMPOTENT.0
+        | Self::COMPACTION_IDEMPOTENT.0
+        | Self::NO_ORPHAN_FILES.0
+        | Self::BYTE_BUDGET.0
+        | Self::MANIFEST_EQUALS_REALITY.0
+        | Self::CHECKSUM_COVERAGE.0
+        | Self::MONOTONIC_SEQ.0
+        | Self::FSYNC_ORDERING.0
+        | Self::TOMBSTONE_BOUND.0;
+
+    pub const fn contains(self, other: Self) -> bool {
+        (self.0 & other.0) == other.0
+    }
+
+    pub const fn union(self, other: Self) -> Self {
+        Self(self.0 | other.0)
+    }
+
+    pub const fn without(self, other: Self) -> Self {
+        Self(self.0 & !other.0)
+    }
+
+    pub const fn unknown_bits(self) -> u32 {
+        self.0 & !Self::ALL_KNOWN
+    }
+}
+
+impl std::ops::BitOr for InvariantSet {
+    type Output = Self;
+    fn bitor(self, rhs: Self) -> Self {
+        self.union(rhs)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct InvariantViolation {
+    pub invariant: &'static str,
+    pub detail: String,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct SnapshotEvent {
+    pub seq: EventSequence,
+    pub timestamp_us: u64,
+    pub event_type_raw: u8,
+    pub did_hash: u32,
+}
+
+#[derive(Debug, Clone)]
+pub struct EventLogSnapshot {
+    pub segments_dir: PathBuf,
+    pub max_segment_size: u64,
+    pub synced_seq: EventSequence,
+    pub segments: Vec<SegmentId>,
+    pub events: Vec<SnapshotEvent>,
+    pub segment_last_ts: Vec<(SegmentId, u64)>,
+}
+
+pub struct InvariantCtx<'a, S: StorageIO + Send + Sync + 'static = RealIO> {
+    pub store: &'a Arc<TranquilBlockStore<S>>,
+    pub oracle: &'a Oracle,
+    pub root: Option<Cid>,
+    pub eventlog: Option<&'a EventLogSnapshot>,
+}
+
+#[async_trait]
+pub trait Invariant<S: StorageIO + Send + Sync + 'static>: Send + Sync {
+    fn name(&self) -> &'static str;
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation>;
+}
+
+pub struct RefcountConservation;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for RefcountConservation {
+    fn name(&self) -> &'static str {
+        "RefcountConservation"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let live: Vec<(String, CidBytes)> = ctx.oracle.live_cids_labeled();
+        let live_set: HashSet<CidBytes> = live.iter().map(|(_, c)| *c).collect();
+        let index: HashMap<CidBytes, u32> = ctx
+            .store
+            .block_index()
+            .live_entries_snapshot()
+            .into_iter()
+            .map(|(c, r)| (c, r.raw()))
+            .collect();
+
+        let forward: Vec<String> = live
+            .iter()
+            .filter_map(|(label, cid)| match index.get(cid) {
+                Some(&r) if r >= 1 => None,
+                Some(&r) => Some(format!("{label}: refcount {r}")),
+                None => Some(format!("{label}: missing from index")),
+            })
+            .collect();
+
+        let inverse: Vec<String> = index
+            .iter()
+            .filter(|(cid, _)| !live_set.contains(*cid))
+            .map(|(cid, r)| format!("orphan cid {} refcount {}", hex_short(cid), r))
+            .collect();
+
+        let violations: Vec<String> = forward.into_iter().chain(inverse).collect();
+        if violations.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "RefcountConservation",
+                detail: violations.join("; "),
+            })
+        }
+    }
+}
+
+pub struct Reachability;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for Reachability {
+    fn name(&self) -> &'static str {
+        "Reachability"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let violations: Vec<String> = ctx
+            .oracle
+            .live_cids_labeled()
+            .into_iter()
+            .filter_map(|(label, fixed)| match ctx.store.get_block_sync(&fixed) {
+                Ok(Some(_)) => None,
+                Ok(None) => Some(format!("{label}: missing")),
+                Err(e) => Some(format!("{label}: read error {e}")),
+            })
+            .collect();
+
+        if violations.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "Reachability",
+                detail: violations.join("; "),
+            })
+        }
+    }
+}
+
+pub struct AckedWritePersistence;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for AckedWritePersistence {
+    fn name(&self) -> &'static str {
+        "AckedWritePersistence"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let Some(root) = ctx.root else {
+            if ctx.oracle.live_count() == 0 {
+                return Ok(());
+            }
+            return Err(InvariantViolation {
+                invariant: "AckedWritePersistence",
+                detail: format!(
+                    "oracle has {} live records but reopened store has no root",
+                    ctx.oracle.live_count()
+                ),
+            });
+        };
+        let mst = Mst::load(ctx.store.clone(), root, None);
+        let keys: Vec<String> = ctx
+            .oracle
+            .live_records()
+            .map(|(c, r, _)| format!("{}/{}", c.0, r.0))
+            .collect();
+
+        let mut missing: Vec<String> = Vec::new();
+        for key in &keys {
+            match mst.get(key).await {
+                Ok(Some(_)) => {}
+                Ok(None) => missing.push(format!("{key}: missing after reopen")),
+                Err(e) => missing.push(format!("{key}: mst.get error after reopen: {e}")),
+            }
+        }
+
+        if missing.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "AckedWritePersistence",
+                detail: missing.join("; "),
+            })
+        }
+    }
+}
+
+pub struct ReadAfterWrite;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for ReadAfterWrite {
+    fn name(&self) -> &'static str {
+        "ReadAfterWrite"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let Some(root) = ctx.root else {
+            return Ok(());
+        };
+        let mst = Mst::load(ctx.store.clone(), root, None);
+
+        let entries: Vec<(String, CidBytes)> = ctx
+            .oracle
+            .live_records()
+            .map(|(c, r, v)| (format!("{}/{}", c.0, r.0), *v))
+            .collect();
+
+        let mut violations: Vec<String> = Vec::new();
+        for (key, expected) in &entries {
+            match mst.get(key).await {
+                Ok(Some(cid)) => match try_cid_to_fixed(&cid) {
+                    Ok(actual) if actual == *expected => match ctx.store.get_block_sync(&actual) {
+                        Ok(Some(_)) => {}
+                        Ok(None) => violations.push(format!("{key}: block missing for cid")),
+                        Err(e) => violations.push(format!("{key}: block read error {e}")),
+                    },
+                    Ok(actual) => violations.push(format!(
+                        "{key}: MST cid {} != oracle cid {}",
+                        hex_short(&actual),
+                        hex_short(expected),
+                    )),
+                    Err(e) => {
+                        violations.push(format!("{key}: unexpected CID format from MST: {e}"))
+                    }
+                },
+                Ok(None) => violations.push(format!("{key}: MST returned None")),
+                Err(e) => violations.push(format!("{key}: mst.get error {e}")),
+            }
+        }
+
+        if violations.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "ReadAfterWrite",
+                detail: violations.join("; "),
+            })
+        }
+    }
+}
+
+pub struct CompactionIdempotent;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for CompactionIdempotent {
+    fn name(&self) -> &'static str {
+        "CompactionIdempotent"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let store_a = ctx.store.clone();
+        let first = tokio::task::spawn_blocking(move || compact_by_liveness(&store_a))
+            .await
+            .map_err(|e| InvariantViolation {
+                invariant: "CompactionIdempotent",
+                detail: format!("first compaction join: {e}"),
+            })?;
+        if let Err(e) = first {
+            return Err(InvariantViolation {
+                invariant: "CompactionIdempotent",
+                detail: format!("first compaction: {e}"),
+            });
+        }
+
+        let pre = snapshot(ctx.store);
+
+        let store_b = ctx.store.clone();
+        let second = tokio::task::spawn_blocking(move || compact_by_liveness(&store_b))
+            .await
+            .map_err(|e| InvariantViolation {
+                invariant: "CompactionIdempotent",
+                detail: format!("second compaction join: {e}"),
+            })?;
+        if let Err(e) = second {
+            return Err(InvariantViolation {
+                invariant: "CompactionIdempotent",
+                detail: format!("second compaction: {e}"),
+            });
+        }
+
+        let post = snapshot(ctx.store);
+
+        if pre == post {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "CompactionIdempotent",
+                detail: format!(
+                    "second compaction changed observable state: pre={} entries, post={} entries",
+                    pre.len(),
+                    post.len(),
+                ),
+            })
+        }
+    }
+}
+
+fn snapshot<S: StorageIO + Send + Sync + 'static>(
+    store: &Arc<TranquilBlockStore<S>>,
+) -> Vec<(CidBytes, u32)> {
+    let mut v: Vec<(CidBytes, u32)> = store
+        .block_index()
+        .live_entries_snapshot()
+        .into_iter()
+        .map(|(c, r)| (c, r.raw()))
+        .collect();
+    v.sort_unstable_by_key(|a| a.0);
+    v
+}
+
+const COMPACT_LIVENESS_CEILING: f64 = 0.99;
+
+fn compact_by_liveness<S: StorageIO + Send + Sync + 'static>(
+    store: &TranquilBlockStore<S>,
+) -> Result<(), String> {
+    let liveness = store
+        .compaction_liveness(0)
+        .map_err(|e| format!("compaction_liveness: {e}"))?;
+    let targets: Vec<_> = liveness
+        .iter()
+        .filter(|(_, info)| info.total_blocks > 0 && info.ratio() < COMPACT_LIVENESS_CEILING)
+        .map(|(&fid, _)| fid)
+        .collect();
+    targets
+        .into_iter()
+        .try_for_each(|fid| match store.compact_file(fid, 0) {
+            Ok(_) => Ok(()),
+            Err(CompactionError::ActiveFileCannotBeCompacted) => Ok(()),
+            Err(e) => Err(format!("{fid}: {e}")),
+        })
+}
+
+pub struct NoOrphanFiles;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for NoOrphanFiles {
+    fn name(&self) -> &'static str {
+        "NoOrphanFiles"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let store_c = ctx.store.clone();
+        let result = tokio::task::spawn_blocking(move || {
+            let disk = store_c.list_data_files().map_err(|e| e.to_string())?;
+            let liveness = store_c.compaction_liveness(0).map_err(|e| e.to_string())?;
+            let header = BLOCK_HEADER_SIZE as u64;
+            let orphans: Vec<String> = disk
+                .iter()
+                .filter(|fid| !liveness.contains_key(fid))
+                .filter_map(|fid| {
+                    let path = store_c.data_file_path(*fid);
+                    let size = std::fs::metadata(&path).map(|m| m.len()).unwrap_or(0);
+                    match size > header {
+                        true => Some(format!("{fid} ({size} B)")),
+                        false => None,
+                    }
+                })
+                .collect();
+            Ok::<_, String>(orphans)
+        })
+        .await
+        .map_err(|e| InvariantViolation {
+            invariant: "NoOrphanFiles",
+            detail: format!("join: {e}"),
+        })?;
+
+        let orphans = result.map_err(|e| InvariantViolation {
+            invariant: "NoOrphanFiles",
+            detail: e,
+        })?;
+
+        if orphans.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "NoOrphanFiles",
+                detail: format!("files on disk missing from index: {}", orphans.join(", ")),
+            })
+        }
+    }
+}
+
+pub struct ByteBudget {
+    pub overhead_factor: f64,
+    pub floor_bytes: u64,
+}
+
+impl Default for ByteBudget {
+    fn default() -> Self {
+        Self {
+            overhead_factor: 8.0,
+            floor_bytes: 1 << 20,
+        }
+    }
+}
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for ByteBudget {
+    fn name(&self) -> &'static str {
+        "ByteBudget"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let store = ctx.store.clone();
+        let factor = self.overhead_factor;
+        let floor = self.floor_bytes;
+        tokio::task::spawn_blocking(move || {
+            let liveness = store.compaction_liveness(0).map_err(|e| e.to_string())?;
+            let live: u64 = liveness.values().map(|i| i.live_bytes).sum();
+            let total: u64 = liveness.values().map(|i| i.total_bytes).sum();
+            let budget = (live as f64 * factor) as u64 + floor;
+            if total <= budget {
+                Ok(())
+            } else {
+                Err(format!(
+                    "total_bytes {total} exceeds budget {budget}: live_bytes {live}, factor {factor}, floor {floor}"
+                ))
+            }
+        })
+        .await
+        .map_err(|e| InvariantViolation {
+            invariant: "ByteBudget",
+            detail: format!("join: {e}"),
+        })?
+        .map_err(|e| InvariantViolation {
+            invariant: "ByteBudget",
+            detail: e,
+        })
+    }
+}
+
+pub struct ManifestEqualsReality;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for ManifestEqualsReality {
+    fn name(&self) -> &'static str {
+        "ManifestEqualsReality"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let store = ctx.store.clone();
+        tokio::task::spawn_blocking(move || {
+            let listed = store.list_data_files().map_err(|e| e.to_string())?;
+            let liveness = store.compaction_liveness(0).map_err(|e| e.to_string())?;
+            let header = BLOCK_HEADER_SIZE as u64;
+
+            let mut violations: Vec<String> = Vec::new();
+            listed.iter().for_each(|fid| {
+                let path = store.data_file_path(*fid);
+                match std::fs::metadata(&path) {
+                    Err(e) => violations.push(format!("{fid}: metadata {e}")),
+                    Ok(meta) => {
+                        let on_disk = meta.len();
+                        let content = on_disk.saturating_sub(header);
+                        match liveness.get(fid) {
+                            None if on_disk > header => violations.push(format!(
+                                "{fid}: listed on disk at {on_disk} B but not in index liveness"
+                            )),
+                            None => {}
+                            Some(info) if content < info.total_bytes => {
+                                violations.push(format!(
+                                    "{fid}: on-disk {on_disk} B (content {content}) < index total_bytes {}",
+                                    info.total_bytes
+                                ));
+                            }
+                            Some(info) if content > info.total_bytes => {
+                                violations.push(format!(
+                                    "{fid}: on-disk {on_disk} B (content {content}) > index total_bytes {}, {} B unaccounted",
+                                    info.total_bytes,
+                                    content - info.total_bytes
+                                ));
+                            }
+                            Some(_) => {}
+                        }
+                    }
+                }
+            });
+
+            let listed_set: std::collections::HashSet<_> = listed.into_iter().collect();
+            liveness.keys().for_each(|fid| {
+                if !listed_set.contains(fid) {
+                    violations.push(format!("{fid}: in index liveness but missing on disk"));
+                }
+            });
+
+            if violations.is_empty() {
+                Ok(())
+            } else {
+                Err(violations.join("; "))
+            }
+        })
+        .await
+        .map_err(|e| InvariantViolation {
+            invariant: "ManifestEqualsReality",
+            detail: format!("join: {e}"),
+        })?
+        .map_err(|e| InvariantViolation {
+            invariant: "ManifestEqualsReality",
+            detail: e,
+        })
+    }
+}
+
+pub struct ChecksumCoverage;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for ChecksumCoverage {
+    fn name(&self) -> &'static str {
+        "ChecksumCoverage"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let violations: Vec<String> = ctx
+            .oracle
+            .live_cids_labeled()
+            .into_iter()
+            .filter_map(|(label, expected)| match ctx.store.get_block_sync(&expected) {
+                Ok(Some(bytes)) => {
+                    let actual = hash_to_cid_bytes(&bytes);
+                    (actual != expected).then(|| {
+                        format!(
+                            "{label}: silent corruption, bytes hash to {} but store returned them under {}",
+                            hex_short(&actual),
+                            hex_short(&expected),
+                        )
+                    })
+                }
+                Ok(None) => Some(format!(
+                    "{label}: live CID {} missing from store",
+                    hex_short(&expected)
+                )),
+                Err(e) => Some(format!(
+                    "{label}: read error for live CID {}: {e}",
+                    hex_short(&expected)
+                )),
+            })
+            .collect();
+
+        if violations.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "ChecksumCoverage",
+                detail: violations.join("; "),
+            })
+        }
+    }
+}
+
+pub struct MonotonicSeq;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for MonotonicSeq {
+    fn name(&self) -> &'static str {
+        "MonotonicSeq"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let Some(el) = ctx.eventlog else {
+            return Ok(());
+        };
+        let mut violations: Vec<String> = Vec::new();
+        el.events
+            .iter()
+            .zip(el.events.iter().skip(1))
+            .for_each(|(prev, next)| match next.seq.raw() {
+                n if n == prev.seq.raw() + 1 => {}
+                n if n == prev.seq.raw() => violations.push(format!("duplicate seq {n}")),
+                n => violations.push(format!(
+                    "gap: seq {} followed by {n}, expected {}",
+                    prev.seq.raw(),
+                    prev.seq.raw() + 1
+                )),
+            });
+        if ctx.oracle.last_retention_cutoff_us().is_none()
+            && let Some(first) = el.events.first()
+            && first.seq.raw() != 1
+        {
+            violations.push(format!(
+                "first persisted seq is {}, expected 1",
+                first.seq.raw()
+            ));
+        }
+        let acked_max = ctx
+            .oracle
+            .synced_events()
+            .iter()
+            .map(|e| e.seq.raw())
+            .max()
+            .unwrap_or(0);
+        let disk_max = el.events.last().map(|e| e.seq.raw()).unwrap_or(0);
+        if disk_max < acked_max {
+            violations.push(format!(
+                "acked seq {acked_max} missing on disk, disk max {disk_max}"
+            ));
+        }
+        if violations.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "MonotonicSeq",
+                detail: violations.join("; "),
+            })
+        }
+    }
+}
+
+pub struct FsyncOrdering;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for FsyncOrdering {
+    fn name(&self) -> &'static str {
+        "FsyncOrdering"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let Some(el) = ctx.eventlog else {
+            return Ok(());
+        };
+        let mut violations: Vec<String> = Vec::new();
+
+        let acked_seqs: HashSet<u64> = ctx
+            .oracle
+            .synced_events()
+            .iter()
+            .map(|e| e.seq.raw())
+            .collect();
+        let disk_seqs: HashSet<u64> = el.events.iter().map(|e| e.seq.raw()).collect();
+        let missing: Vec<u64> = acked_seqs.difference(&disk_seqs).copied().collect();
+        if !missing.is_empty() {
+            let mut sorted = missing;
+            sorted.sort_unstable();
+            violations.push(format!(
+                "{} acked events lost on disk, lowest missing seq {}",
+                sorted.len(),
+                sorted[0]
+            ));
+        }
+
+        if let Some(last_synced) = ctx.oracle.last_synced_seq()
+            && el.synced_seq.raw() != 0
+            && el.synced_seq.raw() < last_synced.raw()
+        {
+            violations.push(format!(
+                "writer synced_seq {} below oracle last_synced_seq {}",
+                el.synced_seq.raw(),
+                last_synced.raw()
+            ));
+        }
+
+        if violations.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "FsyncOrdering",
+                detail: violations.join("; "),
+            })
+        }
+    }
+}
+
+pub struct TombstoneBound;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for TombstoneBound {
+    fn name(&self) -> &'static str {
+        "TombstoneBound"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let Some(el) = ctx.eventlog else {
+            return Ok(());
+        };
+        let Some(cutoff_us) = ctx.oracle.last_retention_cutoff_us() else {
+            return Ok(());
+        };
+
+        let active = el.segments.last().copied();
+
+        let stale: Vec<String> = el
+            .segment_last_ts
+            .iter()
+            .filter(|(id, last_ts)| Some(*id) != active && *last_ts < cutoff_us)
+            .map(|(id, last_ts)| format!("segment {id} last_ts {last_ts} < cutoff {cutoff_us}"))
+            .collect();
+
+        if stale.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "TombstoneBound",
+                detail: stale.join("; "),
+            })
+        }
+    }
+}
+
+pub fn invariants_for<S: StorageIO + Send + Sync + 'static>(
+    set: InvariantSet,
+) -> Vec<Box<dyn Invariant<S>>> {
+    let unknown = set.unknown_bits();
+    assert!(
+        unknown == 0,
+        "invariants_for: unknown InvariantSet bits 0x{unknown:x}; all bits must map to an impl"
+    );
+    let candidates: Vec<(InvariantSet, Box<dyn Invariant<S>>)> = vec![
+        (
+            InvariantSet::REFCOUNT_CONSERVATION,
+            Box::new(RefcountConservation),
+        ),
+        (InvariantSet::REACHABILITY, Box::new(Reachability)),
+        (
+            InvariantSet::ACKED_WRITE_PERSISTENCE,
+            Box::new(AckedWritePersistence),
+        ),
+        (InvariantSet::READ_AFTER_WRITE, Box::new(ReadAfterWrite)),
+        (
+            InvariantSet::COMPACTION_IDEMPOTENT,
+            Box::new(CompactionIdempotent),
+        ),
+        (InvariantSet::NO_ORPHAN_FILES, Box::new(NoOrphanFiles)),
+        (InvariantSet::BYTE_BUDGET, Box::new(ByteBudget::default())),
+        (
+            InvariantSet::MANIFEST_EQUALS_REALITY,
+            Box::new(ManifestEqualsReality),
+        ),
+        (InvariantSet::CHECKSUM_COVERAGE, Box::new(ChecksumCoverage)),
+        (InvariantSet::MONOTONIC_SEQ, Box::new(MonotonicSeq)),
+        (InvariantSet::FSYNC_ORDERING, Box::new(FsyncOrdering)),
+        (InvariantSet::TOMBSTONE_BOUND, Box::new(TombstoneBound)),
+    ];
+    candidates
+        .into_iter()
+        .filter_map(|(flag, inv)| set.contains(flag).then_some(inv))
+        .collect()
+}
@@ -0,0 +1,234 @@
+use std::num::NonZeroU64;
+use std::time::Duration;
+
+use serde::{Deserialize, Serialize};
+
+use super::metrics::{MetricName, MetricsSample};
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub struct LeakGateConfig {
+    pub warmup_ms: u64,
+    pub window_ms: NonZeroU64,
+    pub growth_limit_pct: f64,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct LeakGateBuildError(pub &'static str);
+
+impl std::fmt::Display for LeakGateBuildError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(self.0)
+    }
+}
+
+impl std::error::Error for LeakGateBuildError {}
+
+impl LeakGateConfig {
+    pub fn try_new(
+        warmup_ms: u64,
+        window_ms: u64,
+        growth_limit_pct: f64,
+    ) -> Result<Self, LeakGateBuildError> {
+        let window_ms = NonZeroU64::new(window_ms)
+            .ok_or(LeakGateBuildError("leak gate window_ms must be > 0"))?;
+        if !growth_limit_pct.is_finite() || growth_limit_pct < 0.0 {
+            return Err(LeakGateBuildError(
+                "leak gate growth_limit_pct must be finite and non-negative",
+            ));
+        }
+        Ok(Self {
+            warmup_ms,
+            window_ms,
+            growth_limit_pct,
+        })
+    }
+
+    pub fn standard() -> Self {
+        Self::try_new(60 * 60 * 1_000, 4 * 60 * 60 * 1_000, 5.0)
+            .expect("standard leak gate config is valid")
+    }
+
+    pub fn short_for_tests() -> Self {
+        Self::try_new(60_000, 4 * 60_000, 5.0).expect("short_for_tests leak gate config is valid")
+    }
+
+    pub fn warmup(&self) -> Duration {
+        Duration::from_millis(self.warmup_ms)
+    }
+
+    pub fn window(&self) -> Duration {
+        Duration::from_millis(self.window_ms.get())
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct LeakViolation {
+    pub metric: String,
+    pub start_ms: u64,
+    pub end_ms: u64,
+    pub start_value: u64,
+    pub end_value: u64,
+    pub growth_pct: f64,
+    pub limit_pct: f64,
+}
+
+pub fn evaluate(samples: &[MetricsSample], cfg: LeakGateConfig) -> Vec<LeakViolation> {
+    if samples.len() < 2 {
+        return Vec::new();
+    }
+    MetricName::ALL
+        .iter()
+        .flat_map(|&m| evaluate_metric(samples, m, cfg))
+        .collect()
+}
+
+fn evaluate_metric(
+    samples: &[MetricsSample],
+    metric: MetricName,
+    cfg: LeakGateConfig,
+) -> Option<LeakViolation> {
+    let post_warmup: Vec<&MetricsSample> = samples
+        .iter()
+        .filter(|s| s.elapsed_ms >= cfg.warmup_ms)
+        .collect();
+    if post_warmup.len() < 2 {
+        return None;
+    }
+
+    let min_delta = metric.min_absolute_delta();
+    let window = cfg.window_ms.get();
+    let mut worst: Option<LeakViolation> = None;
+    for (i, start) in post_warmup.iter().enumerate() {
+        let Some(start_v) = start.metric(metric) else {
+            continue;
+        };
+        if start_v == 0 {
+            continue;
+        }
+        let deadline = start.elapsed_ms.saturating_add(window);
+        for end in post_warmup.iter().skip(i + 1) {
+            if end.elapsed_ms > deadline {
+                break;
+            }
+            let Some(end_v) = end.metric(metric) else {
+                continue;
+            };
+            if end_v <= start_v {
+                continue;
+            }
+            let delta = end_v - start_v;
+            if delta < min_delta {
+                continue;
+            }
+            let growth = (delta as f64 / start_v as f64) * 100.0;
+            if growth <= cfg.growth_limit_pct {
+                continue;
+            }
+            let candidate = LeakViolation {
+                metric: metric.as_str().to_string(),
+                start_ms: start.elapsed_ms,
+                end_ms: end.elapsed_ms,
+                start_value: start_v,
+                end_value: end_v,
+                growth_pct: growth,
+                limit_pct: cfg.growth_limit_pct,
+            };
+            match &worst {
+                Some(w) if w.growth_pct >= candidate.growth_pct => {}
+                _ => worst = Some(candidate),
+            }
+        }
+    }
+    worst
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const GIB: u64 = 1024 * 1024 * 1024;
+
+    fn sample(elapsed_ms: u64, rss: u64) -> MetricsSample {
+        MetricsSample {
+            elapsed_ms,
+            rss_bytes: Some(rss),
+            fd_count: Some(10),
+            data_dir_bytes: 0,
+            index_dir_bytes: 0,
+            segments_dir_bytes: 0,
+            data_file_count: Some(0),
+            segment_count: Some(0),
+            block_index_entries: 0,
+            hint_file_bytes: 0,
+        }
+    }
+
+    #[test]
+    fn flat_metrics_no_violation() {
+        let cfg = LeakGateConfig::short_for_tests();
+        let series: Vec<MetricsSample> =
+            (0..20).map(|i| sample(60_000 + i * 60_000, GIB)).collect();
+        assert!(evaluate(&series, cfg).is_empty());
+    }
+
+    #[test]
+    fn growing_rss_flagged() {
+        let cfg = LeakGateConfig::short_for_tests();
+        let series: Vec<MetricsSample> = (0..20)
+            .map(|i| sample(60_000 + i * 60_000, GIB + i * 64 * 1024 * 1024))
+            .collect();
+        let v = evaluate(&series, cfg);
+        assert!(!v.is_empty());
+        assert_eq!(v[0].metric, "rss_bytes");
+        assert!(v[0].growth_pct > 5.0);
+    }
+
+    #[test]
+    fn warmup_samples_ignored() {
+        let cfg = LeakGateConfig::short_for_tests();
+        let mut series: Vec<MetricsSample> = Vec::new();
+        series.push(sample(10_000, 1));
+        series.push(sample(30_000, GIB));
+        (0..10).for_each(|i| {
+            series.push(sample(60_000 + i * 60_000, GIB));
+        });
+        assert!(evaluate(&series, cfg).is_empty());
+    }
+
+    #[test]
+    fn window_bound_honored() {
+        let cfg = LeakGateConfig::try_new(0, 2 * 60_000, 5.0).unwrap();
+        let series = vec![sample(0, GIB), sample(200_000, 2 * GIB)];
+        assert!(
+            evaluate(&series, cfg).is_empty(),
+            "200s gap exceeds 120s window, growth must not be flagged"
+        );
+    }
+
+    #[test]
+    fn small_absolute_delta_not_flagged() {
+        let cfg = LeakGateConfig::short_for_tests();
+        let series: Vec<MetricsSample> = (0..10)
+            .map(|i| sample(60_000 + i * 60_000, GIB + i * 1024))
+            .collect();
+        assert!(
+            evaluate(&series, cfg).is_empty(),
+            "kilobyte growth is below the RSS absolute-delta floor"
+        );
+    }
+
+    #[test]
+    fn missing_metric_samples_skipped() {
+        let cfg = LeakGateConfig::short_for_tests();
+        let mut series: Vec<MetricsSample> =
+            (0..10).map(|i| sample(60_000 + i * 60_000, GIB)).collect();
+        series[3].rss_bytes = None;
+        series[7].rss_bytes = None;
+        assert!(evaluate(&series, cfg).is_empty());
+    }
+
+    #[test]
+    fn zero_window_rejected_at_construction() {
+        assert!(LeakGateConfig::try_new(0, 0, 5.0).is_err());
+    }
+}
@@ -0,0 +1,247 @@
+use std::path::Path;
+use std::sync::Arc;
+use std::time::Duration;
+
+use serde::{Deserialize, Serialize};
+use tracing::warn;
+
+use super::runner::{EventLogState, Harness};
+use crate::blockstore::TranquilBlockStore;
+use crate::io::StorageIO;
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub struct MetricsSample {
+    pub elapsed_ms: u64,
+    pub rss_bytes: Option<u64>,
+    pub fd_count: Option<u64>,
+    pub data_dir_bytes: u64,
+    pub index_dir_bytes: u64,
+    pub segments_dir_bytes: u64,
+    pub data_file_count: Option<u64>,
+    pub segment_count: Option<u64>,
+    pub block_index_entries: u64,
+    pub hint_file_bytes: u64,
+}
+
+impl MetricsSample {
+    pub fn metric(&self, name: MetricName) -> Option<u64> {
+        match name {
+            MetricName::RssBytes => self.rss_bytes,
+            MetricName::FdCount => self.fd_count,
+            MetricName::DataDirBytes => Some(self.data_dir_bytes),
+            MetricName::IndexDirBytes => Some(self.index_dir_bytes),
+            MetricName::SegmentsDirBytes => Some(self.segments_dir_bytes),
+            MetricName::DataFileCount => self.data_file_count,
+            MetricName::SegmentCount => self.segment_count,
+            MetricName::BlockIndexEntries => Some(self.block_index_entries),
+            MetricName::HintFileBytes => Some(self.hint_file_bytes),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum MetricName {
+    RssBytes,
+    FdCount,
+    DataDirBytes,
+    IndexDirBytes,
+    SegmentsDirBytes,
+    DataFileCount,
+    SegmentCount,
+    BlockIndexEntries,
+    HintFileBytes,
+}
+
+impl MetricName {
+    pub const ALL: &'static [MetricName] = &[
+        Self::RssBytes,
+        Self::FdCount,
+        Self::DataDirBytes,
+        Self::IndexDirBytes,
+        Self::SegmentsDirBytes,
+        Self::DataFileCount,
+        Self::SegmentCount,
+        Self::BlockIndexEntries,
+        Self::HintFileBytes,
+    ];
+
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            Self::RssBytes => "rss_bytes",
+            Self::FdCount => "fd_count",
+            Self::DataDirBytes => "data_dir_bytes",
+            Self::IndexDirBytes => "index_dir_bytes",
+            Self::SegmentsDirBytes => "segments_dir_bytes",
+            Self::DataFileCount => "data_file_count",
+            Self::SegmentCount => "segment_count",
+            Self::BlockIndexEntries => "block_index_entries",
+            Self::HintFileBytes => "hint_file_bytes",
+        }
+    }
+
+    pub const fn min_absolute_delta(self) -> u64 {
+        match self {
+            Self::RssBytes => 16 * 1024 * 1024,
+            Self::FdCount => 16,
+            Self::DataDirBytes => 16 * 1024 * 1024,
+            Self::IndexDirBytes => 1024 * 1024,
+            Self::SegmentsDirBytes => 16 * 1024 * 1024,
+            Self::DataFileCount => 16,
+            Self::SegmentCount => 4,
+            Self::BlockIndexEntries => 1024,
+            Self::HintFileBytes => 1024 * 1024,
+        }
+    }
+}
+
+pub fn sample_harness<S: StorageIO + Send + Sync + 'static>(
+    harness: &Harness<S>,
+    elapsed: Duration,
+) -> MetricsSample {
+    MetricsSample {
+        elapsed_ms: u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX),
+        rss_bytes: read_rss(),
+        fd_count: count_open_fds(),
+        data_dir_bytes: dir_bytes(harness.store.data_dir()),
+        index_dir_bytes: dir_bytes(harness.store.block_index().index_dir()),
+        segments_dir_bytes: harness
+            .eventlog
+            .as_ref()
+            .map(|el| dir_bytes(&el.segments_dir))
+            .unwrap_or(0),
+        data_file_count: data_file_count(&harness.store),
+        segment_count: harness.eventlog.as_ref().and_then(segment_count),
+        block_index_entries: harness.store.block_index().approximate_block_count(),
+        hint_file_bytes: hint_bytes(harness.store.data_dir()),
+    }
+}
+
+fn data_file_count<S: StorageIO + Send + Sync + 'static>(
+    store: &Arc<TranquilBlockStore<S>>,
+) -> Option<u64> {
+    match store.list_data_files() {
+        Ok(v) => Some(v.len() as u64),
+        Err(e) => {
+            warn!(error = %e, "gauntlet metrics: list_data_files failed");
+            None
+        }
+    }
+}
+
+fn segment_count<S: StorageIO + Send + Sync + 'static>(el: &EventLogState<S>) -> Option<u64> {
+    match el.manager.list_segments() {
+        Ok(v) => Some(v.len() as u64),
+        Err(e) => {
+            warn!(error = %e, "gauntlet metrics: list_segments failed");
+            None
+        }
+    }
+}
+
+fn dir_bytes(path: &Path) -> u64 {
+    let Ok(entries) = std::fs::read_dir(path) else {
+        return 0;
+    };
+    entries
+        .filter_map(Result::ok)
+        .map(|entry| match entry.file_type() {
+            Ok(ft) if ft.is_dir() => dir_bytes(&entry.path()),
+            Ok(_) => entry.metadata().map(|m| m.len()).unwrap_or(0),
+            Err(_) => 0,
+        })
+        .sum()
+}
+
+fn hint_bytes(data_dir: &Path) -> u64 {
+    let Ok(entries) = std::fs::read_dir(data_dir) else {
+        return 0;
+    };
+    entries
+        .filter_map(Result::ok)
+        .filter(|entry| {
+            entry
+                .path()
+                .extension()
+                .and_then(|e| e.to_str())
+                .map(|e| e == "tqh")
+                .unwrap_or(false)
+        })
+        .map(|entry| entry.metadata().map(|m| m.len()).unwrap_or(0))
+        .sum()
+}
+
+#[cfg(target_os = "linux")]
+fn read_rss() -> Option<u64> {
+    let status = match std::fs::read_to_string("/proc/self/status") {
+        Ok(s) => s,
+        Err(e) => {
+            warn!(error = %e, "gauntlet metrics: read /proc/self/status failed");
+            return None;
+        }
+    };
+    let parsed = status.lines().find_map(|line| {
+        let rest = line.strip_prefix("VmRSS:")?;
+        let kb: u64 = rest.split_whitespace().next()?.parse().ok()?;
+        Some(kb * 1024)
+    });
+    if parsed.is_none() {
+        warn!("gauntlet metrics: VmRSS line missing from /proc/self/status");
+    }
+    parsed
+}
+
+#[cfg(not(target_os = "linux"))]
+fn read_rss() -> Option<u64> {
+    None
+}
+
+#[cfg(target_os = "linux")]
+fn count_open_fds() -> Option<u64> {
+    match std::fs::read_dir("/proc/self/fd") {
+        Ok(entries) => Some(entries.filter_map(Result::ok).count() as u64),
+        Err(e) => {
+            warn!(error = %e, "gauntlet metrics: read /proc/self/fd failed");
+            None
+        }
+    }
+}
+
+#[cfg(not(target_os = "linux"))]
+fn count_open_fds() -> Option<u64> {
+    None
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn metric_names_roundtrip_strings() {
+        MetricName::ALL.iter().for_each(|m| {
+            let s = m.as_str();
+            assert!(!s.is_empty());
+        });
+    }
+
+    #[test]
+    #[cfg_attr(not(target_os = "linux"), ignore = "linux /proc only")]
+    fn rss_reads_nonzero() {
+        let rss = read_rss().expect("rss");
+        assert!(rss > 0, "rss should be positive, got {rss}");
+    }
+
+    #[test]
+    #[cfg_attr(not(target_os = "linux"), ignore = "linux /proc only")]
+    fn fd_count_reads_nonzero() {
+        let fd = count_open_fds().expect("fd");
+        assert!(fd > 0);
+    }
+
+    #[test]
+    fn dir_bytes_sums_entries() {
+        let dir = tempfile::TempDir::new().unwrap();
+        std::fs::write(dir.path().join("a"), b"1234").unwrap();
+        std::fs::write(dir.path().join("b"), b"5678").unwrap();
+        assert_eq!(dir_bytes(dir.path()), 8);
+    }
+}
@@ -0,0 +1,45 @@
+pub mod farm;
+pub mod flaky;
+pub mod invariants;
+pub mod leak;
+pub mod metrics;
+pub mod op;
+pub mod oracle;
+pub mod overrides;
+pub mod regression;
+pub mod runner;
+pub mod scenarios;
+pub mod shrink;
+pub mod soak;
+pub mod workload;
+
+pub use flaky::{
+    BackingMegabytes, DownIntervalSecs, FlakyConfig, FlakyError, FlakyMount, UpIntervalSecs,
+};
+pub use invariants::{
+    EventLogSnapshot, Invariant, InvariantSet, InvariantViolation, SnapshotEvent, invariants_for,
+};
+pub use leak::{LeakGateBuildError, LeakGateConfig, LeakViolation, evaluate as evaluate_leak_gate};
+pub use metrics::{MetricName, MetricsSample, sample_harness};
+pub use op::{
+    CollectionName, DidSeed, EventKind, Op, OpStream, PayloadSeed, RecordKey, RetentionSecs, Seed,
+    ValueSeed,
+};
+pub use oracle::{EventExpectation, Oracle};
+pub use overrides::{ConfigOverrides, GroupCommitOverrides, StoreOverrides};
+pub use regression::{RegressionRecord, RegressionViolation, default_root as regression_root};
+pub use runner::{
+    EventLogConfig, Gauntlet, GauntletBuildError, GauntletConfig, GauntletReport, Harness,
+    IoBackend, MaxFileSize, MaxSegmentSize, OpErrorCount, OpIndex, OpInterval, OpsExecuted,
+    RestartCount, RestartPolicy, RunLimits, ShardCount, StoreConfig, WallMs, WriterConcurrency,
+};
+pub use scenarios::{Scenario, UnknownScenario, config_for};
+pub use shrink::{ShrinkOutcome, shrink_failure};
+pub use soak::{
+    DEFAULT_CHUNK_OPS, DEFAULT_SAMPLE_INTERVAL_MS, InvariantViolationRecord, SoakConfig, SoakError,
+    SoakEvent, SoakReport, run_soak,
+};
+pub use workload::{
+    ByteRange, DidSpaceSize, KeySpaceSize, OpCount, OpWeights, RetentionMaxSecs, SizeDistribution,
+    ValueBytes, WorkloadModel,
+};
@@ -0,0 +1,198 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct Seed(pub u64);
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct CollectionName(pub String);
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct RecordKey(pub String);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct ValueSeed(pub u32);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct DidSeed(pub u32);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct PayloadSeed(pub u32);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct RetentionSecs(pub u32);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub enum EventKind {
+    Commit,
+    Identity,
+    Account,
+    Sync,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum Op {
+    AddRecord {
+        collection: CollectionName,
+        rkey: RecordKey,
+        value_seed: ValueSeed,
+    },
+    DeleteRecord {
+        collection: CollectionName,
+        rkey: RecordKey,
+    },
+    Compact,
+    Checkpoint,
+    AppendEvent {
+        did_seed: DidSeed,
+        event_kind: EventKind,
+        payload_seed: PayloadSeed,
+    },
+    SyncEventLog,
+    RunRetention {
+        max_age_secs: RetentionSecs,
+    },
+    ReadRecord {
+        collection: CollectionName,
+        rkey: RecordKey,
+    },
+    ReadBlock {
+        value_seed: ValueSeed,
+    },
+}
+
+impl Op {
+    pub const fn is_read_only(&self) -> bool {
+        matches!(self, Op::ReadRecord { .. } | Op::ReadBlock { .. })
+    }
+}
+
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct OpStream {
+    ops: Vec<Op>,
+}
+
+impl OpStream {
+    pub fn from_vec(ops: Vec<Op>) -> Self {
+        Self { ops }
+    }
+
+    pub fn empty() -> Self {
+        Self { ops: Vec::new() }
+    }
+
+    pub fn as_slice(&self) -> &[Op] {
+        &self.ops
+    }
+
+    pub fn into_vec(self) -> Vec<Op> {
+        self.ops
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = &Op> {
+        self.ops.iter()
+    }
+
+    pub fn len(&self) -> usize {
+        self.ops.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.ops.is_empty()
+    }
+
+    pub fn shrink_candidates(&self) -> impl Iterator<Item = OpStream> + '_ {
+        let len = self.ops.len();
+        let chunk_sizes: Vec<usize> = std::iter::successors((len >= 2).then_some(len / 2), |&s| {
+            (s >= 2).then_some(s / 2)
+        })
+        .collect();
+
+        let chunk_candidates = chunk_sizes.into_iter().flat_map(move |chunk_size| {
+            let count = len.div_ceil(chunk_size);
+            (0..count).map(move |i| {
+                let start = i * chunk_size;
+                let end = (start + chunk_size).min(len);
+                let mut reduced = Vec::with_capacity(len - (end - start));
+                reduced.extend_from_slice(&self.ops[..start]);
+                reduced.extend_from_slice(&self.ops[end..]);
+                OpStream::from_vec(reduced)
+            })
+        });
+
+        let single_candidates = (0..len).map(move |i| {
+            let mut reduced = self.ops.clone();
+            reduced.remove(i);
+            OpStream::from_vec(reduced)
+        });
+
+        chunk_candidates.chain(single_candidates)
+    }
+
+    pub fn shrink_to_fixpoint(mut self, mut fails: impl FnMut(&OpStream) -> bool) -> OpStream {
+        loop {
+            let next = self.shrink_candidates().find(|c| !c.is_empty() && fails(c));
+            match next {
+                Some(smaller) => self = smaller,
+                None => return self,
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn stream(n: usize) -> OpStream {
+        OpStream::from_vec(
+            (0..n)
+                .map(|i| Op::AddRecord {
+                    collection: CollectionName("c".into()),
+                    rkey: RecordKey(format!("{i:04}")),
+                    value_seed: ValueSeed(i as u32),
+                })
+                .collect(),
+        )
+    }
+
+    fn contains_index(s: &OpStream, target: u32) -> bool {
+        s.iter()
+            .any(|op| matches!(op, Op::AddRecord { value_seed, .. } if value_seed.0 == target))
+    }
+
+    #[test]
+    fn shrink_candidates_nonempty_for_len_ge_2() {
+        let s = stream(8);
+        let count = s.shrink_candidates().count();
+        assert!(count > 0);
+    }
+
+    #[test]
+    fn shrink_candidates_empty_for_len_0() {
+        let s = OpStream::from_vec(Vec::new());
+        assert_eq!(s.shrink_candidates().count(), 0);
+    }
+
+    #[test]
+    fn shrink_candidates_includes_every_single_removal() {
+        let s = stream(5);
+        let singles: Vec<_> = s.shrink_candidates().filter(|c| c.len() == 4).collect();
+        assert!(
+            singles.len() >= 5,
+            "expected at least 5 size-4 candidates, got {}",
+            singles.len()
+        );
+    }
+
+    #[test]
+    fn shrink_to_fixpoint_converges_to_culprit() {
+        let s = stream(64);
+        let shrunk = s.shrink_to_fixpoint(|c| contains_index(c, 17));
+        assert!(contains_index(&shrunk, 17));
+        assert!(
+            shrunk.len() < 4,
+            "expected shrink to close on culprit, got {} ops",
+            shrunk.len()
+        );
+    }
+}
@@ -0,0 +1,143 @@
+use std::collections::HashMap;
+
+use cid::Cid;
+
+use super::op::{CollectionName, EventKind, RecordKey};
+use crate::blockstore::CidBytes;
+use crate::eventlog::EventSequence;
+
+#[derive(Debug, thiserror::Error, PartialEq, Eq)]
+#[error("unexpected CID encoding: got {actual} bytes, expected 36 for sha256 CIDv1")]
+pub struct CidFormatError {
+    pub actual: usize,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct EventExpectation {
+    pub seq: EventSequence,
+    pub timestamp_us: u64,
+    pub kind: EventKind,
+    pub did_hash: u32,
+}
+
+#[derive(Debug, Default)]
+pub struct Oracle {
+    live: HashMap<(CollectionName, RecordKey), CidBytes>,
+    current_root: Option<Cid>,
+    mst_node_cids: Vec<CidBytes>,
+    synced_events: Vec<EventExpectation>,
+    unsynced_events: Vec<EventExpectation>,
+    last_synced_seq: Option<EventSequence>,
+    last_retention_cutoff_us: Option<u64>,
+}
+
+impl Oracle {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    pub fn add(
+        &mut self,
+        coll: CollectionName,
+        rkey: RecordKey,
+        record_cid: CidBytes,
+    ) -> Option<CidBytes> {
+        self.live.insert((coll, rkey), record_cid)
+    }
+
+    pub fn delete(&mut self, coll: &CollectionName, rkey: &RecordKey) -> Option<CidBytes> {
+        self.live.remove(&(coll.clone(), rkey.clone()))
+    }
+
+    pub fn contains_record(&self, coll: &CollectionName, rkey: &RecordKey) -> bool {
+        self.live.contains_key(&(coll.clone(), rkey.clone()))
+    }
+
+    pub fn set_root(&mut self, root: Cid) {
+        self.current_root = Some(root);
+    }
+
+    pub fn root(&self) -> Option<Cid> {
+        self.current_root
+    }
+
+    pub fn set_mst_node_cids(&mut self, cids: Vec<CidBytes>) {
+        self.mst_node_cids = cids;
+    }
+
+    pub fn clear_mst_state(&mut self) {
+        self.current_root = None;
+        self.mst_node_cids.clear();
+    }
+
+    pub fn live_records(&self) -> impl Iterator<Item = (&CollectionName, &RecordKey, &CidBytes)> {
+        self.live.iter().map(|((c, r), v)| (c, r, v))
+    }
+
+    pub fn live_count(&self) -> usize {
+        self.live.len()
+    }
+
+    pub fn live_cids_labeled(&self) -> Vec<(String, CidBytes)> {
+        let nodes = self
+            .mst_node_cids
+            .iter()
+            .map(|bytes| (format!("mst {}", hex_short(bytes)), *bytes));
+        let records = self
+            .live_records()
+            .map(|(c, r, v)| (format!("record {}/{}", c.0, r.0), *v));
+        nodes.chain(records).collect()
+    }
+
+    pub fn record_event_append(&mut self, event: EventExpectation) {
+        self.unsynced_events.push(event);
+    }
+
+    pub fn record_event_sync(&mut self, synced_through: EventSequence) {
+        let (promoted, remaining): (Vec<_>, Vec<_>) = self
+            .unsynced_events
+            .drain(..)
+            .partition(|e| e.seq <= synced_through);
+        self.synced_events.extend(promoted);
+        self.unsynced_events = remaining;
+        self.last_synced_seq = Some(synced_through);
+    }
+
+    pub fn record_crash(&mut self) {
+        self.unsynced_events.clear();
+    }
+
+    pub fn record_retention(&mut self, cutoff_us: u64) {
+        self.synced_events.retain(|e| e.timestamp_us >= cutoff_us);
+        self.last_retention_cutoff_us = Some(cutoff_us);
+    }
+
+    pub fn synced_events(&self) -> &[EventExpectation] {
+        &self.synced_events
+    }
+
+    pub fn unsynced_events(&self) -> &[EventExpectation] {
+        &self.unsynced_events
+    }
+
+    pub fn last_synced_seq(&self) -> Option<EventSequence> {
+        self.last_synced_seq
+    }
+
+    pub fn last_retention_cutoff_us(&self) -> Option<u64> {
+        self.last_retention_cutoff_us
+    }
+}
+
+pub(super) fn try_cid_to_fixed(cid: &Cid) -> Result<CidBytes, CidFormatError> {
+    let bytes = cid.to_bytes();
+    let actual = bytes.len();
+    bytes.try_into().map_err(|_| CidFormatError { actual })
+}
+
+pub(super) fn hex_short(cid: &CidBytes) -> String {
+    cid[cid.len() - 6..]
+        .iter()
+        .map(|b| format!("{b:02x}"))
+        .collect()
+}
@@ -0,0 +1,247 @@
+use serde::{Deserialize, Serialize};
+
+use super::runner::{
+    GauntletConfig, IoBackend, MaxFileSize, OpInterval, RestartPolicy, RunLimits, ShardCount,
+    WallMs, WriterConcurrency,
+};
+use super::workload::{KeySpaceSize, OpCount, SizeDistribution, ValueBytes};
+use crate::sim::FaultConfig;
+
+#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
+#[serde(deny_unknown_fields)]
+pub struct ConfigOverrides {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub op_count: Option<usize>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub max_wall_ms: Option<u64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub writer_concurrency: Option<usize>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub key_space: Option<u32>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub value_bytes: Option<u32>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub fault_density_scale: Option<f64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub fault_density_uniform: Option<f64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub restart_every_n_ops: Option<usize>,
+    #[serde(default, skip_serializing_if = "StoreOverrides::is_empty")]
+    pub store: StoreOverrides,
+}
+
+#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
+pub struct StoreOverrides {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub max_file_size: Option<u64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub shard_count: Option<u8>,
+    #[serde(default, skip_serializing_if = "GroupCommitOverrides::is_empty")]
+    pub group_commit: GroupCommitOverrides,
+}
+
+impl StoreOverrides {
+    pub fn is_empty(&self) -> bool {
+        self.max_file_size.is_none() && self.shard_count.is_none() && self.group_commit.is_empty()
+    }
+}
+
+#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
+pub struct GroupCommitOverrides {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub max_batch_size: Option<usize>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub channel_capacity: Option<usize>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub checkpoint_interval_ms: Option<u64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub checkpoint_write_threshold: Option<u64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub verify_persisted_blocks: Option<bool>,
+}
+
+impl GroupCommitOverrides {
+    pub fn is_empty(&self) -> bool {
+        self.max_batch_size.is_none()
+            && self.channel_capacity.is_none()
+            && self.checkpoint_interval_ms.is_none()
+            && self.checkpoint_write_threshold.is_none()
+            && self.verify_persisted_blocks.is_none()
+    }
+}
+
+impl ConfigOverrides {
+    pub fn apply_to(&self, cfg: &mut GauntletConfig) {
+        if let Some(n) = self.op_count {
+            cfg.op_count = OpCount(n);
+        }
+        if let Some(ms) = self.max_wall_ms {
+            cfg.limits = RunLimits {
+                max_wall_ms: Some(WallMs(ms)),
+            };
+        }
+        if let Some(n) = self.writer_concurrency {
+            cfg.writer_concurrency = WriterConcurrency(n.max(1));
+        }
+        if let Some(n) = self.key_space {
+            cfg.workload.key_space = KeySpaceSize(n.max(1));
+        }
+        if let Some(n) = self.value_bytes {
+            cfg.workload.size_distribution = SizeDistribution::Fixed(ValueBytes(n));
+        }
+        if let Some(m) = self.fault_density_scale
+            && let IoBackend::Simulated { fault } = cfg.io
+        {
+            cfg.io = IoBackend::Simulated {
+                fault: fault.scale_probabilities(m),
+            };
+        }
+        if let Some(d) = self.fault_density_uniform {
+            cfg.io = IoBackend::Simulated {
+                fault: FaultConfig::uniform_density(d.clamp(0.0, 1.0)),
+            };
+        }
+        if let Some(n) = self.restart_every_n_ops {
+            cfg.restart_policy = if n == 0 {
+                RestartPolicy::Never
+            } else {
+                RestartPolicy::EveryNOps(OpInterval(n))
+            };
+        }
+        if let Some(n) = self.store.max_file_size {
+            cfg.store.max_file_size = MaxFileSize(n);
+        }
+        if let Some(n) = self.store.shard_count {
+            cfg.store.shard_count = ShardCount(n);
+        }
+        let gc = &self.store.group_commit;
+        if let Some(n) = gc.max_batch_size {
+            cfg.store.group_commit.max_batch_size = n;
+        }
+        if let Some(n) = gc.channel_capacity {
+            cfg.store.group_commit.channel_capacity = n;
+        }
+        if let Some(n) = gc.checkpoint_interval_ms {
+            cfg.store.group_commit.checkpoint_interval_ms = n;
+        }
+        if let Some(n) = gc.checkpoint_write_threshold {
+            cfg.store.group_commit.checkpoint_write_threshold = n;
+        }
+        if let Some(b) = gc.verify_persisted_blocks {
+            cfg.store.group_commit.verify_persisted_blocks = b;
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn default_overrides_serialize_empty() {
+        let o = ConfigOverrides::default();
+        let json = serde_json::to_string(&o).unwrap();
+        assert_eq!(json, "{}");
+    }
+
+    #[test]
+    fn round_trip_preserves_set_fields() {
+        let o = ConfigOverrides {
+            op_count: Some(42),
+            writer_concurrency: Some(16),
+            key_space: Some(1_000_000),
+            value_bytes: Some(4096),
+            fault_density_scale: Some(1e-3),
+            fault_density_uniform: Some(5e-4),
+            restart_every_n_ops: Some(10_000),
+            store: StoreOverrides {
+                max_file_size: Some(4096),
+                group_commit: GroupCommitOverrides {
+                    max_batch_size: Some(16),
+                    ..GroupCommitOverrides::default()
+                },
+                ..StoreOverrides::default()
+            },
+            ..ConfigOverrides::default()
+        };
+        let json = serde_json::to_string(&o).unwrap();
+        let back: ConfigOverrides = serde_json::from_str(&json).unwrap();
+        assert_eq!(o, back);
+    }
+
+    #[test]
+    fn fault_density_scale_scales_moderate() {
+        use crate::gauntlet::op::Seed;
+        use crate::gauntlet::scenarios::{Scenario, config_for};
+        let mut cfg = config_for(Scenario::ModerateFaults, Seed(1));
+        let o = ConfigOverrides {
+            fault_density_scale: Some(0.1),
+            ..ConfigOverrides::default()
+        };
+        o.apply_to(&mut cfg);
+        match cfg.io {
+            IoBackend::Simulated { fault } => {
+                assert!(fault.torn_page_probability.raw() < 0.02);
+                assert!(fault.torn_page_probability.raw() > 0.0);
+            }
+            _ => panic!("expected simulated io"),
+        }
+    }
+
+    #[test]
+    fn fault_density_scale_zero_kills_probabilities() {
+        use crate::gauntlet::op::Seed;
+        use crate::gauntlet::scenarios::{Scenario, config_for};
+        let mut cfg = config_for(Scenario::ModerateFaults, Seed(1));
+        let o = ConfigOverrides {
+            fault_density_scale: Some(0.0),
+            ..ConfigOverrides::default()
+        };
+        o.apply_to(&mut cfg);
+        match cfg.io {
+            IoBackend::Simulated { fault } => {
+                assert_eq!(fault.partial_write_probability.raw(), 0.0);
+                assert_eq!(fault.torn_page_probability.raw(), 0.0);
+                assert_eq!(fault.io_error_probability.raw(), 0.0);
+                assert_eq!(fault.sync_failure_probability.raw(), 0.0);
+            }
+            _ => panic!("expected simulated io"),
+        }
+    }
+
+    #[test]
+    fn fault_density_scale_is_noop_on_real_backend() {
+        use crate::gauntlet::op::Seed;
+        use crate::gauntlet::scenarios::{Scenario, config_for};
+        let mut cfg = config_for(Scenario::SmokePR, Seed(1));
+        assert!(matches!(cfg.io, IoBackend::Real));
+        let o = ConfigOverrides {
+            fault_density_scale: Some(0.5),
+            ..ConfigOverrides::default()
+        };
+        o.apply_to(&mut cfg);
+        assert!(matches!(cfg.io, IoBackend::Real));
+    }
+
+    #[test]
+    fn fault_density_uniform_forces_simulated_backend() {
+        use crate::gauntlet::op::Seed;
+        use crate::gauntlet::scenarios::{Scenario, config_for};
+        let mut cfg = config_for(Scenario::SmokePR, Seed(1));
+        assert!(matches!(cfg.io, IoBackend::Real));
+        let o = ConfigOverrides {
+            fault_density_uniform: Some(0.25),
+            ..ConfigOverrides::default()
+        };
+        o.apply_to(&mut cfg);
+        match cfg.io {
+            IoBackend::Simulated { fault } => {
+                assert_eq!(fault.torn_page_probability.raw(), 0.25);
+                assert_eq!(fault.io_error_probability.raw(), 0.25);
+            }
+            _ => panic!("expected simulated io"),
+        }
+    }
+}
@@ -0,0 +1,290 @@
+use std::io;
+use std::path::{Path, PathBuf};
+
+use serde::{Deserialize, Serialize};
+
+use super::invariants::InvariantViolation;
+use super::op::{Op, OpStream, Seed};
+use super::overrides::ConfigOverrides;
+use super::runner::{GauntletConfig, GauntletReport};
+use super::scenarios::{Scenario, UnknownScenario, config_for};
+
+pub const SCHEMA_VERSION: u32 = 1;
+pub const MIN_SUPPORTED_SCHEMA_VERSION: u32 = 1;
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct RegressionViolation {
+    pub invariant: String,
+    pub detail: String,
+}
+
+impl From<&InvariantViolation> for RegressionViolation {
+    fn from(v: &InvariantViolation) -> Self {
+        Self {
+            invariant: v.invariant.to_string(),
+            detail: v.detail.clone(),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct RegressionRecord {
+    pub schema_version: u32,
+    pub scenario: String,
+    pub seed: Seed,
+    #[serde(default)]
+    pub overrides: ConfigOverrides,
+    pub violations: Vec<RegressionViolation>,
+    pub ops: Vec<Op>,
+    #[serde(default)]
+    pub original_ops_len: usize,
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum RegressionLoadError {
+    #[error("read {path}: {source}")]
+    Read { path: PathBuf, source: io::Error },
+    #[error("parse {path}: {source}")]
+    Parse {
+        path: PathBuf,
+        source: serde_json::Error,
+    },
+    #[error("schema version {found} outside supported range {min}..={max}")]
+    UnsupportedVersion { found: u32, min: u32, max: u32 },
+    #[error(transparent)]
+    UnknownScenario(#[from] UnknownScenario),
+}
+
+impl RegressionRecord {
+    pub fn from_report(
+        scenario: Scenario,
+        overrides: ConfigOverrides,
+        report: &GauntletReport,
+        original_ops_len: usize,
+        shrunk_ops: OpStream,
+    ) -> Self {
+        Self {
+            schema_version: SCHEMA_VERSION,
+            scenario: scenario.name().to_string(),
+            seed: report.seed,
+            overrides,
+            violations: report
+                .violations
+                .iter()
+                .map(RegressionViolation::from)
+                .collect(),
+            ops: shrunk_ops.into_vec(),
+            original_ops_len,
+        }
+    }
+
+    pub fn file_path(&self, root: &Path) -> PathBuf {
+        root.join("gauntlet")
+            .join(sanitize(&self.scenario))
+            .join(format!("{:016x}.json", self.seed.0))
+    }
+
+    pub fn write_to(&self, root: &Path) -> io::Result<PathBuf> {
+        let path = self.file_path(root);
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent)?;
+        }
+        let json = serde_json::to_vec_pretty(self).map_err(io::Error::other)?;
+        let tmp = path.with_extension("json.tmp");
+        {
+            let mut f = std::fs::File::create(&tmp)?;
+            io::Write::write_all(&mut f, &json)?;
+            f.sync_all()?;
+        }
+        std::fs::rename(&tmp, &path)?;
+        if let Some(parent) = path.parent()
+            && let Ok(dir) = std::fs::File::open(parent)
+        {
+            let _ = dir.sync_all();
+        }
+        Ok(path)
+    }
+
+    pub fn load(path: &Path) -> Result<Self, RegressionLoadError> {
+        let raw = std::fs::read(path).map_err(|source| RegressionLoadError::Read {
+            path: path.to_path_buf(),
+            source,
+        })?;
+        let record: RegressionRecord =
+            serde_json::from_slice(&raw).map_err(|source| RegressionLoadError::Parse {
+                path: path.to_path_buf(),
+                source,
+            })?;
+        if record.schema_version < MIN_SUPPORTED_SCHEMA_VERSION
+            || record.schema_version > SCHEMA_VERSION
+        {
+            return Err(RegressionLoadError::UnsupportedVersion {
+                found: record.schema_version,
+                min: MIN_SUPPORTED_SCHEMA_VERSION,
+                max: SCHEMA_VERSION,
+            });
+        }
+        Ok(record)
+    }
+
+    pub fn scenario_enum(&self) -> Result<Scenario, UnknownScenario> {
+        self.scenario.parse::<Scenario>()
+    }
+
+    pub fn build_config(&self) -> Result<GauntletConfig, UnknownScenario> {
+        let scenario = self.scenario_enum()?;
+        let mut cfg = config_for(scenario, self.seed);
+        self.overrides.apply_to(&mut cfg);
+        Ok(cfg)
+    }
+
+    pub fn op_stream(&self) -> OpStream {
+        OpStream::from_vec(self.ops.clone())
+    }
+}
+
+fn sanitize(s: &str) -> String {
+    s.chars()
+        .map(|c| match c {
+            'a'..='z' | 'A'..='Z' | '0'..='9' | '_' | '-' => c,
+            _ => '_',
+        })
+        .collect()
+}
+
+pub fn default_root() -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("proptest-regressions")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::gauntlet::op::{CollectionName, RecordKey, ValueSeed};
+    use crate::gauntlet::overrides::ConfigOverrides;
+
+    fn sample_record() -> RegressionRecord {
+        use crate::gauntlet::overrides::StoreOverrides;
+
+        let ops = vec![
+            Op::AddRecord {
+                collection: CollectionName("c".into()),
+                rkey: RecordKey("r".into()),
+                value_seed: ValueSeed(1),
+            },
+            Op::Compact,
+        ];
+        let overrides = ConfigOverrides {
+            op_count: Some(128),
+            store: StoreOverrides {
+                max_file_size: Some(4096),
+                ..StoreOverrides::default()
+            },
+            ..ConfigOverrides::default()
+        };
+        RegressionRecord {
+            schema_version: SCHEMA_VERSION,
+            scenario: "HugeValues".to_string(),
+            seed: Seed(0xdeadbeef),
+            overrides,
+            violations: vec![RegressionViolation {
+                invariant: "ByteBudget".to_string(),
+                detail: "exceeded".to_string(),
+            }],
+            ops,
+            original_ops_len: 500,
+        }
+    }
+
+    #[test]
+    fn round_trip_preserves_all_fields() {
+        let dir = tempfile::TempDir::new().unwrap();
+        let original = sample_record();
+        let path = original.write_to(dir.path()).unwrap();
+        assert!(path.exists());
+        let loaded = RegressionRecord::load(&path).unwrap();
+        assert_eq!(loaded.schema_version, original.schema_version);
+        assert_eq!(loaded.scenario, original.scenario);
+        assert_eq!(loaded.seed.0, original.seed.0);
+        assert_eq!(loaded.overrides, original.overrides);
+        assert_eq!(loaded.violations, original.violations);
+        assert_eq!(loaded.ops.len(), original.ops.len());
+        assert_eq!(loaded.original_ops_len, original.original_ops_len);
+    }
+
+    #[test]
+    fn build_config_applies_overrides() {
+        let record = sample_record();
+        let cfg = record.build_config().unwrap();
+        assert_eq!(cfg.op_count.0, 128);
+        assert_eq!(cfg.store.max_file_size.0, 4096);
+    }
+
+    #[test]
+    fn rejects_future_schema_version() {
+        let dir = tempfile::TempDir::new().unwrap();
+        let mut r = sample_record();
+        r.schema_version = SCHEMA_VERSION + 1;
+        let path = r.write_to(dir.path()).unwrap();
+        match RegressionRecord::load(&path) {
+            Err(RegressionLoadError::UnsupportedVersion { found, min, max }) => {
+                assert_eq!(found, SCHEMA_VERSION + 1);
+                assert_eq!(min, MIN_SUPPORTED_SCHEMA_VERSION);
+                assert_eq!(max, SCHEMA_VERSION);
+            }
+            other => panic!("expected UnsupportedVersion, got {other:?}"),
+        }
+    }
+
+    #[test]
+    fn rejects_past_schema_version_below_min() {
+        let dir = tempfile::TempDir::new().unwrap();
+        let mut r = sample_record();
+        r.schema_version = MIN_SUPPORTED_SCHEMA_VERSION.saturating_sub(1);
+        let path = r.write_to(dir.path()).unwrap();
+        match RegressionRecord::load(&path) {
+            Err(RegressionLoadError::UnsupportedVersion { found, min, max }) => {
+                assert_eq!(found, MIN_SUPPORTED_SCHEMA_VERSION.saturating_sub(1));
+                assert_eq!(min, MIN_SUPPORTED_SCHEMA_VERSION);
+                assert_eq!(max, SCHEMA_VERSION);
+            }
+            other => panic!("expected UnsupportedVersion, got {other:?}"),
+        }
+    }
+
+    #[test]
+    fn atomic_write_leaves_no_tmp_file() {
+        let dir = tempfile::TempDir::new().unwrap();
+        let r = sample_record();
+        let path = r.write_to(dir.path()).unwrap();
+        assert!(path.exists());
+        let tmp = path.with_extension("json.tmp");
+        assert!(
+            !tmp.exists(),
+            "tmp sibling {tmp:?} should have been renamed"
+        );
+    }
+
+    #[test]
+    fn rejects_malformed_json() {
+        let dir = tempfile::TempDir::new().unwrap();
+        let path = dir.path().join("bad.json");
+        std::fs::write(&path, b"{not json").unwrap();
+        assert!(matches!(
+            RegressionRecord::load(&path),
+            Err(RegressionLoadError::Parse { .. })
+        ));
+    }
+
+    #[test]
+    fn sanitize_strips_slashes_and_traversal() {
+        assert_eq!(sanitize("foo/bar baz"), "foo_bar_baz");
+        assert_eq!(sanitize("../etc"), "___etc");
+    }
+
+    #[test]
+    fn unknown_scenario_name_errors() {
+        let mut r = sample_record();
+        r.scenario = "BogusScenario".to_string();
+        assert!(r.build_config().is_err());
+    }
+}
@@ -0,0 +1,757 @@
+use super::flaky::FlakyConfig;
+use super::invariants::InvariantSet;
+use super::op::{CollectionName, Seed};
+use super::runner::{
+    EventLogConfig, GauntletConfig, IoBackend, MaxFileSize, MaxSegmentSize, OpInterval,
+    RestartPolicy, RunLimits, ShardCount, StoreConfig, WallMs, WriterConcurrency,
+};
+use super::workload::{
+    ByteRange, DidSpaceSize, KeySpaceSize, OpCount, OpWeights, RetentionMaxSecs, SizeDistribution,
+    ValueBytes, WorkloadModel,
+};
+use crate::blockstore::{GroupCommitConfig, MAX_BLOCK_SIZE};
+use crate::sim::FaultConfig;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Scenario {
+    SmokePR,
+    MstChurn,
+    MstRestartChurn,
+    FullStackRestart,
+    CatastrophicChurn,
+    HugeValues,
+    TinyBatches,
+    GiantBatches,
+    ManyFiles,
+    ModerateFaults,
+    AggressiveFaults,
+    TornPages,
+    Fsyncgate,
+    FirehoseFanout,
+    ContendedReaders,
+    ContendedWriters,
+    FlakyDevice,
+}
+
+impl Scenario {
+    pub const fn name(self) -> &'static str {
+        match self {
+            Self::SmokePR => "SmokePR",
+            Self::MstChurn => "MstChurn",
+            Self::MstRestartChurn => "MstRestartChurn",
+            Self::FullStackRestart => "FullStackRestart",
+            Self::CatastrophicChurn => "CatastrophicChurn",
+            Self::HugeValues => "HugeValues",
+            Self::TinyBatches => "TinyBatches",
+            Self::GiantBatches => "GiantBatches",
+            Self::ManyFiles => "ManyFiles",
+            Self::ModerateFaults => "ModerateFaults",
+            Self::AggressiveFaults => "AggressiveFaults",
+            Self::TornPages => "TornPages",
+            Self::Fsyncgate => "Fsyncgate",
+            Self::FirehoseFanout => "FirehoseFanout",
+            Self::ContendedReaders => "ContendedReaders",
+            Self::ContendedWriters => "ContendedWriters",
+            Self::FlakyDevice => "FlakyDevice",
+        }
+    }
+
+    pub const fn cli_name(self) -> &'static str {
+        match self {
+            Self::SmokePR => "smoke-pr",
+            Self::MstChurn => "mst-churn",
+            Self::MstRestartChurn => "mst-restart-churn",
+            Self::FullStackRestart => "full-stack-restart",
+            Self::CatastrophicChurn => "catastrophic-churn",
+            Self::HugeValues => "huge-values",
+            Self::TinyBatches => "tiny-batches",
+            Self::GiantBatches => "giant-batches",
+            Self::ManyFiles => "many-files",
+            Self::ModerateFaults => "moderate-faults",
+            Self::AggressiveFaults => "aggressive-faults",
+            Self::TornPages => "torn-pages",
+            Self::Fsyncgate => "fsyncgate",
+            Self::FirehoseFanout => "firehose-fanout",
+            Self::ContendedReaders => "contended-readers",
+            Self::ContendedWriters => "contended-writers",
+            Self::FlakyDevice => "flaky-device",
+        }
+    }
+
+    pub const fn description(self) -> &'static str {
+        match self {
+            Self::SmokePR => "60s canary, 10k ops, core invariants. Default PR gate.",
+            Self::MstChurn => "100k churn, no restart. Refcount + reachability focus.",
+            Self::MstRestartChurn => "100k churn with Poisson restart bursts every ~5k ops.",
+            Self::FullStackRestart => "5k ops, deterministic restart every 500 ops.",
+            Self::CatastrophicChurn => {
+                "1M ops, phase-2 invariants, Poisson restart. 30 min budget."
+            }
+            Self::HugeValues => "Heavy-tail values up to 16 MiB. 32 MiB file cap.",
+            Self::TinyBatches => "Group-commit batch size 1, tight checkpoints, 4 KiB files.",
+            Self::GiantBatches => "Group-commit batch size 100k, 16 MiB files.",
+            Self::ManyFiles => "256-byte file cap, many segments, delete-heavy.",
+            Self::ModerateFaults => {
+                "Simulated IO with moderate fault config. CrashAtSyscall restarts."
+            }
+            Self::AggressiveFaults => {
+                "Simulated IO with aggressive fault config. CrashAtSyscall restarts."
+            }
+            Self::TornPages => "Torn-page faults only, 20k ops.",
+            Self::Fsyncgate => "Fsync-drop faults only, 10k ops.",
+            Self::FirehoseFanout => {
+                "Eventlog-heavy workload with FSYNC_ORDERING / MONOTONIC_SEQ / TOMBSTONE_BOUND invariants."
+            }
+            Self::ContendedReaders => "60% reads, 64 writer tasks, simulated moderate faults.",
+            Self::ContendedWriters => {
+                "Add/delete heavy, 32 writer tasks, simulated moderate faults."
+            }
+            Self::FlakyDevice => {
+                "Real IO on ext4 atop dm-flakey. Requires root with dm-flakey available, skips otherwise."
+            }
+        }
+    }
+
+    pub fn from_name(name: &str) -> Option<Self> {
+        Self::ALL.iter().copied().find(|s| s.name() == name)
+    }
+
+    pub fn from_cli_name(name: &str) -> Option<Self> {
+        Self::ALL.iter().copied().find(|s| s.cli_name() == name)
+    }
+
+    pub const ALL: &'static [Scenario] = &[
+        Self::SmokePR,
+        Self::MstChurn,
+        Self::MstRestartChurn,
+        Self::FullStackRestart,
+        Self::CatastrophicChurn,
+        Self::HugeValues,
+        Self::TinyBatches,
+        Self::GiantBatches,
+        Self::ManyFiles,
+        Self::ModerateFaults,
+        Self::AggressiveFaults,
+        Self::TornPages,
+        Self::Fsyncgate,
+        Self::FirehoseFanout,
+        Self::ContendedReaders,
+        Self::ContendedWriters,
+        Self::FlakyDevice,
+    ];
+}
+
+impl serde::Serialize for Scenario {
+    fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        serializer.serialize_str(self.cli_name())
+    }
+}
+
+impl<'de> serde::Deserialize<'de> for Scenario {
+    fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+        let s = <std::borrow::Cow<'de, str>>::deserialize(deserializer)?;
+        Self::from_cli_name(&s).ok_or_else(|| {
+            serde::de::Error::custom(format!(
+                "unknown scenario {s:?}; expected one of {}",
+                Self::ALL
+                    .iter()
+                    .map(|s| s.cli_name())
+                    .collect::<Vec<_>>()
+                    .join(", ")
+            ))
+        })
+    }
+}
+
+#[cfg(feature = "gauntlet-cli")]
+impl clap::ValueEnum for Scenario {
+    fn value_variants<'a>() -> &'a [Self] {
+        Self::ALL
+    }
+
+    fn to_possible_value(&self) -> Option<clap::builder::PossibleValue> {
+        Some(clap::builder::PossibleValue::new(self.cli_name()).help(self.description()))
+    }
+}
+
+impl std::fmt::Display for Scenario {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(self.name())
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+#[error("unknown scenario: {0}")]
+pub struct UnknownScenario(pub String);
+
+impl std::str::FromStr for Scenario {
+    type Err = UnknownScenario;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Self::from_name(s).ok_or_else(|| UnknownScenario(s.to_string()))
+    }
+}
+
+pub fn config_for(scenario: Scenario, seed: Seed) -> GauntletConfig {
+    match scenario {
+        Scenario::SmokePR => smoke_pr(seed),
+        Scenario::MstChurn => mst_churn(seed),
+        Scenario::MstRestartChurn => mst_restart_churn(seed),
+        Scenario::FullStackRestart => full_stack_restart(seed),
+        Scenario::CatastrophicChurn => catastrophic_churn(seed),
+        Scenario::HugeValues => huge_values(seed),
+        Scenario::TinyBatches => tiny_batches(seed),
+        Scenario::GiantBatches => giant_batches(seed),
+        Scenario::ManyFiles => many_files(seed),
+        Scenario::ModerateFaults => moderate_faults(seed),
+        Scenario::AggressiveFaults => aggressive_faults(seed),
+        Scenario::TornPages => torn_pages(seed),
+        Scenario::Fsyncgate => fsyncgate(seed),
+        Scenario::FirehoseFanout => firehose_fanout(seed),
+        Scenario::ContendedReaders => contended_readers(seed),
+        Scenario::ContendedWriters => contended_writers(seed),
+        Scenario::FlakyDevice => flaky_device(seed),
+    }
+}
+
+fn default_collections() -> Vec<CollectionName> {
+    vec![
+        CollectionName("app.bsky.feed.post".to_string()),
+        CollectionName("app.bsky.feed.like".to_string()),
+    ]
+}
+
+fn block_weights(add: u32, delete: u32, compact: u32, checkpoint: u32) -> OpWeights {
+    OpWeights {
+        add,
+        delete,
+        compact,
+        checkpoint,
+        ..OpWeights::default()
+    }
+}
+
+fn block_workload(
+    weights: OpWeights,
+    size_distribution: SizeDistribution,
+    key_space: KeySpaceSize,
+) -> WorkloadModel {
+    WorkloadModel {
+        weights,
+        size_distribution,
+        collections: default_collections(),
+        key_space,
+        did_space: DidSpaceSize(32),
+        retention_max_secs: RetentionMaxSecs(3600),
+    }
+}
+
+fn tiny_store() -> StoreConfig {
+    StoreConfig {
+        max_file_size: MaxFileSize(4096),
+        group_commit: GroupCommitConfig {
+            checkpoint_interval_ms: 100,
+            checkpoint_write_threshold: 10,
+            ..GroupCommitConfig::default()
+        },
+        shard_count: ShardCount(1),
+    }
+}
+
+fn smoke_pr(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Real,
+        workload: block_workload(
+            block_weights(80, 0, 10, 10),
+            SizeDistribution::Fixed(ValueBytes(64)),
+            KeySpaceSize(200),
+        ),
+        op_count: OpCount(10_000),
+        invariants: InvariantSet::REFCOUNT_CONSERVATION
+            | InvariantSet::REACHABILITY
+            | InvariantSet::ACKED_WRITE_PERSISTENCE
+            | InvariantSet::READ_AFTER_WRITE
+            | InvariantSet::RESTART_IDEMPOTENT,
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(60_000)),
+        },
+        restart_policy: RestartPolicy::EveryNOps(OpInterval(2_000)),
+        store: tiny_store(),
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn mst_churn(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Real,
+        workload: block_workload(
+            block_weights(85, 0, 10, 5),
+            SizeDistribution::Fixed(ValueBytes(64)),
+            KeySpaceSize(2_000),
+        ),
+        op_count: OpCount(100_000),
+        invariants: InvariantSet::REFCOUNT_CONSERVATION
+            | InvariantSet::REACHABILITY
+            | InvariantSet::ACKED_WRITE_PERSISTENCE
+            | InvariantSet::READ_AFTER_WRITE
+            | InvariantSet::RESTART_IDEMPOTENT,
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(600_000)),
+        },
+        restart_policy: RestartPolicy::Never,
+        store: tiny_store(),
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn mst_restart_churn(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Real,
+        workload: block_workload(
+            block_weights(85, 0, 10, 5),
+            SizeDistribution::Fixed(ValueBytes(64)),
+            KeySpaceSize(2_000),
+        ),
+        op_count: OpCount(100_000),
+        invariants: InvariantSet::REFCOUNT_CONSERVATION
+            | InvariantSet::REACHABILITY
+            | InvariantSet::ACKED_WRITE_PERSISTENCE
+            | InvariantSet::READ_AFTER_WRITE
+            | InvariantSet::RESTART_IDEMPOTENT,
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(600_000)),
+        },
+        restart_policy: RestartPolicy::PoissonByOps(OpInterval(5_000)),
+        store: tiny_store(),
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn full_stack_restart(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Real,
+        workload: block_workload(
+            block_weights(80, 0, 15, 5),
+            SizeDistribution::Fixed(ValueBytes(80)),
+            KeySpaceSize(500),
+        ),
+        op_count: OpCount(5_000),
+        invariants: InvariantSet::REFCOUNT_CONSERVATION
+            | InvariantSet::REACHABILITY
+            | InvariantSet::ACKED_WRITE_PERSISTENCE
+            | InvariantSet::READ_AFTER_WRITE
+            | InvariantSet::RESTART_IDEMPOTENT,
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(120_000)),
+        },
+        restart_policy: RestartPolicy::EveryNOps(OpInterval(500)),
+        store: StoreConfig {
+            max_file_size: MaxFileSize(4096),
+            group_commit: GroupCommitConfig::default(),
+            shard_count: ShardCount(1),
+        },
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn phase2_invariants() -> InvariantSet {
+    InvariantSet::REFCOUNT_CONSERVATION
+        | InvariantSet::REACHABILITY
+        | InvariantSet::ACKED_WRITE_PERSISTENCE
+        | InvariantSet::READ_AFTER_WRITE
+        | InvariantSet::RESTART_IDEMPOTENT
+        | InvariantSet::COMPACTION_IDEMPOTENT
+        | InvariantSet::BYTE_BUDGET
+        | InvariantSet::MANIFEST_EQUALS_REALITY
+        | InvariantSet::CHECKSUM_COVERAGE
+}
+
+fn catastrophic_churn(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Real,
+        workload: block_workload(
+            block_weights(94, 0, 5, 1),
+            SizeDistribution::Fixed(ValueBytes(64)),
+            KeySpaceSize(200),
+        ),
+        op_count: OpCount(1_000_000),
+        invariants: phase2_invariants(),
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(30 * 60_000)),
+        },
+        restart_policy: RestartPolicy::PoissonByOps(OpInterval(50_000)),
+        store: tiny_store(),
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn huge_values(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Real,
+        workload: block_workload(
+            block_weights(85, 5, 8, 2),
+            SizeDistribution::HeavyTail(
+                ByteRange::new(ValueBytes(256), ValueBytes(MAX_BLOCK_SIZE))
+                    .expect("huge_values ByteRange"),
+            ),
+            KeySpaceSize(64),
+        ),
+        op_count: OpCount(2_000),
+        invariants: InvariantSet::REFCOUNT_CONSERVATION
+            | InvariantSet::REACHABILITY
+            | InvariantSet::ACKED_WRITE_PERSISTENCE
+            | InvariantSet::READ_AFTER_WRITE
+            | InvariantSet::RESTART_IDEMPOTENT,
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(10 * 60_000)),
+        },
+        restart_policy: RestartPolicy::EveryNOps(OpInterval(500)),
+        store: StoreConfig {
+            max_file_size: MaxFileSize(32 * 1024 * 1024),
+            group_commit: GroupCommitConfig::default(),
+            shard_count: ShardCount(1),
+        },
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn tiny_batches(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Real,
+        workload: block_workload(
+            block_weights(85, 0, 5, 10),
+            SizeDistribution::Fixed(ValueBytes(64)),
+            KeySpaceSize(500),
+        ),
+        op_count: OpCount(10_000),
+        invariants: phase2_invariants(),
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(120_000)),
+        },
+        restart_policy: RestartPolicy::EveryNOps(OpInterval(2_000)),
+        store: StoreConfig {
+            max_file_size: MaxFileSize(4096),
+            group_commit: GroupCommitConfig {
+                max_batch_size: 1,
+                checkpoint_interval_ms: 100,
+                checkpoint_write_threshold: 1,
+                ..GroupCommitConfig::default()
+            },
+            shard_count: ShardCount(1),
+        },
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn giant_batches(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Real,
+        workload: block_workload(
+            block_weights(95, 0, 3, 2),
+            SizeDistribution::Fixed(ValueBytes(64)),
+            KeySpaceSize(5_000),
+        ),
+        op_count: OpCount(50_000),
+        invariants: phase2_invariants(),
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(10 * 60_000)),
+        },
+        restart_policy: RestartPolicy::EveryNOps(OpInterval(10_000)),
+        store: StoreConfig {
+            max_file_size: MaxFileSize(16 * 1024 * 1024),
+            group_commit: GroupCommitConfig {
+                max_batch_size: 100_000,
+                checkpoint_interval_ms: 5_000,
+                checkpoint_write_threshold: 100_000,
+                ..GroupCommitConfig::default()
+            },
+            shard_count: ShardCount(1),
+        },
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn many_files(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Real,
+        workload: block_workload(
+            block_weights(80, 10, 5, 5),
+            SizeDistribution::Fixed(ValueBytes(128)),
+            KeySpaceSize(2_000),
+        ),
+        op_count: OpCount(200_000),
+        invariants: phase2_invariants(),
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(20 * 60_000)),
+        },
+        restart_policy: RestartPolicy::PoissonByOps(OpInterval(5_000)),
+        store: StoreConfig {
+            max_file_size: MaxFileSize(256),
+            group_commit: GroupCommitConfig::default(),
+            shard_count: ShardCount(1),
+        },
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn sim_invariants() -> InvariantSet {
+    InvariantSet::REFCOUNT_CONSERVATION
+        | InvariantSet::REACHABILITY
+        | InvariantSet::ACKED_WRITE_PERSISTENCE
+        | InvariantSet::READ_AFTER_WRITE
+        | InvariantSet::RESTART_IDEMPOTENT
+        | InvariantSet::NO_ORPHAN_FILES
+        | InvariantSet::BYTE_BUDGET
+        | InvariantSet::CHECKSUM_COVERAGE
+}
+
+fn sim_microbench_workload() -> WorkloadModel {
+    block_workload(
+        block_weights(80, 10, 5, 5),
+        SizeDistribution::Fixed(ValueBytes(128)),
+        KeySpaceSize(500),
+    )
+}
+
+fn sim_store() -> StoreConfig {
+    StoreConfig {
+        max_file_size: MaxFileSize(16 * 1024),
+        group_commit: GroupCommitConfig {
+            verify_persisted_blocks: true,
+            ..GroupCommitConfig::default()
+        },
+        shard_count: ShardCount(1),
+    }
+}
+
+fn moderate_faults(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Simulated {
+            fault: FaultConfig::moderate(),
+        },
+        workload: sim_microbench_workload(),
+        op_count: OpCount(50_000),
+        invariants: sim_invariants(),
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(10 * 60_000)),
+        },
+        restart_policy: RestartPolicy::CrashAtSyscall(OpInterval(2_000)),
+        store: sim_store(),
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn aggressive_faults(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Simulated {
+            fault: FaultConfig::aggressive(),
+        },
+        workload: sim_microbench_workload(),
+        op_count: OpCount(50_000),
+        invariants: sim_invariants(),
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(10 * 60_000)),
+        },
+        restart_policy: RestartPolicy::CrashAtSyscall(OpInterval(2_000)),
+        store: sim_store(),
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn torn_pages(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Simulated {
+            fault: FaultConfig::torn_pages_only(),
+        },
+        workload: sim_microbench_workload(),
+        op_count: OpCount(20_000),
+        invariants: sim_invariants(),
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(5 * 60_000)),
+        },
+        restart_policy: RestartPolicy::CrashAtSyscall(OpInterval(1_000)),
+        store: sim_store(),
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn fsyncgate(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Simulated {
+            fault: FaultConfig::fsyncgate_only(),
+        },
+        workload: sim_microbench_workload(),
+        op_count: OpCount(10_000),
+        invariants: sim_invariants(),
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(5 * 60_000)),
+        },
+        restart_policy: RestartPolicy::CrashAtSyscall(OpInterval(500)),
+        store: sim_store(),
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn firehose_fanout(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Simulated {
+            fault: FaultConfig::moderate(),
+        },
+        workload: WorkloadModel {
+            weights: OpWeights {
+                add: 20,
+                compact: 2,
+                checkpoint: 3,
+                append_event: 60,
+                sync_event_log: 10,
+                run_retention: 5,
+                ..OpWeights::default()
+            },
+            size_distribution: SizeDistribution::Fixed(ValueBytes(128)),
+            collections: default_collections(),
+            key_space: KeySpaceSize(500),
+            did_space: DidSpaceSize(64),
+            retention_max_secs: RetentionMaxSecs(60),
+        },
+        op_count: OpCount(20_000),
+        invariants: sim_invariants()
+            | InvariantSet::MONOTONIC_SEQ
+            | InvariantSet::FSYNC_ORDERING
+            | InvariantSet::TOMBSTONE_BOUND,
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(10 * 60_000)),
+        },
+        restart_policy: RestartPolicy::CrashAtSyscall(OpInterval(2_000)),
+        store: sim_store(),
+        eventlog: Some(EventLogConfig {
+            max_segment_size: MaxSegmentSize(64 * 1024),
+        }),
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn contended_readers(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Simulated {
+            fault: FaultConfig::moderate(),
+        },
+        workload: WorkloadModel {
+            weights: OpWeights {
+                add: 15,
+                delete: 1,
+                compact: 2,
+                checkpoint: 2,
+                read_record: 60,
+                read_block: 20,
+                ..OpWeights::default()
+            },
+            size_distribution: SizeDistribution::Fixed(ValueBytes(128)),
+            collections: default_collections(),
+            key_space: KeySpaceSize(400),
+            did_space: DidSpaceSize(32),
+            retention_max_secs: RetentionMaxSecs(3600),
+        },
+        op_count: OpCount(20_000),
+        invariants: sim_invariants(),
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(10 * 60_000)),
+        },
+        restart_policy: RestartPolicy::CrashAtSyscall(OpInterval(2_000)),
+        store: sim_store(),
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(64),
+    }
+}
+
+fn flaky_device(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::RealWithFlaky {
+            flaky: FlakyConfig::default_stress(),
+        },
+        workload: block_workload(
+            block_weights(80, 5, 10, 5),
+            SizeDistribution::Fixed(ValueBytes(128)),
+            KeySpaceSize(500),
+        ),
+        op_count: OpCount(20_000),
+        invariants: InvariantSet::REFCOUNT_CONSERVATION
+            | InvariantSet::REACHABILITY
+            | InvariantSet::ACKED_WRITE_PERSISTENCE
+            | InvariantSet::READ_AFTER_WRITE
+            | InvariantSet::RESTART_IDEMPOTENT
+            | InvariantSet::NO_ORPHAN_FILES
+            | InvariantSet::MANIFEST_EQUALS_REALITY
+            | InvariantSet::BYTE_BUDGET
+            | InvariantSet::CHECKSUM_COVERAGE,
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(30 * 60_000)),
+        },
+        restart_policy: RestartPolicy::EveryNOps(OpInterval(1_000)),
+        store: tiny_store(),
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn contended_writers(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Simulated {
+            fault: FaultConfig::moderate(),
+        },
+        workload: WorkloadModel {
+            weights: OpWeights {
+                add: 85,
+                delete: 5,
+                compact: 3,
+                checkpoint: 2,
+                read_record: 4,
+                read_block: 1,
+                ..OpWeights::default()
+            },
+            size_distribution: SizeDistribution::Fixed(ValueBytes(128)),
+            collections: default_collections(),
+            key_space: KeySpaceSize(1_000),
+            did_space: DidSpaceSize(32),
+            retention_max_secs: RetentionMaxSecs(3600),
+        },
+        op_count: OpCount(20_000),
+        invariants: sim_invariants(),
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(10 * 60_000)),
+        },
+        restart_policy: RestartPolicy::CrashAtSyscall(OpInterval(2_000)),
+        store: sim_store(),
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(32),
+    }
+}
@@ -0,0 +1,191 @@
+use std::collections::BTreeSet;
+
+use super::op::OpStream;
+use super::runner::{Gauntlet, GauntletConfig, GauntletReport};
+
+pub const DEFAULT_MAX_SHRINK_ITERATIONS: usize = 256;
+
+#[derive(Debug)]
+pub struct ShrinkOutcome {
+    pub ops: OpStream,
+    pub report: GauntletReport,
+    pub iterations: usize,
+}
+
+pub async fn shrink_failure(
+    config: GauntletConfig,
+    initial_ops: OpStream,
+    initial_report: GauntletReport,
+    max_iterations: usize,
+) -> ShrinkOutcome {
+    let target: BTreeSet<&'static str> = initial_report.violation_invariants();
+    if target.is_empty() {
+        return ShrinkOutcome {
+            ops: initial_ops,
+            report: initial_report,
+            iterations: 0,
+        };
+    }
+
+    let mut current_ops = initial_ops;
+    let mut current_report = initial_report;
+    let mut iterations = 0usize;
+
+    while iterations < max_iterations {
+        match try_one_shrink_round(&config, &current_ops, &target, max_iterations - iterations)
+            .await
+        {
+            ShrinkRound::Progress {
+                ops,
+                report,
+                runs_used,
+            } => {
+                current_ops = ops;
+                current_report = report;
+                iterations += runs_used;
+            }
+            ShrinkRound::Exhausted { runs_used } => {
+                iterations += runs_used;
+                break;
+            }
+        }
+    }
+
+    ShrinkOutcome {
+        ops: current_ops,
+        report: current_report,
+        iterations,
+    }
+}
+
+enum ShrinkRound {
+    Progress {
+        ops: OpStream,
+        report: GauntletReport,
+        runs_used: usize,
+    },
+    Exhausted {
+        runs_used: usize,
+    },
+}
+
+async fn try_one_shrink_round(
+    config: &GauntletConfig,
+    current_ops: &OpStream,
+    target: &BTreeSet<&'static str>,
+    budget: usize,
+) -> ShrinkRound {
+    let mut runs_used = 0usize;
+    for candidate in current_ops.shrink_candidates() {
+        if candidate.is_empty() || candidate.len() >= current_ops.len() {
+            continue;
+        }
+        if runs_used >= budget {
+            return ShrinkRound::Exhausted { runs_used };
+        }
+        runs_used += 1;
+        let gauntlet = match Gauntlet::new(config.clone()) {
+            Ok(g) => g,
+            Err(_) => continue,
+        };
+        let report = gauntlet.run_with_ops(candidate.clone()).await;
+        let got: BTreeSet<&'static str> = report.violation_invariants();
+        if !got.is_disjoint(target) {
+            return ShrinkRound::Progress {
+                ops: candidate,
+                report,
+                runs_used,
+            };
+        }
+    }
+    ShrinkRound::Exhausted { runs_used }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::blockstore::GroupCommitConfig;
+    use crate::gauntlet::invariants::{InvariantSet, InvariantViolation};
+    use crate::gauntlet::op::{CollectionName, Op, OpStream, RecordKey, Seed, ValueSeed};
+    use crate::gauntlet::runner::{
+        GauntletConfig, IoBackend, MaxFileSize, OpErrorCount, OpsExecuted, RestartCount,
+        RestartPolicy, RunLimits, ShardCount, StoreConfig, WriterConcurrency,
+    };
+    use crate::gauntlet::workload::{
+        DidSpaceSize, KeySpaceSize, OpCount, OpWeights, RetentionMaxSecs, SizeDistribution,
+        ValueBytes, WorkloadModel,
+    };
+    use crate::sim::FaultConfig;
+
+    fn dummy_config() -> GauntletConfig {
+        GauntletConfig {
+            seed: Seed(1),
+            io: IoBackend::Simulated {
+                fault: FaultConfig::none(),
+            },
+            workload: WorkloadModel {
+                weights: OpWeights::default(),
+                size_distribution: SizeDistribution::Fixed(ValueBytes(16)),
+                collections: vec![CollectionName("c".into())],
+                key_space: KeySpaceSize(4),
+                did_space: DidSpaceSize(1),
+                retention_max_secs: RetentionMaxSecs(60),
+            },
+            op_count: OpCount(4),
+            invariants: InvariantSet::EMPTY,
+            limits: RunLimits { max_wall_ms: None },
+            restart_policy: RestartPolicy::Never,
+            store: StoreConfig {
+                max_file_size: MaxFileSize(4096),
+                group_commit: GroupCommitConfig::default(),
+                shard_count: ShardCount(1),
+            },
+            eventlog: None,
+            writer_concurrency: WriterConcurrency(1),
+        }
+    }
+
+    fn fake_report(seed: u64, names: &[&'static str]) -> GauntletReport {
+        GauntletReport {
+            seed: Seed(seed),
+            ops_executed: OpsExecuted(0),
+            op_errors: OpErrorCount(0),
+            restarts: RestartCount(0),
+            violations: names
+                .iter()
+                .copied()
+                .map(|n| InvariantViolation {
+                    invariant: n,
+                    detail: "x".to_string(),
+                })
+                .collect(),
+            ops: OpStream::empty(),
+        }
+    }
+
+    fn sample_stream() -> OpStream {
+        OpStream::from_vec(vec![
+            Op::AddRecord {
+                collection: CollectionName("c".into()),
+                rkey: RecordKey("a".into()),
+                value_seed: ValueSeed(1),
+            },
+            Op::Compact,
+        ])
+    }
+
+    #[test]
+    fn clean_report_returns_input_unchanged() {
+        let cfg = dummy_config();
+        let ops = sample_stream();
+        let clean = fake_report(1, &[]);
+        let rt = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .unwrap();
+        let before_len = ops.len();
+        let out = rt.block_on(shrink_failure(cfg, ops, clean, 8));
+        assert_eq!(out.iterations, 0);
+        assert_eq!(out.ops.len(), before_len);
+    }
+}
@@ -0,0 +1,326 @@
+use std::io::{self, Write};
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use cid::Cid;
+use serde::{Deserialize, Serialize};
+use tracing::warn;
+
+use super::invariants::{InvariantSet, InvariantViolation};
+use super::leak::{LeakGateConfig, LeakViolation, evaluate as evaluate_leak_gate};
+use super::metrics::{MetricsSample, sample_harness};
+use super::op::{OpStream, Seed};
+use super::oracle::Oracle;
+use super::runner::{
+    EventLogState, GauntletConfig, Harness, IoBackend, apply_op, blockstore_config,
+    eventlog_snapshot, open_eventlog, refresh_oracle_graph, run_invariants, segments_subdir,
+};
+use super::workload::OpCount;
+use crate::blockstore::TranquilBlockStore;
+use crate::io::{RealIO, StorageIO};
+
+const OP_ERROR_LOG_THROTTLE: u64 = 1024;
+
+pub const DEFAULT_CHUNK_OPS: usize = 5_000;
+pub const DEFAULT_SAMPLE_INTERVAL_MS: u64 = 60_000;
+
+#[derive(Debug, Clone)]
+pub struct SoakConfig {
+    pub gauntlet: GauntletConfig,
+    pub total_duration: Duration,
+    pub sample_interval: Duration,
+    pub chunk_ops: usize,
+    pub leak_gate: LeakGateConfig,
+}
+
+impl SoakConfig {
+    pub fn new(gauntlet: GauntletConfig, total_duration: Duration) -> Self {
+        Self {
+            gauntlet,
+            total_duration,
+            sample_interval: Duration::from_millis(DEFAULT_SAMPLE_INTERVAL_MS),
+            chunk_ops: DEFAULT_CHUNK_OPS,
+            leak_gate: LeakGateConfig::standard(),
+        }
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum SoakError {
+    #[error(transparent)]
+    Io(#[from] io::Error),
+    #[error("soak requires IoBackend::Real; scenario configured Simulated")]
+    SimulatedBackendRejected,
+    #[error("open block store: {0}")]
+    StoreOpen(String),
+    #[error("open event log: {0}")]
+    EventLogOpen(String),
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SoakReport {
+    pub seed: Seed,
+    pub ops_executed: u64,
+    pub op_errors: u64,
+    pub chunks: u64,
+    pub samples: Vec<MetricsSample>,
+    pub invariant_violations: Vec<InvariantViolationRecord>,
+    pub leak_violations: Vec<LeakViolation>,
+    pub total_wall_ms: u64,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct InvariantViolationRecord {
+    pub invariant: String,
+    pub detail: String,
+}
+
+impl SoakReport {
+    pub fn is_clean(&self) -> bool {
+        self.invariant_violations.is_empty() && self.leak_violations.is_empty()
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(tag = "type")]
+pub enum SoakEvent {
+    #[serde(rename = "sample")]
+    Sample {
+        seed: u64,
+        chunk: u64,
+        ops_executed: u64,
+        sample: MetricsSample,
+    },
+    #[serde(rename = "invariant_violation")]
+    Invariant {
+        seed: u64,
+        invariant: String,
+        detail: String,
+    },
+    #[serde(rename = "summary")]
+    Summary {
+        seed: u64,
+        total_wall_ms: u64,
+        ops_executed: u64,
+        op_errors: u64,
+        chunks: u64,
+        clean: bool,
+        invariant_violations: usize,
+        leak_violations: Vec<LeakViolation>,
+    },
+}
+
+pub async fn run_soak<W: Write + Send>(
+    cfg: SoakConfig,
+    mut emitter: W,
+) -> Result<SoakReport, SoakError> {
+    if !matches!(cfg.gauntlet.io, IoBackend::Real) {
+        return Err(SoakError::SimulatedBackendRejected);
+    }
+    let dir = tempfile::TempDir::new()?;
+    let store_cfg = blockstore_config(dir.path(), &cfg.gauntlet.store);
+    let segments_dir: PathBuf = segments_subdir(dir.path());
+    let store = TranquilBlockStore::open(store_cfg)
+        .map(Arc::new)
+        .map_err(|e| SoakError::StoreOpen(e.to_string()))?;
+    let eventlog: Option<EventLogState<RealIO>> = match cfg.gauntlet.eventlog {
+        None => None,
+        Some(ec) => Some(
+            open_eventlog(RealIO::new(), segments_dir, ec.max_segment_size.0)
+                .map_err(|e| SoakError::EventLogOpen(e.to_string()))?,
+        ),
+    };
+    let mut harness = Harness { store, eventlog };
+    let outcome = drive_soak(&mut harness, &cfg, &mut emitter).await;
+    shutdown_harness(&mut harness);
+    outcome
+}
+
+fn shutdown_harness<S: StorageIO + Send + Sync + 'static>(harness: &mut Harness<S>) {
+    if let Some(el) = harness.eventlog.as_mut() {
+        if let Err(e) = el.writer.shutdown() {
+            warn!(error = %e, "soak: event log writer shutdown failed");
+        }
+        el.manager.shutdown();
+    }
+}
+
+async fn drive_soak<S, W>(
+    harness: &mut Harness<S>,
+    cfg: &SoakConfig,
+    emitter: &mut W,
+) -> Result<SoakReport, SoakError>
+where
+    S: StorageIO + Send + Sync + 'static,
+    W: Write + Send,
+{
+    let mut oracle = Oracle::new();
+    let mut root: Option<Cid> = None;
+
+    let start = Instant::now();
+    let mut samples: Vec<MetricsSample> = Vec::new();
+    let mut invariant_records: Vec<InvariantViolationRecord> = Vec::new();
+    let mut ops_executed: u64 = 0;
+    let mut op_errors: u64 = 0;
+    let mut chunks: u64 = 0;
+    let mut last_sample = start;
+    let mut next_error_log_at: u64 = 1;
+
+    let initial = sample_harness(harness, Duration::ZERO);
+    emit_event(
+        emitter,
+        &SoakEvent::Sample {
+            seed: cfg.gauntlet.seed.0,
+            chunk: 0,
+            ops_executed: 0,
+            sample: initial,
+        },
+    )?;
+    samples.push(initial);
+
+    while start.elapsed() < cfg.total_duration {
+        let chunk_seed = Seed(
+            cfg.gauntlet
+                .seed
+                .0
+                .wrapping_add(chunks.wrapping_mul(0x9E37_79B9_7F4A_7C15)),
+        );
+        let stream: OpStream = cfg
+            .gauntlet
+            .workload
+            .generate(chunk_seed, OpCount(cfg.chunk_ops));
+        for op in stream.iter() {
+            if start.elapsed() >= cfg.total_duration {
+                break;
+            }
+            match apply_op(harness, &mut root, &mut oracle, op, &cfg.gauntlet.workload).await {
+                Ok(()) => {
+                    ops_executed = ops_executed.saturating_add(1);
+                }
+                Err(e) => {
+                    op_errors = op_errors.saturating_add(1);
+                    if op_errors >= next_error_log_at {
+                        warn!(
+                            op_errors,
+                            ops_executed,
+                            elapsed_ms = u64::try_from(start.elapsed().as_millis())
+                                .unwrap_or(u64::MAX),
+                            error = %e,
+                            "soak: op error milestone"
+                        );
+                        next_error_log_at = next_error_log_at
+                            .saturating_mul(2)
+                            .max(OP_ERROR_LOG_THROTTLE);
+                    }
+                }
+            }
+            if last_sample.elapsed() >= cfg.sample_interval {
+                let elapsed = start.elapsed();
+                let s = sample_harness(harness, elapsed);
+                emit_event(
+                    emitter,
+                    &SoakEvent::Sample {
+                        seed: cfg.gauntlet.seed.0,
+                        chunk: chunks,
+                        ops_executed,
+                        sample: s,
+                    },
+                )?;
+                samples.push(s);
+                last_sample = Instant::now();
+            }
+        }
+        chunks = chunks.saturating_add(1);
+        tokio::task::yield_now().await;
+    }
+
+    let final_elapsed = start.elapsed();
+    let final_sample = sample_harness(harness, final_elapsed);
+    emit_event(
+        emitter,
+        &SoakEvent::Sample {
+            seed: cfg.gauntlet.seed.0,
+            chunk: chunks,
+            ops_executed,
+            sample: final_sample,
+        },
+    )?;
+    samples.push(final_sample);
+
+    let invariants = match refresh_oracle_graph(&harness.store, &mut oracle, root).await {
+        Ok(()) => {
+            let snapshot = eventlog_snapshot(harness.eventlog.as_ref());
+            let set = cfg
+                .gauntlet
+                .invariants
+                .without(InvariantSet::RESTART_IDEMPOTENT);
+            run_invariants(&harness.store, &oracle, root, snapshot, set).await
+        }
+        Err(e) => vec![InvariantViolation {
+            invariant: "MstRootDurability",
+            detail: format!("refresh: {e}"),
+        }],
+    };
+    for v in invariants.iter() {
+        let rec = InvariantViolationRecord {
+            invariant: v.invariant.to_string(),
+            detail: v.detail.clone(),
+        };
+        emit_event(
+            emitter,
+            &SoakEvent::Invariant {
+                seed: cfg.gauntlet.seed.0,
+                invariant: rec.invariant.clone(),
+                detail: rec.detail.clone(),
+            },
+        )?;
+        invariant_records.push(rec);
+    }
+
+    let leak_violations = evaluate_leak_gate(&samples, cfg.leak_gate);
+    let total_wall_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
+    let clean = invariant_records.is_empty() && leak_violations.is_empty();
+    emit_event(
+        emitter,
+        &SoakEvent::Summary {
+            seed: cfg.gauntlet.seed.0,
+            total_wall_ms,
+            ops_executed,
+            op_errors,
+            chunks,
+            clean,
+            invariant_violations: invariant_records.len(),
+            leak_violations: leak_violations.clone(),
+        },
+    )?;
+
+    Ok(SoakReport {
+        seed: cfg.gauntlet.seed,
+        ops_executed,
+        op_errors,
+        chunks,
+        samples,
+        invariant_violations: invariant_records,
+        leak_violations,
+        total_wall_ms,
+    })
+}
+
+fn emit_event<W: Write>(emitter: &mut W, event: &SoakEvent) -> io::Result<()> {
+    let line = serde_json::to_string(event).map_err(io::Error::other)?;
+    writeln!(emitter, "{line}")?;
+    emitter.flush()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn send_sync<T: Send + Sync>() {}
+
+    #[test]
+    fn soak_error_is_send_sync() {
+        send_sync::<SoakError>();
+    }
+}
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
LewisandTangled	b1d86caa78	feat(tranquil-comms): prework for email Lewis: May this revision serve well! <lu5a@proton.me>	2026-05-02 22:28:59 +03:00
Lewis	9b2cfb3a7e	fix(tranquil-store): durable-tail recovery + sync semantics Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-30 11:11:28 +03:00
Lewis	efd499bb26	fix(tranquil-store): barrier durability + torn-header recovery Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-29 15:35:42 +03:00
nelindandTangled	d4dfe838eb	fix(ci): use kaniko to build	2026-04-28 23:06:36 +03:00
LewisandTangled	af3821514f	test(tranquil-pds): same-rkey batch coverage and inductive inverse for in-batch dups Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-28 22:05:03 +03:00
LewisandTangled	8f7aad3756	fix(tranquil-pds): same-rkey batch semantics and firehose lag recovery Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-28 22:05:03 +03:00
LewisandTangled	75b9e3165f	refactor(deploy): container-first cleanup, drop debian malware-style install Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-27 00:13:27 +03:00
LewisandTangled	ccc9916109	test(tranquil-pds): websocket firehose end-to-end mst verification Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-26 20:11:27 +03:00
LewisandTangled	bc8fd66a45	test(tranquil-pds): mst fuzz + repo integrity properties Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-26 20:11:27 +03:00
LewisandTangled	180de29984	fix(tranquil-pds): firehose car carries inductive proof Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-26 20:11:27 +03:00
max.andTangled	0455dc20bd	fix(validation): discord usernames may contain periods at start or end	2026-04-25 09:59:12 +03:00
Lewis	2770b9b14a	feat(tranquil-store): cargo-fuzz targets with asan+ubsan Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-24 10:50:08 +03:00
Lewis	d436597184	feat(tranquil-store): flaky-device scenario, jemalloc heap-prof Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-24 10:50:08 +03:00
Lewis	4cfca6d956	feat(tranquil-store): soak harness driving leak gate, signal tweaks Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-23 08:39:16 +03:00
Lewis	98b94fb170	feat(tranquil-store): leak gate and metrics sampling for gauntlet Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-22 21:00:17 +03:00
Lewis	4fe01cff72	feat(tranquil-store): sweep subcommand with axis override fan-out Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-22 17:23:15 +03:00
Lewis	00c9eb732f	fix(tranquil-store): arc-counted cache handles, reader-eviction race Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-21 22:04:24 +03:00
Lewis	6d2d3b4be4	fix(tranquil-store): commit-marker batch replay, batch-boundary rotation Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-21 14:55:15 +03:00
Lewis	9d81e58803	fix(tranquil-store): no more orphan data files, recover torn-hint tails, header-safe resume Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-21 09:59:37 +03:00
Lewis	2afd075496	fix(tranquil-store): atomic record commits, hint-as-truth recovery Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-20 21:00:54 +03:00
Hayden YoungandTangled	c74bf967cf	fix(auth): don't require 2fa for app passwords	2026-04-20 18:20:20 +03:00
Lewis	1285d5c675	fix(tranquil-store): bound writer fd usage across rotations Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-20 16:50:19 +03:00
Lewis	c30d73cd4d	test(tranquil-store): migrate some tests to gauntlet Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-19 23:50:27 +03:00
Lewis	0fab8f2eb9	feat(tranquil-store): tranquil-gauntlet CLI, config overrides, profiles Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-19 23:50:27 +03:00
Lewis	57336fa124	feat(tranquil-store/gauntlet): new invariants & scenarios Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-19 10:25:07 +03:00
Lewis	ace105899f	feat(tranquil-store/gauntlet): concurrent executor, eventlog, fault recovery Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-19 00:19:08 +03:00
Lewis	c80a525e0d	feat(tranquil-store/gauntlet): op surface, oracle, workload for eventlog & reads Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-18 10:36:30 +03:00
Lewis	7edb76507b	feat(tranquil-store/gauntlet): simulated io fault modes, shrinker, regression dump Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-18 10:36:30 +03:00
LewisandTangled	a20e4e05e6	fix(oauth): indigo client send empty assertions Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-17 15:18:26 +03:00
teqandTangled	3e7a199736	docs: nixos and binary cache	2026-04-17 01:32:30 +03:00
teqandTangled	7ed9c8ce59	feat(ci): supply cache by arg	2026-04-17 01:32:30 +03:00
teqandTangled	b0c3402a82	feat(ci): add cachix publishing workflows	2026-04-17 01:32:30 +03:00
LewisandTangled	b9574f3ec1	fix(lexicons): cache ttl Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-16 19:45:27 +03:00
isabelandTangled	baef2be844	feat: allow max prefs to be configured some people (me) have a large amount of prefs and moving from ref pds to tranquil ment i lost a few of these. so in this pr i bumped the max prefs to 1000 and made it configurable via the config	2026-04-16 19:45:17 +03:00
Lewis	1f21b85069	fix(example.toml): lewis forgot about confique Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-16 14:49:57 +03:00
Lewis	09d437b3e3	feat(tranquil-store): gauntlet persistence & restart invariants Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-15 22:28:31 +03:00
Lewis	7f2e83e92f	feat(tranquil-store): beginnings of the gauntlet test suite Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-15 20:29:34 +03:00
nelind	ec273fa814	feat(nix): cross compiled aarch64 package	2026-04-14 23:26:34 +02:00
Lewis	d51bfd59da	fix(tranquil-store): exclude 0 refcount blocks from has() Lewis: May this revision serve well! <lu5a@proton.me>	2026-04-14 08:40:21 +03:00