feat(test): test that websockets get closed

Lewis: May this revision serve well! <lu5a@proton.me>

.config/nextest.toml

@@ -25,6 +25,28 @@ fail-fast = false
 test-threads = "num-cpus"
 slow-timeout = { period = "300s", terminate-after = 2 }
 
+[profile.gauntlet-pr]
+retries = 0
+fail-fast = true
+test-threads = "num-cpus"
+slow-timeout = { period = "60s", terminate-after = 5 }
+
+[[profile.gauntlet-pr.overrides]]
+filter = "binary(gauntlet_smoke)"
+slow-timeout = { period = "300s", terminate-after = 2 }
+
+[profile.gauntlet-nightly]
+retries = 0
+fail-fast = false
+test-threads = "num-cpus"
+slow-timeout = { period = "600s", terminate-after = 1 }
+
+[profile.gauntlet-soak]
+retries = 0
+fail-fast = false
+test-threads = 1
+slow-timeout = { period = "5m", terminate-after = 1000 }
+
 [test-groups]
 serial-env-tests = { max-threads = 1 }
 heavy-load-tests = { max-threads = 4 }
@@ -72,6 +94,10 @@ test-group = "io-heavy-sim"
 filter = "test(/test_scale_/) | test(/full_backup_and_restore/)"
 slow-timeout = { period = "120s", terminate-after = 4 }
 
+[[profile.default.overrides]]
+filter = "binary(gauntlet_smoke)"
+slow-timeout = { period = "300s", terminate-after = 8 }
+
 [[profile.default.overrides]]
 filter = "binary(compaction_restart) | binary(mst_refcount_integrity) | binary(gc_compaction_restart)"
 slow-timeout = { period = "120s", terminate-after = 4 }

.gitignore

@@ -1,4 +1,4 @@
-/target
+target/
 .env
 .direnv
 result

.tangled/workflows/publish-cachix-aarch64.yml

@@ -0,0 +1,21 @@
+when:
+  - event: ["push", "manual"]
+    branch: ["main"]
+
+engine: nixery
+
+dependencies:
+  nixpkgs:
+    - nix
+    - cachix
+    - gnugrep
+
+steps:
+  - name: Authenticate
+    command: |
+      cachix authtoken "$CACHIX_AUTH_TOKEN"
+
+  - name: Build and push aarch64
+    command: |
+      SUBS="--option extra-substituters https://tranquil.cachix.org --option extra-trusted-public-keys tranquil.cachix.org-1:PoO+mGL6a6LcJiPakMDHN4E218/ei/7v2sxeDtNkSRg="
+      nix-store -qR --include-outputs $(nix-store -qd $(nix build .#packages.x86_64-linux.tranquil-pds-aarch64 $SUBS --print-out-paths --no-link)) | grep -v '\.drv$' | cachix push "$CACHIX_CACHE_NAME"

.tangled/workflows/publish-cachix-x86_64.yml

@@ -0,0 +1,28 @@
+when:
+  - event: ["push", "manual"]
+    branch: ["main"]
+
+engine: nixery
+
+dependencies:
+  nixpkgs:
+    - nix
+    - cachix
+    - gnugrep
+
+steps:
+  - name: Authenticate
+    command: |
+      cachix authtoken "$CACHIX_AUTH_TOKEN"
+
+  - name: Build and push x86_64
+    command: |
+      SUBS="--option extra-substituters https://tranquil.cachix.org --option extra-trusted-public-keys tranquil.cachix.org-1:PoO+mGL6a6LcJiPakMDHN4E218/ei/7v2sxeDtNkSRg="
+      nix-store -qR --include-outputs $(nix-store -qd $(nix build .#packages.x86_64-linux.tranquil-pds $SUBS --print-out-paths --no-link)) | grep -v '\.drv$' | cachix push "$CACHIX_CACHE_NAME"
+      nix-store -qR --include-outputs $(nix-store -qd $(nix build .#packages.x86_64-linux.tranquil-frontend $SUBS --print-out-paths --no-link)) | grep -v '\.drv$' | cachix push "$CACHIX_CACHE_NAME"
+
+  - name: Build and push devShell
+    command: |
+      SUBS="--option extra-substituters https://tranquil.cachix.org --option extra-trusted-public-keys tranquil.cachix.org-1:PoO+mGL6a6LcJiPakMDHN4E218/ei/7v2sxeDtNkSRg="
+      nix develop $SUBS --profile dev-profile -c true
+      cachix push "$CACHIX_CACHE_NAME" dev-profile

.tangled/workflows/publish-image.yml

@@ -1,24 +1,36 @@
 when:
-  - event: []
-    branch: []
+  - event: [ "manual" ]
+  - event: [ "push" ]
+    branch: [ "main" ]
 
 engine: nixery
 
 dependencies:
   nixpkgs:
-    - podman
+    - kaniko
+
+environment:
+  DOCKER_CONFIG: "/kaniko/.docker"
 
 steps:
-  - name: Create podman config
+  - name: Configure Kaniko
     command: |
-      mkdir -p ~/.config/containers
-      echo "unqualified-search-registries = [\"docker.io\"]" >> ~/.config/containers/registries.conf
+      mkdir -p /kaniko/.docker/
+      echo "{
+        \"auths\": {
+          \"https://atcr.io/v1\":{
+            \"auth\": \"$ATCR_CREDENTIALS\"
+          }
+        }
+      }" > /kaniko/.docker/config.json
 
   - name: Build image
     command: |
-      podman build . -t tranquil-pds:latest -t "tranquil-pds:$TANGLED_COMMIT_SHA"
-
-  - name: Publish image
-    command: |
-      podman push --creds "$ATCR_USERNAME:$ATCR_PASSWORD" tranquil-pds:latest "atcr.io/tranquil.farm/tranquil-pds:latest"
-      podman push --creds "$ATCR_USERNAME:$ATCR_PASSWORD" "tranquil-pds:$TANGLED_COMMIT_SHA" "atcr.io/tranquil.farm/tranquil-pds:$TANGLED_COMMIT_SHA"
+      executor \
+        --context=$(pwd) \
+        --ignore-path=$(pwd) \
+        --dockerfile=$(pwd)/Dockerfile \
+        --destination="atcr.io/tranquil.farm/tranquil-pds:latest" \
+        --destination="atcr.io/tranquil.farm/tranquil-pds:$TANGLED_COMMIT_SHA" \
+        --push-retry=3 \
+        --skip-push-permission-check

Cargo.toml

@@ -26,7 +26,7 @@ members = [
 ]
 
 [workspace.package]
-version = "0.5.4"
+version = "0.5.7"
 edition = "2024"
 license = "AGPL-3.0-or-later"
 

Dockerfile

@@ -35,7 +35,7 @@ COPY crates/tranquil-oauth-server ./crates/tranquil-oauth-server
 COPY crates/tranquil-store ./crates/tranquil-store
 COPY crates/tranquil-signal ./crates/tranquil-signal
 COPY crates/tranquil-server ./crates/tranquil-server
-COPY migrations ./crates/tranquil-pds/migrations
+COPY migrations ./migrations
 RUN --mount=type=cache,target=/usr/local/cargo/registry \
     --mount=type=cache,target=/app/target \
     if [ "$SLIM" = "true" ]; then \
@@ -50,7 +50,6 @@ RUN apk add --no-cache msmtp ca-certificates \
     && ln -sf /usr/bin/msmtp /usr/sbin/sendmail
 COPY --from=builder /tmp/tranquil-pds /usr/local/bin/tranquil-pds
 COPY --from=frontend /app/dist /var/lib/tranquil-pds/frontend
-COPY migrations /app/migrations
 WORKDIR /app
 ENV SERVER_HOST=0.0.0.0
 ENV SERVER_PORT=3000

README.md

@@ -46,6 +46,8 @@ just test
 just lint
 ```
 
+Nix users can enter a devshell with `nix develop`, or `direnv allow` to auto-enter via the bundled `.envrc`. Pre-built artifacts (including the devshell) are available from our [binary cache](docs/install-nix.md#binary-cache).
+
 ## Production Deployment
 
 ### Quick Deploy (Docker/Podman Compose)
@@ -59,7 +61,7 @@ podman-compose -f docker-compose.prod.yaml up -d
 
 ### Installation Guides
 
-- [Debian](docs/install-debian.md)
+- [Nix](docs/install-nix.md)
 - [Containers](docs/install-containers.md)
 - [Kubernetes](docs/install-kubernetes.md)
 
@@ -99,4 +101,3 @@ This project is very grateful to [@nonbinary.computer](https://tangled.org/did:p
 ## License
 
 AGPL-3.0-or-later. Documentation is CC BY-SA 4.0. See [LICENSE](LICENSE) for details.
-

crates/tranquil-api/src/actor/preferences.rs

@@ -12,7 +12,6 @@ use tranquil_pds::auth::{Auth, NotTakendown, Permissive};
 use tranquil_pds::state::AppState;
 
 const APP_BSKY_NAMESPACE: &str = "app.bsky";
-const MAX_PREFERENCES_COUNT: usize = 100;
 const MAX_PREFERENCE_SIZE: usize = 10_000;
 const PERSONAL_DETAILS_PREF: &str = "app.bsky.actor.defs#personalDetailsPref";
 const DECLARED_AGE_PREF: &str = "app.bsky.actor.defs#declaredAgePref";
@@ -92,6 +91,8 @@ pub async fn put_preferences(
     auth: Auth<NotTakendown>,
     Json(input): Json<PutPreferencesInput>,
 ) -> Response {
+    let max_preferences_count: usize = tranquil_config::get().server.max_preferences_count;
+
     let has_full_access = auth.permissions().has_full_access();
     let user_id: uuid::Uuid = match state.repos.user.get_id_by_did(&auth.did).await {
         Ok(Some(id)) => id,
@@ -99,11 +100,11 @@ pub async fn put_preferences(
             return ApiError::InternalError(Some("User not found".into())).into_response();
         }
     };
-    if input.preferences.len() > MAX_PREFERENCES_COUNT {
+    if input.preferences.len() > max_preferences_count {
         return ApiError::InvalidRequest(format!(
             "Too many preferences: {} exceeds limit of {}",
             input.preferences.len(),
-            MAX_PREFERENCES_COUNT
+            max_preferences_count
         ))
         .into_response();
     }

crates/tranquil-api/src/admin/signal.rs

@@ -76,7 +76,7 @@ pub async fn link_signal_device(
         let result = tokio::select! {
             biased;
             _ = shutdown.cancelled() => {
-                tracing::info!("Signal linking aborted due to server shutdown");
+                tracing::info!("server shutting down, aborting signal linking");
                 return;
             }
             r = link_result.completion => r,
@@ -84,10 +84,10 @@ pub async fn link_signal_device(
         match result {
             Ok(Ok(client)) => {
                 if slot_for_task.complete_link(generation, client).await {
-                    tracing::info!("Signal device linked successfully");
+                    tracing::info!("signal device linked");
                 } else {
                     tracing::warn!(
-                        "Signal link completed but generation mismatch or already linked; discarding"
+                        "discarding completed signal link, generation mismatch or already linked"
                     );
                 }
             }

crates/tranquil-api/src/notification_prefs.rs

@@ -282,7 +282,7 @@ async fn process_messaging_channel_update(
                 "Invalid Telegram username. Must be 5-32 characters, alphanumeric or underscore".into(),
             ),
             CommsChannel::Signal => ApiError::InvalidRequest(
-                "Invalid Signal username. Must be 3-32 characters followed by .XX (e.g. username.01)".into(),
+                "Invalid Signal username. Must be a 3-32 character nickname, a dot, then a 2-20 digit discriminator".into(),
             ),
             CommsChannel::Email => ApiError::InvalidEmail,
         });

crates/tranquil-api/src/repo/record/batch.rs

@@ -27,7 +27,6 @@ struct WriteAccumulator {
     mst: Mst<TrackingBlockStore>,
     results: Vec<WriteResult>,
     ops: Vec<RecordOp>,
-    modified_keys: Vec<String>,
     all_blob_cids: Vec<String>,
     backlinks_to_add: Vec<Backlink>,
     backlinks_to_remove: Vec<AtUri>,
@@ -44,7 +43,6 @@ async fn process_single_write(
         mst,
         mut results,
         mut ops,
-        mut modified_keys,
         mut all_blob_cids,
         mut backlinks_to_add,
         mut backlinks_to_remove,
@@ -69,8 +67,19 @@ async fn process_single_write(
                     .await?,
                 )
             };
-            all_blob_cids.extend(extract_blob_cids(value));
             let rkey = rkey.clone().unwrap_or_else(Rkey::generate);
+            let key = format!("{}/{}", collection, rkey);
+            if mst
+                .get(&key)
+                .await
+                .map_err(|e| ApiError::InternalError(Some(format!("Failed to read MST: {e}"))))?
+                .is_some()
+            {
+                return Err(ApiError::InvalidRequest(format!(
+                    "Record already exists at {key}"
+                )));
+            }
+            all_blob_cids.extend(extract_blob_cids(value));
             let record_ipld = tranquil_pds::util::json_to_ipld(value);
             let record_bytes = serde_ipld_dagcbor::to_vec(&record_ipld)
                 .map_err(|_| ApiError::InvalidRecord("Failed to serialize record".into()))?;
@@ -78,8 +87,6 @@ async fn process_single_write(
                 .put(&record_bytes)
                 .await
                 .map_err(|_| ApiError::InternalError(Some("Failed to store record".into())))?;
-            let key = format!("{}/{}", collection, rkey);
-            modified_keys.push(key.clone());
             let new_mst = mst
                 .add(&key, record_cid)
                 .await
@@ -100,7 +107,6 @@ async fn process_single_write(
                 mst: new_mst,
                 results,
                 ops,
-                modified_keys,
                 all_blob_cids,
                 backlinks_to_add,
                 backlinks_to_remove,
@@ -124,16 +130,7 @@ async fn process_single_write(
                     .await?,
                 )
             };
-            all_blob_cids.extend(extract_blob_cids(value));
-            let record_ipld = tranquil_pds::util::json_to_ipld(value);
-            let record_bytes = serde_ipld_dagcbor::to_vec(&record_ipld)
-                .map_err(|_| ApiError::InvalidRecord("Failed to serialize record".into()))?;
-            let record_cid = tracking_store
-                .put(&record_bytes)
-                .await
-                .map_err(|_| ApiError::InternalError(Some("Failed to store record".into())))?;
             let key = format!("{}/{}", collection, rkey);
-            modified_keys.push(key.clone());
             let prev_record_cid = mst
                 .get(&key)
                 .await
@@ -143,6 +140,14 @@ async fn process_single_write(
                 .ok_or_else(|| {
                     ApiError::InvalidRequest("Update target record does not exist".into())
                 })?;
+            all_blob_cids.extend(extract_blob_cids(value));
+            let record_ipld = tranquil_pds::util::json_to_ipld(value);
+            let record_bytes = serde_ipld_dagcbor::to_vec(&record_ipld)
+                .map_err(|_| ApiError::InvalidRecord("Failed to serialize record".into()))?;
+            let record_cid = tracking_store
+                .put(&record_bytes)
+                .await
+                .map_err(|_| ApiError::InternalError(Some("Failed to store record".into())))?;
             let new_mst = mst
                 .update(&key, record_cid)
                 .await
@@ -165,7 +170,6 @@ async fn process_single_write(
                 mst: new_mst,
                 results,
                 ops,
-                modified_keys,
                 all_blob_cids,
                 backlinks_to_add,
                 backlinks_to_remove,
@@ -173,7 +177,6 @@ async fn process_single_write(
         }
         WriteOp::Delete { collection, rkey } => {
             let key = format!("{}/{}", collection, rkey);
-            modified_keys.push(key.clone());
             let prev_record_cid = mst
                 .get(&key)
                 .await
@@ -198,7 +201,6 @@ async fn process_single_write(
                 mst: new_mst,
                 results,
                 ops,
-                modified_keys,
                 all_blob_cids,
                 backlinks_to_add,
                 backlinks_to_remove,
@@ -219,7 +221,6 @@ async fn process_writes(
         mst: initial_mst,
         results: Vec::new(),
         ops: Vec::new(),
-        modified_keys: Vec::new(),
         all_blob_cids: Vec::new(),
         backlinks_to_add: Vec::new(),
         backlinks_to_remove: Vec::new(),
@@ -351,7 +352,6 @@ pub async fn apply_writes(
         mst: final_mst,
         results,
         ops,
-        modified_keys,
         all_blob_cids,
         backlinks_to_add,
         backlinks_to_remove,
@@ -407,7 +407,6 @@ pub async fn apply_writes(
             controller_did: controller_did.as_ref(),
             delegation_detail: write_summary,
             ops,
-            modified_keys: &modified_keys,
             blob_cids: &all_blob_cids,
             backlinks_to_add,
             backlinks_to_remove,

crates/tranquil-api/src/repo/record/delete.rs

@@ -74,7 +74,6 @@ pub async fn delete_record(
         prev: RecordCid::from(prev_record_cid),
     };
 
-    let modified_keys = [key];
     let deleted_uri = AtUri::from_parts(&did, &input.collection, &input.rkey);
 
     let commit_result = finalize_repo_write(
@@ -93,7 +92,6 @@ pub async fn delete_record(
                 })
             }),
             ops: vec![op],
-            modified_keys: &modified_keys,
             blob_cids: &[],
             backlinks_to_add: vec![],
             backlinks_to_remove: vec![deleted_uri],

crates/tranquil-api/src/repo/record/write.rs

@@ -179,6 +179,18 @@ pub async fn create_record(
         }
     }
 
+    let key = format!("{}/{}", input.collection, rkey);
+    if mst
+        .get(&key)
+        .await
+        .map_err(|e| ApiError::InternalError(Some(format!("Failed to read MST: {e}"))))?
+        .is_some()
+    {
+        return Err(ApiError::InvalidRequest(format!(
+            "Record already exists at {key}"
+        )));
+    }
+
     let record_ipld = tranquil_pds::util::json_to_ipld(&input.record);
     let record_bytes = serde_ipld_dagcbor::to_vec(&record_ipld)
         .map_err(|_| ApiError::InvalidRecord("Failed to serialize record".into()))?;
@@ -187,8 +199,6 @@ pub async fn create_record(
         .put(&record_bytes)
         .await
         .map_err(|_| ApiError::InternalError(Some("Failed to save record block".into())))?;
-
-    let key = format!("{}/{}", input.collection, rkey);
     mst = mst
         .add(&key, record_cid)
         .await
@@ -200,20 +210,6 @@ pub async fn create_record(
         cid: tranquil_pds::cid_types::RecordCid::from(record_cid),
     });
 
-    let modified_keys: Vec<String> = ops
-        .iter()
-        .map(|op| match op {
-            RecordOp::Create {
-                collection, rkey, ..
-            }
-            | RecordOp::Update {
-                collection, rkey, ..
-            }
-            | RecordOp::Delete {
-                collection, rkey, ..
-            } => format!("{}/{}", collection, rkey),
-        })
-        .collect();
     let blob_cids = extract_blob_cids(&input.record);
 
     let created_uri = AtUri::from_parts(&did, &input.collection, &rkey);
@@ -235,7 +231,6 @@ pub async fn create_record(
                 })
             }),
             ops,
-            modified_keys: &modified_keys,
             blob_cids: &blob_cids,
             backlinks_to_add,
             backlinks_to_remove: conflict_uris_to_cleanup,
@@ -367,7 +362,6 @@ pub async fn put_record(
         }
     };
 
-    let modified_keys = [key];
     let blob_cids = extract_blob_cids(&input.record);
     let backlinks_to_add = extract_backlinks(&record_uri, &input.record);
 
@@ -387,7 +381,6 @@ pub async fn put_record(
                 })
             }),
             ops: vec![op],
-            modified_keys: &modified_keys,
             blob_cids: &blob_cids,
             backlinks_to_add,
             backlinks_to_remove,

crates/tranquil-api/src/server/session.rs

@@ -163,6 +163,7 @@ pub async fn create_session(
     let email_2fa_enabled = row.email_2fa_enabled;
     let is_legacy_login = has_totp || email_2fa_enabled;
     let twofa_ctx = tranquil_pds::auth::legacy_2fa::Legacy2faContext {
+        is_app_password: app_password_name.is_some(),
         email_2fa_enabled,
         has_totp,
         allow_legacy_login: row.allow_legacy_login,

crates/tranquil-comms/src/sender.rs

@@ -299,7 +299,7 @@ impl DiscordSender {
             "type": 1,
             "options": [{
                 "name": "handle",
-                "description": "Your PDS handle (e.g. alice.example.com)",
+                "description": "Your PDS handle",
                 "type": 3,
                 "required": false
             }]

crates/tranquil-config/src/lib.rs

@@ -65,6 +65,9 @@ pub fn ensure_test_defaults() {
             if env::var("ENABLE_PDS_HOSTED_DID_WEB").is_err() {
                 env::set_var("ENABLE_PDS_HOSTED_DID_WEB", "true");
             }
+            if env::var("TRANQUIL_LEXICON_OFFLINE").is_err() {
+                env::set_var("TRANQUIL_LEXICON_OFFLINE", "1");
+            }
         }
         TranquilConfig::builder()
             .env()
@@ -80,8 +83,9 @@ pub fn ensure_test_defaults() {
 ///
 /// Precedence (highest to lowest):
 /// 1. Environment variables
-/// 2. TOML config file (if provided)
-/// 3. Built-in defaults
+/// 2. Toml config file passed as `config_path`, if provided
+/// 3. `/etc/tranquil-pds/config.toml` - hardcoded fallback, silently skipped if absent
+/// 4. Built-in defaults
 pub fn load(config_path: Option<&PathBuf>) -> Result<TranquilConfig, confique::Error> {
     let mut builder = TranquilConfig::builder().env();
     if let Some(path) = config_path {
@@ -108,6 +112,9 @@ pub struct TranquilConfig {
     #[config(nested)]
     pub storage: StorageConfig,
 
+    #[config(nested)]
+    pub tranquil_store: TranquilStoreConfig,
+
     #[config(nested)]
     pub cache: CacheConfig,
 
@@ -143,9 +150,6 @@ pub struct TranquilConfig {
 
     #[config(nested)]
     pub scheduled: ScheduledConfig,
-
-    #[config(nested)]
-    pub tranquil_store: TranquilStoreConfig,
 }
 
 impl TranquilConfig {
@@ -394,7 +398,7 @@ impl TranquilConfig {
 
 #[derive(Debug, Config)]
 pub struct ServerConfig {
-    /// Public hostname of the PDS (e.g. `pds.example.com`).
+    /// Public hostname of the PDS, such as `pds.example.com`.
     #[config(env = "PDS_HOSTNAME")]
     pub hostname: String,
 
@@ -451,6 +455,10 @@ pub struct ServerConfig {
     /// Maximum allowed blob size in bytes (default 10 GiB).
     #[config(env = "MAX_BLOB_SIZE", default = 10_737_418_240u64)]
     pub max_blob_size: u64,
+
+    /// Maximum allowed number of preferences
+    #[config(env = "MAX_PREFERENCES_COUNT", default = 1000)]
+    pub max_preferences_count: usize,
 }
 
 impl ServerConfig {
@@ -459,8 +467,7 @@ impl ServerConfig {
         format!("https://{}", self.hostname)
     }
 
-    /// Hostname without port suffix (e.g. `pds.example.com` from
-    /// `pds.example.com:443`).
+    /// Hostname without port suffix. Returns `pds.example.com` from `pds.example.com:443`.
     pub fn hostname_without_port(&self) -> &str {
         self.hostname.split(':').next().unwrap_or(&self.hostname)
     }
@@ -656,10 +663,12 @@ pub struct StorageConfig {
     #[config(env = "S3_BUCKET")]
     pub s3_bucket: Option<String>,
 
-    /// Custom S3 endpoint URL (for MinIO, R2, etc.).
+    /// Custom S3 endpoint URL.
     #[config(env = "S3_ENDPOINT")]
     pub s3_endpoint: Option<String>,
 
+    /// Repository backend: `postgres` by default, or `tranquil-store`, our embedded db.
+    /// tranquil-store is EXPERIMENTAL!!!! RISK OF TOTAL DATA LOSS.
     #[config(env = "REPO_BACKEND", default = "postgres")]
     pub repo_backend: String,
 }
@@ -674,7 +683,7 @@ impl StorageConfig {
 
 #[derive(Debug, Config)]
 pub struct CacheConfig {
-    /// Cache backend: `ripple` (default, built-in gossip) or `valkey`.
+    /// Cache backend: `ripple` by default, or `valkey`.
     #[config(env = "CACHE_BACKEND", default = "ripple")]
     pub backend: String,
 
@@ -716,10 +725,6 @@ pub struct FirehoseConfig {
     #[config(env = "FIREHOSE_BACKFILL_HOURS", default = 72)]
     pub backfill_hours: i64,
 
-    /// Maximum number of lagged events before disconnecting a slow consumer.
-    #[config(env = "FIREHOSE_MAX_LAG", default = 5000)]
-    pub max_lag: u64,
-
     /// Maximum concurrent full-repo exports, eg. getRepo without `since`.
     #[config(env = "MAX_CONCURRENT_REPO_EXPORTS", default = 4)]
     pub max_concurrent_repo_exports: usize,
@@ -774,6 +779,8 @@ pub struct TelegramConfig {
 
 #[derive(Debug, Config)]
 pub struct SignalConfig {
+    /// Protocol state is stored in postgres' signal_* tables.
+    /// Link a device via the admin API before enabling.
     #[config(env = "SIGNAL_ENABLED", default = false)]
     pub enabled: bool,
 }
@@ -1124,27 +1131,26 @@ pub struct ScheduledConfig {
 
     /// Maximum age of events retained in the eventlog before pruning.
     /// Per the atproto firehose spec, the relay backfill window only needs
-    /// to cover "hours or days". Default: 7 days.
+    /// to cover "hours or days".
     #[config(env = "EVENT_RETENTION_MAX_AGE_SECS", default = 604800)]
     pub event_retention_max_age_secs: u64,
 
     /// Interval in seconds between event retention prune passes.
-    /// Set to 0 to disable. Default: hourly.
+    /// Set to 0 to disable.
     #[config(env = "EVENT_RETENTION_INTERVAL_SECS", default = 3600)]
     pub event_retention_interval_secs: u64,
 }
 
 #[derive(Debug, Config)]
 pub struct TranquilStoreConfig {
-    /// Directory for tranquil-store data (metastore, eventlog).
+    /// Directory for tranquil-store data: the metastore, eventlog, and blockstore.
     #[config(
         env = "TRANQUIL_STORE_DATA_DIR",
         default = "/var/lib/tranquil-pds/store"
     )]
     pub data_dir: String,
 
-    /// Fjall block cache size in megabytes. Defaults to 20% of system RAM
-    /// when unset.
+    /// Fjall block cache size in megabytes. Defaults to 20% of system RAM when unset.
     #[config(env = "TRANQUIL_STORE_MEMORY_BUDGET_MB")]
     pub memory_budget_mb: Option<u64>,
 
@@ -1152,9 +1158,9 @@ pub struct TranquilStoreConfig {
     #[config(env = "TRANQUIL_STORE_HANDLER_THREADS")]
     pub handler_threads: Option<usize>,
 
-    /// Maximum total bytes of pending (unsynced) eventlog payloads. Appenders
-    /// block once this budget is exhausted until in-flight events drain via
-    /// fsync. Set to 0 to disable backpressure (unbounded). Default: 1 GiB.
+    /// Maximum total bytes of pending (unsynced) eventlog payloads. Appenders block
+    /// once this budget is exhausted until in-flight events drain via fsync. Set to
+    /// 0 to disable backpressure. Default: 1 GiB.
     #[config(
         env = "TRANQUIL_STORE_EVENTLOG_PENDING_BYTES_BUDGET",
         default = 1_073_741_824

crates/tranquil-db-traits/src/backlink.rs

@@ -7,7 +7,7 @@ use uuid::Uuid;
 
 use crate::DbError;
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub enum BacklinkPath {
     Subject,
     SubjectUri,

crates/tranquil-lexicon/Cargo.toml

@@ -24,3 +24,4 @@ urlencoding = { workspace = true, optional = true }
 [dev-dependencies]
 wiremock = { workspace = true }
 tokio = { workspace = true }
+futures = { workspace = true }

crates/tranquil-lexicon/src/dynamic.rs

@@ -5,46 +5,102 @@ use std::collections::{HashMap, VecDeque};
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::time::{Duration, Instant};
+use tokio::sync::Notify;
 
 const NEGATIVE_CACHE_TTL: Duration = Duration::from_secs(24 * 60 * 60);
+const POSITIVE_CACHE_TTL: Duration = Duration::from_secs(24 * 60 * 60);
+const REFRESH_FAILURE_BACKOFF: Duration = Duration::from_secs(60);
 const MAX_DYNAMIC_SCHEMAS: usize = 1024;
 
 struct NegativeEntry {
     expires_at: Instant,
 }
 
+struct PositiveEntry {
+    doc: Arc<LexiconDoc>,
+    expires_at: Instant,
+}
+
+pub(crate) enum CacheEntry {
+    Fresh(Arc<LexiconDoc>),
+    Stale(Arc<LexiconDoc>),
+}
+
+impl CacheEntry {
+    #[cfg(test)]
+    fn is_fresh(&self) -> bool {
+        matches!(self, Self::Fresh(_))
+    }
+}
+
 struct SchemaStore {
-    schemas: HashMap<String, Arc<LexiconDoc>>,
+    schemas: HashMap<String, PositiveEntry>,
     insertion_order: VecDeque<String>,
 }
 
 pub struct DynamicRegistry {
     store: RwLock<SchemaStore>,
     negative_cache: RwLock<HashMap<String, NegativeEntry>>,
+    in_flight: RwLock<HashMap<String, Arc<Notify>>>,
     network_disabled: AtomicBool,
 }
 
+struct InFlightGuard<'a> {
+    registry: &'a DynamicRegistry,
+    nsid: String,
+}
+
+impl Drop for InFlightGuard<'_> {
+    fn drop(&mut self) {
+        let notify = self.registry.in_flight.write().remove(&self.nsid);
+        if let Some(n) = notify {
+            n.notify_waiters();
+        }
+    }
+}
+
 impl DynamicRegistry {
     pub fn new() -> Self {
-        let network_disabled =
-            std::env::var("TRANQUIL_LEXICON_OFFLINE").is_ok_and(|v| v == "1" || v == "true");
         Self {
             store: RwLock::new(SchemaStore {
                 schemas: HashMap::new(),
                 insertion_order: VecDeque::new(),
             }),
             negative_cache: RwLock::new(HashMap::new()),
-            network_disabled: AtomicBool::new(network_disabled),
+            in_flight: RwLock::new(HashMap::new()),
+            network_disabled: AtomicBool::new(false),
         }
     }
 
-    #[allow(dead_code)]
+    pub fn from_env() -> Self {
+        let registry = Self::new();
+        let disabled =
+            std::env::var("TRANQUIL_LEXICON_OFFLINE").is_ok_and(|v| v == "1" || v == "true");
+        registry.set_network_disabled(disabled);
+        registry
+    }
+
     pub fn set_network_disabled(&self, disabled: bool) {
         self.network_disabled.store(disabled, Ordering::Relaxed);
     }
 
-    pub fn get(&self, nsid: &str) -> Option<Arc<LexiconDoc>> {
-        self.store.read().schemas.get(nsid).cloned()
+    pub fn get_cached(&self, nsid: &str) -> Option<Arc<LexiconDoc>> {
+        self.store
+            .read()
+            .schemas
+            .get(nsid)
+            .map(|e| Arc::clone(&e.doc))
+    }
+
+    pub(crate) fn get_entry(&self, nsid: &str) -> Option<CacheEntry> {
+        let now = Instant::now();
+        self.store.read().schemas.get(nsid).map(|e| {
+            if e.expires_at > now {
+                CacheEntry::Fresh(Arc::clone(&e.doc))
+            } else {
+                CacheEntry::Stale(Arc::clone(&e.doc))
+            }
+        })
     }
 
     pub fn is_negative_cached(&self, nsid: &str) -> bool {
@@ -56,7 +112,7 @@ impl DynamicRegistry {
 
     fn insert_negative(&self, nsid: &str) {
         let mut cache = self.negative_cache.write();
-        if cache.len() > MAX_DYNAMIC_SCHEMAS {
+        if cache.len() >= MAX_DYNAMIC_SCHEMAS {
             let now = Instant::now();
             cache.retain(|_, entry| entry.expires_at > now);
         }
@@ -87,29 +143,95 @@ impl DynamicRegistry {
             });
         }
 
-        if store
-            .schemas
-            .insert(nsid.clone(), Arc::clone(&arc))
-            .is_some()
-        {
+        let entry = PositiveEntry {
+            doc: Arc::clone(&arc),
+            expires_at: Instant::now() + POSITIVE_CACHE_TTL,
+        };
+        if store.schemas.insert(nsid.clone(), entry).is_some() {
             store.insertion_order.retain(|k| k != &nsid);
         }
         store.insertion_order.push_back(nsid.clone());
+        drop(store);
 
         self.negative_cache.write().remove(&arc.id);
 
         arc
     }
 
+    fn bump_expiry(&self, nsid: &str, duration: Duration) {
+        let mut store = self.store.write();
+        if let Some(entry) = store.schemas.get_mut(nsid) {
+            entry.expires_at = Instant::now() + duration;
+        }
+    }
+
     pub async fn resolve_and_cache(&self, nsid: &str) -> Result<Arc<LexiconDoc>, ResolveError> {
-        if let Some(doc) = self.get(nsid) {
-            return Ok(doc);
+        self.resolve_and_cache_with(nsid, |n| async move { resolve_lexicon(&n).await })
+            .await
+    }
+
+    async fn resolve_and_cache_with<F, Fut>(
+        &self,
+        nsid: &str,
+        resolver: F,
+    ) -> Result<Arc<LexiconDoc>, ResolveError>
+    where
+        F: FnOnce(String) -> Fut,
+        Fut: std::future::Future<Output = Result<LexiconDoc, ResolveError>>,
+    {
+        match self.get_entry(nsid) {
+            Some(CacheEntry::Fresh(doc)) => Ok(doc),
+            Some(CacheEntry::Stale(stale)) => self.refresh_stale(nsid, stale, resolver).await,
+            None => self.resolve_fresh(nsid, resolver).await,
+        }
+    }
+
+    async fn refresh_stale<F, Fut>(
+        &self,
+        nsid: &str,
+        stale: Arc<LexiconDoc>,
+        resolver: F,
+    ) -> Result<Arc<LexiconDoc>, ResolveError>
+    where
+        F: FnOnce(String) -> Fut,
+        Fut: std::future::Future<Output = Result<LexiconDoc, ResolveError>>,
+    {
+        if self.network_disabled.load(Ordering::Relaxed) {
+            return Ok(stale);
         }
 
+        match self.acquire_leadership(nsid) {
+            Some(_guard) => match resolver(nsid.to_string()).await {
+                Ok(doc) => Ok(self.insert_schema(doc)),
+                Err(e) => {
+                    self.bump_expiry(nsid, REFRESH_FAILURE_BACKOFF);
+                    tracing::warn!(
+                        nsid = nsid,
+                        error = %e,
+                        "lexicon refresh failed, serving stale cached entry"
+                    );
+                    Ok(stale)
+                }
+            },
+            None => {
+                self.wait_for_leader(nsid).await;
+                Ok(self.get_cached(nsid).unwrap_or(stale))
+            }
+        }
+    }
+
+    async fn resolve_fresh<F, Fut>(
+        &self,
+        nsid: &str,
+        resolver: F,
+    ) -> Result<Arc<LexiconDoc>, ResolveError>
+    where
+        F: FnOnce(String) -> Fut,
+        Fut: std::future::Future<Output = Result<LexiconDoc, ResolveError>>,
+    {
         if self.network_disabled.load(Ordering::Relaxed) {
             return Err(ResolveError::NetworkDisabled);
         }
-
         if self.is_negative_cached(nsid) {
             return Err(ResolveError::NegativelyCached {
                 nsid: nsid.to_string(),
@@ -117,19 +239,73 @@ impl DynamicRegistry {
             });
         }
 
-        match resolve_lexicon(nsid).await {
-            Ok(doc) => Ok(self.insert_schema(doc)),
-            Err(e) => {
-                tracing::debug!(nsid = nsid, error = %e, "caching negative resolution result");
-                self.insert_negative(nsid);
-                Err(e)
+        match self.acquire_leadership(nsid) {
+            Some(_guard) => match resolver(nsid.to_string()).await {
+                Ok(doc) => Ok(self.insert_schema(doc)),
+                Err(e) => {
+                    self.insert_negative(nsid);
+                    tracing::debug!(nsid = nsid, error = %e, "caching negative resolution result");
+                    Err(e)
+                }
+            },
+            None => {
+                self.wait_for_leader(nsid).await;
+                match self.get_cached(nsid) {
+                    Some(doc) => Ok(doc),
+                    None if self.is_negative_cached(nsid) => Err(ResolveError::NegativelyCached {
+                        nsid: nsid.to_string(),
+                        ttl_secs: NEGATIVE_CACHE_TTL.as_secs(),
+                    }),
+                    None => Err(ResolveError::LeaderAborted {
+                        nsid: nsid.to_string(),
+                    }),
+                }
             }
         }
     }
 
+    fn acquire_leadership(&self, nsid: &str) -> Option<InFlightGuard<'_>> {
+        let mut map = self.in_flight.write();
+        if map.contains_key(nsid) {
+            None
+        } else {
+            map.insert(nsid.to_string(), Arc::new(Notify::new()));
+            Some(InFlightGuard {
+                registry: self,
+                nsid: nsid.to_string(),
+            })
+        }
+    }
+
+    async fn wait_for_leader(&self, nsid: &str) {
+        let notify = {
+            let map = self.in_flight.read();
+            match map.get(nsid) {
+                Some(n) => Arc::clone(n),
+                None => return,
+            }
+        };
+        let notified = notify.notified();
+        tokio::pin!(notified);
+        notified.as_mut().enable();
+        let still_active = self.in_flight.read().contains_key(nsid);
+        if !still_active {
+            return;
+        }
+        notified.as_mut().await;
+    }
+
     pub fn schema_count(&self) -> usize {
         self.store.read().schemas.len()
     }
+
+    #[cfg(test)]
+    fn expire_now(&self, nsid: &str) {
+        let mut store = self.store.write();
+        if let Some(entry) = store.schemas.get_mut(nsid) {
+            entry.expires_at = Instant::now();
+        }
+    }
 }
 
 impl Default for DynamicRegistry {
@@ -171,7 +347,7 @@ mod tests {
     #[test]
     fn test_empty_lookup() {
         let registry = DynamicRegistry::new();
-        assert!(registry.get("com.example.nonexistent").is_none());
+        assert!(registry.get_cached("com.example.nonexistent").is_none());
         assert_eq!(registry.schema_count(), 0);
     }
 
@@ -188,9 +364,12 @@ mod tests {
         assert_eq!(arc.id, "com.example.test");
         assert_eq!(registry.schema_count(), 1);
 
-        let retrieved = registry.get("com.example.test");
+        let retrieved = registry.get_cached("com.example.test");
         assert!(retrieved.is_some());
         assert_eq!(retrieved.unwrap().id, "com.example.test");
+
+        let entry = registry.get_entry("com.example.test").unwrap();
+        assert!(entry.is_fresh(), "freshly inserted entry must be fresh");
     }
 
     #[test]
@@ -210,6 +389,218 @@ mod tests {
         assert!(!registry.is_negative_cached("com.example.test"));
     }
 
+    #[test]
+    fn test_positive_entry_reports_stale_after_ttl() {
+        let registry = DynamicRegistry::new();
+        let doc = LexiconDoc {
+            lexicon: 1,
+            id: "pet.nel.stale".to_string(),
+            defs: HashMap::new(),
+        };
+        registry.insert_schema(doc);
+
+        assert!(registry.get_entry("pet.nel.stale").unwrap().is_fresh());
+
+        registry.expire_now("pet.nel.stale");
+
+        assert!(
+            !registry.get_entry("pet.nel.stale").unwrap().is_fresh(),
+            "entry past expiry must be reported stale"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_stale_served_on_resolve_failure() {
+        let registry = DynamicRegistry::new();
+        let doc = LexiconDoc {
+            lexicon: 1,
+            id: "pet.nel.flaky".to_string(),
+            defs: HashMap::new(),
+        };
+        registry.insert_schema(doc);
+        registry.expire_now("pet.nel.flaky");
+
+        let result = registry
+            .resolve_and_cache_with("pet.nel.flaky", |n| async move {
+                Err::<LexiconDoc, _>(ResolveError::DnsLookup {
+                    domain: n,
+                    reason: "simulated failure".to_string(),
+                })
+            })
+            .await;
+
+        let served = result.expect("stale entry must be served when refresh fails");
+        assert_eq!(served.id, "pet.nel.flaky");
+        assert!(
+            registry.get_entry("pet.nel.flaky").unwrap().is_fresh(),
+            "failed refresh must bump expiry so subsequent lookups skip the resolver"
+        );
+        assert!(
+            !registry.is_negative_cached("pet.nel.flaky"),
+            "stale refresh failure must not poison negative cache"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_fresh_hit_skips_resolver() {
+        let registry = DynamicRegistry::new();
+        let doc = LexiconDoc {
+            lexicon: 1,
+            id: "pet.nel.fresh".to_string(),
+            defs: HashMap::new(),
+        };
+        registry.insert_schema(doc);
+
+        let result = registry
+            .resolve_and_cache_with("pet.nel.fresh", |_| async move {
+                panic!("resolver must not run on fresh hit")
+            })
+            .await;
+
+        assert!(result.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_stale_served_when_network_disabled() {
+        let registry = DynamicRegistry::new();
+        let doc = LexiconDoc {
+            lexicon: 1,
+            id: "pet.nel.offline".to_string(),
+            defs: HashMap::new(),
+        };
+        registry.insert_schema(doc);
+        registry.expire_now("pet.nel.offline");
+        registry.set_network_disabled(true);
+
+        let result = registry
+            .resolve_and_cache_with("pet.nel.offline", |_| async move {
+                panic!("resolver must not run when network disabled")
+            })
+            .await;
+
+        assert!(result.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_successful_refresh_updates_cached_at() {
+        let registry = DynamicRegistry::new();
+        let doc = LexiconDoc {
+            lexicon: 1,
+            id: "pet.nel.refresh".to_string(),
+            defs: HashMap::new(),
+        };
+        registry.insert_schema(doc);
+        registry.expire_now("pet.nel.refresh");
+
+        assert!(!registry.get_entry("pet.nel.refresh").unwrap().is_fresh());
+
+        let refreshed = registry
+            .resolve_and_cache_with("pet.nel.refresh", |n| async move {
+                Ok(LexiconDoc {
+                    lexicon: 1,
+                    id: n,
+                    defs: HashMap::new(),
+                })
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(refreshed.id, "pet.nel.refresh");
+        assert!(
+            registry.get_entry("pet.nel.refresh").unwrap().is_fresh(),
+            "refresh must restore freshness"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_single_flight_dedups_concurrent_resolves() {
+        use std::sync::atomic::AtomicUsize;
+        let registry = Arc::new(DynamicRegistry::new());
+        let calls = Arc::new(AtomicUsize::new(0));
+
+        let tasks: Vec<_> = (0..16)
+            .map(|_| {
+                let registry = Arc::clone(&registry);
+                let calls = Arc::clone(&calls);
+                tokio::spawn(async move {
+                    registry
+                        .resolve_and_cache_with("pet.nel.herd", |n| {
+                            let calls = Arc::clone(&calls);
+                            async move {
+                                calls.fetch_add(1, Ordering::SeqCst);
+                                tokio::time::sleep(Duration::from_millis(50)).await;
+                                Ok(LexiconDoc {
+                                    lexicon: 1,
+                                    id: n,
+                                    defs: HashMap::new(),
+                                })
+                            }
+                        })
+                        .await
+                })
+            })
+            .collect();
+
+        let results = futures_collect(tasks).await;
+        results
+            .iter()
+            .for_each(|r| assert!(r.is_ok(), "all single-flight callers must succeed"));
+        assert_eq!(
+            calls.load(Ordering::SeqCst),
+            1,
+            "single-flight must coalesce concurrent resolves"
+        );
+        assert_eq!(registry.schema_count(), 1);
+    }
+
+    #[tokio::test]
+    async fn test_single_flight_followers_observe_leader_failure() {
+        use std::sync::atomic::AtomicUsize;
+        let registry = Arc::new(DynamicRegistry::new());
+        let calls = Arc::new(AtomicUsize::new(0));
+
+        let tasks: Vec<_> = (0..8)
+            .map(|_| {
+                let registry = Arc::clone(&registry);
+                let calls = Arc::clone(&calls);
+                tokio::spawn(async move {
+                    registry
+                        .resolve_and_cache_with("pet.nel.failHerd", |n| {
+                            let calls = Arc::clone(&calls);
+                            async move {
+                                calls.fetch_add(1, Ordering::SeqCst);
+                                tokio::time::sleep(Duration::from_millis(50)).await;
+                                Err::<LexiconDoc, _>(ResolveError::DnsLookup {
+                                    domain: n,
+                                    reason: "simulated".to_string(),
+                                })
+                            }
+                        })
+                        .await
+                })
+            })
+            .collect();
+
+        let results = futures_collect(tasks).await;
+        results
+            .iter()
+            .for_each(|r| assert!(r.is_err(), "all followers must observe leader failure"));
+        assert_eq!(
+            calls.load(Ordering::SeqCst),
+            1,
+            "single-flight must coalesce failing resolves too"
+        );
+        assert!(registry.is_negative_cached("pet.nel.failHerd"));
+    }
+
+    async fn futures_collect<T>(handles: Vec<tokio::task::JoinHandle<T>>) -> Vec<T> {
+        futures::future::join_all(handles)
+            .await
+            .into_iter()
+            .map(|r| r.expect("task panicked"))
+            .collect()
+    }
+
     #[test]
     fn test_eviction_is_fifo() {
         let registry = DynamicRegistry::new();
@@ -217,7 +608,7 @@ mod tests {
         (0..MAX_DYNAMIC_SCHEMAS).for_each(|i| {
             let doc = LexiconDoc {
                 lexicon: 1,
-                id: format!("com.example.schema{}", i),
+                id: format!("pet.nel.schema{}", i),
                 defs: HashMap::new(),
             };
             registry.insert_schema(doc);
@@ -226,23 +617,23 @@ mod tests {
 
         let trigger = LexiconDoc {
             lexicon: 1,
-            id: "com.example.trigger".to_string(),
+            id: "pet.nel.trigger".to_string(),
             defs: HashMap::new(),
         };
         registry.insert_schema(trigger);
 
         assert!(
-            registry.get("com.example.schema0").is_none(),
+            registry.get_cached("pet.nel.schema0").is_none(),
             "oldest entry should be evicted"
         );
         assert!(
-            registry.get("com.example.trigger").is_some(),
+            registry.get_cached("pet.nel.trigger").is_some(),
             "newly inserted entry should exist"
         );
         let evict_count = MAX_DYNAMIC_SCHEMAS / 4;
         assert!(
             registry
-                .get(&format!("com.example.schema{}", evict_count))
+                .get_cached(&format!("pet.nel.schema{}", evict_count))
                 .is_some(),
             "entry after eviction window should survive"
         );
@@ -253,7 +644,7 @@ mod tests {
         let registry = DynamicRegistry::new();
         let doc = LexiconDoc {
             lexicon: 1,
-            id: "com.example.tracked".to_string(),
+            id: "pet.nel.tracked".to_string(),
             defs: HashMap::new(),
         };
         let arc = registry.insert_schema(doc);
@@ -265,7 +656,7 @@ mod tests {
         (0..MAX_DYNAMIC_SCHEMAS).for_each(|i| {
             registry.insert_schema(LexiconDoc {
                 lexicon: 1,
-                id: format!("com.example.filler{}", i),
+                id: format!("pet.nel.filler{}", i),
                 defs: HashMap::new(),
             });
         });

crates/tranquil-lexicon/src/registry.rs

@@ -25,7 +25,7 @@ impl LexiconRegistry {
         Self {
             schemas: HashMap::new(),
             #[cfg(feature = "resolve")]
-            dynamic: crate::dynamic::DynamicRegistry::new(),
+            dynamic: crate::dynamic::DynamicRegistry::from_env(),
         }
     }
 
@@ -43,7 +43,7 @@ impl LexiconRegistry {
         self.schemas.get(nsid).cloned().or_else(|| {
             #[cfg(feature = "resolve")]
             {
-                self.dynamic.get(nsid)
+                self.dynamic.get_cached(nsid)
             }
             #[cfg(not(feature = "resolve"))]
             {

crates/tranquil-lexicon/src/resolve.rs

@@ -70,6 +70,8 @@ pub enum ResolveError {
     NegativelyCached { nsid: String, ttl_secs: u64 },
     #[error("network resolution disabled")]
     NetworkDisabled,
+    #[error("leader task for {nsid} aborted before completion")]
+    LeaderAborted { nsid: String },
 }
 
 pub fn nsid_to_authority(nsid: &str) -> Result<String, ResolveError> {

crates/tranquil-oauth-server/src/endpoints/authorize/passkey.rs

@@ -175,10 +175,7 @@ pub async fn passkey_start(
     }
 }
 
-async fn passkey_start_discoverable(
-    state: AppState,
-    request_id: RequestId,
-) -> Response {
+async fn passkey_start_discoverable(state: AppState, request_id: RequestId) -> Response {
     let (rcr, auth_state) = match state.webauthn_config.start_discoverable_authentication() {
         Ok(result) => result,
         Err(e) => {
@@ -570,16 +567,13 @@ pub async fn passkey_finish(
             Err(response) => return response,
         },
         None => {
-            let result = match passkey_finish_discoverable(
-                &state,
-                &credential,
-                &passkey_finish_request_id,
-            )
-            .await
-            {
-                Ok(result) => result,
-                Err(response) => return response,
-            };
+            let result =
+                match passkey_finish_discoverable(&state, &credential, &passkey_finish_request_id)
+                    .await
+                {
+                    Ok(result) => result,
+                    Err(response) => return response,
+                };
             if state
                 .repos
                 .oauth

crates/tranquil-oauth-server/src/endpoints/par.rs

@@ -138,21 +138,29 @@ pub async fn pushed_authorization_request(
 }
 
 fn determine_client_auth(request: &ParRequest) -> Result<ClientAuth, OAuthError> {
-    if let (Some(assertion), Some(assertion_type)) =
-        (&request.client_assertion, &request.client_assertion_type)
-    {
+    let assertion = request
+        .client_assertion
+        .as_deref()
+        .filter(|s| !s.is_empty());
+    let assertion_type = request
+        .client_assertion_type
+        .as_deref()
+        .filter(|s| !s.is_empty());
+    let secret = request.client_secret.as_deref().filter(|s| !s.is_empty());
+
+    if let (Some(assertion), Some(assertion_type)) = (assertion, assertion_type) {
         if assertion_type != "urn:ietf:params:oauth:client-assertion-type:jwt-bearer" {
             return Err(OAuthError::InvalidRequest(
                 "Unsupported client_assertion_type".to_string(),
             ));
         }
         return Ok(ClientAuth::PrivateKeyJwt {
-            client_assertion: assertion.clone(),
+            client_assertion: assertion.to_string(),
         });
     }
-    if let Some(secret) = &request.client_secret {
+    if let Some(secret) = secret {
         return Ok(ClientAuth::SecretPost {
-            client_secret: secret.clone(),
+            client_secret: secret.to_string(),
         });
     }
     Ok(ClientAuth::None)

crates/tranquil-oauth-server/src/endpoints/token/types.rs

@@ -138,13 +138,17 @@ impl TokenRequest {
             }
         };
 
-        let client_auth = match (self.client_assertion, self.client_assertion_type) {
+        let assertion = self.client_assertion.filter(|s| !s.is_empty());
+        let assertion_type = self.client_assertion_type.filter(|s| !s.is_empty());
+        let client_secret = self.client_secret.filter(|s| !s.is_empty());
+
+        let client_auth = match (assertion, assertion_type) {
             (Some(assertion), Some(assertion_type)) => RequestClientAuth::PrivateKeyJwt {
                 client_id: self.client_id,
                 assertion,
                 assertion_type,
             },
-            _ => match self.client_secret {
+            _ => match client_secret {
                 Some(secret) => RequestClientAuth::SecretPost {
                     client_id: self.client_id,
                     client_secret: secret,

crates/tranquil-pds/src/api/validation.rs

@@ -287,8 +287,6 @@ pub fn is_valid_discord_username(username: &str) -> bool {
         && username
             .chars()
             .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_' || c == '.')
-        && !username.starts_with('.')
-        && !username.ends_with('.')
         && !username.contains("..")
 }
 
@@ -493,8 +491,6 @@ mod tests {
         assert!(!is_valid_discord_username("Alice"));
         assert!(!is_valid_discord_username("ALICE"));
         assert!(!is_valid_discord_username("user-name"));
-        assert!(!is_valid_discord_username(".username"));
-        assert!(!is_valid_discord_username("username."));
         assert!(!is_valid_discord_username("user..name"));
         assert!(!is_valid_discord_username("user name"));
         assert!(!is_valid_discord_username(

crates/tranquil-pds/src/auth/legacy_2fa.rs

@@ -156,6 +156,7 @@ pub enum Legacy2faOutcome {
 }
 
 pub struct Legacy2faContext {
+    pub is_app_password: bool,
     pub email_2fa_enabled: bool,
     pub has_totp: bool,
     pub allow_legacy_login: bool,
@@ -163,7 +164,7 @@ pub struct Legacy2faContext {
 
 impl Legacy2faContext {
     pub fn requires_2fa(&self) -> bool {
-        self.email_2fa_enabled || self.has_totp
+        !self.is_app_password && (self.email_2fa_enabled || self.has_totp)
     }
 
     pub fn is_blocked(&self) -> bool {
@@ -418,6 +419,7 @@ mod tests {
         let cache = MockCache::new();
         let did = Did::new("did:plc:test".to_string()).unwrap();
         let ctx = Legacy2faContext {
+            is_app_password: false,
             email_2fa_enabled: false,
             has_totp: false,
             allow_legacy_login: true,
@@ -427,11 +429,27 @@ mod tests {
         assert!(matches!(outcome, Legacy2faOutcome::NotRequired));
     }
 
+    #[tokio::test]
+    async fn test_process_flow_not_required_because_app_password() {
+        let cache = MockCache::new();
+        let did = Did::new("did:plc:test".to_string()).unwrap();
+        let ctx = Legacy2faContext {
+            is_app_password: true,
+            email_2fa_enabled: false,
+            has_totp: true,
+            allow_legacy_login: true,
+        };
+
+        let outcome = process_legacy_2fa(&cache, &did, &ctx, None).await.unwrap();
+        assert!(matches!(outcome, Legacy2faOutcome::NotRequired));
+    }
+
     #[tokio::test]
     async fn test_process_flow_blocked() {
         let cache = MockCache::new();
         let did = Did::new("did:plc:test".to_string()).unwrap();
         let ctx = Legacy2faContext {
+            is_app_password: false,
             email_2fa_enabled: false,
             has_totp: true,
             allow_legacy_login: false,
@@ -446,6 +464,7 @@ mod tests {
         let cache = MockCache::new();
         let did = Did::new("did:plc:test".to_string()).unwrap();
         let ctx = Legacy2faContext {
+            is_app_password: false,
             email_2fa_enabled: false,
             has_totp: true,
             allow_legacy_login: true,
@@ -460,6 +479,7 @@ mod tests {
         let cache = MockCache::new();
         let did = Did::new("did:plc:test2".to_string()).unwrap();
         let ctx = Legacy2faContext {
+            is_app_password: false,
             email_2fa_enabled: true,
             has_totp: false,
             allow_legacy_login: false,
@@ -474,6 +494,7 @@ mod tests {
         let cache = MockCache::new();
         let did = Did::new("did:plc:test".to_string()).unwrap();
         let ctx = Legacy2faContext {
+            is_app_password: false,
             email_2fa_enabled: true,
             has_totp: false,
             allow_legacy_login: false,

crates/tranquil-pds/src/repo_ops.rs

@@ -6,17 +6,19 @@ use crate::types::{Did, Handle, Nsid, Rkey};
 use backon::{ExponentialBuilder, Retryable};
 use bytes::Bytes;
 use cid::Cid;
+use jacquard_common::smol_str::SmolStr;
 use jacquard_common::types::{integer::LimitedU32, string::Tid};
 use jacquard_repo::commit::Commit;
-use jacquard_repo::mst::Mst;
 use jacquard_repo::mst::util::compute_cid;
+use jacquard_repo::mst::{Mst, VerifiedWriteOp};
 use jacquard_repo::storage::BlockStore;
 use k256::ecdsa::SigningKey;
 use serde_json::{Value, json};
+use std::collections::{BTreeMap, HashMap, HashSet};
 use std::str::FromStr;
 use std::sync::Arc;
 use tokio::sync::OwnedMutexGuard;
-use tracing::error;
+use tracing::{error, warn};
 use tranquil_db_traits::SequenceNumber;
 use uuid::Uuid;
 
@@ -38,6 +40,7 @@ pub enum CommitError {
     MstOperationFailed(String),
     RecordSerializationFailed(String),
     InvalidCid(String),
+    RecordAlreadyExists(String),
 }
 
 impl std::fmt::Display for CommitError {
@@ -63,6 +66,7 @@ impl std::fmt::Display for CommitError {
                 write!(f, "Failed to serialize record: {}", e)
             }
             Self::InvalidCid(e) => write!(f, "Invalid CID: {}", e),
+            Self::RecordAlreadyExists(key) => write!(f, "Record already exists at {}", key),
         }
     }
 }
@@ -77,6 +81,9 @@ impl From<CommitError> for ApiError {
             }
             CommitError::RepoNotFound => ApiError::RepoNotFound(None),
             CommitError::UserNotFound => ApiError::RepoNotFound(Some("User not found".into())),
+            CommitError::RecordAlreadyExists(key) => {
+                ApiError::InvalidRequest(format!("Record already exists at {key}"))
+            }
             other => {
                 error!("Commit failed: {}", other);
                 ApiError::InternalError(Some("Failed to commit changes".into()))
@@ -160,7 +167,6 @@ pub struct FinalizeParams<'a> {
     pub controller_did: Option<&'a Did>,
     pub delegation_detail: Option<serde_json::Value>,
     pub ops: Vec<RecordOp>,
-    pub modified_keys: &'a [String],
     pub blob_cids: &'a [String],
     pub backlinks_to_add: Vec<Backlink>,
     pub backlinks_to_remove: Vec<AtUri>,
@@ -236,19 +242,84 @@ pub async fn finalize_repo_write(
         ApiError::InternalError(None)
     })?;
 
-    let block_bytes = ctx.tracking_store.take_written_blocks();
+    let written_bytes = ctx.tracking_store.take_written_blocks();
+    let new_tree_cids: Vec<Cid> = written_bytes.keys().copied().collect();
 
-    let storage_for_diff = Arc::new(ctx.tracking_store.clone());
-    let original_settled = Mst::load(storage_for_diff.clone(), ctx.prev_data_cid, None);
-    let new_settled = Mst::load(storage_for_diff, new_mst_root, None);
+    let storage_for_proof = Arc::new(ctx.tracking_store.clone());
+    let original_settled = Mst::load(storage_for_proof.clone(), ctx.prev_data_cid, None);
+    let new_settled = Mst::load(storage_for_proof.clone(), new_mst_root, None);
 
-    let new_tree_cids: Vec<Cid> = block_bytes.keys().copied().collect();
+    let mut inverse_trace = new_settled.clone();
+    let mut non_invertible: Vec<String> = Vec::new();
+    let mut invert_errors: Vec<String> = Vec::new();
+    for op in params.ops.iter().rev() {
+        let (collection, rkey) = op.collection_rkey();
+        let key = SmolStr::new(format!("{}/{}", collection, rkey));
+        let verified = match op {
+            RecordOp::Create { cid, .. } => VerifiedWriteOp::Create {
+                key,
+                cid: *cid.as_cid(),
+            },
+            RecordOp::Update { cid, prev, .. } => VerifiedWriteOp::Update {
+                key,
+                cid: *cid.as_cid(),
+                prev: *prev.as_cid(),
+            },
+            RecordOp::Delete { prev, .. } => VerifiedWriteOp::Delete {
+                key,
+                prev: *prev.as_cid(),
+            },
+        };
+        match inverse_trace.invert_op(verified.clone()).await {
+            Ok(true) => {}
+            Ok(false) => non_invertible.push(format!("{:?}", verified)),
+            Err(e) => invert_errors.push(format!("{:?} -> {:?}", verified, e)),
+        }
+    }
+    if !non_invertible.is_empty() {
+        warn!(
+            user_id = %params.user_id,
+            count = non_invertible.len(),
+            ops = ?non_invertible,
+            "firehose proof walk: ops not invertible on new MST, consumer will reject frame"
+        );
+    }
+    if !invert_errors.is_empty() {
+        warn!(
+            user_id = %params.user_id,
+            count = invert_errors.len(),
+            failures = ?invert_errors,
+            "firehose proof walk: invert_op errored, cover blocks may be incomplete"
+        );
+    }
+
+    let read_cid_set: HashSet<Cid> = ctx.tracking_store.get_read_cids().into_iter().collect();
+    let missing_read_cids: Vec<Cid> = read_cid_set
+        .iter()
+        .copied()
+        .filter(|cid| !written_bytes.contains_key(cid))
+        .collect();
+    let mut relevant: BTreeMap<Cid, Bytes> = BTreeMap::new();
+    if !missing_read_cids.is_empty() {
+        let fetched = ctx
+            .tracking_store
+            .get_many(&missing_read_cids)
+            .await
+            .map_err(|e| {
+                error!("fetch cover read bytes: {e}");
+                ApiError::InternalError(None)
+            })?;
+        for (cid, maybe) in missing_read_cids.into_iter().zip(fetched) {
+            if let Some(bytes) = maybe {
+                relevant.insert(cid, bytes);
+            }
+        }
+    }
 
     let obsolete_cids = match original_settled.diff(&new_settled).await {
         Ok(diff) => {
-            let mut obsolete: Vec<Cid> = Vec::with_capacity(
-                1 + diff.removed_mst_blocks.len() + diff.removed_cids.len(),
-            );
+            let mut obsolete: Vec<Cid> =
+                Vec::with_capacity(1 + diff.removed_mst_blocks.len() + diff.removed_cids.len());
             obsolete.push(ctx.current_root_cid);
             obsolete.extend(diff.removed_mst_blocks);
             obsolete.extend(diff.removed_cids);
@@ -264,6 +335,9 @@ pub async fn finalize_repo_write(
         }
     };
 
+    let mut block_bytes = written_bytes;
+    block_bytes.extend(relevant);
+
     let result = commit_and_log(
         state,
         CommitParams {
@@ -347,6 +421,22 @@ pub enum RecordOp {
     },
 }
 
+impl RecordOp {
+    pub fn collection_rkey(&self) -> (&Nsid, &Rkey) {
+        match self {
+            Self::Create {
+                collection, rkey, ..
+            }
+            | Self::Update {
+                collection, rkey, ..
+            }
+            | Self::Delete {
+                collection, rkey, ..
+            } => (collection, rkey),
+        }
+    }
+}
+
 pub struct CommitResult {
     pub commit_cid: Cid,
     pub rev: String,
@@ -377,8 +467,6 @@ pub async fn commit_and_log(
         RecordUpsert, RepoEventType,
     };
 
-    let backlinks_to_add = params.backlinks_to_add;
-    let backlinks_to_remove = params.backlinks_to_remove;
     let CommitParams {
         did,
         user_id,
@@ -391,7 +479,8 @@ pub async fn commit_and_log(
         new_tree_cids,
         blobs,
         obsolete_cids,
-        ..
+        backlinks_to_add,
+        backlinks_to_remove,
     } = params;
     debug_assert_eq!(
         current_root_cid.is_some(),
@@ -437,39 +526,65 @@ pub async fn commit_and_log(
 
     let obsolete_bytes: Vec<Vec<u8>> = obsolete_cids.iter().map(|c| c.to_bytes()).collect();
 
-    let (record_upserts, record_deletes): (Vec<RecordUpsert>, Vec<RecordDelete>) = ops.iter().fold(
-        (Vec::new(), Vec::new()),
-        |(mut upserts, mut deletes), op| {
-            match op {
-                RecordOp::Create {
-                    collection,
-                    rkey,
-                    cid,
-                }
-                | RecordOp::Update {
-                    collection,
-                    rkey,
-                    cid,
-                    ..
-                } => {
-                    upserts.push(RecordUpsert {
-                        collection: collection.clone(),
-                        rkey: rkey.clone(),
-                        cid: crate::types::CidLink::from(cid.as_cid()),
-                    });
-                }
-                RecordOp::Delete {
-                    collection, rkey, ..
-                } => {
-                    deletes.push(RecordDelete {
-                        collection: collection.clone(),
-                        rkey: rkey.clone(),
-                    });
-                }
+    let final_ops: HashMap<(&Nsid, &Rkey), &RecordOp> = ops
+        .iter()
+        .map(|op| (op.collection_rkey(), op))
+        .collect();
+
+    let final_record_uris: HashSet<AtUri> = final_ops
+        .iter()
+        .filter(|(_, op)| !matches!(op, RecordOp::Delete { .. }))
+        .map(|((c, r), _)| AtUri::from_parts(did, c, r))
+        .collect();
+
+    let record_upserts: Vec<RecordUpsert> = final_ops
+        .values()
+        .filter_map(|op| match op {
+            RecordOp::Create {
+                collection,
+                rkey,
+                cid,
             }
-            (upserts, deletes)
-        },
-    );
+            | RecordOp::Update {
+                collection,
+                rkey,
+                cid,
+                ..
+            } => Some(RecordUpsert {
+                collection: collection.clone(),
+                rkey: rkey.clone(),
+                cid: crate::types::CidLink::from(cid.as_cid()),
+            }),
+            RecordOp::Delete { .. } => None,
+        })
+        .collect();
+
+    let record_deletes: Vec<RecordDelete> = final_ops
+        .values()
+        .filter_map(|op| match op {
+            RecordOp::Delete {
+                collection, rkey, ..
+            } => Some(RecordDelete {
+                collection: collection.clone(),
+                rkey: rkey.clone(),
+            }),
+            _ => None,
+        })
+        .collect();
+
+    let backlinks_to_add: Vec<Backlink> = backlinks_to_add
+        .into_iter()
+        .filter(|b| final_record_uris.contains(&b.uri))
+        .map(|b| ((b.uri.clone(), b.path), b))
+        .collect::<HashMap<_, _>>()
+        .into_values()
+        .collect();
+
+    let backlinks_to_remove: Vec<AtUri> = backlinks_to_remove
+        .into_iter()
+        .collect::<HashSet<_>>()
+        .into_iter()
+        .collect();
 
     let ops_json: Vec<serde_json::Value> = ops
         .iter()
@@ -604,6 +719,16 @@ pub async fn create_record_internal(
         .await
         .map_err(to_commit_err)?;
 
+    let key = format!("{}/{}", collection, rkey);
+    if mst
+        .get(&key)
+        .await
+        .map_err(|e| CommitError::MstOperationFailed(e.to_string()))?
+        .is_some()
+    {
+        return Err(CommitError::RecordAlreadyExists(key));
+    }
+
     let record_ipld = crate::util::json_to_ipld(record);
     let mut record_bytes = Vec::new();
     serde_ipld_dagcbor::to_writer(&mut record_bytes, &record_ipld)
@@ -613,8 +738,6 @@ pub async fn create_record_internal(
         .put(&record_bytes)
         .await
         .map_err(|e| CommitError::BlockStoreFailed(e.to_string()))?;
-
-    let key = format!("{}/{}", collection, rkey);
     let new_mst = mst
         .add(&key, record_cid)
         .await
@@ -625,7 +748,6 @@ pub async fn create_record_internal(
         rkey: rkey.clone(),
         cid: RecordCid::from(record_cid),
     };
-    let modified_keys = [key];
     let blob_cids = extract_blob_cids(record);
     let record_uri = AtUri::from_parts(did.as_str(), collection.as_str(), rkey.as_str());
     let backlinks = extract_backlinks(&record_uri, record);
@@ -640,7 +762,6 @@ pub async fn create_record_internal(
             controller_did: None,
             delegation_detail: None,
             ops: vec![op],
-            modified_keys: &modified_keys,
             blob_cids: &blob_cids,
             backlinks_to_add: backlinks,
             backlinks_to_remove: vec![],

crates/tranquil-pds/tests/common/mod.rs

@@ -191,16 +191,12 @@ async fn setup_with_external_infra() -> String {
 async fn setup_with_testcontainers() -> String {
     let temp_dir = std::env::temp_dir().join(format!("tranquil-pds-test-{}", uuid::Uuid::new_v4()));
     let blob_path = temp_dir.join("blobs");
-    let backup_path = temp_dir.join("backups");
     std::fs::create_dir_all(&blob_path).expect("Failed to create blob temp directory");
-    std::fs::create_dir_all(&backup_path).expect("Failed to create backup temp directory");
     TEST_TEMP_DIR.set(temp_dir).ok();
     let plc_url = setup_mock_plc_directory().await;
     unsafe {
         std::env::set_var("BLOB_STORAGE_BACKEND", "filesystem");
         std::env::set_var("BLOB_STORAGE_PATH", blob_path.to_str().unwrap());
-        std::env::set_var("BACKUP_STORAGE_BACKEND", "filesystem");
-        std::env::set_var("BACKUP_STORAGE_PATH", backup_path.to_str().unwrap());
         std::env::set_var("MAX_IMPORT_SIZE", "100000000");
         std::env::set_var("SKIP_IMPORT_VERIFICATION", "true");
         std::env::set_var("PLC_DIRECTORY_URL", &plc_url);
@@ -242,8 +238,6 @@ async fn setup_with_testcontainers() -> String {
     let plc_url = setup_mock_plc_directory().await;
     unsafe {
         std::env::set_var("BLOB_STORAGE_BACKEND", "s3");
-        std::env::set_var("BACKUP_STORAGE_BACKEND", "s3");
-        std::env::set_var("BACKUP_S3_BUCKET", "test-backups");
         std::env::set_var("S3_BUCKET", "test-bucket");
         std::env::set_var("AWS_ACCESS_KEY_ID", "minioadmin");
         std::env::set_var("AWS_SECRET_ACCESS_KEY", "minioadmin");
@@ -333,8 +327,6 @@ unsafe fn configure_external_storage_env() {
         if std::env::var("S3_ENDPOINT").is_ok() {
             let s3_endpoint = std::env::var("S3_ENDPOINT").unwrap();
             std::env::set_var("BLOB_STORAGE_BACKEND", "s3");
-            std::env::set_var("BACKUP_STORAGE_BACKEND", "s3");
-            std::env::set_var("BACKUP_S3_BUCKET", "test-backups");
             std::env::set_var(
                 "S3_BUCKET",
                 std::env::var("S3_BUCKET").unwrap_or_else(|_| "test-bucket".to_string()),
@@ -356,14 +348,10 @@ unsafe fn configure_external_storage_env() {
             let process_dir =
                 std::env::temp_dir().join(format!("tranquil-pds-test-{}", std::process::id()));
             let blob_path = process_dir.join("blobs");
-            let backup_path = process_dir.join("backups");
             std::fs::create_dir_all(&blob_path).expect("Failed to create blob directory");
-            std::fs::create_dir_all(&backup_path).expect("Failed to create backup directory");
             TEST_TEMP_DIR.set(process_dir).ok();
             std::env::set_var("BLOB_STORAGE_BACKEND", "filesystem");
             std::env::set_var("BLOB_STORAGE_PATH", blob_path.to_str().unwrap());
-            std::env::set_var("BACKUP_STORAGE_BACKEND", "filesystem");
-            std::env::set_var("BACKUP_STORAGE_PATH", backup_path.to_str().unwrap());
         }
         std::env::set_var("MAX_IMPORT_SIZE", "100000000");
         std::env::set_var("SKIP_IMPORT_VERIFICATION", "true");
@@ -622,18 +610,14 @@ async fn setup_store_backend() -> String {
     let temp_dir =
         std::env::temp_dir().join(format!("tranquil-pds-store-{}", uuid::Uuid::new_v4()));
     let blob_path = temp_dir.join("blobs");
-    let backup_path = temp_dir.join("backups");
     let store_path = temp_dir.join("store");
     std::fs::create_dir_all(&blob_path).expect("failed to create blob temp directory");
-    std::fs::create_dir_all(&backup_path).expect("failed to create backup temp directory");
     std::fs::create_dir_all(&store_path).expect("failed to create store temp directory");
     TEST_TEMP_DIR.set(temp_dir).ok();
     let plc_url = setup_mock_plc_directory().await;
     unsafe {
         std::env::set_var("BLOB_STORAGE_BACKEND", "filesystem");
         std::env::set_var("BLOB_STORAGE_PATH", blob_path.to_str().unwrap());
-        std::env::set_var("BACKUP_STORAGE_BACKEND", "filesystem");
-        std::env::set_var("BACKUP_STORAGE_PATH", backup_path.to_str().unwrap());
         std::env::set_var("MAX_IMPORT_SIZE", "100000000");
         std::env::set_var("SKIP_IMPORT_VERIFICATION", "true");
         std::env::set_var("PLC_DIRECTORY_URL", &plc_url);
@@ -790,18 +774,14 @@ async fn setup_cluster_store_backend() -> Option<sqlx::PgPool> {
         uuid::Uuid::new_v4()
     ));
     let blob_path = temp_dir.join("blobs");
-    let backup_path = temp_dir.join("backups");
     let store_path = temp_dir.join("store");
     std::fs::create_dir_all(&blob_path).expect("failed to create blob temp directory");
-    std::fs::create_dir_all(&backup_path).expect("failed to create backup temp directory");
     std::fs::create_dir_all(&store_path).expect("failed to create store temp directory");
     TEST_TEMP_DIR.set(temp_dir).ok();
     let plc_url = setup_mock_plc_directory().await;
     unsafe {
         std::env::set_var("BLOB_STORAGE_BACKEND", "filesystem");
         std::env::set_var("BLOB_STORAGE_PATH", blob_path.to_str().unwrap());
-        std::env::set_var("BACKUP_STORAGE_BACKEND", "filesystem");
-        std::env::set_var("BACKUP_STORAGE_PATH", backup_path.to_str().unwrap());
         std::env::set_var("MAX_IMPORT_SIZE", "100000000");
         std::env::set_var("SKIP_IMPORT_VERIFICATION", "true");
         std::env::set_var("PLC_DIRECTORY_URL", &plc_url);
@@ -847,16 +827,12 @@ async fn setup_cluster_testcontainers() -> Option<sqlx::PgPool> {
     let temp_dir =
         std::env::temp_dir().join(format!("tranquil-pds-cluster-{}", uuid::Uuid::new_v4()));
     let blob_path = temp_dir.join("blobs");
-    let backup_path = temp_dir.join("backups");
     std::fs::create_dir_all(&blob_path).expect("Failed to create blob temp directory");
-    std::fs::create_dir_all(&backup_path).expect("Failed to create backup temp directory");
     TEST_TEMP_DIR.set(temp_dir).ok();
     let plc_url = setup_mock_plc_directory().await;
     unsafe {
         std::env::set_var("BLOB_STORAGE_BACKEND", "filesystem");
         std::env::set_var("BLOB_STORAGE_PATH", blob_path.to_str().unwrap());
-        std::env::set_var("BACKUP_STORAGE_BACKEND", "filesystem");
-        std::env::set_var("BACKUP_STORAGE_PATH", backup_path.to_str().unwrap());
         std::env::set_var("MAX_IMPORT_SIZE", "100000000");
         std::env::set_var("SKIP_IMPORT_VERIFICATION", "true");
         std::env::set_var("PLC_DIRECTORY_URL", &plc_url);

crates/tranquil-pds/tests/gc_compaction_restart.rs

@@ -13,12 +13,10 @@ fn run_compaction(store: &tranquil_store::blockstore::TranquilBlockStore) {
         .map(|(&fid, _)| fid)
         .collect::<Vec<_>>()
         .into_iter()
-        .for_each(|fid| {
-            match store.compact_file(fid, 0) {
-                Ok(_) => {}
-                Err(tranquil_store::blockstore::CompactionError::ActiveFileCannotBeCompacted) => {}
-                Err(e) => eprintln!("compaction: {e}"),
-            }
+        .for_each(|fid| match store.compact_file(fid, 0) {
+            Ok(_) => {}
+            Err(tranquil_store::blockstore::CompactionError::ActiveFileCannotBeCompacted) => {}
+            Err(e) => eprintln!("compaction: {e}"),
         });
 }
 
@@ -84,10 +82,7 @@ async fn mst_blocks_survive_full_store_reopen() {
     }
 
     let data_dir = store.data_dir().to_path_buf();
-    let index_dir = data_dir
-        .parent()
-        .unwrap()
-        .join("index");
+    let index_dir = data_dir.parent().unwrap().join("index");
 
     let store_clone = store.clone();
     tokio::task::spawn_blocking(move || {
@@ -107,17 +102,16 @@ async fn mst_blocks_survive_full_store_reopen() {
 
     let head_cid = cid::Cid::try_from(repo_root_str.as_str()).expect("invalid cid");
 
-    let car_blocks =
-        tranquil_pds::scheduled::collect_current_repo_blocks(block_store, &head_cid)
-            .await
-            .expect("collect blocks");
+    let car_blocks = tranquil_pds::scheduled::collect_current_repo_blocks(block_store, &head_cid)
+        .await
+        .expect("collect blocks");
 
     let block_count_before = car_blocks.len();
 
     let max_file_size = store
         .list_data_files()
         .ok()
-        .and_then(|_| Some(4 * 1024 * 1024u64))
+        .map(|_| 4 * 1024 * 1024u64)
         .unwrap_or(4 * 1024 * 1024);
 
     let reopened_missing = tokio::task::spawn_blocking(move || {
@@ -131,8 +125,8 @@ async fn mst_blocks_survive_full_store_reopen() {
             group_commit: tranquil_store::blockstore::GroupCommitConfig::default(),
             shard_count: 1,
         };
-        let fresh = tranquil_store::blockstore::TranquilBlockStore::open(config)
-            .expect("reopen failed");
+        let fresh =
+            tranquil_store::blockstore::TranquilBlockStore::open(config).expect("reopen failed");
 
         let missing: Vec<String> = car_blocks
             .iter()

crates/tranquil-pds/tests/handle_domains.rs

@@ -8,7 +8,6 @@ const HANDLE_DOMAIN: &str = "handles.test";
 
 fn set_handle_domain() {
     unsafe {
-        std::env::set_var("AVAILABLE_USER_DOMAINS", HANDLE_DOMAIN);
         std::env::set_var("PDS_USER_HANDLE_DOMAINS", HANDLE_DOMAIN);
     }
 }

crates/tranquil-pds/tests/lifecycle_record.rs

@@ -456,7 +456,7 @@ async fn test_apply_writes_batch() {
         "writes": [
             { "$type": "com.atproto.repo.applyWrites#create", "collection": "app.bsky.feed.post", "rkey": "batch-post-1", "value": { "$type": "app.bsky.feed.post", "text": "First batch post", "createdAt": now } },
             { "$type": "com.atproto.repo.applyWrites#create", "collection": "app.bsky.feed.post", "rkey": "batch-post-2", "value": { "$type": "app.bsky.feed.post", "text": "Second batch post", "createdAt": now } },
-            { "$type": "com.atproto.repo.applyWrites#create", "collection": "app.bsky.actor.profile", "rkey": "self", "value": { "$type": "app.bsky.actor.profile", "displayName": "Batch User" } }
+            { "$type": "com.atproto.repo.applyWrites#update", "collection": "app.bsky.actor.profile", "rkey": "self", "value": { "$type": "app.bsky.actor.profile", "displayName": "Batch User" } }
         ]
     });
     let apply_res = client

crates/tranquil-pds/tests/lifecycle_session.rs

@@ -626,7 +626,12 @@ async fn create_app_password_session(
         .send()
         .await
         .expect("Failed to login with app password");
-    assert_eq!(login_res.status(), StatusCode::OK, "App password login for '{}' failed", name);
+    assert_eq!(
+        login_res.status(),
+        StatusCode::OK,
+        "App password login for '{}' failed",
+        name
+    );
     let session: Value = login_res.json().await.unwrap();
     let jwt = session["accessJwt"].as_str().unwrap().to_string();
     (jwt, scopes_response)
@@ -635,10 +640,7 @@ async fn create_app_password_session(
 async fn try_chat_service_auth(client: &reqwest::Client, jwt: &str) -> StatusCode {
     let base = base_url().await;
     let res = client
-        .get(format!(
-            "{}/xrpc/com.atproto.server.getServiceAuth",
-            base
-        ))
+        .get(format!("{}/xrpc/com.atproto.server.getServiceAuth", base))
         .bearer_auth(jwt)
         .query(&[
             ("aud", "did:web:api.bsky.app"),

crates/tranquil-pds/tests/mst_diff_equivalence.rs

@@ -15,7 +15,9 @@ fn test_cid(n: u32) -> Cid {
     Cid::new_v1(0x71, mh)
 }
 
-async fn compute_obsolete_full_walk<S: jacquard_repo::storage::BlockStore + Sync + Send + 'static>(
+async fn compute_obsolete_full_walk<
+    S: jacquard_repo::storage::BlockStore + Sync + Send + 'static,
+>(
     old: &Mst<S>,
     new: &Mst<S>,
 ) -> BTreeSet<Cid> {
@@ -34,9 +36,7 @@ async fn compute_obsolete_full_walk<S: jacquard_repo::storage::BlockStore + Sync
         .collect()
 }
 
-fn compute_obsolete_from_diff(
-    diff: &jacquard_repo::mst::diff::MstDiff,
-) -> BTreeSet<Cid> {
+fn compute_obsolete_from_diff(diff: &jacquard_repo::mst::diff::MstDiff) -> BTreeSet<Cid> {
     diff.removed_mst_blocks
         .iter()
         .copied()
@@ -74,12 +74,17 @@ async fn assert_equivalence(
     let diff_obsolete = compute_obsolete_from_diff(&diff);
 
     assert_eq!(
-        full_walk_obsolete, diff_obsolete,
+        full_walk_obsolete,
+        diff_obsolete,
         "MISMATCH in scenario: {scenario}\n  full_walk count: {}\n  diff count: {}\n  in full_walk but not diff: {:?}\n  in diff but not full_walk: {:?}",
         full_walk_obsolete.len(),
         diff_obsolete.len(),
-        full_walk_obsolete.difference(&diff_obsolete).collect::<Vec<_>>(),
-        diff_obsolete.difference(&full_walk_obsolete).collect::<Vec<_>>(),
+        full_walk_obsolete
+            .difference(&diff_obsolete)
+            .collect::<Vec<_>>(),
+        diff_obsolete
+            .difference(&full_walk_obsolete)
+            .collect::<Vec<_>>(),
     );
 }
 
@@ -256,7 +261,12 @@ async fn massive_to_empty() {
 async fn massive_complete_replacement() {
     let old = generate_records("app.bsky.feed.post", 0..1000);
     let new_rec = generate_records("app.bsky.feed.post", 1000..2000);
-    assert_equivalence(&old, &new_rec, "1000 records fully replaced with 1000 different").await;
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "1000 records fully replaced with 1000 different",
+    )
+    .await;
 }
 
 #[tokio::test]
@@ -276,7 +286,12 @@ async fn multi_collection_5_collections_500_each() {
     ];
     let old = generate_multi_collection_records(&collections, 500);
     let new_rec = apply_scattered_updates(&old, 4, 30000);
-    assert_equivalence(&old, &new_rec, "5 collections x 500 records - update every 4th").await;
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "5 collections x 500 records - update every 4th",
+    )
+    .await;
 }
 
 #[tokio::test]
@@ -294,7 +309,12 @@ async fn multi_collection_wipe_one_collection() {
         .filter(|(key, _)| !key.starts_with("app.bsky.feed.repost"))
         .cloned()
         .collect();
-    assert_equivalence(&old, &new_rec, "4 collections x 400 - wipe repost collection").await;
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "4 collections x 400 - wipe repost collection",
+    )
+    .await;
 }
 
 #[tokio::test]
@@ -313,10 +333,7 @@ async fn multi_collection_keep_only_one() {
 
 #[tokio::test]
 async fn multi_collection_add_new_collection() {
-    let old_collections = [
-        "app.bsky.feed.like",
-        "app.bsky.feed.post",
-    ];
+    let old_collections = ["app.bsky.feed.like", "app.bsky.feed.post"];
     let old = generate_multi_collection_records(&old_collections, 500);
     let new_rec = append_records(&old, "app.bsky.graph.follow", 0..500, 40000);
     assert_equivalence(&old, &new_rec, "2 collections x 500 + add 500 follows").await;
@@ -378,7 +395,12 @@ async fn interleaved_keys_disjoint_ranges() {
     let new_rec: Vec<_> = (0..1000u32)
         .map(|i| (make_key("app.bsky.feed.post", i * 2 + 1), i + 10000))
         .collect();
-    assert_equivalence(&old, &new_rec, "1000 even-keyed records replaced by 1000 odd-keyed").await;
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "1000 even-keyed records replaced by 1000 odd-keyed",
+    )
+    .await;
 }
 
 #[tokio::test]
@@ -426,7 +448,12 @@ async fn many_collections_few_records_each() {
         })
         .collect();
 
-    assert_equivalence(&old, &new_rec, "50 collections x 20 records - delete every 15th, update every 7th").await;
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "50 collections x 20 records - delete every 15th, update every 7th",
+    )
+    .await;
 }
 
 #[tokio::test]
@@ -457,7 +484,12 @@ async fn one_to_massive() {
 async fn delete_head_and_tail() {
     let old = generate_records("app.bsky.feed.post", 0..2000);
     let new_rec: Vec<_> = old[200..1800].to_vec();
-    assert_equivalence(&old, &new_rec, "2000 records - delete first 200 and last 200").await;
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "2000 records - delete first 200 and last 200",
+    )
+    .await;
 }
 
 #[tokio::test]
@@ -465,7 +497,12 @@ async fn keep_head_and_tail_only() {
     let old = generate_records("app.bsky.feed.post", 0..2000);
     let mut new_rec: Vec<_> = old[..100].to_vec();
     new_rec.extend_from_slice(&old[1900..]);
-    assert_equivalence(&old, &new_rec, "2000 records - keep only first 100 and last 100").await;
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "2000 records - keep only first 100 and last 100",
+    )
+    .await;
 }
 
 #[tokio::test]
@@ -515,7 +552,12 @@ async fn swiss_cheese_deletions() {
         })
         .map(|(_, r)| r.clone())
         .collect();
-    assert_equivalence(&old, &new_rec, "1500 records - delete every 3rd chunk of 50").await;
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "1500 records - delete every 3rd chunk of 50",
+    )
+    .await;
 }
 
 #[tokio::test]
@@ -529,9 +571,7 @@ async fn mixed_ops_with_key_density_change() {
         .filter(|(_, val)| val % 4 != 0)
         .cloned()
         .collect();
-    new_rec.extend((0..500u32).map(|i| {
-        (make_key("app.bsky.feed.post", i * 3 + 1), i + 100000)
-    }));
+    new_rec.extend((0..500u32).map(|i| (make_key("app.bsky.feed.post", i * 3 + 1), i + 100000)));
     new_rec.sort_by(|(a, _), (b, _)| a.cmp(b));
 
     assert_equivalence(

crates/tranquil-pds/tests/mst_firehose_e2e.rs

@@ -0,0 +1,475 @@
+mod common;
+mod firehose;
+
+use std::collections::BTreeMap;
+use std::io::Cursor;
+use std::str::FromStr;
+use std::sync::Arc;
+use std::time::Duration;
+
+use bytes::Bytes;
+use cid::Cid;
+use common::*;
+use firehose::{FirehoseConsumer, ParsedCommitFrame};
+use iroh_car::CarReader;
+use jacquard_common::smol_str::SmolStr;
+use jacquard_repo::commit::Commit;
+use jacquard_repo::mst::{Mst, VerifiedWriteOp};
+use jacquard_repo::storage::{BlockStore, MemoryBlockStore};
+use reqwest::StatusCode;
+use serde_json::{Value, json};
+use tranquil_scopes::RepoAction;
+
+async fn car_to_blocks(car_bytes: &[u8]) -> BTreeMap<Cid, Bytes> {
+    let mut reader = CarReader::new(Cursor::new(car_bytes))
+        .await
+        .expect("parse CAR header");
+    let mut blocks = BTreeMap::new();
+    while let Ok(Some((cid, data))) = reader.next_block().await {
+        blocks.insert(cid, Bytes::from(data));
+    }
+    blocks
+}
+
+fn op_to_verified(op: &firehose::ParsedRepoOp) -> Result<VerifiedWriteOp, String> {
+    let key = SmolStr::new(&op.path);
+    match op.action {
+        RepoAction::Create => {
+            let cid = op.cid.ok_or("create op missing cid")?;
+            Ok(VerifiedWriteOp::Create { key, cid })
+        }
+        RepoAction::Update => {
+            let cid = op.cid.ok_or("update op missing cid")?;
+            let prev = op.prev.ok_or("update op missing prev")?;
+            Ok(VerifiedWriteOp::Update { key, cid, prev })
+        }
+        RepoAction::Delete => {
+            let prev = op.prev.ok_or("delete op missing prev")?;
+            Ok(VerifiedWriteOp::Delete { key, prev })
+        }
+    }
+}
+
+async fn verify_frame_forward(frame: &ParsedCommitFrame) -> Result<(), String> {
+    let prev_data = frame
+        .prev_data
+        .ok_or_else(|| "frame missing prev_data (v1.1 required)".to_string())?;
+
+    let blocks = car_to_blocks(&frame.blocks).await;
+    let storage = Arc::new(MemoryBlockStore::new_from_blocks(blocks));
+
+    let commit_bytes = storage
+        .get(&frame.commit)
+        .await
+        .map_err(|e| format!("get commit: {e:?}"))?
+        .ok_or_else(|| format!("CAR missing commit {}", frame.commit))?;
+    let commit = Commit::from_cbor(&commit_bytes).map_err(|e| format!("parse commit: {e:?}"))?;
+    let expected = *commit.data();
+
+    let mut mst = Mst::load(storage, prev_data, None);
+    for op in &frame.ops {
+        let path = &op.path;
+        match op.action {
+            RepoAction::Create | RepoAction::Update => {
+                let cid = op.cid.ok_or_else(|| format!("{path}: op missing cid"))?;
+                mst = mst
+                    .add(path, cid)
+                    .await
+                    .map_err(|e| format!("forward {path}: {e:?}"))?;
+            }
+            RepoAction::Delete => {
+                mst = mst
+                    .delete(path)
+                    .await
+                    .map_err(|e| format!("forward delete {path}: {e:?}"))?;
+            }
+        }
+    }
+    let computed = mst.persist().await.map_err(|e| format!("persist: {e:?}"))?;
+    if computed != expected {
+        return Err(format!(
+            "root mismatch expected={expected} computed={computed}"
+        ));
+    }
+    Ok(())
+}
+
+async fn verify_frame_inverse(frame: &ParsedCommitFrame) -> Result<(), String> {
+    let prev_data = frame
+        .prev_data
+        .ok_or_else(|| "frame missing prev_data (v1.1 required)".to_string())?;
+
+    let blocks = car_to_blocks(&frame.blocks).await;
+    let storage = Arc::new(MemoryBlockStore::new_from_blocks(blocks));
+
+    let commit_bytes = storage
+        .get(&frame.commit)
+        .await
+        .map_err(|e| format!("get commit: {e:?}"))?
+        .ok_or_else(|| format!("CAR missing commit {}", frame.commit))?;
+    let commit = Commit::from_cbor(&commit_bytes).map_err(|e| format!("parse commit: {e:?}"))?;
+    let new_data = *commit.data();
+
+    let mut mst = Mst::load(storage, new_data, None);
+    for op in &frame.ops {
+        let verified = op_to_verified(op)?;
+        let inverted = mst
+            .invert_op(verified.clone())
+            .await
+            .map_err(|e| format!("invert {verified:?}: {e:?}"))?;
+        if !inverted {
+            return Err(format!("op not invertible: {verified:?}"));
+        }
+    }
+    let computed_prev = mst
+        .get_pointer()
+        .await
+        .map_err(|e| format!("get_pointer: {e:?}"))?;
+    if computed_prev != prev_data {
+        return Err(format!(
+            "inverse root mismatch expected={prev_data} computed={computed_prev}"
+        ));
+    }
+    Ok(())
+}
+
+async fn create_record(client: &reqwest::Client, token: &str, did: &str, rkey: &str, text: &str) {
+    let now = chrono::Utc::now().to_rfc3339();
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.createRecord",
+            base_url().await
+        ))
+        .bearer_auth(token)
+        .json(&json!({
+            "repo": did,
+            "collection": "app.bsky.feed.post",
+            "rkey": rkey,
+            "record": {
+                "$type": "app.bsky.feed.post",
+                "text": text,
+                "createdAt": now,
+            }
+        }))
+        .send()
+        .await
+        .expect("createRecord");
+    assert_eq!(res.status(), StatusCode::OK);
+}
+
+async fn put_record(client: &reqwest::Client, token: &str, did: &str, rkey: &str, text: &str) {
+    let now = chrono::Utc::now().to_rfc3339();
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.putRecord",
+            base_url().await
+        ))
+        .bearer_auth(token)
+        .json(&json!({
+            "repo": did,
+            "collection": "app.bsky.feed.post",
+            "rkey": rkey,
+            "record": {
+                "$type": "app.bsky.feed.post",
+                "text": text,
+                "createdAt": now,
+            }
+        }))
+        .send()
+        .await
+        .expect("putRecord");
+    assert_eq!(res.status(), StatusCode::OK);
+}
+
+async fn delete_record(client: &reqwest::Client, token: &str, did: &str, rkey: &str) {
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.deleteRecord",
+            base_url().await
+        ))
+        .bearer_auth(token)
+        .json(&json!({
+            "repo": did,
+            "collection": "app.bsky.feed.post",
+            "rkey": rkey,
+        }))
+        .send()
+        .await
+        .expect("deleteRecord");
+    assert_eq!(res.status(), StatusCode::OK);
+}
+
+async fn apply_writes_batch(client: &reqwest::Client, token: &str, did: &str, writes: Vec<Value>) {
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.applyWrites",
+            base_url().await
+        ))
+        .bearer_auth(token)
+        .json(&json!({ "repo": did, "writes": writes }))
+        .send()
+        .await
+        .expect("applyWrites");
+    assert_eq!(res.status(), StatusCode::OK);
+}
+
+fn rkey_for(i: usize) -> String {
+    format!("3ke2e{:08}", i)
+}
+
+#[tokio::test]
+async fn websocket_firehose_frames_pass_inductive_forward_and_inverse() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let repos = get_test_repos().await;
+    let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
+    let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    let now = chrono::Utc::now().to_rfc3339();
+    let seed: Vec<Value> = (0..120)
+        .map(|i| {
+            json!({
+                "$type": "com.atproto.repo.applyWrites#create",
+                "collection": "app.bsky.feed.post",
+                "rkey": rkey_for(i),
+                "value": {
+                    "$type": "app.bsky.feed.post",
+                    "text": format!("e2e {i}"),
+                    "createdAt": now,
+                }
+            })
+        })
+        .collect();
+    for chunk in seed.chunks(40) {
+        apply_writes_batch(&client, &token, &did, chunk.to_vec()).await;
+    }
+    for i in (0..120).step_by(6) {
+        put_record(&client, &token, &did, &rkey_for(i), &format!("upd {i}")).await;
+    }
+    for i in (2..120).step_by(11) {
+        delete_record(&client, &token, &did, &rkey_for(i)).await;
+    }
+    create_record(&client, &token, &did, "3ke2efinal001", "final").await;
+
+    let target_commits = 3 + 20 + 11 + 1;
+    let frames = consumer
+        .wait_for_commits(&did, target_commits, Duration::from_secs(90))
+        .await;
+    assert!(
+        frames.len() >= target_commits,
+        "expected {} commit frames, got {}",
+        target_commits,
+        frames.len()
+    );
+
+    let mut forward_failures = Vec::new();
+    let mut inverse_failures = Vec::new();
+    for frame in &frames {
+        if frame.prev_data.is_none() {
+            continue;
+        }
+        if frame.ops.is_empty() {
+            continue;
+        }
+        if let Err(msg) = verify_frame_forward(frame).await {
+            forward_failures.push(format!("seq={}: {msg}", frame.seq));
+        }
+        if let Err(msg) = verify_frame_inverse(frame).await {
+            inverse_failures.push(format!("seq={}: {msg}", frame.seq));
+        }
+    }
+    assert!(
+        forward_failures.is_empty(),
+        "forward verification failures:\n  - {}",
+        forward_failures.join("\n  - ")
+    );
+    assert!(
+        inverse_failures.is_empty(),
+        "inverse verification failures:\n  - {}",
+        inverse_failures.join("\n  - ")
+    );
+}
+
+#[tokio::test]
+async fn websocket_firehose_car_root_matches_commit_cid() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let repos = get_test_repos().await;
+    let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
+    let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    for i in 0..4 {
+        create_record(&client, &token, &did, &rkey_for(i), "ck").await;
+    }
+
+    let frames = consumer
+        .wait_for_commits(&did, 4, Duration::from_secs(10))
+        .await;
+
+    for frame in &frames {
+        let mut reader = CarReader::new(Cursor::new(&frame.blocks))
+            .await
+            .expect("CAR header");
+        let roots = reader.header().roots();
+        assert_eq!(roots.len(), 1, "CAR must have exactly one root");
+        assert_eq!(
+            roots[0], frame.commit,
+            "CAR root must equal frame commit CID"
+        );
+        let mut found = false;
+        while let Ok(Some((cid, _))) = reader.next_block().await {
+            if cid == frame.commit {
+                found = true;
+            }
+        }
+        assert!(found, "CAR body must contain commit block");
+    }
+}
+
+#[tokio::test]
+async fn websocket_firehose_resumption_from_cursor_yields_valid_frames() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+    let repos = get_test_repos().await;
+
+    for i in 0..5 {
+        create_record(&client, &token, &did, &rkey_for(i), "pre").await;
+    }
+
+    let resume_cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
+
+    for i in 5..12 {
+        create_record(&client, &token, &did, &rkey_for(i), "post").await;
+    }
+
+    let consumer = FirehoseConsumer::connect_with_cursor(app_port(), resume_cursor).await;
+    let frames = consumer
+        .wait_for_commits(&did, 7, Duration::from_secs(20))
+        .await;
+    assert!(
+        frames.len() >= 7,
+        "expected 7+ frames after cursor resume, got {}",
+        frames.len()
+    );
+
+    for frame in &frames {
+        if frame.prev_data.is_none() || frame.ops.is_empty() {
+            continue;
+        }
+        verify_frame_forward(frame)
+            .await
+            .unwrap_or_else(|e| panic!("resumed frame seq={} invalid: {e}", frame.seq));
+    }
+}
+
+#[tokio::test]
+async fn websocket_firehose_ops_include_prev_field_for_update_delete() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+    let repos = get_test_repos().await;
+    let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
+    let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    create_record(&client, &token, &did, "3ke2eprev01", "v1").await;
+    put_record(&client, &token, &did, "3ke2eprev01", "v2").await;
+    delete_record(&client, &token, &did, "3ke2eprev01").await;
+
+    let frames = consumer
+        .wait_for_commits(&did, 3, Duration::from_secs(10))
+        .await;
+    assert!(frames.len() >= 3);
+
+    for frame in &frames {
+        for op in &frame.ops {
+            match op.action {
+                RepoAction::Create => {
+                    assert!(op.cid.is_some(), "create must have cid");
+                    assert!(op.prev.is_none(), "create must not have prev");
+                }
+                RepoAction::Update => {
+                    assert!(op.cid.is_some(), "update must have cid");
+                    assert!(
+                        op.prev.is_some(),
+                        "v1.1 update must carry prev CID (seq={})",
+                        frame.seq
+                    );
+                }
+                RepoAction::Delete => {
+                    assert!(op.cid.is_none(), "delete must have null cid");
+                    assert!(
+                        op.prev.is_some(),
+                        "v1.1 delete must carry prev CID (seq={})",
+                        frame.seq
+                    );
+                }
+            }
+        }
+    }
+}
+
+#[tokio::test]
+async fn websocket_firehose_rebuild_new_mst_from_car_matches_commit_data() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+    let repos = get_test_repos().await;
+    let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
+    let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    let now = chrono::Utc::now().to_rfc3339();
+    let writes: Vec<Value> = (0..30)
+        .map(|i| {
+            json!({
+                "$type": "com.atproto.repo.applyWrites#create",
+                "collection": "app.bsky.feed.post",
+                "rkey": rkey_for(i),
+                "value": {
+                    "$type": "app.bsky.feed.post",
+                    "text": format!("rb {i}"),
+                    "createdAt": now,
+                }
+            })
+        })
+        .collect();
+    apply_writes_batch(&client, &token, &did, writes).await;
+
+    let frames = consumer
+        .wait_for_commits(&did, 1, Duration::from_secs(10))
+        .await;
+    let last = frames.last().expect("frame");
+
+    let blocks = car_to_blocks(&last.blocks).await;
+    let storage = Arc::new(MemoryBlockStore::new_from_blocks(blocks));
+    let commit_bytes = storage
+        .get(&last.commit)
+        .await
+        .unwrap()
+        .expect("commit block");
+    let commit = Commit::from_cbor(&commit_bytes).unwrap();
+
+    let new_root_cid = *commit.data();
+    let mst = Mst::load(storage, new_root_cid, None);
+    let rehydrated_cid = mst.get_pointer().await.expect("rebuild mst");
+    assert_eq!(
+        rehydrated_cid, new_root_cid,
+        "MST loaded from CAR must yield same root as commit.data()"
+    );
+
+    for op in &last.ops {
+        if op.action == RepoAction::Create {
+            let expected_cid = op.cid.unwrap();
+            let got = mst
+                .get(&op.path)
+                .await
+                .expect("mst.get")
+                .unwrap_or_else(|| panic!("key {} missing from rebuilt tree", op.path));
+            assert_eq!(got, expected_cid, "record CID mismatch for {}", op.path);
+            let _ = Cid::from_str(&expected_cid.to_string()).unwrap();
+        }
+    }
+}

crates/tranquil-pds/tests/mst_inductive_firehose.rs

@@ -0,0 +1,697 @@
+mod common;
+mod mst_verify;
+
+use std::collections::BTreeMap;
+use std::str::FromStr;
+use std::sync::Arc;
+
+use cid::Cid;
+use common::*;
+use jacquard_common::smol_str::SmolStr;
+use jacquard_repo::commit::Commit;
+use jacquard_repo::mst::{Mst, VerifiedWriteOp};
+use jacquard_repo::storage::{BlockStore, MemoryBlockStore};
+use mst_verify::{extract_event_blocks, inline_to_store};
+use reqwest::StatusCode;
+use serde_json::{Value, json};
+use tranquil_db_traits::{RepoEventType, SequenceNumber, SequencedEvent};
+use tranquil_types::Did;
+
+async fn new_commit_data_cid(
+    storage: &Arc<MemoryBlockStore>,
+    commit_cid: &Cid,
+) -> Result<Cid, String> {
+    let commit_bytes = storage
+        .get(commit_cid)
+        .await
+        .map_err(|e| format!("get commit: {e:?}"))?
+        .ok_or_else(|| format!("CAR missing commit block {commit_cid}"))?;
+    let commit = Commit::from_cbor(&commit_bytes).map_err(|e| format!("parse commit: {e:?}"))?;
+    Ok(*commit.data())
+}
+
+fn ops_json(event: &SequencedEvent) -> Result<&Vec<Value>, String> {
+    event
+        .ops
+        .as_ref()
+        .and_then(|v| v.as_array())
+        .ok_or_else(|| "event.ops not an array".into())
+}
+
+fn parse_op_to_verified(op: &Value) -> Result<VerifiedWriteOp, String> {
+    let action = op["action"].as_str().ok_or("op.action missing")?;
+    let path = op["path"].as_str().ok_or("op.path missing")?;
+    let key = SmolStr::new(path);
+    match action {
+        "create" => {
+            let cid_str = op["cid"].as_str().ok_or("create missing cid")?;
+            let cid = Cid::from_str(cid_str).map_err(|e| format!("parse cid: {e:?}"))?;
+            Ok(VerifiedWriteOp::Create { key, cid })
+        }
+        "update" => {
+            let cid_str = op["cid"].as_str().ok_or("update missing cid")?;
+            let cid = Cid::from_str(cid_str).map_err(|e| format!("parse cid: {e:?}"))?;
+            let prev_str = op["prev"].as_str().ok_or("update missing prev")?;
+            let prev = Cid::from_str(prev_str).map_err(|e| format!("parse prev: {e:?}"))?;
+            Ok(VerifiedWriteOp::Update { key, cid, prev })
+        }
+        "delete" => {
+            let prev_str = op["prev"].as_str().ok_or("delete missing prev")?;
+            let prev = Cid::from_str(prev_str).map_err(|e| format!("parse prev: {e:?}"))?;
+            Ok(VerifiedWriteOp::Delete { key, prev })
+        }
+        other => Err(format!("unknown op action: {other}")),
+    }
+}
+
+async fn verify_inductive_forward(event: &SequencedEvent) -> Result<(Cid, Cid), String> {
+    let prev_data_cid = event
+        .prev_data_cid
+        .as_ref()
+        .and_then(|c| c.to_cid())
+        .ok_or_else(|| "event missing prev_data_cid".to_string())?;
+    let commit_cid = event
+        .commit_cid
+        .as_ref()
+        .and_then(|c| c.to_cid())
+        .ok_or_else(|| "event missing commit_cid".to_string())?;
+
+    let storage = inline_to_store(extract_event_blocks(event)?);
+    let expected_new_data = new_commit_data_cid(&storage, &commit_cid).await?;
+
+    let mut mst = Mst::load(storage.clone(), prev_data_cid, None);
+    for op_value in ops_json(event)? {
+        let action = op_value["action"].as_str().ok_or("op.action missing")?;
+        let path = op_value["path"].as_str().ok_or("op.path missing")?;
+        match action {
+            "create" | "update" => {
+                let cid = Cid::from_str(op_value["cid"].as_str().ok_or("op.cid missing")?)
+                    .map_err(|e| format!("parse op.cid: {e:?}"))?;
+                mst = mst
+                    .add(path, cid)
+                    .await
+                    .map_err(|e| format!("mst.add({path}): {e:?}"))?;
+            }
+            "delete" => {
+                mst = mst
+                    .delete(path)
+                    .await
+                    .map_err(|e| format!("mst.delete({path}): {e:?}"))?;
+            }
+            other => return Err(format!("unknown op action: {other}")),
+        }
+    }
+    let computed = mst
+        .persist()
+        .await
+        .map_err(|e| format!("mst.persist: {e:?}"))?;
+    Ok((expected_new_data, computed))
+}
+
+async fn verify_inductive_inverse(event: &SequencedEvent) -> Result<(Cid, Cid), String> {
+    let prev_data_cid = event
+        .prev_data_cid
+        .as_ref()
+        .and_then(|c| c.to_cid())
+        .ok_or_else(|| "event missing prev_data_cid".to_string())?;
+    let commit_cid = event
+        .commit_cid
+        .as_ref()
+        .and_then(|c| c.to_cid())
+        .ok_or_else(|| "event missing commit_cid".to_string())?;
+
+    let storage = inline_to_store(extract_event_blocks(event)?);
+    let new_data_cid = new_commit_data_cid(&storage, &commit_cid).await?;
+
+    let mut mst = Mst::load(storage.clone(), new_data_cid, None);
+    for op_value in ops_json(event)?.iter().rev() {
+        let verified = parse_op_to_verified(op_value)?;
+        let inverted = mst
+            .invert_op(verified.clone())
+            .await
+            .map_err(|e| format!("invert_op({verified:?}): {e:?}"))?;
+        if !inverted {
+            return Err(format!("op not invertible: {verified:?}"));
+        }
+    }
+    let computed_prev = mst
+        .get_pointer()
+        .await
+        .map_err(|e| format!("get_pointer: {e:?}"))?;
+    Ok((prev_data_cid, computed_prev))
+}
+
+fn report_failures(total: usize, failures: &[String], mode: &str) {
+    assert!(
+        failures.is_empty(),
+        "{} of {total} {mode} commit events failed inductive verification:\n  - {}",
+        failures.len(),
+        failures.join("\n  - "),
+    );
+}
+
+async fn apply_writes_batch(client: &reqwest::Client, token: &str, did: &str, writes: Vec<Value>) {
+    let payload = json!({ "repo": did, "writes": writes });
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.applyWrites",
+            base_url().await
+        ))
+        .bearer_auth(token)
+        .json(&payload)
+        .send()
+        .await
+        .expect("applyWrites request failed");
+    assert_eq!(
+        res.status(),
+        StatusCode::OK,
+        "applyWrites failed: {:?}",
+        res.text().await
+    );
+}
+
+async fn create_record(client: &reqwest::Client, token: &str, did: &str, col: &str, rkey: &str) {
+    let now = chrono::Utc::now().to_rfc3339();
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.createRecord",
+            base_url().await
+        ))
+        .bearer_auth(token)
+        .json(&json!({
+            "repo": did,
+            "collection": col,
+            "rkey": rkey,
+            "record": {
+                "$type": col,
+                "text": format!("post {rkey}"),
+                "createdAt": now,
+            }
+        }))
+        .send()
+        .await
+        .expect("createRecord request failed");
+    assert_eq!(res.status(), StatusCode::OK, "createRecord failed");
+}
+
+async fn put_record(
+    client: &reqwest::Client,
+    token: &str,
+    did: &str,
+    col: &str,
+    rkey: &str,
+    text: &str,
+) {
+    let now = chrono::Utc::now().to_rfc3339();
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.putRecord",
+            base_url().await
+        ))
+        .bearer_auth(token)
+        .json(&json!({
+            "repo": did,
+            "collection": col,
+            "rkey": rkey,
+            "record": {
+                "$type": col,
+                "text": text,
+                "createdAt": now,
+            }
+        }))
+        .send()
+        .await
+        .expect("putRecord request failed");
+    assert_eq!(res.status(), StatusCode::OK, "putRecord failed");
+}
+
+async fn delete_record(client: &reqwest::Client, token: &str, did: &str, col: &str, rkey: &str) {
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.deleteRecord",
+            base_url().await
+        ))
+        .bearer_auth(token)
+        .json(&json!({ "repo": did, "collection": col, "rkey": rkey }))
+        .send()
+        .await
+        .expect("deleteRecord request failed");
+    assert_eq!(res.status(), StatusCode::OK, "deleteRecord failed");
+}
+
+const COLLECTION: &str = "app.bsky.feed.post";
+fn rkey_for(prefix: &str, i: usize) -> String {
+    format!("3k{prefix}{:08}", i)
+}
+
+async fn our_commit_events(did: &str) -> Vec<SequencedEvent> {
+    let repos = get_test_repos().await;
+    let typed_did = Did::new(did.to_string()).unwrap();
+    let events = repos
+        .repo
+        .get_events_since_seq(SequenceNumber::ZERO, None)
+        .await
+        .expect("get_events_since_seq");
+    events
+        .into_iter()
+        .filter(|e| e.did == typed_did && e.event_type == RepoEventType::Commit)
+        .collect()
+}
+
+#[tokio::test]
+async fn inductive_forward_verifies_delete_commits() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let now = chrono::Utc::now().to_rfc3339();
+    const N_CREATE: usize = 200;
+
+    let all_writes: Vec<Value> = (0..N_CREATE)
+        .map(|i| {
+            json!({
+                "$type": "com.atproto.repo.applyWrites#create",
+                "collection": COLLECTION,
+                "rkey": rkey_for("del", i),
+                "value": {
+                    "$type": COLLECTION,
+                    "text": format!("record {i}"),
+                    "createdAt": now,
+                }
+            })
+        })
+        .collect();
+    for chunk in all_writes.chunks(50) {
+        apply_writes_batch(&client, &token, &did, chunk.to_vec()).await;
+    }
+
+    let delete_indices: Vec<usize> = (10..N_CREATE).step_by(7).collect();
+    for i in &delete_indices {
+        delete_record(&client, &token, &did, COLLECTION, &rkey_for("del", *i)).await;
+    }
+
+    let our = our_commit_events(&did).await;
+    let delete_events: Vec<&SequencedEvent> = our
+        .iter()
+        .filter(|e| {
+            ops_json(e)
+                .map(|arr| arr.iter().any(|op| op["action"].as_str() == Some("delete")))
+                .unwrap_or(false)
+        })
+        .collect();
+    assert_eq!(delete_events.len(), delete_indices.len());
+
+    let mut failures = Vec::new();
+    for e in &delete_events {
+        match verify_inductive_forward(e).await {
+            Ok((exp, got)) if exp == got => {}
+            Ok((exp, got)) => failures.push(format!(
+                "seq={}: root mismatch exp={exp} got={got}",
+                e.seq.as_i64()
+            )),
+            Err(msg) => failures.push(format!("seq={}: {msg}", e.seq.as_i64())),
+        }
+    }
+    report_failures(delete_events.len(), &failures, "delete forward");
+}
+
+#[tokio::test]
+async fn inductive_forward_verifies_create_commits() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+    for i in 0..60usize {
+        create_record(&client, &token, &did, COLLECTION, &rkey_for("cre", i)).await;
+    }
+
+    let our = our_commit_events(&did).await;
+    let create_events: Vec<&SequencedEvent> = our
+        .iter()
+        .filter(|e| {
+            ops_json(e)
+                .map(|arr| arr.iter().all(|op| op["action"].as_str() == Some("create")))
+                .unwrap_or(false)
+                && e.prev_data_cid.is_some()
+        })
+        .collect();
+    assert!(!create_events.is_empty());
+
+    let mut failures = Vec::new();
+    for e in &create_events {
+        match verify_inductive_forward(e).await {
+            Ok((exp, got)) if exp == got => {}
+            Ok((exp, got)) => failures.push(format!(
+                "seq={}: root mismatch exp={exp} got={got}",
+                e.seq.as_i64()
+            )),
+            Err(msg) => failures.push(format!("seq={}: {msg}", e.seq.as_i64())),
+        }
+    }
+    report_failures(create_events.len(), &failures, "create forward");
+}
+
+#[tokio::test]
+async fn inductive_forward_verifies_update_commits() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let now = chrono::Utc::now().to_rfc3339();
+    let creates: Vec<Value> = (0..80)
+        .map(|i| {
+            json!({
+                "$type": "com.atproto.repo.applyWrites#create",
+                "collection": COLLECTION,
+                "rkey": rkey_for("upd", i),
+                "value": {
+                    "$type": COLLECTION,
+                    "text": format!("original {i}"),
+                    "createdAt": now,
+                }
+            })
+        })
+        .collect();
+    for chunk in creates.chunks(40) {
+        apply_writes_batch(&client, &token, &did, chunk.to_vec()).await;
+    }
+
+    for i in (0..80).step_by(3) {
+        put_record(
+            &client,
+            &token,
+            &did,
+            COLLECTION,
+            &rkey_for("upd", i),
+            &format!("updated {i}"),
+        )
+        .await;
+    }
+
+    let our = our_commit_events(&did).await;
+    let update_events: Vec<&SequencedEvent> = our
+        .iter()
+        .filter(|e| {
+            ops_json(e)
+                .map(|arr| arr.iter().any(|op| op["action"].as_str() == Some("update")))
+                .unwrap_or(false)
+        })
+        .collect();
+    assert!(!update_events.is_empty());
+
+    let mut failures = Vec::new();
+    for e in &update_events {
+        match verify_inductive_forward(e).await {
+            Ok((exp, got)) if exp == got => {}
+            Ok((exp, got)) => failures.push(format!(
+                "seq={}: root mismatch exp={exp} got={got}",
+                e.seq.as_i64()
+            )),
+            Err(msg) => failures.push(format!("seq={}: {msg}", e.seq.as_i64())),
+        }
+    }
+    report_failures(update_events.len(), &failures, "update forward");
+}
+
+#[tokio::test]
+async fn inductive_forward_verifies_mixed_applywrites() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let now = chrono::Utc::now().to_rfc3339();
+    let seed: Vec<Value> = (0..120)
+        .map(|i| {
+            json!({
+                "$type": "com.atproto.repo.applyWrites#create",
+                "collection": COLLECTION,
+                "rkey": rkey_for("mix", i),
+                "value": {
+                    "$type": COLLECTION,
+                    "text": format!("seed {i}"),
+                    "createdAt": now,
+                }
+            })
+        })
+        .collect();
+    for chunk in seed.chunks(40) {
+        apply_writes_batch(&client, &token, &did, chunk.to_vec()).await;
+    }
+
+    let mixed: Vec<Value> = (0..40)
+        .flat_map(|i| {
+            vec![
+                json!({
+                    "$type": "com.atproto.repo.applyWrites#create",
+                    "collection": COLLECTION,
+                    "rkey": rkey_for("mxc", i),
+                    "value": {
+                        "$type": COLLECTION,
+                        "text": format!("new {i}"),
+                        "createdAt": now,
+                    }
+                }),
+                json!({
+                    "$type": "com.atproto.repo.applyWrites#update",
+                    "collection": COLLECTION,
+                    "rkey": rkey_for("mix", i),
+                    "value": {
+                        "$type": COLLECTION,
+                        "text": format!("updated-mix {i}"),
+                        "createdAt": now,
+                    }
+                }),
+                json!({
+                    "$type": "com.atproto.repo.applyWrites#delete",
+                    "collection": COLLECTION,
+                    "rkey": rkey_for("mix", i + 60),
+                }),
+            ]
+        })
+        .collect();
+    apply_writes_batch(&client, &token, &did, mixed).await;
+
+    let our = our_commit_events(&did).await;
+    let last = our
+        .iter()
+        .rfind(|e| e.prev_data_cid.is_some())
+        .expect("at least one non-genesis commit");
+
+    let actions: Vec<&str> = ops_json(last)
+        .unwrap()
+        .iter()
+        .filter_map(|op| op["action"].as_str())
+        .collect();
+    assert!(actions.contains(&"create"));
+    assert!(actions.contains(&"update"));
+    assert!(actions.contains(&"delete"));
+
+    let (exp, got) = verify_inductive_forward(last)
+        .await
+        .expect("mixed applyWrites forward verify");
+    assert_eq!(exp, got, "mixed applyWrites commit forward-verify mismatch");
+}
+
+#[tokio::test]
+async fn inductive_inverse_verifies_every_commit() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let now = chrono::Utc::now().to_rfc3339();
+    let seed: Vec<Value> = (0..100)
+        .map(|i| {
+            json!({
+                "$type": "com.atproto.repo.applyWrites#create",
+                "collection": COLLECTION,
+                "rkey": rkey_for("inv", i),
+                "value": {
+                    "$type": COLLECTION,
+                    "text": format!("seed {i}"),
+                    "createdAt": now,
+                }
+            })
+        })
+        .collect();
+    for chunk in seed.chunks(50) {
+        apply_writes_batch(&client, &token, &did, chunk.to_vec()).await;
+    }
+    for i in (0..100).step_by(5) {
+        put_record(
+            &client,
+            &token,
+            &did,
+            COLLECTION,
+            &rkey_for("inv", i),
+            &format!("upd {i}"),
+        )
+        .await;
+    }
+    for i in (2..100).step_by(11) {
+        delete_record(&client, &token, &did, COLLECTION, &rkey_for("inv", i)).await;
+    }
+
+    let our = our_commit_events(&did).await;
+    let non_genesis: Vec<&SequencedEvent> = our
+        .iter()
+        .filter(|e| e.prev_data_cid.is_some() && ops_json(e).is_ok())
+        .collect();
+    assert!(!non_genesis.is_empty());
+
+    let mut failures = Vec::new();
+    for e in &non_genesis {
+        match verify_inductive_inverse(e).await {
+            Ok((exp, got)) if exp == got => {}
+            Ok((exp, got)) => failures.push(format!(
+                "seq={}: inverse root mismatch exp={exp} got={got}",
+                e.seq.as_i64()
+            )),
+            Err(msg) => failures.push(format!("seq={}: {msg}", e.seq.as_i64())),
+        }
+    }
+    report_failures(non_genesis.len(), &failures, "any inverse");
+}
+
+#[tokio::test]
+async fn inductive_inverse_handles_same_rkey_in_batch() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let now = chrono::Utc::now().to_rfc3339();
+    let rkey = rkey_for("dup", 0);
+    create_record(&client, &token, &did, COLLECTION, &rkey).await;
+
+    let writes = vec![
+        json!({
+            "$type": "com.atproto.repo.applyWrites#update",
+            "collection": COLLECTION,
+            "rkey": rkey,
+            "value": {
+                "$type": COLLECTION,
+                "text": "v1",
+                "createdAt": now,
+            }
+        }),
+        json!({
+            "$type": "com.atproto.repo.applyWrites#update",
+            "collection": COLLECTION,
+            "rkey": rkey,
+            "value": {
+                "$type": COLLECTION,
+                "text": "v2",
+                "createdAt": now,
+            }
+        }),
+    ];
+    apply_writes_batch(&client, &token, &did, writes).await;
+
+    let our = our_commit_events(&did).await;
+    let dup_event = our
+        .iter()
+        .find(|e| {
+            ops_json(e)
+                .map(|arr| {
+                    arr.iter()
+                        .filter(|op| op["action"].as_str() == Some("update"))
+                        .count()
+                        == 2
+                })
+                .unwrap_or(false)
+        })
+        .expect("commit event with two same-rkey updates");
+
+    let (exp, got) = verify_inductive_inverse(dup_event)
+        .await
+        .expect("inverse verify should succeed for same-rkey batch");
+    assert_eq!(
+        exp, got,
+        "inverse root mismatch for same-rkey batch: exp={exp} got={got}"
+    );
+}
+
+#[tokio::test]
+async fn prev_cid_chain_walks_to_genesis() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+    for i in 0..8 {
+        create_record(&client, &token, &did, COLLECTION, &rkey_for("cha", i)).await;
+    }
+
+    let our = our_commit_events(&did).await;
+    assert!(our.len() >= 2);
+
+    let last = our.last().unwrap();
+    let mut current_prev: Option<Cid> = last.prev_cid.as_ref().and_then(|c| c.to_cid());
+    let head_commit_cid = last
+        .commit_cid
+        .as_ref()
+        .and_then(|c| c.to_cid())
+        .expect("head commit_cid");
+
+    let by_commit: BTreeMap<Cid, &SequencedEvent> = our
+        .iter()
+        .filter_map(|e| {
+            e.commit_cid
+                .as_ref()
+                .and_then(|c| c.to_cid())
+                .map(|c| (c, e))
+        })
+        .collect();
+
+    let mut visited = 1;
+    while let Some(prev) = current_prev {
+        let e = by_commit
+            .get(&prev)
+            .unwrap_or_else(|| panic!("prev commit {prev} missing from event list"));
+        visited += 1;
+        current_prev = e.prev_cid.as_ref().and_then(|c| c.to_cid());
+    }
+    assert!(
+        visited >= 2,
+        "chain too short: visited={visited}, head_commit={head_commit_cid}"
+    );
+    assert_eq!(
+        visited,
+        our.len(),
+        "chain did not reach genesis: walked {visited}, have {}",
+        our.len()
+    );
+}
+
+#[tokio::test]
+async fn record_bytes_present_in_car_for_creates() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let now = chrono::Utc::now().to_rfc3339();
+    let writes: Vec<Value> = (0..5)
+        .map(|i| {
+            json!({
+                "$type": "com.atproto.repo.applyWrites#create",
+                "collection": COLLECTION,
+                "rkey": rkey_for("rec", i),
+                "value": {
+                    "$type": COLLECTION,
+                    "text": format!("rec {i}"),
+                    "createdAt": now,
+                }
+            })
+        })
+        .collect();
+    apply_writes_batch(&client, &token, &did, writes).await;
+
+    let our = our_commit_events(&did).await;
+    let latest = our.iter().rfind(|e| e.prev_data_cid.is_some()).unwrap();
+
+    let inline = extract_event_blocks(latest).unwrap();
+    let have_cids: std::collections::HashSet<Cid> = inline
+        .iter()
+        .map(|b| Cid::read_bytes(b.cid_bytes.as_slice()).unwrap())
+        .collect();
+
+    for op in ops_json(latest).unwrap() {
+        if op["action"].as_str() == Some("create")
+            && let Some(cid_str) = op["cid"].as_str()
+        {
+            let cid = Cid::from_str(cid_str).unwrap();
+            assert!(
+                have_cids.contains(&cid),
+                "create op record CID {cid} not present in CAR inline blocks"
+            );
+        }
+    }
+}

crates/tranquil-pds/tests/mst_integrity.rs

@@ -0,0 +1,334 @@
+mod common;
+mod firehose;
+mod helpers;
+
+use std::collections::BTreeMap;
+use std::io::Cursor;
+use std::sync::Arc;
+use std::time::Duration;
+
+use bytes::Bytes;
+use cid::Cid;
+use common::*;
+use firehose::FirehoseConsumer;
+use helpers::build_car_with_signature;
+use iroh_car::CarReader;
+use jacquard_repo::commit::Commit;
+use jacquard_repo::mst::Mst;
+use jacquard_repo::storage::{BlockStore, MemoryBlockStore};
+use k256::ecdsa::SigningKey;
+use reqwest::StatusCode;
+use serde_json::{Value, json};
+use tranquil_db_traits::{EventBlocks, RepoEventType, SequenceNumber, SequencedEvent};
+use tranquil_scopes::RepoAction;
+use tranquil_types::Did;
+
+async fn car_to_blocks(car_bytes: &[u8]) -> (Vec<Cid>, BTreeMap<Cid, Bytes>) {
+    let mut reader = CarReader::new(Cursor::new(car_bytes))
+        .await
+        .expect("parse CAR");
+    let roots = reader.header().roots().to_vec();
+    let mut blocks = BTreeMap::new();
+    while let Ok(Some((cid, data))) = reader.next_block().await {
+        blocks.insert(cid, Bytes::from(data));
+    }
+    (roots, blocks)
+}
+
+async fn create_post(client: &reqwest::Client, token: &str, did: &str, rkey: &str, text: &str) {
+    let now = chrono::Utc::now().to_rfc3339();
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.createRecord",
+            base_url().await
+        ))
+        .bearer_auth(token)
+        .json(&json!({
+            "repo": did,
+            "collection": "app.bsky.feed.post",
+            "rkey": rkey,
+            "record": {
+                "$type": "app.bsky.feed.post",
+                "text": text,
+                "createdAt": now,
+            }
+        }))
+        .send()
+        .await
+        .expect("createRecord");
+    assert_eq!(res.status(), StatusCode::OK);
+}
+
+#[tokio::test]
+async fn getrepo_car_roundtrips_mst_structure_and_records() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let expected_records: Vec<(String, String)> = (0..20)
+        .map(|i| {
+            let rkey = format!("3krtp{:08}", i);
+            let text = format!("roundtrip record {i}");
+            (rkey, text)
+        })
+        .collect();
+    for (rkey, text) in &expected_records {
+        create_post(&client, &token, &did, rkey, text).await;
+    }
+
+    let res = client
+        .get(format!(
+            "{}/xrpc/com.atproto.sync.getRepo",
+            base_url().await
+        ))
+        .query(&[("did", did.as_str())])
+        .send()
+        .await
+        .expect("getRepo");
+    assert_eq!(res.status(), StatusCode::OK);
+    let car_bytes = res.bytes().await.unwrap();
+
+    let (roots, block_map) = car_to_blocks(&car_bytes).await;
+    assert_eq!(roots.len(), 1, "CAR must have exactly one root");
+    let commit_cid = roots[0];
+    let storage = Arc::new(MemoryBlockStore::new_from_blocks(block_map));
+
+    let commit_bytes = storage
+        .get(&commit_cid)
+        .await
+        .unwrap()
+        .expect("CAR contains commit block");
+    let commit = Commit::from_cbor(&commit_bytes).expect("parse commit");
+    let data_cid = *commit.data();
+
+    let mst = Mst::load(storage.clone(), data_cid, None);
+    let loaded_root = mst.get_pointer().await.expect("load root");
+    assert_eq!(loaded_root, data_cid, "loaded MST pointer == commit.data()");
+
+    for (rkey, _) in &expected_records {
+        let path = format!("app.bsky.feed.post/{rkey}");
+        let leaf = mst
+            .get(&path)
+            .await
+            .expect("mst.get")
+            .unwrap_or_else(|| panic!("record {path} missing from exported MST"));
+        let leaf_bytes = storage
+            .get(&leaf)
+            .await
+            .unwrap()
+            .unwrap_or_else(|| panic!("record block {leaf} missing from CAR"));
+        assert!(!leaf_bytes.is_empty(), "record bytes empty");
+    }
+}
+
+#[tokio::test]
+async fn concurrent_swap_commit_writes_serialize() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    create_post(&client, &token, &did, "3kswap00000001", "anchor").await;
+
+    let latest_res = client
+        .get(format!(
+            "{}/xrpc/com.atproto.sync.getLatestCommit",
+            base_url().await
+        ))
+        .query(&[("did", did.as_str())])
+        .send()
+        .await
+        .expect("getLatestCommit");
+    assert_eq!(latest_res.status(), StatusCode::OK);
+    let latest: Value = latest_res.json().await.unwrap();
+    let swap_cid = latest["cid"].as_str().unwrap().to_string();
+
+    let now = chrono::Utc::now().to_rfc3339();
+    let payload_a = json!({
+        "repo": did,
+        "collection": "app.bsky.feed.post",
+        "rkey": "3kswap00000002",
+        "record": {
+            "$type": "app.bsky.feed.post",
+            "text": "writer A",
+            "createdAt": now,
+        },
+        "swapCommit": swap_cid,
+    });
+    let payload_b = json!({
+        "repo": did,
+        "collection": "app.bsky.feed.post",
+        "rkey": "3kswap00000003",
+        "record": {
+            "$type": "app.bsky.feed.post",
+            "text": "writer B",
+            "createdAt": now,
+        },
+        "swapCommit": swap_cid,
+    });
+
+    let base = base_url().await;
+    let (res_a, res_b) = tokio::join!(
+        client
+            .post(format!("{base}/xrpc/com.atproto.repo.putRecord"))
+            .bearer_auth(&token)
+            .json(&payload_a)
+            .send(),
+        client
+            .post(format!("{base}/xrpc/com.atproto.repo.putRecord"))
+            .bearer_auth(&token)
+            .json(&payload_b)
+            .send(),
+    );
+    let status_a = res_a.expect("A send").status();
+    let status_b = res_b.expect("B send").status();
+
+    let ok_a = status_a == StatusCode::OK;
+    let ok_b = status_b == StatusCode::OK;
+    assert!(
+        ok_a ^ ok_b,
+        "exactly one swap_commit write must succeed: status_a={status_a}, status_b={status_b}"
+    );
+}
+
+#[tokio::test]
+async fn imported_repo_emits_commit_event_with_valid_car() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let signing_key = SigningKey::random(&mut rand::thread_rng());
+    let (car_bytes, _car_root_cid) = build_car_with_signature(&did, &signing_key);
+
+    let import_res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.importRepo",
+            base_url().await
+        ))
+        .bearer_auth(&token)
+        .header("Content-Type", "application/vnd.ipld.car")
+        .body(car_bytes)
+        .send()
+        .await
+        .expect("importRepo");
+    assert_eq!(
+        import_res.status(),
+        StatusCode::OK,
+        "import failed: {:?}",
+        import_res.text().await.unwrap_or_default()
+    );
+
+    let repos = get_test_repos().await;
+    let typed_did = Did::new(did.clone()).unwrap();
+    let events = repos
+        .repo
+        .get_events_since_seq(SequenceNumber::ZERO, None)
+        .await
+        .expect("events");
+    let our: Vec<&SequencedEvent> = events
+        .iter()
+        .filter(|e| e.did == typed_did && e.event_type == RepoEventType::Commit)
+        .collect();
+    let last = our.last().expect("at least one commit event after import");
+
+    let inline = match last.blocks.as_ref().expect("blocks present") {
+        EventBlocks::Inline(v) => v,
+        _ => panic!("expected inline blocks"),
+    };
+    assert!(
+        !inline.is_empty(),
+        "import event inline blocks must not be empty"
+    );
+
+    let have_commit = inline.iter().any(|b| {
+        let cid = Cid::read_bytes(b.cid_bytes.as_slice()).unwrap();
+        last.commit_cid
+            .as_ref()
+            .and_then(|c| c.to_cid())
+            .map(|commit_cid| cid == commit_cid)
+            .unwrap_or(false)
+    });
+    assert!(have_commit, "import event CAR must include commit block");
+}
+
+#[tokio::test]
+async fn firehose_commit_block_bytes_roundtrip_to_same_cid() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let repos = get_test_repos().await;
+    let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
+    let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    create_post(&client, &token, &did, "3krt001", "round-trip me").await;
+    let frames = consumer
+        .wait_for_commits(&did, 1, Duration::from_secs(10))
+        .await;
+    let frame = frames.last().expect("frame");
+
+    let (_, block_map) = car_to_blocks(&frame.blocks).await;
+    use sha2::{Digest, Sha256};
+    for (cid, bytes) in &block_map {
+        let mut hasher = Sha256::new();
+        hasher.update(bytes);
+        let hash = hasher.finalize();
+        let mh = multihash::Multihash::wrap(0x12, hash.as_slice()).expect("wrap");
+        let recomputed = Cid::new_v1(cid.codec(), mh);
+        assert_eq!(
+            recomputed, *cid,
+            "CAR block {cid} bytes do not hash back to same CID"
+        );
+    }
+}
+
+#[tokio::test]
+async fn firehose_commit_car_contains_new_record_bytes_for_every_create() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let repos = get_test_repos().await;
+    let cursor = repos.repo.get_max_seq().await.unwrap().as_i64();
+    let consumer = FirehoseConsumer::connect_with_cursor(app_port(), cursor).await;
+    tokio::time::sleep(Duration::from_millis(100)).await;
+
+    let now = chrono::Utc::now().to_rfc3339();
+    let writes: Vec<Value> = (0..8)
+        .map(|i| {
+            json!({
+                "$type": "com.atproto.repo.applyWrites#create",
+                "collection": "app.bsky.feed.post",
+                "rkey": format!("3krec{:08}", i),
+                "value": {
+                    "$type": "app.bsky.feed.post",
+                    "text": format!("rec {i}"),
+                    "createdAt": now,
+                }
+            })
+        })
+        .collect();
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.applyWrites",
+            base_url().await
+        ))
+        .bearer_auth(&token)
+        .json(&json!({ "repo": did, "writes": writes }))
+        .send()
+        .await
+        .expect("applyWrites");
+    assert_eq!(res.status(), StatusCode::OK);
+
+    let frames = consumer
+        .wait_for_commits(&did, 1, Duration::from_secs(10))
+        .await;
+    let frame = frames.last().expect("frame");
+
+    let (_, block_map) = car_to_blocks(&frame.blocks).await;
+    for op in &frame.ops {
+        if op.action == RepoAction::Create {
+            let cid = op.cid.expect("create cid");
+            assert!(
+                block_map.contains_key(&cid),
+                "record CID {cid} for path {} missing from CAR",
+                op.path
+            );
+        }
+    }
+}

crates/tranquil-pds/tests/mst_property_fuzz.rs

@@ -0,0 +1,348 @@
+mod common;
+mod mst_verify;
+
+use std::collections::HashMap;
+use std::str::FromStr;
+
+use cid::Cid;
+use common::*;
+use jacquard_common::smol_str::SmolStr;
+use jacquard_repo::commit::Commit;
+use jacquard_repo::mst::{Mst, VerifiedWriteOp};
+use jacquard_repo::storage::BlockStore;
+use mst_verify::{extract_event_blocks, inline_to_store};
+use rand::rngs::StdRng;
+use rand::{Rng, SeedableRng};
+use reqwest::StatusCode;
+use serde_json::{Value, json};
+use tranquil_db_traits::{RepoEventType, SequenceNumber, SequencedEvent};
+use tranquil_types::Did;
+
+const COLLECTIONS: &[&str] = &[
+    "app.bsky.feed.post",
+    "app.bsky.feed.like",
+    "app.bsky.graph.follow",
+    "app.bsky.feed.repost",
+];
+
+#[derive(Copy, Clone, Debug)]
+enum FuzzOp {
+    Create,
+    Update,
+    Delete,
+}
+
+fn pick_op(rng: &mut StdRng, have_keys: bool) -> FuzzOp {
+    match (have_keys, rng.gen_range(0..10)) {
+        (false, _) => FuzzOp::Create,
+        (_, 0..=5) => FuzzOp::Create,
+        (_, 6..=7) => FuzzOp::Update,
+        _ => FuzzOp::Delete,
+    }
+}
+
+fn random_rkey(rng: &mut StdRng) -> String {
+    let tid_char_pool = b"234567abcdefghijklmnopqrstuvwxyz";
+    let mut out = Vec::with_capacity(13);
+    (0..13).for_each(|_| {
+        let c = tid_char_pool[rng.gen_range(0..tid_char_pool.len())];
+        out.push(c);
+    });
+    String::from_utf8(out).unwrap()
+}
+
+fn random_collection(rng: &mut StdRng) -> &'static str {
+    COLLECTIONS[rng.gen_range(0..COLLECTIONS.len())]
+}
+
+fn record_for_collection(col: &str, text: &str, now: &str) -> Value {
+    match col {
+        "app.bsky.feed.post" | "app.bsky.feed.repost" | "app.bsky.feed.like" => json!({
+            "$type": col,
+            "text": text,
+            "createdAt": now,
+        }),
+        _ => json!({
+            "$type": col,
+            "subject": format!("did:plc:synthetic{text}"),
+            "createdAt": now,
+        }),
+    }
+}
+
+async fn verify_commit_forward_and_inverse(event: &SequencedEvent) -> Result<(), String> {
+    let prev_data = event
+        .prev_data_cid
+        .as_ref()
+        .and_then(|c| c.to_cid())
+        .ok_or("no prev_data_cid")?;
+    let commit_cid = event
+        .commit_cid
+        .as_ref()
+        .and_then(|c| c.to_cid())
+        .ok_or("no commit_cid")?;
+    let inline = extract_event_blocks(event)?;
+    let ops = event
+        .ops
+        .as_ref()
+        .and_then(|v| v.as_array())
+        .ok_or("ops not array")?;
+
+    let storage = inline_to_store(inline);
+    let commit_bytes = storage
+        .get(&commit_cid)
+        .await
+        .map_err(|e| format!("get commit: {e:?}"))?
+        .ok_or("missing commit block")?;
+    let commit = Commit::from_cbor(&commit_bytes).map_err(|e| format!("parse commit: {e:?}"))?;
+    let new_data = *commit.data();
+
+    let mut forward = Mst::load(storage.clone(), prev_data, None);
+    for op in ops {
+        let action = op["action"].as_str().ok_or("op.action")?;
+        let path = op["path"].as_str().ok_or("op.path")?;
+        match action {
+            "create" | "update" => {
+                let cid = Cid::from_str(op["cid"].as_str().ok_or("op.cid")?)
+                    .map_err(|e| format!("{e:?}"))?;
+                forward = forward
+                    .add(path, cid)
+                    .await
+                    .map_err(|e| format!("fwd add {path}: {e:?}"))?;
+            }
+            "delete" => {
+                forward = forward
+                    .delete(path)
+                    .await
+                    .map_err(|e| format!("fwd delete {path}: {e:?}"))?;
+            }
+            other => return Err(format!("unknown action {other}")),
+        }
+    }
+    let got = forward
+        .persist()
+        .await
+        .map_err(|e| format!("persist: {e:?}"))?;
+    if got != new_data {
+        return Err(format!("forward root mismatch exp={new_data} got={got}"));
+    }
+
+    let mut inverse = Mst::load(storage, new_data, None);
+    for op in ops {
+        let action = op["action"].as_str().ok_or("op.action")?;
+        let path = op["path"].as_str().ok_or("op.path")?;
+        let key = SmolStr::new(path);
+        let verified = match action {
+            "create" => {
+                let cid = Cid::from_str(op["cid"].as_str().ok_or("op.cid")?)
+                    .map_err(|e| format!("{e:?}"))?;
+                VerifiedWriteOp::Create { key, cid }
+            }
+            "update" => {
+                let cid = Cid::from_str(op["cid"].as_str().ok_or("op.cid")?)
+                    .map_err(|e| format!("{e:?}"))?;
+                let prev = Cid::from_str(op["prev"].as_str().ok_or("op.prev")?)
+                    .map_err(|e| format!("{e:?}"))?;
+                VerifiedWriteOp::Update { key, cid, prev }
+            }
+            "delete" => {
+                let prev = Cid::from_str(op["prev"].as_str().ok_or("op.prev")?)
+                    .map_err(|e| format!("{e:?}"))?;
+                VerifiedWriteOp::Delete { key, prev }
+            }
+            other => return Err(format!("unknown action {other}")),
+        };
+        let inverted = inverse
+            .invert_op(verified.clone())
+            .await
+            .map_err(|e| format!("invert {verified:?}: {e:?}"))?;
+        if !inverted {
+            return Err(format!("op not invertible: {verified:?}"));
+        }
+    }
+    let got_prev = inverse
+        .get_pointer()
+        .await
+        .map_err(|e| format!("get_pointer: {e:?}"))?;
+    if got_prev != prev_data {
+        return Err(format!(
+            "inverse root mismatch exp={prev_data} got={got_prev}"
+        ));
+    }
+    Ok(())
+}
+
+async fn fuzz_run_with_seed(seed: u64, steps: usize) -> Vec<String> {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+    let mut rng = StdRng::seed_from_u64(seed);
+
+    let mut live_keys: HashMap<String, String> = HashMap::new();
+
+    for step in 0..steps {
+        let now = chrono::Utc::now().to_rfc3339();
+        let op = pick_op(&mut rng, !live_keys.is_empty());
+        match op {
+            FuzzOp::Create => {
+                let col = random_collection(&mut rng);
+                let rkey = random_rkey(&mut rng);
+                let path = format!("{col}/{rkey}");
+                if live_keys.contains_key(&path) {
+                    continue;
+                }
+                let record = record_for_collection(col, &format!("s{seed}-n{step}"), &now);
+                let res = client
+                    .post(format!(
+                        "{}/xrpc/com.atproto.repo.createRecord",
+                        base_url().await
+                    ))
+                    .bearer_auth(&token)
+                    .json(&json!({
+                        "repo": did,
+                        "collection": col,
+                        "rkey": rkey,
+                        "record": record,
+                    }))
+                    .send()
+                    .await
+                    .expect("createRecord");
+                if res.status() == StatusCode::OK {
+                    live_keys.insert(path, col.to_string());
+                }
+            }
+            FuzzOp::Update => {
+                let keys: Vec<&String> = live_keys.keys().collect();
+                if keys.is_empty() {
+                    continue;
+                }
+                let path = keys[rng.gen_range(0..keys.len())].clone();
+                let col = live_keys.get(&path).unwrap().clone();
+                let rkey = path.split('/').nth(1).unwrap().to_string();
+                let record = record_for_collection(&col, &format!("s{seed}-u{step}"), &now);
+                let res = client
+                    .post(format!(
+                        "{}/xrpc/com.atproto.repo.putRecord",
+                        base_url().await
+                    ))
+                    .bearer_auth(&token)
+                    .json(&json!({
+                        "repo": did,
+                        "collection": col,
+                        "rkey": rkey,
+                        "record": record,
+                    }))
+                    .send()
+                    .await
+                    .expect("putRecord");
+                assert_eq!(res.status(), StatusCode::OK, "putRecord failed");
+            }
+            FuzzOp::Delete => {
+                let keys: Vec<String> = live_keys.keys().cloned().collect();
+                if keys.is_empty() {
+                    continue;
+                }
+                let path = keys[rng.gen_range(0..keys.len())].clone();
+                let col = live_keys.get(&path).unwrap().clone();
+                let rkey = path.split('/').nth(1).unwrap().to_string();
+                let res = client
+                    .post(format!(
+                        "{}/xrpc/com.atproto.repo.deleteRecord",
+                        base_url().await
+                    ))
+                    .bearer_auth(&token)
+                    .json(&json!({
+                        "repo": did,
+                        "collection": col,
+                        "rkey": rkey,
+                    }))
+                    .send()
+                    .await
+                    .expect("deleteRecord");
+                if res.status() == StatusCode::OK {
+                    live_keys.remove(&path);
+                }
+            }
+        }
+    }
+
+    let repos = get_test_repos().await;
+    let typed_did = Did::new(did.clone()).unwrap();
+    let events = repos
+        .repo
+        .get_events_since_seq(SequenceNumber::ZERO, None)
+        .await
+        .expect("get_events_since_seq");
+
+    let our: Vec<SequencedEvent> = events
+        .into_iter()
+        .filter(|e| {
+            e.did == typed_did
+                && e.event_type == RepoEventType::Commit
+                && e.prev_data_cid.is_some()
+                && e.ops
+                    .as_ref()
+                    .and_then(|v| v.as_array())
+                    .is_some_and(|a| !a.is_empty())
+        })
+        .collect();
+
+    let mut failures = Vec::new();
+    for event in &our {
+        if let Err(msg) = verify_commit_forward_and_inverse(event).await {
+            failures.push(format!(
+                "seed={seed} seq={} ops={:?}: {msg}",
+                event.seq.as_i64(),
+                event
+                    .ops
+                    .as_ref()
+                    .and_then(|v| v.as_array())
+                    .map(|a| a.len())
+            ));
+        }
+    }
+    failures
+}
+
+#[tokio::test]
+async fn mst_property_fuzz_seed_1() {
+    let failures = fuzz_run_with_seed(1, 150).await;
+    assert!(
+        failures.is_empty(),
+        "fuzz seed=1 found {} invalid commits:\n  - {}",
+        failures.len(),
+        failures.join("\n  - ")
+    );
+}
+
+#[tokio::test]
+async fn mst_property_fuzz_seed_42() {
+    let failures = fuzz_run_with_seed(42, 150).await;
+    assert!(
+        failures.is_empty(),
+        "fuzz seed=42 found {} invalid commits:\n  - {}",
+        failures.len(),
+        failures.join("\n  - ")
+    );
+}
+
+#[tokio::test]
+async fn mst_property_fuzz_seed_9001() {
+    let failures = fuzz_run_with_seed(9001, 150).await;
+    assert!(
+        failures.is_empty(),
+        "fuzz seed=9001 found {} invalid commits:\n  - {}",
+        failures.len(),
+        failures.join("\n  - ")
+    );
+}
+
+#[tokio::test]
+async fn mst_property_fuzz_deep_tree_seed_7() {
+    let failures = fuzz_run_with_seed(7, 400).await;
+    assert!(
+        failures.is_empty(),
+        "fuzz deep seed=7 found {} invalid commits:\n  - {}",
+        failures.len(),
+        failures.join("\n  - ")
+    );
+}

crates/tranquil-pds/tests/mst_verify/mod.rs

@@ -0,0 +1,26 @@
+use std::collections::BTreeMap;
+use std::sync::Arc;
+
+use bytes::Bytes;
+use cid::Cid;
+use jacquard_repo::storage::MemoryBlockStore;
+use tranquil_db_traits::{EventBlockInline, EventBlocks, SequencedEvent};
+
+pub fn extract_event_blocks(event: &SequencedEvent) -> Result<&[EventBlockInline], String> {
+    match event.blocks.as_ref() {
+        Some(EventBlocks::Inline(v)) => Ok(v.as_slice()),
+        Some(EventBlocks::LegacyCids(_)) => Err("legacy cids, not inline".into()),
+        None => Err("event missing blocks".into()),
+    }
+}
+
+pub fn inline_to_store(inline: &[EventBlockInline]) -> Arc<MemoryBlockStore> {
+    let map: BTreeMap<Cid, Bytes> = inline
+        .iter()
+        .map(|b| {
+            let cid = Cid::read_bytes(b.cid_bytes.as_slice()).expect("valid cid bytes");
+            (cid, Bytes::from(b.data.clone()))
+        })
+        .collect();
+    Arc::new(MemoryBlockStore::new_from_blocks(map))
+}

crates/tranquil-pds/tests/oauth.rs

@@ -187,6 +187,37 @@ async fn test_par_and_authorize() {
     );
 }
 
+#[tokio::test]
+async fn test_par_public_client_empty_assertion_fields() {
+    let url = base_url().await;
+    let client = client();
+    let redirect_uri = "https://nels.evil.oauth.pet/callback";
+    let mock_client = setup_mock_client_metadata(redirect_uri).await;
+    let client_id = mock_client.uri();
+    let (_, code_challenge) = generate_pkce();
+    let par_res = client
+        .post(format!("{}/oauth/par", url))
+        .form(&[
+            ("response_type", "code"),
+            ("client_id", &client_id),
+            ("redirect_uri", redirect_uri),
+            ("code_challenge", &code_challenge),
+            ("code_challenge_method", "S256"),
+            ("scope", "atproto"),
+            ("state", "test-state"),
+            ("client_assertion", ""),
+            ("client_assertion_type", ""),
+        ])
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(
+        par_res.status(),
+        StatusCode::CREATED,
+        "PAR with empty assertion fields from a public client should succeed"
+    );
+}
+
 #[tokio::test]
 async fn test_full_oauth_flow() {
     let url = base_url().await;

crates/tranquil-pds/tests/security_fixes.rs

@@ -88,12 +88,17 @@ fn test_signal_username_validation() {
     assert!(is_valid_signal_username("bob_smith.99"));
     assert!(is_valid_signal_username("user123.42"));
     assert!(is_valid_signal_username("lu1.01"));
-    assert!(is_valid_signal_username("abc.00"));
     assert!(is_valid_signal_username("a_very_long_username_here.55"));
+    assert!(is_valid_signal_username("alice.123"));
+    assert!(is_valid_signal_username("alice.999999999"));
+    assert!(is_valid_signal_username("alice.18446744073709551615"));
 
     assert!(!is_valid_signal_username("alice"));
     assert!(!is_valid_signal_username("alice.1"));
     assert!(!is_valid_signal_username("alice.001"));
+    assert!(!is_valid_signal_username("abc.00"));
+    assert!(!is_valid_signal_username("alice.0"));
+    assert!(!is_valid_signal_username("alice.999999999999999999999"));
     assert!(!is_valid_signal_username(".01"));
     assert!(!is_valid_signal_username("ab.01"));
     assert!(!is_valid_signal_username(""));

crates/tranquil-scopes/src/permissions.rs

@@ -164,7 +164,10 @@ impl ScopePermissions {
             if self.has_transition_generic && !self.has_transition_chat {
                 return Err(ScopeError::InsufficientScope {
                     required: "transition:chat.bsky".to_string(),
-                    message: format!("Chat access requires transition:chat.bsky scope to call {}", lxm),
+                    message: format!(
+                        "Chat access requires transition:chat.bsky scope to call {}",
+                        lxm
+                    ),
                 });
             }
         }

crates/tranquil-signal/src/client.rs

@@ -33,15 +33,11 @@ impl SignalUsername {
     pub fn parse(username: &str) -> Result<Self, InvalidSignalUsername> {
         let reject = || Err(InvalidSignalUsername(username.to_string()));
 
-        if username.len() < 6 || username.len() > 35 {
-            return reject();
-        }
-
         let Some((base, discriminator)) = username.rsplit_once('.') else {
             return reject();
         };
 
-        if base.len() < 3 || base.len() > 32 {
+        if !matches!(base.len(), 3..=32) {
             return reject();
         }
 
@@ -53,7 +49,7 @@ impl SignalUsername {
             return reject();
         }
 
-        if discriminator.len() != 2 || !discriminator.chars().all(|c| c.is_ascii_digit()) {
+        if !is_valid_discriminator(discriminator) {
             return reject();
         }
 
@@ -65,6 +61,19 @@ impl SignalUsername {
     }
 }
 
+fn is_valid_discriminator(s: &str) -> bool {
+    if !s.chars().all(|c| c.is_ascii_digit()) {
+        return false;
+    }
+    if !matches!(s.len(), 2..=20) {
+        return false;
+    }
+    if s.len() > 2 && s.starts_with('0') {
+        return false;
+    }
+    s.parse::<u64>().is_ok_and(|n| n != 0)
+}
+
 impl fmt::Display for SignalUsername {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.write_str(&self.0)
@@ -113,11 +122,7 @@ pub struct MessageTooLong {
 
 impl fmt::Display for MessageTooLong {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(
-            f,
-            "message body too long: {} bytes (max {})",
-            self.len, self.max
-        )
+        write!(f, "message body is {} bytes, max {}", self.len, self.max)
     }
 }
 
@@ -368,13 +373,13 @@ impl SignalClient {
             let req = tokio::select! {
                 biased;
                 _ = shutdown.cancelled() => {
-                    tracing::info!("signal worker shutting down (cancellation)");
+                    tracing::info!("signal worker cancelled, shutting down");
                     break;
                 }
                 msg = rx.recv() => match msg {
                     Some(r) => r,
                     None => {
-                        tracing::info!("signal worker shutting down (channel closed)");
+                        tracing::info!("signal worker channel closed, shutting down");
                         break;
                     }
                 },

crates/tranquil-signal/src/fjall_store.rs

@@ -1050,7 +1050,7 @@ impl ContentsStore for FjallSignalStore {
             .and_then(|v| match <[u8; 32]>::try_from(v.as_ref()) {
                 Ok(arr) => Some(ProfileKey { bytes: arr }),
                 Err(_) => {
-                    warn!(%uuid, len = v.len(), "corrupted profile key (expected 32 bytes)");
+                    warn!(%uuid, len = v.len(), "corrupted profile key, expected 32 bytes");
                     None
                 }
             }))

crates/tranquil-signal/src/store.rs

@@ -1117,7 +1117,7 @@ impl ContentsStore for PgSignalStore {
             Some(r) => match <[u8; 32]>::try_from(r.key.as_slice()) {
                 Ok(arr) => Some(ProfileKey { bytes: arr }),
                 Err(_) => {
-                    warn!(%uuid, len = r.key.len(), "corrupted profile key (expected 32 bytes)");
+                    warn!(%uuid, len = r.key.len(), "corrupted profile key, expected 32 bytes");
                     None
                 }
             },

crates/tranquil-store/Cargo.toml

@@ -14,7 +14,7 @@ parking_lot = { workspace = true }
 fjall = "3"
 lsm-tree = "3"
 flume = "0.11"
-tokio = { workspace = true, features = ["sync", "rt"] }
+tokio = { workspace = true, features = ["sync", "rt", "time"] }
 bytes = "1"
 memmap2 = "0.9"
 tracing = { workspace = true }
@@ -34,9 +34,21 @@ dashmap = "6"
 rayon = "1"
 smallvec = "1"
 uuid = { workspace = true }
+tempfile = { version = "3", optional = true }
+clap = { workspace = true, optional = true }
+toml = { version = "0.8", optional = true }
+tracing-subscriber = { workspace = true, features = ["env-filter"], optional = true }
+libc = "0.2"
 
 [features]
-test-harness = []
+test-harness = ["dep:tempfile"]
+gauntlet-cli = ["test-harness", "dep:clap", "dep:toml", "dep:tracing-subscriber"]
+gauntlet-jemalloc-prof = []
+
+[[bin]]
+name = "tranquil-gauntlet"
+path = "src/bin/tranquil_gauntlet.rs"
+required-features = ["gauntlet-cli"]
 
 [dev-dependencies]
 tranquil-store = { path = ".", features = ["test-harness"] }
@@ -50,9 +62,8 @@ tranquil-db = { workspace = true }
 sqlx = { workspace = true }
 k256 = { workspace = true }
 rand = { workspace = true }
-tikv-jemallocator = "0.6"
+tikv-jemallocator = { version = "0.6", features = ["profiling", "unprefixed_malloc_on_supported_platforms"] }
 tracing-subscriber = { workspace = true, features = ["env-filter"] }
-libc = "0.2"
 
 [[bench]]
 name = "blockstore"

crates/tranquil-store/fuzz/.gitignore

@@ -0,0 +1,4 @@
+target/
+artifacts/
+coverage/
+Cargo.lock

crates/tranquil-store/fuzz/Cargo.toml

@@ -0,0 +1,52 @@
+[package]
+name = "tranquil-store-fuzz"
+version = "0.0.0"
+publish = false
+edition = "2024"
+
+[package.metadata]
+cargo-fuzz = true
+
+[dependencies]
+libfuzzer-sys = "0.4"
+arbitrary = { version = "1", features = ["derive"] }
+tranquil-store = { path = "..", features = ["test-harness"] }
+tempfile = "3"
+tokio = { version = "1", features = ["rt", "time", "macros", "sync"] }
+
+[[bin]]
+name = "decode_block_record"
+path = "fuzz_targets/decode_block_record.rs"
+test = false
+doc = false
+bench = false
+
+[[bin]]
+name = "decode_hint_record"
+path = "fuzz_targets/decode_hint_record.rs"
+test = false
+doc = false
+bench = false
+
+[[bin]]
+name = "segment_scan"
+path = "fuzz_targets/segment_scan.rs"
+test = false
+doc = false
+bench = false
+
+[[bin]]
+name = "metastore_key_codec"
+path = "fuzz_targets/metastore_key_codec.rs"
+test = false
+doc = false
+bench = false
+
+[[bin]]
+name = "gauntlet_micro"
+path = "fuzz_targets/gauntlet_micro.rs"
+test = false
+doc = false
+bench = false
+
+[workspace]

crates/tranquil-store/fuzz/corpus/decode_block_record/cid_only

@@ -0,0 +1 @@
+

crates/tranquil-store/fuzz/corpus/segment_scan/bad_magic

@@ -0,0 +1 @@
+BADX

crates/tranquil-store/fuzz/corpus/segment_scan/header_only

@@ -0,0 +1 @@
+TQEV

crates/tranquil-store/fuzz/fuzz_targets/decode_block_record.rs

@@ -0,0 +1,28 @@
+#![no_main]
+
+use std::path::Path;
+
+use libfuzzer_sys::fuzz_target;
+use tranquil_store::blockstore::{BlockOffset, decode_block_record};
+use tranquil_store::{FaultConfig, OpenOptions, SimulatedIO, StorageIO};
+
+fuzz_target!(|data: &[u8]| {
+    let sim = SimulatedIO::new(0, FaultConfig::none());
+    let opts = OpenOptions {
+        read: true,
+        write: true,
+        create: true,
+        truncate: false,
+    };
+    let fd = match sim.open(Path::new("/fuzz/block.tqb"), opts) {
+        Ok(fd) => fd,
+        Err(_) => return,
+    };
+    if !data.is_empty() {
+        let _ = sim.write_all_at(fd, 0, data);
+        let _ = sim.sync(fd);
+    }
+    let file_size = data.len() as u64;
+    let _ = decode_block_record(&sim, fd, BlockOffset::new(0), file_size);
+    let _ = sim.close(fd);
+});

crates/tranquil-store/fuzz/fuzz_targets/decode_hint_record.rs

@@ -0,0 +1,41 @@
+#![no_main]
+
+use std::path::Path;
+
+use libfuzzer_sys::fuzz_target;
+use tranquil_store::blockstore::{HintOffset, decode_hint_record};
+use tranquil_store::{FaultConfig, OpenOptions, SimulatedIO, StorageIO};
+
+fuzz_target!(|data: &[u8]| {
+    let sim = SimulatedIO::new(0, FaultConfig::none());
+    let opts = OpenOptions {
+        read: true,
+        write: true,
+        create: true,
+        truncate: false,
+    };
+    let fd = match sim.open(Path::new("/fuzz/hint.tqh"), opts) {
+        Ok(fd) => fd,
+        Err(_) => return,
+    };
+    if !data.is_empty() {
+        let _ = sim.write_all_at(fd, 0, data);
+        let _ = sim.sync(fd);
+    }
+    let file_size = data.len() as u64;
+    let cursor = std::cell::Cell::new(0u64);
+    std::iter::from_fn(|| {
+        if cursor.get() >= file_size {
+            return None;
+        }
+        match decode_hint_record(&sim, fd, HintOffset::new(cursor.get()), file_size) {
+            Ok(Some(_)) => {
+                cursor.set(cursor.get() + 64);
+                Some(())
+            }
+            _ => None,
+        }
+    })
+    .for_each(|()| {});
+    let _ = sim.close(fd);
+});

crates/tranquil-store/fuzz/fuzz_targets/gauntlet_micro.rs

@@ -0,0 +1,109 @@
+#![no_main]
+
+use std::sync::OnceLock;
+
+use arbitrary::{Arbitrary, Unstructured};
+use libfuzzer_sys::fuzz_target;
+use tokio::runtime::Runtime;
+use tranquil_store::blockstore::GroupCommitConfig;
+use tranquil_store::gauntlet::{
+    CollectionName, DidSpaceSize, Gauntlet, GauntletConfig, InvariantSet, IoBackend, KeySpaceSize,
+    MaxFileSize, Op, OpCount, OpInterval, OpStream, OpWeights, RecordKey, RestartPolicy,
+    RetentionMaxSecs, RunLimits, Seed, ShardCount, SizeDistribution, StoreConfig, ValueBytes,
+    ValueSeed, WallMs, WorkloadModel, WriterConcurrency,
+};
+
+#[derive(Arbitrary, Debug)]
+enum FuzzOp {
+    Add { rkey: u8, value: u16 },
+    Delete { rkey: u8 },
+    Compact,
+    Checkpoint,
+    Read { rkey: u8 },
+    ReadBlock { value: u16 },
+}
+
+const COLLECTION: &str = "app.bsky.feed.post";
+const MAX_OPS: usize = 128;
+
+fn to_op(fuzz_op: FuzzOp) -> Op {
+    match fuzz_op {
+        FuzzOp::Add { rkey, value } => Op::AddRecord {
+            collection: CollectionName(COLLECTION.to_string()),
+            rkey: RecordKey(format!("k{rkey:03}")),
+            value_seed: ValueSeed(u32::from(value)),
+        },
+        FuzzOp::Delete { rkey } => Op::DeleteRecord {
+            collection: CollectionName(COLLECTION.to_string()),
+            rkey: RecordKey(format!("k{rkey:03}")),
+        },
+        FuzzOp::Compact => Op::Compact,
+        FuzzOp::Checkpoint => Op::Checkpoint,
+        FuzzOp::Read { rkey } => Op::ReadRecord {
+            collection: CollectionName(COLLECTION.to_string()),
+            rkey: RecordKey(format!("k{rkey:03}")),
+        },
+        FuzzOp::ReadBlock { value } => Op::ReadBlock {
+            value_seed: ValueSeed(u32::from(value)),
+        },
+    }
+}
+
+fn tiny_config() -> GauntletConfig {
+    GauntletConfig {
+        seed: Seed(0),
+        io: IoBackend::Real,
+        workload: WorkloadModel {
+            weights: OpWeights::default(),
+            size_distribution: SizeDistribution::Fixed(ValueBytes(64)),
+            collections: vec![CollectionName(COLLECTION.to_string())],
+            key_space: KeySpaceSize(256),
+            did_space: DidSpaceSize(8),
+            retention_max_secs: RetentionMaxSecs(3600),
+        },
+        op_count: OpCount(0),
+        invariants: InvariantSet::REFCOUNT_CONSERVATION
+            | InvariantSet::REACHABILITY
+            | InvariantSet::READ_AFTER_WRITE,
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(2_000)),
+        },
+        restart_policy: RestartPolicy::EveryNOps(OpInterval(32)),
+        store: StoreConfig {
+            max_file_size: MaxFileSize(4096),
+            group_commit: GroupCommitConfig::default(),
+            shard_count: ShardCount(1),
+        },
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn shared_runtime() -> &'static Runtime {
+    static RUNTIME: OnceLock<Runtime> = OnceLock::new();
+    RUNTIME.get_or_init(|| {
+        tokio::runtime::Builder::new_current_thread()
+            .enable_time()
+            .build()
+            .expect("build tokio runtime")
+    })
+}
+
+fuzz_target!(|data: &[u8]| {
+    if data.is_empty() {
+        return;
+    }
+    let mut u = Unstructured::new(data);
+    let ops: Vec<FuzzOp> = match Vec::<FuzzOp>::arbitrary(&mut u) {
+        Ok(ops) => ops.into_iter().take(MAX_OPS).collect(),
+        Err(_) => return,
+    };
+    if ops.is_empty() {
+        return;
+    }
+    let stream = OpStream::from_vec(ops.into_iter().map(to_op).collect());
+
+    let cfg = tiny_config();
+    let gauntlet = Gauntlet::new(cfg).expect("build gauntlet");
+    let _ = shared_runtime().block_on(gauntlet.run_with_ops(stream));
+});

crates/tranquil-store/fuzz/fuzz_targets/metastore_key_codec.rs

@@ -0,0 +1,77 @@
+#![no_main]
+
+use arbitrary::Arbitrary;
+use libfuzzer_sys::fuzz_target;
+use tranquil_store::metastore::encoding::{KeyBuilder, KeyReader};
+
+#[derive(Arbitrary, Debug, PartialEq, Eq)]
+enum Field {
+    U64(u64),
+    I64(i64),
+    U32(u32),
+    U16(u16),
+    Bool(bool),
+    Bytes(Vec<u8>),
+    String(String),
+}
+
+fn append(builder: KeyBuilder, field: &Field) -> KeyBuilder {
+    match field {
+        Field::U64(v) => builder.u64(*v),
+        Field::I64(v) => builder.i64(*v),
+        Field::U32(v) => builder.u32(*v),
+        Field::U16(v) => builder.u16(*v),
+        Field::Bool(v) => builder.bool(*v),
+        Field::Bytes(v) => builder.bytes(v),
+        Field::String(v) => builder.string(v),
+    }
+}
+
+fn consume(reader: &mut KeyReader<'_>, field: &Field) -> bool {
+    match field {
+        Field::U64(v) => reader.u64() == Some(*v),
+        Field::I64(v) => reader.i64() == Some(*v),
+        Field::U32(v) => reader.u32() == Some(*v),
+        Field::U16(v) => reader.u16() == Some(*v),
+        Field::Bool(v) => reader.bool() == Some(*v),
+        Field::Bytes(v) => reader.bytes().as_deref() == Some(v.as_slice()),
+        Field::String(v) => reader.string().as_deref() == Some(v.as_str()),
+    }
+}
+
+#[derive(Arbitrary, Debug)]
+enum Mode {
+    Roundtrip(Vec<Field>),
+    Raw(Vec<u8>),
+}
+
+fuzz_target!(|mode: Mode| {
+    match mode {
+        Mode::Roundtrip(fields) => {
+            let encoded1 = fields.iter().fold(KeyBuilder::new(), append).build();
+
+            let mut reader = KeyReader::new(encoded1.as_slice());
+            let all_match = fields.iter().all(|f| consume(&mut reader, f));
+            assert!(all_match, "roundtrip decode failed");
+            assert!(reader.is_empty(), "trailing bytes after decode");
+
+            let encoded2 = fields.iter().fold(KeyBuilder::new(), append).build();
+            assert_eq!(
+                encoded1.as_slice(),
+                encoded2.as_slice(),
+                "encoding not deterministic",
+            );
+        }
+        Mode::Raw(data) => {
+            let mut reader = KeyReader::new(&data);
+            let _ = reader.u64();
+            let _ = reader.i64();
+            let _ = reader.u32();
+            let _ = reader.u16();
+            let _ = reader.bool();
+            let _ = reader.bytes();
+            let _ = reader.string();
+            let _ = reader.tag();
+        }
+    }
+});

crates/tranquil-store/fuzz/fuzz_targets/segment_scan.rs

@@ -0,0 +1,31 @@
+#![no_main]
+
+use std::path::Path;
+
+use libfuzzer_sys::fuzz_target;
+use tranquil_store::eventlog::SegmentReader;
+
+const FUZZ_MAX_PAYLOAD: u32 = 1 << 20;
+use tranquil_store::{FaultConfig, OpenOptions, SimulatedIO, StorageIO};
+
+fuzz_target!(|data: &[u8]| {
+    let sim = SimulatedIO::new(0, FaultConfig::none());
+    let opts = OpenOptions {
+        read: true,
+        write: true,
+        create: true,
+        truncate: false,
+    };
+    let fd = match sim.open(Path::new("/fuzz/segment.tqe"), opts) {
+        Ok(fd) => fd,
+        Err(_) => return,
+    };
+    if !data.is_empty() {
+        let _ = sim.write_all_at(fd, 0, data);
+        let _ = sim.sync(fd);
+    }
+    if let Ok(reader) = SegmentReader::open(&sim, fd, FUZZ_MAX_PAYLOAD) {
+        reader.for_each(|_result| {});
+    }
+    let _ = sim.close(fd);
+});

crates/tranquil-store/src/blockstore/cid_util.rs

@@ -0,0 +1,47 @@
+use cid::Cid;
+use multihash::Multihash;
+use sha2::{Digest, Sha256};
+
+use super::data_file::CID_SIZE;
+
+pub const DAG_CBOR_CODEC: u64 = 0x71;
+pub const SHA2_256_CODE: u64 = 0x12;
+
+pub fn hash_to_cid(data: &[u8]) -> Cid {
+    let mut hasher = Sha256::new();
+    hasher.update(data);
+    let digest = hasher.finalize();
+    let mh = Multihash::wrap(SHA2_256_CODE, &digest)
+        .expect("SHA-256 digest is 32 bytes, well within multihash capacity");
+    Cid::new_v1(DAG_CBOR_CODEC, mh)
+}
+
+pub fn hash_to_cid_bytes(data: &[u8]) -> [u8; CID_SIZE] {
+    let raw = hash_to_cid(data).to_bytes();
+    raw.try_into()
+        .expect("CIDv1 + DAG-CBOR + SHA-256 always encodes to CID_SIZE bytes")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn hash_to_cid_bytes_is_deterministic() {
+        let a = hash_to_cid_bytes(b"hello");
+        let b = hash_to_cid_bytes(b"hello");
+        assert_eq!(a, b);
+    }
+
+    #[test]
+    fn hash_to_cid_bytes_diverges_on_single_byte_change() {
+        assert_ne!(hash_to_cid_bytes(b"abc"), hash_to_cid_bytes(b"abd"));
+    }
+
+    #[test]
+    fn hash_to_cid_and_bytes_agree() {
+        let cid = hash_to_cid(b"payload");
+        let raw: [u8; CID_SIZE] = cid.to_bytes().try_into().expect("36 bytes");
+        assert_eq!(raw, hash_to_cid_bytes(b"payload"));
+    }
+}

crates/tranquil-store/src/blockstore/compaction.rs

@@ -68,8 +68,8 @@ pub(super) fn compact_on_writer_thread<S: StorageIO>(
         return Err(CompactionError::ActiveFileCannotBeCompacted);
     }
 
-    let source_fd = manager.open_for_read(source_file_id)?;
-    let source_size = manager.io().file_size(source_fd)?;
+    let source_handle = manager.open_for_read(source_file_id)?;
+    let source_size = manager.io().file_size(source_handle.fd())?;
 
     let new_file_id = file_ids.allocate();
 
@@ -77,7 +77,7 @@ pub(super) fn compact_on_writer_thread<S: StorageIO>(
         manager,
         index,
         source_file_id,
-        source_fd,
+        source_handle.fd(),
         new_file_id,
         current_epoch,
         grace_period_ms,
@@ -102,6 +102,13 @@ pub(super) fn compact_on_writer_thread<S: StorageIO>(
                 .io()
                 .delete(&hint_file_path(manager.data_dir(), source_file_id))
                 .ok();
+            if live_count == 0 {
+                manager.delete_data_file(new_file_id).ok();
+                manager
+                    .io()
+                    .delete(&hint_file_path(manager.data_dir(), new_file_id))
+                    .ok();
+            }
             manager.io().sync_dir(manager.data_dir())?;
 
             let reclaimed_bytes = source_size.saturating_sub(new_size);
@@ -141,8 +148,8 @@ fn stream_compact<S: StorageIO>(
     let mut reader = DataFileReader::open(manager.io(), source_fd)?;
     let now = crate::wall_clock_ms();
 
-    let new_fd = manager.open_for_append(new_file_id)?;
-    let mut writer = DataFileWriter::new(manager.io(), new_fd, new_file_id)?;
+    let new_handle = manager.open_for_append(new_file_id)?;
+    let mut writer = DataFileWriter::new(manager.io(), new_handle.fd(), new_file_id)?;
 
     let hint_path = hint_file_path(manager.data_dir(), new_file_id);
     let hint_fd = manager.io().open(&hint_path, OpenOptions::read_write())?;
@@ -153,7 +160,7 @@ fn stream_compact<S: StorageIO>(
     let mut live_count: u64 = 0;
     let mut dead_count: u64 = 0;
 
-    reader.try_for_each(|r| {
+    let scan_result = reader.try_for_each(|r| {
         let record = r?;
         match record {
             ReadBlockRecord::Valid {
@@ -192,11 +199,34 @@ fn stream_compact<S: StorageIO>(
             ReadBlockRecord::Corrupted { .. } | ReadBlockRecord::Truncated { .. } => {}
         }
         Ok::<_, CompactionError>(())
-    })?;
+    });
 
-    writer.sync()?;
-    hint_writer.sync()?;
-    manager.io().sync_dir(manager.data_dir())?;
+    let record_count =
+        u32::try_from((live_count as u128).saturating_add(dead_count as u128)).unwrap_or(u32::MAX);
+    let writer_position = writer.position();
+    let finalize_result = scan_result
+        .and_then(|()| writer.sync().map_err(CompactionError::from))
+        .and_then(|()| {
+            hint_writer
+                .append_commit_marker(
+                    current_epoch.raw(),
+                    record_count,
+                    new_file_id,
+                    writer_position,
+                )
+                .map_err(CompactionError::from)
+        })
+        .and_then(|()| hint_writer.sync().map_err(CompactionError::from))
+        .and_then(|()| {
+            manager
+                .io()
+                .sync_dir(manager.data_dir())
+                .map_err(CompactionError::from)
+        });
+
+    let _ = manager.io().close(hint_fd);
+
+    finalize_result?;
 
     let new_size = writer.position().raw();
 

crates/tranquil-store/src/blockstore/group_commit.rs

@@ -11,7 +11,7 @@ use crate::fsync_order::PostBlockstoreHook;
 use super::BlocksSynced;
 use crate::io::{FileId, OpenOptions, StorageIO};
 
-use super::data_file::{CID_SIZE, DataFileWriter};
+use super::data_file::{CID_SIZE, DataFileWriter, ReadBlockRecord, decode_block_record};
 use super::hash_index::{BlockIndex, BlockIndexError, CheckpointPositions};
 use super::hint::{HintFileWriter, hint_file_path};
 use super::manager::DataFileManager;
@@ -106,6 +106,10 @@ pub enum CommitError {
     Io(Arc<io::Error>),
     Index(String),
     ChannelClosed,
+    VerifyFailed {
+        file_id: DataFileId,
+        offset: BlockOffset,
+    },
 }
 
 impl std::fmt::Display for CommitError {
@@ -114,6 +118,11 @@ impl std::fmt::Display for CommitError {
             Self::Io(e) => write!(f, "io: {}", e.as_ref()),
             Self::Index(e) => write!(f, "index: {e}"),
             Self::ChannelClosed => write!(f, "commit channel closed"),
+            Self::VerifyFailed { file_id, offset } => write!(
+                f,
+                "post-sync verify failed at {file_id}:{} (misdirected write or durable corruption)",
+                offset.raw()
+            ),
         }
     }
 }
@@ -122,7 +131,7 @@ impl std::error::Error for CommitError {
     fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
         match self {
             Self::Io(e) => Some(e.as_ref()),
-            Self::Index(_) | Self::ChannelClosed => None,
+            Self::Index(_) | Self::ChannelClosed | Self::VerifyFailed { .. } => None,
         }
     }
 }
@@ -181,6 +190,7 @@ pub struct GroupCommitConfig {
     pub channel_capacity: usize,
     pub checkpoint_interval_ms: u64,
     pub checkpoint_write_threshold: u64,
+    pub verify_persisted_blocks: bool,
 }
 
 impl Default for GroupCommitConfig {
@@ -190,6 +200,7 @@ impl Default for GroupCommitConfig {
             channel_capacity: 4096,
             checkpoint_interval_ms: 60_000,
             checkpoint_write_threshold: 100_000,
+            verify_persisted_blocks: false,
         }
     }
 }
@@ -200,6 +211,7 @@ struct ShardContext {
     file_ids: Arc<FileIdAllocator>,
     active_files: Arc<ActiveFileSet>,
     hint_positions: Arc<ShardHintPositions>,
+    verify_persisted_blocks: bool,
 }
 
 struct ActiveState {
@@ -403,6 +415,7 @@ impl GroupCommitWriter {
                     file_ids: Arc::clone(&file_ids),
                     active_files: Arc::clone(&active_files),
                     hint_positions: Arc::clone(&hint_positions),
+                    verify_persisted_blocks: config.verify_persisted_blocks,
                 };
                 SingleShardWriter::spawn(
                     ctx,
@@ -540,7 +553,8 @@ fn initialize_active_state<S: StorageIO>(
 
     match cursor {
         Some(wc) => {
-            let fd = manager.open_for_append(wc.file_id)?;
+            let handle = manager.open_for_append(wc.file_id)?;
+            let fd = handle.fd();
             let file_size = manager.io().file_size(fd)?;
 
             if file_size < wc.offset.raw() {
@@ -550,6 +564,16 @@ fn initialize_active_state<S: StorageIO>(
                 )));
             }
 
+            let header_end = super::data_file::BLOCK_HEADER_SIZE as u64;
+            let position = match file_size < header_end {
+                true => {
+                    let writer = DataFileWriter::new(manager.io(), fd, wc.file_id)?;
+                    writer.sync()?;
+                    writer.position()
+                }
+                false => BlockOffset::new(file_size),
+            };
+
             let hint_path = hint_file_path(data_dir, wc.file_id);
             let hint_fd = manager.io().open(&hint_path, OpenOptions::read_write())?;
             let hint_size = manager.io().file_size(hint_fd)?;
@@ -557,7 +581,7 @@ fn initialize_active_state<S: StorageIO>(
             Ok(ActiveState {
                 file_id: wc.file_id,
                 fd,
-                position: BlockOffset::new(file_size),
+                position,
                 hint_fd,
                 hint_position: HintOffset::new(hint_size),
             })
@@ -565,7 +589,8 @@ fn initialize_active_state<S: StorageIO>(
         None => {
             let file_id = file_ids.allocate();
 
-            let fd = manager.open_for_append(file_id)?;
+            let handle = manager.open_for_append(file_id)?;
+            let fd = handle.fd();
             let writer = DataFileWriter::new(manager.io(), fd, file_id)?;
             writer.sync()?;
             let position = writer.position();
@@ -1051,9 +1076,124 @@ fn drain_and_process_remaining<S: StorageIO>(
     shutdown_checkpoint(index, epoch, &ctx.hint_positions);
 }
 
-struct RotationState {
+struct RotationState<S: StorageIO> {
     file_id: DataFileId,
+    handle: Arc<super::manager::CachedHandle<S>>,
+    hint_fd: FileId,
+}
+
+fn verify_persisted_blocks<S: StorageIO>(
+    manager: &DataFileManager<S>,
+    entries: &[([u8; CID_SIZE], BlockLocation)],
+) -> Result<(), CommitError> {
+    use std::collections::BTreeMap;
+    let by_file: BTreeMap<DataFileId, Vec<(&[u8; CID_SIZE], BlockLocation)>> =
+        entries.iter().fold(BTreeMap::new(), |mut acc, (cid, loc)| {
+            acc.entry(loc.file_id).or_default().push((cid, *loc));
+            acc
+        });
+
+    by_file.into_iter().try_for_each(|(file_id, locations)| {
+        let path = manager.data_file_path(file_id);
+        let fd = match manager.io().open(&path, OpenOptions::read_only_existing()) {
+            Ok(fd) => fd,
+            Err(_) => return Ok(()),
+        };
+        let file_size = match manager.io().file_size(fd) {
+            Ok(s) => s,
+            Err(_) => {
+                let _ = manager.io().close(fd);
+                return Ok(());
+            }
+        };
+        let result = locations.into_iter().try_for_each(|(expected_cid, loc)| {
+            verify_block_at(manager, fd, file_size, expected_cid, loc)
+        });
+        let _ = manager.io().close(fd);
+        result
+    })
+}
+
+#[derive(Debug)]
+enum VerifyOutcome {
+    NoFaultDetected,
+    Faulted,
+}
+
+fn verify_block_at<S: StorageIO>(
+    manager: &DataFileManager<S>,
     fd: FileId,
+    file_size: u64,
+    expected_cid: &[u8; CID_SIZE],
+    loc: BlockLocation,
+) -> Result<(), CommitError> {
+    let passed = (0..VERIFY_RETRY_ATTEMPTS).any(|_| {
+        matches!(
+            verify_once(manager, fd, file_size, expected_cid, loc),
+            VerifyOutcome::NoFaultDetected
+        )
+    });
+    match passed {
+        true => Ok(()),
+        false => Err(CommitError::VerifyFailed {
+            file_id: loc.file_id,
+            offset: loc.offset,
+        }),
+    }
+}
+
+fn verify_once<S: StorageIO>(
+    manager: &DataFileManager<S>,
+    fd: FileId,
+    file_size: u64,
+    expected_cid: &[u8; CID_SIZE],
+    loc: BlockLocation,
+) -> VerifyOutcome {
+    match decode_block_record(manager.io(), fd, loc.offset, file_size) {
+        Ok(Some(ReadBlockRecord::Valid { cid_bytes, .. })) if cid_bytes == *expected_cid => {
+            VerifyOutcome::NoFaultDetected
+        }
+        Ok(Some(ReadBlockRecord::Valid { .. })) => {
+            tracing::warn!(
+                file_id = %loc.file_id,
+                offset = loc.offset.raw(),
+                "verify: stored CID mismatch (misdirected write)"
+            );
+            VerifyOutcome::Faulted
+        }
+        Ok(Some(ReadBlockRecord::Corrupted { .. } | ReadBlockRecord::Truncated { .. }))
+        | Ok(None) => {
+            tracing::warn!(
+                file_id = %loc.file_id,
+                offset = loc.offset.raw(),
+                "verify: block undecodable at location"
+            );
+            VerifyOutcome::Faulted
+        }
+        Err(_) => VerifyOutcome::NoFaultDetected,
+    }
+}
+
+const VERIFY_RETRY_ATTEMPTS: u32 = 4;
+
+fn rollback_batch<S: StorageIO>(
+    manager: &DataFileManager<S>,
+    state: &ActiveState,
+    rotations: &[RotationState<S>],
+) {
+    let _ = manager.io().truncate(state.fd, state.position.raw());
+    let _ = manager.io().sync(state.fd);
+    let _ = manager
+        .io()
+        .truncate(state.hint_fd, state.hint_position.raw());
+    let _ = manager.io().sync(state.hint_fd);
+    rotations.iter().for_each(|rot| {
+        manager.rollback_rotation(rot.file_id);
+        let _ = manager.io().close(rot.hint_fd);
+        let _ = manager
+            .io()
+            .delete(&hint_file_path(manager.data_dir(), rot.file_id));
+    });
 }
 
 fn process_batch<S: StorageIO>(
@@ -1071,13 +1211,46 @@ fn process_batch<S: StorageIO>(
     let mut all_decrements: Vec<[u8; CID_SIZE]> = Vec::new();
 
     let mut current_hint_fd = state.hint_fd;
-    let mut rotation: Option<RotationState> = None;
+    let mut rotations: Vec<RotationState<S>> = Vec::new();
 
     let mut data_writer =
         DataFileWriter::resume(manager.io(), state.fd, state.file_id, state.position);
     let mut hint_writer =
         HintFileWriter::resume(manager.io(), current_hint_fd, state.hint_position);
 
+    if manager.should_rotate(data_writer.position()) {
+        data_writer.sync().map_err(CommitError::from)?;
+        hint_writer.sync().map_err(CommitError::from)?;
+
+        let next_id = ctx.file_ids.allocate();
+        let next_handle = manager.open_for_append(next_id)?;
+        let next_fd = next_handle.fd();
+
+        tracing::info!(
+            from = %data_writer.file_id(),
+            to = %next_id,
+            trigger = "batch_boundary",
+            "data file rotation"
+        );
+
+        data_writer = DataFileWriter::new(manager.io(), next_fd, next_id)?;
+
+        let new_hint_path = hint_file_path(manager.data_dir(), next_id);
+        let new_hint_fd = manager
+            .io()
+            .open(&new_hint_path, OpenOptions::read_write())?;
+
+        manager.io().sync_dir(manager.data_dir())?;
+
+        current_hint_fd = new_hint_fd;
+        hint_writer = HintFileWriter::new(manager.io(), new_hint_fd);
+        rotations.push(RotationState {
+            file_id: next_id,
+            handle: next_handle,
+            hint_fd: new_hint_fd,
+        });
+    }
+
     let mut block_bytes: u64 = 0;
     let mut block_count: u64 = 0;
     let mut dedup_hits: u64 = 0;
@@ -1099,45 +1272,24 @@ fn process_batch<S: StorageIO>(
                     hint_writer.append_hint(cid_bytes, &loc)?;
                     loc
                 }
-                None => {
-                    if manager.should_rotate(data_writer.position()) {
-                        data_writer.sync()?;
-                        hint_writer.sync()?;
-
-                        let next_id = ctx.file_ids.allocate();
-                        let next_fd = manager.open_for_append(next_id)?;
-
-                        tracing::info!(
-                            from = %data_writer.file_id(),
-                            to = %next_id,
-                            "data file rotation"
-                        );
-
-                        data_writer = DataFileWriter::new(manager.io(), next_fd, next_id)?;
-
-                        let new_hint_path = hint_file_path(manager.data_dir(), next_id);
-                        let new_hint_fd = manager
-                            .io()
-                            .open(&new_hint_path, OpenOptions::read_write())?;
-
-                        manager.io().sync_dir(manager.data_dir())?;
-
-                        current_hint_fd = new_hint_fd;
-                        hint_writer = HintFileWriter::new(manager.io(), new_hint_fd);
-                        rotation = Some(RotationState {
-                            file_id: next_id,
-                            fd: next_fd,
-                        });
+                None => match index.get(cid_bytes) {
+                    Some(existing) => {
+                        dedup_hits = dedup_hits.saturating_add(1);
+                        let loc = existing.location;
+                        hint_writer.append_hint(cid_bytes, &loc)?;
+                        dedup.insert(*cid_bytes, loc);
+                        loc
                     }
+                    None => {
+                        let loc = data_writer.append_block(cid_bytes, data)?;
+                        hint_writer.append_hint(cid_bytes, &loc)?;
 
-                    let loc = data_writer.append_block(cid_bytes, data)?;
-                    hint_writer.append_hint(cid_bytes, &loc)?;
-
-                    block_bytes = block_bytes.saturating_add(data.len() as u64);
-                    block_count = block_count.saturating_add(1);
-                    dedup.insert(*cid_bytes, loc);
-                    loc
-                }
+                        block_bytes = block_bytes.saturating_add(data.len() as u64);
+                        block_count = block_count.saturating_add(1);
+                        dedup.insert(*cid_bytes, loc);
+                        loc
+                    }
+                },
             };
 
             index_entries.push((*cid_bytes, location));
@@ -1152,9 +1304,7 @@ fn process_batch<S: StorageIO>(
     });
 
     if let Err(e) = write_result {
-        if let Some(rot) = rotation {
-            manager.rollback_rotation(rot.file_id, rot.fd);
-        }
+        rollback_batch(manager, state, &rotations);
         return Err(e);
     }
 
@@ -1163,18 +1313,53 @@ fn process_batch<S: StorageIO>(
     let current_epoch = epoch.current();
     let now = crate::wall_clock_ms();
 
+    let rollback_on_err = |e: CommitError| -> CommitError {
+        rollback_batch(manager, state, &rotations);
+        e
+    };
+
     all_decrements
         .iter()
-        .try_for_each(|cid| hint_writer.append_decrement(cid, current_epoch, now))?;
+        .try_for_each(|cid| hint_writer.append_decrement(cid, current_epoch, now))
+        .map_err(|e| rollback_on_err(CommitError::from(e)))?;
 
     let t = std::time::Instant::now();
-    data_writer.sync()?;
-    hint_writer.sync()?;
+    data_writer.sync().map_err(|e| rollback_on_err(e.into()))?;
+    if ctx.verify_persisted_blocks {
+        verify_persisted_blocks(manager, &index_entries).map_err(rollback_on_err)?;
+    }
+    let batch_record_count = u32::try_from(
+        block_count
+            .saturating_add(dedup_hits)
+            .saturating_add(all_decrements.len() as u64),
+    )
+    .unwrap_or(u32::MAX);
+    hint_writer
+        .append_commit_marker(
+            current_epoch.raw(),
+            batch_record_count,
+            data_writer.file_id(),
+            data_writer.position(),
+        )
+        .map_err(|e| rollback_on_err(CommitError::from(e)))?;
+    hint_writer.sync().map_err(|e| rollback_on_err(e.into()))?;
     let sync_nanos = t.elapsed().as_nanos() as u64;
 
-    if let Some(ref rot) = rotation {
-        manager.commit_rotation(rot.file_id, rot.fd);
-        ctx.active_files.register(ctx.shard_id, rot.file_id);
+    if !rotations.is_empty() {
+        let old_file_id = state.file_id;
+        let old_hint_fd = state.hint_fd;
+        let last_idx = rotations.len() - 1;
+        rotations.iter().enumerate().for_each(|(i, rot)| {
+            if i == last_idx {
+                manager.commit_rotation(rot.file_id, &rot.handle);
+                ctx.active_files.register(ctx.shard_id, rot.file_id);
+            } else {
+                let _ = manager.io().close(rot.hint_fd);
+                manager.evict_handle(rot.file_id);
+            }
+        });
+        manager.evict_handle(old_file_id);
+        let _ = manager.io().close(old_hint_fd);
     }
 
     state.file_id = data_writer.file_id();

crates/tranquil-store/src/blockstore/hash_index.rs

@@ -704,6 +704,7 @@ impl HashTable {
 const CHECKPOINT_MAGIC: [u8; 8] = *b"TQCKPT01";
 const CHECKPOINT_VERSION_V1: u32 = 1;
 const CHECKPOINT_VERSION_V2: u32 = 2;
+const CHECKPOINT_VERSION_V3: u32 = 3;
 const CHECKPOINT_HEADER_SIZE: usize = 128;
 const TRAILER_MAGIC: u64 = 0xDEAD_BEEF_CAFE_F00D;
 const SLOT_SIZE: usize = std::mem::size_of::<Slot>();
@@ -733,6 +734,7 @@ const H_CHECKPOINT_EPOCH: usize = 56;
 const H_HINT_FILE_ID: usize = 64;
 const H_HINT_OFFSET: usize = 72;
 const H_HEADER_CHECKSUM: usize = 80;
+const H_GENERATION: usize = 88;
 
 fn header_checksum(buf: &[u8; CHECKPOINT_HEADER_SIZE]) -> u64 {
     xxhash_rust::xxh3::xxh3_64(&buf[..H_HEADER_CHECKSUM])
@@ -745,10 +747,11 @@ fn serialize_header(
     cursor_offset: u64,
     checkpoint_epoch: u64,
     shard_count: u16,
+    generation: u64,
 ) -> [u8; CHECKPOINT_HEADER_SIZE] {
     let mut buf = [0u8; CHECKPOINT_HEADER_SIZE];
     buf[H_MAGIC..H_MAGIC + 8].copy_from_slice(&CHECKPOINT_MAGIC);
-    buf[H_VERSION..H_VERSION + 4].copy_from_slice(&CHECKPOINT_VERSION_V2.to_le_bytes());
+    buf[H_VERSION..H_VERSION + 4].copy_from_slice(&CHECKPOINT_VERSION_V3.to_le_bytes());
     buf[H_SHARD_COUNT..H_SHARD_COUNT + 2].copy_from_slice(&shard_count.to_le_bytes());
     buf[H_SLOT_COUNT..H_SLOT_COUNT + 8].copy_from_slice(&slot_count.to_le_bytes());
     buf[H_ENTRY_COUNT..H_ENTRY_COUNT + 8].copy_from_slice(&entry_count.to_le_bytes());
@@ -756,6 +759,7 @@ fn serialize_header(
     buf[H_CURSOR_OFFSET..H_CURSOR_OFFSET + 8].copy_from_slice(&cursor_offset.to_le_bytes());
     buf[H_CHECKPOINT_EPOCH..H_CHECKPOINT_EPOCH + 8]
         .copy_from_slice(&checkpoint_epoch.to_le_bytes());
+    buf[H_GENERATION..H_GENERATION + 8].copy_from_slice(&generation.to_le_bytes());
     let checksum = header_checksum(&buf);
     buf[H_HEADER_CHECKSUM..H_HEADER_CHECKSUM + 8].copy_from_slice(&checksum.to_le_bytes());
     buf
@@ -796,6 +800,7 @@ pub fn write_checkpoint(
     table: &HashTable,
     path: &Path,
     epoch: CommitEpoch,
+    generation: u64,
     positions: &CheckpointPositions,
 ) -> io::Result<()> {
     use std::io::Write;
@@ -817,6 +822,7 @@ pub fn write_checkpoint(
         cursor_offset,
         epoch.raw(),
         shard_count,
+        generation,
     );
 
     let slot_bytes = slots_as_bytes(&table.slots);
@@ -844,7 +850,7 @@ pub fn write_checkpoint(
     Ok(())
 }
 
-fn parse_checkpoint_header(data: &[u8]) -> io::Result<(usize, usize, u32, u64, u64, u16)> {
+fn parse_checkpoint_header(data: &[u8]) -> io::Result<(usize, usize, u32, u64, u64, u16, u64)> {
     if data.len() < CHECKPOINT_HEADER_SIZE + 16 {
         return Err(io::Error::new(
             io::ErrorKind::InvalidData,
@@ -863,7 +869,10 @@ fn parse_checkpoint_header(data: &[u8]) -> io::Result<(usize, usize, u32, u64, u
     }
 
     let version = u32::from_le_bytes(hdr[H_VERSION..H_VERSION + 4].try_into().unwrap());
-    if version != CHECKPOINT_VERSION_V1 && version != CHECKPOINT_VERSION_V2 {
+    if version != CHECKPOINT_VERSION_V1
+        && version != CHECKPOINT_VERSION_V2
+        && version != CHECKPOINT_VERSION_V3
+    {
         return Err(io::Error::new(
             io::ErrorKind::InvalidData,
             format!("checkpoint version {version} unsupported"),
@@ -921,12 +930,19 @@ fn parse_checkpoint_header(data: &[u8]) -> io::Result<(usize, usize, u32, u64, u
     );
 
     let shard_count = match version {
-        CHECKPOINT_VERSION_V2 => {
+        CHECKPOINT_VERSION_V2 | CHECKPOINT_VERSION_V3 => {
             u16::from_le_bytes(hdr[H_SHARD_COUNT..H_SHARD_COUNT + 2].try_into().unwrap())
         }
         _ => 0,
     };
 
+    let generation = match version {
+        CHECKPOINT_VERSION_V3 => {
+            u64::from_le_bytes(hdr[H_GENERATION..H_GENERATION + 8].try_into().unwrap())
+        }
+        _ => 0,
+    };
+
     Ok((
         slot_count,
         entry_count,
@@ -934,6 +950,7 @@ fn parse_checkpoint_header(data: &[u8]) -> io::Result<(usize, usize, u32, u64, u
         cursor_offset,
         checkpoint_epoch,
         shard_count,
+        generation,
     ))
 }
 
@@ -948,11 +965,20 @@ fn deserialize_shard_positions(data: &[u8], count: usize) -> Vec<(DataFileId, Hi
         .collect()
 }
 
-pub fn read_checkpoint(path: &Path) -> io::Result<(HashTable, CommitEpoch, CheckpointPositions)> {
+pub fn read_checkpoint(
+    path: &Path,
+) -> io::Result<(HashTable, CommitEpoch, CheckpointPositions, u64)> {
     let data = std::fs::read(path)?;
 
-    let (slot_count, entry_count, cursor_file_id, cursor_offset, checkpoint_epoch, shard_count) =
-        parse_checkpoint_header(&data)?;
+    let (
+        slot_count,
+        entry_count,
+        cursor_file_id,
+        cursor_offset,
+        checkpoint_epoch,
+        shard_count,
+        generation,
+    ) = parse_checkpoint_header(&data)?;
 
     let hdr: &[u8; CHECKPOINT_HEADER_SIZE] = data[..CHECKPOINT_HEADER_SIZE].try_into().unwrap();
     let version = u32::from_le_bytes(hdr[H_VERSION..H_VERSION + 4].try_into().unwrap());
@@ -972,7 +998,7 @@ pub fn read_checkpoint(path: &Path) -> io::Result<(HashTable, CommitEpoch, Check
     let shard_pos_region = &data[shard_pos_start..shard_pos_start + shard_pos_size];
 
     let data_checksum = match version {
-        CHECKPOINT_VERSION_V2 => {
+        CHECKPOINT_VERSION_V2 | CHECKPOINT_VERSION_V3 => {
             let mut hasher = xxhash_rust::xxh3::Xxh3::new();
             hasher.update(slot_region);
             hasher.update(shard_pos_region);
@@ -1035,7 +1061,7 @@ pub fn read_checkpoint(path: &Path) -> io::Result<(HashTable, CommitEpoch, Check
     let epoch = CommitEpoch::new(checkpoint_epoch);
 
     let positions = match version {
-        CHECKPOINT_VERSION_V2 if shard_count > 0 => CheckpointPositions(
+        CHECKPOINT_VERSION_V2 | CHECKPOINT_VERSION_V3 if shard_count > 0 => CheckpointPositions(
             deserialize_shard_positions(shard_pos_region, shard_count as usize),
         ),
         _ => {
@@ -1047,12 +1073,12 @@ pub fn read_checkpoint(path: &Path) -> io::Result<(HashTable, CommitEpoch, Check
         }
     };
 
-    Ok((table, epoch, positions))
+    Ok((table, epoch, positions, generation))
 }
 
 pub fn load_best_checkpoint(
     index_dir: &Path,
-) -> Option<(HashTable, CommitEpoch, CheckpointPositions)> {
+) -> Option<(HashTable, CommitEpoch, CheckpointPositions, u64)> {
     let path_a = index_dir.join("checkpoint_a.tqc");
     let path_b = index_dir.join("checkpoint_b.tqc");
 
@@ -1060,7 +1086,7 @@ pub fn load_best_checkpoint(
     let result_b = read_checkpoint(&path_b).ok();
 
     match (result_a, result_b) {
-        (Some(a), Some(b)) => match a.1.raw() >= b.1.raw() {
+        (Some(a), Some(b)) => match (a.3, a.1.raw()) >= (b.3, b.1.raw()) {
             true => Some(a),
             false => Some(b),
         },
@@ -1070,7 +1096,7 @@ pub fn load_best_checkpoint(
     }
 }
 
-fn read_checkpoint_epoch(path: &Path) -> Option<u64> {
+fn read_checkpoint_meta(path: &Path) -> Option<(u64, u64)> {
     let mut file = std::fs::File::open(path).ok()?;
     let mut buf = [0u8; CHECKPOINT_HEADER_SIZE];
     std::io::Read::read_exact(&mut file, &mut buf).ok()?;
@@ -1081,7 +1107,10 @@ fn read_checkpoint_epoch(path: &Path) -> Option<u64> {
     }
 
     let version = u32::from_le_bytes(buf[H_VERSION..H_VERSION + 4].try_into().ok()?);
-    if version != CHECKPOINT_VERSION_V1 && version != CHECKPOINT_VERSION_V2 {
+    if version != CHECKPOINT_VERSION_V1
+        && version != CHECKPOINT_VERSION_V2
+        && version != CHECKPOINT_VERSION_V3
+    {
         return None;
     }
 
@@ -1094,33 +1123,41 @@ fn read_checkpoint_epoch(path: &Path) -> Option<u64> {
         return None;
     }
 
-    Some(u64::from_le_bytes(
+    let epoch = u64::from_le_bytes(
         buf[H_CHECKPOINT_EPOCH..H_CHECKPOINT_EPOCH + 8]
             .try_into()
             .ok()?,
-    ))
+    );
+    let generation = match version {
+        CHECKPOINT_VERSION_V3 => {
+            u64::from_le_bytes(buf[H_GENERATION..H_GENERATION + 8].try_into().ok()?)
+        }
+        _ => 0,
+    };
+    Some((epoch, generation))
 }
 
 pub fn write_checkpoint_ab(
     table: &HashTable,
     index_dir: &Path,
     epoch: CommitEpoch,
+    generation: u64,
     positions: &CheckpointPositions,
 ) -> io::Result<()> {
     let path_a = index_dir.join("checkpoint_a.tqc");
     let path_b = index_dir.join("checkpoint_b.tqc");
 
-    let epoch_a = read_checkpoint_epoch(&path_a);
-    let epoch_b = read_checkpoint_epoch(&path_b);
+    let meta_a = read_checkpoint_meta(&path_a);
+    let meta_b = read_checkpoint_meta(&path_b);
 
-    let target_path = match (epoch_a, epoch_b) {
-        (Some(a), Some(b)) if a >= b => path_b,
+    let target_path = match (meta_a, meta_b) {
+        (Some(a), Some(b)) if (a.1, a.0) >= (b.1, b.0) => path_b,
         (Some(_), Some(_)) => path_a,
         (Some(_), None) => path_b,
         (None, _) => path_a,
     };
 
-    write_checkpoint(table, &target_path, epoch, positions)
+    write_checkpoint(table, &target_path, epoch, generation, positions)
 }
 
 #[derive(Debug)]
@@ -1146,6 +1183,7 @@ pub struct BlockIndex {
     checkpoint_lock: parking_lot::Mutex<()>,
     loaded_checkpoint_positions: Option<CheckpointPositions>,
     loaded_checkpoint_epoch: Option<CommitEpoch>,
+    next_generation: std::sync::atomic::AtomicU64,
 }
 
 impl BlockIndex {
@@ -1156,33 +1194,36 @@ impl BlockIndex {
             checkpoint_lock: parking_lot::Mutex::new(()),
             loaded_checkpoint_positions: None,
             loaded_checkpoint_epoch: None,
+            next_generation: std::sync::atomic::AtomicU64::new(1),
         }
     }
 
     pub fn open(index_dir: &Path) -> io::Result<Self> {
         std::fs::create_dir_all(index_dir)?;
-        let (table, checkpoint_positions, checkpoint_epoch) = match load_best_checkpoint(index_dir)
-        {
-            Some((table, epoch, positions)) => {
-                tracing::info!(
-                    blocks = table.len(),
-                    epoch = epoch.raw(),
-                    shard_positions = positions.0.len(),
-                    "loaded block index from checkpoint"
-                );
-                (table, Some(positions), Some(epoch))
-            }
-            None => {
-                tracing::info!("no valid checkpoint found, starting with empty index");
-                (HashTable::with_capacity(64), None, None)
-            }
-        };
+        let (table, checkpoint_positions, checkpoint_epoch, loaded_generation) =
+            match load_best_checkpoint(index_dir) {
+                Some((table, epoch, positions, gen_value)) => {
+                    tracing::info!(
+                        blocks = table.len(),
+                        epoch = epoch.raw(),
+                        shard_positions = positions.0.len(),
+                        generation = gen_value,
+                        "loaded block index from checkpoint"
+                    );
+                    (table, Some(positions), Some(epoch), gen_value)
+                }
+                None => {
+                    tracing::info!("no valid checkpoint found, starting with empty index");
+                    (HashTable::with_capacity(64), None, None, 0)
+                }
+            };
         Ok(Self {
             table: RwLock::new(table),
             index_dir: index_dir.to_path_buf(),
             checkpoint_lock: parking_lot::Mutex::new(()),
             loaded_checkpoint_positions: checkpoint_positions,
             loaded_checkpoint_epoch: checkpoint_epoch,
+            next_generation: std::sync::atomic::AtomicU64::new(loaded_generation + 1),
         })
     }
 
@@ -1202,6 +1243,15 @@ impl BlockIndex {
         self.table.read().contains_live(cid)
     }
 
+    pub fn live_entries_snapshot(&self) -> Vec<([u8; CID_SIZE], RefCount)> {
+        self.table
+            .read()
+            .iter()
+            .filter(|s| !s.refcount.is_zero())
+            .map(|s| (s.cid, s.refcount))
+            .collect()
+    }
+
     pub fn batch_put(
         &self,
         entries: &[([u8; CID_SIZE], BlockLocation)],
@@ -1222,7 +1272,14 @@ impl BlockIndex {
         now: WallClockMs,
         position_update: PositionUpdate<'_>,
     ) -> Result<(), BlockIndexError> {
-        self.batch_put_inner(entries, decrements, cursor, epoch, now, Some(position_update))
+        self.batch_put_inner(
+            entries,
+            decrements,
+            cursor,
+            epoch,
+            now,
+            Some(position_update),
+        )
     }
 
     fn batch_put_inner(
@@ -1465,9 +1522,12 @@ impl BlockIndex {
         hint_positions: &ShardHintPositions,
     ) -> io::Result<()> {
         let _guard = self.checkpoint_lock.lock();
+        let generation = self
+            .next_generation
+            .fetch_add(1, std::sync::atomic::Ordering::AcqRel);
         let table = self.table.read();
         let positions = hint_positions.snapshot();
-        write_checkpoint_ab(&table, &self.index_dir, epoch, &positions)
+        write_checkpoint_ab(&table, &self.index_dir, epoch, generation, &positions)
     }
 
     pub fn write_checkpoint_with_positions(
@@ -1476,8 +1536,11 @@ impl BlockIndex {
         positions: &CheckpointPositions,
     ) -> io::Result<()> {
         let _guard = self.checkpoint_lock.lock();
+        let generation = self
+            .next_generation
+            .fetch_add(1, std::sync::atomic::Ordering::AcqRel);
         let table = self.table.read();
-        write_checkpoint_ab(&table, &self.index_dir, epoch, positions)
+        write_checkpoint_ab(&table, &self.index_dir, epoch, generation, positions)
     }
 
     pub fn index_dir(&self) -> &Path {
@@ -1574,7 +1637,8 @@ impl BlockIndex {
                     ReadHintRecord::Remove { cid_bytes } => {
                         let _ = table.remove(&cid_bytes);
                     }
-                    ReadHintRecord::UnknownVersion { .. }
+                    ReadHintRecord::CommitMarker { .. }
+                    | ReadHintRecord::UnknownVersion { .. }
                     | ReadHintRecord::UnknownType { .. }
                     | ReadHintRecord::Corrupted
                     | ReadHintRecord::Truncated => {}
@@ -2029,8 +2093,8 @@ mod tests {
         let epoch = CommitEpoch::new(42);
         let positions = CheckpointPositions::single(DataFileId::new(5), HintOffset::new(12345));
 
-        write_checkpoint(&table, &path, epoch, &positions).unwrap();
-        let (restored, restored_epoch, restored_pos) = read_checkpoint(&path).unwrap();
+        write_checkpoint(&table, &path, epoch, 7, &positions).unwrap();
+        let (restored, restored_epoch, restored_pos, _gen) = read_checkpoint(&path).unwrap();
 
         assert_eq!(restored.len(), 10);
         assert_eq!(restored_epoch.raw(), 42);
@@ -2062,14 +2126,14 @@ mod tests {
         table
             .insert_or_increment(&test_cid(1), test_loc(0, 0, 10))
             .unwrap();
-        write_checkpoint_ab(&table, dir.path(), CommitEpoch::new(1), &pos).unwrap();
+        write_checkpoint_ab(&table, dir.path(), CommitEpoch::new(1), 1, &pos).unwrap();
 
         table
             .insert_or_increment(&test_cid(2), test_loc(0, 100, 10))
             .unwrap();
-        write_checkpoint_ab(&table, dir.path(), CommitEpoch::new(2), &pos).unwrap();
+        write_checkpoint_ab(&table, dir.path(), CommitEpoch::new(2), 2, &pos).unwrap();
 
-        let (best, epoch, _) = load_best_checkpoint(dir.path()).unwrap();
+        let (best, epoch, _, _) = load_best_checkpoint(dir.path()).unwrap();
         assert_eq!(epoch.raw(), 2);
         assert_eq!(best.len(), 2);
     }
@@ -2083,16 +2147,16 @@ mod tests {
         table
             .insert_or_increment(&test_cid(1), test_loc(0, 0, 10))
             .unwrap();
-        write_checkpoint_ab(&table, dir.path(), CommitEpoch::new(1), &pos).unwrap();
+        write_checkpoint_ab(&table, dir.path(), CommitEpoch::new(1), 1, &pos).unwrap();
 
         table
             .insert_or_increment(&test_cid(2), test_loc(0, 100, 10))
             .unwrap();
-        write_checkpoint_ab(&table, dir.path(), CommitEpoch::new(2), &pos).unwrap();
+        write_checkpoint_ab(&table, dir.path(), CommitEpoch::new(2), 2, &pos).unwrap();
 
         std::fs::write(dir.path().join("checkpoint_b.tqc"), b"corrupt").unwrap();
 
-        let (best, epoch, _) = load_best_checkpoint(dir.path()).unwrap();
+        let (best, epoch, _, _) = load_best_checkpoint(dir.path()).unwrap();
         assert_eq!(epoch.raw(), 1);
         assert_eq!(best.len(), 1);
     }

crates/tranquil-store/src/blockstore/hint.rs

@@ -22,6 +22,7 @@ const RECORD_TYPE_PUT: u8 = 0x01;
 const RECORD_TYPE_DECREMENT: u8 = 0x02;
 const RECORD_TYPE_RELOCATE: u8 = 0x03;
 const RECORD_TYPE_REMOVE: u8 = 0x04;
+const RECORD_TYPE_COMMIT_MARKER: u8 = 0x05;
 
 const HINT_FORMAT_VERSION: u8 = 1;
 
@@ -57,10 +58,8 @@ fn write_hint_record<S: StorageIO>(
 
 fn encode_location_fields(record: &mut [u8; HINT_RECORD_SIZE], loc: &BlockLocation) {
     record[FIELD_A_OFFSET..FIELD_A_OFFSET + 4].copy_from_slice(&loc.file_id.raw().to_le_bytes());
-    record[FIELD_A_OFFSET + 4..FIELD_A_OFFSET + 8]
-        .copy_from_slice(&loc.length.raw().to_le_bytes());
-    record[FIELD_B_OFFSET..FIELD_B_OFFSET + 8]
-        .copy_from_slice(&loc.offset.raw().to_le_bytes());
+    record[FIELD_A_OFFSET + 4..FIELD_A_OFFSET + 8].copy_from_slice(&loc.length.raw().to_le_bytes());
+    record[FIELD_B_OFFSET..FIELD_B_OFFSET + 8].copy_from_slice(&loc.offset.raw().to_le_bytes());
 }
 
 pub(crate) fn encode_hint_record<S: StorageIO>(
@@ -144,6 +143,36 @@ pub(crate) fn encode_decrement_record<S: StorageIO>(
     write_hint_record(io, fd, write_offset, &record)
 }
 
+const MARKER_DATA_OFFSET_POS: usize = CID_OFFSET;
+const MARKER_DATA_FILE_ID_POS: usize = CID_OFFSET + 8;
+const MARKER_RECORD_COUNT_POS: usize = FIELD_B_OFFSET;
+
+pub(crate) fn encode_commit_marker_record<S: StorageIO>(
+    io: &S,
+    fd: FileId,
+    write_offset: HintOffset,
+    batch_seq: u64,
+    record_count: u32,
+    data_file_id: DataFileId,
+    data_offset: BlockOffset,
+) -> io::Result<()> {
+    let mut record = [0u8; HINT_RECORD_SIZE];
+    record[TYPE_OFFSET] = RECORD_TYPE_COMMIT_MARKER;
+    record[VERSION_OFFSET] = HINT_FORMAT_VERSION;
+    record[MARKER_DATA_OFFSET_POS..MARKER_DATA_OFFSET_POS + 8]
+        .copy_from_slice(&data_offset.raw().to_le_bytes());
+    record[MARKER_DATA_FILE_ID_POS..MARKER_DATA_FILE_ID_POS + 4]
+        .copy_from_slice(&data_file_id.raw().to_le_bytes());
+    record[FIELD_A_OFFSET..FIELD_A_OFFSET + 8].copy_from_slice(&batch_seq.to_le_bytes());
+    record[MARKER_RECORD_COUNT_POS..MARKER_RECORD_COUNT_POS + 4]
+        .copy_from_slice(&record_count.to_le_bytes());
+
+    let checksum = hint_checksum(&record[..HINT_PAYLOAD_SIZE]);
+    record[CHECKSUM_OFFSET..].copy_from_slice(&checksum.to_le_bytes());
+
+    write_hint_record(io, fd, write_offset, &record)
+}
+
 #[must_use]
 #[derive(Debug)]
 pub enum ReadHintRecord {
@@ -168,6 +197,12 @@ pub enum ReadHintRecord {
     Remove {
         cid_bytes: [u8; CID_SIZE],
     },
+    CommitMarker {
+        batch_seq: u64,
+        record_count: u32,
+        data_file_id: DataFileId,
+        data_offset: BlockOffset,
+    },
     UnknownVersion {
         version: u8,
     },
@@ -297,6 +332,34 @@ pub fn decode_hint_record<S: StorageIO>(
             }))
         }
         RECORD_TYPE_REMOVE => Ok(Some(ReadHintRecord::Remove { cid_bytes })),
+        RECORD_TYPE_COMMIT_MARKER => {
+            let data_offset = BlockOffset::new(u64::from_le_bytes(
+                record[MARKER_DATA_OFFSET_POS..MARKER_DATA_OFFSET_POS + 8]
+                    .try_into()
+                    .unwrap(),
+            ));
+            let data_file_id = DataFileId::new(u32::from_le_bytes(
+                record[MARKER_DATA_FILE_ID_POS..MARKER_DATA_FILE_ID_POS + 4]
+                    .try_into()
+                    .unwrap(),
+            ));
+            let batch_seq = u64::from_le_bytes(
+                record[FIELD_A_OFFSET..FIELD_A_OFFSET + 8]
+                    .try_into()
+                    .unwrap(),
+            );
+            let record_count = u32::from_le_bytes(
+                record[MARKER_RECORD_COUNT_POS..MARKER_RECORD_COUNT_POS + 4]
+                    .try_into()
+                    .unwrap(),
+            );
+            Ok(Some(ReadHintRecord::CommitMarker {
+                batch_seq,
+                record_count,
+                data_file_id,
+                data_offset,
+            }))
+        }
         other => Ok(Some(ReadHintRecord::UnknownType { record_type: other })),
     }
 }
@@ -358,6 +421,26 @@ impl<'a, S: StorageIO> HintFileWriter<'a, S> {
         Ok(())
     }
 
+    pub fn append_commit_marker(
+        &mut self,
+        batch_seq: u64,
+        record_count: u32,
+        data_file_id: DataFileId,
+        data_offset: BlockOffset,
+    ) -> io::Result<()> {
+        encode_commit_marker_record(
+            self.io,
+            self.fd,
+            self.position,
+            batch_seq,
+            record_count,
+            data_file_id,
+            data_offset,
+        )?;
+        self.position = self.position.advance(HINT_RECORD_SIZE as u64);
+        Ok(())
+    }
+
     pub fn sync(&self) -> io::Result<()> {
         self.io.sync(self.fd)
     }
@@ -418,12 +501,13 @@ impl<S: StorageIO> Iterator for HintFileReader<'_, S> {
                     | ReadHintRecord::Decrement { .. }
                     | ReadHintRecord::Relocate { .. }
                     | ReadHintRecord::Remove { .. }
-                    | ReadHintRecord::UnknownType { .. } => {
+                    | ReadHintRecord::CommitMarker { .. }
+                    | ReadHintRecord::UnknownType { .. }
+                    | ReadHintRecord::UnknownVersion { .. }
+                    | ReadHintRecord::Corrupted => {
                         self.position = self.position.advance(HINT_RECORD_SIZE as u64);
                     }
-                    ReadHintRecord::UnknownVersion { .. }
-                    | ReadHintRecord::Corrupted
-                    | ReadHintRecord::Truncated => {
+                    ReadHintRecord::Truncated => {
                         self.position = HintOffset::new(self.file_size);
                     }
                 }
@@ -531,6 +615,7 @@ fn scan_single_hint_file<S: StorageIO>(
                 ReadHintRecord::Decrement { .. }
                 | ReadHintRecord::Relocate { .. }
                 | ReadHintRecord::Remove { .. }
+                | ReadHintRecord::CommitMarker { .. }
                 | ReadHintRecord::UnknownVersion { .. }
                 | ReadHintRecord::UnknownType { .. }
                 | ReadHintRecord::Corrupted
@@ -544,7 +629,95 @@ fn scan_single_hint_file<S: StorageIO>(
     entries
 }
 
-const REPLAY_BATCH_SIZE: usize = 10_000;
+#[derive(Default)]
+struct PendingBatch {
+    puts: Vec<([u8; CID_SIZE], BlockLocation)>,
+    relocates: Vec<([u8; CID_SIZE], BlockLocation, u32)>,
+    removes: Vec<[u8; CID_SIZE]>,
+    decrements: Vec<([u8; CID_SIZE], CommitEpoch, WallClockMs)>,
+    file_cursors: HashMap<DataFileId, BlockOffset>,
+    max_cursor: Option<WriteCursor>,
+    record_count: u32,
+    boundary_lost: bool,
+}
+
+impl PendingBatch {
+    fn reset(&mut self) {
+        self.puts.clear();
+        self.relocates.clear();
+        self.removes.clear();
+        self.decrements.clear();
+        self.file_cursors.clear();
+        self.max_cursor = None;
+        self.record_count = 0;
+        self.boundary_lost = false;
+    }
+
+    fn note_record(&mut self) {
+        self.record_count = self.record_count.saturating_add(1);
+    }
+
+    fn track_cursor(&mut self, file_id: DataFileId, end: BlockOffset) {
+        let candidate = WriteCursor {
+            file_id,
+            offset: end,
+        };
+        self.max_cursor = Some(match self.max_cursor {
+            Some(c) => std::cmp::max_by_key(c, candidate, |w| (w.file_id, w.offset)),
+            None => candidate,
+        });
+        self.file_cursors
+            .entry(file_id)
+            .and_modify(|existing| {
+                if end > *existing {
+                    *existing = end;
+                }
+            })
+            .or_insert(end);
+    }
+}
+
+fn commit_pending_batch(
+    pending: &mut PendingBatch,
+    index: &super::hash_index::BlockIndex,
+    file_cursors: &mut HashMap<DataFileId, BlockOffset>,
+    max_cursor: &mut Option<WriteCursor>,
+    replayed: &mut u64,
+) -> Result<(), RebuildError> {
+    if !pending.puts.is_empty() {
+        index.batch_insert_buffered(&pending.puts)?;
+    }
+    if !pending.relocates.is_empty() {
+        index.batch_relocate(&pending.relocates)?;
+    }
+    if !pending.removes.is_empty() {
+        index.batch_remove(&pending.removes);
+    }
+    pending
+        .decrements
+        .iter()
+        .try_for_each(|(cid, epoch, ts)| index.batch_decrement(&[*cid], *epoch, *ts))?;
+
+    pending.file_cursors.iter().for_each(|(fid, end)| {
+        file_cursors
+            .entry(*fid)
+            .and_modify(|existing| {
+                if *end > *existing {
+                    *existing = *end;
+                }
+            })
+            .or_insert(*end);
+    });
+    if let Some(c) = pending.max_cursor {
+        *max_cursor = Some(match *max_cursor {
+            Some(m) => std::cmp::max_by_key(m, c, |w| (w.file_id, w.offset)),
+            None => c,
+        });
+    }
+    *replayed = replayed.saturating_add(u64::from(pending.record_count));
+    pending.reset();
+    Ok(())
+}
 
 pub fn replay_hints_into_block_index<S: StorageIO>(
     io: &S,
@@ -570,11 +743,7 @@ pub fn replay_hints_into_block_index<S: StorageIO>(
     let mut max_cursor: Option<WriteCursor> = None;
     let mut file_cursors: HashMap<DataFileId, BlockOffset> = HashMap::new();
     let mut replayed: u64 = 0;
-    let mut put_buffer: Vec<([u8; CID_SIZE], BlockLocation)> =
-        Vec::with_capacity(REPLAY_BATCH_SIZE);
-    let mut relocate_buffer: Vec<([u8; CID_SIZE], BlockLocation, u32)> =
-        Vec::with_capacity(REPLAY_BATCH_SIZE);
-    let mut remove_buffer: Vec<[u8; CID_SIZE]> = Vec::with_capacity(REPLAY_BATCH_SIZE);
+    let mut pending = PendingBatch::default();
 
     hint_files
         .iter()
@@ -606,54 +775,19 @@ pub fn replay_hints_into_block_index<S: StorageIO>(
                             offset,
                             length,
                         };
-                        put_buffer.push((cid_bytes, loc));
-
                         let record_end =
                             offset.advance(BLOCK_RECORD_OVERHEAD as u64 + length.as_u64());
-                        let candidate = WriteCursor {
-                            file_id,
-                            offset: record_end,
-                        };
-                        max_cursor = Some(match max_cursor {
-                            Some(c) => {
-                                std::cmp::max_by_key(c, candidate, |w| (w.file_id, w.offset))
-                            }
-                            None => candidate,
-                        });
-                        file_cursors
-                            .entry(file_id)
-                            .and_modify(|existing| {
-                                if record_end > *existing {
-                                    *existing = record_end;
-                                }
-                            })
-                            .or_insert(record_end);
-
-                        replayed = replayed.saturating_add(1);
-                        if put_buffer.len() >= REPLAY_BATCH_SIZE {
-                            index.batch_insert_buffered(&put_buffer)?;
-                            put_buffer.clear();
-                        }
+                        pending.puts.push((cid_bytes, loc));
+                        pending.track_cursor(file_id, record_end);
+                        pending.note_record();
                     }
                     ReadHintRecord::Decrement {
                         cid_bytes,
                         epoch,
                         timestamp,
                     } => {
-                        if !put_buffer.is_empty() {
-                            index.batch_insert_buffered(&put_buffer)?;
-                            put_buffer.clear();
-                        }
-                        if !relocate_buffer.is_empty() {
-                            index.batch_relocate(&relocate_buffer)?;
-                            relocate_buffer.clear();
-                        }
-                        if !remove_buffer.is_empty() {
-                            index.batch_remove(&remove_buffer);
-                            remove_buffer.clear();
-                        }
-                        index.batch_decrement(&[cid_bytes], epoch, timestamp)?;
-                        replayed = replayed.saturating_add(1);
+                        pending.decrements.push((cid_bytes, epoch, timestamp));
+                        pending.note_record();
                     }
                     ReadHintRecord::Relocate {
                         cid_bytes,
@@ -667,75 +801,71 @@ pub fn replay_hints_into_block_index<S: StorageIO>(
                             offset,
                             length,
                         };
-                        relocate_buffer.push((cid_bytes, loc, refcount));
-
                         let record_end =
                             offset.advance(BLOCK_RECORD_OVERHEAD as u64 + length.as_u64());
-                        file_cursors
-                            .entry(file_id)
-                            .and_modify(|existing| {
-                                if record_end > *existing {
-                                    *existing = record_end;
-                                }
-                            })
-                            .or_insert(record_end);
-
-                        replayed = replayed.saturating_add(1);
-                        if relocate_buffer.len() >= REPLAY_BATCH_SIZE {
-                            if !put_buffer.is_empty() {
-                                index.batch_insert_buffered(&put_buffer)?;
-                                put_buffer.clear();
-                            }
-                            index.batch_relocate(&relocate_buffer)?;
-                            relocate_buffer.clear();
-                        }
+                        pending.relocates.push((cid_bytes, loc, refcount));
+                        pending.track_cursor(file_id, record_end);
+                        pending.note_record();
                     }
                     ReadHintRecord::Remove { cid_bytes } => {
-                        remove_buffer.push(cid_bytes);
-                        replayed = replayed.saturating_add(1);
-                        if remove_buffer.len() >= REPLAY_BATCH_SIZE {
-                            if !put_buffer.is_empty() {
-                                index.batch_insert_buffered(&put_buffer)?;
-                                put_buffer.clear();
+                        pending.removes.push(cid_bytes);
+                        pending.note_record();
+                    }
+                    ReadHintRecord::CommitMarker {
+                        batch_seq,
+                        record_count,
+                        data_file_id,
+                        data_offset,
+                    } => {
+                        let accepts =
+                            !pending.boundary_lost && pending.record_count == record_count;
+                        match accepts {
+                            true => {
+                                pending.track_cursor(data_file_id, data_offset);
+                                commit_pending_batch(
+                                    &mut pending,
+                                    index,
+                                    &mut file_cursors,
+                                    &mut max_cursor,
+                                    &mut replayed,
+                                )?;
                             }
-                            if !relocate_buffer.is_empty() {
-                                index.batch_relocate(&relocate_buffer)?;
-                                relocate_buffer.clear();
+                            false => {
+                                tracing::warn!(
+                                    file_id = %fid,
+                                    batch_seq,
+                                    expected_count = record_count,
+                                    observed_count = pending.record_count,
+                                    boundary_lost = pending.boundary_lost,
+                                    "rolling back torn hint batch"
+                                );
+                                pending.reset();
                             }
-                            index.batch_remove(&remove_buffer);
-                            remove_buffer.clear();
                         }
                     }
-                    ReadHintRecord::Corrupted => {
-                        tracing::warn!(
-                            file_id = %fid,
-                            "corrupted hint record during replay, skipping"
-                        );
+                    ReadHintRecord::Corrupted
+                    | ReadHintRecord::UnknownVersion { .. }
+                    | ReadHintRecord::UnknownType { .. } => {
+                        pending.boundary_lost = true;
                     }
-                    ReadHintRecord::UnknownVersion { .. }
-                    | ReadHintRecord::UnknownType { .. }
-                    | ReadHintRecord::Truncated => {}
+                    ReadHintRecord::Truncated => {}
                 }
                 Ok::<_, RebuildError>(())
             })?;
 
-            if !put_buffer.is_empty() {
-                index.batch_insert_buffered(&put_buffer)?;
-                put_buffer.clear();
-            }
-            if !relocate_buffer.is_empty() {
-                index.batch_relocate(&relocate_buffer)?;
-                relocate_buffer.clear();
-            }
-            if !remove_buffer.is_empty() {
-                index.batch_remove(&remove_buffer);
-                remove_buffer.clear();
-            }
-
             let _ = io.close(fd);
             Ok(())
         })?;
 
+    if pending.record_count > 0 || pending.boundary_lost {
+        tracing::warn!(
+            record_count = pending.record_count,
+            boundary_lost = pending.boundary_lost,
+            "discarding unterminated hint batch at replay end"
+        );
+        pending.reset();
+    }
+
     if let Some(cursor) = max_cursor {
         index.set_write_cursor(cursor)?;
     }
@@ -841,7 +971,11 @@ mod tests {
         let offset = BlockOffset::new(1024);
         let length = BlockLength::new(256);
 
-        let loc = BlockLocation { file_id, offset, length };
+        let loc = BlockLocation {
+            file_id,
+            offset,
+            length,
+        };
         encode_hint_record(&sim, fd, HintOffset::new(0), &cid, &loc).unwrap();
 
         let file_size = sim.file_size(fd).unwrap();
@@ -1095,7 +1229,7 @@ mod tests {
     }
 
     #[test]
-    fn hint_reader_stops_on_corrupted() {
+    fn hint_reader_reports_corrupted_and_continues() {
         let (sim, fd) = setup();
         let mut writer = HintFileWriter::new(&sim, fd);
 
@@ -1113,9 +1247,48 @@ mod tests {
 
         let reader = HintFileReader::open(&sim, fd).unwrap();
         let records: Vec<_> = reader.map(|r| r.unwrap()).collect();
-        assert_eq!(records.len(), 2);
+        assert_eq!(records.len(), 3);
         assert!(matches!(records[0], ReadHintRecord::Put { .. }));
         assert!(matches!(records[1], ReadHintRecord::Corrupted));
+        assert!(matches!(records[2], ReadHintRecord::Put { .. }));
+    }
+
+    #[test]
+    fn commit_marker_round_trip() {
+        let (sim, fd) = setup();
+        let data_file_id = DataFileId::new(7);
+        let data_offset = BlockOffset::new(9_876);
+
+        encode_commit_marker_record(
+            &sim,
+            fd,
+            HintOffset::new(0),
+            42,
+            128,
+            data_file_id,
+            data_offset,
+        )
+        .unwrap();
+
+        let file_size = sim.file_size(fd).unwrap();
+        let record = decode_hint_record(&sim, fd, HintOffset::new(0), file_size)
+            .unwrap()
+            .unwrap();
+
+        match record {
+            ReadHintRecord::CommitMarker {
+                batch_seq,
+                record_count,
+                data_file_id: fid,
+                data_offset: off,
+            } => {
+                assert_eq!(batch_seq, 42);
+                assert_eq!(record_count, 128);
+                assert_eq!(fid, data_file_id);
+                assert_eq!(off, data_offset);
+            }
+            other => panic!("expected CommitMarker, got {other:?}"),
+        }
     }
 
     #[test]

crates/tranquil-store/src/blockstore/manager.rs

@@ -1,6 +1,7 @@
 use std::collections::HashMap;
 use std::io;
 use std::path::{Path, PathBuf};
+use std::sync::Arc;
 
 use parking_lot::RwLock;
 
@@ -13,22 +14,39 @@ pub const DEFAULT_MAX_FILE_SIZE: u64 = 256 * 1024 * 1024;
 
 pub(crate) const DATA_FILE_EXTENSION: &str = "tqb";
 
-struct CachedHandle {
+pub struct CachedHandle<S: StorageIO> {
     fd: FileId,
+    io: Arc<S>,
     writable: bool,
 }
 
+impl<S: StorageIO> CachedHandle<S> {
+    pub fn fd(&self) -> FileId {
+        self.fd
+    }
+
+    pub fn is_writable(&self) -> bool {
+        self.writable
+    }
+}
+
+impl<S: StorageIO> Drop for CachedHandle<S> {
+    fn drop(&mut self) {
+        let _ = self.io.close(self.fd);
+    }
+}
+
 pub struct DataFileManager<S: StorageIO> {
-    io: S,
+    io: Arc<S>,
     data_dir: PathBuf,
     max_file_size: u64,
-    handles: RwLock<HashMap<DataFileId, CachedHandle>>,
+    handles: RwLock<HashMap<DataFileId, Arc<CachedHandle<S>>>>,
 }
 
 impl<S: StorageIO> DataFileManager<S> {
     pub fn new(io: S, data_dir: PathBuf, max_file_size: u64) -> Self {
         Self {
-            io,
+            io: Arc::new(io),
             data_dir,
             max_file_size,
             handles: RwLock::new(HashMap::new()),
@@ -40,7 +58,7 @@ impl<S: StorageIO> DataFileManager<S> {
     }
 
     pub fn io(&self) -> &S {
-        &self.io
+        self.io.as_ref()
     }
 
     pub fn data_dir(&self) -> &Path {
@@ -56,77 +74,81 @@ impl<S: StorageIO> DataFileManager<S> {
             .join(format!("{file_id}.{DATA_FILE_EXTENSION}"))
     }
 
-    pub fn open_for_append(&self, file_id: DataFileId) -> io::Result<FileId> {
+    pub fn open_for_append(&self, file_id: DataFileId) -> io::Result<Arc<CachedHandle<S>>> {
         {
             let cache = self.handles.read();
             if let Some(entry) = cache.get(&file_id)
                 && entry.writable
             {
-                return Ok(entry.fd);
+                return Ok(Arc::clone(entry));
             }
         }
         let path = self.data_file_path(file_id);
         let fd = self.io.open(&path, OpenOptions::read_write())?;
         let mut cache = self.handles.write();
-        match cache.get(&file_id) {
+        match cache.get(&file_id).cloned() {
             Some(entry) if entry.writable => {
                 let _ = self.io.close(fd);
-                Ok(entry.fd)
+                Ok(entry)
             }
-            Some(entry) => {
-                let old_fd = entry.fd;
-                cache.insert(file_id, CachedHandle { fd, writable: true });
-                let _ = self.io.close(old_fd);
-                Ok(fd)
-            }
-            None => {
-                cache.insert(file_id, CachedHandle { fd, writable: true });
-                Ok(fd)
+            _ => {
+                let handle = Arc::new(CachedHandle {
+                    fd,
+                    io: Arc::clone(&self.io),
+                    writable: true,
+                });
+                cache.insert(file_id, Arc::clone(&handle));
+                Ok(handle)
             }
         }
     }
 
-    pub fn open_for_read(&self, file_id: DataFileId) -> io::Result<FileId> {
+    pub fn open_for_read(&self, file_id: DataFileId) -> io::Result<Arc<CachedHandle<S>>> {
         if let Some(entry) = self.handles.read().get(&file_id) {
-            return Ok(entry.fd);
+            return Ok(Arc::clone(entry));
         }
         let path = self.data_file_path(file_id);
         let fd = self.io.open(&path, OpenOptions::read_only_existing())?;
         let mut cache = self.handles.write();
-        match cache.get(&file_id) {
+        match cache.get(&file_id).cloned() {
             Some(entry) => {
                 let _ = self.io.close(fd);
-                Ok(entry.fd)
+                Ok(entry)
             }
             None => {
-                cache.insert(
-                    file_id,
-                    CachedHandle {
-                        fd,
-                        writable: false,
-                    },
-                );
-                Ok(fd)
+                let handle = Arc::new(CachedHandle {
+                    fd,
+                    io: Arc::clone(&self.io),
+                    writable: false,
+                });
+                cache.insert(file_id, Arc::clone(&handle));
+                Ok(handle)
             }
         }
     }
 
-    pub fn prepare_rotation(&self, current: DataFileId) -> io::Result<(DataFileId, FileId)> {
+    pub fn prepare_rotation(
+        &self,
+        current: DataFileId,
+    ) -> io::Result<(DataFileId, Arc<CachedHandle<S>>)> {
         let next = current.next();
         let path = self.data_file_path(next);
         let fd = self.io.open(&path, OpenOptions::read_write())?;
-        Ok((next, fd))
+        let handle = Arc::new(CachedHandle {
+            fd,
+            io: Arc::clone(&self.io),
+            writable: true,
+        });
+        Ok((next, handle))
     }
 
-    pub fn commit_rotation(&self, file_id: DataFileId, fd: FileId) {
-        self.handles
-            .write()
-            .insert(file_id, CachedHandle { fd, writable: true });
+    pub fn commit_rotation(&self, file_id: DataFileId, handle: &Arc<CachedHandle<S>>) {
+        self.handles.write().insert(file_id, Arc::clone(handle));
     }
 
-    pub fn rollback_rotation(&self, file_id: DataFileId, fd: FileId) {
-        let _ = self.io.close(fd);
+    pub fn rollback_rotation(&self, file_id: DataFileId) {
         self.handles.write().remove(&file_id);
+        let _ = self.io.delete(&self.data_file_path(file_id));
     }
 
     pub fn should_rotate(&self, position: BlockOffset) -> bool {
@@ -134,14 +156,11 @@ impl<S: StorageIO> DataFileManager<S> {
     }
 
     pub fn list_files(&self) -> io::Result<Vec<DataFileId>> {
-        list_files_by_extension(&self.io, &self.data_dir, DATA_FILE_EXTENSION)
+        list_files_by_extension(&*self.io, &self.data_dir, DATA_FILE_EXTENSION)
     }
 
     pub fn evict_handle(&self, file_id: DataFileId) {
-        let removed = self.handles.write().remove(&file_id);
-        if let Some(entry) = removed {
-            let _ = self.io.close(entry.fd);
-        }
+        self.handles.write().remove(&file_id);
     }
 
     pub fn delete_data_file(&self, file_id: DataFileId) -> io::Result<()> {
@@ -151,14 +170,6 @@ impl<S: StorageIO> DataFileManager<S> {
     }
 }
 
-impl<S: StorageIO> Drop for DataFileManager<S> {
-    fn drop(&mut self) {
-        self.handles.write().drain().for_each(|(_, entry)| {
-            let _ = self.io.close(entry.fd);
-        });
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -177,8 +188,8 @@ mod tests {
     #[test]
     fn open_for_append_creates_file() {
         let mgr = setup_manager(1024);
-        let fd = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        assert_eq!(mgr.io().file_size(fd).unwrap(), 0);
+        let handle = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        assert_eq!(mgr.io().file_size(handle.fd()).unwrap(), 0);
     }
 
     #[test]
@@ -190,45 +201,46 @@ mod tests {
     #[test]
     fn handle_cache_returns_same_fd() {
         let mgr = setup_manager(1024);
-        let fd1 = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        let fd2 = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        assert_eq!(fd1, fd2);
+        let h1 = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        let h2 = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        assert_eq!(h1.fd(), h2.fd());
     }
 
     #[test]
     fn open_for_read_uses_cache_from_append() {
         let mgr = setup_manager(1024);
-        let fd_write = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        let fd_read = mgr.open_for_read(DataFileId::new(0)).unwrap();
-        assert_eq!(fd_write, fd_read);
+        let h_write = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        let h_read = mgr.open_for_read(DataFileId::new(0)).unwrap();
+        assert_eq!(h_write.fd(), h_read.fd());
     }
 
     #[test]
     fn rotation_lifecycle_prepare_commit() {
         let mgr = setup_manager(1024);
-        let _fd0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        let (next_id, next_fd) = mgr.prepare_rotation(DataFileId::new(0)).unwrap();
+        let _h0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        let (next_id, next_handle) = mgr.prepare_rotation(DataFileId::new(0)).unwrap();
         assert_eq!(next_id, DataFileId::new(1));
-        assert_eq!(mgr.io().file_size(next_fd).unwrap(), 0);
+        assert_eq!(mgr.io().file_size(next_handle.fd()).unwrap(), 0);
         mgr.io().sync_dir(mgr.data_dir()).unwrap();
-        mgr.commit_rotation(next_id, next_fd);
-        assert_eq!(mgr.open_for_read(next_id).unwrap(), next_fd);
+        mgr.commit_rotation(next_id, &next_handle);
+        assert_eq!(mgr.open_for_read(next_id).unwrap().fd(), next_handle.fd());
     }
 
     #[test]
-    fn rotation_rollback_cleans_handle() {
+    fn rotation_rollback_cleans_handle_and_deletes_file() {
         let mgr = setup_manager(1024);
-        let _fd0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        let (next_id, next_fd) = mgr.prepare_rotation(DataFileId::new(0)).unwrap();
-        mgr.commit_rotation(next_id, next_fd);
+        let _h0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        let (next_id, next_handle) = mgr.prepare_rotation(DataFileId::new(0)).unwrap();
+        mgr.commit_rotation(next_id, &next_handle);
 
-        assert_eq!(mgr.open_for_read(next_id).unwrap(), next_fd);
-        mgr.rollback_rotation(next_id, next_fd);
+        assert_eq!(mgr.open_for_read(next_id).unwrap().fd(), next_handle.fd());
+        drop(next_handle);
+        mgr.rollback_rotation(next_id);
 
-        let reopened_fd = mgr.open_for_read(next_id).unwrap();
-        assert_ne!(
-            reopened_fd, next_fd,
-            "rollback should have closed the cached fd"
+        let reopen = mgr.open_for_read(next_id);
+        assert!(
+            reopen.is_err_and(|e| e.kind() == io::ErrorKind::NotFound),
+            "rollback_rotation must delete the data file so recovery cannot resurrect uncommitted bytes"
         );
     }
 
@@ -244,8 +256,8 @@ mod tests {
     #[test]
     fn list_files_finds_data_files() {
         let mgr = setup_manager(1024);
-        let _fd0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        let _fd3 = mgr.open_for_append(DataFileId::new(3)).unwrap();
+        let _h0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        let _h3 = mgr.open_for_append(DataFileId::new(3)).unwrap();
 
         let files = mgr.list_files().unwrap();
         assert_eq!(files, vec![DataFileId::new(0), DataFileId::new(3)]);
@@ -254,7 +266,7 @@ mod tests {
     #[test]
     fn list_files_ignores_non_data_files() {
         let mgr = setup_manager(1024);
-        let _fd0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        let _h0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
         mgr.io()
             .open(Path::new("/data/notes.txt"), OpenOptions::read_write())
             .unwrap();
@@ -279,32 +291,32 @@ mod tests {
     #[test]
     fn rotate_and_write_across_files() {
         let mgr = setup_manager(1024);
-        let fd0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        let mut writer0 = DataFileWriter::new(mgr.io(), fd0, DataFileId::new(0)).unwrap();
+        let h0 = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        let mut writer0 = DataFileWriter::new(mgr.io(), h0.fd(), DataFileId::new(0)).unwrap();
         let _ = writer0
             .append_block(&test_cid(1), b"first file data")
             .unwrap();
         writer0.sync().unwrap();
 
-        let (id1, fd1) = mgr.prepare_rotation(DataFileId::new(0)).unwrap();
+        let (id1, h1) = mgr.prepare_rotation(DataFileId::new(0)).unwrap();
         mgr.io().sync_dir(mgr.data_dir()).unwrap();
-        mgr.commit_rotation(id1, fd1);
-        let mut writer1 = DataFileWriter::new(mgr.io(), fd1, id1).unwrap();
+        mgr.commit_rotation(id1, &h1);
+        let mut writer1 = DataFileWriter::new(mgr.io(), h1.fd(), id1).unwrap();
         let _ = writer1
             .append_block(&test_cid(2), b"second file data")
             .unwrap();
         writer1.sync().unwrap();
 
-        let fd0_read = mgr.open_for_read(DataFileId::new(0)).unwrap();
-        let blocks0 = DataFileReader::open(mgr.io(), fd0_read)
+        let h0_read = mgr.open_for_read(DataFileId::new(0)).unwrap();
+        let blocks0 = DataFileReader::open(mgr.io(), h0_read.fd())
             .unwrap()
             .valid_blocks()
             .unwrap();
         assert_eq!(blocks0.len(), 1);
         assert_eq!(blocks0[0].2, b"first file data");
 
-        let fd1_read = mgr.open_for_read(id1).unwrap();
-        let blocks1 = DataFileReader::open(mgr.io(), fd1_read)
+        let h1_read = mgr.open_for_read(id1).unwrap();
+        let blocks1 = DataFileReader::open(mgr.io(), h1_read.fd())
             .unwrap()
             .valid_blocks()
             .unwrap();
@@ -315,11 +327,11 @@ mod tests {
     #[test]
     fn read_cache_hit_from_writable_entry() {
         let mgr = setup_manager(1024);
-        let fd_write = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        DataFileWriter::new(mgr.io(), fd_write, DataFileId::new(0)).unwrap();
+        let h_write = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        DataFileWriter::new(mgr.io(), h_write.fd(), DataFileId::new(0)).unwrap();
 
-        let fd_read = mgr.open_for_read(DataFileId::new(0)).unwrap();
-        assert_eq!(fd_write, fd_read);
+        let h_read = mgr.open_for_read(DataFileId::new(0)).unwrap();
+        assert_eq!(h_write.fd(), h_read.fd());
     }
 
     #[test]
@@ -338,15 +350,15 @@ mod tests {
         mgr.io().sync_dir(mgr.data_dir()).unwrap();
         mgr.io().close(raw_fd).unwrap();
 
-        let fd_read = mgr.open_for_read(DataFileId::new(0)).unwrap();
-        let _reader = DataFileReader::open(mgr.io(), fd_read).unwrap();
+        let h_read = mgr.open_for_read(DataFileId::new(0)).unwrap();
+        let _reader = DataFileReader::open(mgr.io(), h_read.fd()).unwrap();
 
-        let fd_append = mgr.open_for_append(DataFileId::new(0)).unwrap();
-        assert_ne!(fd_read, fd_append);
+        let h_append = mgr.open_for_append(DataFileId::new(0)).unwrap();
+        assert_ne!(h_read.fd(), h_append.fd());
 
         let mut writer = DataFileWriter::resume(
             mgr.io(),
-            fd_append,
+            h_append.fd(),
             DataFileId::new(0),
             BlockOffset::new(BLOCK_HEADER_SIZE as u64),
         );
@@ -355,7 +367,7 @@ mod tests {
             .unwrap();
         writer.sync().unwrap();
 
-        let blocks = DataFileReader::open(mgr.io(), fd_append)
+        let blocks = DataFileReader::open(mgr.io(), h_append.fd())
             .unwrap()
             .valid_blocks()
             .unwrap();

crates/tranquil-store/src/blockstore/mod.rs

@@ -1,3 +1,4 @@
+mod cid_util;
 mod compaction;
 mod data_file;
 mod group_commit;
@@ -8,6 +9,7 @@ mod reader;
 mod store;
 mod types;
 
+pub use cid_util::{DAG_CBOR_CODEC, SHA2_256_CODE, hash_to_cid, hash_to_cid_bytes};
 pub use compaction::CompactionError;
 pub use data_file::{
     BLOCK_FORMAT_VERSION, BLOCK_HEADER_SIZE, BLOCK_MAGIC, BLOCK_RECORD_OVERHEAD, CID_SIZE,
@@ -22,7 +24,7 @@ pub use hint::{
     HINT_FILE_EXTENSION, HINT_RECORD_SIZE, HintFileReader, HintFileWriter, HintIndex,
     ReadHintRecord, RebuildError, decode_hint_record, hint_file_path, scan_hints_to_memory,
 };
-pub use manager::{DEFAULT_MAX_FILE_SIZE, DataFileManager};
+pub use manager::{CachedHandle, DEFAULT_MAX_FILE_SIZE, DataFileManager};
 pub use reader::{BlockStoreReader, ReadError};
 pub use store::QuiesceGuard;
 pub use store::{BlockStoreConfig, DEFAULT_SHARD_COUNT, TranquilBlockStore};

crates/tranquil-store/src/blockstore/reader.rs

@@ -104,11 +104,11 @@ impl<S: StorageIO> BlockStoreReader<S> {
         });
 
         by_file.into_iter().try_for_each(|(file_id, mut entries)| {
-            let fd = self.manager.open_for_read(file_id)?;
-            let file_size = self.manager.io().file_size(fd)?;
+            let handle = self.manager.open_for_read(file_id)?;
+            let file_size = self.manager.io().file_size(handle.fd())?;
             entries.sort_by_key(|(_, loc)| loc.offset);
             entries.into_iter().try_for_each(|(orig_idx, loc)| {
-                let data = self.decode_and_validate(fd, file_size, loc)?;
+                let data = self.decode_and_validate(handle.fd(), file_size, loc)?;
                 results[orig_idx] = Some(data);
                 Ok::<_, ReadError>(())
             })
@@ -116,9 +116,9 @@ impl<S: StorageIO> BlockStoreReader<S> {
     }
 
     fn read_block_at(&self, location: BlockLocation) -> Result<Bytes, ReadError> {
-        let fd = self.manager.open_for_read(location.file_id)?;
-        let file_size = self.manager.io().file_size(fd)?;
-        self.decode_and_validate(fd, file_size, location)
+        let handle = self.manager.open_for_read(location.file_id)?;
+        let file_size = self.manager.io().file_size(handle.fd())?;
+        self.decode_and_validate(handle.fd(), file_size, location)
     }
 
     fn decode_and_validate(
@@ -127,26 +127,37 @@ impl<S: StorageIO> BlockStoreReader<S> {
         file_size: u64,
         location: BlockLocation,
     ) -> Result<Bytes, ReadError> {
-        match decode_block_record(self.manager.io(), fd, location.offset, file_size)? {
-            Some(ReadBlockRecord::Valid { data, .. })
-                if data.len() == location.length.raw() as usize =>
-            {
-                Ok(Bytes::from(data))
-            }
-            Some(ReadBlockRecord::Valid { .. }) => Err(ReadError::Corrupted {
-                file_id: location.file_id,
-                offset: location.offset,
-            }),
-            Some(ReadBlockRecord::Corrupted { offset } | ReadBlockRecord::Truncated { offset }) => {
-                Err(ReadError::Corrupted {
+        let attempt_once = || -> Result<Bytes, ReadError> {
+            match decode_block_record(self.manager.io(), fd, location.offset, file_size)? {
+                Some(ReadBlockRecord::Valid { data, .. })
+                    if data.len() == location.length.raw() as usize =>
+                {
+                    Ok(Bytes::from(data))
+                }
+                Some(ReadBlockRecord::Valid { .. }) => Err(ReadError::Corrupted {
+                    file_id: location.file_id,
+                    offset: location.offset,
+                }),
+                Some(
+                    ReadBlockRecord::Corrupted { offset } | ReadBlockRecord::Truncated { offset },
+                ) => Err(ReadError::Corrupted {
                     file_id: location.file_id,
                     offset,
-                })
+                }),
+                None => Err(ReadError::Corrupted {
+                    file_id: location.file_id,
+                    offset: location.offset,
+                }),
             }
-            None => Err(ReadError::Corrupted {
-                file_id: location.file_id,
-                offset: location.offset,
-            }),
-        }
+        };
+        (0..READ_RETRY_ATTEMPTS.saturating_sub(1))
+            .find_map(|_| match attempt_once() {
+                Ok(bytes) => Some(Ok(bytes)),
+                Err(ReadError::Corrupted { .. }) => None,
+                Err(e) => Some(Err(e)),
+            })
+            .unwrap_or_else(attempt_once)
     }
 }
+
+const READ_RETRY_ATTEMPTS: u32 = 4;

crates/tranquil-store/src/blockstore/store.rs

@@ -8,12 +8,11 @@ use cid::Cid;
 use jacquard_repo::error::RepoError;
 use jacquard_repo::repo::CommitData;
 use jacquard_repo::storage::BlockStore;
-use multihash::Multihash;
-use sha2::{Digest, Sha256};
 
 use crate::fsync_order::PostBlockstoreHook;
 use crate::io::{OpenOptions, RealIO, StorageIO};
 
+use super::cid_util::hash_to_cid;
 use super::compaction::CompactionError;
 use super::data_file::{BLOCK_RECORD_OVERHEAD, CID_SIZE, ReadBlockRecord};
 use super::group_commit::{CommitError, CommitRequest, GroupCommitConfig, GroupCommitWriter};
@@ -21,13 +20,10 @@ use super::hash_index::BlockIndex;
 use super::manager::DataFileManager;
 use super::reader::{BlockStoreReader, ReadError};
 use super::types::{
-    BlockLength, BlockLocation, BlockOffset, CollectionResult, CompactionResult, DataFileId,
-    EpochCounter, LivenessInfo, WallClockMs, WriteCursor,
+    BlockLocation, BlockOffset, CollectionResult, CompactionResult, DataFileId, EpochCounter,
+    LivenessInfo, WallClockMs,
 };
 
-const DAG_CBOR_CODEC: u64 = 0x71;
-const SHA2_256_CODE: u64 = 0x12;
-
 fn cid_to_bytes(cid: &Cid) -> Result<[u8; CID_SIZE], RepoError> {
     let raw = cid.to_bytes();
     let len = raw.len();
@@ -41,20 +37,6 @@ fn cid_to_bytes(cid: &Cid) -> Result<[u8; CID_SIZE], RepoError> {
     })
 }
 
-fn hash_and_cid(data: &[u8]) -> Result<Cid, RepoError> {
-    let mut hasher = Sha256::new();
-    hasher.update(data);
-    let hash = hasher.finalize();
-    let multihash = Multihash::wrap(SHA2_256_CODE, &hash).map_err(|e| {
-        RepoError::storage(io::Error::new(io::ErrorKind::InvalidData, e.to_string()))
-    })?;
-    Ok(Cid::new_v1(DAG_CBOR_CODEC, multihash))
-}
-
-fn block_index_err_to_repo(e: super::hash_index::BlockIndexError) -> RepoError {
-    RepoError::storage(io::Error::other(e.to_string()))
-}
-
 fn commit_error_to_repo(e: CommitError) -> RepoError {
     match e {
         CommitError::Io(io_err) => {
@@ -65,6 +47,10 @@ fn commit_error_to_repo(e: CommitError) -> RepoError {
             io::ErrorKind::BrokenPipe,
             "blockstore commit channel closed",
         )),
+        CommitError::VerifyFailed { file_id, offset } => RepoError::storage(io::Error::new(
+            io::ErrorKind::InvalidData,
+            format!("post-sync verify failed at {file_id}:{}", offset.raw()),
+        )),
     }
 }
 
@@ -123,15 +109,26 @@ impl Drop for QuiesceGuard {
     }
 }
 
-#[derive(Clone)]
-pub struct TranquilBlockStore {
+pub struct TranquilBlockStore<S: StorageIO + Send + Sync + 'static = RealIO> {
     writer: Arc<WriterHandle>,
-    reader: Arc<BlockStoreReader<RealIO>>,
+    reader: Arc<BlockStoreReader<S>>,
     index: Arc<BlockIndex>,
     epoch: EpochCounter,
     data_dir: PathBuf,
 }
 
+impl<S: StorageIO + Send + Sync + 'static> Clone for TranquilBlockStore<S> {
+    fn clone(&self) -> Self {
+        Self {
+            writer: Arc::clone(&self.writer),
+            reader: Arc::clone(&self.reader),
+            index: Arc::clone(&self.index),
+            epoch: self.epoch.clone(),
+            data_dir: self.data_dir.clone(),
+        }
+    }
+}
+
 struct WriterHandle {
     inner: parking_lot::Mutex<Option<GroupCommitWriter>>,
 }
@@ -153,7 +150,7 @@ impl Drop for WriterHandle {
     }
 }
 
-impl TranquilBlockStore {
+impl TranquilBlockStore<RealIO> {
     pub fn open(config: BlockStoreConfig) -> Result<Self, RepoError> {
         Self::open_with_hook(config, None)
     }
@@ -162,6 +159,26 @@ impl TranquilBlockStore {
         config: BlockStoreConfig,
         post_sync_hook: Option<Arc<dyn PostBlockstoreHook>>,
     ) -> Result<Self, RepoError> {
+        Self::open_with_io_hook(config, RealIO::new, post_sync_hook)
+    }
+}
+
+impl<S: StorageIO + Send + Sync + 'static> TranquilBlockStore<S> {
+    pub fn open_with_io<F>(config: BlockStoreConfig, make_io: F) -> Result<Self, RepoError>
+    where
+        F: Fn() -> S + Send + Sync + Clone + 'static,
+    {
+        Self::open_with_io_hook(config, make_io, None)
+    }
+
+    pub fn open_with_io_hook<F>(
+        config: BlockStoreConfig,
+        make_io: F,
+        post_sync_hook: Option<Arc<dyn PostBlockstoreHook>>,
+    ) -> Result<Self, RepoError>
+    where
+        F: Fn() -> S + Send + Sync + Clone + 'static,
+    {
         if config.data_dir == config.index_dir {
             return Err(RepoError::storage(io::Error::new(
                 io::ErrorKind::InvalidInput,
@@ -173,7 +190,7 @@ impl TranquilBlockStore {
 
         let index = BlockIndex::open(&config.index_dir).map_err(RepoError::storage)?;
 
-        let io = RealIO::new();
+        let io = make_io();
 
         let (replayed, file_cursors) = super::hint::replay_hints_into_block_index(
             &io,
@@ -195,8 +212,13 @@ impl TranquilBlockStore {
         let max_file_size = config.max_file_size;
         let shard_count = config.shard_count;
         let data_dir_for_closure = data_dir.clone();
+        let make_io_for_manager = make_io.clone();
         let make_manager = move || {
-            DataFileManager::new(RealIO::new(), data_dir_for_closure.clone(), max_file_size)
+            DataFileManager::new(
+                make_io_for_manager(),
+                data_dir_for_closure.clone(),
+                max_file_size,
+            )
         };
 
         let checkpoint_epoch = index.loaded_checkpoint_epoch();
@@ -214,7 +236,7 @@ impl TranquilBlockStore {
         let epoch = writer.epoch().clone();
 
         let manager_for_reader = Arc::new(DataFileManager::new(
-            RealIO::new(),
+            make_io(),
             data_dir.clone(),
             max_file_size,
         ));
@@ -234,7 +256,7 @@ impl TranquilBlockStore {
         })
     }
 
-    fn recover_from_file_cursors<S: StorageIO>(
+    fn recover_from_file_cursors(
         io: &S,
         data_dir: &Path,
         index: &BlockIndex,
@@ -256,7 +278,7 @@ impl TranquilBlockStore {
         })
     }
 
-    fn replay_single_file<S: StorageIO>(
+    fn replay_single_file(
         io: &S,
         data_dir: &Path,
         index: &BlockIndex,
@@ -277,19 +299,28 @@ impl TranquilBlockStore {
             Err(e) => return Err(RepoError::storage(e)),
         };
 
-        let result = Self::scan_and_index(io, index, fd, file_id, start_offset);
+        let hint_path = super::hint::hint_file_path(data_dir, file_id);
+        let hint_exists = io
+            .open(&hint_path, OpenOptions::read_only_existing())
+            .map(|fd| {
+                let _ = io.close(fd);
+            })
+            .is_ok();
+
+        let result = Self::scan_and_index(io, index, fd, file_id, start_offset, hint_exists);
 
         let _ = io.close(fd);
 
         result
     }
 
-    fn scan_and_index<S: StorageIO>(
+    fn scan_and_index(
         io: &S,
         index: &BlockIndex,
         fd: crate::io::FileId,
         file_id: DataFileId,
         start_offset: BlockOffset,
+        hint_exists: bool,
     ) -> Result<(), RepoError> {
         let file_size = io.file_size(fd).map_err(RepoError::storage)?;
 
@@ -298,7 +329,7 @@ impl TranquilBlockStore {
         }
 
         let scan_pos = &mut { start_offset };
-        let (recovered_entries, last_valid_end) = std::iter::from_fn(|| {
+        let (scanned_entries, last_valid_end) = std::iter::from_fn(|| {
             match super::data_file::decode_block_record(io, fd, *scan_pos, file_size) {
                 Err(e) => {
                     tracing::warn!(
@@ -319,9 +350,10 @@ impl TranquilBlockStore {
                         Ok(n) if n <= super::types::MAX_BLOCK_SIZE => n,
                         _ => return None,
                     };
-                    let length = BlockLength::new(raw_len);
+                    let length = super::types::BlockLength::new(raw_len);
                     let record_size = BLOCK_RECORD_OVERHEAD as u64 + u64::from(raw_len);
-                    *scan_pos = scan_pos.advance(record_size);
+                    let new_end = offset.advance(record_size);
+                    *scan_pos = new_end;
                     Some((
                         cid_bytes,
                         BlockLocation {
@@ -329,6 +361,7 @@ impl TranquilBlockStore {
                             offset,
                             length,
                         },
+                        new_end,
                     ))
                 }
                 Ok(Some(ReadBlockRecord::Corrupted { .. } | ReadBlockRecord::Truncated { .. })) => {
@@ -338,11 +371,8 @@ impl TranquilBlockStore {
         })
         .fold(
             (Vec::new(), start_offset),
-            |(mut entries, _), (cid_bytes, location)| {
-                let new_end = location
-                    .offset
-                    .advance(BLOCK_RECORD_OVERHEAD as u64 + location.length.as_u64());
-                entries.push((cid_bytes, location));
+            |(mut entries, _), (cid, loc, new_end)| {
+                entries.push((cid, loc));
                 (entries, new_end)
             },
         );
@@ -352,28 +382,28 @@ impl TranquilBlockStore {
                 file_id = %file_id,
                 truncating_from = last_valid_end.raw(),
                 file_size,
-                "truncating partial/corrupted tail"
+                scanned_count = scanned_entries.len(),
+                "truncating partial/unacked tail"
             );
             io.truncate(fd, last_valid_end.raw())
                 .map_err(RepoError::storage)?;
             io.sync(fd).map_err(RepoError::storage)?;
         }
 
-        if !recovered_entries.is_empty() {
-            let new_cursor = WriteCursor {
+        if !scanned_entries.is_empty() {
+            tracing::info!(
+                file_id = %file_id,
+                scanned = scanned_entries.len(),
+                hint_exists,
+                "reindexing blocks past hint coverage"
+            );
+            let cursor = super::types::WriteCursor {
                 file_id,
                 offset: last_valid_end,
             };
-            let inserted = index
-                .batch_put_if_absent(&recovered_entries, new_cursor)
-                .map_err(block_index_err_to_repo)?;
-            tracing::info!(
-                file_id = %file_id,
-                scanned = recovered_entries.len(),
-                inserted,
-                new_cursor_offset = last_valid_end.raw(),
-                "recovery data file scan"
-            );
+            index
+                .batch_put_if_absent(&scanned_entries, cursor)
+                .map_err(|e| RepoError::storage(io::Error::other(e.to_string())))?;
         }
 
         Ok(())
@@ -594,7 +624,7 @@ impl TranquilBlockStore {
     }
 }
 
-impl BlockStore for TranquilBlockStore {
+impl<S: StorageIO + Send + Sync + 'static> BlockStore for TranquilBlockStore<S> {
     async fn get(&self, cid: &Cid) -> Result<Option<Bytes>, RepoError> {
         let cid_bytes = cid_to_bytes(cid)?;
         let reader = Arc::clone(&self.reader);
@@ -605,7 +635,7 @@ impl BlockStore for TranquilBlockStore {
     }
 
     async fn put(&self, data: &[u8]) -> Result<Cid, RepoError> {
-        let cid = hash_and_cid(data)?;
+        let cid = hash_to_cid(data);
         let cid_bytes = cid_to_bytes(&cid)?;
         self.send_put_blocks(vec![(cid_bytes, data.to_vec())])
             .await?;
@@ -666,7 +696,7 @@ impl BlockStore for TranquilBlockStore {
     }
 }
 
-impl TranquilBlockStore {
+impl<S: StorageIO + Send + Sync + 'static> TranquilBlockStore<S> {
     pub async fn decrement_refs(&self, cids: &[Cid]) -> Result<(), RepoError> {
         if cids.is_empty() {
             return Ok(());

crates/tranquil-store/src/eventlog/manager.rs

@@ -1,7 +1,8 @@
 use std::collections::HashMap;
 use std::io;
 use std::path::{Path, PathBuf};
-use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 
 use parking_lot::RwLock;
 
@@ -25,17 +26,38 @@ pub fn parse_segment_id(path: &Path) -> Option<SegmentId> {
     (ext == SEGMENT_FILE_EXTENSION).then(|| stem.parse::<u32>().ok().map(SegmentId::new))?
 }
 
-struct CachedSegmentHandle {
+pub struct CachedSegmentHandle<S: StorageIO> {
     fd: FileId,
-    sealed: bool,
+    io: Arc<S>,
+    sealed: AtomicBool,
     writable: bool,
 }
 
+impl<S: StorageIO> CachedSegmentHandle<S> {
+    pub fn fd(&self) -> FileId {
+        self.fd
+    }
+
+    pub fn is_sealed(&self) -> bool {
+        self.sealed.load(Ordering::Acquire)
+    }
+
+    pub fn is_writable(&self) -> bool {
+        self.writable
+    }
+}
+
+impl<S: StorageIO> Drop for CachedSegmentHandle<S> {
+    fn drop(&mut self) {
+        let _ = self.io.close(self.fd);
+    }
+}
+
 pub struct SegmentManager<S: StorageIO> {
-    io: S,
+    io: Arc<S>,
     segments_dir: PathBuf,
     max_segment_size: u64,
-    handles: RwLock<HashMap<SegmentId, CachedSegmentHandle>>,
+    handles: RwLock<HashMap<SegmentId, Arc<CachedSegmentHandle<S>>>>,
     retention_epoch: AtomicU64,
 }
 
@@ -51,7 +73,7 @@ impl<S: StorageIO> SegmentManager<S> {
         );
         io.mkdir(&segments_dir)?;
         Ok(Self {
-            io,
+            io: Arc::new(io),
             segments_dir,
             max_segment_size,
             handles: RwLock::new(HashMap::new()),
@@ -60,7 +82,7 @@ impl<S: StorageIO> SegmentManager<S> {
     }
 
     pub fn io(&self) -> &S {
-        &self.io
+        self.io.as_ref()
     }
 
     pub fn segments_dir(&self) -> &Path {
@@ -92,52 +114,51 @@ impl<S: StorageIO> SegmentManager<S> {
         Ok(ids)
     }
 
-    pub fn open_for_read(&self, id: SegmentId) -> io::Result<FileId> {
+    pub fn open_for_read(&self, id: SegmentId) -> io::Result<Arc<CachedSegmentHandle<S>>> {
         if let Some(entry) = self.handles.read().get(&id) {
-            return Ok(entry.fd);
+            return Ok(Arc::clone(entry));
         }
         let path = self.segment_path(id);
         let fd = self.io.open(&path, OpenOptions::read_only_existing())?;
         let mut cache = self.handles.write();
-        match cache.get(&id) {
+        match cache.get(&id).cloned() {
             Some(entry) => {
                 let _ = self.io.close(fd);
-                Ok(entry.fd)
+                Ok(entry)
             }
             None => {
-                cache.insert(
-                    id,
-                    CachedSegmentHandle {
-                        fd,
-                        sealed: false,
-                        writable: false,
-                    },
-                );
-                Ok(fd)
+                let handle = Arc::new(CachedSegmentHandle {
+                    fd,
+                    io: Arc::clone(&self.io),
+                    sealed: AtomicBool::new(false),
+                    writable: false,
+                });
+                cache.insert(id, Arc::clone(&handle));
+                Ok(handle)
             }
         }
     }
 
-    pub fn open_for_append(&self, id: SegmentId) -> io::Result<FileId> {
+    pub fn open_for_append(&self, id: SegmentId) -> io::Result<Arc<CachedSegmentHandle<S>>> {
         {
             let cache = self.handles.read();
             if let Some(entry) = cache.get(&id) {
-                if entry.sealed {
+                if entry.is_sealed() {
                     return Err(io::Error::new(
                         io::ErrorKind::InvalidInput,
                         format!("cannot append to sealed segment {id}"),
                     ));
                 }
                 if entry.writable {
-                    return Ok(entry.fd);
+                    return Ok(Arc::clone(entry));
                 }
             }
         }
         let path = self.segment_path(id);
         let fd = self.io.open(&path, OpenOptions::read_write())?;
         let mut cache = self.handles.write();
-        match cache.get(&id) {
-            Some(entry) if entry.sealed => {
+        match cache.get(&id).cloned() {
+            Some(entry) if entry.is_sealed() => {
                 let _ = self.io.close(fd);
                 Err(io::Error::new(
                     io::ErrorKind::InvalidInput,
@@ -146,31 +167,17 @@ impl<S: StorageIO> SegmentManager<S> {
             }
             Some(entry) if entry.writable => {
                 let _ = self.io.close(fd);
-                Ok(entry.fd)
+                Ok(entry)
             }
-            Some(entry) => {
-                let old_fd = entry.fd;
-                cache.insert(
-                    id,
-                    CachedSegmentHandle {
-                        fd,
-                        sealed: false,
-                        writable: true,
-                    },
-                );
-                let _ = self.io.close(old_fd);
-                Ok(fd)
-            }
-            None => {
-                cache.insert(
-                    id,
-                    CachedSegmentHandle {
-                        fd,
-                        sealed: false,
-                        writable: true,
-                    },
-                );
-                Ok(fd)
+            _ => {
+                let handle = Arc::new(CachedSegmentHandle {
+                    fd,
+                    io: Arc::clone(&self.io),
+                    sealed: AtomicBool::new(false),
+                    writable: true,
+                });
+                cache.insert(id, Arc::clone(&handle));
+                Ok(handle)
             }
         }
     }
@@ -179,37 +186,39 @@ impl<S: StorageIO> SegmentManager<S> {
         position.raw() >= self.max_segment_size
     }
 
-    pub fn prepare_rotation(&self, current_id: SegmentId) -> io::Result<(SegmentId, FileId)> {
+    pub fn prepare_rotation(
+        &self,
+        current_id: SegmentId,
+    ) -> io::Result<(SegmentId, Arc<CachedSegmentHandle<S>>)> {
         let next = current_id.next();
         let path = self.segment_path(next);
         let fd = self.io.open(&path, OpenOptions::read_write())?;
         self.io.truncate(fd, 0)?;
         self.io.sync_dir(&self.segments_dir)?;
-        Ok((next, fd))
+        let handle = Arc::new(CachedSegmentHandle {
+            fd,
+            io: Arc::clone(&self.io),
+            sealed: AtomicBool::new(false),
+            writable: true,
+        });
+        Ok((next, handle))
     }
 
-    pub fn commit_rotation(&self, new_id: SegmentId, fd: FileId) {
-        self.handles.write().insert(
-            new_id,
-            CachedSegmentHandle {
-                fd,
-                sealed: false,
-                writable: true,
-            },
-        );
+    pub fn commit_rotation(&self, new_id: SegmentId, handle: &Arc<CachedSegmentHandle<S>>) {
+        self.handles.write().insert(new_id, Arc::clone(handle));
     }
 
     pub fn seal_segment(&self, id: SegmentId, index: &SegmentIndex) -> io::Result<()> {
         let path = self.index_path(id);
-        index.save(&self.io, &path)?;
-        let mut cache = self.handles.write();
-        let entry = cache.get_mut(&id).ok_or_else(|| {
+        index.save(self.io.as_ref(), &path)?;
+        let cache = self.handles.read();
+        let entry = cache.get(&id).ok_or_else(|| {
             io::Error::new(
                 io::ErrorKind::InvalidInput,
                 format!("seal_segment: segment {id} not in handle cache"),
             )
         })?;
-        entry.sealed = true;
+        entry.sealed.store(true, Ordering::Release);
         Ok(())
     }
 
@@ -217,22 +226,16 @@ impl<S: StorageIO> SegmentManager<S> {
         self.handles
             .read()
             .get(&id)
-            .is_some_and(|entry| entry.sealed)
+            .is_some_and(|entry| entry.is_sealed())
     }
 
-    pub fn rollback_rotation(&self, new_id: SegmentId, fd: FileId) {
-        let _ = self.io.close(fd);
+    pub fn rollback_rotation(&self, new_id: SegmentId) {
         self.handles.write().remove(&new_id);
         let _ = self.io.delete(&self.segment_path(new_id));
     }
 
     pub fn delete_segment(&self, id: SegmentId) -> io::Result<()> {
-        {
-            let mut cache = self.handles.write();
-            if let Some(entry) = cache.remove(&id) {
-                let _ = self.io.close(entry.fd);
-            }
-        }
+        self.handles.write().remove(&id);
         [self.index_path(id), self.sidecar_path(id)]
             .iter()
             .try_for_each(|path| match self.io.delete(path) {
@@ -255,15 +258,7 @@ impl<S: StorageIO> SegmentManager<S> {
     }
 
     pub fn shutdown(&self) {
-        self.handles.write().drain().for_each(|(_, handle)| {
-            let _ = self.io.close(handle.fd);
-        });
-    }
-}
-
-impl<S: StorageIO> Drop for SegmentManager<S> {
-    fn drop(&mut self) {
-        self.shutdown();
+        self.handles.write().clear();
     }
 }
 
@@ -329,7 +324,7 @@ mod tests {
     #[test]
     fn open_for_append_creates_file() {
         let mgr = setup_manager(1024);
-        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
         assert_eq!(mgr.io().file_size(fd).unwrap(), 0);
     }
 
@@ -342,24 +337,24 @@ mod tests {
     #[test]
     fn handle_cache_returns_same_fd() {
         let mgr = setup_manager(1024);
-        let fd1 = mgr.open_for_append(SegmentId::new(1)).unwrap();
-        let fd2 = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd1 = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        let fd2 = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
         assert_eq!(fd1, fd2);
     }
 
     #[test]
     fn open_for_read_uses_cache_from_append() {
         let mgr = setup_manager(1024);
-        let fd_write = mgr.open_for_append(SegmentId::new(1)).unwrap();
-        let fd_read = mgr.open_for_read(SegmentId::new(1)).unwrap();
+        let fd_write = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        let fd_read = mgr.open_for_read(SegmentId::new(1)).unwrap().fd();
         assert_eq!(fd_write, fd_read);
     }
 
     #[test]
     fn list_segments_finds_segment_files() {
         let mgr = setup_manager(1024);
-        mgr.open_for_append(SegmentId::new(1)).unwrap();
-        mgr.open_for_append(SegmentId::new(3)).unwrap();
+        mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        mgr.open_for_append(SegmentId::new(3)).unwrap().fd();
 
         let segments = mgr.list_segments().unwrap();
         assert_eq!(segments, vec![SegmentId::new(1), SegmentId::new(3)]);
@@ -368,7 +363,7 @@ mod tests {
     #[test]
     fn list_segments_ignores_non_segment_files() {
         let mgr = setup_manager(1024);
-        mgr.open_for_append(SegmentId::new(1)).unwrap();
+        mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
         mgr.io()
             .open(Path::new("/segments/notes.txt"), OpenOptions::read_write())
             .unwrap();
@@ -380,7 +375,7 @@ mod tests {
     #[test]
     fn list_segments_ignores_index_files() {
         let mgr = setup_manager(1024);
-        mgr.open_for_append(SegmentId::new(1)).unwrap();
+        mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
         mgr.io()
             .open(
                 Path::new("/segments/00000001.tqi"),
@@ -395,9 +390,9 @@ mod tests {
     #[test]
     fn list_segments_sorted_ascending() {
         let mgr = setup_manager(1024);
-        mgr.open_for_append(SegmentId::new(5)).unwrap();
-        mgr.open_for_append(SegmentId::new(1)).unwrap();
-        mgr.open_for_append(SegmentId::new(3)).unwrap();
+        mgr.open_for_append(SegmentId::new(5)).unwrap().fd();
+        mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        mgr.open_for_append(SegmentId::new(3)).unwrap().fd();
 
         let segments = mgr.list_segments().unwrap();
         assert_eq!(
@@ -418,23 +413,24 @@ mod tests {
     #[test]
     fn rotation_lifecycle_prepare_commit() {
         let mgr = setup_manager(1024);
-        let _fd0 = mgr.open_for_append(SegmentId::new(1)).unwrap();
-        let (next_id, next_fd) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
+        let _h0 = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        let (next_id, next_handle) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
         assert_eq!(next_id, SegmentId::new(2));
-        assert_eq!(mgr.io().file_size(next_fd).unwrap(), 0);
-        mgr.commit_rotation(next_id, next_fd);
-        assert_eq!(mgr.open_for_read(next_id).unwrap(), next_fd);
+        assert_eq!(mgr.io().file_size(next_handle.fd()).unwrap(), 0);
+        mgr.commit_rotation(next_id, &next_handle);
+        assert_eq!(mgr.open_for_read(next_id).unwrap().fd(), next_handle.fd());
     }
 
     #[test]
     fn rotation_rollback_cleans_up() {
         let mgr = setup_manager(1024);
-        let _fd0 = mgr.open_for_append(SegmentId::new(1)).unwrap();
-        let (next_id, next_fd) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
-        mgr.commit_rotation(next_id, next_fd);
+        let _h0 = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        let (next_id, next_handle) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
+        mgr.commit_rotation(next_id, &next_handle);
 
-        assert_eq!(mgr.open_for_read(next_id).unwrap(), next_fd);
-        mgr.rollback_rotation(next_id, next_fd);
+        assert_eq!(mgr.open_for_read(next_id).unwrap().fd(), next_handle.fd());
+        drop(next_handle);
+        mgr.rollback_rotation(next_id);
 
         let segments = mgr.list_segments().unwrap();
         assert_eq!(segments, vec![SegmentId::new(1)]);
@@ -443,7 +439,7 @@ mod tests {
     #[test]
     fn seal_segment_persists_index_and_marks_sealed() {
         let mgr = setup_manager(64 * 1024);
-        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
         let mut writer = SegmentWriter::new(
             mgr.io(),
             fd,
@@ -476,7 +472,7 @@ mod tests {
     #[test]
     fn delete_segment_removes_files_and_handle() {
         let mgr = setup_manager(64 * 1024);
-        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
         let mut writer = SegmentWriter::new(
             mgr.io(),
             fd,
@@ -507,9 +503,9 @@ mod tests {
         let mgr = setup_manager(1024);
         assert_eq!(mgr.oldest_segment().unwrap(), None);
 
-        mgr.open_for_append(SegmentId::new(3)).unwrap();
-        mgr.open_for_append(SegmentId::new(1)).unwrap();
-        mgr.open_for_append(SegmentId::new(5)).unwrap();
+        mgr.open_for_append(SegmentId::new(3)).unwrap().fd();
+        mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        mgr.open_for_append(SegmentId::new(5)).unwrap().fd();
 
         assert_eq!(mgr.oldest_segment().unwrap(), Some(SegmentId::new(1)));
     }
@@ -524,7 +520,7 @@ mod tests {
     fn rotate_and_write_across_segments() {
         let mgr = setup_manager(1024);
 
-        let fd1 = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd1 = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
         let mut writer1 = SegmentWriter::new(
             mgr.io(),
             fd1,
@@ -538,8 +534,9 @@ mod tests {
             .unwrap();
         writer1.sync(mgr.io()).unwrap();
 
-        let (id2, fd2) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
-        mgr.commit_rotation(id2, fd2);
+        let (id2, handle2) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
+        let fd2 = handle2.fd();
+        mgr.commit_rotation(id2, &handle2);
 
         let mut writer2 =
             SegmentWriter::new(mgr.io(), fd2, id2, EventSequence::new(2), MAX_EVENT_PAYLOAD)
@@ -549,7 +546,7 @@ mod tests {
             .unwrap();
         writer2.sync(mgr.io()).unwrap();
 
-        let fd1_read = mgr.open_for_read(SegmentId::new(1)).unwrap();
+        let fd1_read = mgr.open_for_read(SegmentId::new(1)).unwrap().fd();
         let events1 = crate::eventlog::SegmentReader::open(mgr.io(), fd1_read, MAX_EVENT_PAYLOAD)
             .unwrap()
             .valid_prefix()
@@ -557,7 +554,7 @@ mod tests {
         assert_eq!(events1.len(), 1);
         assert_eq!(events1[0].payload, b"first segment");
 
-        let fd2_read = mgr.open_for_read(id2).unwrap();
+        let fd2_read = mgr.open_for_read(id2).unwrap().fd();
         let events2 = crate::eventlog::SegmentReader::open(mgr.io(), fd2_read, MAX_EVENT_PAYLOAD)
             .unwrap()
             .valid_prefix()
@@ -569,7 +566,7 @@ mod tests {
     #[test]
     fn seal_then_append_errors() {
         let mgr = setup_manager(64 * 1024);
-        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
         SegmentWriter::new(
             mgr.io(),
             fd,
@@ -596,9 +593,9 @@ mod tests {
     #[test]
     fn multiple_deletions_increment_epoch() {
         let mgr = setup_manager(1024);
-        mgr.open_for_append(SegmentId::new(1)).unwrap();
-        mgr.open_for_append(SegmentId::new(2)).unwrap();
-        mgr.open_for_append(SegmentId::new(3)).unwrap();
+        mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        mgr.open_for_append(SegmentId::new(2)).unwrap().fd();
+        mgr.open_for_append(SegmentId::new(3)).unwrap().fd();
 
         assert_eq!(mgr.retention_epoch(), 0);
         mgr.delete_segment(SegmentId::new(1)).unwrap();
@@ -610,7 +607,7 @@ mod tests {
     #[test]
     fn open_for_read_does_not_infer_sealed_from_index_file() {
         let mgr = setup_manager(64 * 1024);
-        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
         let mut writer = SegmentWriter::new(
             mgr.io(),
             fd,
@@ -630,26 +627,26 @@ mod tests {
 
         mgr.handles.write().remove(&SegmentId::new(1));
 
-        let _read_fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
+        let _read_fd = mgr.open_for_read(SegmentId::new(1)).unwrap().fd();
         assert!(!mgr.is_sealed(SegmentId::new(1)));
     }
 
     #[test]
     fn open_for_read_unsealed_allows_append() {
         let mgr = setup_manager(1024);
-        let _fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let _fd = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
 
         mgr.handles.write().remove(&SegmentId::new(1));
 
-        let _read_fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
+        let _read_fd = mgr.open_for_read(SegmentId::new(1)).unwrap().fd();
         assert!(!mgr.is_sealed(SegmentId::new(1)));
     }
 
     #[test]
     fn shutdown_clears_handles() {
         let mgr = setup_manager(1024);
-        mgr.open_for_append(SegmentId::new(1)).unwrap();
-        mgr.open_for_append(SegmentId::new(2)).unwrap();
+        mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
+        mgr.open_for_append(SegmentId::new(2)).unwrap().fd();
 
         mgr.shutdown();
         assert!(mgr.handles.read().is_empty());
@@ -665,7 +662,7 @@ mod tests {
     #[test]
     fn prepare_rotation_truncates_stale_file() {
         let mgr = setup_manager(1024);
-        let _fd0 = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let _fd0 = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
 
         let stale_path = mgr.segment_path(SegmentId::new(2));
         let stale_fd = mgr
@@ -677,23 +674,23 @@ mod tests {
         assert_eq!(mgr.io().file_size(stale_fd).unwrap(), 4096);
         mgr.io().close(stale_fd).unwrap();
 
-        let (next_id, next_fd) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
+        let (next_id, next_handle) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
         assert_eq!(next_id, SegmentId::new(2));
-        assert_eq!(mgr.io().file_size(next_fd).unwrap(), 0);
+        assert_eq!(mgr.io().file_size(next_handle.fd()).unwrap(), 0);
     }
 
     #[test]
     fn open_for_append_upgrades_read_only_handle() {
         let mgr = setup_manager(1024);
-        let fd_append = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd_append = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
 
         mgr.handles.write().remove(&SegmentId::new(1));
 
-        let fd_read = mgr.open_for_read(SegmentId::new(1)).unwrap();
+        let fd_read = mgr.open_for_read(SegmentId::new(1)).unwrap().fd();
         assert_ne!(fd_read, fd_append);
         assert!(!mgr.handles.read().get(&SegmentId::new(1)).unwrap().writable);
 
-        let fd_upgraded = mgr.open_for_append(SegmentId::new(1)).unwrap();
+        let fd_upgraded = mgr.open_for_append(SegmentId::new(1)).unwrap().fd();
         assert_ne!(fd_upgraded, fd_read);
         assert!(mgr.handles.read().get(&SegmentId::new(1)).unwrap().writable);
     }

crates/tranquil-store/src/eventlog/mod.rs

@@ -411,8 +411,8 @@ impl<S: StorageIO + 'static> EventLog<S> {
     pub fn disk_usage(&self) -> io::Result<u64> {
         let segments = self.manager.list_segments()?;
         segments.iter().try_fold(0u64, |acc, &id| {
-            let fd = self.manager.open_for_read(id)?;
-            let size = self.manager.io().file_size(fd)?;
+            let handle = self.manager.open_for_read(id)?;
+            let size = self.manager.io().file_size(handle.fd())?;
             Ok(acc.saturating_add(size))
         })
     }

crates/tranquil-store/src/eventlog/reader.rs

@@ -145,10 +145,10 @@ impl<S: StorageIO> EventLogReader<S> {
     }
 
     fn rebuild_index(&self, segment_id: SegmentId) -> io::Result<SegmentIndex> {
-        let fd = self.manager.open_for_read(segment_id)?;
+        let handle = self.manager.open_for_read(segment_id)?;
         let (idx, _) = rebuild_from_segment(
             self.manager.io(),
-            fd,
+            handle.fd(),
             DEFAULT_INDEX_INTERVAL,
             self.max_payload,
         )?;
@@ -243,8 +243,8 @@ impl<S: StorageIO> EventLogReader<S> {
             return Ok(Arc::clone(m));
         }
 
-        let fd = self.manager.open_for_read(segment_id)?;
-        let mapped = self.manager.io().mmap_file(fd)?;
+        let handle = self.manager.open_for_read(segment_id)?;
+        let mapped = self.manager.io().mmap_file(handle.fd())?;
         let arc = Arc::new(mapped);
         self.mmaps.write().insert(segment_id, Arc::clone(&arc));
         Ok(arc)
@@ -269,10 +269,10 @@ impl<S: StorageIO> EventLogReader<S> {
                 predicate,
             )
         } else {
-            let fd = self.manager.open_for_read(segment_id)?;
-            let file_size = self.manager.io().file_size(fd)?;
+            let handle = self.manager.open_for_read(segment_id)?;
+            let file_size = self.manager.io().file_size(handle.fd())?;
             self.scan_direct(
-                fd,
+                handle.fd(),
                 file_size,
                 start_offset,
                 start_seq,
@@ -513,8 +513,8 @@ impl<S: StorageIO> EventLogReader<S> {
     }
 
     fn rebuild_sidecar(&self, segment_id: SegmentId) -> io::Result<SidecarIndex> {
-        let fd = self.manager.open_for_read(segment_id)?;
-        let sidecar = build_sidecar_from_segment(self.manager.io(), fd, self.max_payload)?;
+        let handle = self.manager.open_for_read(segment_id)?;
+        let sidecar = build_sidecar_from_segment(self.manager.io(), handle.fd(), self.max_payload)?;
         let _ = sidecar.save(self.manager.io(), &self.manager.sidecar_path(segment_id));
         Ok(sidecar)
     }
@@ -552,7 +552,7 @@ fn decode_mmap_event(
             segment = %segment_id,
             offset = raw,
             file_size,
-            "decode offset past file size (corrupt index?)"
+            "decode offset past file size, index likely corrupt"
         );
         return Ok(MmapDecodeResult::Corrupted);
     }

crates/tranquil-store/src/eventlog/writer.rs

@@ -66,9 +66,10 @@ impl<S: StorageIO> EventLogWriter<S> {
         index_interval: usize,
         max_payload: u32,
     ) -> io::Result<Self> {
-        let fd = manager.open_for_append(segment_id)?;
-        manager.io().truncate(fd, 0)?;
-        let writer = SegmentWriter::new(manager.io(), fd, segment_id, next_seq, max_payload)?;
+        let handle = manager.open_for_append(segment_id)?;
+        manager.io().truncate(handle.fd(), 0)?;
+        let writer =
+            SegmentWriter::new(manager.io(), handle.fd(), segment_id, next_seq, max_payload)?;
         writer.sync(manager.io())?;
         manager.io().sync_dir(manager.segments_dir())?;
 
@@ -93,7 +94,8 @@ impl<S: StorageIO> EventLogWriter<S> {
         index_interval: usize,
         max_payload: u32,
     ) -> io::Result<Self> {
-        let fd = manager.open_for_append(active_id)?;
+        let handle = manager.open_for_append(active_id)?;
+        let fd = handle.fd();
 
         let (index, last_seq_in_active) = match rebuild_from_segment(
             manager.io(),
@@ -305,14 +307,14 @@ impl<S: StorageIO> EventLogWriter<S> {
 
         match self.build_sidecar_for_segment(old_id) {
             Ok(()) => {}
-            Err(e) => warn!(segment = %old_id, error = %e, "sidecar build failed (non-fatal)"),
+            Err(e) => warn!(segment = %old_id, error = %e, "non-fatal sidecar build failure"),
         }
 
-        let (new_id, new_fd) = self.manager.prepare_rotation(old_id)?;
+        let (new_id, new_handle) = self.manager.prepare_rotation(old_id)?;
 
         match SegmentWriter::new::<S>(
             self.manager.io(),
-            new_fd,
+            new_handle.fd(),
             new_id,
             self.next_seq,
             self.max_payload,
@@ -322,11 +324,12 @@ impl<S: StorageIO> EventLogWriter<S> {
                 self.active_index = SegmentIndex::new();
                 self.event_count_in_segment = 0;
                 self.last_event_offset = None;
-                self.manager.commit_rotation(new_id, new_fd);
+                self.manager.commit_rotation(new_id, &new_handle);
                 Ok(Some(old_id))
             }
             Err(e) => {
-                self.manager.rollback_rotation(new_id, new_fd);
+                drop(new_handle);
+                self.manager.rollback_rotation(new_id);
                 Err(e)
             }
         }
@@ -361,8 +364,8 @@ impl<S: StorageIO> EventLogWriter<S> {
     }
 
     fn build_sidecar_for_segment(&self, segment_id: SegmentId) -> io::Result<()> {
-        let fd = self.manager.open_for_read(segment_id)?;
-        let sidecar = build_sidecar_from_segment(self.manager.io(), fd, self.max_payload)?;
+        let handle = self.manager.open_for_read(segment_id)?;
+        let sidecar = build_sidecar_from_segment(self.manager.io(), handle.fd(), self.max_payload)?;
         let path = self.manager.sidecar_path(segment_id);
         sidecar.save(self.manager.io(), &path)
     }
@@ -397,9 +400,13 @@ fn find_last_seq_from_segments<S: StorageIO>(
             Ok(Some(idx)) => Ok(idx.last_seq()),
             Err(e) if e.kind() != io::ErrorKind::InvalidData => Err(e),
             _ => {
-                let fd = manager.open_for_read(seg_id)?;
-                let (_, last_seq) =
-                    rebuild_from_segment(manager.io(), fd, DEFAULT_INDEX_INTERVAL, max_payload)?;
+                let handle = manager.open_for_read(seg_id)?;
+                let (_, last_seq) = rebuild_from_segment(
+                    manager.io(),
+                    handle.fd(),
+                    DEFAULT_INDEX_INTERVAL,
+                    max_payload,
+                )?;
                 Ok(last_seq)
             }
         }
@@ -551,7 +558,7 @@ mod tests {
         assert_eq!(writer.synced_seq(), EventSequence::new(5));
         assert_eq!(writer.active_segment_id(), SegmentId::new(1));
 
-        let fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
+        let fd = mgr.open_for_read(SegmentId::new(1)).unwrap().fd();
         let events = SegmentReader::open(mgr.io(), fd, MAX_EVENT_PAYLOAD)
             .unwrap()
             .valid_prefix()
@@ -796,7 +803,7 @@ mod tests {
                 .unwrap();
         assert_eq!(writer.next_seq, EventSequence::new(3));
 
-        let fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
+        let fd = mgr.open_for_read(SegmentId::new(1)).unwrap().fd();
         let events = SegmentReader::open(mgr.io(), fd, MAX_EVENT_PAYLOAD)
             .unwrap()
             .valid_prefix()
@@ -1003,7 +1010,7 @@ mod tests {
         assert_eq!(seq, EventSequence::new(4));
         writer.sync().unwrap();
 
-        let fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
+        let fd = mgr.open_for_read(SegmentId::new(1)).unwrap().fd();
         let events = SegmentReader::open(mgr.io(), fd, MAX_EVENT_PAYLOAD)
             .unwrap()
             .valid_prefix()

crates/tranquil-store/src/gauntlet/farm.rs

@@ -0,0 +1,86 @@
+use std::cell::RefCell;
+use std::panic::{AssertUnwindSafe, catch_unwind};
+use std::time::{Duration, Instant};
+
+use rayon::prelude::*;
+use tokio::runtime::Runtime;
+
+use super::invariants::InvariantViolation;
+use super::op::{OpStream, Seed};
+use super::runner::{
+    Gauntlet, GauntletConfig, GauntletReport, OpErrorCount, OpsExecuted, RestartCount,
+};
+
+thread_local! {
+    static RUNTIME: RefCell<Option<Runtime>> = const { RefCell::new(None) };
+}
+
+fn with_runtime<R>(f: impl FnOnce(&Runtime) -> R) -> R {
+    RUNTIME.with(|cell| {
+        let mut slot = cell.borrow_mut();
+        if slot.is_none() {
+            *slot = Some(
+                tokio::runtime::Builder::new_current_thread()
+                    .enable_all()
+                    .build()
+                    .expect("build rt"),
+            );
+        }
+        f(slot.as_ref().expect("runtime present"))
+    })
+}
+
+pub fn run_many<F>(make_config: F, seeds: impl IntoIterator<Item = Seed>) -> Vec<GauntletReport>
+where
+    F: Fn(Seed) -> GauntletConfig + Sync + Send,
+{
+    run_many_timed(make_config, seeds)
+        .into_iter()
+        .map(|(r, _)| r)
+        .collect()
+}
+
+pub fn run_many_timed<F>(
+    make_config: F,
+    seeds: impl IntoIterator<Item = Seed>,
+) -> Vec<(GauntletReport, Duration)>
+where
+    F: Fn(Seed) -> GauntletConfig + Sync + Send,
+{
+    let seeds: Vec<Seed> = seeds.into_iter().collect();
+    seeds
+        .into_par_iter()
+        .map(|s| {
+            let start = Instant::now();
+            let outcome = catch_unwind(AssertUnwindSafe(|| {
+                let cfg = make_config(s);
+                let gauntlet = Gauntlet::new(cfg).expect("build gauntlet");
+                with_runtime(|rt| rt.block_on(gauntlet.run()))
+            }));
+            let report = outcome.unwrap_or_else(|payload| {
+                RUNTIME.with(|cell| cell.borrow_mut().take());
+                panic_report(s, payload)
+            });
+            (report, start.elapsed())
+        })
+        .collect()
+}
+
+fn panic_report(seed: Seed, payload: Box<dyn std::any::Any + Send>) -> GauntletReport {
+    let msg = payload
+        .downcast_ref::<&'static str>()
+        .map(|s| (*s).to_string())
+        .or_else(|| payload.downcast_ref::<String>().cloned())
+        .unwrap_or_else(|| "non-string panic payload".to_string());
+    GauntletReport {
+        seed,
+        ops_executed: OpsExecuted(0),
+        op_errors: OpErrorCount(0),
+        restarts: RestartCount(0),
+        violations: vec![InvariantViolation {
+            invariant: "FarmPanic",
+            detail: msg,
+        }],
+        ops: OpStream::empty(),
+    }
+}

crates/tranquil-store/src/gauntlet/flaky.rs

@@ -0,0 +1,448 @@
+use std::env;
+use std::num::NonZeroU32;
+use std::path::{Path, PathBuf};
+use std::process::{Command, Output};
+use std::sync::atomic::{AtomicU64, Ordering};
+
+use tempfile::TempDir;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct UpIntervalSecs(pub NonZeroU32);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct DownIntervalSecs(pub NonZeroU32);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct BackingMegabytes(pub u32);
+
+const fn nz(n: u32) -> NonZeroU32 {
+    match NonZeroU32::new(n) {
+        Some(v) => v,
+        None => panic!("zero interval not permitted"),
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct FlakyConfig {
+    pub up_interval: UpIntervalSecs,
+    pub down_interval: DownIntervalSecs,
+    pub backing_mb: BackingMegabytes,
+}
+
+impl FlakyConfig {
+    pub const fn default_stress() -> Self {
+        Self {
+            up_interval: UpIntervalSecs(nz(8)),
+            down_interval: DownIntervalSecs(nz(2)),
+            backing_mb: BackingMegabytes(256),
+        }
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum FlakyError {
+    #[error("not running as root, EUID != 0")]
+    NotRoot,
+    #[error("tool missing: {0}")]
+    ToolMissing(&'static str),
+    #[error("kernel target dm-flakey unavailable: {0}")]
+    DmFlakeyMissing(String),
+    #[error("{tool} failed: status={status}, stderr={stderr}")]
+    CommandFailed {
+        tool: &'static str,
+        status: String,
+        stderr: String,
+    },
+    #[error("io: {0}")]
+    Io(#[from] std::io::Error),
+}
+
+impl FlakyError {
+    pub const fn is_env_absent(&self) -> bool {
+        matches!(
+            self,
+            Self::NotRoot | Self::ToolMissing(_) | Self::DmFlakeyMissing(_)
+        )
+    }
+}
+
+static MOUNT_COUNTER: AtomicU64 = AtomicU64::new(0);
+
+pub struct FlakyMount {
+    mount_point: TempDir,
+    mapper_name: String,
+    mapper_path: PathBuf,
+    loop_device: PathBuf,
+    backing_file: PathBuf,
+    _backing_tempdir: TempDir,
+}
+
+impl FlakyMount {
+    pub fn try_new(cfg: &FlakyConfig) -> Result<Self, FlakyError> {
+        ensure_root()?;
+        ["losetup", "dmsetup", "mkfs.ext4", "mount", "umount"]
+            .iter()
+            .copied()
+            .try_for_each(ensure_tool)?;
+        probe_dm_flakey()?;
+        let _ = reap_stale_mounts();
+
+        let backing_dir = TempDir::new()?;
+        let backing_file = backing_dir.path().join("backing.img");
+        allocate_backing(&backing_file, cfg.backing_mb)?;
+
+        let loop_device = attach_loop(&backing_file)?;
+        let sectors = sector_count(&loop_device)?;
+
+        if let Err(e) = mkfs_ext4(&loop_device) {
+            let _ = detach_loop(&loop_device);
+            return Err(e);
+        }
+
+        let mapper_name = format!(
+            "tranquil-flaky-{}-{}",
+            std::process::id(),
+            MOUNT_COUNTER.fetch_add(1, Ordering::Relaxed),
+        );
+        match dm_create(
+            &mapper_name,
+            &loop_device,
+            sectors,
+            cfg.up_interval,
+            cfg.down_interval,
+        ) {
+            Ok(()) => {}
+            Err(e) => {
+                let _ = detach_loop(&loop_device);
+                return Err(e);
+            }
+        }
+        let mapper_path = PathBuf::from(format!("/dev/mapper/{mapper_name}"));
+
+        let mount_point = TempDir::new()?;
+        if let Err(e) = mount_ext4(&mapper_path, mount_point.path()) {
+            let _ = dm_remove(&mapper_name);
+            let _ = detach_loop(&loop_device);
+            return Err(e);
+        }
+
+        Ok(Self {
+            mount_point,
+            mapper_name,
+            mapper_path,
+            loop_device,
+            backing_file,
+            _backing_tempdir: backing_dir,
+        })
+    }
+
+    pub fn path(&self) -> &Path {
+        self.mount_point.path()
+    }
+
+    pub fn mapper_name(&self) -> &str {
+        &self.mapper_name
+    }
+
+    pub fn mapper_path(&self) -> &Path {
+        &self.mapper_path
+    }
+
+    pub fn loop_device(&self) -> &Path {
+        &self.loop_device
+    }
+
+    pub fn backing_file(&self) -> &Path {
+        &self.backing_file
+    }
+}
+
+impl Drop for FlakyMount {
+    fn drop(&mut self) {
+        if let Err(e) = umount(self.mount_point.path()) {
+            tracing::warn!(
+                mount = %self.mount_point.path().display(),
+                error = %e,
+                "flaky umount failed, trying lazy unmount",
+            );
+            if let Err(e2) = umount_lazy(self.mount_point.path()) {
+                tracing::warn!(
+                    mount = %self.mount_point.path().display(),
+                    error = %e2,
+                    "flaky lazy unmount also failed, device may leak",
+                );
+            }
+        }
+        if let Err(e) = dm_remove(&self.mapper_name) {
+            tracing::warn!(
+                name = %self.mapper_name,
+                error = %e,
+                "flaky dm remove failed, mapper device may leak",
+            );
+        }
+        if let Err(e) = detach_loop(&self.loop_device) {
+            tracing::warn!(
+                device = %self.loop_device.display(),
+                error = %e,
+                "flaky loop detach failed, loop device may leak",
+            );
+        }
+    }
+}
+
+#[cfg(unix)]
+fn ensure_root() -> Result<(), FlakyError> {
+    if unsafe { libc::geteuid() } == 0 {
+        Ok(())
+    } else {
+        Err(FlakyError::NotRoot)
+    }
+}
+
+#[cfg(not(unix))]
+fn ensure_root() -> Result<(), FlakyError> {
+    Err(FlakyError::NotRoot)
+}
+
+fn ensure_tool(tool: &'static str) -> Result<(), FlakyError> {
+    match find_in_path(tool) {
+        Some(_) => Ok(()),
+        None => Err(FlakyError::ToolMissing(tool)),
+    }
+}
+
+fn find_in_path(tool: &str) -> Option<PathBuf> {
+    let path = env::var_os("PATH")?;
+    env::split_paths(&path).find_map(|dir| {
+        let candidate = dir.join(tool);
+        is_executable_file(&candidate).then_some(candidate)
+    })
+}
+
+#[cfg(unix)]
+fn is_executable_file(p: &Path) -> bool {
+    use std::os::unix::fs::PermissionsExt;
+    std::fs::metadata(p)
+        .map(|m| m.is_file() && (m.permissions().mode() & 0o111) != 0)
+        .unwrap_or(false)
+}
+
+#[cfg(not(unix))]
+fn is_executable_file(p: &Path) -> bool {
+    p.is_file()
+}
+
+fn probe_dm_flakey() -> Result<(), FlakyError> {
+    let out = Command::new("dmsetup").arg("targets").output()?;
+    if !out.status.success() {
+        return Err(FlakyError::DmFlakeyMissing(stringify_output(&out)));
+    }
+    let stdout = String::from_utf8_lossy(&out.stdout);
+    if !stdout.lines().any(|l| l.starts_with("flakey")) {
+        return Err(FlakyError::DmFlakeyMissing(stdout.into_owned()));
+    }
+    Ok(())
+}
+
+fn reap_stale_mounts() -> Result<(), FlakyError> {
+    let out = Command::new("dmsetup")
+        .arg("ls")
+        .arg("--target")
+        .arg("flakey")
+        .output()?;
+    if !out.status.success() {
+        return Ok(());
+    }
+    String::from_utf8_lossy(&out.stdout)
+        .lines()
+        .filter_map(parse_flaky_entry)
+        .filter(|(_, pid)| !pid_alive(*pid))
+        .for_each(|(name, _)| {
+            let loop_device = mapper_backing_loop(&name);
+            if let Err(e) = dm_remove(&name) {
+                tracing::warn!(name, error = %e, "reap: dm_remove stale mapper failed");
+                return;
+            }
+            if let Some(loop_dev) = loop_device
+                && let Err(e) = detach_loop(&loop_dev)
+            {
+                tracing::warn!(
+                    device = %loop_dev.display(),
+                    error = %e,
+                    "reap: detach_loop stale device failed",
+                );
+            }
+        });
+    Ok(())
+}
+
+fn parse_flaky_entry(line: &str) -> Option<(String, u32)> {
+    let name = line.split_whitespace().next()?;
+    let suffix = name.strip_prefix("tranquil-flaky-")?;
+    let pid_str = suffix.split('-').next()?;
+    let pid = pid_str.parse::<u32>().ok()?;
+    Some((name.to_string(), pid))
+}
+
+fn pid_alive(pid: u32) -> bool {
+    Path::new(&format!("/proc/{pid}")).exists()
+}
+
+fn mapper_backing_loop(name: &str) -> Option<PathBuf> {
+    let out = Command::new("dmsetup")
+        .arg("deps")
+        .arg("-o")
+        .arg("devname")
+        .arg(name)
+        .output()
+        .ok()?;
+    if !out.status.success() {
+        return None;
+    }
+    let text = String::from_utf8_lossy(&out.stdout);
+    let inner = text.split('(').nth(1)?;
+    let dev = inner.split(')').next()?.trim();
+    if dev.is_empty() {
+        None
+    } else {
+        Some(PathBuf::from(format!("/dev/{dev}")))
+    }
+}
+
+fn allocate_backing(path: &Path, size: BackingMegabytes) -> Result<(), FlakyError> {
+    let out = Command::new("truncate")
+        .arg("-s")
+        .arg(format!("{}M", size.0))
+        .arg(path)
+        .output()?;
+    check_status("truncate", &out)
+}
+
+fn attach_loop(backing: &Path) -> Result<PathBuf, FlakyError> {
+    let out = Command::new("losetup")
+        .arg("--find")
+        .arg("--show")
+        .arg(backing)
+        .output()?;
+    check_status("losetup", &out)?;
+    let raw = String::from_utf8_lossy(&out.stdout).trim().to_string();
+    if raw.is_empty() {
+        return Err(FlakyError::CommandFailed {
+            tool: "losetup",
+            status: "exit 0".to_string(),
+            stderr: "no device path on stdout".to_string(),
+        });
+    }
+    Ok(PathBuf::from(raw))
+}
+
+fn detach_loop(device: &Path) -> Result<(), FlakyError> {
+    let out = Command::new("losetup").arg("-d").arg(device).output()?;
+    check_status("losetup -d", &out)
+}
+
+fn sector_count(device: &Path) -> Result<u64, FlakyError> {
+    let out = Command::new("blockdev")
+        .arg("--getsz")
+        .arg(device)
+        .output()?;
+    check_status("blockdev", &out)?;
+    let raw = String::from_utf8_lossy(&out.stdout).trim().to_string();
+    raw.parse::<u64>().map_err(|_| FlakyError::CommandFailed {
+        tool: "blockdev",
+        status: "exit 0".to_string(),
+        stderr: format!("could not parse sector count: {raw:?}"),
+    })
+}
+
+fn dm_create(
+    name: &str,
+    loop_device: &Path,
+    sectors: u64,
+    up: UpIntervalSecs,
+    down: DownIntervalSecs,
+) -> Result<(), FlakyError> {
+    let table = format!(
+        "0 {sectors} flakey {} 0 {} {}",
+        loop_device.display(),
+        up.0.get(),
+        down.0.get(),
+    );
+    let mut child = Command::new("dmsetup")
+        .arg("create")
+        .arg(name)
+        .stdin(std::process::Stdio::piped())
+        .stdout(std::process::Stdio::piped())
+        .stderr(std::process::Stdio::piped())
+        .spawn()?;
+    if let Some(stdin) = child.stdin.as_mut() {
+        use std::io::Write;
+        stdin.write_all(table.as_bytes())?;
+    }
+    let out = child.wait_with_output()?;
+    check_status("dmsetup create", &out)
+}
+
+fn dm_remove(name: &str) -> Result<(), FlakyError> {
+    let out = Command::new("dmsetup")
+        .arg("remove")
+        .arg("--retry")
+        .arg(name)
+        .output()?;
+    check_status("dmsetup remove", &out)
+}
+
+fn mkfs_ext4(device: &Path) -> Result<(), FlakyError> {
+    let out = Command::new("mkfs.ext4")
+        .arg("-q")
+        .arg("-F")
+        .arg(device)
+        .output()?;
+    check_status("mkfs.ext4", &out)
+}
+
+fn mount_ext4(device: &Path, target: &Path) -> Result<(), FlakyError> {
+    let out = Command::new("mount")
+        .arg("-t")
+        .arg("ext4")
+        .arg(device)
+        .arg(target)
+        .output()?;
+    check_status("mount", &out)
+}
+
+fn umount(target: &Path) -> Result<(), FlakyError> {
+    let out = Command::new("umount").arg(target).output()?;
+    check_status("umount", &out)
+}
+
+fn umount_lazy(target: &Path) -> Result<(), FlakyError> {
+    let out = Command::new("umount").arg("-l").arg(target).output()?;
+    check_status("umount -l", &out)
+}
+
+fn check_status(tool: &'static str, out: &Output) -> Result<(), FlakyError> {
+    if out.status.success() {
+        Ok(())
+    } else {
+        Err(FlakyError::CommandFailed {
+            tool,
+            status: format!("{}", out.status),
+            stderr: String::from_utf8_lossy(&out.stderr).into_owned(),
+        })
+    }
+}
+
+fn stringify_output(out: &Output) -> String {
+    let mut s = String::new();
+    if !out.stdout.is_empty() {
+        s.push_str(&String::from_utf8_lossy(&out.stdout));
+    }
+    if !out.stderr.is_empty() {
+        if !s.is_empty() {
+            s.push('\n');
+        }
+        s.push_str(&String::from_utf8_lossy(&out.stderr));
+    }
+    s
+}

crates/tranquil-store/src/gauntlet/invariants.rs

@@ -0,0 +1,783 @@
+use std::collections::{HashMap, HashSet};
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use cid::Cid;
+use jacquard_repo::mst::Mst;
+
+use super::oracle::{Oracle, hex_short, try_cid_to_fixed};
+use crate::blockstore::{
+    BLOCK_HEADER_SIZE, CidBytes, CompactionError, TranquilBlockStore, hash_to_cid_bytes,
+};
+use crate::eventlog::{EventSequence, SegmentId};
+use crate::io::{RealIO, StorageIO};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct InvariantSet(u32);
+
+impl InvariantSet {
+    pub const EMPTY: Self = Self(0);
+    pub const REFCOUNT_CONSERVATION: Self = Self(1 << 0);
+    pub const REACHABILITY: Self = Self(1 << 1);
+    pub const ACKED_WRITE_PERSISTENCE: Self = Self(1 << 2);
+    pub const READ_AFTER_WRITE: Self = Self(1 << 3);
+    pub const RESTART_IDEMPOTENT: Self = Self(1 << 4);
+    pub const COMPACTION_IDEMPOTENT: Self = Self(1 << 5);
+    pub const NO_ORPHAN_FILES: Self = Self(1 << 6);
+    pub const BYTE_BUDGET: Self = Self(1 << 7);
+    pub const MANIFEST_EQUALS_REALITY: Self = Self(1 << 8);
+    pub const CHECKSUM_COVERAGE: Self = Self(1 << 9);
+    pub const MONOTONIC_SEQ: Self = Self(1 << 10);
+    pub const FSYNC_ORDERING: Self = Self(1 << 11);
+    pub const TOMBSTONE_BOUND: Self = Self(1 << 12);
+
+    const ALL_KNOWN: u32 = Self::REFCOUNT_CONSERVATION.0
+        | Self::REACHABILITY.0
+        | Self::ACKED_WRITE_PERSISTENCE.0
+        | Self::READ_AFTER_WRITE.0
+        | Self::RESTART_IDEMPOTENT.0
+        | Self::COMPACTION_IDEMPOTENT.0
+        | Self::NO_ORPHAN_FILES.0
+        | Self::BYTE_BUDGET.0
+        | Self::MANIFEST_EQUALS_REALITY.0
+        | Self::CHECKSUM_COVERAGE.0
+        | Self::MONOTONIC_SEQ.0
+        | Self::FSYNC_ORDERING.0
+        | Self::TOMBSTONE_BOUND.0;
+
+    pub const fn contains(self, other: Self) -> bool {
+        (self.0 & other.0) == other.0
+    }
+
+    pub const fn union(self, other: Self) -> Self {
+        Self(self.0 | other.0)
+    }
+
+    pub const fn without(self, other: Self) -> Self {
+        Self(self.0 & !other.0)
+    }
+
+    pub const fn unknown_bits(self) -> u32 {
+        self.0 & !Self::ALL_KNOWN
+    }
+}
+
+impl std::ops::BitOr for InvariantSet {
+    type Output = Self;
+    fn bitor(self, rhs: Self) -> Self {
+        self.union(rhs)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct InvariantViolation {
+    pub invariant: &'static str,
+    pub detail: String,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct SnapshotEvent {
+    pub seq: EventSequence,
+    pub timestamp_us: u64,
+    pub event_type_raw: u8,
+    pub did_hash: u32,
+}
+
+#[derive(Debug, Clone)]
+pub struct EventLogSnapshot {
+    pub segments_dir: PathBuf,
+    pub max_segment_size: u64,
+    pub synced_seq: EventSequence,
+    pub segments: Vec<SegmentId>,
+    pub events: Vec<SnapshotEvent>,
+    pub segment_last_ts: Vec<(SegmentId, u64)>,
+}
+
+pub struct InvariantCtx<'a, S: StorageIO + Send + Sync + 'static = RealIO> {
+    pub store: &'a Arc<TranquilBlockStore<S>>,
+    pub oracle: &'a Oracle,
+    pub root: Option<Cid>,
+    pub eventlog: Option<&'a EventLogSnapshot>,
+}
+
+#[async_trait]
+pub trait Invariant<S: StorageIO + Send + Sync + 'static>: Send + Sync {
+    fn name(&self) -> &'static str;
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation>;
+}
+
+pub struct RefcountConservation;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for RefcountConservation {
+    fn name(&self) -> &'static str {
+        "RefcountConservation"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let live: Vec<(String, CidBytes)> = ctx.oracle.live_cids_labeled();
+        let live_set: HashSet<CidBytes> = live.iter().map(|(_, c)| *c).collect();
+        let index: HashMap<CidBytes, u32> = ctx
+            .store
+            .block_index()
+            .live_entries_snapshot()
+            .into_iter()
+            .map(|(c, r)| (c, r.raw()))
+            .collect();
+
+        let forward: Vec<String> = live
+            .iter()
+            .filter_map(|(label, cid)| match index.get(cid) {
+                Some(&r) if r >= 1 => None,
+                Some(&r) => Some(format!("{label}: refcount {r}")),
+                None => Some(format!("{label}: missing from index")),
+            })
+            .collect();
+
+        let inverse: Vec<String> = index
+            .iter()
+            .filter(|(cid, _)| !live_set.contains(*cid))
+            .map(|(cid, r)| format!("orphan cid {} refcount {}", hex_short(cid), r))
+            .collect();
+
+        let violations: Vec<String> = forward.into_iter().chain(inverse).collect();
+        if violations.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "RefcountConservation",
+                detail: violations.join("; "),
+            })
+        }
+    }
+}
+
+pub struct Reachability;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for Reachability {
+    fn name(&self) -> &'static str {
+        "Reachability"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let violations: Vec<String> = ctx
+            .oracle
+            .live_cids_labeled()
+            .into_iter()
+            .filter_map(|(label, fixed)| match ctx.store.get_block_sync(&fixed) {
+                Ok(Some(_)) => None,
+                Ok(None) => Some(format!("{label}: missing")),
+                Err(e) => Some(format!("{label}: read error {e}")),
+            })
+            .collect();
+
+        if violations.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "Reachability",
+                detail: violations.join("; "),
+            })
+        }
+    }
+}
+
+pub struct AckedWritePersistence;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for AckedWritePersistence {
+    fn name(&self) -> &'static str {
+        "AckedWritePersistence"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let Some(root) = ctx.root else {
+            if ctx.oracle.live_count() == 0 {
+                return Ok(());
+            }
+            return Err(InvariantViolation {
+                invariant: "AckedWritePersistence",
+                detail: format!(
+                    "oracle has {} live records but reopened store has no root",
+                    ctx.oracle.live_count()
+                ),
+            });
+        };
+        let mst = Mst::load(ctx.store.clone(), root, None);
+        let keys: Vec<String> = ctx
+            .oracle
+            .live_records()
+            .map(|(c, r, _)| format!("{}/{}", c.0, r.0))
+            .collect();
+
+        let mut missing: Vec<String> = Vec::new();
+        for key in &keys {
+            match mst.get(key).await {
+                Ok(Some(_)) => {}
+                Ok(None) => missing.push(format!("{key}: missing after reopen")),
+                Err(e) => missing.push(format!("{key}: mst.get error after reopen: {e}")),
+            }
+        }
+
+        if missing.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "AckedWritePersistence",
+                detail: missing.join("; "),
+            })
+        }
+    }
+}
+
+pub struct ReadAfterWrite;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for ReadAfterWrite {
+    fn name(&self) -> &'static str {
+        "ReadAfterWrite"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let Some(root) = ctx.root else {
+            return Ok(());
+        };
+        let mst = Mst::load(ctx.store.clone(), root, None);
+
+        let entries: Vec<(String, CidBytes)> = ctx
+            .oracle
+            .live_records()
+            .map(|(c, r, v)| (format!("{}/{}", c.0, r.0), *v))
+            .collect();
+
+        let mut violations: Vec<String> = Vec::new();
+        for (key, expected) in &entries {
+            match mst.get(key).await {
+                Ok(Some(cid)) => match try_cid_to_fixed(&cid) {
+                    Ok(actual) if actual == *expected => match ctx.store.get_block_sync(&actual) {
+                        Ok(Some(_)) => {}
+                        Ok(None) => violations.push(format!("{key}: block missing for cid")),
+                        Err(e) => violations.push(format!("{key}: block read error {e}")),
+                    },
+                    Ok(actual) => violations.push(format!(
+                        "{key}: MST cid {} != oracle cid {}",
+                        hex_short(&actual),
+                        hex_short(expected),
+                    )),
+                    Err(e) => {
+                        violations.push(format!("{key}: unexpected CID format from MST: {e}"))
+                    }
+                },
+                Ok(None) => violations.push(format!("{key}: MST returned None")),
+                Err(e) => violations.push(format!("{key}: mst.get error {e}")),
+            }
+        }
+
+        if violations.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "ReadAfterWrite",
+                detail: violations.join("; "),
+            })
+        }
+    }
+}
+
+pub struct CompactionIdempotent;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for CompactionIdempotent {
+    fn name(&self) -> &'static str {
+        "CompactionIdempotent"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let store_a = ctx.store.clone();
+        let first = tokio::task::spawn_blocking(move || compact_by_liveness(&store_a))
+            .await
+            .map_err(|e| InvariantViolation {
+                invariant: "CompactionIdempotent",
+                detail: format!("first compaction join: {e}"),
+            })?;
+        if let Err(e) = first {
+            return Err(InvariantViolation {
+                invariant: "CompactionIdempotent",
+                detail: format!("first compaction: {e}"),
+            });
+        }
+
+        let pre = snapshot(ctx.store);
+
+        let store_b = ctx.store.clone();
+        let second = tokio::task::spawn_blocking(move || compact_by_liveness(&store_b))
+            .await
+            .map_err(|e| InvariantViolation {
+                invariant: "CompactionIdempotent",
+                detail: format!("second compaction join: {e}"),
+            })?;
+        if let Err(e) = second {
+            return Err(InvariantViolation {
+                invariant: "CompactionIdempotent",
+                detail: format!("second compaction: {e}"),
+            });
+        }
+
+        let post = snapshot(ctx.store);
+
+        if pre == post {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "CompactionIdempotent",
+                detail: format!(
+                    "second compaction changed observable state: pre={} entries, post={} entries",
+                    pre.len(),
+                    post.len(),
+                ),
+            })
+        }
+    }
+}
+
+fn snapshot<S: StorageIO + Send + Sync + 'static>(
+    store: &Arc<TranquilBlockStore<S>>,
+) -> Vec<(CidBytes, u32)> {
+    let mut v: Vec<(CidBytes, u32)> = store
+        .block_index()
+        .live_entries_snapshot()
+        .into_iter()
+        .map(|(c, r)| (c, r.raw()))
+        .collect();
+    v.sort_unstable_by_key(|a| a.0);
+    v
+}
+
+const COMPACT_LIVENESS_CEILING: f64 = 0.99;
+
+fn compact_by_liveness<S: StorageIO + Send + Sync + 'static>(
+    store: &TranquilBlockStore<S>,
+) -> Result<(), String> {
+    let liveness = store
+        .compaction_liveness(0)
+        .map_err(|e| format!("compaction_liveness: {e}"))?;
+    let targets: Vec<_> = liveness
+        .iter()
+        .filter(|(_, info)| info.total_blocks > 0 && info.ratio() < COMPACT_LIVENESS_CEILING)
+        .map(|(&fid, _)| fid)
+        .collect();
+    targets
+        .into_iter()
+        .try_for_each(|fid| match store.compact_file(fid, 0) {
+            Ok(_) => Ok(()),
+            Err(CompactionError::ActiveFileCannotBeCompacted) => Ok(()),
+            Err(e) => Err(format!("{fid}: {e}")),
+        })
+}
+
+pub struct NoOrphanFiles;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for NoOrphanFiles {
+    fn name(&self) -> &'static str {
+        "NoOrphanFiles"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let store_c = ctx.store.clone();
+        let result = tokio::task::spawn_blocking(move || {
+            let disk = store_c.list_data_files().map_err(|e| e.to_string())?;
+            let liveness = store_c.compaction_liveness(0).map_err(|e| e.to_string())?;
+            let header = BLOCK_HEADER_SIZE as u64;
+            let orphans: Vec<String> = disk
+                .iter()
+                .filter(|fid| !liveness.contains_key(fid))
+                .filter_map(|fid| {
+                    let path = store_c.data_file_path(*fid);
+                    let size = std::fs::metadata(&path).map(|m| m.len()).unwrap_or(0);
+                    match size > header {
+                        true => Some(format!("{fid} ({size} B)")),
+                        false => None,
+                    }
+                })
+                .collect();
+            Ok::<_, String>(orphans)
+        })
+        .await
+        .map_err(|e| InvariantViolation {
+            invariant: "NoOrphanFiles",
+            detail: format!("join: {e}"),
+        })?;
+
+        let orphans = result.map_err(|e| InvariantViolation {
+            invariant: "NoOrphanFiles",
+            detail: e,
+        })?;
+
+        if orphans.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "NoOrphanFiles",
+                detail: format!("files on disk missing from index: {}", orphans.join(", ")),
+            })
+        }
+    }
+}
+
+pub struct ByteBudget {
+    pub overhead_factor: f64,
+    pub floor_bytes: u64,
+}
+
+impl Default for ByteBudget {
+    fn default() -> Self {
+        Self {
+            overhead_factor: 8.0,
+            floor_bytes: 1 << 20,
+        }
+    }
+}
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for ByteBudget {
+    fn name(&self) -> &'static str {
+        "ByteBudget"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let store = ctx.store.clone();
+        let factor = self.overhead_factor;
+        let floor = self.floor_bytes;
+        tokio::task::spawn_blocking(move || {
+            let liveness = store.compaction_liveness(0).map_err(|e| e.to_string())?;
+            let live: u64 = liveness.values().map(|i| i.live_bytes).sum();
+            let total: u64 = liveness.values().map(|i| i.total_bytes).sum();
+            let budget = (live as f64 * factor) as u64 + floor;
+            if total <= budget {
+                Ok(())
+            } else {
+                Err(format!(
+                    "total_bytes {total} exceeds budget {budget}: live_bytes {live}, factor {factor}, floor {floor}"
+                ))
+            }
+        })
+        .await
+        .map_err(|e| InvariantViolation {
+            invariant: "ByteBudget",
+            detail: format!("join: {e}"),
+        })?
+        .map_err(|e| InvariantViolation {
+            invariant: "ByteBudget",
+            detail: e,
+        })
+    }
+}
+
+pub struct ManifestEqualsReality;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for ManifestEqualsReality {
+    fn name(&self) -> &'static str {
+        "ManifestEqualsReality"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let store = ctx.store.clone();
+        tokio::task::spawn_blocking(move || {
+            let listed = store.list_data_files().map_err(|e| e.to_string())?;
+            let liveness = store.compaction_liveness(0).map_err(|e| e.to_string())?;
+            let header = BLOCK_HEADER_SIZE as u64;
+
+            let mut violations: Vec<String> = Vec::new();
+            listed.iter().for_each(|fid| {
+                let path = store.data_file_path(*fid);
+                match std::fs::metadata(&path) {
+                    Err(e) => violations.push(format!("{fid}: metadata {e}")),
+                    Ok(meta) => {
+                        let on_disk = meta.len();
+                        let content = on_disk.saturating_sub(header);
+                        match liveness.get(fid) {
+                            None if on_disk > header => violations.push(format!(
+                                "{fid}: listed on disk at {on_disk} B but not in index liveness"
+                            )),
+                            None => {}
+                            Some(info) if content < info.total_bytes => {
+                                violations.push(format!(
+                                    "{fid}: on-disk {on_disk} B (content {content}) < index total_bytes {}",
+                                    info.total_bytes
+                                ));
+                            }
+                            Some(info) if content > info.total_bytes => {
+                                violations.push(format!(
+                                    "{fid}: on-disk {on_disk} B (content {content}) > index total_bytes {}, {} B unaccounted",
+                                    info.total_bytes,
+                                    content - info.total_bytes
+                                ));
+                            }
+                            Some(_) => {}
+                        }
+                    }
+                }
+            });
+
+            let listed_set: std::collections::HashSet<_> = listed.into_iter().collect();
+            liveness.keys().for_each(|fid| {
+                if !listed_set.contains(fid) {
+                    violations.push(format!("{fid}: in index liveness but missing on disk"));
+                }
+            });
+
+            if violations.is_empty() {
+                Ok(())
+            } else {
+                Err(violations.join("; "))
+            }
+        })
+        .await
+        .map_err(|e| InvariantViolation {
+            invariant: "ManifestEqualsReality",
+            detail: format!("join: {e}"),
+        })?
+        .map_err(|e| InvariantViolation {
+            invariant: "ManifestEqualsReality",
+            detail: e,
+        })
+    }
+}
+
+pub struct ChecksumCoverage;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for ChecksumCoverage {
+    fn name(&self) -> &'static str {
+        "ChecksumCoverage"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let violations: Vec<String> = ctx
+            .oracle
+            .live_cids_labeled()
+            .into_iter()
+            .filter_map(|(label, expected)| match ctx.store.get_block_sync(&expected) {
+                Ok(Some(bytes)) => {
+                    let actual = hash_to_cid_bytes(&bytes);
+                    (actual != expected).then(|| {
+                        format!(
+                            "{label}: silent corruption, bytes hash to {} but store returned them under {}",
+                            hex_short(&actual),
+                            hex_short(&expected),
+                        )
+                    })
+                }
+                Ok(None) => Some(format!(
+                    "{label}: live CID {} missing from store",
+                    hex_short(&expected)
+                )),
+                Err(e) => Some(format!(
+                    "{label}: read error for live CID {}: {e}",
+                    hex_short(&expected)
+                )),
+            })
+            .collect();
+
+        if violations.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "ChecksumCoverage",
+                detail: violations.join("; "),
+            })
+        }
+    }
+}
+
+pub struct MonotonicSeq;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for MonotonicSeq {
+    fn name(&self) -> &'static str {
+        "MonotonicSeq"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let Some(el) = ctx.eventlog else {
+            return Ok(());
+        };
+        let mut violations: Vec<String> = Vec::new();
+        el.events
+            .iter()
+            .zip(el.events.iter().skip(1))
+            .for_each(|(prev, next)| match next.seq.raw() {
+                n if n == prev.seq.raw() + 1 => {}
+                n if n == prev.seq.raw() => violations.push(format!("duplicate seq {n}")),
+                n => violations.push(format!(
+                    "gap: seq {} followed by {n}, expected {}",
+                    prev.seq.raw(),
+                    prev.seq.raw() + 1
+                )),
+            });
+        if ctx.oracle.last_retention_cutoff_us().is_none()
+            && let Some(first) = el.events.first()
+            && first.seq.raw() != 1
+        {
+            violations.push(format!(
+                "first persisted seq is {}, expected 1",
+                first.seq.raw()
+            ));
+        }
+        let acked_max = ctx
+            .oracle
+            .synced_events()
+            .iter()
+            .map(|e| e.seq.raw())
+            .max()
+            .unwrap_or(0);
+        let disk_max = el.events.last().map(|e| e.seq.raw()).unwrap_or(0);
+        if disk_max < acked_max {
+            violations.push(format!(
+                "acked seq {acked_max} missing on disk, disk max {disk_max}"
+            ));
+        }
+        if violations.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "MonotonicSeq",
+                detail: violations.join("; "),
+            })
+        }
+    }
+}
+
+pub struct FsyncOrdering;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for FsyncOrdering {
+    fn name(&self) -> &'static str {
+        "FsyncOrdering"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let Some(el) = ctx.eventlog else {
+            return Ok(());
+        };
+        let mut violations: Vec<String> = Vec::new();
+
+        let acked_seqs: HashSet<u64> = ctx
+            .oracle
+            .synced_events()
+            .iter()
+            .map(|e| e.seq.raw())
+            .collect();
+        let disk_seqs: HashSet<u64> = el.events.iter().map(|e| e.seq.raw()).collect();
+        let missing: Vec<u64> = acked_seqs.difference(&disk_seqs).copied().collect();
+        if !missing.is_empty() {
+            let mut sorted = missing;
+            sorted.sort_unstable();
+            violations.push(format!(
+                "{} acked events lost on disk, lowest missing seq {}",
+                sorted.len(),
+                sorted[0]
+            ));
+        }
+
+        if let Some(last_synced) = ctx.oracle.last_synced_seq()
+            && el.synced_seq.raw() != 0
+            && el.synced_seq.raw() < last_synced.raw()
+        {
+            violations.push(format!(
+                "writer synced_seq {} below oracle last_synced_seq {}",
+                el.synced_seq.raw(),
+                last_synced.raw()
+            ));
+        }
+
+        if violations.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "FsyncOrdering",
+                detail: violations.join("; "),
+            })
+        }
+    }
+}
+
+pub struct TombstoneBound;
+
+#[async_trait]
+impl<S: StorageIO + Send + Sync + 'static> Invariant<S> for TombstoneBound {
+    fn name(&self) -> &'static str {
+        "TombstoneBound"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_, S>) -> Result<(), InvariantViolation> {
+        let Some(el) = ctx.eventlog else {
+            return Ok(());
+        };
+        let Some(cutoff_us) = ctx.oracle.last_retention_cutoff_us() else {
+            return Ok(());
+        };
+
+        let active = el.segments.last().copied();
+
+        let stale: Vec<String> = el
+            .segment_last_ts
+            .iter()
+            .filter(|(id, last_ts)| Some(*id) != active && *last_ts < cutoff_us)
+            .map(|(id, last_ts)| format!("segment {id} last_ts {last_ts} < cutoff {cutoff_us}"))
+            .collect();
+
+        if stale.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "TombstoneBound",
+                detail: stale.join("; "),
+            })
+        }
+    }
+}
+
+pub fn invariants_for<S: StorageIO + Send + Sync + 'static>(
+    set: InvariantSet,
+) -> Vec<Box<dyn Invariant<S>>> {
+    let unknown = set.unknown_bits();
+    assert!(
+        unknown == 0,
+        "invariants_for: unknown InvariantSet bits 0x{unknown:x}; all bits must map to an impl"
+    );
+    let candidates: Vec<(InvariantSet, Box<dyn Invariant<S>>)> = vec![
+        (
+            InvariantSet::REFCOUNT_CONSERVATION,
+            Box::new(RefcountConservation),
+        ),
+        (InvariantSet::REACHABILITY, Box::new(Reachability)),
+        (
+            InvariantSet::ACKED_WRITE_PERSISTENCE,
+            Box::new(AckedWritePersistence),
+        ),
+        (InvariantSet::READ_AFTER_WRITE, Box::new(ReadAfterWrite)),
+        (
+            InvariantSet::COMPACTION_IDEMPOTENT,
+            Box::new(CompactionIdempotent),
+        ),
+        (InvariantSet::NO_ORPHAN_FILES, Box::new(NoOrphanFiles)),
+        (InvariantSet::BYTE_BUDGET, Box::new(ByteBudget::default())),
+        (
+            InvariantSet::MANIFEST_EQUALS_REALITY,
+            Box::new(ManifestEqualsReality),
+        ),
+        (InvariantSet::CHECKSUM_COVERAGE, Box::new(ChecksumCoverage)),
+        (InvariantSet::MONOTONIC_SEQ, Box::new(MonotonicSeq)),
+        (InvariantSet::FSYNC_ORDERING, Box::new(FsyncOrdering)),
+        (InvariantSet::TOMBSTONE_BOUND, Box::new(TombstoneBound)),
+    ];
+    candidates
+        .into_iter()
+        .filter_map(|(flag, inv)| set.contains(flag).then_some(inv))
+        .collect()
+}

crates/tranquil-store/src/gauntlet/leak.rs

@@ -0,0 +1,234 @@
+use std::num::NonZeroU64;
+use std::time::Duration;
+
+use serde::{Deserialize, Serialize};
+
+use super::metrics::{MetricName, MetricsSample};
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub struct LeakGateConfig {
+    pub warmup_ms: u64,
+    pub window_ms: NonZeroU64,
+    pub growth_limit_pct: f64,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct LeakGateBuildError(pub &'static str);
+
+impl std::fmt::Display for LeakGateBuildError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(self.0)
+    }
+}
+
+impl std::error::Error for LeakGateBuildError {}
+
+impl LeakGateConfig {
+    pub fn try_new(
+        warmup_ms: u64,
+        window_ms: u64,
+        growth_limit_pct: f64,
+    ) -> Result<Self, LeakGateBuildError> {
+        let window_ms = NonZeroU64::new(window_ms)
+            .ok_or(LeakGateBuildError("leak gate window_ms must be > 0"))?;
+        if !growth_limit_pct.is_finite() || growth_limit_pct < 0.0 {
+            return Err(LeakGateBuildError(
+                "leak gate growth_limit_pct must be finite and non-negative",
+            ));
+        }
+        Ok(Self {
+            warmup_ms,
+            window_ms,
+            growth_limit_pct,
+        })
+    }
+
+    pub fn standard() -> Self {
+        Self::try_new(60 * 60 * 1_000, 4 * 60 * 60 * 1_000, 5.0)
+            .expect("standard leak gate config is valid")
+    }
+
+    pub fn short_for_tests() -> Self {
+        Self::try_new(60_000, 4 * 60_000, 5.0).expect("short_for_tests leak gate config is valid")
+    }
+
+    pub fn warmup(&self) -> Duration {
+        Duration::from_millis(self.warmup_ms)
+    }
+
+    pub fn window(&self) -> Duration {
+        Duration::from_millis(self.window_ms.get())
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct LeakViolation {
+    pub metric: String,
+    pub start_ms: u64,
+    pub end_ms: u64,
+    pub start_value: u64,
+    pub end_value: u64,
+    pub growth_pct: f64,
+    pub limit_pct: f64,
+}
+
+pub fn evaluate(samples: &[MetricsSample], cfg: LeakGateConfig) -> Vec<LeakViolation> {
+    if samples.len() < 2 {
+        return Vec::new();
+    }
+    MetricName::ALL
+        .iter()
+        .flat_map(|&m| evaluate_metric(samples, m, cfg))
+        .collect()
+}
+
+fn evaluate_metric(
+    samples: &[MetricsSample],
+    metric: MetricName,
+    cfg: LeakGateConfig,
+) -> Option<LeakViolation> {
+    let post_warmup: Vec<&MetricsSample> = samples
+        .iter()
+        .filter(|s| s.elapsed_ms >= cfg.warmup_ms)
+        .collect();
+    if post_warmup.len() < 2 {
+        return None;
+    }
+
+    let min_delta = metric.min_absolute_delta();
+    let window = cfg.window_ms.get();
+    let mut worst: Option<LeakViolation> = None;
+    for (i, start) in post_warmup.iter().enumerate() {
+        let Some(start_v) = start.metric(metric) else {
+            continue;
+        };
+        if start_v == 0 {
+            continue;
+        }
+        let deadline = start.elapsed_ms.saturating_add(window);
+        for end in post_warmup.iter().skip(i + 1) {
+            if end.elapsed_ms > deadline {
+                break;
+            }
+            let Some(end_v) = end.metric(metric) else {
+                continue;
+            };
+            if end_v <= start_v {
+                continue;
+            }
+            let delta = end_v - start_v;
+            if delta < min_delta {
+                continue;
+            }
+            let growth = (delta as f64 / start_v as f64) * 100.0;
+            if growth <= cfg.growth_limit_pct {
+                continue;
+            }
+            let candidate = LeakViolation {
+                metric: metric.as_str().to_string(),
+                start_ms: start.elapsed_ms,
+                end_ms: end.elapsed_ms,
+                start_value: start_v,
+                end_value: end_v,
+                growth_pct: growth,
+                limit_pct: cfg.growth_limit_pct,
+            };
+            match &worst {
+                Some(w) if w.growth_pct >= candidate.growth_pct => {}
+                _ => worst = Some(candidate),
+            }
+        }
+    }
+    worst
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const GIB: u64 = 1024 * 1024 * 1024;
+
+    fn sample(elapsed_ms: u64, rss: u64) -> MetricsSample {
+        MetricsSample {
+            elapsed_ms,
+            rss_bytes: Some(rss),
+            fd_count: Some(10),
+            data_dir_bytes: 0,
+            index_dir_bytes: 0,
+            segments_dir_bytes: 0,
+            data_file_count: Some(0),
+            segment_count: Some(0),
+            block_index_entries: 0,
+            hint_file_bytes: 0,
+        }
+    }
+
+    #[test]
+    fn flat_metrics_no_violation() {
+        let cfg = LeakGateConfig::short_for_tests();
+        let series: Vec<MetricsSample> =
+            (0..20).map(|i| sample(60_000 + i * 60_000, GIB)).collect();
+        assert!(evaluate(&series, cfg).is_empty());
+    }
+
+    #[test]
+    fn growing_rss_flagged() {
+        let cfg = LeakGateConfig::short_for_tests();
+        let series: Vec<MetricsSample> = (0..20)
+            .map(|i| sample(60_000 + i * 60_000, GIB + i * 64 * 1024 * 1024))
+            .collect();
+        let v = evaluate(&series, cfg);
+        assert!(!v.is_empty());
+        assert_eq!(v[0].metric, "rss_bytes");
+        assert!(v[0].growth_pct > 5.0);
+    }
+
+    #[test]
+    fn warmup_samples_ignored() {
+        let cfg = LeakGateConfig::short_for_tests();
+        let mut series: Vec<MetricsSample> = Vec::new();
+        series.push(sample(10_000, 1));
+        series.push(sample(30_000, GIB));
+        (0..10).for_each(|i| {
+            series.push(sample(60_000 + i * 60_000, GIB));
+        });
+        assert!(evaluate(&series, cfg).is_empty());
+    }
+
+    #[test]
+    fn window_bound_honored() {
+        let cfg = LeakGateConfig::try_new(0, 2 * 60_000, 5.0).unwrap();
+        let series = vec![sample(0, GIB), sample(200_000, 2 * GIB)];
+        assert!(
+            evaluate(&series, cfg).is_empty(),
+            "200s gap exceeds 120s window, growth must not be flagged"
+        );
+    }
+
+    #[test]
+    fn small_absolute_delta_not_flagged() {
+        let cfg = LeakGateConfig::short_for_tests();
+        let series: Vec<MetricsSample> = (0..10)
+            .map(|i| sample(60_000 + i * 60_000, GIB + i * 1024))
+            .collect();
+        assert!(
+            evaluate(&series, cfg).is_empty(),
+            "kilobyte growth is below the RSS absolute-delta floor"
+        );
+    }
+
+    #[test]
+    fn missing_metric_samples_skipped() {
+        let cfg = LeakGateConfig::short_for_tests();
+        let mut series: Vec<MetricsSample> =
+            (0..10).map(|i| sample(60_000 + i * 60_000, GIB)).collect();
+        series[3].rss_bytes = None;
+        series[7].rss_bytes = None;
+        assert!(evaluate(&series, cfg).is_empty());
+    }
+
+    #[test]
+    fn zero_window_rejected_at_construction() {
+        assert!(LeakGateConfig::try_new(0, 0, 5.0).is_err());
+    }
+}

crates/tranquil-store/src/gauntlet/metrics.rs

@@ -0,0 +1,247 @@
+use std::path::Path;
+use std::sync::Arc;
+use std::time::Duration;
+
+use serde::{Deserialize, Serialize};
+use tracing::warn;
+
+use super::runner::{EventLogState, Harness};
+use crate::blockstore::TranquilBlockStore;
+use crate::io::StorageIO;
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub struct MetricsSample {
+    pub elapsed_ms: u64,
+    pub rss_bytes: Option<u64>,
+    pub fd_count: Option<u64>,
+    pub data_dir_bytes: u64,
+    pub index_dir_bytes: u64,
+    pub segments_dir_bytes: u64,
+    pub data_file_count: Option<u64>,
+    pub segment_count: Option<u64>,
+    pub block_index_entries: u64,
+    pub hint_file_bytes: u64,
+}
+
+impl MetricsSample {
+    pub fn metric(&self, name: MetricName) -> Option<u64> {
+        match name {
+            MetricName::RssBytes => self.rss_bytes,
+            MetricName::FdCount => self.fd_count,
+            MetricName::DataDirBytes => Some(self.data_dir_bytes),
+            MetricName::IndexDirBytes => Some(self.index_dir_bytes),
+            MetricName::SegmentsDirBytes => Some(self.segments_dir_bytes),
+            MetricName::DataFileCount => self.data_file_count,
+            MetricName::SegmentCount => self.segment_count,
+            MetricName::BlockIndexEntries => Some(self.block_index_entries),
+            MetricName::HintFileBytes => Some(self.hint_file_bytes),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum MetricName {
+    RssBytes,
+    FdCount,
+    DataDirBytes,
+    IndexDirBytes,
+    SegmentsDirBytes,
+    DataFileCount,
+    SegmentCount,
+    BlockIndexEntries,
+    HintFileBytes,
+}
+
+impl MetricName {
+    pub const ALL: &'static [MetricName] = &[
+        Self::RssBytes,
+        Self::FdCount,
+        Self::DataDirBytes,
+        Self::IndexDirBytes,
+        Self::SegmentsDirBytes,
+        Self::DataFileCount,
+        Self::SegmentCount,
+        Self::BlockIndexEntries,
+        Self::HintFileBytes,
+    ];
+
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            Self::RssBytes => "rss_bytes",
+            Self::FdCount => "fd_count",
+            Self::DataDirBytes => "data_dir_bytes",
+            Self::IndexDirBytes => "index_dir_bytes",
+            Self::SegmentsDirBytes => "segments_dir_bytes",
+            Self::DataFileCount => "data_file_count",
+            Self::SegmentCount => "segment_count",
+            Self::BlockIndexEntries => "block_index_entries",
+            Self::HintFileBytes => "hint_file_bytes",
+        }
+    }
+
+    pub const fn min_absolute_delta(self) -> u64 {
+        match self {
+            Self::RssBytes => 16 * 1024 * 1024,
+            Self::FdCount => 16,
+            Self::DataDirBytes => 16 * 1024 * 1024,
+            Self::IndexDirBytes => 1024 * 1024,
+            Self::SegmentsDirBytes => 16 * 1024 * 1024,
+            Self::DataFileCount => 16,
+            Self::SegmentCount => 4,
+            Self::BlockIndexEntries => 1024,
+            Self::HintFileBytes => 1024 * 1024,
+        }
+    }
+}
+
+pub fn sample_harness<S: StorageIO + Send + Sync + 'static>(
+    harness: &Harness<S>,
+    elapsed: Duration,
+) -> MetricsSample {
+    MetricsSample {
+        elapsed_ms: u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX),
+        rss_bytes: read_rss(),
+        fd_count: count_open_fds(),
+        data_dir_bytes: dir_bytes(harness.store.data_dir()),
+        index_dir_bytes: dir_bytes(harness.store.block_index().index_dir()),
+        segments_dir_bytes: harness
+            .eventlog
+            .as_ref()
+            .map(|el| dir_bytes(&el.segments_dir))
+            .unwrap_or(0),
+        data_file_count: data_file_count(&harness.store),
+        segment_count: harness.eventlog.as_ref().and_then(segment_count),
+        block_index_entries: harness.store.block_index().approximate_block_count(),
+        hint_file_bytes: hint_bytes(harness.store.data_dir()),
+    }
+}
+
+fn data_file_count<S: StorageIO + Send + Sync + 'static>(
+    store: &Arc<TranquilBlockStore<S>>,
+) -> Option<u64> {
+    match store.list_data_files() {
+        Ok(v) => Some(v.len() as u64),
+        Err(e) => {
+            warn!(error = %e, "gauntlet metrics: list_data_files failed");
+            None
+        }
+    }
+}
+
+fn segment_count<S: StorageIO + Send + Sync + 'static>(el: &EventLogState<S>) -> Option<u64> {
+    match el.manager.list_segments() {
+        Ok(v) => Some(v.len() as u64),
+        Err(e) => {
+            warn!(error = %e, "gauntlet metrics: list_segments failed");
+            None
+        }
+    }
+}
+
+fn dir_bytes(path: &Path) -> u64 {
+    let Ok(entries) = std::fs::read_dir(path) else {
+        return 0;
+    };
+    entries
+        .filter_map(Result::ok)
+        .map(|entry| match entry.file_type() {
+            Ok(ft) if ft.is_dir() => dir_bytes(&entry.path()),
+            Ok(_) => entry.metadata().map(|m| m.len()).unwrap_or(0),
+            Err(_) => 0,
+        })
+        .sum()
+}
+
+fn hint_bytes(data_dir: &Path) -> u64 {
+    let Ok(entries) = std::fs::read_dir(data_dir) else {
+        return 0;
+    };
+    entries
+        .filter_map(Result::ok)
+        .filter(|entry| {
+            entry
+                .path()
+                .extension()
+                .and_then(|e| e.to_str())
+                .map(|e| e == "tqh")
+                .unwrap_or(false)
+        })
+        .map(|entry| entry.metadata().map(|m| m.len()).unwrap_or(0))
+        .sum()
+}
+
+#[cfg(target_os = "linux")]
+fn read_rss() -> Option<u64> {
+    let status = match std::fs::read_to_string("/proc/self/status") {
+        Ok(s) => s,
+        Err(e) => {
+            warn!(error = %e, "gauntlet metrics: read /proc/self/status failed");
+            return None;
+        }
+    };
+    let parsed = status.lines().find_map(|line| {
+        let rest = line.strip_prefix("VmRSS:")?;
+        let kb: u64 = rest.split_whitespace().next()?.parse().ok()?;
+        Some(kb * 1024)
+    });
+    if parsed.is_none() {
+        warn!("gauntlet metrics: VmRSS line missing from /proc/self/status");
+    }
+    parsed
+}
+
+#[cfg(not(target_os = "linux"))]
+fn read_rss() -> Option<u64> {
+    None
+}
+
+#[cfg(target_os = "linux")]
+fn count_open_fds() -> Option<u64> {
+    match std::fs::read_dir("/proc/self/fd") {
+        Ok(entries) => Some(entries.filter_map(Result::ok).count() as u64),
+        Err(e) => {
+            warn!(error = %e, "gauntlet metrics: read /proc/self/fd failed");
+            None
+        }
+    }
+}
+
+#[cfg(not(target_os = "linux"))]
+fn count_open_fds() -> Option<u64> {
+    None
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn metric_names_roundtrip_strings() {
+        MetricName::ALL.iter().for_each(|m| {
+            let s = m.as_str();
+            assert!(!s.is_empty());
+        });
+    }
+
+    #[test]
+    #[cfg_attr(not(target_os = "linux"), ignore = "linux /proc only")]
+    fn rss_reads_nonzero() {
+        let rss = read_rss().expect("rss");
+        assert!(rss > 0, "rss should be positive, got {rss}");
+    }
+
+    #[test]
+    #[cfg_attr(not(target_os = "linux"), ignore = "linux /proc only")]
+    fn fd_count_reads_nonzero() {
+        let fd = count_open_fds().expect("fd");
+        assert!(fd > 0);
+    }
+
+    #[test]
+    fn dir_bytes_sums_entries() {
+        let dir = tempfile::TempDir::new().unwrap();
+        std::fs::write(dir.path().join("a"), b"1234").unwrap();
+        std::fs::write(dir.path().join("b"), b"5678").unwrap();
+        assert_eq!(dir_bytes(dir.path()), 8);
+    }
+}

crates/tranquil-store/src/gauntlet/mod.rs

@@ -0,0 +1,45 @@
+pub mod farm;
+pub mod flaky;
+pub mod invariants;
+pub mod leak;
+pub mod metrics;
+pub mod op;
+pub mod oracle;
+pub mod overrides;
+pub mod regression;
+pub mod runner;
+pub mod scenarios;
+pub mod shrink;
+pub mod soak;
+pub mod workload;
+
+pub use flaky::{
+    BackingMegabytes, DownIntervalSecs, FlakyConfig, FlakyError, FlakyMount, UpIntervalSecs,
+};
+pub use invariants::{
+    EventLogSnapshot, Invariant, InvariantSet, InvariantViolation, SnapshotEvent, invariants_for,
+};
+pub use leak::{LeakGateBuildError, LeakGateConfig, LeakViolation, evaluate as evaluate_leak_gate};
+pub use metrics::{MetricName, MetricsSample, sample_harness};
+pub use op::{
+    CollectionName, DidSeed, EventKind, Op, OpStream, PayloadSeed, RecordKey, RetentionSecs, Seed,
+    ValueSeed,
+};
+pub use oracle::{EventExpectation, Oracle};
+pub use overrides::{ConfigOverrides, GroupCommitOverrides, StoreOverrides};
+pub use regression::{RegressionRecord, RegressionViolation, default_root as regression_root};
+pub use runner::{
+    EventLogConfig, Gauntlet, GauntletBuildError, GauntletConfig, GauntletReport, Harness,
+    IoBackend, MaxFileSize, MaxSegmentSize, OpErrorCount, OpIndex, OpInterval, OpsExecuted,
+    RestartCount, RestartPolicy, RunLimits, ShardCount, StoreConfig, WallMs, WriterConcurrency,
+};
+pub use scenarios::{Scenario, UnknownScenario, config_for};
+pub use shrink::{ShrinkOutcome, shrink_failure};
+pub use soak::{
+    DEFAULT_CHUNK_OPS, DEFAULT_SAMPLE_INTERVAL_MS, InvariantViolationRecord, SoakConfig, SoakError,
+    SoakEvent, SoakReport, run_soak,
+};
+pub use workload::{
+    ByteRange, DidSpaceSize, KeySpaceSize, OpCount, OpWeights, RetentionMaxSecs, SizeDistribution,
+    ValueBytes, WorkloadModel,
+};

crates/tranquil-store/src/gauntlet/op.rs

@@ -0,0 +1,198 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct Seed(pub u64);
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct CollectionName(pub String);
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct RecordKey(pub String);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct ValueSeed(pub u32);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct DidSeed(pub u32);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct PayloadSeed(pub u32);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct RetentionSecs(pub u32);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub enum EventKind {
+    Commit,
+    Identity,
+    Account,
+    Sync,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum Op {
+    AddRecord {
+        collection: CollectionName,
+        rkey: RecordKey,
+        value_seed: ValueSeed,
+    },
+    DeleteRecord {
+        collection: CollectionName,
+        rkey: RecordKey,
+    },
+    Compact,
+    Checkpoint,
+    AppendEvent {
+        did_seed: DidSeed,
+        event_kind: EventKind,
+        payload_seed: PayloadSeed,
+    },
+    SyncEventLog,
+    RunRetention {
+        max_age_secs: RetentionSecs,
+    },
+    ReadRecord {
+        collection: CollectionName,
+        rkey: RecordKey,
+    },
+    ReadBlock {
+        value_seed: ValueSeed,
+    },
+}
+
+impl Op {
+    pub const fn is_read_only(&self) -> bool {
+        matches!(self, Op::ReadRecord { .. } | Op::ReadBlock { .. })
+    }
+}
+
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct OpStream {
+    ops: Vec<Op>,
+}
+
+impl OpStream {
+    pub fn from_vec(ops: Vec<Op>) -> Self {
+        Self { ops }
+    }
+
+    pub fn empty() -> Self {
+        Self { ops: Vec::new() }
+    }
+
+    pub fn as_slice(&self) -> &[Op] {
+        &self.ops
+    }
+
+    pub fn into_vec(self) -> Vec<Op> {
+        self.ops
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = &Op> {
+        self.ops.iter()
+    }
+
+    pub fn len(&self) -> usize {
+        self.ops.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.ops.is_empty()
+    }
+
+    pub fn shrink_candidates(&self) -> impl Iterator<Item = OpStream> + '_ {
+        let len = self.ops.len();
+        let chunk_sizes: Vec<usize> = std::iter::successors((len >= 2).then_some(len / 2), |&s| {
+            (s >= 2).then_some(s / 2)
+        })
+        .collect();
+
+        let chunk_candidates = chunk_sizes.into_iter().flat_map(move |chunk_size| {
+            let count = len.div_ceil(chunk_size);
+            (0..count).map(move |i| {
+                let start = i * chunk_size;
+                let end = (start + chunk_size).min(len);
+                let mut reduced = Vec::with_capacity(len - (end - start));
+                reduced.extend_from_slice(&self.ops[..start]);
+                reduced.extend_from_slice(&self.ops[end..]);
+                OpStream::from_vec(reduced)
+            })
+        });
+
+        let single_candidates = (0..len).map(move |i| {
+            let mut reduced = self.ops.clone();
+            reduced.remove(i);
+            OpStream::from_vec(reduced)
+        });
+
+        chunk_candidates.chain(single_candidates)
+    }
+
+    pub fn shrink_to_fixpoint(mut self, mut fails: impl FnMut(&OpStream) -> bool) -> OpStream {
+        loop {
+            let next = self.shrink_candidates().find(|c| !c.is_empty() && fails(c));
+            match next {
+                Some(smaller) => self = smaller,
+                None => return self,
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn stream(n: usize) -> OpStream {
+        OpStream::from_vec(
+            (0..n)
+                .map(|i| Op::AddRecord {
+                    collection: CollectionName("c".into()),
+                    rkey: RecordKey(format!("{i:04}")),
+                    value_seed: ValueSeed(i as u32),
+                })
+                .collect(),
+        )
+    }
+
+    fn contains_index(s: &OpStream, target: u32) -> bool {
+        s.iter()
+            .any(|op| matches!(op, Op::AddRecord { value_seed, .. } if value_seed.0 == target))
+    }
+
+    #[test]
+    fn shrink_candidates_nonempty_for_len_ge_2() {
+        let s = stream(8);
+        let count = s.shrink_candidates().count();
+        assert!(count > 0);
+    }
+
+    #[test]
+    fn shrink_candidates_empty_for_len_0() {
+        let s = OpStream::from_vec(Vec::new());
+        assert_eq!(s.shrink_candidates().count(), 0);
+    }
+
+    #[test]
+    fn shrink_candidates_includes_every_single_removal() {
+        let s = stream(5);
+        let singles: Vec<_> = s.shrink_candidates().filter(|c| c.len() == 4).collect();
+        assert!(
+            singles.len() >= 5,
+            "expected at least 5 size-4 candidates, got {}",
+            singles.len()
+        );
+    }
+
+    #[test]
+    fn shrink_to_fixpoint_converges_to_culprit() {
+        let s = stream(64);
+        let shrunk = s.shrink_to_fixpoint(|c| contains_index(c, 17));
+        assert!(contains_index(&shrunk, 17));
+        assert!(
+            shrunk.len() < 4,
+            "expected shrink to close on culprit, got {} ops",
+            shrunk.len()
+        );
+    }
+}

crates/tranquil-store/src/gauntlet/oracle.rs

@@ -0,0 +1,143 @@
+use std::collections::HashMap;
+
+use cid::Cid;
+
+use super::op::{CollectionName, EventKind, RecordKey};
+use crate::blockstore::CidBytes;
+use crate::eventlog::EventSequence;
+
+#[derive(Debug, thiserror::Error, PartialEq, Eq)]
+#[error("unexpected CID encoding: got {actual} bytes, expected 36 for sha256 CIDv1")]
+pub struct CidFormatError {
+    pub actual: usize,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct EventExpectation {
+    pub seq: EventSequence,
+    pub timestamp_us: u64,
+    pub kind: EventKind,
+    pub did_hash: u32,
+}
+
+#[derive(Debug, Default)]
+pub struct Oracle {
+    live: HashMap<(CollectionName, RecordKey), CidBytes>,
+    current_root: Option<Cid>,
+    mst_node_cids: Vec<CidBytes>,
+    synced_events: Vec<EventExpectation>,
+    unsynced_events: Vec<EventExpectation>,
+    last_synced_seq: Option<EventSequence>,
+    last_retention_cutoff_us: Option<u64>,
+}
+
+impl Oracle {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    pub fn add(
+        &mut self,
+        coll: CollectionName,
+        rkey: RecordKey,
+        record_cid: CidBytes,
+    ) -> Option<CidBytes> {
+        self.live.insert((coll, rkey), record_cid)
+    }
+
+    pub fn delete(&mut self, coll: &CollectionName, rkey: &RecordKey) -> Option<CidBytes> {
+        self.live.remove(&(coll.clone(), rkey.clone()))
+    }
+
+    pub fn contains_record(&self, coll: &CollectionName, rkey: &RecordKey) -> bool {
+        self.live.contains_key(&(coll.clone(), rkey.clone()))
+    }
+
+    pub fn set_root(&mut self, root: Cid) {
+        self.current_root = Some(root);
+    }
+
+    pub fn root(&self) -> Option<Cid> {
+        self.current_root
+    }
+
+    pub fn set_mst_node_cids(&mut self, cids: Vec<CidBytes>) {
+        self.mst_node_cids = cids;
+    }
+
+    pub fn clear_mst_state(&mut self) {
+        self.current_root = None;
+        self.mst_node_cids.clear();
+    }
+
+    pub fn live_records(&self) -> impl Iterator<Item = (&CollectionName, &RecordKey, &CidBytes)> {
+        self.live.iter().map(|((c, r), v)| (c, r, v))
+    }
+
+    pub fn live_count(&self) -> usize {
+        self.live.len()
+    }
+
+    pub fn live_cids_labeled(&self) -> Vec<(String, CidBytes)> {
+        let nodes = self
+            .mst_node_cids
+            .iter()
+            .map(|bytes| (format!("mst {}", hex_short(bytes)), *bytes));
+        let records = self
+            .live_records()
+            .map(|(c, r, v)| (format!("record {}/{}", c.0, r.0), *v));
+        nodes.chain(records).collect()
+    }
+
+    pub fn record_event_append(&mut self, event: EventExpectation) {
+        self.unsynced_events.push(event);
+    }
+
+    pub fn record_event_sync(&mut self, synced_through: EventSequence) {
+        let (promoted, remaining): (Vec<_>, Vec<_>) = self
+            .unsynced_events
+            .drain(..)
+            .partition(|e| e.seq <= synced_through);
+        self.synced_events.extend(promoted);
+        self.unsynced_events = remaining;
+        self.last_synced_seq = Some(synced_through);
+    }
+
+    pub fn record_crash(&mut self) {
+        self.unsynced_events.clear();
+    }
+
+    pub fn record_retention(&mut self, cutoff_us: u64) {
+        self.synced_events.retain(|e| e.timestamp_us >= cutoff_us);
+        self.last_retention_cutoff_us = Some(cutoff_us);
+    }
+
+    pub fn synced_events(&self) -> &[EventExpectation] {
+        &self.synced_events
+    }
+
+    pub fn unsynced_events(&self) -> &[EventExpectation] {
+        &self.unsynced_events
+    }
+
+    pub fn last_synced_seq(&self) -> Option<EventSequence> {
+        self.last_synced_seq
+    }
+
+    pub fn last_retention_cutoff_us(&self) -> Option<u64> {
+        self.last_retention_cutoff_us
+    }
+}
+
+pub(super) fn try_cid_to_fixed(cid: &Cid) -> Result<CidBytes, CidFormatError> {
+    let bytes = cid.to_bytes();
+    let actual = bytes.len();
+    bytes.try_into().map_err(|_| CidFormatError { actual })
+}
+
+pub(super) fn hex_short(cid: &CidBytes) -> String {
+    cid[cid.len() - 6..]
+        .iter()
+        .map(|b| format!("{b:02x}"))
+        .collect()
+}

crates/tranquil-store/src/gauntlet/overrides.rs

@@ -0,0 +1,247 @@
+use serde::{Deserialize, Serialize};
+
+use super::runner::{
+    GauntletConfig, IoBackend, MaxFileSize, OpInterval, RestartPolicy, RunLimits, ShardCount,
+    WallMs, WriterConcurrency,
+};
+use super::workload::{KeySpaceSize, OpCount, SizeDistribution, ValueBytes};
+use crate::sim::FaultConfig;
+
+#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
+#[serde(deny_unknown_fields)]
+pub struct ConfigOverrides {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub op_count: Option<usize>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub max_wall_ms: Option<u64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub writer_concurrency: Option<usize>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub key_space: Option<u32>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub value_bytes: Option<u32>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub fault_density_scale: Option<f64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub fault_density_uniform: Option<f64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub restart_every_n_ops: Option<usize>,
+    #[serde(default, skip_serializing_if = "StoreOverrides::is_empty")]
+    pub store: StoreOverrides,
+}
+
+#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
+pub struct StoreOverrides {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub max_file_size: Option<u64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub shard_count: Option<u8>,
+    #[serde(default, skip_serializing_if = "GroupCommitOverrides::is_empty")]
+    pub group_commit: GroupCommitOverrides,
+}
+
+impl StoreOverrides {
+    pub fn is_empty(&self) -> bool {
+        self.max_file_size.is_none() && self.shard_count.is_none() && self.group_commit.is_empty()
+    }
+}
+
+#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
+pub struct GroupCommitOverrides {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub max_batch_size: Option<usize>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub channel_capacity: Option<usize>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub checkpoint_interval_ms: Option<u64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub checkpoint_write_threshold: Option<u64>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub verify_persisted_blocks: Option<bool>,
+}
+
+impl GroupCommitOverrides {
+    pub fn is_empty(&self) -> bool {
+        self.max_batch_size.is_none()
+            && self.channel_capacity.is_none()
+            && self.checkpoint_interval_ms.is_none()
+            && self.checkpoint_write_threshold.is_none()
+            && self.verify_persisted_blocks.is_none()
+    }
+}
+
+impl ConfigOverrides {
+    pub fn apply_to(&self, cfg: &mut GauntletConfig) {
+        if let Some(n) = self.op_count {
+            cfg.op_count = OpCount(n);
+        }
+        if let Some(ms) = self.max_wall_ms {
+            cfg.limits = RunLimits {
+                max_wall_ms: Some(WallMs(ms)),
+            };
+        }
+        if let Some(n) = self.writer_concurrency {
+            cfg.writer_concurrency = WriterConcurrency(n.max(1));
+        }
+        if let Some(n) = self.key_space {
+            cfg.workload.key_space = KeySpaceSize(n.max(1));
+        }
+        if let Some(n) = self.value_bytes {
+            cfg.workload.size_distribution = SizeDistribution::Fixed(ValueBytes(n));
+        }
+        if let Some(m) = self.fault_density_scale
+            && let IoBackend::Simulated { fault } = cfg.io
+        {
+            cfg.io = IoBackend::Simulated {
+                fault: fault.scale_probabilities(m),
+            };
+        }
+        if let Some(d) = self.fault_density_uniform {
+            cfg.io = IoBackend::Simulated {
+                fault: FaultConfig::uniform_density(d.clamp(0.0, 1.0)),
+            };
+        }
+        if let Some(n) = self.restart_every_n_ops {
+            cfg.restart_policy = if n == 0 {
+                RestartPolicy::Never
+            } else {
+                RestartPolicy::EveryNOps(OpInterval(n))
+            };
+        }
+        if let Some(n) = self.store.max_file_size {
+            cfg.store.max_file_size = MaxFileSize(n);
+        }
+        if let Some(n) = self.store.shard_count {
+            cfg.store.shard_count = ShardCount(n);
+        }
+        let gc = &self.store.group_commit;
+        if let Some(n) = gc.max_batch_size {
+            cfg.store.group_commit.max_batch_size = n;
+        }
+        if let Some(n) = gc.channel_capacity {
+            cfg.store.group_commit.channel_capacity = n;
+        }
+        if let Some(n) = gc.checkpoint_interval_ms {
+            cfg.store.group_commit.checkpoint_interval_ms = n;
+        }
+        if let Some(n) = gc.checkpoint_write_threshold {
+            cfg.store.group_commit.checkpoint_write_threshold = n;
+        }
+        if let Some(b) = gc.verify_persisted_blocks {
+            cfg.store.group_commit.verify_persisted_blocks = b;
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn default_overrides_serialize_empty() {
+        let o = ConfigOverrides::default();
+        let json = serde_json::to_string(&o).unwrap();
+        assert_eq!(json, "{}");
+    }
+
+    #[test]
+    fn round_trip_preserves_set_fields() {
+        let o = ConfigOverrides {
+            op_count: Some(42),
+            writer_concurrency: Some(16),
+            key_space: Some(1_000_000),
+            value_bytes: Some(4096),
+            fault_density_scale: Some(1e-3),
+            fault_density_uniform: Some(5e-4),
+            restart_every_n_ops: Some(10_000),
+            store: StoreOverrides {
+                max_file_size: Some(4096),
+                group_commit: GroupCommitOverrides {
+                    max_batch_size: Some(16),
+                    ..GroupCommitOverrides::default()
+                },
+                ..StoreOverrides::default()
+            },
+            ..ConfigOverrides::default()
+        };
+        let json = serde_json::to_string(&o).unwrap();
+        let back: ConfigOverrides = serde_json::from_str(&json).unwrap();
+        assert_eq!(o, back);
+    }
+
+    #[test]
+    fn fault_density_scale_scales_moderate() {
+        use crate::gauntlet::op::Seed;
+        use crate::gauntlet::scenarios::{Scenario, config_for};
+        let mut cfg = config_for(Scenario::ModerateFaults, Seed(1));
+        let o = ConfigOverrides {
+            fault_density_scale: Some(0.1),
+            ..ConfigOverrides::default()
+        };
+        o.apply_to(&mut cfg);
+        match cfg.io {
+            IoBackend::Simulated { fault } => {
+                assert!(fault.torn_page_probability.raw() < 0.02);
+                assert!(fault.torn_page_probability.raw() > 0.0);
+            }
+            _ => panic!("expected simulated io"),
+        }
+    }
+
+    #[test]
+    fn fault_density_scale_zero_kills_probabilities() {
+        use crate::gauntlet::op::Seed;
+        use crate::gauntlet::scenarios::{Scenario, config_for};
+        let mut cfg = config_for(Scenario::ModerateFaults, Seed(1));
+        let o = ConfigOverrides {
+            fault_density_scale: Some(0.0),
+            ..ConfigOverrides::default()
+        };
+        o.apply_to(&mut cfg);
+        match cfg.io {
+            IoBackend::Simulated { fault } => {
+                assert_eq!(fault.partial_write_probability.raw(), 0.0);
+                assert_eq!(fault.torn_page_probability.raw(), 0.0);
+                assert_eq!(fault.io_error_probability.raw(), 0.0);
+                assert_eq!(fault.sync_failure_probability.raw(), 0.0);
+            }
+            _ => panic!("expected simulated io"),
+        }
+    }
+
+    #[test]
+    fn fault_density_scale_is_noop_on_real_backend() {
+        use crate::gauntlet::op::Seed;
+        use crate::gauntlet::scenarios::{Scenario, config_for};
+        let mut cfg = config_for(Scenario::SmokePR, Seed(1));
+        assert!(matches!(cfg.io, IoBackend::Real));
+        let o = ConfigOverrides {
+            fault_density_scale: Some(0.5),
+            ..ConfigOverrides::default()
+        };
+        o.apply_to(&mut cfg);
+        assert!(matches!(cfg.io, IoBackend::Real));
+    }
+
+    #[test]
+    fn fault_density_uniform_forces_simulated_backend() {
+        use crate::gauntlet::op::Seed;
+        use crate::gauntlet::scenarios::{Scenario, config_for};
+        let mut cfg = config_for(Scenario::SmokePR, Seed(1));
+        assert!(matches!(cfg.io, IoBackend::Real));
+        let o = ConfigOverrides {
+            fault_density_uniform: Some(0.25),
+            ..ConfigOverrides::default()
+        };
+        o.apply_to(&mut cfg);
+        match cfg.io {
+            IoBackend::Simulated { fault } => {
+                assert_eq!(fault.torn_page_probability.raw(), 0.25);
+                assert_eq!(fault.io_error_probability.raw(), 0.25);
+            }
+            _ => panic!("expected simulated io"),
+        }
+    }
+}

crates/tranquil-store/src/gauntlet/regression.rs

@@ -0,0 +1,290 @@
+use std::io;
+use std::path::{Path, PathBuf};
+
+use serde::{Deserialize, Serialize};
+
+use super::invariants::InvariantViolation;
+use super::op::{Op, OpStream, Seed};
+use super::overrides::ConfigOverrides;
+use super::runner::{GauntletConfig, GauntletReport};
+use super::scenarios::{Scenario, UnknownScenario, config_for};
+
+pub const SCHEMA_VERSION: u32 = 1;
+pub const MIN_SUPPORTED_SCHEMA_VERSION: u32 = 1;
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct RegressionViolation {
+    pub invariant: String,
+    pub detail: String,
+}
+
+impl From<&InvariantViolation> for RegressionViolation {
+    fn from(v: &InvariantViolation) -> Self {
+        Self {
+            invariant: v.invariant.to_string(),
+            detail: v.detail.clone(),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct RegressionRecord {
+    pub schema_version: u32,
+    pub scenario: String,
+    pub seed: Seed,
+    #[serde(default)]
+    pub overrides: ConfigOverrides,
+    pub violations: Vec<RegressionViolation>,
+    pub ops: Vec<Op>,
+    #[serde(default)]
+    pub original_ops_len: usize,
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum RegressionLoadError {
+    #[error("read {path}: {source}")]
+    Read { path: PathBuf, source: io::Error },
+    #[error("parse {path}: {source}")]
+    Parse {
+        path: PathBuf,
+        source: serde_json::Error,
+    },
+    #[error("schema version {found} outside supported range {min}..={max}")]
+    UnsupportedVersion { found: u32, min: u32, max: u32 },
+    #[error(transparent)]
+    UnknownScenario(#[from] UnknownScenario),
+}
+
+impl RegressionRecord {
+    pub fn from_report(
+        scenario: Scenario,
+        overrides: ConfigOverrides,
+        report: &GauntletReport,
+        original_ops_len: usize,
+        shrunk_ops: OpStream,
+    ) -> Self {
+        Self {
+            schema_version: SCHEMA_VERSION,
+            scenario: scenario.name().to_string(),
+            seed: report.seed,
+            overrides,
+            violations: report
+                .violations
+                .iter()
+                .map(RegressionViolation::from)
+                .collect(),
+            ops: shrunk_ops.into_vec(),
+            original_ops_len,
+        }
+    }
+
+    pub fn file_path(&self, root: &Path) -> PathBuf {
+        root.join("gauntlet")
+            .join(sanitize(&self.scenario))
+            .join(format!("{:016x}.json", self.seed.0))
+    }
+
+    pub fn write_to(&self, root: &Path) -> io::Result<PathBuf> {
+        let path = self.file_path(root);
+        if let Some(parent) = path.parent() {
+            std::fs::create_dir_all(parent)?;
+        }
+        let json = serde_json::to_vec_pretty(self).map_err(io::Error::other)?;
+        let tmp = path.with_extension("json.tmp");
+        {
+            let mut f = std::fs::File::create(&tmp)?;
+            io::Write::write_all(&mut f, &json)?;
+            f.sync_all()?;
+        }
+        std::fs::rename(&tmp, &path)?;
+        if let Some(parent) = path.parent()
+            && let Ok(dir) = std::fs::File::open(parent)
+        {
+            let _ = dir.sync_all();
+        }
+        Ok(path)
+    }
+
+    pub fn load(path: &Path) -> Result<Self, RegressionLoadError> {
+        let raw = std::fs::read(path).map_err(|source| RegressionLoadError::Read {
+            path: path.to_path_buf(),
+            source,
+        })?;
+        let record: RegressionRecord =
+            serde_json::from_slice(&raw).map_err(|source| RegressionLoadError::Parse {
+                path: path.to_path_buf(),
+                source,
+            })?;
+        if record.schema_version < MIN_SUPPORTED_SCHEMA_VERSION
+            || record.schema_version > SCHEMA_VERSION
+        {
+            return Err(RegressionLoadError::UnsupportedVersion {
+                found: record.schema_version,
+                min: MIN_SUPPORTED_SCHEMA_VERSION,
+                max: SCHEMA_VERSION,
+            });
+        }
+        Ok(record)
+    }
+
+    pub fn scenario_enum(&self) -> Result<Scenario, UnknownScenario> {
+        self.scenario.parse::<Scenario>()
+    }
+
+    pub fn build_config(&self) -> Result<GauntletConfig, UnknownScenario> {
+        let scenario = self.scenario_enum()?;
+        let mut cfg = config_for(scenario, self.seed);
+        self.overrides.apply_to(&mut cfg);
+        Ok(cfg)
+    }
+
+    pub fn op_stream(&self) -> OpStream {
+        OpStream::from_vec(self.ops.clone())
+    }
+}
+
+fn sanitize(s: &str) -> String {
+    s.chars()
+        .map(|c| match c {
+            'a'..='z' | 'A'..='Z' | '0'..='9' | '_' | '-' => c,
+            _ => '_',
+        })
+        .collect()
+}
+
+pub fn default_root() -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("proptest-regressions")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::gauntlet::op::{CollectionName, RecordKey, ValueSeed};
+    use crate::gauntlet::overrides::ConfigOverrides;
+
+    fn sample_record() -> RegressionRecord {
+        use crate::gauntlet::overrides::StoreOverrides;
+
+        let ops = vec![
+            Op::AddRecord {
+                collection: CollectionName("c".into()),
+                rkey: RecordKey("r".into()),
+                value_seed: ValueSeed(1),
+            },
+            Op::Compact,
+        ];
+        let overrides = ConfigOverrides {
+            op_count: Some(128),
+            store: StoreOverrides {
+                max_file_size: Some(4096),
+                ..StoreOverrides::default()
+            },
+            ..ConfigOverrides::default()
+        };
+        RegressionRecord {
+            schema_version: SCHEMA_VERSION,
+            scenario: "HugeValues".to_string(),
+            seed: Seed(0xdeadbeef),
+            overrides,
+            violations: vec![RegressionViolation {
+                invariant: "ByteBudget".to_string(),
+                detail: "exceeded".to_string(),
+            }],
+            ops,
+            original_ops_len: 500,
+        }
+    }
+
+    #[test]
+    fn round_trip_preserves_all_fields() {
+        let dir = tempfile::TempDir::new().unwrap();
+        let original = sample_record();
+        let path = original.write_to(dir.path()).unwrap();
+        assert!(path.exists());
+        let loaded = RegressionRecord::load(&path).unwrap();
+        assert_eq!(loaded.schema_version, original.schema_version);
+        assert_eq!(loaded.scenario, original.scenario);
+        assert_eq!(loaded.seed.0, original.seed.0);
+        assert_eq!(loaded.overrides, original.overrides);
+        assert_eq!(loaded.violations, original.violations);
+        assert_eq!(loaded.ops.len(), original.ops.len());
+        assert_eq!(loaded.original_ops_len, original.original_ops_len);
+    }
+
+    #[test]
+    fn build_config_applies_overrides() {
+        let record = sample_record();
+        let cfg = record.build_config().unwrap();
+        assert_eq!(cfg.op_count.0, 128);
+        assert_eq!(cfg.store.max_file_size.0, 4096);
+    }
+
+    #[test]
+    fn rejects_future_schema_version() {
+        let dir = tempfile::TempDir::new().unwrap();
+        let mut r = sample_record();
+        r.schema_version = SCHEMA_VERSION + 1;
+        let path = r.write_to(dir.path()).unwrap();
+        match RegressionRecord::load(&path) {
+            Err(RegressionLoadError::UnsupportedVersion { found, min, max }) => {
+                assert_eq!(found, SCHEMA_VERSION + 1);
+                assert_eq!(min, MIN_SUPPORTED_SCHEMA_VERSION);
+                assert_eq!(max, SCHEMA_VERSION);
+            }
+            other => panic!("expected UnsupportedVersion, got {other:?}"),
+        }
+    }
+
+    #[test]
+    fn rejects_past_schema_version_below_min() {
+        let dir = tempfile::TempDir::new().unwrap();
+        let mut r = sample_record();
+        r.schema_version = MIN_SUPPORTED_SCHEMA_VERSION.saturating_sub(1);
+        let path = r.write_to(dir.path()).unwrap();
+        match RegressionRecord::load(&path) {
+            Err(RegressionLoadError::UnsupportedVersion { found, min, max }) => {
+                assert_eq!(found, MIN_SUPPORTED_SCHEMA_VERSION.saturating_sub(1));
+                assert_eq!(min, MIN_SUPPORTED_SCHEMA_VERSION);
+                assert_eq!(max, SCHEMA_VERSION);
+            }
+            other => panic!("expected UnsupportedVersion, got {other:?}"),
+        }
+    }
+
+    #[test]
+    fn atomic_write_leaves_no_tmp_file() {
+        let dir = tempfile::TempDir::new().unwrap();
+        let r = sample_record();
+        let path = r.write_to(dir.path()).unwrap();
+        assert!(path.exists());
+        let tmp = path.with_extension("json.tmp");
+        assert!(
+            !tmp.exists(),
+            "tmp sibling {tmp:?} should have been renamed"
+        );
+    }
+
+    #[test]
+    fn rejects_malformed_json() {
+        let dir = tempfile::TempDir::new().unwrap();
+        let path = dir.path().join("bad.json");
+        std::fs::write(&path, b"{not json").unwrap();
+        assert!(matches!(
+            RegressionRecord::load(&path),
+            Err(RegressionLoadError::Parse { .. })
+        ));
+    }
+
+    #[test]
+    fn sanitize_strips_slashes_and_traversal() {
+        assert_eq!(sanitize("foo/bar baz"), "foo_bar_baz");
+        assert_eq!(sanitize("../etc"), "___etc");
+    }
+
+    #[test]
+    fn unknown_scenario_name_errors() {
+        let mut r = sample_record();
+        r.scenario = "BogusScenario".to_string();
+        assert!(r.build_config().is_err());
+    }
+}

crates/tranquil-store/src/gauntlet/scenarios.rs

@@ -0,0 +1,757 @@
+use super::flaky::FlakyConfig;
+use super::invariants::InvariantSet;
+use super::op::{CollectionName, Seed};
+use super::runner::{
+    EventLogConfig, GauntletConfig, IoBackend, MaxFileSize, MaxSegmentSize, OpInterval,
+    RestartPolicy, RunLimits, ShardCount, StoreConfig, WallMs, WriterConcurrency,
+};
+use super::workload::{
+    ByteRange, DidSpaceSize, KeySpaceSize, OpCount, OpWeights, RetentionMaxSecs, SizeDistribution,
+    ValueBytes, WorkloadModel,
+};
+use crate::blockstore::GroupCommitConfig;
+use crate::sim::FaultConfig;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Scenario {
+    SmokePR,
+    MstChurn,
+    MstRestartChurn,
+    FullStackRestart,
+    CatastrophicChurn,
+    HugeValues,
+    TinyBatches,
+    GiantBatches,
+    ManyFiles,
+    ModerateFaults,
+    AggressiveFaults,
+    TornPages,
+    Fsyncgate,
+    FirehoseFanout,
+    ContendedReaders,
+    ContendedWriters,
+    FlakyDevice,
+}
+
+impl Scenario {
+    pub const fn name(self) -> &'static str {
+        match self {
+            Self::SmokePR => "SmokePR",
+            Self::MstChurn => "MstChurn",
+            Self::MstRestartChurn => "MstRestartChurn",
+            Self::FullStackRestart => "FullStackRestart",
+            Self::CatastrophicChurn => "CatastrophicChurn",
+            Self::HugeValues => "HugeValues",
+            Self::TinyBatches => "TinyBatches",
+            Self::GiantBatches => "GiantBatches",
+            Self::ManyFiles => "ManyFiles",
+            Self::ModerateFaults => "ModerateFaults",
+            Self::AggressiveFaults => "AggressiveFaults",
+            Self::TornPages => "TornPages",
+            Self::Fsyncgate => "Fsyncgate",
+            Self::FirehoseFanout => "FirehoseFanout",
+            Self::ContendedReaders => "ContendedReaders",
+            Self::ContendedWriters => "ContendedWriters",
+            Self::FlakyDevice => "FlakyDevice",
+        }
+    }
+
+    pub const fn cli_name(self) -> &'static str {
+        match self {
+            Self::SmokePR => "smoke-pr",
+            Self::MstChurn => "mst-churn",
+            Self::MstRestartChurn => "mst-restart-churn",
+            Self::FullStackRestart => "full-stack-restart",
+            Self::CatastrophicChurn => "catastrophic-churn",
+            Self::HugeValues => "huge-values",
+            Self::TinyBatches => "tiny-batches",
+            Self::GiantBatches => "giant-batches",
+            Self::ManyFiles => "many-files",
+            Self::ModerateFaults => "moderate-faults",
+            Self::AggressiveFaults => "aggressive-faults",
+            Self::TornPages => "torn-pages",
+            Self::Fsyncgate => "fsyncgate",
+            Self::FirehoseFanout => "firehose-fanout",
+            Self::ContendedReaders => "contended-readers",
+            Self::ContendedWriters => "contended-writers",
+            Self::FlakyDevice => "flaky-device",
+        }
+    }
+
+    pub const fn description(self) -> &'static str {
+        match self {
+            Self::SmokePR => "60s canary, 10k ops, core invariants. Default PR gate.",
+            Self::MstChurn => "100k churn, no restart. Refcount + reachability focus.",
+            Self::MstRestartChurn => "100k churn with Poisson restart bursts every ~5k ops.",
+            Self::FullStackRestart => "5k ops, deterministic restart every 500 ops.",
+            Self::CatastrophicChurn => {
+                "1M ops, phase-2 invariants, Poisson restart. 30 min budget."
+            }
+            Self::HugeValues => "Heavy-tail values up to 16 MiB. 32 MiB file cap.",
+            Self::TinyBatches => "Group-commit batch size 1, tight checkpoints, 4 KiB files.",
+            Self::GiantBatches => "Group-commit batch size 100k, 16 MiB files.",
+            Self::ManyFiles => "256-byte file cap, many segments, delete-heavy.",
+            Self::ModerateFaults => {
+                "Simulated IO with moderate fault config. CrashAtSyscall restarts."
+            }
+            Self::AggressiveFaults => {
+                "Simulated IO with aggressive fault config. CrashAtSyscall restarts."
+            }
+            Self::TornPages => "Torn-page faults only, 20k ops.",
+            Self::Fsyncgate => "Fsync-drop faults only, 10k ops.",
+            Self::FirehoseFanout => {
+                "Eventlog-heavy workload with FSYNC_ORDERING / MONOTONIC_SEQ / TOMBSTONE_BOUND invariants."
+            }
+            Self::ContendedReaders => "60% reads, 64 writer tasks, simulated moderate faults.",
+            Self::ContendedWriters => {
+                "Add/delete heavy, 32 writer tasks, simulated moderate faults."
+            }
+            Self::FlakyDevice => {
+                "Real IO on ext4 atop dm-flakey. Requires root with dm-flakey available, skips otherwise."
+            }
+        }
+    }
+
+    pub fn from_name(name: &str) -> Option<Self> {
+        Self::ALL.iter().copied().find(|s| s.name() == name)
+    }
+
+    pub fn from_cli_name(name: &str) -> Option<Self> {
+        Self::ALL.iter().copied().find(|s| s.cli_name() == name)
+    }
+
+    pub const ALL: &'static [Scenario] = &[
+        Self::SmokePR,
+        Self::MstChurn,
+        Self::MstRestartChurn,
+        Self::FullStackRestart,
+        Self::CatastrophicChurn,
+        Self::HugeValues,
+        Self::TinyBatches,
+        Self::GiantBatches,
+        Self::ManyFiles,
+        Self::ModerateFaults,
+        Self::AggressiveFaults,
+        Self::TornPages,
+        Self::Fsyncgate,
+        Self::FirehoseFanout,
+        Self::ContendedReaders,
+        Self::ContendedWriters,
+        Self::FlakyDevice,
+    ];
+}
+
+impl serde::Serialize for Scenario {
+    fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        serializer.serialize_str(self.cli_name())
+    }
+}
+
+impl<'de> serde::Deserialize<'de> for Scenario {
+    fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+        let s = <std::borrow::Cow<'de, str>>::deserialize(deserializer)?;
+        Self::from_cli_name(&s).ok_or_else(|| {
+            serde::de::Error::custom(format!(
+                "unknown scenario {s:?}; expected one of {}",
+                Self::ALL
+                    .iter()
+                    .map(|s| s.cli_name())
+                    .collect::<Vec<_>>()
+                    .join(", ")
+            ))
+        })
+    }
+}
+
+#[cfg(feature = "gauntlet-cli")]
+impl clap::ValueEnum for Scenario {
+    fn value_variants<'a>() -> &'a [Self] {
+        Self::ALL
+    }
+
+    fn to_possible_value(&self) -> Option<clap::builder::PossibleValue> {
+        Some(clap::builder::PossibleValue::new(self.cli_name()).help(self.description()))
+    }
+}
+
+impl std::fmt::Display for Scenario {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str(self.name())
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+#[error("unknown scenario: {0}")]
+pub struct UnknownScenario(pub String);
+
+impl std::str::FromStr for Scenario {
+    type Err = UnknownScenario;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Self::from_name(s).ok_or_else(|| UnknownScenario(s.to_string()))
+    }
+}
+
+pub fn config_for(scenario: Scenario, seed: Seed) -> GauntletConfig {
+    match scenario {
+        Scenario::SmokePR => smoke_pr(seed),
+        Scenario::MstChurn => mst_churn(seed),
+        Scenario::MstRestartChurn => mst_restart_churn(seed),
+        Scenario::FullStackRestart => full_stack_restart(seed),
+        Scenario::CatastrophicChurn => catastrophic_churn(seed),
+        Scenario::HugeValues => huge_values(seed),
+        Scenario::TinyBatches => tiny_batches(seed),
+        Scenario::GiantBatches => giant_batches(seed),
+        Scenario::ManyFiles => many_files(seed),
+        Scenario::ModerateFaults => moderate_faults(seed),
+        Scenario::AggressiveFaults => aggressive_faults(seed),
+        Scenario::TornPages => torn_pages(seed),
+        Scenario::Fsyncgate => fsyncgate(seed),
+        Scenario::FirehoseFanout => firehose_fanout(seed),
+        Scenario::ContendedReaders => contended_readers(seed),
+        Scenario::ContendedWriters => contended_writers(seed),
+        Scenario::FlakyDevice => flaky_device(seed),
+    }
+}
+
+fn default_collections() -> Vec<CollectionName> {
+    vec![
+        CollectionName("app.bsky.feed.post".to_string()),
+        CollectionName("app.bsky.feed.like".to_string()),
+    ]
+}
+
+fn block_weights(add: u32, delete: u32, compact: u32, checkpoint: u32) -> OpWeights {
+    OpWeights {
+        add,
+        delete,
+        compact,
+        checkpoint,
+        ..OpWeights::default()
+    }
+}
+
+fn block_workload(
+    weights: OpWeights,
+    size_distribution: SizeDistribution,
+    key_space: KeySpaceSize,
+) -> WorkloadModel {
+    WorkloadModel {
+        weights,
+        size_distribution,
+        collections: default_collections(),
+        key_space,
+        did_space: DidSpaceSize(32),
+        retention_max_secs: RetentionMaxSecs(3600),
+    }
+}
+
+fn tiny_store() -> StoreConfig {
+    StoreConfig {
+        max_file_size: MaxFileSize(4096),
+        group_commit: GroupCommitConfig {
+            checkpoint_interval_ms: 100,
+            checkpoint_write_threshold: 10,
+            ..GroupCommitConfig::default()
+        },
+        shard_count: ShardCount(1),
+    }
+}
+
+fn smoke_pr(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Real,
+        workload: block_workload(
+            block_weights(80, 0, 10, 10),
+            SizeDistribution::Fixed(ValueBytes(64)),
+            KeySpaceSize(200),
+        ),
+        op_count: OpCount(10_000),
+        invariants: InvariantSet::REFCOUNT_CONSERVATION
+            | InvariantSet::REACHABILITY
+            | InvariantSet::ACKED_WRITE_PERSISTENCE
+            | InvariantSet::READ_AFTER_WRITE
+            | InvariantSet::RESTART_IDEMPOTENT,
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(60_000)),
+        },
+        restart_policy: RestartPolicy::EveryNOps(OpInterval(2_000)),
+        store: tiny_store(),
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn mst_churn(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Real,
+        workload: block_workload(
+            block_weights(85, 0, 10, 5),
+            SizeDistribution::Fixed(ValueBytes(64)),
+            KeySpaceSize(2_000),
+        ),
+        op_count: OpCount(100_000),
+        invariants: InvariantSet::REFCOUNT_CONSERVATION
+            | InvariantSet::REACHABILITY
+            | InvariantSet::ACKED_WRITE_PERSISTENCE
+            | InvariantSet::READ_AFTER_WRITE
+            | InvariantSet::RESTART_IDEMPOTENT,
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(600_000)),
+        },
+        restart_policy: RestartPolicy::Never,
+        store: tiny_store(),
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn mst_restart_churn(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Real,
+        workload: block_workload(
+            block_weights(85, 0, 10, 5),
+            SizeDistribution::Fixed(ValueBytes(64)),
+            KeySpaceSize(2_000),
+        ),
+        op_count: OpCount(100_000),
+        invariants: InvariantSet::REFCOUNT_CONSERVATION
+            | InvariantSet::REACHABILITY
+            | InvariantSet::ACKED_WRITE_PERSISTENCE
+            | InvariantSet::READ_AFTER_WRITE
+            | InvariantSet::RESTART_IDEMPOTENT,
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(600_000)),
+        },
+        restart_policy: RestartPolicy::PoissonByOps(OpInterval(5_000)),
+        store: tiny_store(),
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn full_stack_restart(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Real,
+        workload: block_workload(
+            block_weights(80, 0, 15, 5),
+            SizeDistribution::Fixed(ValueBytes(80)),
+            KeySpaceSize(500),
+        ),
+        op_count: OpCount(5_000),
+        invariants: InvariantSet::REFCOUNT_CONSERVATION
+            | InvariantSet::REACHABILITY
+            | InvariantSet::ACKED_WRITE_PERSISTENCE
+            | InvariantSet::READ_AFTER_WRITE
+            | InvariantSet::RESTART_IDEMPOTENT,
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(120_000)),
+        },
+        restart_policy: RestartPolicy::EveryNOps(OpInterval(500)),
+        store: StoreConfig {
+            max_file_size: MaxFileSize(4096),
+            group_commit: GroupCommitConfig::default(),
+            shard_count: ShardCount(1),
+        },
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn phase2_invariants() -> InvariantSet {
+    InvariantSet::REFCOUNT_CONSERVATION
+        | InvariantSet::REACHABILITY
+        | InvariantSet::ACKED_WRITE_PERSISTENCE
+        | InvariantSet::READ_AFTER_WRITE
+        | InvariantSet::RESTART_IDEMPOTENT
+        | InvariantSet::COMPACTION_IDEMPOTENT
+        | InvariantSet::BYTE_BUDGET
+        | InvariantSet::MANIFEST_EQUALS_REALITY
+        | InvariantSet::CHECKSUM_COVERAGE
+}
+
+fn catastrophic_churn(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Real,
+        workload: block_workload(
+            block_weights(94, 0, 5, 1),
+            SizeDistribution::Fixed(ValueBytes(64)),
+            KeySpaceSize(200),
+        ),
+        op_count: OpCount(1_000_000),
+        invariants: phase2_invariants(),
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(30 * 60_000)),
+        },
+        restart_policy: RestartPolicy::PoissonByOps(OpInterval(50_000)),
+        store: tiny_store(),
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn huge_values(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Real,
+        workload: block_workload(
+            block_weights(85, 5, 8, 2),
+            SizeDistribution::HeavyTail(
+                ByteRange::new(ValueBytes(256), ValueBytes(16 * 1024 * 1024))
+                    .expect("huge_values ByteRange"),
+            ),
+            KeySpaceSize(64),
+        ),
+        op_count: OpCount(2_000),
+        invariants: InvariantSet::REFCOUNT_CONSERVATION
+            | InvariantSet::REACHABILITY
+            | InvariantSet::ACKED_WRITE_PERSISTENCE
+            | InvariantSet::READ_AFTER_WRITE
+            | InvariantSet::RESTART_IDEMPOTENT,
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(10 * 60_000)),
+        },
+        restart_policy: RestartPolicy::EveryNOps(OpInterval(500)),
+        store: StoreConfig {
+            max_file_size: MaxFileSize(32 * 1024 * 1024),
+            group_commit: GroupCommitConfig::default(),
+            shard_count: ShardCount(1),
+        },
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn tiny_batches(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Real,
+        workload: block_workload(
+            block_weights(85, 0, 5, 10),
+            SizeDistribution::Fixed(ValueBytes(64)),
+            KeySpaceSize(500),
+        ),
+        op_count: OpCount(10_000),
+        invariants: phase2_invariants(),
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(120_000)),
+        },
+        restart_policy: RestartPolicy::EveryNOps(OpInterval(2_000)),
+        store: StoreConfig {
+            max_file_size: MaxFileSize(4096),
+            group_commit: GroupCommitConfig {
+                max_batch_size: 1,
+                checkpoint_interval_ms: 100,
+                checkpoint_write_threshold: 1,
+                ..GroupCommitConfig::default()
+            },
+            shard_count: ShardCount(1),
+        },
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn giant_batches(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Real,
+        workload: block_workload(
+            block_weights(95, 0, 3, 2),
+            SizeDistribution::Fixed(ValueBytes(64)),
+            KeySpaceSize(5_000),
+        ),
+        op_count: OpCount(50_000),
+        invariants: phase2_invariants(),
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(10 * 60_000)),
+        },
+        restart_policy: RestartPolicy::EveryNOps(OpInterval(10_000)),
+        store: StoreConfig {
+            max_file_size: MaxFileSize(16 * 1024 * 1024),
+            group_commit: GroupCommitConfig {
+                max_batch_size: 100_000,
+                checkpoint_interval_ms: 5_000,
+                checkpoint_write_threshold: 100_000,
+                ..GroupCommitConfig::default()
+            },
+            shard_count: ShardCount(1),
+        },
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn many_files(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Real,
+        workload: block_workload(
+            block_weights(80, 10, 5, 5),
+            SizeDistribution::Fixed(ValueBytes(128)),
+            KeySpaceSize(2_000),
+        ),
+        op_count: OpCount(200_000),
+        invariants: phase2_invariants(),
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(20 * 60_000)),
+        },
+        restart_policy: RestartPolicy::PoissonByOps(OpInterval(5_000)),
+        store: StoreConfig {
+            max_file_size: MaxFileSize(256),
+            group_commit: GroupCommitConfig::default(),
+            shard_count: ShardCount(1),
+        },
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn sim_invariants() -> InvariantSet {
+    InvariantSet::REFCOUNT_CONSERVATION
+        | InvariantSet::REACHABILITY
+        | InvariantSet::ACKED_WRITE_PERSISTENCE
+        | InvariantSet::READ_AFTER_WRITE
+        | InvariantSet::RESTART_IDEMPOTENT
+        | InvariantSet::NO_ORPHAN_FILES
+        | InvariantSet::BYTE_BUDGET
+        | InvariantSet::CHECKSUM_COVERAGE
+}
+
+fn sim_microbench_workload() -> WorkloadModel {
+    block_workload(
+        block_weights(80, 10, 5, 5),
+        SizeDistribution::Fixed(ValueBytes(128)),
+        KeySpaceSize(500),
+    )
+}
+
+fn sim_store() -> StoreConfig {
+    StoreConfig {
+        max_file_size: MaxFileSize(16 * 1024),
+        group_commit: GroupCommitConfig {
+            verify_persisted_blocks: true,
+            ..GroupCommitConfig::default()
+        },
+        shard_count: ShardCount(1),
+    }
+}
+
+fn moderate_faults(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Simulated {
+            fault: FaultConfig::moderate(),
+        },
+        workload: sim_microbench_workload(),
+        op_count: OpCount(50_000),
+        invariants: sim_invariants(),
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(10 * 60_000)),
+        },
+        restart_policy: RestartPolicy::CrashAtSyscall(OpInterval(2_000)),
+        store: sim_store(),
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn aggressive_faults(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Simulated {
+            fault: FaultConfig::aggressive(),
+        },
+        workload: sim_microbench_workload(),
+        op_count: OpCount(50_000),
+        invariants: sim_invariants(),
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(10 * 60_000)),
+        },
+        restart_policy: RestartPolicy::CrashAtSyscall(OpInterval(2_000)),
+        store: sim_store(),
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn torn_pages(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Simulated {
+            fault: FaultConfig::torn_pages_only(),
+        },
+        workload: sim_microbench_workload(),
+        op_count: OpCount(20_000),
+        invariants: sim_invariants(),
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(5 * 60_000)),
+        },
+        restart_policy: RestartPolicy::CrashAtSyscall(OpInterval(1_000)),
+        store: sim_store(),
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn fsyncgate(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Simulated {
+            fault: FaultConfig::fsyncgate_only(),
+        },
+        workload: sim_microbench_workload(),
+        op_count: OpCount(10_000),
+        invariants: sim_invariants(),
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(5 * 60_000)),
+        },
+        restart_policy: RestartPolicy::CrashAtSyscall(OpInterval(500)),
+        store: sim_store(),
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn firehose_fanout(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Simulated {
+            fault: FaultConfig::moderate(),
+        },
+        workload: WorkloadModel {
+            weights: OpWeights {
+                add: 20,
+                compact: 2,
+                checkpoint: 3,
+                append_event: 60,
+                sync_event_log: 10,
+                run_retention: 5,
+                ..OpWeights::default()
+            },
+            size_distribution: SizeDistribution::Fixed(ValueBytes(128)),
+            collections: default_collections(),
+            key_space: KeySpaceSize(500),
+            did_space: DidSpaceSize(64),
+            retention_max_secs: RetentionMaxSecs(60),
+        },
+        op_count: OpCount(20_000),
+        invariants: sim_invariants()
+            | InvariantSet::MONOTONIC_SEQ
+            | InvariantSet::FSYNC_ORDERING
+            | InvariantSet::TOMBSTONE_BOUND,
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(10 * 60_000)),
+        },
+        restart_policy: RestartPolicy::CrashAtSyscall(OpInterval(2_000)),
+        store: sim_store(),
+        eventlog: Some(EventLogConfig {
+            max_segment_size: MaxSegmentSize(64 * 1024),
+        }),
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn contended_readers(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Simulated {
+            fault: FaultConfig::moderate(),
+        },
+        workload: WorkloadModel {
+            weights: OpWeights {
+                add: 15,
+                delete: 1,
+                compact: 2,
+                checkpoint: 2,
+                read_record: 60,
+                read_block: 20,
+                ..OpWeights::default()
+            },
+            size_distribution: SizeDistribution::Fixed(ValueBytes(128)),
+            collections: default_collections(),
+            key_space: KeySpaceSize(400),
+            did_space: DidSpaceSize(32),
+            retention_max_secs: RetentionMaxSecs(3600),
+        },
+        op_count: OpCount(20_000),
+        invariants: sim_invariants(),
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(10 * 60_000)),
+        },
+        restart_policy: RestartPolicy::CrashAtSyscall(OpInterval(2_000)),
+        store: sim_store(),
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(64),
+    }
+}
+
+fn flaky_device(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::RealWithFlaky {
+            flaky: FlakyConfig::default_stress(),
+        },
+        workload: block_workload(
+            block_weights(80, 5, 10, 5),
+            SizeDistribution::Fixed(ValueBytes(128)),
+            KeySpaceSize(500),
+        ),
+        op_count: OpCount(20_000),
+        invariants: InvariantSet::REFCOUNT_CONSERVATION
+            | InvariantSet::REACHABILITY
+            | InvariantSet::ACKED_WRITE_PERSISTENCE
+            | InvariantSet::READ_AFTER_WRITE
+            | InvariantSet::RESTART_IDEMPOTENT
+            | InvariantSet::NO_ORPHAN_FILES
+            | InvariantSet::MANIFEST_EQUALS_REALITY
+            | InvariantSet::BYTE_BUDGET
+            | InvariantSet::CHECKSUM_COVERAGE,
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(30 * 60_000)),
+        },
+        restart_policy: RestartPolicy::EveryNOps(OpInterval(1_000)),
+        store: tiny_store(),
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(1),
+    }
+}
+
+fn contended_writers(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Simulated {
+            fault: FaultConfig::moderate(),
+        },
+        workload: WorkloadModel {
+            weights: OpWeights {
+                add: 85,
+                delete: 5,
+                compact: 3,
+                checkpoint: 2,
+                read_record: 4,
+                read_block: 1,
+                ..OpWeights::default()
+            },
+            size_distribution: SizeDistribution::Fixed(ValueBytes(128)),
+            collections: default_collections(),
+            key_space: KeySpaceSize(1_000),
+            did_space: DidSpaceSize(32),
+            retention_max_secs: RetentionMaxSecs(3600),
+        },
+        op_count: OpCount(20_000),
+        invariants: sim_invariants(),
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(10 * 60_000)),
+        },
+        restart_policy: RestartPolicy::CrashAtSyscall(OpInterval(2_000)),
+        store: sim_store(),
+        eventlog: None,
+        writer_concurrency: WriterConcurrency(32),
+    }
+}

crates/tranquil-store/src/gauntlet/shrink.rs

@@ -0,0 +1,191 @@
+use std::collections::BTreeSet;
+
+use super::op::OpStream;
+use super::runner::{Gauntlet, GauntletConfig, GauntletReport};
+
+pub const DEFAULT_MAX_SHRINK_ITERATIONS: usize = 256;
+
+#[derive(Debug)]
+pub struct ShrinkOutcome {
+    pub ops: OpStream,
+    pub report: GauntletReport,
+    pub iterations: usize,
+}
+
+pub async fn shrink_failure(
+    config: GauntletConfig,
+    initial_ops: OpStream,
+    initial_report: GauntletReport,
+    max_iterations: usize,
+) -> ShrinkOutcome {
+    let target: BTreeSet<&'static str> = initial_report.violation_invariants();
+    if target.is_empty() {
+        return ShrinkOutcome {
+            ops: initial_ops,
+            report: initial_report,
+            iterations: 0,
+        };
+    }
+
+    let mut current_ops = initial_ops;
+    let mut current_report = initial_report;
+    let mut iterations = 0usize;
+
+    while iterations < max_iterations {
+        match try_one_shrink_round(&config, &current_ops, &target, max_iterations - iterations)
+            .await
+        {
+            ShrinkRound::Progress {
+                ops,
+                report,
+                runs_used,
+            } => {
+                current_ops = ops;
+                current_report = report;
+                iterations += runs_used;
+            }
+            ShrinkRound::Exhausted { runs_used } => {
+                iterations += runs_used;
+                break;
+            }
+        }
+    }
+
+    ShrinkOutcome {
+        ops: current_ops,
+        report: current_report,
+        iterations,
+    }
+}
+
+enum ShrinkRound {
+    Progress {
+        ops: OpStream,
+        report: GauntletReport,
+        runs_used: usize,
+    },
+    Exhausted {
+        runs_used: usize,
+    },
+}
+
+async fn try_one_shrink_round(
+    config: &GauntletConfig,
+    current_ops: &OpStream,
+    target: &BTreeSet<&'static str>,
+    budget: usize,
+) -> ShrinkRound {
+    let mut runs_used = 0usize;
+    for candidate in current_ops.shrink_candidates() {
+        if candidate.is_empty() || candidate.len() >= current_ops.len() {
+            continue;
+        }
+        if runs_used >= budget {
+            return ShrinkRound::Exhausted { runs_used };
+        }
+        runs_used += 1;
+        let gauntlet = match Gauntlet::new(config.clone()) {
+            Ok(g) => g,
+            Err(_) => continue,
+        };
+        let report = gauntlet.run_with_ops(candidate.clone()).await;
+        let got: BTreeSet<&'static str> = report.violation_invariants();
+        if !got.is_disjoint(target) {
+            return ShrinkRound::Progress {
+                ops: candidate,
+                report,
+                runs_used,
+            };
+        }
+    }
+    ShrinkRound::Exhausted { runs_used }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::blockstore::GroupCommitConfig;
+    use crate::gauntlet::invariants::{InvariantSet, InvariantViolation};
+    use crate::gauntlet::op::{CollectionName, Op, OpStream, RecordKey, Seed, ValueSeed};
+    use crate::gauntlet::runner::{
+        GauntletConfig, IoBackend, MaxFileSize, OpErrorCount, OpsExecuted, RestartCount,
+        RestartPolicy, RunLimits, ShardCount, StoreConfig, WriterConcurrency,
+    };
+    use crate::gauntlet::workload::{
+        DidSpaceSize, KeySpaceSize, OpCount, OpWeights, RetentionMaxSecs, SizeDistribution,
+        ValueBytes, WorkloadModel,
+    };
+    use crate::sim::FaultConfig;
+
+    fn dummy_config() -> GauntletConfig {
+        GauntletConfig {
+            seed: Seed(1),
+            io: IoBackend::Simulated {
+                fault: FaultConfig::none(),
+            },
+            workload: WorkloadModel {
+                weights: OpWeights::default(),
+                size_distribution: SizeDistribution::Fixed(ValueBytes(16)),
+                collections: vec![CollectionName("c".into())],
+                key_space: KeySpaceSize(4),
+                did_space: DidSpaceSize(1),
+                retention_max_secs: RetentionMaxSecs(60),
+            },
+            op_count: OpCount(4),
+            invariants: InvariantSet::EMPTY,
+            limits: RunLimits { max_wall_ms: None },
+            restart_policy: RestartPolicy::Never,
+            store: StoreConfig {
+                max_file_size: MaxFileSize(4096),
+                group_commit: GroupCommitConfig::default(),
+                shard_count: ShardCount(1),
+            },
+            eventlog: None,
+            writer_concurrency: WriterConcurrency(1),
+        }
+    }
+
+    fn fake_report(seed: u64, names: &[&'static str]) -> GauntletReport {
+        GauntletReport {
+            seed: Seed(seed),
+            ops_executed: OpsExecuted(0),
+            op_errors: OpErrorCount(0),
+            restarts: RestartCount(0),
+            violations: names
+                .iter()
+                .copied()
+                .map(|n| InvariantViolation {
+                    invariant: n,
+                    detail: "x".to_string(),
+                })
+                .collect(),
+            ops: OpStream::empty(),
+        }
+    }
+
+    fn sample_stream() -> OpStream {
+        OpStream::from_vec(vec![
+            Op::AddRecord {
+                collection: CollectionName("c".into()),
+                rkey: RecordKey("a".into()),
+                value_seed: ValueSeed(1),
+            },
+            Op::Compact,
+        ])
+    }
+
+    #[test]
+    fn clean_report_returns_input_unchanged() {
+        let cfg = dummy_config();
+        let ops = sample_stream();
+        let clean = fake_report(1, &[]);
+        let rt = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .unwrap();
+        let before_len = ops.len();
+        let out = rt.block_on(shrink_failure(cfg, ops, clean, 8));
+        assert_eq!(out.iterations, 0);
+        assert_eq!(out.ops.len(), before_len);
+    }
+}

crates/tranquil-store/src/gauntlet/soak.rs

@@ -0,0 +1,326 @@
+use std::io::{self, Write};
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use cid::Cid;
+use serde::{Deserialize, Serialize};
+use tracing::warn;
+
+use super::invariants::{InvariantSet, InvariantViolation};
+use super::leak::{LeakGateConfig, LeakViolation, evaluate as evaluate_leak_gate};
+use super::metrics::{MetricsSample, sample_harness};
+use super::op::{OpStream, Seed};
+use super::oracle::Oracle;
+use super::runner::{
+    EventLogState, GauntletConfig, Harness, IoBackend, apply_op, blockstore_config,
+    eventlog_snapshot, open_eventlog, refresh_oracle_graph, run_invariants, segments_subdir,
+};
+use super::workload::OpCount;
+use crate::blockstore::TranquilBlockStore;
+use crate::io::{RealIO, StorageIO};
+
+const OP_ERROR_LOG_THROTTLE: u64 = 1024;
+
+pub const DEFAULT_CHUNK_OPS: usize = 5_000;
+pub const DEFAULT_SAMPLE_INTERVAL_MS: u64 = 60_000;
+
+#[derive(Debug, Clone)]
+pub struct SoakConfig {
+    pub gauntlet: GauntletConfig,
+    pub total_duration: Duration,
+    pub sample_interval: Duration,
+    pub chunk_ops: usize,
+    pub leak_gate: LeakGateConfig,
+}
+
+impl SoakConfig {
+    pub fn new(gauntlet: GauntletConfig, total_duration: Duration) -> Self {
+        Self {
+            gauntlet,
+            total_duration,
+            sample_interval: Duration::from_millis(DEFAULT_SAMPLE_INTERVAL_MS),
+            chunk_ops: DEFAULT_CHUNK_OPS,
+            leak_gate: LeakGateConfig::standard(),
+        }
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum SoakError {
+    #[error(transparent)]
+    Io(#[from] io::Error),
+    #[error("soak requires IoBackend::Real; scenario configured Simulated")]
+    SimulatedBackendRejected,
+    #[error("open block store: {0}")]
+    StoreOpen(String),
+    #[error("open event log: {0}")]
+    EventLogOpen(String),
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SoakReport {
+    pub seed: Seed,
+    pub ops_executed: u64,
+    pub op_errors: u64,
+    pub chunks: u64,
+    pub samples: Vec<MetricsSample>,
+    pub invariant_violations: Vec<InvariantViolationRecord>,
+    pub leak_violations: Vec<LeakViolation>,
+    pub total_wall_ms: u64,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct InvariantViolationRecord {
+    pub invariant: String,
+    pub detail: String,
+}
+
+impl SoakReport {
+    pub fn is_clean(&self) -> bool {
+        self.invariant_violations.is_empty() && self.leak_violations.is_empty()
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(tag = "type")]
+pub enum SoakEvent {
+    #[serde(rename = "sample")]
+    Sample {
+        seed: u64,
+        chunk: u64,
+        ops_executed: u64,
+        sample: MetricsSample,
+    },
+    #[serde(rename = "invariant_violation")]
+    Invariant {
+        seed: u64,
+        invariant: String,
+        detail: String,
+    },
+    #[serde(rename = "summary")]
+    Summary {
+        seed: u64,
+        total_wall_ms: u64,
+        ops_executed: u64,
+        op_errors: u64,
+        chunks: u64,
+        clean: bool,
+        invariant_violations: usize,
+        leak_violations: Vec<LeakViolation>,
+    },
+}
+
+pub async fn run_soak<W: Write + Send>(
+    cfg: SoakConfig,
+    mut emitter: W,
+) -> Result<SoakReport, SoakError> {
+    if !matches!(cfg.gauntlet.io, IoBackend::Real) {
+        return Err(SoakError::SimulatedBackendRejected);
+    }
+    let dir = tempfile::TempDir::new()?;
+    let store_cfg = blockstore_config(dir.path(), &cfg.gauntlet.store);
+    let segments_dir: PathBuf = segments_subdir(dir.path());
+    let store = TranquilBlockStore::open(store_cfg)
+        .map(Arc::new)
+        .map_err(|e| SoakError::StoreOpen(e.to_string()))?;
+    let eventlog: Option<EventLogState<RealIO>> = match cfg.gauntlet.eventlog {
+        None => None,
+        Some(ec) => Some(
+            open_eventlog(RealIO::new(), segments_dir, ec.max_segment_size.0)
+                .map_err(|e| SoakError::EventLogOpen(e.to_string()))?,
+        ),
+    };
+    let mut harness = Harness { store, eventlog };
+    let outcome = drive_soak(&mut harness, &cfg, &mut emitter).await;
+    shutdown_harness(&mut harness);
+    outcome
+}
+
+fn shutdown_harness<S: StorageIO + Send + Sync + 'static>(harness: &mut Harness<S>) {
+    if let Some(el) = harness.eventlog.as_mut() {
+        if let Err(e) = el.writer.shutdown() {
+            warn!(error = %e, "soak: event log writer shutdown failed");
+        }
+        el.manager.shutdown();
+    }
+}
+
+async fn drive_soak<S, W>(
+    harness: &mut Harness<S>,
+    cfg: &SoakConfig,
+    emitter: &mut W,
+) -> Result<SoakReport, SoakError>
+where
+    S: StorageIO + Send + Sync + 'static,
+    W: Write + Send,
+{
+    let mut oracle = Oracle::new();
+    let mut root: Option<Cid> = None;
+
+    let start = Instant::now();
+    let mut samples: Vec<MetricsSample> = Vec::new();
+    let mut invariant_records: Vec<InvariantViolationRecord> = Vec::new();
+    let mut ops_executed: u64 = 0;
+    let mut op_errors: u64 = 0;
+    let mut chunks: u64 = 0;
+    let mut last_sample = start;
+    let mut next_error_log_at: u64 = 1;
+
+    let initial = sample_harness(harness, Duration::ZERO);
+    emit_event(
+        emitter,
+        &SoakEvent::Sample {
+            seed: cfg.gauntlet.seed.0,
+            chunk: 0,
+            ops_executed: 0,
+            sample: initial,
+        },
+    )?;
+    samples.push(initial);
+
+    while start.elapsed() < cfg.total_duration {
+        let chunk_seed = Seed(
+            cfg.gauntlet
+                .seed
+                .0
+                .wrapping_add(chunks.wrapping_mul(0x9E37_79B9_7F4A_7C15)),
+        );
+        let stream: OpStream = cfg
+            .gauntlet
+            .workload
+            .generate(chunk_seed, OpCount(cfg.chunk_ops));
+        for op in stream.iter() {
+            if start.elapsed() >= cfg.total_duration {
+                break;
+            }
+            match apply_op(harness, &mut root, &mut oracle, op, &cfg.gauntlet.workload).await {
+                Ok(()) => {
+                    ops_executed = ops_executed.saturating_add(1);
+                }
+                Err(e) => {
+                    op_errors = op_errors.saturating_add(1);
+                    if op_errors >= next_error_log_at {
+                        warn!(
+                            op_errors,
+                            ops_executed,
+                            elapsed_ms = u64::try_from(start.elapsed().as_millis())
+                                .unwrap_or(u64::MAX),
+                            error = %e,
+                            "soak: op error milestone"
+                        );
+                        next_error_log_at = next_error_log_at
+                            .saturating_mul(2)
+                            .max(OP_ERROR_LOG_THROTTLE);
+                    }
+                }
+            }
+            if last_sample.elapsed() >= cfg.sample_interval {
+                let elapsed = start.elapsed();
+                let s = sample_harness(harness, elapsed);
+                emit_event(
+                    emitter,
+                    &SoakEvent::Sample {
+                        seed: cfg.gauntlet.seed.0,
+                        chunk: chunks,
+                        ops_executed,
+                        sample: s,
+                    },
+                )?;
+                samples.push(s);
+                last_sample = Instant::now();
+            }
+        }
+        chunks = chunks.saturating_add(1);
+        tokio::task::yield_now().await;
+    }
+
+    let final_elapsed = start.elapsed();
+    let final_sample = sample_harness(harness, final_elapsed);
+    emit_event(
+        emitter,
+        &SoakEvent::Sample {
+            seed: cfg.gauntlet.seed.0,
+            chunk: chunks,
+            ops_executed,
+            sample: final_sample,
+        },
+    )?;
+    samples.push(final_sample);
+
+    let invariants = match refresh_oracle_graph(&harness.store, &mut oracle, root).await {
+        Ok(()) => {
+            let snapshot = eventlog_snapshot(harness.eventlog.as_ref());
+            let set = cfg
+                .gauntlet
+                .invariants
+                .without(InvariantSet::RESTART_IDEMPOTENT);
+            run_invariants(&harness.store, &oracle, root, snapshot, set).await
+        }
+        Err(e) => vec![InvariantViolation {
+            invariant: "MstRootDurability",
+            detail: format!("refresh: {e}"),
+        }],
+    };
+    for v in invariants.iter() {
+        let rec = InvariantViolationRecord {
+            invariant: v.invariant.to_string(),
+            detail: v.detail.clone(),
+        };
+        emit_event(
+            emitter,
+            &SoakEvent::Invariant {
+                seed: cfg.gauntlet.seed.0,
+                invariant: rec.invariant.clone(),
+                detail: rec.detail.clone(),
+            },
+        )?;
+        invariant_records.push(rec);
+    }
+
+    let leak_violations = evaluate_leak_gate(&samples, cfg.leak_gate);
+    let total_wall_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
+    let clean = invariant_records.is_empty() && leak_violations.is_empty();
+    emit_event(
+        emitter,
+        &SoakEvent::Summary {
+            seed: cfg.gauntlet.seed.0,
+            total_wall_ms,
+            ops_executed,
+            op_errors,
+            chunks,
+            clean,
+            invariant_violations: invariant_records.len(),
+            leak_violations: leak_violations.clone(),
+        },
+    )?;
+
+    Ok(SoakReport {
+        seed: cfg.gauntlet.seed,
+        ops_executed,
+        op_errors,
+        chunks,
+        samples,
+        invariant_violations: invariant_records,
+        leak_violations,
+        total_wall_ms,
+    })
+}
+
+fn emit_event<W: Write>(emitter: &mut W, event: &SoakEvent) -> io::Result<()> {
+    let line = serde_json::to_string(event).map_err(io::Error::other)?;
+    writeln!(emitter, "{line}")?;
+    emitter.flush()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn send_sync<T: Send + Sync>() {}
+
+    #[test]
+    fn soak_error_is_send_sync() {
+        send_sync::<SoakError>();
+    }
+}

crates/tranquil-store/src/gauntlet/workload.rs

@@ -0,0 +1,217 @@
+use super::op::{
+    CollectionName, DidSeed, EventKind, Op, OpStream, PayloadSeed, RecordKey, RetentionSecs, Seed,
+    ValueSeed,
+};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct ValueBytes(pub u32);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct KeySpaceSize(pub u32);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct OpCount(pub usize);
+
+#[derive(Debug, Clone, Copy, Default)]
+pub struct OpWeights {
+    pub add: u32,
+    pub delete: u32,
+    pub compact: u32,
+    pub checkpoint: u32,
+    pub append_event: u32,
+    pub sync_event_log: u32,
+    pub run_retention: u32,
+    pub read_record: u32,
+    pub read_block: u32,
+}
+
+impl OpWeights {
+    pub const fn total(&self) -> u32 {
+        self.add
+            + self.delete
+            + self.compact
+            + self.checkpoint
+            + self.append_event
+            + self.sync_event_log
+            + self.run_retention
+            + self.read_record
+            + self.read_block
+    }
+
+    pub const fn touches_eventlog(&self) -> bool {
+        self.append_event > 0 || self.sync_event_log > 0 || self.run_retention > 0
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct ByteRange {
+    min: ValueBytes,
+    max: ValueBytes,
+}
+
+impl ByteRange {
+    pub fn new(min: ValueBytes, max: ValueBytes) -> Result<Self, String> {
+        if max.0 < min.0 {
+            Err(format!("ByteRange: max {} < min {}", max.0, min.0))
+        } else {
+            Ok(Self { min, max })
+        }
+    }
+
+    pub fn min(&self) -> ValueBytes {
+        self.min
+    }
+
+    pub fn max(&self) -> ValueBytes {
+        self.max
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub enum SizeDistribution {
+    Fixed(ValueBytes),
+    Uniform(ByteRange),
+    HeavyTail(ByteRange),
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct DidSpaceSize(pub u32);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct RetentionMaxSecs(pub u32);
+
+#[derive(Debug, Clone)]
+pub struct WorkloadModel {
+    pub weights: OpWeights,
+    pub size_distribution: SizeDistribution,
+    pub collections: Vec<CollectionName>,
+    pub key_space: KeySpaceSize,
+    pub did_space: DidSpaceSize,
+    pub retention_max_secs: RetentionMaxSecs,
+}
+
+impl Default for WorkloadModel {
+    fn default() -> Self {
+        Self {
+            weights: OpWeights {
+                add: 80,
+                delete: 10,
+                compact: 5,
+                checkpoint: 5,
+                append_event: 0,
+                sync_event_log: 0,
+                run_retention: 0,
+                read_record: 0,
+                read_block: 0,
+            },
+            size_distribution: SizeDistribution::Fixed(ValueBytes(64)),
+            collections: vec![CollectionName("app.bsky.feed.post".to_string())],
+            key_space: KeySpaceSize(200),
+            did_space: DidSpaceSize(32),
+            retention_max_secs: RetentionMaxSecs(3600),
+        }
+    }
+}
+
+impl WorkloadModel {
+    pub fn generate(&self, seed: Seed, op_count: OpCount) -> OpStream {
+        let mut rng = Lcg::new(seed);
+        let total = self.weights.total();
+        assert!(total > 0, "workload weights must sum to > 0");
+        assert!(
+            !self.collections.is_empty(),
+            "workload needs at least 1 collection"
+        );
+
+        let ops: Vec<Op> = (0..op_count.0)
+            .map(|_| {
+                let bucket = rng.next_u32() % total;
+                let coll = self.collections[rng.next_usize() % self.collections.len()].clone();
+                let rkey = RecordKey(format!("{:06}", rng.next_u32() % self.key_space.0.max(1)));
+
+                let w = &self.weights;
+                let t1 = w.add;
+                let t2 = t1 + w.delete;
+                let t3 = t2 + w.compact;
+                let t4 = t3 + w.checkpoint;
+                let t5 = t4 + w.append_event;
+                let t6 = t5 + w.sync_event_log;
+                let t7 = t6 + w.run_retention;
+                let t8 = t7 + w.read_record;
+
+                match bucket {
+                    b if b < t1 => Op::AddRecord {
+                        collection: coll,
+                        rkey,
+                        value_seed: ValueSeed(rng.next_u32()),
+                    },
+                    b if b < t2 => Op::DeleteRecord {
+                        collection: coll,
+                        rkey,
+                    },
+                    b if b < t3 => Op::Compact,
+                    b if b < t4 => Op::Checkpoint,
+                    b if b < t5 => Op::AppendEvent {
+                        did_seed: DidSeed(rng.next_u32() % self.did_space.0.max(1)),
+                        event_kind: event_kind_for(rng.next_u32()),
+                        payload_seed: PayloadSeed(rng.next_u32()),
+                    },
+                    b if b < t6 => Op::SyncEventLog,
+                    b if b < t7 => Op::RunRetention {
+                        max_age_secs: RetentionSecs(
+                            rng.next_u32() % self.retention_max_secs.0.max(1),
+                        ),
+                    },
+                    b if b < t8 => Op::ReadRecord {
+                        collection: coll,
+                        rkey,
+                    },
+                    _ => Op::ReadBlock {
+                        value_seed: ValueSeed(rng.next_u32()),
+                    },
+                }
+            })
+            .collect();
+        OpStream::from_vec(ops)
+    }
+}
+
+fn event_kind_for(n: u32) -> EventKind {
+    match n & 0b11 {
+        0 => EventKind::Commit,
+        1 => EventKind::Identity,
+        2 => EventKind::Account,
+        _ => EventKind::Sync,
+    }
+}
+
+pub struct Lcg {
+    state: u64,
+}
+
+impl Lcg {
+    pub fn new(seed: Seed) -> Self {
+        Self {
+            state: seed
+                .0
+                .wrapping_mul(6364136223846793005)
+                .wrapping_add(1442695040888963407),
+        }
+    }
+
+    pub fn next_u64(&mut self) -> u64 {
+        self.state = self
+            .state
+            .wrapping_mul(6364136223846793005)
+            .wrapping_add(1442695040888963407);
+        self.state
+    }
+
+    pub fn next_u32(&mut self) -> u32 {
+        (self.next_u64() >> 16) as u32
+    }
+
+    pub fn next_usize(&mut self) -> usize {
+        self.next_u32() as usize
+    }
+}

crates/tranquil-store/src/lib.rs

@@ -6,6 +6,8 @@ pub mod consistency;
 pub mod eventlog;
 pub mod fsync_order;
 #[cfg(any(test, feature = "test-harness"))]
+pub mod gauntlet;
+#[cfg(any(test, feature = "test-harness"))]
 mod harness;
 mod io;
 pub mod metastore;
@@ -26,8 +28,8 @@ pub use record::{
 };
 #[cfg(any(test, feature = "test-harness"))]
 pub use sim::{
-    FaultConfig, OpRecord, SimulatedIO, sim_proptest_cases, sim_seed_count, sim_seed_range,
-    sim_single_seed,
+    FaultConfig, LatencyNs, OpRecord, Probability, SimulatedIO, SyncReorderWindow,
+    sim_proptest_cases, sim_seed_count, sim_seed_range, sim_single_seed,
 };
 
 pub(crate) fn wall_clock_ms() -> blockstore::WallClockMs {

crates/tranquil-store/src/metastore/handler.rs

@@ -5964,7 +5964,7 @@ fn handler_loop<S: StorageIO + 'static>(
                         None => "unknown panic payload".to_owned(),
                     },
                 };
-                tracing::error!(thread_index, msg, "metastore handler panic (recovered)");
+                tracing::error!(thread_index, msg, "recovered metastore handler panic");
             }
         }
     });