fix(lexicons): cache ttl

Lewis: May this revision serve well! <lu5a@proton.me>

.config/nextest.toml

@@ -72,6 +72,14 @@ test-group = "io-heavy-sim"
 filter = "test(/test_scale_/) | test(/full_backup_and_restore/)"
 slow-timeout = { period = "120s", terminate-after = 4 }
 
+[[profile.default.overrides]]
+filter = "binary(gauntlet_smoke)"
+slow-timeout = { period = "300s", terminate-after = 8 }
+
+[[profile.default.overrides]]
+filter = "binary(compaction_restart) | binary(mst_refcount_integrity) | binary(gc_compaction_restart)"
+slow-timeout = { period = "120s", terminate-after = 4 }
+
 [[profile.ci.overrides]]
 filter = "test(/import_with_verification/) | test(/plc_migration/)"
 test-group = "serial-env-tests"

Cargo.lock

@@ -7405,7 +7405,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-api"
-version = "0.5.1"
+version = "0.5.5"
 dependencies = [
  "anyhow",
  "axum",
@@ -7456,7 +7456,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-auth"
-version = "0.5.1"
+version = "0.5.5"
 dependencies = [
  "anyhow",
  "base32",
@@ -7479,7 +7479,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-cache"
-version = "0.5.1"
+version = "0.5.5"
 dependencies = [
  "async-trait",
  "base64 0.22.1",
@@ -7493,7 +7493,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-comms"
-version = "0.5.1"
+version = "0.5.5"
 dependencies = [
  "async-trait",
  "base64 0.22.1",
@@ -7511,7 +7511,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-config"
-version = "0.5.1"
+version = "0.5.5"
 dependencies = [
  "confique",
  "serde",
@@ -7519,7 +7519,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-crypto"
-version = "0.5.1"
+version = "0.5.5"
 dependencies = [
  "aes-gcm",
  "base64 0.22.1",
@@ -7535,7 +7535,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-db"
-version = "0.5.1"
+version = "0.5.5"
 dependencies = [
  "async-trait",
  "chrono",
@@ -7552,7 +7552,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-db-traits"
-version = "0.5.1"
+version = "0.5.5"
 dependencies = [
  "async-trait",
  "base64 0.22.1",
@@ -7568,7 +7568,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-infra"
-version = "0.5.1"
+version = "0.5.5"
 dependencies = [
  "async-trait",
  "bytes",
@@ -7579,9 +7579,10 @@ dependencies = [
 
 [[package]]
 name = "tranquil-lexicon"
-version = "0.5.1"
+version = "0.5.5"
 dependencies = [
  "chrono",
+ "futures",
  "hickory-resolver",
  "parking_lot",
  "reqwest",
@@ -7597,7 +7598,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-oauth"
-version = "0.5.1"
+version = "0.5.5"
 dependencies = [
  "anyhow",
  "axum",
@@ -7620,7 +7621,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-oauth-server"
-version = "0.5.1"
+version = "0.5.5"
 dependencies = [
  "axum",
  "base64 0.22.1",
@@ -7653,7 +7654,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-pds"
-version = "0.5.1"
+version = "0.5.5"
 dependencies = [
  "aes-gcm",
  "anyhow",
@@ -7745,7 +7746,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-repo"
-version = "0.5.1"
+version = "0.5.5"
 dependencies = [
  "bytes",
  "cid",
@@ -7757,7 +7758,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-ripple"
-version = "0.5.1"
+version = "0.5.5"
 dependencies = [
  "async-trait",
  "backon",
@@ -7782,7 +7783,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-scopes"
-version = "0.5.1"
+version = "0.5.5"
 dependencies = [
  "axum",
  "futures",
@@ -7798,7 +7799,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-server"
-version = "0.5.1"
+version = "0.5.5"
 dependencies = [
  "axum",
  "clap",
@@ -7819,7 +7820,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-signal"
-version = "0.5.1"
+version = "0.5.5"
 dependencies = [
  "async-trait",
  "chrono",
@@ -7842,7 +7843,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-storage"
-version = "0.5.1"
+version = "0.5.5"
 dependencies = [
  "async-trait",
  "aws-config",
@@ -7859,7 +7860,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-store"
-version = "0.5.1"
+version = "0.5.5"
 dependencies = [
  "async-trait",
  "bytes",
@@ -7872,6 +7873,7 @@ dependencies = [
  "jacquard-common",
  "jacquard-repo",
  "k256",
+ "libc",
  "lsm-tree",
  "memmap2",
  "multihash",
@@ -7905,7 +7907,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-sync"
-version = "0.5.1"
+version = "0.5.5"
 dependencies = [
  "anyhow",
  "axum",
@@ -7927,7 +7929,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-types"
-version = "0.5.1"
+version = "0.5.5"
 dependencies = [
  "chrono",
  "cid",

Cargo.toml

@@ -26,7 +26,7 @@ members = [
 ]
 
 [workspace.package]
-version = "0.5.1"
+version = "0.5.5"
 edition = "2024"
 license = "AGPL-3.0-or-later"
 

Dockerfile

@@ -1,7 +1,10 @@
-FROM denoland/deno:alpine AS frontend
+FROM node:24-alpine AS frontend
+RUN corepack enable && corepack prepare pnpm@latest --activate
 WORKDIR /app
+COPY frontend/package.json frontend/pnpm-lock.yaml ./
+RUN pnpm install --frozen-lockfile
 COPY frontend/ ./
-RUN deno task build
+RUN pnpm build
 
 FROM rust:1.92-alpine AS builder
 RUN apk add --no-cache ca-certificates musl-dev pkgconfig openssl-dev openssl-libs-static mold clang protoc

crates/tranquil-api/src/actor/preferences.rs

@@ -12,7 +12,6 @@ use tranquil_pds::auth::{Auth, NotTakendown, Permissive};
 use tranquil_pds::state::AppState;
 
 const APP_BSKY_NAMESPACE: &str = "app.bsky";
-const MAX_PREFERENCES_COUNT: usize = 100;
 const MAX_PREFERENCE_SIZE: usize = 10_000;
 const PERSONAL_DETAILS_PREF: &str = "app.bsky.actor.defs#personalDetailsPref";
 const DECLARED_AGE_PREF: &str = "app.bsky.actor.defs#declaredAgePref";
@@ -92,6 +91,8 @@ pub async fn put_preferences(
     auth: Auth<NotTakendown>,
     Json(input): Json<PutPreferencesInput>,
 ) -> Response {
+    let max_preferences_count: usize = tranquil_config::get().server.max_preferences_count;
+
     let has_full_access = auth.permissions().has_full_access();
     let user_id: uuid::Uuid = match state.repos.user.get_id_by_did(&auth.did).await {
         Ok(Some(id)) => id,
@@ -99,11 +100,11 @@ pub async fn put_preferences(
             return ApiError::InternalError(Some("User not found".into())).into_response();
         }
     };
-    if input.preferences.len() > MAX_PREFERENCES_COUNT {
+    if input.preferences.len() > max_preferences_count {
         return ApiError::InvalidRequest(format!(
             "Too many preferences: {} exceeds limit of {}",
             input.preferences.len(),
-            MAX_PREFERENCES_COUNT
+            max_preferences_count
         ))
         .into_response();
     }

crates/tranquil-api/src/common.rs

@@ -231,10 +231,19 @@ pub async fn verify_credential(
     app_passwords
         .into_iter()
         .find(|app| bcrypt::verify(password, &app.password_hash).unwrap_or(false))
-        .map(|app| CredentialMatch::AppPassword {
-            name: app.name,
-            scopes: app.scopes,
-            controller_did: app.created_by_controller_did,
+        .map(|app| {
+            let scopes = app.scopes.unwrap_or_else(|| {
+                if app.privilege.is_privileged() {
+                    "transition:generic transition:chat.bsky".to_string()
+                } else {
+                    "transition:generic".to_string()
+                }
+            });
+            CredentialMatch::AppPassword {
+                name: app.name,
+                scopes: Some(scopes),
+                controller_did: app.created_by_controller_did,
+            }
         })
 }
 

crates/tranquil-api/src/server/app_password.rs

@@ -132,7 +132,14 @@ pub async fn create_app_password(
         };
         (scope_result, Some(controller.clone()))
     } else {
-        (input.scopes.clone(), None)
+        let scopes = match input.scopes {
+            Some(ref s) => s.clone(),
+            None => match input.privileged {
+                Some(false) => "transition:generic".to_string(),
+                _ => "transition:generic transition:chat.bsky".to_string(),
+            },
+        };
+        (Some(scopes), None)
     };
 
     let password = generate_app_password();

crates/tranquil-api/src/server/passkey_account.rs

@@ -401,7 +401,7 @@ pub async fn create_passkey_account(
                     refresh_expires_at: refresh_expires,
                     login_type: tranquil_db_traits::LoginType::Modern,
                     mfa_verified: false,
-                    scope: Some("transition:generic".to_string()),
+                    scope: Some("transition:generic transition:chat.bsky".to_string()),
                     controller_did: None,
                     app_password_name: None,
                 };

crates/tranquil-config/src/lib.rs

@@ -108,6 +108,9 @@ pub struct TranquilConfig {
     #[config(nested)]
     pub storage: StorageConfig,
 
+    #[config(nested)]
+    pub tranquil_store: TranquilStoreConfig,
+
     #[config(nested)]
     pub cache: CacheConfig,
 
@@ -143,9 +146,6 @@ pub struct TranquilConfig {
 
     #[config(nested)]
     pub scheduled: ScheduledConfig,
-
-    #[config(nested)]
-    pub tranquil_store: TranquilStoreConfig,
 }
 
 impl TranquilConfig {
@@ -451,6 +451,10 @@ pub struct ServerConfig {
     /// Maximum allowed blob size in bytes (default 10 GiB).
     #[config(env = "MAX_BLOB_SIZE", default = 10_737_418_240u64)]
     pub max_blob_size: u64,
+
+    /// Maximum allowed number of preferences
+    #[config(env = "MAX_PREFERENCES_COUNT", default = 1000)]
+    pub max_preferences_count: usize,
 }
 
 impl ServerConfig {
@@ -656,10 +660,12 @@ pub struct StorageConfig {
     #[config(env = "S3_BUCKET")]
     pub s3_bucket: Option<String>,
 
-    /// Custom S3 endpoint URL (for MinIO, R2, etc.).
+    /// Custom S3 endpoint URL.
     #[config(env = "S3_ENDPOINT")]
     pub s3_endpoint: Option<String>,
 
+    /// Repository backend: `postgres` by default, or `tranquil-store`, our embedded db.
+    /// tranquil-store is EXPERIMENTAL!!!! RISK OF TOTAL DATA LOSS.
     #[config(env = "REPO_BACKEND", default = "postgres")]
     pub repo_backend: String,
 }
@@ -674,7 +680,7 @@ impl StorageConfig {
 
 #[derive(Debug, Config)]
 pub struct CacheConfig {
-    /// Cache backend: `ripple` (default, built-in gossip) or `valkey`.
+    /// Cache backend: `ripple` by default, or `valkey`.
     #[config(env = "CACHE_BACKEND", default = "ripple")]
     pub backend: String,
 
@@ -720,6 +726,10 @@ pub struct FirehoseConfig {
     #[config(env = "FIREHOSE_MAX_LAG", default = 5000)]
     pub max_lag: u64,
 
+    /// Maximum concurrent full-repo exports, eg. getRepo without `since`.
+    #[config(env = "MAX_CONCURRENT_REPO_EXPORTS", default = 4)]
+    pub max_concurrent_repo_exports: usize,
+
     /// List of relay / crawler notification URLs.
     #[config(env = "CRAWLERS", parse_env = split_comma_list)]
     pub crawlers: Option<Vec<String>>,
@@ -770,6 +780,8 @@ pub struct TelegramConfig {
 
 #[derive(Debug, Config)]
 pub struct SignalConfig {
+    /// Protocol state is stored in postgres' signal_* tables.
+    /// Link a device via the admin API before enabling.
     #[config(env = "SIGNAL_ENABLED", default = false)]
     pub enabled: bool,
 }
@@ -1120,27 +1132,26 @@ pub struct ScheduledConfig {
 
     /// Maximum age of events retained in the eventlog before pruning.
     /// Per the atproto firehose spec, the relay backfill window only needs
-    /// to cover "hours or days". Default: 7 days.
+    /// to cover "hours or days".
     #[config(env = "EVENT_RETENTION_MAX_AGE_SECS", default = 604800)]
     pub event_retention_max_age_secs: u64,
 
     /// Interval in seconds between event retention prune passes.
-    /// Set to 0 to disable. Default: hourly.
+    /// Set to 0 to disable.
     #[config(env = "EVENT_RETENTION_INTERVAL_SECS", default = 3600)]
     pub event_retention_interval_secs: u64,
 }
 
 #[derive(Debug, Config)]
 pub struct TranquilStoreConfig {
-    /// Directory for tranquil-store data (metastore, eventlog).
+    /// Directory for tranquil-store data: the metastore, eventlog, and blockstore.
     #[config(
         env = "TRANQUIL_STORE_DATA_DIR",
         default = "/var/lib/tranquil-pds/store"
     )]
     pub data_dir: String,
 
-    /// Fjall block cache size in megabytes. Defaults to 20% of system RAM
-    /// when unset.
+    /// Fjall block cache size in megabytes. Defaults to 20% of system RAM when unset.
     #[config(env = "TRANQUIL_STORE_MEMORY_BUDGET_MB")]
     pub memory_budget_mb: Option<u64>,
 
@@ -1148,9 +1159,9 @@ pub struct TranquilStoreConfig {
     #[config(env = "TRANQUIL_STORE_HANDLER_THREADS")]
     pub handler_threads: Option<usize>,
 
-    /// Maximum total bytes of pending (unsynced) eventlog payloads. Appenders
-    /// block once this budget is exhausted until in-flight events drain via
-    /// fsync. Set to 0 to disable backpressure (unbounded). Default: 1 GiB.
+    /// Maximum total bytes of pending (unsynced) eventlog payloads. Appenders block
+    /// once this budget is exhausted until in-flight events drain via fsync. Set to
+    /// 0 to disable backpressure. Default: 1 GiB.
     #[config(
         env = "TRANQUIL_STORE_EVENTLOG_PENDING_BYTES_BUDGET",
         default = 1_073_741_824

crates/tranquil-lexicon/Cargo.toml

@@ -24,3 +24,4 @@ urlencoding = { workspace = true, optional = true }
 [dev-dependencies]
 wiremock = { workspace = true }
 tokio = { workspace = true }
+futures = { workspace = true }

crates/tranquil-lexicon/src/dynamic.rs

@@ -5,25 +5,60 @@ use std::collections::{HashMap, VecDeque};
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::time::{Duration, Instant};
+use tokio::sync::Notify;
 
 const NEGATIVE_CACHE_TTL: Duration = Duration::from_secs(24 * 60 * 60);
+const POSITIVE_CACHE_TTL: Duration = Duration::from_secs(24 * 60 * 60);
+const REFRESH_FAILURE_BACKOFF: Duration = Duration::from_secs(60);
 const MAX_DYNAMIC_SCHEMAS: usize = 1024;
 
 struct NegativeEntry {
     expires_at: Instant,
 }
 
+struct PositiveEntry {
+    doc: Arc<LexiconDoc>,
+    expires_at: Instant,
+}
+
+pub(crate) enum CacheEntry {
+    Fresh(Arc<LexiconDoc>),
+    Stale(Arc<LexiconDoc>),
+}
+
+impl CacheEntry {
+    #[cfg(test)]
+    fn is_fresh(&self) -> bool {
+        matches!(self, Self::Fresh(_))
+    }
+}
+
 struct SchemaStore {
-    schemas: HashMap<String, Arc<LexiconDoc>>,
+    schemas: HashMap<String, PositiveEntry>,
     insertion_order: VecDeque<String>,
 }
 
 pub struct DynamicRegistry {
     store: RwLock<SchemaStore>,
     negative_cache: RwLock<HashMap<String, NegativeEntry>>,
+    in_flight: RwLock<HashMap<String, Arc<Notify>>>,
     network_disabled: AtomicBool,
 }
 
+struct InFlightGuard<'a> {
+    registry: &'a DynamicRegistry,
+    nsid: String,
+}
+
+impl Drop for InFlightGuard<'_> {
+    fn drop(&mut self) {
+        let notify = self.registry.in_flight.write().remove(&self.nsid);
+        if let Some(n) = notify {
+            n.notify_waiters();
+        }
+    }
+}
+
 impl DynamicRegistry {
     pub fn new() -> Self {
         let network_disabled =
@@ -34,6 +69,7 @@ impl DynamicRegistry {
                 insertion_order: VecDeque::new(),
             }),
             negative_cache: RwLock::new(HashMap::new()),
+            in_flight: RwLock::new(HashMap::new()),
             network_disabled: AtomicBool::new(network_disabled),
         }
     }
@@ -43,8 +79,23 @@ impl DynamicRegistry {
         self.network_disabled.store(disabled, Ordering::Relaxed);
     }
 
-    pub fn get(&self, nsid: &str) -> Option<Arc<LexiconDoc>> {
-        self.store.read().schemas.get(nsid).cloned()
+    pub fn get_cached(&self, nsid: &str) -> Option<Arc<LexiconDoc>> {
+        self.store
+            .read()
+            .schemas
+            .get(nsid)
+            .map(|e| Arc::clone(&e.doc))
+    }
+
+    pub(crate) fn get_entry(&self, nsid: &str) -> Option<CacheEntry> {
+        let now = Instant::now();
+        self.store.read().schemas.get(nsid).map(|e| {
+            if e.expires_at > now {
+                CacheEntry::Fresh(Arc::clone(&e.doc))
+            } else {
+                CacheEntry::Stale(Arc::clone(&e.doc))
+            }
+        })
     }
 
     pub fn is_negative_cached(&self, nsid: &str) -> bool {
@@ -56,7 +107,7 @@ impl DynamicRegistry {
 
     fn insert_negative(&self, nsid: &str) {
         let mut cache = self.negative_cache.write();
-        if cache.len() > MAX_DYNAMIC_SCHEMAS {
+        if cache.len() >= MAX_DYNAMIC_SCHEMAS {
             let now = Instant::now();
             cache.retain(|_, entry| entry.expires_at > now);
         }
@@ -87,29 +138,95 @@ impl DynamicRegistry {
             });
         }
 
-        if store
-            .schemas
-            .insert(nsid.clone(), Arc::clone(&arc))
-            .is_some()
-        {
+        let entry = PositiveEntry {
+            doc: Arc::clone(&arc),
+            expires_at: Instant::now() + POSITIVE_CACHE_TTL,
+        };
+        if store.schemas.insert(nsid.clone(), entry).is_some() {
             store.insertion_order.retain(|k| k != &nsid);
         }
         store.insertion_order.push_back(nsid.clone());
+        drop(store);
 
         self.negative_cache.write().remove(&arc.id);
 
         arc
     }
 
+    fn bump_expiry(&self, nsid: &str, duration: Duration) {
+        let mut store = self.store.write();
+        if let Some(entry) = store.schemas.get_mut(nsid) {
+            entry.expires_at = Instant::now() + duration;
+        }
+    }
+
     pub async fn resolve_and_cache(&self, nsid: &str) -> Result<Arc<LexiconDoc>, ResolveError> {
-        if let Some(doc) = self.get(nsid) {
-            return Ok(doc);
+        self.resolve_and_cache_with(nsid, |n| async move { resolve_lexicon(&n).await })
+            .await
+    }
+
+    async fn resolve_and_cache_with<F, Fut>(
+        &self,
+        nsid: &str,
+        resolver: F,
+    ) -> Result<Arc<LexiconDoc>, ResolveError>
+    where
+        F: FnOnce(String) -> Fut,
+        Fut: std::future::Future<Output = Result<LexiconDoc, ResolveError>>,
+    {
+        match self.get_entry(nsid) {
+            Some(CacheEntry::Fresh(doc)) => Ok(doc),
+            Some(CacheEntry::Stale(stale)) => self.refresh_stale(nsid, stale, resolver).await,
+            None => self.resolve_fresh(nsid, resolver).await,
+        }
+    }
+
+    async fn refresh_stale<F, Fut>(
+        &self,
+        nsid: &str,
+        stale: Arc<LexiconDoc>,
+        resolver: F,
+    ) -> Result<Arc<LexiconDoc>, ResolveError>
+    where
+        F: FnOnce(String) -> Fut,
+        Fut: std::future::Future<Output = Result<LexiconDoc, ResolveError>>,
+    {
+        if self.network_disabled.load(Ordering::Relaxed) {
+            return Ok(stale);
         }
 
+        match self.acquire_leadership(nsid) {
+            Some(_guard) => match resolver(nsid.to_string()).await {
+                Ok(doc) => Ok(self.insert_schema(doc)),
+                Err(e) => {
+                    self.bump_expiry(nsid, REFRESH_FAILURE_BACKOFF);
+                    tracing::warn!(
+                        nsid = nsid,
+                        error = %e,
+                        "lexicon refresh failed, serving stale cached entry"
+                    );
+                    Ok(stale)
+                }
+            },
+            None => {
+                self.wait_for_leader(nsid).await;
+                Ok(self.get_cached(nsid).unwrap_or(stale))
+            }
+        }
+    }
+
+    async fn resolve_fresh<F, Fut>(
+        &self,
+        nsid: &str,
+        resolver: F,
+    ) -> Result<Arc<LexiconDoc>, ResolveError>
+    where
+        F: FnOnce(String) -> Fut,
+        Fut: std::future::Future<Output = Result<LexiconDoc, ResolveError>>,
+    {
         if self.network_disabled.load(Ordering::Relaxed) {
             return Err(ResolveError::NetworkDisabled);
         }
-
         if self.is_negative_cached(nsid) {
             return Err(ResolveError::NegativelyCached {
                 nsid: nsid.to_string(),
@@ -117,19 +234,75 @@ impl DynamicRegistry {
             });
         }
 
-        match resolve_lexicon(nsid).await {
-            Ok(doc) => Ok(self.insert_schema(doc)),
-            Err(e) => {
-                tracing::debug!(nsid = nsid, error = %e, "caching negative resolution result");
-                self.insert_negative(nsid);
-                Err(e)
+        match self.acquire_leadership(nsid) {
+            Some(_guard) => match resolver(nsid.to_string()).await {
+                Ok(doc) => Ok(self.insert_schema(doc)),
+                Err(e) => {
+                    self.insert_negative(nsid);
+                    tracing::debug!(nsid = nsid, error = %e, "caching negative resolution result");
+                    Err(e)
+                }
+            },
+            None => {
+                self.wait_for_leader(nsid).await;
+                match self.get_cached(nsid) {
+                    Some(doc) => Ok(doc),
+                    None if self.is_negative_cached(nsid) => {
+                        Err(ResolveError::NegativelyCached {
+                            nsid: nsid.to_string(),
+                            ttl_secs: NEGATIVE_CACHE_TTL.as_secs(),
+                        })
+                    }
+                    None => Err(ResolveError::LeaderAborted {
+                        nsid: nsid.to_string(),
+                    }),
+                }
             }
         }
     }
 
+    fn acquire_leadership(&self, nsid: &str) -> Option<InFlightGuard<'_>> {
+        let mut map = self.in_flight.write();
+        if map.contains_key(nsid) {
+            None
+        } else {
+            map.insert(nsid.to_string(), Arc::new(Notify::new()));
+            Some(InFlightGuard {
+                registry: self,
+                nsid: nsid.to_string(),
+            })
+        }
+    }
+
+    async fn wait_for_leader(&self, nsid: &str) {
+        let notify = {
+            let map = self.in_flight.read();
+            match map.get(nsid) {
+                Some(n) => Arc::clone(n),
+                None => return,
+            }
+        };
+        let notified = notify.notified();
+        tokio::pin!(notified);
+        notified.as_mut().enable();
+        let still_active = self.in_flight.read().contains_key(nsid);
+        if !still_active {
+            return;
+        }
+        notified.as_mut().await;
+    }
+
     pub fn schema_count(&self) -> usize {
         self.store.read().schemas.len()
     }
+
+    #[cfg(test)]
+    fn expire_now(&self, nsid: &str) {
+        let mut store = self.store.write();
+        if let Some(entry) = store.schemas.get_mut(nsid) {
+            entry.expires_at = Instant::now();
+        }
+    }
 }
 
 impl Default for DynamicRegistry {
@@ -171,7 +344,7 @@ mod tests {
     #[test]
     fn test_empty_lookup() {
         let registry = DynamicRegistry::new();
-        assert!(registry.get("com.example.nonexistent").is_none());
+        assert!(registry.get_cached("com.example.nonexistent").is_none());
         assert_eq!(registry.schema_count(), 0);
     }
 
@@ -188,9 +361,12 @@ mod tests {
         assert_eq!(arc.id, "com.example.test");
         assert_eq!(registry.schema_count(), 1);
 
-        let retrieved = registry.get("com.example.test");
+        let retrieved = registry.get_cached("com.example.test");
         assert!(retrieved.is_some());
         assert_eq!(retrieved.unwrap().id, "com.example.test");
+
+        let entry = registry.get_entry("com.example.test").unwrap();
+        assert!(entry.is_fresh(), "freshly inserted entry must be fresh");
     }
 
     #[test]
@@ -210,6 +386,231 @@ mod tests {
         assert!(!registry.is_negative_cached("com.example.test"));
     }
 
+    #[test]
+    fn test_positive_entry_reports_stale_after_ttl() {
+        let registry = DynamicRegistry::new();
+        let doc = LexiconDoc {
+            lexicon: 1,
+            id: "pet.nel.stale".to_string(),
+            defs: HashMap::new(),
+        };
+        registry.insert_schema(doc);
+
+        assert!(registry.get_entry("pet.nel.stale").unwrap().is_fresh());
+
+        registry.expire_now("pet.nel.stale");
+
+        assert!(
+            !registry.get_entry("pet.nel.stale").unwrap().is_fresh(),
+            "entry past expiry must be reported stale"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_stale_served_on_resolve_failure() {
+        let registry = DynamicRegistry::new();
+        let doc = LexiconDoc {
+            lexicon: 1,
+            id: "pet.nel.flaky".to_string(),
+            defs: HashMap::new(),
+        };
+        registry.insert_schema(doc);
+        registry.expire_now("pet.nel.flaky");
+
+        let result = registry
+            .resolve_and_cache_with("pet.nel.flaky", |n| async move {
+                Err::<LexiconDoc, _>(ResolveError::DnsLookup {
+                    domain: n,
+                    reason: "simulated failure".to_string(),
+                })
+            })
+            .await;
+
+        let served = result.expect("stale entry must be served when refresh fails");
+        assert_eq!(served.id, "pet.nel.flaky");
+        assert!(
+            registry
+                .get_entry("pet.nel.flaky")
+                .unwrap()
+                .is_fresh(),
+            "failed refresh must bump expiry so subsequent lookups skip the resolver"
+        );
+        assert!(
+            !registry.is_negative_cached("pet.nel.flaky"),
+            "stale refresh failure must not poison negative cache"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_fresh_hit_skips_resolver() {
+        let registry = DynamicRegistry::new();
+        let doc = LexiconDoc {
+            lexicon: 1,
+            id: "pet.nel.fresh".to_string(),
+            defs: HashMap::new(),
+        };
+        registry.insert_schema(doc);
+
+        let result = registry
+            .resolve_and_cache_with("pet.nel.fresh", |_| async move {
+                panic!("resolver must not run on fresh hit")
+            })
+            .await;
+
+        assert!(result.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_stale_served_when_network_disabled() {
+        let registry = DynamicRegistry::new();
+        let doc = LexiconDoc {
+            lexicon: 1,
+            id: "pet.nel.offline".to_string(),
+            defs: HashMap::new(),
+        };
+        registry.insert_schema(doc);
+        registry.expire_now("pet.nel.offline");
+        registry.set_network_disabled(true);
+
+        let result = registry
+            .resolve_and_cache_with("pet.nel.offline", |_| async move {
+                panic!("resolver must not run when network disabled")
+            })
+            .await;
+
+        assert!(result.is_ok());
+    }
+
+    #[tokio::test]
+    async fn test_successful_refresh_updates_cached_at() {
+        let registry = DynamicRegistry::new();
+        let doc = LexiconDoc {
+            lexicon: 1,
+            id: "pet.nel.refresh".to_string(),
+            defs: HashMap::new(),
+        };
+        registry.insert_schema(doc);
+        registry.expire_now("pet.nel.refresh");
+
+        assert!(
+            !registry
+                .get_entry("pet.nel.refresh")
+                .unwrap()
+                .is_fresh()
+        );
+
+        let refreshed = registry
+            .resolve_and_cache_with("pet.nel.refresh", |n| async move {
+                Ok(LexiconDoc {
+                    lexicon: 1,
+                    id: n,
+                    defs: HashMap::new(),
+                })
+            })
+            .await
+            .unwrap();
+
+        assert_eq!(refreshed.id, "pet.nel.refresh");
+        assert!(
+            registry
+                .get_entry("pet.nel.refresh")
+                .unwrap()
+                .is_fresh(),
+            "refresh must restore freshness"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_single_flight_dedups_concurrent_resolves() {
+        use std::sync::atomic::AtomicUsize;
+        let registry = Arc::new(DynamicRegistry::new());
+        let calls = Arc::new(AtomicUsize::new(0));
+
+        let tasks: Vec<_> = (0..16)
+            .map(|_| {
+                let registry = Arc::clone(&registry);
+                let calls = Arc::clone(&calls);
+                tokio::spawn(async move {
+                    registry
+                        .resolve_and_cache_with("pet.nel.herd", |n| {
+                            let calls = Arc::clone(&calls);
+                            async move {
+                                calls.fetch_add(1, Ordering::SeqCst);
+                                tokio::time::sleep(Duration::from_millis(50)).await;
+                                Ok(LexiconDoc {
+                                    lexicon: 1,
+                                    id: n,
+                                    defs: HashMap::new(),
+                                })
+                            }
+                        })
+                        .await
+                })
+            })
+            .collect();
+
+        let results = futures_collect(tasks).await;
+        results
+            .iter()
+            .for_each(|r| assert!(r.is_ok(), "all single-flight callers must succeed"));
+        assert_eq!(
+            calls.load(Ordering::SeqCst),
+            1,
+            "single-flight must coalesce concurrent resolves"
+        );
+        assert_eq!(registry.schema_count(), 1);
+    }
+
+    #[tokio::test]
+    async fn test_single_flight_followers_observe_leader_failure() {
+        use std::sync::atomic::AtomicUsize;
+        let registry = Arc::new(DynamicRegistry::new());
+        let calls = Arc::new(AtomicUsize::new(0));
+
+        let tasks: Vec<_> = (0..8)
+            .map(|_| {
+                let registry = Arc::clone(&registry);
+                let calls = Arc::clone(&calls);
+                tokio::spawn(async move {
+                    registry
+                        .resolve_and_cache_with("pet.nel.failHerd", |n| {
+                            let calls = Arc::clone(&calls);
+                            async move {
+                                calls.fetch_add(1, Ordering::SeqCst);
+                                tokio::time::sleep(Duration::from_millis(50)).await;
+                                Err::<LexiconDoc, _>(ResolveError::DnsLookup {
+                                    domain: n,
+                                    reason: "simulated".to_string(),
+                                })
+                            }
+                        })
+                        .await
+                })
+            })
+            .collect();
+
+        let results = futures_collect(tasks).await;
+        results
+            .iter()
+            .for_each(|r| assert!(r.is_err(), "all followers must observe leader failure"));
+        assert_eq!(
+            calls.load(Ordering::SeqCst),
+            1,
+            "single-flight must coalesce failing resolves too"
+        );
+        assert!(registry.is_negative_cached("pet.nel.failHerd"));
+    }
+
+    async fn futures_collect<T>(
+        handles: Vec<tokio::task::JoinHandle<T>>,
+    ) -> Vec<T> {
+        futures::future::join_all(handles)
+            .await
+            .into_iter()
+            .map(|r| r.expect("task panicked"))
+            .collect()
+    }
+
     #[test]
     fn test_eviction_is_fifo() {
         let registry = DynamicRegistry::new();
@@ -217,7 +618,7 @@ mod tests {
         (0..MAX_DYNAMIC_SCHEMAS).for_each(|i| {
             let doc = LexiconDoc {
                 lexicon: 1,
-                id: format!("com.example.schema{}", i),
+                id: format!("pet.nel.schema{}", i),
                 defs: HashMap::new(),
             };
             registry.insert_schema(doc);
@@ -226,23 +627,23 @@ mod tests {
 
         let trigger = LexiconDoc {
             lexicon: 1,
-            id: "com.example.trigger".to_string(),
+            id: "pet.nel.trigger".to_string(),
             defs: HashMap::new(),
         };
         registry.insert_schema(trigger);
 
         assert!(
-            registry.get("com.example.schema0").is_none(),
+            registry.get_cached("pet.nel.schema0").is_none(),
             "oldest entry should be evicted"
         );
         assert!(
-            registry.get("com.example.trigger").is_some(),
+            registry.get_cached("pet.nel.trigger").is_some(),
             "newly inserted entry should exist"
         );
         let evict_count = MAX_DYNAMIC_SCHEMAS / 4;
         assert!(
             registry
-                .get(&format!("com.example.schema{}", evict_count))
+                .get_cached(&format!("pet.nel.schema{}", evict_count))
                 .is_some(),
             "entry after eviction window should survive"
         );
@@ -253,7 +654,7 @@ mod tests {
         let registry = DynamicRegistry::new();
         let doc = LexiconDoc {
             lexicon: 1,
-            id: "com.example.tracked".to_string(),
+            id: "pet.nel.tracked".to_string(),
             defs: HashMap::new(),
         };
         let arc = registry.insert_schema(doc);
@@ -265,7 +666,7 @@ mod tests {
         (0..MAX_DYNAMIC_SCHEMAS).for_each(|i| {
             registry.insert_schema(LexiconDoc {
                 lexicon: 1,
-                id: format!("com.example.filler{}", i),
+                id: format!("pet.nel.filler{}", i),
                 defs: HashMap::new(),
             });
         });

crates/tranquil-lexicon/src/registry.rs

@@ -43,7 +43,7 @@ impl LexiconRegistry {
         self.schemas.get(nsid).cloned().or_else(|| {
             #[cfg(feature = "resolve")]
             {
-                self.dynamic.get(nsid)
+                self.dynamic.get_cached(nsid)
             }
             #[cfg(not(feature = "resolve"))]
             {

crates/tranquil-lexicon/src/resolve.rs

@@ -70,6 +70,8 @@ pub enum ResolveError {
     NegativelyCached { nsid: String, ttl_secs: u64 },
     #[error("network resolution disabled")]
     NetworkDisabled,
+    #[error("leader task for {nsid} aborted before completion")]
+    LeaderAborted { nsid: String },
 }
 
 pub fn nsid_to_authority(nsid: &str) -> Result<String, ResolveError> {

crates/tranquil-oauth-server/src/endpoints/authorize/passkey.rs

@@ -175,10 +175,7 @@ pub async fn passkey_start(
     }
 }
 
-async fn passkey_start_discoverable(
-    state: AppState,
-    request_id: RequestId,
-) -> Response {
+async fn passkey_start_discoverable(state: AppState, request_id: RequestId) -> Response {
     let (rcr, auth_state) = match state.webauthn_config.start_discoverable_authentication() {
         Ok(result) => result,
         Err(e) => {
@@ -570,16 +567,13 @@ pub async fn passkey_finish(
             Err(response) => return response,
         },
         None => {
-            let result = match passkey_finish_discoverable(
-                &state,
-                &credential,
-                &passkey_finish_request_id,
-            )
-            .await
-            {
-                Ok(result) => result,
-                Err(response) => return response,
-            };
+            let result =
+                match passkey_finish_discoverable(&state, &credential, &passkey_finish_request_id)
+                    .await
+                {
+                    Ok(result) => result,
+                    Err(response) => return response,
+                };
             if state
                 .repos
                 .oauth

crates/tranquil-oauth-server/src/sso_endpoints.rs

@@ -1339,7 +1339,7 @@ pub async fn complete_registration(
                 refresh_expires_at: refresh_meta.expires_at,
                 login_type: tranquil_db_traits::LoginType::Modern,
                 mfa_verified: false,
-                scope: Some("transition:generic".to_string()),
+                scope: Some("transition:generic transition:chat.bsky".to_string()),
                 controller_did: None,
                 app_password_name: None,
             };

crates/tranquil-pds/src/comms/service.rs

@@ -3,7 +3,7 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use chrono::Utc;
-use tokio::time::interval;
+
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, info, warn};
 use tranquil_comms::{
@@ -75,17 +75,28 @@ impl CommsService {
             );
         }
         info!(
-            poll_interval_secs = self.poll_interval.as_secs(),
+            poll_interval_ms = self.poll_interval.as_millis() as u64,
             batch_size = self.batch_size,
             channels = ?self.senders.keys().collect::<Vec<_>>(),
             "Starting comms service"
         );
-        let mut ticker = interval(self.poll_interval);
+        let base = self.poll_interval;
+        let max_backoff = Duration::from_secs(30);
+        let mut current_delay = base;
         loop {
             tokio::select! {
-                _ = ticker.tick() => {
-                    if let Err(e) = self.process_batch().await {
-                        error!(error = %e, "Failed to process comms batch");
+                _ = tokio::time::sleep(current_delay) => {
+                    match self.process_batch().await {
+                        Ok(had_work) => {
+                            current_delay = match had_work {
+                                true => base,
+                                false => max_backoff.min(current_delay.saturating_mul(2)),
+                            };
+                        }
+                        Err(e) => {
+                            error!(error = %e, "Failed to process comms batch");
+                            current_delay = max_backoff.min(current_delay.saturating_mul(2));
+                        }
                     }
                 }
                 _ = shutdown.cancelled() => {
@@ -96,14 +107,14 @@ impl CommsService {
         }
     }
 
-    async fn process_batch(&self) -> Result<(), tranquil_db_traits::DbError> {
+    async fn process_batch(&self) -> Result<bool, tranquil_db_traits::DbError> {
         let items = self.fetch_pending().await?;
         if items.is_empty() {
-            return Ok(());
+            return Ok(false);
         }
         debug!(count = items.len(), "Processing comms batch");
         futures::future::join_all(items.into_iter().map(|item| self.process_item(item))).await;
-        Ok(())
+        Ok(true)
     }
 
     async fn fetch_pending(&self) -> Result<Vec<QueuedComms>, tranquil_db_traits::DbError> {

crates/tranquil-pds/src/repo_ops.rs

@@ -13,7 +13,6 @@ use jacquard_repo::mst::util::compute_cid;
 use jacquard_repo::storage::BlockStore;
 use k256::ecdsa::SigningKey;
 use serde_json::{Value, json};
-use std::collections::BTreeSet;
 use std::str::FromStr;
 use std::sync::Arc;
 use tokio::sync::OwnedMutexGuard;
@@ -226,30 +225,6 @@ pub async fn begin_repo_write(
     Ok((ctx, mst))
 }
 
-pub async fn compute_obsolete_cids(
-    original_mst: &Mst<TrackingBlockStore>,
-    new_mst: &Mst<TrackingBlockStore>,
-    original_root_cid: CommitCid,
-) -> Result<Vec<Cid>, jacquard_repo::error::RepoError> {
-    let (old_nodes, new_nodes, old_leaves, new_leaves) = tokio::try_join!(
-        original_mst.collect_node_cids(),
-        new_mst.collect_node_cids(),
-        original_mst.leaves(),
-        new_mst.leaves(),
-    )?;
-    let old_nodes_set: BTreeSet<Cid> = old_nodes.into_iter().collect();
-    let new_nodes_set: BTreeSet<Cid> = new_nodes.into_iter().collect();
-    let old_leaf_set: BTreeSet<Cid> = old_leaves.iter().map(|(_, cid)| *cid).collect();
-    let new_leaf_set: BTreeSet<Cid> = new_leaves.iter().map(|(_, cid)| *cid).collect();
-    let removed_nodes = old_nodes_set.difference(&new_nodes_set).copied();
-    let removed_leaves = old_leaf_set.difference(&new_leaf_set).copied();
-    let obsolete: BTreeSet<Cid> = std::iter::once(original_root_cid.into_cid())
-        .chain(removed_nodes)
-        .chain(removed_leaves)
-        .collect();
-    Ok(obsolete.into_iter().collect())
-}
-
 pub async fn finalize_repo_write(
     state: &AppState,
     ctx: RepoWriteContext,
@@ -266,35 +241,27 @@ pub async fn finalize_repo_write(
     let storage_for_diff = Arc::new(ctx.tracking_store.clone());
     let original_settled = Mst::load(storage_for_diff.clone(), ctx.prev_data_cid, None);
     let new_settled = Mst::load(storage_for_diff, new_mst_root, None);
-    let (obsolete_cids, new_tree_cids) = tokio::try_join!(
-        async {
-            compute_obsolete_cids(
-                &original_settled,
-                &new_settled,
-                CommitCid::from(ctx.current_root_cid),
-            )
-            .await
-            .map_err(|e| {
-                error!("MST diff failed during finalize_repo_write: {}", e);
-                ApiError::InternalError(Some("MST diff failed".into()))
-            })
-        },
-        async {
-            let (nodes, leaves) =
-                tokio::try_join!(new_settled.collect_node_cids(), new_settled.leaves(),).map_err(
-                    |e| {
-                        error!("new tree walk failed: {}", e);
-                        ApiError::InternalError(None)
-                    },
-                )?;
-            Ok::<Vec<Cid>, ApiError>(
-                nodes
-                    .into_iter()
-                    .chain(leaves.iter().map(|(_, cid)| *cid))
-                    .collect(),
-            )
-        },
-    )?;
+
+    let new_tree_cids: Vec<Cid> = block_bytes.keys().copied().collect();
+
+    let obsolete_cids = match original_settled.diff(&new_settled).await {
+        Ok(diff) => {
+            let mut obsolete: Vec<Cid> =
+                Vec::with_capacity(1 + diff.removed_mst_blocks.len() + diff.removed_cids.len());
+            obsolete.push(ctx.current_root_cid);
+            obsolete.extend(diff.removed_mst_blocks);
+            obsolete.extend(diff.removed_cids);
+            obsolete
+        }
+        Err(e) => {
+            error!(
+                "MST diff failed during finalize_repo_write: {e}. \
+                 Proceeding with commit CID only; leaked blocks \
+                 will be reclaimed by reachability GC."
+            );
+            vec![ctx.current_root_cid]
+        }
+    };
 
     let result = commit_and_log(
         state,

crates/tranquil-pds/src/scheduled.rs

@@ -667,6 +667,8 @@ async fn delete_account_data(
     Ok(())
 }
 
+const CAR_BLOCK_BATCH_SIZE: usize = 500;
+
 pub async fn generate_repo_car(
     block_store: &AnyBlockStore,
     head_cid: &Cid,
@@ -683,21 +685,20 @@ pub async fn generate_repo_car(
         })
         .collect();
 
-    let car_bytes = encode_car_header(head_cid).context("Failed to encode CAR header")?;
+    let mut car_bytes = encode_car_header(head_cid).context("Failed to encode CAR header")?;
 
-    let blocks = block_store
-        .get_many(&block_cids)
-        .await
-        .context("Failed to fetch blocks")?;
+    for chunk in block_cids.chunks(CAR_BLOCK_BATCH_SIZE) {
+        let blocks = block_store
+            .get_many(chunk)
+            .await
+            .context("Failed to fetch blocks")?;
 
-    let car_bytes = block_cids
-        .iter()
-        .zip(blocks.iter())
-        .filter_map(|(cid, block_opt)| block_opt.as_ref().map(|block| (cid, block)))
-        .fold(car_bytes, |mut acc, (cid, block)| {
-            acc.extend(encode_car_block(cid, block));
-            acc
-        });
+        chunk
+            .iter()
+            .zip(blocks.iter())
+            .filter_map(|(cid, block_opt)| block_opt.as_ref().map(|block| (cid, block)))
+            .for_each(|(cid, block)| car_bytes.extend(encode_car_block(cid, block)));
+    }
 
     Ok(car_bytes)
 }

crates/tranquil-pds/src/state.rs

@@ -50,6 +50,7 @@ pub struct AppState {
     pub signal_sender: Option<Arc<tranquil_signal::SignalSlot>>,
     pub signal_store_provider: Option<Arc<dyn tranquil_signal::SignalStoreProvider>>,
     pub eventlog_segments_dir: Option<PathBuf>,
+    pub repo_export_semaphore: Arc<tokio::sync::Semaphore>,
 }
 
 #[derive(Debug, Clone, Copy)]
@@ -394,6 +395,9 @@ impl AppState {
             signal_sender: None,
             signal_store_provider,
             eventlog_segments_dir,
+            repo_export_semaphore: Arc::new(tokio::sync::Semaphore::new(
+                cfg.firehose.max_concurrent_repo_exports,
+            )),
         }
     }
 

crates/tranquil-pds/tests/account_lifecycle.rs

@@ -99,9 +99,8 @@ async fn test_check_account_status_returns_correct_block_count() {
         after_delete_blocks
     );
     assert!(
-        after_delete_blocks >= initial_blocks,
-        "Block count after delete should be at least initial count (initial {}, now {})",
-        initial_blocks,
+        after_delete_blocks >= 2,
+        "Block count after delete should have at least commit + MST root (got {})",
         after_delete_blocks
     );
 }

crates/tranquil-pds/tests/gc_compaction_restart.rs

@@ -0,0 +1,169 @@
+mod common;
+
+use chrono::Utc;
+use common::*;
+use reqwest::StatusCode;
+use serde_json::{Value, json};
+
+fn run_compaction(store: &tranquil_store::blockstore::TranquilBlockStore) {
+    let liveness = store.compaction_liveness(0).unwrap();
+    liveness
+        .iter()
+        .filter(|(_, info)| info.total_blocks > 0 && info.ratio() < 0.95)
+        .map(|(&fid, _)| fid)
+        .collect::<Vec<_>>()
+        .into_iter()
+        .for_each(|fid| match store.compact_file(fid, 0) {
+            Ok(_) => {}
+            Err(tranquil_store::blockstore::CompactionError::ActiveFileCannotBeCompacted) => {}
+            Err(e) => eprintln!("compaction: {e}"),
+        });
+}
+
+#[tokio::test]
+async fn mst_blocks_survive_full_store_reopen() {
+    if !is_store_backend() {
+        eprintln!("skipping: only meaningful with tranquil-store backend");
+        return;
+    }
+
+    let client = client();
+    let base = base_url().await;
+    let block_store = get_test_block_store().await;
+
+    let store = block_store
+        .as_tranquil_store()
+        .expect("expected tranquil-store backend");
+
+    let (jwt, did) = create_account_and_login(&client).await;
+
+    let mut posts = Vec::new();
+    for i in 0..30 {
+        let res = client
+            .post(format!("{base}/xrpc/com.atproto.repo.createRecord"))
+            .bearer_auth(&jwt)
+            .json(&json!({
+                "repo": did,
+                "collection": "app.bsky.feed.post",
+                "record": {
+                    "$type": "app.bsky.feed.post",
+                    "text": format!("compaction test post {i}"),
+                    "createdAt": Utc::now().to_rfc3339()
+                }
+            }))
+            .send()
+            .await
+            .unwrap();
+        assert_eq!(res.status(), StatusCode::OK);
+        let body: Value = res.json().await.unwrap();
+        posts.push((
+            body["uri"].as_str().unwrap().to_string(),
+            body["cid"].as_str().unwrap().to_string(),
+        ));
+    }
+
+    for (uri, cid) in &posts[..20] {
+        let res = client
+            .post(format!("{base}/xrpc/com.atproto.repo.createRecord"))
+            .bearer_auth(&jwt)
+            .json(&json!({
+                "repo": did,
+                "collection": "app.bsky.feed.like",
+                "record": {
+                    "$type": "app.bsky.feed.like",
+                    "subject": { "uri": uri, "cid": cid },
+                    "createdAt": Utc::now().to_rfc3339()
+                }
+            }))
+            .send()
+            .await
+            .unwrap();
+        assert_eq!(res.status(), StatusCode::OK, "like failed for {uri}");
+    }
+
+    let data_dir = store.data_dir().to_path_buf();
+    let index_dir = data_dir.parent().unwrap().join("index");
+
+    let store_clone = store.clone();
+    tokio::task::spawn_blocking(move || {
+        (0..40).for_each(|_| run_compaction(&store_clone));
+    })
+    .await
+    .unwrap();
+
+    let repo_root_str: String = get_test_repos()
+        .await
+        .repo
+        .get_repo_root_by_did(&tranquil_types::Did::new(did.clone()).unwrap())
+        .await
+        .expect("db error")
+        .expect("no repo root")
+        .to_string();
+
+    let head_cid = cid::Cid::try_from(repo_root_str.as_str()).expect("invalid cid");
+
+    let car_blocks = tranquil_pds::scheduled::collect_current_repo_blocks(block_store, &head_cid)
+        .await
+        .expect("collect blocks");
+
+    let block_count_before = car_blocks.len();
+
+    let max_file_size = store
+        .list_data_files()
+        .ok()
+        .and_then(|_| Some(4 * 1024 * 1024u64))
+        .unwrap_or(4 * 1024 * 1024);
+
+    let reopened_missing = tokio::task::spawn_blocking(move || {
+        let rt = tokio::runtime::Handle::current();
+        let _guard = rt.enter();
+
+        let config = tranquil_store::blockstore::BlockStoreConfig {
+            data_dir: data_dir.clone(),
+            index_dir,
+            max_file_size,
+            group_commit: tranquil_store::blockstore::GroupCommitConfig::default(),
+            shard_count: 1,
+        };
+        let fresh =
+            tranquil_store::blockstore::TranquilBlockStore::open(config).expect("reopen failed");
+
+        let missing: Vec<String> = car_blocks
+            .iter()
+            .filter_map(|cid_bytes| {
+                if cid_bytes.len() < 36 {
+                    return None;
+                }
+                let mut arr = [0u8; 36];
+                arr.copy_from_slice(&cid_bytes[..36]);
+                match fresh.get_block_sync(&arr) {
+                    Ok(Some(_)) => None,
+                    Ok(None) => Some(format!(
+                        "missing {}",
+                        cid::Cid::try_from(cid_bytes.as_slice())
+                            .map(|c| c.to_string())
+                            .unwrap_or_else(|_| hex::encode(cid_bytes))
+                    )),
+                    Err(e) => Some(format!("error: {e}")),
+                }
+            })
+            .collect();
+
+        drop(fresh);
+        missing
+    })
+    .await
+    .unwrap();
+
+    assert!(
+        reopened_missing.is_empty(),
+        "{} of {block_count_before} blocks missing after blockstore reopen:\n{}",
+        reopened_missing.len(),
+        reopened_missing
+            .iter()
+            .take(20)
+            .map(|s| s.as_str())
+            .collect::<Vec<_>>()
+            .join("\n"),
+    );
+}

crates/tranquil-pds/tests/lifecycle_session.rs

@@ -597,3 +597,157 @@ async fn test_request_account_delete() {
         "Token should not be expired"
     );
 }
+
+async fn create_app_password_session(
+    client: &reqwest::Client,
+    did: &str,
+    main_jwt: &str,
+    name: &str,
+    body: Value,
+) -> (String, Value) {
+    let base = base_url().await;
+    let create_res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.server.createAppPassword",
+            base
+        ))
+        .bearer_auth(main_jwt)
+        .json(&body)
+        .send()
+        .await
+        .expect("Failed to create app password");
+    assert_eq!(create_res.status(), StatusCode::OK);
+    let app_pass: Value = create_res.json().await.unwrap();
+    let password = app_pass["password"].as_str().unwrap().to_string();
+    let scopes_response = app_pass.clone();
+    let login_res = client
+        .post(format!("{}/xrpc/com.atproto.server.createSession", base))
+        .json(&json!({ "identifier": did, "password": password }))
+        .send()
+        .await
+        .expect("Failed to login with app password");
+    assert_eq!(
+        login_res.status(),
+        StatusCode::OK,
+        "App password login for '{}' failed",
+        name
+    );
+    let session: Value = login_res.json().await.unwrap();
+    let jwt = session["accessJwt"].as_str().unwrap().to_string();
+    (jwt, scopes_response)
+}
+
+async fn try_chat_service_auth(client: &reqwest::Client, jwt: &str) -> StatusCode {
+    let base = base_url().await;
+    let res = client
+        .get(format!("{}/xrpc/com.atproto.server.getServiceAuth", base))
+        .bearer_auth(jwt)
+        .query(&[
+            ("aud", "did:web:api.bsky.app"),
+            ("lxm", "chat.bsky.convo.listConvos"),
+        ])
+        .send()
+        .await
+        .expect("Failed to call getServiceAuth");
+    res.status()
+}
+
+#[tokio::test]
+async fn test_app_password_non_privileged_blocks_chat() {
+    let client = client();
+    let (did, jwt) = setup_new_user("appscope-nonchat").await;
+    let (app_jwt, create_body) = create_app_password_session(
+        &client,
+        &did,
+        &jwt,
+        "non-privileged",
+        json!({ "name": "NoChatApp", "privileged": false }),
+    )
+    .await;
+    assert_eq!(
+        create_body["scopes"].as_str().unwrap(),
+        "transition:generic",
+        "Non-privileged app password should not have chat scope"
+    );
+    let status = try_chat_service_auth(&client, &app_jwt).await;
+    assert_eq!(
+        status,
+        StatusCode::FORBIDDEN,
+        "Non-privileged app password must not access chat methods"
+    );
+}
+
+#[tokio::test]
+async fn test_app_password_privileged_allows_chat() {
+    let client = client();
+    let (did, jwt) = setup_new_user("appscope-chat").await;
+    let (app_jwt, create_body) = create_app_password_session(
+        &client,
+        &did,
+        &jwt,
+        "privileged",
+        json!({ "name": "ChatApp", "privileged": true }),
+    )
+    .await;
+    assert_eq!(
+        create_body["scopes"].as_str().unwrap(),
+        "transition:generic transition:chat.bsky",
+        "Privileged app password should have chat scope"
+    );
+    let status = try_chat_service_auth(&client, &app_jwt).await;
+    assert_eq!(
+        status,
+        StatusCode::OK,
+        "Privileged app password should access chat methods"
+    );
+}
+
+#[tokio::test]
+async fn test_app_password_no_privileged_field_allows_chat() {
+    let client = client();
+    let (did, jwt) = setup_new_user("appscope-full").await;
+    let (app_jwt, create_body) = create_app_password_session(
+        &client,
+        &did,
+        &jwt,
+        "full-access",
+        json!({ "name": "FullApp" }),
+    )
+    .await;
+    assert_eq!(
+        create_body["scopes"].as_str().unwrap(),
+        "transition:generic transition:chat.bsky",
+        "App password without privileged field should default to full access"
+    );
+    let status = try_chat_service_auth(&client, &app_jwt).await;
+    assert_eq!(
+        status,
+        StatusCode::OK,
+        "Full-access app password should access chat methods"
+    );
+}
+
+#[tokio::test]
+async fn test_app_password_explicit_scopes_respected() {
+    let client = client();
+    let (did, jwt) = setup_new_user("appscope-explicit").await;
+    let (app_jwt, create_body) = create_app_password_session(
+        &client,
+        &did,
+        &jwt,
+        "explicit-scopes",
+        json!({ "name": "ScopedApp", "scopes": "transition:generic" }),
+    )
+    .await;
+    assert_eq!(
+        create_body["scopes"].as_str().unwrap(),
+        "transition:generic",
+        "Explicit scopes should be stored as-is"
+    );
+    let status = try_chat_service_auth(&client, &app_jwt).await;
+    assert_eq!(
+        status,
+        StatusCode::FORBIDDEN,
+        "App password with only transition:generic should not access chat"
+    );
+}

crates/tranquil-pds/tests/mst_diff_equivalence.rs

@@ -0,0 +1,583 @@
+use std::collections::BTreeSet;
+use std::sync::Arc;
+
+use cid::Cid;
+use jacquard_repo::mst::Mst;
+use jacquard_repo::storage::MemoryBlockStore;
+
+fn test_cid(n: u32) -> Cid {
+    let data = n.to_be_bytes();
+    let mut buf = [0u8; 32];
+    buf[..4].copy_from_slice(&data);
+    buf[4] = (n >> 8) as u8 ^ 0xAB;
+    buf[5] = (n & 0xFF) as u8 ^ 0xCD;
+    let mh = multihash::Multihash::wrap(0x12, &buf).unwrap();
+    Cid::new_v1(0x71, mh)
+}
+
+async fn compute_obsolete_full_walk<
+    S: jacquard_repo::storage::BlockStore + Sync + Send + 'static,
+>(
+    old: &Mst<S>,
+    new: &Mst<S>,
+) -> BTreeSet<Cid> {
+    let old_nodes = old.collect_node_cids().await.unwrap();
+    let new_nodes = new.collect_node_cids().await.unwrap();
+    let old_leaves = old.leaves().await.unwrap();
+    let new_leaves = new.leaves().await.unwrap();
+    let old_nodes_set: BTreeSet<Cid> = old_nodes.into_iter().collect();
+    let new_nodes_set: BTreeSet<Cid> = new_nodes.into_iter().collect();
+    let old_leaf_set: BTreeSet<Cid> = old_leaves.iter().map(|(_, cid)| *cid).collect();
+    let new_leaf_set: BTreeSet<Cid> = new_leaves.iter().map(|(_, cid)| *cid).collect();
+    old_nodes_set
+        .difference(&new_nodes_set)
+        .copied()
+        .chain(old_leaf_set.difference(&new_leaf_set).copied())
+        .collect()
+}
+
+fn compute_obsolete_from_diff(diff: &jacquard_repo::mst::diff::MstDiff) -> BTreeSet<Cid> {
+    diff.removed_mst_blocks
+        .iter()
+        .copied()
+        .chain(diff.removed_cids.iter().copied())
+        .collect()
+}
+
+async fn assert_equivalence(
+    old_records: &[(String, u32)],
+    new_records: &[(String, u32)],
+    scenario: &str,
+) {
+    let storage = Arc::new(MemoryBlockStore::new());
+
+    let mut old_tree = Mst::new(storage.clone());
+    for (key, val) in old_records {
+        old_tree = old_tree.add(key, test_cid(*val)).await.unwrap();
+    }
+    let old_root = old_tree.persist().await.unwrap();
+
+    let mut new_tree = Mst::new(storage.clone());
+    for (key, val) in new_records {
+        new_tree = new_tree.add(key, test_cid(*val)).await.unwrap();
+    }
+    let new_root = new_tree.persist().await.unwrap();
+
+    let old_settled = Mst::load(storage.clone(), old_root, None);
+    let new_settled = Mst::load(storage.clone(), new_root, None);
+
+    let full_walk_obsolete = compute_obsolete_full_walk(&old_settled, &new_settled).await;
+
+    let old_for_diff = Mst::load(storage.clone(), old_root, None);
+    let new_for_diff = Mst::load(storage, new_root, None);
+    let diff = old_for_diff.diff(&new_for_diff).await.unwrap();
+    let diff_obsolete = compute_obsolete_from_diff(&diff);
+
+    assert_eq!(
+        full_walk_obsolete,
+        diff_obsolete,
+        "MISMATCH in scenario: {scenario}\n  full_walk count: {}\n  diff count: {}\n  in full_walk but not diff: {:?}\n  in diff but not full_walk: {:?}",
+        full_walk_obsolete.len(),
+        diff_obsolete.len(),
+        full_walk_obsolete
+            .difference(&diff_obsolete)
+            .collect::<Vec<_>>(),
+        diff_obsolete
+            .difference(&full_walk_obsolete)
+            .collect::<Vec<_>>(),
+    );
+}
+
+fn make_key(collection: &str, i: u32) -> String {
+    format!("{collection}/{i:06}")
+}
+
+fn generate_records(collection: &str, range: std::ops::Range<u32>) -> Vec<(String, u32)> {
+    range.map(|i| (make_key(collection, i), i)).collect()
+}
+
+fn generate_multi_collection_records(
+    collections: &[&str],
+    per_collection: u32,
+) -> Vec<(String, u32)> {
+    collections
+        .iter()
+        .enumerate()
+        .flat_map(|(ci, coll)| {
+            let base = ci as u32 * per_collection;
+            (0..per_collection).map(move |i| (make_key(coll, i), base + i))
+        })
+        .collect()
+}
+
+fn apply_scattered_updates(
+    records: &[(String, u32)],
+    stride: usize,
+    cid_offset: u32,
+) -> Vec<(String, u32)> {
+    records
+        .iter()
+        .enumerate()
+        .map(|(idx, (key, val))| {
+            if idx % stride == 0 {
+                (key.clone(), val + cid_offset)
+            } else {
+                (key.clone(), *val)
+            }
+        })
+        .collect()
+}
+
+fn remove_every_nth(records: &[(String, u32)], n: usize) -> Vec<(String, u32)> {
+    records
+        .iter()
+        .enumerate()
+        .filter(|(idx, _)| idx % n != 0)
+        .map(|(_, r)| r.clone())
+        .collect()
+}
+
+fn remove_range(records: &[(String, u32)], start: usize, count: usize) -> Vec<(String, u32)> {
+    records
+        .iter()
+        .enumerate()
+        .filter(|(idx, _)| *idx < start || *idx >= start + count)
+        .map(|(_, r)| r.clone())
+        .collect()
+}
+
+fn keep_only_collection(records: &[(String, u32)], collection: &str) -> Vec<(String, u32)> {
+    records
+        .iter()
+        .filter(|(key, _)| key.starts_with(collection))
+        .cloned()
+        .collect()
+}
+
+fn append_records(
+    base: &[(String, u32)],
+    collection: &str,
+    range: std::ops::Range<u32>,
+    cid_base: u32,
+) -> Vec<(String, u32)> {
+    let mut result = base.to_vec();
+    result.extend(range.map(|i| (make_key(collection, i), cid_base + i)));
+    result.sort_by(|(a, _), (b, _)| a.cmp(b));
+    result
+}
+
+#[tokio::test]
+async fn massive_tree_single_create() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let new_rec = append_records(&old, "app.bsky.feed.post", 2000..2001, 2000);
+    assert_equivalence(&old, &new_rec, "2000 records + 1 create").await;
+}
+
+#[tokio::test]
+async fn massive_tree_single_delete() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let new_rec = remove_range(&old, 1000, 1);
+    assert_equivalence(&old, &new_rec, "2000 records - 1 delete from middle").await;
+}
+
+#[tokio::test]
+async fn massive_tree_single_update() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let new_rec: Vec<_> = old
+        .iter()
+        .map(|(k, v)| {
+            if k == "app.bsky.feed.post/001000" {
+                (k.clone(), v + 50000)
+            } else {
+                (k.clone(), *v)
+            }
+        })
+        .collect();
+    assert_equivalence(&old, &new_rec, "2000 records - 1 update in middle").await;
+}
+
+#[tokio::test]
+async fn massive_tree_scattered_updates_every_3rd() {
+    let old = generate_records("app.bsky.feed.post", 0..1500);
+    let new_rec = apply_scattered_updates(&old, 3, 10000);
+    assert_equivalence(&old, &new_rec, "1500 records - update every 3rd").await;
+}
+
+#[tokio::test]
+async fn massive_tree_scattered_updates_every_7th() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let new_rec = apply_scattered_updates(&old, 7, 20000);
+    assert_equivalence(&old, &new_rec, "2000 records - update every 7th").await;
+}
+
+#[tokio::test]
+async fn massive_tree_delete_every_2nd() {
+    let old = generate_records("app.bsky.feed.post", 0..1000);
+    let new_rec = remove_every_nth(&old, 2);
+    assert_equivalence(&old, &new_rec, "1000 records - delete every 2nd").await;
+}
+
+#[tokio::test]
+async fn massive_tree_delete_every_5th() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let new_rec = remove_every_nth(&old, 5);
+    assert_equivalence(&old, &new_rec, "2000 records - delete every 5th").await;
+}
+
+#[tokio::test]
+async fn massive_tree_delete_first_half() {
+    let old = generate_records("app.bsky.feed.post", 0..1500);
+    let new_rec = remove_range(&old, 0, 750);
+    assert_equivalence(&old, &new_rec, "1500 records - delete first 750").await;
+}
+
+#[tokio::test]
+async fn massive_tree_delete_last_half() {
+    let old = generate_records("app.bsky.feed.post", 0..1500);
+    let new_rec = remove_range(&old, 750, 750);
+    assert_equivalence(&old, &new_rec, "1500 records - delete last 750").await;
+}
+
+#[tokio::test]
+async fn massive_tree_delete_middle_chunk() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let new_rec = remove_range(&old, 800, 400);
+    assert_equivalence(&old, &new_rec, "2000 records - delete 400 from middle").await;
+}
+
+#[tokio::test]
+async fn empty_to_massive() {
+    let new_rec = generate_records("app.bsky.feed.post", 0..1500);
+    assert_equivalence(&[], &new_rec, "empty to 1500 records").await;
+}
+
+#[tokio::test]
+async fn massive_to_empty() {
+    let old = generate_records("app.bsky.feed.post", 0..1500);
+    assert_equivalence(&old, &[], "1500 records to empty").await;
+}
+
+#[tokio::test]
+async fn massive_complete_replacement() {
+    let old = generate_records("app.bsky.feed.post", 0..1000);
+    let new_rec = generate_records("app.bsky.feed.post", 1000..2000);
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "1000 records fully replaced with 1000 different",
+    )
+    .await;
+}
+
+#[tokio::test]
+async fn massive_no_change() {
+    let records = generate_records("app.bsky.feed.post", 0..1500);
+    assert_equivalence(&records, &records, "1500 records unchanged").await;
+}
+
+#[tokio::test]
+async fn multi_collection_5_collections_500_each() {
+    let collections = [
+        "app.bsky.feed.like",
+        "app.bsky.feed.post",
+        "app.bsky.feed.repost",
+        "app.bsky.graph.follow",
+        "app.bsky.graph.block",
+    ];
+    let old = generate_multi_collection_records(&collections, 500);
+    let new_rec = apply_scattered_updates(&old, 4, 30000);
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "5 collections x 500 records - update every 4th",
+    )
+    .await;
+}
+
+#[tokio::test]
+async fn multi_collection_wipe_one_collection() {
+    let collections = [
+        "app.bsky.feed.like",
+        "app.bsky.feed.post",
+        "app.bsky.feed.repost",
+        "app.bsky.graph.follow",
+    ];
+    let old = generate_multi_collection_records(&collections, 400);
+
+    let new_rec: Vec<_> = old
+        .iter()
+        .filter(|(key, _)| !key.starts_with("app.bsky.feed.repost"))
+        .cloned()
+        .collect();
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "4 collections x 400 - wipe repost collection",
+    )
+    .await;
+}
+
+#[tokio::test]
+async fn multi_collection_keep_only_one() {
+    let collections = [
+        "app.bsky.feed.like",
+        "app.bsky.feed.post",
+        "app.bsky.feed.repost",
+        "app.bsky.graph.follow",
+        "app.bsky.graph.block",
+    ];
+    let old = generate_multi_collection_records(&collections, 300);
+    let new_rec = keep_only_collection(&old, "app.bsky.feed.post");
+    assert_equivalence(&old, &new_rec, "5 collections x 300 - keep only posts").await;
+}
+
+#[tokio::test]
+async fn multi_collection_add_new_collection() {
+    let old_collections = ["app.bsky.feed.like", "app.bsky.feed.post"];
+    let old = generate_multi_collection_records(&old_collections, 500);
+    let new_rec = append_records(&old, "app.bsky.graph.follow", 0..500, 40000);
+    assert_equivalence(&old, &new_rec, "2 collections x 500 + add 500 follows").await;
+}
+
+#[tokio::test]
+async fn mixed_ops_massive_tree() {
+    let collections = [
+        "app.bsky.feed.like",
+        "app.bsky.feed.post",
+        "app.bsky.feed.repost",
+        "app.bsky.graph.follow",
+    ];
+    let old = generate_multi_collection_records(&collections, 400);
+
+    let mut new_rec: Vec<_> = old
+        .iter()
+        .filter(|(key, _)| !key.starts_with("app.bsky.feed.repost"))
+        .enumerate()
+        .map(|(idx, (key, val))| {
+            if key.starts_with("app.bsky.feed.like") && idx % 3 == 0 {
+                (key.clone(), val + 50000)
+            } else {
+                (key.clone(), *val)
+            }
+        })
+        .collect();
+
+    new_rec.extend((0..200u32).map(|i| (make_key("app.bsky.graph.block", i), 60000 + i)));
+    new_rec.sort_by(|(a, _), (b, _)| a.cmp(b));
+
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "4 collections x 400: wipe reposts, update every 3rd like, add 200 blocks",
+    )
+    .await;
+}
+
+#[tokio::test]
+async fn grow_tree_by_double() {
+    let old = generate_records("app.bsky.feed.post", 0..1000);
+    let new_rec = generate_records("app.bsky.feed.post", 0..2000);
+    assert_equivalence(&old, &new_rec, "grow from 1000 to 2000").await;
+}
+
+#[tokio::test]
+async fn shrink_tree_by_half() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let new_rec = generate_records("app.bsky.feed.post", 0..1000);
+    assert_equivalence(&old, &new_rec, "shrink from 2000 to 1000").await;
+}
+
+#[tokio::test]
+async fn interleaved_keys_disjoint_ranges() {
+    let old: Vec<_> = (0..1000u32)
+        .map(|i| (make_key("app.bsky.feed.post", i * 2), i))
+        .collect();
+    let new_rec: Vec<_> = (0..1000u32)
+        .map(|i| (make_key("app.bsky.feed.post", i * 2 + 1), i + 10000))
+        .collect();
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "1000 even-keyed records replaced by 1000 odd-keyed",
+    )
+    .await;
+}
+
+#[tokio::test]
+async fn sparse_keys_wide_gaps() {
+    let old: Vec<_> = (0..500u32)
+        .map(|i| (make_key("app.bsky.feed.post", i * 100), i))
+        .collect();
+    let new_rec: Vec<_> = (0..500u32)
+        .map(|i| {
+            if i % 10 == 0 {
+                (make_key("app.bsky.feed.post", i * 100), i + 70000)
+            } else {
+                (make_key("app.bsky.feed.post", i * 100), i)
+            }
+        })
+        .collect();
+    assert_equivalence(&old, &new_rec, "500 sparse keys - update every 10th").await;
+}
+
+#[tokio::test]
+async fn many_collections_few_records_each() {
+    let collections: Vec<String> = (0..50u32)
+        .map(|i| format!("com.example.lexicon{i:02}.record"))
+        .collect();
+    let old: Vec<_> = collections
+        .iter()
+        .enumerate()
+        .flat_map(|(ci, coll)| {
+            let base = ci as u32 * 20;
+            (0..20u32).map(move |i| (make_key(coll, i), base + i))
+        })
+        .collect();
+
+    let new_rec: Vec<_> = old
+        .iter()
+        .enumerate()
+        .filter_map(|(idx, (key, val))| {
+            if idx % 15 == 0 {
+                None
+            } else if idx % 7 == 0 {
+                Some((key.clone(), val + 80000))
+            } else {
+                Some((key.clone(), *val))
+            }
+        })
+        .collect();
+
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "50 collections x 20 records - delete every 15th, update every 7th",
+    )
+    .await;
+}
+
+#[tokio::test]
+async fn update_all_records() {
+    let old = generate_records("app.bsky.feed.post", 0..1000);
+    let new_rec: Vec<_> = old
+        .iter()
+        .map(|(key, val)| (key.clone(), val + 90000))
+        .collect();
+    assert_equivalence(&old, &new_rec, "1000 records - update every single one").await;
+}
+
+#[tokio::test]
+async fn delete_all_but_one() {
+    let old = generate_records("app.bsky.feed.post", 0..1500);
+    let new_rec = vec![old[750].clone()];
+    assert_equivalence(&old, &new_rec, "1500 records - delete all but middle one").await;
+}
+
+#[tokio::test]
+async fn one_to_massive() {
+    let old = vec![(make_key("app.bsky.feed.post", 500), 500u32)];
+    let new_rec = generate_records("app.bsky.feed.post", 0..1500);
+    assert_equivalence(&old, &new_rec, "1 record to 1500 records").await;
+}
+
+#[tokio::test]
+async fn delete_head_and_tail() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let new_rec: Vec<_> = old[200..1800].to_vec();
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "2000 records - delete first 200 and last 200",
+    )
+    .await;
+}
+
+#[tokio::test]
+async fn keep_head_and_tail_only() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let mut new_rec: Vec<_> = old[..100].to_vec();
+    new_rec.extend_from_slice(&old[1900..]);
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "2000 records - keep only first 100 and last 100",
+    )
+    .await;
+}
+
+#[tokio::test]
+async fn massive_tree_update_first_and_last() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let mut new_rec = old.clone();
+    new_rec[0].1 += 99000;
+    new_rec[1999].1 += 99000;
+    assert_equivalence(&old, &new_rec, "2000 records - update only first and last").await;
+}
+
+#[tokio::test]
+async fn overlapping_collection_swap() {
+    let old_collections = [
+        "app.bsky.feed.like",
+        "app.bsky.feed.post",
+        "app.bsky.feed.repost",
+    ];
+    let old = generate_multi_collection_records(&old_collections, 500);
+
+    let mut new_rec: Vec<_> = old
+        .iter()
+        .filter(|(key, _)| key.starts_with("app.bsky.feed.post"))
+        .cloned()
+        .collect();
+    new_rec.extend((0..500u32).map(|i| (make_key("app.bsky.graph.follow", i), 70000 + i)));
+    new_rec.extend((0..500u32).map(|i| (make_key("app.bsky.graph.block", i), 71000 + i)));
+    new_rec.sort_by(|(a, _), (b, _)| a.cmp(b));
+
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "swap 2 of 3 collections, keep 1 (posts), 500 each",
+    )
+    .await;
+}
+
+#[tokio::test]
+async fn swiss_cheese_deletions() {
+    let old = generate_records("app.bsky.feed.post", 0..1500);
+    let new_rec: Vec<_> = old
+        .iter()
+        .enumerate()
+        .filter(|(idx, _)| {
+            let bucket = idx / 50;
+            bucket % 3 != 0
+        })
+        .map(|(_, r)| r.clone())
+        .collect();
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "1500 records - delete every 3rd chunk of 50",
+    )
+    .await;
+}
+
+#[tokio::test]
+async fn mixed_ops_with_key_density_change() {
+    let old: Vec<_> = (0..1000u32)
+        .map(|i| (make_key("app.bsky.feed.post", i * 3), i))
+        .collect();
+
+    let mut new_rec: Vec<_> = old
+        .iter()
+        .filter(|(_, val)| val % 4 != 0)
+        .cloned()
+        .collect();
+    new_rec.extend((0..500u32).map(|i| (make_key("app.bsky.feed.post", i * 3 + 1), i + 100000)));
+    new_rec.sort_by(|(a, _), (b, _)| a.cmp(b));
+
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "1000 sparse records: delete every 4th, insert 500 in gaps",
+    )
+    .await;
+}

crates/tranquil-pds/tests/sync_resource_limits.rs

@@ -0,0 +1,147 @@
+mod common;
+mod helpers;
+use common::*;
+use helpers::*;
+use reqwest::StatusCode;
+use std::sync::Once;
+
+static SET_SEMAPHORE: Once = Once::new();
+
+fn ensure_low_semaphore() {
+    SET_SEMAPHORE.call_once(|| unsafe {
+        std::env::set_var("MAX_CONCURRENT_REPO_EXPORTS", "1");
+    });
+}
+
+#[tokio::test]
+async fn test_get_repo_succeeds_with_many_records() {
+    ensure_low_semaphore();
+    let client = client();
+    let (did, jwt) = setup_new_user("sync-batched-car").await;
+
+    let create_futures = (0..20).map(|i| {
+        let client = &client;
+        let did = &did;
+        let jwt = &jwt;
+        async move {
+            create_post(client, did, jwt, &format!("Batch test post {}", i)).await;
+        }
+    });
+    futures::future::join_all(create_futures).await;
+
+    let res = client
+        .get(format!(
+            "{}/xrpc/com.atproto.sync.getRepo",
+            base_url().await
+        ))
+        .query(&[("did", did.as_str())])
+        .send()
+        .await
+        .expect("Failed to send getRepo request");
+
+    assert_eq!(res.status(), StatusCode::OK);
+    assert_eq!(
+        res.headers()
+            .get("content-type")
+            .and_then(|h| h.to_str().ok()),
+        Some("application/vnd.ipld.car")
+    );
+    let car_bytes = res.bytes().await.expect("Failed to read response body");
+    assert!(
+        car_bytes.len() > 200,
+        "CAR with 20 records should have substantial data, got {} bytes",
+        car_bytes.len()
+    );
+}
+
+#[tokio::test]
+async fn test_get_repo_semaphore_rejects_excess_concurrency() {
+    ensure_low_semaphore();
+    let client = client();
+    let (did, jwt) = setup_new_user("sync-semaphore").await;
+
+    for i in 0..50 {
+        create_post(&client, &did, &jwt, &format!("Padding post {}", i)).await;
+    }
+
+    let base = base_url().await;
+    let concurrent_requests = 10;
+
+    let request_futures = (0..concurrent_requests).map(|_| {
+        let client = client.clone();
+        let did = did.clone();
+        async move {
+            client
+                .get(format!("{}/xrpc/com.atproto.sync.getRepo", base))
+                .query(&[("did", did.as_str())])
+                .send()
+                .await
+                .expect("Failed to send request")
+                .status()
+        }
+    });
+
+    let statuses: Vec<StatusCode> = futures::future::join_all(request_futures).await;
+    let ok_count = statuses.iter().filter(|s| **s == StatusCode::OK).count();
+    let rejected_count = statuses
+        .iter()
+        .filter(|s| **s == StatusCode::SERVICE_UNAVAILABLE)
+        .count();
+
+    assert!(ok_count >= 1, "at least one request should succeed");
+    assert!(
+        rejected_count > 0,
+        "semaphore=1 with {} concurrent requests, expected some 503 rejections",
+        concurrent_requests
+    );
+    assert!(
+        ok_count + rejected_count == statuses.len(),
+        "expected only 200 or 503 responses: {:?}",
+        statuses
+    );
+}
+
+#[tokio::test]
+async fn test_get_repo_since_not_affected_by_semaphore() {
+    ensure_low_semaphore();
+    let client = client();
+    let (did, jwt) = setup_new_user("sync-since-no-sem").await;
+    create_post(&client, &did, &jwt, "First post").await;
+
+    let latest_res = client
+        .get(format!(
+            "{}/xrpc/com.atproto.sync.getLatestCommit",
+            base_url().await
+        ))
+        .query(&[("did", did.as_str())])
+        .send()
+        .await
+        .expect("Failed to get latest commit");
+    let body: serde_json::Value = latest_res.json().await.unwrap();
+    let rev = body["rev"].as_str().unwrap();
+
+    create_post(&client, &did, &jwt, "Second post").await;
+
+    let base = base_url().await;
+    let request_futures = (0..10).map(|_| {
+        let client = client.clone();
+        let did = did.clone();
+        let rev = rev.to_string();
+        async move {
+            client
+                .get(format!("{}/xrpc/com.atproto.sync.getRepo", base))
+                .query(&[("did", did.as_str()), ("since", rev.as_str())])
+                .send()
+                .await
+                .expect("Failed to send request")
+                .status()
+        }
+    });
+
+    let statuses: Vec<StatusCode> = futures::future::join_all(request_futures).await;
+    assert!(
+        statuses.iter().all(|s| *s == StatusCode::OK),
+        "getRepo with since should bypass semaphore, got: {:?}",
+        statuses
+    );
+}

crates/tranquil-scopes/src/permissions.rs

@@ -157,11 +157,22 @@ impl ScopePermissions {
     }
 
     pub fn assert_rpc(&self, aud: &str, lxm: &str) -> Result<(), ScopeError> {
-        if self.has_transition_generic {
-            return Ok(());
+        if lxm.starts_with("chat.bsky.") {
+            if self.has_transition_chat {
+                return Ok(());
+            }
+            if self.has_transition_generic && !self.has_transition_chat {
+                return Err(ScopeError::InsufficientScope {
+                    required: "transition:chat.bsky".to_string(),
+                    message: format!(
+                        "Chat access requires transition:chat.bsky scope to call {}",
+                        lxm
+                    ),
+                });
+            }
         }
 
-        if lxm.starts_with("chat.bsky.") && self.has_transition_chat {
+        if self.has_transition_generic {
             return Ok(());
         }
 
@@ -347,6 +358,23 @@ mod tests {
         assert!(perms.allows_blob("image/png"));
     }
 
+    #[test]
+    fn test_transition_generic_without_chat_blocks_chat() {
+        let perms = ScopePermissions::from_scope_string(Some("transition:generic"));
+        assert!(perms.allows_rpc("did:web:api.bsky.app", "app.bsky.feed.getTimeline"));
+        assert!(!perms.allows_rpc("did:web:api.bsky.app", "chat.bsky.convo.listConvos"));
+        assert!(!perms.allows_rpc("did:web:api.bsky.app", "chat.bsky.convo.getMessages"));
+    }
+
+    #[test]
+    fn test_transition_generic_with_chat_allows_chat() {
+        let perms =
+            ScopePermissions::from_scope_string(Some("transition:generic transition:chat.bsky"));
+        assert!(perms.allows_rpc("did:web:api.bsky.app", "app.bsky.feed.getTimeline"));
+        assert!(perms.allows_rpc("did:web:api.bsky.app", "chat.bsky.convo.listConvos"));
+        assert!(perms.allows_rpc("did:web:api.bsky.app", "chat.bsky.convo.getMessages"));
+    }
+
     #[test]
     fn test_transition_chat_only_allows_chat() {
         let perms = ScopePermissions::from_scope_string(Some("transition:chat.bsky"));

crates/tranquil-store/Cargo.toml

@@ -14,7 +14,7 @@ parking_lot = { workspace = true }
 fjall = "3"
 lsm-tree = "3"
 flume = "0.11"
-tokio = { workspace = true, features = ["sync", "rt"] }
+tokio = { workspace = true, features = ["sync", "rt", "time"] }
 bytes = "1"
 memmap2 = "0.9"
 tracing = { workspace = true }
@@ -34,9 +34,10 @@ dashmap = "6"
 rayon = "1"
 smallvec = "1"
 uuid = { workspace = true }
+tempfile = { version = "3", optional = true }
 
 [features]
-test-harness = []
+test-harness = ["dep:tempfile"]
 
 [dev-dependencies]
 tranquil-store = { path = ".", features = ["test-harness"] }
@@ -52,6 +53,7 @@ k256 = { workspace = true }
 rand = { workspace = true }
 tikv-jemallocator = "0.6"
 tracing-subscriber = { workspace = true, features = ["env-filter"] }
+libc = "0.2"
 
 [[bench]]
 name = "blockstore"

crates/tranquil-store/benches/recovery.rs

@@ -100,9 +100,7 @@ impl<'a> DirectSeeder<'a> {
         }
 
         let loc = self.data_writer.append_block(cid, data).unwrap();
-        self.hint_writer
-            .append_hint(cid, loc.file_id, loc.offset, loc.length)
-            .unwrap();
+        self.hint_writer.append_hint(cid, &loc).unwrap();
         self.blocks_in_file += 1;
 
         if self.blocks_in_file.is_multiple_of(10_000) {

crates/tranquil-store/src/blockstore/compaction.rs

@@ -93,8 +93,7 @@ pub(super) fn compact_on_writer_thread<S: StorageIO>(
             Err(e)
         }
         Ok((new_size, live_count, dead_count)) => {
-            let positions = hint_positions.snapshot();
-            if let Err(e) = index.write_checkpoint(epoch.current(), &positions) {
+            if let Err(e) = index.write_checkpoint(epoch.current(), hint_positions) {
                 tracing::warn!(error = %e, "pre-delete checkpoint failed during compaction");
             }
 
@@ -162,13 +161,7 @@ fn stream_compact<S: StorageIO>(
             } => match index.get(&cid_bytes) {
                 Some(e) if e.location.file_id == source_file_id && !e.refcount.is_zero() => {
                     let loc = writer.append_block(&cid_bytes, &data)?;
-                    hint_writer.append_relocate(
-                        &cid_bytes,
-                        loc.file_id,
-                        loc.offset,
-                        loc.length,
-                        e.refcount.raw(),
-                    )?;
+                    hint_writer.append_relocate(&cid_bytes, &loc, e.refcount.raw())?;
                     relocations.push((cid_bytes, loc));
                     live_count = live_count.saturating_add(1);
                 }
@@ -188,13 +181,7 @@ fn stream_compact<S: StorageIO>(
                         }
                         false => {
                             let loc = writer.append_block(&cid_bytes, &data)?;
-                            hint_writer.append_relocate(
-                                &cid_bytes,
-                                loc.file_id,
-                                loc.offset,
-                                loc.length,
-                                e.refcount.raw(),
-                            )?;
+                            hint_writer.append_relocate(&cid_bytes, &loc, e.refcount.raw())?;
                             relocations.push((cid_bytes, loc));
                             live_count = live_count.saturating_add(1);
                         }

crates/tranquil-store/src/blockstore/group_commit.rs

@@ -813,8 +813,7 @@ fn maybe_checkpoint(
     if !elapsed && !threshold {
         return;
     }
-    let positions = hint_positions.snapshot();
-    match index.write_checkpoint(epoch.current(), &positions) {
+    match index.write_checkpoint(epoch.current(), hint_positions) {
         Ok(()) => {
             *last_checkpoint = std::time::Instant::now();
             *writes_since_checkpoint = 0;
@@ -831,8 +830,7 @@ fn shutdown_checkpoint(
     epoch: &EpochCounter,
     hint_positions: &ShardHintPositions,
 ) {
-    let positions = hint_positions.snapshot();
-    match index.write_checkpoint(epoch.current(), &positions) {
+    match index.write_checkpoint(epoch.current(), hint_positions) {
         Ok(()) => tracing::debug!("shutdown checkpoint written"),
         Err(e) => tracing::warn!(error = %e, "shutdown checkpoint failed"),
     }
@@ -924,8 +922,6 @@ fn commit_loop<S: StorageIO>(
             if let Ok((ref dedup, _)) = result {
                 writes_since_checkpoint =
                     writes_since_checkpoint.saturating_add(dedup.len() as u64);
-                ctx.hint_positions
-                    .update(ctx.shard_id, state.file_id, state.hint_position);
             }
 
             dispatch_responses(drain.entries, result.map(|(dedup, _proof)| dedup));
@@ -1024,8 +1020,6 @@ fn drain_and_process_remaining<S: StorageIO>(
 
         if let Ok((ref _dedup, ref proof)) = result {
             run_post_sync_hook(post_sync_hook, proof);
-            ctx.hint_positions
-                .update(ctx.shard_id, state.file_id, state.hint_position);
         }
 
         dispatch_responses(entries, result.map(|(dedup, _proof)| dedup));
@@ -1102,6 +1096,7 @@ fn process_batch<S: StorageIO>(
             let location = match dedup.get(cid_bytes) {
                 Some(&loc) => {
                     dedup_hits = dedup_hits.saturating_add(1);
+                    hint_writer.append_hint(cid_bytes, &loc)?;
                     loc
                 }
                 None => {
@@ -1136,7 +1131,7 @@ fn process_batch<S: StorageIO>(
                     }
 
                     let loc = data_writer.append_block(cid_bytes, data)?;
-                    hint_writer.append_hint(cid_bytes, loc.file_id, loc.offset, loc.length)?;
+                    hint_writer.append_hint(cid_bytes, &loc)?;
 
                     block_bytes = block_bytes.saturating_add(data.len() as u64);
                     block_count = block_count.saturating_add(1);
@@ -1194,7 +1189,19 @@ fn process_batch<S: StorageIO>(
     };
     let t = std::time::Instant::now();
     index
-        .batch_put(&index_entries, &all_decrements, cursor, current_epoch, now)
+        .batch_put_and_advance_position(
+            &index_entries,
+            &all_decrements,
+            cursor,
+            current_epoch,
+            now,
+            super::hash_index::PositionUpdate {
+                hint_positions: &ctx.hint_positions,
+                shard_id: ctx.shard_id,
+                file_id: state.file_id,
+                offset: state.hint_position,
+            },
+        )
         .map_err(CommitError::from)?;
     let index_nanos = t.elapsed().as_nanos() as u64;
 

crates/tranquil-store/src/blockstore/hash_index.rs

@@ -5,11 +5,19 @@ use std::path::{Path, PathBuf};
 use parking_lot::RwLock;
 
 use super::data_file::CID_SIZE;
+use super::group_commit::ShardHintPositions;
 use super::types::{
     BlockLength, BlockLocation, BlockOffset, CidBytes, CollectionResult, CommitEpoch, DataFileId,
-    HintOffset, IndexEntry, LivenessInfo, RefCount, WallClockMs, WriteCursor,
+    HintOffset, IndexEntry, LivenessInfo, RefCount, ShardId, WallClockMs, WriteCursor,
 };
 
+pub struct PositionUpdate<'a> {
+    pub hint_positions: &'a ShardHintPositions,
+    pub shard_id: ShardId,
+    pub file_id: DataFileId,
+    pub offset: HintOffset,
+}
+
 const EMPTY_CID: [u8; CID_SIZE] = [0u8; CID_SIZE];
 
 fn is_empty(cid: &[u8; CID_SIZE]) -> bool {
@@ -207,6 +215,10 @@ impl HashTable {
         self.get(cid).is_some()
     }
 
+    pub fn contains_live(&self, cid: &[u8; CID_SIZE]) -> bool {
+        self.get(cid).is_some_and(|s| !s.refcount.is_zero())
+    }
+
     pub fn insert(&mut self, new_slot: Slot) -> Result<Option<Slot>, CapacityExhausted> {
         if is_empty(&new_slot.cid) {
             tracing::error!("attempted to insert all-zero CID into hash table");
@@ -1187,7 +1199,16 @@ impl BlockIndex {
     }
 
     pub fn has(&self, cid: &[u8; CID_SIZE]) -> bool {
-        self.table.read().contains(cid)
+        self.table.read().contains_live(cid)
+    }
+
+    pub fn live_entries_snapshot(&self) -> Vec<([u8; CID_SIZE], RefCount)> {
+        self.table
+            .read()
+            .iter()
+            .filter(|s| !s.refcount.is_zero())
+            .map(|s| (s.cid, s.refcount))
+            .collect()
     }
 
     pub fn batch_put(
@@ -1197,6 +1218,37 @@ impl BlockIndex {
         cursor: WriteCursor,
         epoch: CommitEpoch,
         now: WallClockMs,
+    ) -> Result<(), BlockIndexError> {
+        self.batch_put_inner(entries, decrements, cursor, epoch, now, None)
+    }
+
+    pub fn batch_put_and_advance_position(
+        &self,
+        entries: &[([u8; CID_SIZE], BlockLocation)],
+        decrements: &[[u8; CID_SIZE]],
+        cursor: WriteCursor,
+        epoch: CommitEpoch,
+        now: WallClockMs,
+        position_update: PositionUpdate<'_>,
+    ) -> Result<(), BlockIndexError> {
+        self.batch_put_inner(
+            entries,
+            decrements,
+            cursor,
+            epoch,
+            now,
+            Some(position_update),
+        )
+    }
+
+    fn batch_put_inner(
+        &self,
+        entries: &[([u8; CID_SIZE], BlockLocation)],
+        decrements: &[[u8; CID_SIZE]],
+        cursor: WriteCursor,
+        epoch: CommitEpoch,
+        now: WallClockMs,
+        position_update: Option<PositionUpdate<'_>>,
     ) -> Result<(), BlockIndexError> {
         let mut table = self.table.write();
 
@@ -1217,6 +1269,12 @@ impl BlockIndex {
         });
 
         table.set_write_cursor(cursor);
+
+        if let Some(pos) = position_update {
+            pos.hint_positions
+                .update(pos.shard_id, pos.file_id, pos.offset);
+        }
+
         Ok(())
     }
 
@@ -1418,6 +1476,17 @@ impl BlockIndex {
     }
 
     pub fn write_checkpoint(
+        &self,
+        epoch: CommitEpoch,
+        hint_positions: &ShardHintPositions,
+    ) -> io::Result<()> {
+        let _guard = self.checkpoint_lock.lock();
+        let table = self.table.read();
+        let positions = hint_positions.snapshot();
+        write_checkpoint_ab(&table, &self.index_dir, epoch, &positions)
+    }
+
+    pub fn write_checkpoint_with_positions(
         &self,
         epoch: CommitEpoch,
         positions: &CheckpointPositions,

crates/tranquil-store/src/blockstore/hint.rs

@@ -55,22 +55,24 @@ fn write_hint_record<S: StorageIO>(
     io.write_all_at(fd, write_offset.raw(), record)
 }
 
+fn encode_location_fields(record: &mut [u8; HINT_RECORD_SIZE], loc: &BlockLocation) {
+    record[FIELD_A_OFFSET..FIELD_A_OFFSET + 4].copy_from_slice(&loc.file_id.raw().to_le_bytes());
+    record[FIELD_A_OFFSET + 4..FIELD_A_OFFSET + 8].copy_from_slice(&loc.length.raw().to_le_bytes());
+    record[FIELD_B_OFFSET..FIELD_B_OFFSET + 8].copy_from_slice(&loc.offset.raw().to_le_bytes());
+}
+
 pub(crate) fn encode_hint_record<S: StorageIO>(
     io: &S,
     fd: FileId,
     write_offset: HintOffset,
     cid_bytes: &[u8; CID_SIZE],
-    file_id: DataFileId,
-    block_offset: BlockOffset,
-    length: BlockLength,
+    loc: &BlockLocation,
 ) -> io::Result<()> {
     let mut record = [0u8; HINT_RECORD_SIZE];
     record[TYPE_OFFSET] = RECORD_TYPE_PUT;
     record[VERSION_OFFSET] = HINT_FORMAT_VERSION;
     record[CID_OFFSET..CID_OFFSET + CID_SIZE].copy_from_slice(cid_bytes);
-    record[FIELD_A_OFFSET..FIELD_A_OFFSET + 4].copy_from_slice(&file_id.raw().to_le_bytes());
-    record[FIELD_A_OFFSET + 4..FIELD_A_OFFSET + 8].copy_from_slice(&length.raw().to_le_bytes());
-    record[FIELD_B_OFFSET..FIELD_B_OFFSET + 8].copy_from_slice(&block_offset.raw().to_le_bytes());
+    encode_location_fields(&mut record, loc);
 
     let checksum = hint_checksum(&record[..HINT_PAYLOAD_SIZE]);
     record[CHECKSUM_OFFSET..].copy_from_slice(&checksum.to_le_bytes());
@@ -85,9 +87,7 @@ pub(crate) fn encode_relocate_record<S: StorageIO>(
     fd: FileId,
     write_offset: HintOffset,
     cid_bytes: &[u8; CID_SIZE],
-    file_id: DataFileId,
-    block_offset: BlockOffset,
-    length: BlockLength,
+    loc: &BlockLocation,
     refcount: u32,
 ) -> io::Result<()> {
     let mut record = [0u8; HINT_RECORD_SIZE];
@@ -96,9 +96,7 @@ pub(crate) fn encode_relocate_record<S: StorageIO>(
     let rc16 = u16::try_from(refcount).unwrap_or(u16::MAX);
     record[REFCOUNT_OFFSET..REFCOUNT_OFFSET + 2].copy_from_slice(&rc16.to_le_bytes());
     record[CID_OFFSET..CID_OFFSET + CID_SIZE].copy_from_slice(cid_bytes);
-    record[FIELD_A_OFFSET..FIELD_A_OFFSET + 4].copy_from_slice(&file_id.raw().to_le_bytes());
-    record[FIELD_A_OFFSET + 4..FIELD_A_OFFSET + 8].copy_from_slice(&length.raw().to_le_bytes());
-    record[FIELD_B_OFFSET..FIELD_B_OFFSET + 8].copy_from_slice(&block_offset.raw().to_le_bytes());
+    encode_location_fields(&mut record, loc);
 
     let checksum = hint_checksum(&record[..HINT_PAYLOAD_SIZE]);
     record[CHECKSUM_OFFSET..].copy_from_slice(&checksum.to_le_bytes());
@@ -323,19 +321,9 @@ impl<'a, S: StorageIO> HintFileWriter<'a, S> {
     pub fn append_hint(
         &mut self,
         cid_bytes: &[u8; CID_SIZE],
-        file_id: DataFileId,
-        offset: BlockOffset,
-        length: BlockLength,
+        loc: &BlockLocation,
     ) -> io::Result<()> {
-        encode_hint_record(
-            self.io,
-            self.fd,
-            self.position,
-            cid_bytes,
-            file_id,
-            offset,
-            length,
-        )?;
+        encode_hint_record(self.io, self.fd, self.position, cid_bytes, loc)?;
         self.position = self.position.advance(HINT_RECORD_SIZE as u64);
         Ok(())
     }
@@ -354,21 +342,10 @@ impl<'a, S: StorageIO> HintFileWriter<'a, S> {
     pub fn append_relocate(
         &mut self,
         cid_bytes: &[u8; CID_SIZE],
-        file_id: DataFileId,
-        offset: BlockOffset,
-        length: BlockLength,
+        loc: &BlockLocation,
         refcount: u32,
     ) -> io::Result<()> {
-        encode_relocate_record(
-            self.io,
-            self.fd,
-            self.position,
-            cid_bytes,
-            file_id,
-            offset,
-            length,
-            refcount,
-        )?;
+        encode_relocate_record(self.io, self.fd, self.position, cid_bytes, loc, refcount)?;
         self.position = self.position.advance(HINT_RECORD_SIZE as u64);
         Ok(())
     }
@@ -862,7 +839,12 @@ mod tests {
         let offset = BlockOffset::new(1024);
         let length = BlockLength::new(256);
 
-        encode_hint_record(&sim, fd, HintOffset::new(0), &cid, file_id, offset, length).unwrap();
+        let loc = BlockLocation {
+            file_id,
+            offset,
+            length,
+        };
+        encode_hint_record(&sim, fd, HintOffset::new(0), &cid, &loc).unwrap();
 
         let file_size = sim.file_size(fd).unwrap();
         let record = decode_hint_record(&sim, fd, HintOffset::new(0), file_size)
@@ -920,16 +902,12 @@ mod tests {
         (0u8..5).for_each(|i| {
             let cid = test_cid(i);
             let write_offset = HintOffset::new(i as u64 * HINT_RECORD_SIZE as u64);
-            encode_hint_record(
-                &sim,
-                fd,
-                write_offset,
-                &cid,
-                DataFileId::new(i as u32),
-                BlockOffset::new(i as u64 * 100),
-                BlockLength::new(50 + i as u32),
-            )
-            .unwrap();
+            let loc = BlockLocation {
+                file_id: DataFileId::new(i as u32),
+                offset: BlockOffset::new(i as u64 * 100),
+                length: BlockLength::new(50 + i as u32),
+            };
+            encode_hint_record(&sim, fd, write_offset, &cid, &loc).unwrap();
         });
 
         let file_size = sim.file_size(fd).unwrap();
@@ -971,16 +949,12 @@ mod tests {
     fn detects_corrupted_hint() {
         let (sim, fd) = setup();
         let cid = test_cid(1);
-        encode_hint_record(
-            &sim,
-            fd,
-            HintOffset::new(0),
-            &cid,
-            DataFileId::new(0),
-            BlockOffset::new(0),
-            BlockLength::new(100),
-        )
-        .unwrap();
+        let loc = BlockLocation {
+            file_id: DataFileId::new(0),
+            offset: BlockOffset::new(0),
+            length: BlockLength::new(100),
+        };
+        encode_hint_record(&sim, fd, HintOffset::new(0), &cid, &loc).unwrap();
 
         sim.write_all_at(fd, 10, &[0xFF]).unwrap();
 
@@ -1006,16 +980,12 @@ mod tests {
     fn oversized_length_treated_as_corrupted() {
         let (sim, fd) = setup();
         let cid = test_cid(1);
-        encode_hint_record(
-            &sim,
-            fd,
-            HintOffset::new(0),
-            &cid,
-            DataFileId::new(0),
-            BlockOffset::new(0),
-            BlockLength::new(100),
-        )
-        .unwrap();
+        let loc = BlockLocation {
+            file_id: DataFileId::new(0),
+            offset: BlockOffset::new(0),
+            length: BlockLength::new(100),
+        };
+        encode_hint_record(&sim, fd, HintOffset::new(0), &cid, &loc).unwrap();
 
         let length_offset = FIELD_A_OFFSET as u64 + 4;
         let oversized = (MAX_BLOCK_SIZE + 1).to_le_bytes();
@@ -1040,14 +1010,12 @@ mod tests {
         let mut writer = HintFileWriter::new(&sim, fd);
 
         (0u8..5).for_each(|i| {
-            writer
-                .append_hint(
-                    &test_cid(i),
-                    DataFileId::new(0),
-                    BlockOffset::new(i as u64 * 100),
-                    BlockLength::new(50 + i as u32),
-                )
-                .unwrap();
+            let loc = BlockLocation {
+                file_id: DataFileId::new(0),
+                offset: BlockOffset::new(i as u64 * 100),
+                length: BlockLength::new(50 + i as u32),
+            };
+            writer.append_hint(&test_cid(i), &loc).unwrap();
         });
 
         assert_eq!(
@@ -1074,25 +1042,21 @@ mod tests {
     fn hint_writer_resume_continues_at_position() {
         let (sim, fd) = setup();
         let mut writer = HintFileWriter::new(&sim, fd);
-        writer
-            .append_hint(
-                &test_cid(0),
-                DataFileId::new(0),
-                BlockOffset::new(0),
-                BlockLength::new(100),
-            )
-            .unwrap();
+        let loc0 = BlockLocation {
+            file_id: DataFileId::new(0),
+            offset: BlockOffset::new(0),
+            length: BlockLength::new(100),
+        };
+        writer.append_hint(&test_cid(0), &loc0).unwrap();
 
         let pos = writer.position();
         let mut writer2 = HintFileWriter::resume(&sim, fd, pos);
-        writer2
-            .append_hint(
-                &test_cid(1),
-                DataFileId::new(0),
-                BlockOffset::new(100),
-                BlockLength::new(200),
-            )
-            .unwrap();
+        let loc1 = BlockLocation {
+            file_id: DataFileId::new(0),
+            offset: BlockOffset::new(100),
+            length: BlockLength::new(200),
+        };
+        writer2.append_hint(&test_cid(1), &loc1).unwrap();
 
         let reader = HintFileReader::open(&sim, fd).unwrap();
         let valid_count = reader
@@ -1115,14 +1079,12 @@ mod tests {
     fn hint_reader_stops_on_truncated() {
         let (sim, fd) = setup();
         let mut writer = HintFileWriter::new(&sim, fd);
-        writer
-            .append_hint(
-                &test_cid(0),
-                DataFileId::new(0),
-                BlockOffset::new(0),
-                BlockLength::new(100),
-            )
-            .unwrap();
+        let loc = BlockLocation {
+            file_id: DataFileId::new(0),
+            offset: BlockOffset::new(0),
+            length: BlockLength::new(100),
+        };
+        writer.append_hint(&test_cid(0), &loc).unwrap();
 
         sim.write_all_at(fd, writer.position().raw(), &[0u8; HINT_RECORD_SIZE - 1])
             .unwrap();
@@ -1140,14 +1102,12 @@ mod tests {
         let mut writer = HintFileWriter::new(&sim, fd);
 
         (0u8..3).for_each(|i| {
-            writer
-                .append_hint(
-                    &test_cid(i),
-                    DataFileId::new(0),
-                    BlockOffset::new(i as u64 * 100),
-                    BlockLength::new(50),
-                )
-                .unwrap();
+            let loc = BlockLocation {
+                file_id: DataFileId::new(0),
+                offset: BlockOffset::new(i as u64 * 100),
+                length: BlockLength::new(50),
+            };
+            writer.append_hint(&test_cid(i), &loc).unwrap();
         });
 
         sim.write_all_at(fd, HINT_RECORD_SIZE as u64 + 5, &[0xFF])

crates/tranquil-store/src/eventlog/reader.rs

@@ -552,7 +552,7 @@ fn decode_mmap_event(
             segment = %segment_id,
             offset = raw,
             file_size,
-            "decode offset past file size (corrupt index?)"
+            "decode offset past file size, index likely corrupt"
         );
         return Ok(MmapDecodeResult::Corrupted);
     }

crates/tranquil-store/src/eventlog/writer.rs

@@ -305,7 +305,7 @@ impl<S: StorageIO> EventLogWriter<S> {
 
         match self.build_sidecar_for_segment(old_id) {
             Ok(()) => {}
-            Err(e) => warn!(segment = %old_id, error = %e, "sidecar build failed (non-fatal)"),
+            Err(e) => warn!(segment = %old_id, error = %e, "non-fatal sidecar build failure"),
         }
 
         let (new_id, new_fd) = self.manager.prepare_rotation(old_id)?;

crates/tranquil-store/src/gauntlet/farm.rs

@@ -0,0 +1,41 @@
+use std::cell::RefCell;
+
+use rayon::prelude::*;
+use tokio::runtime::Runtime;
+
+use super::op::Seed;
+use super::runner::{Gauntlet, GauntletConfig, GauntletReport};
+
+thread_local! {
+    static RUNTIME: RefCell<Option<Runtime>> = const { RefCell::new(None) };
+}
+
+fn with_runtime<R>(f: impl FnOnce(&Runtime) -> R) -> R {
+    RUNTIME.with(|cell| {
+        let mut slot = cell.borrow_mut();
+        if slot.is_none() {
+            *slot = Some(
+                tokio::runtime::Builder::new_current_thread()
+                    .enable_all()
+                    .build()
+                    .expect("build rt"),
+            );
+        }
+        f(slot.as_ref().expect("runtime present"))
+    })
+}
+
+pub fn run_many<F>(make_config: F, seeds: impl IntoIterator<Item = Seed>) -> Vec<GauntletReport>
+where
+    F: Fn(Seed) -> GauntletConfig + Sync + Send,
+{
+    let seeds: Vec<Seed> = seeds.into_iter().collect();
+    seeds
+        .into_par_iter()
+        .map(|s| {
+            let cfg = make_config(s);
+            let gauntlet = Gauntlet::new(cfg).expect("build gauntlet");
+            with_runtime(|rt| rt.block_on(gauntlet.run()))
+        })
+        .collect()
+}

crates/tranquil-store/src/gauntlet/invariants.rs

@@ -0,0 +1,271 @@
+use std::collections::{HashMap, HashSet};
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use cid::Cid;
+use jacquard_repo::mst::Mst;
+
+use super::oracle::{Oracle, hex_short, try_cid_to_fixed};
+use crate::blockstore::{CidBytes, TranquilBlockStore};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct InvariantSet(u32);
+
+impl InvariantSet {
+    pub const EMPTY: Self = Self(0);
+    pub const REFCOUNT_CONSERVATION: Self = Self(1 << 0);
+    pub const REACHABILITY: Self = Self(1 << 1);
+    pub const ACKED_WRITE_PERSISTENCE: Self = Self(1 << 2);
+    pub const READ_AFTER_WRITE: Self = Self(1 << 3);
+    pub const RESTART_IDEMPOTENT: Self = Self(1 << 4);
+
+    const ALL_KNOWN: u32 = Self::REFCOUNT_CONSERVATION.0
+        | Self::REACHABILITY.0
+        | Self::ACKED_WRITE_PERSISTENCE.0
+        | Self::READ_AFTER_WRITE.0
+        | Self::RESTART_IDEMPOTENT.0;
+
+    pub const fn contains(self, other: Self) -> bool {
+        (self.0 & other.0) == other.0
+    }
+
+    pub const fn union(self, other: Self) -> Self {
+        Self(self.0 | other.0)
+    }
+
+    pub const fn without(self, other: Self) -> Self {
+        Self(self.0 & !other.0)
+    }
+
+    pub const fn unknown_bits(self) -> u32 {
+        self.0 & !Self::ALL_KNOWN
+    }
+}
+
+impl std::ops::BitOr for InvariantSet {
+    type Output = Self;
+    fn bitor(self, rhs: Self) -> Self {
+        self.union(rhs)
+    }
+}
+
+#[derive(Debug)]
+pub struct InvariantViolation {
+    pub invariant: &'static str,
+    pub detail: String,
+}
+
+pub struct InvariantCtx<'a> {
+    pub store: &'a Arc<TranquilBlockStore>,
+    pub oracle: &'a Oracle,
+    pub root: Option<Cid>,
+}
+
+#[async_trait]
+pub trait Invariant: Send + Sync {
+    fn name(&self) -> &'static str;
+    async fn check(&self, ctx: &InvariantCtx<'_>) -> Result<(), InvariantViolation>;
+}
+
+pub struct RefcountConservation;
+
+#[async_trait]
+impl Invariant for RefcountConservation {
+    fn name(&self) -> &'static str {
+        "RefcountConservation"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_>) -> Result<(), InvariantViolation> {
+        let live: Vec<(String, CidBytes)> = ctx.oracle.live_cids_labeled();
+        let live_set: HashSet<CidBytes> = live.iter().map(|(_, c)| *c).collect();
+        let index: HashMap<CidBytes, u32> = ctx
+            .store
+            .block_index()
+            .live_entries_snapshot()
+            .into_iter()
+            .map(|(c, r)| (c, r.raw()))
+            .collect();
+
+        let forward: Vec<String> = live
+            .iter()
+            .filter_map(|(label, cid)| match index.get(cid) {
+                Some(&r) if r >= 1 => None,
+                Some(&r) => Some(format!("{label}: refcount {r}")),
+                None => Some(format!("{label}: missing from index")),
+            })
+            .collect();
+
+        let inverse: Vec<String> = index
+            .iter()
+            .filter(|(cid, _)| !live_set.contains(*cid))
+            .map(|(cid, r)| format!("orphan cid {} refcount {}", hex_short(cid), r))
+            .collect();
+
+        let violations: Vec<String> = forward.into_iter().chain(inverse).collect();
+        if violations.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "RefcountConservation",
+                detail: violations.join("; "),
+            })
+        }
+    }
+}
+
+pub struct Reachability;
+
+#[async_trait]
+impl Invariant for Reachability {
+    fn name(&self) -> &'static str {
+        "Reachability"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_>) -> Result<(), InvariantViolation> {
+        let violations: Vec<String> = ctx
+            .oracle
+            .live_cids_labeled()
+            .into_iter()
+            .filter_map(|(label, fixed)| match ctx.store.get_block_sync(&fixed) {
+                Ok(Some(_)) => None,
+                Ok(None) => Some(format!("{label}: missing")),
+                Err(e) => Some(format!("{label}: read error {e}")),
+            })
+            .collect();
+
+        if violations.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "Reachability",
+                detail: violations.join("; "),
+            })
+        }
+    }
+}
+
+pub struct AckedWritePersistence;
+
+#[async_trait]
+impl Invariant for AckedWritePersistence {
+    fn name(&self) -> &'static str {
+        "AckedWritePersistence"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_>) -> Result<(), InvariantViolation> {
+        let Some(root) = ctx.root else {
+            if ctx.oracle.live_count() == 0 {
+                return Ok(());
+            }
+            return Err(InvariantViolation {
+                invariant: "AckedWritePersistence",
+                detail: format!(
+                    "oracle has {} live records but reopened store has no root",
+                    ctx.oracle.live_count()
+                ),
+            });
+        };
+        let mst = Mst::load(ctx.store.clone(), root, None);
+        let keys: Vec<String> = ctx
+            .oracle
+            .live_records()
+            .map(|(c, r, _)| format!("{}/{}", c.0, r.0))
+            .collect();
+
+        let mut missing: Vec<String> = Vec::new();
+        for key in &keys {
+            match mst.get(key).await {
+                Ok(Some(_)) => {}
+                Ok(None) => missing.push(format!("{key}: missing after reopen")),
+                Err(e) => missing.push(format!("{key}: mst.get error after reopen: {e}")),
+            }
+        }
+
+        if missing.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "AckedWritePersistence",
+                detail: missing.join("; "),
+            })
+        }
+    }
+}
+
+pub struct ReadAfterWrite;
+
+#[async_trait]
+impl Invariant for ReadAfterWrite {
+    fn name(&self) -> &'static str {
+        "ReadAfterWrite"
+    }
+
+    async fn check(&self, ctx: &InvariantCtx<'_>) -> Result<(), InvariantViolation> {
+        let Some(root) = ctx.root else {
+            return Ok(());
+        };
+        let mst = Mst::load(ctx.store.clone(), root, None);
+
+        let entries: Vec<(String, CidBytes)> = ctx
+            .oracle
+            .live_records()
+            .map(|(c, r, v)| (format!("{}/{}", c.0, r.0), *v))
+            .collect();
+
+        let mut violations: Vec<String> = Vec::new();
+        for (key, expected) in &entries {
+            match mst.get(key).await {
+                Ok(Some(cid)) => match try_cid_to_fixed(&cid) {
+                    Ok(actual) if actual == *expected => match ctx.store.get_block_sync(&actual) {
+                        Ok(Some(_)) => {}
+                        Ok(None) => violations.push(format!("{key}: block missing for cid")),
+                        Err(e) => violations.push(format!("{key}: block read error {e}")),
+                    },
+                    Ok(actual) => violations.push(format!(
+                        "{key}: MST cid {} != oracle cid {}",
+                        hex_short(&actual),
+                        hex_short(expected),
+                    )),
+                    Err(e) => {
+                        violations.push(format!("{key}: unexpected CID format from MST: {e}"))
+                    }
+                },
+                Ok(None) => violations.push(format!("{key}: MST returned None")),
+                Err(e) => violations.push(format!("{key}: mst.get error {e}")),
+            }
+        }
+
+        if violations.is_empty() {
+            Ok(())
+        } else {
+            Err(InvariantViolation {
+                invariant: "ReadAfterWrite",
+                detail: violations.join("; "),
+            })
+        }
+    }
+}
+
+pub fn invariants_for(set: InvariantSet) -> Vec<Box<dyn Invariant>> {
+    let unknown = set.unknown_bits();
+    assert!(
+        unknown == 0,
+        "invariants_for: unknown InvariantSet bits 0x{unknown:x}; all bits must map to an impl"
+    );
+    let candidates: Vec<(InvariantSet, Box<dyn Invariant>)> = vec![
+        (
+            InvariantSet::REFCOUNT_CONSERVATION,
+            Box::new(RefcountConservation),
+        ),
+        (InvariantSet::REACHABILITY, Box::new(Reachability)),
+        (
+            InvariantSet::ACKED_WRITE_PERSISTENCE,
+            Box::new(AckedWritePersistence),
+        ),
+        (InvariantSet::READ_AFTER_WRITE, Box::new(ReadAfterWrite)),
+    ];
+    candidates
+        .into_iter()
+        .filter_map(|(flag, inv)| set.contains(flag).then_some(inv))
+        .collect()
+}

crates/tranquil-store/src/gauntlet/mod.rs

@@ -0,0 +1,20 @@
+pub mod farm;
+pub mod invariants;
+pub mod op;
+pub mod oracle;
+pub mod runner;
+pub mod scenarios;
+pub mod workload;
+
+pub use invariants::{Invariant, InvariantSet, InvariantViolation, invariants_for};
+pub use op::{CollectionName, Op, OpStream, RecordKey, Seed, ValueSeed};
+pub use oracle::Oracle;
+pub use runner::{
+    Gauntlet, GauntletBuildError, GauntletConfig, GauntletReport, IoBackend, MaxFileSize, OpIndex,
+    OpInterval, OpsExecuted, RestartCount, RestartPolicy, RunLimits, ShardCount, StoreConfig,
+    WallMs,
+};
+pub use scenarios::{Scenario, config_for};
+pub use workload::{
+    ByteRange, KeySpaceSize, OpCount, OpWeights, SizeDistribution, ValueBytes, WorkloadModel,
+};

crates/tranquil-store/src/gauntlet/op.rs

@@ -0,0 +1,60 @@
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct Seed(pub u64);
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct CollectionName(pub String);
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct RecordKey(pub String);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct ValueSeed(pub u32);
+
+#[derive(Debug, Clone)]
+pub enum Op {
+    AddRecord {
+        collection: CollectionName,
+        rkey: RecordKey,
+        value_seed: ValueSeed,
+    },
+    DeleteRecord {
+        collection: CollectionName,
+        rkey: RecordKey,
+    },
+    Compact,
+    Checkpoint,
+}
+
+#[derive(Debug, Clone)]
+pub struct OpStream {
+    ops: Vec<Op>,
+}
+
+impl OpStream {
+    pub fn from_vec(ops: Vec<Op>) -> Self {
+        Self { ops }
+    }
+
+    pub fn into_vec(self) -> Vec<Op> {
+        self.ops
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = &Op> {
+        self.ops.iter()
+    }
+
+    pub fn len(&self) -> usize {
+        self.ops.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.ops.is_empty()
+    }
+
+    pub fn shrink(&self) -> Option<OpStream> {
+        (self.ops.len() >= 2).then(|| {
+            let half = self.ops.len() / 2;
+            OpStream::from_vec(self.ops[..half].to_vec())
+        })
+    }
+}

crates/tranquil-store/src/gauntlet/oracle.rs

@@ -0,0 +1,87 @@
+use std::collections::HashMap;
+
+use cid::Cid;
+
+use super::op::{CollectionName, RecordKey};
+use crate::blockstore::CidBytes;
+
+#[derive(Debug, thiserror::Error, PartialEq, Eq)]
+#[error("unexpected CID encoding: got {actual} bytes, expected 36 for sha256 CIDv1")]
+pub struct CidFormatError {
+    pub actual: usize,
+}
+
+#[derive(Debug, Default)]
+pub struct Oracle {
+    live: HashMap<(CollectionName, RecordKey), CidBytes>,
+    current_root: Option<Cid>,
+    mst_node_cids: Vec<CidBytes>,
+}
+
+impl Oracle {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    pub fn add(
+        &mut self,
+        coll: CollectionName,
+        rkey: RecordKey,
+        record_cid: CidBytes,
+    ) -> Option<CidBytes> {
+        self.live.insert((coll, rkey), record_cid)
+    }
+
+    pub fn delete(&mut self, coll: &CollectionName, rkey: &RecordKey) -> Option<CidBytes> {
+        self.live.remove(&(coll.clone(), rkey.clone()))
+    }
+
+    pub fn set_root(&mut self, root: Cid) {
+        self.current_root = Some(root);
+    }
+
+    pub fn root(&self) -> Option<Cid> {
+        self.current_root
+    }
+
+    pub fn set_mst_node_cids(&mut self, cids: Vec<CidBytes>) {
+        self.mst_node_cids = cids;
+    }
+
+    pub fn clear_mst_state(&mut self) {
+        self.current_root = None;
+        self.mst_node_cids.clear();
+    }
+
+    pub fn live_records(&self) -> impl Iterator<Item = (&CollectionName, &RecordKey, &CidBytes)> {
+        self.live.iter().map(|((c, r), v)| (c, r, v))
+    }
+
+    pub fn live_count(&self) -> usize {
+        self.live.len()
+    }
+
+    pub fn live_cids_labeled(&self) -> Vec<(String, CidBytes)> {
+        let nodes = self
+            .mst_node_cids
+            .iter()
+            .map(|bytes| (format!("mst {}", hex_short(bytes)), *bytes));
+        let records = self
+            .live_records()
+            .map(|(c, r, v)| (format!("record {}/{}", c.0, r.0), *v));
+        nodes.chain(records).collect()
+    }
+}
+
+pub(super) fn try_cid_to_fixed(cid: &Cid) -> Result<CidBytes, CidFormatError> {
+    let bytes = cid.to_bytes();
+    let actual = bytes.len();
+    bytes.try_into().map_err(|_| CidFormatError { actual })
+}
+
+pub(super) fn hex_short(cid: &CidBytes) -> String {
+    cid[cid.len() - 6..]
+        .iter()
+        .map(|b| format!("{b:02x}"))
+        .collect()
+}

crates/tranquil-store/src/gauntlet/runner.rs

@@ -0,0 +1,593 @@
+use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::time::Duration;
+
+use cid::Cid;
+use jacquard_repo::mst::Mst;
+use jacquard_repo::storage::BlockStore;
+
+use super::invariants::{InvariantCtx, InvariantSet, InvariantViolation, invariants_for};
+use super::op::{Op, OpStream, Seed, ValueSeed};
+use super::oracle::{CidFormatError, Oracle, hex_short, try_cid_to_fixed};
+use super::workload::{Lcg, OpCount, SizeDistribution, ValueBytes, WorkloadModel};
+use crate::blockstore::{
+    BlockStoreConfig, CidBytes, CompactionError, GroupCommitConfig, TranquilBlockStore,
+};
+
+#[derive(Debug, Clone, Copy)]
+pub enum IoBackend {
+    Real,
+    Simulated,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct OpInterval(pub usize);
+
+#[derive(Debug, Clone, Copy)]
+pub enum RestartPolicy {
+    Never,
+    EveryNOps(OpInterval),
+    PoissonByOps(OpInterval),
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct WallMs(pub u64);
+
+#[derive(Debug, Clone, Copy)]
+pub struct RunLimits {
+    pub max_wall_ms: Option<WallMs>,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct MaxFileSize(pub u64);
+
+#[derive(Debug, Clone, Copy)]
+pub struct ShardCount(pub u8);
+
+#[derive(Debug, Clone)]
+pub struct StoreConfig {
+    pub max_file_size: MaxFileSize,
+    pub group_commit: GroupCommitConfig,
+    pub shard_count: ShardCount,
+}
+
+#[derive(Debug, Clone)]
+pub struct GauntletConfig {
+    pub seed: Seed,
+    pub io: IoBackend,
+    pub workload: WorkloadModel,
+    pub op_count: OpCount,
+    pub invariants: InvariantSet,
+    pub limits: RunLimits,
+    pub restart_policy: RestartPolicy,
+    pub store: StoreConfig,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct OpsExecuted(pub usize);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct RestartCount(pub usize);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct OpIndex(pub usize);
+
+#[derive(Debug)]
+pub struct GauntletReport {
+    pub seed: Seed,
+    pub ops_executed: OpsExecuted,
+    pub restarts: RestartCount,
+    pub violations: Vec<InvariantViolation>,
+}
+
+impl GauntletReport {
+    pub fn is_clean(&self) -> bool {
+        self.violations.is_empty()
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+enum OpError {
+    #[error("put record: {0}")]
+    PutRecord(String),
+    #[error("mst add: {0}")]
+    MstAdd(String),
+    #[error("mst delete: {0}")]
+    MstDelete(String),
+    #[error("mst persist: {0}")]
+    MstPersist(String),
+    #[error("mst diff: {0}")]
+    MstDiff(String),
+    #[error("apply commit: {0}")]
+    ApplyCommit(String),
+    #[error("compact_file: {0}")]
+    CompactFile(String),
+    #[error("join: {0}")]
+    Join(String),
+    #[error("cid format: {0}")]
+    CidFormat(#[from] CidFormatError),
+}
+
+pub struct Gauntlet {
+    config: GauntletConfig,
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum GauntletBuildError {
+    #[error("IoBackend::Simulated not wired yet")]
+    UnsupportedIoBackend,
+}
+
+impl Gauntlet {
+    pub fn new(config: GauntletConfig) -> Result<Self, GauntletBuildError> {
+        match config.io {
+            IoBackend::Real => Ok(Self { config }),
+            IoBackend::Simulated => Err(GauntletBuildError::UnsupportedIoBackend),
+        }
+    }
+
+    pub async fn run(self) -> GauntletReport {
+        let deadline = self
+            .config
+            .limits
+            .max_wall_ms
+            .map(|WallMs(ms)| Duration::from_millis(ms));
+
+        let seed = self.config.seed;
+        let ops_counter = Arc::new(AtomicUsize::new(0));
+        let restarts_counter = Arc::new(AtomicUsize::new(0));
+        let fut = run_real_inner(self.config, ops_counter.clone(), restarts_counter.clone());
+        match deadline {
+            Some(d) => match tokio::time::timeout(d, fut).await {
+                Ok(r) => r,
+                Err(_) => GauntletReport {
+                    seed,
+                    ops_executed: OpsExecuted(ops_counter.load(Ordering::Relaxed)),
+                    restarts: RestartCount(restarts_counter.load(Ordering::Relaxed)),
+                    violations: vec![InvariantViolation {
+                        invariant: "WallClockBudget",
+                        detail: format!("exceeded max_wall_ms of {} ms", d.as_millis()),
+                    }],
+                },
+            },
+            None => fut.await,
+        }
+    }
+}
+
+async fn run_real_inner(
+    config: GauntletConfig,
+    ops_counter: Arc<AtomicUsize>,
+    restarts_counter: Arc<AtomicUsize>,
+) -> GauntletReport {
+    let dir = tempfile::TempDir::new().expect("tempdir");
+    let op_stream: OpStream = config.workload.generate(config.seed, config.op_count);
+
+    let mut oracle = Oracle::new();
+    let mut violations: Vec<InvariantViolation> = Vec::new();
+
+    let mut store = Arc::new(
+        TranquilBlockStore::open(blockstore_config(dir.path(), &config.store)).expect("open store"),
+    );
+    let mut root: Option<Cid> = None;
+    let mut restart_rng = Lcg::new(Seed(config.seed.0 ^ 0xA5A5_A5A5_A5A5_A5A5));
+    let mut halt_ops = false;
+
+    let mid_run_set = config
+        .invariants
+        .without(InvariantSet::RESTART_IDEMPOTENT)
+        .without(InvariantSet::ACKED_WRITE_PERSISTENCE);
+    let post_reopen_set = config.invariants.without(InvariantSet::RESTART_IDEMPOTENT);
+
+    for (idx, op) in op_stream.iter().enumerate() {
+        if halt_ops {
+            break;
+        }
+        match apply_op(&store, &mut root, &mut oracle, op, &config.workload).await {
+            Ok(()) => {}
+            Err(e) => {
+                violations.push(InvariantViolation {
+                    invariant: "OpExecution",
+                    detail: format!("op {idx}: {e}"),
+                });
+                halt_ops = true;
+                continue;
+            }
+        }
+        ops_counter.store(idx + 1, Ordering::Relaxed);
+
+        if should_restart(config.restart_policy, OpIndex(idx), &mut restart_rng) {
+            drop(store);
+            store = Arc::new(
+                TranquilBlockStore::open(blockstore_config(dir.path(), &config.store))
+                    .expect("reopen store"),
+            );
+            let n = restarts_counter.fetch_add(1, Ordering::Relaxed) + 1;
+
+            if let Err(e) = refresh_oracle_graph(&store, &mut oracle, root).await {
+                violations.push(InvariantViolation {
+                    invariant: "OpExecution",
+                    detail: format!("refresh after restart {n}: {e}"),
+                });
+                halt_ops = true;
+                continue;
+            }
+            let before = violations.len();
+            violations.extend(run_invariants(&store, &oracle, root, mid_run_set).await);
+            if violations.len() > before {
+                halt_ops = true;
+            }
+        }
+    }
+
+    if !halt_ops {
+        match refresh_oracle_graph(&store, &mut oracle, root).await {
+            Ok(()) => {
+                let before = violations.len();
+                violations.extend(run_invariants(&store, &oracle, root, mid_run_set).await);
+                if violations.len() > before {
+                    halt_ops = true;
+                }
+            }
+            Err(e) => {
+                violations.push(InvariantViolation {
+                    invariant: "OpExecution",
+                    detail: format!("refresh at end: {e}"),
+                });
+                halt_ops = true;
+            }
+        }
+    }
+
+    if config.invariants.contains(InvariantSet::RESTART_IDEMPOTENT) && !halt_ops {
+        let pre_snapshot = snapshot_block_index(&store);
+        drop(store);
+        let reopened = Arc::new(
+            TranquilBlockStore::open(blockstore_config(dir.path(), &config.store))
+                .expect("reopen for RestartIdempotent"),
+        );
+        let post_snapshot = snapshot_block_index(&reopened);
+        if let Some(detail) = diff_snapshots(&pre_snapshot, &post_snapshot) {
+            violations.push(InvariantViolation {
+                invariant: "RestartIdempotent",
+                detail,
+            });
+        } else {
+            violations.extend(run_invariants(&reopened, &oracle, root, post_reopen_set).await);
+        }
+    }
+
+    GauntletReport {
+        seed: config.seed,
+        ops_executed: OpsExecuted(ops_counter.load(Ordering::Relaxed)),
+        restarts: RestartCount(restarts_counter.load(Ordering::Relaxed)),
+        violations,
+    }
+}
+
+async fn run_invariants(
+    store: &Arc<TranquilBlockStore>,
+    oracle: &Oracle,
+    root: Option<Cid>,
+    set: InvariantSet,
+) -> Vec<InvariantViolation> {
+    let ctx = InvariantCtx {
+        store,
+        oracle,
+        root,
+    };
+    let mut out = Vec::new();
+    for inv in invariants_for(set) {
+        if let Err(v) = inv.check(&ctx).await {
+            out.push(v);
+        }
+    }
+    out
+}
+
+fn snapshot_block_index(store: &TranquilBlockStore) -> Vec<(CidBytes, u32)> {
+    let mut v: Vec<(CidBytes, u32)> = store
+        .block_index()
+        .live_entries_snapshot()
+        .into_iter()
+        .map(|(c, r)| (c, r.raw()))
+        .collect();
+    v.sort_unstable_by(|a, b| a.0.cmp(&b.0));
+    v
+}
+
+const SNAPSHOT_DIFF_ITEMS: usize = 16;
+
+fn diff_snapshots(pre: &[(CidBytes, u32)], post: &[(CidBytes, u32)]) -> Option<String> {
+    if pre == post {
+        return None;
+    }
+    let pre_map: std::collections::HashMap<CidBytes, u32> = pre.iter().copied().collect();
+    let post_map: std::collections::HashMap<CidBytes, u32> = post.iter().copied().collect();
+
+    let only_pre: Vec<String> = pre_map
+        .iter()
+        .filter(|(c, _)| !post_map.contains_key(*c))
+        .map(|(c, r)| format!("lost {} refcount {}", hex_short(c), r))
+        .collect();
+    let only_post: Vec<String> = post_map
+        .iter()
+        .filter(|(c, _)| !pre_map.contains_key(*c))
+        .map(|(c, r)| format!("gained {} refcount {}", hex_short(c), r))
+        .collect();
+    let changed: Vec<String> = pre_map
+        .iter()
+        .filter_map(|(c, pre_r)| match post_map.get(c) {
+            Some(post_r) if post_r != pre_r => {
+                Some(format!("{} refcount {} -> {}", hex_short(c), pre_r, post_r))
+            }
+            _ => None,
+        })
+        .collect();
+
+    let total = only_pre.len() + only_post.len() + changed.len();
+    let mut items: Vec<String> = only_pre
+        .into_iter()
+        .chain(only_post)
+        .chain(changed)
+        .take(SNAPSHOT_DIFF_ITEMS)
+        .collect();
+    if total > items.len() {
+        items.push(format!("+{} more", total - items.len()));
+    }
+    Some(format!(
+        "block index changed across clean reopen: pre={} entries, post={} entries; {}",
+        pre.len(),
+        post.len(),
+        items.join("; "),
+    ))
+}
+
+async fn refresh_oracle_graph(
+    store: &Arc<TranquilBlockStore>,
+    oracle: &mut Oracle,
+    root: Option<Cid>,
+) -> Result<(), String> {
+    match root {
+        None => {
+            oracle.clear_mst_state();
+            Ok(())
+        }
+        Some(r) => {
+            let settled = Mst::load(store.clone(), r, None);
+            let cids = settled
+                .collect_node_cids()
+                .await
+                .map_err(|e| format!("collect_node_cids: {e}"))?;
+            let fixed: Vec<CidBytes> = cids
+                .iter()
+                .map(try_cid_to_fixed)
+                .collect::<Result<_, _>>()
+                .map_err(|e| format!("mst node cid: {e}"))?;
+            oracle.set_root(r);
+            oracle.set_mst_node_cids(fixed);
+            Ok(())
+        }
+    }
+}
+
+fn should_restart(policy: RestartPolicy, idx: OpIndex, rng: &mut Lcg) -> bool {
+    match policy {
+        RestartPolicy::Never => false,
+        RestartPolicy::EveryNOps(OpInterval(n)) => n > 0 && (idx.0 + 1).is_multiple_of(n),
+        RestartPolicy::PoissonByOps(OpInterval(n)) => {
+            if n == 0 {
+                false
+            } else {
+                rng.next_u64().is_multiple_of(n as u64)
+            }
+        }
+    }
+}
+
+fn blockstore_config(dir: &std::path::Path, s: &StoreConfig) -> BlockStoreConfig {
+    BlockStoreConfig {
+        data_dir: dir.join("data"),
+        index_dir: dir.join("index"),
+        max_file_size: s.max_file_size.0,
+        group_commit: s.group_commit.clone(),
+        shard_count: s.shard_count.0,
+    }
+}
+
+fn make_record_bytes(value_seed: ValueSeed, dist: SizeDistribution) -> Vec<u8> {
+    let raw = value_seed.0;
+    let target_len: usize = match dist {
+        SizeDistribution::Fixed(ValueBytes(n)) => n as usize,
+        SizeDistribution::Uniform(range) => {
+            let ValueBytes(lo) = range.min();
+            let ValueBytes(hi) = range.max();
+            let span = u64::from(hi.saturating_sub(lo)).max(1);
+            (lo as usize) + (u64::from(raw) % span) as usize
+        }
+    };
+    let target_len = target_len.max(8);
+    let seed_bytes = raw.to_le_bytes();
+    (0..target_len)
+        .map(|i| seed_bytes[i % 4] ^ (i as u8).wrapping_mul(31))
+        .collect()
+}
+
+async fn apply_op(
+    store: &Arc<TranquilBlockStore>,
+    root: &mut Option<Cid>,
+    oracle: &mut Oracle,
+    op: &Op,
+    workload: &WorkloadModel,
+) -> Result<(), OpError> {
+    match op {
+        Op::AddRecord {
+            collection,
+            rkey,
+            value_seed,
+        } => {
+            let record_bytes = make_record_bytes(*value_seed, workload.size_distribution);
+            let record_cid = store
+                .put(&record_bytes)
+                .await
+                .map_err(|e| OpError::PutRecord(e.to_string()))?;
+            let record_cid_bytes = try_cid_to_fixed(&record_cid)?;
+
+            let outcome =
+                add_record_inner(store, *root, collection, rkey, record_cid, record_cid_bytes)
+                    .await;
+            match outcome {
+                Ok((new_root, applied)) => {
+                    *root = Some(new_root);
+                    if applied {
+                        oracle.add(collection.clone(), rkey.clone(), record_cid_bytes);
+                    }
+                    Ok(())
+                }
+                Err(e) => {
+                    if let Err(cleanup_err) =
+                        decrement_obsolete(store, vec![record_cid_bytes]).await
+                    {
+                        tracing::warn!(
+                            op_error = %e,
+                            cleanup_error = %cleanup_err,
+                            "AddRecord cleanup decrement failed; refcount may leak",
+                        );
+                    }
+                    Err(e)
+                }
+            }
+        }
+        Op::DeleteRecord { collection, rkey } => {
+            let Some(old_root) = *root else { return Ok(()) };
+            if oracle.delete(collection, rkey).is_none() {
+                return Ok(());
+            }
+            let key = format!("{}/{}", collection.0, rkey.0);
+            let loaded = Mst::load(store.clone(), old_root, None);
+            let updated = loaded
+                .delete(&key)
+                .await
+                .map_err(|e| OpError::MstDelete(e.to_string()))?;
+            let new_root = updated
+                .persist()
+                .await
+                .map_err(|e| OpError::MstPersist(e.to_string()))?;
+            apply_mst_diff(store, old_root, new_root).await?;
+            *root = Some(new_root);
+            Ok(())
+        }
+        Op::Compact => {
+            let s = store.clone();
+            tokio::task::spawn_blocking(move || compact_by_liveness(&s))
+                .await
+                .map_err(|e| OpError::Join(e.to_string()))?
+        }
+        Op::Checkpoint => {
+            let s = store.clone();
+            tokio::task::spawn_blocking(move || {
+                s.apply_commit_blocking(vec![], vec![])
+                    .map_err(|e| e.to_string())
+            })
+            .await
+            .map_err(|e| OpError::Join(e.to_string()))?
+            .map_err(OpError::ApplyCommit)
+        }
+    }
+}
+
+async fn add_record_inner(
+    store: &Arc<TranquilBlockStore>,
+    root: Option<Cid>,
+    collection: &super::op::CollectionName,
+    rkey: &super::op::RecordKey,
+    record_cid: Cid,
+    record_cid_bytes: CidBytes,
+) -> Result<(Cid, bool), OpError> {
+    let key = format!("{}/{}", collection.0, rkey.0);
+    let loaded = match root {
+        None => Mst::new(store.clone()),
+        Some(r) => Mst::load(store.clone(), r, None),
+    };
+    let updated = loaded
+        .add(&key, record_cid)
+        .await
+        .map_err(|e| OpError::MstAdd(e.to_string()))?;
+    let new_root = updated
+        .persist()
+        .await
+        .map_err(|e| OpError::MstPersist(e.to_string()))?;
+
+    match root {
+        Some(old_root) if old_root == new_root => {
+            decrement_obsolete(store, vec![record_cid_bytes]).await?;
+            Ok((new_root, false))
+        }
+        Some(old_root) => {
+            apply_mst_diff(store, old_root, new_root).await?;
+            Ok((new_root, true))
+        }
+        None => Ok((new_root, true)),
+    }
+}
+
+async fn decrement_obsolete(
+    store: &Arc<TranquilBlockStore>,
+    obsolete: Vec<CidBytes>,
+) -> Result<(), OpError> {
+    let s = store.clone();
+    tokio::task::spawn_blocking(move || {
+        s.apply_commit_blocking(vec![], obsolete)
+            .map_err(|e| e.to_string())
+    })
+    .await
+    .map_err(|e| OpError::Join(e.to_string()))?
+    .map_err(OpError::ApplyCommit)
+}
+
+async fn apply_mst_diff(
+    store: &Arc<TranquilBlockStore>,
+    old_root: Cid,
+    new_root: Cid,
+) -> Result<(), OpError> {
+    let old_m = Mst::load(store.clone(), old_root, None);
+    let new_m = Mst::load(store.clone(), new_root, None);
+    let diff = old_m
+        .diff(&new_m)
+        .await
+        .map_err(|e| OpError::MstDiff(e.to_string()))?;
+    let obsolete: Vec<CidBytes> = diff
+        .removed_mst_blocks
+        .into_iter()
+        .chain(diff.removed_cids.into_iter())
+        .map(|c| try_cid_to_fixed(&c))
+        .collect::<Result<_, _>>()?;
+    let s = store.clone();
+    tokio::task::spawn_blocking(move || {
+        s.apply_commit_blocking(vec![], obsolete)
+            .map_err(|e| e.to_string())
+    })
+    .await
+    .map_err(|e| OpError::Join(e.to_string()))?
+    .map_err(OpError::ApplyCommit)
+}
+
+const COMPACT_LIVENESS_CEILING: f64 = 0.99;
+
+fn compact_by_liveness(store: &TranquilBlockStore) -> Result<(), OpError> {
+    let liveness = store
+        .compaction_liveness(0)
+        .map_err(|e| OpError::CompactFile(format!("compaction_liveness: {e}")))?;
+    let targets: Vec<_> = liveness
+        .iter()
+        .filter(|(_, info)| info.total_blocks > 0 && info.ratio() < COMPACT_LIVENESS_CEILING)
+        .map(|(&fid, _)| fid)
+        .collect();
+    targets
+        .into_iter()
+        .try_for_each(|fid| match store.compact_file(fid, 0) {
+            Ok(_) => Ok(()),
+            Err(CompactionError::ActiveFileCannotBeCompacted) => Ok(()),
+            Err(e) => Err(OpError::CompactFile(format!("{fid}: {e}"))),
+        })
+}

crates/tranquil-store/src/gauntlet/scenarios.rs

@@ -0,0 +1,166 @@
+use super::invariants::InvariantSet;
+use super::op::{CollectionName, Seed};
+use super::runner::{
+    GauntletConfig, IoBackend, MaxFileSize, OpInterval, RestartPolicy, RunLimits, ShardCount,
+    StoreConfig, WallMs,
+};
+use super::workload::{
+    KeySpaceSize, OpCount, OpWeights, SizeDistribution, ValueBytes, WorkloadModel,
+};
+use crate::blockstore::GroupCommitConfig;
+
+#[derive(Debug, Clone, Copy)]
+pub enum Scenario {
+    SmokePR,
+    MstChurn,
+    MstRestartChurn,
+    FullStackRestart,
+}
+
+pub fn config_for(scenario: Scenario, seed: Seed) -> GauntletConfig {
+    match scenario {
+        Scenario::SmokePR => smoke_pr(seed),
+        Scenario::MstChurn => mst_churn(seed),
+        Scenario::MstRestartChurn => mst_restart_churn(seed),
+        Scenario::FullStackRestart => full_stack_restart(seed),
+    }
+}
+
+fn default_collections() -> Vec<CollectionName> {
+    vec![
+        CollectionName("app.bsky.feed.post".to_string()),
+        CollectionName("app.bsky.feed.like".to_string()),
+    ]
+}
+
+fn tiny_store() -> StoreConfig {
+    StoreConfig {
+        max_file_size: MaxFileSize(4096),
+        group_commit: GroupCommitConfig {
+            checkpoint_interval_ms: 100,
+            checkpoint_write_threshold: 10,
+            ..GroupCommitConfig::default()
+        },
+        shard_count: ShardCount(1),
+    }
+}
+
+fn smoke_pr(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Real,
+        workload: WorkloadModel {
+            weights: OpWeights {
+                add: 80,
+                delete: 0,
+                compact: 10,
+                checkpoint: 10,
+            },
+            size_distribution: SizeDistribution::Fixed(ValueBytes(64)),
+            collections: default_collections(),
+            key_space: KeySpaceSize(200),
+        },
+        op_count: OpCount(10_000),
+        invariants: InvariantSet::REFCOUNT_CONSERVATION
+            | InvariantSet::REACHABILITY
+            | InvariantSet::ACKED_WRITE_PERSISTENCE
+            | InvariantSet::READ_AFTER_WRITE
+            | InvariantSet::RESTART_IDEMPOTENT,
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(60_000)),
+        },
+        restart_policy: RestartPolicy::EveryNOps(OpInterval(2_000)),
+        store: tiny_store(),
+    }
+}
+
+fn mst_churn(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Real,
+        workload: WorkloadModel {
+            weights: OpWeights {
+                add: 85,
+                delete: 0,
+                compact: 10,
+                checkpoint: 5,
+            },
+            size_distribution: SizeDistribution::Fixed(ValueBytes(64)),
+            collections: default_collections(),
+            key_space: KeySpaceSize(2_000),
+        },
+        op_count: OpCount(100_000),
+        invariants: InvariantSet::REFCOUNT_CONSERVATION
+            | InvariantSet::REACHABILITY
+            | InvariantSet::ACKED_WRITE_PERSISTENCE
+            | InvariantSet::READ_AFTER_WRITE
+            | InvariantSet::RESTART_IDEMPOTENT,
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(600_000)),
+        },
+        restart_policy: RestartPolicy::Never,
+        store: tiny_store(),
+    }
+}
+
+fn mst_restart_churn(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Real,
+        workload: WorkloadModel {
+            weights: OpWeights {
+                add: 85,
+                delete: 0,
+                compact: 10,
+                checkpoint: 5,
+            },
+            size_distribution: SizeDistribution::Fixed(ValueBytes(64)),
+            collections: default_collections(),
+            key_space: KeySpaceSize(2_000),
+        },
+        op_count: OpCount(100_000),
+        invariants: InvariantSet::REFCOUNT_CONSERVATION
+            | InvariantSet::REACHABILITY
+            | InvariantSet::ACKED_WRITE_PERSISTENCE
+            | InvariantSet::READ_AFTER_WRITE
+            | InvariantSet::RESTART_IDEMPOTENT,
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(600_000)),
+        },
+        restart_policy: RestartPolicy::PoissonByOps(OpInterval(5_000)),
+        store: tiny_store(),
+    }
+}
+
+fn full_stack_restart(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Real,
+        workload: WorkloadModel {
+            weights: OpWeights {
+                add: 80,
+                delete: 0,
+                compact: 15,
+                checkpoint: 5,
+            },
+            size_distribution: SizeDistribution::Fixed(ValueBytes(80)),
+            collections: default_collections(),
+            key_space: KeySpaceSize(500),
+        },
+        op_count: OpCount(5_000),
+        invariants: InvariantSet::REFCOUNT_CONSERVATION
+            | InvariantSet::REACHABILITY
+            | InvariantSet::ACKED_WRITE_PERSISTENCE
+            | InvariantSet::READ_AFTER_WRITE
+            | InvariantSet::RESTART_IDEMPOTENT,
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(120_000)),
+        },
+        restart_policy: RestartPolicy::EveryNOps(OpInterval(500)),
+        store: StoreConfig {
+            max_file_size: MaxFileSize(4096),
+            group_commit: GroupCommitConfig::default(),
+            shard_count: ShardCount(1),
+        },
+    }
+}

crates/tranquil-store/src/gauntlet/workload.rs

@@ -0,0 +1,133 @@
+use super::op::{CollectionName, Op, OpStream, RecordKey, Seed, ValueSeed};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct ValueBytes(pub u32);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct KeySpaceSize(pub u32);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct OpCount(pub usize);
+
+#[derive(Debug, Clone, Copy)]
+pub struct OpWeights {
+    pub add: u32,
+    pub delete: u32,
+    pub compact: u32,
+    pub checkpoint: u32,
+}
+
+impl OpWeights {
+    pub const fn total(&self) -> u32 {
+        self.add + self.delete + self.compact + self.checkpoint
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct ByteRange {
+    min: ValueBytes,
+    max: ValueBytes,
+}
+
+impl ByteRange {
+    pub fn new(min: ValueBytes, max: ValueBytes) -> Result<Self, String> {
+        if max.0 < min.0 {
+            Err(format!("ByteRange: max {} < min {}", max.0, min.0))
+        } else {
+            Ok(Self { min, max })
+        }
+    }
+
+    pub fn min(&self) -> ValueBytes {
+        self.min
+    }
+
+    pub fn max(&self) -> ValueBytes {
+        self.max
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub enum SizeDistribution {
+    Fixed(ValueBytes),
+    Uniform(ByteRange),
+}
+
+#[derive(Debug, Clone)]
+pub struct WorkloadModel {
+    pub weights: OpWeights,
+    pub size_distribution: SizeDistribution,
+    pub collections: Vec<CollectionName>,
+    pub key_space: KeySpaceSize,
+}
+
+impl WorkloadModel {
+    pub fn generate(&self, seed: Seed, op_count: OpCount) -> OpStream {
+        let mut rng = Lcg::new(seed);
+        let total = self.weights.total();
+        assert!(total > 0, "workload weights must sum to > 0");
+        assert!(
+            !self.collections.is_empty(),
+            "workload needs at least 1 collection"
+        );
+
+        let ops: Vec<Op> = (0..op_count.0)
+            .map(|_| {
+                let bucket = rng.next_u32() % total;
+                let coll = self.collections[rng.next_usize() % self.collections.len()].clone();
+                let rkey = RecordKey(format!("{:06}", rng.next_u32() % self.key_space.0.max(1)));
+
+                let (a, d, c) = (
+                    self.weights.add,
+                    self.weights.add + self.weights.delete,
+                    self.weights.add + self.weights.delete + self.weights.compact,
+                );
+                match bucket {
+                    b if b < a => Op::AddRecord {
+                        collection: coll,
+                        rkey,
+                        value_seed: ValueSeed(rng.next_u32()),
+                    },
+                    b if b < d => Op::DeleteRecord {
+                        collection: coll,
+                        rkey,
+                    },
+                    b if b < c => Op::Compact,
+                    _ => Op::Checkpoint,
+                }
+            })
+            .collect();
+        OpStream::from_vec(ops)
+    }
+}
+
+pub struct Lcg {
+    state: u64,
+}
+
+impl Lcg {
+    pub fn new(seed: Seed) -> Self {
+        Self {
+            state: seed
+                .0
+                .wrapping_mul(6364136223846793005)
+                .wrapping_add(1442695040888963407),
+        }
+    }
+
+    pub fn next_u64(&mut self) -> u64 {
+        self.state = self
+            .state
+            .wrapping_mul(6364136223846793005)
+            .wrapping_add(1442695040888963407);
+        self.state
+    }
+
+    pub fn next_u32(&mut self) -> u32 {
+        (self.next_u64() >> 16) as u32
+    }
+
+    pub fn next_usize(&mut self) -> usize {
+        self.next_u32() as usize
+    }
+}

crates/tranquil-store/src/lib.rs

@@ -6,6 +6,8 @@ pub mod consistency;
 pub mod eventlog;
 pub mod fsync_order;
 #[cfg(any(test, feature = "test-harness"))]
+pub mod gauntlet;
+#[cfg(any(test, feature = "test-harness"))]
 mod harness;
 mod io;
 pub mod metastore;

crates/tranquil-store/src/metastore/handler.rs

@@ -5964,7 +5964,7 @@ fn handler_loop<S: StorageIO + 'static>(
                         None => "unknown panic payload".to_owned(),
                     },
                 };
-                tracing::error!(thread_index, msg, "metastore handler panic (recovered)");
+                tracing::error!(thread_index, msg, "recovered metastore handler panic");
             }
         }
     });

crates/tranquil-store/tests/checkpoint_race.rs

@@ -0,0 +1,223 @@
+mod common;
+
+use std::io;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+
+use tranquil_store::PostBlockstoreHook;
+use tranquil_store::blockstore::{
+    BlockStoreConfig, BlocksSynced, CidBytes, GroupCommitConfig, TranquilBlockStore,
+};
+
+struct SlowHook;
+
+impl PostBlockstoreHook for SlowHook {
+    fn on_blocks_synced(&self, _proof: &BlocksSynced) -> io::Result<()> {
+        std::thread::sleep(std::time::Duration::from_millis(1));
+        Ok(())
+    }
+}
+
+fn refcount(store: &TranquilBlockStore, cid: &CidBytes) -> Option<u32> {
+    store.block_index().get(cid).map(|e| e.refcount.raw())
+}
+
+fn race_config(dir: &std::path::Path) -> BlockStoreConfig {
+    BlockStoreConfig {
+        data_dir: dir.join("data"),
+        index_dir: dir.join("index"),
+        max_file_size: 256 * 1024,
+        group_commit: GroupCommitConfig {
+            checkpoint_interval_ms: 10,
+            checkpoint_write_threshold: 20,
+            ..GroupCommitConfig::default()
+        },
+        shard_count: 4,
+    }
+}
+
+fn cid_for(shard: u8, seq: u32) -> CidBytes {
+    let mut cid = [0u8; 36];
+    cid[0] = 0x01;
+    cid[1] = 0x71;
+    cid[2] = 0x12;
+    cid[3] = 0x20;
+    cid[4] = shard;
+    cid[8..12].copy_from_slice(&seq.to_le_bytes());
+    (12..36).for_each(|i| cid[i] = (seq as u8).wrapping_add(i as u8));
+    cid
+}
+
+fn write_phase(base: &std::path::Path, use_hook: bool) -> Vec<CidBytes> {
+    let config = race_config(base);
+    let hook: Option<Arc<dyn PostBlockstoreHook>> = use_hook.then(|| Arc::new(SlowHook) as _);
+    let store = Arc::new(TranquilBlockStore::open_with_hook(config, hook).unwrap());
+
+    let running = Arc::new(AtomicBool::new(true));
+    let total_cycles = Arc::new(AtomicU64::new(0));
+
+    let writers: Vec<_> = (0..4u8)
+        .map(|shard| {
+            let store = Arc::clone(&store);
+            let running = Arc::clone(&running);
+            let total_cycles = Arc::clone(&total_cycles);
+            std::thread::spawn(move || {
+                let mut targets = Vec::new();
+                let mut seq = 0u32;
+                while running.load(Ordering::Relaxed) {
+                    let cid = cid_for(shard, seq);
+                    store
+                        .put_blocks_blocking(vec![(cid, vec![shard; 60])])
+                        .unwrap();
+                    store
+                        .put_blocks_blocking(vec![(cid, vec![shard; 60])])
+                        .unwrap();
+                    store.apply_commit_blocking(vec![], vec![cid]).unwrap();
+                    targets.push(cid);
+                    seq += 1;
+                    total_cycles.fetch_add(1, Ordering::Relaxed);
+                }
+                targets
+            })
+        })
+        .collect();
+
+    while total_cycles.load(Ordering::Relaxed) < 500 {
+        std::thread::yield_now();
+    }
+
+    running.store(false, Ordering::Relaxed);
+
+    let all_targets: Vec<CidBytes> = writers
+        .into_iter()
+        .flat_map(|w| w.join().unwrap())
+        .collect();
+
+    all_targets.iter().for_each(|cid| {
+        assert_eq!(refcount(&store, cid), Some(1), "pre-crash sanity");
+    });
+
+    let store = Arc::try_unwrap(store).ok().unwrap();
+    std::mem::forget(store);
+
+    all_targets
+}
+
+fn verify_phase(base: &std::path::Path, targets: &[CidBytes]) -> usize {
+    let config = race_config(base);
+    let store = TranquilBlockStore::open(config).unwrap();
+    let bad = targets
+        .iter()
+        .filter(|cid| refcount(&store, cid) != Some(1))
+        .count();
+    drop(store);
+    bad
+}
+
+#[test]
+fn crash_recovery_preserves_refcounts() {
+    common::with_runtime(|| {
+        let mut corrupted = 0u32;
+        let total = 20u32;
+
+        (0..total).for_each(|_| {
+            let dir = tempfile::TempDir::new().unwrap();
+            let exe = std::env::current_exe().unwrap();
+            let dir_str = dir.path().to_str().unwrap();
+
+            let output = std::process::Command::new(&exe)
+                .arg("--exact")
+                .arg("__crash_write_phase")
+                .env("CRASH_TEST_DIR", dir_str)
+                .env("CRASH_TEST_HOOK", "0")
+                .output()
+                .unwrap();
+
+            assert!(output.status.success() || output.status.code() == Some(0));
+
+            let target_bytes = std::fs::read(dir.path().join("targets.bin")).unwrap();
+            let targets: Vec<CidBytes> = target_bytes
+                .chunks_exact(36)
+                .map(|chunk| {
+                    let mut cid = [0u8; 36];
+                    cid.copy_from_slice(chunk);
+                    cid
+                })
+                .collect();
+
+            if verify_phase(dir.path(), &targets) > 0 {
+                corrupted += 1;
+            }
+        });
+
+        assert_eq!(
+            corrupted, 0,
+            "{corrupted}/{total} iterations had refcount corruption after crash recovery"
+        );
+    });
+}
+
+#[test]
+fn crash_with_slow_hook_preserves_refcounts() {
+    common::with_runtime(|| {
+        let mut corrupted = 0u32;
+        let total = 20u32;
+
+        (0..total).for_each(|_| {
+            let dir = tempfile::TempDir::new().unwrap();
+            let exe = std::env::current_exe().unwrap();
+            let dir_str = dir.path().to_str().unwrap();
+
+            let output = std::process::Command::new(&exe)
+                .arg("--exact")
+                .arg("__crash_write_phase")
+                .env("CRASH_TEST_DIR", dir_str)
+                .env("CRASH_TEST_HOOK", "1")
+                .output()
+                .unwrap();
+
+            assert!(output.status.success() || output.status.code() == Some(0));
+
+            let target_bytes = std::fs::read(dir.path().join("targets.bin")).unwrap();
+            let targets: Vec<CidBytes> = target_bytes
+                .chunks_exact(36)
+                .map(|chunk| {
+                    let mut cid = [0u8; 36];
+                    cid.copy_from_slice(chunk);
+                    cid
+                })
+                .collect();
+
+            if verify_phase(dir.path(), &targets) > 0 {
+                corrupted += 1;
+            }
+        });
+
+        assert_eq!(
+            corrupted, 0,
+            "{corrupted}/{total} iterations had refcount corruption after crash with slow hook"
+        );
+    });
+}
+
+#[test]
+fn __crash_write_phase() {
+    let dir = match std::env::var("CRASH_TEST_DIR") {
+        Ok(d) => d,
+        Err(_) => return,
+    };
+    let use_hook = std::env::var("CRASH_TEST_HOOK")
+        .map(|v| v == "1")
+        .unwrap_or(false);
+    let base = std::path::Path::new(&dir);
+
+    let rt = tokio::runtime::Runtime::new().unwrap();
+    let _guard = rt.enter();
+
+    let targets = write_phase(base, use_hook);
+
+    let target_bytes: Vec<u8> = targets.iter().flat_map(|cid| cid.iter().copied()).collect();
+    std::fs::write(base.join("targets.bin"), &target_bytes).unwrap();
+
+    unsafe { libc::_exit(0) }
+}

crates/tranquil-store/tests/common/mod.rs

@@ -104,6 +104,70 @@ pub fn compact_all_sealed(store: &TranquilBlockStore) {
         });
 }
 
+pub fn tiny_blockstore_config(dir: &std::path::Path) -> BlockStoreConfig {
+    BlockStoreConfig {
+        data_dir: dir.join("data"),
+        index_dir: dir.join("index"),
+        max_file_size: 300,
+        group_commit: GroupCommitConfig {
+            checkpoint_interval_ms: 100,
+            checkpoint_write_threshold: 10,
+            ..GroupCommitConfig::default()
+        },
+        shard_count: 1,
+    }
+}
+
+pub fn compact_by_liveness(store: &TranquilBlockStore) {
+    let liveness = store.compaction_liveness(0).unwrap();
+    liveness
+        .iter()
+        .filter(|(_, info)| info.total_blocks > 0 && info.ratio() < 0.99)
+        .map(|(&fid, _)| fid)
+        .collect::<Vec<_>>()
+        .into_iter()
+        .for_each(|fid| match store.compact_file(fid, 0) {
+            Ok(_) => {}
+            Err(tranquil_store::blockstore::CompactionError::ActiveFileCannotBeCompacted) => {}
+            Err(e) => eprintln!("compaction: {e}"),
+        });
+}
+
+pub fn compact_lowest_liveness(store: &TranquilBlockStore) {
+    let liveness = store.compaction_liveness(0).unwrap();
+    let candidate = liveness
+        .iter()
+        .filter(|(_, info)| info.total_blocks > 0 && info.ratio() < 0.99)
+        .min_by(|(_, a), (_, b)| {
+            a.ratio()
+                .partial_cmp(&b.ratio())
+                .unwrap_or(std::cmp::Ordering::Equal)
+        })
+        .map(|(&fid, _)| fid);
+
+    if let Some(fid) = candidate {
+        match store.compact_file(fid, 0) {
+            Ok(_) => {}
+            Err(tranquil_store::blockstore::CompactionError::ActiveFileCannotBeCompacted) => {}
+            Err(e) => eprintln!("compaction: {e}"),
+        }
+    }
+}
+
+pub fn collect_refcounts(store: &TranquilBlockStore, cids: &[CidBytes]) -> Vec<(u32, u32)> {
+    cids.iter()
+        .map(|cid| {
+            let seed = u32::from_le_bytes([cid[4], cid[5], cid[6], cid[7]]);
+            let rc = store
+                .block_index()
+                .get(cid)
+                .map(|e| e.refcount.raw())
+                .unwrap_or(0);
+            (seed, rc)
+        })
+        .collect()
+}
+
 pub struct TestStores {
     pub blockstore: TranquilBlockStore,
     pub eventlog: Arc<EventLog<RealIO>>,

crates/tranquil-store/tests/compaction_liveness.rs

@@ -226,7 +226,7 @@ fn stress_create_delete_restart_cycle_matches_bug_report() {
                     live.insert(seed_a);
                     live.insert(seed_b);
 
-                    if rng.next_u32() % 2 == 0 {
+                    if rng.next_u32().is_multiple_of(2) {
                         let victim: Option<u32> = live.iter().copied().next();
                         if let Some(v) = victim {
                             store

crates/tranquil-store/tests/compaction_restart.rs

@@ -0,0 +1,567 @@
+mod common;
+
+use std::path::Path;
+use std::sync::Arc;
+
+use common::{
+    collect_refcounts, compact_by_liveness, compact_lowest_liveness, test_cid,
+    tiny_blockstore_config, with_runtime,
+};
+use tranquil_store::RealIO;
+use tranquil_store::blockstore::{CidBytes, TranquilBlockStore};
+use tranquil_store::eventlog::{EventLog, EventLogBridge, EventLogConfig};
+use tranquil_store::metastore::handler::HandlerPool;
+use tranquil_store::metastore::partitions::Partition;
+use tranquil_store::metastore::{Metastore, MetastoreConfig};
+
+struct FullStack {
+    blockstore: TranquilBlockStore,
+    _pool: Arc<HandlerPool>,
+    _event_log: Arc<EventLog<RealIO>>,
+}
+
+fn open_full_stack(base_dir: &Path) -> FullStack {
+    let metastore_dir = base_dir.join("metastore");
+    let segments_dir = base_dir.join("eventlog").join("segments");
+    let blockstore_data = base_dir.join("blockstore").join("data");
+    let blockstore_index = base_dir.join("blockstore").join("index");
+
+    [
+        &metastore_dir,
+        &segments_dir,
+        &blockstore_data,
+        &blockstore_index,
+    ]
+    .iter()
+    .for_each(|d| std::fs::create_dir_all(d).unwrap());
+
+    let metastore = Metastore::open(&metastore_dir, MetastoreConfig::default()).unwrap();
+
+    let blockstore = TranquilBlockStore::open(tranquil_store::blockstore::BlockStoreConfig {
+        data_dir: blockstore_data,
+        index_dir: blockstore_index,
+        max_file_size: 512,
+        group_commit: tranquil_store::blockstore::GroupCommitConfig::default(),
+        shard_count: 1,
+    })
+    .unwrap();
+
+    let event_log = Arc::new(
+        EventLog::open(
+            EventLogConfig {
+                segments_dir,
+                ..EventLogConfig::default()
+            },
+            RealIO::new(),
+        )
+        .unwrap(),
+    );
+
+    let bridge = Arc::new(EventLogBridge::new(Arc::clone(&event_log)));
+
+    let was_clean = tranquil_store::consistency::had_clean_shutdown(base_dir);
+    tranquil_store::consistency::remove_clean_shutdown_marker(base_dir).ok();
+
+    let indexes = metastore.partition(Partition::Indexes).clone();
+    let event_ops = metastore.event_ops(Arc::clone(&bridge));
+    let recovered = event_ops.recover_metastore_mutations(&indexes).unwrap();
+    if recovered > 0 {
+        eprintln!("replayed {recovered} metastore mutations from eventlog");
+    }
+
+    if !was_clean || recovered > 0 {
+        let report = tranquil_store::consistency::verify_store_consistency(
+            &blockstore,
+            &metastore,
+            &event_log,
+        );
+        report.log_findings();
+        if report.has_repairable_issues() {
+            let repair = tranquil_store::consistency::repair_known_issues(&blockstore, &report);
+            if repair.orphan_files_removed > 0 {
+                eprintln!("removed {} orphan files", repair.orphan_files_removed);
+            }
+        }
+    }
+
+    let pool = Arc::new(HandlerPool::spawn::<RealIO>(
+        metastore,
+        bridge,
+        Some(blockstore.clone()),
+        None,
+    ));
+
+    FullStack {
+        blockstore,
+        _pool: pool,
+        _event_log: event_log,
+    }
+}
+
+fn close_full_stack(stack: FullStack, base_dir: &Path) {
+    let rt = tokio::runtime::Handle::current();
+    rt.block_on(stack._pool.close());
+    if let Err(e) = stack._event_log.shutdown() {
+        eprintln!("eventlog shutdown: {e}");
+    }
+    tranquil_store::consistency::write_clean_shutdown_marker(base_dir).ok();
+    drop(stack.blockstore);
+}
+
+fn verify_blocks_and_refcounts(
+    store: &TranquilBlockStore,
+    live_cids: &[CidBytes],
+    expected_refcounts: Option<&[(u32, u32)]>,
+    label: &str,
+) {
+    let missing: Vec<u32> = live_cids
+        .iter()
+        .filter(|cid| store.get_block_sync(cid).unwrap().is_none())
+        .map(|cid| u32::from_le_bytes([cid[4], cid[5], cid[6], cid[7]]))
+        .collect();
+
+    assert!(
+        missing.is_empty(),
+        "{label}: live blocks missing after reopen: {missing:?}"
+    );
+
+    match expected_refcounts {
+        Some(expected) => {
+            let actual = collect_refcounts(store, live_cids);
+            let mismatches: Vec<_> = expected
+                .iter()
+                .zip(actual.iter())
+                .filter(|((_, exp_rc), (_, act_rc))| exp_rc != act_rc)
+                .map(|((seed, exp), (_, act))| format!("seed {seed}: before={exp} after={act}"))
+                .collect();
+
+            assert!(
+                mismatches.is_empty(),
+                "{label}: refcounts changed across reopen:\n{}",
+                mismatches.join("\n"),
+            );
+        }
+        None => {
+            live_cids.iter().for_each(|cid| {
+                let rc = store
+                    .block_index()
+                    .get(cid)
+                    .map(|e| e.refcount.raw())
+                    .unwrap_or(0);
+                assert!(
+                    rc > 0,
+                    "{label}: refcount dropped to 0 for seed {}",
+                    u32::from_le_bytes([cid[4], cid[5], cid[6], cid[7]])
+                );
+            });
+        }
+    }
+}
+
+#[test]
+fn hundreds_of_compaction_cycles() {
+    with_runtime(|| {
+        let dir = tempfile::TempDir::new().unwrap();
+
+        let live_cids: Vec<CidBytes> = (0..15u32).map(test_cid).collect();
+
+        {
+            let store = TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap();
+
+            live_cids.iter().for_each(|cid| {
+                store
+                    .put_blocks_blocking(vec![(*cid, vec![0xAA; 80])])
+                    .unwrap();
+            });
+
+            (0..500u32).for_each(|round| {
+                let churn = test_cid(2000 + round);
+                store
+                    .put_blocks_blocking(vec![(churn, vec![0xDD; 80])])
+                    .unwrap();
+                store.apply_commit_blocking(vec![], vec![churn]).unwrap();
+
+                if round % 3 == 0 {
+                    compact_lowest_liveness(&store);
+                }
+            });
+
+            (0..200).for_each(|_| compact_by_liveness(&store));
+
+            live_cids.iter().for_each(|cid| {
+                assert!(
+                    store.get_block_sync(cid).unwrap().is_some(),
+                    "sanity: block present before drop"
+                );
+            });
+
+            drop(store);
+        }
+
+        let store = TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap();
+        verify_blocks_and_refcounts(&store, &live_cids, None, "500 churn + 200 compact rounds");
+    });
+}
+
+#[test]
+fn commit_style_decrements() {
+    with_runtime(|| {
+        let dir = tempfile::TempDir::new().unwrap();
+
+        let shared_nodes: Vec<CidBytes> = (0..10u32).map(test_cid).collect();
+
+        let refcounts_before = {
+            let store = TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap();
+
+            shared_nodes.iter().for_each(|cid| {
+                store
+                    .put_blocks_blocking(vec![(*cid, vec![0xAA; 80])])
+                    .unwrap();
+            });
+
+            let mut prev_commit = test_cid(5000);
+            store
+                .put_blocks_blocking(vec![(prev_commit, vec![0xCC; 80])])
+                .unwrap();
+
+            (0..500u32).for_each(|round| {
+                let new_commit = test_cid(5001 + round);
+                let new_mst_node = test_cid(6000 + round);
+                let old_mst_node = test_cid(7000 + round);
+
+                store
+                    .put_blocks_blocking(vec![
+                        (new_commit, vec![0xBB; 80]),
+                        (new_mst_node, vec![0xCC; 60]),
+                        (old_mst_node, vec![0xDD; 60]),
+                    ])
+                    .unwrap();
+
+                store
+                    .apply_commit_blocking(vec![], vec![prev_commit, old_mst_node])
+                    .unwrap();
+
+                if round > 0 {
+                    let prev_mst = test_cid(6000 + round - 1);
+                    store.apply_commit_blocking(vec![], vec![prev_mst]).unwrap();
+                }
+
+                prev_commit = new_commit;
+
+                if round % 2 == 0 {
+                    compact_lowest_liveness(&store);
+                }
+            });
+
+            (0..300).for_each(|_| {
+                compact_by_liveness(&store);
+                std::thread::sleep(std::time::Duration::from_millis(1));
+            });
+
+            shared_nodes.iter().for_each(|cid| {
+                assert!(
+                    store.get_block_sync(cid).unwrap().is_some(),
+                    "sanity: shared node present before drop"
+                );
+            });
+
+            let rc = collect_refcounts(&store, &shared_nodes);
+            drop(store);
+            rc
+        };
+
+        let store = TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap();
+        verify_blocks_and_refcounts(
+            &store,
+            &shared_nodes,
+            Some(&refcounts_before),
+            "500 commits + 300 compact rounds",
+        );
+    });
+}
+
+#[test]
+fn extreme_file_churn_with_dedup_hits() {
+    with_runtime(|| {
+        let dir = tempfile::TempDir::new().unwrap();
+
+        let live_cids: Vec<CidBytes> = (0..8u32).map(test_cid).collect();
+        let live_data: Vec<u8> = vec![0xAA; 80];
+
+        let refcounts_before = {
+            let store = TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap();
+
+            live_cids.iter().for_each(|cid| {
+                store
+                    .put_blocks_blocking(vec![(*cid, live_data.clone())])
+                    .unwrap();
+            });
+
+            (0..300u32).for_each(|round| {
+                let churn = test_cid(3000 + round);
+                store
+                    .put_blocks_blocking(vec![(churn, vec![0xEE; 80])])
+                    .unwrap();
+                store.apply_commit_blocking(vec![], vec![churn]).unwrap();
+
+                if round % 50 == 0 {
+                    live_cids.iter().for_each(|cid| {
+                        store
+                            .put_blocks_blocking(vec![(*cid, live_data.clone())])
+                            .unwrap();
+                    });
+                }
+
+                if round % 2 == 0 {
+                    compact_lowest_liveness(&store);
+                }
+            });
+
+            (0..200).for_each(|_| compact_by_liveness(&store));
+
+            let rc = collect_refcounts(&store, &live_cids);
+            drop(store);
+            rc
+        };
+
+        let store = TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap();
+        verify_blocks_and_refcounts(
+            &store,
+            &live_cids,
+            Some(&refcounts_before),
+            "300 churn + dedup re-puts + 200 compacts",
+        );
+    });
+}
+
+#[test]
+fn long_idle_compaction_only_phase() {
+    with_runtime(|| {
+        let dir = tempfile::TempDir::new().unwrap();
+
+        let live_cids: Vec<CidBytes> = (0..20u32).map(test_cid).collect();
+
+        {
+            let store = TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap();
+
+            live_cids.iter().for_each(|cid| {
+                store
+                    .put_blocks_blocking(vec![(*cid, vec![0xAA; 80])])
+                    .unwrap();
+            });
+
+            (0..100u32).for_each(|round| {
+                let churn = test_cid(4000 + round);
+                store
+                    .put_blocks_blocking(vec![(churn, vec![0xFF; 80])])
+                    .unwrap();
+                store.apply_commit_blocking(vec![], vec![churn]).unwrap();
+            });
+
+            (0..500).for_each(|_| {
+                compact_by_liveness(&store);
+                std::thread::sleep(std::time::Duration::from_millis(1));
+            });
+
+            live_cids.iter().for_each(|cid| {
+                assert!(
+                    store.get_block_sync(cid).unwrap().is_some(),
+                    "sanity: block present before drop"
+                );
+            });
+
+            drop(store);
+        }
+
+        let store = TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap();
+        verify_blocks_and_refcounts(
+            &store,
+            &live_cids,
+            None,
+            "idle with 500 compaction-only rounds",
+        );
+    });
+}
+
+#[test]
+fn multiple_restart_cycles_blockstore() {
+    with_runtime(|| {
+        let dir = tempfile::TempDir::new().unwrap();
+
+        let live_cids: Vec<CidBytes> = (0..10u32).map(test_cid).collect();
+
+        {
+            let store = TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap();
+            live_cids.iter().for_each(|cid| {
+                store
+                    .put_blocks_blocking(vec![(*cid, vec![0xAA; 80])])
+                    .unwrap();
+            });
+            (0..50u32).for_each(|round| {
+                let churn = test_cid(8000 + round);
+                store
+                    .put_blocks_blocking(vec![(churn, vec![0xBB; 80])])
+                    .unwrap();
+                store.apply_commit_blocking(vec![], vec![churn]).unwrap();
+            });
+            drop(store);
+        }
+
+        (0..10u32).for_each(|cycle| {
+            {
+                let store = TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap();
+
+                (0..50u32).for_each(|round| {
+                    let churn = test_cid(9000 + cycle * 100 + round);
+                    store
+                        .put_blocks_blocking(vec![(churn, vec![0xCC; 80])])
+                        .unwrap();
+                    store.apply_commit_blocking(vec![], vec![churn]).unwrap();
+                    compact_lowest_liveness(&store);
+                });
+
+                (0..50).for_each(|_| compact_by_liveness(&store));
+
+                live_cids.iter().for_each(|cid| {
+                    assert!(
+                        store.get_block_sync(cid).unwrap().is_some(),
+                        "cycle {cycle}: block missing before drop"
+                    );
+                });
+
+                drop(store);
+            }
+
+            let store = TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap();
+            verify_blocks_and_refcounts(
+                &store,
+                &live_cids,
+                None,
+                &format!("blockstore restart cycle {cycle}"),
+            );
+        });
+    });
+}
+
+#[test]
+fn full_stack_compaction_restart_preserves_refcounts() {
+    with_runtime(|| {
+        let base = tempfile::TempDir::new().unwrap();
+        let base_dir = base.path().to_path_buf();
+
+        let live_cids: Vec<CidBytes> = (0..15u32).map(test_cid).collect();
+
+        let refcounts_before = {
+            let stack = open_full_stack(&base_dir);
+
+            live_cids.iter().for_each(|cid| {
+                stack
+                    .blockstore
+                    .put_blocks_blocking(vec![(*cid, vec![0xAA; 80])])
+                    .unwrap();
+            });
+
+            (0..500u32).for_each(|round| {
+                let churn = test_cid(2000 + round);
+                stack
+                    .blockstore
+                    .put_blocks_blocking(vec![(churn, vec![0xDD; 80])])
+                    .unwrap();
+                stack
+                    .blockstore
+                    .apply_commit_blocking(vec![], vec![churn])
+                    .unwrap();
+
+                if round % 3 == 0 {
+                    compact_lowest_liveness(&stack.blockstore);
+                }
+            });
+
+            (0..200).for_each(|_| compact_by_liveness(&stack.blockstore));
+
+            let rc = collect_refcounts(&stack.blockstore, &live_cids);
+
+            live_cids.iter().for_each(|cid| {
+                assert!(
+                    stack.blockstore.get_block_sync(cid).unwrap().is_some(),
+                    "sanity: block present before shutdown"
+                );
+            });
+
+            close_full_stack(stack, &base_dir);
+            rc
+        };
+
+        let stack = open_full_stack(&base_dir);
+        verify_blocks_and_refcounts(
+            &stack.blockstore,
+            &live_cids,
+            Some(&refcounts_before),
+            "full stack restart",
+        );
+        close_full_stack(stack, &base_dir);
+    });
+}
+
+#[test]
+fn full_stack_multiple_restart_cycles() {
+    with_runtime(|| {
+        let base = tempfile::TempDir::new().unwrap();
+        let base_dir = base.path().to_path_buf();
+
+        let live_cids: Vec<CidBytes> = (0..10u32).map(test_cid).collect();
+
+        {
+            let stack = open_full_stack(&base_dir);
+            live_cids.iter().for_each(|cid| {
+                stack
+                    .blockstore
+                    .put_blocks_blocking(vec![(*cid, vec![0xAA; 80])])
+                    .unwrap();
+            });
+            close_full_stack(stack, &base_dir);
+        }
+
+        (0..10u32).for_each(|cycle| {
+            let refcounts_before = {
+                let stack = open_full_stack(&base_dir);
+
+                (0..50u32).for_each(|round| {
+                    let churn = test_cid(5000 + cycle * 100 + round);
+                    stack
+                        .blockstore
+                        .put_blocks_blocking(vec![(churn, vec![0xBB; 80])])
+                        .unwrap();
+                    stack
+                        .blockstore
+                        .apply_commit_blocking(vec![], vec![churn])
+                        .unwrap();
+                    compact_lowest_liveness(&stack.blockstore);
+                });
+
+                (0..30).for_each(|_| compact_by_liveness(&stack.blockstore));
+
+                let rc = collect_refcounts(&stack.blockstore, &live_cids);
+
+                live_cids.iter().for_each(|cid| {
+                    assert!(
+                        stack.blockstore.get_block_sync(cid).unwrap().is_some(),
+                        "cycle {cycle}: block missing before shutdown"
+                    );
+                });
+
+                close_full_stack(stack, &base_dir);
+                rc
+            };
+
+            let stack = open_full_stack(&base_dir);
+            verify_blocks_and_refcounts(
+                &stack.blockstore,
+                &live_cids,
+                Some(&refcounts_before),
+                &format!("full stack cycle {cycle}"),
+            );
+            close_full_stack(stack, &base_dir);
+        });
+    });
+}

crates/tranquil-store/tests/eventlog_properties.rs

@@ -579,7 +579,7 @@ fn index_checkpoint_accelerates_recovery() {
 
     assert!(
         reads_with_index < reads_without_index,
-        "read with index ({reads_with_index} reads) should require fewer reads than without ({reads_without_index} reads)"
+        "indexed read took {reads_with_index} reads but unindexed took only {reads_without_index}"
     );
 }
 
@@ -654,7 +654,7 @@ fn fsync_ordering_blocks_before_events() {
     assert_eq!(
         event_writer.synced_seq(),
         EventSequence::BEFORE_ALL,
-        "crash between blockstore sync and eventlog sync must not persist the event (blocks exist, event does not = orphan, not inconsistency)"
+        "crash between blockstore sync and eventlog sync must leave blocks orphaned rather than persist the event"
     );
 
     drop(event_writer);

crates/tranquil-store/tests/eventlog_retention.rs

@@ -72,7 +72,7 @@ fn run_retention_at_deletes_sealed_segments_past_cutoff() {
     );
     assert!(
         segments_after >= 1,
-        "active segment must remain (got {segments_after})"
+        "active segment must remain, got {segments_after}"
     );
 }
 

crates/tranquil-store/tests/gauntlet_smoke.rs

@@ -0,0 +1,132 @@
+use tranquil_store::blockstore::GroupCommitConfig;
+use tranquil_store::gauntlet::{
+    CollectionName, Gauntlet, GauntletConfig, InvariantSet, IoBackend, KeySpaceSize, MaxFileSize,
+    OpCount, OpInterval, OpWeights, RestartPolicy, RunLimits, Scenario, Seed, ShardCount,
+    SizeDistribution, StoreConfig, ValueBytes, WallMs, WorkloadModel, config_for, farm,
+};
+
+#[test]
+#[ignore = "long running, 30 seeds of 10k ops each"]
+fn smoke_pr_30_seeds() {
+    let reports = farm::run_many(
+        |seed| config_for(Scenario::SmokePR, seed),
+        (0..30).map(Seed),
+    );
+    let failures: Vec<String> = reports
+        .iter()
+        .filter(|r| !r.is_clean())
+        .map(|r| {
+            format!(
+                "seed {}: {} violations\n  {}",
+                r.seed.0,
+                r.violations.len(),
+                r.violations
+                    .iter()
+                    .map(|v| format!("{}: {}", v.invariant, v.detail))
+                    .collect::<Vec<_>>()
+                    .join("\n  ")
+            )
+        })
+        .collect();
+    assert!(failures.is_empty(), "{}", failures.join("\n---\n"));
+}
+
+fn fast_sanity_config(seed: Seed) -> GauntletConfig {
+    GauntletConfig {
+        seed,
+        io: IoBackend::Real,
+        workload: WorkloadModel {
+            weights: OpWeights {
+                add: 80,
+                delete: 0,
+                compact: 10,
+                checkpoint: 10,
+            },
+            size_distribution: SizeDistribution::Fixed(ValueBytes(64)),
+            collections: vec![CollectionName("app.bsky.feed.post".to_string())],
+            key_space: KeySpaceSize(100),
+        },
+        op_count: OpCount(200),
+        invariants: InvariantSet::REFCOUNT_CONSERVATION
+            | InvariantSet::REACHABILITY
+            | InvariantSet::ACKED_WRITE_PERSISTENCE
+            | InvariantSet::READ_AFTER_WRITE
+            | InvariantSet::RESTART_IDEMPOTENT,
+        limits: RunLimits {
+            max_wall_ms: Some(WallMs(30_000)),
+        },
+        restart_policy: RestartPolicy::EveryNOps(OpInterval(80)),
+        store: StoreConfig {
+            max_file_size: MaxFileSize(512),
+            group_commit: GroupCommitConfig {
+                checkpoint_interval_ms: 50,
+                checkpoint_write_threshold: 8,
+                ..GroupCommitConfig::default()
+            },
+            shard_count: ShardCount(1),
+        },
+    }
+}
+
+#[tokio::test]
+async fn gauntlet_fast_sanity() {
+    let report = Gauntlet::new(fast_sanity_config(Seed(7)))
+        .expect("build gauntlet")
+        .run()
+        .await;
+    assert!(
+        report.is_clean(),
+        "violations: {:?}",
+        report
+            .violations
+            .iter()
+            .map(|v| format!("{}: {}", v.invariant, v.detail))
+            .collect::<Vec<_>>()
+    );
+    assert!(
+        report.restarts.0 >= 2,
+        "expected at least 2 restarts, got {}",
+        report.restarts.0
+    );
+    assert_eq!(report.ops_executed.0, 200);
+}
+
+#[tokio::test]
+async fn full_stack_restart_port() {
+    let cfg = config_for(Scenario::FullStackRestart, Seed(1));
+    let report = Gauntlet::new(cfg).expect("build gauntlet").run().await;
+    assert!(
+        report.is_clean(),
+        "violations: {:?}",
+        report
+            .violations
+            .iter()
+            .map(|v| format!("{}: {}", v.invariant, v.detail))
+            .collect::<Vec<_>>()
+    );
+    assert_eq!(
+        report.restarts.0, 10,
+        "FullStackRestart with EveryNOps(500) over 5000 ops must restart exactly 10 times",
+    );
+}
+
+#[tokio::test]
+#[ignore = "long running, 100k ops with around 20 restarts"]
+async fn mst_restart_churn_single_seed() {
+    let cfg = config_for(Scenario::MstRestartChurn, Seed(42));
+    let report = Gauntlet::new(cfg).expect("build gauntlet").run().await;
+    assert!(
+        report.is_clean(),
+        "violations: {:?}",
+        report
+            .violations
+            .iter()
+            .map(|v| format!("{}: {}", v.invariant, v.detail))
+            .collect::<Vec<_>>()
+    );
+    assert!(
+        report.restarts.0 >= 1,
+        "PoissonByOps(5000) over 100k ops should fire at least 1 restart, got {}",
+        report.restarts.0
+    );
+}

crates/tranquil-store/tests/gc.rs

@@ -160,7 +160,7 @@ fn collect_dead_blocks_respects_epoch_gating() {
             .collect();
         assert!(
             all_cids.contains(&cid_a),
-            "cid_a should be collectible (epoch advanced by subsequent commit)"
+            "cid_a should be collectible after subsequent commit advanced the epoch"
         );
     });
 }

crates/tranquil-store/tests/mst_integration.rs

@@ -268,12 +268,12 @@ async fn mst_create_update_delete_with_refcounts() {
     assert_eq!(&retrieved_a_v2.unwrap()[..], &record_a_v2);
 
     assert!(
-        store.has(&cid_a_v1).await.unwrap(),
-        "cid_a_v1 should still exist, tombstoned but not GC'd"
+        store.get(&cid_a_v1).await.unwrap().is_some(),
+        "cid_a_v1 data should still exist, tombstoned but not GC'd"
     );
     assert!(
-        store.has(&cid_b).await.unwrap(),
-        "cid_b should still exist, tombstoned but not GC'd"
+        store.get(&cid_b).await.unwrap().is_some(),
+        "cid_b data should still exist, tombstoned but not GC'd"
     );
 
     assert!(
@@ -284,8 +284,8 @@ async fn mst_create_update_delete_with_refcounts() {
     assert_eq!(&retrieved_c[..], &record_c);
 
     assert!(
-        store.has(&cid_shared).await.unwrap(),
-        "shared-content block should still exist, tombstoned but not GC'd"
+        store.get(&cid_shared).await.unwrap().is_some(),
+        "shared-content block data should still exist, tombstoned but not GC'd"
     );
 
     let loaded_mst = Mst::load(storage.clone(), mst_root_v2, None);

crates/tranquil-store/tests/mst_refcount_integrity.rs

@@ -0,0 +1,213 @@
+mod common;
+
+use std::sync::Arc;
+
+use cid::Cid;
+use common::{compact_by_liveness, tiny_blockstore_config};
+use jacquard_repo::mst::Mst;
+use jacquard_repo::storage::BlockStore;
+use tranquil_store::blockstore::TranquilBlockStore;
+
+fn cid_to_fixed(cid: &Cid) -> [u8; 36] {
+    let bytes = cid.to_bytes();
+    let mut arr = [0u8; 36];
+    arr.copy_from_slice(&bytes[..36]);
+    arr
+}
+
+fn make_record_bytes(seed: u32) -> Vec<u8> {
+    serde_ipld_dagcbor::to_vec(&serde_json::json!({
+        "$type": "app.bsky.feed.post",
+        "text": format!("record {seed}"),
+        "createdAt": "2026-01-01T00:00:00Z"
+    }))
+    .unwrap()
+}
+
+fn make_fake_commit_cid(counter: u32) -> Cid {
+    let data = format!("commit-{counter}");
+    let mh = multihash::Multihash::wrap(0x12, &{
+        use sha2::Digest;
+        sha2::Sha256::digest(data.as_bytes())
+    })
+    .unwrap();
+    Cid::new_v1(0x71, mh)
+}
+
+async fn compute_obsolete_from_diff<S: BlockStore + Sync + Send + 'static>(
+    old_mst: &Mst<S>,
+    new_mst: &Mst<S>,
+    old_commit_cid: Cid,
+) -> Vec<Cid> {
+    let diff = old_mst.diff(new_mst).await.unwrap();
+    std::iter::once(old_commit_cid)
+        .chain(diff.removed_mst_blocks.into_iter())
+        .chain(diff.removed_cids.into_iter())
+        .collect()
+}
+
+#[tokio::test]
+async fn mst_shared_subtrees_survive_incremental_writes_compaction_restart() {
+    let dir = tempfile::TempDir::new().unwrap();
+
+    let mut commit_counter = 0u32;
+    let final_node_cids: Vec<Cid>;
+
+    {
+        let store = Arc::new(TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap());
+
+        let mut mst = Mst::new(store.clone());
+        let mut root: Option<Cid> = None;
+        let mut prev_commit = make_fake_commit_cid(commit_counter);
+        commit_counter += 1;
+
+        for i in 0..30u32 {
+            let record_bytes = make_record_bytes(i);
+            let record_cid = store.put(&record_bytes).await.unwrap();
+            let key = format!("app.bsky.feed.post/{i:06}");
+            mst = match root {
+                None => mst.add(&key, record_cid).await.unwrap(),
+                Some(r) => {
+                    let loaded = Mst::load(store.clone(), r, None);
+                    loaded.add(&key, record_cid).await.unwrap()
+                }
+            };
+            let new_root = mst.persist().await.unwrap();
+
+            if let Some(old_root) = root {
+                let old_settled = Mst::load(store.clone(), old_root, None);
+                let new_settled = Mst::load(store.clone(), new_root, None);
+
+                let obsolete =
+                    compute_obsolete_from_diff(&old_settled, &new_settled, prev_commit).await;
+                let obsolete_fixed: Vec<[u8; 36]> = obsolete.iter().map(cid_to_fixed).collect();
+                let s = store.clone();
+                tokio::task::spawn_blocking(move || {
+                    s.apply_commit_blocking(vec![], obsolete_fixed).unwrap();
+                })
+                .await
+                .unwrap();
+            }
+
+            root = Some(new_root);
+            prev_commit = make_fake_commit_cid(commit_counter);
+            commit_counter += 1;
+
+            if i % 5 == 0 {
+                let s = store.clone();
+                tokio::task::spawn_blocking(move || compact_by_liveness(&s))
+                    .await
+                    .unwrap();
+            }
+        }
+
+        for i in 0..15u32 {
+            let record_bytes = make_record_bytes(1000 + i);
+            let record_cid = store.put(&record_bytes).await.unwrap();
+            let key = format!("app.bsky.feed.like/{i:06}");
+            let loaded = Mst::load(store.clone(), root.unwrap(), None);
+            mst = loaded.add(&key, record_cid).await.unwrap();
+            let new_root = mst.persist().await.unwrap();
+
+            let old_settled = Mst::load(store.clone(), root.unwrap(), None);
+            let new_settled = Mst::load(store.clone(), new_root, None);
+
+            let obsolete =
+                compute_obsolete_from_diff(&old_settled, &new_settled, prev_commit).await;
+            let obsolete_fixed: Vec<[u8; 36]> = obsolete.iter().map(cid_to_fixed).collect();
+            let s = store.clone();
+            tokio::task::spawn_blocking(move || {
+                s.apply_commit_blocking(vec![], obsolete_fixed).unwrap();
+            })
+            .await
+            .unwrap();
+
+            root = Some(new_root);
+            prev_commit = make_fake_commit_cid(commit_counter);
+            commit_counter += 1;
+
+            if i % 3 == 0 {
+                let s = store.clone();
+                tokio::task::spawn_blocking(move || compact_by_liveness(&s))
+                    .await
+                    .unwrap();
+            }
+        }
+
+        let final_settled = Mst::load(store.clone(), root.unwrap(), None);
+        final_node_cids = final_settled.collect_node_cids().await.unwrap();
+
+        final_node_cids.iter().for_each(|cid| {
+            let fixed = cid_to_fixed(cid);
+            let rc = store.block_index().get(&fixed).map(|e| e.refcount.raw());
+            assert!(
+                rc.is_some_and(|r| r > 0),
+                "MST node {cid} has refcount {rc:?} before shutdown"
+            );
+        });
+
+        let s = store.clone();
+        tokio::task::spawn_blocking(move || {
+            (0..100).for_each(|_| compact_by_liveness(&s));
+        })
+        .await
+        .unwrap();
+
+        final_node_cids.iter().for_each(|cid| {
+            let fixed = cid_to_fixed(cid);
+            let block = store.get_block_sync(&fixed).unwrap();
+            assert!(
+                block.is_some(),
+                "MST node {cid} missing after compaction before shutdown"
+            );
+        });
+
+        drop(store);
+    }
+
+    {
+        let store = Arc::new(TranquilBlockStore::open(tiny_blockstore_config(dir.path())).unwrap());
+
+        let missing: Vec<String> = final_node_cids
+            .iter()
+            .filter_map(|cid| {
+                let fixed = cid_to_fixed(cid);
+                match store.get_block_sync(&fixed) {
+                    Ok(Some(_)) => None,
+                    Ok(None) => {
+                        let rc = store.block_index().get(&fixed).map(|e| e.refcount.raw());
+                        Some(format!("{cid} missing, index refcount {rc:?}"))
+                    }
+                    Err(e) => Some(format!("{cid} error: {e}")),
+                }
+            })
+            .collect();
+
+        assert!(
+            missing.is_empty(),
+            "{} of {} MST nodes missing after reopen:\n{}",
+            missing.len(),
+            final_node_cids.len(),
+            missing.join("\n"),
+        );
+
+        let refcount_issues: Vec<String> = final_node_cids
+            .iter()
+            .filter_map(|cid| {
+                let fixed = cid_to_fixed(cid);
+                let rc = store.block_index().get(&fixed).map(|e| e.refcount.raw());
+                match rc {
+                    Some(0) => Some(format!("{cid} refcount dropped to 0")),
+                    None => Some(format!("{cid} not in index")),
+                    _ => None,
+                }
+            })
+            .collect();
+
+        assert!(
+            refcount_issues.is_empty(),
+            "MST nodes with bad refcounts after reopen:\n{}",
+            refcount_issues.join("\n"),
+        );
+    }
+}

crates/tranquil-store/tests/sim_blockstore.rs

@@ -74,9 +74,7 @@ impl SimHarness {
                 let cid = test_cid(seed as u32);
                 let data = vec![seed as u8; data_size];
                 let loc = writer.append_block(&cid, &data).unwrap();
-                hint_writer
-                    .append_hint(&cid, loc.file_id, loc.offset, loc.length)
-                    .unwrap();
+                hint_writer.append_hint(&cid, &loc).unwrap();
                 (cid, loc)
             })
             .collect();
@@ -112,7 +110,7 @@ impl SimHarness {
             HintOffset::new(entries.len() as u64 * HINT_RECORD_SIZE as u64),
         );
         index
-            .write_checkpoint(CommitEpoch::zero(), &positions)
+            .write_checkpoint_with_positions(CommitEpoch::zero(), &positions)
             .unwrap();
     }
 
@@ -538,9 +536,7 @@ fn sim_aggressive_faults_data_integrity() {
                 let cid = test_cid(i as u32);
                 let data = vec![i as u8; 64];
                 let loc = writer.append_block(&cid, &data).ok()?;
-                hint_writer
-                    .append_hint(&cid, loc.file_id, loc.offset, loc.length)
-                    .ok()?;
+                hint_writer.append_hint(&cid, &loc).ok()?;
                 Some(())
             })?;
 

crates/tranquil-sync/src/repo.rs

@@ -142,6 +142,17 @@ pub async fn get_repo(
         return get_repo_since(&state, &did, &head_cid, since).await;
     }
 
+    let _permit = match state.repo_export_semaphore.try_acquire() {
+        Ok(permit) => permit,
+        Err(_) => {
+            return (
+                StatusCode::SERVICE_UNAVAILABLE,
+                "Too many concurrent repo exports",
+            )
+                .into_response();
+        }
+    };
+
     let car_bytes = match generate_repo_car_from_user_blocks(
         state.repos.repo.as_ref(),
         &state.block_store,
@@ -213,19 +224,24 @@ async fn get_repo_since(state: &AppState, did: &Did, head_cid: &Cid, since: &str
             .into_response();
     }
 
-    let blocks = match state.block_store.get_many(&block_cids).await {
-        Ok(b) => b,
-        Err(e) => {
-            error!("Block store error in get_repo_since: {:?}", e);
-            return ApiError::InternalError(Some("Failed to get blocks".into())).into_response();
-        }
-    };
+    for chunk_start in (0..block_cids.len()).step_by(500) {
+        let chunk_end = (chunk_start + 500).min(block_cids.len());
+        let chunk = &block_cids[chunk_start..chunk_end];
+        let blocks = match state.block_store.get_many(chunk).await {
+            Ok(b) => b,
+            Err(e) => {
+                error!("Block store error in get_repo_since: {:?}", e);
+                return ApiError::InternalError(Some("Failed to get blocks".into()))
+                    .into_response();
+            }
+        };
 
-    blocks
-        .into_iter()
-        .enumerate()
-        .filter_map(|(i, block_opt)| block_opt.map(|block| (block_cids[i], block)))
-        .for_each(|(cid, block)| car_bytes.extend_from_slice(&encode_car_block(&cid, &block)));
+        chunk
+            .iter()
+            .zip(blocks.into_iter())
+            .filter_map(|(cid, block_opt)| block_opt.map(|block| (*cid, block)))
+            .for_each(|(cid, block)| car_bytes.extend_from_slice(&encode_car_block(&cid, &block)));
+    }
 
     (
         StatusCode::OK,

docs/install-debian.md

@@ -49,12 +49,12 @@ mkdir -p /var/lib/tranquil/blobs
 
 We'll set ownership after creating the service user.
 
-## Install deno (for frontend build)
+## Install Node.js and pnpm (for frontend build)
 
 ```bash
-curl -fsSL https://deno.land/install.sh | sh
-export PATH="$HOME/.deno/bin:$PATH"
-echo 'export PATH="$HOME/.deno/bin:$PATH"' >> ~/.bashrc
+curl -fsSL https://deb.nodesource.com/setup_24.x | bash -
+apt install -y nodejs
+npm install -g pnpm
 ```
 
 ## Clone and build Tranquil PDS
@@ -64,7 +64,8 @@ cd /opt
 git clone https://tangled.org/tranquil.farm/tranquil-pds tranquil-pds
 cd tranquil-pds
 cd frontend
-deno task build
+pnpm install --frozen-lockfile
+pnpm build
 cd ..
 cargo build --release
 ```
@@ -330,7 +331,7 @@ Update Tranquil PDS:
 ```bash
 cd /opt/tranquil-pds
 git pull
-cd frontend && deno task build && cd ..
+cd frontend && pnpm install --frozen-lockfile && pnpm build && cd ..
 cargo build --release
 systemctl stop tranquil-pds
 cp target/release/tranquil-pds /usr/local/bin/

example.toml

@@ -26,12 +26,6 @@
 # Can also be specified via environment variable `PDS_USER_HANDLE_DOMAINS`.
 #user_handle_domains =
 
-# List of domains available for user registration.
-# Defaults to the PDS hostname when not set.
-#
-# Can also be specified via environment variable `AVAILABLE_USER_DOMAINS`.
-#available_user_domains =
-
 # Enable PDS-hosted did:web identities.  Hosting did:web requires a
 # long-term commitment to serve DID documents; opt-in only.
 #
@@ -95,6 +89,29 @@
 # Default value: 10737418240
 #max_blob_size = 10737418240
 
+# Maximum allowed number of preferences
+#
+# Can also be specified via environment variable `MAX_PREFERENCES_COUNT`.
+#
+# Default value: 1000
+#max_preferences_count = 1000
+
+[frontend]
+# Whether to enable the built in serving of the frontend.
+#
+# Can also be specified via environment variable `FRONTEND_ENABLED`.
+#
+# Default value: true
+#enabled = true
+
+# Directory to serve as the frontend. The oauth_client_metadata.json will have any references to
+# the frontend hostname replaced by the configured frontend hostname.
+#
+# Can also be specified via environment variable `FRONTEND_DIR`.
+#
+# Default value: "/var/lib/tranquil-pds/frontend"
+#dir = "/var/lib/tranquil-pds/frontend"
+
 [database]
 # PostgreSQL connection URL.
 #
@@ -210,8 +227,7 @@
 # once this budget is exhausted until in-flight events drain via fsync. Set to
 # 0 to disable backpressure. Default: 1 GiB.
 #
-# Can also be specified via environment variable
-# `TRANQUIL_STORE_EVENTLOG_PENDING_BYTES_BUDGET`.
+# Can also be specified via environment variable `TRANQUIL_STORE_EVENTLOG_PENDING_BYTES_BUDGET`.
 #
 # Default value: 1073741824
 #eventlog_pending_bytes_budget = 1073741824
@@ -219,12 +235,29 @@
 # Maximum size of an individual eventlog payload in bytes. Single events
 # larger than this are rejected at append time. Default: 256 MiB.
 #
-# Can also be specified via environment variable
-# `TRANQUIL_STORE_EVENTLOG_MAX_EVENT_PAYLOAD`.
+# Can also be specified via environment variable `TRANQUIL_STORE_EVENTLOG_MAX_EVENT_PAYLOAD`.
 #
 # Default value: 268435456
 #eventlog_max_event_payload = 268435456
 
+# Maximum size of an individual blockstore data file in bytes. When the
+# active data file reaches this size it is rolled over and becomes
+# eligible for compaction. Default: 256 MiB.
+#
+# Can also be specified via environment variable `TRANQUIL_STORE_MAX_BLOCKSTORE_FILE_SIZE`.
+#
+# Default value: 268435456
+#max_blockstore_file_size = 268435456
+
+# Maximum size of an individual eventlog segment file in bytes. When the
+# active segment reaches this size it is sealed and a new one is created.
+# Safe to change on a running instance. Default: 256 MiB.
+#
+# Can also be specified via environment variable `TRANQUIL_STORE_MAX_EVENTLOG_SEGMENT_SIZE`.
+#
+# Default value: 268435456
+#max_eventlog_segment_size = 268435456
+
 [cache]
 # Cache backend: `ripple` by default, or `valkey`.
 #
@@ -322,6 +355,13 @@
 # Default value: 5000
 #max_lag = 5000
 
+# Maximum concurrent full-repo exports, eg. getRepo without `since`.
+#
+# Can also be specified via environment variable `MAX_CONCURRENT_REPO_EXPORTS`.
+#
+# Default value: 4
+#max_concurrent_repo_exports = 4
+
 # List of relay / crawler notification URLs.
 #
 # Can also be specified via environment variable `CRAWLERS`.
@@ -369,6 +409,8 @@
 # Link a device via the admin API before enabling.
 #
 # Can also be specified via environment variable `SIGNAL_ENABLED`.
+#
+# Default value: false
 #enabled = false
 
 [notifications]
@@ -528,13 +570,58 @@
 # Default value: 3600
 #delete_check_interval_secs = 3600
 
+# Interval in seconds between data file compaction scans (tranquil-store only).
+# Set to 0 to disable.
+#
+# Can also be specified via environment variable `COMPACTION_INTERVAL_SECS`.
+#
+# Default value: 3600
+#compaction_interval_secs = 3600
+
+# Liveness ratio threshold below which a data file is compacted (0.0-1.0).
+#
+# Can also be specified via environment variable `COMPACTION_LIVENESS_THRESHOLD`.
+#
+# Default value: 0.7
+#compaction_liveness_threshold = 0.7
+
+# Grace period in milliseconds before a zero-refcount block can be removed by compaction.
+#
+# Can also be specified via environment variable `COMPACTION_GRACE_PERIOD_MS`.
+#
+# Default value: 600000
+#compaction_grace_period_ms = 600000
+
+# Interval in seconds between reachability walk runs (tranquil-store only).
+# Set to 0 to disable. Default: weekly.
+#
+# Can also be specified via environment variable `REACHABILITY_WALK_INTERVAL_SECS`.
+#
+# Default value: 604800
+#reachability_walk_interval_secs = 604800
+
+# Interval in seconds between continuous archival passes (tranquil-store only).
+# Sealed eventlog segments are copied to the archival destination each tick.
+# Set to 0 to disable. Default: 60 seconds.
+#
+# Can also be specified via environment variable `ARCHIVAL_INTERVAL_SECS`.
+#
+# Default value: 60
+#archival_interval_secs = 60
+
+# Archival destination directory for sealed eventlog segments.
+# If unset, archival is disabled.
+#
+# Can also be specified via environment variable `ARCHIVAL_DEST_DIR`.
+#archival_dest_dir =
+
 # Maximum age of events retained in the eventlog before pruning.
 # Per the atproto firehose spec, the relay backfill window only needs
 # to cover "hours or days".
 #
 # Can also be specified via environment variable `EVENT_RETENTION_MAX_AGE_SECS`.
 #
-# Default value: 604800 (7 days)
+# Default value: 604800
 #event_retention_max_age_secs = 604800
 
 # Interval in seconds between event retention prune passes.

flake.lock

@@ -2,11 +2,11 @@
   "nodes": {
     "nixpkgs": {
       "locked": {
-        "lastModified": 1766314097,
-        "narHash": "sha256-laJftWbghBehazn/zxVJ8NdENVgjccsWAdAqKXhErrM=",
+        "lastModified": 1775888245,
+        "narHash": "sha256-nwASzrRDD1JBEu/o8ekKYEXm/oJW6EMCzCRdrwcLe90=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "306ea70f9eb0fb4e040f8540e2deab32ed7e2055",
+        "rev": "13043924aaa7375ce482ebe2494338e058282925",
         "type": "github"
       },
       "original": {
@@ -16,26 +16,9 @@
         "type": "github"
       }
     },
-    "nixpkgs-fetch-deno": {
-      "locked": {
-        "lastModified": 1766410835,
-        "narHash": "sha256-dRhVt0aFDyTqppyzRLxiO1JZEAoIA2fUnaeyJTe+UwU=",
-        "owner": "aMOPel",
-        "repo": "nixpkgs",
-        "rev": "c9801acc8c4fac6377d076bc1c102b15bd9cfa6f",
-        "type": "github"
-      },
-      "original": {
-        "owner": "aMOPel",
-        "ref": "feat/fetchDenoDeps",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
     "root": {
       "inputs": {
-        "nixpkgs": "nixpkgs",
-        "nixpkgs-fetch-deno": "nixpkgs-fetch-deno"
+        "nixpkgs": "nixpkgs"
       }
     }
   },

flake.nix

@@ -1,51 +1,54 @@
 {
   inputs = {
     nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable";
-
-    # tranquil frontend uses deno as its package manager and build time runtime.
-    # nixpkgs does not have deno support yet but its being worked on in https://github.com/NixOS/nixpkgs/pull/419255
-    # for now we important that PR as well purely for its fetchDenoDeps
-    nixpkgs-fetch-deno.url = "github:aMOPel/nixpkgs/feat/fetchDenoDeps";
   };
 
-  outputs = {
-    self,
-    nixpkgs,
-    ...
-  } @ inputs: let
-    forAllSystems = function:
-      nixpkgs.lib.genAttrs nixpkgs.lib.systems.flakeExposed (
-        system: (function system nixpkgs.legacyPackages.${system})
-      );
-  in {
-    packages = forAllSystems (system: pkgs: {
-      tranquil-pds = pkgs.callPackage ./default.nix {};
-      tranquil-frontend = pkgs.callPackage ./frontend.nix {
-        inherit (inputs.nixpkgs-fetch-deno.legacyPackages.${system}) fetchDenoDeps;
+  outputs =
+    {
+      self,
+      nixpkgs,
+    }:
+    let
+      forAllSystems =
+        function:
+        nixpkgs.lib.genAttrs nixpkgs.lib.systems.flakeExposed (
+          system: function nixpkgs.legacyPackages.${system}
+        );
+    in
+    {
+      packages = forAllSystems (pkgs: {
+        tranquil-pds = pkgs.callPackage ./default.nix { };
+        tranquil-pds-aarch64 = pkgs.pkgsCross.aarch64-multiplatform.callPackage ./default.nix { };
+        tranquil-frontend = pkgs.callPackage ./frontend.nix { };
+        default = self.packages.${pkgs.stdenv.hostPlatform.system}.tranquil-pds;
+      });
+
+      devShells = forAllSystems (pkgs: {
+        default = pkgs.callPackage ./shell.nix { };
+      });
+
+      nixosModules = {
+        default = self.nixosModules.tranquil-pds;
+        tranquil-pds =
+          { lib, pkgs, ... }:
+          {
+            _file = "${self.outPath}/flake.nix#nixosModules.tranquil-pds";
+            imports = [ ./module.nix ];
+            config.services.tranquil-pds = {
+              package = self.packages.${pkgs.stdenv.hostPlatform.system}.tranquil-pds;
+              settings.frontend.package = self.packages.${pkgs.stdenv.hostPlatform.system}.tranquil-frontend;
+            };
+          };
       };
-      default = self.packages.${pkgs.stdenv.hostPlatform.system}.tranquil-pds;
-    });
 
-    devShells = forAllSystems (system: pkgs: {
-      default = pkgs.callPackage ./shell.nix {};
-    });
+      checks.x86_64-linux.integration = import ./test.nix {
+        pkgs = nixpkgs.legacyPackages.x86_64-linux;
+        inherit self;
+      };
 
-    nixosModules = {
-      default = self.nixosModules.tranquil-pds;
-      tranquil-pds = {
-        _file = "${self.outPath}/flake.nix#nixosModules.tranquil-pds";
-        imports = [(import ./module.nix self)];
+      checks.aarch64-linux.integration = import ./test.nix {
+        pkgs = nixpkgs.legacyPackages.aarch64-linux;
+        inherit self;
       };
     };
-
-    checks.x86_64-linux.integration = import ./test.nix {
-      pkgs = nixpkgs.legacyPackages.x86_64-linux;
-      inherit self;
-    };
-
-    checks.aarch64-linux.integration = import ./test.nix {
-      pkgs = nixpkgs.legacyPackages.aarch64-linux;
-      inherit self;
-    };
-  };
 }

frontend.nix

@@ -1,66 +1,49 @@
 {
   lib,
   stdenvNoCC,
-  
-  fetchDenoDeps,
-  fetchFromGitHub,
-  
-  buildGoModule,
-
-  deno,
-  esbuild,
-}: let
+  nodejs,
+  pnpm,
+  pnpmConfigHook,
+  fetchPnpmDeps,
+  nix-update-script,
+}:
+let
   toml = (lib.importTOML ./Cargo.toml).workspace.package;
-  deno-deps = fetchDenoDeps {
-    pname = "tranquil-frontend-deno-deps";
-    denoLock = ./frontend/deno.lock;
-    hash = "sha256-UB+E00TjWX0fTUZ7XwcwRJ/OUOSSJpz6Ss04U5i8dGI=";
-  };
-  # the esbuild in upstream nixpkgs is too old.
-  esbuild' = esbuild.override {
-    buildGoModule = args: buildGoModule (
-      args // (
-        let
-          version = "0.27.2";
-        in {
-          inherit version;
-          src = fetchFromGitHub {
-            owner = "evanw";
-            repo = "esbuild";
-            tag = "v${version}";
-            hash = "sha256-JbJB3F1NQlmA5d0rdsLm4RVD24OPdV4QXpxW8VWbESA";
-          };
-          vendorHash = "sha256-+BfxCyg0KkDQpHt/wycy/8CTG6YBA/VJvJFhhzUnSiQ";
-        }
-      )
-    );
-  };
-in stdenvNoCC.mkDerivation {
-    pname = "tranquil-frontend";
-    inherit (toml) version;
-    
-    src = ./frontend;
+in
+stdenvNoCC.mkDerivation (finalAttrs: {
+  pname = "tranquil-frontend";
+  inherit (toml) version;
 
-    nativeBuildInputs = [
-      deno
+  src = ./frontend;
+
+  pnpmDeps = fetchPnpmDeps {
+    inherit (finalAttrs) pname version src;
+    fetcherVersion = 3;
+    hash = "sha256-E0S8dOaTOpY9m7Ft59tUQ6CLlLriWPE4WE1+S45vomY=";
+  };
+
+  nativeBuildInputs = [
+    pnpm
+    nodejs
+    pnpmConfigHook
+  ];
+
+  buildPhase = ''
+    runHook preBuild
+    pnpm build
+    runHook postBuild
+  '';
+
+  installPhase = ''
+    runHook preInstall
+    cp -r ./dist $out
+    runHook postInstall
+  '';
+
+  passthru.updateScript = nix-update-script {
+    extraArgs = [
+      "--version"
+      "SKIP"
     ];
-    # tell vite (through the esbuild api) where the nix provided esbuild binary is
-    env.ESBUILD_BINARY_PATH = lib.getExe esbuild';
-
-    buildPhase = ''
-      # copy the deps to the required location
-      cp -r --no-preserve=mode ${deno-deps.denoDeps}/.deno ./
-      cp -r --no-preserve=mode ${deno-deps.denoDeps}/vendor ./
-
-      pwd
-      ls /build/frontend/vendor
-
-      # Now you can run the project using deps
-      # you need to activate [deno's vendor feature](https://docs.deno.com/runtime/fundamentals/modules/#vendoring-remote-modules)
-      # you need to use the `$DENO_DIR` env var, to point deno to the correct local cache
-      DENO_DIR=./.deno deno run --frozen --cached-only build
-    '';
-    installPhase = ''
-      cp -r ./dist $out
-    '';
-}
+  };
+})

frontend/Dockerfile

@@ -1,7 +1,10 @@
-FROM denoland/deno:alpine AS builder
+FROM node:24-alpine AS builder
+RUN corepack enable && corepack prepare pnpm@latest --activate
 WORKDIR /app
+COPY package.json pnpm-lock.yaml ./
+RUN pnpm install --frozen-lockfile
 COPY . ./
-RUN deno task build
+RUN pnpm build
 
 FROM nginx:1.29-alpine
 COPY --from=builder /app/dist /usr/share/nginx/html

frontend/deno.json

@@ -1,22 +0,0 @@
-{
-  "tasks": {
-    "dev": "deno run -A npm:vite",
-    "build": "deno run -A npm:vite build",
-    "preview": "deno run -A npm:vite preview",
-    "check": "deno run -A npm:svelte-check --tsconfig ./tsconfig.json",
-    "test": "deno run -A npm:vitest",
-    "test:run": "deno run -A npm:vitest run",
-    "test:watch": "deno run -A npm:vitest watch",
-    "test:ui": "deno run -A npm:vitest --ui",
-    "test:coverage": "deno run -A npm:vitest run --coverage"
-  },
-  "nodeModulesDir": "auto",
-  "lint": {
-    "rules": {
-      "exclude": [
-        "require-await",
-        "prefer-const"
-      ]
-    }
-  }
-}

justfile

@@ -56,7 +56,6 @@ test-misc:
     ./scripts/run-tests.sh --test actor --test commit_signing --test image_processing --test lifecycle_social --test notifications --test server --test signing_key --test verify_live_commit
 
 test *args:
-    @just test-store
     @just test-unit
     ./scripts/run-tests.sh {{args}}
 

module.nix

@@ -1,15 +1,17 @@
-self: {
+{
   lib,
   pkgs,
   config,
   ...
-}: let
+}:
+let
   cfg = config.services.tranquil-pds;
 
   inherit (lib) types mkOption;
 
   settingsFormat = pkgs.formats.toml { };
-in {
+in
+{
   _class = "nixos";
 
   options.services.tranquil-pds = {
@@ -17,8 +19,8 @@ in {
 
     package = mkOption {
       type = types.package;
-      default = self.packages.${pkgs.stdenv.hostPlatform.system}.tranquil-pds;
-      defaultText = lib.literalExpression "self.packages.\${pkgs.stdenv.hostPlatform.system}.tranquil-pds";
+      default = pkgs.callPackage ./default.nix { };
+      defaultText = lib.literalExpression "pkgs.tranquil-pds";
       description = "The tranquil-pds package to use";
     };
 
@@ -97,13 +99,16 @@ in {
           };
 
           frontend = {
-            enabled = lib.mkEnableOption "serving the frontend from the backend. Disable to serve the frontend manually"
-              // { default = true; };
+            enabled =
+              lib.mkEnableOption "serving the frontend from the backend. Disable to serve the frontend manually"
+              // {
+                default = true;
+              };
 
             dir = mkOption {
               type = types.nullOr types.package;
-              default = self.packages.${pkgs.stdenv.hostPlatform.system}.tranquil-frontend;
-              defaultText = lib.literalExpression "self.packages.\${pkgs.stdenv.hostPlatform.system}.tranquil-frontend";
+              default = pkgs.callPackage ./frontend.nix { };
+              defaultText = lib.literalExpression "pkgs.tranquil-frontend";
               description = "Frontend package to be served by the backend";
             };
           };
@@ -137,7 +142,7 @@ in {
   };
 
   config = lib.mkIf cfg.enable (
-    lib.mkMerge [            
+    lib.mkMerge [
       (lib.mkIf cfg.database.createLocally {
         services.postgresql = {
           enable = true;
@@ -159,7 +164,7 @@ in {
         };
       })
 
-      {      
+      {
         users.users.${cfg.user} = {
           isSystemUser = true;
           inherit (cfg) group;

scripts/install-debian.sh

@@ -207,11 +207,15 @@ if ! command -v rustc &>/dev/null; then
     source "$HOME/.cargo/env"
 fi
 
-log_info "Installing deno..."
-export PATH="$HOME/.deno/bin:$PATH"
-if ! command -v deno &>/dev/null && [[ ! -f "$HOME/.deno/bin/deno" ]]; then
-    curl -fsSL https://deno.land/install.sh | sh
-    grep -q 'deno/bin' ~/.bashrc 2>/dev/null || echo 'export PATH="$HOME/.deno/bin:$PATH"' >> ~/.bashrc
+log_info "Installing Node.js..."
+if ! command -v node &>/dev/null; then
+    curl -fsSL https://deb.nodesource.com/setup_24.x | bash -
+    apt install -y nodejs
+fi
+
+log_info "Installing pnpm..."
+if ! command -v pnpm &>/dev/null; then
+    npm install -g pnpm
 fi
 
 log_info "Cloning Tranquil PDS..."
@@ -223,7 +227,7 @@ fi
 cd /opt/tranquil-pds
 
 log_info "Building frontend..."
-"$HOME/.deno/bin/deno" task build --filter=frontend
+cd frontend && pnpm install --frozen-lockfile && pnpm build && cd ..
 log_success "Frontend built"
 
 log_info "Building Tranquil PDS (this takes a while)..."

scripts/run-tests.sh

@@ -23,7 +23,7 @@ cargo test --no-run 2>&1 | tail -1
 
 echo "Running tests..."
 echo ""
-cargo nextest run "$@"
+cargo nextest run -E 'not package(tranquil-store)' "$@"
 
 echo ""
 echo "All tests passed."

shell.nix

@@ -1,5 +1,4 @@
 {
-  lib,
   mkShell,
   callPackage,
   rustPlatform,
@@ -17,13 +16,18 @@
   cargo-nextest,
 
   # frontend tooling
-  deno,
   svelte-language-server,
   typescript-language-server,
-}: let
-  defaultPackage = callPackage ./default.nix { };
-in mkShell {
-  inputsFrom = [ defaultPackage ];
+}:
+let
+  pds = callPackage ./default.nix { };
+  frontend = callPackage ./frontend.nix { };
+in
+mkShell {
+  inputsFrom = [
+    pds
+    frontend
+  ];
 
   env = {
     RUST_SRC_PATH = rustPlatform.rustLibSrc;
@@ -39,10 +43,8 @@ in mkShell {
     rust-analyzer
     sqlx-cli
     cargo-nextest
-    
-    deno
+
     svelte-language-server
     typescript-language-server
   ];
 }
-