fix(repo): use mst diff instead of full tree walk for obsolete blocks

Lewis: May this revision serve well! <lu5a@proton.me>

.config/nextest.toml

@@ -13,9 +13,22 @@ fail-fast = false
 test-threads = "num-cpus"
 slow-timeout = { period = "30s", terminate-after = 4 }
 
+[profile.sim-pr]
+retries = 0
+fail-fast = true
+test-threads = "num-cpus"
+slow-timeout = { period = "60s", terminate-after = 4 }
+
+[profile.sim-nightly]
+retries = 0
+fail-fast = false
+test-threads = "num-cpus"
+slow-timeout = { period = "300s", terminate-after = 2 }
+
 [test-groups]
 serial-env-tests = { max-threads = 1 }
 heavy-load-tests = { max-threads = 4 }
+io-heavy-sim = { max-threads = 2 }
 
 [[profile.default.overrides]]
 filter = "test(/import_with_verification/) | test(/plc_migration/)"
@@ -45,6 +58,20 @@ test-group = "heavy-load-tests"
 filter = "binary(repo_lifecycle)"
 test-group = "heavy-load-tests"
 
+[[profile.default.overrides]]
+filter = "binary(sim_soak)"
+slow-timeout = { period = "300s", terminate-after = 10 }
+test-group = "io-heavy-sim"
+
+[[profile.default.overrides]]
+filter = "binary(metastore_crash) | binary(sim_cross_store) | binary(sim_gc_concurrent) | binary(sim_reachability) | binary(sim_eventlog) | test(/sim_/)"
+slow-timeout = { period = "120s", terminate-after = 8 }
+test-group = "io-heavy-sim"
+
+[[profile.default.overrides]]
+filter = "test(/test_scale_/) | test(/full_backup_and_restore/)"
+slow-timeout = { period = "120s", terminate-after = 4 }
+
 [[profile.ci.overrides]]
 filter = "test(/import_with_verification/) | test(/plc_migration/)"
 test-group = "serial-env-tests"

.sqlx/query-033ed5638ef6e30f1648c1acdbc477c173798cb57eb0127c3a582a9633fa5012.json

@@ -1,32 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n            SELECT seq, did, commit_cid\n            FROM repo_seq\n            WHERE event_type = 'commit'\n              AND prev_cid IS NULL\n              AND (blocks_cids IS NULL OR array_length(blocks_cids, 1) IS NULL OR array_length(blocks_cids, 1) = 0)\n            ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "seq",
-        "type_info": "Int8"
-      },
-      {
-        "ordinal": 1,
-        "name": "did",
-        "type_info": "Text"
-      },
-      {
-        "ordinal": 2,
-        "name": "commit_cid",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      false,
-      false,
-      true
-    ]
-  },
-  "hash": "033ed5638ef6e30f1648c1acdbc477c173798cb57eb0127c3a582a9633fa5012"
-}

.sqlx/query-0349b2af4d21105029ecc2bf81ad7be9987ec17ac088730596849d7a45558e57.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n                              ops, blobs, blocks_cids, handle, active, status, rev\n                       FROM repo_seq\n                       WHERE seq > $1\n                       ORDER BY seq ASC\n                       LIMIT $2",
+  "query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n                      ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev\n               FROM repo_seq\n               WHERE seq > $1\n               ORDER BY seq ASC\n               LIMIT $2",
   "describe": {
     "columns": [
       {
@@ -50,26 +50,36 @@
       },
       {
         "ordinal": 9,
+        "name": "block_cids",
+        "type_info": "ByteaArray"
+      },
+      {
+        "ordinal": 10,
+        "name": "block_data",
+        "type_info": "ByteaArray"
+      },
+      {
+        "ordinal": 11,
         "name": "blocks_cids",
         "type_info": "TextArray"
       },
       {
-        "ordinal": 10,
+        "ordinal": 12,
         "name": "handle",
         "type_info": "Text"
       },
       {
-        "ordinal": 11,
+        "ordinal": 13,
         "name": "active",
         "type_info": "Bool"
       },
       {
-        "ordinal": 12,
+        "ordinal": 14,
         "name": "status",
         "type_info": "Text"
       },
       {
-        "ordinal": 13,
+        "ordinal": 15,
         "name": "rev",
         "type_info": "Text"
       }
@@ -94,8 +104,10 @@
       true,
       true,
       true,
+      true,
+      true,
       true
     ]
   },
-  "hash": "b26bf97a27783eb7fb524a92dda3e68ef8470a9751fcaefe5fd2d7909dead54b"
+  "hash": "0349b2af4d21105029ecc2bf81ad7be9987ec17ac088730596849d7a45558e57"
 }

.sqlx/query-053c971024b0d29a441c3597d760b3e21db2383442c3e6f09de4eb49ea437e7c.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT did, password_hash FROM users WHERE handle = $1 OR email = $1",
+  "query": "SELECT did, password_hash FROM users WHERE handle = $1 OR did = $1",
   "describe": {
     "columns": [
       {
@@ -24,5 +24,5 @@
       true
     ]
   },
-  "hash": "c4621f6a8a1ab78a6355b09fdfc2bf8999d276564e93015792ec07cb05e79038"
+  "hash": "053c971024b0d29a441c3597d760b3e21db2383442c3e6f09de4eb49ea437e7c"
 }

.sqlx/query-060c285c93a05252aab7d474df0186e7b5083fafedc582b8eac9916983e8fc2d.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n            SELECT id, did, email, password_hash, password_required, two_factor_enabled,\n                   preferred_comms_channel as \"preferred_comms_channel!: CommsChannel\",\n                   deactivated_at, takedown_ref,\n                   email_verified, discord_verified, telegram_verified, signal_verified,\n                   account_type as \"account_type!: AccountType\"\n            FROM users\n            WHERE handle = $1 OR email = $1\n            ",
+  "query": "\n            SELECT id, did, email, password_hash, password_required, two_factor_enabled,\n                   preferred_comms_channel as \"preferred_comms_channel!: CommsChannel\",\n                   deactivated_at, takedown_ref,\n                   email_verified, discord_verified, telegram_verified, signal_verified,\n                   account_type as \"account_type!: AccountType\"\n            FROM users\n            WHERE handle = $1 OR did = $1\n            ",
   "describe": {
     "columns": [
       {
@@ -118,5 +118,5 @@
       false
     ]
   },
-  "hash": "7061e8763ef7d91ff152ed0124f99e1820172fd06916d225ca6c5137a507b8fa"
+  "hash": "060c285c93a05252aab7d474df0186e7b5083fafedc582b8eac9916983e8fc2d"
 }

.sqlx/query-12f5864ebff622fc52643de7151a40e984082851741b22f63a170728e734763b.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT cid FROM blocks ORDER BY created_at ASC LIMIT $1",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "cid",
-        "type_info": "Bytea"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Int8"
-      ]
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "12f5864ebff622fc52643de7151a40e984082851741b22f63a170728e734763b"
-}

.sqlx/query-18fa821e4bd00ccf5d1d8395ba728e4905d69f9fe527b4d4b49c69deff52cea8.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n                    SELECT t.cid FROM UNNEST($1::bytea[]) AS t(cid)\n                    WHERE NOT EXISTS (\n                        SELECT 1 FROM user_blocks WHERE block_cid = t.cid\n                    )\n                    ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "cid",
-        "type_info": "Bytea"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "ByteaArray"
-      ]
-    },
-    "nullable": [
-      null
-    ]
-  },
-  "hash": "18fa821e4bd00ccf5d1d8395ba728e4905d69f9fe527b4d4b49c69deff52cea8"
-}

.sqlx/query-3155ef4f35698a3fe6aa38d5d976fd51b7f6a0381c81c4907dad61d2f37992bd.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM webauthn_challenges WHERE did = $1 AND challenge_type = 'discoverable'",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "3155ef4f35698a3fe6aa38d5d976fd51b7f6a0381c81c4907dad61d2f37992bd"
+}

.sqlx/query-3b791fdb8e29043c980963d4d18e1e492c73c39818a8648a7af70555418fb5d1.json

@@ -1,15 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "UPDATE repo_seq SET blocks_cids = $1 WHERE seq = $2",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "TextArray",
-        "Int8"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "3b791fdb8e29043c980963d4d18e1e492c73c39818a8648a7af70555418fb5d1"
-}

.sqlx/query-4702a94384ce938d3f5384bc2106a48500f8749134e6bfd1c619a9151f5d3772.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n                      ops, blobs, blocks_cids, handle, active, status, rev\n               FROM repo_seq\n               WHERE seq > $1 AND seq < $2\n               ORDER BY seq ASC",
+  "query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n                      ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev\n               FROM repo_seq\n               WHERE seq = $1",
   "describe": {
     "columns": [
       {
@@ -50,33 +50,42 @@
       },
       {
         "ordinal": 9,
+        "name": "block_cids",
+        "type_info": "ByteaArray"
+      },
+      {
+        "ordinal": 10,
+        "name": "block_data",
+        "type_info": "ByteaArray"
+      },
+      {
+        "ordinal": 11,
         "name": "blocks_cids",
         "type_info": "TextArray"
       },
       {
-        "ordinal": 10,
+        "ordinal": 12,
         "name": "handle",
         "type_info": "Text"
       },
       {
-        "ordinal": 11,
+        "ordinal": 13,
         "name": "active",
         "type_info": "Bool"
       },
       {
-        "ordinal": 12,
+        "ordinal": 14,
         "name": "status",
         "type_info": "Text"
       },
       {
-        "ordinal": 13,
+        "ordinal": 15,
         "name": "rev",
         "type_info": "Text"
       }
     ],
     "parameters": {
       "Left": [
-        "Int8",
         "Int8"
       ]
     },
@@ -94,8 +103,10 @@
       true,
       true,
       true,
+      true,
+      true,
       true
     ]
   },
-  "hash": "d8524ad3f5dc03eb09ed60396a78df5003f804c43ad253d6476523eacdebf811"
+  "hash": "4702a94384ce938d3f5384bc2106a48500f8749134e6bfd1c619a9151f5d3772"
 }

.sqlx/query-5996774d097a484f81353d49e4a99b07c30a003377c924add576bfb48ddde29b.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n            INSERT INTO repo_seq (did, event_type, commit_cid, rev)\n            VALUES ($1, 'sync', $2, $3)\n            RETURNING seq\n            ",
+  "query": "\n            INSERT INTO repo_seq (did, event_type, commit_cid, rev, block_cids, block_data)\n            VALUES ($1, 'sync', $2, $3, $4, $5)\n            RETURNING seq\n            ",
   "describe": {
     "columns": [
       {
@@ -13,12 +13,14 @@
       "Left": [
         "Text",
         "Text",
-        "Text"
+        "Text",
+        "ByteaArray",
+        "ByteaArray"
       ]
     },
     "nullable": [
       false
     ]
   },
-  "hash": "bce9edc597592d2e7d672061883639bcdeff80be85e2ecacb6c75d8c027e6710"
+  "hash": "5996774d097a484f81353d49e4a99b07c30a003377c924add576bfb48ddde29b"
 }

.sqlx/query-6969c478a0922bac4b79902313a0e28c94d6b8d6b16035474dd8f484e6171d60.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT state_json FROM webauthn_challenges\n               WHERE did = $1 AND challenge_type = 'discoverable' AND expires_at > NOW()\n               ORDER BY created_at DESC LIMIT 1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "state_json",
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "6969c478a0922bac4b79902313a0e28c94d6b8d6b16035474dd8f484e6171d60"
+}

.sqlx/query-746ca87e98f6eaa9c060770c0d1bca6f0d4da60df2dc2db088e57bdc914cf233.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "DELETE FROM repo_seq WHERE created_at < $1",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Timestamptz"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "746ca87e98f6eaa9c060770c0d1bca6f0d4da60df2dc2db088e57bdc914cf233"
+}

.sqlx/query-89c8ce80d8c52b4668c34f63e48b51cacc7e8ff300958a192378f1d72de04f9a.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n                              ops, blobs, blocks_cids, handle, active, status, rev\n                       FROM repo_seq\n                       WHERE seq > $1\n                       ORDER BY seq ASC",
+  "query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n                              ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev\n                       FROM repo_seq\n                       WHERE seq > $1\n                       ORDER BY seq ASC\n                       LIMIT $2",
   "describe": {
     "columns": [
       {
@@ -50,32 +50,43 @@
       },
       {
         "ordinal": 9,
+        "name": "block_cids",
+        "type_info": "ByteaArray"
+      },
+      {
+        "ordinal": 10,
+        "name": "block_data",
+        "type_info": "ByteaArray"
+      },
+      {
+        "ordinal": 11,
         "name": "blocks_cids",
         "type_info": "TextArray"
       },
       {
-        "ordinal": 10,
+        "ordinal": 12,
         "name": "handle",
         "type_info": "Text"
       },
       {
-        "ordinal": 11,
+        "ordinal": 13,
         "name": "active",
         "type_info": "Bool"
       },
       {
-        "ordinal": 12,
+        "ordinal": 14,
         "name": "status",
         "type_info": "Text"
       },
       {
-        "ordinal": 13,
+        "ordinal": 15,
         "name": "rev",
         "type_info": "Text"
       }
     ],
     "parameters": {
       "Left": [
+        "Int8",
         "Int8"
       ]
     },
@@ -93,8 +104,10 @@
       true,
       true,
       true,
+      true,
+      true,
       true
     ]
   },
-  "hash": "b8101757a50075d20147014e450cb7deb7e58f84310690c7bde61e1834dc5903"
+  "hash": "89c8ce80d8c52b4668c34f63e48b51cacc7e8ff300958a192378f1d72de04f9a"
 }

.sqlx/query-8eecf8fef308716be88815eb59bb67ec7c534b3c821d55481b110e3e462ee366.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "DELETE FROM blocks WHERE cid = ANY($1)",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "ByteaArray"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "8eecf8fef308716be88815eb59bb67ec7c534b3c821d55481b110e3e462ee366"
-}

.sqlx/query-97dd3cfe72b74b4dac686c030a07f6eb56dabb2df72815f3248760b4e3f7950e.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n            INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, blocks_cids, prev_data_cid, rev)\n            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)\n            RETURNING seq\n            ",
+  "query": "\n            INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, block_cids, block_data, prev_data_cid, rev)\n            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)\n            RETURNING seq\n            ",
   "describe": {
     "columns": [
       {
@@ -17,7 +17,8 @@
         "Text",
         "Jsonb",
         "TextArray",
-        "TextArray",
+        "ByteaArray",
+        "ByteaArray",
         "Text",
         "Text"
       ]
@@ -26,5 +27,5 @@
       false
     ]
   },
-  "hash": "40e848f48afb50f5e4820445aee874a482333e9090ef14ce44db3fd9721cfff0"
+  "hash": "97dd3cfe72b74b4dac686c030a07f6eb56dabb2df72815f3248760b4e3f7950e"
 }

.sqlx/query-a325862f897484bb1a8863fd7a1bc3fd17ab20b6e30498f9900daf80609c413d.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "\n            INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, blocks_cids, rev)\n            VALUES ($1, 'commit', $2, $3::TEXT, $4, $5, $6, $7)\n            RETURNING seq\n            ",
+  "query": "\n            INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, block_cids, block_data, rev)\n            VALUES ($1, 'commit', $2, $3::TEXT, $4, $5, $6, $7, $8)\n            RETURNING seq\n            ",
   "describe": {
     "columns": [
       {
@@ -16,7 +16,8 @@
         "Text",
         "Jsonb",
         "TextArray",
-        "TextArray",
+        "ByteaArray",
+        "ByteaArray",
         "Text"
       ]
     },
@@ -24,5 +25,5 @@
       false
     ]
   },
-  "hash": "52df8ae809b990fa76e3ac45f37e352a9fc8f92f911a7b9c9be2fd9e8032b18c"
+  "hash": "a325862f897484bb1a8863fd7a1bc3fd17ab20b6e30498f9900daf80609c413d"
 }

.sqlx/query-aafc2a7e51200ca1e7071c63c13698bf34ef8b66758ca9ebab4ea706ffb62914.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT\n                u.id, u.did, u.handle, u.password_hash, u.email, u.deactivated_at, u.takedown_ref,\n                u.email_verified, u.discord_verified, u.telegram_verified, u.signal_verified,\n                u.allow_legacy_login, u.migrated_to_pds,\n                u.preferred_comms_channel as \"preferred_comms_channel: CommsChannel\",\n                k.key_bytes, k.encryption_version,\n                (SELECT verified FROM user_totp WHERE did = u.did) as totp_enabled,\n                COALESCE((SELECT (value_json)::boolean FROM account_preferences WHERE user_id = u.id AND name = 'email_auth_factor' ORDER BY created_at DESC LIMIT 1), false) as \"email_2fa_enabled!\"\n            FROM users u\n            JOIN user_keys k ON u.id = k.user_id\n            WHERE u.handle = $1 OR u.email = $1 OR u.did = $1",
+  "query": "SELECT\n                u.id, u.did, u.handle, u.password_hash, u.email, u.deactivated_at, u.takedown_ref,\n                u.email_verified, u.discord_verified, u.telegram_verified, u.signal_verified,\n                u.allow_legacy_login, u.migrated_to_pds,\n                u.preferred_comms_channel as \"preferred_comms_channel: CommsChannel\",\n                k.key_bytes, k.encryption_version,\n                (SELECT verified FROM user_totp WHERE did = u.did) as totp_enabled,\n                COALESCE((SELECT (value_json)::boolean FROM account_preferences WHERE user_id = u.id AND name = 'email_auth_factor' ORDER BY created_at DESC LIMIT 1), false) as \"email_2fa_enabled!\"\n            FROM users u\n            JOIN user_keys k ON u.id = k.user_id\n            WHERE u.handle = $1 OR u.did = $1",
   "describe": {
     "columns": [
       {
@@ -132,5 +132,5 @@
       null
     ]
   },
-  "hash": "a960b981a146a0e422ef53601dfc31e29cf777aa194227c48c6ebc6905ea3249"
+  "hash": "aafc2a7e51200ca1e7071c63c13698bf34ef8b66758ca9ebab4ea706ffb62914"
 }

.sqlx/query-ad784aff9bb5c1f19f90a3b1c3b3694a3e1aa889b0fe3e55cfebb7af8c620256.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n                      ops, blobs, blocks_cids, handle, active, status, rev\n               FROM repo_seq\n               WHERE seq > $1\n               ORDER BY seq ASC\n               LIMIT $2",
+  "query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n                      ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev\n               FROM repo_seq\n               WHERE seq > $1 AND seq < $2\n               ORDER BY seq ASC",
   "describe": {
     "columns": [
       {
@@ -50,26 +50,36 @@
       },
       {
         "ordinal": 9,
+        "name": "block_cids",
+        "type_info": "ByteaArray"
+      },
+      {
+        "ordinal": 10,
+        "name": "block_data",
+        "type_info": "ByteaArray"
+      },
+      {
+        "ordinal": 11,
         "name": "blocks_cids",
         "type_info": "TextArray"
       },
       {
-        "ordinal": 10,
+        "ordinal": 12,
         "name": "handle",
         "type_info": "Text"
       },
       {
-        "ordinal": 11,
+        "ordinal": 13,
         "name": "active",
         "type_info": "Bool"
       },
       {
-        "ordinal": 12,
+        "ordinal": 14,
         "name": "status",
         "type_info": "Text"
       },
       {
-        "ordinal": 13,
+        "ordinal": 15,
         "name": "rev",
         "type_info": "Text"
       }
@@ -94,8 +104,10 @@
       true,
       true,
       true,
+      true,
+      true,
       true
     ]
   },
-  "hash": "e7aa1080be9eb3a8ddf1f050c93dc8afd10478f41e22307014784b4ee3740b4a"
+  "hash": "ad784aff9bb5c1f19f90a3b1c3b3694a3e1aa889b0fe3e55cfebb7af8c620256"
 }

.sqlx/query-c15d538fece4cd7db0056f9e0374a2290e26f5766881fd9dbf4f05788845aac0.json

@@ -1,29 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT blocks_cids, commit_cid\n               FROM repo_seq\n               WHERE did = $1 AND rev > $2\n               ORDER BY seq DESC",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "blocks_cids",
-        "type_info": "TextArray"
-      },
-      {
-        "ordinal": 1,
-        "name": "commit_cid",
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Text",
-        "Text"
-      ]
-    },
-    "nullable": [
-      true,
-      true
-    ]
-  },
-  "hash": "c15d538fece4cd7db0056f9e0374a2290e26f5766881fd9dbf4f05788845aac0"
-}

.sqlx/query-c47746c256fd8c9d386551f3bd270e9a96fecdb713642d07d60fb4aeb02c8426.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n                      ops, blobs, blocks_cids, handle, active, status, rev\n               FROM repo_seq\n               WHERE seq = $1",
+  "query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n                              ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev\n                       FROM repo_seq\n                       WHERE seq > $1\n                       ORDER BY seq ASC",
   "describe": {
     "columns": [
       {
@@ -50,26 +50,36 @@
       },
       {
         "ordinal": 9,
+        "name": "block_cids",
+        "type_info": "ByteaArray"
+      },
+      {
+        "ordinal": 10,
+        "name": "block_data",
+        "type_info": "ByteaArray"
+      },
+      {
+        "ordinal": 11,
         "name": "blocks_cids",
         "type_info": "TextArray"
       },
       {
-        "ordinal": 10,
+        "ordinal": 12,
         "name": "handle",
         "type_info": "Text"
       },
       {
-        "ordinal": 11,
+        "ordinal": 13,
         "name": "active",
         "type_info": "Bool"
       },
       {
-        "ordinal": 12,
+        "ordinal": 14,
         "name": "status",
         "type_info": "Text"
       },
       {
-        "ordinal": 13,
+        "ordinal": 15,
         "name": "rev",
         "type_info": "Text"
       }
@@ -93,8 +103,10 @@
       true,
       true,
       true,
+      true,
+      true,
       true
     ]
   },
-  "hash": "0d32a592a97ad47c65aa37cf0d45417f2966fcbd688be7434626ae5f6971fa1f"
+  "hash": "c47746c256fd8c9d386551f3bd270e9a96fecdb713642d07d60fb4aeb02c8426"
 }

.sqlx/query-c6e3388fc39983f1787917606ba3194c72322d2d1ec54402c262194791a2b06a.json

@@ -0,0 +1,18 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO webauthn_challenges (id, did, challenge, challenge_type, state_json, expires_at)\n               VALUES ($1, $2, $3, 'discoverable', $4, $5)",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Uuid",
+        "Text",
+        "Bytea",
+        "Text",
+        "Timestamptz"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "c6e3388fc39983f1787917606ba3194c72322d2d1ec54402c262194791a2b06a"
+}

Cargo.lock

@@ -7405,7 +7405,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-api"
-version = "0.4.7"
+version = "0.5.3"
 dependencies = [
  "anyhow",
  "axum",
@@ -7456,7 +7456,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-auth"
-version = "0.4.7"
+version = "0.5.3"
 dependencies = [
  "anyhow",
  "base32",
@@ -7479,7 +7479,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-cache"
-version = "0.4.7"
+version = "0.5.3"
 dependencies = [
  "async-trait",
  "base64 0.22.1",
@@ -7493,7 +7493,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-comms"
-version = "0.4.7"
+version = "0.5.3"
 dependencies = [
  "async-trait",
  "base64 0.22.1",
@@ -7511,7 +7511,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-config"
-version = "0.4.7"
+version = "0.5.3"
 dependencies = [
  "confique",
  "serde",
@@ -7519,7 +7519,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-crypto"
-version = "0.4.7"
+version = "0.5.3"
 dependencies = [
  "aes-gcm",
  "base64 0.22.1",
@@ -7535,7 +7535,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-db"
-version = "0.4.7"
+version = "0.5.3"
 dependencies = [
  "async-trait",
  "chrono",
@@ -7552,7 +7552,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-db-traits"
-version = "0.4.7"
+version = "0.5.3"
 dependencies = [
  "async-trait",
  "base64 0.22.1",
@@ -7568,7 +7568,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-infra"
-version = "0.4.7"
+version = "0.5.3"
 dependencies = [
  "async-trait",
  "bytes",
@@ -7579,7 +7579,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-lexicon"
-version = "0.4.7"
+version = "0.5.3"
 dependencies = [
  "chrono",
  "hickory-resolver",
@@ -7597,7 +7597,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-oauth"
-version = "0.4.7"
+version = "0.5.3"
 dependencies = [
  "anyhow",
  "axum",
@@ -7620,7 +7620,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-oauth-server"
-version = "0.4.7"
+version = "0.5.3"
 dependencies = [
  "axum",
  "base64 0.22.1",
@@ -7653,7 +7653,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-pds"
-version = "0.4.7"
+version = "0.5.3"
 dependencies = [
  "aes-gcm",
  "anyhow",
@@ -7738,13 +7738,14 @@ dependencies = [
  "urlencoding",
  "uuid",
  "webauthn-rs",
+ "webauthn-rs-proto",
  "wiremock",
  "zip",
 ]
 
 [[package]]
 name = "tranquil-repo"
-version = "0.4.7"
+version = "0.5.3"
 dependencies = [
  "bytes",
  "cid",
@@ -7756,7 +7757,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-ripple"
-version = "0.4.7"
+version = "0.5.3"
 dependencies = [
  "async-trait",
  "backon",
@@ -7781,7 +7782,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-scopes"
-version = "0.4.7"
+version = "0.5.3"
 dependencies = [
  "axum",
  "futures",
@@ -7797,7 +7798,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-server"
-version = "0.4.7"
+version = "0.5.3"
 dependencies = [
  "axum",
  "clap",
@@ -7818,7 +7819,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-signal"
-version = "0.4.7"
+version = "0.5.3"
 dependencies = [
  "async-trait",
  "chrono",
@@ -7829,17 +7830,19 @@ dependencies = [
  "serde",
  "serde_json",
  "sqlx",
+ "tempfile",
  "thiserror 2.0.18",
  "tokio",
  "tokio-util",
  "tracing",
+ "tranquil-signal",
  "url",
  "uuid",
 ]
 
 [[package]]
 name = "tranquil-storage"
-version = "0.4.7"
+version = "0.5.3"
 dependencies = [
  "async-trait",
  "aws-config",
@@ -7856,7 +7859,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-store"
-version = "0.4.7"
+version = "0.5.3"
 dependencies = [
  "async-trait",
  "bytes",
@@ -7869,6 +7872,7 @@ dependencies = [
  "jacquard-common",
  "jacquard-repo",
  "k256",
+ "libc",
  "lsm-tree",
  "memmap2",
  "multihash",
@@ -7876,6 +7880,7 @@ dependencies = [
  "postcard",
  "proptest",
  "rand 0.8.5",
+ "rayon",
  "serde",
  "serde_ipld_dagcbor",
  "serde_json",
@@ -7888,10 +7893,12 @@ dependencies = [
  "tikv-jemallocator",
  "tokio",
  "tracing",
+ "tracing-subscriber",
  "tranquil-db",
  "tranquil-db-traits",
  "tranquil-oauth",
  "tranquil-repo",
+ "tranquil-store",
  "tranquil-types",
  "uuid",
  "xxhash-rust",
@@ -7899,7 +7906,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-sync"
-version = "0.4.7"
+version = "0.5.3"
 dependencies = [
  "anyhow",
  "axum",
@@ -7921,7 +7928,7 @@ dependencies = [
 
 [[package]]
 name = "tranquil-types"
-version = "0.4.7"
+version = "0.5.3"
 dependencies = [
  "chrono",
  "cid",

Cargo.toml

@@ -26,7 +26,7 @@ members = [
 ]
 
 [workspace.package]
-version = "0.5.0"
+version = "0.5.3"
 edition = "2024"
 license = "AGPL-3.0-or-later"
 
@@ -126,7 +126,7 @@ tracing = "0.1"
 tracing-subscriber = "0.3"
 urlencoding = "2.1"
 uuid = { version = "1.19", features = ["v4", "v5", "v7", "fast-rng", "serde"] }
-webauthn-rs = { version = "0.5", features = ["danger-allow-state-serialisation", "danger-user-presence-only-security-keys"] }
+webauthn-rs = { version = "0.5", features = ["danger-allow-state-serialisation", "danger-user-presence-only-security-keys", "conditional-ui"] }
 webauthn-rs-proto = "0.5"
 zip = { version = "7.0", default-features = false, features = ["deflate"] }
 

Dockerfile

@@ -1,7 +1,10 @@
-FROM denoland/deno:alpine AS frontend
+FROM node:24-alpine AS frontend
+RUN corepack enable && corepack prepare pnpm@latest --activate
 WORKDIR /app
+COPY frontend/package.json frontend/pnpm-lock.yaml ./
+RUN pnpm install --frozen-lockfile
 COPY frontend/ ./
-RUN deno task build
+RUN pnpm build
 
 FROM rust:1.92-alpine AS builder
 RUN apk add --no-cache ca-certificates musl-dev pkgconfig openssl-dev openssl-libs-static mold clang protoc

README.md

@@ -2,17 +2,25 @@
 
 A Personal Data Server for the AT Protocol.
 
-Bluesky runs on a federated protocol called AT Protocol. Your account lives on a PDS, a server that stores your posts, profile, follows, and cryptographic keys. Bluesky hosts one for you at bsky.social, but you can run your own. Self-hosting means you control your data; you're not dependent on any company's servers, and your account + data is actually yours.
+"A what for the what?" -> glad you asked: Bluesky, Tangled, and a bunch of other web applications use a federated protocol called AT Protocol (atproto). Your account lives on a PDS, a server that stores your posts, profile, follows, cryptographic keys, et cetera. The beauty is that a PDS is the *only* place your data lives permanently - so you can navigate any atproto app knowing that your data is yours and not getting locked behind any one app's walls.
 
-This particular PDS thrives under harsh conditions. It is a dandelion growing through the cracks in the sidewalk concrete.
+We came together to make this PDS to enable and empower our users to better host their data on this shared protocol. All of our decisions as a project are guided by their usefulness to the community: PDS hosters and end-users both. 
 
-It has full compatibility with Bluesky's reference PDS.
+Comparatively: Bluesky the company created a "reference PDS" that we can self-host quite easily, and that's great, but Bluesky has an incentive to make software for themselvess first & foremost, then secondly their software can be useful for us self-hosters. In contrast, Tranquil is not from a company, and will never be.
 
 ## What's different about Tranquil PDS
 
-It is a superset of the reference PDS, including: passkeys and 2FA (WebAuthn/FIDO2, TOTP, backup codes, trusted devices), SSO login and signup, did:web support (PDS-hosted subdomains or bring-your-own), multi-channel communication (email, discord, telegram, signal) for verification and alerts, granular OAuth scopes with a consent UI showing human-readable descriptions, app passwords with granular permissions (read-only, post-only, or custom scopes), account delegation (letting others manage an account with configurable permission levels), and a built-in web UI for account management, repo browsing, and admin.
+It is a superset of the reference PDS, including:
+- passkeys and 2FA: WebAuthn/FIDO2, TOTP, backup codes, trusted devices
+- SSO login and signup
+- did:web support: PDS-hosted subdomains or bring-your-own
+- multi-channel communication: you can be notified via email, discord, telegram, and signal for verification and alerts
+- granular OAuth scopes with a consent UI that allows unchecking specific scopes
+- app passwords with the same granular permission scope system as OAuth
+- account delegation: letting others manage an account with configurable permission levels
+- a built-in web UI for account management, repo browsing, and admin
 
-The PDS itself is a single binary with no nodeJS runtime. However, at time of writing, Tranquil requires postgres running separately. Blobs are stored on the local filesystem by default (S3 optional). Valkey is also optional (as an alternative to the built-in cache).
+Unlike the ref PDS, Tranquil itself is compiled to a single binary with no nodeJS runtime. However, at time of writing, Tranquil requires postgres running separately.
 
 ## Quick Start
 
@@ -55,14 +63,38 @@ podman-compose -f docker-compose.prod.yaml up -d
 - [Containers](docs/install-containers.md)
 - [Kubernetes](docs/install-kubernetes.md)
 
-## Maintainers to ping
+## Community
 
-- [@lewis.moe](https://bsky.app/profile/did:plc:3fwecdnvtcscjnrx2p4n7alz)
-- [@nel.pet](https://bsky.app/profile/did:plc:h5wsnqetncv6lu2weom35lg2)
+### "Let's connect!" or whatever linkedin-types say
 
-## Thanks
+We currently don't have a shared space to chat and organize Tranquil things, but we're very interested in changing that in the near future. What do you suggest? Anything but a discord server.
 
-This project is very grateful to [@nonbinary.computer](https://bsky.app/profile/did:plc:yfvwmnlztr4dwkb7hwz55r2g), [@juli.ee](https://bsky.app/profile/did:plc:7vimlesenouvuaqvle42yhvo), [@mary.my.id](https://bsky.app/profile/did:plc:ia76kvnndjutgedggx2ibrem), and [@baileytownsend.dev](https://bsky.app/profile/did:plc:rnpkyqnmsw4ipey6eotbdnnf) for their help and their code to lean on.
+### Core team
+
+- [@oyster.cafe](https://tangled.org/did:plc:3fwecdnvtcscjnrx2p4n7alz)
+- [@nel.pet](https://tangled.org/did:plc:h5wsnqetncv6lu2weom35lg2)
+
+### Amazing contributers
+
+- [@isabelroses.com](https://tangled.org/did:plc:qxichs7jsycphrsmbujwqbfb)
+- [@quilling.dev](https://tangled.org/did:plc:jrtgsidnmxaen4offglr5lsh)
+- [@koi.rip](https://tangled.org/did:plc:b26ewgkrnx3yvsp2cdao3ntu)
+- [@bas.sh](https://tangled.org/did:plc:c52wep6lj4sfbsqiz3yvb55h)
+- [@nekomimi.pet](https://tangled.org/did:plc:ttdrpj45ibqunmfhdsb4zdwq)
+- [@islacant.win](https://tangled.org/did:plc:aut6evcs6d6ngaunqgfhdzzu)
+- [@a.starrysky.fyi](https://tangled.org/did:plc:uuyqs6y3pwtbteet4swt5i5y)
+- [@sans-self.org](https://tangled.org/did:plc:wydyrngmxbcsqdvhmd7whmye)
+- [@tachyonism.tngl.sh](https://tangled.org/did:plc:w6qiwij62bmdugsd3gemhpy2)
+- Could be your name here too!
+
+### Tranquil PDS instances in the wild!
+
+- [Tranquil Farm](https://tranquil.farm)
+- Your instance here!! Don't be a stranger.
+
+### Special thanks
+
+This project is very grateful to [@nonbinary.computer](https://tangled.org/did:plc:yfvwmnlztr4dwkb7hwz55r2g), [@juliet.paris](https://tangled.org/did:plc:hs3aly5l26pozymy4b6hz7ae), [@mary.my.id](https://tangled.org/did:plc:ia76kvnndjutgedggx2ibrem), [@baileytownsend.dev](https://tangled.org/did:plc:rnpkyqnmsw4ipey6eotbdnnf), and [@ptr.pet](https://tangled.org/did:plc:dfl62fgb7wtjj3fcbb72naae) for their help and their code to lean on.
 
 ## License
 

crates/tranquil-api/src/admin/account/mod.rs

@@ -11,6 +11,7 @@ pub use info::{
 };
 pub use search::{SearchAccountsOutput, SearchAccountsParams, search_accounts};
 pub use update::{
-    UpdateAccountEmailInput, UpdateAccountHandleInput, UpdateAccountPasswordInput,
-    update_account_email, update_account_handle, update_account_password,
+    SetAdminStatusInput, UpdateAccountEmailInput, UpdateAccountHandleInput,
+    UpdateAccountPasswordInput, set_admin_status, update_account_email, update_account_handle,
+    update_account_password,
 };

crates/tranquil-api/src/admin/account/update.rs

@@ -1,6 +1,6 @@
 use axum::{Json, extract::State};
 use serde::Deserialize;
-use tracing::{error, warn};
+use tracing::{error, info, warn};
 use tranquil_pds::api::EmptyResponse;
 use tranquil_pds::api::error::ApiError;
 use tranquil_pds::auth::{Admin, Auth};
@@ -166,3 +166,34 @@ pub async fn update_account_password(
         }
     }
 }
+
+#[derive(Deserialize)]
+pub struct SetAdminStatusInput {
+    pub did: Did,
+    pub admin: bool,
+}
+
+pub async fn set_admin_status(
+    State(state): State<AppState>,
+    auth: Auth<Admin>,
+    Json(input): Json<SetAdminStatusInput>,
+) -> Result<Json<EmptyResponse>, ApiError> {
+    info!(
+        actor = %auth.did,
+        target = %input.did,
+        admin = input.admin,
+        "admin status change"
+    );
+
+    state
+        .repos
+        .user
+        .set_admin_status(&input.did, input.admin)
+        .await
+        .map_err(|e| {
+            error!("DB error setting admin status: {:?}", e);
+            ApiError::InternalError(None)
+        })?;
+
+    Ok(Json(EmptyResponse {}))
+}

crates/tranquil-api/src/admin/mod.rs

@@ -7,7 +7,7 @@ pub mod status;
 
 pub use account::{
     delete_account, get_account_info, get_account_infos, search_accounts, send_email,
-    update_account_email, update_account_handle, update_account_password,
+    set_admin_status, update_account_email, update_account_handle, update_account_password,
 };
 pub use config::{get_server_config, update_server_config};
 pub use invite::{

crates/tranquil-api/src/common.rs

@@ -231,10 +231,19 @@ pub async fn verify_credential(
     app_passwords
         .into_iter()
         .find(|app| bcrypt::verify(password, &app.password_hash).unwrap_or(false))
-        .map(|app| CredentialMatch::AppPassword {
-            name: app.name,
-            scopes: app.scopes,
-            controller_did: app.created_by_controller_did,
+        .map(|app| {
+            let scopes = app.scopes.unwrap_or_else(|| {
+                if app.privilege.is_privileged() {
+                    "transition:generic transition:chat.bsky".to_string()
+                } else {
+                    "transition:generic".to_string()
+                }
+            });
+            CredentialMatch::AppPassword {
+                name: app.name,
+                scopes: Some(scopes),
+                controller_did: app.created_by_controller_did,
+            }
         })
 }
 

crates/tranquil-api/src/lib.rs

@@ -330,6 +330,7 @@ pub fn api_routes() -> axum::Router<AppState> {
             get(admin::get_invite_codes),
         )
         .route("/_admin.getServerStats", get(admin::get_server_stats))
+        .route("/_admin.setAdminStatus", post(admin::set_admin_status))
         .route("/_admin.getSignalStatus", get(admin::get_signal_status))
         .route("/_admin.linkSignalDevice", post(admin::link_signal_device))
         .route(

crates/tranquil-api/src/repo/import.rs

@@ -328,7 +328,8 @@ pub async fn import_repo(
                 new_root_str, new_rev_str
             );
             if !is_migration
-                && let Err(e) = sequence_import_event(&state, did, &new_root_cid_link).await
+                && let Err(e) =
+                    sequence_import_event(&state, did, &new_root_cid_link, &commit_bytes).await
             {
                 warn!("Failed to sequence import event: {:?}", e);
             }
@@ -395,15 +396,23 @@ async fn sequence_import_event(
     state: &AppState,
     did: &Did,
     commit_cid: &CidLink,
-) -> Result<(), tranquil_db::DbError> {
-    let data = tranquil_db::CommitEventData {
+    commit_bytes: &[u8],
+) -> Result<(), tranquil_db_traits::DbError> {
+    let commit_cid_parsed = commit_cid
+        .to_cid()
+        .expect("CidLink invariant: validated at construction");
+    let inline_commit = tranquil_db_traits::EventBlockInline {
+        cid_bytes: commit_cid_parsed.to_bytes(),
+        data: commit_bytes.to_vec(),
+    };
+    let data = tranquil_db_traits::CommitEventData {
         did: did.clone(),
-        event_type: tranquil_db::RepoEventType::Commit,
+        event_type: tranquil_db_traits::RepoEventType::Commit,
         commit_cid: Some(commit_cid.clone()),
         prev_cid: None,
         ops: Some(serde_json::json!([])),
         blobs: Some(vec![]),
-        blocks_cids: Some(vec![]),
+        blocks: Some(vec![inline_commit]),
         prev_data_cid: None,
         rev: None,
     };

crates/tranquil-api/src/repo/record/batch.rs

@@ -94,7 +94,7 @@ async fn process_single_write(
             ops.push(RecordOp::Create {
                 collection: collection.clone(),
                 rkey: rkey.clone(),
-                cid: record_cid,
+                cid: tranquil_pds::cid_types::RecordCid::from(record_cid),
             });
             Ok(WriteAccumulator {
                 mst: new_mst,
@@ -134,7 +134,15 @@ async fn process_single_write(
                 .map_err(|_| ApiError::InternalError(Some("Failed to store record".into())))?;
             let key = format!("{}/{}", collection, rkey);
             modified_keys.push(key.clone());
-            let prev_record_cid = mst.get(&key).await.ok().flatten();
+            let prev_record_cid = mst
+                .get(&key)
+                .await
+                .map_err(|e| {
+                    ApiError::InternalError(Some(format!("Failed to read prev record: {}", e)))
+                })?
+                .ok_or_else(|| {
+                    ApiError::InvalidRequest("Update target record does not exist".into())
+                })?;
             let new_mst = mst
                 .update(&key, record_cid)
                 .await
@@ -150,8 +158,8 @@ async fn process_single_write(
             ops.push(RecordOp::Update {
                 collection: collection.clone(),
                 rkey: rkey.clone(),
-                cid: record_cid,
-                prev: prev_record_cid,
+                cid: tranquil_pds::cid_types::RecordCid::from(record_cid),
+                prev: tranquil_pds::cid_types::RecordCid::from(prev_record_cid),
             });
             Ok(WriteAccumulator {
                 mst: new_mst,
@@ -166,7 +174,15 @@ async fn process_single_write(
         WriteOp::Delete { collection, rkey } => {
             let key = format!("{}/{}", collection, rkey);
             modified_keys.push(key.clone());
-            let prev_record_cid = mst.get(&key).await.ok().flatten();
+            let prev_record_cid = mst
+                .get(&key)
+                .await
+                .map_err(|e| {
+                    ApiError::InternalError(Some(format!("Failed to read prev record: {}", e)))
+                })?
+                .ok_or_else(|| {
+                    ApiError::InvalidRequest("Delete target record does not exist".into())
+                })?;
             let new_mst = mst
                 .delete(&key)
                 .await
@@ -176,7 +192,7 @@ async fn process_single_write(
             ops.push(RecordOp::Delete {
                 collection: collection.clone(),
                 rkey: rkey.clone(),
-                prev: prev_record_cid,
+                prev: tranquil_pds::cid_types::RecordCid::from(prev_record_cid),
             });
             Ok(WriteAccumulator {
                 mst: new_mst,

crates/tranquil-api/src/repo/record/delete.rs

@@ -1,20 +1,16 @@
 use crate::repo::record::write::{CommitInfo, prepare_repo_write};
 use axum::{Json, extract::State};
 use cid::Cid;
-use jacquard_repo::{commit::Commit, mst::Mst, storage::BlockStore};
 use serde::{Deserialize, Serialize};
 use serde_json::json;
 use std::str::FromStr;
-use std::sync::Arc;
 use tracing::error;
 use tranquil_pds::api::error::ApiError;
 use tranquil_pds::auth::{Active, Auth, VerifyScope};
-use tranquil_pds::repo::TrackingBlockStore;
-use tranquil_pds::repo_ops::{
-    CommitError, FinalizeParams, RecordOp, begin_repo_write, finalize_repo_write,
-};
+use tranquil_pds::cid_types::RecordCid;
+use tranquil_pds::repo_ops::{FinalizeParams, RecordOp, begin_repo_write, finalize_repo_write};
 use tranquil_pds::state::AppState;
-use tranquil_pds::types::{AtIdentifier, AtUri, Did, Nsid, Rkey};
+use tranquil_pds::types::{AtIdentifier, AtUri, Nsid, Rkey};
 
 #[derive(Deserialize)]
 pub struct DeleteRecordInput {
@@ -59,20 +55,23 @@ pub async fn delete_record(
         }
     }
 
-    let prev_record_cid = mst.get(&key).await.ok().flatten();
-    if prev_record_cid.is_none() {
+    let prev_record_cid = mst.get(&key).await.map_err(|e| {
+        error!("Failed to read prev record from MST: {}", e);
+        ApiError::InternalError(Some("Failed to read MST".into()))
+    })?;
+    let Some(prev_record_cid) = prev_record_cid else {
         return Ok(Json(DeleteRecordOutput { commit: None }));
-    }
+    };
 
     let new_mst = mst.delete(&key).await.map_err(|e| {
-        error!("Failed to delete from MST: {:?}", e);
+        error!("Failed to delete from MST: {}", e);
         ApiError::InternalError(Some("Failed to delete from MST".into()))
     })?;
 
     let op = RecordOp::Delete {
         collection: input.collection.clone(),
         rkey: input.rkey.clone(),
-        prev: prev_record_cid,
+        prev: RecordCid::from(prev_record_cid),
     };
 
     let modified_keys = [key];
@@ -109,124 +108,3 @@ pub async fn delete_record(
         }),
     }))
 }
-
-use uuid::Uuid;
-
-pub async fn delete_record_internal(
-    state: &AppState,
-    did: &Did,
-    user_id: Uuid,
-    collection: &Nsid,
-    rkey: &Rkey,
-) -> Result<(), CommitError> {
-    use tranquil_pds::repo_ops::{CommitParams, RecordOp, commit_and_log};
-
-    let _write_lock = state.repo_write_locks.lock(user_id).await;
-
-    let root_cid_str = state
-        .repos
-        .repo
-        .get_repo_root_cid_by_user_id(user_id)
-        .await
-        .map_err(|e| CommitError::DatabaseError(e.to_string()))?
-        .ok_or(CommitError::RepoNotFound)?;
-
-    let current_root_cid =
-        Cid::from_str(root_cid_str.as_str()).map_err(|e| CommitError::InvalidCid(e.to_string()))?;
-
-    let tracking_store = TrackingBlockStore::new(state.block_store.clone());
-    let commit_bytes = tracking_store
-        .get(&current_root_cid)
-        .await
-        .map_err(|e| CommitError::BlockStoreFailed(format!("{:?}", e)))?
-        .ok_or(CommitError::BlockStoreFailed(
-            "Commit block not found".into(),
-        ))?;
-
-    let commit = Commit::from_cbor(&commit_bytes)
-        .map_err(|e| CommitError::CommitParseFailed(format!("{:?}", e)))?;
-
-    let mst = Mst::load(Arc::new(tracking_store.clone()), commit.data, None);
-    let key = format!("{}/{}", collection, rkey);
-
-    let prev_record_cid = mst
-        .get(&key)
-        .await
-        .map_err(|e| CommitError::MstOperationFailed(format!("{:?}", e)))?;
-
-    let Some(prev_cid) = prev_record_cid else {
-        return Ok(());
-    };
-
-    let new_mst = mst
-        .delete(&key)
-        .await
-        .map_err(|e| CommitError::MstOperationFailed(format!("{:?}", e)))?;
-
-    let new_mst_root = new_mst
-        .persist()
-        .await
-        .map_err(|e| CommitError::MstOperationFailed(format!("{:?}", e)))?;
-
-    let op = RecordOp::Delete {
-        collection: collection.clone(),
-        rkey: rkey.clone(),
-        prev: Some(prev_cid),
-    };
-
-    let mut new_mst_blocks = std::collections::BTreeMap::new();
-    let mut old_mst_blocks = std::collections::BTreeMap::new();
-
-    new_mst
-        .blocks_for_path(&key, &mut new_mst_blocks)
-        .await
-        .map_err(|e| CommitError::MstOperationFailed(format!("{:?}", e)))?;
-
-    mst.blocks_for_path(&key, &mut old_mst_blocks)
-        .await
-        .map_err(|e| CommitError::MstOperationFailed(format!("{:?}", e)))?;
-
-    let obsolete_cids: Vec<Cid> = std::iter::once(current_root_cid)
-        .chain(
-            old_mst_blocks
-                .keys()
-                .filter(|cid| !new_mst_blocks.contains_key(*cid))
-                .copied(),
-        )
-        .chain(std::iter::once(prev_cid))
-        .collect();
-
-    let mut relevant_blocks = new_mst_blocks;
-    relevant_blocks.extend(old_mst_blocks);
-
-    let written_cids: Vec<Cid> = tracking_store
-        .get_all_relevant_cids()
-        .into_iter()
-        .chain(relevant_blocks.keys().copied())
-        .collect::<std::collections::HashSet<_>>()
-        .into_iter()
-        .collect();
-
-    let written_cids_str: Vec<String> = written_cids.iter().map(ToString::to_string).collect();
-
-    let deleted_uri = AtUri::from_parts(did.as_str(), collection.as_str(), rkey.as_str());
-    commit_and_log(
-        state,
-        CommitParams {
-            did,
-            user_id,
-            current_root_cid: Some(current_root_cid),
-            prev_data_cid: Some(commit.data),
-            new_mst_root,
-            ops: vec![op],
-            blocks_cids: &written_cids_str,
-            blobs: &[],
-            obsolete_cids,
-            backlinks_to_add: vec![],
-            backlinks_to_remove: vec![deleted_uri],
-        },
-    )
-    .await?;
-
-    Ok(())
-}

crates/tranquil-api/src/repo/record/mod.rs

@@ -10,7 +10,7 @@ pub use pagination::PaginationDirection;
 pub use validation_mode::ValidationMode;
 
 pub use batch::apply_writes;
-pub use delete::{DeleteRecordInput, delete_record, delete_record_internal};
+pub use delete::{DeleteRecordInput, delete_record};
 pub use read::{GetRecordInput, ListRecordsInput, ListRecordsOutput, get_record, list_records};
 pub use tranquil_pds::repo_ops::*;
 pub use write::{

crates/tranquil-api/src/repo/record/write.rs

@@ -172,7 +172,7 @@ pub async fn create_record(
                 ops.push(RecordOp::Delete {
                     collection: conflict_collection,
                     rkey: conflict_rkey,
-                    prev: Some(prev_cid),
+                    prev: tranquil_pds::cid_types::RecordCid::from(prev_cid),
                 });
                 conflict_uris_to_cleanup.push(conflict_uri);
             }
@@ -197,7 +197,7 @@ pub async fn create_record(
     ops.push(RecordOp::Create {
         collection: input.collection.clone(),
         rkey: rkey.clone(),
-        cid: record_cid,
+        cid: tranquil_pds::cid_types::RecordCid::from(record_cid),
     });
 
     let modified_keys: Vec<String> = ops
@@ -338,41 +338,38 @@ pub async fn put_record(
         }));
     }
 
-    let is_update = existing_cid.is_some();
-    let new_mst = if is_update {
-        mst.update(&key, record_cid)
-            .await
-            .map_err(|_| ApiError::InternalError(Some("Failed to update MST".into())))?
-    } else {
-        mst.add(&key, record_cid)
-            .await
-            .map_err(|_| ApiError::InternalError(Some("Failed to add to MST".into())))?
-    };
-
-    let op = if is_update {
-        RecordOp::Update {
-            collection: input.collection.clone(),
-            rkey: input.rkey.clone(),
-            cid: record_cid,
-            prev: existing_cid,
+    let record_uri = AtUri::from_parts(&did, &input.collection, &input.rkey);
+    let (new_mst, op, is_update, backlinks_to_remove) = match existing_cid {
+        Some(prev_cid) => {
+            let new_mst = mst
+                .update(&key, record_cid)
+                .await
+                .map_err(|_| ApiError::InternalError(Some("Failed to update MST".into())))?;
+            let op = RecordOp::Update {
+                collection: input.collection.clone(),
+                rkey: input.rkey.clone(),
+                cid: tranquil_pds::cid_types::RecordCid::from(record_cid),
+                prev: tranquil_pds::cid_types::RecordCid::from(prev_cid),
+            };
+            (new_mst, op, true, vec![record_uri.clone()])
         }
-    } else {
-        RecordOp::Create {
-            collection: input.collection.clone(),
-            rkey: input.rkey.clone(),
-            cid: record_cid,
+        None => {
+            let new_mst = mst
+                .add(&key, record_cid)
+                .await
+                .map_err(|_| ApiError::InternalError(Some("Failed to add to MST".into())))?;
+            let op = RecordOp::Create {
+                collection: input.collection.clone(),
+                rkey: input.rkey.clone(),
+                cid: tranquil_pds::cid_types::RecordCid::from(record_cid),
+            };
+            (new_mst, op, false, vec![])
         }
     };
 
     let modified_keys = [key];
     let blob_cids = extract_blob_cids(&input.record);
-
-    let record_uri = AtUri::from_parts(&did, &input.collection, &input.rkey);
     let backlinks_to_add = extract_backlinks(&record_uri, &input.record);
-    let backlinks_to_remove = match is_update {
-        true => vec![record_uri.clone()],
-        false => vec![],
-    };
 
     let commit_result = finalize_repo_write(
         &state,

crates/tranquil-api/src/server/app_password.rs

@@ -132,7 +132,14 @@ pub async fn create_app_password(
         };
         (scope_result, Some(controller.clone()))
     } else {
-        (input.scopes.clone(), None)
+        let scopes = match input.scopes {
+            Some(ref s) => s.clone(),
+            None => match input.privileged {
+                Some(false) => "transition:generic".to_string(),
+                _ => "transition:generic transition:chat.bsky".to_string(),
+            },
+        };
+        (Some(scopes), None)
     };
 
     let password = generate_app_password();

crates/tranquil-api/src/server/passkey_account.rs

@@ -393,15 +393,15 @@ pub async fn create_passkey_account(
             Ok(token_meta) => {
                 let refresh_jti = uuid::Uuid::new_v4().to_string();
                 let refresh_expires = chrono::Utc::now() + chrono::Duration::hours(24);
-                let session_data = tranquil_db::SessionTokenCreate {
+                let session_data = tranquil_db_traits::SessionTokenCreate {
                     did: did_typed.clone(),
                     access_jti: token_meta.jti.clone(),
                     refresh_jti,
                     access_expires_at: token_meta.expires_at,
                     refresh_expires_at: refresh_expires,
-                    login_type: tranquil_db::LoginType::Modern,
+                    login_type: tranquil_db_traits::LoginType::Modern,
                     mfa_verified: false,
-                    scope: Some("transition:generic".to_string()),
+                    scope: Some("transition:generic transition:chat.bsky".to_string()),
                     controller_did: None,
                     app_password_name: None,
                 };

crates/tranquil-config/src/lib.rs

@@ -269,6 +269,38 @@ impl TranquilConfig {
         {
             errors.push("tranquil_store.handler_threads must be at least 1".to_string());
         }
+        if self.tranquil_store.eventlog_max_event_payload == 0 {
+            errors.push(
+                "tranquil_store.eventlog_max_event_payload \
+                 (TRANQUIL_STORE_EVENTLOG_MAX_EVENT_PAYLOAD) must be at least 1; \
+                 a value of 0 would reject every event"
+                    .to_string(),
+            );
+        }
+
+        // -- scheduled / event retention --------------------------------------
+        const MAX_RETENTION_SECS: u64 = (i64::MAX / 1000) as u64;
+        if self.scheduled.event_retention_max_age_secs > MAX_RETENTION_SECS {
+            errors.push(format!(
+                "scheduled.event_retention_max_age_secs (EVENT_RETENTION_MAX_AGE_SECS) \
+                 must be at most {MAX_RETENTION_SECS} (chrono::Duration limit); got {}",
+                self.scheduled.event_retention_max_age_secs
+            ));
+        }
+        if self.scheduled.event_retention_interval_secs > 0 {
+            let backfill_secs = u64::try_from(self.firehose.backfill_hours.max(0))
+                .unwrap_or(0)
+                .saturating_mul(3600);
+            if self.scheduled.event_retention_max_age_secs < backfill_secs {
+                errors.push(format!(
+                    "scheduled.event_retention_max_age_secs ({}) is shorter than \
+                     firehose.backfill_hours ({}h = {backfill_secs}s): \
+                     relays would receive cursor responses pointing at pruned events. \
+                     Increase event_retention_max_age_secs or decrease firehose.backfill_hours.",
+                    self.scheduled.event_retention_max_age_secs, self.firehose.backfill_hours,
+                ));
+            }
+        }
 
         // -- cache ------------------------------------------------------------
         match self.cache.backend.as_str() {
@@ -688,6 +720,10 @@ pub struct FirehoseConfig {
     #[config(env = "FIREHOSE_MAX_LAG", default = 5000)]
     pub max_lag: u64,
 
+    /// Maximum concurrent full-repo exports, eg. getRepo without `since`.
+    #[config(env = "MAX_CONCURRENT_REPO_EXPORTS", default = 4)]
+    pub max_concurrent_repo_exports: usize,
+
     /// List of relay / crawler notification URLs.
     #[config(env = "CRAWLERS", parse_env = split_comma_list)]
     pub crawlers: Option<Vec<String>>,
@@ -1057,9 +1093,45 @@ pub struct ScheduledConfig {
     #[config(env = "SCHEDULED_DELETE_CHECK_INTERVAL_SECS", default = 3600)]
     pub delete_check_interval_secs: u64,
 
-    /// Interval in seconds between block garbage collection cycles.
-    #[config(env = "BLOCK_GC_INTERVAL_SECS", default = 21600)]
-    pub block_gc_interval_secs: u64,
+    /// Interval in seconds between data file compaction scans (tranquil-store only).
+    /// Set to 0 to disable.
+    #[config(env = "COMPACTION_INTERVAL_SECS", default = 3600)]
+    pub compaction_interval_secs: u64,
+
+    /// Liveness ratio threshold below which a data file is compacted (0.0-1.0).
+    #[config(env = "COMPACTION_LIVENESS_THRESHOLD", default = 0.7)]
+    pub compaction_liveness_threshold: f64,
+
+    /// Grace period in milliseconds before a zero-refcount block can be removed by compaction.
+    #[config(env = "COMPACTION_GRACE_PERIOD_MS", default = 600000)]
+    pub compaction_grace_period_ms: u64,
+
+    /// Interval in seconds between reachability walk runs (tranquil-store only).
+    /// Set to 0 to disable. Default: weekly.
+    #[config(env = "REACHABILITY_WALK_INTERVAL_SECS", default = 604800)]
+    pub reachability_walk_interval_secs: u64,
+
+    /// Interval in seconds between continuous archival passes (tranquil-store only).
+    /// Sealed eventlog segments are copied to the archival destination each tick.
+    /// Set to 0 to disable. Default: 60 seconds.
+    #[config(env = "ARCHIVAL_INTERVAL_SECS", default = 60)]
+    pub archival_interval_secs: u64,
+
+    /// Archival destination directory for sealed eventlog segments.
+    /// If unset, archival is disabled.
+    #[config(env = "ARCHIVAL_DEST_DIR")]
+    pub archival_dest_dir: Option<String>,
+
+    /// Maximum age of events retained in the eventlog before pruning.
+    /// Per the atproto firehose spec, the relay backfill window only needs
+    /// to cover "hours or days". Default: 7 days.
+    #[config(env = "EVENT_RETENTION_MAX_AGE_SECS", default = 604800)]
+    pub event_retention_max_age_secs: u64,
+
+    /// Interval in seconds between event retention prune passes.
+    /// Set to 0 to disable. Default: hourly.
+    #[config(env = "EVENT_RETENTION_INTERVAL_SECS", default = 3600)]
+    pub event_retention_interval_secs: u64,
 }
 
 #[derive(Debug, Config)]
@@ -1079,6 +1151,38 @@ pub struct TranquilStoreConfig {
     /// Number of handler threads. Defaults to available_parallelism / 2.
     #[config(env = "TRANQUIL_STORE_HANDLER_THREADS")]
     pub handler_threads: Option<usize>,
+
+    /// Maximum total bytes of pending (unsynced) eventlog payloads. Appenders
+    /// block once this budget is exhausted until in-flight events drain via
+    /// fsync. Set to 0 to disable backpressure (unbounded). Default: 1 GiB.
+    #[config(
+        env = "TRANQUIL_STORE_EVENTLOG_PENDING_BYTES_BUDGET",
+        default = 1_073_741_824
+    )]
+    pub eventlog_pending_bytes_budget: u64,
+
+    /// Maximum size of an individual eventlog payload in bytes. Single events
+    /// larger than this are rejected at append time. Default: 256 MiB.
+    #[config(
+        env = "TRANQUIL_STORE_EVENTLOG_MAX_EVENT_PAYLOAD",
+        default = 268_435_456
+    )]
+    pub eventlog_max_event_payload: u32,
+
+    /// Maximum size of an individual blockstore data file in bytes. When the
+    /// active data file reaches this size it is rolled over and becomes
+    /// eligible for compaction. Default: 256 MiB.
+    #[config(env = "TRANQUIL_STORE_MAX_BLOCKSTORE_FILE_SIZE", default = 268_435_456)]
+    pub max_blockstore_file_size: u64,
+
+    /// Maximum size of an individual eventlog segment file in bytes. When the
+    /// active segment reaches this size it is sealed and a new one is created.
+    /// Safe to change on a running instance. Default: 256 MiB.
+    #[config(
+        env = "TRANQUIL_STORE_MAX_EVENTLOG_SEGMENT_SIZE",
+        default = 268_435_456
+    )]
+    pub max_eventlog_segment_size: u64,
 }
 
 /// Generate a TOML configuration template with all available options,

crates/tranquil-db-traits/src/lib.rs

@@ -33,11 +33,12 @@ pub use oauth::{
     ScopePreference, TokenFamilyId, TrustedDeviceRow, TwoFactorChallenge,
 };
 pub use repo::{
-    AccountStatus, ApplyCommitError, ApplyCommitInput, ApplyCommitResult, BrokenGenesisCommit,
-    CommitEventData, EventBlocksCids, FullRecordInfo, ImportBlock, ImportRecord, ImportRepoError,
-    RecordDelete, RecordInfo, RecordUpsert, RecordWithTakedown, RepoAccountInfo, RepoEventNotifier,
-    RepoEventReceiver, RepoEventType, RepoInfo, RepoListItem, RepoRepository, RepoSeqEvent,
-    RepoWithoutRev, SequencedEvent, UserNeedingRecordBlobsBackfill, UserWithoutBlocks,
+    AccountStatus, ApplyCommitError, ApplyCommitInput, ApplyCommitResult, CommitEventData,
+    EventBlockInline, EventBlocks, FullRecordInfo, ImportBlock, ImportRecord, ImportRepoError,
+    PruneCount, RecordDelete, RecordInfo, RecordUpsert, RecordWithTakedown, RepoAccountInfo,
+    RepoEventNotifier, RepoEventReceiver, RepoEventType, RepoInfo, RepoListItem, RepoRepository,
+    RepoSeqEvent, RepoWithoutRev, SequencedEvent, UserNeedingRecordBlobsBackfill,
+    UserWithoutBlocks,
 };
 pub use scope::{DbScope, InvalidScopeError};
 pub use sequence::{SequenceNumber, deserialize_optional_sequence};

crates/tranquil-db-traits/src/repo.rs

@@ -165,13 +165,6 @@ pub struct RepoWithoutRev {
     pub repo_root_cid: CidLink,
 }
 
-#[derive(Debug, Clone)]
-pub struct BrokenGenesisCommit {
-    pub seq: SequenceNumber,
-    pub did: Did,
-    pub commit_cid: Option<CidLink>,
-}
-
 #[derive(Debug, Clone)]
 pub struct UserWithoutBlocks {
     pub user_id: Uuid,
@@ -190,6 +183,51 @@ pub struct RepoSeqEvent {
     pub seq: SequenceNumber,
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PruneCount {
+    Rows(u64),
+    Segments(u64),
+}
+
+impl PruneCount {
+    pub fn is_zero(&self) -> bool {
+        match self {
+            Self::Rows(n) | Self::Segments(n) => *n == 0,
+        }
+    }
+
+    pub fn count(&self) -> u64 {
+        match self {
+            Self::Rows(n) | Self::Segments(n) => *n,
+        }
+    }
+
+    pub fn unit(&self) -> &'static str {
+        match self {
+            Self::Rows(_) => "rows",
+            Self::Segments(_) => "segments",
+        }
+    }
+}
+
+impl std::fmt::Display for PruneCount {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{} {}", self.count(), self.unit())
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct EventBlockInline {
+    pub cid_bytes: Vec<u8>,
+    pub data: Vec<u8>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum EventBlocks {
+    Inline(Vec<EventBlockInline>),
+    LegacyCids(Vec<String>),
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct SequencedEvent {
     pub seq: SequenceNumber,
@@ -201,7 +239,7 @@ pub struct SequencedEvent {
     pub prev_data_cid: Option<CidLink>,
     pub ops: Option<serde_json::Value>,
     pub blobs: Option<Vec<String>>,
-    pub blocks_cids: Option<Vec<String>>,
+    pub blocks: Option<EventBlocks>,
     pub handle: Option<Handle>,
     pub active: Option<bool>,
     pub status: Option<AccountStatus>,
@@ -216,17 +254,11 @@ pub struct CommitEventData {
     pub prev_cid: Option<CidLink>,
     pub ops: Option<serde_json::Value>,
     pub blobs: Option<Vec<String>>,
-    pub blocks_cids: Option<Vec<String>>,
+    pub blocks: Option<Vec<EventBlockInline>>,
     pub prev_data_cid: Option<CidLink>,
     pub rev: Option<String>,
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct EventBlocksCids {
-    pub blocks_cids: Option<Vec<String>>,
-    pub commit_cid: Option<CidLink>,
-}
-
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct RepoListItem {
     pub did: Did,
@@ -413,11 +445,6 @@ pub trait RepoRepository: Send + Sync {
 
     async fn count_user_blocks(&self, user_id: Uuid) -> Result<i64, DbError>;
 
-    async fn find_unreferenced_blocks(
-        &self,
-        candidate_cids: &[Vec<u8>],
-    ) -> Result<Vec<Vec<u8>>, DbError>;
-
     async fn insert_commit_event(&self, data: &CommitEventData) -> Result<SequenceNumber, DbError>;
 
     async fn insert_identity_event(
@@ -437,6 +464,7 @@ pub trait RepoRepository: Send + Sync {
         did: &Did,
         commit_cid: &CidLink,
         rev: Option<&str>,
+        commit_bytes: &[u8],
     ) -> Result<SequenceNumber, DbError>;
 
     async fn insert_genesis_commit_event(
@@ -445,20 +473,18 @@ pub trait RepoRepository: Send + Sync {
         commit_cid: &CidLink,
         mst_root_cid: &CidLink,
         rev: &str,
+        commit_bytes: &[u8],
+        mst_root_bytes: &[u8],
     ) -> Result<SequenceNumber, DbError>;
 
-    async fn update_seq_blocks_cids(
-        &self,
-        seq: SequenceNumber,
-        blocks_cids: &[String],
-    ) -> Result<(), DbError>;
-
     async fn delete_sequences_except(
         &self,
         did: &Did,
         keep_seq: SequenceNumber,
     ) -> Result<(), DbError>;
 
+    async fn prune_events_older_than(&self, cutoff: DateTime<Utc>) -> Result<PruneCount, DbError>;
+
     async fn get_max_seq(&self) -> Result<SequenceNumber, DbError>;
 
     async fn get_min_seq_since(
@@ -491,12 +517,6 @@ pub trait RepoRepository: Send + Sync {
         limit: i64,
     ) -> Result<Vec<SequencedEvent>, DbError>;
 
-    async fn get_events_since_rev(
-        &self,
-        did: &Did,
-        since_rev: &str,
-    ) -> Result<Vec<EventBlocksCids>, DbError>;
-
     async fn list_repos_paginated(
         &self,
         cursor_did: Option<&Did>,
@@ -521,8 +541,6 @@ pub trait RepoRepository: Send + Sync {
         input: ApplyCommitInput,
     ) -> Result<ApplyCommitResult, ApplyCommitError>;
 
-    async fn get_broken_genesis_commits(&self) -> Result<Vec<BrokenGenesisCommit>, DbError>;
-
     async fn get_users_without_blocks(&self) -> Result<Vec<UserWithoutBlocks>, DbError>;
 
     async fn get_users_needing_record_blobs_backfill(

crates/tranquil-db-traits/src/user.rs

@@ -144,12 +144,12 @@ pub trait UserRepository: Send + Sync {
 
     async fn get_by_email(&self, email: &str) -> Result<Option<UserForVerification>, DbError>;
 
-    async fn get_login_check_by_handle_or_email(
+    async fn get_login_check_by_identifier(
         &self,
         identifier: &str,
     ) -> Result<Option<UserLoginCheck>, DbError>;
 
-    async fn get_login_info_by_handle_or_email(
+    async fn get_login_info_by_identifier(
         &self,
         identifier: &str,
     ) -> Result<Option<UserLoginInfo>, DbError>;
@@ -358,6 +358,19 @@ pub trait UserRepository: Send + Sync {
         challenge_type: WebauthnChallengeType,
     ) -> Result<(), DbError>;
 
+    async fn save_discoverable_challenge(
+        &self,
+        request_key: &str,
+        state_json: &str,
+    ) -> Result<Uuid, DbError>;
+
+    async fn load_discoverable_challenge(
+        &self,
+        request_key: &str,
+    ) -> Result<Option<String>, DbError>;
+
+    async fn delete_discoverable_challenge(&self, request_key: &str) -> Result<(), DbError>;
+
     async fn get_totp_record(&self, did: &Did) -> Result<Option<TotpRecord>, DbError>;
 
     async fn get_totp_record_state(&self, did: &Did) -> Result<Option<TotpRecordState>, DbError>;

crates/tranquil-db/src/postgres/repo.rs

@@ -2,10 +2,10 @@ use async_trait::async_trait;
 use chrono::{DateTime, Utc};
 use sqlx::PgPool;
 use tranquil_db_traits::{
-    AccountStatus, BrokenGenesisCommit, CommitEventData, DbError, EventBlocksCids, FullRecordInfo,
-    ImportBlock, ImportRecord, ImportRepoError, RecordInfo, RecordWithTakedown, RepoAccountInfo,
-    RepoEventType, RepoInfo, RepoListItem, RepoRepository, RepoWithoutRev, SequenceNumber,
-    SequencedEvent, UserNeedingRecordBlobsBackfill, UserWithoutBlocks,
+    AccountStatus, CommitEventData, DbError, EventBlockInline, EventBlocks, FullRecordInfo,
+    ImportBlock, ImportRecord, ImportRepoError, PruneCount, RecordInfo, RecordWithTakedown,
+    RepoAccountInfo, RepoEventType, RepoInfo, RepoListItem, RepoRepository, RepoWithoutRev,
+    SequenceNumber, SequencedEvent, UserNeedingRecordBlobsBackfill, UserWithoutBlocks,
 };
 use tranquil_types::{AtUri, CidLink, Did, Handle, Nsid, Rkey};
 use uuid::Uuid;
@@ -27,6 +27,8 @@ struct SequencedEventRow {
     prev_data_cid: Option<String>,
     ops: Option<serde_json::Value>,
     blobs: Option<Vec<String>>,
+    block_cids: Option<Vec<Vec<u8>>>,
+    block_data: Option<Vec<Vec<u8>>>,
     blocks_cids: Option<Vec<String>>,
     handle: Option<String>,
     active: Option<bool>,
@@ -34,6 +36,81 @@ struct SequencedEventRow {
     rev: Option<String>,
 }
 
+fn row_to_event_blocks(
+    block_cids: Option<Vec<Vec<u8>>>,
+    block_data: Option<Vec<Vec<u8>>>,
+    legacy_blocks_cids: Option<Vec<String>>,
+) -> Result<Option<EventBlocks>, DbError> {
+    match (block_cids, block_data) {
+        (Some(cids), Some(data)) if cids.len() == data.len() => match cids.is_empty() {
+            true => Ok(legacy_fallback(legacy_blocks_cids)),
+            false => Ok(Some(EventBlocks::Inline(
+                cids.into_iter()
+                    .zip(data)
+                    .map(|(cid_bytes, data)| EventBlockInline { cid_bytes, data })
+                    .collect(),
+            ))),
+        },
+        (Some(_), Some(_)) => Err(DbError::CorruptData(
+            "repo_seq.block_cids/block_data length mismatch",
+        )),
+        (Some(_), None) | (None, Some(_)) => Err(DbError::CorruptData(
+            "repo_seq.block_cids/block_data partially populated",
+        )),
+        (None, None) => Ok(legacy_fallback(legacy_blocks_cids)),
+    }
+}
+
+fn legacy_fallback(legacy_blocks_cids: Option<Vec<String>>) -> Option<EventBlocks> {
+    match legacy_blocks_cids {
+        Some(cids) if !cids.is_empty() => Some(EventBlocks::LegacyCids(cids)),
+        _ => None,
+    }
+}
+
+fn inline_to_paired_blocks(blocks: Option<&[EventBlockInline]>) -> (Vec<Vec<u8>>, Vec<Vec<u8>>) {
+    blocks
+        .map(|bs| {
+            bs.iter()
+                .map(|b| (b.cid_bytes.clone(), b.data.clone()))
+                .unzip()
+        })
+        .unwrap_or_default()
+}
+
+fn inline_into_paired_blocks(
+    blocks: Option<Vec<EventBlockInline>>,
+) -> (Vec<Vec<u8>>, Vec<Vec<u8>>) {
+    blocks
+        .map(|bs| bs.into_iter().map(|b| (b.cid_bytes, b.data)).unzip())
+        .unwrap_or_default()
+}
+
+fn map_sequenced_row(r: SequencedEventRow) -> Result<SequencedEvent, DbError> {
+    let status = r
+        .status
+        .as_deref()
+        .and_then(AccountStatus::parse)
+        .or_else(|| r.active.filter(|a| *a).map(|_| AccountStatus::Active));
+    let blocks = row_to_event_blocks(r.block_cids, r.block_data, r.blocks_cids)?;
+    Ok(SequencedEvent {
+        seq: r.seq.into(),
+        did: Did::from(r.did),
+        created_at: r.created_at,
+        event_type: r.event_type,
+        commit_cid: r.commit_cid.map(CidLink::from),
+        prev_cid: r.prev_cid.map(CidLink::from),
+        prev_data_cid: r.prev_data_cid.map(CidLink::from),
+        ops: r.ops,
+        blobs: r.blobs,
+        blocks,
+        handle: r.handle.map(Handle::from),
+        active: r.active,
+        status,
+        rev: r.rev,
+    })
+}
+
 pub struct PostgresRepoRepository {
     pool: PgPool,
 }
@@ -618,30 +695,6 @@ impl RepoRepository for PostgresRepoRepository {
         Ok(count)
     }
 
-    async fn find_unreferenced_blocks(
-        &self,
-        candidate_cids: &[Vec<u8>],
-    ) -> Result<Vec<Vec<u8>>, DbError> {
-        match candidate_cids.is_empty() {
-            true => Ok(Vec::new()),
-            false => {
-                let rows = sqlx::query!(
-                    r#"
-                    SELECT t.cid FROM UNNEST($1::bytea[]) AS t(cid)
-                    WHERE NOT EXISTS (
-                        SELECT 1 FROM user_blocks WHERE block_cid = t.cid
-                    )
-                    "#,
-                    candidate_cids,
-                )
-                .fetch_all(&self.pool)
-                .await
-                .map_err(map_sqlx_error)?;
-                Ok(rows.into_iter().filter_map(|r| r.cid).collect())
-            }
-        }
-    }
-
     async fn get_user_block_cids_since_rev(
         &self,
         user_id: Uuid,
@@ -664,10 +717,11 @@ impl RepoRepository for PostgresRepoRepository {
     }
 
     async fn insert_commit_event(&self, data: &CommitEventData) -> Result<SequenceNumber, DbError> {
+        let (block_cids, block_data) = inline_to_paired_blocks(data.blocks.as_deref());
         let seq = sqlx::query_scalar!(
             r#"
-            INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, blocks_cids, prev_data_cid, rev)
-            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+            INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, block_cids, block_data, prev_data_cid, rev)
+            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
             RETURNING seq
             "#,
             data.did.as_str(),
@@ -676,7 +730,8 @@ impl RepoRepository for PostgresRepoRepository {
             data.prev_cid.as_ref().map(|c| c.as_str()),
             data.ops,
             data.blobs.as_deref(),
-            data.blocks_cids.as_deref(),
+            &block_cids as &[Vec<u8>],
+            &block_data as &[Vec<u8>],
             data.prev_data_cid.as_ref().map(|c| c.as_str()),
             data.rev
         )
@@ -748,16 +803,25 @@ impl RepoRepository for PostgresRepoRepository {
         did: &Did,
         commit_cid: &CidLink,
         rev: Option<&str>,
+        commit_bytes: &[u8],
     ) -> Result<SequenceNumber, DbError> {
+        let cid_bytes = commit_cid
+            .to_cid()
+            .map(|c| c.to_bytes())
+            .unwrap_or_default();
+        let block_cids: Vec<Vec<u8>> = vec![cid_bytes];
+        let block_data: Vec<Vec<u8>> = vec![commit_bytes.to_vec()];
         let seq = sqlx::query_scalar!(
             r#"
-            INSERT INTO repo_seq (did, event_type, commit_cid, rev)
-            VALUES ($1, 'sync', $2, $3)
+            INSERT INTO repo_seq (did, event_type, commit_cid, rev, block_cids, block_data)
+            VALUES ($1, 'sync', $2, $3, $4, $5)
             RETURNING seq
             "#,
             did.as_str(),
             commit_cid.as_str(),
-            rev
+            rev,
+            &block_cids as &[Vec<u8>],
+            &block_data as &[Vec<u8>]
         )
         .fetch_one(&self.pool)
         .await
@@ -777,16 +841,27 @@ impl RepoRepository for PostgresRepoRepository {
         commit_cid: &CidLink,
         mst_root_cid: &CidLink,
         rev: &str,
+        commit_bytes: &[u8],
+        mst_root_bytes: &[u8],
     ) -> Result<SequenceNumber, DbError> {
         let ops = serde_json::json!([]);
         let blobs: Vec<String> = vec![];
-        let blocks_cids: Vec<String> = vec![mst_root_cid.to_string(), commit_cid.to_string()];
+        let commit_cid_bytes = commit_cid
+            .to_cid()
+            .map(|c| c.to_bytes())
+            .unwrap_or_default();
+        let mst_cid_bytes = mst_root_cid
+            .to_cid()
+            .map(|c| c.to_bytes())
+            .unwrap_or_default();
+        let block_cids: Vec<Vec<u8>> = vec![commit_cid_bytes, mst_cid_bytes];
+        let block_data: Vec<Vec<u8>> = vec![commit_bytes.to_vec(), mst_root_bytes.to_vec()];
         let prev_cid: Option<&str> = None;
 
         let seq = sqlx::query_scalar!(
             r#"
-            INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, blocks_cids, rev)
-            VALUES ($1, 'commit', $2, $3::TEXT, $4, $5, $6, $7)
+            INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, block_cids, block_data, rev)
+            VALUES ($1, 'commit', $2, $3::TEXT, $4, $5, $6, $7, $8)
             RETURNING seq
             "#,
             did.as_str(),
@@ -794,7 +869,8 @@ impl RepoRepository for PostgresRepoRepository {
             prev_cid,
             ops,
             &blobs,
-            &blocks_cids,
+            &block_cids as &[Vec<u8>],
+            &block_data as &[Vec<u8>],
             rev
         )
         .fetch_one(&self.pool)
@@ -809,23 +885,6 @@ impl RepoRepository for PostgresRepoRepository {
         Ok(seq.into())
     }
 
-    async fn update_seq_blocks_cids(
-        &self,
-        seq: SequenceNumber,
-        blocks_cids: &[String],
-    ) -> Result<(), DbError> {
-        sqlx::query!(
-            "UPDATE repo_seq SET blocks_cids = $1 WHERE seq = $2",
-            blocks_cids,
-            seq.as_i64()
-        )
-        .execute(&self.pool)
-        .await
-        .map_err(map_sqlx_error)?;
-
-        Ok(())
-    }
-
     async fn delete_sequences_except(
         &self,
         did: &Did,
@@ -843,6 +902,15 @@ impl RepoRepository for PostgresRepoRepository {
         Ok(())
     }
 
+    async fn prune_events_older_than(&self, cutoff: DateTime<Utc>) -> Result<PruneCount, DbError> {
+        let result = sqlx::query!("DELETE FROM repo_seq WHERE created_at < $1", cutoff)
+            .execute(&self.pool)
+            .await
+            .map_err(map_sqlx_error)?;
+
+        Ok(PruneCount::Rows(result.rows_affected()))
+    }
+
     async fn get_max_seq(&self) -> Result<SequenceNumber, DbError> {
         let seq = sqlx::query_scalar!(r#"SELECT COALESCE(MAX(seq), 0) as "max!" FROM repo_seq"#)
             .fetch_one(&self.pool)
@@ -893,35 +961,12 @@ impl RepoRepository for PostgresRepoRepository {
         since_seq: SequenceNumber,
         limit: Option<i64>,
     ) -> Result<Vec<SequencedEvent>, DbError> {
-        let map_row = |r: SequencedEventRow| {
-            let status = r
-                .status
-                .as_deref()
-                .and_then(AccountStatus::parse)
-                .or_else(|| r.active.filter(|a| *a).map(|_| AccountStatus::Active));
-            SequencedEvent {
-                seq: r.seq.into(),
-                did: Did::from(r.did),
-                created_at: r.created_at,
-                event_type: r.event_type,
-                commit_cid: r.commit_cid.map(CidLink::from),
-                prev_cid: r.prev_cid.map(CidLink::from),
-                prev_data_cid: r.prev_data_cid.map(CidLink::from),
-                ops: r.ops,
-                blobs: r.blobs,
-                blocks_cids: r.blocks_cids,
-                handle: r.handle.map(Handle::from),
-                active: r.active,
-                status,
-                rev: r.rev,
-            }
-        };
         match limit {
             Some(lim) => {
                 let rows = sqlx::query_as!(
                     SequencedEventRow,
                     r#"SELECT seq, did, created_at, event_type as "event_type: RepoEventType", commit_cid, prev_cid, prev_data_cid,
-                              ops, blobs, blocks_cids, handle, active, status, rev
+                              ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev
                        FROM repo_seq
                        WHERE seq > $1
                        ORDER BY seq ASC
@@ -932,13 +977,13 @@ impl RepoRepository for PostgresRepoRepository {
                 .fetch_all(&self.pool)
                 .await
                 .map_err(map_sqlx_error)?;
-                Ok(rows.into_iter().map(map_row).collect())
+                rows.into_iter().map(map_sequenced_row).collect()
             }
             None => {
                 let rows = sqlx::query_as!(
                     SequencedEventRow,
                     r#"SELECT seq, did, created_at, event_type as "event_type: RepoEventType", commit_cid, prev_cid, prev_data_cid,
-                              ops, blobs, blocks_cids, handle, active, status, rev
+                              ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev
                        FROM repo_seq
                        WHERE seq > $1
                        ORDER BY seq ASC"#,
@@ -947,7 +992,7 @@ impl RepoRepository for PostgresRepoRepository {
                 .fetch_all(&self.pool)
                 .await
                 .map_err(map_sqlx_error)?;
-                Ok(rows.into_iter().map(map_row).collect())
+                rows.into_iter().map(map_sequenced_row).collect()
             }
         }
     }
@@ -957,9 +1002,10 @@ impl RepoRepository for PostgresRepoRepository {
         start_seq: SequenceNumber,
         end_seq: SequenceNumber,
     ) -> Result<Vec<SequencedEvent>, DbError> {
-        let rows = sqlx::query!(
+        let rows = sqlx::query_as!(
+            SequencedEventRow,
             r#"SELECT seq, did, created_at, event_type as "event_type: RepoEventType", commit_cid, prev_cid, prev_data_cid,
-                      ops, blobs, blocks_cids, handle, active, status, rev
+                      ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev
                FROM repo_seq
                WHERE seq > $1 AND seq < $2
                ORDER BY seq ASC"#,
@@ -969,41 +1015,17 @@ impl RepoRepository for PostgresRepoRepository {
         .fetch_all(&self.pool)
         .await
         .map_err(map_sqlx_error)?;
-        Ok(rows
-            .into_iter()
-            .map(|r| {
-                let status = r
-                    .status
-                    .as_deref()
-                    .and_then(AccountStatus::parse)
-                    .or_else(|| r.active.filter(|a| *a).map(|_| AccountStatus::Active));
-                SequencedEvent {
-                    seq: r.seq.into(),
-                    did: Did::from(r.did),
-                    created_at: r.created_at,
-                    event_type: r.event_type,
-                    commit_cid: r.commit_cid.map(CidLink::from),
-                    prev_cid: r.prev_cid.map(CidLink::from),
-                    prev_data_cid: r.prev_data_cid.map(CidLink::from),
-                    ops: r.ops,
-                    blobs: r.blobs,
-                    blocks_cids: r.blocks_cids,
-                    handle: r.handle.map(Handle::from),
-                    active: r.active,
-                    status,
-                    rev: r.rev,
-                }
-            })
-            .collect())
+        rows.into_iter().map(map_sequenced_row).collect()
     }
 
     async fn get_event_by_seq(
         &self,
         seq: SequenceNumber,
     ) -> Result<Option<SequencedEvent>, DbError> {
-        let row = sqlx::query!(
+        let row = sqlx::query_as!(
+            SequencedEventRow,
             r#"SELECT seq, did, created_at, event_type as "event_type: RepoEventType", commit_cid, prev_cid, prev_data_cid,
-                      ops, blobs, blocks_cids, handle, active, status, rev
+                      ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev
                FROM repo_seq
                WHERE seq = $1"#,
             seq.as_i64()
@@ -1011,29 +1033,7 @@ impl RepoRepository for PostgresRepoRepository {
         .fetch_optional(&self.pool)
         .await
         .map_err(map_sqlx_error)?;
-        Ok(row.map(|r| {
-            let status = r
-                .status
-                .as_deref()
-                .and_then(AccountStatus::parse)
-                .or_else(|| r.active.filter(|a| *a).map(|_| AccountStatus::Active));
-            SequencedEvent {
-                seq: r.seq.into(),
-                did: Did::from(r.did),
-                created_at: r.created_at,
-                event_type: r.event_type,
-                commit_cid: r.commit_cid.map(CidLink::from),
-                prev_cid: r.prev_cid.map(CidLink::from),
-                prev_data_cid: r.prev_data_cid.map(CidLink::from),
-                ops: r.ops,
-                blobs: r.blobs,
-                blocks_cids: r.blocks_cids,
-                handle: r.handle.map(Handle::from),
-                active: r.active,
-                status,
-                rev: r.rev,
-            }
-        }))
+        row.map(map_sequenced_row).transpose()
     }
 
     async fn get_events_since_cursor(
@@ -1041,9 +1041,10 @@ impl RepoRepository for PostgresRepoRepository {
         cursor: SequenceNumber,
         limit: i64,
     ) -> Result<Vec<SequencedEvent>, DbError> {
-        let rows = sqlx::query!(
+        let rows = sqlx::query_as!(
+            SequencedEventRow,
             r#"SELECT seq, did, created_at, event_type as "event_type: RepoEventType", commit_cid, prev_cid, prev_data_cid,
-                      ops, blobs, blocks_cids, handle, active, status, rev
+                      ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev
                FROM repo_seq
                WHERE seq > $1
                ORDER BY seq ASC
@@ -1054,58 +1055,7 @@ impl RepoRepository for PostgresRepoRepository {
         .fetch_all(&self.pool)
         .await
         .map_err(map_sqlx_error)?;
-        Ok(rows
-            .into_iter()
-            .map(|r| {
-                let status = r
-                    .status
-                    .as_deref()
-                    .and_then(AccountStatus::parse)
-                    .or_else(|| r.active.filter(|a| *a).map(|_| AccountStatus::Active));
-                SequencedEvent {
-                    seq: r.seq.into(),
-                    did: Did::from(r.did),
-                    created_at: r.created_at,
-                    event_type: r.event_type,
-                    commit_cid: r.commit_cid.map(CidLink::from),
-                    prev_cid: r.prev_cid.map(CidLink::from),
-                    prev_data_cid: r.prev_data_cid.map(CidLink::from),
-                    ops: r.ops,
-                    blobs: r.blobs,
-                    blocks_cids: r.blocks_cids,
-                    handle: r.handle.map(Handle::from),
-                    active: r.active,
-                    status,
-                    rev: r.rev,
-                }
-            })
-            .collect())
-    }
-
-    async fn get_events_since_rev(
-        &self,
-        did: &Did,
-        since_rev: &str,
-    ) -> Result<Vec<EventBlocksCids>, DbError> {
-        let rows = sqlx::query!(
-            r#"SELECT blocks_cids, commit_cid
-               FROM repo_seq
-               WHERE did = $1 AND rev > $2
-               ORDER BY seq DESC"#,
-            did.as_str(),
-            since_rev
-        )
-        .fetch_all(&self.pool)
-        .await
-        .map_err(map_sqlx_error)?;
-
-        Ok(rows
-            .into_iter()
-            .map(|r| EventBlocksCids {
-                blocks_cids: r.blocks_cids,
-                commit_cid: r.commit_cid.map(CidLink::from),
-            })
-            .collect())
+        rows.into_iter().map(map_sequenced_row).collect()
     }
 
     async fn list_repos_paginated(
@@ -1450,22 +1400,24 @@ impl RepoRepository for PostgresRepoRepository {
             .map_err(|e| ApplyCommitError::Database(e.to_string()))?;
         }
 
-        let event = &input.commit_event;
+        let event = input.commit_event;
+        let (event_block_cids, event_block_data) = inline_into_paired_blocks(event.blocks);
         let seq: i64 = sqlx::query_scalar(
             r#"
-            INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, blocks_cids, prev_data_cid, rev)
-            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+            INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, block_cids, block_data, prev_data_cid, rev)
+            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
             RETURNING seq
             "#,
         )
-        .bind(&event.did)
+        .bind(event.did.as_str())
         .bind(event.event_type.as_str())
-        .bind(&event.commit_cid)
-        .bind(&event.prev_cid)
+        .bind(event.commit_cid.as_ref().map(|c| c.as_str()))
+        .bind(event.prev_cid.as_ref().map(|c| c.as_str()))
         .bind(&event.ops)
         .bind(&event.blobs)
-        .bind(&event.blocks_cids)
-        .bind(&event.prev_data_cid)
+        .bind(&event_block_cids)
+        .bind(&event_block_data)
+        .bind(event.prev_data_cid.as_ref().map(|c| c.as_str()))
         .bind(&event.rev)
         .fetch_one(&mut *tx)
         .await
@@ -1486,32 +1438,6 @@ impl RepoRepository for PostgresRepoRepository {
         })
     }
 
-    async fn get_broken_genesis_commits(
-        &self,
-    ) -> Result<Vec<tranquil_db_traits::BrokenGenesisCommit>, DbError> {
-        let rows = sqlx::query!(
-            r#"
-            SELECT seq, did, commit_cid
-            FROM repo_seq
-            WHERE event_type = 'commit'
-              AND prev_cid IS NULL
-              AND (blocks_cids IS NULL OR array_length(blocks_cids, 1) IS NULL OR array_length(blocks_cids, 1) = 0)
-            "#
-        )
-        .fetch_all(&self.pool)
-        .await
-        .map_err(map_sqlx_error)?;
-
-        Ok(rows
-            .into_iter()
-            .map(|r| BrokenGenesisCommit {
-                seq: r.seq.into(),
-                did: Did::from(r.did),
-                commit_cid: r.commit_cid.map(CidLink::from),
-            })
-            .collect())
-    }
-
     async fn get_users_without_blocks(&self) -> Result<Vec<UserWithoutBlocks>, DbError> {
         let rows: Vec<(Uuid, String, Option<String>)> = sqlx::query_as(
             r#"

crates/tranquil-db/src/postgres/user.rs

@@ -1102,6 +1102,59 @@ impl UserRepository for PostgresUserRepository {
         Ok(())
     }
 
+    async fn save_discoverable_challenge(
+        &self,
+        request_key: &str,
+        state_json: &str,
+    ) -> Result<Uuid, DbError> {
+        let id = Uuid::new_v4();
+        let challenge = id.as_bytes().to_vec();
+        let expires_at = chrono::Utc::now() + chrono::Duration::minutes(5);
+        sqlx::query!(
+            r#"INSERT INTO webauthn_challenges (id, did, challenge, challenge_type, state_json, expires_at)
+               VALUES ($1, $2, $3, 'discoverable', $4, $5)"#,
+            id,
+            request_key,
+            challenge,
+            state_json,
+            expires_at,
+        )
+        .execute(&self.pool)
+        .await
+        .map_err(map_sqlx_error)?;
+
+        Ok(id)
+    }
+
+    async fn load_discoverable_challenge(
+        &self,
+        request_key: &str,
+    ) -> Result<Option<String>, DbError> {
+        let row = sqlx::query_scalar!(
+            r#"SELECT state_json FROM webauthn_challenges
+               WHERE did = $1 AND challenge_type = 'discoverable' AND expires_at > NOW()
+               ORDER BY created_at DESC LIMIT 1"#,
+            request_key,
+        )
+        .fetch_optional(&self.pool)
+        .await
+        .map_err(map_sqlx_error)?;
+
+        Ok(row)
+    }
+
+    async fn delete_discoverable_challenge(&self, request_key: &str) -> Result<(), DbError> {
+        sqlx::query!(
+            "DELETE FROM webauthn_challenges WHERE did = $1 AND challenge_type = 'discoverable'",
+            request_key,
+        )
+        .execute(&self.pool)
+        .await
+        .map_err(map_sqlx_error)?;
+
+        Ok(())
+    }
+
     async fn get_totp_record(&self, did: &Did) -> Result<Option<TotpRecord>, DbError> {
         let row = sqlx::query!(
             "SELECT secret_encrypted, encryption_version, verified FROM user_totp WHERE did = $1",
@@ -1330,12 +1383,12 @@ impl UserRepository for PostgresUserRepository {
         Ok(())
     }
 
-    async fn get_login_check_by_handle_or_email(
+    async fn get_login_check_by_identifier(
         &self,
         identifier: &str,
     ) -> Result<Option<UserLoginCheck>, DbError> {
         sqlx::query!(
-            "SELECT did, password_hash FROM users WHERE handle = $1 OR email = $1",
+            "SELECT did, password_hash FROM users WHERE handle = $1 OR did = $1",
             identifier
         )
         .fetch_optional(&self.pool)
@@ -1349,7 +1402,7 @@ impl UserRepository for PostgresUserRepository {
         })
     }
 
-    async fn get_login_info_by_handle_or_email(
+    async fn get_login_info_by_identifier(
         &self,
         identifier: &str,
     ) -> Result<Option<UserLoginInfo>, DbError> {
@@ -1361,7 +1414,7 @@ impl UserRepository for PostgresUserRepository {
                    email_verified, discord_verified, telegram_verified, signal_verified,
                    account_type as "account_type!: AccountType"
             FROM users
-            WHERE handle = $1 OR email = $1
+            WHERE handle = $1 OR did = $1
             "#,
             identifier
         )
@@ -1524,7 +1577,7 @@ impl UserRepository for PostgresUserRepository {
                 COALESCE((SELECT (value_json)::boolean FROM account_preferences WHERE user_id = u.id AND name = 'email_auth_factor' ORDER BY created_at DESC LIMIT 1), false) as "email_2fa_enabled!"
             FROM users u
             JOIN user_keys k ON u.id = k.user_id
-            WHERE u.handle = $1 OR u.email = $1 OR u.did = $1"#,
+            WHERE u.handle = $1 OR u.did = $1"#,
             identifier
         )
         .fetch_optional(&self.pool)

crates/tranquil-lexicon/src/validate.rs

@@ -322,7 +322,9 @@ fn validate_blob_ref(
 
     if let Some(ref accept) = lex_blob.accept {
         let mime_type = obj.get("mimeType").and_then(|v| v.as_str()).unwrap_or("");
-        let matched = accept.iter().any(|pattern| mime_type_matches_accept_pattern(mime_type, pattern));
+        let matched = accept
+            .iter()
+            .any(|pattern| mime_type_matches_accept_pattern(mime_type, pattern));
         if !mime_type.is_empty() && !matched {
             return Err(LexValidationError::field(
                 path,

crates/tranquil-oauth-server/src/endpoints/authorize/login.rs

@@ -108,7 +108,7 @@ pub async fn authorize_get(
         match state
             .repos
             .user
-            .get_login_check_by_handle_or_email(normalized.as_str())
+            .get_login_check_by_identifier(normalized.as_str())
             .await
         {
             Ok(Some(user)) => {
@@ -401,7 +401,7 @@ pub async fn authorize_post(
     let user = match state
         .repos
         .user
-        .get_login_info_by_handle_or_email(normalized_username.as_str())
+        .get_login_info_by_identifier(normalized_username.as_str())
         .await
     {
         Ok(Some(u)) => u,
@@ -410,7 +410,7 @@ pub async fn authorize_post(
                 &form.password,
                 "$2b$12$LQv3c1yqBWVHxkd0LHAkCOYz6TtxMQJqhN8/X4.VTtYw1ZzQKZqmK",
             );
-            return show_login_error("Invalid handle/email or password.", json_response);
+            return show_login_error("Invalid identifier or password.", json_response);
         }
         Err(_) => return show_login_error("An error occurred. Please try again.", json_response),
     };
@@ -486,7 +486,7 @@ pub async fn authorize_post(
         None => false,
     };
     if !password_valid {
-        return show_login_error("Invalid handle/email or password.", json_response);
+        return show_login_error("Invalid identifier or password.", json_response);
     }
     let is_verified = user.channel_verification.has_any_verified();
     if !is_verified {

crates/tranquil-oauth-server/src/endpoints/authorize/passkey.rs

@@ -22,7 +22,7 @@ pub async fn check_user_has_passkeys(
     let user = state
         .repos
         .user
-        .get_login_check_by_handle_or_email(bare_identifier.as_str())
+        .get_login_check_by_identifier(bare_identifier.as_str())
         .await;
 
     let has_passkeys = match user {
@@ -55,7 +55,7 @@ pub async fn check_user_security_status(
     let user = state
         .repos
         .user
-        .get_login_check_by_handle_or_email(normalized_identifier.as_str())
+        .get_login_check_by_identifier(normalized_identifier.as_str())
         .await;
 
     let (has_passkeys, has_totp, has_password, is_delegated, did): (
@@ -99,7 +99,7 @@ pub async fn check_user_security_status(
 #[derive(Debug, Deserialize)]
 pub struct PasskeyStartInput {
     pub request_uri: String,
-    pub identifier: String,
+    pub identifier: Option<String>,
     pub delegated_did: Option<String>,
 }
 
@@ -160,14 +160,91 @@ pub async fn passkey_start(
             .into_response();
     }
 
+    match form.identifier.filter(|s| !s.trim().is_empty()) {
+        Some(identifier) => {
+            passkey_start_named(
+                state,
+                identifier,
+                form.delegated_did,
+                request_data,
+                passkey_start_request_id,
+            )
+            .await
+        }
+        None => passkey_start_discoverable(state, passkey_start_request_id).await,
+    }
+}
+
+async fn passkey_start_discoverable(
+    state: AppState,
+    request_id: RequestId,
+) -> Response {
+    let (rcr, auth_state) = match state.webauthn_config.start_discoverable_authentication() {
+        Ok(result) => result,
+        Err(e) => {
+            tracing::error!(error = %e, "Failed to start discoverable passkey authentication");
+            return (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                Json(serde_json::json!({
+                    "error": "server_error",
+                    "error_description": "Failed to start authentication."
+                })),
+            )
+                .into_response();
+        }
+    };
+
+    let state_json = match serde_json::to_string(&auth_state) {
+        Ok(j) => j,
+        Err(e) => {
+            tracing::error!(error = %e, "Failed to serialize authentication state");
+            return (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                Json(serde_json::json!({
+                    "error": "server_error",
+                    "error_description": "An error occurred."
+                })),
+            )
+                .into_response();
+        }
+    };
+
+    if let Err(e) = state
+        .repos
+        .user
+        .save_discoverable_challenge(request_id.as_str(), &state_json)
+        .await
+    {
+        tracing::error!(error = %e, "Failed to save discoverable authentication state");
+        return (
+            StatusCode::INTERNAL_SERVER_ERROR,
+            Json(serde_json::json!({
+                "error": "server_error",
+                "error_description": "An error occurred."
+            })),
+        )
+            .into_response();
+    }
+
+    let options = serde_json::to_value(&rcr).unwrap_or(serde_json::json!({}));
+    Json(PasskeyStartResponse { options }).into_response()
+}
+
+async fn passkey_start_named(
+    state: AppState,
+    identifier: String,
+    delegated_did: Option<String>,
+    request_data: tranquil_pds::oauth::RequestData,
+    passkey_start_request_id: RequestId,
+) -> Response {
     let hostname_for_handles = tranquil_config::get().server.hostname_without_port();
     let normalized_username =
-        NormalizedLoginIdentifier::normalize(&form.identifier, hostname_for_handles);
+        NormalizedLoginIdentifier::normalize(&identifier, hostname_for_handles);
 
     let user = match state
         .repos
         .user
-        .get_login_info_by_handle_or_email(normalized_username.as_str())
+        .get_login_info_by_identifier(normalized_username.as_str())
         .await
     {
         Ok(Some(u)) => u,
@@ -325,7 +402,7 @@ pub async fn passkey_start(
             .into_response();
     }
 
-    let delegation_from_param = match &form.delegated_did {
+    let delegation_from_param = match &delegated_did {
         Some(delegated_did_str) => match delegated_did_str.parse::<tranquil_types::Did>() {
             Ok(delegated_did) if delegated_did != user.did => {
                 match state
@@ -471,85 +548,6 @@ pub async fn passkey_finish(
             .into_response();
     }
 
-    let did_str = match request_data.did {
-        Some(d) => d,
-        None => {
-            return (
-                StatusCode::BAD_REQUEST,
-                Json(serde_json::json!({
-                    "error": "invalid_request",
-                    "error_description": "No passkey authentication in progress."
-                })),
-            )
-                .into_response();
-        }
-    };
-    let did: tranquil_types::Did = match did_str.parse() {
-        Ok(d) => d,
-        Err(_) => {
-            return (
-                StatusCode::BAD_REQUEST,
-                Json(serde_json::json!({
-                    "error": "invalid_request",
-                    "error_description": "Invalid DID format."
-                })),
-            )
-                .into_response();
-        }
-    };
-
-    let controller_did: Option<tranquil_types::Did> = request_data
-        .controller_did
-        .as_ref()
-        .and_then(|s| s.parse().ok());
-    let passkey_owner_did = controller_did.as_ref().unwrap_or(&did);
-
-    let auth_state_json = match state
-        .repos
-        .user
-        .load_webauthn_challenge(passkey_owner_did, WebauthnChallengeType::Authentication)
-        .await
-    {
-        Ok(Some(s)) => s,
-        Ok(None) => {
-            return (
-                StatusCode::BAD_REQUEST,
-                Json(serde_json::json!({
-                    "error": "invalid_request",
-                    "error_description": "No passkey authentication in progress or challenge expired."
-                })),
-            )
-                .into_response();
-        }
-        Err(e) => {
-            tracing::error!(error = %e, "Failed to load authentication state");
-            return (
-                StatusCode::INTERNAL_SERVER_ERROR,
-                Json(serde_json::json!({
-                    "error": "server_error",
-                    "error_description": "An error occurred."
-                })),
-            )
-                .into_response();
-        }
-    };
-
-    let auth_state: webauthn_rs::prelude::SecurityKeyAuthentication =
-        match serde_json::from_str(&auth_state_json) {
-            Ok(s) => s,
-            Err(e) => {
-                tracing::error!(error = %e, "Failed to deserialize authentication state");
-                return (
-                    StatusCode::INTERNAL_SERVER_ERROR,
-                    Json(serde_json::json!({
-                        "error": "server_error",
-                        "error_description": "An error occurred."
-                    })),
-                )
-                    .into_response();
-            }
-        };
-
     let credential: webauthn_rs::prelude::PublicKeyCredential =
         match serde_json::from_value(form.credential) {
             Ok(c) => c,
@@ -566,33 +564,35 @@ pub async fn passkey_finish(
             }
         };
 
-    let auth_result = match state
-        .webauthn_config
-        .finish_authentication(&credential, &auth_state)
-    {
-        Ok(r) => r,
-        Err(e) => {
-            tracing::warn!(error = %e, did = %did, "Failed to verify passkey authentication");
-            return (
-                StatusCode::FORBIDDEN,
-                Json(serde_json::json!({
-                    "error": "access_denied",
-                    "error_description": "Passkey verification failed."
-                })),
+    let (did, auth_result) = match request_data.did.clone() {
+        Some(did) => match passkey_finish_named(&state, did, &request_data, &credential).await {
+            Ok(result) => result,
+            Err(response) => return response,
+        },
+        None => {
+            let result = match passkey_finish_discoverable(
+                &state,
+                &credential,
+                &passkey_finish_request_id,
             )
-                .into_response();
+            .await
+            {
+                Ok(result) => result,
+                Err(response) => return response,
+            };
+            if state
+                .repos
+                .oauth
+                .set_authorization_did(&passkey_finish_request_id, &result.0, None)
+                .await
+                .is_err()
+            {
+                return OAuthError::ServerError("An error occurred.".into()).into_response();
+            }
+            result
         }
     };
 
-    if let Err(e) = state
-        .repos
-        .user
-        .delete_webauthn_challenge(passkey_owner_did, WebauthnChallengeType::Authentication)
-        .await
-    {
-        tracing::warn!(error = %e, "Failed to delete authentication state");
-    }
-
     if auth_result.needs_update() {
         let cred_id_bytes = auth_result.cred_id().as_slice();
         match state
@@ -691,6 +691,187 @@ pub async fn passkey_finish(
     .into_response()
 }
 
+async fn passkey_finish_named(
+    state: &AppState,
+    did: tranquil_types::Did,
+    request_data: &tranquil_pds::oauth::RequestData,
+    credential: &webauthn_rs::prelude::PublicKeyCredential,
+) -> Result<
+    (
+        tranquil_types::Did,
+        webauthn_rs::prelude::AuthenticationResult,
+    ),
+    Response,
+> {
+    let passkey_owner_did = request_data.controller_did.as_ref().unwrap_or(&did);
+
+    let auth_state_json = state
+        .repos
+        .user
+        .load_webauthn_challenge(passkey_owner_did, WebauthnChallengeType::Authentication)
+        .await
+        .map_err(|e| {
+            tracing::error!(error = %e, "Failed to load authentication state");
+            (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                Json(serde_json::json!({"error": "server_error", "error_description": "An error occurred."})),
+            ).into_response()
+        })?
+        .ok_or_else(|| {
+            (
+                StatusCode::BAD_REQUEST,
+                Json(serde_json::json!({
+                    "error": "invalid_request",
+                    "error_description": "No passkey authentication in progress or challenge expired."
+                })),
+            ).into_response()
+        })?;
+
+    let auth_state: webauthn_rs::prelude::SecurityKeyAuthentication =
+        serde_json::from_str(&auth_state_json).map_err(|e| {
+            tracing::error!(error = %e, "Failed to deserialize authentication state");
+            (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                Json(serde_json::json!({"error": "server_error", "error_description": "An error occurred."})),
+            ).into_response()
+        })?;
+
+    let auth_result = state
+        .webauthn_config
+        .finish_authentication(credential, &auth_state)
+        .map_err(|e| {
+            tracing::warn!(error = %e, did = %did, "Failed to verify passkey authentication");
+            (
+                StatusCode::FORBIDDEN,
+                Json(serde_json::json!({
+                    "error": "access_denied",
+                    "error_description": "Passkey verification failed."
+                })),
+            )
+                .into_response()
+        })?;
+
+    let _ = state
+        .repos
+        .user
+        .delete_webauthn_challenge(passkey_owner_did, WebauthnChallengeType::Authentication)
+        .await;
+
+    Ok((did, auth_result))
+}
+
+async fn passkey_finish_discoverable(
+    state: &AppState,
+    credential: &webauthn_rs::prelude::PublicKeyCredential,
+    request_id: &RequestId,
+) -> Result<
+    (
+        tranquil_types::Did,
+        webauthn_rs::prelude::AuthenticationResult,
+    ),
+    Response,
+> {
+    let auth_state_json = state
+        .repos
+        .user
+        .load_discoverable_challenge(request_id.as_str())
+        .await
+        .map_err(|e| {
+            tracing::error!(error = %e, "Failed to load discoverable authentication state");
+            (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                Json(serde_json::json!({"error": "server_error", "error_description": "An error occurred."})),
+            ).into_response()
+        })?
+        .ok_or_else(|| {
+            (
+                StatusCode::BAD_REQUEST,
+                Json(serde_json::json!({
+                    "error": "invalid_request",
+                    "error_description": "No passkey authentication in progress or challenge expired."
+                })),
+            ).into_response()
+        })?;
+
+    let auth_state: webauthn_rs::prelude::DiscoverableAuthentication =
+        serde_json::from_str(&auth_state_json).map_err(|e| {
+            tracing::error!(error = %e, "Failed to deserialize discoverable authentication state");
+            (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                Json(serde_json::json!({"error": "server_error", "error_description": "An error occurred."})),
+            ).into_response()
+        })?;
+
+    let (_user_uuid, cred_id) = state
+        .webauthn_config
+        .identify_discoverable_authentication(credential)
+        .map_err(|e| {
+            tracing::warn!(error = %e, "Failed to identify discoverable credential");
+            (
+                StatusCode::FORBIDDEN,
+                Json(serde_json::json!({
+                    "error": "access_denied",
+                    "error_description": "Passkey verification failed."
+                })),
+            )
+                .into_response()
+        })?;
+
+    let stored_passkey = state
+        .repos
+        .user
+        .get_passkey_by_credential_id(cred_id)
+        .await
+        .map_err(|e| {
+            tracing::error!(error = %e, "Failed to look up passkey by credential ID");
+            (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                Json(serde_json::json!({"error": "server_error", "error_description": "An error occurred."})),
+            ).into_response()
+        })?
+        .ok_or_else(|| {
+            tracing::warn!("Discoverable credential not found in database");
+            (
+                StatusCode::FORBIDDEN,
+                Json(serde_json::json!({
+                    "error": "access_denied",
+                    "error_description": "Passkey not recognized."
+                })),
+            ).into_response()
+        })?;
+
+    let discoverable_key: webauthn_rs::prelude::DiscoverableKey =
+        serde_json::from_slice(&stored_passkey.public_key).map_err(|e| {
+            tracing::error!(error = %e, "Failed to deserialize stored passkey as DiscoverableKey");
+            (
+                StatusCode::INTERNAL_SERVER_ERROR,
+                Json(serde_json::json!({"error": "server_error", "error_description": "An error occurred."})),
+            ).into_response()
+        })?;
+
+    let auth_result = state
+        .webauthn_config
+        .finish_discoverable_authentication(credential, auth_state, &[discoverable_key])
+        .map_err(|e| {
+            tracing::warn!(error = %e, did = %stored_passkey.did, "Failed to verify discoverable passkey authentication");
+            (
+                StatusCode::FORBIDDEN,
+                Json(serde_json::json!({
+                    "error": "access_denied",
+                    "error_description": "Passkey verification failed."
+                })),
+            ).into_response()
+        })?;
+
+    let _ = state
+        .repos
+        .user
+        .delete_discoverable_challenge(request_id.as_str())
+        .await;
+
+    Ok((stored_passkey.did, auth_result))
+}
+
 #[derive(Debug, Deserialize)]
 pub struct AuthorizePasskeyQuery {
     pub request_uri: String,

crates/tranquil-oauth-server/src/sso_endpoints.rs

@@ -1339,7 +1339,7 @@ pub async fn complete_registration(
                 refresh_expires_at: refresh_meta.expires_at,
                 login_type: tranquil_db_traits::LoginType::Modern,
                 mfa_verified: false,
-                scope: Some("transition:generic".to_string()),
+                scope: Some("transition:generic transition:chat.bsky".to_string()),
                 controller_did: None,
                 app_password_name: None,
             };

crates/tranquil-pds/Cargo.toml

@@ -78,6 +78,7 @@ tracing = { workspace = true }
 urlencoding = { workspace = true }
 uuid = { workspace = true }
 webauthn-rs = { workspace = true }
+webauthn-rs-proto = { workspace = true }
 zip = { workspace = true }
 aws-config = { workspace = true, optional = true }
 aws-sdk-s3 = { workspace = true, optional = true }

crates/tranquil-pds/src/auth/mod.rs

@@ -7,8 +7,7 @@ use crate::api::ApiError;
 use crate::cache::Cache;
 use crate::oauth::scopes::ScopePermissions;
 use crate::types::Did;
-use tranquil_db::UserRepository;
-use tranquil_db_traits::OAuthRepository;
+use tranquil_db_traits::{OAuthRepository, UserRepository};
 
 pub mod account_verified;
 pub mod email_token;

crates/tranquil-pds/src/auth/webauthn.rs

@@ -1,5 +1,8 @@
 use uuid::Uuid;
 use webauthn_rs::prelude::*;
+use webauthn_rs_proto::{
+    AuthenticatorSelectionCriteria, ResidentKeyRequirement, UserVerificationPolicy,
+};
 
 #[derive(Debug, thiserror::Error)]
 pub enum WebauthnError {
@@ -57,6 +60,15 @@ impl WebAuthnConfig {
                 None,
                 None,
             )
+            .map(|(mut ccr, state)| {
+                let sel = ccr
+                    .public_key
+                    .authenticator_selection
+                    .get_or_insert_with(AuthenticatorSelectionCriteria::default);
+                sel.resident_key = Some(ResidentKeyRequirement::Required);
+                sel.require_resident_key = true;
+                (ccr, state)
+            })
             .map_err(|e| WebauthnError::RegistrationFailed(e.to_string()))
     }
 
@@ -88,4 +100,49 @@ impl WebAuthnConfig {
             .finish_securitykey_authentication(auth, state)
             .map_err(|e| WebauthnError::AuthenticationFailed(e.to_string()))
     }
+
+    pub fn start_discoverable_authentication(
+        &self,
+    ) -> Result<(RequestChallengeResponse, DiscoverableAuthentication), WebauthnError> {
+        let (mut rcr, state) = self
+            .webauthn
+            .start_discoverable_authentication()
+            .map_err(|e| WebauthnError::AuthenticationFailed(e.to_string()))?;
+
+        rcr.mediation = None;
+        rcr.public_key.user_verification = UserVerificationPolicy::Discouraged_DO_NOT_USE;
+
+        let mut state_json = serde_json::to_value(&state)
+            .map_err(|e| WebauthnError::AuthenticationFailed(e.to_string()))?;
+        let ast = state_json
+            .get_mut("ast")
+            .ok_or_else(|| WebauthnError::AuthenticationFailed(
+                "webauthn-rs DiscoverableAuthentication missing 'ast' field, library version incompatible".into(),
+            ))?;
+        ast["policy"] = serde_json::json!("discouraged");
+        let patched: DiscoverableAuthentication = serde_json::from_value(state_json)
+            .map_err(|e| WebauthnError::AuthenticationFailed(e.to_string()))?;
+
+        Ok((rcr, patched))
+    }
+
+    pub fn identify_discoverable_authentication<'a>(
+        &self,
+        credential: &'a PublicKeyCredential,
+    ) -> Result<(Uuid, &'a [u8]), WebauthnError> {
+        self.webauthn
+            .identify_discoverable_authentication(credential)
+            .map_err(|e| WebauthnError::AuthenticationFailed(e.to_string()))
+    }
+
+    pub fn finish_discoverable_authentication(
+        &self,
+        credential: &PublicKeyCredential,
+        state: DiscoverableAuthentication,
+        creds: &[DiscoverableKey],
+    ) -> Result<AuthenticationResult, WebauthnError> {
+        self.webauthn
+            .finish_discoverable_authentication(credential, state, creds)
+            .map_err(|e| WebauthnError::AuthenticationFailed(e.to_string()))
+    }
 }

crates/tranquil-pds/src/comms/service.rs

@@ -3,7 +3,7 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use chrono::Utc;
-use tokio::time::interval;
+
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, info, warn};
 use tranquil_comms::{
@@ -75,17 +75,28 @@ impl CommsService {
             );
         }
         info!(
-            poll_interval_secs = self.poll_interval.as_secs(),
+            poll_interval_ms = self.poll_interval.as_millis() as u64,
             batch_size = self.batch_size,
             channels = ?self.senders.keys().collect::<Vec<_>>(),
             "Starting comms service"
         );
-        let mut ticker = interval(self.poll_interval);
+        let base = self.poll_interval;
+        let max_backoff = Duration::from_secs(30);
+        let mut current_delay = base;
         loop {
             tokio::select! {
-                _ = ticker.tick() => {
-                    if let Err(e) = self.process_batch().await {
-                        error!(error = %e, "Failed to process comms batch");
+                _ = tokio::time::sleep(current_delay) => {
+                    match self.process_batch().await {
+                        Ok(had_work) => {
+                            current_delay = match had_work {
+                                true => base,
+                                false => max_backoff.min(current_delay.saturating_mul(2)),
+                            };
+                        }
+                        Err(e) => {
+                            error!(error = %e, "Failed to process comms batch");
+                            current_delay = max_backoff.min(current_delay.saturating_mul(2));
+                        }
                     }
                 }
                 _ = shutdown.cancelled() => {
@@ -96,14 +107,14 @@ impl CommsService {
         }
     }
 
-    async fn process_batch(&self) -> Result<(), tranquil_db_traits::DbError> {
+    async fn process_batch(&self) -> Result<bool, tranquil_db_traits::DbError> {
         let items = self.fetch_pending().await?;
         if items.is_empty() {
-            return Ok(());
+            return Ok(false);
         }
         debug!(count = items.len(), "Processing comms batch");
         futures::future::join_all(items.into_iter().map(|item| self.process_item(item))).await;
-        Ok(())
+        Ok(true)
     }
 
     async fn fetch_pending(&self) -> Result<Vec<QueuedComms>, tranquil_db_traits::DbError> {

crates/tranquil-pds/src/crawlers.rs

@@ -75,8 +75,8 @@ impl Crawlers {
         self.last_notified.store(now, Ordering::Relaxed);
     }
 
-    pub async fn notify_of_update(&self) {
-        if !self.should_notify() {
+    pub async fn notify_of_update(&self, force: bool) {
+        if !force && !self.should_notify() {
             debug!("Skipping crawler notification due to debounce");
             return;
         }
@@ -157,13 +157,17 @@ pub async fn start_crawlers_service(
             result = firehose_rx.recv() => {
                 match result {
                     Ok(event) => {
-                        if event.event_type == RepoEventType::Commit {
-                            crawlers.notify_of_update().await;
+                        match event.event_type {
+                            RepoEventType::Commit => crawlers.notify_of_update(false).await,
+                            RepoEventType::Account | RepoEventType::Identity => {
+                                crawlers.notify_of_update(true).await
+                            }
+                            RepoEventType::Sync => {}
                         }
                     }
                     Err(broadcast::error::RecvError::Lagged(n)) => {
                         warn!(skipped = n, "Crawlers service lagged behind firehose");
-                        crawlers.notify_of_update().await;
+                        crawlers.notify_of_update(false).await;
                     }
                     Err(broadcast::error::RecvError::Closed) => {
                         error!("Firehose channel closed, stopping crawlers service");

crates/tranquil-pds/src/oauth/mod.rs

@@ -3,7 +3,7 @@ pub mod db;
 pub mod scopes;
 pub mod verify;
 
-pub fn db_err_to_oauth(err: tranquil_db::DbError) -> OAuthError {
+pub fn db_err_to_oauth(err: tranquil_db_traits::DbError) -> OAuthError {
     tracing::error!("Database error in OAuth flow: {}", err);
     OAuthError::ServerError("An internal error occurred".to_string())
 }

crates/tranquil-pds/src/repo/mod.rs

@@ -22,6 +22,20 @@ impl AnyBlockStore {
             Self::TranquilStore(_) => None,
         }
     }
+
+    pub fn as_tranquil_store(&self) -> Option<&TranquilBlockStore> {
+        match self {
+            Self::TranquilStore(s) => Some(s),
+            Self::Postgres(_) => None,
+        }
+    }
+
+    pub async fn decrement_refs(&self, cids: &[Cid]) -> Result<(), RepoError> {
+        match self {
+            Self::Postgres(_) => Ok(()),
+            Self::TranquilStore(s) => s.decrement_refs(cids).await,
+        }
+    }
 }
 
 impl BlockStore for AnyBlockStore {

crates/tranquil-pds/src/repo_ops.rs

@@ -1,13 +1,15 @@
 use crate::api::error::ApiError;
-use crate::cid_types::CommitCid;
+use crate::cid_types::{CommitCid, RecordCid};
 use crate::repo::TrackingBlockStore;
 use crate::state::AppState;
 use crate::types::{Did, Handle, Nsid, Rkey};
+use backon::{ExponentialBuilder, Retryable};
 use bytes::Bytes;
 use cid::Cid;
 use jacquard_common::types::{integer::LimitedU32, string::Tid};
 use jacquard_repo::commit::Commit;
 use jacquard_repo::mst::Mst;
+use jacquard_repo::mst::util::compute_cid;
 use jacquard_repo::storage::BlockStore;
 use k256::ecdsa::SigningKey;
 use serde_json::{Value, json};
@@ -147,6 +149,7 @@ pub fn extract_backlinks(uri: &AtUri, record: &Value) -> Vec<Backlink> {
 pub struct RepoWriteContext {
     pub tracking_store: TrackingBlockStore,
     pub current_root_cid: Cid,
+    pub prev_commit_bytes: Bytes,
     pub prev_data_cid: Cid,
     pub write_lock: OwnedMutexGuard<()>,
 }
@@ -197,22 +200,25 @@ pub async fn begin_repo_write(
         .get(&current_root_cid)
         .await
         .map_err(|e| {
-            error!("Failed to load commit block: {:?}", e);
+            error!("Failed to load commit block: {}", e);
             ApiError::InternalError(None)
         })?
         .ok_or_else(|| ApiError::InternalError(Some("Commit block not found".into())))?;
 
-    let commit = Commit::from_cbor(&commit_bytes).map_err(|e| {
-        error!("Failed to parse commit: {:?}", e);
-        ApiError::InternalError(None)
-    })?;
+    let prev_data_cid = Commit::from_cbor(&commit_bytes)
+        .map_err(|e| {
+            error!("Failed to parse commit: {}", e);
+            ApiError::InternalError(None)
+        })?
+        .data;
 
-    let mst = Mst::load(Arc::new(tracking_store.clone()), commit.data, None);
+    let mst = Mst::load(Arc::new(tracking_store.clone()), prev_data_cid, None);
 
     let ctx = RepoWriteContext {
         tracking_store,
         current_root_cid,
-        prev_data_cid: commit.data,
+        prev_commit_bytes: commit_bytes,
+        prev_data_cid,
         write_lock,
     };
 
@@ -226,18 +232,37 @@ pub async fn finalize_repo_write(
     params: FinalizeParams<'_>,
 ) -> Result<CommitResult, ApiError> {
     let new_mst_root = mst.persist().await.map_err(|e| {
-        error!("MST persist failed: {:?}", e);
+        error!("MST persist failed: {}", e);
         ApiError::InternalError(None)
     })?;
 
-    let written_cids: Vec<Cid> = ctx
-        .tracking_store
-        .get_all_relevant_cids()
-        .into_iter()
-        .collect::<std::collections::HashSet<_>>()
-        .into_iter()
-        .collect();
-    let written_cids_str: Vec<String> = written_cids.iter().map(ToString::to_string).collect();
+    let block_bytes = ctx.tracking_store.take_written_blocks();
+
+    let storage_for_diff = Arc::new(ctx.tracking_store.clone());
+    let original_settled = Mst::load(storage_for_diff.clone(), ctx.prev_data_cid, None);
+    let new_settled = Mst::load(storage_for_diff, new_mst_root, None);
+
+    let new_tree_cids: Vec<Cid> = block_bytes.keys().copied().collect();
+
+    let obsolete_cids = match original_settled.diff(&new_settled).await {
+        Ok(diff) => {
+            let mut obsolete: Vec<Cid> = Vec::with_capacity(
+                1 + diff.removed_mst_blocks.len() + diff.removed_cids.len(),
+            );
+            obsolete.push(ctx.current_root_cid);
+            obsolete.extend(diff.removed_mst_blocks);
+            obsolete.extend(diff.removed_cids);
+            obsolete
+        }
+        Err(e) => {
+            error!(
+                "MST diff failed during finalize_repo_write: {e}. \
+                 Proceeding with commit CID only; leaked blocks \
+                 will be reclaimed by reachability GC."
+            );
+            vec![ctx.current_root_cid]
+        }
+    };
 
     let result = commit_and_log(
         state,
@@ -245,12 +270,14 @@ pub async fn finalize_repo_write(
             did: params.did,
             user_id: params.user_id,
             current_root_cid: Some(ctx.current_root_cid),
+            prev_commit_bytes: Some(ctx.prev_commit_bytes),
             prev_data_cid: Some(ctx.prev_data_cid),
             new_mst_root,
             ops: params.ops,
-            blocks_cids: &written_cids_str,
+            block_bytes,
+            new_tree_cids,
             blobs: params.blob_cids,
-            obsolete_cids: vec![ctx.current_root_cid],
+            obsolete_cids,
             backlinks_to_add: params.backlinks_to_add,
             backlinks_to_remove: params.backlinks_to_remove,
         },
@@ -297,7 +324,7 @@ pub fn create_signed_commit(
     let sig_bytes = signed.sig().clone();
     let signed_bytes = signed
         .to_cbor()
-        .map_err(|e| CommitError::SerializationFailed(format!("{:?}", e)))?;
+        .map_err(|e| CommitError::SerializationFailed(e.to_string()))?;
     Ok((signed_bytes, sig_bytes))
 }
 
@@ -305,18 +332,18 @@ pub enum RecordOp {
     Create {
         collection: Nsid,
         rkey: Rkey,
-        cid: Cid,
+        cid: RecordCid,
     },
     Update {
         collection: Nsid,
         rkey: Rkey,
-        cid: Cid,
-        prev: Option<Cid>,
+        cid: RecordCid,
+        prev: RecordCid,
     },
     Delete {
         collection: Nsid,
         rkey: Rkey,
-        prev: Option<Cid>,
+        prev: RecordCid,
     },
 }
 
@@ -329,10 +356,12 @@ pub struct CommitParams<'a> {
     pub did: &'a Did,
     pub user_id: Uuid,
     pub current_root_cid: Option<Cid>,
+    pub prev_commit_bytes: Option<Bytes>,
     pub prev_data_cid: Option<Cid>,
     pub new_mst_root: Cid,
     pub ops: Vec<RecordOp>,
-    pub blocks_cids: &'a [String],
+    pub block_bytes: std::collections::HashMap<Cid, Bytes>,
+    pub new_tree_cids: Vec<Cid>,
     pub blobs: &'a [String],
     pub obsolete_cids: Vec<Cid>,
     pub backlinks_to_add: Vec<Backlink>,
@@ -344,8 +373,8 @@ pub async fn commit_and_log(
     params: CommitParams<'_>,
 ) -> Result<CommitResult, CommitError> {
     use tranquil_db_traits::{
-        ApplyCommitError, ApplyCommitInput, CommitEventData, RecordDelete, RecordUpsert,
-        RepoEventType,
+        ApplyCommitError, ApplyCommitInput, CommitEventData, EventBlockInline, RecordDelete,
+        RecordUpsert, RepoEventType,
     };
 
     let backlinks_to_add = params.backlinks_to_add;
@@ -354,14 +383,21 @@ pub async fn commit_and_log(
         did,
         user_id,
         current_root_cid,
+        prev_commit_bytes,
         prev_data_cid,
         new_mst_root,
         ops,
-        blocks_cids,
+        mut block_bytes,
+        new_tree_cids,
         blobs,
         obsolete_cids,
         ..
     } = params;
+    debug_assert_eq!(
+        current_root_cid.is_some(),
+        prev_commit_bytes.is_some(),
+        "current_root_cid and prev_commit_bytes must be both Some (non-genesis) or both None (genesis)"
+    );
     let key_row = state
         .repos
         .user
@@ -377,18 +413,27 @@ pub async fn commit_and_log(
     let rev_str = rev.to_string();
     let (new_commit_bytes, _sig) =
         create_signed_commit(did, new_mst_root, &rev_str, current_root_cid, &signing_key)?;
-    let new_root_cid = state
+    let new_root_cid =
+        compute_cid(&new_commit_bytes).map_err(|e| CommitError::BlockStoreFailed(e.to_string()))?;
+
+    let commit_bytes_owned = Bytes::from(new_commit_bytes.clone());
+    state
         .block_store
         .put(&new_commit_bytes)
         .await
-        .map_err(|e| CommitError::BlockStoreFailed(format!("{:?}", e)))?;
+        .map_err(|e| CommitError::BlockStoreFailed(format!("failed to write commit block: {e}")))?;
 
-    let mut all_block_cids: Vec<Vec<u8>> = blocks_cids
+    block_bytes.insert(new_root_cid, commit_bytes_owned);
+
+    if let (Some(prev_root), Some(prev_bytes)) = (current_root_cid, prev_commit_bytes) {
+        block_bytes.entry(prev_root).or_insert(prev_bytes);
+    }
+
+    let all_block_cids: Vec<Vec<u8>> = new_tree_cids
         .iter()
-        .filter_map(|s| Cid::from_str(s).ok())
+        .chain(std::iter::once(&new_root_cid))
         .map(|c| c.to_bytes())
         .collect();
-    all_block_cids.push(new_root_cid.to_bytes());
 
     let obsolete_bytes: Vec<Vec<u8>> = obsolete_cids.iter().map(|c| c.to_bytes()).collect();
 
@@ -410,7 +455,7 @@ pub async fn commit_and_log(
                     upserts.push(RecordUpsert {
                         collection: collection.clone(),
                         rkey: rkey.clone(),
-                        cid: crate::types::CidLink::from(cid),
+                        cid: crate::types::CidLink::from(cid.as_cid()),
                     });
                 }
                 RecordOp::Delete {
@@ -443,32 +488,30 @@ pub async fn commit_and_log(
                 rkey,
                 cid,
                 prev,
-            } => {
-                let mut obj = json!({
-                    "action": "update",
-                    "path": format!("{}/{}", collection, rkey),
-                    "cid": cid.to_string()
-                });
-                if let Some(prev_cid) = prev {
-                    obj["prev"] = json!(prev_cid.to_string());
-                }
-                obj
-            }
+            } => json!({
+                "action": "update",
+                "path": format!("{}/{}", collection, rkey),
+                "cid": cid.to_string(),
+                "prev": prev.to_string(),
+            }),
             RecordOp::Delete {
                 collection,
                 rkey,
                 prev,
-            } => {
-                let mut obj = json!({
-                    "action": "delete",
-                    "path": format!("{}/{}", collection, rkey),
-                    "cid": null
-                });
-                if let Some(prev_cid) = prev {
-                    obj["prev"] = json!(prev_cid.to_string());
-                }
-                obj
-            }
+            } => json!({
+                "action": "delete",
+                "path": format!("{}/{}", collection, rkey),
+                "cid": null,
+                "prev": prev.to_string(),
+            }),
+        })
+        .collect();
+
+    let inline_blocks: Vec<EventBlockInline> = block_bytes
+        .iter()
+        .map(|(cid, data)| EventBlockInline {
+            cid_bytes: cid.to_bytes(),
+            data: data.to_vec(),
         })
         .collect();
 
@@ -479,7 +522,7 @@ pub async fn commit_and_log(
         prev_cid: current_root_cid.map(crate::types::CidLink::from),
         ops: Some(json!(ops_json)),
         blobs: Some(blobs.to_vec()),
-        blocks_cids: Some(blocks_cids.to_vec()),
+        blocks: Some(inline_blocks),
         prev_data_cid: prev_data_cid.map(crate::types::CidLink::from),
         rev: Some(rev_str.clone()),
     };
@@ -510,6 +553,31 @@ pub async fn commit_and_log(
             ApplyCommitError::Database(msg) => CommitError::DatabaseError(msg),
         })?;
 
+    let apply_result = (|| {
+        let bs = state.block_store.clone();
+        let decrements = obsolete_cids.clone();
+        async move { bs.decrement_refs(&decrements).await }
+    })
+    .retry(
+        ExponentialBuilder::default()
+            .with_min_delay(std::time::Duration::from_millis(50))
+            .with_max_delay(std::time::Duration::from_secs(2))
+            .with_max_times(5),
+    )
+    .await;
+
+    if let Err(e) = apply_result {
+        let leaked: Vec<String> = obsolete_cids.iter().map(Cid::to_string).collect();
+        tracing::error!(
+            error = %e,
+            user_id = %user_id,
+            new_root = %new_root_cid,
+            leaked_cids = ?leaked,
+            "blockstore decrement_refs failed after metastore commit succeeded \
+             and exhausted retries; blocks may leak refcounts"
+        );
+    }
+
     Ok(CommitResult {
         commit_cid: new_root_cid,
         rev: rev_str,
@@ -530,98 +598,57 @@ pub async fn create_record_internal(
         .map_err(|e| CommitError::DatabaseError(e.to_string()))?
         .ok_or(CommitError::UserNotFound)?;
 
-    let _write_lock = state.repo_write_locks.lock(user_id).await;
+    let to_commit_err = |e: ApiError| CommitError::DatabaseError(format!("{:?}", e));
 
-    let root_cid_link = state
-        .repos
-        .repo
-        .get_repo_root_cid_by_user_id(user_id)
+    let (ctx, mst) = begin_repo_write(state, user_id, None)
         .await
-        .map_err(|e| CommitError::DatabaseError(e.to_string()))?
-        .ok_or(CommitError::RepoNotFound)?;
-    let current_root_cid = Cid::from_str(root_cid_link.as_str())
-        .map_err(|e| CommitError::InvalidCid(e.to_string()))?;
-    let tracking_store = TrackingBlockStore::new(state.block_store.clone());
-    let commit_bytes = tracking_store
-        .get(&current_root_cid)
-        .await
-        .map_err(|e| CommitError::BlockStoreFailed(format!("{:?}", e)))?
-        .ok_or(CommitError::BlockStoreFailed(
-            "Commit block not found".into(),
-        ))?;
-    let commit = jacquard_repo::commit::Commit::from_cbor(&commit_bytes)
-        .map_err(|e| CommitError::CommitParseFailed(format!("{:?}", e)))?;
-    let mst = Mst::load(Arc::new(tracking_store.clone()), commit.data, None);
+        .map_err(to_commit_err)?;
+
     let record_ipld = crate::util::json_to_ipld(record);
     let mut record_bytes = Vec::new();
     serde_ipld_dagcbor::to_writer(&mut record_bytes, &record_ipld)
-        .map_err(|e| CommitError::RecordSerializationFailed(format!("{:?}", e)))?;
-    let record_cid = tracking_store
+        .map_err(|e| CommitError::RecordSerializationFailed(e.to_string()))?;
+    let record_cid = ctx
+        .tracking_store
         .put(&record_bytes)
         .await
-        .map_err(|e| CommitError::BlockStoreFailed(format!("{:?}", e)))?;
+        .map_err(|e| CommitError::BlockStoreFailed(e.to_string()))?;
+
     let key = format!("{}/{}", collection, rkey);
     let new_mst = mst
         .add(&key, record_cid)
         .await
-        .map_err(|e| CommitError::MstOperationFailed(format!("{:?}", e)))?;
-    let new_mst_root = new_mst
-        .persist()
-        .await
-        .map_err(|e| CommitError::MstOperationFailed(format!("{:?}", e)))?;
+        .map_err(|e| CommitError::MstOperationFailed(e.to_string()))?;
+
     let op = RecordOp::Create {
         collection: collection.clone(),
         rkey: rkey.clone(),
-        cid: record_cid,
+        cid: RecordCid::from(record_cid),
     };
-    let mut new_mst_blocks = std::collections::BTreeMap::new();
-    let mut old_mst_blocks = std::collections::BTreeMap::new();
-    new_mst
-        .blocks_for_path(&key, &mut new_mst_blocks)
-        .await
-        .map_err(|e| CommitError::MstOperationFailed(format!("{:?}", e)))?;
-    mst.blocks_for_path(&key, &mut old_mst_blocks)
-        .await
-        .map_err(|e| CommitError::MstOperationFailed(format!("{:?}", e)))?;
-    let obsolete_cids: Vec<Cid> = std::iter::once(current_root_cid)
-        .chain(
-            old_mst_blocks
-                .keys()
-                .filter(|cid| !new_mst_blocks.contains_key(*cid))
-                .copied(),
-        )
-        .collect();
-    let mut relevant_blocks = new_mst_blocks;
-    relevant_blocks.extend(old_mst_blocks);
-    relevant_blocks.insert(record_cid, bytes::Bytes::from(record_bytes));
-    let written_cids: Vec<Cid> = tracking_store
-        .get_all_relevant_cids()
-        .into_iter()
-        .chain(relevant_blocks.keys().copied())
-        .collect::<std::collections::HashSet<_>>()
-        .into_iter()
-        .collect();
-    let written_cids_str: Vec<String> = written_cids.iter().map(|c| c.to_string()).collect();
+    let modified_keys = [key];
     let blob_cids = extract_blob_cids(record);
     let record_uri = AtUri::from_parts(did.as_str(), collection.as_str(), rkey.as_str());
     let backlinks = extract_backlinks(&record_uri, record);
-    let result = commit_and_log(
+
+    let result = finalize_repo_write(
         state,
-        CommitParams {
+        ctx,
+        new_mst,
+        FinalizeParams {
             did,
             user_id,
-            current_root_cid: Some(current_root_cid),
-            prev_data_cid: Some(commit.data),
-            new_mst_root,
+            controller_did: None,
+            delegation_detail: None,
             ops: vec![op],
-            blocks_cids: &written_cids_str,
-            blobs: &blob_cids,
-            obsolete_cids,
+            modified_keys: &modified_keys,
+            blob_cids: &blob_cids,
             backlinks_to_add: backlinks,
             backlinks_to_remove: vec![],
         },
     )
-    .await?;
+    .await
+    .map_err(to_commit_err)?;
+
     let uri = format!("at://{}/{}/{}", did, collection, rkey);
     Ok((uri, result.commit_cid))
 }
@@ -659,10 +686,20 @@ pub async fn sequence_sync_event(
     let cid_link: crate::types::CidLink = commit_cid
         .parse()
         .map_err(|_| CommitError::InvalidCid(commit_cid.to_string()))?;
+    let commit_cid_parsed =
+        Cid::from_str(commit_cid).map_err(|e| CommitError::InvalidCid(e.to_string()))?;
+    let commit_bytes = state
+        .block_store
+        .get(&commit_cid_parsed)
+        .await
+        .map_err(|e| CommitError::BlockStoreFailed(format!("{:?}", e)))?
+        .ok_or(CommitError::BlockStoreFailed(
+            "Commit block not found for sync event".into(),
+        ))?;
     state
         .repos
         .repo
-        .insert_sync_event(did, &cid_link, rev)
+        .insert_sync_event(did, &cid_link, rev, &commit_bytes)
         .await
         .map_err(|e| CommitError::DatabaseError(format!("sync event: {}", e)))
 }
@@ -676,10 +713,33 @@ pub async fn sequence_genesis_commit(
 ) -> Result<SequenceNumber, CommitError> {
     let commit_cid_link = crate::types::CidLink::from(commit_cid);
     let mst_root_cid_link = crate::types::CidLink::from(mst_root_cid);
+    let commit_bytes = state
+        .block_store
+        .get(commit_cid)
+        .await
+        .map_err(|e| CommitError::BlockStoreFailed(format!("{:?}", e)))?
+        .ok_or(CommitError::BlockStoreFailed(
+            "Genesis commit block not found".into(),
+        ))?;
+    let mst_root_bytes = state
+        .block_store
+        .get(mst_root_cid)
+        .await
+        .map_err(|e| CommitError::BlockStoreFailed(format!("{:?}", e)))?
+        .ok_or(CommitError::BlockStoreFailed(
+            "Genesis MST root block not found".into(),
+        ))?;
     state
         .repos
         .repo
-        .insert_genesis_commit_event(did, &commit_cid_link, &mst_root_cid_link, rev)
+        .insert_genesis_commit_event(
+            did,
+            &commit_cid_link,
+            &mst_root_cid_link,
+            rev,
+            &commit_bytes,
+            &mst_root_bytes,
+        )
         .await
         .map_err(|e| CommitError::DatabaseError(format!("genesis commit event: {}", e)))
 }

crates/tranquil-pds/src/scheduled.rs

@@ -9,117 +9,15 @@ use std::time::Duration;
 use tokio::time::interval;
 use tokio_util::sync::CancellationToken;
 use tracing::{debug, error, info, warn};
-use tranquil_db_traits::{
-    BlobRepository, BrokenGenesisCommit, RepoRepository, SequenceNumber, SsoRepository,
-    UserRepository,
-};
+use tranquil_db_traits::{BlobRepository, RepoRepository, SsoRepository, UserRepository};
+use tranquil_store::blockstore::CidBytes;
+use tranquil_store::bloom::BloomFilter;
 use tranquil_types::{AtUri, CidLink, Did};
 
 use crate::repo::AnyBlockStore;
 use crate::storage::BlobStorage;
 use crate::sync::car::encode_car_header;
 
-#[derive(Debug)]
-enum GenesisBackfillError {
-    MissingCommitCid,
-    InvalidCid,
-    BlockFetchFailed,
-    BlockNotFound,
-    CommitParseFailed,
-    UpdateFailed,
-}
-
-impl std::fmt::Display for GenesisBackfillError {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            Self::MissingCommitCid => f.write_str("missing commit_cid"),
-            Self::InvalidCid => f.write_str("invalid CID"),
-            Self::BlockFetchFailed => f.write_str("failed to fetch block"),
-            Self::BlockNotFound => f.write_str("block not found"),
-            Self::CommitParseFailed => f.write_str("failed to parse commit"),
-            Self::UpdateFailed => f.write_str("failed to update"),
-        }
-    }
-}
-
-async fn process_genesis_commit(
-    repo_repo: &dyn RepoRepository,
-    block_store: &AnyBlockStore,
-    row: BrokenGenesisCommit,
-) -> Result<(Did, SequenceNumber), (SequenceNumber, GenesisBackfillError)> {
-    let commit_cid_str = row
-        .commit_cid
-        .ok_or((row.seq, GenesisBackfillError::MissingCommitCid))?;
-    let commit_cid =
-        Cid::from_str(&commit_cid_str).map_err(|_| (row.seq, GenesisBackfillError::InvalidCid))?;
-    let block = block_store
-        .get(&commit_cid)
-        .await
-        .map_err(|_| (row.seq, GenesisBackfillError::BlockFetchFailed))?
-        .ok_or((row.seq, GenesisBackfillError::BlockNotFound))?;
-    let commit = Commit::from_cbor(&block)
-        .map_err(|_| (row.seq, GenesisBackfillError::CommitParseFailed))?;
-    let blocks_cids = vec![commit.data.to_string(), commit_cid.to_string()];
-    repo_repo
-        .update_seq_blocks_cids(row.seq, &blocks_cids)
-        .await
-        .map_err(|_| (row.seq, GenesisBackfillError::UpdateFailed))?;
-    Ok((row.did, row.seq))
-}
-
-pub async fn backfill_genesis_commit_blocks(
-    repo_repo: Arc<dyn RepoRepository>,
-    block_store: AnyBlockStore,
-) {
-    let broken_genesis_commits = match repo_repo.get_broken_genesis_commits().await {
-        Ok(rows) => rows,
-        Err(e) => {
-            error!(
-                "Failed to query repo_seq for genesis commit backfill: {:?}",
-                e
-            );
-            return;
-        }
-    };
-
-    if broken_genesis_commits.is_empty() {
-        debug!("No genesis commits need blocks_cids backfill");
-        return;
-    }
-
-    info!(
-        count = broken_genesis_commits.len(),
-        "Backfilling blocks_cids for genesis commits"
-    );
-
-    let results = futures::future::join_all(broken_genesis_commits.into_iter().map(|row| {
-        let repo_repo = repo_repo.clone();
-        let block_store = block_store.clone();
-        async move { process_genesis_commit(repo_repo.as_ref(), &block_store, row).await }
-    }))
-    .await;
-
-    let (success, failed) = results.iter().fold((0, 0), |(s, f), r| match r {
-        Ok((did, seq)) => {
-            info!(seq = seq.as_i64(), did = %did, "Fixed genesis commit blocks_cids");
-            (s + 1, f)
-        }
-        Err((seq, reason)) => {
-            warn!(
-                seq = seq.as_i64(),
-                reason = %reason,
-                "Failed to process genesis commit"
-            );
-            (s, f + 1)
-        }
-    });
-
-    info!(
-        success,
-        failed, "Completed genesis commit blocks_cids backfill"
-    );
-}
-
 async fn process_repo_rev(
     repo_repo: &dyn RepoRepository,
     block_store: &AnyBlockStore,
@@ -422,6 +320,7 @@ pub async fn backfill_record_blobs(repo_repo: Arc<dyn RepoRepository>, block_sto
     info!(success, failed, "Completed record_blobs backfill");
 }
 
+#[allow(clippy::too_many_arguments)]
 pub async fn start_scheduled_tasks(
     user_repo: Arc<dyn UserRepository>,
     blob_repo: Arc<dyn BlobRepository>,
@@ -429,23 +328,91 @@ pub async fn start_scheduled_tasks(
     sso_repo: Arc<dyn SsoRepository>,
     repo_repo: Arc<dyn RepoRepository>,
     block_store: AnyBlockStore,
+    eventlog_segments_dir: Option<std::path::PathBuf>,
     shutdown: CancellationToken,
 ) {
     let cfg = tranquil_config::get();
     let check_interval = Duration::from_secs(cfg.scheduled.delete_check_interval_secs);
-    let gc_interval = Duration::from_secs(cfg.scheduled.block_gc_interval_secs);
+    let compaction_enabled = cfg.scheduled.compaction_interval_secs > 0;
+    let reachability_enabled = cfg.scheduled.reachability_walk_interval_secs > 0;
+    let archival_enabled_secs = cfg.scheduled.archival_interval_secs > 0;
+    let event_retention_enabled = cfg.scheduled.event_retention_interval_secs > 0;
+    let compaction_interval = Duration::from_secs(cfg.scheduled.compaction_interval_secs.max(60));
+    let reachability_interval =
+        Duration::from_secs(cfg.scheduled.reachability_walk_interval_secs.max(60));
+    let archival_interval = Duration::from_secs(cfg.scheduled.archival_interval_secs.max(60));
+    let event_retention_interval =
+        Duration::from_secs(cfg.scheduled.event_retention_interval_secs.max(60));
+    let event_retention_max_age = Duration::from_secs(cfg.scheduled.event_retention_max_age_secs);
+
+    let archiver: Option<Arc<tranquil_store::archival::ContinuousArchiver>> =
+        match (&eventlog_segments_dir, &cfg.scheduled.archival_dest_dir) {
+            (Some(segments_dir), Some(dest_dir)) if archival_enabled_secs => {
+                let sidecar_path = segments_dir
+                    .parent()
+                    .unwrap_or(segments_dir)
+                    .join("archival.state");
+                match tranquil_store::archival::LocalArchivalDestination::new(
+                    std::path::PathBuf::from(dest_dir),
+                ) {
+                    Ok(dest) => {
+                        info!(
+                            dest_dir = dest_dir,
+                            interval_secs = archival_interval.as_secs(),
+                            "continuous archival enabled"
+                        );
+                        Some(Arc::new(tranquil_store::archival::ContinuousArchiver::new(
+                            segments_dir.clone(),
+                            sidecar_path,
+                            Box::new(dest),
+                        )))
+                    }
+                    Err(e) => {
+                        error!(
+                            dest_dir = dest_dir,
+                            error = %e,
+                            "failed to initialize archival destination, archival disabled"
+                        );
+                        None
+                    }
+                }
+            }
+            _ => None,
+        };
 
     info!(
         check_interval_secs = check_interval.as_secs(),
-        gc_interval_secs = gc_interval.as_secs(),
+        compaction_enabled,
+        compaction_interval_secs = cfg.scheduled.compaction_interval_secs,
+        reachability_enabled,
+        reachability_interval_secs = cfg.scheduled.reachability_walk_interval_secs,
+        archival_enabled = archiver.is_some(),
+        event_retention_enabled,
+        event_retention_interval_secs = cfg.scheduled.event_retention_interval_secs,
+        event_retention_max_age_secs = cfg.scheduled.event_retention_max_age_secs,
         "Starting scheduled tasks service"
     );
 
     let mut ticker = interval(check_interval);
     ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
 
-    let mut gc_ticker = interval(gc_interval);
-    gc_ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+    let mut compaction_ticker = interval(compaction_interval);
+    compaction_ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+
+    let mut reachability_ticker = interval(reachability_interval);
+    reachability_ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+
+    let mut archival_ticker = interval(archival_interval);
+    archival_ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+
+    let mut event_retention_ticker = match event_retention_enabled {
+        true => {
+            let mut t = interval(event_retention_interval);
+            t.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+            Some(t)
+        }
+        false => None,
+    };
 
     loop {
         tokio::select! {
@@ -492,63 +459,139 @@ pub async fn start_scheduled_tasks(
                     }
                 }
             }
-            _ = gc_ticker.tick() => {
-                if let Some(pg) = block_store.as_postgres()
-                    && let Err(e) = run_block_gc(repo_repo.as_ref(), pg).await
-                {
-                    error!("Block GC error: {e}");
+            _ = compaction_ticker.tick(), if compaction_enabled => {
+                if let Some(store) = block_store.as_tranquil_store() {
+                    let store = store.clone();
+                    let threshold = cfg.scheduled.compaction_liveness_threshold;
+                    let grace_ms = cfg.scheduled.compaction_grace_period_ms;
+                    if let Err(e) = tokio::task::spawn_blocking(move || {
+                        run_compaction_pass(&store, threshold, grace_ms)
+                    }).await.unwrap_or_else(|e| Err(anyhow::anyhow!("compaction task panicked: {e}"))) {
+                        error!("Compaction error: {e}");
+                    }
+                }
+            }
+            _ = reachability_ticker.tick(), if reachability_enabled => {
+                if let Some(store) = block_store.as_tranquil_store() {
+                    let store = store.clone();
+                    let repo_repo = repo_repo.clone();
+                    match tokio::task::spawn_blocking(move || {
+                        run_reachability_walk(&store, repo_repo.as_ref())
+                    }).await {
+                        Ok(Ok(result)) => {
+                            info!(
+                                repos_walked = result.repos_walked,
+                                blocks_visited = result.blocks_visited,
+                                live_refcounted = result.live_refcounted,
+                                leaked_blocks = result.leaked_blocks,
+                                repaired_blocks = result.repaired_blocks,
+                                bloom_heap_mb = result.bloom_heap_bytes / (1024 * 1024),
+                                "reachability walk complete"
+                            );
+                        }
+                        Ok(Err(e)) => error!("Reachability walk error: {e}"),
+                        Err(e) => error!("Reachability walk panicked: {e}"),
+                    }
+                }
+            }
+            _ = archival_ticker.tick(), if archival_enabled_secs => {
+                if let Some(ref archiver) = archiver {
+                    let archiver = Arc::clone(archiver);
+                    match tokio::task::spawn_blocking(move || {
+                        archiver.run_pass()
+                    }).await {
+                        Ok(Ok(result)) if result.segments_archived > 0 => {
+                            info!(
+                                segments_archived = result.segments_archived,
+                                bytes_archived = result.bytes_archived,
+                                "archival pass complete"
+                            );
+                        }
+                        Ok(Ok(_)) => {}
+                        Ok(Err(e)) => error!("Archival pass error: {e}"),
+                        Err(e) => error!("Archival task panicked: {e}"),
+                    }
+                }
+            }
+            _ = async {
+                match event_retention_ticker.as_mut() {
+                    Some(t) => { t.tick().await; }
+                    None => std::future::pending::<()>().await,
+                }
+            }, if event_retention_enabled => {
+                let cutoff = chrono::Utc::now()
+                    - chrono::Duration::from_std(event_retention_max_age)
+                        .expect("event_retention_max_age fits chrono::Duration: validated at config load");
+                match repo_repo.prune_events_older_than(cutoff).await {
+                    Ok(count) if count.is_zero() => {
+                        debug!("event retention: nothing past cutoff");
+                    }
+                    Ok(count) => {
+                        info!(deleted = count.count(), unit = count.unit(), "event retention prune complete");
+                    }
+                    Err(e) => error!(error = %e, "event retention error"),
                 }
             }
         }
     }
 }
 
-const BLOCK_GC_BATCH_SIZE: i64 = 1000;
-
-async fn run_block_gc(
-    repo_repo: &dyn RepoRepository,
-    block_store: &crate::repo::PostgresBlockStore,
+fn run_compaction_pass(
+    store: &tranquil_store::blockstore::TranquilBlockStore,
+    liveness_threshold: f64,
+    grace_period_ms: u64,
 ) -> anyhow::Result<()> {
-    let mut total_deleted: u64 = 0;
+    match store.cleanup_gc_meta() {
+        Ok(0) => {}
+        Ok(n) => info!(count = n, "cleaned up stale gc_meta entries"),
+        Err(e) => warn!(error = %e, "gc_meta cleanup failed, continuing"),
+    }
 
-    loop {
-        let candidates = block_store
-            .get_oldest_block_cids(BLOCK_GC_BATCH_SIZE)
-            .await
-            .context("failed to fetch candidate blocks")?;
+    let liveness_map = store
+        .compaction_liveness(grace_period_ms)
+        .context("failed to compute liveness")?;
 
-        match candidates.is_empty() {
-            true => break,
-            false => {
-                let batch_len = candidates.len();
-                let unreferenced = repo_repo
-                    .find_unreferenced_blocks(&candidates)
-                    .await
-                    .context("failed to check block references")?;
+    let candidate = liveness_map
+        .iter()
+        .filter(|(_, info)| info.total_blocks > 0 && info.ratio() < liveness_threshold)
+        .min_by(|(_, a), (_, b)| {
+            a.ratio()
+                .partial_cmp(&b.ratio())
+                .unwrap_or(std::cmp::Ordering::Equal)
+        });
 
-                let deleted = match unreferenced.is_empty() {
-                    true => 0,
-                    false => block_store
-                        .delete_blocks(&unreferenced)
-                        .await
-                        .context("failed to delete unreferenced blocks")?,
-                };
-
-                total_deleted = total_deleted.saturating_add(deleted);
-
-                match unreferenced.len() == batch_len {
-                    true => continue,
-                    false => break,
+    match candidate {
+        None => {
+            debug!("Compaction: no files below liveness threshold");
+            Ok(())
+        }
+        Some((&file_id, info)) => {
+            info!(
+                file_id = %file_id,
+                liveness = format!("{:.1}%", info.ratio() * 100.0),
+                live_blocks = info.live_blocks,
+                total_blocks = info.total_blocks,
+                "compacting data file"
+            );
+            match store.compact_file(file_id, grace_period_ms) {
+                Ok(result) => {
+                    info!(
+                        file_id = %result.file_id,
+                        reclaimed_bytes = result.reclaimed_bytes,
+                        live_blocks = result.live_blocks,
+                        dead_blocks = result.dead_blocks,
+                        "compaction complete"
+                    );
+                    Ok(())
                 }
+                Err(tranquil_store::blockstore::CompactionError::ActiveFileCannotBeCompacted) => {
+                    debug!(file_id = %file_id, "skipped active file");
+                    Ok(())
+                }
+                Err(e) => Err(anyhow::anyhow!("compaction failed: {e}")),
             }
         }
     }
-
-    match total_deleted > 0 {
-        true => info!(total_deleted, "Block GC cycle complete"),
-        false => debug!("Block GC cycle: no orphaned blocks found"),
-    }
-    Ok(())
 }
 
 async fn process_scheduled_deletions(
@@ -624,6 +667,8 @@ async fn delete_account_data(
     Ok(())
 }
 
+const CAR_BLOCK_BATCH_SIZE: usize = 500;
+
 pub async fn generate_repo_car(
     block_store: &AnyBlockStore,
     head_cid: &Cid,
@@ -640,21 +685,20 @@ pub async fn generate_repo_car(
         })
         .collect();
 
-    let car_bytes = encode_car_header(head_cid).context("Failed to encode CAR header")?;
+    let mut car_bytes = encode_car_header(head_cid).context("Failed to encode CAR header")?;
 
-    let blocks = block_store
-        .get_many(&block_cids)
-        .await
-        .context("Failed to fetch blocks")?;
+    for chunk in block_cids.chunks(CAR_BLOCK_BATCH_SIZE) {
+        let blocks = block_store
+            .get_many(chunk)
+            .await
+            .context("Failed to fetch blocks")?;
 
-    let car_bytes = block_cids
-        .iter()
-        .zip(blocks.iter())
-        .filter_map(|(cid, block_opt)| block_opt.as_ref().map(|block| (cid, block)))
-        .fold(car_bytes, |mut acc, (cid, block)| {
-            acc.extend(encode_car_block(cid, block));
-            acc
-        });
+        chunk
+            .iter()
+            .zip(blocks.iter())
+            .filter_map(|(cid, block_opt)| block_opt.as_ref().map(|block| (cid, block)))
+            .for_each(|(cid, block)| car_bytes.extend(encode_car_block(cid, block)));
+    }
 
     Ok(car_bytes)
 }
@@ -694,3 +738,232 @@ pub async fn generate_repo_car_from_user_blocks(
 
     generate_repo_car(block_store, &actual_head_cid).await
 }
+
+pub struct ReachabilityResult {
+    pub repos_walked: u64,
+    pub blocks_visited: u64,
+    pub live_refcounted: u64,
+    pub leaked_blocks: u64,
+    pub repaired_blocks: u64,
+    pub bloom_heap_bytes: usize,
+}
+
+const REPO_PAGE_SIZE: i64 = 500;
+const BLOOM_FALSE_POSITIVE_RATE: f64 = 0.01;
+
+fn cid_to_bytes(cid: &Cid) -> anyhow::Result<CidBytes> {
+    cid.to_bytes()
+        .try_into()
+        .map_err(|_| anyhow::anyhow!("CID byte length mismatch for {cid}"))
+}
+
+fn walk_repo_dag_sync(
+    store: &tranquil_store::blockstore::TranquilBlockStore,
+    head_cid: &Cid,
+    reachable: &mut std::collections::HashSet<CidBytes>,
+) -> anyhow::Result<()> {
+    let mut to_visit = vec![cid_to_bytes(head_cid)?];
+
+    while let Some(cid_bytes) = to_visit.pop() {
+        if !reachable.insert(cid_bytes) {
+            continue;
+        }
+
+        let block = match store.get_block_sync(&cid_bytes)? {
+            Some(b) => b,
+            None => {
+                tracing::warn!(
+                    ?cid_bytes,
+                    "referenced block missing during reachability walk"
+                );
+                continue;
+            }
+        };
+
+        if let Ok(commit) = Commit::from_cbor(&block) {
+            to_visit.push(cid_to_bytes(&commit.data)?);
+            if let Some(prev) = &commit.prev {
+                to_visit.push(cid_to_bytes(prev)?);
+            }
+        } else if let Ok(Ipld::Map(ref obj)) = serde_ipld_dagcbor::from_slice::<Ipld>(&block) {
+            if let Some(Ipld::Link(left_cid)) = obj.get("l")
+                && let Ok(bytes) = <CidBytes>::try_from(left_cid.to_bytes().as_slice())
+            {
+                to_visit.push(bytes);
+            }
+            if let Some(Ipld::List(entries)) = obj.get("e") {
+                entries
+                    .iter()
+                    .filter_map(|entry| match entry {
+                        Ipld::Map(entry_obj) => Some(entry_obj),
+                        _ => None,
+                    })
+                    .flat_map(|entry_obj| {
+                        [entry_obj.get("t"), entry_obj.get("v")]
+                            .into_iter()
+                            .flatten()
+                            .filter_map(|v| match v {
+                                Ipld::Link(link_cid) => {
+                                    <CidBytes>::try_from(link_cid.to_bytes().as_slice()).ok()
+                                }
+                                _ => None,
+                            })
+                    })
+                    .for_each(|bytes| to_visit.push(bytes));
+            }
+        }
+    }
+
+    Ok(())
+}
+
+fn paginate_repos(
+    rt: &tokio::runtime::Handle,
+    repo_repo: &dyn RepoRepository,
+    mut each_page: impl FnMut(&[tranquil_db_traits::RepoListItem]) -> anyhow::Result<()>,
+) -> anyhow::Result<()> {
+    let mut cursor_did: Option<Did> = None;
+
+    std::iter::from_fn(|| {
+        let page = rt
+            .block_on(repo_repo.list_repos_paginated(cursor_did.as_ref(), REPO_PAGE_SIZE))
+            .context("failed to list repos");
+        match &page {
+            Ok(p) => {
+                cursor_did = p.last().map(|r| r.did.clone());
+                cursor_did.as_ref().map(|_| page)
+            }
+            Err(_) => Some(page),
+        }
+    })
+    .try_for_each(|page| each_page(&page?))
+}
+
+pub fn run_reachability_walk(
+    store: &tranquil_store::blockstore::TranquilBlockStore,
+    repo_repo: &dyn RepoRepository,
+) -> anyhow::Result<ReachabilityResult> {
+    let rt = tokio::runtime::Handle::current();
+
+    let approx_blocks = store.approximate_block_count();
+
+    const MAX_PREALLOC: usize = 64_000_000;
+    let mut visited = std::collections::HashSet::with_capacity(
+        usize::try_from(approx_blocks)
+            .unwrap_or(0)
+            .min(MAX_PREALLOC),
+    );
+
+    info!(approx_blocks, "reachability walk starting");
+
+    let mut repos_walked: u64 = 0;
+    let mut seen_heads: std::collections::HashMap<Did, CidLink> = std::collections::HashMap::new();
+
+    paginate_repos(&rt, repo_repo, |page| {
+        page.iter().try_for_each(|repo| -> anyhow::Result<()> {
+            let cid =
+                Cid::from_str(repo.repo_root_cid.as_str()).context("invalid repo_root_cid")?;
+            seen_heads.insert(repo.did.clone(), repo.repo_root_cid.clone());
+            walk_repo_dag_sync(store, &cid, &mut visited)?;
+            repos_walked = repos_walked.saturating_add(1);
+            if repos_walked.is_multiple_of(1000) {
+                info!(
+                    repos_walked,
+                    blocks_so_far = visited.len(),
+                    "reachability walk progress"
+                );
+            }
+            Ok(())
+        })
+    })?;
+
+    let blocks_visited = u64::try_from(visited.len()).unwrap_or(u64::MAX);
+
+    let mut reachable =
+        BloomFilter::with_capacity_and_fpr(blocks_visited.max(1024), BLOOM_FALSE_POSITIVE_RATE);
+    visited.iter().for_each(|cid| reachable.insert(cid));
+    drop(visited);
+
+    let mut stale_repos: u64 = 0;
+    paginate_repos(&rt, repo_repo, |page| {
+        let stale: Vec<_> = page
+            .iter()
+            .filter(|repo| seen_heads.get(&repo.did) != Some(&repo.repo_root_cid))
+            .collect();
+        stale.iter().try_for_each(|repo| -> anyhow::Result<()> {
+            let cid =
+                Cid::from_str(repo.repo_root_cid.as_str()).context("invalid repo_root_cid")?;
+            let mut extra = std::collections::HashSet::new();
+            walk_repo_dag_sync(store, &cid, &mut extra)?;
+            extra.iter().for_each(|c| reachable.insert(c));
+            seen_heads.insert(repo.did.clone(), repo.repo_root_cid.clone());
+            stale_repos = stale_repos.saturating_add(1);
+            Ok(())
+        })
+    })?;
+
+    info!(
+        repos_walked,
+        blocks_visited,
+        stale_repos,
+        bloom_heap_mb = reachable.heap_bytes() / (1024 * 1024),
+        "DAG traversal complete, quiescing blockstore for leak scan"
+    );
+
+    let (_snapshot, quiesce_guard) = store
+        .quiesce()
+        .map_err(|e| anyhow::anyhow!("failed to quiesce blockstore: {e}"))?;
+
+    let mut quiesced_stale: u64 = 0;
+    paginate_repos(&rt, repo_repo, |page| {
+        page.iter()
+            .filter(|repo| seen_heads.get(&repo.did) != Some(&repo.repo_root_cid))
+            .try_for_each(|repo| -> anyhow::Result<()> {
+                let cid =
+                    Cid::from_str(repo.repo_root_cid.as_str()).context("invalid repo_root_cid")?;
+                let mut extra = std::collections::HashSet::new();
+                walk_repo_dag_sync(store, &cid, &mut extra)?;
+                extra.iter().for_each(|c| reachable.insert(c));
+                quiesced_stale = quiesced_stale.saturating_add(1);
+                Ok(())
+            })
+    })?;
+
+    if quiesced_stale > 0 {
+        info!(
+            quiesced_stale,
+            "caught additional stale repos during quiesced re-walk"
+        );
+    }
+
+    let (leaked, live_refcounted) = store
+        .find_leaked_refcounts(|cid| reachable.contains(cid))
+        .map_err(|e| anyhow::anyhow!("failed to scan index: {e}"))?;
+    let leaked_blocks = u64::try_from(leaked.len()).unwrap_or(u64::MAX);
+    let bloom_heap_bytes = reachable.heap_bytes();
+    drop(reachable);
+
+    quiesce_guard.resume();
+
+    let repaired_blocks = match leaked.is_empty() {
+        true => 0,
+        false => {
+            warn!(
+                leaked_blocks,
+                "reachability walk found leaked refcounts, repairing"
+            );
+            store
+                .repair_leaked_refcounts(&leaked)
+                .map_err(|e| anyhow::anyhow!("failed to repair leaked refcounts: {e}"))?
+        }
+    };
+
+    Ok(ReachabilityResult {
+        repos_walked,
+        blocks_visited,
+        live_refcounted,
+        leaked_blocks,
+        repaired_blocks,
+        bloom_heap_bytes,
+    })
+}

crates/tranquil-pds/src/state.rs

@@ -49,6 +49,8 @@ pub struct AppState {
     pub bootstrap_invite_code: Option<String>,
     pub signal_sender: Option<Arc<tranquil_signal::SignalSlot>>,
     pub signal_store_provider: Option<Arc<dyn tranquil_signal::SignalStoreProvider>>,
+    pub eventlog_segments_dir: Option<PathBuf>,
+    pub repo_export_semaphore: Arc<tokio::sync::Semaphore>,
 }
 
 #[derive(Debug, Clone, Copy)]
@@ -214,9 +216,7 @@ impl AppState {
 
         match cfg.storage.repo_backend() {
             tranquil_config::RepoBackend::TranquilStore => {
-                tracing::info!(
-                    "tranquil-store repo backend active. EXPERIMENTAL! No garbage collection, no backup/restore"
-                );
+                tracing::info!("tranquil-store repo backend active. EXPERIMENTAL!");
                 Ok(Self::from_store(shutdown).await)
             }
             tranquil_config::RepoBackend::Postgres => {
@@ -273,10 +273,11 @@ impl AppState {
 
     pub async fn from_db(db: PgPool, shutdown: CancellationToken) -> Self {
         let cfg = tranquil_config::get();
-        let (repos, block_store, signal_store_provider): (
+        let (repos, block_store, signal_store_provider, eventlog_segments_dir): (
             PostgresRepositories,
             crate::repo::AnyBlockStore,
             Option<Arc<dyn tranquil_signal::SignalStoreProvider>>,
+            Option<PathBuf>,
         ) = match cfg.storage.repo_backend() == tranquil_config::RepoBackend::TranquilStore {
             true => {
                 let wiring = wire_tranquil_store(&cfg.tranquil_store, shutdown.clone());
@@ -284,6 +285,7 @@ impl AppState {
                     wiring.repos,
                     crate::repo::AnyBlockStore::TranquilStore(wiring.blockstore),
                     Some(wiring.signal_provider),
+                    Some(wiring.segments_dir),
                 )
             }
             false => {
@@ -294,11 +296,19 @@ impl AppState {
                     repos,
                     crate::repo::AnyBlockStore::Postgres(PostgresBlockStore::new(db)),
                     Some(provider),
+                    None,
                 )
             }
         };
 
-        Self::build(repos, block_store, signal_store_provider, shutdown).await
+        Self::build(
+            repos,
+            block_store,
+            signal_store_provider,
+            eventlog_segments_dir,
+            shutdown,
+        )
+        .await
     }
 
     pub async fn from_store(shutdown: CancellationToken) -> Self {
@@ -309,6 +319,30 @@ impl AppState {
             wiring.repos,
             crate::repo::AnyBlockStore::TranquilStore(wiring.blockstore),
             Some(wiring.signal_provider),
+            Some(wiring.segments_dir),
+            shutdown,
+        )
+        .await
+    }
+
+    pub async fn from_store_at(data_dir: &std::path::Path, shutdown: CancellationToken) -> Self {
+        let base = &tranquil_config::get().tranquil_store;
+        let store_cfg = tranquil_config::TranquilStoreConfig {
+            data_dir: data_dir.to_string_lossy().into_owned(),
+            memory_budget_mb: base.memory_budget_mb,
+            handler_threads: base.handler_threads,
+            eventlog_pending_bytes_budget: base.eventlog_pending_bytes_budget,
+            eventlog_max_event_payload: base.eventlog_max_event_payload,
+            max_blockstore_file_size: base.max_blockstore_file_size,
+            max_eventlog_segment_size: base.max_eventlog_segment_size,
+        };
+        let wiring = wire_tranquil_store(&store_cfg, shutdown.clone());
+
+        Self::build(
+            wiring.repos,
+            crate::repo::AnyBlockStore::TranquilStore(wiring.blockstore),
+            Some(wiring.signal_provider),
+            Some(wiring.segments_dir),
             shutdown,
         )
         .await
@@ -318,6 +352,7 @@ impl AppState {
         repos: PostgresRepositories,
         block_store: crate::repo::AnyBlockStore,
         signal_store_provider: Option<Arc<dyn tranquil_signal::SignalStoreProvider>>,
+        eventlog_segments_dir: Option<PathBuf>,
         shutdown: CancellationToken,
     ) -> Self {
         AuthConfig::init();
@@ -359,6 +394,10 @@ impl AppState {
             bootstrap_invite_code: None,
             signal_sender: None,
             signal_store_provider,
+            eventlog_segments_dir,
+            repo_export_semaphore: Arc::new(tokio::sync::Semaphore::new(
+                cfg.firehose.max_concurrent_repo_exports,
+            )),
         }
     }
 
@@ -442,6 +481,7 @@ struct TranquilStoreWiring {
     blockstore: tranquil_store::blockstore::TranquilBlockStore,
     signal_provider: Arc<dyn tranquil_signal::SignalStoreProvider>,
     repos: PostgresRepositories,
+    segments_dir: PathBuf,
 }
 
 fn wire_tranquil_store(
@@ -486,14 +526,18 @@ fn wire_tranquil_store(
     let blockstore = TranquilBlockStore::open(BlockStoreConfig {
         data_dir: blockstore_data_dir,
         index_dir: blockstore_index_dir,
-        max_file_size: tranquil_store::blockstore::DEFAULT_MAX_FILE_SIZE,
+        max_file_size: store_cfg.max_blockstore_file_size,
         group_commit: Default::default(),
+        shard_count: tranquil_store::blockstore::DEFAULT_SHARD_COUNT,
     })
     .expect("failed to open blockstore");
 
     let event_log = EventLog::open(
         EventLogConfig {
             segments_dir,
+            pending_bytes_budget: store_cfg.eventlog_pending_bytes_budget,
+            max_event_payload: store_cfg.eventlog_max_event_payload,
+            max_segment_size: store_cfg.max_eventlog_segment_size,
             ..EventLogConfig::default()
         },
         RealIO::new(),
@@ -503,6 +547,10 @@ fn wire_tranquil_store(
 
     let bridge = Arc::new(EventLogBridge::new(Arc::clone(&event_log)));
 
+    let was_clean = tranquil_store::consistency::had_clean_shutdown(&data_dir);
+    tranquil_store::consistency::remove_clean_shutdown_marker(&data_dir)
+        .expect("failed to remove clean shutdown marker");
+
     let indexes = metastore.partition(Partition::Indexes).clone();
     let event_ops = metastore.event_ops(Arc::clone(&bridge));
     let recovered = event_ops
@@ -512,6 +560,41 @@ fn wire_tranquil_store(
         tracing::info!(recovered, "replayed metastore mutations from eventlog");
     }
 
+    let skip_check = std::env::var("TRANQUIL_SKIP_CONSISTENCY_CHECK").is_ok_and(|v| v == "1");
+    if (!was_clean || recovered > 0) && !skip_check {
+        let report = tranquil_store::consistency::verify_store_consistency(
+            &blockstore,
+            &metastore,
+            &event_log,
+        );
+        report.log_findings();
+
+        if report.has_repairable_issues() {
+            let repair = tranquil_store::consistency::repair_known_issues(&blockstore, &report);
+            if repair.orphan_files_removed > 0 {
+                tracing::info!(
+                    removed = repair.orphan_files_removed,
+                    "repaired orphan data files"
+                );
+            }
+            if repair.had_errors() {
+                tracing::warn!(errors = repair.repair_errors, "some repairs failed");
+            }
+        }
+
+        if report.has_unrecoverable_issues() {
+            panic!(
+                "unrecoverable store inconsistencies detected: {} dangling root CIDs, {} dangling record CIDs, \
+                 {} deserialization failures, cursor_ahead={}. \
+                 manual intervention required. set TRANQUIL_SKIP_CONSISTENCY_CHECK=1 to bypass.",
+                report.dangling_root_cids.len(),
+                report.dangling_record_cids.len(),
+                report.deserialization_failures,
+                report.cursor_ahead_of_eventlog,
+            );
+        }
+    }
+
     let notifier = bridge.notifier();
     let signal_db = metastore.database().clone();
     let signal_ks = metastore.signal_keyspace();
@@ -525,13 +608,23 @@ fn wire_tranquil_store(
 
     tokio::spawn({
         let pool = Arc::clone(&pool);
+        let shutdown_event_log = Arc::clone(&event_log);
+        let shutdown_data_dir = data_dir.clone();
         async move {
             shutdown.cancelled().await;
             pool.close().await;
+            if let Err(e) = shutdown_event_log.shutdown() {
+                tracing::warn!(error = %e, "eventlog shutdown failed");
+            }
+            if let Err(e) =
+                tranquil_store::consistency::write_clean_shutdown_marker(&shutdown_data_dir)
+            {
+                tracing::warn!(error = %e, "failed to write clean shutdown marker");
+            }
         }
     });
 
-    let client = MetastoreClient::<RealIO>::new(pool);
+    let client = MetastoreClient::<RealIO>::new(pool, Arc::clone(&event_log));
 
     tracing::info!(data_dir = %store_cfg.data_dir, "tranquil-store data directory");
 
@@ -553,9 +646,12 @@ fn wire_tranquil_store(
         tranquil_signal::fjall_store::FjallSignalStoreProvider::new(signal_db, signal_ks),
     );
 
+    let eventlog_segments_dir = event_log.segments_dir().to_path_buf();
+
     TranquilStoreWiring {
         blockstore,
         signal_provider,
         repos,
+        segments_dir: eventlog_segments_dir,
     }
 }

crates/tranquil-pds/src/sync/frame.rs

@@ -142,7 +142,6 @@ pub struct CommitFrameBuilder {
     seq: i64,
     did: Did,
     commit_cid: Cid,
-    prev_cid: Option<Cid>,
     ops_json: serde_json::Value,
     blob_cids: Vec<Cid>,
     time: chrono::DateTime<chrono::Utc>,
@@ -150,12 +149,10 @@ pub struct CommitFrameBuilder {
 }
 
 impl CommitFrameBuilder {
-    #[allow(clippy::too_many_arguments)]
     pub fn new(
         seq: i64,
         did: Did,
         commit_cid_str: &str,
-        prev_cid_str: Option<&str>,
         ops_json: serde_json::Value,
         blob_strs: Vec<String>,
         time: chrono::DateTime<chrono::Utc>,
@@ -163,9 +160,6 @@ impl CommitFrameBuilder {
     ) -> Result<Self, CommitFrameError> {
         let commit_cid = Cid::from_str(commit_cid_str)
             .map_err(|_| CommitFrameError::InvalidCommitCid(commit_cid_str.to_string()))?;
-        let prev_cid = prev_cid_str.map(Cid::from_str).transpose().map_err(|_| {
-            CommitFrameError::InvalidCommitCid(prev_cid_str.unwrap_or("").to_string())
-        })?;
         let blob_cids: Vec<Cid> = blob_strs
             .iter()
             .filter_map(|s| Cid::from_str(s).ok())
@@ -174,7 +168,6 @@ impl CommitFrameBuilder {
             seq,
             did,
             commit_cid,
-            prev_cid,
             ops_json,
             blob_cids,
             time,
@@ -197,7 +190,7 @@ impl CommitFrameBuilder {
             })
             .collect();
         let rev = self.rev.unwrap_or_else(placeholder_rev);
-        let since = self.prev_cid.as_ref().map(|_| rev.clone());
+        let since = None;
         CommitFrame {
             seq: self.seq,
             rebase: false,
@@ -235,7 +228,6 @@ impl TryFrom<SequencedEvent> for CommitFrame {
             event.seq.as_i64(),
             event.did.clone(),
             commit_cid.as_str(),
-            event.prev_cid.as_ref().map(|c| c.as_str()),
             event.ops.unwrap_or_default(),
             event.blobs.unwrap_or_default(),
             event.created_at,

crates/tranquil-pds/src/sync/import.rs

@@ -8,7 +8,7 @@ use std::io::Cursor;
 use std::sync::Arc;
 use thiserror::Error;
 use tracing::debug;
-use tranquil_db::{ImportBlock, ImportRecord, ImportRepoError, RepoRepository};
+use tranquil_db_traits::{ImportBlock, ImportRecord, ImportRepoError, RepoRepository};
 use tranquil_types::CidLink;
 use uuid::Uuid;
 

crates/tranquil-pds/src/sync/util.rs

@@ -15,7 +15,7 @@ use std::collections::{BTreeMap, HashMap};
 use std::io::Cursor;
 use std::str::FromStr;
 use tokio::io::AsyncWriteExt;
-use tranquil_db_traits::{AccountStatus, RepoEventType, RepoRepository};
+use tranquil_db_traits::{AccountStatus, EventBlocks, RepoEventType, RepoRepository};
 use tranquil_types::Did;
 
 #[derive(Debug)]
@@ -25,7 +25,8 @@ pub enum SyncFrameError {
     IoFlush(std::io::Error),
     CborSerialize(String),
     MissingCommitCid,
-    CommitBlockNotFound,
+    MissingInlineCommitBlock,
+    MissingLegacyBlocks(Vec<Cid>),
     RevExtraction,
     InvalidEvent(String),
     BlockStore(tranquil_db_traits::DbError),
@@ -40,7 +41,17 @@ impl std::fmt::Display for SyncFrameError {
             Self::IoFlush(e) => write!(f, "CAR buffer flush failed: {}", e),
             Self::CborSerialize(e) => write!(f, "CBOR serialization failed: {}", e),
             Self::MissingCommitCid => write!(f, "missing commit_cid"),
-            Self::CommitBlockNotFound => write!(f, "commit block not found"),
+            Self::MissingInlineCommitBlock => {
+                write!(f, "event missing inline commit block bytes")
+            }
+            Self::MissingLegacyBlocks(cids) => {
+                write!(
+                    f,
+                    "legacy event references blocks not present in live blockstore (gc race): {} missing cid(s), first: {}",
+                    cids.len(),
+                    cids.first().map(|c| c.to_string()).unwrap_or_default()
+                )
+            }
             Self::RevExtraction => write!(f, "could not extract rev from commit"),
             Self::InvalidEvent(msg) => write!(f, "invalid event: {}", msg),
             Self::BlockStore(e) => write!(f, "block store error: {}", e),
@@ -178,7 +189,7 @@ fn extract_rev_from_commit_bytes(commit_bytes: &[u8]) -> Option<String> {
 
 async fn write_car_blocks(
     commit_cid: Cid,
-    commit_bytes: Option<Bytes>,
+    commit_bytes: Bytes,
     other_blocks: BTreeMap<Cid, Bytes>,
 ) -> Result<Vec<u8>, SyncFrameError> {
     let mut buffer = Cursor::new(Vec::new());
@@ -190,12 +201,10 @@ async fn write_car_blocks(
             .await
             .map_err(SyncFrameError::CarWrite)?;
     }
-    if let Some(data) = commit_bytes {
-        writer
-            .write(commit_cid, data.as_ref())
-            .await
-            .map_err(SyncFrameError::CarWrite)?;
-    }
+    writer
+        .write(commit_cid, commit_bytes.as_ref())
+        .await
+        .map_err(SyncFrameError::CarWrite)?;
     writer.finish().await.map_err(SyncFrameError::CarFinalize)?;
     buffer.flush().await.map_err(SyncFrameError::IoFlush)?;
     Ok(buffer.into_inner())
@@ -265,26 +274,81 @@ fn format_account_event(event: &SequencedEvent) -> Result<Vec<u8>, SyncFrameErro
     Ok(bytes)
 }
 
-async fn format_sync_event(
+async fn event_blocks_to_map(
+    blocks: Option<&EventBlocks>,
+    prefetched: &HashMap<Cid, Bytes>,
     state: &AppState,
+) -> Result<HashMap<Cid, Bytes>, SyncFrameError> {
+    match blocks {
+        None => Ok(HashMap::new()),
+        Some(EventBlocks::Inline(inline)) => inline
+            .iter()
+            .map(|b| {
+                Cid::read_bytes(b.cid_bytes.as_slice())
+                    .map_err(SyncFrameError::CidParse)
+                    .map(|cid| (cid, Bytes::copy_from_slice(&b.data)))
+            })
+            .collect(),
+        Some(EventBlocks::LegacyCids(cid_strs)) => {
+            let cids: Vec<Cid> = cid_strs
+                .iter()
+                .map(|s| Cid::from_str(s).map_err(SyncFrameError::CidParse))
+                .collect::<Result<_, _>>()?;
+            let mut map: HashMap<Cid, Bytes> = HashMap::with_capacity(cids.len());
+            let to_fetch: Vec<Cid> = cids
+                .iter()
+                .filter(|cid| match prefetched.get(cid) {
+                    Some(b) => {
+                        map.insert(**cid, b.clone());
+                        false
+                    }
+                    None => true,
+                })
+                .copied()
+                .collect();
+            if !to_fetch.is_empty() {
+                let fetched = state.block_store.get_many(&to_fetch).await?;
+                let (found, missing): (Vec<_>, Vec<_>) = to_fetch
+                    .into_iter()
+                    .zip(fetched)
+                    .partition(|(_, opt)| opt.is_some());
+                found
+                    .into_iter()
+                    .filter_map(|(cid, opt)| opt.map(|b| (cid, b)))
+                    .for_each(|(cid, b)| {
+                        map.insert(cid, b);
+                    });
+                if !missing.is_empty() {
+                    let missing_cids: Vec<Cid> = missing.into_iter().map(|(cid, _)| cid).collect();
+                    return Err(SyncFrameError::MissingLegacyBlocks(missing_cids));
+                }
+            }
+            Ok(map)
+        }
+    }
+}
+
+async fn format_sync_event(
     event: &SequencedEvent,
+    prefetched: &HashMap<Cid, Bytes>,
+    state: &AppState,
 ) -> Result<Vec<u8>, SyncFrameError> {
     let commit_cid_str = event
         .commit_cid
         .as_ref()
         .ok_or(SyncFrameError::MissingCommitCid)?;
     let commit_cid = Cid::from_str(commit_cid_str)?;
-    let commit_bytes = state
-        .block_store
+    let blocks_map = event_blocks_to_map(event.blocks.as_ref(), prefetched, state).await?;
+    let commit_bytes = blocks_map
         .get(&commit_cid)
-        .await?
-        .ok_or(SyncFrameError::CommitBlockNotFound)?;
+        .cloned()
+        .ok_or(SyncFrameError::MissingInlineCommitBlock)?;
     let rev = if let Some(ref stored_rev) = event.rev {
         stored_rev.clone()
     } else {
         extract_rev_from_commit_bytes(&commit_bytes).ok_or(SyncFrameError::RevExtraction)?
     };
-    let car_bytes = write_car_blocks(commit_cid, Some(commit_bytes), BTreeMap::new()).await?;
+    let car_bytes = write_car_blocks(commit_cid, commit_bytes, BTreeMap::new()).await?;
     serialize_event_frame(
         FrameType::Sync,
         &SyncFrame {
@@ -302,13 +366,17 @@ struct CommitEventContext {
     frame: CommitFrame,
     commit_cid: Cid,
     prev_cid: Option<Cid>,
-    block_cids: Vec<Cid>,
+    inline_blocks: HashMap<Cid, Bytes>,
 }
 
-fn prepare_commit_event(event: SequencedEvent) -> Result<CommitEventContext, SyncFrameError> {
-    let block_cids_str = event.blocks_cids.clone().unwrap_or_default();
+async fn prepare_commit_event(
+    event: SequencedEvent,
+    prefetched: &HashMap<Cid, Bytes>,
+    state: &AppState,
+) -> Result<CommitEventContext, SyncFrameError> {
     let prev_cid_link = event.prev_cid.clone();
     let prev_data_cid_link = event.prev_data_cid.clone();
+    let inline_blocks = event_blocks_to_map(event.blocks.as_ref(), prefetched, state).await?;
     let mut frame: CommitFrame =
         event
             .try_into()
@@ -321,46 +389,43 @@ fn prepare_commit_event(event: SequencedEvent) -> Result<CommitEventContext, Syn
         frame.prev_data = Some(cid);
     }
     let commit_cid = frame.commit;
+    if !inline_blocks.contains_key(&commit_cid) {
+        return Err(SyncFrameError::MissingInlineCommitBlock);
+    }
     let prev_cid = prev_cid_link
         .as_ref()
         .and_then(|c| Cid::from_str(c.as_str()).ok());
-    let mut block_cids: Vec<Cid> = block_cids_str
-        .iter()
-        .filter_map(|s| Cid::from_str(s).ok())
-        .filter(|c| Some(*c) != prev_cid)
-        .collect();
-    if !block_cids.contains(&commit_cid) {
-        block_cids.push(commit_cid);
-    }
     Ok(CommitEventContext {
         frame,
         commit_cid,
         prev_cid,
-        block_cids,
+        inline_blocks,
     })
 }
 
 fn partition_blocks(
     block_cids: impl IntoIterator<Item = (Cid, Bytes)>,
     commit_cid: Cid,
-) -> (Option<Bytes>, BTreeMap<Cid, Bytes>) {
+) -> Result<(Bytes, BTreeMap<Cid, Bytes>), SyncFrameError> {
     let (commit_data, other_blocks): (Vec<_>, Vec<_>) = block_cids
         .into_iter()
         .partition(|(cid, _)| *cid == commit_cid);
-    let commit_bytes = commit_data.into_iter().next().map(|(_, data)| data);
+    let commit_bytes = commit_data
+        .into_iter()
+        .next()
+        .map(|(_, data)| data)
+        .ok_or(SyncFrameError::MissingInlineCommitBlock)?;
     let other = other_blocks.into_iter().collect();
-    (commit_bytes, other)
+    Ok((commit_bytes, other))
 }
 
 async fn finalize_commit_frame(
     mut frame: CommitFrame,
     commit_cid: Cid,
-    commit_bytes: Option<Bytes>,
+    commit_bytes: Bytes,
     other_blocks: BTreeMap<Cid, Bytes>,
 ) -> Result<Vec<u8>, SyncFrameError> {
-    if let Some(ref cb) = commit_bytes
-        && let Some(rev) = extract_rev_from_commit_bytes(cb)
-    {
+    if let Some(rev) = extract_rev_from_commit_bytes(&commit_bytes) {
         frame.rev = rev;
     }
     frame.blocks = write_car_blocks(commit_cid, commit_bytes, other_blocks).await?;
@@ -371,134 +436,63 @@ async fn finalize_commit_frame(
 pub async fn format_event_for_sending(
     state: &AppState,
     event: SequencedEvent,
+) -> Result<Vec<u8>, SyncFrameError> {
+    format_event_inner(event, &HashMap::new(), state).await
+}
+
+async fn format_event_inner(
+    event: SequencedEvent,
+    prefetched: &HashMap<Cid, Bytes>,
+    state: &AppState,
 ) -> Result<Vec<u8>, SyncFrameError> {
     match event.event_type {
         RepoEventType::Identity => return format_identity_event(&event),
         RepoEventType::Account => return format_account_event(&event),
-        RepoEventType::Sync => return format_sync_event(state, &event).await,
+        RepoEventType::Sync => return format_sync_event(&event, prefetched, state).await,
         RepoEventType::Commit => {}
     }
-    let ctx = prepare_commit_event(event)?;
+    let ctx = prepare_commit_event(event, prefetched, state).await?;
     let mut frame = ctx.frame;
     if let Some(ref pc) = ctx.prev_cid
-        && let Ok(Some(prev_bytes)) = state.block_store.get(pc).await
-        && let Some(rev) = extract_rev_from_commit_bytes(&prev_bytes)
+        && let Some(prev_bytes) = ctx.inline_blocks.get(pc)
+        && let Some(rev) = extract_rev_from_commit_bytes(prev_bytes)
     {
         frame.since = Some(rev);
     }
-    if ctx.block_cids.is_empty() {
-        frame.blocks = Vec::new();
-        let capacity = frame.blocks.len() + 512;
-        return serialize_event_frame(FrameType::Commit, &frame, capacity);
-    }
-    let fetched = state.block_store.get_many(&ctx.block_cids).await?;
-    let resolved = ctx
-        .block_cids
-        .iter()
-        .zip(fetched.iter())
-        .filter_map(|(cid, data_opt)| data_opt.as_ref().map(|data| (*cid, data.clone())));
-    let (commit_bytes, other_blocks) = partition_blocks(resolved, ctx.commit_cid);
+    let (commit_bytes, other_blocks) = partition_blocks(ctx.inline_blocks, ctx.commit_cid)?;
     finalize_commit_frame(frame, ctx.commit_cid, commit_bytes, other_blocks).await
 }
 
+pub async fn format_event_with_prefetched_blocks(
+    state: &AppState,
+    event: SequencedEvent,
+    prefetched: &HashMap<Cid, Bytes>,
+) -> Result<Vec<u8>, SyncFrameError> {
+    format_event_inner(event, prefetched, state).await
+}
+
 pub async fn prefetch_blocks_for_events(
     state: &AppState,
     events: &[SequencedEvent],
 ) -> Result<HashMap<Cid, Bytes>, SyncFrameError> {
-    let mut all_cids: Vec<Cid> = events
+    let legacy_cids: Vec<Cid> = events
         .iter()
-        .flat_map(|event| {
-            let commit_cid = event
-                .commit_cid
-                .as_ref()
-                .and_then(|s| Cid::from_str(s).ok());
-            let prev_cid = event.prev_cid.as_ref().and_then(|s| Cid::from_str(s).ok());
-            let block_cids = event
-                .blocks_cids
-                .as_ref()
-                .map(|cids| cids.iter().filter_map(|s| Cid::from_str(s).ok()).collect())
-                .unwrap_or_else(Vec::new);
-            commit_cid.into_iter().chain(prev_cid).chain(block_cids)
+        .filter_map(|e| match e.blocks.as_ref() {
+            Some(EventBlocks::LegacyCids(strs)) => Some(strs.iter()),
+            _ => None,
         })
-        .collect();
-    all_cids.sort();
-    all_cids.dedup();
-    if all_cids.is_empty() {
+        .flatten()
+        .map(|s| Cid::from_str(s).map_err(SyncFrameError::CidParse))
+        .collect::<Result<_, _>>()?;
+    if legacy_cids.is_empty() {
         return Ok(HashMap::new());
     }
-    let fetched = state.block_store.get_many(&all_cids).await?;
-    let blocks_map: HashMap<Cid, Bytes> = all_cids
+    let fetched = state.block_store.get_many(&legacy_cids).await?;
+    Ok(legacy_cids
         .into_iter()
         .zip(fetched)
-        .filter_map(|(cid, data_opt)| data_opt.map(|data| (cid, data)))
-        .collect();
-    Ok(blocks_map)
-}
-
-fn format_sync_event_with_prefetched(
-    event: &SequencedEvent,
-    prefetched: &HashMap<Cid, Bytes>,
-) -> Result<Vec<u8>, SyncFrameError> {
-    let commit_cid_str = event
-        .commit_cid
-        .as_ref()
-        .ok_or(SyncFrameError::MissingCommitCid)?;
-    let commit_cid = Cid::from_str(commit_cid_str)?;
-    let commit_bytes = prefetched
-        .get(&commit_cid)
-        .ok_or(SyncFrameError::CommitBlockNotFound)?;
-    let rev = if let Some(ref stored_rev) = event.rev {
-        stored_rev.clone()
-    } else {
-        extract_rev_from_commit_bytes(commit_bytes).ok_or(SyncFrameError::RevExtraction)?
-    };
-    let car_bytes = futures::executor::block_on(write_car_blocks(
-        commit_cid,
-        Some(commit_bytes.clone()),
-        BTreeMap::new(),
-    ))?;
-    serialize_event_frame(
-        FrameType::Sync,
-        &SyncFrame {
-            did: event.did.clone(),
-            rev,
-            blocks: car_bytes,
-            seq: event.seq.as_i64(),
-            time: format_atproto_time(event.created_at),
-        },
-        512,
-    )
-}
-
-pub async fn format_event_with_prefetched_blocks(
-    event: SequencedEvent,
-    prefetched: &HashMap<Cid, Bytes>,
-) -> Result<Vec<u8>, SyncFrameError> {
-    match event.event_type {
-        RepoEventType::Identity => return format_identity_event(&event),
-        RepoEventType::Account => return format_account_event(&event),
-        RepoEventType::Sync => return format_sync_event_with_prefetched(&event, prefetched),
-        RepoEventType::Commit => {}
-    }
-    let ctx = prepare_commit_event(event)?;
-    let mut frame = ctx.frame;
-    if let Some(ref pc) = ctx.prev_cid
-        && let Some(prev_bytes) = prefetched.get(pc)
-        && let Some(rev) = extract_rev_from_commit_bytes(prev_bytes)
-    {
-        frame.since = Some(rev);
-    }
-    if ctx.block_cids.is_empty() {
-        frame.blocks = Vec::new();
-        let capacity = frame.blocks.len() + 512;
-        return serialize_event_frame(FrameType::Commit, &frame, capacity);
-    }
-    let resolved = ctx
-        .block_cids
-        .into_iter()
-        .filter_map(|cid| prefetched.get(&cid).map(|data| (cid, data.clone())));
-    let (commit_bytes, other_blocks) = partition_blocks(resolved, ctx.commit_cid);
-    finalize_commit_frame(frame, ctx.commit_cid, commit_bytes, other_blocks).await
+        .filter_map(|(cid, opt)| opt.map(|b| (cid, b)))
+        .collect())
 }
 
 pub fn format_info_frame(

crates/tranquil-pds/tests/account_lifecycle.rs

@@ -99,9 +99,8 @@ async fn test_check_account_status_returns_correct_block_count() {
         after_delete_blocks
     );
     assert!(
-        after_delete_blocks >= initial_blocks,
-        "Block count after delete should be at least initial count (initial {}, now {})",
-        initial_blocks,
+        after_delete_blocks >= 2,
+        "Block count after delete should have at least commit + MST root (got {})",
         after_delete_blocks
     );
 }

crates/tranquil-pds/tests/common/mod.rs

@@ -29,6 +29,7 @@ static TEST_DB_POOL: OnceLock<sqlx::PgPool> = OnceLock::new();
 static TEST_TEMP_DIR: OnceLock<PathBuf> = OnceLock::new();
 static CLUSTER: OnceLock<Vec<ServerInstance>> = OnceLock::new();
 static TEST_REPOS: OnceLock<Arc<tranquil_db::PostgresRepositories>> = OnceLock::new();
+static TEST_BLOCK_STORE: OnceLock<tranquil_pds::repo::AnyBlockStore> = OnceLock::new();
 
 #[allow(dead_code)]
 pub fn is_store_backend() -> bool {
@@ -41,6 +42,8 @@ pub fn is_store_backend() -> bool {
 pub struct ServerConfig {
     pub pool: Option<sqlx::PgPool>,
     pub cache: Option<(Arc<dyn Cache>, Arc<dyn DistributedRateLimiter>)>,
+    pub store_path: Option<PathBuf>,
+    pub shared_state: Option<AppState>,
 }
 
 #[allow(dead_code)]
@@ -574,12 +577,17 @@ async fn spawn_server(config: ServerConfig) -> ServerInstance {
         .with_oauth_authorize_limit(10000)
         .with_oauth_token_limit(10000);
     let cache_refs = config.cache.as_ref().map(|(c, r)| (c.clone(), r.clone()));
-    let mut state = match config.pool {
-        Some(pool) => AppState::from_db(pool, CancellationToken::new()).await,
-        None => AppState::from_store(CancellationToken::new()).await,
+    let mut state = match config.shared_state {
+        Some(s) => s,
+        None => match (config.pool, config.store_path) {
+            (Some(pool), _) => AppState::from_db(pool, CancellationToken::new()).await,
+            (None, Some(path)) => AppState::from_store_at(&path, CancellationToken::new()).await,
+            (None, None) => AppState::from_store(CancellationToken::new()).await,
+        },
     };
     state = state.with_rate_limiters(rate_limiters);
     TEST_REPOS.set(state.repos.clone()).ok();
+    TEST_BLOCK_STORE.set(state.block_store.clone()).ok();
     if let Some((cache, distributed_rate_limiter)) = config.cache {
         state = state.with_cache(cache, distributed_rate_limiter);
     }
@@ -637,6 +645,8 @@ async fn setup_store_backend() -> String {
     let instance = spawn_server(ServerConfig {
         pool: None,
         cache: None,
+        store_path: None,
+        shared_state: None,
     })
     .await;
     APP_PORT.set(instance.port).ok();
@@ -664,6 +674,8 @@ async fn spawn_app(database_url: String) -> String {
     let instance = spawn_server(ServerConfig {
         pool: Some(pool),
         cache: None,
+        store_path: None,
+        shared_state: None,
     })
     .await;
     APP_PORT.set(instance.port).ok();
@@ -671,27 +683,9 @@ async fn spawn_app(database_url: String) -> String {
 }
 
 #[allow(dead_code)]
-pub async fn spawn_cluster(database_url: String, node_count: usize) -> Vec<ServerInstance> {
+pub async fn spawn_cluster(pool: Option<sqlx::PgPool>, node_count: usize) -> Vec<ServerInstance> {
     use tranquil_ripple::{RippleConfig, RippleEngine};
 
-    let pool = PgPoolOptions::new()
-        .max_connections(10)
-        .acquire_timeout(std::time::Duration::from_secs(30))
-        .connect(&database_url)
-        .await
-        .expect("Failed to connect to Postgres for cluster");
-    sqlx::migrate!("./migrations")
-        .run(&pool)
-        .await
-        .expect("Failed to run migrations for cluster");
-    let test_pool = PgPoolOptions::new()
-        .max_connections(2)
-        .acquire_timeout(std::time::Duration::from_secs(30))
-        .connect(&database_url)
-        .await
-        .expect("Failed to create test pool for cluster");
-    TEST_DB_POOL.set(test_pool).ok();
-
     let shutdown = CancellationToken::new();
 
     let mut ripple_nodes: Vec<(Arc<dyn Cache>, Arc<dyn DistributedRateLimiter>)> =
@@ -713,11 +707,28 @@ pub async fn spawn_cluster(database_url: String, node_count: usize) -> Vec<Serve
         ripple_nodes.push((cache, rate_limiter));
     }
 
+    unsafe {
+        std::env::set_var("PDS_HOSTNAME", "pds.test");
+    }
+    tranquil_config::ensure_test_defaults();
+    let base_state = match is_store_backend() {
+        true => {
+            let path = std::env::temp_dir().join(format!(
+                "tranquil-pds-cluster-store-{}",
+                uuid::Uuid::new_v4()
+            ));
+            std::fs::create_dir_all(&path).expect("failed to create cluster store dir");
+            Some(AppState::from_store_at(&path, CancellationToken::new()).await)
+        }
+        false => None,
+    };
     let mut instances: Vec<ServerInstance> = Vec::with_capacity(node_count);
     for (cache, rate_limiter) in ripple_nodes {
         let server_config = ServerConfig {
-            pool: Some(pool.clone()),
+            pool: pool.clone(),
             cache: Some((cache, rate_limiter)),
+            store_path: None,
+            shared_state: base_state.clone(),
         };
         let instance = spawn_server(server_config).await;
         instances.push(instance);
@@ -757,12 +768,14 @@ pub async fn cluster() -> &'static [ServerInstance] {
                 unsafe {
                     std::env::remove_var("DISABLE_RATE_LIMITING");
                 }
-                let database_url = if has_external_infra() {
+                let pool = if is_store_backend() {
+                    setup_cluster_store_backend().await
+                } else if has_external_infra() {
                     setup_cluster_external_infra().await
                 } else {
                     setup_cluster_testcontainers().await
                 };
-                let nodes = spawn_cluster(database_url, 3).await;
+                let nodes = spawn_cluster(pool, 3).await;
                 tx.send(nodes).unwrap();
                 std::future::pending::<()>().await;
             });
@@ -771,7 +784,36 @@ pub async fn cluster() -> &'static [ServerInstance] {
     })
 }
 
-async fn setup_cluster_external_infra() -> String {
+async fn setup_cluster_store_backend() -> Option<sqlx::PgPool> {
+    let temp_dir = std::env::temp_dir().join(format!(
+        "tranquil-pds-cluster-store-{}",
+        uuid::Uuid::new_v4()
+    ));
+    let blob_path = temp_dir.join("blobs");
+    let backup_path = temp_dir.join("backups");
+    let store_path = temp_dir.join("store");
+    std::fs::create_dir_all(&blob_path).expect("failed to create blob temp directory");
+    std::fs::create_dir_all(&backup_path).expect("failed to create backup temp directory");
+    std::fs::create_dir_all(&store_path).expect("failed to create store temp directory");
+    TEST_TEMP_DIR.set(temp_dir).ok();
+    let plc_url = setup_mock_plc_directory().await;
+    unsafe {
+        std::env::set_var("BLOB_STORAGE_BACKEND", "filesystem");
+        std::env::set_var("BLOB_STORAGE_PATH", blob_path.to_str().unwrap());
+        std::env::set_var("BACKUP_STORAGE_BACKEND", "filesystem");
+        std::env::set_var("BACKUP_STORAGE_PATH", backup_path.to_str().unwrap());
+        std::env::set_var("MAX_IMPORT_SIZE", "100000000");
+        std::env::set_var("SKIP_IMPORT_VERIFICATION", "true");
+        std::env::set_var("PLC_DIRECTORY_URL", &plc_url);
+        std::env::set_var("REPO_BACKEND", "tranquil-store");
+        std::env::set_var("TRANQUIL_STORE_DATA_DIR", store_path.to_str().unwrap());
+        std::env::set_var("DATABASE_URL", "postgres://unused/unused");
+    }
+    register_mock_appview().await;
+    None
+}
+
+async fn setup_cluster_external_infra() -> Option<sqlx::PgPool> {
     let database_url =
         std::env::var("DATABASE_URL").expect("DATABASE_URL must be set when using external infra");
     let plc_url = setup_mock_plc_directory().await;
@@ -780,11 +822,28 @@ async fn setup_cluster_external_infra() -> String {
         std::env::set_var("PLC_DIRECTORY_URL", &plc_url);
     }
     register_mock_appview().await;
-    database_url
+    let pool = PgPoolOptions::new()
+        .max_connections(10)
+        .acquire_timeout(std::time::Duration::from_secs(30))
+        .connect(&database_url)
+        .await
+        .expect("Failed to connect to Postgres for cluster");
+    sqlx::migrate!("./migrations")
+        .run(&pool)
+        .await
+        .expect("Failed to run migrations for cluster");
+    let test_pool = PgPoolOptions::new()
+        .max_connections(2)
+        .acquire_timeout(std::time::Duration::from_secs(30))
+        .connect(&database_url)
+        .await
+        .expect("Failed to create test pool for cluster");
+    TEST_DB_POOL.set(test_pool).ok();
+    Some(pool)
 }
 
 #[cfg(not(feature = "external-infra"))]
-async fn setup_cluster_testcontainers() -> String {
+async fn setup_cluster_testcontainers() -> Option<sqlx::PgPool> {
     let temp_dir =
         std::env::temp_dir().join(format!("tranquil-pds-cluster-{}", uuid::Uuid::new_v4()));
     let blob_path = temp_dir.join("blobs");
@@ -817,11 +876,28 @@ async fn setup_cluster_testcontainers() -> String {
             .expect("Failed to get port")
     );
     DB_CONTAINER.set(container).ok();
-    connection_string
+    let pool = PgPoolOptions::new()
+        .max_connections(10)
+        .acquire_timeout(std::time::Duration::from_secs(30))
+        .connect(&connection_string)
+        .await
+        .expect("Failed to connect to Postgres for cluster");
+    sqlx::migrate!("./migrations")
+        .run(&pool)
+        .await
+        .expect("Failed to run migrations for cluster");
+    let test_pool = PgPoolOptions::new()
+        .max_connections(2)
+        .acquire_timeout(std::time::Duration::from_secs(30))
+        .connect(&connection_string)
+        .await
+        .expect("Failed to create test pool for cluster");
+    TEST_DB_POOL.set(test_pool).ok();
+    Some(pool)
 }
 
 #[cfg(feature = "external-infra")]
-async fn setup_cluster_testcontainers() -> String {
+async fn setup_cluster_testcontainers() -> Option<sqlx::PgPool> {
     panic!(
         "Testcontainers disabled with external-infra feature. Set DATABASE_URL and BLOB_STORAGE_PATH (or S3_ENDPOINT)."
     );
@@ -861,6 +937,14 @@ pub async fn get_test_repos() -> &'static Arc<tranquil_db::PostgresRepositories>
     TEST_REPOS.get().expect("TEST_REPOS not initialized")
 }
 
+#[allow(dead_code)]
+pub async fn get_test_block_store() -> &'static tranquil_pds::repo::AnyBlockStore {
+    base_url().await;
+    TEST_BLOCK_STORE
+        .get()
+        .expect("TEST_BLOCK_STORE not initialized")
+}
+
 fn extract_verification_code(body_text: &str) -> String {
     let lines: Vec<&str> = body_text.lines().collect();
     lines

crates/tranquil-pds/tests/firehose/mod.rs

@@ -32,6 +32,7 @@ pub struct ParsedCommitFrame {
     pub prev_data: Option<Cid>,
 }
 
+#[allow(dead_code)]
 #[derive(Debug, Clone)]
 pub struct ParsedRepoOp {
     pub action: RepoAction,
@@ -223,6 +224,7 @@ impl FirehoseConsumer {
         self.frames.lock().unwrap().drain(..).collect()
     }
 
+    #[allow(dead_code)]
     pub fn all_commits(&self) -> Vec<ParsedCommitFrame> {
         self.frames
             .lock()

crates/tranquil-pds/tests/firehose_inline_blocks.rs

@@ -0,0 +1,423 @@
+mod common;
+mod firehose;
+mod helpers;
+
+use cid::Cid;
+use common::*;
+use firehose::FirehoseConsumer;
+use helpers::build_car_with_signature;
+use iroh_car::CarReader;
+use k256::ecdsa::SigningKey;
+use multihash::Multihash;
+use reqwest::StatusCode;
+use serde_json::json;
+use sha2::{Digest, Sha256};
+use std::io::Cursor;
+use std::time::Duration;
+use tranquil_db_traits::{EventBlocks, RepoEventType, SequenceNumber};
+use tranquil_types::{CidLink, Did};
+
+fn synthetic_cid(payload: &[u8]) -> Cid {
+    let digest = Sha256::digest(payload);
+    let mh = Multihash::wrap(0x12, digest.as_slice()).expect("multihash wrap");
+    Cid::new_v1(0x71, mh)
+}
+
+fn fresh_synthetic_did(label: &str) -> Did {
+    Did::new(format!(
+        "did:plc:test{}{}",
+        label,
+        uuid::Uuid::new_v4().simple()
+    ))
+    .expect("valid did")
+}
+
+async fn create_post(client: &reqwest::Client, token: &str, did: &str, text: &str) {
+    let payload = json!({
+        "repo": did,
+        "collection": "app.bsky.feed.post",
+        "record": {
+            "$type": "app.bsky.feed.post",
+            "text": text,
+            "createdAt": chrono::Utc::now().to_rfc3339(),
+        }
+    });
+    let res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.createRecord",
+            base_url().await
+        ))
+        .bearer_auth(token)
+        .json(&payload)
+        .send()
+        .await
+        .expect("createRecord request failed");
+    assert_eq!(res.status(), StatusCode::OK, "createRecord failed");
+}
+
+#[tokio::test]
+async fn commit_events_carry_inline_blocks() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    create_post(&client, &token, &did, "commit A: orphans incoming").await;
+    create_post(&client, &token, &did, "commit B: bye bye MST nodes from A").await;
+
+    let repos = get_test_repos().await;
+    let typed_did = tranquil_types::Did::new(did.clone()).unwrap();
+
+    let events = repos
+        .repo
+        .get_events_since_seq(SequenceNumber::ZERO, None)
+        .await
+        .expect("get_events_since_seq failed");
+
+    let our_commits: Vec<_> = events
+        .iter()
+        .filter(|e| e.did == typed_did && e.event_type == RepoEventType::Commit)
+        .collect();
+
+    assert!(
+        our_commits.len() >= 2,
+        "expected at least 2 commit events for our DID, got {}",
+        our_commits.len()
+    );
+
+    our_commits.iter().for_each(|event| {
+        let blocks = event.blocks.as_ref().unwrap_or_else(|| {
+            panic!(
+                "commit event seq={} has no blocks field",
+                event.seq.as_i64()
+            )
+        });
+        let inline = match blocks {
+            EventBlocks::Inline(v) => v,
+            EventBlocks::LegacyCids(_) => panic!(
+                "commit event seq={} resolved as LegacyCids, expected Inline; \
+                 new commits must inline block bytes into the eventlog",
+                event.seq.as_i64()
+            ),
+        };
+        assert!(
+            !inline.is_empty(),
+            "commit event seq={} has empty Inline blocks vec",
+            event.seq.as_i64()
+        );
+        let commit_cid = event
+            .commit_cid
+            .as_ref()
+            .and_then(|c| c.to_cid())
+            .unwrap_or_else(|| {
+                panic!("commit event seq={} missing commit_cid", event.seq.as_i64())
+            });
+        let commit_cid_bytes = commit_cid.to_bytes();
+        assert!(
+            inline.iter().any(|b| b.cid_bytes == commit_cid_bytes),
+            "commit event seq={} inline blocks do not contain the commit block",
+            event.seq.as_i64()
+        );
+        inline.iter().for_each(|b| {
+            let parsed = Cid::read_bytes(b.cid_bytes.as_slice()).unwrap_or_else(|e| {
+                panic!(
+                    "commit event seq={} inline cid_bytes failed to parse as Cid: {e}",
+                    event.seq.as_i64()
+                )
+            });
+            assert_eq!(
+                parsed.to_bytes(),
+                b.cid_bytes,
+                "commit event seq={} cid round-trip mismatch (cid={parsed})",
+                event.seq.as_i64()
+            );
+        });
+    });
+}
+
+#[tokio::test]
+async fn sync_event_carries_inline_commit_block() {
+    let repos = get_test_repos().await;
+    let did = fresh_synthetic_did("sync");
+    let commit_bytes = b"synthetic sync commit block payload".to_vec();
+    let commit_cid = synthetic_cid(&commit_bytes);
+    let cid_link: CidLink = (&commit_cid).into();
+    let rev = "3kabcdefghij2";
+
+    let seq = repos
+        .repo
+        .insert_sync_event(&did, &cid_link, Some(rev), &commit_bytes)
+        .await
+        .expect("insert_sync_event");
+
+    let event = repos
+        .repo
+        .get_event_by_seq(seq)
+        .await
+        .expect("get_event_by_seq")
+        .expect("event present");
+
+    assert_eq!(event.event_type, RepoEventType::Sync);
+    let blocks = event
+        .blocks
+        .as_ref()
+        .expect("sync event must carry inline blocks");
+    let inline = match blocks {
+        EventBlocks::Inline(v) => v,
+        EventBlocks::LegacyCids(_) => {
+            panic!("sync event resolved as LegacyCids; new sync events must inline block bytes")
+        }
+    };
+    assert_eq!(
+        inline.len(),
+        1,
+        "sync event must carry exactly the commit block, got {}",
+        inline.len()
+    );
+    let stored = &inline[0];
+    assert_eq!(
+        stored.cid_bytes,
+        commit_cid.to_bytes(),
+        "sync event inline cid_bytes mismatch"
+    );
+    assert_eq!(stored.data, commit_bytes, "sync event inline data mismatch");
+}
+
+#[tokio::test]
+async fn genesis_commit_event_carries_inline_blocks() {
+    let repos = get_test_repos().await;
+    let did = fresh_synthetic_did("gen");
+    let commit_bytes = b"synthetic genesis commit block payload".to_vec();
+    let mst_root_bytes = b"synthetic genesis mst root block payload".to_vec();
+    let commit_cid = synthetic_cid(&commit_bytes);
+    let mst_root_cid = synthetic_cid(&mst_root_bytes);
+    let commit_link: CidLink = (&commit_cid).into();
+    let mst_link: CidLink = (&mst_root_cid).into();
+    let rev = "3kabcdefghij3";
+
+    let seq = repos
+        .repo
+        .insert_genesis_commit_event(
+            &did,
+            &commit_link,
+            &mst_link,
+            rev,
+            &commit_bytes,
+            &mst_root_bytes,
+        )
+        .await
+        .expect("insert_genesis_commit_event");
+
+    let event = repos
+        .repo
+        .get_event_by_seq(seq)
+        .await
+        .expect("get_event_by_seq")
+        .expect("event present");
+
+    assert_eq!(event.event_type, RepoEventType::Commit);
+    let blocks = event
+        .blocks
+        .as_ref()
+        .expect("genesis commit event must carry inline blocks");
+    let inline = match blocks {
+        EventBlocks::Inline(v) => v,
+        EventBlocks::LegacyCids(_) => {
+            panic!("genesis event resolved as LegacyCids; new genesis events must inline blocks")
+        }
+    };
+    assert_eq!(
+        inline.len(),
+        2,
+        "genesis event must carry commit + mst root blocks, got {}",
+        inline.len()
+    );
+
+    let commit_cid_bytes = commit_cid.to_bytes();
+    let mst_cid_bytes = mst_root_cid.to_bytes();
+
+    let commit_block = inline
+        .iter()
+        .find(|b| b.cid_bytes == commit_cid_bytes)
+        .expect("genesis inline blocks missing commit block");
+    assert_eq!(commit_block.data, commit_bytes);
+
+    let mst_block = inline
+        .iter()
+        .find(|b| b.cid_bytes == mst_cid_bytes)
+        .expect("genesis inline blocks missing mst root block");
+    assert_eq!(mst_block.data, mst_root_bytes);
+}
+
+#[tokio::test]
+async fn backfill_succeeds_from_eventlog_alone() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    create_post(&client, &token, &did, "first").await;
+    create_post(&client, &token, &did, "second").await;
+
+    let consumer = FirehoseConsumer::connect_with_cursor(app_port(), 0).await;
+    let commits = consumer
+        .wait_for_commits(&did, 2, Duration::from_secs(20))
+        .await;
+
+    assert!(
+        commits.len() >= 2,
+        "expected at least 2 backfilled commits for {}, got {}",
+        did,
+        commits.len()
+    );
+
+    for commit in &commits {
+        assert!(
+            !commit.blocks.is_empty(),
+            "backfilled commit seq={} has empty CAR blocks",
+            commit.seq
+        );
+        let mut reader = CarReader::new(Cursor::new(&commit.blocks))
+            .await
+            .unwrap_or_else(|e| panic!("CAR header parse failed for seq={}: {e}", commit.seq));
+        assert!(
+            !reader.header().roots().is_empty(),
+            "CAR for seq={} has no roots",
+            commit.seq
+        );
+        assert_eq!(
+            reader.header().roots()[0],
+            commit.commit,
+            "CAR root mismatch for seq={}",
+            commit.seq
+        );
+        let mut found_commit_block = false;
+        while let Ok(Some((cid, _))) = reader.next_block().await {
+            if cid == commit.commit {
+                found_commit_block = true;
+            }
+        }
+        assert!(
+            found_commit_block,
+            "backfilled commit seq={} CAR missing the commit block",
+            commit.seq
+        );
+    }
+}
+
+#[tokio::test]
+async fn import_event_carries_inline_commit_block() {
+    let client = client();
+    let (token, did) = create_account_and_login(&client).await;
+
+    let signing_key = SigningKey::random(&mut rand::thread_rng());
+    let (car_bytes, _car_root_cid) = build_car_with_signature(&did, &signing_key);
+
+    let import_res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.repo.importRepo",
+            base_url().await
+        ))
+        .bearer_auth(&token)
+        .header("Content-Type", "application/vnd.ipld.car")
+        .body(car_bytes)
+        .send()
+        .await
+        .expect("import request failed");
+    assert_eq!(
+        import_res.status(),
+        StatusCode::OK,
+        "import should succeed: body={:?}",
+        import_res.text().await.unwrap_or_default()
+    );
+
+    let repos = get_test_repos().await;
+    let typed_did = tranquil_types::Did::new(did.clone()).unwrap();
+    let events = repos
+        .repo
+        .get_events_since_seq(SequenceNumber::ZERO, None)
+        .await
+        .expect("get_events_since_seq failed");
+
+    let our_commits: Vec<_> = events
+        .iter()
+        .filter(|e| e.did == typed_did && e.event_type == RepoEventType::Commit)
+        .collect();
+    assert!(
+        !our_commits.is_empty(),
+        "expected at least one commit event for {} after import",
+        did
+    );
+
+    let import_event = our_commits
+        .last()
+        .expect("at least one commit event after import");
+    let blocks = import_event.blocks.as_ref().unwrap_or_else(|| {
+        panic!(
+            "import commit event seq={} missing blocks field",
+            import_event.seq.as_i64()
+        )
+    });
+    let inline = match blocks {
+        EventBlocks::Inline(v) => v,
+        EventBlocks::LegacyCids(_) => panic!(
+            "import event seq={} resolved as LegacyCids; new commits must inline blocks",
+            import_event.seq.as_i64()
+        ),
+    };
+    assert!(
+        !inline.is_empty(),
+        "import event seq={} has empty Inline blocks vec — this is the bug from \
+         sequence_import_event using `blocks: Some(vec![])`",
+        import_event.seq.as_i64()
+    );
+    let commit_cid = import_event
+        .commit_cid
+        .as_ref()
+        .and_then(|c| c.to_cid())
+        .unwrap_or_else(|| {
+            panic!(
+                "import event seq={} missing commit_cid",
+                import_event.seq.as_i64()
+            )
+        });
+    let commit_cid_bytes = commit_cid.to_bytes();
+    assert!(
+        inline.iter().any(|b| b.cid_bytes == commit_cid_bytes),
+        "import event seq={} inline blocks do not contain the freshly-created commit block",
+        import_event.seq.as_i64()
+    );
+
+    let consumer = FirehoseConsumer::connect_with_cursor(app_port(), 0).await;
+    let commits = consumer
+        .wait_for_commits(&did, 1, Duration::from_secs(20))
+        .await;
+    assert!(
+        !commits.is_empty(),
+        "expected at least one backfilled commit after import for {}",
+        did
+    );
+    for commit in &commits {
+        assert!(
+            !commit.blocks.is_empty(),
+            "backfilled import commit seq={} has empty CAR blocks",
+            commit.seq
+        );
+        let mut reader = CarReader::new(Cursor::new(&commit.blocks))
+            .await
+            .unwrap_or_else(|e| panic!("CAR header parse failed for seq={}: {e}", commit.seq));
+        assert_eq!(
+            reader.header().roots()[0],
+            commit.commit,
+            "CAR root mismatch for import commit seq={}",
+            commit.seq
+        );
+        let mut found_commit_block = false;
+        while let Ok(Some((cid, _))) = reader.next_block().await {
+            if cid == commit.commit {
+                found_commit_block = true;
+            }
+        }
+        assert!(
+            found_commit_block,
+            "backfilled import commit seq={} CAR missing the commit block",
+            commit.seq
+        );
+    }
+}

crates/tranquil-pds/tests/firehose_validation.rs

@@ -245,11 +245,11 @@ async fn test_firehose_frame_structure() {
     assert!(timeout.is_ok(), "Timed out waiting for event for our DID");
     let (header, frame) = frame_opt.expect("No matching frame found");
 
-    println!("\n=== Frame Structure Validation ===\n");
+    println!("\n-- frame structure validation --\n");
 
     println!("Header:");
-    println!("  op: {} (expected: 1)", header.op);
-    println!("  t: {} (expected: #commit)", header.t);
+    println!("  op: {}, expected 1", header.op);
+    println!("  t: {}, expected #commit", header.t);
     assert_eq!(header.op, 1, "Header op should be 1");
     assert_eq!(header.t, "#commit", "Header t should be #commit");
 
@@ -260,7 +260,7 @@ async fn test_firehose_frame_structure() {
     println!("  repo: {}", frame.repo);
     println!("  commit: {}", frame.commit);
     println!(
-        "  rev: {} (valid TID: {})",
+        "  rev: {}, valid TID: {}",
         frame.rev,
         is_valid_tid(&frame.rev)
     );
@@ -269,12 +269,12 @@ async fn test_firehose_frame_structure() {
     println!("  ops count: {}", frame.ops.len());
     println!("  blobs count: {}", frame.blobs.len());
     println!(
-        "  time: {} (valid format: {})",
+        "  time: {}, valid format: {}",
         frame.time,
         is_valid_time_format(&frame.time)
     );
     println!(
-        "  prevData: {:?} (IMPORTANT - should have value for updates)",
+        "  prevData: {:?}, should have value for updates",
         frame.prev_data
     );
 
@@ -297,7 +297,7 @@ async fn test_firehose_frame_structure() {
         println!("    path: {}", op.path);
         println!("    cid: {:?}", op.cid);
         println!(
-            "    prev: {:?} (should be Some for updates/deletes)",
+            "    prev: {:?}, should be Some for updates/deletes",
             op.prev
         );
 
@@ -351,7 +351,7 @@ async fn test_firehose_frame_structure() {
         }
     }
 
-    println!("\n=== Validation Complete ===\n");
+    println!("\n-- validation complete --\n");
 
     ws_stream.send(tungstenite::Message::Close(None)).await.ok();
 }
@@ -435,7 +435,7 @@ async fn test_firehose_update_has_prev_field() {
     assert!(timeout.is_ok(), "Timed out waiting for update commit");
     let frame = frame_opt.expect("No matching frame found");
 
-    println!("\n=== Update Operation Validation ===\n");
+    println!("\n-- update operation validation --\n");
     println!("First profile CID: {}", first_cid);
     println!("Frame prevData: {:?}", frame.prev_data);
 
@@ -455,7 +455,7 @@ async fn test_firehose_update_has_prev_field() {
         }
     }
 
-    println!("\n=== Validation Complete ===\n");
+    println!("\n-- validation complete --\n");
 
     ws_stream.send(tungstenite::Message::Close(None)).await.ok();
 }
@@ -512,13 +512,13 @@ async fn test_firehose_commit_has_prev_data() {
     assert!(timeout.is_ok(), "Timed out waiting for first commit");
     let first_frame = first_frame_opt.expect("No first frame found");
 
-    println!("\n=== First Commit ===");
+    println!("\n-- first commit --");
     println!(
-        "  prevData: {:?} (first commit may be None)",
+        "  prevData: {:?}, first commit may be None",
         first_frame.prev_data
     );
     println!(
-        "  since: {:?} (first commit should be None)",
+        "  since: {:?}, first commit should be None",
         first_frame.since
     );
 
@@ -562,13 +562,13 @@ async fn test_firehose_commit_has_prev_data() {
     assert!(timeout.is_ok(), "Timed out waiting for second commit");
     let second_frame = second_frame_opt.expect("No second frame found");
 
-    println!("\n=== Second Commit ===");
+    println!("\n-- second commit --");
     println!(
-        "  prevData: {:?} (should have value - MST root CID)",
+        "  prevData: {:?}, should have value as MST root CID",
         second_frame.prev_data
     );
     println!(
-        "  since: {:?} (should have value - previous rev)",
+        "  since: {:?}, should have value as previous rev",
         second_frame.since
     );
 
@@ -577,7 +577,7 @@ async fn test_firehose_commit_has_prev_data() {
         "Second commit should have 'since' field pointing to first commit rev"
     );
 
-    println!("\n=== Validation Complete ===\n");
+    println!("\n-- validation complete --\n");
 
     ws_stream.send(tungstenite::Message::Close(None)).await.ok();
 }
@@ -634,7 +634,7 @@ async fn test_compare_raw_cbor_encoding() {
     assert!(timeout.is_ok(), "Timed out waiting for event for our DID");
     let raw_bytes = raw_bytes_opt.expect("No matching frame found");
 
-    println!("\n=== Raw CBOR Analysis ===\n");
+    println!("\n-- raw CBOR analysis --\n");
     println!("Total frame size: {} bytes", raw_bytes.len());
 
     fn bytes_to_hex(bytes: &[u8]) -> String {
@@ -657,7 +657,7 @@ async fn test_compare_raw_cbor_encoding() {
 
     println!("\nPayload section: {} bytes", raw_bytes.len() - header_end);
 
-    println!("\n=== Analysis Complete ===\n");
+    println!("\n-- analysis complete --\n");
 
     ws_stream.send(tungstenite::Message::Close(None)).await.ok();
 }

crates/tranquil-pds/tests/gc_after_delete.rs

@@ -0,0 +1,410 @@
+mod common;
+mod helpers;
+use chrono::Utc;
+use common::*;
+use helpers::*;
+use reqwest::StatusCode;
+use serde_json::{Value, json};
+use tranquil_types::Did;
+
+#[tokio::test]
+async fn test_delete_record_marks_blocks_obsolete() {
+    let client = client();
+    let base = base_url().await;
+    let repos = get_test_repos().await;
+    let (did, jwt) = setup_new_user("gc-after-delete").await;
+
+    let user_id = repos
+        .user
+        .get_id_by_did(&Did::new(did.clone()).unwrap())
+        .await
+        .expect("DB error")
+        .expect("User not found");
+
+    let count_baseline = repos
+        .repo
+        .count_user_blocks(user_id)
+        .await
+        .expect("count_user_blocks failed");
+
+    let collection = "app.bsky.feed.post";
+    let rkey = format!("gc_test_{}", Utc::now().timestamp_millis());
+    let create_payload = json!({
+        "repo": did,
+        "collection": collection,
+        "rkey": rkey,
+        "record": {
+            "$type": collection,
+            "text": "this record is destined for deletion",
+            "createdAt": Utc::now().to_rfc3339()
+        }
+    });
+
+    let create_res = client
+        .post(format!("{}/xrpc/com.atproto.repo.createRecord", base))
+        .bearer_auth(&jwt)
+        .json(&create_payload)
+        .send()
+        .await
+        .expect("Failed to send createRecord");
+    assert_eq!(
+        create_res.status(),
+        StatusCode::OK,
+        "createRecord did not return 200"
+    );
+    let create_body: Value = create_res
+        .json()
+        .await
+        .expect("createRecord response was not JSON");
+    let record_uri = create_body["uri"]
+        .as_str()
+        .expect("createRecord response missing uri")
+        .to_string();
+    let record_cid = create_body["cid"]
+        .as_str()
+        .expect("createRecord response missing cid")
+        .to_string();
+
+    let count_after_create = repos
+        .repo
+        .count_user_blocks(user_id)
+        .await
+        .expect("count_user_blocks failed");
+    assert!(
+        count_after_create > count_baseline,
+        "user_blocks count did not grow after createRecord (baseline={}, after_create={})",
+        count_baseline,
+        count_after_create
+    );
+
+    let delete_payload = json!({
+        "repo": did,
+        "collection": collection,
+        "rkey": rkey,
+    });
+    let delete_res = client
+        .post(format!("{}/xrpc/com.atproto.repo.deleteRecord", base))
+        .bearer_auth(&jwt)
+        .json(&delete_payload)
+        .send()
+        .await
+        .expect("Failed to send deleteRecord");
+    assert_eq!(
+        delete_res.status(),
+        StatusCode::OK,
+        "deleteRecord did not return 200: {:?}",
+        delete_res.text().await
+    );
+
+    let count_after_delete = repos
+        .repo
+        .count_user_blocks(user_id)
+        .await
+        .expect("count_user_blocks failed");
+
+    assert!(
+        count_after_delete < count_after_create,
+        "user_blocks count did not shrink after deleteRecord \
+         (baseline={}, after_create={}, after_delete={}). \
+         The delete path produced no obsolete CIDs beyond the prior commit root, \
+         which is the regression this test guards against.",
+        count_baseline,
+        count_after_create,
+        count_after_delete
+    );
+
+    let get_res = client
+        .get(format!("{}/xrpc/com.atproto.repo.getRecord", base))
+        .query(&[
+            ("repo", did.as_str()),
+            ("collection", collection),
+            ("rkey", rkey.as_str()),
+        ])
+        .send()
+        .await
+        .expect("Failed to send getRecord");
+    assert!(
+        !get_res.status().is_success(),
+        "deleted record is still resolvable via getRecord (status={}); uri={} cid={}",
+        get_res.status(),
+        record_uri,
+        record_cid
+    );
+}
+
+#[tokio::test]
+async fn test_update_record_marks_old_record_block_obsolete() {
+    let client = client();
+    let base = base_url().await;
+    let repos = get_test_repos().await;
+    let (did, jwt) = setup_new_user("gc-after-update").await;
+
+    let user_id = repos
+        .user
+        .get_id_by_did(&Did::new(did.clone()).unwrap())
+        .await
+        .expect("DB error")
+        .expect("User not found");
+
+    let collection = "app.bsky.feed.post";
+    let rkey = format!("gc_update_{}", Utc::now().timestamp_millis());
+
+    let put_v1 = json!({
+        "repo": did,
+        "collection": collection,
+        "rkey": rkey,
+        "record": {
+            "$type": collection,
+            "text": "first version",
+            "createdAt": Utc::now().to_rfc3339()
+        }
+    });
+    let res = client
+        .post(format!("{}/xrpc/com.atproto.repo.putRecord", base))
+        .bearer_auth(&jwt)
+        .json(&put_v1)
+        .send()
+        .await
+        .expect("Failed to send putRecord v1");
+    assert_eq!(res.status(), StatusCode::OK, "first putRecord failed");
+
+    let count_after_create = repos
+        .repo
+        .count_user_blocks(user_id)
+        .await
+        .expect("count_user_blocks failed");
+
+    let put_v2 = json!({
+        "repo": did,
+        "collection": collection,
+        "rkey": rkey,
+        "record": {
+            "$type": collection,
+            "text": "second version with new content",
+            "createdAt": Utc::now().to_rfc3339()
+        }
+    });
+    let res = client
+        .post(format!("{}/xrpc/com.atproto.repo.putRecord", base))
+        .bearer_auth(&jwt)
+        .json(&put_v2)
+        .send()
+        .await
+        .expect("Failed to send putRecord v2");
+    assert_eq!(res.status(), StatusCode::OK, "second putRecord failed");
+
+    let count_after_update = repos
+        .repo
+        .count_user_blocks(user_id)
+        .await
+        .expect("count_user_blocks failed");
+
+    assert!(
+        count_after_update <= count_after_create + 1,
+        "user_blocks count grew by more than 1 after putRecord update \
+         (after_create={}, after_update={}). The previous version's record block \
+         should have been marked obsolete; instead it appears to be leaking.",
+        count_after_create,
+        count_after_update
+    );
+}
+
+#[tokio::test]
+async fn test_delete_in_populated_repo_marks_merged_subtree_blocks_obsolete() {
+    let client = client();
+    let base = base_url().await;
+    let repos = get_test_repos().await;
+    let (did, jwt) = setup_new_user("gc-merge").await;
+
+    let user_id = repos
+        .user
+        .get_id_by_did(&Did::new(did.clone()).unwrap())
+        .await
+        .expect("DB error")
+        .expect("User not found");
+
+    let collection = "app.bsky.feed.post";
+    let record_count = 64usize;
+    let now_ms = Utc::now().timestamp_millis();
+
+    let rkeys: Vec<String> = (0..record_count)
+        .map(|i| format!("gc_merge_{}_{:04}", now_ms, i))
+        .collect();
+
+    let create_results =
+        futures::future::try_join_all(rkeys.iter().enumerate().map(|(i, rkey)| {
+            let client = client.clone();
+            let jwt = jwt.clone();
+            let did = did.clone();
+            let base = base.to_string();
+            let payload = json!({
+                "repo": did,
+                "collection": collection,
+                "rkey": rkey,
+                "record": {
+                    "$type": collection,
+                    "text": format!("seed record {}", i),
+                    "createdAt": Utc::now().to_rfc3339()
+                }
+            });
+            async move {
+                let res = client
+                    .post(format!("{}/xrpc/com.atproto.repo.createRecord", base))
+                    .bearer_auth(&jwt)
+                    .json(&payload)
+                    .send()
+                    .await
+                    .expect("Failed to send createRecord");
+                if res.status() != StatusCode::OK {
+                    return Err(format!("seed createRecord failed: {}", res.status()));
+                }
+                Ok::<(), String>(())
+            }
+        }))
+        .await;
+    create_results.expect("seeding records failed");
+
+    let count_after_seed = repos
+        .repo
+        .count_user_blocks(user_id)
+        .await
+        .expect("count_user_blocks failed");
+
+    let target_rkey = &rkeys[record_count / 2];
+    let delete_payload = json!({
+        "repo": did,
+        "collection": collection,
+        "rkey": target_rkey,
+    });
+    let delete_res = client
+        .post(format!("{}/xrpc/com.atproto.repo.deleteRecord", base))
+        .bearer_auth(&jwt)
+        .json(&delete_payload)
+        .send()
+        .await
+        .expect("Failed to send deleteRecord");
+    assert_eq!(
+        delete_res.status(),
+        StatusCode::OK,
+        "deleteRecord did not return 200: {:?}",
+        delete_res.text().await
+    );
+
+    let count_after_delete = repos
+        .repo
+        .count_user_blocks(user_id)
+        .await
+        .expect("count_user_blocks failed");
+    assert!(
+        count_after_delete < count_after_seed,
+        "user_blocks did not shrink after deleting from a populated repo \
+         (after_seed={}, after_delete={}). The path-walk-based obsolete \
+         calculation does not capture sibling subtree blocks orphaned by \
+         delete-merge; only an MST-diff-based calculation does.",
+        count_after_seed,
+        count_after_delete
+    );
+
+    let get_res = client
+        .get(format!("{}/xrpc/com.atproto.repo.getRecord", base))
+        .query(&[
+            ("repo", did.as_str()),
+            ("collection", collection),
+            ("rkey", target_rkey.as_str()),
+        ])
+        .send()
+        .await
+        .expect("Failed to send getRecord");
+    assert!(
+        !get_res.status().is_success(),
+        "deleted record is still resolvable via getRecord (status={})",
+        get_res.status(),
+    );
+}
+
+#[tokio::test]
+async fn test_delete_decrements_tranquil_store_refcounts() {
+    if !is_store_backend() {
+        eprintln!(
+            "skipping test_delete_decrements_tranquil_store_refcounts: \
+             only meaningful with the tranquil-store backend"
+        );
+        return;
+    }
+
+    let client = client();
+    let base = base_url().await;
+    let block_store = get_test_block_store().await;
+    let store = block_store
+        .as_tranquil_store()
+        .expect("tranquil-store backend selected but block_store is not TranquilStore");
+    let (did, jwt) = setup_new_user("gc-store-decrement").await;
+
+    let collection = "app.bsky.feed.post";
+    let rkey = format!("gc_store_{}", Utc::now().timestamp_millis());
+
+    let create_res = client
+        .post(format!("{}/xrpc/com.atproto.repo.createRecord", base))
+        .bearer_auth(&jwt)
+        .json(&json!({
+            "repo": did,
+            "collection": collection,
+            "rkey": rkey,
+            "record": {
+                "$type": collection,
+                "text": "destined for refcount decrement",
+                "createdAt": Utc::now().to_rfc3339()
+            }
+        }))
+        .send()
+        .await
+        .expect("Failed to send createRecord");
+    assert_eq!(create_res.status(), StatusCode::OK, "createRecord failed");
+    let create_body: Value = create_res.json().await.expect("createRecord not JSON");
+    let record_cid_str = create_body["cid"]
+        .as_str()
+        .expect("createRecord response missing cid")
+        .to_string();
+    let record_cid = cid::Cid::try_from(record_cid_str.as_str()).expect("invalid record cid");
+
+    let refcount_after_create = store
+        .refcount_of(&record_cid)
+        .expect("refcount_of failed")
+        .expect("record cid not in blockstore index after create");
+    assert!(
+        refcount_after_create > 0,
+        "record cid had refcount 0 immediately after create (cid={})",
+        record_cid_str
+    );
+
+    let delete_res = client
+        .post(format!("{}/xrpc/com.atproto.repo.deleteRecord", base))
+        .bearer_auth(&jwt)
+        .json(&json!({
+            "repo": did,
+            "collection": collection,
+            "rkey": rkey,
+        }))
+        .send()
+        .await
+        .expect("Failed to send deleteRecord");
+    assert_eq!(
+        delete_res.status(),
+        StatusCode::OK,
+        "deleteRecord did not return 200: {:?}",
+        delete_res.text().await
+    );
+
+    let refcount_after_delete = store
+        .refcount_of(&record_cid)
+        .expect("refcount_of failed")
+        .expect("record cid slot vanished entirely after delete");
+    assert_eq!(
+        refcount_after_delete, 0,
+        "record cid still has nonzero refcount after deleteRecord \
+         (cid={}, before_delete={}, after_delete={}). The hash_index \
+         decrement that drives on-disk reclamation is the regression \
+         this test guards against.",
+        record_cid_str, refcount_after_create, refcount_after_delete
+    );
+}

crates/tranquil-pds/tests/lifecycle_session.rs

@@ -597,3 +597,155 @@ async fn test_request_account_delete() {
         "Token should not be expired"
     );
 }
+
+async fn create_app_password_session(
+    client: &reqwest::Client,
+    did: &str,
+    main_jwt: &str,
+    name: &str,
+    body: Value,
+) -> (String, Value) {
+    let base = base_url().await;
+    let create_res = client
+        .post(format!(
+            "{}/xrpc/com.atproto.server.createAppPassword",
+            base
+        ))
+        .bearer_auth(main_jwt)
+        .json(&body)
+        .send()
+        .await
+        .expect("Failed to create app password");
+    assert_eq!(create_res.status(), StatusCode::OK);
+    let app_pass: Value = create_res.json().await.unwrap();
+    let password = app_pass["password"].as_str().unwrap().to_string();
+    let scopes_response = app_pass.clone();
+    let login_res = client
+        .post(format!("{}/xrpc/com.atproto.server.createSession", base))
+        .json(&json!({ "identifier": did, "password": password }))
+        .send()
+        .await
+        .expect("Failed to login with app password");
+    assert_eq!(login_res.status(), StatusCode::OK, "App password login for '{}' failed", name);
+    let session: Value = login_res.json().await.unwrap();
+    let jwt = session["accessJwt"].as_str().unwrap().to_string();
+    (jwt, scopes_response)
+}
+
+async fn try_chat_service_auth(client: &reqwest::Client, jwt: &str) -> StatusCode {
+    let base = base_url().await;
+    let res = client
+        .get(format!(
+            "{}/xrpc/com.atproto.server.getServiceAuth",
+            base
+        ))
+        .bearer_auth(jwt)
+        .query(&[
+            ("aud", "did:web:api.bsky.app"),
+            ("lxm", "chat.bsky.convo.listConvos"),
+        ])
+        .send()
+        .await
+        .expect("Failed to call getServiceAuth");
+    res.status()
+}
+
+#[tokio::test]
+async fn test_app_password_non_privileged_blocks_chat() {
+    let client = client();
+    let (did, jwt) = setup_new_user("appscope-nonchat").await;
+    let (app_jwt, create_body) = create_app_password_session(
+        &client,
+        &did,
+        &jwt,
+        "non-privileged",
+        json!({ "name": "NoChatApp", "privileged": false }),
+    )
+    .await;
+    assert_eq!(
+        create_body["scopes"].as_str().unwrap(),
+        "transition:generic",
+        "Non-privileged app password should not have chat scope"
+    );
+    let status = try_chat_service_auth(&client, &app_jwt).await;
+    assert_eq!(
+        status,
+        StatusCode::FORBIDDEN,
+        "Non-privileged app password must not access chat methods"
+    );
+}
+
+#[tokio::test]
+async fn test_app_password_privileged_allows_chat() {
+    let client = client();
+    let (did, jwt) = setup_new_user("appscope-chat").await;
+    let (app_jwt, create_body) = create_app_password_session(
+        &client,
+        &did,
+        &jwt,
+        "privileged",
+        json!({ "name": "ChatApp", "privileged": true }),
+    )
+    .await;
+    assert_eq!(
+        create_body["scopes"].as_str().unwrap(),
+        "transition:generic transition:chat.bsky",
+        "Privileged app password should have chat scope"
+    );
+    let status = try_chat_service_auth(&client, &app_jwt).await;
+    assert_eq!(
+        status,
+        StatusCode::OK,
+        "Privileged app password should access chat methods"
+    );
+}
+
+#[tokio::test]
+async fn test_app_password_no_privileged_field_allows_chat() {
+    let client = client();
+    let (did, jwt) = setup_new_user("appscope-full").await;
+    let (app_jwt, create_body) = create_app_password_session(
+        &client,
+        &did,
+        &jwt,
+        "full-access",
+        json!({ "name": "FullApp" }),
+    )
+    .await;
+    assert_eq!(
+        create_body["scopes"].as_str().unwrap(),
+        "transition:generic transition:chat.bsky",
+        "App password without privileged field should default to full access"
+    );
+    let status = try_chat_service_auth(&client, &app_jwt).await;
+    assert_eq!(
+        status,
+        StatusCode::OK,
+        "Full-access app password should access chat methods"
+    );
+}
+
+#[tokio::test]
+async fn test_app_password_explicit_scopes_respected() {
+    let client = client();
+    let (did, jwt) = setup_new_user("appscope-explicit").await;
+    let (app_jwt, create_body) = create_app_password_session(
+        &client,
+        &did,
+        &jwt,
+        "explicit-scopes",
+        json!({ "name": "ScopedApp", "scopes": "transition:generic" }),
+    )
+    .await;
+    assert_eq!(
+        create_body["scopes"].as_str().unwrap(),
+        "transition:generic",
+        "Explicit scopes should be stored as-is"
+    );
+    let status = try_chat_service_auth(&client, &app_jwt).await;
+    assert_eq!(
+        status,
+        StatusCode::FORBIDDEN,
+        "App password with only transition:generic should not access chat"
+    );
+}

crates/tranquil-pds/tests/mst_diff_equivalence.rs

@@ -0,0 +1,543 @@
+use std::collections::BTreeSet;
+use std::sync::Arc;
+
+use cid::Cid;
+use jacquard_repo::mst::Mst;
+use jacquard_repo::storage::MemoryBlockStore;
+
+fn test_cid(n: u32) -> Cid {
+    let data = n.to_be_bytes();
+    let mut buf = [0u8; 32];
+    buf[..4].copy_from_slice(&data);
+    buf[4] = (n >> 8) as u8 ^ 0xAB;
+    buf[5] = (n & 0xFF) as u8 ^ 0xCD;
+    let mh = multihash::Multihash::wrap(0x12, &buf).unwrap();
+    Cid::new_v1(0x71, mh)
+}
+
+async fn compute_obsolete_full_walk<S: jacquard_repo::storage::BlockStore + Sync + Send + 'static>(
+    old: &Mst<S>,
+    new: &Mst<S>,
+) -> BTreeSet<Cid> {
+    let old_nodes = old.collect_node_cids().await.unwrap();
+    let new_nodes = new.collect_node_cids().await.unwrap();
+    let old_leaves = old.leaves().await.unwrap();
+    let new_leaves = new.leaves().await.unwrap();
+    let old_nodes_set: BTreeSet<Cid> = old_nodes.into_iter().collect();
+    let new_nodes_set: BTreeSet<Cid> = new_nodes.into_iter().collect();
+    let old_leaf_set: BTreeSet<Cid> = old_leaves.iter().map(|(_, cid)| *cid).collect();
+    let new_leaf_set: BTreeSet<Cid> = new_leaves.iter().map(|(_, cid)| *cid).collect();
+    old_nodes_set
+        .difference(&new_nodes_set)
+        .copied()
+        .chain(old_leaf_set.difference(&new_leaf_set).copied())
+        .collect()
+}
+
+fn compute_obsolete_from_diff(
+    diff: &jacquard_repo::mst::diff::MstDiff,
+) -> BTreeSet<Cid> {
+    diff.removed_mst_blocks
+        .iter()
+        .copied()
+        .chain(diff.removed_cids.iter().copied())
+        .collect()
+}
+
+async fn assert_equivalence(
+    old_records: &[(String, u32)],
+    new_records: &[(String, u32)],
+    scenario: &str,
+) {
+    let storage = Arc::new(MemoryBlockStore::new());
+
+    let mut old_tree = Mst::new(storage.clone());
+    for (key, val) in old_records {
+        old_tree = old_tree.add(key, test_cid(*val)).await.unwrap();
+    }
+    let old_root = old_tree.persist().await.unwrap();
+
+    let mut new_tree = Mst::new(storage.clone());
+    for (key, val) in new_records {
+        new_tree = new_tree.add(key, test_cid(*val)).await.unwrap();
+    }
+    let new_root = new_tree.persist().await.unwrap();
+
+    let old_settled = Mst::load(storage.clone(), old_root, None);
+    let new_settled = Mst::load(storage.clone(), new_root, None);
+
+    let full_walk_obsolete = compute_obsolete_full_walk(&old_settled, &new_settled).await;
+
+    let old_for_diff = Mst::load(storage.clone(), old_root, None);
+    let new_for_diff = Mst::load(storage, new_root, None);
+    let diff = old_for_diff.diff(&new_for_diff).await.unwrap();
+    let diff_obsolete = compute_obsolete_from_diff(&diff);
+
+    assert_eq!(
+        full_walk_obsolete, diff_obsolete,
+        "MISMATCH in scenario: {scenario}\n  full_walk count: {}\n  diff count: {}\n  in full_walk but not diff: {:?}\n  in diff but not full_walk: {:?}",
+        full_walk_obsolete.len(),
+        diff_obsolete.len(),
+        full_walk_obsolete.difference(&diff_obsolete).collect::<Vec<_>>(),
+        diff_obsolete.difference(&full_walk_obsolete).collect::<Vec<_>>(),
+    );
+}
+
+fn make_key(collection: &str, i: u32) -> String {
+    format!("{collection}/{i:06}")
+}
+
+fn generate_records(collection: &str, range: std::ops::Range<u32>) -> Vec<(String, u32)> {
+    range.map(|i| (make_key(collection, i), i)).collect()
+}
+
+fn generate_multi_collection_records(
+    collections: &[&str],
+    per_collection: u32,
+) -> Vec<(String, u32)> {
+    collections
+        .iter()
+        .enumerate()
+        .flat_map(|(ci, coll)| {
+            let base = ci as u32 * per_collection;
+            (0..per_collection).map(move |i| (make_key(coll, i), base + i))
+        })
+        .collect()
+}
+
+fn apply_scattered_updates(
+    records: &[(String, u32)],
+    stride: usize,
+    cid_offset: u32,
+) -> Vec<(String, u32)> {
+    records
+        .iter()
+        .enumerate()
+        .map(|(idx, (key, val))| {
+            if idx % stride == 0 {
+                (key.clone(), val + cid_offset)
+            } else {
+                (key.clone(), *val)
+            }
+        })
+        .collect()
+}
+
+fn remove_every_nth(records: &[(String, u32)], n: usize) -> Vec<(String, u32)> {
+    records
+        .iter()
+        .enumerate()
+        .filter(|(idx, _)| idx % n != 0)
+        .map(|(_, r)| r.clone())
+        .collect()
+}
+
+fn remove_range(records: &[(String, u32)], start: usize, count: usize) -> Vec<(String, u32)> {
+    records
+        .iter()
+        .enumerate()
+        .filter(|(idx, _)| *idx < start || *idx >= start + count)
+        .map(|(_, r)| r.clone())
+        .collect()
+}
+
+fn keep_only_collection(records: &[(String, u32)], collection: &str) -> Vec<(String, u32)> {
+    records
+        .iter()
+        .filter(|(key, _)| key.starts_with(collection))
+        .cloned()
+        .collect()
+}
+
+fn append_records(
+    base: &[(String, u32)],
+    collection: &str,
+    range: std::ops::Range<u32>,
+    cid_base: u32,
+) -> Vec<(String, u32)> {
+    let mut result = base.to_vec();
+    result.extend(range.map(|i| (make_key(collection, i), cid_base + i)));
+    result.sort_by(|(a, _), (b, _)| a.cmp(b));
+    result
+}
+
+#[tokio::test]
+async fn massive_tree_single_create() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let new_rec = append_records(&old, "app.bsky.feed.post", 2000..2001, 2000);
+    assert_equivalence(&old, &new_rec, "2000 records + 1 create").await;
+}
+
+#[tokio::test]
+async fn massive_tree_single_delete() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let new_rec = remove_range(&old, 1000, 1);
+    assert_equivalence(&old, &new_rec, "2000 records - 1 delete from middle").await;
+}
+
+#[tokio::test]
+async fn massive_tree_single_update() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let new_rec: Vec<_> = old
+        .iter()
+        .map(|(k, v)| {
+            if k == "app.bsky.feed.post/001000" {
+                (k.clone(), v + 50000)
+            } else {
+                (k.clone(), *v)
+            }
+        })
+        .collect();
+    assert_equivalence(&old, &new_rec, "2000 records - 1 update in middle").await;
+}
+
+#[tokio::test]
+async fn massive_tree_scattered_updates_every_3rd() {
+    let old = generate_records("app.bsky.feed.post", 0..1500);
+    let new_rec = apply_scattered_updates(&old, 3, 10000);
+    assert_equivalence(&old, &new_rec, "1500 records - update every 3rd").await;
+}
+
+#[tokio::test]
+async fn massive_tree_scattered_updates_every_7th() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let new_rec = apply_scattered_updates(&old, 7, 20000);
+    assert_equivalence(&old, &new_rec, "2000 records - update every 7th").await;
+}
+
+#[tokio::test]
+async fn massive_tree_delete_every_2nd() {
+    let old = generate_records("app.bsky.feed.post", 0..1000);
+    let new_rec = remove_every_nth(&old, 2);
+    assert_equivalence(&old, &new_rec, "1000 records - delete every 2nd").await;
+}
+
+#[tokio::test]
+async fn massive_tree_delete_every_5th() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let new_rec = remove_every_nth(&old, 5);
+    assert_equivalence(&old, &new_rec, "2000 records - delete every 5th").await;
+}
+
+#[tokio::test]
+async fn massive_tree_delete_first_half() {
+    let old = generate_records("app.bsky.feed.post", 0..1500);
+    let new_rec = remove_range(&old, 0, 750);
+    assert_equivalence(&old, &new_rec, "1500 records - delete first 750").await;
+}
+
+#[tokio::test]
+async fn massive_tree_delete_last_half() {
+    let old = generate_records("app.bsky.feed.post", 0..1500);
+    let new_rec = remove_range(&old, 750, 750);
+    assert_equivalence(&old, &new_rec, "1500 records - delete last 750").await;
+}
+
+#[tokio::test]
+async fn massive_tree_delete_middle_chunk() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let new_rec = remove_range(&old, 800, 400);
+    assert_equivalence(&old, &new_rec, "2000 records - delete 400 from middle").await;
+}
+
+#[tokio::test]
+async fn empty_to_massive() {
+    let new_rec = generate_records("app.bsky.feed.post", 0..1500);
+    assert_equivalence(&[], &new_rec, "empty to 1500 records").await;
+}
+
+#[tokio::test]
+async fn massive_to_empty() {
+    let old = generate_records("app.bsky.feed.post", 0..1500);
+    assert_equivalence(&old, &[], "1500 records to empty").await;
+}
+
+#[tokio::test]
+async fn massive_complete_replacement() {
+    let old = generate_records("app.bsky.feed.post", 0..1000);
+    let new_rec = generate_records("app.bsky.feed.post", 1000..2000);
+    assert_equivalence(&old, &new_rec, "1000 records fully replaced with 1000 different").await;
+}
+
+#[tokio::test]
+async fn massive_no_change() {
+    let records = generate_records("app.bsky.feed.post", 0..1500);
+    assert_equivalence(&records, &records, "1500 records unchanged").await;
+}
+
+#[tokio::test]
+async fn multi_collection_5_collections_500_each() {
+    let collections = [
+        "app.bsky.feed.like",
+        "app.bsky.feed.post",
+        "app.bsky.feed.repost",
+        "app.bsky.graph.follow",
+        "app.bsky.graph.block",
+    ];
+    let old = generate_multi_collection_records(&collections, 500);
+    let new_rec = apply_scattered_updates(&old, 4, 30000);
+    assert_equivalence(&old, &new_rec, "5 collections x 500 records - update every 4th").await;
+}
+
+#[tokio::test]
+async fn multi_collection_wipe_one_collection() {
+    let collections = [
+        "app.bsky.feed.like",
+        "app.bsky.feed.post",
+        "app.bsky.feed.repost",
+        "app.bsky.graph.follow",
+    ];
+    let old = generate_multi_collection_records(&collections, 400);
+
+    let new_rec: Vec<_> = old
+        .iter()
+        .filter(|(key, _)| !key.starts_with("app.bsky.feed.repost"))
+        .cloned()
+        .collect();
+    assert_equivalence(&old, &new_rec, "4 collections x 400 - wipe repost collection").await;
+}
+
+#[tokio::test]
+async fn multi_collection_keep_only_one() {
+    let collections = [
+        "app.bsky.feed.like",
+        "app.bsky.feed.post",
+        "app.bsky.feed.repost",
+        "app.bsky.graph.follow",
+        "app.bsky.graph.block",
+    ];
+    let old = generate_multi_collection_records(&collections, 300);
+    let new_rec = keep_only_collection(&old, "app.bsky.feed.post");
+    assert_equivalence(&old, &new_rec, "5 collections x 300 - keep only posts").await;
+}
+
+#[tokio::test]
+async fn multi_collection_add_new_collection() {
+    let old_collections = [
+        "app.bsky.feed.like",
+        "app.bsky.feed.post",
+    ];
+    let old = generate_multi_collection_records(&old_collections, 500);
+    let new_rec = append_records(&old, "app.bsky.graph.follow", 0..500, 40000);
+    assert_equivalence(&old, &new_rec, "2 collections x 500 + add 500 follows").await;
+}
+
+#[tokio::test]
+async fn mixed_ops_massive_tree() {
+    let collections = [
+        "app.bsky.feed.like",
+        "app.bsky.feed.post",
+        "app.bsky.feed.repost",
+        "app.bsky.graph.follow",
+    ];
+    let old = generate_multi_collection_records(&collections, 400);
+
+    let mut new_rec: Vec<_> = old
+        .iter()
+        .filter(|(key, _)| !key.starts_with("app.bsky.feed.repost"))
+        .enumerate()
+        .map(|(idx, (key, val))| {
+            if key.starts_with("app.bsky.feed.like") && idx % 3 == 0 {
+                (key.clone(), val + 50000)
+            } else {
+                (key.clone(), *val)
+            }
+        })
+        .collect();
+
+    new_rec.extend((0..200u32).map(|i| (make_key("app.bsky.graph.block", i), 60000 + i)));
+    new_rec.sort_by(|(a, _), (b, _)| a.cmp(b));
+
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "4 collections x 400: wipe reposts, update every 3rd like, add 200 blocks",
+    )
+    .await;
+}
+
+#[tokio::test]
+async fn grow_tree_by_double() {
+    let old = generate_records("app.bsky.feed.post", 0..1000);
+    let new_rec = generate_records("app.bsky.feed.post", 0..2000);
+    assert_equivalence(&old, &new_rec, "grow from 1000 to 2000").await;
+}
+
+#[tokio::test]
+async fn shrink_tree_by_half() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let new_rec = generate_records("app.bsky.feed.post", 0..1000);
+    assert_equivalence(&old, &new_rec, "shrink from 2000 to 1000").await;
+}
+
+#[tokio::test]
+async fn interleaved_keys_disjoint_ranges() {
+    let old: Vec<_> = (0..1000u32)
+        .map(|i| (make_key("app.bsky.feed.post", i * 2), i))
+        .collect();
+    let new_rec: Vec<_> = (0..1000u32)
+        .map(|i| (make_key("app.bsky.feed.post", i * 2 + 1), i + 10000))
+        .collect();
+    assert_equivalence(&old, &new_rec, "1000 even-keyed records replaced by 1000 odd-keyed").await;
+}
+
+#[tokio::test]
+async fn sparse_keys_wide_gaps() {
+    let old: Vec<_> = (0..500u32)
+        .map(|i| (make_key("app.bsky.feed.post", i * 100), i))
+        .collect();
+    let new_rec: Vec<_> = (0..500u32)
+        .map(|i| {
+            if i % 10 == 0 {
+                (make_key("app.bsky.feed.post", i * 100), i + 70000)
+            } else {
+                (make_key("app.bsky.feed.post", i * 100), i)
+            }
+        })
+        .collect();
+    assert_equivalence(&old, &new_rec, "500 sparse keys - update every 10th").await;
+}
+
+#[tokio::test]
+async fn many_collections_few_records_each() {
+    let collections: Vec<String> = (0..50u32)
+        .map(|i| format!("com.example.lexicon{i:02}.record"))
+        .collect();
+    let old: Vec<_> = collections
+        .iter()
+        .enumerate()
+        .flat_map(|(ci, coll)| {
+            let base = ci as u32 * 20;
+            (0..20u32).map(move |i| (make_key(coll, i), base + i))
+        })
+        .collect();
+
+    let new_rec: Vec<_> = old
+        .iter()
+        .enumerate()
+        .filter_map(|(idx, (key, val))| {
+            if idx % 15 == 0 {
+                None
+            } else if idx % 7 == 0 {
+                Some((key.clone(), val + 80000))
+            } else {
+                Some((key.clone(), *val))
+            }
+        })
+        .collect();
+
+    assert_equivalence(&old, &new_rec, "50 collections x 20 records - delete every 15th, update every 7th").await;
+}
+
+#[tokio::test]
+async fn update_all_records() {
+    let old = generate_records("app.bsky.feed.post", 0..1000);
+    let new_rec: Vec<_> = old
+        .iter()
+        .map(|(key, val)| (key.clone(), val + 90000))
+        .collect();
+    assert_equivalence(&old, &new_rec, "1000 records - update every single one").await;
+}
+
+#[tokio::test]
+async fn delete_all_but_one() {
+    let old = generate_records("app.bsky.feed.post", 0..1500);
+    let new_rec = vec![old[750].clone()];
+    assert_equivalence(&old, &new_rec, "1500 records - delete all but middle one").await;
+}
+
+#[tokio::test]
+async fn one_to_massive() {
+    let old = vec![(make_key("app.bsky.feed.post", 500), 500u32)];
+    let new_rec = generate_records("app.bsky.feed.post", 0..1500);
+    assert_equivalence(&old, &new_rec, "1 record to 1500 records").await;
+}
+
+#[tokio::test]
+async fn delete_head_and_tail() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let new_rec: Vec<_> = old[200..1800].to_vec();
+    assert_equivalence(&old, &new_rec, "2000 records - delete first 200 and last 200").await;
+}
+
+#[tokio::test]
+async fn keep_head_and_tail_only() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let mut new_rec: Vec<_> = old[..100].to_vec();
+    new_rec.extend_from_slice(&old[1900..]);
+    assert_equivalence(&old, &new_rec, "2000 records - keep only first 100 and last 100").await;
+}
+
+#[tokio::test]
+async fn massive_tree_update_first_and_last() {
+    let old = generate_records("app.bsky.feed.post", 0..2000);
+    let mut new_rec = old.clone();
+    new_rec[0].1 += 99000;
+    new_rec[1999].1 += 99000;
+    assert_equivalence(&old, &new_rec, "2000 records - update only first and last").await;
+}
+
+#[tokio::test]
+async fn overlapping_collection_swap() {
+    let old_collections = [
+        "app.bsky.feed.like",
+        "app.bsky.feed.post",
+        "app.bsky.feed.repost",
+    ];
+    let old = generate_multi_collection_records(&old_collections, 500);
+
+    let mut new_rec: Vec<_> = old
+        .iter()
+        .filter(|(key, _)| key.starts_with("app.bsky.feed.post"))
+        .cloned()
+        .collect();
+    new_rec.extend((0..500u32).map(|i| (make_key("app.bsky.graph.follow", i), 70000 + i)));
+    new_rec.extend((0..500u32).map(|i| (make_key("app.bsky.graph.block", i), 71000 + i)));
+    new_rec.sort_by(|(a, _), (b, _)| a.cmp(b));
+
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "swap 2 of 3 collections, keep 1 (posts), 500 each",
+    )
+    .await;
+}
+
+#[tokio::test]
+async fn swiss_cheese_deletions() {
+    let old = generate_records("app.bsky.feed.post", 0..1500);
+    let new_rec: Vec<_> = old
+        .iter()
+        .enumerate()
+        .filter(|(idx, _)| {
+            let bucket = idx / 50;
+            bucket % 3 != 0
+        })
+        .map(|(_, r)| r.clone())
+        .collect();
+    assert_equivalence(&old, &new_rec, "1500 records - delete every 3rd chunk of 50").await;
+}
+
+#[tokio::test]
+async fn mixed_ops_with_key_density_change() {
+    let old: Vec<_> = (0..1000u32)
+        .map(|i| (make_key("app.bsky.feed.post", i * 3), i))
+        .collect();
+
+    let mut new_rec: Vec<_> = old
+        .iter()
+        .filter(|(_, val)| val % 4 != 0)
+        .cloned()
+        .collect();
+    new_rec.extend((0..500u32).map(|i| {
+        (make_key("app.bsky.feed.post", i * 3 + 1), i + 100000)
+    }));
+    new_rec.sort_by(|(a, _), (b, _)| a.cmp(b));
+
+    assert_equivalence(
+        &old,
+        &new_rec,
+        "1000 sparse records: delete every 4th, insert 500 in gaps",
+    )
+    .await;
+}

crates/tranquil-pds/tests/sync_resource_limits.rs

@@ -0,0 +1,147 @@
+mod common;
+mod helpers;
+use common::*;
+use helpers::*;
+use reqwest::StatusCode;
+use std::sync::Once;
+
+static SET_SEMAPHORE: Once = Once::new();
+
+fn ensure_low_semaphore() {
+    SET_SEMAPHORE.call_once(|| unsafe {
+        std::env::set_var("MAX_CONCURRENT_REPO_EXPORTS", "1");
+    });
+}
+
+#[tokio::test]
+async fn test_get_repo_succeeds_with_many_records() {
+    ensure_low_semaphore();
+    let client = client();
+    let (did, jwt) = setup_new_user("sync-batched-car").await;
+
+    let create_futures = (0..20).map(|i| {
+        let client = &client;
+        let did = &did;
+        let jwt = &jwt;
+        async move {
+            create_post(client, did, jwt, &format!("Batch test post {}", i)).await;
+        }
+    });
+    futures::future::join_all(create_futures).await;
+
+    let res = client
+        .get(format!(
+            "{}/xrpc/com.atproto.sync.getRepo",
+            base_url().await
+        ))
+        .query(&[("did", did.as_str())])
+        .send()
+        .await
+        .expect("Failed to send getRepo request");
+
+    assert_eq!(res.status(), StatusCode::OK);
+    assert_eq!(
+        res.headers()
+            .get("content-type")
+            .and_then(|h| h.to_str().ok()),
+        Some("application/vnd.ipld.car")
+    );
+    let car_bytes = res.bytes().await.expect("Failed to read response body");
+    assert!(
+        car_bytes.len() > 200,
+        "CAR with 20 records should have substantial data, got {} bytes",
+        car_bytes.len()
+    );
+}
+
+#[tokio::test]
+async fn test_get_repo_semaphore_rejects_excess_concurrency() {
+    ensure_low_semaphore();
+    let client = client();
+    let (did, jwt) = setup_new_user("sync-semaphore").await;
+
+    for i in 0..50 {
+        create_post(&client, &did, &jwt, &format!("Padding post {}", i)).await;
+    }
+
+    let base = base_url().await;
+    let concurrent_requests = 10;
+
+    let request_futures = (0..concurrent_requests).map(|_| {
+        let client = client.clone();
+        let did = did.clone();
+        async move {
+            client
+                .get(format!("{}/xrpc/com.atproto.sync.getRepo", base))
+                .query(&[("did", did.as_str())])
+                .send()
+                .await
+                .expect("Failed to send request")
+                .status()
+        }
+    });
+
+    let statuses: Vec<StatusCode> = futures::future::join_all(request_futures).await;
+    let ok_count = statuses.iter().filter(|s| **s == StatusCode::OK).count();
+    let rejected_count = statuses
+        .iter()
+        .filter(|s| **s == StatusCode::SERVICE_UNAVAILABLE)
+        .count();
+
+    assert!(ok_count >= 1, "at least one request should succeed");
+    assert!(
+        rejected_count > 0,
+        "semaphore=1 with {} concurrent requests, expected some 503 rejections",
+        concurrent_requests
+    );
+    assert!(
+        ok_count + rejected_count == statuses.len(),
+        "expected only 200 or 503 responses: {:?}",
+        statuses
+    );
+}
+
+#[tokio::test]
+async fn test_get_repo_since_not_affected_by_semaphore() {
+    ensure_low_semaphore();
+    let client = client();
+    let (did, jwt) = setup_new_user("sync-since-no-sem").await;
+    create_post(&client, &did, &jwt, "First post").await;
+
+    let latest_res = client
+        .get(format!(
+            "{}/xrpc/com.atproto.sync.getLatestCommit",
+            base_url().await
+        ))
+        .query(&[("did", did.as_str())])
+        .send()
+        .await
+        .expect("Failed to get latest commit");
+    let body: serde_json::Value = latest_res.json().await.unwrap();
+    let rev = body["rev"].as_str().unwrap();
+
+    create_post(&client, &did, &jwt, "Second post").await;
+
+    let base = base_url().await;
+    let request_futures = (0..10).map(|_| {
+        let client = client.clone();
+        let did = did.clone();
+        let rev = rev.to_string();
+        async move {
+            client
+                .get(format!("{}/xrpc/com.atproto.sync.getRepo", base))
+                .query(&[("did", did.as_str()), ("since", rev.as_str())])
+                .send()
+                .await
+                .expect("Failed to send request")
+                .status()
+        }
+    });
+
+    let statuses: Vec<StatusCode> = futures::future::join_all(request_futures).await;
+    assert!(
+        statuses.iter().all(|s| *s == StatusCode::OK),
+        "getRepo with since should bypass semaphore, got: {:?}",
+        statuses
+    );
+}

crates/tranquil-pds/tests/whole_story.rs

@@ -1171,7 +1171,7 @@ async fn test_backup_restore_workflow() {
 }
 
 #[tokio::test]
-async fn test_scale_100_posts_with_pagination() {
+async fn test_scale_1000_posts_with_pagination() {
     let client = client();
     let base = base_url().await;
     let (did, jwt) = setup_new_user("scale-posts").await;

crates/tranquil-repo/src/lib.rs

@@ -6,7 +6,7 @@ use jacquard_repo::storage::BlockStore;
 use multihash::Multihash;
 use sha2::{Digest, Sha256};
 use sqlx::PgPool;
-use std::collections::HashSet;
+use std::collections::{HashMap, HashSet};
 use std::sync::{Arc, Mutex};
 
 #[derive(Clone)]
@@ -24,32 +24,6 @@ impl PostgresBlockStore {
     }
 }
 
-impl PostgresBlockStore {
-    pub async fn get_oldest_block_cids(&self, limit: i64) -> Result<Vec<Vec<u8>>, RepoError> {
-        let rows = sqlx::query!(
-            "SELECT cid FROM blocks ORDER BY created_at ASC LIMIT $1",
-            limit,
-        )
-        .fetch_all(&self.pool)
-        .await
-        .map_err(RepoError::storage)?;
-        Ok(rows.into_iter().map(|r| r.cid).collect())
-    }
-
-    pub async fn delete_blocks(&self, cids: &[Vec<u8>]) -> Result<u64, RepoError> {
-        match cids.is_empty() {
-            true => Ok(0),
-            false => {
-                let result = sqlx::query!("DELETE FROM blocks WHERE cid = ANY($1)", cids,)
-                    .execute(&self.pool)
-                    .await
-                    .map_err(RepoError::storage)?;
-                Ok(result.rows_affected())
-            }
-        }
-    }
-}
-
 impl BlockStore for PostgresBlockStore {
     async fn get(&self, cid: &Cid) -> Result<Option<Bytes>, RepoError> {
         let cid_bytes = cid.to_bytes();
@@ -152,7 +126,7 @@ impl BlockStore for PostgresBlockStore {
 #[derive(Clone)]
 pub struct TrackingBlockStore<S: BlockStore> {
     inner: S,
-    written_cids: Arc<Mutex<Vec<Cid>>>,
+    written_blocks: Arc<Mutex<HashMap<Cid, Bytes>>>,
     read_cids: Arc<Mutex<HashSet<Cid>>>,
 }
 
@@ -160,18 +134,26 @@ impl<S: BlockStore + Sync> TrackingBlockStore<S> {
     pub fn new(store: S) -> Self {
         Self {
             inner: store,
-            written_cids: Arc::new(Mutex::new(Vec::new())),
+            written_blocks: Arc::new(Mutex::new(HashMap::new())),
             read_cids: Arc::new(Mutex::new(HashSet::new())),
         }
     }
 
     pub fn get_written_cids(&self) -> Vec<Cid> {
-        match self.written_cids.lock() {
-            Ok(guard) => guard.clone(),
-            Err(poisoned) => poisoned.into_inner().clone(),
+        match self.written_blocks.lock() {
+            Ok(guard) => guard.keys().copied().collect(),
+            Err(poisoned) => poisoned.into_inner().keys().copied().collect(),
         }
     }
 
+    pub fn take_written_blocks(&self) -> HashMap<Cid, Bytes> {
+        let mut guard = match self.written_blocks.lock() {
+            Ok(g) => g,
+            Err(poisoned) => poisoned.into_inner(),
+        };
+        std::mem::take(&mut *guard)
+    }
+
     pub fn get_read_cids(&self) -> Vec<Cid> {
         match self.read_cids.lock() {
             Ok(guard) => guard.iter().cloned().collect(),
@@ -206,9 +188,14 @@ impl<S: BlockStore + Sync> BlockStore for TrackingBlockStore<S> {
 
     async fn put(&self, data: &[u8]) -> Result<Cid, RepoError> {
         let cid = self.inner.put(data).await?;
-        match self.written_cids.lock() {
-            Ok(mut guard) => guard.push(cid),
-            Err(poisoned) => poisoned.into_inner().push(cid),
+        let bytes = Bytes::copy_from_slice(data);
+        match self.written_blocks.lock() {
+            Ok(mut guard) => {
+                guard.insert(cid, bytes);
+            }
+            Err(poisoned) => {
+                poisoned.into_inner().insert(cid, bytes);
+            }
         }
         Ok(cid)
     }
@@ -222,11 +209,10 @@ impl<S: BlockStore + Sync> BlockStore for TrackingBlockStore<S> {
         blocks: impl IntoIterator<Item = (Cid, Bytes)> + Send,
     ) -> Result<(), RepoError> {
         let blocks: Vec<_> = blocks.into_iter().collect();
-        let cids: Vec<Cid> = blocks.iter().map(|(cid, _)| *cid).collect();
-        self.inner.put_many(blocks).await?;
-        match self.written_cids.lock() {
-            Ok(mut guard) => guard.extend(cids),
-            Err(poisoned) => poisoned.into_inner().extend(cids),
+        self.inner.put_many(blocks.clone()).await?;
+        match self.written_blocks.lock() {
+            Ok(mut guard) => guard.extend(blocks),
+            Err(poisoned) => poisoned.into_inner().extend(blocks),
         }
         Ok(())
     }

crates/tranquil-scopes/src/permissions.rs

@@ -157,11 +157,19 @@ impl ScopePermissions {
     }
 
     pub fn assert_rpc(&self, aud: &str, lxm: &str) -> Result<(), ScopeError> {
-        if self.has_transition_generic {
-            return Ok(());
+        if lxm.starts_with("chat.bsky.") {
+            if self.has_transition_chat {
+                return Ok(());
+            }
+            if self.has_transition_generic && !self.has_transition_chat {
+                return Err(ScopeError::InsufficientScope {
+                    required: "transition:chat.bsky".to_string(),
+                    message: format!("Chat access requires transition:chat.bsky scope to call {}", lxm),
+                });
+            }
         }
 
-        if lxm.starts_with("chat.bsky.") && self.has_transition_chat {
+        if self.has_transition_generic {
             return Ok(());
         }
 
@@ -347,6 +355,23 @@ mod tests {
         assert!(perms.allows_blob("image/png"));
     }
 
+    #[test]
+    fn test_transition_generic_without_chat_blocks_chat() {
+        let perms = ScopePermissions::from_scope_string(Some("transition:generic"));
+        assert!(perms.allows_rpc("did:web:api.bsky.app", "app.bsky.feed.getTimeline"));
+        assert!(!perms.allows_rpc("did:web:api.bsky.app", "chat.bsky.convo.listConvos"));
+        assert!(!perms.allows_rpc("did:web:api.bsky.app", "chat.bsky.convo.getMessages"));
+    }
+
+    #[test]
+    fn test_transition_generic_with_chat_allows_chat() {
+        let perms =
+            ScopePermissions::from_scope_string(Some("transition:generic transition:chat.bsky"));
+        assert!(perms.allows_rpc("did:web:api.bsky.app", "app.bsky.feed.getTimeline"));
+        assert!(perms.allows_rpc("did:web:api.bsky.app", "chat.bsky.convo.listConvos"));
+        assert!(perms.allows_rpc("did:web:api.bsky.app", "chat.bsky.convo.getMessages"));
+    }
+
     #[test]
     fn test_transition_chat_only_allows_chat() {
         let perms = ScopePermissions::from_scope_string(Some("transition:chat.bsky"));

crates/tranquil-server/src/main.rs

@@ -10,8 +10,7 @@ use tranquil_pds::comms::{CommsService, DiscordSender, EmailSender, SignalSender
 
 use tranquil_pds::crawlers::{Crawlers, start_crawlers_service};
 use tranquil_pds::scheduled::{
-    backfill_genesis_commit_blocks, backfill_record_blobs, backfill_repo_rev, backfill_user_blocks,
-    start_scheduled_tasks,
+    backfill_record_blobs, backfill_repo_rev, backfill_user_blocks, start_scheduled_tasks,
 };
 use tranquil_pds::state::AppState;
 
@@ -131,10 +130,6 @@ async fn run() -> Result<(), Box<dyn std::error::Error>> {
     let backfill_block_store = state.block_store.clone();
     tokio::spawn(async move {
         tokio::join!(
-            backfill_genesis_commit_blocks(
-                backfill_repo_repo.clone(),
-                backfill_block_store.clone()
-            ),
             backfill_repo_rev(backfill_repo_repo.clone(), backfill_block_store.clone()),
             backfill_user_blocks(backfill_repo_repo.clone(), backfill_block_store.clone()),
             backfill_record_blobs(backfill_repo_repo, backfill_block_store),
@@ -255,6 +250,7 @@ async fn run() -> Result<(), Box<dyn std::error::Error>> {
         state.repos.sso.clone(),
         state.repos.repo.clone(),
         state.block_store.clone(),
+        state.eventlog_segments_dir.clone(),
         shutdown.clone(),
     ));
 

crates/tranquil-signal/Cargo.toml

@@ -24,4 +24,6 @@ uuid = { workspace = true }
 thiserror = { workspace = true }
 
 [dev-dependencies]
+tranquil-signal = { path = ".", features = ["fjall-store"] }
 rand = "0.9"
+tempfile = "3"

crates/tranquil-signal/src/lib.rs

@@ -6,6 +6,8 @@ pub mod fjall_store;
 
 #[cfg(test)]
 mod tests;
+#[cfg(test)]
+mod tests_fjall;
 
 pub use client::{
     DeviceName, InvalidDeviceName, InvalidSignalUsername, LinkGeneration, LinkResult, MessageBody,

crates/tranquil-signal/src/tests.rs

@@ -14,15 +14,16 @@ use uuid::Uuid;
 
 use crate::store::{IdentityType, PgProtocolStore, PgSignalStore};
 
-async fn test_store() -> PgSignalStore {
+async fn test_store() -> Option<PgSignalStore> {
     let url = std::env::var("DATABASE_URL")
         .unwrap_or_else(|_| "postgres://postgres:postgres@127.0.0.1:5432/postgres".into());
 
     let pool = PgPoolOptions::new()
         .max_connections(5)
+        .acquire_timeout(std::time::Duration::from_secs(2))
         .connect(&url)
         .await
-        .unwrap();
+        .ok()?;
 
     sqlx::query("DELETE FROM signal_kv")
         .execute(&pool)
@@ -61,7 +62,7 @@ async fn test_store() -> PgSignalStore {
         .await
         .ok();
 
-    PgSignalStore::new(pool)
+    Some(PgSignalStore::new(pool))
 }
 
 fn protocol_store(store: &PgSignalStore, identity: IdentityType) -> PgProtocolStore {
@@ -70,7 +71,9 @@ fn protocol_store(store: &PgSignalStore, identity: IdentityType) -> PgProtocolSt
 
 #[tokio::test]
 async fn state_store_registration_empty() {
-    let store = test_store().await;
+    let Some(store) = test_store().await else {
+        return;
+    };
 
     assert!(store.load_registration_data().await.unwrap().is_none());
     assert!(!store.is_registered().await);
@@ -78,7 +81,9 @@ async fn state_store_registration_empty() {
 
 #[tokio::test]
 async fn state_store_kv_roundtrip() {
-    let store = test_store().await;
+    let Some(store) = test_store().await else {
+        return;
+    };
 
     let value = b"test-data".to_vec();
     sqlx::query("INSERT INTO signal_kv (key, value) VALUES ('test_key', $1)")
@@ -96,7 +101,9 @@ async fn state_store_kv_roundtrip() {
 
 #[tokio::test]
 async fn state_store_identity_keypairs() {
-    let store = test_store().await;
+    let Some(store) = test_store().await else {
+        return;
+    };
 
     let aci_pair = IdentityKeyPair::generate(&mut rand::rng());
     let pni_pair = IdentityKeyPair::generate(&mut rand::rng());
@@ -116,13 +123,17 @@ async fn state_store_identity_keypairs() {
 
 #[tokio::test]
 async fn state_store_sender_certificate_roundtrip() {
-    let store = test_store().await;
+    let Some(store) = test_store().await else {
+        return;
+    };
     assert!(store.sender_certificate().await.unwrap().is_none());
 }
 
 #[tokio::test]
 async fn state_store_clear_registration() {
-    let mut store = test_store().await;
+    let Some(mut store) = test_store().await else {
+        return;
+    };
 
     sqlx::query("INSERT INTO signal_kv (key, value) VALUES ('registration', $1)")
         .bind(b"dummy-data".as_slice())
@@ -151,7 +162,9 @@ async fn state_store_clear_registration() {
 
 #[tokio::test]
 async fn session_store_crud() {
-    let store = test_store().await;
+    let Some(store) = test_store().await else {
+        return;
+    };
     let mut ps = protocol_store(&store, IdentityType::Aci);
 
     let addr = ProtocolAddress::new("test-uuid".into(), DeviceId::new(1).unwrap());
@@ -170,7 +183,9 @@ async fn session_store_crud() {
 
 #[tokio::test]
 async fn session_store_sub_devices() {
-    let store = test_store().await;
+    let Some(store) = test_store().await else {
+        return;
+    };
     let mut ps = protocol_store(&store, IdentityType::Aci);
 
     let uuid = Uuid::new_v4();
@@ -196,7 +211,9 @@ async fn session_store_sub_devices() {
 
 #[tokio::test]
 async fn pre_key_store_crud() {
-    let store = test_store().await;
+    let Some(store) = test_store().await else {
+        return;
+    };
     let mut ps = protocol_store(&store, IdentityType::Aci);
 
     let keypair = KeyPair::generate(&mut rand::rng());
@@ -213,7 +230,9 @@ async fn pre_key_store_crud() {
 
 #[tokio::test]
 async fn pre_key_store_next_ids() {
-    let store = test_store().await;
+    let Some(store) = test_store().await else {
+        return;
+    };
     let mut ps = protocol_store(&store, IdentityType::Aci);
 
     assert_eq!(ps.next_pre_key_id().await.unwrap(), 1);
@@ -229,7 +248,9 @@ async fn pre_key_store_next_ids() {
 
 #[tokio::test]
 async fn signed_pre_key_store_crud() {
-    let store = test_store().await;
+    let Some(store) = test_store().await else {
+        return;
+    };
     let mut ps = protocol_store(&store, IdentityType::Aci);
 
     let keypair = KeyPair::generate(&mut rand::rng());
@@ -251,7 +272,9 @@ async fn signed_pre_key_store_crud() {
 
 #[tokio::test]
 async fn kyber_pre_key_one_time_mark_used_deletes() {
-    let store = test_store().await;
+    let Some(store) = test_store().await else {
+        return;
+    };
     let mut ps = protocol_store(&store, IdentityType::Aci);
 
     let keypair = KeyPair::generate(&mut rand::rng());
@@ -276,7 +299,9 @@ async fn kyber_pre_key_one_time_mark_used_deletes() {
 
 #[tokio::test]
 async fn kyber_pre_key_last_resort_survives_mark_used() {
-    let store = test_store().await;
+    let Some(store) = test_store().await else {
+        return;
+    };
     let mut ps = protocol_store(&store, IdentityType::Aci);
 
     let keypair = KeyPair::generate(&mut rand::rng());
@@ -303,7 +328,9 @@ async fn kyber_pre_key_last_resort_survives_mark_used() {
 
 #[tokio::test]
 async fn kyber_pre_key_last_resort_rejects_replayed_base_key() {
-    let store = test_store().await;
+    let Some(store) = test_store().await else {
+        return;
+    };
     let mut ps = protocol_store(&store, IdentityType::Aci);
 
     let keypair = KeyPair::generate(&mut rand::rng());
@@ -332,7 +359,9 @@ async fn kyber_pre_key_last_resort_rejects_replayed_base_key() {
 
 #[tokio::test]
 async fn kyber_pre_key_last_resort_list() {
-    let store = test_store().await;
+    let Some(store) = test_store().await else {
+        return;
+    };
     let mut ps = protocol_store(&store, IdentityType::Aci);
 
     let keypair = KeyPair::generate(&mut rand::rng());
@@ -361,7 +390,9 @@ async fn kyber_pre_key_last_resort_list() {
 
 #[tokio::test]
 async fn identity_store_crud() {
-    let store = test_store().await;
+    let Some(store) = test_store().await else {
+        return;
+    };
     let mut ps = protocol_store(&store, IdentityType::Aci);
 
     let addr = ProtocolAddress::new("test-addr".into(), DeviceId::new(1).unwrap());
@@ -383,7 +414,9 @@ async fn identity_store_crud() {
 
 #[tokio::test]
 async fn identity_store_aci_pni_isolation() {
-    let store = test_store().await;
+    let Some(store) = test_store().await else {
+        return;
+    };
     let mut aci_store = protocol_store(&store, IdentityType::Aci);
     let pni_store = protocol_store(&store, IdentityType::Pni);
 
@@ -401,7 +434,9 @@ async fn identity_store_aci_pni_isolation() {
 
 #[tokio::test]
 async fn sender_key_store_load_missing() {
-    let store = test_store().await;
+    let Some(store) = test_store().await else {
+        return;
+    };
     let mut ps = protocol_store(&store, IdentityType::Aci);
 
     let sender = ProtocolAddress::new("sender-uuid".into(), DeviceId::new(1).unwrap());
@@ -417,7 +452,9 @@ async fn sender_key_store_load_missing() {
 
 #[tokio::test]
 async fn profile_key_store_roundtrip() {
-    let mut store = test_store().await;
+    let Some(mut store) = test_store().await else {
+        return;
+    };
 
     let uuid = Uuid::new_v4();
     let service_id: ServiceId = presage::libsignal_service::protocol::Aci::from(uuid).into();
@@ -433,7 +470,9 @@ async fn profile_key_store_roundtrip() {
 
 #[tokio::test]
 async fn client_from_pool_returns_none_without_registration() {
-    let store = test_store().await;
+    let Some(store) = test_store().await else {
+        return;
+    };
     let pool = store.db.clone();
 
     let client =
@@ -443,7 +482,9 @@ async fn client_from_pool_returns_none_without_registration() {
 
 #[tokio::test]
 async fn store_clear_removes_kv() {
-    let mut store = test_store().await;
+    let Some(mut store) = test_store().await else {
+        return;
+    };
 
     store
         .set_aci_identity_key_pair(IdentityKeyPair::generate(&mut rand::rng()))

crates/tranquil-signal/src/tests_fjall.rs

@@ -0,0 +1,373 @@
+use presage::libsignal_service::{
+    pre_keys::{KyberPreKeyStoreExt, PreKeysStore},
+    prelude::{ProfileKey, SessionStoreExt},
+    protocol::{
+        DeviceId, Direction, GenericSignedPreKey, IdentityKeyPair, IdentityKeyStore, KeyPair,
+        KyberPreKeyId, KyberPreKeyRecord, KyberPreKeyStore, PreKeyId, PreKeyRecord, PreKeyStore,
+        ProtocolAddress, SenderKeyStore, ServiceId, SessionRecord, SessionStore, SignedPreKeyId,
+        SignedPreKeyRecord, SignedPreKeyStore, Timestamp,
+    },
+};
+use presage::store::{ContentsStore, StateStore, Store};
+use uuid::Uuid;
+
+use crate::fjall_store::FjallSignalStore;
+
+fn test_store() -> (FjallSignalStore, tempfile::TempDir) {
+    let dir = tempfile::TempDir::new().unwrap();
+    let db = fjall::Database::builder(dir.path()).open().unwrap();
+    let ks = db
+        .keyspace("signal", fjall::KeyspaceCreateOptions::default)
+        .unwrap();
+    (FjallSignalStore::new(db, ks), dir)
+}
+
+#[tokio::test]
+async fn state_store_registration_empty() {
+    let (store, _dir) = test_store();
+    assert!(store.load_registration_data().await.unwrap().is_none());
+    assert!(!store.is_registered().await);
+}
+
+#[tokio::test]
+async fn state_store_identity_keypairs() {
+    let (store, _dir) = test_store();
+    let aci_pair = IdentityKeyPair::generate(&mut rand::rng());
+    let pni_pair = IdentityKeyPair::generate(&mut rand::rng());
+
+    store.set_aci_identity_key_pair(aci_pair).await.unwrap();
+    store.set_pni_identity_key_pair(pni_pair).await.unwrap();
+
+    let aci_store = store.aci_protocol_store();
+    let pni_store = store.pni_protocol_store();
+
+    let loaded_aci = aci_store.get_identity_key_pair().await.unwrap();
+    let loaded_pni = pni_store.get_identity_key_pair().await.unwrap();
+
+    assert_eq!(loaded_aci.serialize(), aci_pair.serialize());
+    assert_eq!(loaded_pni.serialize(), pni_pair.serialize());
+}
+
+#[tokio::test]
+async fn state_store_sender_certificate_roundtrip() {
+    let (store, _dir) = test_store();
+    assert!(store.sender_certificate().await.unwrap().is_none());
+}
+
+#[tokio::test]
+async fn state_store_clear_registration() {
+    let (mut store, _dir) = test_store();
+
+    store
+        .set_aci_identity_key_pair(IdentityKeyPair::generate(&mut rand::rng()))
+        .await
+        .unwrap();
+
+    let mut ps = store.aci_protocol_store();
+    let keypair = KeyPair::generate(&mut rand::rng());
+    let record = PreKeyRecord::new(PreKeyId::from(1u32), &keypair);
+    ps.save_pre_key(PreKeyId::from(1u32), &record)
+        .await
+        .unwrap();
+
+    store.clear_registration().await.unwrap();
+
+    assert!(store.load_registration_data().await.unwrap().is_none());
+    assert!(ps.get_pre_key(PreKeyId::from(1u32)).await.is_err());
+}
+
+#[tokio::test]
+async fn session_store_crud() {
+    let (store, _dir) = test_store();
+    let mut ps = store.aci_protocol_store();
+
+    let addr = ProtocolAddress::new("test-uuid".into(), DeviceId::new(1).unwrap());
+    assert!(ps.load_session(&addr).await.unwrap().is_none());
+
+    let record = SessionRecord::new_fresh();
+    ps.store_session(&addr, &record).await.unwrap();
+
+    let loaded = ps.load_session(&addr).await.unwrap();
+    assert!(loaded.is_some());
+
+    ps.store_session(&addr, &record).await.unwrap();
+    let loaded2 = ps.load_session(&addr).await.unwrap();
+    assert!(loaded2.is_some());
+}
+
+#[tokio::test]
+async fn session_store_sub_devices() {
+    let (store, _dir) = test_store();
+    let mut ps = store.aci_protocol_store();
+
+    let uuid = Uuid::new_v4();
+    let service_id: ServiceId = presage::libsignal_service::protocol::Aci::from(uuid).into();
+    let addr1 = ProtocolAddress::new(uuid.to_string(), DeviceId::new(1).unwrap());
+    let addr2 = ProtocolAddress::new(uuid.to_string(), DeviceId::new(2).unwrap());
+    let addr3 = ProtocolAddress::new(uuid.to_string(), DeviceId::new(3).unwrap());
+
+    let record = SessionRecord::new_fresh();
+    ps.store_session(&addr1, &record).await.unwrap();
+    ps.store_session(&addr2, &record).await.unwrap();
+    ps.store_session(&addr3, &record).await.unwrap();
+
+    let sub_devices = ps.get_sub_device_sessions(&service_id).await.unwrap();
+    assert_eq!(sub_devices.len(), 2);
+
+    let deleted = ps.delete_all_sessions(&service_id).await.unwrap();
+    assert_eq!(deleted, 3);
+
+    let sub_devices = ps.get_sub_device_sessions(&service_id).await.unwrap();
+    assert!(sub_devices.is_empty());
+}
+
+#[tokio::test]
+async fn pre_key_store_crud() {
+    let (store, _dir) = test_store();
+    let mut ps = store.aci_protocol_store();
+
+    let keypair = KeyPair::generate(&mut rand::rng());
+    let id = PreKeyId::from(42u32);
+    let record = PreKeyRecord::new(id, &keypair);
+
+    ps.save_pre_key(id, &record).await.unwrap();
+    let loaded = ps.get_pre_key(id).await.unwrap();
+    assert_eq!(loaded.serialize().unwrap(), record.serialize().unwrap());
+
+    ps.remove_pre_key(id).await.unwrap();
+    assert!(ps.get_pre_key(id).await.is_err());
+}
+
+#[tokio::test]
+async fn pre_key_store_next_ids() {
+    let (store, _dir) = test_store();
+    let mut ps = store.aci_protocol_store();
+
+    assert_eq!(ps.next_pre_key_id().await.unwrap(), 1);
+
+    let keypair = KeyPair::generate(&mut rand::rng());
+    let record = PreKeyRecord::new(PreKeyId::from(5u32), &keypair);
+    ps.save_pre_key(PreKeyId::from(5u32), &record)
+        .await
+        .unwrap();
+
+    assert_eq!(ps.next_pre_key_id().await.unwrap(), 6);
+}
+
+#[tokio::test]
+async fn signed_pre_key_store_crud() {
+    let (store, _dir) = test_store();
+    let mut ps = store.aci_protocol_store();
+
+    let keypair = KeyPair::generate(&mut rand::rng());
+    let id = SignedPreKeyId::from(1u32);
+    let signature = keypair
+        .private_key
+        .calculate_signature(&keypair.public_key.serialize(), &mut rand::rng())
+        .unwrap();
+    let record =
+        SignedPreKeyRecord::new(id, Timestamp::from_epoch_millis(1000), &keypair, &signature);
+
+    ps.save_signed_pre_key(id, &record).await.unwrap();
+    let loaded = ps.get_signed_pre_key(id).await.unwrap();
+    assert_eq!(loaded.serialize().unwrap(), record.serialize().unwrap());
+
+    assert_eq!(ps.signed_pre_keys_count().await.unwrap(), 1);
+    assert_eq!(ps.next_signed_pre_key_id().await.unwrap(), 2);
+}
+
+#[tokio::test]
+async fn kyber_pre_key_one_time_mark_used_deletes() {
+    let (store, _dir) = test_store();
+    let mut ps = store.aci_protocol_store();
+
+    let keypair = KeyPair::generate(&mut rand::rng());
+    let id = KyberPreKeyId::from(1u32);
+    let record = KyberPreKeyRecord::generate(
+        presage::libsignal_service::protocol::kem::KeyType::Kyber1024,
+        id,
+        &keypair.private_key,
+    )
+    .unwrap();
+
+    ps.save_kyber_pre_key(id, &record).await.unwrap();
+    assert!(ps.get_kyber_pre_key(id).await.is_ok());
+
+    let ec_prekey_id = SignedPreKeyId::from(1u32);
+    ps.mark_kyber_pre_key_used(id, ec_prekey_id, &keypair.public_key)
+        .await
+        .unwrap();
+
+    assert!(ps.get_kyber_pre_key(id).await.is_err());
+}
+
+#[tokio::test]
+async fn kyber_pre_key_last_resort_survives_mark_used() {
+    let (store, _dir) = test_store();
+    let mut ps = store.aci_protocol_store();
+
+    let keypair = KeyPair::generate(&mut rand::rng());
+    let id = KyberPreKeyId::from(1u32);
+    let record = KyberPreKeyRecord::generate(
+        presage::libsignal_service::protocol::kem::KeyType::Kyber1024,
+        id,
+        &keypair.private_key,
+    )
+    .unwrap();
+
+    ps.store_last_resort_kyber_pre_key(id, &record)
+        .await
+        .unwrap();
+    assert!(ps.get_kyber_pre_key(id).await.is_ok());
+
+    let ec_prekey_id = SignedPreKeyId::from(1u32);
+    ps.mark_kyber_pre_key_used(id, ec_prekey_id, &keypair.public_key)
+        .await
+        .unwrap();
+
+    assert!(ps.get_kyber_pre_key(id).await.is_ok());
+}
+
+#[tokio::test]
+async fn kyber_pre_key_last_resort_rejects_replayed_base_key() {
+    let (store, _dir) = test_store();
+    let mut ps = store.aci_protocol_store();
+
+    let keypair = KeyPair::generate(&mut rand::rng());
+    let id = KyberPreKeyId::from(1u32);
+    let record = KyberPreKeyRecord::generate(
+        presage::libsignal_service::protocol::kem::KeyType::Kyber1024,
+        id,
+        &keypair.private_key,
+    )
+    .unwrap();
+
+    ps.store_last_resort_kyber_pre_key(id, &record)
+        .await
+        .unwrap();
+
+    let ec_prekey_id = SignedPreKeyId::from(1u32);
+    ps.mark_kyber_pre_key_used(id, ec_prekey_id, &keypair.public_key)
+        .await
+        .unwrap();
+
+    let replay_result = ps
+        .mark_kyber_pre_key_used(id, ec_prekey_id, &keypair.public_key)
+        .await;
+    assert!(replay_result.is_err());
+}
+
+#[tokio::test]
+async fn kyber_pre_key_last_resort_list() {
+    let (store, _dir) = test_store();
+    let mut ps = store.aci_protocol_store();
+
+    let keypair = KeyPair::generate(&mut rand::rng());
+    let id = KyberPreKeyId::from(1u32);
+    let record = KyberPreKeyRecord::generate(
+        presage::libsignal_service::protocol::kem::KeyType::Kyber1024,
+        id,
+        &keypair.private_key,
+    )
+    .unwrap();
+
+    assert!(
+        ps.load_last_resort_kyber_pre_keys()
+            .await
+            .unwrap()
+            .is_empty()
+    );
+
+    ps.store_last_resort_kyber_pre_key(id, &record)
+        .await
+        .unwrap();
+
+    let last_resorts = ps.load_last_resort_kyber_pre_keys().await.unwrap();
+    assert_eq!(last_resorts.len(), 1);
+}
+
+#[tokio::test]
+async fn identity_store_crud() {
+    let (store, _dir) = test_store();
+    let mut ps = store.aci_protocol_store();
+
+    let addr = ProtocolAddress::new("test-addr".into(), DeviceId::new(1).unwrap());
+    let keypair = IdentityKeyPair::generate(&mut rand::rng());
+    let identity_key = keypair.identity_key();
+
+    assert!(ps.get_identity(&addr).await.unwrap().is_none());
+
+    ps.save_identity(&addr, identity_key).await.unwrap();
+    let loaded = ps.get_identity(&addr).await.unwrap().unwrap();
+    assert_eq!(loaded.serialize(), identity_key.serialize());
+
+    assert!(
+        ps.is_trusted_identity(&addr, identity_key, Direction::Receiving)
+            .await
+            .unwrap()
+    );
+}
+
+#[tokio::test]
+async fn identity_store_aci_pni_isolation() {
+    let (store, _dir) = test_store();
+    let mut aci_store = store.aci_protocol_store();
+    let pni_store = store.pni_protocol_store();
+
+    let addr = ProtocolAddress::new("same-addr".into(), DeviceId::new(1).unwrap());
+    let keypair = IdentityKeyPair::generate(&mut rand::rng());
+
+    aci_store
+        .save_identity(&addr, keypair.identity_key())
+        .await
+        .unwrap();
+
+    assert!(aci_store.get_identity(&addr).await.unwrap().is_some());
+    assert!(pni_store.get_identity(&addr).await.unwrap().is_none());
+}
+
+#[tokio::test]
+async fn sender_key_store_load_missing() {
+    let (store, _dir) = test_store();
+    let mut ps = store.aci_protocol_store();
+
+    let sender = ProtocolAddress::new("sender-uuid".into(), DeviceId::new(1).unwrap());
+    let dist_id = Uuid::new_v4();
+
+    assert!(
+        ps.load_sender_key(&sender, dist_id)
+            .await
+            .unwrap()
+            .is_none()
+    );
+}
+
+#[tokio::test]
+async fn profile_key_store_roundtrip() {
+    let (mut store, _dir) = test_store();
+
+    let uuid = Uuid::new_v4();
+    let service_id: ServiceId = presage::libsignal_service::protocol::Aci::from(uuid).into();
+    let key = ProfileKey { bytes: [42u8; 32] };
+
+    assert!(store.profile_key(&service_id).await.unwrap().is_none());
+
+    store.upsert_profile_key(&uuid, key).await.unwrap();
+
+    let loaded = store.profile_key(&service_id).await.unwrap().unwrap();
+    assert_eq!(loaded.bytes, key.bytes);
+}
+
+#[tokio::test]
+async fn store_clear_removes_all() {
+    let (mut store, _dir) = test_store();
+
+    store
+        .set_aci_identity_key_pair(IdentityKeyPair::generate(&mut rand::rng()))
+        .await
+        .unwrap();
+
+    store.clear().await.unwrap();
+
+    assert!(store.load_registration_data().await.unwrap().is_none());
+}

crates/tranquil-store/Cargo.toml

@@ -28,8 +28,10 @@ jacquard-repo = { workspace = true }
 cid = { workspace = true }
 multihash = { workspace = true }
 sha2 = { workspace = true }
+serde_ipld_dagcbor = { workspace = true }
 siphasher = "1"
 dashmap = "6"
+rayon = "1"
 smallvec = "1"
 uuid = { workspace = true }
 
@@ -37,6 +39,7 @@ uuid = { workspace = true }
 test-harness = []
 
 [dev-dependencies]
+tranquil-store = { path = ".", features = ["test-harness"] }
 proptest = "1"
 tempfile = "3"
 futures = { workspace = true }
@@ -47,8 +50,9 @@ tranquil-db = { workspace = true }
 sqlx = { workspace = true }
 k256 = { workspace = true }
 rand = { workspace = true }
-serde_ipld_dagcbor = { workspace = true }
 tikv-jemallocator = "0.6"
+tracing-subscriber = { workspace = true, features = ["env-filter"] }
+libc = "0.2"
 
 [[bench]]
 name = "blockstore"
@@ -69,3 +73,11 @@ harness = false
 [[bench]]
 name = "profile_reads"
 harness = false
+
+[[bench]]
+name = "profile_eventlog_reads"
+harness = false
+
+[[bench]]
+name = "recovery"
+harness = false

crates/tranquil-store/benches/blockstore.rs

@@ -61,11 +61,16 @@ fn compute_stats(durations: &mut [Duration]) -> Option<LatencyStats> {
 }
 
 fn open_store(dir: &Path) -> TranquilBlockStore {
+    open_store_sharded(dir, 1)
+}
+
+fn open_store_sharded(dir: &Path, shard_count: u8) -> TranquilBlockStore {
     TranquilBlockStore::open(BlockStoreConfig {
         data_dir: dir.join("data"),
         index_dir: dir.join("index"),
         max_file_size: DEFAULT_MAX_FILE_SIZE,
         group_commit: GroupCommitConfig::default(),
+        shard_count,
     })
     .unwrap()
 }
@@ -81,7 +86,7 @@ fn format_latency(stats: Option<&LatencyStats>) -> String {
 }
 
 async fn bench_write_throughput(block_count: usize, concurrency: usize) {
-    let dir = tempfile::TempDir::new().unwrap();
+    let dir = bench_temp_dir();
     let store = open_store(dir.path());
 
     let blocks_per_task = block_count / concurrency;
@@ -162,7 +167,7 @@ async fn bench_write_throughput(block_count: usize, concurrency: usize) {
 }
 
 async fn bench_read_throughput(block_count: usize, concurrency: usize) {
-    let dir = tempfile::TempDir::new().unwrap();
+    let dir = bench_temp_dir();
     let store = open_store(dir.path());
 
     let cids_per_task = block_count / concurrency;
@@ -238,7 +243,7 @@ async fn bench_read_throughput(block_count: usize, concurrency: usize) {
 }
 
 async fn bench_mixed_workload(block_count: usize, concurrency: usize) {
-    let dir = tempfile::TempDir::new().unwrap();
+    let dir = bench_temp_dir();
     let store = open_store(dir.path());
 
     let ops_per_task = block_count / concurrency;
@@ -351,7 +356,7 @@ async fn bench_group_commit_effectiveness(block_count: usize) {
     println!("-- group commit effectiveness at {block_count} blocks --");
 
     let baseline_cycle_time = {
-        let dir = tempfile::TempDir::new().unwrap();
+        let dir = bench_temp_dir();
         let store = open_store(dir.path());
 
         let start = Instant::now();
@@ -377,7 +382,7 @@ async fn bench_group_commit_effectiveness(block_count: usize) {
                 if block_count < concurrency {
                     return;
                 }
-                let dir = tempfile::TempDir::new().unwrap();
+                let dir = bench_temp_dir();
                 let store = open_store(dir.path());
                 let blocks_per_task = block_count / concurrency;
                 let actual_count = blocks_per_task * concurrency;
@@ -422,29 +427,10 @@ async fn bench_group_commit_effectiveness(block_count: usize) {
         .await;
 }
 
-async fn bench_postgres_write_throughput(block_count: usize, concurrency: usize) {
-    let database_url = match std::env::var("DATABASE_URL") {
-        Ok(url) => url,
-        Err(_) => {
-            println!("skipped, set DATABASE_URL to enable");
-            return;
-        }
-    };
+async fn bench_sharded_write_throughput(block_count: usize, concurrency: usize, shard_count: u8) {
+    let dir = bench_temp_dir();
+    let store = open_store_sharded(dir.path(), shard_count);
 
-    let max_conns = u32::try_from(concurrency).expect("concurrency exceeds u32") + 5;
-    let pool = sqlx::postgres::PgPoolOptions::new()
-        .max_connections(max_conns)
-        .connect(&database_url)
-        .await
-        .unwrap();
-
-    sqlx::query("CREATE TABLE IF NOT EXISTS blocks (cid bytea PRIMARY KEY, data bytea NOT NULL)")
-        .execute(&pool)
-        .await
-        .unwrap();
-    sqlx::query("TRUNCATE blocks").execute(&pool).await.unwrap();
-
-    let pg_store = tranquil_repo::PostgresBlockStore::new(pool.clone());
     let blocks_per_task = block_count / concurrency;
     let actual_count = blocks_per_task * concurrency;
     let blocks: Vec<Vec<u8>> = (0..actual_count).map(make_block).collect();
@@ -454,7 +440,7 @@ async fn bench_postgres_write_throughput(block_count: usize, concurrency: usize)
 
     let handles: Vec<_> = (0..concurrency)
         .map(|task_id| {
-            let store = pg_store.clone();
+            let store = store.clone();
             let task_blocks: Vec<Vec<u8>> =
                 blocks[task_id * blocks_per_task..(task_id + 1) * blocks_per_task].to_vec();
             tokio::spawn(async move {
@@ -462,38 +448,39 @@ async fn bench_postgres_write_throughput(block_count: usize, concurrency: usize)
                     .then(|block| {
                         let store = store.clone();
                         async move {
-                            let t = Instant::now();
                             store.put(&block).await.unwrap();
-                            t.elapsed()
                         }
                     })
-                    .collect::<Vec<Duration>>()
-                    .await
+                    .collect::<Vec<()>>()
+                    .await;
             })
         })
         .collect();
 
-    let mut all_latencies: Vec<Duration> = futures::future::join_all(handles)
-        .await
-        .into_iter()
-        .flat_map(Result::unwrap)
-        .collect();
+    futures::future::join_all(handles).await;
     let elapsed = start.elapsed();
-    let stats = compute_stats(&mut all_latencies);
 
-    let lat = format_latency(stats.as_ref());
     println!(
-        "{:.0} blocks/sec, {:.1} MB/sec, {:.1}ms{lat}",
+        "{:.0} blocks/sec, {:.1} MB/sec, {:.1}ms",
         actual_count as f64 / elapsed.as_secs_f64(),
         total_bytes as f64 / elapsed.as_secs_f64() / (1024.0 * 1024.0),
         elapsed.as_secs_f64() * 1000.0,
     );
+}
 
-    sqlx::query("TRUNCATE blocks").execute(&pool).await.unwrap();
-    pool.close().await;
+fn bench_temp_dir() -> tempfile::TempDir {
+    match std::env::var("BENCH_DIR") {
+        Ok(dir) => tempfile::TempDir::new_in(dir).unwrap(),
+        Err(_) => tempfile::TempDir::new().unwrap(),
+    }
 }
 
 fn main() {
+    tracing_subscriber::fmt()
+        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
+        .with_writer(std::io::stderr)
+        .init();
+
     let worker_threads = std::env::var("BENCH_WORKER_THREADS")
         .ok()
         .and_then(|s| s.trim().parse::<usize>().ok())
@@ -558,20 +545,23 @@ fn main() {
 
     rt.block_on(bench_group_commit_effectiveness(1000));
 
-    if std::env::var("DATABASE_URL").is_ok() {
-        block_counts.iter().for_each(|&block_count| {
-            concurrency_levels.iter().for_each(|&concurrency| {
-                if block_count < concurrency {
-                    return;
-                }
-                println!(
-                    "-- postgres write: {} blocks, {} writers --",
-                    block_count, concurrency
-                );
-                rt.block_on(bench_postgres_write_throughput(block_count, concurrency));
+    let shard_counts = parse_env_list("BENCH_SHARDS", vec![1, 2, 4]);
+    if shard_counts.iter().any(|&s| s > 1) {
+        println!("\n-- sharded write throughput :p --");
+        shard_counts.iter().for_each(|&shards| {
+            block_counts.iter().for_each(|&block_count| {
+                concurrency_levels.iter().for_each(|&concurrency| {
+                    if block_count < concurrency {
+                        return;
+                    }
+                    let sc = u8::try_from(shards).unwrap_or(4);
+                    println!(
+                        "-- sharded write: {} shards, {} blocks, {} writers --",
+                        sc, block_count, concurrency
+                    );
+                    rt.block_on(bench_sharded_write_throughput(block_count, concurrency, sc));
+                });
             });
         });
-    } else {
-        println!("set DATABASE_URL for postgres comparison");
     }
 }

crates/tranquil-store/benches/eventlog.rs

@@ -42,7 +42,7 @@ fn make_event(index: usize) -> SequencedEvent {
         prev_data_cid: None,
         ops: Some(serde_json::json!({ "data": ops_payload })),
         blobs: None,
-        blocks_cids: None,
+        blocks: None,
         handle: None,
         active: None,
         status: None,
@@ -621,217 +621,6 @@ fn bench_broadcast_fanout(subscriber_count: usize) {
     let _ = log.shutdown();
 }
 
-async fn bench_pg_write_throughput(event_count: usize, concurrency: usize) {
-    let database_url = match std::env::var("DATABASE_URL") {
-        Ok(url) => url,
-        Err(_) => {
-            println!("skipped, set DATABASE_URL to enable");
-            return;
-        }
-    };
-
-    let max_conns = u32::try_from(concurrency)
-        .unwrap_or(u32::MAX)
-        .saturating_add(10);
-    let pool = sqlx::postgres::PgPoolOptions::new()
-        .max_connections(max_conns)
-        .acquire_timeout(Duration::from_secs(30))
-        .connect(&database_url)
-        .await
-        .unwrap();
-
-    sqlx::query(
-        "CREATE TABLE IF NOT EXISTS bench_repo_seq (
-            seq BIGSERIAL PRIMARY KEY,
-            did TEXT NOT NULL,
-            created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
-            event_type TEXT NOT NULL,
-            ops JSONB
-        )",
-    )
-    .execute(&pool)
-    .await
-    .unwrap();
-    sqlx::query("TRUNCATE bench_repo_seq")
-        .execute(&pool)
-        .await
-        .unwrap();
-
-    let events_per_task = event_count / concurrency;
-    let actual_count = events_per_task * concurrency;
-
-    let start = Instant::now();
-
-    let handles: Vec<_> = (0..concurrency)
-        .map(|task_id| {
-            let pool = pool.clone();
-            tokio::spawn(async move {
-                futures::stream::iter(0..events_per_task)
-                    .then(|i| {
-                        let pool = pool.clone();
-                        async move {
-                            let global = task_id * events_per_task + i;
-                            let did = format!("did:plc:{global:024x}");
-                            let ops_size = match global % 4 {
-                                0 => 64,
-                                1 => 256,
-                                2 => 1024,
-                                _ => 4096,
-                            };
-                            let payload: String = (0..ops_size)
-                                .map(|j| {
-                                    ((global.wrapping_mul(31).wrapping_add(j)) % 26 + 97) as u8
-                                        as char
-                                })
-                                .collect();
-                            let ops = serde_json::json!({ "data": payload });
-                            let t = Instant::now();
-                            sqlx::query(
-                                "INSERT INTO bench_repo_seq (did, event_type, ops) VALUES ($1, $2, $3)",
-                            )
-                            .bind(&did)
-                            .bind("commit")
-                            .bind(&ops)
-                            .execute(&pool)
-                            .await
-                            .unwrap();
-                            t.elapsed()
-                        }
-                    })
-                    .collect::<Vec<Duration>>()
-                    .await
-            })
-        })
-        .collect();
-
-    let mut all_latencies: Vec<Duration> = futures::future::join_all(handles)
-        .await
-        .into_iter()
-        .flat_map(Result::unwrap)
-        .collect();
-    let elapsed = start.elapsed();
-
-    let lat = format_latency(compute_stats(&mut all_latencies).as_ref());
-    println!(
-        "{:.0} events/sec, {:.1}ms{lat}",
-        actual_count as f64 / elapsed.as_secs_f64(),
-        elapsed.as_secs_f64() * 1000.0,
-    );
-
-    sqlx::query("TRUNCATE bench_repo_seq")
-        .execute(&pool)
-        .await
-        .unwrap();
-    pool.close().await;
-}
-
-async fn bench_pg_read_throughput(event_count: usize, concurrency: usize) {
-    let database_url = match std::env::var("DATABASE_URL") {
-        Ok(url) => url,
-        Err(_) => {
-            println!("skipped, set DATABASE_URL to enable");
-            return;
-        }
-    };
-
-    let max_conns = u32::try_from(concurrency)
-        .unwrap_or(u32::MAX)
-        .saturating_add(5);
-    let pool = sqlx::postgres::PgPoolOptions::new()
-        .max_connections(max_conns)
-        .connect(&database_url)
-        .await
-        .unwrap();
-
-    sqlx::query(
-        "CREATE TABLE IF NOT EXISTS bench_repo_seq (
-            seq BIGSERIAL PRIMARY KEY,
-            did TEXT NOT NULL,
-            created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
-            event_type TEXT NOT NULL,
-            ops JSONB
-        )",
-    )
-    .execute(&pool)
-    .await
-    .unwrap();
-
-    let row: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM bench_repo_seq")
-        .fetch_one(&pool)
-        .await
-        .unwrap();
-    if (row.0 as usize) < event_count {
-        sqlx::query("TRUNCATE bench_repo_seq")
-            .execute(&pool)
-            .await
-            .unwrap();
-        println!("populating {event_count} events");
-        futures::stream::iter(0..event_count)
-            .map(|i| {
-                let pool = pool.clone();
-                async move {
-                    let did = format!("did:plc:{i:024x}");
-                    let ops = serde_json::json!({ "data": "x".repeat(256) });
-                    sqlx::query(
-                        "INSERT INTO bench_repo_seq (did, event_type, ops) VALUES ($1, $2, $3)",
-                    )
-                    .bind(&did)
-                    .bind("commit")
-                    .bind(&ops)
-                    .execute(&pool)
-                    .await
-                    .unwrap();
-                }
-            })
-            .buffer_unordered(50)
-            .collect::<Vec<()>>()
-            .await;
-    }
-
-    let total_events = Arc::new(AtomicU64::new(0));
-
-    let start = Instant::now();
-
-    let handles: Vec<_> = (0..concurrency)
-        .map(|_| {
-            let pool = pool.clone();
-            let total_events = Arc::clone(&total_events);
-            tokio::spawn(async move {
-                let mut cursor = 0i64;
-                let mut count = 0u64;
-                loop {
-                    let rows: Vec<(i64,)> = sqlx::query_as(
-                        "SELECT seq FROM bench_repo_seq WHERE seq > $1 ORDER BY seq LIMIT $2",
-                    )
-                    .bind(cursor)
-                    .bind(1000i64)
-                    .fetch_all(&pool)
-                    .await
-                    .unwrap();
-                    if rows.is_empty() {
-                        break;
-                    }
-                    count += rows.len() as u64;
-                    cursor = rows.last().unwrap().0;
-                }
-                total_events.fetch_add(count, Ordering::Relaxed);
-            })
-        })
-        .collect();
-
-    futures::future::join_all(handles).await;
-    let elapsed = start.elapsed();
-
-    let total = total_events.load(Ordering::Relaxed);
-    println!(
-        "{:.0} total events/sec across {concurrency} readers, {total} events, {:.1}ms",
-        total as f64 / elapsed.as_secs_f64(),
-        elapsed.as_secs_f64() * 1000.0,
-    );
-
-    pool.close().await;
-}
-
 fn main() {
     println!("-- eventlog benchmarks --");
     let cpus = std::thread::available_parallelism()
@@ -915,45 +704,5 @@ fn main() {
     bench_stampede(100_000, 100, 16, 50);
     bench_stampede(500_000, 100, 16, 50);
 
-    let rt = tokio::runtime::Builder::new_multi_thread()
-        .worker_threads(cpus)
-        .enable_all()
-        .build()
-        .unwrap();
-
-    if std::env::var("DATABASE_URL").is_ok() {
-        println!("-- postgres comparison --");
-
-        event_counts.iter().for_each(|&n| {
-            producer_counts.iter().for_each(|&p| {
-                if n >= p {
-                    println!("-- postgres write: {n} events, {p} writers --",);
-                    rt.block_on(bench_pg_write_throughput(n, p));
-                }
-            });
-        });
-
-        event_counts.iter().for_each(|&n| {
-            [1usize, 4, 16, 32].iter().for_each(|&r| {
-                println!("-- postgres read: {n} events, {r} readers --",);
-                rt.block_on(bench_pg_read_throughput(n, r));
-            });
-        });
-
-        rt.block_on(async {
-            let url = std::env::var("DATABASE_URL").unwrap();
-            let pool = sqlx::postgres::PgPoolOptions::new()
-                .max_connections(5)
-                .connect(&url)
-                .await
-                .unwrap();
-            sqlx::query("DROP TABLE IF EXISTS bench_repo_seq")
-                .execute(&pool)
-                .await
-                .unwrap();
-            pool.close().await;
-        });
-    } else {
-        println!("set DATABASE_URL for postgres comparison");
-    }
+    let _ = (cpus, producer_counts);
 }

crates/tranquil-store/benches/metastore.rs

@@ -180,7 +180,7 @@ fn make_commit_input(
             prev_cid: None,
             ops: None,
             blobs: None,
-            blocks_cids: None,
+            blocks: None,
             prev_data_cid: None,
             rev: Some(make_rev(rev_n)),
         },
@@ -241,7 +241,7 @@ async fn seed_records(
                         prev_cid: None,
                         ops: None,
                         blobs: None,
-                        blocks_cids: None,
+                        blocks: None,
                         prev_data_cid: None,
                         rev: Some(make_rev(rev_n)),
                     },

crates/tranquil-store/benches/metastore_scale.rs

@@ -257,7 +257,7 @@ async fn seed_records_for_user(pool: &HandlerPool, user: &UserInfo, record_count
             prev_cid: None,
             ops: None,
             blobs: None,
-            blocks_cids: None,
+            blocks: None,
             prev_data_cid: None,
             rev: Some(make_rev(1)),
         },
@@ -348,7 +348,7 @@ async fn bench_single_user_commit(pool: &Arc<HandlerPool>, user: &UserInfo, ops:
                         prev_cid: None,
                         ops: None,
                         blobs: None,
-                        blocks_cids: None,
+                        blocks: None,
                         prev_data_cid: None,
                         rev: Some(make_rev(rev_n)),
                     },
@@ -424,7 +424,7 @@ async fn bench_multi_user_commit(
                                     prev_cid: None,
                                     ops: None,
                                     blobs: None,
-                                    blocks_cids: None,
+                                    blocks: None,
                                     prev_data_cid: None,
                                     rev: Some(make_rev(rev_n)),
                                 },

crates/tranquil-store/benches/profile_eventlog_reads.rs

@@ -0,0 +1,342 @@
+use std::path::Path;
+use std::sync::Arc;
+use std::time::Instant;
+
+use chrono::Utc;
+use tranquil_db_traits::{RepoEventType, SequencedEvent};
+use tranquil_types::Did;
+
+use tranquil_store::RealIO;
+use tranquil_store::eventlog::{
+    EventLog, EventLogConfig, EventSequence, decode_payload, to_sequenced_event,
+};
+
+fn make_did(index: usize) -> Did {
+    let suffix: String = format!("{index:024x}");
+    Did::new(format!("did:plc:{suffix}")).unwrap()
+}
+
+fn make_event(index: usize) -> SequencedEvent {
+    let ops_size = match index % 4 {
+        0 => 64,
+        1 => 256,
+        2 => 1024,
+        _ => 4096,
+    };
+
+    let ops_payload: String = (0..ops_size)
+        .map(|i| ((index.wrapping_mul(31).wrapping_add(i)) % 26 + 97) as u8 as char)
+        .collect();
+
+    SequencedEvent {
+        seq: tranquil_db_traits::SequenceNumber::from_raw(
+            i64::try_from(index + 1).expect("event index overflow"),
+        ),
+        did: make_did(index % 10_000),
+        created_at: Utc::now(),
+        event_type: match index % 4 {
+            0 => RepoEventType::Commit,
+            1 => RepoEventType::Identity,
+            2 => RepoEventType::Account,
+            _ => RepoEventType::Sync,
+        },
+        commit_cid: None,
+        prev_cid: None,
+        prev_data_cid: None,
+        ops: Some(serde_json::json!({ "data": ops_payload })),
+        blobs: None,
+        blocks: None,
+        handle: None,
+        active: None,
+        status: None,
+        rev: None,
+    }
+}
+
+fn open_eventlog(dir: &Path) -> EventLog<RealIO> {
+    let segments_dir = dir.join("segments");
+    std::fs::create_dir_all(&segments_dir).unwrap();
+    EventLog::open(
+        EventLogConfig {
+            segments_dir,
+            ..EventLogConfig::default()
+        },
+        RealIO::new(),
+    )
+    .unwrap()
+}
+
+struct PhaseTimings {
+    raw_read_ns: Vec<u64>,
+    decode_payload_ns: Vec<u64>,
+    ops_json_ns: Vec<u64>,
+    did_parse_ns: Vec<u64>,
+    full_conversion_ns: Vec<u64>,
+    total_get_events_ns: Vec<u64>,
+}
+
+impl PhaseTimings {
+    fn new(capacity: usize) -> Self {
+        Self {
+            raw_read_ns: Vec::with_capacity(capacity),
+            decode_payload_ns: Vec::with_capacity(capacity),
+            ops_json_ns: Vec::with_capacity(capacity),
+            did_parse_ns: Vec::with_capacity(capacity),
+            full_conversion_ns: Vec::with_capacity(capacity),
+            total_get_events_ns: Vec::with_capacity(capacity),
+        }
+    }
+}
+
+fn percentile(sorted: &[u64], pct: f64) -> u64 {
+    if sorted.is_empty() {
+        return 0;
+    }
+    let idx = ((sorted.len() - 1) as f64 * pct / 100.0) as usize;
+    sorted[idx]
+}
+
+fn report_phase(name: &str, values_ns: &mut [u64], event_count: usize) {
+    values_ns.sort();
+    let total: u64 = values_ns.iter().sum();
+    let per_event_ns = total as f64 / event_count as f64;
+    let p50 = percentile(values_ns, 50.0);
+    let p99 = percentile(values_ns, 99.0);
+    println!(
+        "{name}: {:.2}ms total, {per_event_ns:.0}ns/event, p50 {p50}ns, p99 {p99}ns",
+        total as f64 / 1_000_000.0,
+    );
+}
+
+fn profile_read_phases(event_count: usize, readers: usize) {
+    println!("-- read path profile: {event_count} events, {readers} readers --");
+
+    let dir = tempfile::TempDir::new().unwrap();
+    let log = Arc::new(open_eventlog(dir.path()));
+
+    let events: Vec<SequencedEvent> = (0..event_count).map(make_event).collect();
+    events.iter().enumerate().for_each(|(i, event)| {
+        log.append_event(&make_did(i % 10_000), RepoEventType::Commit, event)
+            .unwrap();
+    });
+    log.sync().unwrap();
+
+    println!("seeded {event_count} events");
+
+    let batch_size = 4096usize;
+    let iterations = 3;
+
+    (0..iterations).for_each(|iter| {
+        println!("-- iteration {}/{iterations} --", iter + 1);
+
+        let handles: Vec<_> = (0..readers)
+            .map(|_| {
+                let log = Arc::clone(&log);
+                std::thread::spawn(move || {
+                    let reader = log.reader();
+                    let mut timings = PhaseTimings::new(event_count / batch_size + 1);
+                    let mut total_events = 0usize;
+
+                    let mut cursor = EventSequence::BEFORE_ALL;
+                    std::iter::from_fn(|| {
+                        let t_total = Instant::now();
+
+                        let t_raw = Instant::now();
+                        let raw_events = reader.read_events_from(cursor, batch_size).unwrap();
+                        let raw_read_elapsed = t_raw.elapsed();
+
+                        if raw_events.is_empty() {
+                            return None;
+                        }
+
+                        let mut batch_decode_ns = 0u64;
+                        let mut batch_ops_ns = 0u64;
+                        let mut batch_did_ns = 0u64;
+                        let mut batch_conversion_ns = 0u64;
+
+                        raw_events.iter().for_each(|raw| {
+                            let t_decode = Instant::now();
+                            let payload = decode_payload(&raw.payload).unwrap();
+                            batch_decode_ns += t_decode.elapsed().as_nanos() as u64;
+
+                            let t_ops = Instant::now();
+                            let _ops: Option<serde_json::Value> = payload
+                                .ops
+                                .as_ref()
+                                .map(|bytes| serde_ipld_dagcbor::from_slice(bytes).unwrap());
+                            batch_ops_ns += t_ops.elapsed().as_nanos() as u64;
+
+                            let t_did = Instant::now();
+                            let _did = Did::new(&payload.did).unwrap();
+                            batch_did_ns += t_did.elapsed().as_nanos() as u64;
+
+                            let t_conversion = Instant::now();
+                            let payload2 = decode_payload(&raw.payload).unwrap();
+                            let _event = to_sequenced_event(raw, &payload2).unwrap();
+                            batch_conversion_ns += t_conversion.elapsed().as_nanos() as u64;
+                        });
+
+                        let batch_events = raw_events.len();
+                        cursor = EventSequence::new(
+                            u64::try_from(raw_events.last().unwrap().seq.as_i64()).unwrap(),
+                        );
+                        total_events += batch_events;
+
+                        timings.raw_read_ns.push(raw_read_elapsed.as_nanos() as u64);
+                        timings.decode_payload_ns.push(batch_decode_ns);
+                        timings.ops_json_ns.push(batch_ops_ns);
+                        timings.did_parse_ns.push(batch_did_ns);
+                        timings.full_conversion_ns.push(batch_conversion_ns);
+                        timings
+                            .total_get_events_ns
+                            .push(t_total.elapsed().as_nanos() as u64);
+
+                        Some(())
+                    })
+                    .count();
+
+                    (timings, total_events)
+                })
+            })
+            .collect();
+
+        let results: Vec<_> = handles.into_iter().map(|h| h.join().unwrap()).collect();
+
+        let total_events: usize = results.iter().map(|(_, count)| count).sum();
+
+        let mut agg = PhaseTimings::new(0);
+        results.iter().for_each(|(t, _)| {
+            agg.raw_read_ns.extend_from_slice(&t.raw_read_ns);
+            agg.decode_payload_ns
+                .extend_from_slice(&t.decode_payload_ns);
+            agg.ops_json_ns.extend_from_slice(&t.ops_json_ns);
+            agg.did_parse_ns.extend_from_slice(&t.did_parse_ns);
+            agg.full_conversion_ns
+                .extend_from_slice(&t.full_conversion_ns);
+            agg.total_get_events_ns
+                .extend_from_slice(&t.total_get_events_ns);
+        });
+
+        println!("{total_events} events across {readers} readers");
+        report_phase("raw_read", &mut agg.raw_read_ns, total_events);
+        report_phase("full conversion", &mut agg.full_conversion_ns, total_events);
+        report_phase(
+            "postcard decode, isolated",
+            &mut agg.decode_payload_ns,
+            total_events,
+        );
+        report_phase(
+            "DAG-CBOR ops parse, isolated",
+            &mut agg.ops_json_ns,
+            total_events,
+        );
+        report_phase("DID parse, isolated", &mut agg.did_parse_ns, total_events);
+        report_phase(
+            "end-to-end total",
+            &mut agg.total_get_events_ns,
+            total_events,
+        );
+
+        let raw_total: u64 = agg.raw_read_ns.iter().sum();
+        let conversion_total: u64 = agg.full_conversion_ns.iter().sum();
+        let decode_total: u64 = agg.decode_payload_ns.iter().sum();
+        let ops_total: u64 = agg.ops_json_ns.iter().sum();
+        let did_total: u64 = agg.did_parse_ns.iter().sum();
+
+        let pipeline_total = raw_total + conversion_total;
+        let pct = |v: u64| v as f64 / pipeline_total as f64 * 100.0;
+        let conversion_other =
+            conversion_total.saturating_sub(decode_total + ops_total + did_total);
+        println!(
+            "breakdown: raw_read {:.1}%, postcard {:.1}%, dagcbor_ops {:.1}%, did {:.1}%, rest {:.1}%",
+            pct(raw_total),
+            pct(decode_total),
+            pct(ops_total),
+            pct(did_total),
+            pct(conversion_other),
+        );
+    });
+
+    let _ = log.shutdown();
+}
+
+fn profile_decode_phases(event_count: usize) {
+    println!("-- decode phase isolation: {event_count} events --");
+
+    let dir = tempfile::TempDir::new().unwrap();
+    let log = open_eventlog(dir.path());
+
+    let events: Vec<SequencedEvent> = (0..event_count).map(make_event).collect();
+    events.iter().enumerate().for_each(|(i, event)| {
+        log.append_event(&make_did(i % 10_000), RepoEventType::Commit, event)
+            .unwrap();
+    });
+    log.sync().unwrap();
+
+    let reader = log.reader();
+    let raw_events = reader
+        .read_events_from(EventSequence::BEFORE_ALL, event_count)
+        .unwrap();
+
+    println!("{} raw events pre-loaded", raw_events.len());
+
+    (0..5).for_each(|_| {
+        let t_decode = Instant::now();
+        let payloads: Vec<_> = raw_events
+            .iter()
+            .map(|raw| decode_payload(&raw.payload).unwrap())
+            .collect();
+        let decode_elapsed = t_decode.elapsed();
+
+        let t_convert = Instant::now();
+        let _events: Vec<_> = raw_events
+            .iter()
+            .zip(payloads.iter())
+            .map(|(raw, payload)| to_sequenced_event(raw, payload).unwrap())
+            .collect();
+        let convert_elapsed = t_convert.elapsed();
+
+        let t_ops_only = Instant::now();
+        let _: Vec<_> = payloads
+            .iter()
+            .map(|p| {
+                p.ops.as_ref().map(|bytes| {
+                    serde_ipld_dagcbor::from_slice::<serde_json::Value>(bytes).unwrap()
+                })
+            })
+            .collect();
+        let ops_elapsed = t_ops_only.elapsed();
+
+        let n = raw_events.len() as f64;
+        println!(
+            "postcard {:.0}ns/evt, to_sequenced_event {:.0}ns/evt, dagcbor_ops {:.0}ns/evt",
+            decode_elapsed.as_nanos() as f64 / n,
+            convert_elapsed.as_nanos() as f64 / n,
+            ops_elapsed.as_nanos() as f64 / n,
+        );
+    });
+
+    let _ = log.shutdown();
+}
+
+fn main() {
+    println!("-- eventlog read path profiler --");
+    let cpus = std::thread::available_parallelism()
+        .map(|n| n.get())
+        .unwrap_or(8);
+    println!("available parallelism: {cpus}");
+
+    let event_count = std::env::var("PROFILE_EVENTS")
+        .ok()
+        .and_then(|s| s.replace('_', "").parse().ok())
+        .unwrap_or(100_000usize);
+
+    let reader_count = std::env::var("PROFILE_READERS")
+        .ok()
+        .and_then(|s| s.parse().ok())
+        .unwrap_or(4usize);
+
+    profile_read_phases(event_count, reader_count);
+    println!();
+    profile_decode_phases(event_count);
+}

crates/tranquil-store/benches/profile_reads.rs

@@ -128,7 +128,7 @@ async fn seed_records(pool: &Arc<HandlerPool>, users: &[UserInfo], records_per_u
                                     prev_cid: None,
                                     ops: None,
                                     blobs: None,
-                                    blocks_cids: None,
+                                    blocks: None,
                                     prev_data_cid: None,
                                     rev: Some("rev0000000001".to_string()),
                                 },

crates/tranquil-store/benches/recovery.rs

@@ -0,0 +1,268 @@
+use std::path::Path;
+use std::time::Instant;
+
+use tranquil_store::blockstore::hash_index::BlockIndex;
+use tranquil_store::blockstore::{
+    CidBytes, DEFAULT_MAX_FILE_SIZE, DataFileId, DataFileWriter, HintFileWriter, hint_file_path,
+    scan_hints_to_memory,
+};
+use tranquil_store::{OpenOptions, RealIO, StorageIO};
+
+#[global_allocator]
+static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
+
+fn test_cid(seed: u32) -> CidBytes {
+    let le = seed.to_le_bytes();
+    std::array::from_fn(|i| match i {
+        0 => 0x01,
+        1 => 0x71,
+        2 => 0x12,
+        3 => 0x20,
+        4..8 => le[i - 4],
+        _ => (seed as u8).wrapping_add(i as u8),
+    })
+}
+
+fn block_data(seed: u32) -> Vec<u8> {
+    let tag = seed.to_le_bytes();
+    std::iter::repeat(tag).flatten().take(256).collect()
+}
+
+struct DirectSeeder<'a> {
+    io: &'a RealIO,
+    data_dir: &'a Path,
+    file_id: DataFileId,
+    data_writer: DataFileWriter<'a, RealIO>,
+    hint_writer: HintFileWriter<'a, RealIO>,
+    blocks_in_file: u64,
+}
+
+impl<'a> DirectSeeder<'a> {
+    fn new(io: &'a RealIO, data_dir: &'a Path) -> Self {
+        std::fs::create_dir_all(data_dir).unwrap();
+        let file_id = DataFileId::new(0);
+
+        let data_fd = io
+            .open(
+                &data_dir.join(format!("{file_id}.tqb")),
+                OpenOptions::read_write(),
+            )
+            .unwrap();
+        let data_writer = DataFileWriter::new(io, data_fd, file_id).unwrap();
+
+        let hint_fd = io
+            .open(
+                &hint_file_path(data_dir, file_id),
+                OpenOptions::read_write(),
+            )
+            .unwrap();
+        let hint_writer = HintFileWriter::new(io, hint_fd);
+
+        Self {
+            io,
+            data_dir,
+            file_id,
+            data_writer,
+            hint_writer,
+            blocks_in_file: 0,
+        }
+    }
+
+    fn rotate(&mut self) {
+        self.data_writer.sync().unwrap();
+        self.hint_writer.sync().unwrap();
+
+        self.file_id = self.file_id.next();
+
+        let data_fd = self
+            .io
+            .open(
+                &self.data_dir.join(format!("{}.tqb", self.file_id)),
+                OpenOptions::read_write(),
+            )
+            .unwrap();
+        self.data_writer = DataFileWriter::new(self.io, data_fd, self.file_id).unwrap();
+
+        let hint_fd = self
+            .io
+            .open(
+                &hint_file_path(self.data_dir, self.file_id),
+                OpenOptions::read_write(),
+            )
+            .unwrap();
+        self.hint_writer = HintFileWriter::new(self.io, hint_fd);
+        self.blocks_in_file = 0;
+    }
+
+    fn append(&mut self, cid: &CidBytes, data: &[u8]) {
+        if self.data_writer.position().raw() > DEFAULT_MAX_FILE_SIZE {
+            self.rotate();
+        }
+
+        let loc = self.data_writer.append_block(cid, data).unwrap();
+        self.hint_writer.append_hint(cid, &loc).unwrap();
+        self.blocks_in_file += 1;
+
+        if self.blocks_in_file.is_multiple_of(10_000) {
+            self.data_writer.sync().unwrap();
+            self.hint_writer.sync().unwrap();
+        }
+    }
+
+    fn finish(&mut self) {
+        self.data_writer.sync().unwrap();
+        self.hint_writer.sync().unwrap();
+        self.io.sync_dir(self.data_dir).unwrap();
+    }
+}
+
+fn seed_blocks_direct(data_dir: &Path, count: u32) {
+    let io = RealIO::new();
+    let mut seeder = DirectSeeder::new(&io, data_dir);
+    (0..count).for_each(|i| {
+        let cid = test_cid(i);
+        let data = block_data(i);
+        seeder.append(&cid, &data);
+    });
+    seeder.finish();
+}
+
+fn read_rss_mb() -> f64 {
+    std::fs::read_to_string("/proc/self/status")
+        .ok()
+        .and_then(|s| {
+            s.lines().find(|l| l.starts_with("VmRSS:")).and_then(|l| {
+                l.split_whitespace()
+                    .nth(1)
+                    .and_then(|v| v.parse::<f64>().ok())
+            })
+        })
+        .map(|kb| kb / 1024.0)
+        .unwrap_or(0.0)
+}
+
+fn bench_hint_scan_only(data_dir: &Path, block_count: u32) {
+    let io = RealIO::new();
+
+    let rss_before = read_rss_mb();
+    let start = Instant::now();
+    let (hint_index, _cursor) = scan_hints_to_memory(&io, data_dir).unwrap();
+    let elapsed = start.elapsed();
+    let rss_after = read_rss_mb();
+
+    let entry_count = hint_index.len();
+    let rss_delta = rss_after - rss_before;
+    let bytes_per_entry = match entry_count {
+        0 => 0.0,
+        n => (rss_delta * 1024.0 * 1024.0) / n as f64,
+    };
+
+    println!(
+        "hint scan to memory ({block_count} blocks): {:.3}s ({:.0} blocks/sec)",
+        elapsed.as_secs_f64(),
+        block_count as f64 / elapsed.as_secs_f64(),
+    );
+    println!(
+        "  entries: {entry_count}, RSS: {rss_before:.1}MB -> {rss_after:.1}MB (delta: {rss_delta:.1}MB, {bytes_per_entry:.0} bytes/entry)"
+    );
+
+    drop(hint_index);
+    let rss_after_drop = read_rss_mb();
+    println!("  RSS after drop: {rss_after_drop:.1}MB");
+}
+
+fn bench_hash_table_rebuild_from_hints(data_dir: &Path, index_dir: &Path, block_count: u32) {
+    let io = RealIO::new();
+    let index = BlockIndex::open(index_dir).unwrap();
+
+    let rss_before = read_rss_mb();
+    let start = Instant::now();
+    index.rebuild_from_hints(&io, data_dir).unwrap();
+    let elapsed = start.elapsed();
+    let rss_after = read_rss_mb();
+
+    println!(
+        "hash table rebuild from hints ({block_count} blocks): {:.3}s ({:.0} blocks/sec)",
+        elapsed.as_secs_f64(),
+        block_count as f64 / elapsed.as_secs_f64(),
+    );
+    println!(
+        "  RSS: {rss_before:.1}MB -> {rss_after:.1}MB (delta: {:.1}MB)",
+        rss_after - rss_before,
+    );
+}
+
+fn bench_hash_table_rebuild_from_data_files(data_dir: &Path, index_dir: &Path, block_count: u32) {
+    let io = RealIO::new();
+    let index = BlockIndex::open(index_dir).unwrap();
+
+    let start = Instant::now();
+    index.rebuild_from_data_files(&io, data_dir).unwrap();
+    let elapsed = start.elapsed();
+
+    println!(
+        "hash table rebuild from data files ({block_count} blocks): {:.3}s ({:.0} blocks/sec)",
+        elapsed.as_secs_f64(),
+        block_count as f64 / elapsed.as_secs_f64(),
+    );
+}
+
+fn nuke_index(index_dir: &Path) {
+    if index_dir.exists() {
+        std::fs::remove_dir_all(index_dir).unwrap();
+    }
+    std::fs::create_dir_all(index_dir).unwrap();
+}
+
+fn run_scale(block_count: u32) {
+    let label = match block_count {
+        n if n >= 10_000_000 => format!("{}M blocks", n / 1_000_000),
+        n if n >= 1_000_000 => format!("{}M blocks", n / 1_000_000),
+        n => format!("{}K blocks", n / 1_000),
+    };
+    println!("\n-- {label} --");
+
+    let dir = tempfile::TempDir::new().unwrap();
+    let data_dir = dir.path().join("data");
+    let index_dir = dir.path().join("index");
+
+    println!("seeding {block_count} blocks, direct without index...");
+    let seed_start = Instant::now();
+    seed_blocks_direct(&data_dir, block_count);
+    println!(
+        "  blocks seeded in {:.1}s",
+        seed_start.elapsed().as_secs_f64()
+    );
+
+    println!("\n-- hint scan to memory --");
+    bench_hint_scan_only(&data_dir, block_count);
+
+    println!("\n-- hash table rebuild from hints --");
+    nuke_index(&index_dir);
+    bench_hash_table_rebuild_from_hints(&data_dir, &index_dir, block_count);
+
+    println!("\n-- hash table rebuild from data files --");
+    nuke_index(&index_dir);
+    bench_hash_table_rebuild_from_data_files(&data_dir, &index_dir, block_count);
+}
+
+fn parse_scales(input: &str) -> Vec<u32> {
+    input
+        .split(';')
+        .map(|s| s.trim().replace('_', "").parse::<u32>().unwrap())
+        .collect()
+}
+
+fn main() {
+    let scales = parse_scales(
+        &std::env::var("BENCH_RECOVERY_SCALES")
+            .unwrap_or_else(|_| "100_000; 1_000_000; 10_000_000".into()),
+    );
+
+    println!("recovery performance benchmark, hash table index :3");
+    println!("scales: {scales:?}");
+
+    scales.iter().for_each(|&blocks| {
+        run_scale(blocks);
+    });
+}

crates/tranquil-store/src/archival.rs

@@ -0,0 +1,547 @@
+use std::io;
+use std::path::{Path, PathBuf};
+
+use serde::{Deserialize, Serialize};
+use tracing::{debug, info, warn};
+
+use crate::eventlog::{SEGMENT_FILE_EXTENSION, SegmentId, parse_segment_id, segment_path};
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ArchivalState {
+    pub last_archived_segment: Option<SegmentId>,
+}
+
+impl ArchivalState {
+    fn empty() -> Self {
+        Self {
+            last_archived_segment: None,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct ArchivalPassResult {
+    pub segments_archived: u32,
+    pub bytes_archived: u64,
+}
+
+pub trait ArchivalDestination: Send + Sync {
+    fn store_segment(&self, segment_id: SegmentId, data: &[u8]) -> io::Result<()>;
+}
+
+pub struct LocalArchivalDestination {
+    dest_dir: PathBuf,
+}
+
+impl LocalArchivalDestination {
+    pub fn new(dest_dir: PathBuf) -> io::Result<Self> {
+        std::fs::create_dir_all(&dest_dir)?;
+        Ok(Self { dest_dir })
+    }
+}
+
+impl ArchivalDestination for LocalArchivalDestination {
+    fn store_segment(&self, segment_id: SegmentId, data: &[u8]) -> io::Result<()> {
+        let dest_path = segment_path(&self.dest_dir, segment_id);
+        let tmp_path = dest_path.with_extension(format!("{SEGMENT_FILE_EXTENSION}.tmp"));
+
+        std::fs::write(&tmp_path, data)?;
+
+        let f = std::fs::File::open(&tmp_path)?;
+        f.sync_all()?;
+        drop(f);
+
+        std::fs::rename(&tmp_path, &dest_path)?;
+
+        sync_dir(&self.dest_dir)?;
+
+        Ok(())
+    }
+}
+
+fn sync_dir(dir: &Path) -> io::Result<()> {
+    let d = std::fs::File::open(dir)?;
+    d.sync_all()
+}
+
+fn list_segment_files(segments_dir: &Path) -> io::Result<Vec<SegmentId>> {
+    let entries = match std::fs::read_dir(segments_dir) {
+        Ok(entries) => entries,
+        Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()),
+        Err(e) => return Err(e),
+    };
+
+    let mut ids: Vec<SegmentId> = entries
+        .filter_map(|entry| parse_segment_id(&entry.ok()?.path()))
+        .collect();
+    ids.sort();
+    Ok(ids)
+}
+
+pub struct ArchivalSidecar {
+    path: PathBuf,
+}
+
+impl ArchivalSidecar {
+    pub fn new(path: PathBuf) -> Self {
+        Self { path }
+    }
+
+    pub fn load(&self) -> io::Result<ArchivalState> {
+        match std::fs::read(&self.path) {
+            Ok(data) => serde_json::from_slice(&data)
+                .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)),
+            Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(ArchivalState::empty()),
+            Err(e) => Err(e),
+        }
+    }
+
+    pub fn save(&self, state: &ArchivalState) -> io::Result<()> {
+        let json =
+            serde_json::to_vec(state).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
+
+        let tmp_path = self.path.with_extension("tmp");
+        std::fs::write(&tmp_path, &json)?;
+        let f = std::fs::File::open(&tmp_path)?;
+        f.sync_all()?;
+        drop(f);
+        std::fs::rename(&tmp_path, &self.path)?;
+
+        self.path.parent().map(sync_dir).transpose()?;
+
+        Ok(())
+    }
+}
+
+pub struct ContinuousArchiver {
+    segments_dir: PathBuf,
+    sidecar: ArchivalSidecar,
+    destination: Box<dyn ArchivalDestination>,
+}
+
+impl ContinuousArchiver {
+    pub fn new(
+        segments_dir: PathBuf,
+        sidecar_path: PathBuf,
+        destination: Box<dyn ArchivalDestination>,
+    ) -> Self {
+        Self {
+            segments_dir,
+            sidecar: ArchivalSidecar::new(sidecar_path),
+            destination,
+        }
+    }
+
+    pub fn run_pass(&self) -> io::Result<ArchivalPassResult> {
+        let state = self.sidecar.load()?;
+
+        let all_segments = list_segment_files(&self.segments_dir)?;
+
+        let sealed_segments = match all_segments.len() {
+            0 | 1 => Vec::new(),
+            n => all_segments[..n - 1].to_vec(),
+        };
+
+        let new_segments: Vec<SegmentId> = match state.last_archived_segment {
+            Some(last) => sealed_segments
+                .into_iter()
+                .filter(|&id| id > last)
+                .collect(),
+            None => sealed_segments,
+        };
+
+        if new_segments.is_empty() {
+            debug!("no new sealed segments to archive");
+            return Ok(ArchivalPassResult {
+                segments_archived: 0,
+                bytes_archived: 0,
+            });
+        }
+
+        let mut segments_archived = 0u32;
+        let mut bytes_archived = 0u64;
+
+        let result = new_segments.iter().try_for_each(|&seg_id| {
+            let path = segment_path(&self.segments_dir, seg_id);
+            let data = std::fs::read(&path)?;
+            let size = data.len() as u64;
+
+            self.destination.store_segment(seg_id, &data)?;
+
+            self.sidecar.save(&ArchivalState {
+                last_archived_segment: Some(seg_id),
+            })?;
+
+            segments_archived = segments_archived.saturating_add(1);
+            bytes_archived = bytes_archived.saturating_add(size);
+
+            info!(
+                segment_id = %seg_id,
+                size_bytes = size,
+                "archived sealed segment"
+            );
+
+            Ok::<(), io::Error>(())
+        });
+
+        match result {
+            Ok(()) => {}
+            Err(e) => {
+                warn!(
+                    segments_archived,
+                    bytes_archived,
+                    error = %e,
+                    "archival pass interrupted after partial progress"
+                );
+                return Err(e);
+            }
+        }
+
+        Ok(ArchivalPassResult {
+            segments_archived,
+            bytes_archived,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use std::sync::{Arc, Mutex};
+
+    type ArchivedSegments = Arc<Mutex<Vec<(SegmentId, Vec<u8>)>>>;
+
+    #[derive(Clone)]
+    struct CollectingDestination {
+        stored: ArchivedSegments,
+    }
+
+    impl CollectingDestination {
+        fn new() -> Self {
+            Self {
+                stored: Arc::new(Mutex::new(Vec::new())),
+            }
+        }
+
+        fn stored_ids(&self) -> Vec<SegmentId> {
+            self.stored
+                .lock()
+                .unwrap()
+                .iter()
+                .map(|(id, _)| *id)
+                .collect()
+        }
+    }
+
+    impl ArchivalDestination for CollectingDestination {
+        fn store_segment(&self, segment_id: SegmentId, data: &[u8]) -> io::Result<()> {
+            self.stored
+                .lock()
+                .unwrap()
+                .push((segment_id, data.to_vec()));
+            Ok(())
+        }
+    }
+
+    fn create_segment_file(dir: &Path, id: u32, content: &[u8]) {
+        let path = dir.join(format!("{:08}.{SEGMENT_FILE_EXTENSION}", id));
+        std::fs::write(path, content).unwrap();
+    }
+
+    #[test]
+    fn sidecar_round_trip() {
+        let dir = tempfile::tempdir().unwrap();
+        let sidecar = ArchivalSidecar::new(dir.path().join("archival.state"));
+
+        let state = sidecar.load().unwrap();
+        assert!(state.last_archived_segment.is_none());
+
+        let updated = ArchivalState {
+            last_archived_segment: Some(SegmentId::new(42)),
+        };
+        sidecar.save(&updated).unwrap();
+
+        let loaded = sidecar.load().unwrap();
+        assert_eq!(loaded.last_archived_segment, Some(SegmentId::new(42)));
+    }
+
+    #[test]
+    fn sidecar_missing_file_returns_empty() {
+        let dir = tempfile::tempdir().unwrap();
+        let sidecar = ArchivalSidecar::new(dir.path().join("nonexistent.state"));
+        let state = sidecar.load().unwrap();
+        assert!(state.last_archived_segment.is_none());
+    }
+
+    #[test]
+    fn list_segment_files_sorts_ascending() {
+        let dir = tempfile::tempdir().unwrap();
+        create_segment_file(dir.path(), 5, b"e");
+        create_segment_file(dir.path(), 1, b"a");
+        create_segment_file(dir.path(), 3, b"c");
+        std::fs::write(dir.path().join("notes.txt"), b"ignored").unwrap();
+
+        let ids = list_segment_files(dir.path()).unwrap();
+        assert_eq!(
+            ids,
+            vec![SegmentId::new(1), SegmentId::new(3), SegmentId::new(5)]
+        );
+    }
+
+    #[test]
+    fn list_segment_files_empty_dir() {
+        let dir = tempfile::tempdir().unwrap();
+        let ids = list_segment_files(dir.path()).unwrap();
+        assert!(ids.is_empty());
+    }
+
+    #[test]
+    fn list_segment_files_missing_dir() {
+        let ids = list_segment_files(Path::new("/nonexistent/dir")).unwrap();
+        assert!(ids.is_empty());
+    }
+
+    #[test]
+    fn no_segments_no_archival() {
+        let dir = tempfile::tempdir().unwrap();
+        let seg_dir = dir.path().join("segments");
+        std::fs::create_dir_all(&seg_dir).unwrap();
+
+        let dest = CollectingDestination::new();
+        let dest_check = dest.clone();
+
+        let archiver =
+            ContinuousArchiver::new(seg_dir, dir.path().join("archival.state"), Box::new(dest));
+
+        let result = archiver.run_pass().unwrap();
+        assert_eq!(result.segments_archived, 0);
+        assert_eq!(result.bytes_archived, 0);
+        assert!(dest_check.stored_ids().is_empty());
+    }
+
+    #[test]
+    fn single_active_segment_not_archived() {
+        let dir = tempfile::tempdir().unwrap();
+        let seg_dir = dir.path().join("segments");
+        std::fs::create_dir_all(&seg_dir).unwrap();
+        create_segment_file(&seg_dir, 0, b"active segment data");
+
+        let dest = CollectingDestination::new();
+        let dest_check = dest.clone();
+
+        let archiver =
+            ContinuousArchiver::new(seg_dir, dir.path().join("archival.state"), Box::new(dest));
+
+        let result = archiver.run_pass().unwrap();
+        assert_eq!(result.segments_archived, 0);
+        assert!(dest_check.stored_ids().is_empty());
+    }
+
+    #[test]
+    fn archives_sealed_segments() {
+        let dir = tempfile::tempdir().unwrap();
+        let seg_dir = dir.path().join("segments");
+        std::fs::create_dir_all(&seg_dir).unwrap();
+        create_segment_file(&seg_dir, 0, b"sealed-0");
+        create_segment_file(&seg_dir, 1, b"sealed-1");
+        create_segment_file(&seg_dir, 2, b"active");
+
+        let dest = CollectingDestination::new();
+        let dest_check = dest.clone();
+
+        let archiver =
+            ContinuousArchiver::new(seg_dir, dir.path().join("archival.state"), Box::new(dest));
+
+        let result = archiver.run_pass().unwrap();
+        assert_eq!(result.segments_archived, 2);
+        assert_eq!(result.bytes_archived, 16);
+
+        let stored = dest_check.stored_ids();
+        assert_eq!(stored, vec![SegmentId::new(0), SegmentId::new(1)]);
+    }
+
+    #[test]
+    fn incremental_archival_skips_already_archived() {
+        let dir = tempfile::tempdir().unwrap();
+        let seg_dir = dir.path().join("segments");
+        std::fs::create_dir_all(&seg_dir).unwrap();
+        create_segment_file(&seg_dir, 0, b"sealed-0");
+        create_segment_file(&seg_dir, 1, b"sealed-1");
+        create_segment_file(&seg_dir, 2, b"sealed-2");
+        create_segment_file(&seg_dir, 3, b"active");
+
+        let sidecar_path = dir.path().join("archival.state");
+        ArchivalSidecar::new(sidecar_path.clone())
+            .save(&ArchivalState {
+                last_archived_segment: Some(SegmentId::new(0)),
+            })
+            .unwrap();
+
+        let dest = CollectingDestination::new();
+        let dest_check = dest.clone();
+
+        let archiver = ContinuousArchiver::new(seg_dir, sidecar_path.clone(), Box::new(dest));
+
+        let result = archiver.run_pass().unwrap();
+        assert_eq!(result.segments_archived, 2);
+
+        let stored = dest_check.stored_ids();
+        assert_eq!(stored, vec![SegmentId::new(1), SegmentId::new(2)]);
+
+        let final_state = ArchivalSidecar::new(sidecar_path).load().unwrap();
+        assert_eq!(final_state.last_archived_segment, Some(SegmentId::new(2)));
+    }
+
+    #[test]
+    fn sidecar_updated_per_segment_for_crash_safety() {
+        let dir = tempfile::tempdir().unwrap();
+        let seg_dir = dir.path().join("segments");
+        std::fs::create_dir_all(&seg_dir).unwrap();
+        create_segment_file(&seg_dir, 0, b"sealed-0");
+        create_segment_file(&seg_dir, 1, b"sealed-1");
+        create_segment_file(&seg_dir, 2, b"active");
+
+        struct FailOnSecondDestination {
+            call_count: Mutex<u32>,
+        }
+        impl ArchivalDestination for FailOnSecondDestination {
+            fn store_segment(&self, _id: SegmentId, _data: &[u8]) -> io::Result<()> {
+                let mut count = self.call_count.lock().unwrap();
+                *count += 1;
+                match *count {
+                    1 => Ok(()),
+                    _ => Err(io::Error::other("simulated failure")),
+                }
+            }
+        }
+
+        let sidecar_path = dir.path().join("archival.state");
+
+        let archiver = ContinuousArchiver::new(
+            seg_dir,
+            sidecar_path.clone(),
+            Box::new(FailOnSecondDestination {
+                call_count: Mutex::new(0),
+            }),
+        );
+
+        let err = archiver.run_pass().unwrap_err();
+        assert_eq!(err.kind(), io::ErrorKind::Other);
+
+        let state = ArchivalSidecar::new(sidecar_path).load().unwrap();
+        assert_eq!(state.last_archived_segment, Some(SegmentId::new(0)));
+    }
+
+    #[test]
+    fn idempotent_rerun_after_full_archival() {
+        let dir = tempfile::tempdir().unwrap();
+        let seg_dir = dir.path().join("segments");
+        std::fs::create_dir_all(&seg_dir).unwrap();
+        create_segment_file(&seg_dir, 0, b"sealed-0");
+        create_segment_file(&seg_dir, 1, b"sealed-1");
+        create_segment_file(&seg_dir, 2, b"active");
+
+        let sidecar_path = dir.path().join("archival.state");
+
+        let dest1 = CollectingDestination::new();
+        let dest1_check = dest1.clone();
+        let archiver1 =
+            ContinuousArchiver::new(seg_dir.clone(), sidecar_path.clone(), Box::new(dest1));
+        archiver1.run_pass().unwrap();
+        assert_eq!(dest1_check.stored_ids().len(), 2);
+
+        let dest2 = CollectingDestination::new();
+        let dest2_check = dest2.clone();
+        let archiver2 = ContinuousArchiver::new(seg_dir, sidecar_path, Box::new(dest2));
+        let result = archiver2.run_pass().unwrap();
+        assert_eq!(result.segments_archived, 0);
+        assert!(dest2_check.stored_ids().is_empty());
+    }
+
+    #[test]
+    fn new_segments_after_initial_archival() {
+        let dir = tempfile::tempdir().unwrap();
+        let seg_dir = dir.path().join("segments");
+        std::fs::create_dir_all(&seg_dir).unwrap();
+        create_segment_file(&seg_dir, 0, b"sealed-0");
+        create_segment_file(&seg_dir, 1, b"active");
+
+        let sidecar_path = dir.path().join("archival.state");
+
+        let dest1 = CollectingDestination::new();
+        let archiver1 =
+            ContinuousArchiver::new(seg_dir.clone(), sidecar_path.clone(), Box::new(dest1));
+        let r1 = archiver1.run_pass().unwrap();
+        assert_eq!(r1.segments_archived, 1);
+
+        create_segment_file(&seg_dir, 2, b"new-active");
+
+        let dest2 = CollectingDestination::new();
+        let dest2_check = dest2.clone();
+        let archiver2 = ContinuousArchiver::new(seg_dir, sidecar_path, Box::new(dest2));
+        let r2 = archiver2.run_pass().unwrap();
+        assert_eq!(r2.segments_archived, 1);
+        assert_eq!(dest2_check.stored_ids(), vec![SegmentId::new(1)]);
+    }
+
+    #[test]
+    fn local_destination_writes_files() {
+        let dir = tempfile::tempdir().unwrap();
+        let dest_dir = dir.path().join("archive");
+
+        let dest = LocalArchivalDestination::new(dest_dir.clone()).unwrap();
+
+        let payload = b"segment data here";
+        dest.store_segment(SegmentId::new(5), payload).unwrap();
+
+        let written =
+            std::fs::read(dest_dir.join(format!("00000005.{SEGMENT_FILE_EXTENSION}"))).unwrap();
+        assert_eq!(written, payload);
+    }
+
+    #[test]
+    fn local_destination_atomic_overwrite() {
+        let dir = tempfile::tempdir().unwrap();
+        let dest_dir = dir.path().join("archive");
+
+        let dest = LocalArchivalDestination::new(dest_dir.clone()).unwrap();
+
+        dest.store_segment(SegmentId::new(1), b"first").unwrap();
+        dest.store_segment(SegmentId::new(1), b"second").unwrap();
+
+        let written =
+            std::fs::read(dest_dir.join(format!("00000001.{SEGMENT_FILE_EXTENSION}"))).unwrap();
+        assert_eq!(written, b"second");
+
+        assert!(
+            !dest_dir
+                .join(format!("00000001.{SEGMENT_FILE_EXTENSION}.tmp"))
+                .exists()
+        );
+    }
+
+    #[test]
+    fn archived_data_matches_source() {
+        let dir = tempfile::tempdir().unwrap();
+        let seg_dir = dir.path().join("segments");
+        std::fs::create_dir_all(&seg_dir).unwrap();
+
+        let content_0 = b"sealed segment zero content with some bulk data";
+        let content_1 = b"sealed segment one with different content";
+        create_segment_file(&seg_dir, 0, content_0);
+        create_segment_file(&seg_dir, 1, content_1);
+        create_segment_file(&seg_dir, 2, b"active");
+
+        let dest = CollectingDestination::new();
+        let dest_check = dest.clone();
+
+        let archiver =
+            ContinuousArchiver::new(seg_dir, dir.path().join("archival.state"), Box::new(dest));
+        archiver.run_pass().unwrap();
+
+        let stored = dest_check.stored.lock().unwrap();
+        assert_eq!(stored[0].1, content_0);
+        assert_eq!(stored[1].1, content_1);
+    }
+}

crates/tranquil-store/src/blockstore/compaction.rs

@@ -0,0 +1,206 @@
+use std::io;
+
+use crate::io::{FileId, OpenOptions, StorageIO};
+
+use super::data_file::{DataFileReader, DataFileWriter, ReadBlockRecord};
+use super::group_commit::{ActiveFileSet, FileIdAllocator};
+use super::hash_index::{BlockIndex, BlockIndexError};
+use super::hint::{HintFileWriter, hint_file_path};
+use super::manager::DataFileManager;
+use super::types::{BlockLocation, CidBytes, CommitEpoch, CompactionResult, DataFileId};
+
+#[derive(Debug)]
+pub enum CompactionError {
+    Io(io::Error),
+    Index(BlockIndexError),
+    ChannelClosed,
+    ActiveFileCannotBeCompacted,
+}
+
+impl std::fmt::Display for CompactionError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Io(e) => write!(f, "io: {e}"),
+            Self::Index(e) => write!(f, "index: {e}"),
+            Self::ChannelClosed => write!(f, "commit channel closed"),
+            Self::ActiveFileCannotBeCompacted => {
+                write!(f, "cannot compact the active data file")
+            }
+        }
+    }
+}
+
+impl std::error::Error for CompactionError {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        match self {
+            Self::Io(e) => Some(e),
+            Self::Index(e) => Some(e),
+            Self::ChannelClosed | Self::ActiveFileCannotBeCompacted => None,
+        }
+    }
+}
+
+impl From<io::Error> for CompactionError {
+    fn from(e: io::Error) -> Self {
+        Self::Io(e)
+    }
+}
+
+impl From<BlockIndexError> for CompactionError {
+    fn from(e: BlockIndexError) -> Self {
+        Self::Index(e)
+    }
+}
+
+#[allow(clippy::too_many_arguments)]
+pub(super) fn compact_on_writer_thread<S: StorageIO>(
+    manager: &DataFileManager<S>,
+    index: &BlockIndex,
+    source_file_id: DataFileId,
+    current_epoch: CommitEpoch,
+    grace_period_ms: u64,
+    file_ids: &FileIdAllocator,
+    active_files: &ActiveFileSet,
+    hint_positions: &super::group_commit::ShardHintPositions,
+    epoch: &super::types::EpochCounter,
+) -> Result<CompactionResult, CompactionError> {
+    if active_files.contains(source_file_id) {
+        return Err(CompactionError::ActiveFileCannotBeCompacted);
+    }
+
+    let source_fd = manager.open_for_read(source_file_id)?;
+    let source_size = manager.io().file_size(source_fd)?;
+
+    let new_file_id = file_ids.allocate();
+
+    let result = stream_compact(
+        manager,
+        index,
+        source_file_id,
+        source_fd,
+        new_file_id,
+        current_epoch,
+        grace_period_ms,
+    );
+
+    match result {
+        Err(e) => {
+            manager.delete_data_file(new_file_id).ok();
+            manager
+                .io()
+                .delete(&hint_file_path(manager.data_dir(), new_file_id))
+                .ok();
+            Err(e)
+        }
+        Ok((new_size, live_count, dead_count)) => {
+            if let Err(e) = index.write_checkpoint(epoch.current(), hint_positions) {
+                tracing::warn!(error = %e, "pre-delete checkpoint failed during compaction");
+            }
+
+            manager.delete_data_file(source_file_id)?;
+            manager
+                .io()
+                .delete(&hint_file_path(manager.data_dir(), source_file_id))
+                .ok();
+            manager.io().sync_dir(manager.data_dir())?;
+
+            let reclaimed_bytes = source_size.saturating_sub(new_size);
+
+            tracing::info!(
+                source = %source_file_id,
+                dest = %new_file_id,
+                old_size = source_size,
+                new_size,
+                live_count,
+                dead_count,
+                reclaimed_bytes,
+                "compaction complete"
+            );
+
+            Ok(CompactionResult {
+                file_id: source_file_id,
+                old_size: source_size,
+                new_size,
+                live_blocks: live_count,
+                dead_blocks: dead_count,
+                reclaimed_bytes,
+            })
+        }
+    }
+}
+
+fn stream_compact<S: StorageIO>(
+    manager: &DataFileManager<S>,
+    index: &BlockIndex,
+    source_file_id: DataFileId,
+    source_fd: FileId,
+    new_file_id: DataFileId,
+    current_epoch: CommitEpoch,
+    grace_period_ms: u64,
+) -> Result<(u64, u64, u64), CompactionError> {
+    let mut reader = DataFileReader::open(manager.io(), source_fd)?;
+    let now = crate::wall_clock_ms();
+
+    let new_fd = manager.open_for_append(new_file_id)?;
+    let mut writer = DataFileWriter::new(manager.io(), new_fd, new_file_id)?;
+
+    let hint_path = hint_file_path(manager.data_dir(), new_file_id);
+    let hint_fd = manager.io().open(&hint_path, OpenOptions::read_write())?;
+    let mut hint_writer = HintFileWriter::new(manager.io(), hint_fd);
+
+    let mut relocations: Vec<(CidBytes, BlockLocation)> = Vec::new();
+    let mut dead_cids: Vec<CidBytes> = Vec::new();
+    let mut live_count: u64 = 0;
+    let mut dead_count: u64 = 0;
+
+    reader.try_for_each(|r| {
+        let record = r?;
+        match record {
+            ReadBlockRecord::Valid {
+                cid_bytes, data, ..
+            } => match index.get(&cid_bytes) {
+                Some(e) if e.location.file_id == source_file_id && !e.refcount.is_zero() => {
+                    let loc = writer.append_block(&cid_bytes, &data)?;
+                    hint_writer.append_relocate(&cid_bytes, &loc, e.refcount.raw())?;
+                    relocations.push((cid_bytes, loc));
+                    live_count = live_count.saturating_add(1);
+                }
+                Some(e) if e.location.file_id == source_file_id && e.refcount.is_zero() => {
+                    let eligible =
+                        index.is_gc_eligible(&cid_bytes, current_epoch, now, grace_period_ms);
+                    match eligible {
+                        true => {
+                            tracing::debug!(
+                                ?cid_bytes,
+                                file_id = %source_file_id,
+                                "gc: collecting dead block"
+                            );
+                            hint_writer.append_remove(&cid_bytes)?;
+                            dead_cids.push(cid_bytes);
+                            dead_count = dead_count.saturating_add(1);
+                        }
+                        false => {
+                            let loc = writer.append_block(&cid_bytes, &data)?;
+                            hint_writer.append_relocate(&cid_bytes, &loc, e.refcount.raw())?;
+                            relocations.push((cid_bytes, loc));
+                            live_count = live_count.saturating_add(1);
+                        }
+                    }
+                }
+                _ => {}
+            },
+            ReadBlockRecord::Corrupted { .. } | ReadBlockRecord::Truncated { .. } => {}
+        }
+        Ok::<_, CompactionError>(())
+    })?;
+
+    writer.sync()?;
+    hint_writer.sync()?;
+    manager.io().sync_dir(manager.data_dir())?;
+
+    let new_size = writer.position().raw();
+
+    index.apply_compaction(&relocations, &dead_cids);
+
+    Ok((new_size, live_count, dead_count))
+}

crates/tranquil-store/src/blockstore/key_index.rs

@@ -1,539 +0,0 @@
-use std::collections::HashMap;
-use std::path::Path;
-
-use fjall::{
-    Database, Keyspace, KeyspaceCreateOptions, PersistMode,
-    config::{BloomConstructionPolicy, FilterPolicy, FilterPolicyEntry},
-};
-
-use super::data_file::CID_SIZE;
-use super::types::{BlockLocation, IndexEntry, RefCount, WriteCursor};
-
-const WRITE_CURSOR_KEY: &[u8] = b"\x00write_cursor";
-
-const KEYSPACE_NAME: &str = "blocks";
-
-fn bloom_options() -> KeyspaceCreateOptions {
-    KeyspaceCreateOptions::default().filter_policy(FilterPolicy::new([
-        FilterPolicyEntry::Bloom(BloomConstructionPolicy::FalsePositiveRate(0.01)),
-        FilterPolicyEntry::Bloom(BloomConstructionPolicy::FalsePositiveRate(0.01)),
-    ]))
-}
-
-fn is_corruption_error(e: &fjall::Error) -> bool {
-    match e {
-        fjall::Error::Io(io_err) => matches!(
-            io_err.kind(),
-            std::io::ErrorKind::InvalidData | std::io::ErrorKind::UnexpectedEof
-        ),
-        fjall::Error::Locked | fjall::Error::KeyspaceDeleted => false,
-        _ => true,
-    }
-}
-
-fn serialize_entry(entry: &IndexEntry) -> Vec<u8> {
-    postcard::to_allocvec(entry)
-        .expect("IndexEntry serialization is infallible for fixed-layout types")
-}
-
-fn deserialize_entry(bytes: &[u8]) -> Result<IndexEntry, KeyIndexError> {
-    postcard::from_bytes(bytes).map_err(KeyIndexError::Deserialize)
-}
-
-fn serialize_cursor(cursor: &WriteCursor) -> Vec<u8> {
-    postcard::to_allocvec(cursor)
-        .expect("WriteCursor serialization is infallible for fixed-layout types")
-}
-
-fn deserialize_cursor(bytes: &[u8]) -> Result<WriteCursor, KeyIndexError> {
-    postcard::from_bytes(bytes).map_err(KeyIndexError::Deserialize)
-}
-
-#[derive(Debug)]
-pub enum KeyIndexError {
-    Fjall(fjall::Error),
-    Deserialize(postcard::Error),
-    MissingEntry,
-}
-
-impl std::fmt::Display for KeyIndexError {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            Self::Fjall(e) => write!(f, "fjall: {e}"),
-            Self::Deserialize(e) => write!(f, "deserialize: {e}"),
-            Self::MissingEntry => write!(f, "entry not found"),
-        }
-    }
-}
-
-impl std::error::Error for KeyIndexError {
-    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
-        match self {
-            Self::Fjall(e) => Some(e),
-            Self::Deserialize(e) => Some(e),
-            Self::MissingEntry => None,
-        }
-    }
-}
-
-impl From<fjall::Error> for KeyIndexError {
-    fn from(e: fjall::Error) -> Self {
-        Self::Fjall(e)
-    }
-}
-
-pub enum KeyIndexOpenOutcome {
-    Opened(KeyIndex),
-    NeedsRebuild(KeyIndex),
-}
-
-impl KeyIndexOpenOutcome {
-    pub fn into_inner(self) -> KeyIndex {
-        match self {
-            Self::Opened(idx) | Self::NeedsRebuild(idx) => idx,
-        }
-    }
-
-    pub fn needs_rebuild(&self) -> bool {
-        matches!(self, Self::NeedsRebuild(_))
-    }
-}
-
-pub struct KeyIndex {
-    db: Database,
-    blocks: Keyspace,
-}
-
-impl KeyIndex {
-    pub fn open(path: &Path) -> Result<KeyIndexOpenOutcome, KeyIndexError> {
-        match Self::try_open(path) {
-            Ok(idx) => Ok(KeyIndexOpenOutcome::Opened(idx)),
-            Err(KeyIndexError::Fjall(ref e)) if is_corruption_error(e) => {
-                let _ = std::fs::remove_dir_all(path);
-                let idx = Self::try_open(path)?;
-                Ok(KeyIndexOpenOutcome::NeedsRebuild(idx))
-            }
-            Err(e) => Err(e),
-        }
-    }
-
-    fn try_open(path: &Path) -> Result<Self, KeyIndexError> {
-        let db = Database::builder(path).open()?;
-        let blocks = db.keyspace(KEYSPACE_NAME, bloom_options)?;
-        Ok(Self { db, blocks })
-    }
-
-    pub fn get(&self, cid_bytes: &[u8; CID_SIZE]) -> Result<Option<IndexEntry>, KeyIndexError> {
-        self.blocks
-            .get(cid_bytes)?
-            .map(|v| deserialize_entry(&v))
-            .transpose()
-    }
-
-    pub fn has(&self, cid_bytes: &[u8; CID_SIZE]) -> Result<bool, KeyIndexError> {
-        self.blocks.contains_key(cid_bytes).map_err(Into::into)
-    }
-
-    pub fn put(
-        &self,
-        cid_bytes: &[u8; CID_SIZE],
-        location: BlockLocation,
-    ) -> Result<(), KeyIndexError> {
-        let entry = match self.get(cid_bytes)? {
-            Some(existing) => IndexEntry {
-                location: existing.location,
-                refcount: existing.refcount.increment(),
-            },
-            None => IndexEntry {
-                location,
-                refcount: RefCount::one(),
-            },
-        };
-        self.blocks
-            .insert(cid_bytes, serialize_entry(&entry))
-            .map_err(Into::into)
-    }
-
-    pub fn decrement_refcount(
-        &self,
-        cid_bytes: &[u8; CID_SIZE],
-    ) -> Result<RefCount, KeyIndexError> {
-        let existing = self.get(cid_bytes)?.ok_or(KeyIndexError::MissingEntry)?;
-        let new_refcount = match existing.refcount.is_zero() {
-            true => {
-                tracing::warn!(?cid_bytes, "decrement on zero-refcount entry, skipping");
-                existing.refcount
-            }
-            false => existing.refcount.decrement(),
-        };
-        let updated = IndexEntry {
-            location: existing.location,
-            refcount: new_refcount,
-        };
-        self.blocks.insert(cid_bytes, serialize_entry(&updated))?;
-        Ok(new_refcount)
-    }
-
-    pub fn batch_put(
-        &self,
-        entries: &[([u8; CID_SIZE], BlockLocation)],
-        decrements: &[[u8; CID_SIZE]],
-        cursor: WriteCursor,
-    ) -> Result<(), KeyIndexError> {
-        let mut batch = self.db.batch().durability(Some(PersistMode::SyncData));
-        let mut pending: HashMap<[u8; CID_SIZE], IndexEntry> = HashMap::new();
-
-        entries.iter().try_for_each(|(cid_bytes, location)| {
-            let entry = match pending.get(cid_bytes).copied().or(self.get(cid_bytes)?) {
-                Some(existing) => IndexEntry {
-                    location: existing.location,
-                    refcount: existing.refcount.increment(),
-                },
-                None => IndexEntry {
-                    location: *location,
-                    refcount: RefCount::one(),
-                },
-            };
-            pending.insert(*cid_bytes, entry);
-            batch.insert(&self.blocks, cid_bytes.as_slice(), serialize_entry(&entry));
-            Ok::<_, KeyIndexError>(())
-        })?;
-
-        decrements.iter().try_for_each(|cid_bytes| {
-            let existing = pending
-                .get(cid_bytes)
-                .copied()
-                .or(self.get(cid_bytes)?)
-                .ok_or(KeyIndexError::MissingEntry)?;
-            let new_refcount = match existing.refcount.is_zero() {
-                true => {
-                    tracing::warn!(?cid_bytes, "decrement on zero-refcount entry, skipping");
-                    existing.refcount
-                }
-                false => existing.refcount.decrement(),
-            };
-            let updated = IndexEntry {
-                location: existing.location,
-                refcount: new_refcount,
-            };
-            pending.insert(*cid_bytes, updated);
-            batch.insert(
-                &self.blocks,
-                cid_bytes.as_slice(),
-                serialize_entry(&updated),
-            );
-            Ok::<_, KeyIndexError>(())
-        })?;
-
-        batch.insert(&self.blocks, WRITE_CURSOR_KEY, serialize_cursor(&cursor));
-
-        batch.commit().map_err(Into::into)
-    }
-
-    pub fn read_write_cursor(&self) -> Result<Option<WriteCursor>, KeyIndexError> {
-        self.blocks
-            .get(WRITE_CURSOR_KEY)?
-            .map(|v| deserialize_cursor(&v))
-            .transpose()
-    }
-
-    pub fn persist(&self) -> Result<(), KeyIndexError> {
-        self.db.persist(PersistMode::SyncData).map_err(Into::into)
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::blockstore::test_cid;
-    use crate::blockstore::types::{BlockLength, BlockOffset, DataFileId};
-
-    fn test_location(file_id: u32, offset: u64, length: u32) -> BlockLocation {
-        BlockLocation {
-            file_id: DataFileId::new(file_id),
-            offset: BlockOffset::new(offset),
-            length: BlockLength::new(length),
-        }
-    }
-
-    fn open_temp() -> (tempfile::TempDir, KeyIndex) {
-        let dir = tempfile::TempDir::new().unwrap();
-        let outcome = KeyIndex::open(dir.path()).unwrap();
-        assert!(!outcome.needs_rebuild());
-        (dir, outcome.into_inner())
-    }
-
-    #[test]
-    fn put_then_get_round_trips() {
-        let (_dir, idx) = open_temp();
-        let cid = test_cid(1);
-        let loc = test_location(0, 100, 256);
-
-        idx.put(&cid, loc).unwrap();
-        let entry = idx.get(&cid).unwrap().unwrap();
-        assert_eq!(entry.location, loc);
-        assert_eq!(entry.refcount, RefCount::one());
-    }
-
-    #[test]
-    fn get_missing_returns_none() {
-        let (_dir, idx) = open_temp();
-        assert!(idx.get(&test_cid(42)).unwrap().is_none());
-    }
-
-    #[test]
-    fn has_missing_returns_false() {
-        let (_dir, idx) = open_temp();
-        assert!(!idx.has(&test_cid(42)).unwrap());
-    }
-
-    #[test]
-    fn has_existing_returns_true() {
-        let (_dir, idx) = open_temp();
-        let cid = test_cid(1);
-        idx.put(&cid, test_location(0, 0, 10)).unwrap();
-        assert!(idx.has(&cid).unwrap());
-    }
-
-    #[test]
-    fn duplicate_put_increments_refcount() {
-        let (_dir, idx) = open_temp();
-        let cid = test_cid(1);
-        let loc = test_location(0, 100, 256);
-
-        idx.put(&cid, loc).unwrap();
-        idx.put(&cid, test_location(1, 200, 512)).unwrap();
-
-        let entry = idx.get(&cid).unwrap().unwrap();
-        assert_eq!(entry.refcount, RefCount::new(2));
-        assert_eq!(entry.location, loc);
-    }
-
-    #[test]
-    fn decrement_refcount_from_two_to_one() {
-        let (_dir, idx) = open_temp();
-        let cid = test_cid(1);
-        idx.put(&cid, test_location(0, 0, 10)).unwrap();
-        idx.put(&cid, test_location(0, 0, 10)).unwrap();
-
-        let rc = idx.decrement_refcount(&cid).unwrap();
-        assert_eq!(rc, RefCount::one());
-
-        let entry = idx.get(&cid).unwrap().unwrap();
-        assert_eq!(entry.refcount, RefCount::one());
-    }
-
-    #[test]
-    fn decrement_refcount_to_zero_keeps_entry() {
-        let (_dir, idx) = open_temp();
-        let cid = test_cid(1);
-        idx.put(&cid, test_location(0, 0, 10)).unwrap();
-
-        let rc = idx.decrement_refcount(&cid).unwrap();
-        assert!(rc.is_zero());
-
-        let entry = idx.get(&cid).unwrap().unwrap();
-        assert!(entry.refcount.is_zero());
-    }
-
-    #[test]
-    fn decrement_missing_entry_errors() {
-        let (_dir, idx) = open_temp();
-        let result = idx.decrement_refcount(&test_cid(99));
-        assert!(matches!(result, Err(KeyIndexError::MissingEntry)));
-    }
-
-    #[test]
-    fn batch_put_new_entries() {
-        let (_dir, idx) = open_temp();
-        let entries: Vec<_> = (0u8..3)
-            .map(|i| (test_cid(i), test_location(0, i as u64 * 100, 50)))
-            .collect();
-        let cursor = WriteCursor {
-            file_id: DataFileId::new(0),
-            offset: BlockOffset::new(300),
-        };
-
-        idx.batch_put(&entries, &[], cursor).unwrap();
-
-        entries.iter().for_each(|(cid, loc)| {
-            let entry = idx.get(cid).unwrap().unwrap();
-            assert_eq!(entry.location, *loc);
-            assert_eq!(entry.refcount, RefCount::one());
-        });
-    }
-
-    #[test]
-    fn batch_put_increments_existing() {
-        let (_dir, idx) = open_temp();
-        let cid = test_cid(1);
-        let original_loc = test_location(0, 100, 50);
-        idx.put(&cid, original_loc).unwrap();
-
-        let entries = vec![(cid, test_location(1, 200, 60))];
-        let cursor = WriteCursor {
-            file_id: DataFileId::new(1),
-            offset: BlockOffset::new(260),
-        };
-        idx.batch_put(&entries, &[], cursor).unwrap();
-
-        let entry = idx.get(&cid).unwrap().unwrap();
-        assert_eq!(entry.refcount, RefCount::new(2));
-        assert_eq!(entry.location, original_loc);
-    }
-
-    #[test]
-    fn batch_put_with_decrements() {
-        let (_dir, idx) = open_temp();
-        let cid_a = test_cid(1);
-        let cid_b = test_cid(2);
-        idx.put(&cid_b, test_location(0, 0, 10)).unwrap();
-        idx.put(&cid_b, test_location(0, 0, 10)).unwrap();
-
-        let entries = vec![(cid_a, test_location(0, 100, 50))];
-        let decrements = vec![cid_b];
-        let cursor = WriteCursor {
-            file_id: DataFileId::new(0),
-            offset: BlockOffset::new(150),
-        };
-        idx.batch_put(&entries, &decrements, cursor).unwrap();
-
-        let a = idx.get(&cid_a).unwrap().unwrap();
-        assert_eq!(a.refcount, RefCount::one());
-
-        let b = idx.get(&cid_b).unwrap().unwrap();
-        assert_eq!(b.refcount, RefCount::one());
-    }
-
-    #[test]
-    fn batch_put_mixed_new_and_duplicate() {
-        let (_dir, idx) = open_temp();
-        let existing_cid = test_cid(1);
-        let existing_loc = test_location(0, 0, 10);
-        idx.put(&existing_cid, existing_loc).unwrap();
-
-        let entries: Vec<_> = (1u8..=4)
-            .map(|i| (test_cid(i), test_location(0, i as u64 * 100, 50)))
-            .collect();
-        let cursor = WriteCursor {
-            file_id: DataFileId::new(0),
-            offset: BlockOffset::new(500),
-        };
-        idx.batch_put(&entries, &[], cursor).unwrap();
-
-        let existing = idx.get(&existing_cid).unwrap().unwrap();
-        assert_eq!(existing.refcount, RefCount::new(2));
-        assert_eq!(existing.location, existing_loc);
-
-        (2u8..=4).for_each(|i| {
-            let entry = idx.get(&test_cid(i)).unwrap().unwrap();
-            assert_eq!(entry.refcount, RefCount::one());
-        });
-    }
-
-    #[test]
-    fn batch_put_duplicate_cid_in_same_batch() {
-        let (_dir, idx) = open_temp();
-        let cid = test_cid(1);
-        let loc = test_location(0, 100, 50);
-
-        let entries = vec![(cid, loc), (cid, test_location(0, 200, 60))];
-        let cursor = WriteCursor {
-            file_id: DataFileId::new(0),
-            offset: BlockOffset::new(260),
-        };
-        idx.batch_put(&entries, &[], cursor).unwrap();
-
-        let entry = idx.get(&cid).unwrap().unwrap();
-        assert_eq!(entry.refcount, RefCount::new(2));
-        assert_eq!(entry.location, loc);
-    }
-
-    #[test]
-    fn batch_put_entry_then_decrement_same_cid() {
-        let (_dir, idx) = open_temp();
-        let cid = test_cid(1);
-        let loc = test_location(0, 100, 50);
-
-        let entries = vec![(cid, loc)];
-        let decrements = vec![cid];
-        let cursor = WriteCursor {
-            file_id: DataFileId::new(0),
-            offset: BlockOffset::new(150),
-        };
-        idx.batch_put(&entries, &decrements, cursor).unwrap();
-
-        let entry = idx.get(&cid).unwrap().unwrap();
-        assert!(entry.refcount.is_zero());
-    }
-
-    #[test]
-    fn write_cursor_round_trip() {
-        let (_dir, idx) = open_temp();
-        assert!(idx.read_write_cursor().unwrap().is_none());
-
-        let cursor = WriteCursor {
-            file_id: DataFileId::new(3),
-            offset: BlockOffset::new(65536),
-        };
-        let entries = vec![(test_cid(1), test_location(3, 0, 100))];
-        idx.batch_put(&entries, &[], cursor).unwrap();
-
-        let read_back = idx.read_write_cursor().unwrap().unwrap();
-        assert_eq!(read_back, cursor);
-    }
-
-    #[test]
-    fn write_cursor_persists_across_reopen() {
-        let dir = tempfile::TempDir::new().unwrap();
-
-        let cursor = WriteCursor {
-            file_id: DataFileId::new(7),
-            offset: BlockOffset::new(99999),
-        };
-
-        {
-            let idx = KeyIndex::open(dir.path()).unwrap().into_inner();
-            let entries = vec![(test_cid(1), test_location(7, 0, 100))];
-            idx.batch_put(&entries, &[], cursor).unwrap();
-            idx.persist().unwrap();
-        }
-
-        {
-            let idx = KeyIndex::open(dir.path()).unwrap().into_inner();
-            let read_back = idx.read_write_cursor().unwrap().unwrap();
-            assert_eq!(read_back, cursor);
-
-            let entry = idx.get(&test_cid(1)).unwrap().unwrap();
-            assert_eq!(entry.refcount, RefCount::one());
-        }
-    }
-
-    #[test]
-    fn corrupt_index_triggers_needs_rebuild() {
-        let dir = tempfile::TempDir::new().unwrap();
-
-        {
-            let idx = KeyIndex::open(dir.path()).unwrap().into_inner();
-            idx.put(&test_cid(1), test_location(0, 0, 10)).unwrap();
-            idx.persist().unwrap();
-        }
-
-        std::fs::read_dir(dir.path())
-            .unwrap()
-            .filter_map(|e| e.ok())
-            .for_each(|entry| {
-                let path = entry.path();
-                if path.is_file() {
-                    std::fs::write(&path, b"corrupted").unwrap();
-                }
-            });
-
-        let outcome = KeyIndex::open(dir.path()).unwrap();
-        assert!(outcome.needs_rebuild());
-
-        let idx = outcome.into_inner();
-        assert!(idx.get(&test_cid(1)).unwrap().is_none());
-        assert!(idx.read_write_cursor().unwrap().is_none());
-    }
-}

crates/tranquil-store/src/blockstore/manager.rs

@@ -136,6 +136,27 @@ impl<S: StorageIO> DataFileManager<S> {
     pub fn list_files(&self) -> io::Result<Vec<DataFileId>> {
         list_files_by_extension(&self.io, &self.data_dir, DATA_FILE_EXTENSION)
     }
+
+    pub fn evict_handle(&self, file_id: DataFileId) {
+        let removed = self.handles.write().remove(&file_id);
+        if let Some(entry) = removed {
+            let _ = self.io.close(entry.fd);
+        }
+    }
+
+    pub fn delete_data_file(&self, file_id: DataFileId) -> io::Result<()> {
+        self.evict_handle(file_id);
+        let path = self.data_file_path(file_id);
+        self.io.delete(&path)
+    }
+}
+
+impl<S: StorageIO> Drop for DataFileManager<S> {
+    fn drop(&mut self) {
+        self.handles.write().drain().for_each(|(_, entry)| {
+            let _ = self.io.close(entry.fd);
+        });
+    }
 }
 
 #[cfg(test)]

crates/tranquil-store/src/blockstore/mod.rs

@@ -1,30 +1,35 @@
+mod compaction;
 mod data_file;
 mod group_commit;
+pub mod hash_index;
 mod hint;
-mod key_index;
 mod manager;
 mod reader;
 mod store;
 mod types;
 
+pub use compaction::CompactionError;
 pub use data_file::{
     BLOCK_FORMAT_VERSION, BLOCK_HEADER_SIZE, BLOCK_MAGIC, BLOCK_RECORD_OVERHEAD, CID_SIZE,
     DataFileReader, DataFileWriter, ReadBlockRecord, ValidBlock, decode_block_record,
     encode_block_record,
 };
-pub use group_commit::{CommitError, CommitRequest, GroupCommitConfig, GroupCommitWriter};
+pub use group_commit::{
+    ActiveFileSet, CommitError, CommitRequest, FileIdAllocator, GroupCommitConfig,
+    GroupCommitWriter, ShardHintPositions,
+};
 pub use hint::{
-    HINT_FILE_EXTENSION, HINT_RECORD_SIZE, HintFileReader, HintFileWriter, ReadHintRecord,
-    RebuildError, decode_hint_record, hint_file_path, rebuild_index_from_data_files,
-    rebuild_index_from_hints,
+    HINT_FILE_EXTENSION, HINT_RECORD_SIZE, HintFileReader, HintFileWriter, HintIndex,
+    ReadHintRecord, RebuildError, decode_hint_record, hint_file_path, scan_hints_to_memory,
 };
-pub use key_index::{KeyIndex, KeyIndexError, KeyIndexOpenOutcome};
 pub use manager::{DEFAULT_MAX_FILE_SIZE, DataFileManager};
 pub use reader::{BlockStoreReader, ReadError};
-pub use store::{BlockStoreConfig, TranquilBlockStore};
+pub use store::QuiesceGuard;
+pub use store::{BlockStoreConfig, DEFAULT_SHARD_COUNT, TranquilBlockStore};
 pub use types::{
-    BlockLength, BlockLocation, BlockOffset, DataFileId, HintOffset, IndexEntry, MAX_BLOCK_SIZE,
-    RefCount, WriteCursor,
+    BlockLength, BlockLocation, BlockOffset, BlockstoreSnapshot, CidBytes, CollectionResult,
+    CommitEpoch, CompactionResult, DataFileId, EpochCounter, HintOffset, IndexEntry, LivenessInfo,
+    MAX_BLOCK_SIZE, RefCount, ShardId, WallClockMs, WriteCursor,
 };
 
 use std::io;
@@ -40,7 +45,7 @@ impl BlocksSynced {
     }
 }
 
-pub(crate) fn list_files_by_extension<S: StorageIO>(
+pub fn list_files_by_extension<S: StorageIO>(
     io: &S,
     dir: &Path,
     extension: &str,

crates/tranquil-store/src/blockstore/reader.rs

@@ -7,14 +7,13 @@ use bytes::Bytes;
 use crate::io::{FileId, StorageIO};
 
 use super::data_file::{CID_SIZE, ReadBlockRecord, decode_block_record};
-use super::key_index::{KeyIndex, KeyIndexError};
+use super::hash_index::BlockIndex;
 use super::manager::DataFileManager;
 use super::types::{BlockLocation, BlockOffset, DataFileId};
 
 #[derive(Debug, Clone)]
 pub enum ReadError {
     Io(Arc<io::Error>),
-    Index(Arc<KeyIndexError>),
     Corrupted {
         file_id: DataFileId,
         offset: BlockOffset,
@@ -25,7 +24,6 @@ impl std::fmt::Display for ReadError {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             Self::Io(e) => write!(f, "io: {e}"),
-            Self::Index(e) => write!(f, "index: {e}"),
             Self::Corrupted { file_id, offset } => {
                 write!(f, "corrupted block at {file_id}:{}", offset.raw())
             }
@@ -37,7 +35,6 @@ impl std::error::Error for ReadError {
     fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
         match self {
             Self::Io(e) => Some(e.as_ref()),
-            Self::Index(e) => Some(e.as_ref()),
             Self::Corrupted { .. } => None,
         }
     }
@@ -49,14 +46,8 @@ impl From<io::Error> for ReadError {
     }
 }
 
-impl From<KeyIndexError> for ReadError {
-    fn from(e: KeyIndexError) -> Self {
-        Self::Index(Arc::new(e))
-    }
-}
-
 pub struct BlockStoreReader<S: StorageIO> {
-    index: Arc<KeyIndex>,
+    index: Arc<BlockIndex>,
     manager: Arc<DataFileManager<S>>,
 }
 
@@ -70,54 +61,58 @@ impl<S: StorageIO> Clone for BlockStoreReader<S> {
 }
 
 impl<S: StorageIO> BlockStoreReader<S> {
-    pub fn new(index: Arc<KeyIndex>, manager: Arc<DataFileManager<S>>) -> Self {
+    pub fn new(index: Arc<BlockIndex>, manager: Arc<DataFileManager<S>>) -> Self {
         Self { index, manager }
     }
 
+    pub fn manager(&self) -> &DataFileManager<S> {
+        &self.manager
+    }
+
     pub fn get(&self, cid: &[u8; CID_SIZE]) -> Result<Option<Bytes>, ReadError> {
-        let entry = match self.index.get(cid)? {
-            Some(e) => e,
-            None => return Ok(None),
-        };
-        self.read_block_at(entry.location).map(Some)
+        match self.index.get(cid) {
+            Some(e) => self.read_block_at(e.location).map(Some),
+            None => Ok(None),
+        }
     }
 
     pub fn has(&self, cid: &[u8; CID_SIZE]) -> Result<bool, ReadError> {
-        self.index.has(cid).map_err(ReadError::from)
+        Ok(self.index.has(cid))
     }
 
     pub fn get_many(&self, cids: &[[u8; CID_SIZE]]) -> Result<Vec<Option<Bytes>>, ReadError> {
         let mut results: Vec<Option<Bytes>> = vec![None; cids.len()];
 
-        let lookups: Vec<(usize, BlockLocation)> = cids
+        let index_lookups: Vec<(usize, BlockLocation)> = cids
             .iter()
             .enumerate()
-            .filter_map(|(i, cid)| match self.index.get(cid) {
-                Ok(Some(entry)) => Some(Ok((i, entry.location))),
-                Ok(None) => None,
-                Err(e) => Some(Err(ReadError::from(e))),
-            })
-            .collect::<Result<Vec<_>, _>>()?;
+            .filter_map(|(i, cid)| self.index.get(cid).map(|entry| (i, entry.location)))
+            .collect();
+        self.read_locations_into(&index_lookups, &mut results)?;
 
+        Ok(results)
+    }
+
+    fn read_locations_into(
+        &self,
+        lookups: &[(usize, BlockLocation)],
+        results: &mut [Option<Bytes>],
+    ) -> Result<(), ReadError> {
         let mut by_file: HashMap<DataFileId, Vec<(usize, BlockLocation)>> = HashMap::new();
-        lookups.into_iter().for_each(|(idx, loc)| {
+        lookups.iter().for_each(|&(idx, loc)| {
             by_file.entry(loc.file_id).or_default().push((idx, loc));
         });
 
         by_file.into_iter().try_for_each(|(file_id, mut entries)| {
             let fd = self.manager.open_for_read(file_id)?;
             let file_size = self.manager.io().file_size(fd)?;
-
             entries.sort_by_key(|(_, loc)| loc.offset);
-
             entries.into_iter().try_for_each(|(orig_idx, loc)| {
                 let data = self.decode_and_validate(fd, file_size, loc)?;
                 results[orig_idx] = Some(data);
                 Ok::<_, ReadError>(())
             })
-        })?;
-
-        Ok(results)
+        })
     }
 
     fn read_block_at(&self, location: BlockLocation) -> Result<Bytes, ReadError> {
@@ -155,457 +150,3 @@ impl<S: StorageIO> BlockStoreReader<S> {
         }
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::RealIO;
-    use crate::blockstore::data_file::CID_SIZE;
-    use crate::blockstore::group_commit::{CommitRequest, GroupCommitConfig, GroupCommitWriter};
-    use crate::blockstore::key_index::KeyIndex;
-    use crate::blockstore::manager::DataFileManager;
-    use crate::blockstore::test_cid;
-    use futures::StreamExt;
-
-    struct TestHarness {
-        _dir: tempfile::TempDir,
-        index: Arc<KeyIndex>,
-        manager: Arc<DataFileManager<RealIO>>,
-        writer: Option<GroupCommitWriter>,
-        sender: flume::Sender<CommitRequest>,
-    }
-
-    impl TestHarness {
-        fn new() -> Self {
-            let dir = tempfile::TempDir::new().unwrap();
-            let data_dir = dir.path().join("data");
-            std::fs::create_dir_all(&data_dir).unwrap();
-            let index_dir = dir.path().join("index");
-            let manager = Arc::new(DataFileManager::with_default_max_size(
-                RealIO::new(),
-                data_dir,
-            ));
-            let index = Arc::new(KeyIndex::open(&index_dir).unwrap().into_inner());
-            let writer = GroupCommitWriter::spawn(
-                DataFileManager::with_default_max_size(RealIO::new(), dir.path().join("data")),
-                Arc::clone(&index),
-                GroupCommitConfig::default(),
-            )
-            .unwrap();
-            let sender = writer.sender().clone();
-
-            Self {
-                _dir: dir,
-                index,
-                manager,
-                writer: Some(writer),
-                sender,
-            }
-        }
-
-        fn reader(&self) -> BlockStoreReader<RealIO> {
-            BlockStoreReader::new(Arc::clone(&self.index), Arc::clone(&self.manager))
-        }
-
-        async fn put_blocks(
-            &self,
-            blocks: Vec<([u8; CID_SIZE], Vec<u8>)>,
-        ) -> Result<Vec<super::super::types::BlockLocation>, super::super::group_commit::CommitError>
-        {
-            let (tx, rx) = tokio::sync::oneshot::channel();
-            self.sender
-                .send_async(CommitRequest::PutBlocks {
-                    blocks,
-                    response: tx,
-                })
-                .await
-                .map_err(|_| super::super::group_commit::CommitError::ChannelClosed)?;
-            rx.await
-                .map_err(|_| super::super::group_commit::CommitError::ChannelClosed)?
-        }
-
-        fn shutdown(&mut self) {
-            if let Some(w) = self.writer.take() {
-                w.shutdown();
-            }
-        }
-    }
-
-    impl Drop for TestHarness {
-        fn drop(&mut self) {
-            self.shutdown();
-        }
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn get_existing_block() {
-        let mut harness = TestHarness::new();
-        let cid = test_cid(1);
-        let data = vec![0xAB; 256];
-        harness.put_blocks(vec![(cid, data.clone())]).await.unwrap();
-        harness.shutdown();
-
-        let reader = harness.reader();
-        let result = reader.get(&cid).unwrap().unwrap();
-        assert_eq!(&result[..], &data[..]);
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn get_missing_block_returns_none() {
-        let mut harness = TestHarness::new();
-        harness.shutdown();
-
-        let reader = harness.reader();
-        assert!(reader.get(&test_cid(99)).unwrap().is_none());
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn get_many_mixed_hits_and_misses() {
-        let mut harness = TestHarness::new();
-        let blocks: Vec<_> = (0u8..5)
-            .map(|i| (test_cid(i), vec![i; (i as usize + 1) * 32]))
-            .collect();
-        harness.put_blocks(blocks.clone()).await.unwrap();
-        harness.shutdown();
-
-        let reader = harness.reader();
-        let query: Vec<[u8; CID_SIZE]> = vec![
-            test_cid(0),
-            test_cid(99),
-            test_cid(2),
-            test_cid(100),
-            test_cid(4),
-        ];
-        let results = reader.get_many(&query).unwrap();
-
-        assert_eq!(results.len(), 5);
-        assert_eq!(&results[0].as_ref().unwrap()[..], &blocks[0].1[..]);
-        assert!(results[1].is_none());
-        assert_eq!(&results[2].as_ref().unwrap()[..], &blocks[2].1[..]);
-        assert!(results[3].is_none());
-        assert_eq!(&results[4].as_ref().unwrap()[..], &blocks[4].1[..]);
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn has_returns_true_for_existing() {
-        let mut harness = TestHarness::new();
-        let cid = test_cid(1);
-        harness
-            .put_blocks(vec![(cid, vec![0xFF; 64])])
-            .await
-            .unwrap();
-        harness.shutdown();
-
-        let reader = harness.reader();
-        assert!(reader.has(&cid).unwrap());
-        assert!(!reader.has(&test_cid(99)).unwrap());
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn checksum_mismatch_returns_error() {
-        let mut harness = TestHarness::new();
-        let cid = test_cid(1);
-        let data = vec![0xAA; 256];
-        harness.put_blocks(vec![(cid, data)]).await.unwrap();
-        harness.shutdown();
-
-        let entry = harness.index.get(&cid).unwrap().unwrap();
-        let loc = entry.location;
-        let data_file_path = harness.manager.data_file_path(loc.file_id);
-
-        let corrupt_offset = loc.offset.raw() + super::super::data_file::CID_SIZE as u64 + 4 + 128;
-        let file_bytes = std::fs::read(&data_file_path).unwrap();
-        let mut corrupted = file_bytes;
-        corrupted[corrupt_offset as usize] ^= 0xFF;
-        std::fs::write(&data_file_path, &corrupted).unwrap();
-
-        let fresh_manager = Arc::new(DataFileManager::with_default_max_size(
-            RealIO::new(),
-            harness.manager.data_dir().to_path_buf(),
-        ));
-        let reader = BlockStoreReader::new(Arc::clone(&harness.index), fresh_manager);
-        let result = reader.get(&cid);
-        assert!(
-            matches!(result, Err(ReadError::Corrupted { .. })),
-            "expected Corrupted error, got {result:?}"
-        );
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-    async fn spawn_blocking_does_not_block_tokio_workers() {
-        let mut harness = TestHarness::new();
-        let blocks: Vec<_> = (0u8..200).map(|i| (test_cid(i), vec![i; 1024])).collect();
-        harness.put_blocks(blocks).await.unwrap();
-        harness.shutdown();
-
-        let reader = harness.reader();
-        let reader = Arc::new(reader);
-
-        let timer_handle = tokio::spawn(futures::stream::iter(0..100).fold(
-            std::time::Duration::ZERO,
-            |max_drift, _| async move {
-                let start = std::time::Instant::now();
-                tokio::time::sleep(std::time::Duration::from_millis(1)).await;
-                let drift = start
-                    .elapsed()
-                    .saturating_sub(std::time::Duration::from_millis(1));
-                max_drift.max(drift)
-            },
-        ));
-
-        let read_handles: Vec<_> = (0..8)
-            .map(|_| {
-                let reader = Arc::clone(&reader);
-                tokio::spawn(futures::stream::iter(0u8..200).fold(
-                    (0u64, 200u64),
-                    move |(total_us, count), i| {
-                        let reader = Arc::clone(&reader);
-                        async move {
-                            let cid = test_cid(i);
-                            let start = std::time::Instant::now();
-                            let result = tokio::task::spawn_blocking(move || reader.get(&cid))
-                                .await
-                                .unwrap();
-                            let elapsed_us = start.elapsed().as_micros() as u64;
-                            assert!(result.unwrap().is_some());
-                            (total_us.saturating_add(elapsed_us), count)
-                        }
-                    },
-                ))
-            })
-            .collect();
-
-        let timer_drift = timer_handle.await.unwrap();
-        assert!(
-            timer_drift < std::time::Duration::from_millis(5),
-            "timer drift {timer_drift:?} exceeds 5ms, reads may be blocking tokio workers"
-        );
-
-        let stats: Vec<(u64, u64)> = futures::future::join_all(read_handles)
-            .await
-            .into_iter()
-            .map(|r| r.unwrap())
-            .collect();
-        let total_us: u64 = stats.iter().map(|(us, _)| us).sum();
-        let total_count: u64 = stats.iter().map(|(_, c)| c).sum();
-        let avg_us = total_us / total_count.max(1);
-        eprintln!("avg read latency: {avg_us}us across {total_count} reads");
-    }
-
-    use crate::blockstore::test_cid_u16 as stress_cid;
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
-    async fn stress_50_writers_20_readers() {
-        let dir = tempfile::TempDir::new().unwrap();
-        let data_dir = dir.path().join("data");
-        std::fs::create_dir_all(&data_dir).unwrap();
-        let index_dir = dir.path().join("index");
-        let index = Arc::new(KeyIndex::open(&index_dir).unwrap().into_inner());
-        let manager_for_writer =
-            DataFileManager::with_default_max_size(RealIO::new(), data_dir.clone());
-        let writer = GroupCommitWriter::spawn(
-            manager_for_writer,
-            Arc::clone(&index),
-            GroupCommitConfig::default(),
-        )
-        .unwrap();
-        let sender = writer.sender().clone();
-        let manager_for_reader = Arc::new(DataFileManager::with_default_max_size(
-            RealIO::new(),
-            data_dir,
-        ));
-        let reader = BlockStoreReader::new(Arc::clone(&index), manager_for_reader);
-
-        let committed = Arc::new(std::sync::Mutex::new(Vec::<(u16, Vec<u8>)>::new()));
-        let writer_done = Arc::new(std::sync::atomic::AtomicBool::new(false));
-
-        let writer_handles: Vec<_> = (0u16..50)
-            .map(|writer_id| {
-                let sender = sender.clone();
-                let committed = Arc::clone(&committed);
-                tokio::spawn(async move {
-                    futures::stream::iter(0u16..200)
-                        .fold((), |(), block_id| {
-                            let sender = sender.clone();
-                            let committed = Arc::clone(&committed);
-                            async move {
-                                let seed = writer_id * 200 + block_id;
-                                let cid = stress_cid(seed);
-                                let size = ((seed as usize % 256) + 1) * 4;
-                                let data = vec![seed as u8; size];
-                                let (tx, rx) = tokio::sync::oneshot::channel();
-                                sender
-                                    .send_async(CommitRequest::PutBlocks {
-                                        blocks: vec![(cid, data.clone())],
-                                        response: tx,
-                                    })
-                                    .await
-                                    .unwrap();
-                                rx.await.unwrap().unwrap();
-                                committed.lock().unwrap().push((seed, data));
-                            }
-                        })
-                        .await;
-                })
-            })
-            .collect();
-
-        let reader_handles: Vec<_> = (0..20)
-            .map(|_| {
-                let reader = reader.clone();
-                let committed = Arc::clone(&committed);
-                let done = Arc::clone(&writer_done);
-                tokio::spawn(async move {
-                    let reads = std::sync::atomic::AtomicU64::new(0);
-                    (0..5000)
-                        .take_while(|_| {
-                            let is_done = done.load(std::sync::atomic::Ordering::Relaxed);
-                            let has_reads = reads.load(std::sync::atomic::Ordering::Relaxed) > 100;
-                            !(is_done && has_reads)
-                        })
-                        .for_each(|_| {
-                            let snapshot = committed.lock().unwrap().clone();
-                            if let Some((seed, expected)) = snapshot.last() {
-                                let cid = stress_cid(*seed);
-                                match reader.get(&cid) {
-                                    Ok(Some(actual)) => {
-                                        assert_eq!(&actual[..], &expected[..]);
-                                        reads.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-                                    }
-                                    Ok(None) => {}
-                                    Err(e) => panic!("read error: {e}"),
-                                }
-                            }
-                            std::thread::yield_now();
-                        });
-                    reads.load(std::sync::atomic::Ordering::Relaxed)
-                })
-            })
-            .collect();
-
-        futures::future::join_all(writer_handles)
-            .await
-            .into_iter()
-            .for_each(|r| r.unwrap());
-        writer_done.store(true, std::sync::atomic::Ordering::Relaxed);
-
-        let read_counts: Vec<u64> = futures::future::join_all(reader_handles)
-            .await
-            .into_iter()
-            .map(|r| r.unwrap())
-            .collect();
-
-        let total_reads: u64 = read_counts.iter().sum();
-        eprintln!("total reader reads: {total_reads}");
-        assert!(total_reads > 0);
-
-        writer.shutdown();
-
-        let final_committed = committed.lock().unwrap();
-        assert_eq!(final_committed.len(), 10_000);
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
-    async fn concurrent_read_write() {
-        let mut harness = TestHarness::new();
-        let sender = harness.sender.clone();
-        let reader = harness.reader();
-
-        let written_cids = Arc::new(std::sync::Mutex::new(Vec::<(u8, Vec<u8>)>::new()));
-        let writer_done = Arc::new(std::sync::atomic::AtomicBool::new(false));
-
-        let writer_handle = {
-            let written = Arc::clone(&written_cids);
-            tokio::spawn(async move {
-                futures::stream::iter(0u8..50)
-                    .fold((), |(), i| {
-                        let sender = sender.clone();
-                        let written = Arc::clone(&written);
-                        async move {
-                            let cid = test_cid(i);
-                            let data = vec![i; (i as usize + 1) * 16];
-                            let (tx, rx) = tokio::sync::oneshot::channel();
-                            sender
-                                .send_async(CommitRequest::PutBlocks {
-                                    blocks: vec![(cid, data.clone())],
-                                    response: tx,
-                                })
-                                .await
-                                .unwrap();
-                            rx.await.unwrap().unwrap();
-                            written.lock().unwrap().push((i, data));
-                        }
-                    })
-                    .await;
-            })
-        };
-
-        let reader_handles: Vec<_> = (0..4)
-            .map(|_| {
-                let reader = reader.clone();
-                let written = Arc::clone(&written_cids);
-                let done = Arc::clone(&writer_done);
-                tokio::spawn(async move {
-                    let reads = std::sync::atomic::AtomicU64::new(0);
-
-                    (0..2000)
-                        .take_while(|_| {
-                            let is_done = done.load(std::sync::atomic::Ordering::Relaxed);
-                            let has_reads = reads.load(std::sync::atomic::Ordering::Relaxed) > 0;
-                            !(is_done && has_reads)
-                        })
-                        .for_each(|_| {
-                            let snapshot = written.lock().unwrap().clone();
-                            snapshot.iter().for_each(|(seed, expected_data)| {
-                                let cid = test_cid(*seed);
-                                match reader.get(&cid) {
-                                    Ok(Some(actual)) => {
-                                        assert_eq!(
-                                            &actual[..],
-                                            &expected_data[..],
-                                            "data mismatch for block {seed}"
-                                        );
-                                        reads.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-                                    }
-                                    Ok(None) => {}
-                                    Err(e) => panic!("read error for block {seed}: {e}"),
-                                }
-                            });
-                            std::thread::yield_now();
-                        });
-                    reads.load(std::sync::atomic::Ordering::Relaxed)
-                })
-            })
-            .collect();
-
-        writer_handle.await.unwrap();
-        writer_done.store(true, std::sync::atomic::Ordering::Relaxed);
-
-        let read_counts: Vec<u64> = futures::future::join_all(reader_handles)
-            .await
-            .into_iter()
-            .map(|r| r.unwrap())
-            .collect();
-
-        let total_reads: u64 = read_counts.iter().sum();
-        assert!(
-            total_reads > 0,
-            "readers should have completed at least some reads"
-        );
-
-        let final_snapshot = written_cids.lock().unwrap().clone();
-        assert_eq!(final_snapshot.len(), 50);
-
-        final_snapshot.iter().for_each(|(seed, expected_data)| {
-            let cid = test_cid(*seed);
-            let actual = reader.get(&cid).unwrap().unwrap();
-            assert_eq!(
-                &actual[..],
-                &expected_data[..],
-                "final verification failed for block {seed}"
-            );
-        });
-
-        harness.shutdown();
-    }
-}

crates/tranquil-store/src/blockstore/types.rs

@@ -1,10 +1,104 @@
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicU64, Ordering};
+
 use serde::{Deserialize, Serialize};
 
+use super::data_file::CID_SIZE;
+
+pub type CidBytes = [u8; CID_SIZE];
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+#[repr(transparent)]
+pub struct CommitEpoch(u64);
+
+impl CommitEpoch {
+    pub const fn new(value: u64) -> Self {
+        Self(value)
+    }
+
+    pub const fn zero() -> Self {
+        Self(0)
+    }
+
+    pub fn raw(self) -> u64 {
+        self.0
+    }
+
+    pub fn next(self) -> Self {
+        Self(self.0.saturating_add(1))
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct EpochCounter(Arc<AtomicU64>);
+
+impl Default for EpochCounter {
+    fn default() -> Self {
+        Self(Arc::new(AtomicU64::new(0)))
+    }
+}
+
+impl EpochCounter {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    pub fn from_raw(value: u64) -> Self {
+        Self(Arc::new(AtomicU64::new(value)))
+    }
+
+    pub fn current(&self) -> CommitEpoch {
+        CommitEpoch(self.0.load(Ordering::Acquire))
+    }
+
+    pub fn advance(&self) -> CommitEpoch {
+        let prev = self
+            .0
+            .fetch_update(Ordering::AcqRel, Ordering::Acquire, |v| {
+                Some(v.saturating_add(1))
+            })
+            .unwrap_or(u64::MAX);
+        CommitEpoch(prev.saturating_add(1))
+    }
+}
+
+pub struct CollectionResult {
+    pub candidates: HashMap<DataFileId, Vec<CidBytes>>,
+    pub total_bytes: u64,
+}
+
+pub struct CompactionResult {
+    pub file_id: DataFileId,
+    pub old_size: u64,
+    pub new_size: u64,
+    pub live_blocks: u64,
+    pub dead_blocks: u64,
+    pub reclaimed_bytes: u64,
+}
+
+pub struct LivenessInfo {
+    pub live_bytes: u64,
+    pub total_bytes: u64,
+    pub live_blocks: u64,
+    pub total_blocks: u64,
+}
+
+impl LivenessInfo {
+    pub fn ratio(&self) -> f64 {
+        match self.total_bytes {
+            0 => 1.0,
+            total => self.live_bytes as f64 / total as f64,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+#[repr(transparent)]
 pub struct DataFileId(u32);
 
 impl DataFileId {
-    pub fn new(id: u32) -> Self {
+    pub const fn new(id: u32) -> Self {
         Self(id)
     }
 
@@ -24,10 +118,11 @@ impl std::fmt::Display for DataFileId {
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+#[repr(transparent)]
 pub struct BlockOffset(u64);
 
 impl BlockOffset {
-    pub fn new(offset: u64) -> Self {
+    pub const fn new(offset: u64) -> Self {
         Self(offset)
     }
 
@@ -43,6 +138,7 @@ impl BlockOffset {
 pub const MAX_BLOCK_SIZE: u32 = 4 * 1024 * 1024;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+#[repr(transparent)]
 pub struct BlockLength(u32);
 
 impl BlockLength {
@@ -54,6 +150,10 @@ impl BlockLength {
         Self(length)
     }
 
+    pub const fn from_raw(length: u32) -> Self {
+        Self(length)
+    }
+
     pub fn raw(self) -> u32 {
         self.0
     }
@@ -64,10 +164,11 @@ impl BlockLength {
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+#[repr(transparent)]
 pub struct RefCount(u32);
 
 impl RefCount {
-    pub fn new(count: u32) -> Self {
+    pub const fn new(count: u32) -> Self {
         Self(count)
     }
 
@@ -75,7 +176,7 @@ impl RefCount {
         self.0
     }
 
-    pub fn one() -> Self {
+    pub const fn one() -> Self {
         Self(1)
     }
 
@@ -87,6 +188,10 @@ impl RefCount {
         Self(self.0.checked_add(1).expect("RefCount overflow"))
     }
 
+    pub fn saturating_increment(self) -> Self {
+        Self(self.0.saturating_add(1))
+    }
+
     pub fn decrement(self) -> Self {
         Self(self.0.saturating_sub(1))
     }
@@ -113,6 +218,7 @@ pub struct WriteCursor {
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[repr(transparent)]
 pub struct HintOffset(u64);
 
 impl HintOffset {
@@ -129,10 +235,86 @@ impl HintOffset {
     }
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+#[repr(transparent)]
+pub struct WallClockMs(u64);
+
+impl WallClockMs {
+    pub const fn new(ms: u64) -> Self {
+        Self(ms)
+    }
+
+    pub fn now() -> Self {
+        let millis = std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap_or_default()
+            .as_millis();
+        Self(u64::try_from(millis).unwrap_or(u64::MAX))
+    }
+
+    pub fn raw(self) -> u64 {
+        self.0
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+#[repr(transparent)]
+pub struct ShardId(u8);
+
+impl ShardId {
+    pub const fn new(id: u8) -> Self {
+        Self(id)
+    }
+
+    pub fn raw(self) -> u8 {
+        self.0
+    }
+
+    pub fn as_usize(self) -> usize {
+        self.0 as usize
+    }
+}
+
+impl std::fmt::Display for ShardId {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "shard_{}", self.0)
+    }
+}
+
+pub struct BlockstoreSnapshot {
+    pub shard_cursors: Vec<WriteCursor>,
+    pub epoch: CommitEpoch,
+    pub data_files: Vec<DataFileId>,
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
 
+    #[test]
+    fn commit_epoch_advances() {
+        let e = CommitEpoch::zero();
+        assert_eq!(e.raw(), 0);
+        assert_eq!(e.next().raw(), 1);
+    }
+
+    #[test]
+    fn commit_epoch_saturates() {
+        let e = CommitEpoch::new(u64::MAX);
+        assert_eq!(e.next().raw(), u64::MAX);
+    }
+
+    #[test]
+    fn epoch_counter_advance_returns_new_value() {
+        let counter = EpochCounter::new();
+        assert_eq!(counter.current().raw(), 0);
+        let epoch1 = counter.advance();
+        assert_eq!(epoch1.raw(), 1);
+        assert_eq!(counter.current().raw(), 1);
+        let epoch2 = counter.advance();
+        assert_eq!(epoch2.raw(), 2);
+    }
+
     #[test]
     fn index_entry_postcard_round_trip() {
         let entry = IndexEntry {

crates/tranquil-store/src/bloom.rs

@@ -0,0 +1,134 @@
+use xxhash_rust::xxh3::xxh3_64_with_seed;
+
+pub struct BloomFilter {
+    bits: Vec<u64>,
+    num_bits: u64,
+    num_hashes: u32,
+}
+
+impl BloomFilter {
+    const MAX_BITS: u64 = 1 << 34;
+
+    pub fn with_capacity_and_fpr(expected_items: u64, false_positive_rate: f64) -> Self {
+        debug_assert!(
+            false_positive_rate > 0.0 && false_positive_rate < 1.0,
+            "false_positive_rate must be in (0, 1), got {false_positive_rate}"
+        );
+        let expected = expected_items.max(1) as f64;
+        let ln2 = std::f64::consts::LN_2;
+
+        let num_bits_f = -(expected * false_positive_rate.ln()) / (ln2 * ln2);
+        let num_bits = num_bits_f.ceil().clamp(64.0, Self::MAX_BITS as f64) as u64;
+        let num_bits = num_bits.next_power_of_two();
+
+        let optimal_k = ((num_bits as f64 / expected) * ln2).ceil();
+        let num_hashes = (optimal_k as u32).clamp(1, 16);
+
+        let words = (num_bits / 64) as usize;
+
+        Self {
+            bits: vec![0u64; words],
+            num_bits,
+            num_hashes,
+        }
+    }
+
+    pub fn insert(&mut self, key: &[u8]) {
+        let mask = self.num_bits - 1;
+        (0..self.num_hashes).for_each(|i| {
+            let h = xxh3_64_with_seed(key, u64::from(i)) & mask;
+            let word = (h / 64) as usize;
+            let bit = h % 64;
+            self.bits[word] |= 1u64 << bit;
+        });
+    }
+
+    pub fn contains(&self, key: &[u8]) -> bool {
+        let mask = self.num_bits - 1;
+        (0..self.num_hashes).all(|i| {
+            let h = xxh3_64_with_seed(key, u64::from(i)) & mask;
+            let word = (h / 64) as usize;
+            let bit = h % 64;
+            (self.bits[word] >> bit) & 1 == 1
+        })
+    }
+
+    pub fn heap_bytes(&self) -> usize {
+        self.bits.len() * 8
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn insert_and_contains() {
+        let mut bf = BloomFilter::with_capacity_and_fpr(1000, 0.01);
+        bf.insert(b"hello");
+        bf.insert(b"world");
+
+        assert!(bf.contains(b"hello"));
+        assert!(bf.contains(b"world"));
+    }
+
+    #[test]
+    fn missing_key_usually_absent() {
+        let mut bf = BloomFilter::with_capacity_and_fpr(1000, 0.01);
+        (0u32..500).for_each(|i| bf.insert(&i.to_le_bytes()));
+
+        let false_positives = (1000u32..2000)
+            .filter(|i| bf.contains(&i.to_le_bytes()))
+            .count();
+
+        assert!(
+            false_positives < 50,
+            "expected <5% FPR, got {false_positives}/1000"
+        );
+    }
+
+    #[test]
+    fn no_false_negatives() {
+        let mut bf = BloomFilter::with_capacity_and_fpr(10_000, 0.01);
+        let keys: Vec<[u8; 4]> = (0u32..10_000).map(|i| i.to_le_bytes()).collect();
+        keys.iter().for_each(|k| bf.insert(k));
+        assert!(keys.iter().all(|k| bf.contains(k)));
+    }
+
+    #[test]
+    fn empty_filter_contains_nothing() {
+        let bf = BloomFilter::with_capacity_and_fpr(1000, 0.01);
+        assert!(!bf.contains(b"anything"));
+    }
+
+    #[test]
+    fn heap_bytes_reasonable() {
+        let bf = BloomFilter::with_capacity_and_fpr(100_000_000, 0.01);
+        let mb = bf.heap_bytes() / (1024 * 1024);
+        assert!(
+            mb < 256,
+            "100M items at 1% FPR should be <256MB, got {mb}MB"
+        );
+        assert!(mb > 64, "100M items at 1% FPR should be >64MB, got {mb}MB");
+    }
+
+    #[test]
+    fn fpr_empirical() {
+        let n = 50_000u32;
+        let target_fpr = 0.01;
+        let mut bf = BloomFilter::with_capacity_and_fpr(n as u64, target_fpr);
+
+        (0..n).for_each(|i| bf.insert(&i.to_le_bytes()));
+
+        let test_range = 100_000u32;
+        let false_positives = (n..n + test_range)
+            .filter(|i| bf.contains(&i.to_le_bytes()))
+            .count();
+        let measured_fpr = false_positives as f64 / test_range as f64;
+
+        assert!(
+            measured_fpr < target_fpr * 3.0,
+            "measured FPR {measured_fpr:.4} exceeds 3x target {target_fpr}"
+        );
+    }
+}