mirrors/tranquil-pds
1
0
Fork 0
mirror of https://tangled.org/tranquil.farm/tranquil-pds synced 2026-09-16 23:34:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(tranquil-store): gc and hardening

Browse Source 
Lewis: May this revision serve well! <lu5a@proton.me>
This commit is contained in:
Lewis
2026-04-10 13:57:47 +03:00
parent 8d0e0c6810
commit 05246aec58
131 changed files with 23215 additions and 5946 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.config/nextest.toml
+27
View File
@@ -13,9 +13,22 @@ fail-fast = false
test-threads = "num-cpus"
slow-timeout = { period = "30s", terminate-after = 4 }
[profile.sim-pr]
retries = 0
fail-fast = true
test-threads = "num-cpus"
slow-timeout = { period = "60s", terminate-after = 4 }
[profile.sim-nightly]
retries = 0
fail-fast = false
test-threads = "num-cpus"
slow-timeout = { period = "300s", terminate-after = 2 }
[test-groups]
serial-env-tests = { max-threads = 1 }
heavy-load-tests = { max-threads = 4 }
io-heavy-sim = { max-threads = 2 }
[[profile.default.overrides]]
filter = "test(/import_with_verification/) | test(/plc_migration/)"
@@ -45,6 +58,20 @@ test-group = "heavy-load-tests"
filter = "binary(repo_lifecycle)"
test-group = "heavy-load-tests"
[[profile.default.overrides]]
filter = "binary(sim_soak)"
slow-timeout = { period = "300s", terminate-after = 10 }
test-group = "io-heavy-sim"
[[profile.default.overrides]]
filter = "binary(metastore_crash) | binary(sim_cross_store) | binary(sim_gc_concurrent) | binary(sim_reachability) | binary(sim_eventlog) | test(/sim_/)"
slow-timeout = { period = "120s", terminate-after = 8 }
test-group = "io-heavy-sim"
[[profile.default.overrides]]
filter = "test(/test_scale_/) | test(/full_backup_and_restore/)"
slow-timeout = { period = "120s", terminate-after = 4 }
[[profile.ci.overrides]]
filter = "test(/import_with_verification/) | test(/plc_migration/)"
test-group = "serial-env-tests"
.sqlx/query-033ed5638ef6e30f1648c1acdbc477c173798cb57eb0127c3a582a9633fa5012.json
Generated
-32
View File
@@ -1,32 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT seq, did, commit_cid\n FROM repo_seq\n WHERE event_type = 'commit'\n AND prev_cid IS NULL\n AND (blocks_cids IS NULL OR array_length(blocks_cids, 1) IS NULL OR array_length(blocks_cids, 1) = 0)\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "seq",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "did",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "commit_cid",
"type_info": "Text"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
false,
true
]
},
"hash": "033ed5638ef6e30f1648c1acdbc477c173798cb57eb0127c3a582a9633fa5012"
}
.sqlx/query-d8524ad3f5dc03eb09ed60396a78df5003f804c43ad253d6476523eacdebf811.json → .sqlx/query-0349b2af4d21105029ecc2bf81ad7be9987ec17ac088730596849d7a45558e57.json
Generated
+18 -6
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n ops, blobs, blocks_cids, handle, active, status, rev\n FROM repo_seq\n WHERE seq > $1 AND seq < $2\n ORDER BY seq ASC",
"query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev\n FROM repo_seq\n WHERE seq > $1\n ORDER BY seq ASC\n LIMIT $2",
"describe": {
"columns": [
{
@@ -50,26 +50,36 @@
},
{
"ordinal": 9,
"name": "block_cids",
"type_info": "ByteaArray"
},
{
"ordinal": 10,
"name": "block_data",
"type_info": "ByteaArray"
},
{
"ordinal": 11,
"name": "blocks_cids",
"type_info": "TextArray"
},
{
"ordinal": 10,
"ordinal": 12,
"name": "handle",
"type_info": "Text"
},
{
"ordinal": 11,
"ordinal": 13,
"name": "active",
"type_info": "Bool"
},
{
"ordinal": 12,
"ordinal": 14,
"name": "status",
"type_info": "Text"
},
{
"ordinal": 13,
"ordinal": 15,
"name": "rev",
"type_info": "Text"
}
@@ -94,8 +104,10 @@
true,
true,
true,
true,
true,
true
]
},
"hash": "d8524ad3f5dc03eb09ed60396a78df5003f804c43ad253d6476523eacdebf811"
"hash": "0349b2af4d21105029ecc2bf81ad7be9987ec17ac088730596849d7a45558e57"
}
.sqlx/query-12f5864ebff622fc52643de7151a40e984082851741b22f63a170728e734763b.json
Generated
-22
View File
@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT cid FROM blocks ORDER BY created_at ASC LIMIT $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "cid",
"type_info": "Bytea"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
false
]
},
"hash": "12f5864ebff622fc52643de7151a40e984082851741b22f63a170728e734763b"
}
.sqlx/query-18fa821e4bd00ccf5d1d8395ba728e4905d69f9fe527b4d4b49c69deff52cea8.json
Generated
-22
View File
@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT t.cid FROM UNNEST($1::bytea[]) AS t(cid)\n WHERE NOT EXISTS (\n SELECT 1 FROM user_blocks WHERE block_cid = t.cid\n )\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "cid",
"type_info": "Bytea"
}
],
"parameters": {
"Left": [
"ByteaArray"
]
},
"nullable": [
null
]
},
"hash": "18fa821e4bd00ccf5d1d8395ba728e4905d69f9fe527b4d4b49c69deff52cea8"
}
.sqlx/query-3b791fdb8e29043c980963d4d18e1e492c73c39818a8648a7af70555418fb5d1.json
Generated
-15
View File
@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE repo_seq SET blocks_cids = $1 WHERE seq = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"TextArray",
"Int8"
]
},
"nullable": []
},
"hash": "3b791fdb8e29043c980963d4d18e1e492c73c39818a8648a7af70555418fb5d1"
}
.sqlx/query-b26bf97a27783eb7fb524a92dda3e68ef8470a9751fcaefe5fd2d7909dead54b.json → .sqlx/query-4702a94384ce938d3f5384bc2106a48500f8749134e6bfd1c619a9151f5d3772.json
Generated
+18 -7
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n ops, blobs, blocks_cids, handle, active, status, rev\n FROM repo_seq\n WHERE seq > $1\n ORDER BY seq ASC\n LIMIT $2",
"query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev\n FROM repo_seq\n WHERE seq = $1",
"describe": {
"columns": [
{
@@ -50,33 +50,42 @@
},
{
"ordinal": 9,
"name": "block_cids",
"type_info": "ByteaArray"
},
{
"ordinal": 10,
"name": "block_data",
"type_info": "ByteaArray"
},
{
"ordinal": 11,
"name": "blocks_cids",
"type_info": "TextArray"
},
{
"ordinal": 10,
"ordinal": 12,
"name": "handle",
"type_info": "Text"
},
{
"ordinal": 11,
"ordinal": 13,
"name": "active",
"type_info": "Bool"
},
{
"ordinal": 12,
"ordinal": 14,
"name": "status",
"type_info": "Text"
},
{
"ordinal": 13,
"ordinal": 15,
"name": "rev",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Int8",
"Int8"
]
},
@@ -94,8 +103,10 @@
true,
true,
true,
true,
true,
true
]
},
"hash": "b26bf97a27783eb7fb524a92dda3e68ef8470a9751fcaefe5fd2d7909dead54b"
"hash": "4702a94384ce938d3f5384bc2106a48500f8749134e6bfd1c619a9151f5d3772"
}
.sqlx/query-bce9edc597592d2e7d672061883639bcdeff80be85e2ecacb6c75d8c027e6710.json → .sqlx/query-5996774d097a484f81353d49e4a99b07c30a003377c924add576bfb48ddde29b.json
Generated
+5 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO repo_seq (did, event_type, commit_cid, rev)\n VALUES ($1, 'sync', $2, $3)\n RETURNING seq\n ",
"query": "\n INSERT INTO repo_seq (did, event_type, commit_cid, rev, block_cids, block_data)\n VALUES ($1, 'sync', $2, $3, $4, $5)\n RETURNING seq\n ",
"describe": {
"columns": [
{
@@ -13,12 +13,14 @@
"Left": [
"Text",
"Text",
"Text"
"Text",
"ByteaArray",
"ByteaArray"
]
},
"nullable": [
false
]
},
"hash": "bce9edc597592d2e7d672061883639bcdeff80be85e2ecacb6c75d8c027e6710"
"hash": "5996774d097a484f81353d49e4a99b07c30a003377c924add576bfb48ddde29b"
}
.sqlx/query-746ca87e98f6eaa9c060770c0d1bca6f0d4da60df2dc2db088e57bdc914cf233.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM repo_seq WHERE created_at < $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Timestamptz"
]
},
"nullable": []
},
"hash": "746ca87e98f6eaa9c060770c0d1bca6f0d4da60df2dc2db088e57bdc914cf233"
}
.sqlx/query-b8101757a50075d20147014e450cb7deb7e58f84310690c7bde61e1834dc5903.json → .sqlx/query-89c8ce80d8c52b4668c34f63e48b51cacc7e8ff300958a192378f1d72de04f9a.json
Generated
+19 -6
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n ops, blobs, blocks_cids, handle, active, status, rev\n FROM repo_seq\n WHERE seq > $1\n ORDER BY seq ASC",
"query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev\n FROM repo_seq\n WHERE seq > $1\n ORDER BY seq ASC\n LIMIT $2",
"describe": {
"columns": [
{
@@ -50,32 +50,43 @@
},
{
"ordinal": 9,
"name": "block_cids",
"type_info": "ByteaArray"
},
{
"ordinal": 10,
"name": "block_data",
"type_info": "ByteaArray"
},
{
"ordinal": 11,
"name": "blocks_cids",
"type_info": "TextArray"
},
{
"ordinal": 10,
"ordinal": 12,
"name": "handle",
"type_info": "Text"
},
{
"ordinal": 11,
"ordinal": 13,
"name": "active",
"type_info": "Bool"
},
{
"ordinal": 12,
"ordinal": 14,
"name": "status",
"type_info": "Text"
},
{
"ordinal": 13,
"ordinal": 15,
"name": "rev",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Int8",
"Int8"
]
},
@@ -93,8 +104,10 @@
true,
true,
true,
true,
true,
true
]
},
"hash": "b8101757a50075d20147014e450cb7deb7e58f84310690c7bde61e1834dc5903"
"hash": "89c8ce80d8c52b4668c34f63e48b51cacc7e8ff300958a192378f1d72de04f9a"
}
.sqlx/query-8eecf8fef308716be88815eb59bb67ec7c534b3c821d55481b110e3e462ee366.json
Generated
-14
View File
@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM blocks WHERE cid = ANY($1)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"ByteaArray"
]
},
"nullable": []
},
"hash": "8eecf8fef308716be88815eb59bb67ec7c534b3c821d55481b110e3e462ee366"
}
.sqlx/query-40e848f48afb50f5e4820445aee874a482333e9090ef14ce44db3fd9721cfff0.json → .sqlx/query-97dd3cfe72b74b4dac686c030a07f6eb56dabb2df72815f3248760b4e3f7950e.json
Generated
+4 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, blocks_cids, prev_data_cid, rev)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)\n RETURNING seq\n ",
"query": "\n INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, block_cids, block_data, prev_data_cid, rev)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)\n RETURNING seq\n ",
"describe": {
"columns": [
{
@@ -17,7 +17,8 @@
"Text",
"Jsonb",
"TextArray",
"TextArray",
"ByteaArray",
"ByteaArray",
"Text",
"Text"
]
@@ -26,5 +27,5 @@
false
]
},
"hash": "40e848f48afb50f5e4820445aee874a482333e9090ef14ce44db3fd9721cfff0"
"hash": "97dd3cfe72b74b4dac686c030a07f6eb56dabb2df72815f3248760b4e3f7950e"
}
.sqlx/query-52df8ae809b990fa76e3ac45f37e352a9fc8f92f911a7b9c9be2fd9e8032b18c.json → .sqlx/query-a325862f897484bb1a8863fd7a1bc3fd17ab20b6e30498f9900daf80609c413d.json
Generated
+4 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, blocks_cids, rev)\n VALUES ($1, 'commit', $2, $3::TEXT, $4, $5, $6, $7)\n RETURNING seq\n ",
"query": "\n INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, block_cids, block_data, rev)\n VALUES ($1, 'commit', $2, $3::TEXT, $4, $5, $6, $7, $8)\n RETURNING seq\n ",
"describe": {
"columns": [
{
@@ -16,7 +16,8 @@
"Text",
"Jsonb",
"TextArray",
"TextArray",
"ByteaArray",
"ByteaArray",
"Text"
]
},
@@ -24,5 +25,5 @@
false
]
},
"hash": "52df8ae809b990fa76e3ac45f37e352a9fc8f92f911a7b9c9be2fd9e8032b18c"
"hash": "a325862f897484bb1a8863fd7a1bc3fd17ab20b6e30498f9900daf80609c413d"
}
.sqlx/query-e7aa1080be9eb3a8ddf1f050c93dc8afd10478f41e22307014784b4ee3740b4a.json → .sqlx/query-ad784aff9bb5c1f19f90a3b1c3b3694a3e1aa889b0fe3e55cfebb7af8c620256.json
Generated
+18 -6
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n ops, blobs, blocks_cids, handle, active, status, rev\n FROM repo_seq\n WHERE seq > $1\n ORDER BY seq ASC\n LIMIT $2",
"query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev\n FROM repo_seq\n WHERE seq > $1 AND seq < $2\n ORDER BY seq ASC",
"describe": {
"columns": [
{
@@ -50,26 +50,36 @@
},
{
"ordinal": 9,
"name": "block_cids",
"type_info": "ByteaArray"
},
{
"ordinal": 10,
"name": "block_data",
"type_info": "ByteaArray"
},
{
"ordinal": 11,
"name": "blocks_cids",
"type_info": "TextArray"
},
{
"ordinal": 10,
"ordinal": 12,
"name": "handle",
"type_info": "Text"
},
{
"ordinal": 11,
"ordinal": 13,
"name": "active",
"type_info": "Bool"
},
{
"ordinal": 12,
"ordinal": 14,
"name": "status",
"type_info": "Text"
},
{
"ordinal": 13,
"ordinal": 15,
"name": "rev",
"type_info": "Text"
}
@@ -94,8 +104,10 @@
true,
true,
true,
true,
true,
true
]
},
"hash": "e7aa1080be9eb3a8ddf1f050c93dc8afd10478f41e22307014784b4ee3740b4a"
"hash": "ad784aff9bb5c1f19f90a3b1c3b3694a3e1aa889b0fe3e55cfebb7af8c620256"
}
.sqlx/query-c15d538fece4cd7db0056f9e0374a2290e26f5766881fd9dbf4f05788845aac0.json
Generated
-29
View File
@@ -1,29 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT blocks_cids, commit_cid\n FROM repo_seq\n WHERE did = $1 AND rev > $2\n ORDER BY seq DESC",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "blocks_cids",
"type_info": "TextArray"
},
{
"ordinal": 1,
"name": "commit_cid",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
true,
true
]
},
"hash": "c15d538fece4cd7db0056f9e0374a2290e26f5766881fd9dbf4f05788845aac0"
}
.sqlx/query-0d32a592a97ad47c65aa37cf0d45417f2966fcbd688be7434626ae5f6971fa1f.json → .sqlx/query-c47746c256fd8c9d386551f3bd270e9a96fecdb713642d07d60fb4aeb02c8426.json
Generated
+18 -6
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n ops, blobs, blocks_cids, handle, active, status, rev\n FROM repo_seq\n WHERE seq = $1",
"query": "SELECT seq, did, created_at, event_type as \"event_type: RepoEventType\", commit_cid, prev_cid, prev_data_cid,\n ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev\n FROM repo_seq\n WHERE seq > $1\n ORDER BY seq ASC",
"describe": {
"columns": [
{
@@ -50,26 +50,36 @@
},
{
"ordinal": 9,
"name": "block_cids",
"type_info": "ByteaArray"
},
{
"ordinal": 10,
"name": "block_data",
"type_info": "ByteaArray"
},
{
"ordinal": 11,
"name": "blocks_cids",
"type_info": "TextArray"
},
{
"ordinal": 10,
"ordinal": 12,
"name": "handle",
"type_info": "Text"
},
{
"ordinal": 11,
"ordinal": 13,
"name": "active",
"type_info": "Bool"
},
{
"ordinal": 12,
"ordinal": 14,
"name": "status",
"type_info": "Text"
},
{
"ordinal": 13,
"ordinal": 15,
"name": "rev",
"type_info": "Text"
}
@@ -93,8 +103,10 @@
true,
true,
true,
true,
true,
true
]
},
"hash": "0d32a592a97ad47c65aa37cf0d45417f2966fcbd688be7434626ae5f6971fa1f"
"hash": "c47746c256fd8c9d386551f3bd270e9a96fecdb713642d07d60fb4aeb02c8426"
}
Cargo.lock
Generated
+27 -22
View File
@@ -7405,7 +7405,7 @@ dependencies = [
[[package]]
name = "tranquil-api"
version = "0.4.7"
version = "0.5.0"
dependencies = [
"anyhow",
"axum",
@@ -7456,7 +7456,7 @@ dependencies = [
[[package]]
name = "tranquil-auth"
version = "0.4.7"
version = "0.5.0"
dependencies = [
"anyhow",
"base32",
@@ -7479,7 +7479,7 @@ dependencies = [
[[package]]
name = "tranquil-cache"
version = "0.4.7"
version = "0.5.0"
dependencies = [
"async-trait",
"base64 0.22.1",
@@ -7493,7 +7493,7 @@ dependencies = [
[[package]]
name = "tranquil-comms"
version = "0.4.7"
version = "0.5.0"
dependencies = [
"async-trait",
"base64 0.22.1",
@@ -7511,7 +7511,7 @@ dependencies = [
[[package]]
name = "tranquil-config"
version = "0.4.7"
version = "0.5.0"
dependencies = [
"confique",
"serde",
@@ -7519,7 +7519,7 @@ dependencies = [
[[package]]
name = "tranquil-crypto"
version = "0.4.7"
version = "0.5.0"
dependencies = [
"aes-gcm",
"base64 0.22.1",
@@ -7535,7 +7535,7 @@ dependencies = [
[[package]]
name = "tranquil-db"
version = "0.4.7"
version = "0.5.0"
dependencies = [
"async-trait",
"chrono",
@@ -7552,7 +7552,7 @@ dependencies = [
[[package]]
name = "tranquil-db-traits"
version = "0.4.7"
version = "0.5.0"
dependencies = [
"async-trait",
"base64 0.22.1",
@@ -7568,7 +7568,7 @@ dependencies = [
[[package]]
name = "tranquil-infra"
version = "0.4.7"
version = "0.5.0"
dependencies = [
"async-trait",
"bytes",
@@ -7579,7 +7579,7 @@ dependencies = [
[[package]]
name = "tranquil-lexicon"
version = "0.4.7"
version = "0.5.0"
dependencies = [
"chrono",
"hickory-resolver",
@@ -7597,7 +7597,7 @@ dependencies = [
[[package]]
name = "tranquil-oauth"
version = "0.4.7"
version = "0.5.0"
dependencies = [
"anyhow",
"axum",
@@ -7620,7 +7620,7 @@ dependencies = [
[[package]]
name = "tranquil-oauth-server"
version = "0.4.7"
version = "0.5.0"
dependencies = [
"axum",
"base64 0.22.1",
@@ -7653,7 +7653,7 @@ dependencies = [
[[package]]
name = "tranquil-pds"
version = "0.4.7"
version = "0.5.0"
dependencies = [
"aes-gcm",
"anyhow",
@@ -7744,7 +7744,7 @@ dependencies = [
[[package]]
name = "tranquil-repo"
version = "0.4.7"
version = "0.5.0"
dependencies = [
"bytes",
"cid",
@@ -7756,7 +7756,7 @@ dependencies = [
[[package]]
name = "tranquil-ripple"
version = "0.4.7"
version = "0.5.0"
dependencies = [
"async-trait",
"backon",
@@ -7781,7 +7781,7 @@ dependencies = [
[[package]]
name = "tranquil-scopes"
version = "0.4.7"
version = "0.5.0"
dependencies = [
"axum",
"futures",
@@ -7797,7 +7797,7 @@ dependencies = [
[[package]]
name = "tranquil-server"
version = "0.4.7"
version = "0.5.0"
dependencies = [
"axum",
"clap",
@@ -7818,7 +7818,7 @@ dependencies = [
[[package]]
name = "tranquil-signal"
version = "0.4.7"
version = "0.5.0"
dependencies = [
"async-trait",
"chrono",
@@ -7829,17 +7829,19 @@ dependencies = [
"serde",
"serde_json",
"sqlx",
"tempfile",
"thiserror 2.0.18",
"tokio",
"tokio-util",
"tracing",
"tranquil-signal",
"url",
"uuid",
]
[[package]]
name = "tranquil-storage"
version = "0.4.7"
version = "0.5.0"
dependencies = [
"async-trait",
"aws-config",
@@ -7856,7 +7858,7 @@ dependencies = [
[[package]]
name = "tranquil-store"
version = "0.4.7"
version = "0.5.0"
dependencies = [
"async-trait",
"bytes",
@@ -7876,6 +7878,7 @@ dependencies = [
"postcard",
"proptest",
"rand 0.8.5",
"rayon",
"serde",
"serde_ipld_dagcbor",
"serde_json",
@@ -7888,10 +7891,12 @@ dependencies = [
"tikv-jemallocator",
"tokio",
"tracing",
"tracing-subscriber",
"tranquil-db",
"tranquil-db-traits",
"tranquil-oauth",
"tranquil-repo",
"tranquil-store",
"tranquil-types",
"uuid",
"xxhash-rust",
@@ -7899,7 +7904,7 @@ dependencies = [
[[package]]
name = "tranquil-sync"
version = "0.4.7"
version = "0.5.0"
dependencies = [
"anyhow",
"axum",
@@ -7921,7 +7926,7 @@ dependencies = [
[[package]]
name = "tranquil-types"
version = "0.4.7"
version = "0.5.0"
dependencies = [
"chrono",
"cid",
crates/tranquil-api/src/admin/account/mod.rs
+3 -2
View File
@@ -11,6 +11,7 @@ pub use info::{
};
pub use search::{SearchAccountsOutput, SearchAccountsParams, search_accounts};
pub use update::{
UpdateAccountEmailInput, UpdateAccountHandleInput, UpdateAccountPasswordInput,
update_account_email, update_account_handle, update_account_password,
SetAdminStatusInput, UpdateAccountEmailInput, UpdateAccountHandleInput,
UpdateAccountPasswordInput, set_admin_status, update_account_email, update_account_handle,
update_account_password,
};
crates/tranquil-api/src/admin/account/update.rs
+32 -1
View File
@@ -1,6 +1,6 @@
use axum::{Json, extract::State};
use serde::Deserialize;
use tracing::{error, warn};
use tracing::{error, info, warn};
use tranquil_pds::api::EmptyResponse;
use tranquil_pds::api::error::ApiError;
use tranquil_pds::auth::{Admin, Auth};
@@ -166,3 +166,34 @@ pub async fn update_account_password(
}
}
}
#[derive(Deserialize)]
pub struct SetAdminStatusInput {
pub did: Did,
pub admin: bool,
}
pub async fn set_admin_status(
State(state): State<AppState>,
auth: Auth<Admin>,
Json(input): Json<SetAdminStatusInput>,
) -> Result<Json<EmptyResponse>, ApiError> {
info!(
actor = %auth.did,
target = %input.did,
admin = input.admin,
"admin status change"
);
state
.repos
.user
.set_admin_status(&input.did, input.admin)
.await
.map_err(|e| {
error!("DB error setting admin status: {:?}", e);
ApiError::InternalError(None)
})?;
Ok(Json(EmptyResponse {}))
}
crates/tranquil-api/src/admin/mod.rs
+1 -1
View File
@@ -7,7 +7,7 @@ pub mod status;
pub use account::{
delete_account, get_account_info, get_account_infos, search_accounts, send_email,
update_account_email, update_account_handle, update_account_password,
set_admin_status, update_account_email, update_account_handle, update_account_password,
};
pub use config::{get_server_config, update_server_config};
pub use invite::{
crates/tranquil-api/src/lib.rs
+1
View File
@@ -330,6 +330,7 @@ pub fn api_routes() -> axum::Router<AppState> {
get(admin::get_invite_codes),
)
.route("/_admin.getServerStats", get(admin::get_server_stats))
.route("/_admin.setAdminStatus", post(admin::set_admin_status))
.route("/_admin.getSignalStatus", get(admin::get_signal_status))
.route("/_admin.linkSignalDevice", post(admin::link_signal_device))
.route(
crates/tranquil-api/src/repo/import.rs
+14 -5
View File
@@ -328,7 +328,8 @@ pub async fn import_repo(
new_root_str, new_rev_str
);
if !is_migration
&& let Err(e) = sequence_import_event(&state, did, &new_root_cid_link).await
&& let Err(e) =
sequence_import_event(&state, did, &new_root_cid_link, &commit_bytes).await
{
warn!("Failed to sequence import event: {:?}", e);
}
@@ -395,15 +396,23 @@ async fn sequence_import_event(
state: &AppState,
did: &Did,
commit_cid: &CidLink,
) -> Result<(), tranquil_db::DbError> {
let data = tranquil_db::CommitEventData {
commit_bytes: &[u8],
) -> Result<(), tranquil_db_traits::DbError> {
let commit_cid_parsed = commit_cid
.to_cid()
.expect("CidLink invariant: validated at construction");
let inline_commit = tranquil_db_traits::EventBlockInline {
cid_bytes: commit_cid_parsed.to_bytes(),
data: commit_bytes.to_vec(),
};
let data = tranquil_db_traits::CommitEventData {
did: did.clone(),
event_type: tranquil_db::RepoEventType::Commit,
event_type: tranquil_db_traits::RepoEventType::Commit,
commit_cid: Some(commit_cid.clone()),
prev_cid: None,
ops: Some(serde_json::json!([])),
blobs: Some(vec![]),
blocks_cids: Some(vec![]),
blocks: Some(vec![inline_commit]),
prev_data_cid: None,
rev: None,
};
crates/tranquil-api/src/repo/record/batch.rs
+22 -6
View File
@@ -94,7 +94,7 @@ async fn process_single_write(
ops.push(RecordOp::Create {
collection: collection.clone(),
rkey: rkey.clone(),
cid: record_cid,
cid: tranquil_pds::cid_types::RecordCid::from(record_cid),
});
Ok(WriteAccumulator {
mst: new_mst,
@@ -134,7 +134,15 @@ async fn process_single_write(
.map_err(|_| ApiError::InternalError(Some("Failed to store record".into())))?;
let key = format!("{}/{}", collection, rkey);
modified_keys.push(key.clone());
let prev_record_cid = mst.get(&key).await.ok().flatten();
let prev_record_cid = mst
.get(&key)
.await
.map_err(|e| {
ApiError::InternalError(Some(format!("Failed to read prev record: {}", e)))
})?
.ok_or_else(|| {
ApiError::InvalidRequest("Update target record does not exist".into())
})?;
let new_mst = mst
.update(&key, record_cid)
.await
@@ -150,8 +158,8 @@ async fn process_single_write(
ops.push(RecordOp::Update {
collection: collection.clone(),
rkey: rkey.clone(),
cid: record_cid,
prev: prev_record_cid,
cid: tranquil_pds::cid_types::RecordCid::from(record_cid),
prev: tranquil_pds::cid_types::RecordCid::from(prev_record_cid),
});
Ok(WriteAccumulator {
mst: new_mst,
@@ -166,7 +174,15 @@ async fn process_single_write(
WriteOp::Delete { collection, rkey } => {
let key = format!("{}/{}", collection, rkey);
modified_keys.push(key.clone());
let prev_record_cid = mst.get(&key).await.ok().flatten();
let prev_record_cid = mst
.get(&key)
.await
.map_err(|e| {
ApiError::InternalError(Some(format!("Failed to read prev record: {}", e)))
})?
.ok_or_else(|| {
ApiError::InvalidRequest("Delete target record does not exist".into())
})?;
let new_mst = mst
.delete(&key)
.await
@@ -176,7 +192,7 @@ async fn process_single_write(
ops.push(RecordOp::Delete {
collection: collection.clone(),
rkey: rkey.clone(),
prev: prev_record_cid,
prev: tranquil_pds::cid_types::RecordCid::from(prev_record_cid),
});
Ok(WriteAccumulator {
mst: new_mst,
crates/tranquil-api/src/repo/record/delete.rs
+11 -133
View File
@@ -1,20 +1,16 @@
use crate::repo::record::write::{CommitInfo, prepare_repo_write};
use axum::{Json, extract::State};
use cid::Cid;
use jacquard_repo::{commit::Commit, mst::Mst, storage::BlockStore};
use serde::{Deserialize, Serialize};
use serde_json::json;
use std::str::FromStr;
use std::sync::Arc;
use tracing::error;
use tranquil_pds::api::error::ApiError;
use tranquil_pds::auth::{Active, Auth, VerifyScope};
use tranquil_pds::repo::TrackingBlockStore;
use tranquil_pds::repo_ops::{
CommitError, FinalizeParams, RecordOp, begin_repo_write, finalize_repo_write,
};
use tranquil_pds::cid_types::RecordCid;
use tranquil_pds::repo_ops::{FinalizeParams, RecordOp, begin_repo_write, finalize_repo_write};
use tranquil_pds::state::AppState;
use tranquil_pds::types::{AtIdentifier, AtUri, Did, Nsid, Rkey};
use tranquil_pds::types::{AtIdentifier, AtUri, Nsid, Rkey};
#[derive(Deserialize)]
pub struct DeleteRecordInput {
@@ -59,20 +55,23 @@ pub async fn delete_record(
}
}
let prev_record_cid = mst.get(&key).await.ok().flatten();
if prev_record_cid.is_none() {
let prev_record_cid = mst.get(&key).await.map_err(|e| {
error!("Failed to read prev record from MST: {}", e);
ApiError::InternalError(Some("Failed to read MST".into()))
})?;
let Some(prev_record_cid) = prev_record_cid else {
return Ok(Json(DeleteRecordOutput { commit: None }));
}
};
let new_mst = mst.delete(&key).await.map_err(|e| {
error!("Failed to delete from MST: {:?}", e);
error!("Failed to delete from MST: {}", e);
ApiError::InternalError(Some("Failed to delete from MST".into()))
})?;
let op = RecordOp::Delete {
collection: input.collection.clone(),
rkey: input.rkey.clone(),
prev: prev_record_cid,
prev: RecordCid::from(prev_record_cid),
};
let modified_keys = [key];
@@ -109,124 +108,3 @@ pub async fn delete_record(
}),
}))
}
use uuid::Uuid;
pub async fn delete_record_internal(
state: &AppState,
did: &Did,
user_id: Uuid,
collection: &Nsid,
rkey: &Rkey,
) -> Result<(), CommitError> {
use tranquil_pds::repo_ops::{CommitParams, RecordOp, commit_and_log};
let _write_lock = state.repo_write_locks.lock(user_id).await;
let root_cid_str = state
.repos
.repo
.get_repo_root_cid_by_user_id(user_id)
.await
.map_err(|e| CommitError::DatabaseError(e.to_string()))?
.ok_or(CommitError::RepoNotFound)?;
let current_root_cid =
Cid::from_str(root_cid_str.as_str()).map_err(|e| CommitError::InvalidCid(e.to_string()))?;
let tracking_store = TrackingBlockStore::new(state.block_store.clone());
let commit_bytes = tracking_store
.get(&current_root_cid)
.await
.map_err(|e| CommitError::BlockStoreFailed(format!("{:?}", e)))?
.ok_or(CommitError::BlockStoreFailed(
"Commit block not found".into(),
))?;
let commit = Commit::from_cbor(&commit_bytes)
.map_err(|e| CommitError::CommitParseFailed(format!("{:?}", e)))?;
let mst = Mst::load(Arc::new(tracking_store.clone()), commit.data, None);
let key = format!("{}/{}", collection, rkey);
let prev_record_cid = mst
.get(&key)
.await
.map_err(|e| CommitError::MstOperationFailed(format!("{:?}", e)))?;
let Some(prev_cid) = prev_record_cid else {
return Ok(());
};
let new_mst = mst
.delete(&key)
.await
.map_err(|e| CommitError::MstOperationFailed(format!("{:?}", e)))?;
let new_mst_root = new_mst
.persist()
.await
.map_err(|e| CommitError::MstOperationFailed(format!("{:?}", e)))?;
let op = RecordOp::Delete {
collection: collection.clone(),
rkey: rkey.clone(),
prev: Some(prev_cid),
};
let mut new_mst_blocks = std::collections::BTreeMap::new();
let mut old_mst_blocks = std::collections::BTreeMap::new();
new_mst
.blocks_for_path(&key, &mut new_mst_blocks)
.await
.map_err(|e| CommitError::MstOperationFailed(format!("{:?}", e)))?;
mst.blocks_for_path(&key, &mut old_mst_blocks)
.await
.map_err(|e| CommitError::MstOperationFailed(format!("{:?}", e)))?;
let obsolete_cids: Vec<Cid> = std::iter::once(current_root_cid)
.chain(
old_mst_blocks
.keys()
.filter(|cid| !new_mst_blocks.contains_key(*cid))
.copied(),
)
.chain(std::iter::once(prev_cid))
.collect();
let mut relevant_blocks = new_mst_blocks;
relevant_blocks.extend(old_mst_blocks);
let written_cids: Vec<Cid> = tracking_store
.get_all_relevant_cids()
.into_iter()
.chain(relevant_blocks.keys().copied())
.collect::<std::collections::HashSet<_>>()
.into_iter()
.collect();
let written_cids_str: Vec<String> = written_cids.iter().map(ToString::to_string).collect();
let deleted_uri = AtUri::from_parts(did.as_str(), collection.as_str(), rkey.as_str());
commit_and_log(
state,
CommitParams {
did,
user_id,
current_root_cid: Some(current_root_cid),
prev_data_cid: Some(commit.data),
new_mst_root,
ops: vec![op],
blocks_cids: &written_cids_str,
blobs: &[],
obsolete_cids,
backlinks_to_add: vec![],
backlinks_to_remove: vec![deleted_uri],
},
)
.await?;
Ok(())
}
crates/tranquil-api/src/repo/record/mod.rs
+1 -1
View File
@@ -10,7 +10,7 @@ pub use pagination::PaginationDirection;
pub use validation_mode::ValidationMode;
pub use batch::apply_writes;
pub use delete::{DeleteRecordInput, delete_record, delete_record_internal};
pub use delete::{DeleteRecordInput, delete_record};
pub use read::{GetRecordInput, ListRecordsInput, ListRecordsOutput, get_record, list_records};
pub use tranquil_pds::repo_ops::*;
pub use write::{
crates/tranquil-api/src/repo/record/write.rs
+27 -30
View File
@@ -172,7 +172,7 @@ pub async fn create_record(
ops.push(RecordOp::Delete {
collection: conflict_collection,
rkey: conflict_rkey,
prev: Some(prev_cid),
prev: tranquil_pds::cid_types::RecordCid::from(prev_cid),
});
conflict_uris_to_cleanup.push(conflict_uri);
}
@@ -197,7 +197,7 @@ pub async fn create_record(
ops.push(RecordOp::Create {
collection: input.collection.clone(),
rkey: rkey.clone(),
cid: record_cid,
cid: tranquil_pds::cid_types::RecordCid::from(record_cid),
});
let modified_keys: Vec<String> = ops
@@ -338,41 +338,38 @@ pub async fn put_record(
}));
}
let is_update = existing_cid.is_some();
let new_mst = if is_update {
mst.update(&key, record_cid)
.await
.map_err(|_| ApiError::InternalError(Some("Failed to update MST".into())))?
} else {
mst.add(&key, record_cid)
.await
.map_err(|_| ApiError::InternalError(Some("Failed to add to MST".into())))?
};
let op = if is_update {
RecordOp::Update {
collection: input.collection.clone(),
rkey: input.rkey.clone(),
cid: record_cid,
prev: existing_cid,
let record_uri = AtUri::from_parts(&did, &input.collection, &input.rkey);
let (new_mst, op, is_update, backlinks_to_remove) = match existing_cid {
Some(prev_cid) => {
let new_mst = mst
.update(&key, record_cid)
.await
.map_err(|_| ApiError::InternalError(Some("Failed to update MST".into())))?;
let op = RecordOp::Update {
collection: input.collection.clone(),
rkey: input.rkey.clone(),
cid: tranquil_pds::cid_types::RecordCid::from(record_cid),
prev: tranquil_pds::cid_types::RecordCid::from(prev_cid),
};
(new_mst, op, true, vec![record_uri.clone()])
}
} else {
RecordOp::Create {
collection: input.collection.clone(),
rkey: input.rkey.clone(),
cid: record_cid,
None => {
let new_mst = mst
.add(&key, record_cid)
.await
.map_err(|_| ApiError::InternalError(Some("Failed to add to MST".into())))?;
let op = RecordOp::Create {
collection: input.collection.clone(),
rkey: input.rkey.clone(),
cid: tranquil_pds::cid_types::RecordCid::from(record_cid),
};
(new_mst, op, false, vec![])
}
};
let modified_keys = [key];
let blob_cids = extract_blob_cids(&input.record);
let record_uri = AtUri::from_parts(&did, &input.collection, &input.rkey);
let backlinks_to_add = extract_backlinks(&record_uri, &input.record);
let backlinks_to_remove = match is_update {
true => vec![record_uri.clone()],
false => vec![],
};
let commit_result = finalize_repo_write(
&state,
crates/tranquil-api/src/server/passkey_account.rs
+2 -2
View File
@@ -393,13 +393,13 @@ pub async fn create_passkey_account(
Ok(token_meta) => {
let refresh_jti = uuid::Uuid::new_v4().to_string();
let refresh_expires = chrono::Utc::now() + chrono::Duration::hours(24);
let session_data = tranquil_db::SessionTokenCreate {
let session_data = tranquil_db_traits::SessionTokenCreate {
did: did_typed.clone(),
access_jti: token_meta.jti.clone(),
refresh_jti,
access_expires_at: token_meta.expires_at,
refresh_expires_at: refresh_expires,
login_type: tranquil_db::LoginType::Modern,
login_type: tranquil_db_traits::LoginType::Modern,
mfa_verified: false,
scope: Some("transition:generic".to_string()),
controller_did: None,
crates/tranquil-config/src/lib.rs
+103 -3
View File
@@ -269,6 +269,38 @@ impl TranquilConfig {
{
errors.push("tranquil_store.handler_threads must be at least 1".to_string());
}
if self.tranquil_store.eventlog_max_event_payload == 0 {
errors.push(
"tranquil_store.eventlog_max_event_payload \
(TRANQUIL_STORE_EVENTLOG_MAX_EVENT_PAYLOAD) must be at least 1; \
a value of 0 would reject every event"
.to_string(),
);
}
// -- scheduled / event retention --------------------------------------
const MAX_RETENTION_SECS: u64 = (i64::MAX / 1000) as u64;
if self.scheduled.event_retention_max_age_secs > MAX_RETENTION_SECS {
errors.push(format!(
"scheduled.event_retention_max_age_secs (EVENT_RETENTION_MAX_AGE_SECS) \
must be at most {MAX_RETENTION_SECS} (chrono::Duration limit); got {}",
self.scheduled.event_retention_max_age_secs
));
}
if self.scheduled.event_retention_interval_secs > 0 {
let backfill_secs = u64::try_from(self.firehose.backfill_hours.max(0))
.unwrap_or(0)
.saturating_mul(3600);
if self.scheduled.event_retention_max_age_secs < backfill_secs {
errors.push(format!(
"scheduled.event_retention_max_age_secs ({}) is shorter than \
firehose.backfill_hours ({}h = {backfill_secs}s): \
relays would receive cursor responses pointing at pruned events. \
Increase event_retention_max_age_secs or decrease firehose.backfill_hours.",
self.scheduled.event_retention_max_age_secs, self.firehose.backfill_hours,
));
}
}
// -- cache ------------------------------------------------------------
match self.cache.backend.as_str() {
@@ -1057,9 +1089,45 @@ pub struct ScheduledConfig {
#[config(env = "SCHEDULED_DELETE_CHECK_INTERVAL_SECS", default = 3600)]
pub delete_check_interval_secs: u64,
/// Interval in seconds between block garbage collection cycles.
#[config(env = "BLOCK_GC_INTERVAL_SECS", default = 21600)]
pub block_gc_interval_secs: u64,
/// Interval in seconds between data file compaction scans (tranquil-store only).
/// Set to 0 to disable.
#[config(env = "COMPACTION_INTERVAL_SECS", default = 3600)]
pub compaction_interval_secs: u64,
/// Liveness ratio threshold below which a data file is compacted (0.0-1.0).
#[config(env = "COMPACTION_LIVENESS_THRESHOLD", default = 0.7)]
pub compaction_liveness_threshold: f64,
/// Grace period in milliseconds before a zero-refcount block can be removed by compaction.
#[config(env = "COMPACTION_GRACE_PERIOD_MS", default = 600000)]
pub compaction_grace_period_ms: u64,
/// Interval in seconds between reachability walk runs (tranquil-store only).
/// Set to 0 to disable. Default: weekly.
#[config(env = "REACHABILITY_WALK_INTERVAL_SECS", default = 604800)]
pub reachability_walk_interval_secs: u64,
/// Interval in seconds between continuous archival passes (tranquil-store only).
/// Sealed eventlog segments are copied to the archival destination each tick.
/// Set to 0 to disable. Default: 60 seconds.
#[config(env = "ARCHIVAL_INTERVAL_SECS", default = 60)]
pub archival_interval_secs: u64,
/// Archival destination directory for sealed eventlog segments.
/// If unset, archival is disabled.
#[config(env = "ARCHIVAL_DEST_DIR")]
pub archival_dest_dir: Option<String>,
/// Maximum age of events retained in the eventlog before pruning.
/// Per the atproto firehose spec, the relay backfill window only needs
/// to cover "hours or days". Default: 7 days.
#[config(env = "EVENT_RETENTION_MAX_AGE_SECS", default = 604800)]
pub event_retention_max_age_secs: u64,
/// Interval in seconds between event retention prune passes.
/// Set to 0 to disable. Default: hourly.
#[config(env = "EVENT_RETENTION_INTERVAL_SECS", default = 3600)]
pub event_retention_interval_secs: u64,
}
#[derive(Debug, Config)]
@@ -1079,6 +1147,38 @@ pub struct TranquilStoreConfig {
/// Number of handler threads. Defaults to available_parallelism / 2.
#[config(env = "TRANQUIL_STORE_HANDLER_THREADS")]
pub handler_threads: Option<usize>,
/// Maximum total bytes of pending (unsynced) eventlog payloads. Appenders
/// block once this budget is exhausted until in-flight events drain via
/// fsync. Set to 0 to disable backpressure (unbounded). Default: 1 GiB.
#[config(
env = "TRANQUIL_STORE_EVENTLOG_PENDING_BYTES_BUDGET",
default = 1_073_741_824
)]
pub eventlog_pending_bytes_budget: u64,
/// Maximum size of an individual eventlog payload in bytes. Single events
/// larger than this are rejected at append time. Default: 256 MiB.
#[config(
env = "TRANQUIL_STORE_EVENTLOG_MAX_EVENT_PAYLOAD",
default = 268_435_456
)]
pub eventlog_max_event_payload: u32,
/// Maximum size of an individual blockstore data file in bytes. When the
/// active data file reaches this size it is rolled over and becomes
/// eligible for compaction. Default: 256 MiB.
#[config(env = "TRANQUIL_STORE_MAX_BLOCKSTORE_FILE_SIZE", default = 268_435_456)]
pub max_blockstore_file_size: u64,
/// Maximum size of an individual eventlog segment file in bytes. When the
/// active segment reaches this size it is sealed and a new one is created.
/// Safe to change on a running instance. Default: 256 MiB.
#[config(
env = "TRANQUIL_STORE_MAX_EVENTLOG_SEGMENT_SIZE",
default = 268_435_456
)]
pub max_eventlog_segment_size: u64,
}
/// Generate a TOML configuration template with all available options,
crates/tranquil-db-traits/src/lib.rs
+6 -5
View File
@@ -33,11 +33,12 @@ pub use oauth::{
ScopePreference, TokenFamilyId, TrustedDeviceRow, TwoFactorChallenge,
};
pub use repo::{
AccountStatus, ApplyCommitError, ApplyCommitInput, ApplyCommitResult, BrokenGenesisCommit,
CommitEventData, EventBlocksCids, FullRecordInfo, ImportBlock, ImportRecord, ImportRepoError,
RecordDelete, RecordInfo, RecordUpsert, RecordWithTakedown, RepoAccountInfo, RepoEventNotifier,
RepoEventReceiver, RepoEventType, RepoInfo, RepoListItem, RepoRepository, RepoSeqEvent,
RepoWithoutRev, SequencedEvent, UserNeedingRecordBlobsBackfill, UserWithoutBlocks,
AccountStatus, ApplyCommitError, ApplyCommitInput, ApplyCommitResult, CommitEventData,
EventBlockInline, EventBlocks, FullRecordInfo, ImportBlock, ImportRecord, ImportRepoError,
PruneCount, RecordDelete, RecordInfo, RecordUpsert, RecordWithTakedown, RepoAccountInfo,
RepoEventNotifier, RepoEventReceiver, RepoEventType, RepoInfo, RepoListItem, RepoRepository,
RepoSeqEvent, RepoWithoutRev, SequencedEvent, UserNeedingRecordBlobsBackfill,
UserWithoutBlocks,
};
pub use scope::{DbScope, InvalidScopeError};
pub use sequence::{SequenceNumber, deserialize_optional_sequence};
crates/tranquil-db-traits/src/repo.rs
+52 -34
View File
@@ -165,13 +165,6 @@ pub struct RepoWithoutRev {
pub repo_root_cid: CidLink,
}
#[derive(Debug, Clone)]
pub struct BrokenGenesisCommit {
pub seq: SequenceNumber,
pub did: Did,
pub commit_cid: Option<CidLink>,
}
#[derive(Debug, Clone)]
pub struct UserWithoutBlocks {
pub user_id: Uuid,
@@ -190,6 +183,51 @@ pub struct RepoSeqEvent {
pub seq: SequenceNumber,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PruneCount {
Rows(u64),
Segments(u64),
}
impl PruneCount {
pub fn is_zero(&self) -> bool {
match self {
Self::Rows(n) | Self::Segments(n) => *n == 0,
}
}
pub fn count(&self) -> u64 {
match self {
Self::Rows(n) | Self::Segments(n) => *n,
}
}
pub fn unit(&self) -> &'static str {
match self {
Self::Rows(_) => "rows",
Self::Segments(_) => "segments",
}
}
}
impl std::fmt::Display for PruneCount {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{} {}", self.count(), self.unit())
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EventBlockInline {
pub cid_bytes: Vec<u8>,
pub data: Vec<u8>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum EventBlocks {
Inline(Vec<EventBlockInline>),
LegacyCids(Vec<String>),
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SequencedEvent {
pub seq: SequenceNumber,
@@ -201,7 +239,7 @@ pub struct SequencedEvent {
pub prev_data_cid: Option<CidLink>,
pub ops: Option<serde_json::Value>,
pub blobs: Option<Vec<String>>,
pub blocks_cids: Option<Vec<String>>,
pub blocks: Option<EventBlocks>,
pub handle: Option<Handle>,
pub active: Option<bool>,
pub status: Option<AccountStatus>,
@@ -216,17 +254,11 @@ pub struct CommitEventData {
pub prev_cid: Option<CidLink>,
pub ops: Option<serde_json::Value>,
pub blobs: Option<Vec<String>>,
pub blocks_cids: Option<Vec<String>>,
pub blocks: Option<Vec<EventBlockInline>>,
pub prev_data_cid: Option<CidLink>,
pub rev: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EventBlocksCids {
pub blocks_cids: Option<Vec<String>>,
pub commit_cid: Option<CidLink>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RepoListItem {
pub did: Did,
@@ -413,11 +445,6 @@ pub trait RepoRepository: Send + Sync {
async fn count_user_blocks(&self, user_id: Uuid) -> Result<i64, DbError>;
async fn find_unreferenced_blocks(
&self,
candidate_cids: &[Vec<u8>],
) -> Result<Vec<Vec<u8>>, DbError>;
async fn insert_commit_event(&self, data: &CommitEventData) -> Result<SequenceNumber, DbError>;
async fn insert_identity_event(
@@ -437,6 +464,7 @@ pub trait RepoRepository: Send + Sync {
did: &Did,
commit_cid: &CidLink,
rev: Option<&str>,
commit_bytes: &[u8],
) -> Result<SequenceNumber, DbError>;
async fn insert_genesis_commit_event(
@@ -445,20 +473,18 @@ pub trait RepoRepository: Send + Sync {
commit_cid: &CidLink,
mst_root_cid: &CidLink,
rev: &str,
commit_bytes: &[u8],
mst_root_bytes: &[u8],
) -> Result<SequenceNumber, DbError>;
async fn update_seq_blocks_cids(
&self,
seq: SequenceNumber,
blocks_cids: &[String],
) -> Result<(), DbError>;
async fn delete_sequences_except(
&self,
did: &Did,
keep_seq: SequenceNumber,
) -> Result<(), DbError>;
async fn prune_events_older_than(&self, cutoff: DateTime<Utc>) -> Result<PruneCount, DbError>;
async fn get_max_seq(&self) -> Result<SequenceNumber, DbError>;
async fn get_min_seq_since(
@@ -491,12 +517,6 @@ pub trait RepoRepository: Send + Sync {
limit: i64,
) -> Result<Vec<SequencedEvent>, DbError>;
async fn get_events_since_rev(
&self,
did: &Did,
since_rev: &str,
) -> Result<Vec<EventBlocksCids>, DbError>;
async fn list_repos_paginated(
&self,
cursor_did: Option<&Did>,
@@ -521,8 +541,6 @@ pub trait RepoRepository: Send + Sync {
input: ApplyCommitInput,
) -> Result<ApplyCommitResult, ApplyCommitError>;
async fn get_broken_genesis_commits(&self) -> Result<Vec<BrokenGenesisCommit>, DbError>;
async fn get_users_without_blocks(&self) -> Result<Vec<UserWithoutBlocks>, DbError>;
async fn get_users_needing_record_blobs_backfill(
crates/tranquil-db/src/postgres/repo.rs
+149 -223
View File
@@ -2,10 +2,10 @@ use async_trait::async_trait;
use chrono::{DateTime, Utc};
use sqlx::PgPool;
use tranquil_db_traits::{
AccountStatus, BrokenGenesisCommit, CommitEventData, DbError, EventBlocksCids, FullRecordInfo,
ImportBlock, ImportRecord, ImportRepoError, RecordInfo, RecordWithTakedown, RepoAccountInfo,
RepoEventType, RepoInfo, RepoListItem, RepoRepository, RepoWithoutRev, SequenceNumber,
SequencedEvent, UserNeedingRecordBlobsBackfill, UserWithoutBlocks,
AccountStatus, CommitEventData, DbError, EventBlockInline, EventBlocks, FullRecordInfo,
ImportBlock, ImportRecord, ImportRepoError, PruneCount, RecordInfo, RecordWithTakedown,
RepoAccountInfo, RepoEventType, RepoInfo, RepoListItem, RepoRepository, RepoWithoutRev,
SequenceNumber, SequencedEvent, UserNeedingRecordBlobsBackfill, UserWithoutBlocks,
};
use tranquil_types::{AtUri, CidLink, Did, Handle, Nsid, Rkey};
use uuid::Uuid;
@@ -27,6 +27,8 @@ struct SequencedEventRow {
prev_data_cid: Option<String>,
ops: Option<serde_json::Value>,
blobs: Option<Vec<String>>,
block_cids: Option<Vec<Vec<u8>>>,
block_data: Option<Vec<Vec<u8>>>,
blocks_cids: Option<Vec<String>>,
handle: Option<String>,
active: Option<bool>,
@@ -34,6 +36,81 @@ struct SequencedEventRow {
rev: Option<String>,
}
fn row_to_event_blocks(
block_cids: Option<Vec<Vec<u8>>>,
block_data: Option<Vec<Vec<u8>>>,
legacy_blocks_cids: Option<Vec<String>>,
) -> Result<Option<EventBlocks>, DbError> {
match (block_cids, block_data) {
(Some(cids), Some(data)) if cids.len() == data.len() => match cids.is_empty() {
true => Ok(legacy_fallback(legacy_blocks_cids)),
false => Ok(Some(EventBlocks::Inline(
cids.into_iter()
.zip(data)
.map(|(cid_bytes, data)| EventBlockInline { cid_bytes, data })
.collect(),
))),
},
(Some(_), Some(_)) => Err(DbError::CorruptData(
"repo_seq.block_cids/block_data length mismatch",
)),
(Some(_), None) | (None, Some(_)) => Err(DbError::CorruptData(
"repo_seq.block_cids/block_data partially populated",
)),
(None, None) => Ok(legacy_fallback(legacy_blocks_cids)),
}
}
fn legacy_fallback(legacy_blocks_cids: Option<Vec<String>>) -> Option<EventBlocks> {
match legacy_blocks_cids {
Some(cids) if !cids.is_empty() => Some(EventBlocks::LegacyCids(cids)),
_ => None,
}
}
fn inline_to_paired_blocks(blocks: Option<&[EventBlockInline]>) -> (Vec<Vec<u8>>, Vec<Vec<u8>>) {
blocks
.map(|bs| {
bs.iter()
.map(|b| (b.cid_bytes.clone(), b.data.clone()))
.unzip()
})
.unwrap_or_default()
}
fn inline_into_paired_blocks(
blocks: Option<Vec<EventBlockInline>>,
) -> (Vec<Vec<u8>>, Vec<Vec<u8>>) {
blocks
.map(|bs| bs.into_iter().map(|b| (b.cid_bytes, b.data)).unzip())
.unwrap_or_default()
}
fn map_sequenced_row(r: SequencedEventRow) -> Result<SequencedEvent, DbError> {
let status = r
.status
.as_deref()
.and_then(AccountStatus::parse)
.or_else(|| r.active.filter(|a| *a).map(|_| AccountStatus::Active));
let blocks = row_to_event_blocks(r.block_cids, r.block_data, r.blocks_cids)?;
Ok(SequencedEvent {
seq: r.seq.into(),
did: Did::from(r.did),
created_at: r.created_at,
event_type: r.event_type,
commit_cid: r.commit_cid.map(CidLink::from),
prev_cid: r.prev_cid.map(CidLink::from),
prev_data_cid: r.prev_data_cid.map(CidLink::from),
ops: r.ops,
blobs: r.blobs,
blocks,
handle: r.handle.map(Handle::from),
active: r.active,
status,
rev: r.rev,
})
}
pub struct PostgresRepoRepository {
pool: PgPool,
}
@@ -618,30 +695,6 @@ impl RepoRepository for PostgresRepoRepository {
Ok(count)
}
async fn find_unreferenced_blocks(
&self,
candidate_cids: &[Vec<u8>],
) -> Result<Vec<Vec<u8>>, DbError> {
match candidate_cids.is_empty() {
true => Ok(Vec::new()),
false => {
let rows = sqlx::query!(
r#"
SELECT t.cid FROM UNNEST($1::bytea[]) AS t(cid)
WHERE NOT EXISTS (
SELECT 1 FROM user_blocks WHERE block_cid = t.cid
)
"#,
candidate_cids,
)
.fetch_all(&self.pool)
.await
.map_err(map_sqlx_error)?;
Ok(rows.into_iter().filter_map(|r| r.cid).collect())
}
}
}
async fn get_user_block_cids_since_rev(
&self,
user_id: Uuid,
@@ -664,10 +717,11 @@ impl RepoRepository for PostgresRepoRepository {
}
async fn insert_commit_event(&self, data: &CommitEventData) -> Result<SequenceNumber, DbError> {
let (block_cids, block_data) = inline_to_paired_blocks(data.blocks.as_deref());
let seq = sqlx::query_scalar!(
r#"
INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, blocks_cids, prev_data_cid, rev)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, block_cids, block_data, prev_data_cid, rev)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
RETURNING seq
"#,
data.did.as_str(),
@@ -676,7 +730,8 @@ impl RepoRepository for PostgresRepoRepository {
data.prev_cid.as_ref().map(|c| c.as_str()),
data.ops,
data.blobs.as_deref(),
data.blocks_cids.as_deref(),
&block_cids as &[Vec<u8>],
&block_data as &[Vec<u8>],
data.prev_data_cid.as_ref().map(|c| c.as_str()),
data.rev
)
@@ -748,16 +803,25 @@ impl RepoRepository for PostgresRepoRepository {
did: &Did,
commit_cid: &CidLink,
rev: Option<&str>,
commit_bytes: &[u8],
) -> Result<SequenceNumber, DbError> {
let cid_bytes = commit_cid
.to_cid()
.map(|c| c.to_bytes())
.unwrap_or_default();
let block_cids: Vec<Vec<u8>> = vec![cid_bytes];
let block_data: Vec<Vec<u8>> = vec![commit_bytes.to_vec()];
let seq = sqlx::query_scalar!(
r#"
INSERT INTO repo_seq (did, event_type, commit_cid, rev)
VALUES ($1, 'sync', $2, $3)
INSERT INTO repo_seq (did, event_type, commit_cid, rev, block_cids, block_data)
VALUES ($1, 'sync', $2, $3, $4, $5)
RETURNING seq
"#,
did.as_str(),
commit_cid.as_str(),
rev
rev,
&block_cids as &[Vec<u8>],
&block_data as &[Vec<u8>]
)
.fetch_one(&self.pool)
.await
@@ -777,16 +841,27 @@ impl RepoRepository for PostgresRepoRepository {
commit_cid: &CidLink,
mst_root_cid: &CidLink,
rev: &str,
commit_bytes: &[u8],
mst_root_bytes: &[u8],
) -> Result<SequenceNumber, DbError> {
let ops = serde_json::json!([]);
let blobs: Vec<String> = vec![];
let blocks_cids: Vec<String> = vec![mst_root_cid.to_string(), commit_cid.to_string()];
let commit_cid_bytes = commit_cid
.to_cid()
.map(|c| c.to_bytes())
.unwrap_or_default();
let mst_cid_bytes = mst_root_cid
.to_cid()
.map(|c| c.to_bytes())
.unwrap_or_default();
let block_cids: Vec<Vec<u8>> = vec![commit_cid_bytes, mst_cid_bytes];
let block_data: Vec<Vec<u8>> = vec![commit_bytes.to_vec(), mst_root_bytes.to_vec()];
let prev_cid: Option<&str> = None;
let seq = sqlx::query_scalar!(
r#"
INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, blocks_cids, rev)
VALUES ($1, 'commit', $2, $3::TEXT, $4, $5, $6, $7)
INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, block_cids, block_data, rev)
VALUES ($1, 'commit', $2, $3::TEXT, $4, $5, $6, $7, $8)
RETURNING seq
"#,
did.as_str(),
@@ -794,7 +869,8 @@ impl RepoRepository for PostgresRepoRepository {
prev_cid,
ops,
&blobs,
&blocks_cids,
&block_cids as &[Vec<u8>],
&block_data as &[Vec<u8>],
rev
)
.fetch_one(&self.pool)
@@ -809,23 +885,6 @@ impl RepoRepository for PostgresRepoRepository {
Ok(seq.into())
}
async fn update_seq_blocks_cids(
&self,
seq: SequenceNumber,
blocks_cids: &[String],
) -> Result<(), DbError> {
sqlx::query!(
"UPDATE repo_seq SET blocks_cids = $1 WHERE seq = $2",
blocks_cids,
seq.as_i64()
)
.execute(&self.pool)
.await
.map_err(map_sqlx_error)?;
Ok(())
}
async fn delete_sequences_except(
&self,
did: &Did,
@@ -843,6 +902,15 @@ impl RepoRepository for PostgresRepoRepository {
Ok(())
}
async fn prune_events_older_than(&self, cutoff: DateTime<Utc>) -> Result<PruneCount, DbError> {
let result = sqlx::query!("DELETE FROM repo_seq WHERE created_at < $1", cutoff)
.execute(&self.pool)
.await
.map_err(map_sqlx_error)?;
Ok(PruneCount::Rows(result.rows_affected()))
}
async fn get_max_seq(&self) -> Result<SequenceNumber, DbError> {
let seq = sqlx::query_scalar!(r#"SELECT COALESCE(MAX(seq), 0) as "max!" FROM repo_seq"#)
.fetch_one(&self.pool)
@@ -893,35 +961,12 @@ impl RepoRepository for PostgresRepoRepository {
since_seq: SequenceNumber,
limit: Option<i64>,
) -> Result<Vec<SequencedEvent>, DbError> {
let map_row = |r: SequencedEventRow| {
let status = r
.status
.as_deref()
.and_then(AccountStatus::parse)
.or_else(|| r.active.filter(|a| *a).map(|_| AccountStatus::Active));
SequencedEvent {
seq: r.seq.into(),
did: Did::from(r.did),
created_at: r.created_at,
event_type: r.event_type,
commit_cid: r.commit_cid.map(CidLink::from),
prev_cid: r.prev_cid.map(CidLink::from),
prev_data_cid: r.prev_data_cid.map(CidLink::from),
ops: r.ops,
blobs: r.blobs,
blocks_cids: r.blocks_cids,
handle: r.handle.map(Handle::from),
active: r.active,
status,
rev: r.rev,
}
};
match limit {
Some(lim) => {
let rows = sqlx::query_as!(
SequencedEventRow,
r#"SELECT seq, did, created_at, event_type as "event_type: RepoEventType", commit_cid, prev_cid, prev_data_cid,
ops, blobs, blocks_cids, handle, active, status, rev
ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev
FROM repo_seq
WHERE seq > $1
ORDER BY seq ASC
@@ -932,13 +977,13 @@ impl RepoRepository for PostgresRepoRepository {
.fetch_all(&self.pool)
.await
.map_err(map_sqlx_error)?;
Ok(rows.into_iter().map(map_row).collect())
rows.into_iter().map(map_sequenced_row).collect()
}
None => {
let rows = sqlx::query_as!(
SequencedEventRow,
r#"SELECT seq, did, created_at, event_type as "event_type: RepoEventType", commit_cid, prev_cid, prev_data_cid,
ops, blobs, blocks_cids, handle, active, status, rev
ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev
FROM repo_seq
WHERE seq > $1
ORDER BY seq ASC"#,
@@ -947,7 +992,7 @@ impl RepoRepository for PostgresRepoRepository {
.fetch_all(&self.pool)
.await
.map_err(map_sqlx_error)?;
Ok(rows.into_iter().map(map_row).collect())
rows.into_iter().map(map_sequenced_row).collect()
}
}
}
@@ -957,9 +1002,10 @@ impl RepoRepository for PostgresRepoRepository {
start_seq: SequenceNumber,
end_seq: SequenceNumber,
) -> Result<Vec<SequencedEvent>, DbError> {
let rows = sqlx::query!(
let rows = sqlx::query_as!(
SequencedEventRow,
r#"SELECT seq, did, created_at, event_type as "event_type: RepoEventType", commit_cid, prev_cid, prev_data_cid,
ops, blobs, blocks_cids, handle, active, status, rev
ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev
FROM repo_seq
WHERE seq > $1 AND seq < $2
ORDER BY seq ASC"#,
@@ -969,41 +1015,17 @@ impl RepoRepository for PostgresRepoRepository {
.fetch_all(&self.pool)
.await
.map_err(map_sqlx_error)?;
Ok(rows
.into_iter()
.map(|r| {
let status = r
.status
.as_deref()
.and_then(AccountStatus::parse)
.or_else(|| r.active.filter(|a| *a).map(|_| AccountStatus::Active));
SequencedEvent {
seq: r.seq.into(),
did: Did::from(r.did),
created_at: r.created_at,
event_type: r.event_type,
commit_cid: r.commit_cid.map(CidLink::from),
prev_cid: r.prev_cid.map(CidLink::from),
prev_data_cid: r.prev_data_cid.map(CidLink::from),
ops: r.ops,
blobs: r.blobs,
blocks_cids: r.blocks_cids,
handle: r.handle.map(Handle::from),
active: r.active,
status,
rev: r.rev,
}
})
.collect())
rows.into_iter().map(map_sequenced_row).collect()
}
async fn get_event_by_seq(
&self,
seq: SequenceNumber,
) -> Result<Option<SequencedEvent>, DbError> {
let row = sqlx::query!(
let row = sqlx::query_as!(
SequencedEventRow,
r#"SELECT seq, did, created_at, event_type as "event_type: RepoEventType", commit_cid, prev_cid, prev_data_cid,
ops, blobs, blocks_cids, handle, active, status, rev
ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev
FROM repo_seq
WHERE seq = $1"#,
seq.as_i64()
@@ -1011,29 +1033,7 @@ impl RepoRepository for PostgresRepoRepository {
.fetch_optional(&self.pool)
.await
.map_err(map_sqlx_error)?;
Ok(row.map(|r| {
let status = r
.status
.as_deref()
.and_then(AccountStatus::parse)
.or_else(|| r.active.filter(|a| *a).map(|_| AccountStatus::Active));
SequencedEvent {
seq: r.seq.into(),
did: Did::from(r.did),
created_at: r.created_at,
event_type: r.event_type,
commit_cid: r.commit_cid.map(CidLink::from),
prev_cid: r.prev_cid.map(CidLink::from),
prev_data_cid: r.prev_data_cid.map(CidLink::from),
ops: r.ops,
blobs: r.blobs,
blocks_cids: r.blocks_cids,
handle: r.handle.map(Handle::from),
active: r.active,
status,
rev: r.rev,
}
}))
row.map(map_sequenced_row).transpose()
}
async fn get_events_since_cursor(
@@ -1041,9 +1041,10 @@ impl RepoRepository for PostgresRepoRepository {
cursor: SequenceNumber,
limit: i64,
) -> Result<Vec<SequencedEvent>, DbError> {
let rows = sqlx::query!(
let rows = sqlx::query_as!(
SequencedEventRow,
r#"SELECT seq, did, created_at, event_type as "event_type: RepoEventType", commit_cid, prev_cid, prev_data_cid,
ops, blobs, blocks_cids, handle, active, status, rev
ops, blobs, block_cids, block_data, blocks_cids, handle, active, status, rev
FROM repo_seq
WHERE seq > $1
ORDER BY seq ASC
@@ -1054,58 +1055,7 @@ impl RepoRepository for PostgresRepoRepository {
.fetch_all(&self.pool)
.await
.map_err(map_sqlx_error)?;
Ok(rows
.into_iter()
.map(|r| {
let status = r
.status
.as_deref()
.and_then(AccountStatus::parse)
.or_else(|| r.active.filter(|a| *a).map(|_| AccountStatus::Active));
SequencedEvent {
seq: r.seq.into(),
did: Did::from(r.did),
created_at: r.created_at,
event_type: r.event_type,
commit_cid: r.commit_cid.map(CidLink::from),
prev_cid: r.prev_cid.map(CidLink::from),
prev_data_cid: r.prev_data_cid.map(CidLink::from),
ops: r.ops,
blobs: r.blobs,
blocks_cids: r.blocks_cids,
handle: r.handle.map(Handle::from),
active: r.active,
status,
rev: r.rev,
}
})
.collect())
}
async fn get_events_since_rev(
&self,
did: &Did,
since_rev: &str,
) -> Result<Vec<EventBlocksCids>, DbError> {
let rows = sqlx::query!(
r#"SELECT blocks_cids, commit_cid
FROM repo_seq
WHERE did = $1 AND rev > $2
ORDER BY seq DESC"#,
did.as_str(),
since_rev
)
.fetch_all(&self.pool)
.await
.map_err(map_sqlx_error)?;
Ok(rows
.into_iter()
.map(|r| EventBlocksCids {
blocks_cids: r.blocks_cids,
commit_cid: r.commit_cid.map(CidLink::from),
})
.collect())
rows.into_iter().map(map_sequenced_row).collect()
}
async fn list_repos_paginated(
@@ -1450,22 +1400,24 @@ impl RepoRepository for PostgresRepoRepository {
.map_err(|e| ApplyCommitError::Database(e.to_string()))?;
}
let event = &input.commit_event;
let event = input.commit_event;
let (event_block_cids, event_block_data) = inline_into_paired_blocks(event.blocks);
let seq: i64 = sqlx::query_scalar(
r#"
INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, blocks_cids, prev_data_cid, rev)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
INSERT INTO repo_seq (did, event_type, commit_cid, prev_cid, ops, blobs, block_cids, block_data, prev_data_cid, rev)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
RETURNING seq
"#,
)
.bind(&event.did)
.bind(event.did.as_str())
.bind(event.event_type.as_str())
.bind(&event.commit_cid)
.bind(&event.prev_cid)
.bind(event.commit_cid.as_ref().map(|c| c.as_str()))
.bind(event.prev_cid.as_ref().map(|c| c.as_str()))
.bind(&event.ops)
.bind(&event.blobs)
.bind(&event.blocks_cids)
.bind(&event.prev_data_cid)
.bind(&event_block_cids)
.bind(&event_block_data)
.bind(event.prev_data_cid.as_ref().map(|c| c.as_str()))
.bind(&event.rev)
.fetch_one(&mut *tx)
.await
@@ -1486,32 +1438,6 @@ impl RepoRepository for PostgresRepoRepository {
})
}
async fn get_broken_genesis_commits(
&self,
) -> Result<Vec<tranquil_db_traits::BrokenGenesisCommit>, DbError> {
let rows = sqlx::query!(
r#"
SELECT seq, did, commit_cid
FROM repo_seq
WHERE event_type = 'commit'
AND prev_cid IS NULL
AND (blocks_cids IS NULL OR array_length(blocks_cids, 1) IS NULL OR array_length(blocks_cids, 1) = 0)
"#
)
.fetch_all(&self.pool)
.await
.map_err(map_sqlx_error)?;
Ok(rows
.into_iter()
.map(|r| BrokenGenesisCommit {
seq: r.seq.into(),
did: Did::from(r.did),
commit_cid: r.commit_cid.map(CidLink::from),
})
.collect())
}
async fn get_users_without_blocks(&self) -> Result<Vec<UserWithoutBlocks>, DbError> {
let rows: Vec<(Uuid, String, Option<String>)> = sqlx::query_as(
r#"
crates/tranquil-pds/src/auth/mod.rs
+1 -2
View File
@@ -7,8 +7,7 @@ use crate::api::ApiError;
use crate::cache::Cache;
use crate::oauth::scopes::ScopePermissions;
use crate::types::Did;
use tranquil_db::UserRepository;
use tranquil_db_traits::OAuthRepository;
use tranquil_db_traits::{OAuthRepository, UserRepository};
pub mod account_verified;
pub mod email_token;
crates/tranquil-pds/src/crawlers.rs
+9 -5
View File
@@ -75,8 +75,8 @@ impl Crawlers {
self.last_notified.store(now, Ordering::Relaxed);
}
pub async fn notify_of_update(&self) {
if !self.should_notify() {
pub async fn notify_of_update(&self, force: bool) {
if !force && !self.should_notify() {
debug!("Skipping crawler notification due to debounce");
return;
}
@@ -157,13 +157,17 @@ pub async fn start_crawlers_service(
result = firehose_rx.recv() => {
match result {
Ok(event) => {
if event.event_type == RepoEventType::Commit {
crawlers.notify_of_update().await;
match event.event_type {
RepoEventType::Commit => crawlers.notify_of_update(false).await,
RepoEventType::Account | RepoEventType::Identity => {
crawlers.notify_of_update(true).await
}
RepoEventType::Sync => {}
}
}
Err(broadcast::error::RecvError::Lagged(n)) => {
warn!(skipped = n, "Crawlers service lagged behind firehose");
crawlers.notify_of_update().await;
crawlers.notify_of_update(false).await;
}
Err(broadcast::error::RecvError::Closed) => {
error!("Firehose channel closed, stopping crawlers service");
crates/tranquil-pds/src/oauth/mod.rs
+1 -1
View File
@@ -3,7 +3,7 @@ pub mod db;
pub mod scopes;
pub mod verify;
pub fn db_err_to_oauth(err: tranquil_db::DbError) -> OAuthError {
pub fn db_err_to_oauth(err: tranquil_db_traits::DbError) -> OAuthError {
tracing::error!("Database error in OAuth flow: {}", err);
OAuthError::ServerError("An internal error occurred".to_string())
}
crates/tranquil-pds/src/repo/mod.rs
+14
View File
@@ -22,6 +22,20 @@ impl AnyBlockStore {
Self::TranquilStore(_) => None,
}
}
pub fn as_tranquil_store(&self) -> Option<&TranquilBlockStore> {
match self {
Self::TranquilStore(s) => Some(s),
Self::Postgres(_) => None,
}
}
pub async fn decrement_refs(&self, cids: &[Cid]) -> Result<(), RepoError> {
match self {
Self::Postgres(_) => Ok(()),
Self::TranquilStore(s) => s.decrement_refs(cids).await,
}
}
}
impl BlockStore for AnyBlockStore {
crates/tranquil-pds/src/repo_ops.rs
+217 -125
View File
@@ -1,16 +1,19 @@
use crate::api::error::ApiError;
use crate::cid_types::CommitCid;
use crate::cid_types::{CommitCid, RecordCid};
use crate::repo::TrackingBlockStore;
use crate::state::AppState;
use crate::types::{Did, Handle, Nsid, Rkey};
use backon::{ExponentialBuilder, Retryable};
use bytes::Bytes;
use cid::Cid;
use jacquard_common::types::{integer::LimitedU32, string::Tid};
use jacquard_repo::commit::Commit;
use jacquard_repo::mst::Mst;
use jacquard_repo::mst::util::compute_cid;
use jacquard_repo::storage::BlockStore;
use k256::ecdsa::SigningKey;
use serde_json::{Value, json};
use std::collections::BTreeSet;
use std::str::FromStr;
use std::sync::Arc;
use tokio::sync::OwnedMutexGuard;
@@ -147,6 +150,7 @@ pub fn extract_backlinks(uri: &AtUri, record: &Value) -> Vec<Backlink> {
pub struct RepoWriteContext {
pub tracking_store: TrackingBlockStore,
pub current_root_cid: Cid,
pub prev_commit_bytes: Bytes,
pub prev_data_cid: Cid,
pub write_lock: OwnedMutexGuard<()>,
}
@@ -197,28 +201,55 @@ pub async fn begin_repo_write(
.get(&current_root_cid)
.await
.map_err(|e| {
error!("Failed to load commit block: {:?}", e);
error!("Failed to load commit block: {}", e);
ApiError::InternalError(None)
})?
.ok_or_else(|| ApiError::InternalError(Some("Commit block not found".into())))?;
let commit = Commit::from_cbor(&commit_bytes).map_err(|e| {
error!("Failed to parse commit: {:?}", e);
ApiError::InternalError(None)
})?;
let prev_data_cid = Commit::from_cbor(&commit_bytes)
.map_err(|e| {
error!("Failed to parse commit: {}", e);
ApiError::InternalError(None)
})?
.data;
let mst = Mst::load(Arc::new(tracking_store.clone()), commit.data, None);
let mst = Mst::load(Arc::new(tracking_store.clone()), prev_data_cid, None);
let ctx = RepoWriteContext {
tracking_store,
current_root_cid,
prev_data_cid: commit.data,
prev_commit_bytes: commit_bytes,
prev_data_cid,
write_lock,
};
Ok((ctx, mst))
}
pub async fn compute_obsolete_cids(
original_mst: &Mst<TrackingBlockStore>,
new_mst: &Mst<TrackingBlockStore>,
original_root_cid: CommitCid,
) -> Result<Vec<Cid>, jacquard_repo::error::RepoError> {
let (old_nodes, new_nodes, old_leaves, new_leaves) = tokio::try_join!(
original_mst.collect_node_cids(),
new_mst.collect_node_cids(),
original_mst.leaves(),
new_mst.leaves(),
)?;
let old_nodes_set: BTreeSet<Cid> = old_nodes.into_iter().collect();
let new_nodes_set: BTreeSet<Cid> = new_nodes.into_iter().collect();
let old_leaf_set: BTreeSet<Cid> = old_leaves.iter().map(|(_, cid)| *cid).collect();
let new_leaf_set: BTreeSet<Cid> = new_leaves.iter().map(|(_, cid)| *cid).collect();
let removed_nodes = old_nodes_set.difference(&new_nodes_set).copied();
let removed_leaves = old_leaf_set.difference(&new_leaf_set).copied();
let obsolete: BTreeSet<Cid> = std::iter::once(original_root_cid.into_cid())
.chain(removed_nodes)
.chain(removed_leaves)
.collect();
Ok(obsolete.into_iter().collect())
}
pub async fn finalize_repo_write(
state: &AppState,
ctx: RepoWriteContext,
@@ -226,18 +257,44 @@ pub async fn finalize_repo_write(
params: FinalizeParams<'_>,
) -> Result<CommitResult, ApiError> {
let new_mst_root = mst.persist().await.map_err(|e| {
error!("MST persist failed: {:?}", e);
error!("MST persist failed: {}", e);
ApiError::InternalError(None)
})?;
let written_cids: Vec<Cid> = ctx
.tracking_store
.get_all_relevant_cids()
.into_iter()
.collect::<std::collections::HashSet<_>>()
.into_iter()
.collect();
let written_cids_str: Vec<String> = written_cids.iter().map(ToString::to_string).collect();
let block_bytes = ctx.tracking_store.take_written_blocks();
let storage_for_diff = Arc::new(ctx.tracking_store.clone());
let original_settled = Mst::load(storage_for_diff.clone(), ctx.prev_data_cid, None);
let new_settled = Mst::load(storage_for_diff, new_mst_root, None);
let (obsolete_cids, new_tree_cids) = tokio::try_join!(
async {
compute_obsolete_cids(
&original_settled,
&new_settled,
CommitCid::from(ctx.current_root_cid),
)
.await
.map_err(|e| {
error!("MST diff failed during finalize_repo_write: {}", e);
ApiError::InternalError(Some("MST diff failed".into()))
})
},
async {
let (nodes, leaves) =
tokio::try_join!(new_settled.collect_node_cids(), new_settled.leaves(),).map_err(
|e| {
error!("new tree walk failed: {}", e);
ApiError::InternalError(None)
},
)?;
Ok::<Vec<Cid>, ApiError>(
nodes
.into_iter()
.chain(leaves.iter().map(|(_, cid)| *cid))
.collect(),
)
},
)?;
let result = commit_and_log(
state,
@@ -245,12 +302,14 @@ pub async fn finalize_repo_write(
did: params.did,
user_id: params.user_id,
current_root_cid: Some(ctx.current_root_cid),
prev_commit_bytes: Some(ctx.prev_commit_bytes),
prev_data_cid: Some(ctx.prev_data_cid),
new_mst_root,
ops: params.ops,
blocks_cids: &written_cids_str,
block_bytes,
new_tree_cids,
blobs: params.blob_cids,
obsolete_cids: vec![ctx.current_root_cid],
obsolete_cids,
backlinks_to_add: params.backlinks_to_add,
backlinks_to_remove: params.backlinks_to_remove,
},
@@ -297,7 +356,7 @@ pub fn create_signed_commit(
let sig_bytes = signed.sig().clone();
let signed_bytes = signed
.to_cbor()
.map_err(|e| CommitError::SerializationFailed(format!("{:?}", e)))?;
.map_err(|e| CommitError::SerializationFailed(e.to_string()))?;
Ok((signed_bytes, sig_bytes))
}
@@ -305,18 +364,18 @@ pub enum RecordOp {
Create {
collection: Nsid,
rkey: Rkey,
cid: Cid,
cid: RecordCid,
},
Update {
collection: Nsid,
rkey: Rkey,
cid: Cid,
prev: Option<Cid>,
cid: RecordCid,
prev: RecordCid,
},
Delete {
collection: Nsid,
rkey: Rkey,
prev: Option<Cid>,
prev: RecordCid,
},
}
@@ -329,10 +388,12 @@ pub struct CommitParams<'a> {
pub did: &'a Did,
pub user_id: Uuid,
pub current_root_cid: Option<Cid>,
pub prev_commit_bytes: Option<Bytes>,
pub prev_data_cid: Option<Cid>,
pub new_mst_root: Cid,
pub ops: Vec<RecordOp>,
pub blocks_cids: &'a [String],
pub block_bytes: std::collections::HashMap<Cid, Bytes>,
pub new_tree_cids: Vec<Cid>,
pub blobs: &'a [String],
pub obsolete_cids: Vec<Cid>,
pub backlinks_to_add: Vec<Backlink>,
@@ -344,8 +405,8 @@ pub async fn commit_and_log(
params: CommitParams<'_>,
) -> Result<CommitResult, CommitError> {
use tranquil_db_traits::{
ApplyCommitError, ApplyCommitInput, CommitEventData, RecordDelete, RecordUpsert,
RepoEventType,
ApplyCommitError, ApplyCommitInput, CommitEventData, EventBlockInline, RecordDelete,
RecordUpsert, RepoEventType,
};
let backlinks_to_add = params.backlinks_to_add;
@@ -354,14 +415,21 @@ pub async fn commit_and_log(
did,
user_id,
current_root_cid,
prev_commit_bytes,
prev_data_cid,
new_mst_root,
ops,
blocks_cids,
mut block_bytes,
new_tree_cids,
blobs,
obsolete_cids,
..
} = params;
debug_assert_eq!(
current_root_cid.is_some(),
prev_commit_bytes.is_some(),
"current_root_cid and prev_commit_bytes must be both Some (non-genesis) or both None (genesis)"
);
let key_row = state
.repos
.user
@@ -377,18 +445,27 @@ pub async fn commit_and_log(
let rev_str = rev.to_string();
let (new_commit_bytes, _sig) =
create_signed_commit(did, new_mst_root, &rev_str, current_root_cid, &signing_key)?;
let new_root_cid = state
let new_root_cid =
compute_cid(&new_commit_bytes).map_err(|e| CommitError::BlockStoreFailed(e.to_string()))?;
let commit_bytes_owned = Bytes::from(new_commit_bytes.clone());
state
.block_store
.put(&new_commit_bytes)
.await
.map_err(|e| CommitError::BlockStoreFailed(format!("{:?}", e)))?;
.map_err(|e| CommitError::BlockStoreFailed(format!("failed to write commit block: {e}")))?;
let mut all_block_cids: Vec<Vec<u8>> = blocks_cids
block_bytes.insert(new_root_cid, commit_bytes_owned);
if let (Some(prev_root), Some(prev_bytes)) = (current_root_cid, prev_commit_bytes) {
block_bytes.entry(prev_root).or_insert(prev_bytes);
}
let all_block_cids: Vec<Vec<u8>> = new_tree_cids
.iter()
.filter_map(|s| Cid::from_str(s).ok())
.chain(std::iter::once(&new_root_cid))
.map(|c| c.to_bytes())
.collect();
all_block_cids.push(new_root_cid.to_bytes());
let obsolete_bytes: Vec<Vec<u8>> = obsolete_cids.iter().map(|c| c.to_bytes()).collect();
@@ -410,7 +487,7 @@ pub async fn commit_and_log(
upserts.push(RecordUpsert {
collection: collection.clone(),
rkey: rkey.clone(),
cid: crate::types::CidLink::from(cid),
cid: crate::types::CidLink::from(cid.as_cid()),
});
}
RecordOp::Delete {
@@ -443,32 +520,30 @@ pub async fn commit_and_log(
rkey,
cid,
prev,
} => {
let mut obj = json!({
"action": "update",
"path": format!("{}/{}", collection, rkey),
"cid": cid.to_string()
});
if let Some(prev_cid) = prev {
obj["prev"] = json!(prev_cid.to_string());
}
obj
}
} => json!({
"action": "update",
"path": format!("{}/{}", collection, rkey),
"cid": cid.to_string(),
"prev": prev.to_string(),
}),
RecordOp::Delete {
collection,
rkey,
prev,
} => {
let mut obj = json!({
"action": "delete",
"path": format!("{}/{}", collection, rkey),
"cid": null
});
if let Some(prev_cid) = prev {
obj["prev"] = json!(prev_cid.to_string());
}
obj
}
} => json!({
"action": "delete",
"path": format!("{}/{}", collection, rkey),
"cid": null,
"prev": prev.to_string(),
}),
})
.collect();
let inline_blocks: Vec<EventBlockInline> = block_bytes
.iter()
.map(|(cid, data)| EventBlockInline {
cid_bytes: cid.to_bytes(),
data: data.to_vec(),
})
.collect();
@@ -479,7 +554,7 @@ pub async fn commit_and_log(
prev_cid: current_root_cid.map(crate::types::CidLink::from),
ops: Some(json!(ops_json)),
blobs: Some(blobs.to_vec()),
blocks_cids: Some(blocks_cids.to_vec()),
blocks: Some(inline_blocks),
prev_data_cid: prev_data_cid.map(crate::types::CidLink::from),
rev: Some(rev_str.clone()),
};
@@ -510,6 +585,31 @@ pub async fn commit_and_log(
ApplyCommitError::Database(msg) => CommitError::DatabaseError(msg),
})?;
let apply_result = (|| {
let bs = state.block_store.clone();
let decrements = obsolete_cids.clone();
async move { bs.decrement_refs(&decrements).await }
})
.retry(
ExponentialBuilder::default()
.with_min_delay(std::time::Duration::from_millis(50))
.with_max_delay(std::time::Duration::from_secs(2))
.with_max_times(5),
)
.await;
if let Err(e) = apply_result {
let leaked: Vec<String> = obsolete_cids.iter().map(Cid::to_string).collect();
tracing::error!(
error = %e,
user_id = %user_id,
new_root = %new_root_cid,
leaked_cids = ?leaked,
"blockstore decrement_refs failed after metastore commit succeeded \
and exhausted retries; blocks may leak refcounts"
);
}
Ok(CommitResult {
commit_cid: new_root_cid,
rev: rev_str,
@@ -530,98 +630,57 @@ pub async fn create_record_internal(
.map_err(|e| CommitError::DatabaseError(e.to_string()))?
.ok_or(CommitError::UserNotFound)?;
let _write_lock = state.repo_write_locks.lock(user_id).await;
let to_commit_err = |e: ApiError| CommitError::DatabaseError(format!("{:?}", e));
let root_cid_link = state
.repos
.repo
.get_repo_root_cid_by_user_id(user_id)
let (ctx, mst) = begin_repo_write(state, user_id, None)
.await
.map_err(|e| CommitError::DatabaseError(e.to_string()))?
.ok_or(CommitError::RepoNotFound)?;
let current_root_cid = Cid::from_str(root_cid_link.as_str())
.map_err(|e| CommitError::InvalidCid(e.to_string()))?;
let tracking_store = TrackingBlockStore::new(state.block_store.clone());
let commit_bytes = tracking_store
.get(&current_root_cid)
.await
.map_err(|e| CommitError::BlockStoreFailed(format!("{:?}", e)))?
.ok_or(CommitError::BlockStoreFailed(
"Commit block not found".into(),
))?;
let commit = jacquard_repo::commit::Commit::from_cbor(&commit_bytes)
.map_err(|e| CommitError::CommitParseFailed(format!("{:?}", e)))?;
let mst = Mst::load(Arc::new(tracking_store.clone()), commit.data, None);
.map_err(to_commit_err)?;
let record_ipld = crate::util::json_to_ipld(record);
let mut record_bytes = Vec::new();
serde_ipld_dagcbor::to_writer(&mut record_bytes, &record_ipld)
.map_err(|e| CommitError::RecordSerializationFailed(format!("{:?}", e)))?;
let record_cid = tracking_store
.map_err(|e| CommitError::RecordSerializationFailed(e.to_string()))?;
let record_cid = ctx
.tracking_store
.put(&record_bytes)
.await
.map_err(|e| CommitError::BlockStoreFailed(format!("{:?}", e)))?;
.map_err(|e| CommitError::BlockStoreFailed(e.to_string()))?;
let key = format!("{}/{}", collection, rkey);
let new_mst = mst
.add(&key, record_cid)
.await
.map_err(|e| CommitError::MstOperationFailed(format!("{:?}", e)))?;
let new_mst_root = new_mst
.persist()
.await
.map_err(|e| CommitError::MstOperationFailed(format!("{:?}", e)))?;
.map_err(|e| CommitError::MstOperationFailed(e.to_string()))?;
let op = RecordOp::Create {
collection: collection.clone(),
rkey: rkey.clone(),
cid: record_cid,
cid: RecordCid::from(record_cid),
};
let mut new_mst_blocks = std::collections::BTreeMap::new();
let mut old_mst_blocks = std::collections::BTreeMap::new();
new_mst
.blocks_for_path(&key, &mut new_mst_blocks)
.await
.map_err(|e| CommitError::MstOperationFailed(format!("{:?}", e)))?;
mst.blocks_for_path(&key, &mut old_mst_blocks)
.await
.map_err(|e| CommitError::MstOperationFailed(format!("{:?}", e)))?;
let obsolete_cids: Vec<Cid> = std::iter::once(current_root_cid)
.chain(
old_mst_blocks
.keys()
.filter(|cid| !new_mst_blocks.contains_key(*cid))
.copied(),
)
.collect();
let mut relevant_blocks = new_mst_blocks;
relevant_blocks.extend(old_mst_blocks);
relevant_blocks.insert(record_cid, bytes::Bytes::from(record_bytes));
let written_cids: Vec<Cid> = tracking_store
.get_all_relevant_cids()
.into_iter()
.chain(relevant_blocks.keys().copied())
.collect::<std::collections::HashSet<_>>()
.into_iter()
.collect();
let written_cids_str: Vec<String> = written_cids.iter().map(|c| c.to_string()).collect();
let modified_keys = [key];
let blob_cids = extract_blob_cids(record);
let record_uri = AtUri::from_parts(did.as_str(), collection.as_str(), rkey.as_str());
let backlinks = extract_backlinks(&record_uri, record);
let result = commit_and_log(
let result = finalize_repo_write(
state,
CommitParams {
ctx,
new_mst,
FinalizeParams {
did,
user_id,
current_root_cid: Some(current_root_cid),
prev_data_cid: Some(commit.data),
new_mst_root,
controller_did: None,
delegation_detail: None,
ops: vec![op],
blocks_cids: &written_cids_str,
blobs: &blob_cids,
obsolete_cids,
modified_keys: &modified_keys,
blob_cids: &blob_cids,
backlinks_to_add: backlinks,
backlinks_to_remove: vec![],
},
)
.await?;
.await
.map_err(to_commit_err)?;
let uri = format!("at://{}/{}/{}", did, collection, rkey);
Ok((uri, result.commit_cid))
}
@@ -659,10 +718,20 @@ pub async fn sequence_sync_event(
let cid_link: crate::types::CidLink = commit_cid
.parse()
.map_err(|_| CommitError::InvalidCid(commit_cid.to_string()))?;
let commit_cid_parsed =
Cid::from_str(commit_cid).map_err(|e| CommitError::InvalidCid(e.to_string()))?;
let commit_bytes = state
.block_store
.get(&commit_cid_parsed)
.await
.map_err(|e| CommitError::BlockStoreFailed(format!("{:?}", e)))?
.ok_or(CommitError::BlockStoreFailed(
"Commit block not found for sync event".into(),
))?;
state
.repos
.repo
.insert_sync_event(did, &cid_link, rev)
.insert_sync_event(did, &cid_link, rev, &commit_bytes)
.await
.map_err(|e| CommitError::DatabaseError(format!("sync event: {}", e)))
}
@@ -676,10 +745,33 @@ pub async fn sequence_genesis_commit(
) -> Result<SequenceNumber, CommitError> {
let commit_cid_link = crate::types::CidLink::from(commit_cid);
let mst_root_cid_link = crate::types::CidLink::from(mst_root_cid);
let commit_bytes = state
.block_store
.get(commit_cid)
.await
.map_err(|e| CommitError::BlockStoreFailed(format!("{:?}", e)))?
.ok_or(CommitError::BlockStoreFailed(
"Genesis commit block not found".into(),
))?;
let mst_root_bytes = state
.block_store
.get(mst_root_cid)
.await
.map_err(|e| CommitError::BlockStoreFailed(format!("{:?}", e)))?
.ok_or(CommitError::BlockStoreFailed(
"Genesis MST root block not found".into(),
))?;
state
.repos
.repo
.insert_genesis_commit_event(did, &commit_cid_link, &mst_root_cid_link, rev)
.insert_genesis_commit_event(
did,
&commit_cid_link,
&mst_root_cid_link,
rev,
&commit_bytes,
&mst_root_bytes,
)
.await
.map_err(|e| CommitError::DatabaseError(format!("genesis commit event: {}", e)))
}
crates/tranquil-pds/src/scheduled.rs
+424 -152
View File
@@ -9,117 +9,15 @@ use std::time::Duration;
use tokio::time::interval;
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn};
use tranquil_db_traits::{
BlobRepository, BrokenGenesisCommit, RepoRepository, SequenceNumber, SsoRepository,
UserRepository,
};
use tranquil_db_traits::{BlobRepository, RepoRepository, SsoRepository, UserRepository};
use tranquil_store::blockstore::CidBytes;
use tranquil_store::bloom::BloomFilter;
use tranquil_types::{AtUri, CidLink, Did};
use crate::repo::AnyBlockStore;
use crate::storage::BlobStorage;
use crate::sync::car::encode_car_header;
#[derive(Debug)]
enum GenesisBackfillError {
MissingCommitCid,
InvalidCid,
BlockFetchFailed,
BlockNotFound,
CommitParseFailed,
UpdateFailed,
}
impl std::fmt::Display for GenesisBackfillError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::MissingCommitCid => f.write_str("missing commit_cid"),
Self::InvalidCid => f.write_str("invalid CID"),
Self::BlockFetchFailed => f.write_str("failed to fetch block"),
Self::BlockNotFound => f.write_str("block not found"),
Self::CommitParseFailed => f.write_str("failed to parse commit"),
Self::UpdateFailed => f.write_str("failed to update"),
}
}
}
async fn process_genesis_commit(
repo_repo: &dyn RepoRepository,
block_store: &AnyBlockStore,
row: BrokenGenesisCommit,
) -> Result<(Did, SequenceNumber), (SequenceNumber, GenesisBackfillError)> {
let commit_cid_str = row
.commit_cid
.ok_or((row.seq, GenesisBackfillError::MissingCommitCid))?;
let commit_cid =
Cid::from_str(&commit_cid_str).map_err(|_| (row.seq, GenesisBackfillError::InvalidCid))?;
let block = block_store
.get(&commit_cid)
.await
.map_err(|_| (row.seq, GenesisBackfillError::BlockFetchFailed))?
.ok_or((row.seq, GenesisBackfillError::BlockNotFound))?;
let commit = Commit::from_cbor(&block)
.map_err(|_| (row.seq, GenesisBackfillError::CommitParseFailed))?;
let blocks_cids = vec![commit.data.to_string(), commit_cid.to_string()];
repo_repo
.update_seq_blocks_cids(row.seq, &blocks_cids)
.await
.map_err(|_| (row.seq, GenesisBackfillError::UpdateFailed))?;
Ok((row.did, row.seq))
}
pub async fn backfill_genesis_commit_blocks(
repo_repo: Arc<dyn RepoRepository>,
block_store: AnyBlockStore,
) {
let broken_genesis_commits = match repo_repo.get_broken_genesis_commits().await {
Ok(rows) => rows,
Err(e) => {
error!(
"Failed to query repo_seq for genesis commit backfill: {:?}",
e
);
return;
}
};
if broken_genesis_commits.is_empty() {
debug!("No genesis commits need blocks_cids backfill");
return;
}
info!(
count = broken_genesis_commits.len(),
"Backfilling blocks_cids for genesis commits"
);
let results = futures::future::join_all(broken_genesis_commits.into_iter().map(|row| {
let repo_repo = repo_repo.clone();
let block_store = block_store.clone();
async move { process_genesis_commit(repo_repo.as_ref(), &block_store, row).await }
}))
.await;
let (success, failed) = results.iter().fold((0, 0), |(s, f), r| match r {
Ok((did, seq)) => {
info!(seq = seq.as_i64(), did = %did, "Fixed genesis commit blocks_cids");
(s + 1, f)
}
Err((seq, reason)) => {
warn!(
seq = seq.as_i64(),
reason = %reason,
"Failed to process genesis commit"
);
(s, f + 1)
}
});
info!(
success,
failed, "Completed genesis commit blocks_cids backfill"
);
}
async fn process_repo_rev(
repo_repo: &dyn RepoRepository,
block_store: &AnyBlockStore,
@@ -422,6 +320,7 @@ pub async fn backfill_record_blobs(repo_repo: Arc<dyn RepoRepository>, block_sto
info!(success, failed, "Completed record_blobs backfill");
}
#[allow(clippy::too_many_arguments)]
pub async fn start_scheduled_tasks(
user_repo: Arc<dyn UserRepository>,
blob_repo: Arc<dyn BlobRepository>,
@@ -429,23 +328,91 @@ pub async fn start_scheduled_tasks(
sso_repo: Arc<dyn SsoRepository>,
repo_repo: Arc<dyn RepoRepository>,
block_store: AnyBlockStore,
eventlog_segments_dir: Option<std::path::PathBuf>,
shutdown: CancellationToken,
) {
let cfg = tranquil_config::get();
let check_interval = Duration::from_secs(cfg.scheduled.delete_check_interval_secs);
let gc_interval = Duration::from_secs(cfg.scheduled.block_gc_interval_secs);
let compaction_enabled = cfg.scheduled.compaction_interval_secs > 0;
let reachability_enabled = cfg.scheduled.reachability_walk_interval_secs > 0;
let archival_enabled_secs = cfg.scheduled.archival_interval_secs > 0;
let event_retention_enabled = cfg.scheduled.event_retention_interval_secs > 0;
let compaction_interval = Duration::from_secs(cfg.scheduled.compaction_interval_secs.max(60));
let reachability_interval =
Duration::from_secs(cfg.scheduled.reachability_walk_interval_secs.max(60));
let archival_interval = Duration::from_secs(cfg.scheduled.archival_interval_secs.max(60));
let event_retention_interval =
Duration::from_secs(cfg.scheduled.event_retention_interval_secs.max(60));
let event_retention_max_age = Duration::from_secs(cfg.scheduled.event_retention_max_age_secs);
let archiver: Option<Arc<tranquil_store::archival::ContinuousArchiver>> =
match (&eventlog_segments_dir, &cfg.scheduled.archival_dest_dir) {
(Some(segments_dir), Some(dest_dir)) if archival_enabled_secs => {
let sidecar_path = segments_dir
.parent()
.unwrap_or(segments_dir)
.join("archival.state");
match tranquil_store::archival::LocalArchivalDestination::new(
std::path::PathBuf::from(dest_dir),
) {
Ok(dest) => {
info!(
dest_dir = dest_dir,
interval_secs = archival_interval.as_secs(),
"continuous archival enabled"
);
Some(Arc::new(tranquil_store::archival::ContinuousArchiver::new(
segments_dir.clone(),
sidecar_path,
Box::new(dest),
)))
}
Err(e) => {
error!(
dest_dir = dest_dir,
error = %e,
"failed to initialize archival destination, archival disabled"
);
None
}
}
}
_ => None,
};
info!(
check_interval_secs = check_interval.as_secs(),
gc_interval_secs = gc_interval.as_secs(),
compaction_enabled,
compaction_interval_secs = cfg.scheduled.compaction_interval_secs,
reachability_enabled,
reachability_interval_secs = cfg.scheduled.reachability_walk_interval_secs,
archival_enabled = archiver.is_some(),
event_retention_enabled,
event_retention_interval_secs = cfg.scheduled.event_retention_interval_secs,
event_retention_max_age_secs = cfg.scheduled.event_retention_max_age_secs,
"Starting scheduled tasks service"
);
let mut ticker = interval(check_interval);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
let mut gc_ticker = interval(gc_interval);
gc_ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
let mut compaction_ticker = interval(compaction_interval);
compaction_ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
let mut reachability_ticker = interval(reachability_interval);
reachability_ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
let mut archival_ticker = interval(archival_interval);
archival_ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
let mut event_retention_ticker = match event_retention_enabled {
true => {
let mut t = interval(event_retention_interval);
t.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
Some(t)
}
false => None,
};
loop {
tokio::select! {
@@ -492,63 +459,139 @@ pub async fn start_scheduled_tasks(
}
}
}
_ = gc_ticker.tick() => {
if let Some(pg) = block_store.as_postgres()
&& let Err(e) = run_block_gc(repo_repo.as_ref(), pg).await
{
error!("Block GC error: {e}");
_ = compaction_ticker.tick(), if compaction_enabled => {
if let Some(store) = block_store.as_tranquil_store() {
let store = store.clone();
let threshold = cfg.scheduled.compaction_liveness_threshold;
let grace_ms = cfg.scheduled.compaction_grace_period_ms;
if let Err(e) = tokio::task::spawn_blocking(move || {
run_compaction_pass(&store, threshold, grace_ms)
}).await.unwrap_or_else(|e| Err(anyhow::anyhow!("compaction task panicked: {e}"))) {
error!("Compaction error: {e}");
}
}
}
_ = reachability_ticker.tick(), if reachability_enabled => {
if let Some(store) = block_store.as_tranquil_store() {
let store = store.clone();
let repo_repo = repo_repo.clone();
match tokio::task::spawn_blocking(move || {
run_reachability_walk(&store, repo_repo.as_ref())
}).await {
Ok(Ok(result)) => {
info!(
repos_walked = result.repos_walked,
blocks_visited = result.blocks_visited,
live_refcounted = result.live_refcounted,
leaked_blocks = result.leaked_blocks,
repaired_blocks = result.repaired_blocks,
bloom_heap_mb = result.bloom_heap_bytes / (1024 * 1024),
"reachability walk complete"
);
}
Ok(Err(e)) => error!("Reachability walk error: {e}"),
Err(e) => error!("Reachability walk panicked: {e}"),
}
}
}
_ = archival_ticker.tick(), if archival_enabled_secs => {
if let Some(ref archiver) = archiver {
let archiver = Arc::clone(archiver);
match tokio::task::spawn_blocking(move || {
archiver.run_pass()
}).await {
Ok(Ok(result)) if result.segments_archived > 0 => {
info!(
segments_archived = result.segments_archived,
bytes_archived = result.bytes_archived,
"archival pass complete"
);
}
Ok(Ok(_)) => {}
Ok(Err(e)) => error!("Archival pass error: {e}"),
Err(e) => error!("Archival task panicked: {e}"),
}
}
}
_ = async {
match event_retention_ticker.as_mut() {
Some(t) => { t.tick().await; }
None => std::future::pending::<()>().await,
}
}, if event_retention_enabled => {
let cutoff = chrono::Utc::now()
- chrono::Duration::from_std(event_retention_max_age)
.expect("event_retention_max_age fits chrono::Duration: validated at config load");
match repo_repo.prune_events_older_than(cutoff).await {
Ok(count) if count.is_zero() => {
debug!("event retention: nothing past cutoff");
}
Ok(count) => {
info!(deleted = count.count(), unit = count.unit(), "event retention prune complete");
}
Err(e) => error!(error = %e, "event retention error"),
}
}
}
}
}
const BLOCK_GC_BATCH_SIZE: i64 = 1000;
async fn run_block_gc(
repo_repo: &dyn RepoRepository,
block_store: &crate::repo::PostgresBlockStore,
fn run_compaction_pass(
store: &tranquil_store::blockstore::TranquilBlockStore,
liveness_threshold: f64,
grace_period_ms: u64,
) -> anyhow::Result<()> {
let mut total_deleted: u64 = 0;
match store.cleanup_gc_meta() {
Ok(0) => {}
Ok(n) => info!(count = n, "cleaned up stale gc_meta entries"),
Err(e) => warn!(error = %e, "gc_meta cleanup failed, continuing"),
}
loop {
let candidates = block_store
.get_oldest_block_cids(BLOCK_GC_BATCH_SIZE)
.await
.context("failed to fetch candidate blocks")?;
let liveness_map = store
.compaction_liveness(grace_period_ms)
.context("failed to compute liveness")?;
match candidates.is_empty() {
true => break,
false => {
let batch_len = candidates.len();
let unreferenced = repo_repo
.find_unreferenced_blocks(&candidates)
.await
.context("failed to check block references")?;
let candidate = liveness_map
.iter()
.filter(|(_, info)| info.total_blocks > 0 && info.ratio() < liveness_threshold)
.min_by(|(_, a), (_, b)| {
a.ratio()
.partial_cmp(&b.ratio())
.unwrap_or(std::cmp::Ordering::Equal)
});
let deleted = match unreferenced.is_empty() {
true => 0,
false => block_store
.delete_blocks(&unreferenced)
.await
.context("failed to delete unreferenced blocks")?,
};
total_deleted = total_deleted.saturating_add(deleted);
match unreferenced.len() == batch_len {
true => continue,
false => break,
match candidate {
None => {
debug!("Compaction: no files below liveness threshold");
Ok(())
}
Some((&file_id, info)) => {
info!(
file_id = %file_id,
liveness = format!("{:.1}%", info.ratio() * 100.0),
live_blocks = info.live_blocks,
total_blocks = info.total_blocks,
"compacting data file"
);
match store.compact_file(file_id, grace_period_ms) {
Ok(result) => {
info!(
file_id = %result.file_id,
reclaimed_bytes = result.reclaimed_bytes,
live_blocks = result.live_blocks,
dead_blocks = result.dead_blocks,
"compaction complete"
);
Ok(())
}
Err(tranquil_store::blockstore::CompactionError::ActiveFileCannotBeCompacted) => {
debug!(file_id = %file_id, "skipped active file");
Ok(())
}
Err(e) => Err(anyhow::anyhow!("compaction failed: {e}")),
}
}
}
match total_deleted > 0 {
true => info!(total_deleted, "Block GC cycle complete"),
false => debug!("Block GC cycle: no orphaned blocks found"),
}
Ok(())
}
async fn process_scheduled_deletions(
@@ -694,3 +737,232 @@ pub async fn generate_repo_car_from_user_blocks(
generate_repo_car(block_store, &actual_head_cid).await
}
pub struct ReachabilityResult {
pub repos_walked: u64,
pub blocks_visited: u64,
pub live_refcounted: u64,
pub leaked_blocks: u64,
pub repaired_blocks: u64,
pub bloom_heap_bytes: usize,
}
const REPO_PAGE_SIZE: i64 = 500;
const BLOOM_FALSE_POSITIVE_RATE: f64 = 0.01;
fn cid_to_bytes(cid: &Cid) -> anyhow::Result<CidBytes> {
cid.to_bytes()
.try_into()
.map_err(|_| anyhow::anyhow!("CID byte length mismatch for {cid}"))
}
fn walk_repo_dag_sync(
store: &tranquil_store::blockstore::TranquilBlockStore,
head_cid: &Cid,
reachable: &mut std::collections::HashSet<CidBytes>,
) -> anyhow::Result<()> {
let mut to_visit = vec![cid_to_bytes(head_cid)?];
while let Some(cid_bytes) = to_visit.pop() {
if !reachable.insert(cid_bytes) {
continue;
}
let block = match store.get_block_sync(&cid_bytes)? {
Some(b) => b,
None => {
tracing::warn!(
?cid_bytes,
"referenced block missing during reachability walk"
);
continue;
}
};
if let Ok(commit) = Commit::from_cbor(&block) {
to_visit.push(cid_to_bytes(&commit.data)?);
if let Some(prev) = &commit.prev {
to_visit.push(cid_to_bytes(prev)?);
}
} else if let Ok(Ipld::Map(ref obj)) = serde_ipld_dagcbor::from_slice::<Ipld>(&block) {
if let Some(Ipld::Link(left_cid)) = obj.get("l")
&& let Ok(bytes) = <CidBytes>::try_from(left_cid.to_bytes().as_slice())
{
to_visit.push(bytes);
}
if let Some(Ipld::List(entries)) = obj.get("e") {
entries
.iter()
.filter_map(|entry| match entry {
Ipld::Map(entry_obj) => Some(entry_obj),
_ => None,
})
.flat_map(|entry_obj| {
[entry_obj.get("t"), entry_obj.get("v")]
.into_iter()
.flatten()
.filter_map(|v| match v {
Ipld::Link(link_cid) => {
<CidBytes>::try_from(link_cid.to_bytes().as_slice()).ok()
}
_ => None,
})
})
.for_each(|bytes| to_visit.push(bytes));
}
}
}
Ok(())
}
fn paginate_repos(
rt: &tokio::runtime::Handle,
repo_repo: &dyn RepoRepository,
mut each_page: impl FnMut(&[tranquil_db_traits::RepoListItem]) -> anyhow::Result<()>,
) -> anyhow::Result<()> {
let mut cursor_did: Option<Did> = None;
std::iter::from_fn(|| {
let page = rt
.block_on(repo_repo.list_repos_paginated(cursor_did.as_ref(), REPO_PAGE_SIZE))
.context("failed to list repos");
match &page {
Ok(p) => {
cursor_did = p.last().map(|r| r.did.clone());
cursor_did.as_ref().map(|_| page)
}
Err(_) => Some(page),
}
})
.try_for_each(|page| each_page(&page?))
}
pub fn run_reachability_walk(
store: &tranquil_store::blockstore::TranquilBlockStore,
repo_repo: &dyn RepoRepository,
) -> anyhow::Result<ReachabilityResult> {
let rt = tokio::runtime::Handle::current();
let approx_blocks = store.approximate_block_count();
const MAX_PREALLOC: usize = 64_000_000;
let mut visited = std::collections::HashSet::with_capacity(
usize::try_from(approx_blocks)
.unwrap_or(0)
.min(MAX_PREALLOC),
);
info!(approx_blocks, "reachability walk starting");
let mut repos_walked: u64 = 0;
let mut seen_heads: std::collections::HashMap<Did, CidLink> = std::collections::HashMap::new();
paginate_repos(&rt, repo_repo, |page| {
page.iter().try_for_each(|repo| -> anyhow::Result<()> {
let cid =
Cid::from_str(repo.repo_root_cid.as_str()).context("invalid repo_root_cid")?;
seen_heads.insert(repo.did.clone(), repo.repo_root_cid.clone());
walk_repo_dag_sync(store, &cid, &mut visited)?;
repos_walked = repos_walked.saturating_add(1);
if repos_walked.is_multiple_of(1000) {
info!(
repos_walked,
blocks_so_far = visited.len(),
"reachability walk progress"
);
}
Ok(())
})
})?;
let blocks_visited = u64::try_from(visited.len()).unwrap_or(u64::MAX);
let mut reachable =
BloomFilter::with_capacity_and_fpr(blocks_visited.max(1024), BLOOM_FALSE_POSITIVE_RATE);
visited.iter().for_each(|cid| reachable.insert(cid));
drop(visited);
let mut stale_repos: u64 = 0;
paginate_repos(&rt, repo_repo, |page| {
let stale: Vec<_> = page
.iter()
.filter(|repo| seen_heads.get(&repo.did) != Some(&repo.repo_root_cid))
.collect();
stale.iter().try_for_each(|repo| -> anyhow::Result<()> {
let cid =
Cid::from_str(repo.repo_root_cid.as_str()).context("invalid repo_root_cid")?;
let mut extra = std::collections::HashSet::new();
walk_repo_dag_sync(store, &cid, &mut extra)?;
extra.iter().for_each(|c| reachable.insert(c));
seen_heads.insert(repo.did.clone(), repo.repo_root_cid.clone());
stale_repos = stale_repos.saturating_add(1);
Ok(())
})
})?;
info!(
repos_walked,
blocks_visited,
stale_repos,
bloom_heap_mb = reachable.heap_bytes() / (1024 * 1024),
"DAG traversal complete, quiescing blockstore for leak scan"
);
let (_snapshot, quiesce_guard) = store
.quiesce()
.map_err(|e| anyhow::anyhow!("failed to quiesce blockstore: {e}"))?;
let mut quiesced_stale: u64 = 0;
paginate_repos(&rt, repo_repo, |page| {
page.iter()
.filter(|repo| seen_heads.get(&repo.did) != Some(&repo.repo_root_cid))
.try_for_each(|repo| -> anyhow::Result<()> {
let cid =
Cid::from_str(repo.repo_root_cid.as_str()).context("invalid repo_root_cid")?;
let mut extra = std::collections::HashSet::new();
walk_repo_dag_sync(store, &cid, &mut extra)?;
extra.iter().for_each(|c| reachable.insert(c));
quiesced_stale = quiesced_stale.saturating_add(1);
Ok(())
})
})?;
if quiesced_stale > 0 {
info!(
quiesced_stale,
"caught additional stale repos during quiesced re-walk"
);
}
let (leaked, live_refcounted) = store
.find_leaked_refcounts(|cid| reachable.contains(cid))
.map_err(|e| anyhow::anyhow!("failed to scan index: {e}"))?;
let leaked_blocks = u64::try_from(leaked.len()).unwrap_or(u64::MAX);
let bloom_heap_bytes = reachable.heap_bytes();
drop(reachable);
quiesce_guard.resume();
let repaired_blocks = match leaked.is_empty() {
true => 0,
false => {
warn!(
leaked_blocks,
"reachability walk found leaked refcounts, repairing"
);
store
.repair_leaked_refcounts(&leaked)
.map_err(|e| anyhow::anyhow!("failed to repair leaked refcounts: {e}"))?
}
};
Ok(ReachabilityResult {
repos_walked,
blocks_visited,
live_refcounted,
leaked_blocks,
repaired_blocks,
bloom_heap_bytes,
})
}
crates/tranquil-pds/src/state.rs
+99 -7
View File
@@ -49,6 +49,7 @@ pub struct AppState {
pub bootstrap_invite_code: Option<String>,
pub signal_sender: Option<Arc<tranquil_signal::SignalSlot>>,
pub signal_store_provider: Option<Arc<dyn tranquil_signal::SignalStoreProvider>>,
pub eventlog_segments_dir: Option<PathBuf>,
}
#[derive(Debug, Clone, Copy)]
@@ -214,9 +215,7 @@ impl AppState {
match cfg.storage.repo_backend() {
tranquil_config::RepoBackend::TranquilStore => {
tracing::info!(
"tranquil-store repo backend active. EXPERIMENTAL! No garbage collection, no backup/restore"
);
tracing::info!("tranquil-store repo backend active. EXPERIMENTAL!");
Ok(Self::from_store(shutdown).await)
}
tranquil_config::RepoBackend::Postgres => {
@@ -273,10 +272,11 @@ impl AppState {
pub async fn from_db(db: PgPool, shutdown: CancellationToken) -> Self {
let cfg = tranquil_config::get();
let (repos, block_store, signal_store_provider): (
let (repos, block_store, signal_store_provider, eventlog_segments_dir): (
PostgresRepositories,
crate::repo::AnyBlockStore,
Option<Arc<dyn tranquil_signal::SignalStoreProvider>>,
Option<PathBuf>,
) = match cfg.storage.repo_backend() == tranquil_config::RepoBackend::TranquilStore {
true => {
let wiring = wire_tranquil_store(&cfg.tranquil_store, shutdown.clone());
@@ -284,6 +284,7 @@ impl AppState {
wiring.repos,
crate::repo::AnyBlockStore::TranquilStore(wiring.blockstore),
Some(wiring.signal_provider),
Some(wiring.segments_dir),
)
}
false => {
@@ -294,11 +295,19 @@ impl AppState {
repos,
crate::repo::AnyBlockStore::Postgres(PostgresBlockStore::new(db)),
Some(provider),
None,
)
}
};
Self::build(repos, block_store, signal_store_provider, shutdown).await
Self::build(
repos,
block_store,
signal_store_provider,
eventlog_segments_dir,
shutdown,
)
.await
}
pub async fn from_store(shutdown: CancellationToken) -> Self {
@@ -309,6 +318,30 @@ impl AppState {
wiring.repos,
crate::repo::AnyBlockStore::TranquilStore(wiring.blockstore),
Some(wiring.signal_provider),
Some(wiring.segments_dir),
shutdown,
)
.await
}
pub async fn from_store_at(data_dir: &std::path::Path, shutdown: CancellationToken) -> Self {
let base = &tranquil_config::get().tranquil_store;
let store_cfg = tranquil_config::TranquilStoreConfig {
data_dir: data_dir.to_string_lossy().into_owned(),
memory_budget_mb: base.memory_budget_mb,
handler_threads: base.handler_threads,
eventlog_pending_bytes_budget: base.eventlog_pending_bytes_budget,
eventlog_max_event_payload: base.eventlog_max_event_payload,
max_blockstore_file_size: base.max_blockstore_file_size,
max_eventlog_segment_size: base.max_eventlog_segment_size,
};
let wiring = wire_tranquil_store(&store_cfg, shutdown.clone());
Self::build(
wiring.repos,
crate::repo::AnyBlockStore::TranquilStore(wiring.blockstore),
Some(wiring.signal_provider),
Some(wiring.segments_dir),
shutdown,
)
.await
@@ -318,6 +351,7 @@ impl AppState {
repos: PostgresRepositories,
block_store: crate::repo::AnyBlockStore,
signal_store_provider: Option<Arc<dyn tranquil_signal::SignalStoreProvider>>,
eventlog_segments_dir: Option<PathBuf>,
shutdown: CancellationToken,
) -> Self {
AuthConfig::init();
@@ -359,6 +393,7 @@ impl AppState {
bootstrap_invite_code: None,
signal_sender: None,
signal_store_provider,
eventlog_segments_dir,
}
}
@@ -442,6 +477,7 @@ struct TranquilStoreWiring {
blockstore: tranquil_store::blockstore::TranquilBlockStore,
signal_provider: Arc<dyn tranquil_signal::SignalStoreProvider>,
repos: PostgresRepositories,
segments_dir: PathBuf,
}
fn wire_tranquil_store(
@@ -486,14 +522,18 @@ fn wire_tranquil_store(
let blockstore = TranquilBlockStore::open(BlockStoreConfig {
data_dir: blockstore_data_dir,
index_dir: blockstore_index_dir,
max_file_size: tranquil_store::blockstore::DEFAULT_MAX_FILE_SIZE,
max_file_size: store_cfg.max_blockstore_file_size,
group_commit: Default::default(),
shard_count: tranquil_store::blockstore::DEFAULT_SHARD_COUNT,
})
.expect("failed to open blockstore");
let event_log = EventLog::open(
EventLogConfig {
segments_dir,
pending_bytes_budget: store_cfg.eventlog_pending_bytes_budget,
max_event_payload: store_cfg.eventlog_max_event_payload,
max_segment_size: store_cfg.max_eventlog_segment_size,
..EventLogConfig::default()
},
RealIO::new(),
@@ -503,6 +543,10 @@ fn wire_tranquil_store(
let bridge = Arc::new(EventLogBridge::new(Arc::clone(&event_log)));
let was_clean = tranquil_store::consistency::had_clean_shutdown(&data_dir);
tranquil_store::consistency::remove_clean_shutdown_marker(&data_dir)
.expect("failed to remove clean shutdown marker");
let indexes = metastore.partition(Partition::Indexes).clone();
let event_ops = metastore.event_ops(Arc::clone(&bridge));
let recovered = event_ops
@@ -512,6 +556,41 @@ fn wire_tranquil_store(
tracing::info!(recovered, "replayed metastore mutations from eventlog");
}
let skip_check = std::env::var("TRANQUIL_SKIP_CONSISTENCY_CHECK").is_ok_and(|v| v == "1");
if (!was_clean || recovered > 0) && !skip_check {
let report = tranquil_store::consistency::verify_store_consistency(
&blockstore,
&metastore,
&event_log,
);
report.log_findings();
if report.has_repairable_issues() {
let repair = tranquil_store::consistency::repair_known_issues(&blockstore, &report);
if repair.orphan_files_removed > 0 {
tracing::info!(
removed = repair.orphan_files_removed,
"repaired orphan data files"
);
}
if repair.had_errors() {
tracing::warn!(errors = repair.repair_errors, "some repairs failed");
}
}
if report.has_unrecoverable_issues() {
panic!(
"unrecoverable store inconsistencies detected: {} dangling root CIDs, {} dangling record CIDs, \
{} deserialization failures, cursor_ahead={}. \
manual intervention required. set TRANQUIL_SKIP_CONSISTENCY_CHECK=1 to bypass.",
report.dangling_root_cids.len(),
report.dangling_record_cids.len(),
report.deserialization_failures,
report.cursor_ahead_of_eventlog,
);
}
}
let notifier = bridge.notifier();
let signal_db = metastore.database().clone();
let signal_ks = metastore.signal_keyspace();
@@ -525,13 +604,23 @@ fn wire_tranquil_store(
tokio::spawn({
let pool = Arc::clone(&pool);
let shutdown_event_log = Arc::clone(&event_log);
let shutdown_data_dir = data_dir.clone();
async move {
shutdown.cancelled().await;
pool.close().await;
if let Err(e) = shutdown_event_log.shutdown() {
tracing::warn!(error = %e, "eventlog shutdown failed");
}
if let Err(e) =
tranquil_store::consistency::write_clean_shutdown_marker(&shutdown_data_dir)
{
tracing::warn!(error = %e, "failed to write clean shutdown marker");
}
}
});
let client = MetastoreClient::<RealIO>::new(pool);
let client = MetastoreClient::<RealIO>::new(pool, Arc::clone(&event_log));
tracing::info!(data_dir = %store_cfg.data_dir, "tranquil-store data directory");
@@ -553,9 +642,12 @@ fn wire_tranquil_store(
tranquil_signal::fjall_store::FjallSignalStoreProvider::new(signal_db, signal_ks),
);
let eventlog_segments_dir = event_log.segments_dir().to_path_buf();
TranquilStoreWiring {
blockstore,
signal_provider,
repos,
segments_dir: eventlog_segments_dir,
}
}
crates/tranquil-pds/src/sync/frame.rs
+1 -9
View File
@@ -142,7 +142,6 @@ pub struct CommitFrameBuilder {
seq: i64,
did: Did,
commit_cid: Cid,
prev_cid: Option<Cid>,
ops_json: serde_json::Value,
blob_cids: Vec<Cid>,
time: chrono::DateTime<chrono::Utc>,
@@ -150,12 +149,10 @@ pub struct CommitFrameBuilder {
}
impl CommitFrameBuilder {
#[allow(clippy::too_many_arguments)]
pub fn new(
seq: i64,
did: Did,
commit_cid_str: &str,
prev_cid_str: Option<&str>,
ops_json: serde_json::Value,
blob_strs: Vec<String>,
time: chrono::DateTime<chrono::Utc>,
@@ -163,9 +160,6 @@ impl CommitFrameBuilder {
) -> Result<Self, CommitFrameError> {
let commit_cid = Cid::from_str(commit_cid_str)
.map_err(|_| CommitFrameError::InvalidCommitCid(commit_cid_str.to_string()))?;
let prev_cid = prev_cid_str.map(Cid::from_str).transpose().map_err(|_| {
CommitFrameError::InvalidCommitCid(prev_cid_str.unwrap_or("").to_string())
})?;
let blob_cids: Vec<Cid> = blob_strs
.iter()
.filter_map(|s| Cid::from_str(s).ok())
@@ -174,7 +168,6 @@ impl CommitFrameBuilder {
seq,
did,
commit_cid,
prev_cid,
ops_json,
blob_cids,
time,
@@ -197,7 +190,7 @@ impl CommitFrameBuilder {
})
.collect();
let rev = self.rev.unwrap_or_else(placeholder_rev);
let since = self.prev_cid.as_ref().map(|_| rev.clone());
let since = None;
CommitFrame {
seq: self.seq,
rebase: false,
@@ -235,7 +228,6 @@ impl TryFrom<SequencedEvent> for CommitFrame {
event.seq.as_i64(),
event.did.clone(),
commit_cid.as_str(),
event.prev_cid.as_ref().map(|c| c.as_str()),
event.ops.unwrap_or_default(),
event.blobs.unwrap_or_default(),
event.created_at,
crates/tranquil-pds/src/sync/import.rs
+1 -1
View File
@@ -8,7 +8,7 @@ use std::io::Cursor;
use std::sync::Arc;
use thiserror::Error;
use tracing::debug;
use tranquil_db::{ImportBlock, ImportRecord, ImportRepoError, RepoRepository};
use tranquil_db_traits::{ImportBlock, ImportRecord, ImportRepoError, RepoRepository};
use tranquil_types::CidLink;
use uuid::Uuid;
crates/tranquil-pds/src/sync/util.rs
+133 -139
View File
@@ -15,7 +15,7 @@ use std::collections::{BTreeMap, HashMap};
use std::io::Cursor;
use std::str::FromStr;
use tokio::io::AsyncWriteExt;
use tranquil_db_traits::{AccountStatus, RepoEventType, RepoRepository};
use tranquil_db_traits::{AccountStatus, EventBlocks, RepoEventType, RepoRepository};
use tranquil_types::Did;
#[derive(Debug)]
@@ -25,7 +25,8 @@ pub enum SyncFrameError {
IoFlush(std::io::Error),
CborSerialize(String),
MissingCommitCid,
CommitBlockNotFound,
MissingInlineCommitBlock,
MissingLegacyBlocks(Vec<Cid>),
RevExtraction,
InvalidEvent(String),
BlockStore(tranquil_db_traits::DbError),
@@ -40,7 +41,17 @@ impl std::fmt::Display for SyncFrameError {
Self::IoFlush(e) => write!(f, "CAR buffer flush failed: {}", e),
Self::CborSerialize(e) => write!(f, "CBOR serialization failed: {}", e),
Self::MissingCommitCid => write!(f, "missing commit_cid"),
Self::CommitBlockNotFound => write!(f, "commit block not found"),
Self::MissingInlineCommitBlock => {
write!(f, "event missing inline commit block bytes")
}
Self::MissingLegacyBlocks(cids) => {
write!(
f,
"legacy event references blocks not present in live blockstore (gc race): {} missing cid(s), first: {}",
cids.len(),
cids.first().map(|c| c.to_string()).unwrap_or_default()
)
}
Self::RevExtraction => write!(f, "could not extract rev from commit"),
Self::InvalidEvent(msg) => write!(f, "invalid event: {}", msg),
Self::BlockStore(e) => write!(f, "block store error: {}", e),
@@ -178,7 +189,7 @@ fn extract_rev_from_commit_bytes(commit_bytes: &[u8]) -> Option<String> {
async fn write_car_blocks(
commit_cid: Cid,
commit_bytes: Option<Bytes>,
commit_bytes: Bytes,
other_blocks: BTreeMap<Cid, Bytes>,
) -> Result<Vec<u8>, SyncFrameError> {
let mut buffer = Cursor::new(Vec::new());
@@ -190,12 +201,10 @@ async fn write_car_blocks(
.await
.map_err(SyncFrameError::CarWrite)?;
}
if let Some(data) = commit_bytes {
writer
.write(commit_cid, data.as_ref())
.await
.map_err(SyncFrameError::CarWrite)?;
}
writer
.write(commit_cid, commit_bytes.as_ref())
.await
.map_err(SyncFrameError::CarWrite)?;
writer.finish().await.map_err(SyncFrameError::CarFinalize)?;
buffer.flush().await.map_err(SyncFrameError::IoFlush)?;
Ok(buffer.into_inner())
@@ -265,26 +274,81 @@ fn format_account_event(event: &SequencedEvent) -> Result<Vec<u8>, SyncFrameErro
Ok(bytes)
}
async fn format_sync_event(
async fn event_blocks_to_map(
blocks: Option<&EventBlocks>,
prefetched: &HashMap<Cid, Bytes>,
state: &AppState,
) -> Result<HashMap<Cid, Bytes>, SyncFrameError> {
match blocks {
None => Ok(HashMap::new()),
Some(EventBlocks::Inline(inline)) => inline
.iter()
.map(|b| {
Cid::read_bytes(b.cid_bytes.as_slice())
.map_err(SyncFrameError::CidParse)
.map(|cid| (cid, Bytes::copy_from_slice(&b.data)))
})
.collect(),
Some(EventBlocks::LegacyCids(cid_strs)) => {
let cids: Vec<Cid> = cid_strs
.iter()
.map(|s| Cid::from_str(s).map_err(SyncFrameError::CidParse))
.collect::<Result<_, _>>()?;
let mut map: HashMap<Cid, Bytes> = HashMap::with_capacity(cids.len());
let to_fetch: Vec<Cid> = cids
.iter()
.filter(|cid| match prefetched.get(cid) {
Some(b) => {
map.insert(**cid, b.clone());
false
}
None => true,
})
.copied()
.collect();
if !to_fetch.is_empty() {
let fetched = state.block_store.get_many(&to_fetch).await?;
let (found, missing): (Vec<_>, Vec<_>) = to_fetch
.into_iter()
.zip(fetched)
.partition(|(_, opt)| opt.is_some());
found
.into_iter()
.filter_map(|(cid, opt)| opt.map(|b| (cid, b)))
.for_each(|(cid, b)| {
map.insert(cid, b);
});
if !missing.is_empty() {
let missing_cids: Vec<Cid> = missing.into_iter().map(|(cid, _)| cid).collect();
return Err(SyncFrameError::MissingLegacyBlocks(missing_cids));
}
}
Ok(map)
}
}
}
async fn format_sync_event(
event: &SequencedEvent,
prefetched: &HashMap<Cid, Bytes>,
state: &AppState,
) -> Result<Vec<u8>, SyncFrameError> {
let commit_cid_str = event
.commit_cid
.as_ref()
.ok_or(SyncFrameError::MissingCommitCid)?;
let commit_cid = Cid::from_str(commit_cid_str)?;
let commit_bytes = state
.block_store
let blocks_map = event_blocks_to_map(event.blocks.as_ref(), prefetched, state).await?;
let commit_bytes = blocks_map
.get(&commit_cid)
.await?
.ok_or(SyncFrameError::CommitBlockNotFound)?;
.cloned()
.ok_or(SyncFrameError::MissingInlineCommitBlock)?;
let rev = if let Some(ref stored_rev) = event.rev {
stored_rev.clone()
} else {
extract_rev_from_commit_bytes(&commit_bytes).ok_or(SyncFrameError::RevExtraction)?
};
let car_bytes = write_car_blocks(commit_cid, Some(commit_bytes), BTreeMap::new()).await?;
let car_bytes = write_car_blocks(commit_cid, commit_bytes, BTreeMap::new()).await?;
serialize_event_frame(
FrameType::Sync,
&SyncFrame {
@@ -302,13 +366,17 @@ struct CommitEventContext {
frame: CommitFrame,
commit_cid: Cid,
prev_cid: Option<Cid>,
block_cids: Vec<Cid>,
inline_blocks: HashMap<Cid, Bytes>,
}
fn prepare_commit_event(event: SequencedEvent) -> Result<CommitEventContext, SyncFrameError> {
let block_cids_str = event.blocks_cids.clone().unwrap_or_default();
async fn prepare_commit_event(
event: SequencedEvent,
prefetched: &HashMap<Cid, Bytes>,
state: &AppState,
) -> Result<CommitEventContext, SyncFrameError> {
let prev_cid_link = event.prev_cid.clone();
let prev_data_cid_link = event.prev_data_cid.clone();
let inline_blocks = event_blocks_to_map(event.blocks.as_ref(), prefetched, state).await?;
let mut frame: CommitFrame =
event
.try_into()
@@ -321,46 +389,43 @@ fn prepare_commit_event(event: SequencedEvent) -> Result<CommitEventContext, Syn
frame.prev_data = Some(cid);
}
let commit_cid = frame.commit;
if !inline_blocks.contains_key(&commit_cid) {
return Err(SyncFrameError::MissingInlineCommitBlock);
}
let prev_cid = prev_cid_link
.as_ref()
.and_then(|c| Cid::from_str(c.as_str()).ok());
let mut block_cids: Vec<Cid> = block_cids_str
.iter()
.filter_map(|s| Cid::from_str(s).ok())
.filter(|c| Some(*c) != prev_cid)
.collect();
if !block_cids.contains(&commit_cid) {
block_cids.push(commit_cid);
}
Ok(CommitEventContext {
frame,
commit_cid,
prev_cid,
block_cids,
inline_blocks,
})
}
fn partition_blocks(
block_cids: impl IntoIterator<Item = (Cid, Bytes)>,
commit_cid: Cid,
) -> (Option<Bytes>, BTreeMap<Cid, Bytes>) {
) -> Result<(Bytes, BTreeMap<Cid, Bytes>), SyncFrameError> {
let (commit_data, other_blocks): (Vec<_>, Vec<_>) = block_cids
.into_iter()
.partition(|(cid, _)| *cid == commit_cid);
let commit_bytes = commit_data.into_iter().next().map(|(_, data)| data);
let commit_bytes = commit_data
.into_iter()
.next()
.map(|(_, data)| data)
.ok_or(SyncFrameError::MissingInlineCommitBlock)?;
let other = other_blocks.into_iter().collect();
(commit_bytes, other)
Ok((commit_bytes, other))
}
async fn finalize_commit_frame(
mut frame: CommitFrame,
commit_cid: Cid,
commit_bytes: Option<Bytes>,
commit_bytes: Bytes,
other_blocks: BTreeMap<Cid, Bytes>,
) -> Result<Vec<u8>, SyncFrameError> {
if let Some(ref cb) = commit_bytes
&& let Some(rev) = extract_rev_from_commit_bytes(cb)
{
if let Some(rev) = extract_rev_from_commit_bytes(&commit_bytes) {
frame.rev = rev;
}
frame.blocks = write_car_blocks(commit_cid, commit_bytes, other_blocks).await?;
@@ -371,134 +436,63 @@ async fn finalize_commit_frame(
pub async fn format_event_for_sending(
state: &AppState,
event: SequencedEvent,
) -> Result<Vec<u8>, SyncFrameError> {
format_event_inner(event, &HashMap::new(), state).await
}
async fn format_event_inner(
event: SequencedEvent,
prefetched: &HashMap<Cid, Bytes>,
state: &AppState,
) -> Result<Vec<u8>, SyncFrameError> {
match event.event_type {
RepoEventType::Identity => return format_identity_event(&event),
RepoEventType::Account => return format_account_event(&event),
RepoEventType::Sync => return format_sync_event(state, &event).await,
RepoEventType::Sync => return format_sync_event(&event, prefetched, state).await,
RepoEventType::Commit => {}
}
let ctx = prepare_commit_event(event)?;
let ctx = prepare_commit_event(event, prefetched, state).await?;
let mut frame = ctx.frame;
if let Some(ref pc) = ctx.prev_cid
&& let Ok(Some(prev_bytes)) = state.block_store.get(pc).await
&& let Some(rev) = extract_rev_from_commit_bytes(&prev_bytes)
&& let Some(prev_bytes) = ctx.inline_blocks.get(pc)
&& let Some(rev) = extract_rev_from_commit_bytes(prev_bytes)
{
frame.since = Some(rev);
}
if ctx.block_cids.is_empty() {
frame.blocks = Vec::new();
let capacity = frame.blocks.len() + 512;
return serialize_event_frame(FrameType::Commit, &frame, capacity);
}
let fetched = state.block_store.get_many(&ctx.block_cids).await?;
let resolved = ctx
.block_cids
.iter()
.zip(fetched.iter())
.filter_map(|(cid, data_opt)| data_opt.as_ref().map(|data| (*cid, data.clone())));
let (commit_bytes, other_blocks) = partition_blocks(resolved, ctx.commit_cid);
let (commit_bytes, other_blocks) = partition_blocks(ctx.inline_blocks, ctx.commit_cid)?;
finalize_commit_frame(frame, ctx.commit_cid, commit_bytes, other_blocks).await
}
pub async fn format_event_with_prefetched_blocks(
state: &AppState,
event: SequencedEvent,
prefetched: &HashMap<Cid, Bytes>,
) -> Result<Vec<u8>, SyncFrameError> {
format_event_inner(event, prefetched, state).await
}
pub async fn prefetch_blocks_for_events(
state: &AppState,
events: &[SequencedEvent],
) -> Result<HashMap<Cid, Bytes>, SyncFrameError> {
let mut all_cids: Vec<Cid> = events
let legacy_cids: Vec<Cid> = events
.iter()
.flat_map(|event| {
let commit_cid = event
.commit_cid
.as_ref()
.and_then(|s| Cid::from_str(s).ok());
let prev_cid = event.prev_cid.as_ref().and_then(|s| Cid::from_str(s).ok());
let block_cids = event
.blocks_cids
.as_ref()
.map(|cids| cids.iter().filter_map(|s| Cid::from_str(s).ok()).collect())
.unwrap_or_else(Vec::new);
commit_cid.into_iter().chain(prev_cid).chain(block_cids)
.filter_map(|e| match e.blocks.as_ref() {
Some(EventBlocks::LegacyCids(strs)) => Some(strs.iter()),
_ => None,
})
.collect();
all_cids.sort();
all_cids.dedup();
if all_cids.is_empty() {
.flatten()
.map(|s| Cid::from_str(s).map_err(SyncFrameError::CidParse))
.collect::<Result<_, _>>()?;
if legacy_cids.is_empty() {
return Ok(HashMap::new());
}
let fetched = state.block_store.get_many(&all_cids).await?;
let blocks_map: HashMap<Cid, Bytes> = all_cids
let fetched = state.block_store.get_many(&legacy_cids).await?;
Ok(legacy_cids
.into_iter()
.zip(fetched)
.filter_map(|(cid, data_opt)| data_opt.map(|data| (cid, data)))
.collect();
Ok(blocks_map)
}
fn format_sync_event_with_prefetched(
event: &SequencedEvent,
prefetched: &HashMap<Cid, Bytes>,
) -> Result<Vec<u8>, SyncFrameError> {
let commit_cid_str = event
.commit_cid
.as_ref()
.ok_or(SyncFrameError::MissingCommitCid)?;
let commit_cid = Cid::from_str(commit_cid_str)?;
let commit_bytes = prefetched
.get(&commit_cid)
.ok_or(SyncFrameError::CommitBlockNotFound)?;
let rev = if let Some(ref stored_rev) = event.rev {
stored_rev.clone()
} else {
extract_rev_from_commit_bytes(commit_bytes).ok_or(SyncFrameError::RevExtraction)?
};
let car_bytes = futures::executor::block_on(write_car_blocks(
commit_cid,
Some(commit_bytes.clone()),
BTreeMap::new(),
))?;
serialize_event_frame(
FrameType::Sync,
&SyncFrame {
did: event.did.clone(),
rev,
blocks: car_bytes,
seq: event.seq.as_i64(),
time: format_atproto_time(event.created_at),
},
512,
)
}
pub async fn format_event_with_prefetched_blocks(
event: SequencedEvent,
prefetched: &HashMap<Cid, Bytes>,
) -> Result<Vec<u8>, SyncFrameError> {
match event.event_type {
RepoEventType::Identity => return format_identity_event(&event),
RepoEventType::Account => return format_account_event(&event),
RepoEventType::Sync => return format_sync_event_with_prefetched(&event, prefetched),
RepoEventType::Commit => {}
}
let ctx = prepare_commit_event(event)?;
let mut frame = ctx.frame;
if let Some(ref pc) = ctx.prev_cid
&& let Some(prev_bytes) = prefetched.get(pc)
&& let Some(rev) = extract_rev_from_commit_bytes(prev_bytes)
{
frame.since = Some(rev);
}
if ctx.block_cids.is_empty() {
frame.blocks = Vec::new();
let capacity = frame.blocks.len() + 512;
return serialize_event_frame(FrameType::Commit, &frame, capacity);
}
let resolved = ctx
.block_cids
.into_iter()
.filter_map(|cid| prefetched.get(&cid).map(|data| (cid, data.clone())));
let (commit_bytes, other_blocks) = partition_blocks(resolved, ctx.commit_cid);
finalize_commit_frame(frame, ctx.commit_cid, commit_bytes, other_blocks).await
.filter_map(|(cid, opt)| opt.map(|b| (cid, b)))
.collect())
}
pub fn format_info_frame(
crates/tranquil-pds/tests/common/mod.rs
+114 -30
View File
@@ -29,6 +29,7 @@ static TEST_DB_POOL: OnceLock<sqlx::PgPool> = OnceLock::new();
static TEST_TEMP_DIR: OnceLock<PathBuf> = OnceLock::new();
static CLUSTER: OnceLock<Vec<ServerInstance>> = OnceLock::new();
static TEST_REPOS: OnceLock<Arc<tranquil_db::PostgresRepositories>> = OnceLock::new();
static TEST_BLOCK_STORE: OnceLock<tranquil_pds::repo::AnyBlockStore> = OnceLock::new();
#[allow(dead_code)]
pub fn is_store_backend() -> bool {
@@ -41,6 +42,8 @@ pub fn is_store_backend() -> bool {
pub struct ServerConfig {
pub pool: Option<sqlx::PgPool>,
pub cache: Option<(Arc<dyn Cache>, Arc<dyn DistributedRateLimiter>)>,
pub store_path: Option<PathBuf>,
pub shared_state: Option<AppState>,
}
#[allow(dead_code)]
@@ -574,12 +577,17 @@ async fn spawn_server(config: ServerConfig) -> ServerInstance {
.with_oauth_authorize_limit(10000)
.with_oauth_token_limit(10000);
let cache_refs = config.cache.as_ref().map(|(c, r)| (c.clone(), r.clone()));
let mut state = match config.pool {
Some(pool) => AppState::from_db(pool, CancellationToken::new()).await,
None => AppState::from_store(CancellationToken::new()).await,
let mut state = match config.shared_state {
Some(s) => s,
None => match (config.pool, config.store_path) {
(Some(pool), _) => AppState::from_db(pool, CancellationToken::new()).await,
(None, Some(path)) => AppState::from_store_at(&path, CancellationToken::new()).await,
(None, None) => AppState::from_store(CancellationToken::new()).await,
},
};
state = state.with_rate_limiters(rate_limiters);
TEST_REPOS.set(state.repos.clone()).ok();
TEST_BLOCK_STORE.set(state.block_store.clone()).ok();
if let Some((cache, distributed_rate_limiter)) = config.cache {
state = state.with_cache(cache, distributed_rate_limiter);
}
@@ -637,6 +645,8 @@ async fn setup_store_backend() -> String {
let instance = spawn_server(ServerConfig {
pool: None,
cache: None,
store_path: None,
shared_state: None,
})
.await;
APP_PORT.set(instance.port).ok();
@@ -664,6 +674,8 @@ async fn spawn_app(database_url: String) -> String {
let instance = spawn_server(ServerConfig {
pool: Some(pool),
cache: None,
store_path: None,
shared_state: None,
})
.await;
APP_PORT.set(instance.port).ok();
@@ -671,27 +683,9 @@ async fn spawn_app(database_url: String) -> String {
}
#[allow(dead_code)]
pub async fn spawn_cluster(database_url: String, node_count: usize) -> Vec<ServerInstance> {
pub async fn spawn_cluster(pool: Option<sqlx::PgPool>, node_count: usize) -> Vec<ServerInstance> {
use tranquil_ripple::{RippleConfig, RippleEngine};
let pool = PgPoolOptions::new()
.max_connections(10)
.acquire_timeout(std::time::Duration::from_secs(30))
.connect(&database_url)
.await
.expect("Failed to connect to Postgres for cluster");
sqlx::migrate!("./migrations")
.run(&pool)
.await
.expect("Failed to run migrations for cluster");
let test_pool = PgPoolOptions::new()
.max_connections(2)
.acquire_timeout(std::time::Duration::from_secs(30))
.connect(&database_url)
.await
.expect("Failed to create test pool for cluster");
TEST_DB_POOL.set(test_pool).ok();
let shutdown = CancellationToken::new();
let mut ripple_nodes: Vec<(Arc<dyn Cache>, Arc<dyn DistributedRateLimiter>)> =
@@ -713,11 +707,28 @@ pub async fn spawn_cluster(database_url: String, node_count: usize) -> Vec<Serve
ripple_nodes.push((cache, rate_limiter));
}
unsafe {
std::env::set_var("PDS_HOSTNAME", "pds.test");
}
tranquil_config::ensure_test_defaults();
let base_state = match is_store_backend() {
true => {
let path = std::env::temp_dir().join(format!(
"tranquil-pds-cluster-store-{}",
uuid::Uuid::new_v4()
));
std::fs::create_dir_all(&path).expect("failed to create cluster store dir");
Some(AppState::from_store_at(&path, CancellationToken::new()).await)
}
false => None,
};
let mut instances: Vec<ServerInstance> = Vec::with_capacity(node_count);
for (cache, rate_limiter) in ripple_nodes {
let server_config = ServerConfig {
pool: Some(pool.clone()),
pool: pool.clone(),
cache: Some((cache, rate_limiter)),
store_path: None,
shared_state: base_state.clone(),
};
let instance = spawn_server(server_config).await;
instances.push(instance);
@@ -757,12 +768,14 @@ pub async fn cluster() -> &'static [ServerInstance] {
unsafe {
std::env::remove_var("DISABLE_RATE_LIMITING");
}
let database_url = if has_external_infra() {
let pool = if is_store_backend() {
setup_cluster_store_backend().await
} else if has_external_infra() {
setup_cluster_external_infra().await
} else {
setup_cluster_testcontainers().await
};
let nodes = spawn_cluster(database_url, 3).await;
let nodes = spawn_cluster(pool, 3).await;
tx.send(nodes).unwrap();
std::future::pending::<()>().await;
});
@@ -771,7 +784,36 @@ pub async fn cluster() -> &'static [ServerInstance] {
})
}
async fn setup_cluster_external_infra() -> String {
async fn setup_cluster_store_backend() -> Option<sqlx::PgPool> {
let temp_dir = std::env::temp_dir().join(format!(
"tranquil-pds-cluster-store-{}",
uuid::Uuid::new_v4()
));
let blob_path = temp_dir.join("blobs");
let backup_path = temp_dir.join("backups");
let store_path = temp_dir.join("store");
std::fs::create_dir_all(&blob_path).expect("failed to create blob temp directory");
std::fs::create_dir_all(&backup_path).expect("failed to create backup temp directory");
std::fs::create_dir_all(&store_path).expect("failed to create store temp directory");
TEST_TEMP_DIR.set(temp_dir).ok();
let plc_url = setup_mock_plc_directory().await;
unsafe {
std::env::set_var("BLOB_STORAGE_BACKEND", "filesystem");
std::env::set_var("BLOB_STORAGE_PATH", blob_path.to_str().unwrap());
std::env::set_var("BACKUP_STORAGE_BACKEND", "filesystem");
std::env::set_var("BACKUP_STORAGE_PATH", backup_path.to_str().unwrap());
std::env::set_var("MAX_IMPORT_SIZE", "100000000");
std::env::set_var("SKIP_IMPORT_VERIFICATION", "true");
std::env::set_var("PLC_DIRECTORY_URL", &plc_url);
std::env::set_var("REPO_BACKEND", "tranquil-store");
std::env::set_var("TRANQUIL_STORE_DATA_DIR", store_path.to_str().unwrap());
std::env::set_var("DATABASE_URL", "postgres://unused/unused");
}
register_mock_appview().await;
None
}
async fn setup_cluster_external_infra() -> Option<sqlx::PgPool> {
let database_url =
std::env::var("DATABASE_URL").expect("DATABASE_URL must be set when using external infra");
let plc_url = setup_mock_plc_directory().await;
@@ -780,11 +822,28 @@ async fn setup_cluster_external_infra() -> String {
std::env::set_var("PLC_DIRECTORY_URL", &plc_url);
}
register_mock_appview().await;
database_url
let pool = PgPoolOptions::new()
.max_connections(10)
.acquire_timeout(std::time::Duration::from_secs(30))
.connect(&database_url)
.await
.expect("Failed to connect to Postgres for cluster");
sqlx::migrate!("./migrations")
.run(&pool)
.await
.expect("Failed to run migrations for cluster");
let test_pool = PgPoolOptions::new()
.max_connections(2)
.acquire_timeout(std::time::Duration::from_secs(30))
.connect(&database_url)
.await
.expect("Failed to create test pool for cluster");
TEST_DB_POOL.set(test_pool).ok();
Some(pool)
}
#[cfg(not(feature = "external-infra"))]
async fn setup_cluster_testcontainers() -> String {
async fn setup_cluster_testcontainers() -> Option<sqlx::PgPool> {
let temp_dir =
std::env::temp_dir().join(format!("tranquil-pds-cluster-{}", uuid::Uuid::new_v4()));
let blob_path = temp_dir.join("blobs");
@@ -817,11 +876,28 @@ async fn setup_cluster_testcontainers() -> String {
.expect("Failed to get port")
);
DB_CONTAINER.set(container).ok();
connection_string
let pool = PgPoolOptions::new()
.max_connections(10)
.acquire_timeout(std::time::Duration::from_secs(30))
.connect(&connection_string)
.await
.expect("Failed to connect to Postgres for cluster");
sqlx::migrate!("./migrations")
.run(&pool)
.await
.expect("Failed to run migrations for cluster");
let test_pool = PgPoolOptions::new()
.max_connections(2)
.acquire_timeout(std::time::Duration::from_secs(30))
.connect(&connection_string)
.await
.expect("Failed to create test pool for cluster");
TEST_DB_POOL.set(test_pool).ok();
Some(pool)
}
#[cfg(feature = "external-infra")]
async fn setup_cluster_testcontainers() -> String {
async fn setup_cluster_testcontainers() -> Option<sqlx::PgPool> {
panic!(
"Testcontainers disabled with external-infra feature. Set DATABASE_URL and BLOB_STORAGE_PATH (or S3_ENDPOINT)."
);
@@ -861,6 +937,14 @@ pub async fn get_test_repos() -> &'static Arc<tranquil_db::PostgresRepositories>
TEST_REPOS.get().expect("TEST_REPOS not initialized")
}
#[allow(dead_code)]
pub async fn get_test_block_store() -> &'static tranquil_pds::repo::AnyBlockStore {
base_url().await;
TEST_BLOCK_STORE
.get()
.expect("TEST_BLOCK_STORE not initialized")
}
fn extract_verification_code(body_text: &str) -> String {
let lines: Vec<&str> = body_text.lines().collect();
lines
crates/tranquil-pds/tests/firehose/mod.rs
+2
View File
@@ -32,6 +32,7 @@ pub struct ParsedCommitFrame {
pub prev_data: Option<Cid>,
}
#[allow(dead_code)]
#[derive(Debug, Clone)]
pub struct ParsedRepoOp {
pub action: RepoAction,
@@ -223,6 +224,7 @@ impl FirehoseConsumer {
self.frames.lock().unwrap().drain(..).collect()
}
#[allow(dead_code)]
pub fn all_commits(&self) -> Vec<ParsedCommitFrame> {
self.frames
.lock()
crates/tranquil-pds/tests/firehose_inline_blocks.rs
+423
View File
@@ -0,0 +1,423 @@
mod common;
mod firehose;
mod helpers;
use cid::Cid;
use common::*;
use firehose::FirehoseConsumer;
use helpers::build_car_with_signature;
use iroh_car::CarReader;
use k256::ecdsa::SigningKey;
use multihash::Multihash;
use reqwest::StatusCode;
use serde_json::json;
use sha2::{Digest, Sha256};
use std::io::Cursor;
use std::time::Duration;
use tranquil_db_traits::{EventBlocks, RepoEventType, SequenceNumber};
use tranquil_types::{CidLink, Did};
fn synthetic_cid(payload: &[u8]) -> Cid {
let digest = Sha256::digest(payload);
let mh = Multihash::wrap(0x12, digest.as_slice()).expect("multihash wrap");
Cid::new_v1(0x71, mh)
}
fn fresh_synthetic_did(label: &str) -> Did {
Did::new(format!(
"did:plc:test{}{}",
label,
uuid::Uuid::new_v4().simple()
))
.expect("valid did")
}
async fn create_post(client: &reqwest::Client, token: &str, did: &str, text: &str) {
let payload = json!({
"repo": did,
"collection": "app.bsky.feed.post",
"record": {
"$type": "app.bsky.feed.post",
"text": text,
"createdAt": chrono::Utc::now().to_rfc3339(),
}
});
let res = client
.post(format!(
"{}/xrpc/com.atproto.repo.createRecord",
base_url().await
))
.bearer_auth(token)
.json(&payload)
.send()
.await
.expect("createRecord request failed");
assert_eq!(res.status(), StatusCode::OK, "createRecord failed");
}
#[tokio::test]
async fn commit_events_carry_inline_blocks() {
let client = client();
let (token, did) = create_account_and_login(&client).await;
create_post(&client, &token, &did, "commit A: orphans incoming").await;
create_post(&client, &token, &did, "commit B: bye bye MST nodes from A").await;
let repos = get_test_repos().await;
let typed_did = tranquil_types::Did::new(did.clone()).unwrap();
let events = repos
.repo
.get_events_since_seq(SequenceNumber::ZERO, None)
.await
.expect("get_events_since_seq failed");
let our_commits: Vec<_> = events
.iter()
.filter(|e| e.did == typed_did && e.event_type == RepoEventType::Commit)
.collect();
assert!(
our_commits.len() >= 2,
"expected at least 2 commit events for our DID, got {}",
our_commits.len()
);
our_commits.iter().for_each(|event| {
let blocks = event.blocks.as_ref().unwrap_or_else(|| {
panic!(
"commit event seq={} has no blocks field",
event.seq.as_i64()
)
});
let inline = match blocks {
EventBlocks::Inline(v) => v,
EventBlocks::LegacyCids(_) => panic!(
"commit event seq={} resolved as LegacyCids, expected Inline; \
new commits must inline block bytes into the eventlog",
event.seq.as_i64()
),
};
assert!(
!inline.is_empty(),
"commit event seq={} has empty Inline blocks vec",
event.seq.as_i64()
);
let commit_cid = event
.commit_cid
.as_ref()
.and_then(|c| c.to_cid())
.unwrap_or_else(|| {
panic!("commit event seq={} missing commit_cid", event.seq.as_i64())
});
let commit_cid_bytes = commit_cid.to_bytes();
assert!(
inline.iter().any(|b| b.cid_bytes == commit_cid_bytes),
"commit event seq={} inline blocks do not contain the commit block",
event.seq.as_i64()
);
inline.iter().for_each(|b| {
let parsed = Cid::read_bytes(b.cid_bytes.as_slice()).unwrap_or_else(|e| {
panic!(
"commit event seq={} inline cid_bytes failed to parse as Cid: {e}",
event.seq.as_i64()
)
});
assert_eq!(
parsed.to_bytes(),
b.cid_bytes,
"commit event seq={} cid round-trip mismatch (cid={parsed})",
event.seq.as_i64()
);
});
});
}
#[tokio::test]
async fn sync_event_carries_inline_commit_block() {
let repos = get_test_repos().await;
let did = fresh_synthetic_did("sync");
let commit_bytes = b"synthetic sync commit block payload".to_vec();
let commit_cid = synthetic_cid(&commit_bytes);
let cid_link: CidLink = (&commit_cid).into();
let rev = "3kabcdefghij2";
let seq = repos
.repo
.insert_sync_event(&did, &cid_link, Some(rev), &commit_bytes)
.await
.expect("insert_sync_event");
let event = repos
.repo
.get_event_by_seq(seq)
.await
.expect("get_event_by_seq")
.expect("event present");
assert_eq!(event.event_type, RepoEventType::Sync);
let blocks = event
.blocks
.as_ref()
.expect("sync event must carry inline blocks");
let inline = match blocks {
EventBlocks::Inline(v) => v,
EventBlocks::LegacyCids(_) => {
panic!("sync event resolved as LegacyCids; new sync events must inline block bytes")
}
};
assert_eq!(
inline.len(),
1,
"sync event must carry exactly the commit block, got {}",
inline.len()
);
let stored = &inline[0];
assert_eq!(
stored.cid_bytes,
commit_cid.to_bytes(),
"sync event inline cid_bytes mismatch"
);
assert_eq!(stored.data, commit_bytes, "sync event inline data mismatch");
}
#[tokio::test]
async fn genesis_commit_event_carries_inline_blocks() {
let repos = get_test_repos().await;
let did = fresh_synthetic_did("gen");
let commit_bytes = b"synthetic genesis commit block payload".to_vec();
let mst_root_bytes = b"synthetic genesis mst root block payload".to_vec();
let commit_cid = synthetic_cid(&commit_bytes);
let mst_root_cid = synthetic_cid(&mst_root_bytes);
let commit_link: CidLink = (&commit_cid).into();
let mst_link: CidLink = (&mst_root_cid).into();
let rev = "3kabcdefghij3";
let seq = repos
.repo
.insert_genesis_commit_event(
&did,
&commit_link,
&mst_link,
rev,
&commit_bytes,
&mst_root_bytes,
)
.await
.expect("insert_genesis_commit_event");
let event = repos
.repo
.get_event_by_seq(seq)
.await
.expect("get_event_by_seq")
.expect("event present");
assert_eq!(event.event_type, RepoEventType::Commit);
let blocks = event
.blocks
.as_ref()
.expect("genesis commit event must carry inline blocks");
let inline = match blocks {
EventBlocks::Inline(v) => v,
EventBlocks::LegacyCids(_) => {
panic!("genesis event resolved as LegacyCids; new genesis events must inline blocks")
}
};
assert_eq!(
inline.len(),
2,
"genesis event must carry commit + mst root blocks, got {}",
inline.len()
);
let commit_cid_bytes = commit_cid.to_bytes();
let mst_cid_bytes = mst_root_cid.to_bytes();
let commit_block = inline
.iter()
.find(|b| b.cid_bytes == commit_cid_bytes)
.expect("genesis inline blocks missing commit block");
assert_eq!(commit_block.data, commit_bytes);
let mst_block = inline
.iter()
.find(|b| b.cid_bytes == mst_cid_bytes)
.expect("genesis inline blocks missing mst root block");
assert_eq!(mst_block.data, mst_root_bytes);
}
#[tokio::test]
async fn backfill_succeeds_from_eventlog_alone() {
let client = client();
let (token, did) = create_account_and_login(&client).await;
create_post(&client, &token, &did, "first").await;
create_post(&client, &token, &did, "second").await;
let consumer = FirehoseConsumer::connect_with_cursor(app_port(), 0).await;
let commits = consumer
.wait_for_commits(&did, 2, Duration::from_secs(20))
.await;
assert!(
commits.len() >= 2,
"expected at least 2 backfilled commits for {}, got {}",
did,
commits.len()
);
for commit in &commits {
assert!(
!commit.blocks.is_empty(),
"backfilled commit seq={} has empty CAR blocks",
commit.seq
);
let mut reader = CarReader::new(Cursor::new(&commit.blocks))
.await
.unwrap_or_else(|e| panic!("CAR header parse failed for seq={}: {e}", commit.seq));
assert!(
!reader.header().roots().is_empty(),
"CAR for seq={} has no roots",
commit.seq
);
assert_eq!(
reader.header().roots()[0],
commit.commit,
"CAR root mismatch for seq={}",
commit.seq
);
let mut found_commit_block = false;
while let Ok(Some((cid, _))) = reader.next_block().await {
if cid == commit.commit {
found_commit_block = true;
}
}
assert!(
found_commit_block,
"backfilled commit seq={} CAR missing the commit block",
commit.seq
);
}
}
#[tokio::test]
async fn import_event_carries_inline_commit_block() {
let client = client();
let (token, did) = create_account_and_login(&client).await;
let signing_key = SigningKey::random(&mut rand::thread_rng());
let (car_bytes, _car_root_cid) = build_car_with_signature(&did, &signing_key);
let import_res = client
.post(format!(
"{}/xrpc/com.atproto.repo.importRepo",
base_url().await
))
.bearer_auth(&token)
.header("Content-Type", "application/vnd.ipld.car")
.body(car_bytes)
.send()
.await
.expect("import request failed");
assert_eq!(
import_res.status(),
StatusCode::OK,
"import should succeed: body={:?}",
import_res.text().await.unwrap_or_default()
);
let repos = get_test_repos().await;
let typed_did = tranquil_types::Did::new(did.clone()).unwrap();
let events = repos
.repo
.get_events_since_seq(SequenceNumber::ZERO, None)
.await
.expect("get_events_since_seq failed");
let our_commits: Vec<_> = events
.iter()
.filter(|e| e.did == typed_did && e.event_type == RepoEventType::Commit)
.collect();
assert!(
!our_commits.is_empty(),
"expected at least one commit event for {} after import",
did
);
let import_event = our_commits
.last()
.expect("at least one commit event after import");
let blocks = import_event.blocks.as_ref().unwrap_or_else(|| {
panic!(
"import commit event seq={} missing blocks field",
import_event.seq.as_i64()
)
});
let inline = match blocks {
EventBlocks::Inline(v) => v,
EventBlocks::LegacyCids(_) => panic!(
"import event seq={} resolved as LegacyCids; new commits must inline blocks",
import_event.seq.as_i64()
),
};
assert!(
!inline.is_empty(),
"import event seq={} has empty Inline blocks vec — this is the bug from \
sequence_import_event using `blocks: Some(vec![])`",
import_event.seq.as_i64()
);
let commit_cid = import_event
.commit_cid
.as_ref()
.and_then(|c| c.to_cid())
.unwrap_or_else(|| {
panic!(
"import event seq={} missing commit_cid",
import_event.seq.as_i64()
)
});
let commit_cid_bytes = commit_cid.to_bytes();
assert!(
inline.iter().any(|b| b.cid_bytes == commit_cid_bytes),
"import event seq={} inline blocks do not contain the freshly-created commit block",
import_event.seq.as_i64()
);
let consumer = FirehoseConsumer::connect_with_cursor(app_port(), 0).await;
let commits = consumer
.wait_for_commits(&did, 1, Duration::from_secs(20))
.await;
assert!(
!commits.is_empty(),
"expected at least one backfilled commit after import for {}",
did
);
for commit in &commits {
assert!(
!commit.blocks.is_empty(),
"backfilled import commit seq={} has empty CAR blocks",
commit.seq
);
let mut reader = CarReader::new(Cursor::new(&commit.blocks))
.await
.unwrap_or_else(|e| panic!("CAR header parse failed for seq={}: {e}", commit.seq));
assert_eq!(
reader.header().roots()[0],
commit.commit,
"CAR root mismatch for import commit seq={}",
commit.seq
);
let mut found_commit_block = false;
while let Ok(Some((cid, _))) = reader.next_block().await {
if cid == commit.commit {
found_commit_block = true;
}
}
assert!(
found_commit_block,
"backfilled import commit seq={} CAR missing the commit block",
commit.seq
);
}
}
crates/tranquil-pds/tests/firehose_validation.rs
+19 -19
View File
@@ -245,11 +245,11 @@ async fn test_firehose_frame_structure() {
assert!(timeout.is_ok(), "Timed out waiting for event for our DID");
let (header, frame) = frame_opt.expect("No matching frame found");
println!("\n=== Frame Structure Validation ===\n");
println!("\n-- frame structure validation --\n");
println!("Header:");
println!(" op: {} (expected: 1)", header.op);
println!(" t: {} (expected: #commit)", header.t);
println!(" op: {}, expected 1", header.op);
println!(" t: {}, expected #commit", header.t);
assert_eq!(header.op, 1, "Header op should be 1");
assert_eq!(header.t, "#commit", "Header t should be #commit");
@@ -260,7 +260,7 @@ async fn test_firehose_frame_structure() {
println!(" repo: {}", frame.repo);
println!(" commit: {}", frame.commit);
println!(
" rev: {} (valid TID: {})",
" rev: {}, valid TID: {}",
frame.rev,
is_valid_tid(&frame.rev)
);
@@ -269,12 +269,12 @@ async fn test_firehose_frame_structure() {
println!(" ops count: {}", frame.ops.len());
println!(" blobs count: {}", frame.blobs.len());
println!(
" time: {} (valid format: {})",
" time: {}, valid format: {}",
frame.time,
is_valid_time_format(&frame.time)
);
println!(
" prevData: {:?} (IMPORTANT - should have value for updates)",
" prevData: {:?}, should have value for updates",
frame.prev_data
);
@@ -297,7 +297,7 @@ async fn test_firehose_frame_structure() {
println!(" path: {}", op.path);
println!(" cid: {:?}", op.cid);
println!(
" prev: {:?} (should be Some for updates/deletes)",
" prev: {:?}, should be Some for updates/deletes",
op.prev
);
@@ -351,7 +351,7 @@ async fn test_firehose_frame_structure() {
}
}
println!("\n=== Validation Complete ===\n");
println!("\n-- validation complete --\n");
ws_stream.send(tungstenite::Message::Close(None)).await.ok();
}
@@ -435,7 +435,7 @@ async fn test_firehose_update_has_prev_field() {
assert!(timeout.is_ok(), "Timed out waiting for update commit");
let frame = frame_opt.expect("No matching frame found");
println!("\n=== Update Operation Validation ===\n");
println!("\n-- update operation validation --\n");
println!("First profile CID: {}", first_cid);
println!("Frame prevData: {:?}", frame.prev_data);
@@ -455,7 +455,7 @@ async fn test_firehose_update_has_prev_field() {
}
}
println!("\n=== Validation Complete ===\n");
println!("\n-- validation complete --\n");
ws_stream.send(tungstenite::Message::Close(None)).await.ok();
}
@@ -512,13 +512,13 @@ async fn test_firehose_commit_has_prev_data() {
assert!(timeout.is_ok(), "Timed out waiting for first commit");
let first_frame = first_frame_opt.expect("No first frame found");
println!("\n=== First Commit ===");
println!("\n-- first commit --");
println!(
" prevData: {:?} (first commit may be None)",
" prevData: {:?}, first commit may be None",
first_frame.prev_data
);
println!(
" since: {:?} (first commit should be None)",
" since: {:?}, first commit should be None",
first_frame.since
);
@@ -562,13 +562,13 @@ async fn test_firehose_commit_has_prev_data() {
assert!(timeout.is_ok(), "Timed out waiting for second commit");
let second_frame = second_frame_opt.expect("No second frame found");
println!("\n=== Second Commit ===");
println!("\n-- second commit --");
println!(
" prevData: {:?} (should have value - MST root CID)",
" prevData: {:?}, should have value as MST root CID",
second_frame.prev_data
);
println!(
" since: {:?} (should have value - previous rev)",
" since: {:?}, should have value as previous rev",
second_frame.since
);
@@ -577,7 +577,7 @@ async fn test_firehose_commit_has_prev_data() {
"Second commit should have 'since' field pointing to first commit rev"
);
println!("\n=== Validation Complete ===\n");
println!("\n-- validation complete --\n");
ws_stream.send(tungstenite::Message::Close(None)).await.ok();
}
@@ -634,7 +634,7 @@ async fn test_compare_raw_cbor_encoding() {
assert!(timeout.is_ok(), "Timed out waiting for event for our DID");
let raw_bytes = raw_bytes_opt.expect("No matching frame found");
println!("\n=== Raw CBOR Analysis ===\n");
println!("\n-- raw CBOR analysis --\n");
println!("Total frame size: {} bytes", raw_bytes.len());
fn bytes_to_hex(bytes: &[u8]) -> String {
@@ -657,7 +657,7 @@ async fn test_compare_raw_cbor_encoding() {
println!("\nPayload section: {} bytes", raw_bytes.len() - header_end);
println!("\n=== Analysis Complete ===\n");
println!("\n-- analysis complete --\n");
ws_stream.send(tungstenite::Message::Close(None)).await.ok();
}
crates/tranquil-pds/tests/gc_after_delete.rs
+410
View File
@@ -0,0 +1,410 @@
mod common;
mod helpers;
use chrono::Utc;
use common::*;
use helpers::*;
use reqwest::StatusCode;
use serde_json::{Value, json};
use tranquil_types::Did;
#[tokio::test]
async fn test_delete_record_marks_blocks_obsolete() {
let client = client();
let base = base_url().await;
let repos = get_test_repos().await;
let (did, jwt) = setup_new_user("gc-after-delete").await;
let user_id = repos
.user
.get_id_by_did(&Did::new(did.clone()).unwrap())
.await
.expect("DB error")
.expect("User not found");
let count_baseline = repos
.repo
.count_user_blocks(user_id)
.await
.expect("count_user_blocks failed");
let collection = "app.bsky.feed.post";
let rkey = format!("gc_test_{}", Utc::now().timestamp_millis());
let create_payload = json!({
"repo": did,
"collection": collection,
"rkey": rkey,
"record": {
"$type": collection,
"text": "this record is destined for deletion",
"createdAt": Utc::now().to_rfc3339()
}
});
let create_res = client
.post(format!("{}/xrpc/com.atproto.repo.createRecord", base))
.bearer_auth(&jwt)
.json(&create_payload)
.send()
.await
.expect("Failed to send createRecord");
assert_eq!(
create_res.status(),
StatusCode::OK,
"createRecord did not return 200"
);
let create_body: Value = create_res
.json()
.await
.expect("createRecord response was not JSON");
let record_uri = create_body["uri"]
.as_str()
.expect("createRecord response missing uri")
.to_string();
let record_cid = create_body["cid"]
.as_str()
.expect("createRecord response missing cid")
.to_string();
let count_after_create = repos
.repo
.count_user_blocks(user_id)
.await
.expect("count_user_blocks failed");
assert!(
count_after_create > count_baseline,
"user_blocks count did not grow after createRecord (baseline={}, after_create={})",
count_baseline,
count_after_create
);
let delete_payload = json!({
"repo": did,
"collection": collection,
"rkey": rkey,
});
let delete_res = client
.post(format!("{}/xrpc/com.atproto.repo.deleteRecord", base))
.bearer_auth(&jwt)
.json(&delete_payload)
.send()
.await
.expect("Failed to send deleteRecord");
assert_eq!(
delete_res.status(),
StatusCode::OK,
"deleteRecord did not return 200: {:?}",
delete_res.text().await
);
let count_after_delete = repos
.repo
.count_user_blocks(user_id)
.await
.expect("count_user_blocks failed");
assert!(
count_after_delete < count_after_create,
"user_blocks count did not shrink after deleteRecord \
(baseline={}, after_create={}, after_delete={}). \
The delete path produced no obsolete CIDs beyond the prior commit root, \
which is the regression this test guards against.",
count_baseline,
count_after_create,
count_after_delete
);
let get_res = client
.get(format!("{}/xrpc/com.atproto.repo.getRecord", base))
.query(&[
("repo", did.as_str()),
("collection", collection),
("rkey", rkey.as_str()),
])
.send()
.await
.expect("Failed to send getRecord");
assert!(
!get_res.status().is_success(),
"deleted record is still resolvable via getRecord (status={}); uri={} cid={}",
get_res.status(),
record_uri,
record_cid
);
}
#[tokio::test]
async fn test_update_record_marks_old_record_block_obsolete() {
let client = client();
let base = base_url().await;
let repos = get_test_repos().await;
let (did, jwt) = setup_new_user("gc-after-update").await;
let user_id = repos
.user
.get_id_by_did(&Did::new(did.clone()).unwrap())
.await
.expect("DB error")
.expect("User not found");
let collection = "app.bsky.feed.post";
let rkey = format!("gc_update_{}", Utc::now().timestamp_millis());
let put_v1 = json!({
"repo": did,
"collection": collection,
"rkey": rkey,
"record": {
"$type": collection,
"text": "first version",
"createdAt": Utc::now().to_rfc3339()
}
});
let res = client
.post(format!("{}/xrpc/com.atproto.repo.putRecord", base))
.bearer_auth(&jwt)
.json(&put_v1)
.send()
.await
.expect("Failed to send putRecord v1");
assert_eq!(res.status(), StatusCode::OK, "first putRecord failed");
let count_after_create = repos
.repo
.count_user_blocks(user_id)
.await
.expect("count_user_blocks failed");
let put_v2 = json!({
"repo": did,
"collection": collection,
"rkey": rkey,
"record": {
"$type": collection,
"text": "second version with new content",
"createdAt": Utc::now().to_rfc3339()
}
});
let res = client
.post(format!("{}/xrpc/com.atproto.repo.putRecord", base))
.bearer_auth(&jwt)
.json(&put_v2)
.send()
.await
.expect("Failed to send putRecord v2");
assert_eq!(res.status(), StatusCode::OK, "second putRecord failed");
let count_after_update = repos
.repo
.count_user_blocks(user_id)
.await
.expect("count_user_blocks failed");
assert!(
count_after_update <= count_after_create + 1,
"user_blocks count grew by more than 1 after putRecord update \
(after_create={}, after_update={}). The previous version's record block \
should have been marked obsolete; instead it appears to be leaking.",
count_after_create,
count_after_update
);
}
#[tokio::test]
async fn test_delete_in_populated_repo_marks_merged_subtree_blocks_obsolete() {
let client = client();
let base = base_url().await;
let repos = get_test_repos().await;
let (did, jwt) = setup_new_user("gc-merge").await;
let user_id = repos
.user
.get_id_by_did(&Did::new(did.clone()).unwrap())
.await
.expect("DB error")
.expect("User not found");
let collection = "app.bsky.feed.post";
let record_count = 64usize;
let now_ms = Utc::now().timestamp_millis();
let rkeys: Vec<String> = (0..record_count)
.map(|i| format!("gc_merge_{}_{:04}", now_ms, i))
.collect();
let create_results =
futures::future::try_join_all(rkeys.iter().enumerate().map(|(i, rkey)| {
let client = client.clone();
let jwt = jwt.clone();
let did = did.clone();
let base = base.to_string();
let payload = json!({
"repo": did,
"collection": collection,
"rkey": rkey,
"record": {
"$type": collection,
"text": format!("seed record {}", i),
"createdAt": Utc::now().to_rfc3339()
}
});
async move {
let res = client
.post(format!("{}/xrpc/com.atproto.repo.createRecord", base))
.bearer_auth(&jwt)
.json(&payload)
.send()
.await
.expect("Failed to send createRecord");
if res.status() != StatusCode::OK {
return Err(format!("seed createRecord failed: {}", res.status()));
}
Ok::<(), String>(())
}
}))
.await;
create_results.expect("seeding records failed");
let count_after_seed = repos
.repo
.count_user_blocks(user_id)
.await
.expect("count_user_blocks failed");
let target_rkey = &rkeys[record_count / 2];
let delete_payload = json!({
"repo": did,
"collection": collection,
"rkey": target_rkey,
});
let delete_res = client
.post(format!("{}/xrpc/com.atproto.repo.deleteRecord", base))
.bearer_auth(&jwt)
.json(&delete_payload)
.send()
.await
.expect("Failed to send deleteRecord");
assert_eq!(
delete_res.status(),
StatusCode::OK,
"deleteRecord did not return 200: {:?}",
delete_res.text().await
);
let count_after_delete = repos
.repo
.count_user_blocks(user_id)
.await
.expect("count_user_blocks failed");
assert!(
count_after_delete < count_after_seed,
"user_blocks did not shrink after deleting from a populated repo \
(after_seed={}, after_delete={}). The path-walk-based obsolete \
calculation does not capture sibling subtree blocks orphaned by \
delete-merge; only an MST-diff-based calculation does.",
count_after_seed,
count_after_delete
);
let get_res = client
.get(format!("{}/xrpc/com.atproto.repo.getRecord", base))
.query(&[
("repo", did.as_str()),
("collection", collection),
("rkey", target_rkey.as_str()),
])
.send()
.await
.expect("Failed to send getRecord");
assert!(
!get_res.status().is_success(),
"deleted record is still resolvable via getRecord (status={})",
get_res.status(),
);
}
#[tokio::test]
async fn test_delete_decrements_tranquil_store_refcounts() {
if !is_store_backend() {
eprintln!(
"skipping test_delete_decrements_tranquil_store_refcounts: \
only meaningful with the tranquil-store backend"
);
return;
}
let client = client();
let base = base_url().await;
let block_store = get_test_block_store().await;
let store = block_store
.as_tranquil_store()
.expect("tranquil-store backend selected but block_store is not TranquilStore");
let (did, jwt) = setup_new_user("gc-store-decrement").await;
let collection = "app.bsky.feed.post";
let rkey = format!("gc_store_{}", Utc::now().timestamp_millis());
let create_res = client
.post(format!("{}/xrpc/com.atproto.repo.createRecord", base))
.bearer_auth(&jwt)
.json(&json!({
"repo": did,
"collection": collection,
"rkey": rkey,
"record": {
"$type": collection,
"text": "destined for refcount decrement",
"createdAt": Utc::now().to_rfc3339()
}
}))
.send()
.await
.expect("Failed to send createRecord");
assert_eq!(create_res.status(), StatusCode::OK, "createRecord failed");
let create_body: Value = create_res.json().await.expect("createRecord not JSON");
let record_cid_str = create_body["cid"]
.as_str()
.expect("createRecord response missing cid")
.to_string();
let record_cid = cid::Cid::try_from(record_cid_str.as_str()).expect("invalid record cid");
let refcount_after_create = store
.refcount_of(&record_cid)
.expect("refcount_of failed")
.expect("record cid not in blockstore index after create");
assert!(
refcount_after_create > 0,
"record cid had refcount 0 immediately after create (cid={})",
record_cid_str
);
let delete_res = client
.post(format!("{}/xrpc/com.atproto.repo.deleteRecord", base))
.bearer_auth(&jwt)
.json(&json!({
"repo": did,
"collection": collection,
"rkey": rkey,
}))
.send()
.await
.expect("Failed to send deleteRecord");
assert_eq!(
delete_res.status(),
StatusCode::OK,
"deleteRecord did not return 200: {:?}",
delete_res.text().await
);
let refcount_after_delete = store
.refcount_of(&record_cid)
.expect("refcount_of failed")
.expect("record cid slot vanished entirely after delete");
assert_eq!(
refcount_after_delete, 0,
"record cid still has nonzero refcount after deleteRecord \
(cid={}, before_delete={}, after_delete={}). The hash_index \
decrement that drives on-disk reclamation is the regression \
this test guards against.",
record_cid_str, refcount_after_create, refcount_after_delete
);
}
crates/tranquil-pds/tests/store_parity.rs
+351 -210
View File
File diff suppressed because it is too large Load Diff
crates/tranquil-pds/tests/whole_story.rs
+1 -1
View File
@@ -1171,7 +1171,7 @@ async fn test_backup_restore_workflow() {
}
#[tokio::test]
async fn test_scale_100_posts_with_pagination() {
async fn test_scale_1000_posts_with_pagination() {
let client = client();
let base = base_url().await;
let (did, jwt) = setup_new_user("scale-posts").await;
crates/tranquil-repo/src/lib.rs
+26 -40
View File
@@ -6,7 +6,7 @@ use jacquard_repo::storage::BlockStore;
use multihash::Multihash;
use sha2::{Digest, Sha256};
use sqlx::PgPool;
use std::collections::HashSet;
use std::collections::{HashMap, HashSet};
use std::sync::{Arc, Mutex};
#[derive(Clone)]
@@ -24,32 +24,6 @@ impl PostgresBlockStore {
}
}
impl PostgresBlockStore {
pub async fn get_oldest_block_cids(&self, limit: i64) -> Result<Vec<Vec<u8>>, RepoError> {
let rows = sqlx::query!(
"SELECT cid FROM blocks ORDER BY created_at ASC LIMIT $1",
limit,
)
.fetch_all(&self.pool)
.await
.map_err(RepoError::storage)?;
Ok(rows.into_iter().map(|r| r.cid).collect())
}
pub async fn delete_blocks(&self, cids: &[Vec<u8>]) -> Result<u64, RepoError> {
match cids.is_empty() {
true => Ok(0),
false => {
let result = sqlx::query!("DELETE FROM blocks WHERE cid = ANY($1)", cids,)
.execute(&self.pool)
.await
.map_err(RepoError::storage)?;
Ok(result.rows_affected())
}
}
}
}
impl BlockStore for PostgresBlockStore {
async fn get(&self, cid: &Cid) -> Result<Option<Bytes>, RepoError> {
let cid_bytes = cid.to_bytes();
@@ -152,7 +126,7 @@ impl BlockStore for PostgresBlockStore {
#[derive(Clone)]
pub struct TrackingBlockStore<S: BlockStore> {
inner: S,
written_cids: Arc<Mutex<Vec<Cid>>>,
written_blocks: Arc<Mutex<HashMap<Cid, Bytes>>>,
read_cids: Arc<Mutex<HashSet<Cid>>>,
}
@@ -160,18 +134,26 @@ impl<S: BlockStore + Sync> TrackingBlockStore<S> {
pub fn new(store: S) -> Self {
Self {
inner: store,
written_cids: Arc::new(Mutex::new(Vec::new())),
written_blocks: Arc::new(Mutex::new(HashMap::new())),
read_cids: Arc::new(Mutex::new(HashSet::new())),
}
}
pub fn get_written_cids(&self) -> Vec<Cid> {
match self.written_cids.lock() {
Ok(guard) => guard.clone(),
Err(poisoned) => poisoned.into_inner().clone(),
match self.written_blocks.lock() {
Ok(guard) => guard.keys().copied().collect(),
Err(poisoned) => poisoned.into_inner().keys().copied().collect(),
}
}
pub fn take_written_blocks(&self) -> HashMap<Cid, Bytes> {
let mut guard = match self.written_blocks.lock() {
Ok(g) => g,
Err(poisoned) => poisoned.into_inner(),
};
std::mem::take(&mut *guard)
}
pub fn get_read_cids(&self) -> Vec<Cid> {
match self.read_cids.lock() {
Ok(guard) => guard.iter().cloned().collect(),
@@ -206,9 +188,14 @@ impl<S: BlockStore + Sync> BlockStore for TrackingBlockStore<S> {
async fn put(&self, data: &[u8]) -> Result<Cid, RepoError> {
let cid = self.inner.put(data).await?;
match self.written_cids.lock() {
Ok(mut guard) => guard.push(cid),
Err(poisoned) => poisoned.into_inner().push(cid),
let bytes = Bytes::copy_from_slice(data);
match self.written_blocks.lock() {
Ok(mut guard) => {
guard.insert(cid, bytes);
}
Err(poisoned) => {
poisoned.into_inner().insert(cid, bytes);
}
}
Ok(cid)
}
@@ -222,11 +209,10 @@ impl<S: BlockStore + Sync> BlockStore for TrackingBlockStore<S> {
blocks: impl IntoIterator<Item = (Cid, Bytes)> + Send,
) -> Result<(), RepoError> {
let blocks: Vec<_> = blocks.into_iter().collect();
let cids: Vec<Cid> = blocks.iter().map(|(cid, _)| *cid).collect();
self.inner.put_many(blocks).await?;
match self.written_cids.lock() {
Ok(mut guard) => guard.extend(cids),
Err(poisoned) => poisoned.into_inner().extend(cids),
self.inner.put_many(blocks.clone()).await?;
match self.written_blocks.lock() {
Ok(mut guard) => guard.extend(blocks),
Err(poisoned) => poisoned.into_inner().extend(blocks),
}
Ok(())
}
crates/tranquil-server/src/main.rs
+2 -6
View File
@@ -10,8 +10,7 @@ use tranquil_pds::comms::{CommsService, DiscordSender, EmailSender, SignalSender
use tranquil_pds::crawlers::{Crawlers, start_crawlers_service};
use tranquil_pds::scheduled::{
backfill_genesis_commit_blocks, backfill_record_blobs, backfill_repo_rev, backfill_user_blocks,
start_scheduled_tasks,
backfill_record_blobs, backfill_repo_rev, backfill_user_blocks, start_scheduled_tasks,
};
use tranquil_pds::state::AppState;
@@ -131,10 +130,6 @@ async fn run() -> Result<(), Box<dyn std::error::Error>> {
let backfill_block_store = state.block_store.clone();
tokio::spawn(async move {
tokio::join!(
backfill_genesis_commit_blocks(
backfill_repo_repo.clone(),
backfill_block_store.clone()
),
backfill_repo_rev(backfill_repo_repo.clone(), backfill_block_store.clone()),
backfill_user_blocks(backfill_repo_repo.clone(), backfill_block_store.clone()),
backfill_record_blobs(backfill_repo_repo, backfill_block_store),
@@ -255,6 +250,7 @@ async fn run() -> Result<(), Box<dyn std::error::Error>> {
state.repos.sso.clone(),
state.repos.repo.clone(),
state.block_store.clone(),
state.eventlog_segments_dir.clone(),
shutdown.clone(),
));
crates/tranquil-signal/Cargo.toml
+2
View File
@@ -24,4 +24,6 @@ uuid = { workspace = true }
thiserror = { workspace = true }
[dev-dependencies]
tranquil-signal = { path = ".", features = ["fjall-store"] }
rand = "0.9"
tempfile = "3"
crates/tranquil-signal/src/lib.rs
+2
View File
@@ -6,6 +6,8 @@ pub mod fjall_store;
#[cfg(test)]
mod tests;
#[cfg(test)]
mod tests_fjall;
pub use client::{
DeviceName, InvalidDeviceName, InvalidSignalUsername, LinkGeneration, LinkResult, MessageBody,
crates/tranquil-signal/src/tests.rs
+64 -23
View File
@@ -14,15 +14,16 @@ use uuid::Uuid;
use crate::store::{IdentityType, PgProtocolStore, PgSignalStore};
async fn test_store() -> PgSignalStore {
async fn test_store() -> Option<PgSignalStore> {
let url = std::env::var("DATABASE_URL")
.unwrap_or_else(|_| "postgres://postgres:postgres@127.0.0.1:5432/postgres".into());
let pool = PgPoolOptions::new()
.max_connections(5)
.acquire_timeout(std::time::Duration::from_secs(2))
.connect(&url)
.await
.unwrap();
.ok()?;
sqlx::query("DELETE FROM signal_kv")
.execute(&pool)
@@ -61,7 +62,7 @@ async fn test_store() -> PgSignalStore {
.await
.ok();
PgSignalStore::new(pool)
Some(PgSignalStore::new(pool))
}
fn protocol_store(store: &PgSignalStore, identity: IdentityType) -> PgProtocolStore {
@@ -70,7 +71,9 @@ fn protocol_store(store: &PgSignalStore, identity: IdentityType) -> PgProtocolSt
#[tokio::test]
async fn state_store_registration_empty() {
let store = test_store().await;
let Some(store) = test_store().await else {
return;
};
assert!(store.load_registration_data().await.unwrap().is_none());
assert!(!store.is_registered().await);
@@ -78,7 +81,9 @@ async fn state_store_registration_empty() {
#[tokio::test]
async fn state_store_kv_roundtrip() {
let store = test_store().await;
let Some(store) = test_store().await else {
return;
};
let value = b"test-data".to_vec();
sqlx::query("INSERT INTO signal_kv (key, value) VALUES ('test_key', $1)")
@@ -96,7 +101,9 @@ async fn state_store_kv_roundtrip() {
#[tokio::test]
async fn state_store_identity_keypairs() {
let store = test_store().await;
let Some(store) = test_store().await else {
return;
};
let aci_pair = IdentityKeyPair::generate(&mut rand::rng());
let pni_pair = IdentityKeyPair::generate(&mut rand::rng());
@@ -116,13 +123,17 @@ async fn state_store_identity_keypairs() {
#[tokio::test]
async fn state_store_sender_certificate_roundtrip() {
let store = test_store().await;
let Some(store) = test_store().await else {
return;
};
assert!(store.sender_certificate().await.unwrap().is_none());
}
#[tokio::test]
async fn state_store_clear_registration() {
let mut store = test_store().await;
let Some(mut store) = test_store().await else {
return;
};
sqlx::query("INSERT INTO signal_kv (key, value) VALUES ('registration', $1)")
.bind(b"dummy-data".as_slice())
@@ -151,7 +162,9 @@ async fn state_store_clear_registration() {
#[tokio::test]
async fn session_store_crud() {
let store = test_store().await;
let Some(store) = test_store().await else {
return;
};
let mut ps = protocol_store(&store, IdentityType::Aci);
let addr = ProtocolAddress::new("test-uuid".into(), DeviceId::new(1).unwrap());
@@ -170,7 +183,9 @@ async fn session_store_crud() {
#[tokio::test]
async fn session_store_sub_devices() {
let store = test_store().await;
let Some(store) = test_store().await else {
return;
};
let mut ps = protocol_store(&store, IdentityType::Aci);
let uuid = Uuid::new_v4();
@@ -196,7 +211,9 @@ async fn session_store_sub_devices() {
#[tokio::test]
async fn pre_key_store_crud() {
let store = test_store().await;
let Some(store) = test_store().await else {
return;
};
let mut ps = protocol_store(&store, IdentityType::Aci);
let keypair = KeyPair::generate(&mut rand::rng());
@@ -213,7 +230,9 @@ async fn pre_key_store_crud() {
#[tokio::test]
async fn pre_key_store_next_ids() {
let store = test_store().await;
let Some(store) = test_store().await else {
return;
};
let mut ps = protocol_store(&store, IdentityType::Aci);
assert_eq!(ps.next_pre_key_id().await.unwrap(), 1);
@@ -229,7 +248,9 @@ async fn pre_key_store_next_ids() {
#[tokio::test]
async fn signed_pre_key_store_crud() {
let store = test_store().await;
let Some(store) = test_store().await else {
return;
};
let mut ps = protocol_store(&store, IdentityType::Aci);
let keypair = KeyPair::generate(&mut rand::rng());
@@ -251,7 +272,9 @@ async fn signed_pre_key_store_crud() {
#[tokio::test]
async fn kyber_pre_key_one_time_mark_used_deletes() {
let store = test_store().await;
let Some(store) = test_store().await else {
return;
};
let mut ps = protocol_store(&store, IdentityType::Aci);
let keypair = KeyPair::generate(&mut rand::rng());
@@ -276,7 +299,9 @@ async fn kyber_pre_key_one_time_mark_used_deletes() {
#[tokio::test]
async fn kyber_pre_key_last_resort_survives_mark_used() {
let store = test_store().await;
let Some(store) = test_store().await else {
return;
};
let mut ps = protocol_store(&store, IdentityType::Aci);
let keypair = KeyPair::generate(&mut rand::rng());
@@ -303,7 +328,9 @@ async fn kyber_pre_key_last_resort_survives_mark_used() {
#[tokio::test]
async fn kyber_pre_key_last_resort_rejects_replayed_base_key() {
let store = test_store().await;
let Some(store) = test_store().await else {
return;
};
let mut ps = protocol_store(&store, IdentityType::Aci);
let keypair = KeyPair::generate(&mut rand::rng());
@@ -332,7 +359,9 @@ async fn kyber_pre_key_last_resort_rejects_replayed_base_key() {
#[tokio::test]
async fn kyber_pre_key_last_resort_list() {
let store = test_store().await;
let Some(store) = test_store().await else {
return;
};
let mut ps = protocol_store(&store, IdentityType::Aci);
let keypair = KeyPair::generate(&mut rand::rng());
@@ -361,7 +390,9 @@ async fn kyber_pre_key_last_resort_list() {
#[tokio::test]
async fn identity_store_crud() {
let store = test_store().await;
let Some(store) = test_store().await else {
return;
};
let mut ps = protocol_store(&store, IdentityType::Aci);
let addr = ProtocolAddress::new("test-addr".into(), DeviceId::new(1).unwrap());
@@ -383,7 +414,9 @@ async fn identity_store_crud() {
#[tokio::test]
async fn identity_store_aci_pni_isolation() {
let store = test_store().await;
let Some(store) = test_store().await else {
return;
};
let mut aci_store = protocol_store(&store, IdentityType::Aci);
let pni_store = protocol_store(&store, IdentityType::Pni);
@@ -401,7 +434,9 @@ async fn identity_store_aci_pni_isolation() {
#[tokio::test]
async fn sender_key_store_load_missing() {
let store = test_store().await;
let Some(store) = test_store().await else {
return;
};
let mut ps = protocol_store(&store, IdentityType::Aci);
let sender = ProtocolAddress::new("sender-uuid".into(), DeviceId::new(1).unwrap());
@@ -417,7 +452,9 @@ async fn sender_key_store_load_missing() {
#[tokio::test]
async fn profile_key_store_roundtrip() {
let mut store = test_store().await;
let Some(mut store) = test_store().await else {
return;
};
let uuid = Uuid::new_v4();
let service_id: ServiceId = presage::libsignal_service::protocol::Aci::from(uuid).into();
@@ -433,7 +470,9 @@ async fn profile_key_store_roundtrip() {
#[tokio::test]
async fn client_from_pool_returns_none_without_registration() {
let store = test_store().await;
let Some(store) = test_store().await else {
return;
};
let pool = store.db.clone();
let client =
@@ -443,7 +482,9 @@ async fn client_from_pool_returns_none_without_registration() {
#[tokio::test]
async fn store_clear_removes_kv() {
let mut store = test_store().await;
let Some(mut store) = test_store().await else {
return;
};
store
.set_aci_identity_key_pair(IdentityKeyPair::generate(&mut rand::rng()))
crates/tranquil-signal/src/tests_fjall.rs
+373
View File
@@ -0,0 +1,373 @@
use presage::libsignal_service::{
pre_keys::{KyberPreKeyStoreExt, PreKeysStore},
prelude::{ProfileKey, SessionStoreExt},
protocol::{
DeviceId, Direction, GenericSignedPreKey, IdentityKeyPair, IdentityKeyStore, KeyPair,
KyberPreKeyId, KyberPreKeyRecord, KyberPreKeyStore, PreKeyId, PreKeyRecord, PreKeyStore,
ProtocolAddress, SenderKeyStore, ServiceId, SessionRecord, SessionStore, SignedPreKeyId,
SignedPreKeyRecord, SignedPreKeyStore, Timestamp,
},
};
use presage::store::{ContentsStore, StateStore, Store};
use uuid::Uuid;
use crate::fjall_store::FjallSignalStore;
fn test_store() -> (FjallSignalStore, tempfile::TempDir) {
let dir = tempfile::TempDir::new().unwrap();
let db = fjall::Database::builder(dir.path()).open().unwrap();
let ks = db
.keyspace("signal", fjall::KeyspaceCreateOptions::default)
.unwrap();
(FjallSignalStore::new(db, ks), dir)
}
#[tokio::test]
async fn state_store_registration_empty() {
let (store, _dir) = test_store();
assert!(store.load_registration_data().await.unwrap().is_none());
assert!(!store.is_registered().await);
}
#[tokio::test]
async fn state_store_identity_keypairs() {
let (store, _dir) = test_store();
let aci_pair = IdentityKeyPair::generate(&mut rand::rng());
let pni_pair = IdentityKeyPair::generate(&mut rand::rng());
store.set_aci_identity_key_pair(aci_pair).await.unwrap();
store.set_pni_identity_key_pair(pni_pair).await.unwrap();
let aci_store = store.aci_protocol_store();
let pni_store = store.pni_protocol_store();
let loaded_aci = aci_store.get_identity_key_pair().await.unwrap();
let loaded_pni = pni_store.get_identity_key_pair().await.unwrap();
assert_eq!(loaded_aci.serialize(), aci_pair.serialize());
assert_eq!(loaded_pni.serialize(), pni_pair.serialize());
}
#[tokio::test]
async fn state_store_sender_certificate_roundtrip() {
let (store, _dir) = test_store();
assert!(store.sender_certificate().await.unwrap().is_none());
}
#[tokio::test]
async fn state_store_clear_registration() {
let (mut store, _dir) = test_store();
store
.set_aci_identity_key_pair(IdentityKeyPair::generate(&mut rand::rng()))
.await
.unwrap();
let mut ps = store.aci_protocol_store();
let keypair = KeyPair::generate(&mut rand::rng());
let record = PreKeyRecord::new(PreKeyId::from(1u32), &keypair);
ps.save_pre_key(PreKeyId::from(1u32), &record)
.await
.unwrap();
store.clear_registration().await.unwrap();
assert!(store.load_registration_data().await.unwrap().is_none());
assert!(ps.get_pre_key(PreKeyId::from(1u32)).await.is_err());
}
#[tokio::test]
async fn session_store_crud() {
let (store, _dir) = test_store();
let mut ps = store.aci_protocol_store();
let addr = ProtocolAddress::new("test-uuid".into(), DeviceId::new(1).unwrap());
assert!(ps.load_session(&addr).await.unwrap().is_none());
let record = SessionRecord::new_fresh();
ps.store_session(&addr, &record).await.unwrap();
let loaded = ps.load_session(&addr).await.unwrap();
assert!(loaded.is_some());
ps.store_session(&addr, &record).await.unwrap();
let loaded2 = ps.load_session(&addr).await.unwrap();
assert!(loaded2.is_some());
}
#[tokio::test]
async fn session_store_sub_devices() {
let (store, _dir) = test_store();
let mut ps = store.aci_protocol_store();
let uuid = Uuid::new_v4();
let service_id: ServiceId = presage::libsignal_service::protocol::Aci::from(uuid).into();
let addr1 = ProtocolAddress::new(uuid.to_string(), DeviceId::new(1).unwrap());
let addr2 = ProtocolAddress::new(uuid.to_string(), DeviceId::new(2).unwrap());
let addr3 = ProtocolAddress::new(uuid.to_string(), DeviceId::new(3).unwrap());
let record = SessionRecord::new_fresh();
ps.store_session(&addr1, &record).await.unwrap();
ps.store_session(&addr2, &record).await.unwrap();
ps.store_session(&addr3, &record).await.unwrap();
let sub_devices = ps.get_sub_device_sessions(&service_id).await.unwrap();
assert_eq!(sub_devices.len(), 2);
let deleted = ps.delete_all_sessions(&service_id).await.unwrap();
assert_eq!(deleted, 3);
let sub_devices = ps.get_sub_device_sessions(&service_id).await.unwrap();
assert!(sub_devices.is_empty());
}
#[tokio::test]
async fn pre_key_store_crud() {
let (store, _dir) = test_store();
let mut ps = store.aci_protocol_store();
let keypair = KeyPair::generate(&mut rand::rng());
let id = PreKeyId::from(42u32);
let record = PreKeyRecord::new(id, &keypair);
ps.save_pre_key(id, &record).await.unwrap();
let loaded = ps.get_pre_key(id).await.unwrap();
assert_eq!(loaded.serialize().unwrap(), record.serialize().unwrap());
ps.remove_pre_key(id).await.unwrap();
assert!(ps.get_pre_key(id).await.is_err());
}
#[tokio::test]
async fn pre_key_store_next_ids() {
let (store, _dir) = test_store();
let mut ps = store.aci_protocol_store();
assert_eq!(ps.next_pre_key_id().await.unwrap(), 1);
let keypair = KeyPair::generate(&mut rand::rng());
let record = PreKeyRecord::new(PreKeyId::from(5u32), &keypair);
ps.save_pre_key(PreKeyId::from(5u32), &record)
.await
.unwrap();
assert_eq!(ps.next_pre_key_id().await.unwrap(), 6);
}
#[tokio::test]
async fn signed_pre_key_store_crud() {
let (store, _dir) = test_store();
let mut ps = store.aci_protocol_store();
let keypair = KeyPair::generate(&mut rand::rng());
let id = SignedPreKeyId::from(1u32);
let signature = keypair
.private_key
.calculate_signature(&keypair.public_key.serialize(), &mut rand::rng())
.unwrap();
let record =
SignedPreKeyRecord::new(id, Timestamp::from_epoch_millis(1000), &keypair, &signature);
ps.save_signed_pre_key(id, &record).await.unwrap();
let loaded = ps.get_signed_pre_key(id).await.unwrap();
assert_eq!(loaded.serialize().unwrap(), record.serialize().unwrap());
assert_eq!(ps.signed_pre_keys_count().await.unwrap(), 1);
assert_eq!(ps.next_signed_pre_key_id().await.unwrap(), 2);
}
#[tokio::test]
async fn kyber_pre_key_one_time_mark_used_deletes() {
let (store, _dir) = test_store();
let mut ps = store.aci_protocol_store();
let keypair = KeyPair::generate(&mut rand::rng());
let id = KyberPreKeyId::from(1u32);
let record = KyberPreKeyRecord::generate(
presage::libsignal_service::protocol::kem::KeyType::Kyber1024,
id,
&keypair.private_key,
)
.unwrap();
ps.save_kyber_pre_key(id, &record).await.unwrap();
assert!(ps.get_kyber_pre_key(id).await.is_ok());
let ec_prekey_id = SignedPreKeyId::from(1u32);
ps.mark_kyber_pre_key_used(id, ec_prekey_id, &keypair.public_key)
.await
.unwrap();
assert!(ps.get_kyber_pre_key(id).await.is_err());
}
#[tokio::test]
async fn kyber_pre_key_last_resort_survives_mark_used() {
let (store, _dir) = test_store();
let mut ps = store.aci_protocol_store();
let keypair = KeyPair::generate(&mut rand::rng());
let id = KyberPreKeyId::from(1u32);
let record = KyberPreKeyRecord::generate(
presage::libsignal_service::protocol::kem::KeyType::Kyber1024,
id,
&keypair.private_key,
)
.unwrap();
ps.store_last_resort_kyber_pre_key(id, &record)
.await
.unwrap();
assert!(ps.get_kyber_pre_key(id).await.is_ok());
let ec_prekey_id = SignedPreKeyId::from(1u32);
ps.mark_kyber_pre_key_used(id, ec_prekey_id, &keypair.public_key)
.await
.unwrap();
assert!(ps.get_kyber_pre_key(id).await.is_ok());
}
#[tokio::test]
async fn kyber_pre_key_last_resort_rejects_replayed_base_key() {
let (store, _dir) = test_store();
let mut ps = store.aci_protocol_store();
let keypair = KeyPair::generate(&mut rand::rng());
let id = KyberPreKeyId::from(1u32);
let record = KyberPreKeyRecord::generate(
presage::libsignal_service::protocol::kem::KeyType::Kyber1024,
id,
&keypair.private_key,
)
.unwrap();
ps.store_last_resort_kyber_pre_key(id, &record)
.await
.unwrap();
let ec_prekey_id = SignedPreKeyId::from(1u32);
ps.mark_kyber_pre_key_used(id, ec_prekey_id, &keypair.public_key)
.await
.unwrap();
let replay_result = ps
.mark_kyber_pre_key_used(id, ec_prekey_id, &keypair.public_key)
.await;
assert!(replay_result.is_err());
}
#[tokio::test]
async fn kyber_pre_key_last_resort_list() {
let (store, _dir) = test_store();
let mut ps = store.aci_protocol_store();
let keypair = KeyPair::generate(&mut rand::rng());
let id = KyberPreKeyId::from(1u32);
let record = KyberPreKeyRecord::generate(
presage::libsignal_service::protocol::kem::KeyType::Kyber1024,
id,
&keypair.private_key,
)
.unwrap();
assert!(
ps.load_last_resort_kyber_pre_keys()
.await
.unwrap()
.is_empty()
);
ps.store_last_resort_kyber_pre_key(id, &record)
.await
.unwrap();
let last_resorts = ps.load_last_resort_kyber_pre_keys().await.unwrap();
assert_eq!(last_resorts.len(), 1);
}
#[tokio::test]
async fn identity_store_crud() {
let (store, _dir) = test_store();
let mut ps = store.aci_protocol_store();
let addr = ProtocolAddress::new("test-addr".into(), DeviceId::new(1).unwrap());
let keypair = IdentityKeyPair::generate(&mut rand::rng());
let identity_key = keypair.identity_key();
assert!(ps.get_identity(&addr).await.unwrap().is_none());
ps.save_identity(&addr, identity_key).await.unwrap();
let loaded = ps.get_identity(&addr).await.unwrap().unwrap();
assert_eq!(loaded.serialize(), identity_key.serialize());
assert!(
ps.is_trusted_identity(&addr, identity_key, Direction::Receiving)
.await
.unwrap()
);
}
#[tokio::test]
async fn identity_store_aci_pni_isolation() {
let (store, _dir) = test_store();
let mut aci_store = store.aci_protocol_store();
let pni_store = store.pni_protocol_store();
let addr = ProtocolAddress::new("same-addr".into(), DeviceId::new(1).unwrap());
let keypair = IdentityKeyPair::generate(&mut rand::rng());
aci_store
.save_identity(&addr, keypair.identity_key())
.await
.unwrap();
assert!(aci_store.get_identity(&addr).await.unwrap().is_some());
assert!(pni_store.get_identity(&addr).await.unwrap().is_none());
}
#[tokio::test]
async fn sender_key_store_load_missing() {
let (store, _dir) = test_store();
let mut ps = store.aci_protocol_store();
let sender = ProtocolAddress::new("sender-uuid".into(), DeviceId::new(1).unwrap());
let dist_id = Uuid::new_v4();
assert!(
ps.load_sender_key(&sender, dist_id)
.await
.unwrap()
.is_none()
);
}
#[tokio::test]
async fn profile_key_store_roundtrip() {
let (mut store, _dir) = test_store();
let uuid = Uuid::new_v4();
let service_id: ServiceId = presage::libsignal_service::protocol::Aci::from(uuid).into();
let key = ProfileKey { bytes: [42u8; 32] };
assert!(store.profile_key(&service_id).await.unwrap().is_none());
store.upsert_profile_key(&uuid, key).await.unwrap();
let loaded = store.profile_key(&service_id).await.unwrap().unwrap();
assert_eq!(loaded.bytes, key.bytes);
}
#[tokio::test]
async fn store_clear_removes_all() {
let (mut store, _dir) = test_store();
store
.set_aci_identity_key_pair(IdentityKeyPair::generate(&mut rand::rng()))
.await
.unwrap();
store.clear().await.unwrap();
assert!(store.load_registration_data().await.unwrap().is_none());
}
crates/tranquil-store/Cargo.toml
+12 -1
View File
@@ -28,8 +28,10 @@ jacquard-repo = { workspace = true }
cid = { workspace = true }
multihash = { workspace = true }
sha2 = { workspace = true }
serde_ipld_dagcbor = { workspace = true }
siphasher = "1"
dashmap = "6"
rayon = "1"
smallvec = "1"
uuid = { workspace = true }
@@ -37,6 +39,7 @@ uuid = { workspace = true }
test-harness = []
[dev-dependencies]
tranquil-store = { path = ".", features = ["test-harness"] }
proptest = "1"
tempfile = "3"
futures = { workspace = true }
@@ -47,8 +50,8 @@ tranquil-db = { workspace = true }
sqlx = { workspace = true }
k256 = { workspace = true }
rand = { workspace = true }
serde_ipld_dagcbor = { workspace = true }
tikv-jemallocator = "0.6"
tracing-subscriber = { workspace = true, features = ["env-filter"] }
[[bench]]
name = "blockstore"
@@ -69,3 +72,11 @@ harness = false
[[bench]]
name = "profile_reads"
harness = false
[[bench]]
name = "profile_eventlog_reads"
harness = false
[[bench]]
name = "recovery"
harness = false
crates/tranquil-store/benches/blockstore.rs
+45 -55
View File
@@ -61,11 +61,16 @@ fn compute_stats(durations: &mut [Duration]) -> Option<LatencyStats> {
}
fn open_store(dir: &Path) -> TranquilBlockStore {
open_store_sharded(dir, 1)
}
fn open_store_sharded(dir: &Path, shard_count: u8) -> TranquilBlockStore {
TranquilBlockStore::open(BlockStoreConfig {
data_dir: dir.join("data"),
index_dir: dir.join("index"),
max_file_size: DEFAULT_MAX_FILE_SIZE,
group_commit: GroupCommitConfig::default(),
shard_count,
})
.unwrap()
}
@@ -81,7 +86,7 @@ fn format_latency(stats: Option<&LatencyStats>) -> String {
}
async fn bench_write_throughput(block_count: usize, concurrency: usize) {
let dir = tempfile::TempDir::new().unwrap();
let dir = bench_temp_dir();
let store = open_store(dir.path());
let blocks_per_task = block_count / concurrency;
@@ -162,7 +167,7 @@ async fn bench_write_throughput(block_count: usize, concurrency: usize) {
}
async fn bench_read_throughput(block_count: usize, concurrency: usize) {
let dir = tempfile::TempDir::new().unwrap();
let dir = bench_temp_dir();
let store = open_store(dir.path());
let cids_per_task = block_count / concurrency;
@@ -238,7 +243,7 @@ async fn bench_read_throughput(block_count: usize, concurrency: usize) {
}
async fn bench_mixed_workload(block_count: usize, concurrency: usize) {
let dir = tempfile::TempDir::new().unwrap();
let dir = bench_temp_dir();
let store = open_store(dir.path());
let ops_per_task = block_count / concurrency;
@@ -351,7 +356,7 @@ async fn bench_group_commit_effectiveness(block_count: usize) {
println!("-- group commit effectiveness at {block_count} blocks --");
let baseline_cycle_time = {
let dir = tempfile::TempDir::new().unwrap();
let dir = bench_temp_dir();
let store = open_store(dir.path());
let start = Instant::now();
@@ -377,7 +382,7 @@ async fn bench_group_commit_effectiveness(block_count: usize) {
if block_count < concurrency {
return;
}
let dir = tempfile::TempDir::new().unwrap();
let dir = bench_temp_dir();
let store = open_store(dir.path());
let blocks_per_task = block_count / concurrency;
let actual_count = blocks_per_task * concurrency;
@@ -422,29 +427,10 @@ async fn bench_group_commit_effectiveness(block_count: usize) {
.await;
}
async fn bench_postgres_write_throughput(block_count: usize, concurrency: usize) {
let database_url = match std::env::var("DATABASE_URL") {
Ok(url) => url,
Err(_) => {
println!("skipped, set DATABASE_URL to enable");
return;
}
};
async fn bench_sharded_write_throughput(block_count: usize, concurrency: usize, shard_count: u8) {
let dir = bench_temp_dir();
let store = open_store_sharded(dir.path(), shard_count);
let max_conns = u32::try_from(concurrency).expect("concurrency exceeds u32") + 5;
let pool = sqlx::postgres::PgPoolOptions::new()
.max_connections(max_conns)
.connect(&database_url)
.await
.unwrap();
sqlx::query("CREATE TABLE IF NOT EXISTS blocks (cid bytea PRIMARY KEY, data bytea NOT NULL)")
.execute(&pool)
.await
.unwrap();
sqlx::query("TRUNCATE blocks").execute(&pool).await.unwrap();
let pg_store = tranquil_repo::PostgresBlockStore::new(pool.clone());
let blocks_per_task = block_count / concurrency;
let actual_count = blocks_per_task * concurrency;
let blocks: Vec<Vec<u8>> = (0..actual_count).map(make_block).collect();
@@ -454,7 +440,7 @@ async fn bench_postgres_write_throughput(block_count: usize, concurrency: usize)
let handles: Vec<_> = (0..concurrency)
.map(|task_id| {
let store = pg_store.clone();
let store = store.clone();
let task_blocks: Vec<Vec<u8>> =
blocks[task_id * blocks_per_task..(task_id + 1) * blocks_per_task].to_vec();
tokio::spawn(async move {
@@ -462,38 +448,39 @@ async fn bench_postgres_write_throughput(block_count: usize, concurrency: usize)
.then(|block| {
let store = store.clone();
async move {
let t = Instant::now();
store.put(&block).await.unwrap();
t.elapsed()
}
})
.collect::<Vec<Duration>>()
.await
.collect::<Vec<()>>()
.await;
})
})
.collect();
let mut all_latencies: Vec<Duration> = futures::future::join_all(handles)
.await
.into_iter()
.flat_map(Result::unwrap)
.collect();
futures::future::join_all(handles).await;
let elapsed = start.elapsed();
let stats = compute_stats(&mut all_latencies);
let lat = format_latency(stats.as_ref());
println!(
"{:.0} blocks/sec, {:.1} MB/sec, {:.1}ms{lat}",
"{:.0} blocks/sec, {:.1} MB/sec, {:.1}ms",
actual_count as f64 / elapsed.as_secs_f64(),
total_bytes as f64 / elapsed.as_secs_f64() / (1024.0 * 1024.0),
elapsed.as_secs_f64() * 1000.0,
);
}
sqlx::query("TRUNCATE blocks").execute(&pool).await.unwrap();
pool.close().await;
fn bench_temp_dir() -> tempfile::TempDir {
match std::env::var("BENCH_DIR") {
Ok(dir) => tempfile::TempDir::new_in(dir).unwrap(),
Err(_) => tempfile::TempDir::new().unwrap(),
}
}
fn main() {
tracing_subscriber::fmt()
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
.with_writer(std::io::stderr)
.init();
let worker_threads = std::env::var("BENCH_WORKER_THREADS")
.ok()
.and_then(|s| s.trim().parse::<usize>().ok())
@@ -558,20 +545,23 @@ fn main() {
rt.block_on(bench_group_commit_effectiveness(1000));
if std::env::var("DATABASE_URL").is_ok() {
block_counts.iter().for_each(|&block_count| {
concurrency_levels.iter().for_each(|&concurrency| {
if block_count < concurrency {
return;
}
println!(
"-- postgres write: {} blocks, {} writers --",
block_count, concurrency
);
rt.block_on(bench_postgres_write_throughput(block_count, concurrency));
let shard_counts = parse_env_list("BENCH_SHARDS", vec![1, 2, 4]);
if shard_counts.iter().any(|&s| s > 1) {
println!("\n-- sharded write throughput :p --");
shard_counts.iter().for_each(|&shards| {
block_counts.iter().for_each(|&block_count| {
concurrency_levels.iter().for_each(|&concurrency| {
if block_count < concurrency {
return;
}
let sc = u8::try_from(shards).unwrap_or(4);
println!(
"-- sharded write: {} shards, {} blocks, {} writers --",
sc, block_count, concurrency
);
rt.block_on(bench_sharded_write_throughput(block_count, concurrency, sc));
});
});
});
} else {
println!("set DATABASE_URL for postgres comparison");
}
}
crates/tranquil-store/benches/eventlog.rs
+2 -253
View File
@@ -42,7 +42,7 @@ fn make_event(index: usize) -> SequencedEvent {
prev_data_cid: None,
ops: Some(serde_json::json!({ "data": ops_payload })),
blobs: None,
blocks_cids: None,
blocks: None,
handle: None,
active: None,
status: None,
@@ -621,217 +621,6 @@ fn bench_broadcast_fanout(subscriber_count: usize) {
let _ = log.shutdown();
}
async fn bench_pg_write_throughput(event_count: usize, concurrency: usize) {
let database_url = match std::env::var("DATABASE_URL") {
Ok(url) => url,
Err(_) => {
println!("skipped, set DATABASE_URL to enable");
return;
}
};
let max_conns = u32::try_from(concurrency)
.unwrap_or(u32::MAX)
.saturating_add(10);
let pool = sqlx::postgres::PgPoolOptions::new()
.max_connections(max_conns)
.acquire_timeout(Duration::from_secs(30))
.connect(&database_url)
.await
.unwrap();
sqlx::query(
"CREATE TABLE IF NOT EXISTS bench_repo_seq (
seq BIGSERIAL PRIMARY KEY,
did TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
event_type TEXT NOT NULL,
ops JSONB
)",
)
.execute(&pool)
.await
.unwrap();
sqlx::query("TRUNCATE bench_repo_seq")
.execute(&pool)
.await
.unwrap();
let events_per_task = event_count / concurrency;
let actual_count = events_per_task * concurrency;
let start = Instant::now();
let handles: Vec<_> = (0..concurrency)
.map(|task_id| {
let pool = pool.clone();
tokio::spawn(async move {
futures::stream::iter(0..events_per_task)
.then(|i| {
let pool = pool.clone();
async move {
let global = task_id * events_per_task + i;
let did = format!("did:plc:{global:024x}");
let ops_size = match global % 4 {
0 => 64,
1 => 256,
2 => 1024,
_ => 4096,
};
let payload: String = (0..ops_size)
.map(|j| {
((global.wrapping_mul(31).wrapping_add(j)) % 26 + 97) as u8
as char
})
.collect();
let ops = serde_json::json!({ "data": payload });
let t = Instant::now();
sqlx::query(
"INSERT INTO bench_repo_seq (did, event_type, ops) VALUES ($1, $2, $3)",
)
.bind(&did)
.bind("commit")
.bind(&ops)
.execute(&pool)
.await
.unwrap();
t.elapsed()
}
})
.collect::<Vec<Duration>>()
.await
})
})
.collect();
let mut all_latencies: Vec<Duration> = futures::future::join_all(handles)
.await
.into_iter()
.flat_map(Result::unwrap)
.collect();
let elapsed = start.elapsed();
let lat = format_latency(compute_stats(&mut all_latencies).as_ref());
println!(
"{:.0} events/sec, {:.1}ms{lat}",
actual_count as f64 / elapsed.as_secs_f64(),
elapsed.as_secs_f64() * 1000.0,
);
sqlx::query("TRUNCATE bench_repo_seq")
.execute(&pool)
.await
.unwrap();
pool.close().await;
}
async fn bench_pg_read_throughput(event_count: usize, concurrency: usize) {
let database_url = match std::env::var("DATABASE_URL") {
Ok(url) => url,
Err(_) => {
println!("skipped, set DATABASE_URL to enable");
return;
}
};
let max_conns = u32::try_from(concurrency)
.unwrap_or(u32::MAX)
.saturating_add(5);
let pool = sqlx::postgres::PgPoolOptions::new()
.max_connections(max_conns)
.connect(&database_url)
.await
.unwrap();
sqlx::query(
"CREATE TABLE IF NOT EXISTS bench_repo_seq (
seq BIGSERIAL PRIMARY KEY,
did TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
event_type TEXT NOT NULL,
ops JSONB
)",
)
.execute(&pool)
.await
.unwrap();
let row: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM bench_repo_seq")
.fetch_one(&pool)
.await
.unwrap();
if (row.0 as usize) < event_count {
sqlx::query("TRUNCATE bench_repo_seq")
.execute(&pool)
.await
.unwrap();
println!("populating {event_count} events");
futures::stream::iter(0..event_count)
.map(|i| {
let pool = pool.clone();
async move {
let did = format!("did:plc:{i:024x}");
let ops = serde_json::json!({ "data": "x".repeat(256) });
sqlx::query(
"INSERT INTO bench_repo_seq (did, event_type, ops) VALUES ($1, $2, $3)",
)
.bind(&did)
.bind("commit")
.bind(&ops)
.execute(&pool)
.await
.unwrap();
}
})
.buffer_unordered(50)
.collect::<Vec<()>>()
.await;
}
let total_events = Arc::new(AtomicU64::new(0));
let start = Instant::now();
let handles: Vec<_> = (0..concurrency)
.map(|_| {
let pool = pool.clone();
let total_events = Arc::clone(&total_events);
tokio::spawn(async move {
let mut cursor = 0i64;
let mut count = 0u64;
loop {
let rows: Vec<(i64,)> = sqlx::query_as(
"SELECT seq FROM bench_repo_seq WHERE seq > $1 ORDER BY seq LIMIT $2",
)
.bind(cursor)
.bind(1000i64)
.fetch_all(&pool)
.await
.unwrap();
if rows.is_empty() {
break;
}
count += rows.len() as u64;
cursor = rows.last().unwrap().0;
}
total_events.fetch_add(count, Ordering::Relaxed);
})
})
.collect();
futures::future::join_all(handles).await;
let elapsed = start.elapsed();
let total = total_events.load(Ordering::Relaxed);
println!(
"{:.0} total events/sec across {concurrency} readers, {total} events, {:.1}ms",
total as f64 / elapsed.as_secs_f64(),
elapsed.as_secs_f64() * 1000.0,
);
pool.close().await;
}
fn main() {
println!("-- eventlog benchmarks --");
let cpus = std::thread::available_parallelism()
@@ -915,45 +704,5 @@ fn main() {
bench_stampede(100_000, 100, 16, 50);
bench_stampede(500_000, 100, 16, 50);
let rt = tokio::runtime::Builder::new_multi_thread()
.worker_threads(cpus)
.enable_all()
.build()
.unwrap();
if std::env::var("DATABASE_URL").is_ok() {
println!("-- postgres comparison --");
event_counts.iter().for_each(|&n| {
producer_counts.iter().for_each(|&p| {
if n >= p {
println!("-- postgres write: {n} events, {p} writers --",);
rt.block_on(bench_pg_write_throughput(n, p));
}
});
});
event_counts.iter().for_each(|&n| {
[1usize, 4, 16, 32].iter().for_each(|&r| {
println!("-- postgres read: {n} events, {r} readers --",);
rt.block_on(bench_pg_read_throughput(n, r));
});
});
rt.block_on(async {
let url = std::env::var("DATABASE_URL").unwrap();
let pool = sqlx::postgres::PgPoolOptions::new()
.max_connections(5)
.connect(&url)
.await
.unwrap();
sqlx::query("DROP TABLE IF EXISTS bench_repo_seq")
.execute(&pool)
.await
.unwrap();
pool.close().await;
});
} else {
println!("set DATABASE_URL for postgres comparison");
}
let _ = (cpus, producer_counts);
}
crates/tranquil-store/benches/metastore.rs
+2 -2
View File
@@ -180,7 +180,7 @@ fn make_commit_input(
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some(make_rev(rev_n)),
},
@@ -241,7 +241,7 @@ async fn seed_records(
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some(make_rev(rev_n)),
},
crates/tranquil-store/benches/metastore_scale.rs
+3 -3
View File
@@ -257,7 +257,7 @@ async fn seed_records_for_user(pool: &HandlerPool, user: &UserInfo, record_count
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some(make_rev(1)),
},
@@ -348,7 +348,7 @@ async fn bench_single_user_commit(pool: &Arc<HandlerPool>, user: &UserInfo, ops:
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some(make_rev(rev_n)),
},
@@ -424,7 +424,7 @@ async fn bench_multi_user_commit(
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some(make_rev(rev_n)),
},
crates/tranquil-store/benches/profile_eventlog_reads.rs
+342
View File
@@ -0,0 +1,342 @@
use std::path::Path;
use std::sync::Arc;
use std::time::Instant;
use chrono::Utc;
use tranquil_db_traits::{RepoEventType, SequencedEvent};
use tranquil_types::Did;
use tranquil_store::RealIO;
use tranquil_store::eventlog::{
EventLog, EventLogConfig, EventSequence, decode_payload, to_sequenced_event,
};
fn make_did(index: usize) -> Did {
let suffix: String = format!("{index:024x}");
Did::new(format!("did:plc:{suffix}")).unwrap()
}
fn make_event(index: usize) -> SequencedEvent {
let ops_size = match index % 4 {
0 => 64,
1 => 256,
2 => 1024,
_ => 4096,
};
let ops_payload: String = (0..ops_size)
.map(|i| ((index.wrapping_mul(31).wrapping_add(i)) % 26 + 97) as u8 as char)
.collect();
SequencedEvent {
seq: tranquil_db_traits::SequenceNumber::from_raw(
i64::try_from(index + 1).expect("event index overflow"),
),
did: make_did(index % 10_000),
created_at: Utc::now(),
event_type: match index % 4 {
0 => RepoEventType::Commit,
1 => RepoEventType::Identity,
2 => RepoEventType::Account,
_ => RepoEventType::Sync,
},
commit_cid: None,
prev_cid: None,
prev_data_cid: None,
ops: Some(serde_json::json!({ "data": ops_payload })),
blobs: None,
blocks: None,
handle: None,
active: None,
status: None,
rev: None,
}
}
fn open_eventlog(dir: &Path) -> EventLog<RealIO> {
let segments_dir = dir.join("segments");
std::fs::create_dir_all(&segments_dir).unwrap();
EventLog::open(
EventLogConfig {
segments_dir,
..EventLogConfig::default()
},
RealIO::new(),
)
.unwrap()
}
struct PhaseTimings {
raw_read_ns: Vec<u64>,
decode_payload_ns: Vec<u64>,
ops_json_ns: Vec<u64>,
did_parse_ns: Vec<u64>,
full_conversion_ns: Vec<u64>,
total_get_events_ns: Vec<u64>,
}
impl PhaseTimings {
fn new(capacity: usize) -> Self {
Self {
raw_read_ns: Vec::with_capacity(capacity),
decode_payload_ns: Vec::with_capacity(capacity),
ops_json_ns: Vec::with_capacity(capacity),
did_parse_ns: Vec::with_capacity(capacity),
full_conversion_ns: Vec::with_capacity(capacity),
total_get_events_ns: Vec::with_capacity(capacity),
}
}
}
fn percentile(sorted: &[u64], pct: f64) -> u64 {
if sorted.is_empty() {
return 0;
}
let idx = ((sorted.len() - 1) as f64 * pct / 100.0) as usize;
sorted[idx]
}
fn report_phase(name: &str, values_ns: &mut [u64], event_count: usize) {
values_ns.sort();
let total: u64 = values_ns.iter().sum();
let per_event_ns = total as f64 / event_count as f64;
let p50 = percentile(values_ns, 50.0);
let p99 = percentile(values_ns, 99.0);
println!(
"{name}: {:.2}ms total, {per_event_ns:.0}ns/event, p50 {p50}ns, p99 {p99}ns",
total as f64 / 1_000_000.0,
);
}
fn profile_read_phases(event_count: usize, readers: usize) {
println!("-- read path profile: {event_count} events, {readers} readers --");
let dir = tempfile::TempDir::new().unwrap();
let log = Arc::new(open_eventlog(dir.path()));
let events: Vec<SequencedEvent> = (0..event_count).map(make_event).collect();
events.iter().enumerate().for_each(|(i, event)| {
log.append_event(&make_did(i % 10_000), RepoEventType::Commit, event)
.unwrap();
});
log.sync().unwrap();
println!("seeded {event_count} events");
let batch_size = 4096usize;
let iterations = 3;
(0..iterations).for_each(|iter| {
println!("-- iteration {}/{iterations} --", iter + 1);
let handles: Vec<_> = (0..readers)
.map(|_| {
let log = Arc::clone(&log);
std::thread::spawn(move || {
let reader = log.reader();
let mut timings = PhaseTimings::new(event_count / batch_size + 1);
let mut total_events = 0usize;
let mut cursor = EventSequence::BEFORE_ALL;
std::iter::from_fn(|| {
let t_total = Instant::now();
let t_raw = Instant::now();
let raw_events = reader.read_events_from(cursor, batch_size).unwrap();
let raw_read_elapsed = t_raw.elapsed();
if raw_events.is_empty() {
return None;
}
let mut batch_decode_ns = 0u64;
let mut batch_ops_ns = 0u64;
let mut batch_did_ns = 0u64;
let mut batch_conversion_ns = 0u64;
raw_events.iter().for_each(|raw| {
let t_decode = Instant::now();
let payload = decode_payload(&raw.payload).unwrap();
batch_decode_ns += t_decode.elapsed().as_nanos() as u64;
let t_ops = Instant::now();
let _ops: Option<serde_json::Value> = payload
.ops
.as_ref()
.map(|bytes| serde_ipld_dagcbor::from_slice(bytes).unwrap());
batch_ops_ns += t_ops.elapsed().as_nanos() as u64;
let t_did = Instant::now();
let _did = Did::new(&payload.did).unwrap();
batch_did_ns += t_did.elapsed().as_nanos() as u64;
let t_conversion = Instant::now();
let payload2 = decode_payload(&raw.payload).unwrap();
let _event = to_sequenced_event(raw, &payload2).unwrap();
batch_conversion_ns += t_conversion.elapsed().as_nanos() as u64;
});
let batch_events = raw_events.len();
cursor = EventSequence::new(
u64::try_from(raw_events.last().unwrap().seq.as_i64()).unwrap(),
);
total_events += batch_events;
timings.raw_read_ns.push(raw_read_elapsed.as_nanos() as u64);
timings.decode_payload_ns.push(batch_decode_ns);
timings.ops_json_ns.push(batch_ops_ns);
timings.did_parse_ns.push(batch_did_ns);
timings.full_conversion_ns.push(batch_conversion_ns);
timings
.total_get_events_ns
.push(t_total.elapsed().as_nanos() as u64);
Some(())
})
.count();
(timings, total_events)
})
})
.collect();
let results: Vec<_> = handles.into_iter().map(|h| h.join().unwrap()).collect();
let total_events: usize = results.iter().map(|(_, count)| count).sum();
let mut agg = PhaseTimings::new(0);
results.iter().for_each(|(t, _)| {
agg.raw_read_ns.extend_from_slice(&t.raw_read_ns);
agg.decode_payload_ns
.extend_from_slice(&t.decode_payload_ns);
agg.ops_json_ns.extend_from_slice(&t.ops_json_ns);
agg.did_parse_ns.extend_from_slice(&t.did_parse_ns);
agg.full_conversion_ns
.extend_from_slice(&t.full_conversion_ns);
agg.total_get_events_ns
.extend_from_slice(&t.total_get_events_ns);
});
println!("{total_events} events across {readers} readers");
report_phase("raw_read", &mut agg.raw_read_ns, total_events);
report_phase("full conversion", &mut agg.full_conversion_ns, total_events);
report_phase(
"postcard decode, isolated",
&mut agg.decode_payload_ns,
total_events,
);
report_phase(
"DAG-CBOR ops parse, isolated",
&mut agg.ops_json_ns,
total_events,
);
report_phase("DID parse, isolated", &mut agg.did_parse_ns, total_events);
report_phase(
"end-to-end total",
&mut agg.total_get_events_ns,
total_events,
);
let raw_total: u64 = agg.raw_read_ns.iter().sum();
let conversion_total: u64 = agg.full_conversion_ns.iter().sum();
let decode_total: u64 = agg.decode_payload_ns.iter().sum();
let ops_total: u64 = agg.ops_json_ns.iter().sum();
let did_total: u64 = agg.did_parse_ns.iter().sum();
let pipeline_total = raw_total + conversion_total;
let pct = |v: u64| v as f64 / pipeline_total as f64 * 100.0;
let conversion_other =
conversion_total.saturating_sub(decode_total + ops_total + did_total);
println!(
"breakdown: raw_read {:.1}%, postcard {:.1}%, dagcbor_ops {:.1}%, did {:.1}%, rest {:.1}%",
pct(raw_total),
pct(decode_total),
pct(ops_total),
pct(did_total),
pct(conversion_other),
);
});
let _ = log.shutdown();
}
fn profile_decode_phases(event_count: usize) {
println!("-- decode phase isolation: {event_count} events --");
let dir = tempfile::TempDir::new().unwrap();
let log = open_eventlog(dir.path());
let events: Vec<SequencedEvent> = (0..event_count).map(make_event).collect();
events.iter().enumerate().for_each(|(i, event)| {
log.append_event(&make_did(i % 10_000), RepoEventType::Commit, event)
.unwrap();
});
log.sync().unwrap();
let reader = log.reader();
let raw_events = reader
.read_events_from(EventSequence::BEFORE_ALL, event_count)
.unwrap();
println!("{} raw events pre-loaded", raw_events.len());
(0..5).for_each(|_| {
let t_decode = Instant::now();
let payloads: Vec<_> = raw_events
.iter()
.map(|raw| decode_payload(&raw.payload).unwrap())
.collect();
let decode_elapsed = t_decode.elapsed();
let t_convert = Instant::now();
let _events: Vec<_> = raw_events
.iter()
.zip(payloads.iter())
.map(|(raw, payload)| to_sequenced_event(raw, payload).unwrap())
.collect();
let convert_elapsed = t_convert.elapsed();
let t_ops_only = Instant::now();
let _: Vec<_> = payloads
.iter()
.map(|p| {
p.ops.as_ref().map(|bytes| {
serde_ipld_dagcbor::from_slice::<serde_json::Value>(bytes).unwrap()
})
})
.collect();
let ops_elapsed = t_ops_only.elapsed();
let n = raw_events.len() as f64;
println!(
"postcard {:.0}ns/evt, to_sequenced_event {:.0}ns/evt, dagcbor_ops {:.0}ns/evt",
decode_elapsed.as_nanos() as f64 / n,
convert_elapsed.as_nanos() as f64 / n,
ops_elapsed.as_nanos() as f64 / n,
);
});
let _ = log.shutdown();
}
fn main() {
println!("-- eventlog read path profiler --");
let cpus = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(8);
println!("available parallelism: {cpus}");
let event_count = std::env::var("PROFILE_EVENTS")
.ok()
.and_then(|s| s.replace('_', "").parse().ok())
.unwrap_or(100_000usize);
let reader_count = std::env::var("PROFILE_READERS")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(4usize);
profile_read_phases(event_count, reader_count);
println!();
profile_decode_phases(event_count);
}
crates/tranquil-store/benches/profile_reads.rs
+1 -1
View File
@@ -128,7 +128,7 @@ async fn seed_records(pool: &Arc<HandlerPool>, users: &[UserInfo], records_per_u
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev0000000001".to_string()),
},
crates/tranquil-store/benches/recovery.rs
+270
View File
@@ -0,0 +1,270 @@
use std::path::Path;
use std::time::Instant;
use tranquil_store::blockstore::hash_index::BlockIndex;
use tranquil_store::blockstore::{
CidBytes, DEFAULT_MAX_FILE_SIZE, DataFileId, DataFileWriter, HintFileWriter, hint_file_path,
scan_hints_to_memory,
};
use tranquil_store::{OpenOptions, RealIO, StorageIO};
#[global_allocator]
static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
fn test_cid(seed: u32) -> CidBytes {
let le = seed.to_le_bytes();
std::array::from_fn(|i| match i {
0 => 0x01,
1 => 0x71,
2 => 0x12,
3 => 0x20,
4..8 => le[i - 4],
_ => (seed as u8).wrapping_add(i as u8),
})
}
fn block_data(seed: u32) -> Vec<u8> {
let tag = seed.to_le_bytes();
std::iter::repeat(tag).flatten().take(256).collect()
}
struct DirectSeeder<'a> {
io: &'a RealIO,
data_dir: &'a Path,
file_id: DataFileId,
data_writer: DataFileWriter<'a, RealIO>,
hint_writer: HintFileWriter<'a, RealIO>,
blocks_in_file: u64,
}
impl<'a> DirectSeeder<'a> {
fn new(io: &'a RealIO, data_dir: &'a Path) -> Self {
std::fs::create_dir_all(data_dir).unwrap();
let file_id = DataFileId::new(0);
let data_fd = io
.open(
&data_dir.join(format!("{file_id}.tqb")),
OpenOptions::read_write(),
)
.unwrap();
let data_writer = DataFileWriter::new(io, data_fd, file_id).unwrap();
let hint_fd = io
.open(
&hint_file_path(data_dir, file_id),
OpenOptions::read_write(),
)
.unwrap();
let hint_writer = HintFileWriter::new(io, hint_fd);
Self {
io,
data_dir,
file_id,
data_writer,
hint_writer,
blocks_in_file: 0,
}
}
fn rotate(&mut self) {
self.data_writer.sync().unwrap();
self.hint_writer.sync().unwrap();
self.file_id = self.file_id.next();
let data_fd = self
.io
.open(
&self.data_dir.join(format!("{}.tqb", self.file_id)),
OpenOptions::read_write(),
)
.unwrap();
self.data_writer = DataFileWriter::new(self.io, data_fd, self.file_id).unwrap();
let hint_fd = self
.io
.open(
&hint_file_path(self.data_dir, self.file_id),
OpenOptions::read_write(),
)
.unwrap();
self.hint_writer = HintFileWriter::new(self.io, hint_fd);
self.blocks_in_file = 0;
}
fn append(&mut self, cid: &CidBytes, data: &[u8]) {
if self.data_writer.position().raw() > DEFAULT_MAX_FILE_SIZE {
self.rotate();
}
let loc = self.data_writer.append_block(cid, data).unwrap();
self.hint_writer
.append_hint(cid, loc.file_id, loc.offset, loc.length)
.unwrap();
self.blocks_in_file += 1;
if self.blocks_in_file.is_multiple_of(10_000) {
self.data_writer.sync().unwrap();
self.hint_writer.sync().unwrap();
}
}
fn finish(&mut self) {
self.data_writer.sync().unwrap();
self.hint_writer.sync().unwrap();
self.io.sync_dir(self.data_dir).unwrap();
}
}
fn seed_blocks_direct(data_dir: &Path, count: u32) {
let io = RealIO::new();
let mut seeder = DirectSeeder::new(&io, data_dir);
(0..count).for_each(|i| {
let cid = test_cid(i);
let data = block_data(i);
seeder.append(&cid, &data);
});
seeder.finish();
}
fn read_rss_mb() -> f64 {
std::fs::read_to_string("/proc/self/status")
.ok()
.and_then(|s| {
s.lines().find(|l| l.starts_with("VmRSS:")).and_then(|l| {
l.split_whitespace()
.nth(1)
.and_then(|v| v.parse::<f64>().ok())
})
})
.map(|kb| kb / 1024.0)
.unwrap_or(0.0)
}
fn bench_hint_scan_only(data_dir: &Path, block_count: u32) {
let io = RealIO::new();
let rss_before = read_rss_mb();
let start = Instant::now();
let (hint_index, _cursor) = scan_hints_to_memory(&io, data_dir).unwrap();
let elapsed = start.elapsed();
let rss_after = read_rss_mb();
let entry_count = hint_index.len();
let rss_delta = rss_after - rss_before;
let bytes_per_entry = match entry_count {
0 => 0.0,
n => (rss_delta * 1024.0 * 1024.0) / n as f64,
};
println!(
"hint scan to memory ({block_count} blocks): {:.3}s ({:.0} blocks/sec)",
elapsed.as_secs_f64(),
block_count as f64 / elapsed.as_secs_f64(),
);
println!(
" entries: {entry_count}, RSS: {rss_before:.1}MB -> {rss_after:.1}MB (delta: {rss_delta:.1}MB, {bytes_per_entry:.0} bytes/entry)"
);
drop(hint_index);
let rss_after_drop = read_rss_mb();
println!(" RSS after drop: {rss_after_drop:.1}MB");
}
fn bench_hash_table_rebuild_from_hints(data_dir: &Path, index_dir: &Path, block_count: u32) {
let io = RealIO::new();
let index = BlockIndex::open(index_dir).unwrap();
let rss_before = read_rss_mb();
let start = Instant::now();
index.rebuild_from_hints(&io, data_dir).unwrap();
let elapsed = start.elapsed();
let rss_after = read_rss_mb();
println!(
"hash table rebuild from hints ({block_count} blocks): {:.3}s ({:.0} blocks/sec)",
elapsed.as_secs_f64(),
block_count as f64 / elapsed.as_secs_f64(),
);
println!(
" RSS: {rss_before:.1}MB -> {rss_after:.1}MB (delta: {:.1}MB)",
rss_after - rss_before,
);
}
fn bench_hash_table_rebuild_from_data_files(data_dir: &Path, index_dir: &Path, block_count: u32) {
let io = RealIO::new();
let index = BlockIndex::open(index_dir).unwrap();
let start = Instant::now();
index.rebuild_from_data_files(&io, data_dir).unwrap();
let elapsed = start.elapsed();
println!(
"hash table rebuild from data files ({block_count} blocks): {:.3}s ({:.0} blocks/sec)",
elapsed.as_secs_f64(),
block_count as f64 / elapsed.as_secs_f64(),
);
}
fn nuke_index(index_dir: &Path) {
if index_dir.exists() {
std::fs::remove_dir_all(index_dir).unwrap();
}
std::fs::create_dir_all(index_dir).unwrap();
}
fn run_scale(block_count: u32) {
let label = match block_count {
n if n >= 10_000_000 => format!("{}M blocks", n / 1_000_000),
n if n >= 1_000_000 => format!("{}M blocks", n / 1_000_000),
n => format!("{}K blocks", n / 1_000),
};
println!("\n-- {label} --");
let dir = tempfile::TempDir::new().unwrap();
let data_dir = dir.path().join("data");
let index_dir = dir.path().join("index");
println!("seeding {block_count} blocks, direct without index...");
let seed_start = Instant::now();
seed_blocks_direct(&data_dir, block_count);
println!(
" blocks seeded in {:.1}s",
seed_start.elapsed().as_secs_f64()
);
println!("\n-- hint scan to memory --");
bench_hint_scan_only(&data_dir, block_count);
println!("\n-- hash table rebuild from hints --");
nuke_index(&index_dir);
bench_hash_table_rebuild_from_hints(&data_dir, &index_dir, block_count);
println!("\n-- hash table rebuild from data files --");
nuke_index(&index_dir);
bench_hash_table_rebuild_from_data_files(&data_dir, &index_dir, block_count);
}
fn parse_scales(input: &str) -> Vec<u32> {
input
.split(';')
.map(|s| s.trim().replace('_', "").parse::<u32>().unwrap())
.collect()
}
fn main() {
let scales = parse_scales(
&std::env::var("BENCH_RECOVERY_SCALES")
.unwrap_or_else(|_| "100_000; 1_000_000; 10_000_000".into()),
);
println!("recovery performance benchmark, hash table index :3");
println!("scales: {scales:?}");
scales.iter().for_each(|&blocks| {
run_scale(blocks);
});
}
crates/tranquil-store/src/archival.rs
+547
View File
@@ -0,0 +1,547 @@
use std::io;
use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use tracing::{debug, info, warn};
use crate::eventlog::{SEGMENT_FILE_EXTENSION, SegmentId, parse_segment_id, segment_path};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ArchivalState {
pub last_archived_segment: Option<SegmentId>,
}
impl ArchivalState {
fn empty() -> Self {
Self {
last_archived_segment: None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ArchivalPassResult {
pub segments_archived: u32,
pub bytes_archived: u64,
}
pub trait ArchivalDestination: Send + Sync {
fn store_segment(&self, segment_id: SegmentId, data: &[u8]) -> io::Result<()>;
}
pub struct LocalArchivalDestination {
dest_dir: PathBuf,
}
impl LocalArchivalDestination {
pub fn new(dest_dir: PathBuf) -> io::Result<Self> {
std::fs::create_dir_all(&dest_dir)?;
Ok(Self { dest_dir })
}
}
impl ArchivalDestination for LocalArchivalDestination {
fn store_segment(&self, segment_id: SegmentId, data: &[u8]) -> io::Result<()> {
let dest_path = segment_path(&self.dest_dir, segment_id);
let tmp_path = dest_path.with_extension(format!("{SEGMENT_FILE_EXTENSION}.tmp"));
std::fs::write(&tmp_path, data)?;
let f = std::fs::File::open(&tmp_path)?;
f.sync_all()?;
drop(f);
std::fs::rename(&tmp_path, &dest_path)?;
sync_dir(&self.dest_dir)?;
Ok(())
}
}
fn sync_dir(dir: &Path) -> io::Result<()> {
let d = std::fs::File::open(dir)?;
d.sync_all()
}
fn list_segment_files(segments_dir: &Path) -> io::Result<Vec<SegmentId>> {
let entries = match std::fs::read_dir(segments_dir) {
Ok(entries) => entries,
Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()),
Err(e) => return Err(e),
};
let mut ids: Vec<SegmentId> = entries
.filter_map(|entry| parse_segment_id(&entry.ok()?.path()))
.collect();
ids.sort();
Ok(ids)
}
pub struct ArchivalSidecar {
path: PathBuf,
}
impl ArchivalSidecar {
pub fn new(path: PathBuf) -> Self {
Self { path }
}
pub fn load(&self) -> io::Result<ArchivalState> {
match std::fs::read(&self.path) {
Ok(data) => serde_json::from_slice(&data)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)),
Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(ArchivalState::empty()),
Err(e) => Err(e),
}
}
pub fn save(&self, state: &ArchivalState) -> io::Result<()> {
let json =
serde_json::to_vec(state).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
let tmp_path = self.path.with_extension("tmp");
std::fs::write(&tmp_path, &json)?;
let f = std::fs::File::open(&tmp_path)?;
f.sync_all()?;
drop(f);
std::fs::rename(&tmp_path, &self.path)?;
self.path.parent().map(sync_dir).transpose()?;
Ok(())
}
}
pub struct ContinuousArchiver {
segments_dir: PathBuf,
sidecar: ArchivalSidecar,
destination: Box<dyn ArchivalDestination>,
}
impl ContinuousArchiver {
pub fn new(
segments_dir: PathBuf,
sidecar_path: PathBuf,
destination: Box<dyn ArchivalDestination>,
) -> Self {
Self {
segments_dir,
sidecar: ArchivalSidecar::new(sidecar_path),
destination,
}
}
pub fn run_pass(&self) -> io::Result<ArchivalPassResult> {
let state = self.sidecar.load()?;
let all_segments = list_segment_files(&self.segments_dir)?;
let sealed_segments = match all_segments.len() {
0 | 1 => Vec::new(),
n => all_segments[..n - 1].to_vec(),
};
let new_segments: Vec<SegmentId> = match state.last_archived_segment {
Some(last) => sealed_segments
.into_iter()
.filter(|&id| id > last)
.collect(),
None => sealed_segments,
};
if new_segments.is_empty() {
debug!("no new sealed segments to archive");
return Ok(ArchivalPassResult {
segments_archived: 0,
bytes_archived: 0,
});
}
let mut segments_archived = 0u32;
let mut bytes_archived = 0u64;
let result = new_segments.iter().try_for_each(|&seg_id| {
let path = segment_path(&self.segments_dir, seg_id);
let data = std::fs::read(&path)?;
let size = data.len() as u64;
self.destination.store_segment(seg_id, &data)?;
self.sidecar.save(&ArchivalState {
last_archived_segment: Some(seg_id),
})?;
segments_archived = segments_archived.saturating_add(1);
bytes_archived = bytes_archived.saturating_add(size);
info!(
segment_id = %seg_id,
size_bytes = size,
"archived sealed segment"
);
Ok::<(), io::Error>(())
});
match result {
Ok(()) => {}
Err(e) => {
warn!(
segments_archived,
bytes_archived,
error = %e,
"archival pass interrupted after partial progress"
);
return Err(e);
}
}
Ok(ArchivalPassResult {
segments_archived,
bytes_archived,
})
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::{Arc, Mutex};
type ArchivedSegments = Arc<Mutex<Vec<(SegmentId, Vec<u8>)>>>;
#[derive(Clone)]
struct CollectingDestination {
stored: ArchivedSegments,
}
impl CollectingDestination {
fn new() -> Self {
Self {
stored: Arc::new(Mutex::new(Vec::new())),
}
}
fn stored_ids(&self) -> Vec<SegmentId> {
self.stored
.lock()
.unwrap()
.iter()
.map(|(id, _)| *id)
.collect()
}
}
impl ArchivalDestination for CollectingDestination {
fn store_segment(&self, segment_id: SegmentId, data: &[u8]) -> io::Result<()> {
self.stored
.lock()
.unwrap()
.push((segment_id, data.to_vec()));
Ok(())
}
}
fn create_segment_file(dir: &Path, id: u32, content: &[u8]) {
let path = dir.join(format!("{:08}.{SEGMENT_FILE_EXTENSION}", id));
std::fs::write(path, content).unwrap();
}
#[test]
fn sidecar_round_trip() {
let dir = tempfile::tempdir().unwrap();
let sidecar = ArchivalSidecar::new(dir.path().join("archival.state"));
let state = sidecar.load().unwrap();
assert!(state.last_archived_segment.is_none());
let updated = ArchivalState {
last_archived_segment: Some(SegmentId::new(42)),
};
sidecar.save(&updated).unwrap();
let loaded = sidecar.load().unwrap();
assert_eq!(loaded.last_archived_segment, Some(SegmentId::new(42)));
}
#[test]
fn sidecar_missing_file_returns_empty() {
let dir = tempfile::tempdir().unwrap();
let sidecar = ArchivalSidecar::new(dir.path().join("nonexistent.state"));
let state = sidecar.load().unwrap();
assert!(state.last_archived_segment.is_none());
}
#[test]
fn list_segment_files_sorts_ascending() {
let dir = tempfile::tempdir().unwrap();
create_segment_file(dir.path(), 5, b"e");
create_segment_file(dir.path(), 1, b"a");
create_segment_file(dir.path(), 3, b"c");
std::fs::write(dir.path().join("notes.txt"), b"ignored").unwrap();
let ids = list_segment_files(dir.path()).unwrap();
assert_eq!(
ids,
vec![SegmentId::new(1), SegmentId::new(3), SegmentId::new(5)]
);
}
#[test]
fn list_segment_files_empty_dir() {
let dir = tempfile::tempdir().unwrap();
let ids = list_segment_files(dir.path()).unwrap();
assert!(ids.is_empty());
}
#[test]
fn list_segment_files_missing_dir() {
let ids = list_segment_files(Path::new("/nonexistent/dir")).unwrap();
assert!(ids.is_empty());
}
#[test]
fn no_segments_no_archival() {
let dir = tempfile::tempdir().unwrap();
let seg_dir = dir.path().join("segments");
std::fs::create_dir_all(&seg_dir).unwrap();
let dest = CollectingDestination::new();
let dest_check = dest.clone();
let archiver =
ContinuousArchiver::new(seg_dir, dir.path().join("archival.state"), Box::new(dest));
let result = archiver.run_pass().unwrap();
assert_eq!(result.segments_archived, 0);
assert_eq!(result.bytes_archived, 0);
assert!(dest_check.stored_ids().is_empty());
}
#[test]
fn single_active_segment_not_archived() {
let dir = tempfile::tempdir().unwrap();
let seg_dir = dir.path().join("segments");
std::fs::create_dir_all(&seg_dir).unwrap();
create_segment_file(&seg_dir, 0, b"active segment data");
let dest = CollectingDestination::new();
let dest_check = dest.clone();
let archiver =
ContinuousArchiver::new(seg_dir, dir.path().join("archival.state"), Box::new(dest));
let result = archiver.run_pass().unwrap();
assert_eq!(result.segments_archived, 0);
assert!(dest_check.stored_ids().is_empty());
}
#[test]
fn archives_sealed_segments() {
let dir = tempfile::tempdir().unwrap();
let seg_dir = dir.path().join("segments");
std::fs::create_dir_all(&seg_dir).unwrap();
create_segment_file(&seg_dir, 0, b"sealed-0");
create_segment_file(&seg_dir, 1, b"sealed-1");
create_segment_file(&seg_dir, 2, b"active");
let dest = CollectingDestination::new();
let dest_check = dest.clone();
let archiver =
ContinuousArchiver::new(seg_dir, dir.path().join("archival.state"), Box::new(dest));
let result = archiver.run_pass().unwrap();
assert_eq!(result.segments_archived, 2);
assert_eq!(result.bytes_archived, 16);
let stored = dest_check.stored_ids();
assert_eq!(stored, vec![SegmentId::new(0), SegmentId::new(1)]);
}
#[test]
fn incremental_archival_skips_already_archived() {
let dir = tempfile::tempdir().unwrap();
let seg_dir = dir.path().join("segments");
std::fs::create_dir_all(&seg_dir).unwrap();
create_segment_file(&seg_dir, 0, b"sealed-0");
create_segment_file(&seg_dir, 1, b"sealed-1");
create_segment_file(&seg_dir, 2, b"sealed-2");
create_segment_file(&seg_dir, 3, b"active");
let sidecar_path = dir.path().join("archival.state");
ArchivalSidecar::new(sidecar_path.clone())
.save(&ArchivalState {
last_archived_segment: Some(SegmentId::new(0)),
})
.unwrap();
let dest = CollectingDestination::new();
let dest_check = dest.clone();
let archiver = ContinuousArchiver::new(seg_dir, sidecar_path.clone(), Box::new(dest));
let result = archiver.run_pass().unwrap();
assert_eq!(result.segments_archived, 2);
let stored = dest_check.stored_ids();
assert_eq!(stored, vec![SegmentId::new(1), SegmentId::new(2)]);
let final_state = ArchivalSidecar::new(sidecar_path).load().unwrap();
assert_eq!(final_state.last_archived_segment, Some(SegmentId::new(2)));
}
#[test]
fn sidecar_updated_per_segment_for_crash_safety() {
let dir = tempfile::tempdir().unwrap();
let seg_dir = dir.path().join("segments");
std::fs::create_dir_all(&seg_dir).unwrap();
create_segment_file(&seg_dir, 0, b"sealed-0");
create_segment_file(&seg_dir, 1, b"sealed-1");
create_segment_file(&seg_dir, 2, b"active");
struct FailOnSecondDestination {
call_count: Mutex<u32>,
}
impl ArchivalDestination for FailOnSecondDestination {
fn store_segment(&self, _id: SegmentId, _data: &[u8]) -> io::Result<()> {
let mut count = self.call_count.lock().unwrap();
*count += 1;
match *count {
1 => Ok(()),
_ => Err(io::Error::other("simulated failure")),
}
}
}
let sidecar_path = dir.path().join("archival.state");
let archiver = ContinuousArchiver::new(
seg_dir,
sidecar_path.clone(),
Box::new(FailOnSecondDestination {
call_count: Mutex::new(0),
}),
);
let err = archiver.run_pass().unwrap_err();
assert_eq!(err.kind(), io::ErrorKind::Other);
let state = ArchivalSidecar::new(sidecar_path).load().unwrap();
assert_eq!(state.last_archived_segment, Some(SegmentId::new(0)));
}
#[test]
fn idempotent_rerun_after_full_archival() {
let dir = tempfile::tempdir().unwrap();
let seg_dir = dir.path().join("segments");
std::fs::create_dir_all(&seg_dir).unwrap();
create_segment_file(&seg_dir, 0, b"sealed-0");
create_segment_file(&seg_dir, 1, b"sealed-1");
create_segment_file(&seg_dir, 2, b"active");
let sidecar_path = dir.path().join("archival.state");
let dest1 = CollectingDestination::new();
let dest1_check = dest1.clone();
let archiver1 =
ContinuousArchiver::new(seg_dir.clone(), sidecar_path.clone(), Box::new(dest1));
archiver1.run_pass().unwrap();
assert_eq!(dest1_check.stored_ids().len(), 2);
let dest2 = CollectingDestination::new();
let dest2_check = dest2.clone();
let archiver2 = ContinuousArchiver::new(seg_dir, sidecar_path, Box::new(dest2));
let result = archiver2.run_pass().unwrap();
assert_eq!(result.segments_archived, 0);
assert!(dest2_check.stored_ids().is_empty());
}
#[test]
fn new_segments_after_initial_archival() {
let dir = tempfile::tempdir().unwrap();
let seg_dir = dir.path().join("segments");
std::fs::create_dir_all(&seg_dir).unwrap();
create_segment_file(&seg_dir, 0, b"sealed-0");
create_segment_file(&seg_dir, 1, b"active");
let sidecar_path = dir.path().join("archival.state");
let dest1 = CollectingDestination::new();
let archiver1 =
ContinuousArchiver::new(seg_dir.clone(), sidecar_path.clone(), Box::new(dest1));
let r1 = archiver1.run_pass().unwrap();
assert_eq!(r1.segments_archived, 1);
create_segment_file(&seg_dir, 2, b"new-active");
let dest2 = CollectingDestination::new();
let dest2_check = dest2.clone();
let archiver2 = ContinuousArchiver::new(seg_dir, sidecar_path, Box::new(dest2));
let r2 = archiver2.run_pass().unwrap();
assert_eq!(r2.segments_archived, 1);
assert_eq!(dest2_check.stored_ids(), vec![SegmentId::new(1)]);
}
#[test]
fn local_destination_writes_files() {
let dir = tempfile::tempdir().unwrap();
let dest_dir = dir.path().join("archive");
let dest = LocalArchivalDestination::new(dest_dir.clone()).unwrap();
let payload = b"segment data here";
dest.store_segment(SegmentId::new(5), payload).unwrap();
let written =
std::fs::read(dest_dir.join(format!("00000005.{SEGMENT_FILE_EXTENSION}"))).unwrap();
assert_eq!(written, payload);
}
#[test]
fn local_destination_atomic_overwrite() {
let dir = tempfile::tempdir().unwrap();
let dest_dir = dir.path().join("archive");
let dest = LocalArchivalDestination::new(dest_dir.clone()).unwrap();
dest.store_segment(SegmentId::new(1), b"first").unwrap();
dest.store_segment(SegmentId::new(1), b"second").unwrap();
let written =
std::fs::read(dest_dir.join(format!("00000001.{SEGMENT_FILE_EXTENSION}"))).unwrap();
assert_eq!(written, b"second");
assert!(
!dest_dir
.join(format!("00000001.{SEGMENT_FILE_EXTENSION}.tmp"))
.exists()
);
}
#[test]
fn archived_data_matches_source() {
let dir = tempfile::tempdir().unwrap();
let seg_dir = dir.path().join("segments");
std::fs::create_dir_all(&seg_dir).unwrap();
let content_0 = b"sealed segment zero content with some bulk data";
let content_1 = b"sealed segment one with different content";
create_segment_file(&seg_dir, 0, content_0);
create_segment_file(&seg_dir, 1, content_1);
create_segment_file(&seg_dir, 2, b"active");
let dest = CollectingDestination::new();
let dest_check = dest.clone();
let archiver =
ContinuousArchiver::new(seg_dir, dir.path().join("archival.state"), Box::new(dest));
archiver.run_pass().unwrap();
let stored = dest_check.stored.lock().unwrap();
assert_eq!(stored[0].1, content_0);
assert_eq!(stored[1].1, content_1);
}
}
crates/tranquil-store/src/backup.rs
+1756
View File
File diff suppressed because it is too large Load Diff
crates/tranquil-store/src/blockstore/compaction.rs
+212
View File
@@ -0,0 +1,212 @@
use std::io;
use crate::io::{FileId, OpenOptions, StorageIO};
use super::data_file::{DataFileReader, DataFileWriter, ReadBlockRecord};
use super::group_commit::{ActiveFileSet, FileIdAllocator};
use super::hash_index::{BlockIndex, BlockIndexError};
use super::hint::{HintFileWriter, hint_file_path};
use super::manager::DataFileManager;
use super::types::{BlockLocation, CidBytes, CommitEpoch, CompactionResult, DataFileId};
#[derive(Debug)]
pub enum CompactionError {
Io(io::Error),
Index(BlockIndexError),
ChannelClosed,
ActiveFileCannotBeCompacted,
}
impl std::fmt::Display for CompactionError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Io(e) => write!(f, "io: {e}"),
Self::Index(e) => write!(f, "index: {e}"),
Self::ChannelClosed => write!(f, "commit channel closed"),
Self::ActiveFileCannotBeCompacted => {
write!(f, "cannot compact the active data file")
}
}
}
}
impl std::error::Error for CompactionError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
Self::Io(e) => Some(e),
Self::Index(e) => Some(e),
Self::ChannelClosed | Self::ActiveFileCannotBeCompacted => None,
}
}
}
impl From<io::Error> for CompactionError {
fn from(e: io::Error) -> Self {
Self::Io(e)
}
}
impl From<BlockIndexError> for CompactionError {
fn from(e: BlockIndexError) -> Self {
Self::Index(e)
}
}
#[allow(clippy::too_many_arguments)]
pub(super) fn compact_on_writer_thread<S: StorageIO>(
manager: &DataFileManager<S>,
index: &BlockIndex,
source_file_id: DataFileId,
current_epoch: CommitEpoch,
grace_period_ms: u64,
file_ids: &FileIdAllocator,
active_files: &ActiveFileSet,
hint_positions: &super::group_commit::ShardHintPositions,
epoch: &super::types::EpochCounter,
) -> Result<CompactionResult, CompactionError> {
if active_files.contains(source_file_id) {
return Err(CompactionError::ActiveFileCannotBeCompacted);
}
let source_fd = manager.open_for_read(source_file_id)?;
let source_size = manager.io().file_size(source_fd)?;
let new_file_id = file_ids.allocate();
let result = stream_compact(
manager,
index,
source_file_id,
source_fd,
new_file_id,
current_epoch,
grace_period_ms,
);
match result {
Err(e) => {
manager.delete_data_file(new_file_id).ok();
manager
.io()
.delete(&hint_file_path(manager.data_dir(), new_file_id))
.ok();
Err(e)
}
Ok((new_size, live_count, dead_count)) => {
let positions = hint_positions.snapshot();
if let Err(e) = index.write_checkpoint(epoch.current(), &positions) {
tracing::warn!(error = %e, "pre-delete checkpoint failed during compaction");
}
manager.delete_data_file(source_file_id)?;
manager
.io()
.delete(&hint_file_path(manager.data_dir(), source_file_id))
.ok();
manager.io().sync_dir(manager.data_dir())?;
let reclaimed_bytes = source_size.saturating_sub(new_size);
tracing::info!(
source = %source_file_id,
dest = %new_file_id,
old_size = source_size,
new_size,
live_count,
dead_count,
reclaimed_bytes,
"compaction complete"
);
Ok(CompactionResult {
file_id: source_file_id,
old_size: source_size,
new_size,
live_blocks: live_count,
dead_blocks: dead_count,
reclaimed_bytes,
})
}
}
}
fn stream_compact<S: StorageIO>(
manager: &DataFileManager<S>,
index: &BlockIndex,
source_file_id: DataFileId,
source_fd: FileId,
new_file_id: DataFileId,
current_epoch: CommitEpoch,
grace_period_ms: u64,
) -> Result<(u64, u64, u64), CompactionError> {
let mut reader = DataFileReader::open(manager.io(), source_fd)?;
let now = crate::wall_clock_ms();
let new_fd = manager.open_for_append(new_file_id)?;
let mut writer = DataFileWriter::new(manager.io(), new_fd, new_file_id)?;
let hint_path = hint_file_path(manager.data_dir(), new_file_id);
let hint_fd = manager.io().open(&hint_path, OpenOptions::read_write())?;
let mut hint_writer = HintFileWriter::new(manager.io(), hint_fd);
let mut relocations: Vec<(CidBytes, BlockLocation)> = Vec::new();
let mut dead_cids: Vec<CidBytes> = Vec::new();
let mut live_count: u64 = 0;
let mut dead_count: u64 = 0;
reader.try_for_each(|r| {
let record = r?;
match record {
ReadBlockRecord::Valid {
cid_bytes, data, ..
} => match index.get(&cid_bytes) {
Some(e) if e.location.file_id == source_file_id && !e.refcount.is_zero() => {
let loc = writer.append_block(&cid_bytes, &data)?;
hint_writer.append_relocate(&cid_bytes, loc.file_id, loc.offset, loc.length)?;
relocations.push((cid_bytes, loc));
live_count = live_count.saturating_add(1);
}
Some(e) if e.location.file_id == source_file_id && e.refcount.is_zero() => {
let eligible =
index.is_gc_eligible(&cid_bytes, current_epoch, now, grace_period_ms);
match eligible {
true => {
tracing::debug!(
?cid_bytes,
file_id = %source_file_id,
"gc: collecting dead block"
);
hint_writer.append_remove(&cid_bytes)?;
dead_cids.push(cid_bytes);
dead_count = dead_count.saturating_add(1);
}
false => {
let loc = writer.append_block(&cid_bytes, &data)?;
hint_writer.append_relocate(
&cid_bytes,
loc.file_id,
loc.offset,
loc.length,
)?;
relocations.push((cid_bytes, loc));
live_count = live_count.saturating_add(1);
}
}
}
_ => {}
},
ReadBlockRecord::Corrupted { .. } | ReadBlockRecord::Truncated { .. } => {}
}
Ok::<_, CompactionError>(())
})?;
writer.sync()?;
hint_writer.sync()?;
manager.io().sync_dir(manager.data_dir())?;
let new_size = writer.position().raw();
index.apply_compaction(&relocations, &dead_cids);
Ok((new_size, live_count, dead_count))
}
crates/tranquil-store/src/blockstore/group_commit.rs
+829 -553
View File
File diff suppressed because it is too large Load Diff
crates/tranquil-store/src/blockstore/hash_index.rs
+2051
View File
File diff suppressed because it is too large Load Diff
crates/tranquil-store/src/blockstore/hint.rs
+611 -486
View File
File diff suppressed because it is too large Load Diff
crates/tranquil-store/src/blockstore/key_index.rs
-539
View File
@@ -1,539 +0,0 @@
use std::collections::HashMap;
use std::path::Path;
use fjall::{
Database, Keyspace, KeyspaceCreateOptions, PersistMode,
config::{BloomConstructionPolicy, FilterPolicy, FilterPolicyEntry},
};
use super::data_file::CID_SIZE;
use super::types::{BlockLocation, IndexEntry, RefCount, WriteCursor};
const WRITE_CURSOR_KEY: &[u8] = b"\x00write_cursor";
const KEYSPACE_NAME: &str = "blocks";
fn bloom_options() -> KeyspaceCreateOptions {
KeyspaceCreateOptions::default().filter_policy(FilterPolicy::new([
FilterPolicyEntry::Bloom(BloomConstructionPolicy::FalsePositiveRate(0.01)),
FilterPolicyEntry::Bloom(BloomConstructionPolicy::FalsePositiveRate(0.01)),
]))
}
fn is_corruption_error(e: &fjall::Error) -> bool {
match e {
fjall::Error::Io(io_err) => matches!(
io_err.kind(),
std::io::ErrorKind::InvalidData | std::io::ErrorKind::UnexpectedEof
),
fjall::Error::Locked | fjall::Error::KeyspaceDeleted => false,
_ => true,
}
}
fn serialize_entry(entry: &IndexEntry) -> Vec<u8> {
postcard::to_allocvec(entry)
.expect("IndexEntry serialization is infallible for fixed-layout types")
}
fn deserialize_entry(bytes: &[u8]) -> Result<IndexEntry, KeyIndexError> {
postcard::from_bytes(bytes).map_err(KeyIndexError::Deserialize)
}
fn serialize_cursor(cursor: &WriteCursor) -> Vec<u8> {
postcard::to_allocvec(cursor)
.expect("WriteCursor serialization is infallible for fixed-layout types")
}
fn deserialize_cursor(bytes: &[u8]) -> Result<WriteCursor, KeyIndexError> {
postcard::from_bytes(bytes).map_err(KeyIndexError::Deserialize)
}
#[derive(Debug)]
pub enum KeyIndexError {
Fjall(fjall::Error),
Deserialize(postcard::Error),
MissingEntry,
}
impl std::fmt::Display for KeyIndexError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Fjall(e) => write!(f, "fjall: {e}"),
Self::Deserialize(e) => write!(f, "deserialize: {e}"),
Self::MissingEntry => write!(f, "entry not found"),
}
}
}
impl std::error::Error for KeyIndexError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
Self::Fjall(e) => Some(e),
Self::Deserialize(e) => Some(e),
Self::MissingEntry => None,
}
}
}
impl From<fjall::Error> for KeyIndexError {
fn from(e: fjall::Error) -> Self {
Self::Fjall(e)
}
}
pub enum KeyIndexOpenOutcome {
Opened(KeyIndex),
NeedsRebuild(KeyIndex),
}
impl KeyIndexOpenOutcome {
pub fn into_inner(self) -> KeyIndex {
match self {
Self::Opened(idx) | Self::NeedsRebuild(idx) => idx,
}
}
pub fn needs_rebuild(&self) -> bool {
matches!(self, Self::NeedsRebuild(_))
}
}
pub struct KeyIndex {
db: Database,
blocks: Keyspace,
}
impl KeyIndex {
pub fn open(path: &Path) -> Result<KeyIndexOpenOutcome, KeyIndexError> {
match Self::try_open(path) {
Ok(idx) => Ok(KeyIndexOpenOutcome::Opened(idx)),
Err(KeyIndexError::Fjall(ref e)) if is_corruption_error(e) => {
let _ = std::fs::remove_dir_all(path);
let idx = Self::try_open(path)?;
Ok(KeyIndexOpenOutcome::NeedsRebuild(idx))
}
Err(e) => Err(e),
}
}
fn try_open(path: &Path) -> Result<Self, KeyIndexError> {
let db = Database::builder(path).open()?;
let blocks = db.keyspace(KEYSPACE_NAME, bloom_options)?;
Ok(Self { db, blocks })
}
pub fn get(&self, cid_bytes: &[u8; CID_SIZE]) -> Result<Option<IndexEntry>, KeyIndexError> {
self.blocks
.get(cid_bytes)?
.map(|v| deserialize_entry(&v))
.transpose()
}
pub fn has(&self, cid_bytes: &[u8; CID_SIZE]) -> Result<bool, KeyIndexError> {
self.blocks.contains_key(cid_bytes).map_err(Into::into)
}
pub fn put(
&self,
cid_bytes: &[u8; CID_SIZE],
location: BlockLocation,
) -> Result<(), KeyIndexError> {
let entry = match self.get(cid_bytes)? {
Some(existing) => IndexEntry {
location: existing.location,
refcount: existing.refcount.increment(),
},
None => IndexEntry {
location,
refcount: RefCount::one(),
},
};
self.blocks
.insert(cid_bytes, serialize_entry(&entry))
.map_err(Into::into)
}
pub fn decrement_refcount(
&self,
cid_bytes: &[u8; CID_SIZE],
) -> Result<RefCount, KeyIndexError> {
let existing = self.get(cid_bytes)?.ok_or(KeyIndexError::MissingEntry)?;
let new_refcount = match existing.refcount.is_zero() {
true => {
tracing::warn!(?cid_bytes, "decrement on zero-refcount entry, skipping");
existing.refcount
}
false => existing.refcount.decrement(),
};
let updated = IndexEntry {
location: existing.location,
refcount: new_refcount,
};
self.blocks.insert(cid_bytes, serialize_entry(&updated))?;
Ok(new_refcount)
}
pub fn batch_put(
&self,
entries: &[([u8; CID_SIZE], BlockLocation)],
decrements: &[[u8; CID_SIZE]],
cursor: WriteCursor,
) -> Result<(), KeyIndexError> {
let mut batch = self.db.batch().durability(Some(PersistMode::SyncData));
let mut pending: HashMap<[u8; CID_SIZE], IndexEntry> = HashMap::new();
entries.iter().try_for_each(|(cid_bytes, location)| {
let entry = match pending.get(cid_bytes).copied().or(self.get(cid_bytes)?) {
Some(existing) => IndexEntry {
location: existing.location,
refcount: existing.refcount.increment(),
},
None => IndexEntry {
location: *location,
refcount: RefCount::one(),
},
};
pending.insert(*cid_bytes, entry);
batch.insert(&self.blocks, cid_bytes.as_slice(), serialize_entry(&entry));
Ok::<_, KeyIndexError>(())
})?;
decrements.iter().try_for_each(|cid_bytes| {
let existing = pending
.get(cid_bytes)
.copied()
.or(self.get(cid_bytes)?)
.ok_or(KeyIndexError::MissingEntry)?;
let new_refcount = match existing.refcount.is_zero() {
true => {
tracing::warn!(?cid_bytes, "decrement on zero-refcount entry, skipping");
existing.refcount
}
false => existing.refcount.decrement(),
};
let updated = IndexEntry {
location: existing.location,
refcount: new_refcount,
};
pending.insert(*cid_bytes, updated);
batch.insert(
&self.blocks,
cid_bytes.as_slice(),
serialize_entry(&updated),
);
Ok::<_, KeyIndexError>(())
})?;
batch.insert(&self.blocks, WRITE_CURSOR_KEY, serialize_cursor(&cursor));
batch.commit().map_err(Into::into)
}
pub fn read_write_cursor(&self) -> Result<Option<WriteCursor>, KeyIndexError> {
self.blocks
.get(WRITE_CURSOR_KEY)?
.map(|v| deserialize_cursor(&v))
.transpose()
}
pub fn persist(&self) -> Result<(), KeyIndexError> {
self.db.persist(PersistMode::SyncData).map_err(Into::into)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::blockstore::test_cid;
use crate::blockstore::types::{BlockLength, BlockOffset, DataFileId};
fn test_location(file_id: u32, offset: u64, length: u32) -> BlockLocation {
BlockLocation {
file_id: DataFileId::new(file_id),
offset: BlockOffset::new(offset),
length: BlockLength::new(length),
}
}
fn open_temp() -> (tempfile::TempDir, KeyIndex) {
let dir = tempfile::TempDir::new().unwrap();
let outcome = KeyIndex::open(dir.path()).unwrap();
assert!(!outcome.needs_rebuild());
(dir, outcome.into_inner())
}
#[test]
fn put_then_get_round_trips() {
let (_dir, idx) = open_temp();
let cid = test_cid(1);
let loc = test_location(0, 100, 256);
idx.put(&cid, loc).unwrap();
let entry = idx.get(&cid).unwrap().unwrap();
assert_eq!(entry.location, loc);
assert_eq!(entry.refcount, RefCount::one());
}
#[test]
fn get_missing_returns_none() {
let (_dir, idx) = open_temp();
assert!(idx.get(&test_cid(42)).unwrap().is_none());
}
#[test]
fn has_missing_returns_false() {
let (_dir, idx) = open_temp();
assert!(!idx.has(&test_cid(42)).unwrap());
}
#[test]
fn has_existing_returns_true() {
let (_dir, idx) = open_temp();
let cid = test_cid(1);
idx.put(&cid, test_location(0, 0, 10)).unwrap();
assert!(idx.has(&cid).unwrap());
}
#[test]
fn duplicate_put_increments_refcount() {
let (_dir, idx) = open_temp();
let cid = test_cid(1);
let loc = test_location(0, 100, 256);
idx.put(&cid, loc).unwrap();
idx.put(&cid, test_location(1, 200, 512)).unwrap();
let entry = idx.get(&cid).unwrap().unwrap();
assert_eq!(entry.refcount, RefCount::new(2));
assert_eq!(entry.location, loc);
}
#[test]
fn decrement_refcount_from_two_to_one() {
let (_dir, idx) = open_temp();
let cid = test_cid(1);
idx.put(&cid, test_location(0, 0, 10)).unwrap();
idx.put(&cid, test_location(0, 0, 10)).unwrap();
let rc = idx.decrement_refcount(&cid).unwrap();
assert_eq!(rc, RefCount::one());
let entry = idx.get(&cid).unwrap().unwrap();
assert_eq!(entry.refcount, RefCount::one());
}
#[test]
fn decrement_refcount_to_zero_keeps_entry() {
let (_dir, idx) = open_temp();
let cid = test_cid(1);
idx.put(&cid, test_location(0, 0, 10)).unwrap();
let rc = idx.decrement_refcount(&cid).unwrap();
assert!(rc.is_zero());
let entry = idx.get(&cid).unwrap().unwrap();
assert!(entry.refcount.is_zero());
}
#[test]
fn decrement_missing_entry_errors() {
let (_dir, idx) = open_temp();
let result = idx.decrement_refcount(&test_cid(99));
assert!(matches!(result, Err(KeyIndexError::MissingEntry)));
}
#[test]
fn batch_put_new_entries() {
let (_dir, idx) = open_temp();
let entries: Vec<_> = (0u8..3)
.map(|i| (test_cid(i), test_location(0, i as u64 * 100, 50)))
.collect();
let cursor = WriteCursor {
file_id: DataFileId::new(0),
offset: BlockOffset::new(300),
};
idx.batch_put(&entries, &[], cursor).unwrap();
entries.iter().for_each(|(cid, loc)| {
let entry = idx.get(cid).unwrap().unwrap();
assert_eq!(entry.location, *loc);
assert_eq!(entry.refcount, RefCount::one());
});
}
#[test]
fn batch_put_increments_existing() {
let (_dir, idx) = open_temp();
let cid = test_cid(1);
let original_loc = test_location(0, 100, 50);
idx.put(&cid, original_loc).unwrap();
let entries = vec![(cid, test_location(1, 200, 60))];
let cursor = WriteCursor {
file_id: DataFileId::new(1),
offset: BlockOffset::new(260),
};
idx.batch_put(&entries, &[], cursor).unwrap();
let entry = idx.get(&cid).unwrap().unwrap();
assert_eq!(entry.refcount, RefCount::new(2));
assert_eq!(entry.location, original_loc);
}
#[test]
fn batch_put_with_decrements() {
let (_dir, idx) = open_temp();
let cid_a = test_cid(1);
let cid_b = test_cid(2);
idx.put(&cid_b, test_location(0, 0, 10)).unwrap();
idx.put(&cid_b, test_location(0, 0, 10)).unwrap();
let entries = vec![(cid_a, test_location(0, 100, 50))];
let decrements = vec![cid_b];
let cursor = WriteCursor {
file_id: DataFileId::new(0),
offset: BlockOffset::new(150),
};
idx.batch_put(&entries, &decrements, cursor).unwrap();
let a = idx.get(&cid_a).unwrap().unwrap();
assert_eq!(a.refcount, RefCount::one());
let b = idx.get(&cid_b).unwrap().unwrap();
assert_eq!(b.refcount, RefCount::one());
}
#[test]
fn batch_put_mixed_new_and_duplicate() {
let (_dir, idx) = open_temp();
let existing_cid = test_cid(1);
let existing_loc = test_location(0, 0, 10);
idx.put(&existing_cid, existing_loc).unwrap();
let entries: Vec<_> = (1u8..=4)
.map(|i| (test_cid(i), test_location(0, i as u64 * 100, 50)))
.collect();
let cursor = WriteCursor {
file_id: DataFileId::new(0),
offset: BlockOffset::new(500),
};
idx.batch_put(&entries, &[], cursor).unwrap();
let existing = idx.get(&existing_cid).unwrap().unwrap();
assert_eq!(existing.refcount, RefCount::new(2));
assert_eq!(existing.location, existing_loc);
(2u8..=4).for_each(|i| {
let entry = idx.get(&test_cid(i)).unwrap().unwrap();
assert_eq!(entry.refcount, RefCount::one());
});
}
#[test]
fn batch_put_duplicate_cid_in_same_batch() {
let (_dir, idx) = open_temp();
let cid = test_cid(1);
let loc = test_location(0, 100, 50);
let entries = vec![(cid, loc), (cid, test_location(0, 200, 60))];
let cursor = WriteCursor {
file_id: DataFileId::new(0),
offset: BlockOffset::new(260),
};
idx.batch_put(&entries, &[], cursor).unwrap();
let entry = idx.get(&cid).unwrap().unwrap();
assert_eq!(entry.refcount, RefCount::new(2));
assert_eq!(entry.location, loc);
}
#[test]
fn batch_put_entry_then_decrement_same_cid() {
let (_dir, idx) = open_temp();
let cid = test_cid(1);
let loc = test_location(0, 100, 50);
let entries = vec![(cid, loc)];
let decrements = vec![cid];
let cursor = WriteCursor {
file_id: DataFileId::new(0),
offset: BlockOffset::new(150),
};
idx.batch_put(&entries, &decrements, cursor).unwrap();
let entry = idx.get(&cid).unwrap().unwrap();
assert!(entry.refcount.is_zero());
}
#[test]
fn write_cursor_round_trip() {
let (_dir, idx) = open_temp();
assert!(idx.read_write_cursor().unwrap().is_none());
let cursor = WriteCursor {
file_id: DataFileId::new(3),
offset: BlockOffset::new(65536),
};
let entries = vec![(test_cid(1), test_location(3, 0, 100))];
idx.batch_put(&entries, &[], cursor).unwrap();
let read_back = idx.read_write_cursor().unwrap().unwrap();
assert_eq!(read_back, cursor);
}
#[test]
fn write_cursor_persists_across_reopen() {
let dir = tempfile::TempDir::new().unwrap();
let cursor = WriteCursor {
file_id: DataFileId::new(7),
offset: BlockOffset::new(99999),
};
{
let idx = KeyIndex::open(dir.path()).unwrap().into_inner();
let entries = vec![(test_cid(1), test_location(7, 0, 100))];
idx.batch_put(&entries, &[], cursor).unwrap();
idx.persist().unwrap();
}
{
let idx = KeyIndex::open(dir.path()).unwrap().into_inner();
let read_back = idx.read_write_cursor().unwrap().unwrap();
assert_eq!(read_back, cursor);
let entry = idx.get(&test_cid(1)).unwrap().unwrap();
assert_eq!(entry.refcount, RefCount::one());
}
}
#[test]
fn corrupt_index_triggers_needs_rebuild() {
let dir = tempfile::TempDir::new().unwrap();
{
let idx = KeyIndex::open(dir.path()).unwrap().into_inner();
idx.put(&test_cid(1), test_location(0, 0, 10)).unwrap();
idx.persist().unwrap();
}
std::fs::read_dir(dir.path())
.unwrap()
.filter_map(|e| e.ok())
.for_each(|entry| {
let path = entry.path();
if path.is_file() {
std::fs::write(&path, b"corrupted").unwrap();
}
});
let outcome = KeyIndex::open(dir.path()).unwrap();
assert!(outcome.needs_rebuild());
let idx = outcome.into_inner();
assert!(idx.get(&test_cid(1)).unwrap().is_none());
assert!(idx.read_write_cursor().unwrap().is_none());
}
}
crates/tranquil-store/src/blockstore/manager.rs
+21
View File
@@ -136,6 +136,27 @@ impl<S: StorageIO> DataFileManager<S> {
pub fn list_files(&self) -> io::Result<Vec<DataFileId>> {
list_files_by_extension(&self.io, &self.data_dir, DATA_FILE_EXTENSION)
}
pub fn evict_handle(&self, file_id: DataFileId) {
let removed = self.handles.write().remove(&file_id);
if let Some(entry) = removed {
let _ = self.io.close(entry.fd);
}
}
pub fn delete_data_file(&self, file_id: DataFileId) -> io::Result<()> {
self.evict_handle(file_id);
let path = self.data_file_path(file_id);
self.io.delete(&path)
}
}
impl<S: StorageIO> Drop for DataFileManager<S> {
fn drop(&mut self) {
self.handles.write().drain().for_each(|(_, entry)| {
let _ = self.io.close(entry.fd);
});
}
}
#[cfg(test)]
crates/tranquil-store/src/blockstore/mod.rs
+15 -10
View File
@@ -1,30 +1,35 @@
mod compaction;
mod data_file;
mod group_commit;
pub mod hash_index;
mod hint;
mod key_index;
mod manager;
mod reader;
mod store;
mod types;
pub use compaction::CompactionError;
pub use data_file::{
BLOCK_FORMAT_VERSION, BLOCK_HEADER_SIZE, BLOCK_MAGIC, BLOCK_RECORD_OVERHEAD, CID_SIZE,
DataFileReader, DataFileWriter, ReadBlockRecord, ValidBlock, decode_block_record,
encode_block_record,
};
pub use group_commit::{CommitError, CommitRequest, GroupCommitConfig, GroupCommitWriter};
pub use group_commit::{
ActiveFileSet, CommitError, CommitRequest, FileIdAllocator, GroupCommitConfig,
GroupCommitWriter, ShardHintPositions,
};
pub use hint::{
HINT_FILE_EXTENSION, HINT_RECORD_SIZE, HintFileReader, HintFileWriter, ReadHintRecord,
RebuildError, decode_hint_record, hint_file_path, rebuild_index_from_data_files,
rebuild_index_from_hints,
HINT_FILE_EXTENSION, HINT_RECORD_SIZE, HintFileReader, HintFileWriter, HintIndex,
ReadHintRecord, RebuildError, decode_hint_record, hint_file_path, scan_hints_to_memory,
};
pub use key_index::{KeyIndex, KeyIndexError, KeyIndexOpenOutcome};
pub use manager::{DEFAULT_MAX_FILE_SIZE, DataFileManager};
pub use reader::{BlockStoreReader, ReadError};
pub use store::{BlockStoreConfig, TranquilBlockStore};
pub use store::QuiesceGuard;
pub use store::{BlockStoreConfig, DEFAULT_SHARD_COUNT, TranquilBlockStore};
pub use types::{
BlockLength, BlockLocation, BlockOffset, DataFileId, HintOffset, IndexEntry, MAX_BLOCK_SIZE,
RefCount, WriteCursor,
BlockLength, BlockLocation, BlockOffset, BlockstoreSnapshot, CidBytes, CollectionResult,
CommitEpoch, CompactionResult, DataFileId, EpochCounter, HintOffset, IndexEntry, LivenessInfo,
MAX_BLOCK_SIZE, RefCount, ShardId, WallClockMs, WriteCursor,
};
use std::io;
@@ -40,7 +45,7 @@ impl BlocksSynced {
}
}
pub(crate) fn list_files_by_extension<S: StorageIO>(
pub fn list_files_by_extension<S: StorageIO>(
io: &S,
dir: &Path,
extension: &str,
crates/tranquil-store/src/blockstore/reader.rs
+26 -485
View File
@@ -7,14 +7,13 @@ use bytes::Bytes;
use crate::io::{FileId, StorageIO};
use super::data_file::{CID_SIZE, ReadBlockRecord, decode_block_record};
use super::key_index::{KeyIndex, KeyIndexError};
use super::hash_index::BlockIndex;
use super::manager::DataFileManager;
use super::types::{BlockLocation, BlockOffset, DataFileId};
#[derive(Debug, Clone)]
pub enum ReadError {
Io(Arc<io::Error>),
Index(Arc<KeyIndexError>),
Corrupted {
file_id: DataFileId,
offset: BlockOffset,
@@ -25,7 +24,6 @@ impl std::fmt::Display for ReadError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Io(e) => write!(f, "io: {e}"),
Self::Index(e) => write!(f, "index: {e}"),
Self::Corrupted { file_id, offset } => {
write!(f, "corrupted block at {file_id}:{}", offset.raw())
}
@@ -37,7 +35,6 @@ impl std::error::Error for ReadError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
Self::Io(e) => Some(e.as_ref()),
Self::Index(e) => Some(e.as_ref()),
Self::Corrupted { .. } => None,
}
}
@@ -49,14 +46,8 @@ impl From<io::Error> for ReadError {
}
}
impl From<KeyIndexError> for ReadError {
fn from(e: KeyIndexError) -> Self {
Self::Index(Arc::new(e))
}
}
pub struct BlockStoreReader<S: StorageIO> {
index: Arc<KeyIndex>,
index: Arc<BlockIndex>,
manager: Arc<DataFileManager<S>>,
}
@@ -70,54 +61,58 @@ impl<S: StorageIO> Clone for BlockStoreReader<S> {
}
impl<S: StorageIO> BlockStoreReader<S> {
pub fn new(index: Arc<KeyIndex>, manager: Arc<DataFileManager<S>>) -> Self {
pub fn new(index: Arc<BlockIndex>, manager: Arc<DataFileManager<S>>) -> Self {
Self { index, manager }
}
pub fn manager(&self) -> &DataFileManager<S> {
&self.manager
}
pub fn get(&self, cid: &[u8; CID_SIZE]) -> Result<Option<Bytes>, ReadError> {
let entry = match self.index.get(cid)? {
Some(e) => e,
None => return Ok(None),
};
self.read_block_at(entry.location).map(Some)
match self.index.get(cid) {
Some(e) => self.read_block_at(e.location).map(Some),
None => Ok(None),
}
}
pub fn has(&self, cid: &[u8; CID_SIZE]) -> Result<bool, ReadError> {
self.index.has(cid).map_err(ReadError::from)
Ok(self.index.has(cid))
}
pub fn get_many(&self, cids: &[[u8; CID_SIZE]]) -> Result<Vec<Option<Bytes>>, ReadError> {
let mut results: Vec<Option<Bytes>> = vec![None; cids.len()];
let lookups: Vec<(usize, BlockLocation)> = cids
let index_lookups: Vec<(usize, BlockLocation)> = cids
.iter()
.enumerate()
.filter_map(|(i, cid)| match self.index.get(cid) {
Ok(Some(entry)) => Some(Ok((i, entry.location))),
Ok(None) => None,
Err(e) => Some(Err(ReadError::from(e))),
})
.collect::<Result<Vec<_>, _>>()?;
.filter_map(|(i, cid)| self.index.get(cid).map(|entry| (i, entry.location)))
.collect();
self.read_locations_into(&index_lookups, &mut results)?;
Ok(results)
}
fn read_locations_into(
&self,
lookups: &[(usize, BlockLocation)],
results: &mut [Option<Bytes>],
) -> Result<(), ReadError> {
let mut by_file: HashMap<DataFileId, Vec<(usize, BlockLocation)>> = HashMap::new();
lookups.into_iter().for_each(|(idx, loc)| {
lookups.iter().for_each(|&(idx, loc)| {
by_file.entry(loc.file_id).or_default().push((idx, loc));
});
by_file.into_iter().try_for_each(|(file_id, mut entries)| {
let fd = self.manager.open_for_read(file_id)?;
let file_size = self.manager.io().file_size(fd)?;
entries.sort_by_key(|(_, loc)| loc.offset);
entries.into_iter().try_for_each(|(orig_idx, loc)| {
let data = self.decode_and_validate(fd, file_size, loc)?;
results[orig_idx] = Some(data);
Ok::<_, ReadError>(())
})
})?;
Ok(results)
})
}
fn read_block_at(&self, location: BlockLocation) -> Result<Bytes, ReadError> {
@@ -155,457 +150,3 @@ impl<S: StorageIO> BlockStoreReader<S> {
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::RealIO;
use crate::blockstore::data_file::CID_SIZE;
use crate::blockstore::group_commit::{CommitRequest, GroupCommitConfig, GroupCommitWriter};
use crate::blockstore::key_index::KeyIndex;
use crate::blockstore::manager::DataFileManager;
use crate::blockstore::test_cid;
use futures::StreamExt;
struct TestHarness {
_dir: tempfile::TempDir,
index: Arc<KeyIndex>,
manager: Arc<DataFileManager<RealIO>>,
writer: Option<GroupCommitWriter>,
sender: flume::Sender<CommitRequest>,
}
impl TestHarness {
fn new() -> Self {
let dir = tempfile::TempDir::new().unwrap();
let data_dir = dir.path().join("data");
std::fs::create_dir_all(&data_dir).unwrap();
let index_dir = dir.path().join("index");
let manager = Arc::new(DataFileManager::with_default_max_size(
RealIO::new(),
data_dir,
));
let index = Arc::new(KeyIndex::open(&index_dir).unwrap().into_inner());
let writer = GroupCommitWriter::spawn(
DataFileManager::with_default_max_size(RealIO::new(), dir.path().join("data")),
Arc::clone(&index),
GroupCommitConfig::default(),
)
.unwrap();
let sender = writer.sender().clone();
Self {
_dir: dir,
index,
manager,
writer: Some(writer),
sender,
}
}
fn reader(&self) -> BlockStoreReader<RealIO> {
BlockStoreReader::new(Arc::clone(&self.index), Arc::clone(&self.manager))
}
async fn put_blocks(
&self,
blocks: Vec<([u8; CID_SIZE], Vec<u8>)>,
) -> Result<Vec<super::super::types::BlockLocation>, super::super::group_commit::CommitError>
{
let (tx, rx) = tokio::sync::oneshot::channel();
self.sender
.send_async(CommitRequest::PutBlocks {
blocks,
response: tx,
})
.await
.map_err(|_| super::super::group_commit::CommitError::ChannelClosed)?;
rx.await
.map_err(|_| super::super::group_commit::CommitError::ChannelClosed)?
}
fn shutdown(&mut self) {
if let Some(w) = self.writer.take() {
w.shutdown();
}
}
}
impl Drop for TestHarness {
fn drop(&mut self) {
self.shutdown();
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn get_existing_block() {
let mut harness = TestHarness::new();
let cid = test_cid(1);
let data = vec![0xAB; 256];
harness.put_blocks(vec![(cid, data.clone())]).await.unwrap();
harness.shutdown();
let reader = harness.reader();
let result = reader.get(&cid).unwrap().unwrap();
assert_eq!(&result[..], &data[..]);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn get_missing_block_returns_none() {
let mut harness = TestHarness::new();
harness.shutdown();
let reader = harness.reader();
assert!(reader.get(&test_cid(99)).unwrap().is_none());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn get_many_mixed_hits_and_misses() {
let mut harness = TestHarness::new();
let blocks: Vec<_> = (0u8..5)
.map(|i| (test_cid(i), vec![i; (i as usize + 1) * 32]))
.collect();
harness.put_blocks(blocks.clone()).await.unwrap();
harness.shutdown();
let reader = harness.reader();
let query: Vec<[u8; CID_SIZE]> = vec![
test_cid(0),
test_cid(99),
test_cid(2),
test_cid(100),
test_cid(4),
];
let results = reader.get_many(&query).unwrap();
assert_eq!(results.len(), 5);
assert_eq!(&results[0].as_ref().unwrap()[..], &blocks[0].1[..]);
assert!(results[1].is_none());
assert_eq!(&results[2].as_ref().unwrap()[..], &blocks[2].1[..]);
assert!(results[3].is_none());
assert_eq!(&results[4].as_ref().unwrap()[..], &blocks[4].1[..]);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn has_returns_true_for_existing() {
let mut harness = TestHarness::new();
let cid = test_cid(1);
harness
.put_blocks(vec![(cid, vec![0xFF; 64])])
.await
.unwrap();
harness.shutdown();
let reader = harness.reader();
assert!(reader.has(&cid).unwrap());
assert!(!reader.has(&test_cid(99)).unwrap());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn checksum_mismatch_returns_error() {
let mut harness = TestHarness::new();
let cid = test_cid(1);
let data = vec![0xAA; 256];
harness.put_blocks(vec![(cid, data)]).await.unwrap();
harness.shutdown();
let entry = harness.index.get(&cid).unwrap().unwrap();
let loc = entry.location;
let data_file_path = harness.manager.data_file_path(loc.file_id);
let corrupt_offset = loc.offset.raw() + super::super::data_file::CID_SIZE as u64 + 4 + 128;
let file_bytes = std::fs::read(&data_file_path).unwrap();
let mut corrupted = file_bytes;
corrupted[corrupt_offset as usize] ^= 0xFF;
std::fs::write(&data_file_path, &corrupted).unwrap();
let fresh_manager = Arc::new(DataFileManager::with_default_max_size(
RealIO::new(),
harness.manager.data_dir().to_path_buf(),
));
let reader = BlockStoreReader::new(Arc::clone(&harness.index), fresh_manager);
let result = reader.get(&cid);
assert!(
matches!(result, Err(ReadError::Corrupted { .. })),
"expected Corrupted error, got {result:?}"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn spawn_blocking_does_not_block_tokio_workers() {
let mut harness = TestHarness::new();
let blocks: Vec<_> = (0u8..200).map(|i| (test_cid(i), vec![i; 1024])).collect();
harness.put_blocks(blocks).await.unwrap();
harness.shutdown();
let reader = harness.reader();
let reader = Arc::new(reader);
let timer_handle = tokio::spawn(futures::stream::iter(0..100).fold(
std::time::Duration::ZERO,
|max_drift, _| async move {
let start = std::time::Instant::now();
tokio::time::sleep(std::time::Duration::from_millis(1)).await;
let drift = start
.elapsed()
.saturating_sub(std::time::Duration::from_millis(1));
max_drift.max(drift)
},
));
let read_handles: Vec<_> = (0..8)
.map(|_| {
let reader = Arc::clone(&reader);
tokio::spawn(futures::stream::iter(0u8..200).fold(
(0u64, 200u64),
move |(total_us, count), i| {
let reader = Arc::clone(&reader);
async move {
let cid = test_cid(i);
let start = std::time::Instant::now();
let result = tokio::task::spawn_blocking(move || reader.get(&cid))
.await
.unwrap();
let elapsed_us = start.elapsed().as_micros() as u64;
assert!(result.unwrap().is_some());
(total_us.saturating_add(elapsed_us), count)
}
},
))
})
.collect();
let timer_drift = timer_handle.await.unwrap();
assert!(
timer_drift < std::time::Duration::from_millis(5),
"timer drift {timer_drift:?} exceeds 5ms, reads may be blocking tokio workers"
);
let stats: Vec<(u64, u64)> = futures::future::join_all(read_handles)
.await
.into_iter()
.map(|r| r.unwrap())
.collect();
let total_us: u64 = stats.iter().map(|(us, _)| us).sum();
let total_count: u64 = stats.iter().map(|(_, c)| c).sum();
let avg_us = total_us / total_count.max(1);
eprintln!("avg read latency: {avg_us}us across {total_count} reads");
}
use crate::blockstore::test_cid_u16 as stress_cid;
#[tokio::test(flavor = "multi_thread", worker_threads = 8)]
async fn stress_50_writers_20_readers() {
let dir = tempfile::TempDir::new().unwrap();
let data_dir = dir.path().join("data");
std::fs::create_dir_all(&data_dir).unwrap();
let index_dir = dir.path().join("index");
let index = Arc::new(KeyIndex::open(&index_dir).unwrap().into_inner());
let manager_for_writer =
DataFileManager::with_default_max_size(RealIO::new(), data_dir.clone());
let writer = GroupCommitWriter::spawn(
manager_for_writer,
Arc::clone(&index),
GroupCommitConfig::default(),
)
.unwrap();
let sender = writer.sender().clone();
let manager_for_reader = Arc::new(DataFileManager::with_default_max_size(
RealIO::new(),
data_dir,
));
let reader = BlockStoreReader::new(Arc::clone(&index), manager_for_reader);
let committed = Arc::new(std::sync::Mutex::new(Vec::<(u16, Vec<u8>)>::new()));
let writer_done = Arc::new(std::sync::atomic::AtomicBool::new(false));
let writer_handles: Vec<_> = (0u16..50)
.map(|writer_id| {
let sender = sender.clone();
let committed = Arc::clone(&committed);
tokio::spawn(async move {
futures::stream::iter(0u16..200)
.fold((), |(), block_id| {
let sender = sender.clone();
let committed = Arc::clone(&committed);
async move {
let seed = writer_id * 200 + block_id;
let cid = stress_cid(seed);
let size = ((seed as usize % 256) + 1) * 4;
let data = vec![seed as u8; size];
let (tx, rx) = tokio::sync::oneshot::channel();
sender
.send_async(CommitRequest::PutBlocks {
blocks: vec![(cid, data.clone())],
response: tx,
})
.await
.unwrap();
rx.await.unwrap().unwrap();
committed.lock().unwrap().push((seed, data));
}
})
.await;
})
})
.collect();
let reader_handles: Vec<_> = (0..20)
.map(|_| {
let reader = reader.clone();
let committed = Arc::clone(&committed);
let done = Arc::clone(&writer_done);
tokio::spawn(async move {
let reads = std::sync::atomic::AtomicU64::new(0);
(0..5000)
.take_while(|_| {
let is_done = done.load(std::sync::atomic::Ordering::Relaxed);
let has_reads = reads.load(std::sync::atomic::Ordering::Relaxed) > 100;
!(is_done && has_reads)
})
.for_each(|_| {
let snapshot = committed.lock().unwrap().clone();
if let Some((seed, expected)) = snapshot.last() {
let cid = stress_cid(*seed);
match reader.get(&cid) {
Ok(Some(actual)) => {
assert_eq!(&actual[..], &expected[..]);
reads.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
Ok(None) => {}
Err(e) => panic!("read error: {e}"),
}
}
std::thread::yield_now();
});
reads.load(std::sync::atomic::Ordering::Relaxed)
})
})
.collect();
futures::future::join_all(writer_handles)
.await
.into_iter()
.for_each(|r| r.unwrap());
writer_done.store(true, std::sync::atomic::Ordering::Relaxed);
let read_counts: Vec<u64> = futures::future::join_all(reader_handles)
.await
.into_iter()
.map(|r| r.unwrap())
.collect();
let total_reads: u64 = read_counts.iter().sum();
eprintln!("total reader reads: {total_reads}");
assert!(total_reads > 0);
writer.shutdown();
let final_committed = committed.lock().unwrap();
assert_eq!(final_committed.len(), 10_000);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn concurrent_read_write() {
let mut harness = TestHarness::new();
let sender = harness.sender.clone();
let reader = harness.reader();
let written_cids = Arc::new(std::sync::Mutex::new(Vec::<(u8, Vec<u8>)>::new()));
let writer_done = Arc::new(std::sync::atomic::AtomicBool::new(false));
let writer_handle = {
let written = Arc::clone(&written_cids);
tokio::spawn(async move {
futures::stream::iter(0u8..50)
.fold((), |(), i| {
let sender = sender.clone();
let written = Arc::clone(&written);
async move {
let cid = test_cid(i);
let data = vec![i; (i as usize + 1) * 16];
let (tx, rx) = tokio::sync::oneshot::channel();
sender
.send_async(CommitRequest::PutBlocks {
blocks: vec![(cid, data.clone())],
response: tx,
})
.await
.unwrap();
rx.await.unwrap().unwrap();
written.lock().unwrap().push((i, data));
}
})
.await;
})
};
let reader_handles: Vec<_> = (0..4)
.map(|_| {
let reader = reader.clone();
let written = Arc::clone(&written_cids);
let done = Arc::clone(&writer_done);
tokio::spawn(async move {
let reads = std::sync::atomic::AtomicU64::new(0);
(0..2000)
.take_while(|_| {
let is_done = done.load(std::sync::atomic::Ordering::Relaxed);
let has_reads = reads.load(std::sync::atomic::Ordering::Relaxed) > 0;
!(is_done && has_reads)
})
.for_each(|_| {
let snapshot = written.lock().unwrap().clone();
snapshot.iter().for_each(|(seed, expected_data)| {
let cid = test_cid(*seed);
match reader.get(&cid) {
Ok(Some(actual)) => {
assert_eq!(
&actual[..],
&expected_data[..],
"data mismatch for block {seed}"
);
reads.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
Ok(None) => {}
Err(e) => panic!("read error for block {seed}: {e}"),
}
});
std::thread::yield_now();
});
reads.load(std::sync::atomic::Ordering::Relaxed)
})
})
.collect();
writer_handle.await.unwrap();
writer_done.store(true, std::sync::atomic::Ordering::Relaxed);
let read_counts: Vec<u64> = futures::future::join_all(reader_handles)
.await
.into_iter()
.map(|r| r.unwrap())
.collect();
let total_reads: u64 = read_counts.iter().sum();
assert!(
total_reads > 0,
"readers should have completed at least some reads"
);
let final_snapshot = written_cids.lock().unwrap().clone();
assert_eq!(final_snapshot.len(), 50);
final_snapshot.iter().for_each(|(seed, expected_data)| {
let cid = test_cid(*seed);
let actual = reader.get(&cid).unwrap().unwrap();
assert_eq!(
&actual[..],
&expected_data[..],
"final verification failed for block {seed}"
);
});
harness.shutdown();
}
}
crates/tranquil-store/src/blockstore/store.rs
+292 -804
View File
File diff suppressed because it is too large Load Diff
crates/tranquil-store/src/blockstore/types.rs
+186 -4
View File
@@ -1,10 +1,104 @@
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use serde::{Deserialize, Serialize};
use super::data_file::CID_SIZE;
pub type CidBytes = [u8; CID_SIZE];
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[repr(transparent)]
pub struct CommitEpoch(u64);
impl CommitEpoch {
pub const fn new(value: u64) -> Self {
Self(value)
}
pub const fn zero() -> Self {
Self(0)
}
pub fn raw(self) -> u64 {
self.0
}
pub fn next(self) -> Self {
Self(self.0.saturating_add(1))
}
}
#[derive(Debug, Clone)]
pub struct EpochCounter(Arc<AtomicU64>);
impl Default for EpochCounter {
fn default() -> Self {
Self(Arc::new(AtomicU64::new(0)))
}
}
impl EpochCounter {
pub fn new() -> Self {
Self::default()
}
pub fn from_raw(value: u64) -> Self {
Self(Arc::new(AtomicU64::new(value)))
}
pub fn current(&self) -> CommitEpoch {
CommitEpoch(self.0.load(Ordering::Acquire))
}
pub fn advance(&self) -> CommitEpoch {
let prev = self
.0
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |v| {
Some(v.saturating_add(1))
})
.unwrap_or(u64::MAX);
CommitEpoch(prev.saturating_add(1))
}
}
pub struct CollectionResult {
pub candidates: HashMap<DataFileId, Vec<CidBytes>>,
pub total_bytes: u64,
}
pub struct CompactionResult {
pub file_id: DataFileId,
pub old_size: u64,
pub new_size: u64,
pub live_blocks: u64,
pub dead_blocks: u64,
pub reclaimed_bytes: u64,
}
pub struct LivenessInfo {
pub live_bytes: u64,
pub total_bytes: u64,
pub live_blocks: u64,
pub total_blocks: u64,
}
impl LivenessInfo {
pub fn ratio(&self) -> f64 {
match self.total_bytes {
0 => 1.0,
total => self.live_bytes as f64 / total as f64,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[repr(transparent)]
pub struct DataFileId(u32);
impl DataFileId {
pub fn new(id: u32) -> Self {
pub const fn new(id: u32) -> Self {
Self(id)
}
@@ -24,10 +118,11 @@ impl std::fmt::Display for DataFileId {
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[repr(transparent)]
pub struct BlockOffset(u64);
impl BlockOffset {
pub fn new(offset: u64) -> Self {
pub const fn new(offset: u64) -> Self {
Self(offset)
}
@@ -43,6 +138,7 @@ impl BlockOffset {
pub const MAX_BLOCK_SIZE: u32 = 4 * 1024 * 1024;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[repr(transparent)]
pub struct BlockLength(u32);
impl BlockLength {
@@ -54,6 +150,10 @@ impl BlockLength {
Self(length)
}
pub const fn from_raw(length: u32) -> Self {
Self(length)
}
pub fn raw(self) -> u32 {
self.0
}
@@ -64,10 +164,11 @@ impl BlockLength {
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[repr(transparent)]
pub struct RefCount(u32);
impl RefCount {
pub fn new(count: u32) -> Self {
pub const fn new(count: u32) -> Self {
Self(count)
}
@@ -75,7 +176,7 @@ impl RefCount {
self.0
}
pub fn one() -> Self {
pub const fn one() -> Self {
Self(1)
}
@@ -87,6 +188,10 @@ impl RefCount {
Self(self.0.checked_add(1).expect("RefCount overflow"))
}
pub fn saturating_increment(self) -> Self {
Self(self.0.saturating_add(1))
}
pub fn decrement(self) -> Self {
Self(self.0.saturating_sub(1))
}
@@ -113,6 +218,7 @@ pub struct WriteCursor {
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[repr(transparent)]
pub struct HintOffset(u64);
impl HintOffset {
@@ -129,10 +235,86 @@ impl HintOffset {
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[repr(transparent)]
pub struct WallClockMs(u64);
impl WallClockMs {
pub const fn new(ms: u64) -> Self {
Self(ms)
}
pub fn now() -> Self {
let millis = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_millis();
Self(u64::try_from(millis).unwrap_or(u64::MAX))
}
pub fn raw(self) -> u64 {
self.0
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[repr(transparent)]
pub struct ShardId(u8);
impl ShardId {
pub const fn new(id: u8) -> Self {
Self(id)
}
pub fn raw(self) -> u8 {
self.0
}
pub fn as_usize(self) -> usize {
self.0 as usize
}
}
impl std::fmt::Display for ShardId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "shard_{}", self.0)
}
}
pub struct BlockstoreSnapshot {
pub shard_cursors: Vec<WriteCursor>,
pub epoch: CommitEpoch,
pub data_files: Vec<DataFileId>,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn commit_epoch_advances() {
let e = CommitEpoch::zero();
assert_eq!(e.raw(), 0);
assert_eq!(e.next().raw(), 1);
}
#[test]
fn commit_epoch_saturates() {
let e = CommitEpoch::new(u64::MAX);
assert_eq!(e.next().raw(), u64::MAX);
}
#[test]
fn epoch_counter_advance_returns_new_value() {
let counter = EpochCounter::new();
assert_eq!(counter.current().raw(), 0);
let epoch1 = counter.advance();
assert_eq!(epoch1.raw(), 1);
assert_eq!(counter.current().raw(), 1);
let epoch2 = counter.advance();
assert_eq!(epoch2.raw(), 2);
}
#[test]
fn index_entry_postcard_round_trip() {
let entry = IndexEntry {
crates/tranquil-store/src/bloom.rs
+134
View File
@@ -0,0 +1,134 @@
use xxhash_rust::xxh3::xxh3_64_with_seed;
pub struct BloomFilter {
bits: Vec<u64>,
num_bits: u64,
num_hashes: u32,
}
impl BloomFilter {
const MAX_BITS: u64 = 1 << 34;
pub fn with_capacity_and_fpr(expected_items: u64, false_positive_rate: f64) -> Self {
debug_assert!(
false_positive_rate > 0.0 && false_positive_rate < 1.0,
"false_positive_rate must be in (0, 1), got {false_positive_rate}"
);
let expected = expected_items.max(1) as f64;
let ln2 = std::f64::consts::LN_2;
let num_bits_f = -(expected * false_positive_rate.ln()) / (ln2 * ln2);
let num_bits = num_bits_f.ceil().clamp(64.0, Self::MAX_BITS as f64) as u64;
let num_bits = num_bits.next_power_of_two();
let optimal_k = ((num_bits as f64 / expected) * ln2).ceil();
let num_hashes = (optimal_k as u32).clamp(1, 16);
let words = (num_bits / 64) as usize;
Self {
bits: vec![0u64; words],
num_bits,
num_hashes,
}
}
pub fn insert(&mut self, key: &[u8]) {
let mask = self.num_bits - 1;
(0..self.num_hashes).for_each(|i| {
let h = xxh3_64_with_seed(key, u64::from(i)) & mask;
let word = (h / 64) as usize;
let bit = h % 64;
self.bits[word] |= 1u64 << bit;
});
}
pub fn contains(&self, key: &[u8]) -> bool {
let mask = self.num_bits - 1;
(0..self.num_hashes).all(|i| {
let h = xxh3_64_with_seed(key, u64::from(i)) & mask;
let word = (h / 64) as usize;
let bit = h % 64;
(self.bits[word] >> bit) & 1 == 1
})
}
pub fn heap_bytes(&self) -> usize {
self.bits.len() * 8
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn insert_and_contains() {
let mut bf = BloomFilter::with_capacity_and_fpr(1000, 0.01);
bf.insert(b"hello");
bf.insert(b"world");
assert!(bf.contains(b"hello"));
assert!(bf.contains(b"world"));
}
#[test]
fn missing_key_usually_absent() {
let mut bf = BloomFilter::with_capacity_and_fpr(1000, 0.01);
(0u32..500).for_each(|i| bf.insert(&i.to_le_bytes()));
let false_positives = (1000u32..2000)
.filter(|i| bf.contains(&i.to_le_bytes()))
.count();
assert!(
false_positives < 50,
"expected <5% FPR, got {false_positives}/1000"
);
}
#[test]
fn no_false_negatives() {
let mut bf = BloomFilter::with_capacity_and_fpr(10_000, 0.01);
let keys: Vec<[u8; 4]> = (0u32..10_000).map(|i| i.to_le_bytes()).collect();
keys.iter().for_each(|k| bf.insert(k));
assert!(keys.iter().all(|k| bf.contains(k)));
}
#[test]
fn empty_filter_contains_nothing() {
let bf = BloomFilter::with_capacity_and_fpr(1000, 0.01);
assert!(!bf.contains(b"anything"));
}
#[test]
fn heap_bytes_reasonable() {
let bf = BloomFilter::with_capacity_and_fpr(100_000_000, 0.01);
let mb = bf.heap_bytes() / (1024 * 1024);
assert!(
mb < 256,
"100M items at 1% FPR should be <256MB, got {mb}MB"
);
assert!(mb > 64, "100M items at 1% FPR should be >64MB, got {mb}MB");
}
#[test]
fn fpr_empirical() {
let n = 50_000u32;
let target_fpr = 0.01;
let mut bf = BloomFilter::with_capacity_and_fpr(n as u64, target_fpr);
(0..n).for_each(|i| bf.insert(&i.to_le_bytes()));
let test_range = 100_000u32;
let false_positives = (n..n + test_range)
.filter(|i| bf.contains(&i.to_le_bytes()))
.count();
let measured_fpr = false_positives as f64 / test_range as f64;
assert!(
measured_fpr < target_fpr * 3.0,
"measured FPR {measured_fpr:.4} exceeds 3x target {target_fpr}"
);
}
}
crates/tranquil-store/src/consistency.rs
+657
View File
@@ -0,0 +1,657 @@
use std::collections::HashSet;
use std::fmt;
use std::path::Path;
use crate::blockstore::CID_SIZE;
use crate::blockstore::hash_index::BlockIndex;
use crate::blockstore::{DataFileId, TranquilBlockStore};
use crate::eventlog::{EventLog, EventSequence, SequenceContiguityResult};
use crate::io::StorageIO;
use crate::metastore::Metastore;
use crate::metastore::encoding::KeyBuilder;
use crate::metastore::event_keys::metastore_cursor_key;
use crate::metastore::keys::{KeyTag, UserHash};
use crate::metastore::partitions::Partition;
use crate::metastore::records::RecordValue;
use crate::metastore::repo_meta::RepoMetaValue;
const CLEAN_SHUTDOWN_MARKER: &str = ".clean_shutdown";
#[derive(Debug, Default)]
pub struct ConsistencyReport {
pub repos_checked: u64,
pub records_checked: u64,
pub user_blocks_checked: u64,
pub handles_checked: u64,
pub dangling_record_cids: Vec<DanglingCid>,
pub dangling_root_cids: Vec<DanglingRootCid>,
pub orphaned_user_repos: Vec<OrphanedUserRepo>,
pub inconsistent_handles: Vec<InconsistentHandle>,
pub orphan_data_files: Vec<DataFileId>,
pub deserialization_failures: u64,
pub eventlog_contiguity: Option<SequenceContiguityResult>,
pub cursor_ahead_of_eventlog: bool,
pub metastore_cursor: Option<EventSequence>,
pub eventlog_max_seq: Option<EventSequence>,
}
#[derive(Debug, Clone)]
pub struct DanglingCid {
pub user_hash: UserHash,
pub collection: String,
pub rkey: String,
pub cid_bytes: Vec<u8>,
}
#[derive(Debug, Clone)]
pub struct DanglingRootCid {
pub user_hash: UserHash,
pub root_cid_bytes: Vec<u8>,
}
#[derive(Debug, Clone)]
pub struct OrphanedUserRepo {
pub user_hash: UserHash,
}
#[derive(Debug, Clone)]
pub struct InconsistentHandle {
pub handle: String,
pub mapped_user_hash: UserHash,
pub problem: HandleProblem,
}
#[derive(Debug, Clone)]
pub enum HandleProblem {
NoRepoMeta,
HandleMismatch { repo_handle: String },
}
impl ConsistencyReport {
pub fn is_consistent(&self) -> bool {
self.dangling_record_cids.is_empty()
&& self.dangling_root_cids.is_empty()
&& self.orphaned_user_repos.is_empty()
&& self.inconsistent_handles.is_empty()
&& self.orphan_data_files.is_empty()
&& self.deserialization_failures == 0
&& self
.eventlog_contiguity
.as_ref()
.is_none_or(|c| c.is_contiguous())
&& !self.cursor_ahead_of_eventlog
}
pub fn has_repairable_issues(&self) -> bool {
!self.orphan_data_files.is_empty()
}
pub fn has_unrecoverable_issues(&self) -> bool {
!self.dangling_root_cids.is_empty()
|| !self.dangling_record_cids.is_empty()
|| self.deserialization_failures > 0
|| self.cursor_ahead_of_eventlog
}
pub fn log_findings(&self) {
if self.is_consistent() {
tracing::info!(
repos = self.repos_checked,
records = self.records_checked,
user_blocks = self.user_blocks_checked,
handles = self.handles_checked,
"consistency check passed"
);
return;
}
if !self.dangling_record_cids.is_empty() {
tracing::warn!(
count = self.dangling_record_cids.len(),
"records reference missing blocks"
);
}
if !self.dangling_root_cids.is_empty() {
tracing::warn!(
count = self.dangling_root_cids.len(),
"repo roots reference missing blocks"
);
}
if !self.orphaned_user_repos.is_empty() {
tracing::warn!(
count = self.orphaned_user_repos.len(),
"repos with user_blocks but no repo_meta"
);
}
if !self.inconsistent_handles.is_empty() {
tracing::warn!(
count = self.inconsistent_handles.len(),
"handle index inconsistencies"
);
}
if !self.orphan_data_files.is_empty() {
tracing::warn!(
count = self.orphan_data_files.len(),
files = ?self.orphan_data_files,
"orphan data files with no index references"
);
}
if self.deserialization_failures > 0 {
tracing::error!(
count = self.deserialization_failures,
"metastore values failed to deserialize"
);
}
if let Some(c) = &self.eventlog_contiguity
&& !c.is_contiguous()
{
tracing::warn!(gaps = c.gaps.len(), "eventlog sequence gaps detected");
c.gaps.iter().take(5).for_each(|gap| {
tracing::warn!(
after_segment = %gap.after_segment,
expected = ?gap.expected_seq,
actual = ?gap.actual_seq,
"eventlog gap"
);
});
}
if self.cursor_ahead_of_eventlog {
tracing::error!(
cursor = ?self.metastore_cursor,
eventlog_max = ?self.eventlog_max_seq,
"metastore cursor is ahead of eventlog max sequence"
);
}
}
}
impl fmt::Display for ConsistencyReport {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
if self.is_consistent() {
return write!(
f,
"consistent (repos={}, records={}, user_blocks={}, handles={})",
self.repos_checked,
self.records_checked,
self.user_blocks_checked,
self.handles_checked,
);
}
write!(
f,
"INCONSISTENT: dangling_roots={}, dangling_records={}, orphaned_repos={}, \
inconsistent_handles={}, orphan_files={}, deserialize_failures={}, \
eventlog_gaps={}, cursor_ahead={}",
self.dangling_root_cids.len(),
self.dangling_record_cids.len(),
self.orphaned_user_repos.len(),
self.inconsistent_handles.len(),
self.orphan_data_files.len(),
self.deserialization_failures,
self.eventlog_contiguity
.as_ref()
.map_or(0, |c| c.gaps.len()),
self.cursor_ahead_of_eventlog,
)
}
}
#[derive(Debug, Clone, Copy)]
pub struct ConsistencyCheckOptions {
pub check_block_references: bool,
pub check_handles: bool,
pub check_user_blocks: bool,
pub check_eventlog: bool,
pub check_orphan_files: bool,
}
impl Default for ConsistencyCheckOptions {
fn default() -> Self {
Self {
check_block_references: true,
check_handles: true,
check_user_blocks: true,
check_eventlog: true,
check_orphan_files: true,
}
}
}
pub fn verify_store_consistency<S: StorageIO + 'static>(
blockstore: &TranquilBlockStore,
metastore: &Metastore,
eventlog: &EventLog<S>,
) -> ConsistencyReport {
verify_store_consistency_with_options(
blockstore,
metastore,
eventlog,
ConsistencyCheckOptions::default(),
)
}
pub fn verify_store_consistency_with_options<S: StorageIO + 'static>(
blockstore: &TranquilBlockStore,
metastore: &Metastore,
eventlog: &EventLog<S>,
options: ConsistencyCheckOptions,
) -> ConsistencyReport {
let mut report = ConsistencyReport::default();
let block_index = blockstore.block_index();
let repo_data = metastore.partition(Partition::RepoData);
let known_user_hashes = if options.check_block_references {
let hashes = check_repo_root_cids(repo_data, block_index, &mut report);
check_record_cids(repo_data, block_index, &mut report);
hashes
} else if options.check_user_blocks {
collect_known_user_hashes(repo_data)
} else {
HashSet::new()
};
if options.check_user_blocks {
check_user_blocks(repo_data, &known_user_hashes, &mut report);
}
if options.check_handles {
check_handle_consistency(repo_data, &mut report);
}
if options.check_eventlog {
check_eventlog_contiguity(eventlog, &mut report);
check_cursor_vs_eventlog(repo_data, eventlog, &mut report);
}
if options.check_orphan_files {
check_orphan_data_files(blockstore, block_index, &mut report);
}
report
}
fn check_repo_root_cids(
repo_data: &fjall::Keyspace,
block_index: &BlockIndex,
report: &mut ConsistencyReport,
) -> HashSet<UserHash> {
let prefix = KeyBuilder::new().tag(KeyTag::REPO_META).build();
let mut known_user_hashes = HashSet::new();
repo_data.prefix(prefix.as_slice()).for_each(|guard| {
let Ok((key_bytes, value_bytes)) = guard.into_inner() else {
return;
};
report.repos_checked = report.repos_checked.saturating_add(1);
if let Some(h) = extract_user_hash(&key_bytes) {
known_user_hashes.insert(h);
}
let Some(meta) = RepoMetaValue::deserialize(&value_bytes) else {
tracing::warn!(
user_hash = ?extract_user_hash(&key_bytes),
"repo_meta value failed to deserialize"
);
report.deserialization_failures = report.deserialization_failures.saturating_add(1);
return;
};
if meta.repo_root_cid.is_empty() {
return;
}
let Some(cid_fixed) = try_cid_bytes_to_fixed(&meta.repo_root_cid) else {
let Some(user_hash) = extract_user_hash(&key_bytes) else {
return;
};
tracing::warn!(
%user_hash,
cid_len = meta.repo_root_cid.len(),
"repo_meta has non-standard CID length"
);
report.dangling_root_cids.push(DanglingRootCid {
user_hash,
root_cid_bytes: meta.repo_root_cid,
});
return;
};
if !block_index.has(&cid_fixed) {
let Some(user_hash) = extract_user_hash(&key_bytes) else {
return;
};
report.dangling_root_cids.push(DanglingRootCid {
user_hash,
root_cid_bytes: meta.repo_root_cid,
});
}
});
known_user_hashes
}
fn collect_known_user_hashes(repo_data: &fjall::Keyspace) -> HashSet<UserHash> {
let prefix = KeyBuilder::new().tag(KeyTag::REPO_META).build();
let mut hashes = HashSet::new();
repo_data.prefix(prefix.as_slice()).for_each(|guard| {
if let Ok((key_bytes, _)) = guard.into_inner()
&& let Some(h) = extract_user_hash(&key_bytes)
{
hashes.insert(h);
}
});
hashes
}
fn check_record_cids(
repo_data: &fjall::Keyspace,
block_index: &BlockIndex,
report: &mut ConsistencyReport,
) {
let prefix = KeyBuilder::new().tag(KeyTag::RECORDS).build();
repo_data.prefix(prefix.as_slice()).for_each(|guard| {
let Ok((key_bytes, value_bytes)) = guard.into_inner() else {
return;
};
report.records_checked = report.records_checked.saturating_add(1);
let Some(record) = RecordValue::deserialize(&value_bytes) else {
tracing::warn!(
user_hash = ?extract_user_hash(&key_bytes),
"record value failed to deserialize"
);
report.deserialization_failures = report.deserialization_failures.saturating_add(1);
return;
};
let Some(cid_fixed) = try_cid_bytes_to_fixed(&record.record_cid) else {
let (user_hash, collection, rkey) = parse_record_key(&key_bytes);
let Some(user_hash) = user_hash else {
return;
};
tracing::warn!(
%user_hash,
collection,
rkey,
cid_len = record.record_cid.len(),
"record has non-standard CID length"
);
report.dangling_record_cids.push(DanglingCid {
user_hash,
collection,
rkey,
cid_bytes: record.record_cid,
});
return;
};
if !block_index.has(&cid_fixed) {
let (user_hash, collection, rkey) = parse_record_key(&key_bytes);
let Some(user_hash) = user_hash else {
return;
};
report.dangling_record_cids.push(DanglingCid {
user_hash,
collection,
rkey,
cid_bytes: record.record_cid,
});
}
});
}
fn check_user_blocks(
repo_data: &fjall::Keyspace,
known_user_hashes: &HashSet<UserHash>,
report: &mut ConsistencyReport,
) {
let prefix = KeyBuilder::new().tag(KeyTag::USER_BLOCKS).build();
let mut seen_orphan_hashes: HashSet<UserHash> = HashSet::new();
repo_data.prefix(prefix.as_slice()).for_each(|guard| {
let Ok((key_bytes, _)) = guard.into_inner() else {
return;
};
report.user_blocks_checked = report.user_blocks_checked.saturating_add(1);
let Some(user_hash) = extract_user_hash(&key_bytes) else {
return;
};
if !known_user_hashes.contains(&user_hash) && seen_orphan_hashes.insert(user_hash) {
report
.orphaned_user_repos
.push(OrphanedUserRepo { user_hash });
}
});
}
fn check_handle_consistency(repo_data: &fjall::Keyspace, report: &mut ConsistencyReport) {
let prefix = KeyBuilder::new().tag(KeyTag::HANDLES).build();
repo_data.prefix(prefix.as_slice()).for_each(|guard| {
let Ok((key_bytes, value_bytes)) = guard.into_inner() else {
return;
};
report.handles_checked = report.handles_checked.saturating_add(1);
let handle = parse_handle_from_key(&key_bytes);
let Some(mapped_hash) = parse_user_hash_from_value(&value_bytes) else {
return;
};
let meta_key = crate::metastore::repo_meta::repo_meta_key(mapped_hash);
match repo_data.get(meta_key.as_slice()) {
Ok(Some(meta_bytes)) => {
let Some(meta) = RepoMetaValue::deserialize(&meta_bytes) else {
tracing::warn!(
%mapped_hash,
handle,
"repo_meta value failed to deserialize during handle check"
);
report.deserialization_failures =
report.deserialization_failures.saturating_add(1);
return;
};
let meta_handle_lower = meta.handle.to_lowercase();
let handle_lower = handle.to_lowercase();
if meta_handle_lower != handle_lower {
report.inconsistent_handles.push(InconsistentHandle {
handle,
mapped_user_hash: mapped_hash,
problem: HandleProblem::HandleMismatch {
repo_handle: meta.handle,
},
});
}
}
Ok(None) => {
report.inconsistent_handles.push(InconsistentHandle {
handle,
mapped_user_hash: mapped_hash,
problem: HandleProblem::NoRepoMeta,
});
}
Err(e) => {
tracing::warn!(error = %e, handle, "repo_meta lookup failed during handle check");
}
}
});
}
fn check_eventlog_contiguity<S: StorageIO + 'static>(
eventlog: &EventLog<S>,
report: &mut ConsistencyReport,
) {
let reader = eventlog.reader();
if let Err(e) = reader.refresh_segment_ranges() {
tracing::warn!(error = %e, "failed to refresh segment ranges for contiguity check");
return;
}
report.eventlog_contiguity = Some(reader.check_sequence_contiguity());
}
fn check_cursor_vs_eventlog<S: StorageIO + 'static>(
repo_data: &fjall::Keyspace,
eventlog: &EventLog<S>,
report: &mut ConsistencyReport,
) {
let cursor_key = metastore_cursor_key();
let cursor_seq = repo_data
.get(cursor_key.as_slice())
.ok()
.flatten()
.and_then(|bytes| {
let arr: [u8; 8] = bytes.as_ref().try_into().ok()?;
Some(match u64::from_be_bytes(arr) {
0 => EventSequence::BEFORE_ALL,
n => EventSequence::new(n),
})
});
let max_seq = eventlog.max_seq();
report.metastore_cursor = cursor_seq;
report.eventlog_max_seq = (max_seq != EventSequence::BEFORE_ALL).then_some(max_seq);
if let Some(cursor) = cursor_seq
&& max_seq != EventSequence::BEFORE_ALL
&& cursor > max_seq
{
report.cursor_ahead_of_eventlog = true;
}
}
fn check_orphan_data_files(
blockstore: &TranquilBlockStore,
block_index: &BlockIndex,
report: &mut ConsistencyReport,
) {
let disk_files = match blockstore.list_data_files() {
Ok(files) => files,
Err(e) => {
tracing::warn!(error = %e, "failed to list data files for orphan check");
return;
}
};
let epoch = blockstore.epoch().current();
let now = crate::wall_clock_ms();
let indexed_files = block_index.liveness_by_file(epoch, now, 0);
let indexed_file_ids: HashSet<DataFileId> = indexed_files.keys().copied().collect();
let active_file_id = block_index.read_write_cursor().map(|c| c.file_id);
if active_file_id.is_none() && indexed_file_ids.is_empty() {
return;
}
disk_files.iter().for_each(|&fid| {
let is_active = active_file_id.is_some_and(|active| fid >= active);
if !is_active && !indexed_file_ids.contains(&fid) {
report.orphan_data_files.push(fid);
}
});
}
fn try_cid_bytes_to_fixed(bytes: &[u8]) -> Option<[u8; CID_SIZE]> {
bytes.try_into().ok()
}
fn extract_user_hash(key_bytes: &[u8]) -> Option<UserHash> {
key_bytes
.get(1..9)?
.try_into()
.ok()
.map(|arr| UserHash::from_raw(u64::from_be_bytes(arr)))
}
fn parse_record_key(key_bytes: &[u8]) -> (Option<UserHash>, String, String) {
let user_hash = extract_user_hash(key_bytes);
let mut reader = crate::metastore::encoding::KeyReader::new(key_bytes);
let _ = reader.tag();
let _ = reader.u64();
let collection = reader.string().unwrap_or_default();
let rkey = reader.string().unwrap_or_default();
(user_hash, collection, rkey)
}
fn parse_handle_from_key(key_bytes: &[u8]) -> String {
let mut reader = crate::metastore::encoding::KeyReader::new(key_bytes);
let _ = reader.tag();
reader.string().unwrap_or_default()
}
fn parse_user_hash_from_value(value_bytes: &[u8]) -> Option<UserHash> {
value_bytes
.get(..8)?
.try_into()
.ok()
.map(|arr| UserHash::from_raw(u64::from_be_bytes(arr)))
}
pub fn repair_known_issues(
blockstore: &TranquilBlockStore,
report: &ConsistencyReport,
) -> RepairResult {
let mut result = RepairResult::default();
report.orphan_data_files.iter().for_each(|&file_id| {
let path = blockstore.data_file_path(file_id);
match std::fs::remove_file(&path) {
Ok(()) => {
tracing::info!(%file_id, "removed orphan data file");
result.orphan_files_removed = result.orphan_files_removed.saturating_add(1);
}
Err(e) => {
tracing::warn!(%file_id, error = %e, "failed to remove orphan data file");
result.repair_errors = result.repair_errors.saturating_add(1);
}
}
});
result
}
#[derive(Debug, Default)]
pub struct RepairResult {
pub orphan_files_removed: u64,
pub repair_errors: u64,
}
impl RepairResult {
pub fn had_errors(&self) -> bool {
self.repair_errors > 0
}
}
pub fn write_clean_shutdown_marker(data_dir: &Path) -> std::io::Result<()> {
let marker_path = data_dir.join(CLEAN_SHUTDOWN_MARKER);
let f = std::fs::File::create(&marker_path)?;
f.sync_all()?;
std::fs::File::open(data_dir)?.sync_all()
}
pub fn remove_clean_shutdown_marker(data_dir: &Path) -> std::io::Result<()> {
let marker_path = data_dir.join(CLEAN_SHUTDOWN_MARKER);
match std::fs::remove_file(&marker_path) {
Ok(()) => std::fs::File::open(data_dir)?.sync_all(),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(e) => Err(e),
}
}
pub fn had_clean_shutdown(data_dir: &Path) -> bool {
data_dir.join(CLEAN_SHUTDOWN_MARKER).exists()
}
crates/tranquil-store/src/eventlog/bridge.rs
+19 -36
View File
@@ -7,11 +7,10 @@ use tranquil_db_traits::{DbError, SequenceNumber, SequencedEvent};
use super::notifier::EventLogNotifier;
use super::types::{EventSequence, TimestampMicros};
use super::writer::SyncResult;
use super::{EventLog, EventWithMutations, decode_payload, to_sequenced_event};
use crate::io::StorageIO;
pub struct DeferredBroadcast(SyncResult);
pub struct DeferredBroadcast;
fn io_to_db(e: io::Error) -> DbError {
DbError::Query(e.to_string())
@@ -39,7 +38,7 @@ pub struct EventLogBridge<S: StorageIO> {
log: Arc<EventLog<S>>,
}
impl<S: StorageIO> EventLogBridge<S> {
impl<S: StorageIO + 'static> EventLogBridge<S> {
pub fn new(log: Arc<EventLog<S>>) -> Self {
Self { log }
}
@@ -156,21 +155,22 @@ impl<S: StorageIO> EventLogBridge<S> {
fn scan_for_timestamp(
&self,
reader: &super::EventLogReader<S>,
cursor: EventSequence,
mut cursor: EventSequence,
target_ts: TimestampMicros,
batch_size: usize,
) -> Result<Option<SequenceNumber>, DbError> {
let batch = reader
.read_events_from(cursor, batch_size)
.map_err(io_to_db)?;
if batch.is_empty() {
return Ok(None);
}
match batch.iter().find(|e| e.timestamp >= target_ts) {
Some(e) => Ok(Some(SequenceNumber::from_raw(e.seq.as_i64()))),
None => {
let next_cursor = batch.last().map(|e| e.seq).unwrap_or(cursor);
self.scan_for_timestamp(reader, next_cursor, target_ts, batch_size)
loop {
let batch = reader
.read_events_from(cursor, batch_size)
.map_err(io_to_db)?;
if batch.is_empty() {
return Ok(None);
}
match batch.iter().find(|e| e.timestamp >= target_ts) {
Some(e) => return Ok(Some(SequenceNumber::from_raw(e.seq.as_i64()))),
None => {
cursor = batch.last().map(|e| e.seq).unwrap_or(cursor);
}
}
}
}
@@ -264,33 +264,16 @@ impl<S: StorageIO> EventLogBridge<S> {
Ok(SequenceNumber::from_raw(seq.as_i64()))
}
pub fn insert_event_deferred(
&self,
event: &SequencedEvent,
) -> Result<(SequenceNumber, DeferredBroadcast), io::Error> {
let seq = self.log.append_event(&event.did, event.event_type, event)?;
let sync_result = self.log.sync_data()?;
Ok((
SequenceNumber::from_raw(seq.as_i64()),
DeferredBroadcast(sync_result),
))
}
pub fn insert_event_deferred_raw(
pub fn insert_event_group_commit_raw(
&self,
did: &tranquil_types::Did,
event_type: tranquil_db_traits::RepoEventType,
payload: Vec<u8>,
) -> Result<(SequenceNumber, DeferredBroadcast), io::Error> {
let seq = self.log.append_raw_payload(did, event_type, payload)?;
let sync_result = self.log.sync_data()?;
Ok((
SequenceNumber::from_raw(seq.as_i64()),
DeferredBroadcast(sync_result),
))
self.log.group_sync(seq)?;
Ok((SequenceNumber::from_raw(seq.as_i64()), DeferredBroadcast))
}
pub fn complete_broadcast(&self, deferred: DeferredBroadcast) {
self.log.broadcast_result(&deferred.0);
}
pub fn complete_broadcast(&self, _deferred: DeferredBroadcast) {}
}
crates/tranquil-store/src/eventlog/commit_loop.rs
+632
View File
@@ -0,0 +1,632 @@
use std::collections::BTreeMap;
use std::io;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::time::{Duration, Instant};
use parking_lot::{Condvar, Mutex};
use tokio::sync::broadcast;
use tracing::warn;
use super::reader::{EventLogReader, RawEvent};
use super::segment_file::ValidEvent;
use super::types::{EventSequence, SegmentId, SegmentOffset};
use super::valid_event_to_raw;
use super::writer::{EventLogWriter, SyncResult};
use crate::io::StorageIO;
const MAX_BATCH_SIZE: usize = 1024;
const MAX_REORDER_PENDING: usize = 65536;
const SYNC_TIMEOUT: Duration = Duration::from_secs(30);
const REORDER_TIMEOUT: Duration = Duration::from_millis(100);
const GAP_ABANDON_TIMEOUT: Duration = Duration::from_secs(5);
pub struct FreezeResponse {
pub synced_through: EventSequence,
pub segment_id: SegmentId,
pub position: SegmentOffset,
}
pub enum WriterRequest {
Append(ValidEvent),
SyncBarrier {
response: flume::Sender<io::Result<SyncResult>>,
},
Freeze {
response: flume::Sender<io::Result<FreezeResponse>>,
resume: flume::Receiver<()>,
},
Shutdown,
}
pub struct WriterNotify {
synced_seq: AtomicU64,
poisoned: AtomicBool,
mutex: Mutex<()>,
cond: Condvar,
}
pub struct PendingBytesBudget {
budget: u64,
state: Mutex<BudgetState>,
cond: Condvar,
}
struct BudgetState {
in_flight: u64,
closed: bool,
}
impl PendingBytesBudget {
pub fn new(budget: u64) -> Self {
let effective = match budget {
0 => u64::MAX,
n => n,
};
Self {
budget: effective,
state: Mutex::new(BudgetState {
in_flight: 0,
closed: false,
}),
cond: Condvar::new(),
}
}
pub fn budget(&self) -> u64 {
self.budget
}
pub fn in_flight(&self) -> u64 {
self.state.lock().in_flight
}
pub fn acquire(&self, bytes: u64) -> io::Result<()> {
let oversized = bytes > self.budget;
let mut guard = self.state.lock();
loop {
if guard.closed {
return Err(io::Error::other("eventlog writer pending budget closed"));
}
let admit = match oversized {
true => guard.in_flight == 0,
false => guard.in_flight.saturating_add(bytes) <= self.budget,
};
if admit {
guard.in_flight = guard.in_flight.saturating_add(bytes);
if oversized {
warn!(
bytes,
budget = self.budget,
"eventlog admitting oversized event past pending budget"
);
}
return Ok(());
}
self.cond.wait(&mut guard);
}
}
pub fn release(&self, bytes: u64) {
if bytes == 0 {
return;
}
let mut guard = self.state.lock();
guard.in_flight = guard.in_flight.saturating_sub(bytes);
self.cond.notify_all();
}
pub fn close(&self) {
let mut guard = self.state.lock();
guard.closed = true;
self.cond.notify_all();
}
}
impl WriterNotify {
pub fn new(initial_synced: u64) -> Self {
Self {
synced_seq: AtomicU64::new(initial_synced),
poisoned: AtomicBool::new(false),
mutex: Mutex::new(()),
cond: Condvar::new(),
}
}
pub fn wait_for_sync(&self, target: EventSequence) -> io::Result<()> {
let target_raw = target.raw();
if self.synced_seq.load(Ordering::Acquire) >= target_raw {
return Ok(());
}
if self.poisoned.load(Ordering::Acquire) {
return Err(io::Error::other("eventlog writer poisoned"));
}
let deadline = Instant::now() + SYNC_TIMEOUT;
let mut guard = self.mutex.lock();
loop {
if self.synced_seq.load(Ordering::Acquire) >= target_raw {
return Ok(());
}
if self.poisoned.load(Ordering::Acquire) {
return Err(io::Error::other("eventlog writer poisoned"));
}
let now = Instant::now();
if now >= deadline {
return Err(io::Error::new(
io::ErrorKind::TimedOut,
"eventlog sync timed out",
));
}
self.cond.wait_for(&mut guard, deadline - now);
}
}
fn update_synced(&self, synced_through: u64) {
self.synced_seq.store(synced_through, Ordering::Release);
let _guard = self.mutex.lock();
self.cond.notify_all();
}
fn poison(&self) {
self.poisoned.store(true, Ordering::Release);
let _guard = self.mutex.lock();
self.cond.notify_all();
}
}
struct ReorderBuffer {
pending: BTreeMap<u64, ValidEvent>,
next_write_seq: u64,
gap_since: Option<Instant>,
}
impl ReorderBuffer {
fn new(next_write_seq: u64) -> Self {
Self {
pending: BTreeMap::new(),
next_write_seq,
gap_since: None,
}
}
fn insert(&mut self, event: ValidEvent) {
if event.seq.raw() < self.next_write_seq {
warn!(
event_seq = event.seq.raw(),
next_write_seq = self.next_write_seq,
"dropping late-arriving event after gap skip"
);
return;
}
self.pending.insert(event.seq.raw(), event);
}
fn is_full(&self) -> bool {
self.pending.len() >= MAX_REORDER_PENDING
}
fn pending_count(&self) -> usize {
self.pending.len()
}
fn drain_contiguous(&mut self) -> Vec<ValidEvent> {
let mut batch = Vec::new();
while let Some(event) = self.pending.remove(&self.next_write_seq) {
self.next_write_seq = event.seq.next().raw();
batch.push(event);
}
if batch.is_empty() && !self.pending.is_empty() {
if self.gap_since.is_none() {
self.gap_since = Some(Instant::now());
}
} else {
self.gap_since = None;
}
batch
}
fn should_skip_gap(&self) -> bool {
self.gap_since
.is_some_and(|since| since.elapsed() >= GAP_ABANDON_TIMEOUT)
}
fn skip_to_first_available(&mut self) -> Vec<ValidEvent> {
let first_available = match self.pending.keys().next() {
Some(&seq) => seq,
None => return Vec::new(),
};
warn!(
expected = self.next_write_seq,
skipping_to = first_available,
"eventlog writer skipping gap after timeout"
);
self.next_write_seq = first_available;
self.gap_since = None;
self.drain_contiguous()
}
fn has_pending(&self) -> bool {
!self.pending.is_empty()
}
}
struct WriterCtx<'a, S: StorageIO> {
reader: &'a EventLogReader<S>,
broadcast_tx: &'a broadcast::Sender<RawEvent>,
notify: &'a WriterNotify,
synced_seq: &'a AtomicU64,
pending_bytes: &'a PendingBytesBudget,
}
fn post_sync<S: StorageIO>(result: &mut SyncResult, ctx: &WriterCtx<'_, S>) {
let synced = result.synced_through.raw();
let flushed = std::mem::take(&mut result.flushed_events);
if let (Some(first), Some(last)) = (flushed.first(), flushed.last()) {
ctx.reader.extend_active_range(first.seq, last.seq);
}
ctx.synced_seq.store(synced, Ordering::Release);
ctx.notify.update_synced(synced);
let released = flushed
.iter()
.map(|e| e.payload.len() as u64)
.fold(0u64, u64::saturating_add);
flushed.into_iter().for_each(|e| {
let _ = ctx.broadcast_tx.send(valid_event_to_raw(e));
});
ctx.pending_bytes.release(released);
}
fn flush_and_notify<S: StorageIO>(writer: &mut EventLogWriter<S>, ctx: &WriterCtx<'_, S>) -> bool {
match writer.sync() {
Ok(mut result) => {
post_sync(&mut result, ctx);
match writer.rotate_if_needed() {
Ok(Some(sealed_id)) => {
let new_id = writer.active_segment_id();
if let Err(e) = ctx.reader.on_segment_rotated(sealed_id, new_id) {
warn!(error = %e, "eventlog rotation notification failed");
}
}
Ok(None) => {}
Err(e) => {
warn!(error = %e, "eventlog rotation deferred");
}
}
true
}
Err(e) => {
warn!(error = %e, "eventlog sync failed, poisoning writer");
ctx.notify.poison();
false
}
}
}
fn append_batch<S: StorageIO>(
writer: &mut EventLogWriter<S>,
events: Vec<ValidEvent>,
notify: &WriterNotify,
) -> bool {
let ok = events
.into_iter()
.try_for_each(|event| writer.append_valid_event(event));
if let Err(e) = ok {
warn!(error = %e, "eventlog append failed, poisoning writer");
notify.poison();
return false;
}
true
}
fn handle_sync_barrier<S: StorageIO>(
writer: &mut EventLogWriter<S>,
response: flume::Sender<io::Result<SyncResult>>,
ctx: &WriterCtx<'_, S>,
) {
let mut result = writer.sync();
if let Ok(ref mut sync_result) = result {
post_sync(sync_result, ctx);
}
let _ = response.send(result);
}
fn handle_freeze<S: StorageIO>(
writer: &mut EventLogWriter<S>,
response: flume::Sender<io::Result<FreezeResponse>>,
resume: flume::Receiver<()>,
ctx: &WriterCtx<'_, S>,
) {
let result = writer.sync().map(|mut sync_result| {
post_sync(&mut sync_result, ctx);
FreezeResponse {
synced_through: sync_result.synced_through,
segment_id: sync_result.segment_id,
position: sync_result.position,
}
});
let _ = response.send(result);
let _ = resume.recv();
}
struct CloseOnDrop<'a>(&'a PendingBytesBudget);
impl<'a> Drop for CloseOnDrop<'a> {
fn drop(&mut self) {
self.0.close();
}
}
fn writer_loop<S: StorageIO>(
receiver: &flume::Receiver<WriterRequest>,
writer: &mut EventLogWriter<S>,
ctx: &WriterCtx<'_, S>,
) {
let _close = CloseOnDrop(ctx.pending_bytes);
let mut reorder = ReorderBuffer::new(writer.current_seq().next().raw());
loop {
if ctx.notify.poisoned.load(Ordering::Acquire) {
let _ = writer.shutdown();
break;
}
let recv_result = match reorder.has_pending() {
true => receiver.recv_timeout(REORDER_TIMEOUT),
false => receiver
.recv()
.map_err(|_| flume::RecvTimeoutError::Disconnected),
};
match recv_result {
Err(flume::RecvTimeoutError::Disconnected) => {
if reorder.has_pending() {
let batch = reorder.skip_to_first_available();
if !batch.is_empty() && append_batch(writer, batch, ctx.notify) {
flush_and_notify(writer, ctx);
}
}
let _ = writer.shutdown();
break;
}
Err(flume::RecvTimeoutError::Timeout) => {
if reorder.should_skip_gap() {
let batch = reorder.skip_to_first_available();
if !batch.is_empty() && append_batch(writer, batch, ctx.notify) {
flush_and_notify(writer, ctx);
}
}
continue;
}
Ok(WriterRequest::Shutdown) => {
if reorder.has_pending() {
let batch = reorder.skip_to_first_available();
if !batch.is_empty() {
append_batch(writer, batch, ctx.notify);
}
}
let _ = writer.shutdown();
break;
}
Ok(WriterRequest::SyncBarrier { response }) => {
let batch = reorder.drain_contiguous();
if !batch.is_empty() {
append_batch(writer, batch, ctx.notify);
}
handle_sync_barrier(writer, response, ctx);
}
Ok(WriterRequest::Freeze { response, resume }) => {
if reorder.has_pending() {
let batch = reorder.skip_to_first_available();
if !batch.is_empty() {
append_batch(writer, batch, ctx.notify);
}
}
handle_freeze(writer, response, resume, ctx);
reorder = ReorderBuffer::new(writer.current_seq().next().raw());
}
Ok(WriterRequest::Append(event)) => {
reorder.insert(event);
while reorder.pending_count() < MAX_BATCH_SIZE {
match receiver.try_recv() {
Ok(WriterRequest::Append(e)) => reorder.insert(e),
Ok(WriterRequest::Shutdown) => {
let batch = reorder.skip_to_first_available();
if !batch.is_empty() {
append_batch(writer, batch, ctx.notify);
}
let _ = writer.shutdown();
return;
}
Ok(WriterRequest::SyncBarrier { response }) => {
let batch = reorder.drain_contiguous();
if !batch.is_empty() && append_batch(writer, batch, ctx.notify) {
flush_and_notify(writer, ctx);
}
handle_sync_barrier(writer, response, ctx);
break;
}
Ok(WriterRequest::Freeze { response, resume }) => {
let batch = reorder.drain_contiguous();
if !batch.is_empty() && append_batch(writer, batch, ctx.notify) {
flush_and_notify(writer, ctx);
}
handle_freeze(writer, response, resume, ctx);
reorder = ReorderBuffer::new(writer.current_seq().next().raw());
break;
}
Err(_) => break,
}
}
let batch = reorder.drain_contiguous();
if !batch.is_empty() && append_batch(writer, batch, ctx.notify) {
flush_and_notify(writer, ctx);
}
if reorder.is_full() {
warn!(
pending = reorder.pending_count(),
"reorder buffer at capacity, force-skipping gap"
);
let batch = reorder.skip_to_first_available();
if !batch.is_empty() && append_batch(writer, batch, ctx.notify) {
flush_and_notify(writer, ctx);
}
}
}
}
}
}
fn log_thread_panic(payload: Box<dyn std::any::Any + Send>) {
let msg = payload
.downcast_ref::<&str>()
.copied()
.or_else(|| payload.downcast_ref::<String>().map(|s| s.as_str()))
.unwrap_or("unknown panic");
tracing::error!(panic = msg, "eventlog commit thread panicked");
}
pub struct CommitThread {
sender: flume::Sender<WriterRequest>,
handle: Mutex<Option<std::thread::JoinHandle<()>>>,
}
impl CommitThread {
pub fn spawn<S: StorageIO + 'static>(
mut writer: EventLogWriter<S>,
reader: Arc<EventLogReader<S>>,
broadcast_tx: broadcast::Sender<RawEvent>,
notify: Arc<WriterNotify>,
synced_seq: Arc<AtomicU64>,
pending_bytes: Arc<PendingBytesBudget>,
) -> io::Result<Self> {
let (sender, receiver) = flume::unbounded();
let handle = std::thread::Builder::new()
.name("eventlog-commit".into())
.spawn(move || {
let ctx = WriterCtx {
reader: &reader,
broadcast_tx: &broadcast_tx,
notify: &notify,
synced_seq: &synced_seq,
pending_bytes: &pending_bytes,
};
writer_loop(&receiver, &mut writer, &ctx);
})
.map_err(io::Error::other)?;
Ok(Self {
sender,
handle: Mutex::new(Some(handle)),
})
}
pub fn sender(&self) -> &flume::Sender<WriterRequest> {
&self.sender
}
pub fn shutdown(&self) {
let _ = self.sender.send(WriterRequest::Shutdown);
if let Some(handle) = self.handle.lock().take()
&& let Err(payload) = handle.join()
{
log_thread_panic(payload);
}
}
}
impl Drop for CommitThread {
fn drop(&mut self) {
let _ = self.sender.try_send(WriterRequest::Shutdown);
if let Some(handle) = self.handle.lock().take()
&& let Err(payload) = handle.join()
{
log_thread_panic(payload);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
use std::time::Duration;
#[test]
fn close_unblocks_waiter() {
let budget = Arc::new(PendingBytesBudget::new(1024));
budget.acquire(1024).unwrap();
assert_eq!(budget.in_flight(), 1024);
let budget_blocked = Arc::clone(&budget);
let blocked = std::thread::spawn(move || budget_blocked.acquire(256));
std::thread::sleep(Duration::from_millis(100));
assert!(
!blocked.is_finished(),
"acquire must block when budget exhausted"
);
budget.close();
let result = blocked.join().expect("blocked thread must not panic");
assert!(
result.is_err(),
"acquire must error after close, got {result:?}"
);
}
#[test]
fn close_makes_subsequent_acquire_fail_immediately() {
let budget = PendingBytesBudget::new(1024);
budget.close();
assert!(budget.acquire(1).is_err());
}
#[test]
fn release_after_full_unblocks_waiter() {
let budget = Arc::new(PendingBytesBudget::new(1024));
budget.acquire(1024).unwrap();
let budget_blocked = Arc::clone(&budget);
let blocked = std::thread::spawn(move || budget_blocked.acquire(256));
std::thread::sleep(Duration::from_millis(100));
assert!(!blocked.is_finished());
budget.release(1024);
blocked
.join()
.expect("thread panic")
.expect("acquire after release must succeed");
assert_eq!(budget.in_flight(), 256);
}
#[test]
fn zero_budget_means_unbounded() {
let budget = PendingBytesBudget::new(0);
budget.acquire(u64::MAX / 2).unwrap();
budget.acquire(u64::MAX / 4).unwrap();
assert!(budget.in_flight() >= u64::MAX / 2);
}
}
crates/tranquil-store/src/eventlog/manager.rs
+82 -33
View File
@@ -11,8 +11,19 @@ use super::segment_file::SEGMENT_HEADER_SIZE;
use super::segment_index::SegmentIndex;
use super::types::{SegmentId, SegmentOffset};
pub(crate) const SEGMENT_FILE_EXTENSION: &str = "tqe";
pub const SEGMENT_FILE_EXTENSION: &str = "tqe";
pub(crate) const INDEX_FILE_EXTENSION: &str = "tqi";
pub(crate) const SIDECAR_FILE_EXTENSION: &str = "tqs";
pub fn segment_path(dir: &Path, id: SegmentId) -> PathBuf {
dir.join(format!("{id}.{SEGMENT_FILE_EXTENSION}"))
}
pub fn parse_segment_id(path: &Path) -> Option<SegmentId> {
let stem = path.file_stem()?.to_str()?;
let ext = path.extension()?.to_str()?;
(ext == SEGMENT_FILE_EXTENSION).then(|| stem.parse::<u32>().ok().map(SegmentId::new))?
}
struct CachedSegmentHandle {
fd: FileId,
@@ -34,6 +45,10 @@ impl<S: StorageIO> SegmentManager<S> {
max_segment_size > SEGMENT_HEADER_SIZE as u64,
"max_segment_size ({max_segment_size}) must exceed SEGMENT_HEADER_SIZE ({SEGMENT_HEADER_SIZE})"
);
assert!(
max_segment_size <= u32::MAX as u64,
"max_segment_size ({max_segment_size}) must not exceed u32::MAX (sidecar offsets are u32)"
);
io.mkdir(&segments_dir)?;
Ok(Self {
io,
@@ -57,8 +72,7 @@ impl<S: StorageIO> SegmentManager<S> {
}
pub fn segment_path(&self, id: SegmentId) -> PathBuf {
self.segments_dir
.join(format!("{id}.{SEGMENT_FILE_EXTENSION}"))
segment_path(&self.segments_dir, id)
}
pub fn index_path(&self, id: SegmentId) -> PathBuf {
@@ -66,17 +80,14 @@ impl<S: StorageIO> SegmentManager<S> {
.join(format!("{id}.{INDEX_FILE_EXTENSION}"))
}
pub fn sidecar_path(&self, id: SegmentId) -> PathBuf {
self.segments_dir
.join(format!("{id}.{SIDECAR_FILE_EXTENSION}"))
}
pub fn list_segments(&self) -> io::Result<Vec<SegmentId>> {
let entries = self.io.list_dir(&self.segments_dir)?;
let mut ids: Vec<SegmentId> = entries
.iter()
.filter_map(|path| {
let stem = path.file_stem()?.to_str()?;
let ext = path.extension()?.to_str()?;
(ext == SEGMENT_FILE_EXTENSION)
.then(|| stem.parse::<u32>().ok().map(SegmentId::new))?
})
.collect();
let mut ids: Vec<SegmentId> = entries.iter().filter_map(|p| parse_segment_id(p)).collect();
ids.sort();
Ok(ids)
}
@@ -222,11 +233,13 @@ impl<S: StorageIO> SegmentManager<S> {
let _ = self.io.close(entry.fd);
}
}
match self.io.delete(&self.index_path(id)) {
Ok(()) => {}
Err(e) if e.kind() == io::ErrorKind::NotFound => {}
Err(e) => return Err(e),
}
[self.index_path(id), self.sidecar_path(id)]
.iter()
.try_for_each(|path| match self.io.delete(path) {
Ok(()) => Ok(()),
Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()),
Err(e) => Err(e),
})?;
self.io.delete(&self.segment_path(id))?;
self.io.sync_dir(&self.segments_dir)?;
self.retention_epoch.fetch_add(1, Ordering::Relaxed);
@@ -260,7 +273,7 @@ mod tests {
use crate::eventlog::segment_file::{SegmentWriter, ValidEvent};
use crate::eventlog::segment_index::{DEFAULT_INDEX_INTERVAL, rebuild_from_segment};
use crate::eventlog::types::{
DidHash, EventSequence, EventTypeTag, SegmentOffset, TimestampMicros,
DidHash, EventSequence, EventTypeTag, MAX_EVENT_PAYLOAD, SegmentOffset, TimestampMicros,
};
use crate::sim::SimulatedIO;
@@ -431,8 +444,14 @@ mod tests {
fn seal_segment_persists_index_and_marks_sealed() {
let mgr = setup_manager(64 * 1024);
let fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
let mut writer =
SegmentWriter::new(mgr.io(), fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let mut writer = SegmentWriter::new(
mgr.io(),
fd,
SegmentId::new(1),
EventSequence::new(1),
MAX_EVENT_PAYLOAD,
)
.unwrap();
(1u64..=10).for_each(|i| {
writer
@@ -441,7 +460,8 @@ mod tests {
});
writer.sync(mgr.io()).unwrap();
let (index, _) = rebuild_from_segment(mgr.io(), fd, DEFAULT_INDEX_INTERVAL).unwrap();
let (index, _) =
rebuild_from_segment(mgr.io(), fd, DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD).unwrap();
assert!(!mgr.is_sealed(SegmentId::new(1)));
mgr.seal_segment(SegmentId::new(1), &index).unwrap();
@@ -457,14 +477,21 @@ mod tests {
fn delete_segment_removes_files_and_handle() {
let mgr = setup_manager(64 * 1024);
let fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
let mut writer =
SegmentWriter::new(mgr.io(), fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let mut writer = SegmentWriter::new(
mgr.io(),
fd,
SegmentId::new(1),
EventSequence::new(1),
MAX_EVENT_PAYLOAD,
)
.unwrap();
writer
.append_event(mgr.io(), &test_event(1, b"will be deleted"))
.unwrap();
writer.sync(mgr.io()).unwrap();
let (index, _) = rebuild_from_segment(mgr.io(), fd, DEFAULT_INDEX_INTERVAL).unwrap();
let (index, _) =
rebuild_from_segment(mgr.io(), fd, DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD).unwrap();
mgr.seal_segment(SegmentId::new(1), &index).unwrap();
let epoch_before = mgr.retention_epoch();
@@ -498,8 +525,14 @@ mod tests {
let mgr = setup_manager(1024);
let fd1 = mgr.open_for_append(SegmentId::new(1)).unwrap();
let mut writer1 =
SegmentWriter::new(mgr.io(), fd1, SegmentId::new(1), EventSequence::new(1)).unwrap();
let mut writer1 = SegmentWriter::new(
mgr.io(),
fd1,
SegmentId::new(1),
EventSequence::new(1),
MAX_EVENT_PAYLOAD,
)
.unwrap();
writer1
.append_event(mgr.io(), &test_event(1, b"first segment"))
.unwrap();
@@ -508,14 +541,16 @@ mod tests {
let (id2, fd2) = mgr.prepare_rotation(SegmentId::new(1)).unwrap();
mgr.commit_rotation(id2, fd2);
let mut writer2 = SegmentWriter::new(mgr.io(), fd2, id2, EventSequence::new(2)).unwrap();
let mut writer2 =
SegmentWriter::new(mgr.io(), fd2, id2, EventSequence::new(2), MAX_EVENT_PAYLOAD)
.unwrap();
writer2
.append_event(mgr.io(), &test_event(2, b"second segment"))
.unwrap();
writer2.sync(mgr.io()).unwrap();
let fd1_read = mgr.open_for_read(SegmentId::new(1)).unwrap();
let events1 = crate::eventlog::SegmentReader::open(mgr.io(), fd1_read)
let events1 = crate::eventlog::SegmentReader::open(mgr.io(), fd1_read, MAX_EVENT_PAYLOAD)
.unwrap()
.valid_prefix()
.unwrap();
@@ -523,7 +558,7 @@ mod tests {
assert_eq!(events1[0].payload, b"first segment");
let fd2_read = mgr.open_for_read(id2).unwrap();
let events2 = crate::eventlog::SegmentReader::open(mgr.io(), fd2_read)
let events2 = crate::eventlog::SegmentReader::open(mgr.io(), fd2_read, MAX_EVENT_PAYLOAD)
.unwrap()
.valid_prefix()
.unwrap();
@@ -535,7 +570,14 @@ mod tests {
fn seal_then_append_errors() {
let mgr = setup_manager(64 * 1024);
let fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
SegmentWriter::new(mgr.io(), fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
SegmentWriter::new(
mgr.io(),
fd,
SegmentId::new(1),
EventSequence::new(1),
MAX_EVENT_PAYLOAD,
)
.unwrap();
let index = SegmentIndex::new();
mgr.seal_segment(SegmentId::new(1), &index).unwrap();
@@ -569,14 +611,21 @@ mod tests {
fn open_for_read_does_not_infer_sealed_from_index_file() {
let mgr = setup_manager(64 * 1024);
let fd = mgr.open_for_append(SegmentId::new(1)).unwrap();
let mut writer =
SegmentWriter::new(mgr.io(), fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let mut writer = SegmentWriter::new(
mgr.io(),
fd,
SegmentId::new(1),
EventSequence::new(1),
MAX_EVENT_PAYLOAD,
)
.unwrap();
writer
.append_event(mgr.io(), &test_event(1, b"sealed test"))
.unwrap();
writer.sync(mgr.io()).unwrap();
let (index, _) = rebuild_from_segment(mgr.io(), fd, DEFAULT_INDEX_INTERVAL).unwrap();
let (index, _) =
rebuild_from_segment(mgr.io(), fd, DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD).unwrap();
mgr.seal_segment(SegmentId::new(1), &index).unwrap();
mgr.handles.write().remove(&SegmentId::new(1));
crates/tranquil-store/src/eventlog/mod.rs
+237 -102
View File
@@ -1,10 +1,12 @@
mod bridge;
mod commit_loop;
mod manager;
mod notifier;
mod payload;
mod reader;
mod segment_file;
mod segment_index;
mod sidecar;
mod types;
mod writer;
@@ -12,10 +14,9 @@ use std::collections::VecDeque;
use std::io;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use parking_lot::Mutex;
use tokio::sync::broadcast;
use tracing::warn;
use tranquil_db_traits::{RepoEventType, SequencedEvent};
@@ -25,27 +26,31 @@ use crate::blockstore::BlocksSynced;
use crate::fsync_order::PostBlockstoreHook;
use crate::io::StorageIO;
use commit_loop::{CommitThread, FreezeResponse, PendingBytesBudget, WriterNotify, WriterRequest};
pub use bridge::{DeferredBroadcast, EventLogBridge};
pub use manager::SegmentManager;
pub use manager::{SEGMENT_FILE_EXTENSION, SegmentManager, parse_segment_id, segment_path};
pub use notifier::EventLogNotifier;
pub use payload::{
EventPayload, PayloadError, decode_payload, encode_payload, encode_payload_with_mutations,
to_sequenced_event, validate_payload_size,
};
pub use reader::{EventLogReader, RawEvent};
pub use reader::{EventLogReader, RawEvent, SequenceContiguityResult, SequenceGap};
pub use segment_file::{
EVENT_HEADER_SIZE, EVENT_RECORD_OVERHEAD, ReadEventRecord, SEGMENT_FORMAT_VERSION,
SEGMENT_HEADER_SIZE, SEGMENT_MAGIC, SegmentReader, SegmentWriter, ValidEvent,
ValidateEventRecord, decode_event_record, encode_event_record, validate_event_record,
};
pub use segment_index::{DEFAULT_INDEX_INTERVAL, SegmentIndex, rebuild_from_segment};
pub use sidecar::{SidecarEntry, SidecarIndex, build_sidecar_from_segment};
pub use types::{
DEFAULT_SEGMENT_SIZE, DidHash, EventLength, EventSequence, EventTypeTag, MAX_EVENT_PAYLOAD,
SegmentId, SegmentOffset, TimestampMicros,
DEFAULT_MAX_EVENT_PAYLOAD, DEFAULT_SEGMENT_SIZE, DidHash, EventLength, EventSequence,
EventTypeTag, MAX_EVENT_PAYLOAD, SegmentId, SegmentOffset, TimestampMicros,
};
pub use writer::{EventLogWriter, SyncResult};
const DEFAULT_BROADCAST_BUFFER: usize = 16384;
pub const DEFAULT_PENDING_BYTES_BUDGET: u64 = 1024 * 1024 * 1024;
pub struct EventWithMutations {
pub event: SequencedEvent,
@@ -58,6 +63,9 @@ pub struct EventLogConfig {
pub index_interval: usize,
pub broadcast_buffer: usize,
pub use_mmap: bool,
pub skip_sealed_checksum: bool,
pub pending_bytes_budget: u64,
pub max_event_payload: u32,
}
impl Default for EventLogConfig {
@@ -68,47 +76,120 @@ impl Default for EventLogConfig {
index_interval: DEFAULT_INDEX_INTERVAL,
broadcast_buffer: DEFAULT_BROADCAST_BUFFER,
use_mmap: true,
skip_sealed_checksum: false,
pending_bytes_budget: DEFAULT_PENDING_BYTES_BUDGET,
max_event_payload: DEFAULT_MAX_EVENT_PAYLOAD,
}
}
}
pub struct EventLogSnapshotState {
pub max_seq: EventSequence,
pub active_segment_id: SegmentId,
pub active_segment_position: SegmentOffset,
pub sealed_segments: Vec<SegmentId>,
}
pub struct EventLogFreezeGuard {
_resume: Option<flume::Sender<()>>,
}
impl Drop for EventLogFreezeGuard {
fn drop(&mut self) {
if let Some(resume) = self._resume.take() {
let _ = resume.send(());
}
}
}
pub struct EventLog<S: StorageIO> {
writer: Mutex<EventLogWriter<S>>,
commit_thread: CommitThread,
reader: Arc<EventLogReader<S>>,
manager: Arc<SegmentManager<S>>,
broadcast_tx: broadcast::Sender<RawEvent>,
synced_seq: AtomicU64,
consecutive_sync_failures: AtomicU32,
synced_seq: Arc<AtomicU64>,
notify: Arc<WriterNotify>,
pending_bytes: Arc<PendingBytesBudget>,
next_seq: AtomicU64,
max_payload: u32,
}
impl<S: StorageIO> EventLog<S> {
impl<S: StorageIO + 'static> EventLog<S> {
pub fn open(config: EventLogConfig, io: S) -> io::Result<Self> {
let max_payload = config.max_event_payload;
let manager = Arc::new(SegmentManager::new(
io,
config.segments_dir,
config.max_segment_size,
)?);
let writer = EventLogWriter::open(Arc::clone(&manager), config.index_interval)?;
let writer =
EventLogWriter::open(Arc::clone(&manager), config.index_interval, max_payload)?;
let synced = writer.synced_seq();
let initial_next_seq = writer.current_seq().next();
let reader = Arc::new(EventLogReader::new(Arc::clone(&manager), config.use_mmap));
let reader = Arc::new(EventLogReader::new(
Arc::clone(&manager),
config.use_mmap,
config.skip_sealed_checksum,
max_payload,
));
reader.set_active_segment(writer.active_segment_id());
reader.seed_index(writer.active_segment_id(), writer.active_index_snapshot());
reader.refresh_segment_ranges()?;
let (broadcast_tx, _) = broadcast::channel(config.broadcast_buffer);
let synced_seq = Arc::new(AtomicU64::new(synced.raw()));
let notify = Arc::new(WriterNotify::new(synced.raw()));
let pending_bytes = Arc::new(PendingBytesBudget::new(config.pending_bytes_budget));
let commit_thread = CommitThread::spawn(
writer,
Arc::clone(&reader),
broadcast_tx.clone(),
Arc::clone(&notify),
Arc::clone(&synced_seq),
Arc::clone(&pending_bytes),
)?;
Ok(Self {
writer: Mutex::new(writer),
commit_thread,
reader,
manager,
broadcast_tx,
synced_seq: AtomicU64::new(synced.raw()),
consecutive_sync_failures: AtomicU32::new(0),
synced_seq,
notify,
pending_bytes,
next_seq: AtomicU64::new(initial_next_seq.raw()),
max_payload,
})
}
pub fn max_payload(&self) -> u32 {
self.max_payload
}
pub fn pending_bytes_in_flight(&self) -> u64 {
self.pending_bytes.in_flight()
}
pub fn pending_bytes_budget(&self) -> u64 {
self.pending_bytes.budget()
}
fn reserve_seq(&self) -> EventSequence {
let raw = self.next_seq.fetch_add(1, Ordering::Relaxed);
EventSequence::new(raw)
}
fn send_append(&self, event: ValidEvent) -> io::Result<()> {
self.commit_thread
.sender()
.send(WriterRequest::Append(event))
.map_err(|_| io::Error::other("eventlog writer thread terminated"))
}
pub fn append_event(
&self,
did: &Did,
@@ -127,13 +208,31 @@ impl<S: StorageIO> EventLog<S> {
) -> io::Result<EventSequence> {
let did_hash = DidHash::from_did(did.as_str());
let tag = repo_event_type_to_tag(event_type);
validate_payload_size(&payload)
validate_payload_size(&payload, self.max_payload)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
self.writer.lock().append(did_hash, tag, payload)
self.pending_bytes.acquire(payload.len() as u64)?;
let seq = self.reserve_seq();
let timestamp = TimestampMicros::now();
let event = ValidEvent {
seq,
timestamp,
did_hash,
event_type: tag,
payload,
};
self.send_append(event)?;
Ok(seq)
}
pub fn group_sync(&self, my_seq: EventSequence) -> io::Result<()> {
self.notify.wait_for_sync(my_seq)
}
pub fn sync(&self) -> io::Result<SyncResult> {
self.sync_and_broadcast()
self.sync_data()
}
pub fn append_and_sync(
@@ -143,7 +242,7 @@ impl<S: StorageIO> EventLog<S> {
event: &SequencedEvent,
) -> io::Result<EventSequence> {
let seq = self.append_event(did, event_type, event)?;
self.sync_and_broadcast()?;
self.group_sync(seq)?;
Ok(seq)
}
@@ -151,44 +250,21 @@ impl<S: StorageIO> EventLog<S> {
&self,
events: Vec<(&Did, RepoEventType, &SequencedEvent)>,
) -> io::Result<Vec<EventSequence>> {
let mut writer = self.writer.lock();
events
.iter()
.map(|(did, event_type, event)| {
let did_hash = DidHash::from_did(did.as_str());
let tag = repo_event_type_to_tag(*event_type);
let payload = encode_payload(event);
validate_payload_size(&payload)
.map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?;
writer.append(did_hash, tag, payload)
})
.map(|(did, event_type, event)| self.append_event(did, *event_type, event))
.collect()
}
pub fn sync_data(&self) -> io::Result<SyncResult> {
let mut writer = self.writer.lock();
let result = writer.sync()?;
self.synced_seq
.store(result.synced_through.raw(), Ordering::Release);
if let (Some(first), Some(last)) =
(result.flushed_events.first(), result.flushed_events.last())
{
self.reader.extend_active_range(first.seq, last.seq);
}
Ok(result)
}
pub fn broadcast_result(&self, result: &SyncResult) {
result.flushed_events.iter().for_each(|e| {
let _ = self.broadcast_tx.send(valid_event_to_raw(e));
});
}
pub fn sync_and_broadcast(&self) -> io::Result<SyncResult> {
let result = self.sync_data()?;
self.broadcast_result(&result);
Ok(result)
let (resp_tx, resp_rx) = flume::bounded(1);
self.commit_thread
.sender()
.send(WriterRequest::SyncBarrier { response: resp_tx })
.map_err(|_| io::Error::other("eventlog writer thread terminated"))?;
resp_rx
.recv()
.map_err(|_| io::Error::other("eventlog writer thread terminated"))?
}
pub fn get_events_since(
@@ -253,40 +329,25 @@ impl<S: StorageIO> EventLog<S> {
}
pub fn maybe_rotate(&self) -> io::Result<bool> {
let (sealed_id, new_active_id) = {
let mut writer = self.writer.lock();
match writer.rotate_if_needed()? {
None => return Ok(false),
Some(sealed_id) => (sealed_id, writer.active_segment_id()),
}
};
self.reader.on_segment_rotated(sealed_id, new_active_id)?;
Ok(true)
Ok(false)
}
pub fn run_retention(&self, max_age: Duration) -> io::Result<usize> {
self.run_retention_at(TimestampMicros::now(), max_age)
}
pub fn run_retention_at(&self, now: TimestampMicros, max_age: Duration) -> io::Result<usize> {
let max_age_us = u64::try_from(max_age.as_micros()).unwrap_or(u64::MAX);
let cutoff_us = TimestampMicros::now().raw().saturating_sub(max_age_us);
let active_id = self.writer.lock().active_segment_id();
let cutoff_us = now.raw().saturating_sub(max_age_us);
let sync_result = self.sync_data()?;
let active_id = sync_result.segment_id;
let segments = self.manager.list_segments()?;
let deleted = segments
.iter()
.take_while(|&&id| id != active_id)
.filter(|&&id| {
self.reader
.load_index(id)
.ok()
.and_then(|idx| idx.last_seq())
.and_then(|seq| {
self.reader
.read_event_at(seq)
.ok()
.flatten()
.map(|e| e.timestamp.raw() < cutoff_us)
})
.unwrap_or(false)
})
.filter(|&&id| self.segment_past_cutoff(id, cutoff_us))
.copied()
.collect::<Vec<_>>();
@@ -294,6 +355,7 @@ impl<S: StorageIO> EventLog<S> {
self.manager.delete_segment(id)?;
self.reader.invalidate_index(id);
self.reader.invalidate_mmap(id);
self.reader.invalidate_sidecar(id);
Ok(())
})?;
@@ -304,6 +366,44 @@ impl<S: StorageIO> EventLog<S> {
Ok(deleted.len())
}
fn segment_past_cutoff(&self, id: SegmentId, cutoff_us: u64) -> bool {
let idx = match self.reader.load_index(id) {
Ok(idx) => idx,
Err(e) => {
warn!(
segment_id = id.raw(),
error = %e,
"eventlog retention: failed to load segment index, keeping segment"
);
return false;
}
};
let last_seq = match idx.last_seq() {
Some(seq) => seq,
None => return false,
};
match self.reader.read_event_at(last_seq) {
Ok(Some(event)) => event.timestamp.raw() < cutoff_us,
Ok(None) => {
warn!(
segment_id = id.raw(),
last_seq = last_seq.raw(),
"eventlog retention: index reports last_seq but read_event_at returned None, keeping segment"
);
false
}
Err(e) => {
warn!(
segment_id = id.raw(),
last_seq = last_seq.raw(),
error = %e,
"eventlog retention: failed to read last event, keeping segment"
);
false
}
}
}
pub fn segment_count(&self) -> usize {
self.manager.list_segments().map_or(0, |s| s.len())
}
@@ -317,8 +417,55 @@ impl<S: StorageIO> EventLog<S> {
})
}
pub fn snapshot_state(&self) -> io::Result<EventLogSnapshotState> {
let (state, _guard) = self.freeze()?;
Ok(state)
}
pub fn freeze(&self) -> io::Result<(EventLogSnapshotState, EventLogFreezeGuard)> {
let (resp_tx, resp_rx) = flume::bounded(1);
let (resume_tx, resume_rx) = flume::bounded(1);
self.commit_thread
.sender()
.send(WriterRequest::Freeze {
response: resp_tx,
resume: resume_rx,
})
.map_err(|_| io::Error::other("eventlog writer thread terminated"))?;
let freeze_resp: FreezeResponse = resp_rx
.recv()
.map_err(|_| io::Error::other("eventlog writer thread terminated"))??;
let all_segments = self.manager.list_segments()?;
let sealed_segments: Vec<SegmentId> = all_segments
.into_iter()
.filter(|&id| id != freeze_resp.segment_id)
.collect();
let state = EventLogSnapshotState {
max_seq: freeze_resp.synced_through,
active_segment_id: freeze_resp.segment_id,
active_segment_position: freeze_resp.position,
sealed_segments,
};
Ok((
state,
EventLogFreezeGuard {
_resume: Some(resume_tx),
},
))
}
pub fn segments_dir(&self) -> &std::path::Path {
self.manager.segments_dir()
}
pub fn shutdown(&self) -> io::Result<()> {
self.writer.lock().shutdown()
self.commit_thread.shutdown();
Ok(())
}
pub fn subscriber(&self, start_seq: EventSequence) -> EventLogSubscriber<S> {
@@ -337,34 +484,22 @@ impl<S: StorageIO> EventLog<S> {
&self.manager
}
pub fn consecutive_sync_failures(&self) -> u32 {
self.consecutive_sync_failures.load(Ordering::Relaxed)
fn last_assigned_seq(&self) -> EventSequence {
let raw = self.next_seq.load(Ordering::Acquire);
match raw.checked_sub(1) {
Some(0) | None => EventSequence::BEFORE_ALL,
Some(n) => EventSequence::new(n),
}
}
}
impl<S: StorageIO + Send + Sync> PostBlockstoreHook for EventLog<S> {
impl<S: StorageIO + Send + Sync + 'static> PostBlockstoreHook for EventLog<S> {
fn on_blocks_synced(&self, _proof: &BlocksSynced) -> io::Result<()> {
match self.sync_and_broadcast() {
Ok(_) => {
self.consecutive_sync_failures.store(0, Ordering::Relaxed);
if let Err(e) = self.maybe_rotate() {
warn!(error = %e, "eventlog rotation deferred");
}
Ok(())
}
Err(e) => {
let count = self
.consecutive_sync_failures
.fetch_add(1, Ordering::Relaxed)
.saturating_add(1);
warn!(
error = %e,
consecutive_failures = count,
"eventlog sync failed after blockstore commit"
);
Err(e)
}
let target = self.last_assigned_seq();
if target == EventSequence::BEFORE_ALL {
return Ok(());
}
self.notify.wait_for_sync(target)
}
}
@@ -469,13 +604,13 @@ impl<S: StorageIO> EventLogSubscriber<S> {
}
}
fn valid_event_to_raw(e: &ValidEvent) -> RawEvent {
fn valid_event_to_raw(e: ValidEvent) -> RawEvent {
RawEvent {
seq: e.seq,
timestamp: e.timestamp,
did_hash: e.did_hash,
event_type: e.event_type,
payload: bytes::Bytes::from(e.payload.clone()),
payload: bytes::Bytes::from(e.payload),
}
}
crates/tranquil-store/src/eventlog/payload.rs
+49 -31
View File
@@ -1,14 +1,16 @@
use serde::{Deserialize, Serialize};
use tranquil_db_traits::{AccountStatus, SequenceNumber, SequencedEvent};
use tranquil_db_traits::{
AccountStatus, EventBlockInline, EventBlocks, SequenceNumber, SequencedEvent,
};
use tranquil_types::{CidLink, Did, Handle};
use crate::eventlog::reader::RawEvent;
use crate::eventlog::types::MAX_EVENT_PAYLOAD;
const PAYLOAD_VERSION: u8 = 1;
const LARGE_PAYLOAD_WARNING_THRESHOLD: usize = 1024 * 1024;
pub(crate) const PAYLOAD_VERSION_V1: u8 = 1;
const CURRENT_PAYLOAD_VERSION: u8 = PAYLOAD_VERSION_V1;
const LARGE_PAYLOAD_WARNING_THRESHOLD: usize = 4 * 1024 * 1024;
const CID_BYTE_LEN: usize = 36;
pub(crate) const CID_BYTE_LEN: usize = 36;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EventPayload {
@@ -18,7 +20,7 @@ pub struct EventPayload {
pub prev_data_cid: Option<Vec<u8>>,
pub ops: Option<Vec<u8>>,
pub blobs: Option<Vec<String>>,
pub blocks_cids: Option<Vec<String>>,
pub blocks: Option<Vec<EventBlockInline>>,
pub handle: Option<String>,
pub active: Option<bool>,
pub status: Option<u8>,
@@ -38,8 +40,8 @@ pub enum PayloadError {
InvalidDid(String),
#[error("invalid timestamp: {0}")]
InvalidTimestamp(u64),
#[error("invalid ops JSON in payload: {0}")]
InvalidOps(serde_json::Error),
#[error("invalid ops DAG-CBOR in payload: {0}")]
InvalidDagCborOps(String),
#[error("invalid handle in payload: {0}")]
InvalidHandle(String),
#[error("invalid CID length: got {got}, expected {expected}")]
@@ -96,7 +98,7 @@ pub fn encode_payload_with_mutations(
let ops_bytes = event
.ops
.as_ref()
.map(|v| serde_json::to_vec(v).expect("serde_json::Value always serializes"));
.map(|v| serde_ipld_dagcbor::to_vec(v).expect("serde_json::Value serializes to DAG-CBOR"));
let payload = EventPayload {
did: event.did.as_str().to_owned(),
@@ -105,7 +107,10 @@ pub fn encode_payload_with_mutations(
prev_data_cid: event.prev_data_cid.as_ref().and_then(cid_link_to_bytes),
ops: ops_bytes,
blobs: event.blobs.clone(),
blocks_cids: event.blocks_cids.clone(),
blocks: match event.blocks.as_ref() {
Some(EventBlocks::Inline(v)) => Some(v.clone()),
Some(EventBlocks::LegacyCids(_)) | None => None,
},
handle: event
.handle
.as_ref()
@@ -127,7 +132,7 @@ pub fn encode_payload_with_mutations(
}
let mut buf = Vec::with_capacity(1 + body.len());
buf.push(PAYLOAD_VERSION);
buf.push(CURRENT_PAYLOAD_VERSION);
buf.extend_from_slice(&body);
buf
}
@@ -137,15 +142,15 @@ pub fn decode_payload(bytes: &[u8]) -> Result<EventPayload, PayloadError> {
postcard::Error::DeserializeUnexpectedEnd,
))?;
if version != PAYLOAD_VERSION {
if version != PAYLOAD_VERSION_V1 {
return Err(PayloadError::UnknownVersion(version));
}
postcard::from_bytes(body).map_err(PayloadError::DeserializeFailed)
}
pub fn validate_payload_size(payload: &[u8]) -> Result<(), PayloadError> {
let max = MAX_EVENT_PAYLOAD as usize;
pub fn validate_payload_size(payload: &[u8], max_payload: u32) -> Result<(), PayloadError> {
let max = max_payload as usize;
if payload.len() > max {
return Err(PayloadError::TooLarge {
size: payload.len(),
@@ -174,9 +179,11 @@ pub fn to_sequenced_event(
let ops = payload
.ops
.as_ref()
.map(|bytes| serde_json::from_slice(bytes))
.transpose()
.map_err(PayloadError::InvalidOps)?;
.map(|bytes| {
serde_ipld_dagcbor::from_slice(bytes)
.map_err(|e| PayloadError::InvalidDagCborOps(e.to_string()))
})
.transpose()?;
let handle = payload
.handle
@@ -209,7 +216,7 @@ pub fn to_sequenced_event(
.flatten(),
ops,
blobs: payload.blobs.clone(),
blocks_cids: payload.blocks_cids.clone(),
blocks: payload.blocks.clone().map(EventBlocks::Inline),
handle,
active: payload.active,
status: payload.status.and_then(u8_to_account_status),
@@ -248,7 +255,7 @@ mod tests {
prev_data_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
handle: None,
active: Some(true),
status: Some(AccountStatus::Active),
@@ -256,7 +263,7 @@ mod tests {
};
let encoded = encode_payload(&event);
assert_eq!(encoded[0], PAYLOAD_VERSION);
assert_eq!(encoded[0], CURRENT_PAYLOAD_VERSION);
let decoded = decode_payload(&encoded).unwrap();
assert_eq!(decoded.did, event.did.as_str());
@@ -280,7 +287,10 @@ mod tests {
prev_data_cid: Some(cid.clone()),
ops: Some(ops.clone()),
blobs: Some(vec!["bafkreibtest".to_owned()]),
blocks_cids: Some(vec!["bafyreiblock".to_owned()]),
blocks: Some(EventBlocks::Inline(vec![EventBlockInline {
cid_bytes: cid_link_to_bytes(&cid).unwrap(),
data: b"hello block".to_vec(),
}])),
handle: Some(Handle::new("test.bsky.social").unwrap()),
active: None,
status: None,
@@ -304,7 +314,14 @@ mod tests {
assert_eq!(reconstructed.prev_cid, event.prev_cid);
assert_eq!(reconstructed.prev_data_cid, event.prev_data_cid);
assert_eq!(reconstructed.blobs, event.blobs);
assert_eq!(reconstructed.blocks_cids, event.blocks_cids);
let inline_len = |b: &EventBlocks| match b {
EventBlocks::Inline(v) => v.len(),
EventBlocks::LegacyCids(_) => 0,
};
assert_eq!(
reconstructed.blocks.as_ref().map(inline_len),
event.blocks.as_ref().map(inline_len)
);
assert_eq!(
reconstructed.handle.as_ref().map(|h: &Handle| h.as_str()),
event.handle.as_ref().map(|h: &Handle| h.as_str())
@@ -328,7 +345,7 @@ mod tests {
prev_data_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
handle: None,
active: None,
status: None,
@@ -352,17 +369,18 @@ mod tests {
#[test]
fn validate_payload_size_accepts_within_limit() {
let data = vec![0u8; MAX_EVENT_PAYLOAD as usize];
assert!(validate_payload_size(&data).is_ok());
let data = vec![0u8; 1024];
assert!(validate_payload_size(&data, 4096).is_ok());
}
#[test]
fn validate_payload_size_rejects_oversized() {
let data = vec![0u8; MAX_EVENT_PAYLOAD as usize + 1];
match validate_payload_size(&data) {
let limit: u32 = 1024;
let data = vec![0u8; limit as usize + 1];
match validate_payload_size(&data, limit) {
Err(PayloadError::TooLarge { size, max }) => {
assert_eq!(size, MAX_EVENT_PAYLOAD as usize + 1);
assert_eq!(max, MAX_EVENT_PAYLOAD as usize);
assert_eq!(size, limit as usize + 1);
assert_eq!(max, limit as usize);
}
other => panic!("expected TooLarge, got {other:?}"),
}
@@ -436,7 +454,7 @@ mod tests {
timestamp: TimestampMicros::new(us),
did_hash: DidHash::from_did("did:plc:test"),
event_type: EventTypeTag::COMMIT,
payload: Bytes::new(),
payload: Bytes::from_static(&[PAYLOAD_VERSION_V1]),
};
let payload = EventPayload {
@@ -446,7 +464,7 @@ mod tests {
prev_data_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
handle: None,
active: None,
status: None,
crates/tranquil-store/src/eventlog/reader.rs
+193 -52
View File
@@ -12,12 +12,34 @@ use crate::io::{MappedFile, StorageIO};
use super::manager::SegmentManager;
use super::segment_file::{ReadEventRecord, SEGMENT_HEADER_SIZE, decode_event_record};
use super::segment_index::{DEFAULT_INDEX_INTERVAL, SegmentIndex, rebuild_from_segment};
use super::sidecar::{SidecarIndex, build_sidecar_from_segment};
use super::types::{
DidHash, EventSequence, EventTypeTag, SegmentId, SegmentOffset, TimestampMicros,
};
const FIRST_EVENT_OFFSET: SegmentOffset = SegmentOffset::new(SEGMENT_HEADER_SIZE as u64);
#[derive(Debug, Clone)]
pub struct SequenceGap {
pub after_segment: SegmentId,
pub expected_seq: EventSequence,
pub actual_seq: EventSequence,
}
#[derive(Debug, Clone)]
pub struct SequenceContiguityResult {
pub total_segments: u64,
pub min_seq: Option<EventSequence>,
pub max_seq: Option<EventSequence>,
pub gaps: Vec<SequenceGap>,
}
impl SequenceContiguityResult {
pub fn is_contiguous(&self) -> bool {
self.gaps.is_empty()
}
}
#[derive(Debug, Clone)]
pub struct RawEvent {
pub seq: EventSequence,
@@ -37,29 +59,45 @@ struct SegmentRange {
pub struct EventLogReader<S: StorageIO> {
manager: Arc<SegmentManager<S>>,
indexes: RwLock<HashMap<SegmentId, Arc<SegmentIndex>>>,
sidecars: RwLock<HashMap<SegmentId, Arc<SidecarIndex>>>,
ranges: RwLock<Vec<SegmentRange>>,
mmaps: RwLock<HashMap<SegmentId, Arc<MappedFile>>>,
active_segment: RwLock<Option<SegmentId>>,
use_mmap: bool,
skip_sealed_checksum: bool,
max_payload: u32,
}
impl<S: StorageIO> EventLogReader<S> {
pub fn new(manager: Arc<SegmentManager<S>>, use_mmap: bool) -> Self {
pub fn new(
manager: Arc<SegmentManager<S>>,
use_mmap: bool,
skip_sealed_checksum: bool,
max_payload: u32,
) -> Self {
Self {
manager,
indexes: RwLock::new(HashMap::new()),
sidecars: RwLock::new(HashMap::new()),
ranges: RwLock::new(Vec::new()),
mmaps: RwLock::new(HashMap::new()),
active_segment: RwLock::new(None),
use_mmap,
skip_sealed_checksum,
max_payload,
}
}
pub fn max_payload(&self) -> u32 {
self.max_payload
}
pub fn set_active_segment(&self, id: SegmentId) {
*self.active_segment.write() = Some(id);
}
pub fn extend_active_range(&self, first_seq: EventSequence, last_seq: EventSequence) {
debug_assert!(first_seq <= last_seq);
let active_id = match *self.active_segment.read() {
Some(id) => id,
None => return,
@@ -108,7 +146,12 @@ impl<S: StorageIO> EventLogReader<S> {
fn rebuild_index(&self, segment_id: SegmentId) -> io::Result<SegmentIndex> {
let fd = self.manager.open_for_read(segment_id)?;
let (idx, _) = rebuild_from_segment(self.manager.io(), fd, DEFAULT_INDEX_INTERVAL)?;
let (idx, _) = rebuild_from_segment(
self.manager.io(),
fd,
DEFAULT_INDEX_INTERVAL,
self.max_payload,
)?;
let _ = idx.save(self.manager.io(), &self.manager.index_path(segment_id));
Ok(idx)
}
@@ -159,6 +202,34 @@ impl<S: StorageIO> EventLogReader<S> {
Ok(())
}
pub fn check_sequence_contiguity(&self) -> SequenceContiguityResult {
let ranges = self.ranges.read();
let mut gaps: Vec<SequenceGap> = Vec::new();
let total_segments = ranges.len() as u64;
ranges.windows(2).for_each(|pair| {
let expected_next = pair[0].last.next();
let actual_next = pair[1].first;
if actual_next != expected_next {
gaps.push(SequenceGap {
after_segment: pair[0].id,
expected_seq: expected_next,
actual_seq: actual_next,
});
}
});
let max_seq = ranges.last().map(|r| r.last);
let min_seq = ranges.first().map(|r| r.first);
SequenceContiguityResult {
total_segments,
min_seq,
max_seq,
gaps,
}
}
fn is_mmap_eligible(&self, segment_id: SegmentId) -> bool {
self.use_mmap
&& self
@@ -222,15 +293,27 @@ impl<S: StorageIO> EventLogReader<S> {
mut predicate: impl FnMut(&EventSequence) -> bool,
) -> io::Result<bool> {
let mmap = self.get_mmap(segment_id)?;
let mmap_bytes = Bytes::from_owner(OwnedMmap(Arc::clone(&mmap)));
let data: &[u8] = (*mmap).as_ref();
let file_size = data.len() as u64;
let skip_checksum = self.skip_sealed_checksum && self.is_mmap_eligible(segment_id);
let offset = Cell::new(start_offset);
let collected = Cell::new(0usize);
let max_payload = self.max_payload;
std::iter::from_fn(|| {
let cur = offset.get();
(cur.raw() < file_size && collected.get() < limit)
.then(|| decode_mmap_event(data, cur, file_size, segment_id))
(cur.raw() < file_size && collected.get() < limit).then(|| {
decode_mmap_event(
data,
&mmap_bytes,
cur,
file_size,
segment_id,
skip_checksum,
max_payload,
)
})
})
.try_for_each(|result| -> io::Result<()> {
match result? {
@@ -268,8 +351,9 @@ impl<S: StorageIO> EventLogReader<S> {
std::iter::from_fn(|| {
let cur = offset.get();
(cur.raw() < file_size && collected.get() < limit)
.then(|| decode_event_record(self.manager.io(), fd, cur, file_size))
(cur.raw() < file_size && collected.get() < limit).then(|| {
decode_event_record(self.manager.io(), fd, cur, file_size, self.max_payload)
})
})
.try_for_each(|result| -> io::Result<()> {
match result? {
@@ -320,14 +404,13 @@ impl<S: StorageIO> EventLogReader<S> {
};
let mut events = Vec::with_capacity(limit.min(1024));
let done = Cell::new(false);
ranges[start_idx..].iter().enumerate().try_fold(
false,
|limit_reached, (i, range)| -> io::Result<bool> {
if limit_reached {
return Ok(true);
}
ranges[start_idx..]
.iter()
.enumerate()
.take_while(|_| !done.get())
.try_for_each(|(i, range)| -> io::Result<()> {
let remaining = limit - events.len();
let is_first = i == 0;
@@ -341,16 +424,16 @@ impl<S: StorageIO> EventLogReader<S> {
}
};
self.scan_events_from_offset(
done.set(self.scan_events_from_offset(
range.id,
scan_offset,
effective_seq,
remaining,
&mut events,
|_| true,
)
},
)?;
)?);
Ok(())
})?;
Ok(events)
}
@@ -384,6 +467,7 @@ impl<S: StorageIO> EventLogReader<S> {
) -> io::Result<()> {
self.invalidate_index(sealed_id);
self.invalidate_mmap(sealed_id);
self.invalidate_sidecar(sealed_id);
self.set_active_segment(new_active_id);
self.refresh_segment_ranges()
}
@@ -395,6 +479,53 @@ impl<S: StorageIO> EventLogReader<S> {
pub fn invalidate_index(&self, segment_id: SegmentId) {
self.indexes.write().remove(&segment_id);
}
pub fn invalidate_sidecar(&self, segment_id: SegmentId) {
self.sidecars.write().remove(&segment_id);
}
pub fn load_sidecar(&self, segment_id: SegmentId) -> io::Result<Option<Arc<SidecarIndex>>> {
if let Some(sc) = self.sidecars.read().get(&segment_id) {
return Ok(Some(Arc::clone(sc)));
}
let sidecar = match SidecarIndex::load(
self.manager.io(),
&self.manager.sidecar_path(segment_id),
) {
Ok(Some(sc)) => sc,
Ok(None) => return Ok(None),
Err(e) => {
warn!(segment = %segment_id, error = %e, "sidecar load failed, attempting rebuild");
match self.rebuild_sidecar(segment_id) {
Ok(sc) => sc,
Err(rebuild_err) => {
warn!(segment = %segment_id, error = %rebuild_err, "sidecar rebuild also failed");
return Ok(None);
}
}
}
};
let arc = Arc::new(sidecar);
self.sidecars.write().insert(segment_id, Arc::clone(&arc));
Ok(Some(arc))
}
fn rebuild_sidecar(&self, segment_id: SegmentId) -> io::Result<SidecarIndex> {
let fd = self.manager.open_for_read(segment_id)?;
let sidecar = build_sidecar_from_segment(self.manager.io(), fd, self.max_payload)?;
let _ = sidecar.save(self.manager.io(), &self.manager.sidecar_path(segment_id));
Ok(sidecar)
}
}
struct OwnedMmap(Arc<MappedFile>);
impl AsRef<[u8]> for OwnedMmap {
fn as_ref(&self) -> &[u8] {
(*self.0).as_ref()
}
}
enum MmapDecodeResult {
@@ -406,12 +537,14 @@ enum MmapDecodeResult {
fn decode_mmap_event(
data: &[u8],
mmap_bytes: &Bytes,
offset: SegmentOffset,
file_size: u64,
segment_id: SegmentId,
skip_checksum: bool,
max_payload: u32,
) -> io::Result<MmapDecodeResult> {
use super::segment_file::EVENT_HEADER_SIZE;
use super::types::MAX_EVENT_PAYLOAD;
let raw = offset.raw();
if raw > file_size {
@@ -469,7 +602,7 @@ fn decode_mmap_event(
};
let payload_len = u32::from_le_bytes(header_slice[21..25].try_into().unwrap());
if payload_len > MAX_EVENT_PAYLOAD {
if payload_len > max_payload {
warn!(
segment = %segment_id,
offset = raw,
@@ -494,25 +627,27 @@ fn decode_mmap_event(
let payload_start = base + EVENT_HEADER_SIZE;
let payload_end = payload_start + usize::try_from(payload_len).expect("payload_len fits usize");
let checksum_start = payload_end;
let stored_checksum =
u32::from_le_bytes(data[checksum_start..checksum_start + 4].try_into().unwrap());
if !skip_checksum {
let checksum_start = payload_end;
let stored_checksum =
u32::from_le_bytes(data[checksum_start..checksum_start + 4].try_into().unwrap());
let mut hasher = xxhash_rust::xxh3::Xxh3::new();
hasher.update(header_slice);
hasher.update(&data[payload_start..payload_end]);
let computed = hasher.digest() as u32;
let mut hasher = xxhash_rust::xxh3::Xxh3::new();
hasher.update(header_slice);
hasher.update(&data[payload_start..payload_end]);
let computed = hasher.digest() as u32;
if stored_checksum != computed {
warn!(
segment = %segment_id,
offset = raw,
seq = %seq,
stored = stored_checksum,
computed,
"corrupted record in sealed segment: checksum mismatch"
);
return Ok(MmapDecodeResult::Corrupted);
if stored_checksum != computed {
warn!(
segment = %segment_id,
offset = raw,
seq = %seq,
stored = stored_checksum,
computed,
"corrupted record in sealed segment: checksum mismatch"
);
return Ok(MmapDecodeResult::Corrupted);
}
}
let next_offset = offset.advance(record_size);
@@ -522,7 +657,7 @@ fn decode_mmap_event(
timestamp,
did_hash,
event_type,
payload: Bytes::copy_from_slice(&data[payload_start..payload_end]),
payload: mmap_bytes.slice(payload_start..payload_end),
},
next_offset,
))
@@ -532,6 +667,7 @@ fn decode_mmap_event(
mod tests {
use super::*;
use crate::eventlog::segment_file::EVENT_RECORD_OVERHEAD;
use crate::eventlog::types::MAX_EVENT_PAYLOAD;
use crate::eventlog::writer::EventLogWriter;
use crate::sim::SimulatedIO;
use std::path::PathBuf;
@@ -552,7 +688,8 @@ mod tests {
let mgr = setup_manager(max_segment_size);
{
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
(1..=event_count).for_each(|i| {
writer
.append(
@@ -566,7 +703,7 @@ mod tests {
}
mgr.shutdown();
let reader = EventLogReader::new(Arc::clone(&mgr), false);
let reader = EventLogReader::new(Arc::clone(&mgr), false, false, MAX_EVENT_PAYLOAD);
reader.refresh_segment_ranges().unwrap();
(mgr, reader)
}
@@ -586,7 +723,8 @@ mod tests {
let mgr = setup_manager(max_segment_size);
{
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
let total = events_per_segment * num_segments;
(1..=total).for_each(|i| {
writer
@@ -605,7 +743,7 @@ mod tests {
}
mgr.shutdown();
let reader = EventLogReader::new(Arc::clone(&mgr), false);
let reader = EventLogReader::new(Arc::clone(&mgr), false, false, MAX_EVENT_PAYLOAD);
reader.refresh_segment_ranges().unwrap();
(mgr, reader)
}
@@ -777,7 +915,7 @@ mod tests {
fn mmap_read_matches_direct_read() {
let (mgr, direct_reader) = setup_with_events(10, 50, 64 * 1024);
let mmap_reader = EventLogReader::new(Arc::clone(&mgr), true);
let mmap_reader = EventLogReader::new(Arc::clone(&mgr), true, false, MAX_EVENT_PAYLOAD);
mmap_reader.refresh_segment_ranges().unwrap();
let direct_events = direct_reader
@@ -820,7 +958,7 @@ mod tests {
#[test]
fn empty_reader_returns_empty() {
let mgr = setup_manager(64 * 1024);
let reader = EventLogReader::new(Arc::clone(&mgr), false);
let reader = EventLogReader::new(Arc::clone(&mgr), false, false, MAX_EVENT_PAYLOAD);
reader.refresh_segment_ranges().unwrap();
let events = reader
@@ -858,7 +996,8 @@ mod tests {
let mgr = setup_manager(64 * 1024);
{
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
let types = [
EventTypeTag::COMMIT,
EventTypeTag::IDENTITY,
@@ -878,7 +1017,7 @@ mod tests {
}
mgr.shutdown();
let reader = EventLogReader::new(Arc::clone(&mgr), false);
let reader = EventLogReader::new(Arc::clone(&mgr), false, false, MAX_EVENT_PAYLOAD);
reader.refresh_segment_ranges().unwrap();
let events = reader
@@ -894,7 +1033,7 @@ mod tests {
fn active_segment_excludes_mmap() {
let (mgr, _) = setup_with_events(10, 50, 64 * 1024);
let reader = EventLogReader::new(Arc::clone(&mgr), true);
let reader = EventLogReader::new(Arc::clone(&mgr), true, false, MAX_EVENT_PAYLOAD);
reader.set_active_segment(SegmentId::new(1));
reader.refresh_segment_ranges().unwrap();
@@ -905,7 +1044,7 @@ mod tests {
#[test]
fn no_active_segment_mmaps_all() {
let reader: EventLogReader<SimulatedIO> =
EventLogReader::new(setup_manager(64 * 1024), true);
EventLogReader::new(setup_manager(64 * 1024), true, false, MAX_EVENT_PAYLOAD);
assert!(reader.is_mmap_eligible(SegmentId::new(1)));
assert!(reader.is_mmap_eligible(SegmentId::new(99)));
@@ -916,7 +1055,8 @@ mod tests {
let mgr = setup_manager(64 * 1024);
{
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
(1..=5).for_each(|i| {
writer
.append(
@@ -937,7 +1077,7 @@ mod tests {
.save(mgr.io(), &mgr.index_path(SegmentId::new(1)))
.unwrap();
let reader = EventLogReader::new(Arc::clone(&mgr), false);
let reader = EventLogReader::new(Arc::clone(&mgr), false, false, MAX_EVENT_PAYLOAD);
reader.refresh_segment_ranges().unwrap();
let events = reader
@@ -954,7 +1094,8 @@ mod tests {
let mgr = setup_manager(64 * 1024);
{
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
(1..=5).for_each(|i| {
writer
.append(
@@ -975,7 +1116,7 @@ mod tests {
.save(mgr.io(), &mgr.index_path(SegmentId::new(1)))
.unwrap();
let reader = EventLogReader::new(Arc::clone(&mgr), true);
let reader = EventLogReader::new(Arc::clone(&mgr), true, false, MAX_EVENT_PAYLOAD);
reader.refresh_segment_ranges().unwrap();
let events = reader
@@ -988,7 +1129,7 @@ mod tests {
fn on_segment_rotated_updates_state() {
let (mgr, _direct_reader) = setup_multi_segment(3, 2, 50);
let reader = EventLogReader::new(Arc::clone(&mgr), true);
let reader = EventLogReader::new(Arc::clone(&mgr), true, false, MAX_EVENT_PAYLOAD);
reader.refresh_segment_ranges().unwrap();
assert_eq!(
crates/tranquil-store/src/eventlog/segment_file.rs
+218 -59
View File
@@ -3,8 +3,7 @@ use std::io;
use crate::io::{FileId, StorageIO};
use super::types::{
DidHash, EventSequence, EventTypeTag, MAX_EVENT_PAYLOAD, SegmentId, SegmentOffset,
TimestampMicros,
DidHash, EventSequence, EventTypeTag, SegmentId, SegmentOffset, TimestampMicros,
};
pub const SEGMENT_MAGIC: [u8; 4] = *b"TQEV";
@@ -46,6 +45,7 @@ pub fn encode_event_record<S: StorageIO>(
fd: FileId,
offset: SegmentOffset,
event: &ValidEvent,
max_payload: u32,
) -> io::Result<u64> {
let payload_len = u32::try_from(event.payload.len()).map_err(|_| {
io::Error::new(
@@ -53,10 +53,10 @@ pub fn encode_event_record<S: StorageIO>(
"event payload exceeds u32::MAX",
)
})?;
if payload_len > MAX_EVENT_PAYLOAD {
if payload_len > max_payload {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"event payload exceeds MAX_EVENT_PAYLOAD",
format!("event payload {payload_len} exceeds configured max_payload {max_payload}"),
));
}
@@ -96,6 +96,7 @@ pub fn decode_event_record<S: StorageIO>(
fd: FileId,
offset: SegmentOffset,
file_size: u64,
max_payload: u32,
) -> io::Result<Option<ReadEventRecord>> {
let raw = offset.raw();
if raw > file_size {
@@ -128,7 +129,7 @@ pub fn decode_event_record<S: StorageIO>(
};
let payload_len = u32::from_le_bytes(header[21..25].try_into().unwrap());
if payload_len > MAX_EVENT_PAYLOAD {
if payload_len > max_payload {
return Ok(Some(ReadEventRecord::Corrupted { offset }));
}
@@ -186,6 +187,7 @@ pub fn validate_event_record<S: StorageIO>(
fd: FileId,
offset: SegmentOffset,
file_size: u64,
max_payload: u32,
) -> io::Result<Option<ValidateEventRecord>> {
let raw = offset.raw();
assert!(
@@ -216,7 +218,7 @@ pub fn validate_event_record<S: StorageIO>(
}
let payload_len = u32::from_le_bytes(header[21..25].try_into().unwrap());
if payload_len > MAX_EVENT_PAYLOAD {
if payload_len > max_payload {
return Ok(Some(ValidateEventRecord::Corrupted));
}
@@ -266,6 +268,7 @@ pub struct SegmentWriter {
position: SegmentOffset,
base_seq: EventSequence,
last_seq: Option<EventSequence>,
max_payload: u32,
}
impl SegmentWriter {
@@ -274,6 +277,7 @@ impl SegmentWriter {
fd: FileId,
segment_id: SegmentId,
base_seq: EventSequence,
max_payload: u32,
) -> io::Result<Self> {
let mut header = [0u8; SEGMENT_HEADER_SIZE];
header[..4].copy_from_slice(&SEGMENT_MAGIC);
@@ -285,6 +289,7 @@ impl SegmentWriter {
position: SegmentOffset::new(SEGMENT_HEADER_SIZE as u64),
base_seq,
last_seq: None,
max_payload,
})
}
@@ -295,6 +300,7 @@ impl SegmentWriter {
position: SegmentOffset,
base_seq: EventSequence,
last_seq: Option<EventSequence>,
max_payload: u32,
) -> Self {
assert!(
position.raw() >= SEGMENT_HEADER_SIZE as u64,
@@ -315,9 +321,14 @@ impl SegmentWriter {
position,
base_seq,
last_seq,
max_payload,
}
}
pub fn max_payload(&self) -> u32 {
self.max_payload
}
pub fn append_event<S: StorageIO>(
&mut self,
io: &S,
@@ -330,7 +341,8 @@ impl SegmentWriter {
self.last_seq.unwrap()
);
let record_offset = self.position;
let bytes_written = encode_event_record(io, self.fd, record_offset, event)?;
let bytes_written =
encode_event_record(io, self.fd, record_offset, event, self.max_payload)?;
self.position = self.position.advance(bytes_written);
self.last_seq = Some(event.seq);
Ok(record_offset)
@@ -362,10 +374,11 @@ pub struct SegmentReader<'a, S: StorageIO> {
fd: FileId,
position: SegmentOffset,
file_size: u64,
max_payload: u32,
}
impl<'a, S: StorageIO> SegmentReader<'a, S> {
pub fn open(io: &'a S, fd: FileId) -> io::Result<Self> {
pub fn open(io: &'a S, fd: FileId, max_payload: u32) -> io::Result<Self> {
let file_size = io.file_size(fd)?;
if file_size < SEGMENT_HEADER_SIZE as u64 {
return Err(io::Error::new(
@@ -395,9 +408,14 @@ impl<'a, S: StorageIO> SegmentReader<'a, S> {
fd,
position: SegmentOffset::new(SEGMENT_HEADER_SIZE as u64),
file_size,
max_payload,
})
}
pub fn max_payload(&self) -> u32 {
self.max_payload
}
pub fn valid_prefix(self) -> io::Result<Vec<ValidEvent>> {
self.map(|result| {
result.map(|record| match record {
@@ -430,7 +448,13 @@ impl<S: StorageIO> Iterator for SegmentReader<'_, S> {
type Item = io::Result<ReadEventRecord>;
fn next(&mut self) -> Option<Self::Item> {
match decode_event_record(self.io, self.fd, self.position, self.file_size) {
match decode_event_record(
self.io,
self.fd,
self.position,
self.file_size,
self.max_payload,
) {
Err(e) => {
self.position = SegmentOffset::new(self.file_size);
Some(Err(e))
@@ -455,6 +479,7 @@ impl<S: StorageIO> Iterator for SegmentReader<'_, S> {
mod tests {
use super::*;
use crate::OpenOptions;
use crate::eventlog::types::MAX_EVENT_PAYLOAD;
use crate::sim::SimulatedIO;
use proptest::prelude::*;
use std::path::Path;
@@ -487,8 +512,14 @@ mod tests {
#[test]
fn write_and_read_single_event() {
let (sim, fd) = setup();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let mut writer = SegmentWriter::new(
&sim,
fd,
SegmentId::new(1),
EventSequence::new(1),
MAX_EVENT_PAYLOAD,
)
.unwrap();
let event = test_event(1, b"test event payload");
let offset = writer.append_event(&sim, &event).unwrap();
@@ -496,7 +527,7 @@ mod tests {
assert_eq!(offset, SegmentOffset::new(SEGMENT_HEADER_SIZE as u64));
let reader = SegmentReader::open(&sim, fd).unwrap();
let reader = SegmentReader::open(&sim, fd, MAX_EVENT_PAYLOAD).unwrap();
let events = reader.valid_prefix().unwrap();
assert_eq!(events.len(), 1);
assert_eq!(events[0], event);
@@ -505,8 +536,14 @@ mod tests {
#[test]
fn write_and_read_multiple_events() {
let (sim, fd) = setup();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let mut writer = SegmentWriter::new(
&sim,
fd,
SegmentId::new(1),
EventSequence::new(1),
MAX_EVENT_PAYLOAD,
)
.unwrap();
let written: Vec<ValidEvent> = (1u64..=3)
.map(|i| {
@@ -517,7 +554,7 @@ mod tests {
.collect();
writer.sync(&sim).unwrap();
let reader = SegmentReader::open(&sim, fd).unwrap();
let reader = SegmentReader::open(&sim, fd, MAX_EVENT_PAYLOAD).unwrap();
let events = reader.valid_prefix().unwrap();
assert_eq!(events, written);
}
@@ -525,9 +562,16 @@ mod tests {
#[test]
fn empty_segment_has_no_events() {
let (sim, fd) = setup();
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
SegmentWriter::new(
&sim,
fd,
SegmentId::new(1),
EventSequence::new(1),
MAX_EVENT_PAYLOAD,
)
.unwrap();
let reader = SegmentReader::open(&sim, fd).unwrap();
let reader = SegmentReader::open(&sim, fd, MAX_EVENT_PAYLOAD).unwrap();
let events = reader.valid_prefix().unwrap();
assert!(events.is_empty());
}
@@ -535,8 +579,14 @@ mod tests {
#[test]
fn detects_truncated_event() {
let (sim, fd) = setup();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let mut writer = SegmentWriter::new(
&sim,
fd,
SegmentId::new(1),
EventSequence::new(1),
MAX_EVENT_PAYLOAD,
)
.unwrap();
writer
.append_event(&sim, &test_event(1, b"complete event"))
.unwrap();
@@ -546,7 +596,7 @@ mod tests {
.unwrap();
sim.sync(fd).unwrap();
let mut reader = SegmentReader::open(&sim, fd).unwrap();
let mut reader = SegmentReader::open(&sim, fd, MAX_EVENT_PAYLOAD).unwrap();
let first = reader.next().unwrap().unwrap();
assert!(matches!(first, ReadEventRecord::Valid { .. }));
@@ -557,8 +607,14 @@ mod tests {
#[test]
fn checksum_detects_corruption() {
let (sim, fd) = setup();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let mut writer = SegmentWriter::new(
&sim,
fd,
SegmentId::new(1),
EventSequence::new(1),
MAX_EVENT_PAYLOAD,
)
.unwrap();
writer
.append_event(&sim, &test_event(1, &vec![0xAA; 256]))
.unwrap();
@@ -567,7 +623,7 @@ mod tests {
let corrupt_offset = SEGMENT_HEADER_SIZE as u64 + EVENT_HEADER_SIZE as u64 + 128;
sim.write_all_at(fd, corrupt_offset, &[0x00]).unwrap();
let mut reader = SegmentReader::open(&sim, fd).unwrap();
let mut reader = SegmentReader::open(&sim, fd, MAX_EVENT_PAYLOAD).unwrap();
let record = reader.next().unwrap().unwrap();
assert!(matches!(record, ReadEventRecord::Corrupted { .. }));
}
@@ -575,8 +631,14 @@ mod tests {
#[test]
fn crash_before_sync_loses_events() {
let (sim, fd) = setup();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let mut writer = SegmentWriter::new(
&sim,
fd,
SegmentId::new(1),
EventSequence::new(1),
MAX_EVENT_PAYLOAD,
)
.unwrap();
writer
.append_event(&sim, &test_event(1, b"synced"))
.unwrap();
@@ -592,7 +654,7 @@ mod tests {
let fd = sim
.open(Path::new("/test/segment.tqe"), OpenOptions::read())
.unwrap();
let reader = SegmentReader::open(&sim, fd).unwrap();
let reader = SegmentReader::open(&sim, fd, MAX_EVENT_PAYLOAD).unwrap();
let events = reader.valid_prefix().unwrap();
assert_eq!(events.len(), 1);
assert_eq!(events[0].payload, b"synced");
@@ -601,20 +663,76 @@ mod tests {
#[test]
fn rejects_oversized_payload() {
let (sim, fd) = setup();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let result = writer.append_event(
const SMALL_MAX: u32 = 1024;
let mut writer = SegmentWriter::new(
&sim,
&test_event(1, &vec![0u8; MAX_EVENT_PAYLOAD as usize + 1]),
);
fd,
SegmentId::new(1),
EventSequence::new(1),
SMALL_MAX,
)
.unwrap();
let result = writer.append_event(&sim, &test_event(1, &vec![0u8; SMALL_MAX as usize + 1]));
assert!(result.is_err());
}
#[test]
fn reader_rejects_corrupt_header_claiming_oversize() {
let (sim, fd) = setup();
const SMALL_MAX: u32 = 1024;
let mut writer = SegmentWriter::new(
&sim,
fd,
SegmentId::new(1),
EventSequence::new(1),
MAX_EVENT_PAYLOAD,
)
.unwrap();
writer
.append_event(&sim, &test_event(1, &vec![0xAA; 2048]))
.unwrap();
writer.sync(&sim).unwrap();
let mut reader = SegmentReader::open(&sim, fd, SMALL_MAX).unwrap();
let record = reader.next().unwrap().unwrap();
assert!(matches!(record, ReadEventRecord::Corrupted { .. }));
}
#[test]
fn reader_with_larger_max_reads_writer_segment() {
let (sim, fd) = setup();
const WRITER_MAX: u32 = 16 * 1024;
const READER_MAX: u32 = 1024 * 1024 * 1024;
let mut writer = SegmentWriter::new(
&sim,
fd,
SegmentId::new(1),
EventSequence::new(1),
WRITER_MAX,
)
.unwrap();
writer
.append_event(&sim, &test_event(1, &vec![0xCD; 8 * 1024]))
.unwrap();
writer.sync(&sim).unwrap();
let reader = SegmentReader::open(&sim, fd, READER_MAX).unwrap();
let events = reader.valid_prefix().unwrap();
assert_eq!(events.len(), 1);
assert_eq!(events[0].payload.len(), 8 * 1024);
}
#[test]
fn zero_length_payload_round_trips() {
let (sim, fd) = setup();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let mut writer = SegmentWriter::new(
&sim,
fd,
SegmentId::new(1),
EventSequence::new(1),
MAX_EVENT_PAYLOAD,
)
.unwrap();
let event = ValidEvent {
seq: EventSequence::new(1),
timestamp: TimestampMicros::new(1_000_000),
@@ -625,7 +743,7 @@ mod tests {
writer.append_event(&sim, &event).unwrap();
writer.sync(&sim).unwrap();
let reader = SegmentReader::open(&sim, fd).unwrap();
let reader = SegmentReader::open(&sim, fd, MAX_EVENT_PAYLOAD).unwrap();
let events = reader.valid_prefix().unwrap();
assert_eq!(events, vec![event]);
}
@@ -633,12 +751,16 @@ mod tests {
#[test]
fn accepts_exact_max_payload() {
let (sim, fd) = setup();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let result = writer.append_event(
const SMALL_MAX: u32 = 4096;
let mut writer = SegmentWriter::new(
&sim,
&test_event(1, &vec![0xBB; MAX_EVENT_PAYLOAD as usize]),
);
fd,
SegmentId::new(1),
EventSequence::new(1),
SMALL_MAX,
)
.unwrap();
let result = writer.append_event(&sim, &test_event(1, &vec![0xBB; SMALL_MAX as usize]));
assert!(result.is_ok());
}
@@ -653,7 +775,7 @@ mod tests {
.unwrap();
sim.write_all_at(fd, 0, b"NOPE\x01").unwrap();
let result = SegmentReader::open(&sim, fd);
let result = SegmentReader::open(&sim, fd, MAX_EVENT_PAYLOAD);
assert!(result.is_err());
}
@@ -671,12 +793,13 @@ mod tests {
event_type: EventTypeTag::ACCOUNT,
payload: b"round trip test data".to_vec(),
};
let bytes_written = encode_event_record(&sim, fd, offset, &event).unwrap();
let bytes_written =
encode_event_record(&sim, fd, offset, &event, MAX_EVENT_PAYLOAD).unwrap();
let expected_size = EVENT_RECORD_OVERHEAD as u64 + event.payload.len() as u64;
assert_eq!(bytes_written, expected_size);
let file_size = sim.file_size(fd).unwrap();
let record = decode_event_record(&sim, fd, offset, file_size)
let record = decode_event_record(&sim, fd, offset, file_size, MAX_EVENT_PAYLOAD)
.unwrap()
.unwrap();
match record {
@@ -688,8 +811,14 @@ mod tests {
#[test]
fn resume_writer_continues_at_position() {
let (sim, fd) = setup();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let mut writer = SegmentWriter::new(
&sim,
fd,
SegmentId::new(1),
EventSequence::new(1),
MAX_EVENT_PAYLOAD,
)
.unwrap();
writer.append_event(&sim, &test_event(1, b"first")).unwrap();
writer.sync(&sim).unwrap();
@@ -701,13 +830,14 @@ mod tests {
resume_pos,
EventSequence::new(1),
Some(EventSequence::new(1)),
MAX_EVENT_PAYLOAD,
);
writer2
.append_event(&sim, &test_event(2, b"second"))
.unwrap();
writer2.sync(&sim).unwrap();
let reader = SegmentReader::open(&sim, fd).unwrap();
let reader = SegmentReader::open(&sim, fd, MAX_EVENT_PAYLOAD).unwrap();
let events = reader.valid_prefix().unwrap();
assert_eq!(events.len(), 2);
assert_eq!(events[0].payload, b"first");
@@ -717,8 +847,14 @@ mod tests {
#[test]
fn all_event_types_round_trip() {
let (sim, fd) = setup();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let mut writer = SegmentWriter::new(
&sim,
fd,
SegmentId::new(1),
EventSequence::new(1),
MAX_EVENT_PAYLOAD,
)
.unwrap();
let types = [
EventTypeTag::COMMIT,
@@ -739,7 +875,7 @@ mod tests {
});
writer.sync(&sim).unwrap();
let reader = SegmentReader::open(&sim, fd).unwrap();
let reader = SegmentReader::open(&sim, fd, MAX_EVENT_PAYLOAD).unwrap();
let events = reader.valid_prefix().unwrap();
assert_eq!(events.len(), 4);
events
@@ -753,7 +889,14 @@ mod tests {
#[test]
fn seq_zero_detected_as_corrupted() {
let (sim, fd) = setup();
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
SegmentWriter::new(
&sim,
fd,
SegmentId::new(1),
EventSequence::new(1),
MAX_EVENT_PAYLOAD,
)
.unwrap();
let mut raw_header = [0u8; EVENT_HEADER_SIZE];
raw_header[0..8].copy_from_slice(&0u64.to_le_bytes());
@@ -777,7 +920,7 @@ mod tests {
)
.unwrap();
let mut reader = SegmentReader::open(&sim, fd).unwrap();
let mut reader = SegmentReader::open(&sim, fd, MAX_EVENT_PAYLOAD).unwrap();
let record = reader.next().unwrap().unwrap();
assert!(matches!(record, ReadEventRecord::Corrupted { .. }));
}
@@ -785,8 +928,14 @@ mod tests {
#[test]
fn writer_accessors() {
let (sim, fd) = setup();
let writer =
SegmentWriter::new(&sim, fd, SegmentId::new(7), EventSequence::new(100)).unwrap();
let writer = SegmentWriter::new(
&sim,
fd,
SegmentId::new(7),
EventSequence::new(100),
MAX_EVENT_PAYLOAD,
)
.unwrap();
assert_eq!(writer.segment_id(), SegmentId::new(7));
assert_eq!(writer.base_seq(), EventSequence::new(100));
assert_eq!(
@@ -804,9 +953,13 @@ mod tests {
let written_count =
if let Ok(fd) = sim.open(Path::new("/data/segment.tqe"), OpenOptions::read_write()) {
if let Ok(mut writer) =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1))
{
if let Ok(mut writer) = SegmentWriter::new(
&sim,
fd,
SegmentId::new(1),
EventSequence::new(1),
MAX_EVENT_PAYLOAD,
) {
let count = (1u64..=20).fold(0u64, |count, i| {
let event = ValidEvent {
seq: EventSequence::new(i),
@@ -833,7 +986,7 @@ mod tests {
sim.crash();
if let Ok(fd) = sim.open(Path::new("/data/segment.tqe"), OpenOptions::read())
&& let Ok(reader) = SegmentReader::open(&sim, fd)
&& let Ok(reader) = SegmentReader::open(&sim, fd, MAX_EVENT_PAYLOAD)
{
let recovered: Vec<_> = reader
.map_while(|r| match r {
@@ -878,8 +1031,14 @@ mod tests {
let fd = sim
.open(Path::new("/data/segment.tqe"), OpenOptions::read_write())
.unwrap();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(1), EventSequence::new(1)).unwrap();
let mut writer = SegmentWriter::new(
&sim,
fd,
SegmentId::new(1),
EventSequence::new(1),
MAX_EVENT_PAYLOAD,
)
.unwrap();
let data_len = ((seed % 256) as usize).max(1);
let event = ValidEvent {
@@ -902,7 +1061,7 @@ mod tests {
byte_buf[0] ^= 1 << flip_bit;
sim.write_all_at(fd, flip_pos, &byte_buf).unwrap();
let mut reader = SegmentReader::open(&sim, fd).unwrap();
let mut reader = SegmentReader::open(&sim, fd, MAX_EVENT_PAYLOAD).unwrap();
let record = reader.next().unwrap().unwrap();
assert!(
!matches!(record, ReadEventRecord::Valid { .. }),
crates/tranquil-store/src/eventlog/segment_index.rs
+46 -19
View File
@@ -159,6 +159,7 @@ pub fn rebuild_from_segment<S: StorageIO>(
io: &S,
segment_fd: FileId,
index_interval: usize,
max_payload: u32,
) -> io::Result<(SegmentIndex, Option<EventSequence>)> {
assert!(index_interval > 0, "index_interval must be positive");
let file_size = io.file_size(segment_fd)?;
@@ -193,7 +194,7 @@ pub fn rebuild_from_segment<S: StorageIO>(
if offset.raw() >= file_size {
return None;
}
match validate_event_record(io, segment_fd, offset, file_size) {
match validate_event_record(io, segment_fd, offset, file_size, max_payload) {
Err(e) => Some(Err(e)),
Ok(None) => None,
Ok(Some(ValidateEventRecord::Valid { seq, next_offset })) => {
@@ -254,7 +255,8 @@ mod tests {
EVENT_HEADER_SIZE, SegmentWriter, ValidEvent, encode_event_record,
};
use crate::eventlog::types::{
DidHash, EventSequence, EventTypeTag, SegmentId, SegmentOffset, TimestampMicros,
DidHash, EventSequence, EventTypeTag, MAX_EVENT_PAYLOAD, SegmentId, SegmentOffset,
TimestampMicros,
};
use crate::sim::SimulatedIO;
use std::path::Path;
@@ -285,8 +287,14 @@ mod tests {
fd: FileId,
count: u64,
) -> Vec<(EventSequence, SegmentOffset)> {
let mut writer =
SegmentWriter::new(io, fd, SegmentId::new(0), EventSequence::new(1)).unwrap();
let mut writer = SegmentWriter::new(
io,
fd,
SegmentId::new(0),
EventSequence::new(1),
MAX_EVENT_PAYLOAD,
)
.unwrap();
let offsets: Vec<_> = (1..=count)
.map(|i| {
let event = test_event(i, format!("payload-{i}").as_bytes());
@@ -436,10 +444,18 @@ mod tests {
#[test]
fn rebuild_empty_segment() {
let (sim, fd) = setup();
SegmentWriter::new(&sim, fd, SegmentId::new(0), EventSequence::new(1)).unwrap();
SegmentWriter::new(
&sim,
fd,
SegmentId::new(0),
EventSequence::new(1),
MAX_EVENT_PAYLOAD,
)
.unwrap();
sim.sync(fd).unwrap();
let (index, last_seq) = rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL).unwrap();
let (index, last_seq) =
rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD).unwrap();
assert_eq!(index.entry_count(), 0);
assert_eq!(last_seq, None);
}
@@ -450,7 +466,8 @@ mod tests {
let offsets = write_n_events(&sim, fd, 1);
sim.sync(fd).unwrap();
let (index, last_seq) = rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL).unwrap();
let (index, last_seq) =
rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD).unwrap();
assert_eq!(last_seq, Some(EventSequence::new(1)));
assert_eq!(index.entry_count(), 1);
assert_eq!(index.first_seq(), Some(EventSequence::new(1)));
@@ -463,7 +480,8 @@ mod tests {
let offsets = write_n_events(&sim, fd, 10);
sim.sync(fd).unwrap();
let (index, last_seq) = rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL).unwrap();
let (index, last_seq) =
rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD).unwrap();
assert_eq!(last_seq, Some(EventSequence::new(10)));
assert_eq!(index.entry_count(), 2);
assert_eq!(index.first_seq(), Some(EventSequence::new(1)));
@@ -478,7 +496,7 @@ mod tests {
let offsets = write_n_events(&sim, fd, 600);
sim.sync(fd).unwrap();
let (index, last_seq) = rebuild_from_segment(&sim, fd, 256).unwrap();
let (index, last_seq) = rebuild_from_segment(&sim, fd, 256, MAX_EVENT_PAYLOAD).unwrap();
assert_eq!(last_seq, Some(EventSequence::new(600)));
assert_eq!(index.first_seq(), Some(EventSequence::new(1)));
assert_eq!(index.last_seq(), Some(EventSequence::new(600)));
@@ -504,7 +522,8 @@ mod tests {
let file_size_with_garbage = sim.file_size(fd).unwrap();
assert!(file_size_with_garbage > file_size_before);
let (index, last_seq) = rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL).unwrap();
let (index, last_seq) =
rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD).unwrap();
assert_eq!(last_seq, Some(EventSequence::new(5)));
assert_eq!(index.first_seq(), Some(EventSequence::new(1)));
@@ -524,7 +543,8 @@ mod tests {
sim.write_all_at(fd, valid_end, &partial_header).unwrap();
sim.sync(fd).unwrap();
let (_, last_seq) = rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL).unwrap();
let (_, last_seq) =
rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD).unwrap();
assert_eq!(last_seq, Some(EventSequence::new(3)));
assert_eq!(sim.file_size(fd).unwrap(), valid_end);
}
@@ -532,8 +552,14 @@ mod tests {
#[test]
fn rebuild_truncates_at_non_monotonic_seq() {
let (sim, fd) = setup();
let mut writer =
SegmentWriter::new(&sim, fd, SegmentId::new(0), EventSequence::new(1)).unwrap();
let mut writer = SegmentWriter::new(
&sim,
fd,
SegmentId::new(0),
EventSequence::new(1),
MAX_EVENT_PAYLOAD,
)
.unwrap();
let event1 = test_event(1, b"first");
let event2 = test_event(2, b"second");
@@ -553,10 +579,11 @@ mod tests {
event_type: EventTypeTag::COMMIT,
payload: b"regressed".to_vec(),
};
encode_event_record(&sim, fd, offset_after_two, &regressed).unwrap();
encode_event_record(&sim, fd, offset_after_two, &regressed, MAX_EVENT_PAYLOAD).unwrap();
sim.sync(fd).unwrap();
let (index, last_seq) = rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL).unwrap();
let (index, last_seq) =
rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD).unwrap();
assert_eq!(last_seq, Some(EventSequence::new(2)));
assert_eq!(index.first_seq(), Some(EventSequence::new(1)));
assert_eq!(index.last_seq(), Some(EventSequence::new(2)));
@@ -569,7 +596,7 @@ mod tests {
let offsets = write_n_events(&sim, fd, 10);
sim.sync(fd).unwrap();
let (index, _) = rebuild_from_segment(&sim, fd, 1).unwrap();
let (index, _) = rebuild_from_segment(&sim, fd, 1, MAX_EVENT_PAYLOAD).unwrap();
assert_eq!(index.entry_count(), 10);
offsets.iter().enumerate().for_each(|(i, (seq, offset))| {
@@ -590,7 +617,7 @@ mod tests {
write_n_events(&sim, fd, 300);
sim.sync(fd).unwrap();
let (index, last_seq) = rebuild_from_segment(&sim, fd, 256).unwrap();
let (index, last_seq) = rebuild_from_segment(&sim, fd, 256, MAX_EVENT_PAYLOAD).unwrap();
assert_eq!(last_seq, Some(EventSequence::new(300)));
let index_path = Path::new("/test/00000000.tqi");
@@ -640,7 +667,7 @@ mod tests {
sim.write_all_at(fd, 0, b"NOPE\x01").unwrap();
sim.sync(fd).unwrap();
let result = rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL);
let result = rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD);
assert!(result.is_err());
}
@@ -651,7 +678,7 @@ mod tests {
sim.sync(fd).unwrap();
let size_before = sim.file_size(fd).unwrap();
rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL).unwrap();
rebuild_from_segment(&sim, fd, DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD).unwrap();
let size_after = sim.file_size(fd).unwrap();
assert_eq!(size_before, size_after);
}
crates/tranquil-store/src/eventlog/sidecar.rs
+1076
View File
File diff suppressed because it is too large Load Diff
crates/tranquil-store/src/eventlog/types.rs
+4 -32
View File
@@ -1,8 +1,9 @@
use serde::{Deserialize, Serialize};
use tranquil_db_traits::SequenceNumber;
pub const MAX_EVENT_PAYLOAD: u32 = 4 * 1024 * 1024;
pub const DEFAULT_SEGMENT_SIZE: u64 = 64 * 1024 * 1024;
pub const MAX_EVENT_PAYLOAD: u32 = u32::MAX;
pub const DEFAULT_MAX_EVENT_PAYLOAD: u32 = 256 * 1024 * 1024;
pub const DEFAULT_SEGMENT_SIZE: u64 = 256 * 1024 * 1024;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct EventSequence(u64);
@@ -114,15 +115,11 @@ impl SegmentOffset {
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct EventLength(u32);
impl EventLength {
pub fn new(length: u32) -> Self {
assert!(
length <= MAX_EVENT_PAYLOAD,
"EventLength {length} exceeds MAX_EVENT_PAYLOAD {MAX_EVENT_PAYLOAD}"
);
Self(length)
}
@@ -135,18 +132,6 @@ impl EventLength {
}
}
impl<'de> Deserialize<'de> for EventLength {
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
let raw = u32::deserialize(deserializer)?;
if raw > MAX_EVENT_PAYLOAD {
return Err(serde::de::Error::custom(format_args!(
"EventLength {raw} exceeds MAX_EVENT_PAYLOAD {MAX_EVENT_PAYLOAD}"
)));
}
Ok(Self(raw))
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct DidHash(u32);
@@ -359,12 +344,6 @@ mod tests {
assert_eq!(len.raw(), MAX_EVENT_PAYLOAD);
}
#[test]
#[should_panic(expected = "exceeds MAX_EVENT_PAYLOAD")]
fn event_length_overflow_panics() {
EventLength::new(MAX_EVENT_PAYLOAD + 1);
}
#[test]
fn did_hash_deterministic() {
let hash1 = DidHash::from_did("did:plc:abc123");
@@ -474,13 +453,6 @@ mod tests {
assert!(postcard::from_bytes::<EventTypeTag>(&bytes).is_err());
}
#[test]
fn postcard_rejects_oversized_event_length() {
let oversized = MAX_EVENT_PAYLOAD + 1;
let bytes = postcard::to_allocvec(&oversized).unwrap();
assert!(postcard::from_bytes::<EventLength>(&bytes).is_err());
}
#[test]
fn event_sequence_to_sequence_number() {
let es = EventSequence::new(42);
crates/tranquil-store/src/eventlog/writer.rs
+169 -51
View File
@@ -8,9 +8,9 @@ use crate::io::StorageIO;
use super::manager::SegmentManager;
use super::segment_file::{SEGMENT_HEADER_SIZE, SegmentWriter, ValidEvent};
use super::segment_index::{DEFAULT_INDEX_INTERVAL, SegmentIndex, rebuild_from_segment};
use super::sidecar::build_sidecar_from_segment;
use super::types::{
DidHash, EventSequence, EventTypeTag, MAX_EVENT_PAYLOAD, SegmentId, SegmentOffset,
TimestampMicros,
DidHash, EventSequence, EventTypeTag, SegmentId, SegmentOffset, TimestampMicros,
};
#[derive(Debug)]
@@ -28,14 +28,20 @@ pub struct EventLogWriter<S: StorageIO> {
next_seq: EventSequence,
synced_seq: EventSequence,
index_interval: usize,
max_payload: u32,
event_count_in_segment: usize,
last_event_offset: Option<SegmentOffset>,
pending_events: Vec<ValidEvent>,
}
impl<S: StorageIO> EventLogWriter<S> {
pub fn open(manager: Arc<SegmentManager<S>>, index_interval: usize) -> io::Result<Self> {
pub fn open(
manager: Arc<SegmentManager<S>>,
index_interval: usize,
max_payload: u32,
) -> io::Result<Self> {
assert!(index_interval > 0, "index_interval must be positive");
assert!(max_payload > 0, "max_payload must be positive");
let segments = manager.list_segments()?;
@@ -45,8 +51,11 @@ impl<S: StorageIO> EventLogWriter<S> {
SegmentId::new(1),
EventSequence::new(1),
index_interval,
max_payload,
),
Some(&last_id) => Self::recover_active(manager, &segments, last_id, index_interval),
Some(&last_id) => {
Self::recover_active(manager, &segments, last_id, index_interval, max_payload)
}
}
}
@@ -55,10 +64,11 @@ impl<S: StorageIO> EventLogWriter<S> {
segment_id: SegmentId,
next_seq: EventSequence,
index_interval: usize,
max_payload: u32,
) -> io::Result<Self> {
let fd = manager.open_for_append(segment_id)?;
manager.io().truncate(fd, 0)?;
let writer = SegmentWriter::new(manager.io(), fd, segment_id, next_seq)?;
let writer = SegmentWriter::new(manager.io(), fd, segment_id, next_seq, max_payload)?;
writer.sync(manager.io())?;
manager.io().sync_dir(manager.segments_dir())?;
@@ -69,6 +79,7 @@ impl<S: StorageIO> EventLogWriter<S> {
next_seq,
synced_seq: next_seq.prev_or_before_all(),
index_interval,
max_payload,
event_count_in_segment: 0,
last_event_offset: None,
pending_events: Vec::new(),
@@ -80,6 +91,7 @@ impl<S: StorageIO> EventLogWriter<S> {
segments: &[SegmentId],
active_id: SegmentId,
index_interval: usize,
max_payload: u32,
) -> io::Result<Self> {
let fd = manager.open_for_append(active_id)?;
@@ -87,6 +99,7 @@ impl<S: StorageIO> EventLogWriter<S> {
manager.io(),
fd,
index_interval,
max_payload,
) {
Ok(result) => result,
Err(rebuild_err) => {
@@ -94,13 +107,15 @@ impl<S: StorageIO> EventLogWriter<S> {
if file_size <= SEGMENT_HEADER_SIZE as u64 {
manager.io().truncate(fd, 0)?;
let prev_segments = &segments[..segments.len().saturating_sub(1)];
let next_seq = find_last_seq_from_segments(&manager, prev_segments)?
.map_or(EventSequence::new(1), |s| s.next());
let next_seq =
find_last_seq_from_segments(&manager, prev_segments, max_payload)?
.map_or(EventSequence::new(1), |s| s.next());
return Self::init_fresh(
Arc::clone(&manager),
active_id,
next_seq,
index_interval,
max_payload,
);
}
return Err(io::Error::new(
@@ -118,7 +133,8 @@ impl<S: StorageIO> EventLogWriter<S> {
let next_seq = match last_seq_in_active {
Some(seq) => {
if let Some(sealed_last) = find_last_seq_from_segments(&manager, prev_segments)?
if let Some(sealed_last) =
find_last_seq_from_segments(&manager, prev_segments, max_payload)?
&& seq <= sealed_last
{
return Err(io::Error::new(
@@ -131,7 +147,7 @@ impl<S: StorageIO> EventLogWriter<S> {
}
seq.next()
}
None => find_last_seq_from_segments(&manager, prev_segments)?
None => find_last_seq_from_segments(&manager, prev_segments, max_payload)?
.map_or(EventSequence::new(1), |s| s.next()),
};
@@ -159,6 +175,7 @@ impl<S: StorageIO> EventLogWriter<S> {
position,
base_seq,
last_seq_in_active,
max_payload,
);
if let Err(e) = manager.io().delete(&manager.index_path(active_id))
@@ -174,6 +191,7 @@ impl<S: StorageIO> EventLogWriter<S> {
next_seq,
synced_seq,
index_interval,
max_payload,
event_count_in_segment,
last_event_offset,
pending_events: Vec::new(),
@@ -188,12 +206,11 @@ impl<S: StorageIO> EventLogWriter<S> {
) -> io::Result<EventSequence> {
let payload_len = u32::try_from(payload.len())
.map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "payload exceeds u32::MAX"))?;
if payload_len > MAX_EVENT_PAYLOAD {
if payload_len > self.max_payload {
let max = self.max_payload;
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!(
"payload length {payload_len} exceeds MAX_EVENT_PAYLOAD {MAX_EVENT_PAYLOAD}"
),
format!("payload length {payload_len} exceeds configured max_payload {max}"),
));
}
@@ -229,6 +246,32 @@ impl<S: StorageIO> EventLogWriter<S> {
Ok(seq)
}
pub fn append_valid_event(&mut self, event: ValidEvent) -> io::Result<()> {
let offset = self.active_writer.append_event(self.manager.io(), &event)?;
let should_index = self.event_count_in_segment == 0
|| self
.event_count_in_segment
.is_multiple_of(self.index_interval);
if should_index {
self.active_index.record(event.seq, offset);
}
self.event_count_in_segment = self
.event_count_in_segment
.checked_add(1)
.expect("event_count_in_segment overflow");
self.last_event_offset = Some(offset);
self.next_seq = event.seq.next();
self.pending_events.push(event);
Ok(())
}
pub fn peek_pending_event(&self, seq: EventSequence) -> Option<&ValidEvent> {
self.pending_events.iter().find(|e| e.seq == seq)
}
pub fn sync(&mut self) -> io::Result<SyncResult> {
if !self.pending_events.is_empty() {
self.active_writer.sync(self.manager.io())?;
@@ -260,9 +303,20 @@ impl<S: StorageIO> EventLogWriter<S> {
self.manager.seal_segment(old_id, &self.active_index)?;
match self.build_sidecar_for_segment(old_id) {
Ok(()) => {}
Err(e) => warn!(segment = %old_id, error = %e, "sidecar build failed (non-fatal)"),
}
let (new_id, new_fd) = self.manager.prepare_rotation(old_id)?;
match SegmentWriter::new::<S>(self.manager.io(), new_fd, new_id, self.next_seq) {
match SegmentWriter::new::<S>(
self.manager.io(),
new_fd,
new_id,
self.next_seq,
self.max_payload,
) {
Ok(writer) => {
self.active_writer = writer;
self.active_index = SegmentIndex::new();
@@ -306,6 +360,13 @@ impl<S: StorageIO> EventLogWriter<S> {
self.active_writer.position()
}
fn build_sidecar_for_segment(&self, segment_id: SegmentId) -> io::Result<()> {
let fd = self.manager.open_for_read(segment_id)?;
let sidecar = build_sidecar_from_segment(self.manager.io(), fd, self.max_payload)?;
let path = self.manager.sidecar_path(segment_id);
sidecar.save(self.manager.io(), &path)
}
pub fn shutdown(&mut self) -> io::Result<()> {
let _ = self.sync()?;
self.ensure_last_event_indexed();
@@ -325,6 +386,7 @@ impl<S: StorageIO> EventLogWriter<S> {
fn find_last_seq_from_segments<S: StorageIO>(
manager: &SegmentManager<S>,
segments: &[SegmentId],
max_payload: u32,
) -> io::Result<Option<EventSequence>> {
segments.iter().rev().try_fold(None, |acc, &seg_id| {
if acc.is_some() {
@@ -336,7 +398,8 @@ fn find_last_seq_from_segments<S: StorageIO>(
Err(e) if e.kind() != io::ErrorKind::InvalidData => Err(e),
_ => {
let fd = manager.open_for_read(seg_id)?;
let (_, last_seq) = rebuild_from_segment(manager.io(), fd, DEFAULT_INDEX_INTERVAL)?;
let (_, last_seq) =
rebuild_from_segment(manager.io(), fd, DEFAULT_INDEX_INTERVAL, max_payload)?;
Ok(last_seq)
}
}
@@ -348,6 +411,7 @@ mod tests {
use super::*;
use crate::eventlog::segment_file::{EVENT_RECORD_OVERHEAD, SegmentReader};
use crate::eventlog::segment_index::DEFAULT_INDEX_INTERVAL;
use crate::eventlog::types::MAX_EVENT_PAYLOAD;
use crate::sim::SimulatedIO;
use std::path::{Path, PathBuf};
@@ -372,7 +436,9 @@ mod tests {
#[test]
fn open_fresh_creates_segment() {
let mgr = setup_manager(64 * 1024);
let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
assert_eq!(writer.active_segment_id(), SegmentId::new(1));
assert_eq!(writer.current_seq(), EventSequence::BEFORE_ALL);
@@ -389,7 +455,9 @@ mod tests {
#[test]
fn append_assigns_contiguous_sequences() {
let mgr = setup_manager(64 * 1024);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
let seqs: Vec<EventSequence> = (1..=5)
.map(|i| append_test_event(&mut writer, &format!("did:plc:user{i}")))
@@ -402,7 +470,9 @@ mod tests {
#[test]
fn sync_returns_flushed_events() {
let mgr = setup_manager(64 * 1024);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
(1..=3).for_each(|i| {
append_test_event(&mut writer, &format!("did:plc:user{i}"));
@@ -427,7 +497,9 @@ mod tests {
#[test]
fn sync_without_pending_is_noop() {
let mgr = setup_manager(64 * 1024);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
let result = writer.sync().unwrap();
assert_eq!(result.synced_through, EventSequence::BEFORE_ALL);
@@ -437,7 +509,9 @@ mod tests {
#[test]
fn second_sync_returns_only_new_events() {
let mgr = setup_manager(64 * 1024);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
(1..=3).for_each(|i| {
append_test_event(&mut writer, &format!("did:plc:user{i}"));
@@ -460,7 +534,8 @@ mod tests {
{
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
(1..=5).for_each(|i| {
append_test_event(&mut writer, &format!("did:plc:user{i}"));
});
@@ -469,13 +544,15 @@ mod tests {
mgr.shutdown();
let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
assert_eq!(writer.current_seq(), EventSequence::new(5));
assert_eq!(writer.synced_seq(), EventSequence::new(5));
assert_eq!(writer.active_segment_id(), SegmentId::new(1));
let fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
let events = SegmentReader::open(mgr.io(), fd)
let events = SegmentReader::open(mgr.io(), fd, MAX_EVENT_PAYLOAD)
.unwrap()
.valid_prefix()
.unwrap();
@@ -488,7 +565,8 @@ mod tests {
{
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
(1..=3).for_each(|i| {
append_test_event(&mut writer, &format!("did:plc:user{i}"));
});
@@ -503,7 +581,9 @@ mod tests {
mgr.shutdown();
mgr.io().crash();
let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
assert_eq!(writer.current_seq(), EventSequence::new(3));
assert_eq!(writer.next_seq, EventSequence::new(4));
}
@@ -515,7 +595,9 @@ mod tests {
let max_segment_size = SEGMENT_HEADER_SIZE + record_size * 3;
let mgr = setup_manager(max_segment_size as u64);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
(1..=3).for_each(|i| {
writer
@@ -546,7 +628,9 @@ mod tests {
let max_segment_size = SEGMENT_HEADER_SIZE + record_size * 2;
let mgr = setup_manager(max_segment_size as u64);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
(1..=2).for_each(|i| {
writer
@@ -576,7 +660,9 @@ mod tests {
let max_segment_size = SEGMENT_HEADER_SIZE + record_size * 2;
let mgr = setup_manager(max_segment_size as u64);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
(1..=2).for_each(|i| {
writer
@@ -610,7 +696,8 @@ mod tests {
{
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
(1..=2).for_each(|i| {
writer
.append(
@@ -635,7 +722,9 @@ mod tests {
mgr.shutdown();
let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
assert_eq!(writer.active_segment_id(), SegmentId::new(2));
assert_eq!(writer.current_seq(), EventSequence::new(3));
assert_eq!(writer.next_seq, EventSequence::new(4));
@@ -651,7 +740,8 @@ mod tests {
{
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
(1..=2).for_each(|i| {
writer
.append(
@@ -668,7 +758,9 @@ mod tests {
mgr.shutdown();
mgr.io().crash();
let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
assert_eq!(writer.next_seq, EventSequence::new(3));
}
@@ -682,7 +774,8 @@ mod tests {
{
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
(1..=2).for_each(|i| {
writer
.append(
@@ -698,11 +791,13 @@ mod tests {
mgr.shutdown();
let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
assert_eq!(writer.next_seq, EventSequence::new(3));
let fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
let events = SegmentReader::open(mgr.io(), fd)
let events = SegmentReader::open(mgr.io(), fd, MAX_EVENT_PAYLOAD)
.unwrap()
.valid_prefix()
.unwrap();
@@ -712,7 +807,9 @@ mod tests {
#[test]
fn checkpoint_creates_index_file() {
let mgr = setup_manager(64 * 1024);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
(1..=10).for_each(|i| {
append_test_event(&mut writer, &format!("did:plc:user{i}"));
@@ -729,7 +826,9 @@ mod tests {
#[test]
fn checkpoint_empty_index_is_noop() {
let mgr = setup_manager(64 * 1024);
let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
writer.checkpoint_index().unwrap();
@@ -741,7 +840,9 @@ mod tests {
#[test]
fn current_seq_and_synced_seq_diverge_before_sync() {
let mgr = setup_manager(64 * 1024);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
append_test_event(&mut writer, "did:plc:user1");
append_test_event(&mut writer, "did:plc:user2");
@@ -758,7 +859,7 @@ mod tests {
#[test]
fn sparse_index_built_at_intervals() {
let mgr = setup_manager(64 * 1024);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), 4).unwrap();
let mut writer = EventLogWriter::open(Arc::clone(&mgr), 4, MAX_EVENT_PAYLOAD).unwrap();
(1..=10).for_each(|i| {
append_test_event(&mut writer, &format!("did:plc:user{i}"));
@@ -781,7 +882,8 @@ mod tests {
{
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
(1..=9).for_each(|i| {
writer
.append(
@@ -801,7 +903,9 @@ mod tests {
mgr.shutdown();
let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
assert_eq!(writer.next_seq, EventSequence::new(10));
let segments = mgr.list_segments().unwrap();
@@ -811,7 +915,9 @@ mod tests {
#[test]
fn shutdown_syncs_and_checkpoints() {
let mgr = setup_manager(64 * 1024);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
(1..=5).for_each(|i| {
append_test_event(&mut writer, &format!("did:plc:user{i}"));
@@ -834,7 +940,7 @@ mod tests {
let max_segment_size = SEGMENT_HEADER_SIZE + record_size * 5;
let mgr = setup_manager(max_segment_size as u64);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256).unwrap();
let mut writer = EventLogWriter::open(Arc::clone(&mgr), 256, MAX_EVENT_PAYLOAD).unwrap();
(1..=5).for_each(|i| {
writer
@@ -861,11 +967,15 @@ mod tests {
let mgr = setup_manager(64 * 1024);
{
let _writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let _writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
}
mgr.shutdown();
let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
assert_eq!(writer.active_segment_id(), SegmentId::new(1));
assert_eq!(writer.current_seq(), EventSequence::BEFORE_ALL);
}
@@ -876,7 +986,8 @@ mod tests {
{
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
(1..=3).for_each(|i| {
append_test_event(&mut writer, &format!("did:plc:user{i}"));
});
@@ -885,13 +996,15 @@ mod tests {
mgr.shutdown();
let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
let seq = append_test_event(&mut writer, "did:plc:user4");
assert_eq!(seq, EventSequence::new(4));
writer.sync().unwrap();
let fd = mgr.open_for_read(SegmentId::new(1)).unwrap();
let events = SegmentReader::open(mgr.io(), fd)
let events = SegmentReader::open(mgr.io(), fd, MAX_EVENT_PAYLOAD)
.unwrap()
.valid_prefix()
.unwrap();
@@ -909,7 +1022,8 @@ mod tests {
{
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
(1..=2).for_each(|i| {
writer
.append(
@@ -955,14 +1069,18 @@ mod tests {
mgr.io().sync(fd2).unwrap();
mgr.io().close(fd2).unwrap();
let writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
assert_eq!(writer.next_seq, EventSequence::new(5));
}
#[test]
fn rotation_not_needed_returns_false() {
let mgr = setup_manager(64 * 1024);
let mut writer = EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL).unwrap();
let mut writer =
EventLogWriter::open(Arc::clone(&mgr), DEFAULT_INDEX_INTERVAL, MAX_EVENT_PAYLOAD)
.unwrap();
append_test_event(&mut writer, "did:plc:user1");
writer.sync().unwrap();
crates/tranquil-store/src/harness.rs
+4 -2
View File
@@ -207,6 +207,8 @@ pub struct PristineComparisonResult {
#[cfg(test)]
mod tests {
use super::*;
use crate::sim::sim_seed_range;
use rayon::prelude::*;
#[test]
fn no_fault_recovers_all_synced() {
@@ -242,7 +244,7 @@ mod tests {
#[test]
fn faulted_recovery_never_exceeds_written() {
(0..1000).for_each(|seed| {
sim_seed_range().into_par_iter().for_each(|seed| {
let payloads: Vec<Vec<u8>> = (0..5).map(|i| format!("data-{i}").into_bytes()).collect();
let Ok(result) = run_crash_test(seed, FaultConfig::moderate(), &payloads, 2) else {
return;
@@ -258,7 +260,7 @@ mod tests {
#[test]
fn pristine_comparison_with_faults() {
(0..1000).for_each(|seed| {
sim_seed_range().into_par_iter().for_each(|seed| {
let payloads: Vec<Vec<u8>> = (0..8).map(|i| format!("item-{i}").into_bytes()).collect();
let Ok(result) = run_pristine_comparison(seed, FaultConfig::moderate(), &payloads, 4)
else {
crates/tranquil-store/src/io.rs
+1
View File
@@ -25,6 +25,7 @@ impl AsRef<[u8]> for MappedFile {
pub struct FileId(u64);
impl FileId {
#[cfg(any(test, feature = "test-harness"))]
pub(crate) fn new(id: u64) -> Self {
Self(id)
}
crates/tranquil-store/src/lib.rs
+12 -1
View File
@@ -1,4 +1,8 @@
pub mod archival;
pub mod backup;
pub mod blockstore;
pub mod bloom;
pub mod consistency;
pub mod eventlog;
pub mod fsync_order;
#[cfg(any(test, feature = "test-harness"))]
@@ -21,4 +25,11 @@ pub use record::{
RecordReader, RecordWriter,
};
#[cfg(any(test, feature = "test-harness"))]
pub use sim::{FaultConfig, OpRecord, SimulatedIO};
pub use sim::{
FaultConfig, OpRecord, SimulatedIO, sim_proptest_cases, sim_seed_count, sim_seed_range,
sim_single_seed,
};
pub(crate) fn wall_clock_ms() -> blockstore::WallClockMs {
blockstore::WallClockMs::now()
}
crates/tranquil-store/src/metastore/backlink_ops.rs
+30 -35
View File
@@ -175,7 +175,6 @@ impl BacklinkOps {
)?;
let uri: AtUri = val.source_uri.into();
let is_self = uri.as_str() == bl.uri.as_str();
let matches_collection = uri.collection().is_some_and(|c| c == collection_str);
let matches_path = match discriminant_to_path(val.path) {
Some(p) => p == bl.path,
@@ -188,11 +187,7 @@ impl BacklinkOps {
false
}
};
if !is_self
&& matches_collection
&& matches_path
&& !seen.contains(uri.as_str())
{
if matches_collection && matches_path && !seen.contains(uri.as_str()) {
seen.insert(uri.as_str().to_owned());
conflicts.push(uri);
}
@@ -263,9 +258,9 @@ mod tests {
fn add_and_query_by_target() {
let h = setup();
let ops = h.metastore.backlink_ops();
let (_user_id, user_hash) = create_repo(&h, "alice", 1);
let (_user_id, user_hash) = create_repo(&h, "olaren", 1);
let uri = AtUri::from_parts("did:plc:alice", "app.bsky.feed.like", "3k2abc");
let uri = AtUri::from_parts("did:plc:olaren", "app.bsky.feed.like", "3k2abc");
let backlinks = vec![Backlink {
uri: uri.clone(),
path: BacklinkPath::SubjectUri,
@@ -290,9 +285,9 @@ mod tests {
fn remove_by_uri_deletes_both_indexes() {
let h = setup();
let ops = h.metastore.backlink_ops();
let (_user_id, user_hash) = create_repo(&h, "bob", 2);
let (_user_id, user_hash) = create_repo(&h, "teq", 2);
let uri = AtUri::from_parts("did:plc:bob", "app.bsky.graph.follow", "3k2fol");
let uri = AtUri::from_parts("did:plc:teq", "app.bsky.graph.follow", "3k2fol");
let backlinks = vec![Backlink {
uri: uri.clone(),
path: BacklinkPath::Subject,
@@ -335,11 +330,11 @@ mod tests {
fn remove_by_repo_deletes_all_user_backlinks() {
let h = setup();
let ops = h.metastore.backlink_ops();
let (_user_id, user_hash) = create_repo(&h, "carol", 3);
let (_user_id, user_hash) = create_repo(&h, "nel", 3);
let backlinks: Vec<Backlink> = (0..5)
.map(|i| Backlink {
uri: AtUri::from_parts("did:plc:carol", "app.bsky.feed.like", &format!("3k2r{i}")),
uri: AtUri::from_parts("did:plc:nel", "app.bsky.feed.like", &format!("3k2r{i}")),
path: BacklinkPath::SubjectUri,
link_to: format!("at://did:plc:target{i}/app.bsky.feed.post/3k2p{i}"),
})
@@ -376,10 +371,10 @@ mod tests {
fn get_backlink_conflicts_finds_matching() {
let h = setup();
let ops = h.metastore.backlink_ops();
let (user_id, user_hash) = create_repo(&h, "dave", 4);
let (user_id, user_hash) = create_repo(&h, "lyna", 4);
let existing = Backlink {
uri: AtUri::from_parts("did:plc:dave", "app.bsky.feed.like", "3k2old"),
uri: AtUri::from_parts("did:plc:lyna", "app.bsky.feed.like", "3k2old"),
path: BacklinkPath::SubjectUri,
link_to: "at://did:plc:someone/app.bsky.feed.post/3k2p1".to_string(),
};
@@ -390,7 +385,7 @@ mod tests {
batch.commit().unwrap();
let proposed = vec![Backlink {
uri: AtUri::from_parts("did:plc:dave", "app.bsky.feed.like", "3k2new"),
uri: AtUri::from_parts("did:plc:lyna", "app.bsky.feed.like", "3k2new"),
path: BacklinkPath::SubjectUri,
link_to: "at://did:plc:someone/app.bsky.feed.post/3k2p1".to_string(),
}];
@@ -403,7 +398,7 @@ mod tests {
assert_eq!(conflicts.len(), 1);
assert_eq!(
conflicts[0].as_str(),
"at://did:plc:dave/app.bsky.feed.like/3k2old"
"at://did:plc:lyna/app.bsky.feed.like/3k2old"
);
}
@@ -411,10 +406,10 @@ mod tests {
fn get_backlink_conflicts_ignores_different_collection() {
let h = setup();
let ops = h.metastore.backlink_ops();
let (user_id, user_hash) = create_repo(&h, "eve", 5);
let (user_id, user_hash) = create_repo(&h, "bailey", 5);
let existing = Backlink {
uri: AtUri::from_parts("did:plc:eve", "app.bsky.feed.like", "3k2old"),
uri: AtUri::from_parts("did:plc:bailey", "app.bsky.feed.like", "3k2old"),
path: BacklinkPath::SubjectUri,
link_to: "at://did:plc:someone/app.bsky.feed.post/3k2p1".to_string(),
};
@@ -425,7 +420,7 @@ mod tests {
batch.commit().unwrap();
let proposed = vec![Backlink {
uri: AtUri::from_parts("did:plc:eve", "app.bsky.feed.repost", "3k2new"),
uri: AtUri::from_parts("did:plc:bailey", "app.bsky.feed.repost", "3k2new"),
path: BacklinkPath::SubjectUri,
link_to: "at://did:plc:someone/app.bsky.feed.post/3k2p1".to_string(),
}];
@@ -442,10 +437,10 @@ mod tests {
fn get_backlink_conflicts_ignores_different_path() {
let h = setup();
let ops = h.metastore.backlink_ops();
let (user_id, user_hash) = create_repo(&h, "frank", 6);
let (user_id, user_hash) = create_repo(&h, "olaren", 6);
let existing = Backlink {
uri: AtUri::from_parts("did:plc:frank", "app.bsky.graph.follow", "3k2old"),
uri: AtUri::from_parts("did:plc:olaren", "app.bsky.graph.follow", "3k2old"),
path: BacklinkPath::Subject,
link_to: "did:plc:target".to_string(),
};
@@ -456,7 +451,7 @@ mod tests {
batch.commit().unwrap();
let proposed = vec![Backlink {
uri: AtUri::from_parts("did:plc:frank", "app.bsky.graph.follow", "3k2new"),
uri: AtUri::from_parts("did:plc:olaren", "app.bsky.graph.follow", "3k2new"),
path: BacklinkPath::SubjectUri,
link_to: "did:plc:target".to_string(),
}];
@@ -473,11 +468,11 @@ mod tests {
fn get_backlink_conflicts_ignores_other_users() {
let h = setup();
let ops = h.metastore.backlink_ops();
let (_user_id_a, user_hash_a) = create_repo(&h, "grace", 7);
let (user_id_b, _user_hash_b) = create_repo(&h, "henry", 8);
let (_user_id_a, user_hash_a) = create_repo(&h, "teq", 7);
let (user_id_b, _user_hash_b) = create_repo(&h, "nel", 8);
let existing = Backlink {
uri: AtUri::from_parts("did:plc:grace", "app.bsky.feed.like", "3k2old"),
uri: AtUri::from_parts("did:plc:teq", "app.bsky.feed.like", "3k2old"),
path: BacklinkPath::SubjectUri,
link_to: "at://did:plc:target/app.bsky.feed.post/3k2p1".to_string(),
};
@@ -488,7 +483,7 @@ mod tests {
batch.commit().unwrap();
let proposed = vec![Backlink {
uri: AtUri::from_parts("did:plc:henry", "app.bsky.feed.like", "3k2new"),
uri: AtUri::from_parts("did:plc:nel", "app.bsky.feed.like", "3k2new"),
path: BacklinkPath::SubjectUri,
link_to: "at://did:plc:target/app.bsky.feed.post/3k2p1".to_string(),
}];
@@ -502,13 +497,13 @@ mod tests {
}
#[test]
fn get_backlink_conflicts_excludes_self_match() {
fn get_backlink_conflicts_includes_self_match() {
let h = setup();
let ops = h.metastore.backlink_ops();
let (user_id, user_hash) = create_repo(&h, "luna", 12);
let (user_id, user_hash) = create_repo(&h, "lyna", 12);
let existing = Backlink {
uri: AtUri::from_parts("did:plc:luna", "app.bsky.feed.like", "3k2same"),
uri: AtUri::from_parts("did:plc:lyna", "app.bsky.feed.like", "3k2same"),
path: BacklinkPath::SubjectUri,
link_to: "at://did:plc:someone/app.bsky.feed.post/3k2p1".to_string(),
};
@@ -519,7 +514,7 @@ mod tests {
batch.commit().unwrap();
let proposed = vec![Backlink {
uri: AtUri::from_parts("did:plc:luna", "app.bsky.feed.like", "3k2same"),
uri: AtUri::from_parts("did:plc:lyna", "app.bsky.feed.like", "3k2same"),
path: BacklinkPath::SubjectUri,
link_to: "at://did:plc:someone/app.bsky.feed.post/3k2p1".to_string(),
}];
@@ -529,14 +524,14 @@ mod tests {
.get_backlink_conflicts(user_id, &collection, &proposed)
.unwrap();
assert!(conflicts.is_empty());
assert_eq!(conflicts.len(), 1);
}
#[test]
fn empty_backlinks_returns_empty_conflicts() {
let h = setup();
let ops = h.metastore.backlink_ops();
let (user_id, _user_hash) = create_repo(&h, "ivan", 9);
let (user_id, _user_hash) = create_repo(&h, "bailey", 9);
let collection = Nsid::from("app.bsky.feed.like".to_string());
let conflicts = ops
@@ -549,15 +544,15 @@ mod tests {
fn remove_by_uri_only_removes_matching_rkey() {
let h = setup();
let ops = h.metastore.backlink_ops();
let (_user_id, user_hash) = create_repo(&h, "julia", 10);
let (_user_id, user_hash) = create_repo(&h, "bailey", 10);
let bl1 = Backlink {
uri: AtUri::from_parts("did:plc:julia", "app.bsky.feed.like", "3k2aaa"),
uri: AtUri::from_parts("did:plc:bailey", "app.bsky.feed.like", "3k2aaa"),
path: BacklinkPath::SubjectUri,
link_to: "at://did:plc:t1/app.bsky.feed.post/p1".to_string(),
};
let bl2 = Backlink {
uri: AtUri::from_parts("did:plc:julia", "app.bsky.feed.like", "3k2bbb"),
uri: AtUri::from_parts("did:plc:bailey", "app.bsky.feed.like", "3k2bbb"),
path: BacklinkPath::SubjectUri,
link_to: "at://did:plc:t2/app.bsky.feed.post/p2".to_string(),
};
crates/tranquil-store/src/metastore/client.rs
+59 -70
View File
@@ -6,26 +6,25 @@ use chrono::{DateTime, Utc};
use tokio::sync::oneshot;
use tranquil_db_traits::{
AccountSearchResult, AccountStatus, AdminAccountInfo, ApplyCommitError, ApplyCommitInput,
ApplyCommitResult, Backlink, BrokenGenesisCommit, CommitEventData, CommsChannel, CommsType,
ApplyCommitResult, Backlink, CommitEventData, CommsChannel, CommsType,
CompletePasskeySetupInput, CreateAccountError, CreateDelegatedAccountInput,
CreatePasskeyAccountInput, CreatePasswordAccountInput, CreatePasswordAccountResult,
CreateSsoAccountInput, DbError, DeletionRequest, DeletionRequestWithToken, DidWebOverrides,
EventBlocksCids, ImportBlock, ImportRecord, ImportRepoError, InviteCodeError, InviteCodeInfo,
InviteCodeRow, InviteCodeSortOrder, InviteCodeUse, MigrationReactivationError,
MigrationReactivationInput, NotificationHistoryRow, NotificationPrefs, OAuthTokenWithUser,
PasswordResetResult, PlcTokenInfo, QueuedComms, ReactivatedAccountInfo,
RecoverPasskeyAccountInput, RecoverPasskeyAccountResult, RepoAccountInfo, RepoInfo,
RepoListItem, RepoWithoutRev, ReservedSigningKey, ReservedSigningKeyFull,
ScheduledDeletionAccount, ScopePreference, SequenceNumber, SequencedEvent, StoredBackupCode,
StoredPasskey, TokenFamilyId, TotpRecord, TotpRecordState, User2faStatus, UserAuthInfo,
UserCommsPrefs, UserConfirmSignup, UserDidWebInfo, UserEmailInfo, UserForDeletion,
UserForDidDoc, UserForDidDocBuild, UserForPasskeyRecovery, UserForPasskeySetup,
UserForRecovery, UserForVerification, UserIdAndHandle, UserIdAndPasswordHash,
UserIdHandleEmail, UserInfoForAuth, UserKeyInfo, UserKeyWithId, UserLegacyLoginPref,
UserLoginCheck, UserLoginFull, UserLoginInfo, UserNeedingRecordBlobsBackfill, UserPasswordInfo,
UserResendVerification, UserResetCodeInfo, UserRow, UserSessionInfo, UserStatus,
UserVerificationInfo, UserWithKey, UserWithoutBlocks, ValidatedInviteCode,
WebauthnChallengeType,
ImportBlock, ImportRecord, ImportRepoError, InviteCodeError, InviteCodeInfo, InviteCodeRow,
InviteCodeSortOrder, InviteCodeUse, MigrationReactivationError, MigrationReactivationInput,
NotificationHistoryRow, NotificationPrefs, OAuthTokenWithUser, PasswordResetResult,
PlcTokenInfo, PruneCount, QueuedComms, ReactivatedAccountInfo, RecoverPasskeyAccountInput,
RecoverPasskeyAccountResult, RepoAccountInfo, RepoInfo, RepoListItem, RepoWithoutRev,
ReservedSigningKey, ReservedSigningKeyFull, ScheduledDeletionAccount, ScopePreference,
SequenceNumber, SequencedEvent, StoredBackupCode, StoredPasskey, TokenFamilyId, TotpRecord,
TotpRecordState, User2faStatus, UserAuthInfo, UserCommsPrefs, UserConfirmSignup,
UserDidWebInfo, UserEmailInfo, UserForDeletion, UserForDidDoc, UserForDidDocBuild,
UserForPasskeyRecovery, UserForPasskeySetup, UserForRecovery, UserForVerification,
UserIdAndHandle, UserIdAndPasswordHash, UserIdHandleEmail, UserInfoForAuth, UserKeyInfo,
UserKeyWithId, UserLegacyLoginPref, UserLoginCheck, UserLoginFull, UserLoginInfo,
UserNeedingRecordBlobsBackfill, UserPasswordInfo, UserResendVerification, UserResetCodeInfo,
UserRow, UserSessionInfo, UserStatus, UserVerificationInfo, UserWithKey, UserWithoutBlocks,
ValidatedInviteCode, WebauthnChallengeType,
};
use tranquil_oauth::{AuthorizedClientData, DeviceData, RequestData, TokenData};
use tranquil_types::{
@@ -40,6 +39,7 @@ use super::handler::{
SsoRequest, UserBlockRequest, UserRequest,
};
use super::keys::UserHash;
use crate::eventlog::{EventLog, TimestampMicros};
use crate::io::StorageIO;
async fn recv<T>(rx: oneshot::Receiver<Result<T, DbError>>) -> Result<T, DbError> {
@@ -88,6 +88,7 @@ async fn recv_migration_reactivation(
pub struct MetastoreClient<S: StorageIO> {
pool: Arc<HandlerPool>,
event_log: Arc<EventLog<S>>,
_phantom: PhantomData<S>,
}
@@ -95,15 +96,17 @@ impl<S: StorageIO> Clone for MetastoreClient<S> {
fn clone(&self) -> Self {
Self {
pool: Arc::clone(&self.pool),
event_log: Arc::clone(&self.event_log),
_phantom: PhantomData,
}
}
}
impl<S: StorageIO> MetastoreClient<S> {
pub fn new(pool: Arc<HandlerPool>) -> Self {
pub fn new(pool: Arc<HandlerPool>, event_log: Arc<EventLog<S>>) -> Self {
Self {
pool,
event_log,
_phantom: PhantomData,
}
}
@@ -112,6 +115,10 @@ impl<S: StorageIO> MetastoreClient<S> {
&self.pool
}
pub fn event_log(&self) -> &Arc<EventLog<S>> {
&self.event_log
}
pub async fn create_repo_full(
&self,
user_id: Uuid,
@@ -467,20 +474,6 @@ impl<S: StorageIO + 'static> tranquil_db_traits::RepoRepository for MetastoreCli
recv(rx).await
}
async fn find_unreferenced_blocks(
&self,
candidate_cids: &[Vec<u8>],
) -> Result<Vec<Vec<u8>>, DbError> {
let (tx, rx) = oneshot::channel();
self.pool.send(MetastoreRequest::UserBlock(
UserBlockRequest::FindUnreferencedBlocks {
candidate_cids: candidate_cids.to_vec(),
tx,
},
))?;
recv(rx).await
}
async fn insert_commit_event(&self, data: &CommitEventData) -> Result<SequenceNumber, DbError> {
let (tx, rx) = oneshot::channel();
self.pool
@@ -526,6 +519,7 @@ impl<S: StorageIO + 'static> tranquil_db_traits::RepoRepository for MetastoreCli
did: &Did,
commit_cid: &CidLink,
rev: Option<&str>,
commit_bytes: &[u8],
) -> Result<SequenceNumber, DbError> {
let (tx, rx) = oneshot::channel();
self.pool
@@ -533,6 +527,7 @@ impl<S: StorageIO + 'static> tranquil_db_traits::RepoRepository for MetastoreCli
did: did.clone(),
commit_cid: commit_cid.clone(),
rev: rev.map(str::to_owned),
commit_bytes: commit_bytes.to_vec(),
tx,
}))?;
recv(rx).await
@@ -544,6 +539,8 @@ impl<S: StorageIO + 'static> tranquil_db_traits::RepoRepository for MetastoreCli
commit_cid: &CidLink,
mst_root_cid: &CidLink,
rev: &str,
commit_bytes: &[u8],
mst_root_bytes: &[u8],
) -> Result<SequenceNumber, DbError> {
let (tx, rx) = oneshot::channel();
self.pool.send(MetastoreRequest::Event(
@@ -552,27 +549,14 @@ impl<S: StorageIO + 'static> tranquil_db_traits::RepoRepository for MetastoreCli
commit_cid: commit_cid.clone(),
mst_root_cid: mst_root_cid.clone(),
rev: rev.to_string(),
commit_bytes: commit_bytes.to_vec(),
mst_root_bytes: mst_root_bytes.to_vec(),
tx,
},
))?;
recv(rx).await
}
async fn update_seq_blocks_cids(
&self,
seq: SequenceNumber,
blocks_cids: &[String],
) -> Result<(), DbError> {
let (tx, rx) = oneshot::channel();
self.pool
.send(MetastoreRequest::Event(EventRequest::UpdateSeqBlocksCids {
seq,
blocks_cids: blocks_cids.to_vec(),
tx,
}))?;
recv(rx).await
}
async fn delete_sequences_except(
&self,
did: &Did,
@@ -589,6 +573,34 @@ impl<S: StorageIO + 'static> tranquil_db_traits::RepoRepository for MetastoreCli
recv(rx).await
}
async fn prune_events_older_than(&self, cutoff: DateTime<Utc>) -> Result<PruneCount, DbError> {
let cutoff_micros = cutoff.timestamp_micros();
if cutoff_micros < 0 {
return Err(DbError::Query(format!(
"eventlog retention: refusing pre-epoch cutoff {cutoff_micros} us (would prune entire log)"
)));
}
let now_micros = Utc::now().timestamp_micros();
let now_us = u64::try_from(now_micros).map_err(|_| {
DbError::Query(format!(
"eventlog retention: current wall time {now_micros} us out of u64 range"
))
})?;
let cutoff_us = u64::try_from(cutoff_micros).map_err(|_| {
DbError::Query(format!(
"eventlog retention: cutoff {cutoff_micros} us out of u64 range"
))
})?;
let max_age = std::time::Duration::from_micros(now_us.saturating_sub(cutoff_us));
let event_log = Arc::clone(&self.event_log);
let now = TimestampMicros::new(now_us);
tokio::task::spawn_blocking(move || event_log.run_retention_at(now, max_age))
.await
.map_err(|e| DbError::Connection(format!("retention task panicked: {e}")))?
.map(|n| PruneCount::Segments(n as u64))
.map_err(|e| DbError::Query(format!("eventlog retention failed: {e}")))
}
async fn get_max_seq(&self) -> Result<SequenceNumber, DbError> {
let (tx, rx) = oneshot::channel();
self.pool
@@ -674,21 +686,6 @@ impl<S: StorageIO + 'static> tranquil_db_traits::RepoRepository for MetastoreCli
recv(rx).await
}
async fn get_events_since_rev(
&self,
did: &Did,
since_rev: &str,
) -> Result<Vec<EventBlocksCids>, DbError> {
let (tx, rx) = oneshot::channel();
self.pool
.send(MetastoreRequest::Event(EventRequest::GetEventsSinceRev {
did: did.clone(),
since_rev: since_rev.to_string(),
tx,
}))?;
recv(rx).await
}
async fn list_repos_paginated(
&self,
cursor_did: Option<&Did>,
@@ -764,14 +761,6 @@ impl<S: StorageIO + 'static> tranquil_db_traits::RepoRepository for MetastoreCli
recv_commit(rx).await
}
async fn get_broken_genesis_commits(&self) -> Result<Vec<BrokenGenesisCommit>, DbError> {
let (tx, rx) = oneshot::channel();
self.pool.send(MetastoreRequest::Commit(Box::new(
CommitRequest::GetBrokenGenesisCommits { tx },
)))?;
recv(rx).await
}
async fn get_users_without_blocks(&self) -> Result<Vec<UserWithoutBlocks>, DbError> {
let (tx, rx) = oneshot::channel();
self.pool.send(MetastoreRequest::Commit(Box::new(
crates/tranquil-store/src/metastore/commit_ops.rs
+31 -72
View File
@@ -24,9 +24,8 @@ use crate::eventlog::EventLogBridge;
use crate::io::StorageIO;
use tranquil_db_traits::{
ApplyCommitError, ApplyCommitInput, ApplyCommitResult, BrokenGenesisCommit, ImportBlock,
ImportRecord, ImportRepoError, RepoEventType, SequenceNumber, UserNeedingRecordBlobsBackfill,
UserWithoutBlocks,
ApplyCommitError, ApplyCommitInput, ApplyCommitResult, ImportBlock, ImportRecord,
ImportRepoError, UserNeedingRecordBlobsBackfill, UserWithoutBlocks,
};
use tranquil_types::{AtUri, CidLink, Did};
@@ -85,7 +84,7 @@ pub struct CommitOps<S: StorageIO> {
blockstore: Option<TranquilBlockStore>,
}
impl<S: StorageIO> CommitOps<S> {
impl<S: StorageIO + 'static> CommitOps<S> {
pub fn new(
db: Database,
repo_data: Keyspace,
@@ -382,46 +381,6 @@ impl<S: StorageIO> CommitOps<S> {
)
}
pub fn get_broken_genesis_commits(&self) -> Result<Vec<BrokenGenesisCommit>, MetastoreError> {
const PAGE_SIZE: usize = 4096;
self.collect_broken_genesis_page(SequenceNumber::ZERO, Vec::new(), PAGE_SIZE)
}
fn collect_broken_genesis_page(
&self,
cursor: SequenceNumber,
acc: Vec<BrokenGenesisCommit>,
page_size: usize,
) -> Result<Vec<BrokenGenesisCommit>, MetastoreError> {
let limit = i64::try_from(page_size).unwrap_or(i64::MAX);
let events = self
.event_ops
.get_events_since_seq(cursor, Some(limit))
.map_err(|_| MetastoreError::CorruptData("failed to read events"))?;
let page_len = events.len();
let page_high_seq = events.last().map(|e| e.seq).unwrap_or(cursor);
let results = events.into_iter().fold(acc, |mut results, e| {
if e.event_type == RepoEventType::Commit
&& e.prev_cid.is_none()
&& e.commit_cid.is_none()
{
results.push(BrokenGenesisCommit {
seq: e.seq,
did: e.did,
commit_cid: e.commit_cid,
});
}
results
});
match page_len < page_size {
true => Ok(results),
false => self.collect_broken_genesis_page(page_high_seq, results, page_size),
}
}
pub fn get_users_without_blocks(&self) -> Result<Vec<UserWithoutBlocks>, MetastoreError> {
const MAX_RESULTS: usize = 10_000;
@@ -518,7 +477,7 @@ mod tests {
use crate::eventlog::{EventLog, EventLogConfig};
use crate::io::RealIO;
use crate::metastore::{Metastore, MetastoreConfig};
use tranquil_db_traits::CommitEventData;
use tranquil_db_traits::{CommitEventData, RepoEventType, SequenceNumber};
use tranquil_types::{Handle, Nsid, Rkey};
struct TestHarness {
@@ -603,7 +562,7 @@ mod tests {
fn apply_commit_updates_records_and_meta() {
let h = setup();
let ops = make_commit_ops(&h);
let (user_id, did, root_cid) = create_test_repo(&h, "alice", 1);
let (user_id, did, root_cid) = create_test_repo(&h, "olaren", 1);
let new_root = test_cid_link(2);
let record_cid = test_cid_link(3);
@@ -633,7 +592,7 @@ mod tests {
prev_cid: Some(root_cid.clone()),
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev1".to_string()),
},
@@ -660,7 +619,7 @@ mod tests {
fn apply_commit_cas_rejects_stale_root() {
let h = setup();
let ops = make_commit_ops(&h);
let (user_id, did, _root_cid) = create_test_repo(&h, "bob", 10);
let (user_id, did, _root_cid) = create_test_repo(&h, "teq", 10);
let stale_root = test_cid_link(99);
let new_root = test_cid_link(11);
@@ -678,13 +637,13 @@ mod tests {
backlinks_to_add: vec![],
backlinks_to_remove: vec![],
commit_event: CommitEventData {
did: test_did("bob"),
did: test_did("teq"),
event_type: RepoEventType::Commit,
commit_cid: None,
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev1".to_string()),
},
@@ -704,7 +663,7 @@ mod tests {
let input = ApplyCommitInput {
user_id: Uuid::new_v4(),
did: test_did("nobody"),
did: test_did("nonexistent"),
expected_root_cid: None,
new_root_cid: test_cid_link(1),
new_rev: "rev1".to_string(),
@@ -715,13 +674,13 @@ mod tests {
backlinks_to_add: vec![],
backlinks_to_remove: vec![],
commit_event: CommitEventData {
did: test_did("nobody"),
did: test_did("nonexistent"),
event_type: RepoEventType::Commit,
commit_cid: None,
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: None,
},
@@ -737,7 +696,7 @@ mod tests {
fn apply_commit_record_deletes() {
let h = setup();
let ops = make_commit_ops(&h);
let (user_id, did, root_cid) = create_test_repo(&h, "carol", 20);
let (user_id, did, root_cid) = create_test_repo(&h, "nel", 20);
let mid_root = test_cid_link(21);
let record_cid = test_cid_link(22);
@@ -767,7 +726,7 @@ mod tests {
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev1".to_string()),
},
@@ -805,7 +764,7 @@ mod tests {
prev_cid: Some(mid_root.clone()),
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev2".to_string()),
},
@@ -825,7 +784,7 @@ mod tests {
fn apply_commit_event_visible_after_commit() {
let h = setup();
let ops = make_commit_ops(&h);
let (user_id, did, root_cid) = create_test_repo(&h, "dave", 30);
let (user_id, did, root_cid) = create_test_repo(&h, "lyna", 30);
let new_root = test_cid_link(31);
let input = ApplyCommitInput {
@@ -847,7 +806,7 @@ mod tests {
prev_cid: Some(root_cid.clone()),
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev1".to_string()),
},
@@ -866,7 +825,7 @@ mod tests {
fn import_repo_data_inserts_records() {
let h = setup();
let ops = make_commit_ops(&h);
let (user_id, _did, root_cid) = create_test_repo(&h, "eve", 40);
let (user_id, _did, root_cid) = create_test_repo(&h, "bailey", 40);
let collection = Nsid::from("app.bsky.feed.post".to_string());
let rkey = Rkey::from("3k2import".to_string());
@@ -897,7 +856,7 @@ mod tests {
fn import_repo_data_cas_rejects_stale_root() {
let h = setup();
let ops = make_commit_ops(&h);
let (user_id, _did, _root_cid) = create_test_repo(&h, "frank", 50);
let (user_id, _did, _root_cid) = create_test_repo(&h, "olaren", 50);
let stale = test_cid_link(99);
let result = ops.import_repo_data(user_id, &[], &[], Some(&stale));
@@ -908,8 +867,8 @@ mod tests {
fn insert_record_blobs_and_backfill_query() {
let h = setup();
let ops = make_commit_ops(&h);
let (user_id_a, did_a, _) = create_test_repo(&h, "grace", 60);
let (user_id_b, _did_b, _) = create_test_repo(&h, "henry", 61);
let (user_id_a, did_a, _) = create_test_repo(&h, "teq", 60);
let (user_id_b, _did_b, _) = create_test_repo(&h, "nel", 61);
let needing = ops.get_users_needing_record_blobs_backfill(100).unwrap();
assert_eq!(needing.len(), 2);
@@ -928,8 +887,8 @@ mod tests {
fn get_users_without_blocks_returns_users_with_no_blocks() {
let h = setup();
let ops = make_commit_ops(&h);
let (user_id_a, did_a, root_a) = create_test_repo(&h, "ivan", 70);
let (user_id_b, _did_b, _root_b) = create_test_repo(&h, "julia", 71);
let (user_id_a, did_a, root_a) = create_test_repo(&h, "lyna", 70);
let (user_id_b, _did_b, _root_b) = create_test_repo(&h, "bailey", 71);
let new_root = test_cid_link(72);
let input = ApplyCommitInput {
@@ -951,7 +910,7 @@ mod tests {
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev1".to_string()),
},
@@ -989,7 +948,7 @@ mod tests {
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev_force".to_string()),
},
@@ -1041,7 +1000,7 @@ mod tests {
prev_cid: Some(root_cid.clone()),
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev1".to_string()),
},
@@ -1085,7 +1044,7 @@ mod tests {
prev_cid: Some(mid_root.clone()),
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev2".to_string()),
},
@@ -1179,7 +1138,7 @@ mod tests {
prev_cid: Some(initial_root.clone()),
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev1".to_string()),
},
@@ -1305,7 +1264,7 @@ mod tests {
prev_cid: Some(initial_root.clone()),
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev1".to_string()),
};
@@ -1432,7 +1391,7 @@ mod tests {
prev_cid: Some(root_cid.clone()),
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev1".to_string()),
},
@@ -1479,7 +1438,7 @@ mod tests {
prev_cid: Some(mid_root.clone()),
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev2".to_string()),
},
crates/tranquil-store/src/metastore/event_keys.rs
-92
View File
@@ -1,34 +1,8 @@
use serde::{Deserialize, Serialize};
use smallvec::SmallVec;
use super::encoding::KeyBuilder;
use super::keys::{KeyTag, UserHash};
const SEQ_META_SCHEMA_VERSION: u8 = 1;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SeqMetaValue {
pub blocks_cids: Vec<String>,
}
impl SeqMetaValue {
pub fn serialize(&self) -> Vec<u8> {
let payload = postcard::to_allocvec(self).expect("SeqMetaValue serialization cannot fail");
let mut buf = Vec::with_capacity(1 + payload.len());
buf.push(SEQ_META_SCHEMA_VERSION);
buf.extend_from_slice(&payload);
buf
}
pub fn deserialize(bytes: &[u8]) -> Option<Self> {
let (&version, payload) = bytes.split_first()?;
match version {
SEQ_META_SCHEMA_VERSION => postcard::from_bytes(payload).ok(),
_ => None,
}
}
}
pub fn rev_to_seq_key(user_hash: UserHash, rev: &str) -> SmallVec<[u8; 128]> {
KeyBuilder::new()
.tag(KeyTag::REV_TO_SEQ)
@@ -44,10 +18,6 @@ pub fn rev_to_seq_user_prefix(user_hash: UserHash) -> SmallVec<[u8; 128]> {
.build()
}
pub fn seq_meta_key(seq: u64) -> SmallVec<[u8; 128]> {
KeyBuilder::new().tag(KeyTag::SEQ_META).u64(seq).build()
}
pub fn seq_tombstone_key(seq: u64) -> SmallVec<[u8; 128]> {
KeyBuilder::new()
.tag(KeyTag::SEQ_TOMBSTONE)
@@ -82,50 +52,6 @@ mod tests {
use super::*;
use crate::metastore::encoding::KeyReader;
#[test]
fn seq_meta_value_roundtrip() {
let value = SeqMetaValue {
blocks_cids: vec!["bafyreiblock1".to_owned(), "bafyreiblock2".to_owned()],
};
let bytes = value.serialize();
let decoded = SeqMetaValue::deserialize(&bytes).unwrap();
assert_eq!(decoded, value);
}
#[test]
fn seq_meta_value_empty_blocks() {
let value = SeqMetaValue {
blocks_cids: vec![],
};
let bytes = value.serialize();
let decoded = SeqMetaValue::deserialize(&bytes).unwrap();
assert_eq!(decoded, value);
}
#[test]
fn seq_meta_schema_version_first_byte() {
let value = SeqMetaValue {
blocks_cids: vec![],
};
let bytes = value.serialize();
assert_eq!(bytes[0], SEQ_META_SCHEMA_VERSION);
}
#[test]
fn seq_meta_rejects_unknown_version() {
let value = SeqMetaValue {
blocks_cids: vec![],
};
let mut bytes = value.serialize();
bytes[0] = 99;
assert!(SeqMetaValue::deserialize(&bytes).is_none());
}
#[test]
fn seq_meta_rejects_empty_input() {
assert!(SeqMetaValue::deserialize(&[]).is_none());
}
#[test]
fn rev_to_seq_key_roundtrip() {
let hash = UserHash::from_raw(0xDEAD_BEEF_CAFE_BABE);
@@ -156,24 +82,6 @@ mod tests {
assert!(full.as_slice().starts_with(prefix.as_slice()));
}
#[test]
fn seq_meta_key_roundtrip() {
let key = seq_meta_key(12345);
let mut reader = KeyReader::new(&key);
assert_eq!(reader.tag(), Some(KeyTag::SEQ_META.raw()));
assert_eq!(reader.u64(), Some(12345));
assert!(reader.is_empty());
}
#[test]
fn seq_meta_keys_sort_by_seq() {
let k1 = seq_meta_key(1);
let k2 = seq_meta_key(2);
let k3 = seq_meta_key(100);
assert!(k1.as_slice() < k2.as_slice());
assert!(k2.as_slice() < k3.as_slice());
}
#[test]
fn seq_tombstone_key_roundtrip() {
let key = seq_tombstone_key(999);
crates/tranquil-store/src/metastore/event_ops.rs
+87 -357
View File
@@ -5,15 +5,14 @@ use chrono::{DateTime, Utc};
use fjall::{Database, Keyspace};
use tracing::warn;
use tranquil_db_traits::{
AccountStatus, CommitEventData, DbError, EventBlocksCids, RepoEventType, SequenceNumber,
SequencedEvent,
AccountStatus, CommitEventData, DbError, RepoEventType, SequenceNumber, SequencedEvent,
};
use tranquil_types::{CidLink, Did, Handle};
use super::encoding::{KeyReader, exclusive_upper_bound};
use super::event_keys::{
SeqMetaValue, did_events_key, did_events_prefix, metastore_cursor_key, rev_to_seq_key,
rev_to_seq_user_prefix, seq_meta_key, seq_tombstone_key,
did_events_key, did_events_prefix, metastore_cursor_key, rev_to_seq_key,
rev_to_seq_user_prefix, seq_tombstone_key,
};
use super::keys::UserHash;
use super::recovery::CommitMutationSet;
@@ -29,7 +28,7 @@ pub struct EventOps<S: StorageIO> {
bridge: Arc<EventLogBridge<S>>,
}
impl<S: StorageIO> EventOps<S> {
impl<S: StorageIO + 'static> EventOps<S> {
pub fn new(db: Database, repo_data: Keyspace, bridge: Arc<EventLogBridge<S>>) -> Self {
Self {
db,
@@ -61,7 +60,7 @@ impl<S: StorageIO> EventOps<S> {
let payload = crate::eventlog::encode_payload_with_mutations(&event, mutation_set_bytes);
let (seq, deferred) = self
.bridge
.insert_event_deferred_raw(&data.did, data.event_type, payload)
.insert_event_group_commit_raw(&data.did, data.event_type, payload)
.map_err(|e| DbError::Query(e.to_string()))?;
let seq_u64 = seq_to_u64(seq)?;
@@ -90,7 +89,10 @@ impl<S: StorageIO> EventOps<S> {
prev_data_cid: data.prev_data_cid.clone(),
ops: data.ops.clone(),
blobs: data.blobs.clone(),
blocks_cids: data.blocks_cids.clone(),
blocks: data
.blocks
.clone()
.map(tranquil_db_traits::EventBlocks::Inline),
handle: None,
active: None,
status: None,
@@ -113,7 +115,7 @@ impl<S: StorageIO> EventOps<S> {
prev_data_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
handle: handle.cloned(),
active: None,
status: None,
@@ -139,7 +141,7 @@ impl<S: StorageIO> EventOps<S> {
prev_data_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
handle: None,
active,
status: Some(status),
@@ -154,7 +156,15 @@ impl<S: StorageIO> EventOps<S> {
did: &Did,
commit_cid: &CidLink,
rev: Option<&str>,
commit_bytes: &[u8],
) -> Result<SequenceNumber, DbError> {
let inline = tranquil_db_traits::EventBlockInline {
cid_bytes: commit_cid
.to_cid()
.expect("CidLink invariant: validated at construction")
.to_bytes(),
data: commit_bytes.to_vec(),
};
let event = SequencedEvent {
seq: SequenceNumber::ZERO,
did: did.clone(),
@@ -165,7 +175,7 @@ impl<S: StorageIO> EventOps<S> {
prev_data_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: Some(tranquil_db_traits::EventBlocks::Inline(vec![inline])),
handle: None,
active: None,
status: None,
@@ -181,7 +191,23 @@ impl<S: StorageIO> EventOps<S> {
commit_cid: &CidLink,
mst_root_cid: &CidLink,
rev: &str,
commit_bytes: &[u8],
mst_root_bytes: &[u8],
) -> Result<SequenceNumber, DbError> {
let commit_block = tranquil_db_traits::EventBlockInline {
cid_bytes: commit_cid
.to_cid()
.expect("CidLink invariant: validated at construction")
.to_bytes(),
data: commit_bytes.to_vec(),
};
let mst_block = tranquil_db_traits::EventBlockInline {
cid_bytes: mst_root_cid
.to_cid()
.expect("CidLink invariant: validated at construction")
.to_bytes(),
data: mst_root_bytes.to_vec(),
};
let event = SequencedEvent {
seq: SequenceNumber::ZERO,
did: did.clone(),
@@ -192,7 +218,10 @@ impl<S: StorageIO> EventOps<S> {
prev_data_cid: Some(mst_root_cid.clone()),
ops: None,
blobs: None,
blocks_cids: None,
blocks: Some(tranquil_db_traits::EventBlocks::Inline(vec![
commit_block,
mst_block,
])),
handle: None,
active: None,
status: None,
@@ -208,7 +237,7 @@ impl<S: StorageIO> EventOps<S> {
limit: Option<i64>,
) -> Result<Vec<SequencedEvent>, DbError> {
let events = self.bridge.get_events_since_seq(since, limit)?;
self.apply_sidecars_and_filter(events)
self.filter_tombstoned(events)
}
pub fn get_events_in_seq_range(
@@ -217,7 +246,7 @@ impl<S: StorageIO> EventOps<S> {
end: SequenceNumber,
) -> Result<Vec<SequencedEvent>, DbError> {
let events = self.bridge.get_events_in_seq_range(start, end)?;
self.apply_sidecars_and_filter(events)
self.filter_tombstoned(events)
}
pub fn get_event_by_seq(&self, seq: SequenceNumber) -> Result<Option<SequencedEvent>, DbError> {
@@ -230,9 +259,7 @@ impl<S: StorageIO> EventOps<S> {
return Ok(None);
}
self.bridge
.get_event_by_seq(seq)
.map(|opt| opt.map(|e| self.merge_sidecar(e)))
self.bridge.get_event_by_seq(seq)
}
pub fn get_events_since_cursor(
@@ -241,7 +268,7 @@ impl<S: StorageIO> EventOps<S> {
limit: i64,
) -> Result<Vec<SequencedEvent>, DbError> {
let events = self.bridge.get_events_since_cursor(cursor, limit)?;
self.apply_sidecars_and_filter(events)
self.filter_tombstoned(events)
}
pub fn get_max_seq(&self) -> SequenceNumber {
@@ -255,56 +282,6 @@ impl<S: StorageIO> EventOps<S> {
self.bridge.get_min_seq_since(since)
}
pub fn get_events_since_rev(
&self,
did: &Did,
since_rev: &str,
) -> Result<Vec<EventBlocksCids>, DbError> {
let user_hash = UserHash::from_did(did.as_str());
let key = rev_to_seq_key(user_hash, since_rev);
let since_seq_u64 = match self.repo_data.get(key).map_err(fjall_to_db)? {
Some(bytes) => {
let arr: [u8; 8] = bytes
.as_ref()
.try_into()
.map_err(|_| DbError::Query("corrupt rev_to_seq value".to_owned()))?;
u64::from_be_bytes(arr)
}
None => return Ok(Vec::new()),
};
let start_seq = match since_seq_u64.checked_add(1) {
Some(s) => s,
None => return Ok(Vec::new()),
};
let user_seqs = self.scan_did_events(user_hash, start_seq)?;
user_seqs
.into_iter()
.try_fold(Vec::new(), |mut acc, seq_u64| {
if self.is_tombstoned(seq_u64)? {
return Ok(acc);
}
let seq_sn = SequenceNumber::from_raw(
i64::try_from(seq_u64)
.map_err(|_| DbError::Query("seq exceeds i64::MAX".to_owned()))?,
);
match self.bridge.get_event_by_seq(seq_sn)? {
Some(event) if event.rev.is_some() => {
let merged = self.merge_sidecar(event);
acc.push(EventBlocksCids {
blocks_cids: merged.blocks_cids,
commit_cid: merged.commit_cid,
});
Ok(acc)
}
_ => Ok(acc),
}
})
}
pub fn get_blob_cids_since_rev(
&self,
did: &Did,
@@ -359,23 +336,6 @@ impl<S: StorageIO> EventOps<S> {
})
}
pub fn update_seq_blocks_cids(
&self,
seq: SequenceNumber,
blocks_cids: &[String],
) -> Result<(), DbError> {
let seq_u64 = seq
.as_u64()
.ok_or_else(|| DbError::Query("invalid sequence number".to_owned()))?;
let key = seq_meta_key(seq_u64);
let value = SeqMetaValue {
blocks_cids: blocks_cids.to_vec(),
};
self.repo_data
.insert(key.as_slice(), value.serialize())
.map_err(fjall_to_db)
}
pub fn delete_sequences_except(
&self,
did: &Did,
@@ -413,7 +373,6 @@ impl<S: StorageIO> EventOps<S> {
seqs.iter().for_each(|&seq| {
batch.insert(&self.repo_data, seq_tombstone_key(seq).as_slice(), []);
batch.remove(&self.repo_data, did_events_key(user_hash, seq).as_slice());
batch.remove(&self.repo_data, seq_meta_key(seq).as_slice());
});
stale_rev_keys.iter().for_each(|key| {
batch.remove(&self.repo_data, key.as_slice());
@@ -707,29 +666,7 @@ impl<S: StorageIO> EventOps<S> {
}
}
fn merge_sidecar(&self, mut event: SequencedEvent) -> SequencedEvent {
let seq_u64 = match event.seq.as_u64() {
Some(v) => v,
None => return event,
};
let key = seq_meta_key(seq_u64);
match self.repo_data.get(key.as_slice()) {
Ok(Some(sidecar_bytes)) => {
if let Some(sidecar) = SeqMetaValue::deserialize(sidecar_bytes.as_ref()) {
event.blocks_cids = Some(sidecar.blocks_cids);
}
}
Ok(None) => {}
Err(e) => {
warn!(seq = seq_u64, error = %e, "failed to read seq sidecar, returning event without sidecar merge");
}
}
event
}
fn apply_sidecars_and_filter(
fn filter_tombstoned(
&self,
events: Vec<SequencedEvent>,
) -> Result<Vec<SequencedEvent>, DbError> {
@@ -739,7 +676,7 @@ impl<S: StorageIO> EventOps<S> {
None => false,
};
if !tombstoned {
acc.push(self.merge_sidecar(e));
acc.push(e);
}
Ok(acc)
})
@@ -832,7 +769,7 @@ mod tests {
prev_cid: None,
ops: Some(serde_json::json!([{"action": "create", "path": "app.bsky.feed.post/abc"}])),
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("3k2abcde".to_owned()),
};
@@ -850,7 +787,7 @@ mod tests {
#[test]
fn insert_and_query_identity_event() {
let h = setup();
let handle = Handle::new("alice.test").unwrap();
let handle = Handle::new("olaren.test").unwrap();
let seq = h
.event_ops
@@ -862,7 +799,7 @@ mod tests {
assert_eq!(event.event_type, RepoEventType::Identity);
assert_eq!(
event.handle.as_ref().map(|h| h.as_str()),
Some("alice.test")
Some("olaren.test")
);
}
@@ -888,7 +825,7 @@ mod tests {
let seq = h
.event_ops
.insert_sync_event(&test_did(), &cid, Some("rev1"))
.insert_sync_event(&test_did(), &cid, Some("rev1"), b"sync_commit_bytes")
.unwrap();
assert!(seq.as_i64() > 0);
@@ -906,7 +843,14 @@ mod tests {
let seq = h
.event_ops
.insert_genesis_commit_event(&test_did(), &commit_cid, &mst_cid, "genesis_rev")
.insert_genesis_commit_event(
&test_did(),
&commit_cid,
&mst_cid,
"genesis_rev",
b"genesis_commit_bytes",
b"genesis_mst_bytes",
)
.unwrap();
assert!(seq.as_i64() > 0);
@@ -1019,30 +963,6 @@ mod tests {
assert_eq!(h.event_ops.get_max_seq(), seq);
}
#[test]
fn update_seq_blocks_cids_merges_on_query() {
let h = setup();
let data = CommitEventData {
did: test_did(),
event_type: RepoEventType::Commit,
commit_cid: Some(test_cid_link()),
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
prev_data_cid: None,
rev: Some("rev1".to_owned()),
};
let seq = h.event_ops.insert_commit_event(&data).unwrap();
let blocks = vec!["bafyblock1".to_owned(), "bafyblock2".to_owned()];
h.event_ops.update_seq_blocks_cids(seq, &blocks).unwrap();
let event = h.event_ops.get_event_by_seq(seq).unwrap().unwrap();
assert_eq!(event.blocks_cids, Some(blocks));
}
#[test]
fn delete_sequences_except_tombstones_others() {
let h = setup();
@@ -1058,7 +978,7 @@ mod tests {
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev_a".to_owned()),
})
@@ -1073,7 +993,7 @@ mod tests {
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev_b".to_owned()),
})
@@ -1100,7 +1020,7 @@ mod tests {
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev_x".to_owned()),
})
@@ -1115,7 +1035,7 @@ mod tests {
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev_y".to_owned()),
})
@@ -1131,56 +1051,6 @@ mod tests {
assert_eq!(events[0].seq, seq2);
}
#[test]
fn get_events_since_rev() {
let h = setup();
let did = test_did();
let cid = test_cid_link();
h.event_ops
.insert_commit_event(&CommitEventData {
did: did.clone(),
event_type: RepoEventType::Commit,
commit_cid: Some(cid.clone()),
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: Some(vec!["block_a".to_owned()]),
prev_data_cid: None,
rev: Some("rev_1".to_owned()),
})
.unwrap();
h.event_ops
.insert_commit_event(&CommitEventData {
did: did.clone(),
event_type: RepoEventType::Commit,
commit_cid: Some(cid.clone()),
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: Some(vec!["block_b".to_owned()]),
prev_data_cid: None,
rev: Some("rev_2".to_owned()),
})
.unwrap();
let events = h.event_ops.get_events_since_rev(&did, "rev_1").unwrap();
assert_eq!(events.len(), 1);
assert_eq!(events[0].blocks_cids, Some(vec!["block_b".to_owned()]));
}
#[test]
fn get_events_since_rev_unknown_rev_returns_empty() {
let h = setup();
let events = h
.event_ops
.get_events_since_rev(&test_did(), "nonexistent_rev")
.unwrap();
assert!(events.is_empty());
}
#[test]
fn metastore_cursor_read_write() {
let h = setup();
@@ -1251,7 +1121,7 @@ mod tests {
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("r1".to_owned()),
})
@@ -1263,7 +1133,7 @@ mod tests {
.unwrap();
let s4 = h
.event_ops
.insert_sync_event(&did, &cid, Some("r2"))
.insert_sync_event(&did, &cid, Some("r2"), b"sync_commit_bytes")
.unwrap();
let events = h
@@ -1296,7 +1166,7 @@ mod tests {
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev_keep".to_owned()),
})
@@ -1311,7 +1181,7 @@ mod tests {
let keep_seq = h
.event_ops
.insert_sync_event(&did, &cid, Some("rev_sync"))
.insert_sync_event(&did, &cid, Some("rev_sync"), b"sync_commit_bytes")
.unwrap();
h.event_ops.delete_sequences_except(&did, keep_seq).unwrap();
@@ -1351,7 +1221,7 @@ mod tests {
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev_old".to_owned()),
})
@@ -1366,7 +1236,7 @@ mod tests {
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev_keep".to_owned()),
})
@@ -1417,7 +1287,7 @@ mod tests {
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev_a".to_owned()),
})
@@ -1432,7 +1302,7 @@ mod tests {
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev_b".to_owned()),
})
@@ -1460,154 +1330,6 @@ mod tests {
);
}
#[test]
fn delete_sequences_except_cleans_seq_meta_entries() {
let h = setup();
let did = test_did();
let cid = test_cid_link();
let seq1 = h
.event_ops
.insert_commit_event(&CommitEventData {
did: did.clone(),
event_type: RepoEventType::Commit,
commit_cid: Some(cid.clone()),
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
prev_data_cid: None,
rev: Some("rev_a".to_owned()),
})
.unwrap();
let seq2 = h
.event_ops
.insert_commit_event(&CommitEventData {
did: did.clone(),
event_type: RepoEventType::Commit,
commit_cid: Some(cid.clone()),
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
prev_data_cid: None,
rev: Some("rev_b".to_owned()),
})
.unwrap();
h.event_ops
.update_seq_blocks_cids(seq1, &["block1".to_owned()])
.unwrap();
h.event_ops
.update_seq_blocks_cids(seq2, &["block2".to_owned()])
.unwrap();
let stale_key = super::super::event_keys::seq_meta_key(seq1.as_u64().unwrap());
assert!(
h.event_ops
.repo_data
.get(stale_key.as_slice())
.unwrap()
.is_some()
);
h.event_ops.delete_sequences_except(&did, seq2).unwrap();
assert!(
h.event_ops
.repo_data
.get(stale_key.as_slice())
.unwrap()
.is_none()
);
let kept_key = super::super::event_keys::seq_meta_key(seq2.as_u64().unwrap());
assert!(
h.event_ops
.repo_data
.get(kept_key.as_slice())
.unwrap()
.is_some()
);
}
#[test]
fn get_events_since_rev_excludes_events_without_rev() {
let h = setup();
let did = test_did();
let cid = test_cid_link();
h.event_ops
.insert_commit_event(&CommitEventData {
did: did.clone(),
event_type: RepoEventType::Commit,
commit_cid: Some(cid.clone()),
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: Some(vec!["block_1".to_owned()]),
prev_data_cid: None,
rev: Some("rev_1".to_owned()),
})
.unwrap();
h.event_ops.insert_identity_event(&did, None).unwrap();
h.event_ops
.insert_account_event(&did, AccountStatus::Active)
.unwrap();
h.event_ops
.insert_commit_event(&CommitEventData {
did: did.clone(),
event_type: RepoEventType::Commit,
commit_cid: Some(cid.clone()),
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: Some(vec!["block_2".to_owned()]),
prev_data_cid: None,
rev: Some("rev_2".to_owned()),
})
.unwrap();
let events = h.event_ops.get_events_since_rev(&did, "rev_1").unwrap();
assert_eq!(events.len(), 1);
assert_eq!(events[0].blocks_cids, Some(vec!["block_2".to_owned()]));
}
#[test]
fn sync_event_with_rev_appears_in_get_events_since_rev() {
let h = setup();
let did = test_did();
let cid = test_cid_link();
h.event_ops
.insert_commit_event(&CommitEventData {
did: did.clone(),
event_type: RepoEventType::Commit,
commit_cid: Some(cid.clone()),
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
prev_data_cid: None,
rev: Some("rev_a".to_owned()),
})
.unwrap();
h.event_ops
.insert_sync_event(&did, &cid, Some("rev_b"))
.unwrap();
let events = h.event_ops.get_events_since_rev(&did, "rev_a").unwrap();
assert_eq!(events.len(), 1);
assert_eq!(events[0].commit_cid, Some(cid));
}
#[test]
fn recover_sidecar_indexes_no_gap() {
let h = setup();
@@ -1622,7 +1344,7 @@ mod tests {
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev_1".to_owned()),
})
@@ -1651,7 +1373,7 @@ mod tests {
prev_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
prev_data_cid: None,
rev: Some("rev_1".to_owned()),
})
@@ -1671,7 +1393,7 @@ mod tests {
prev_data_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
handle: None,
active: None,
status: None,
@@ -1689,7 +1411,7 @@ mod tests {
prev_data_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
handle: None,
active: None,
status: None,
@@ -1707,7 +1429,7 @@ mod tests {
prev_data_cid: None,
ops: None,
blobs: None,
blocks_cids: None,
blocks: None,
handle: None,
active: None,
status: None,
@@ -1715,18 +1437,26 @@ mod tests {
};
h.event_ops.bridge.insert_event(&crash_event_3).unwrap();
let user_hash = super::UserHash::from_did(did.as_str());
let rev2_key = super::super::event_keys::rev_to_seq_key(user_hash, "rev_2");
assert!(
h.event_ops
.get_events_since_rev(&did, "rev_2")
.repo_data
.get(rev2_key.as_slice())
.unwrap()
.is_empty()
.is_none()
);
let recovered = h.event_ops.recover_sidecar_indexes().unwrap();
assert_eq!(recovered, 3);
let events = h.event_ops.get_events_since_rev(&did, "rev_2").unwrap();
assert_eq!(events.len(), 1);
assert!(
h.event_ops
.repo_data
.get(rev2_key.as_slice())
.unwrap()
.is_some()
);
let cursor = h.event_ops.read_last_applied_cursor().unwrap();
assert!(cursor.is_some());
crates/tranquil-store/src/metastore/handler.rs
+83 -70
View File
@@ -7,13 +7,13 @@ use tokio::sync::oneshot;
use tranquil_db_traits::DbScope;
use tranquil_db_traits::{
AccountSearchResult, AccountStatus, AdminAccountInfo, ApplyCommitError, ApplyCommitInput,
ApplyCommitResult, Backlink, BrokenGenesisCommit, CommitEventData, CommsChannel, CommsType,
ApplyCommitResult, Backlink, CommitEventData, CommsChannel, CommsType,
CompletePasskeySetupInput, CreateAccountError, CreateDelegatedAccountInput,
CreatePasskeyAccountInput, CreatePasswordAccountInput, CreatePasswordAccountResult,
CreateSsoAccountInput, DbError, DelegationActionType, DeletionRequest,
DeletionRequestWithToken, DidWebOverrides, EventBlocksCids, ImportBlock, ImportRecord,
ImportRepoError, InviteCodeError, InviteCodeInfo, InviteCodeRow, InviteCodeSortOrder,
InviteCodeUse, MigrationReactivationError, MigrationReactivationInput, NotificationHistoryRow,
DeletionRequestWithToken, DidWebOverrides, ImportBlock, ImportRecord, ImportRepoError,
InviteCodeError, InviteCodeInfo, InviteCodeRow, InviteCodeSortOrder, InviteCodeUse,
MigrationReactivationError, MigrationReactivationInput, NotificationHistoryRow,
NotificationPrefs, OAuthTokenWithUser, PasswordResetResult, PlcTokenInfo, QueuedComms,
ReactivatedAccountInfo, RecoverPasskeyAccountInput, RecoverPasskeyAccountResult,
RefreshSessionResult, ReservedSigningKey, ReservedSigningKeyFull, ScheduledDeletionAccount,
@@ -64,6 +64,9 @@ fn metastore_to_db(e: MetastoreError) -> DbError {
} => DbError::Constraint(format!(
"user hash collision: {hash} maps to both {existing_uuid} and {new_uuid}"
)),
MetastoreError::UniqueViolation(constraint) => {
DbError::Constraint(format!("unique constraint violated: {constraint}"))
}
}
}
@@ -315,10 +318,6 @@ pub enum UserBlockRequest {
user_id: Uuid,
tx: Tx<i64>,
},
FindUnreferencedBlocks {
candidate_cids: Vec<Vec<u8>>,
tx: Tx<Vec<Vec<u8>>>,
},
}
impl UserBlockRequest {
@@ -328,7 +327,6 @@ impl UserBlockRequest {
| Self::DeleteUserBlocks { user_id, .. }
| Self::GetUserBlockCidsSinceRev { user_id, .. }
| Self::CountUserBlocks { user_id, .. } => uuid_to_routing(user_hashes, user_id),
Self::FindUnreferencedBlocks { .. } => Routing::Global,
}
}
}
@@ -352,6 +350,7 @@ pub enum EventRequest {
did: Did,
commit_cid: CidLink,
rev: Option<String>,
commit_bytes: Vec<u8>,
tx: Tx<SequenceNumber>,
},
InsertGenesisCommitEvent {
@@ -359,13 +358,10 @@ pub enum EventRequest {
commit_cid: CidLink,
mst_root_cid: CidLink,
rev: String,
commit_bytes: Vec<u8>,
mst_root_bytes: Vec<u8>,
tx: Tx<SequenceNumber>,
},
UpdateSeqBlocksCids {
seq: SequenceNumber,
blocks_cids: Vec<String>,
tx: Tx<()>,
},
DeleteSequencesExcept {
did: Did,
keep_seq: SequenceNumber,
@@ -397,11 +393,6 @@ pub enum EventRequest {
limit: i64,
tx: Tx<Vec<SequencedEvent>>,
},
GetEventsSinceRev {
did: Did,
since_rev: String,
tx: Tx<Vec<EventBlocksCids>>,
},
NotifyUpdate {
seq: SequenceNumber,
tx: Tx<()>,
@@ -418,12 +409,10 @@ impl EventRequest {
| Self::InsertAccountEvent { did, .. }
| Self::InsertSyncEvent { did, .. }
| Self::InsertGenesisCommitEvent { did, .. }
| Self::DeleteSequencesExcept { did, .. }
| Self::GetEventsSinceRev { did, .. } => {
| Self::DeleteSequencesExcept { did, .. } => {
Routing::Sharded(UserHash::from_did(did.as_str()).raw())
}
Self::UpdateSeqBlocksCids { .. }
| Self::GetMaxSeq { .. }
Self::GetMaxSeq { .. }
| Self::GetMinSeqSince { .. }
| Self::GetEventsSinceSeq { .. }
| Self::GetEventsInSeqRange { .. }
@@ -446,9 +435,6 @@ pub enum CommitRequest {
expected_root_cid: Option<CidLink>,
tx: oneshot::Sender<Result<(), ImportRepoError>>,
},
GetBrokenGenesisCommits {
tx: Tx<Vec<BrokenGenesisCommit>>,
},
GetUsersWithoutBlocks {
tx: Tx<Vec<UserWithoutBlocks>>,
},
@@ -472,8 +458,7 @@ impl CommitRequest {
| Self::InsertRecordBlobs {
repo_id: user_id, ..
} => uuid_to_routing(user_hashes, user_id),
Self::GetBrokenGenesisCommits { .. }
| Self::GetUsersWithoutBlocks { .. }
Self::GetUsersWithoutBlocks { .. }
| Self::GetUsersNeedingRecordBlobsBackfill { .. } => Routing::Global,
}
}
@@ -2912,17 +2897,10 @@ fn dispatch_user_block<S: StorageIO>(state: &HandlerState<S>, req: UserBlockRequ
.map_err(metastore_to_db);
let _ = tx.send(result);
}
UserBlockRequest::FindUnreferencedBlocks { candidate_cids, tx } => {
let result = state
.metastore
.user_block_ops()
.find_unreferenced(&candidate_cids);
let _ = tx.send(Ok(result));
}
}
}
fn dispatch_event<S: StorageIO>(state: &HandlerState<S>, req: EventRequest) {
fn dispatch_event<S: StorageIO + 'static>(state: &HandlerState<S>, req: EventRequest) {
match req {
EventRequest::InsertCommitEvent { data, tx } => {
let result = state.event_ops.insert_commit_event(&data);
@@ -2940,11 +2918,13 @@ fn dispatch_event<S: StorageIO>(state: &HandlerState<S>, req: EventRequest) {
did,
commit_cid,
rev,
commit_bytes,
tx,
} => {
let result = state
.event_ops
.insert_sync_event(&did, &commit_cid, rev.as_deref());
let result =
state
.event_ops
.insert_sync_event(&did, &commit_cid, rev.as_deref(), &commit_bytes);
let _ = tx.send(result);
}
EventRequest::InsertGenesisCommitEvent {
@@ -2952,20 +2932,18 @@ fn dispatch_event<S: StorageIO>(state: &HandlerState<S>, req: EventRequest) {
commit_cid,
mst_root_cid,
rev,
commit_bytes,
mst_root_bytes,
tx,
} => {
let result =
state
.event_ops
.insert_genesis_commit_event(&did, &commit_cid, &mst_root_cid, &rev);
let _ = tx.send(result);
}
EventRequest::UpdateSeqBlocksCids {
seq,
blocks_cids,
tx,
} => {
let result = state.event_ops.update_seq_blocks_cids(seq, &blocks_cids);
let result = state.event_ops.insert_genesis_commit_event(
&did,
&commit_cid,
&mst_root_cid,
&rev,
&commit_bytes,
&mst_root_bytes,
);
let _ = tx.send(result);
}
EventRequest::DeleteSequencesExcept { did, keep_seq, tx } => {
@@ -2998,16 +2976,13 @@ fn dispatch_event<S: StorageIO>(state: &HandlerState<S>, req: EventRequest) {
EventRequest::GetEventsSinceCursor { cursor, limit, tx } => {
let _ = tx.send(state.event_ops.get_events_since_cursor(cursor, limit));
}
EventRequest::GetEventsSinceRev { did, since_rev, tx } => {
let _ = tx.send(state.event_ops.get_events_since_rev(&did, &since_rev));
}
EventRequest::NotifyUpdate { seq, tx } => {
let _ = tx.send(state.event_ops.notify_update(seq));
}
}
}
fn dispatch_commit<S: StorageIO>(state: &HandlerState<S>, req: CommitRequest) {
fn dispatch_commit<S: StorageIO + 'static>(state: &HandlerState<S>, req: CommitRequest) {
match req {
CommitRequest::ApplyCommit { input, tx } => {
let _ = tx.send(state.commit_ops.apply_commit(*input));
@@ -3026,14 +3001,6 @@ fn dispatch_commit<S: StorageIO>(state: &HandlerState<S>, req: CommitRequest) {
expected_root_cid.as_ref(),
));
}
CommitRequest::GetBrokenGenesisCommits { tx } => {
let _ = tx.send(
state
.commit_ops
.get_broken_genesis_commits()
.map_err(metastore_to_db),
);
}
CommitRequest::GetUsersWithoutBlocks { tx } => {
let _ = tx.send(
state
@@ -3138,7 +3105,7 @@ fn dispatch_backlink<S: StorageIO>(state: &HandlerState<S>, req: BacklinkRequest
}
}
fn dispatch_blob<S: StorageIO>(state: &HandlerState<S>, req: BlobRequest) {
fn dispatch_blob<S: StorageIO + 'static>(state: &HandlerState<S>, req: BlobRequest) {
match req {
BlobRequest::InsertBlob {
cid,
@@ -4160,6 +4127,13 @@ fn dispatch_infra<S: StorageIO>(state: &HandlerState<S>, req: InfraRequest) {
value_json,
tx,
} => {
if name == "email_auth_factor" {
let enabled = value_json.as_bool().unwrap_or(false);
let _ = state
.metastore
.user_ops()
.set_email_2fa_enabled(user_id, enabled);
}
let result = state
.metastore
.infra_ops()
@@ -5005,7 +4979,7 @@ fn dispatch_oauth<S: StorageIO>(state: &HandlerState<S>, req: OAuthRequest) {
}
}
fn dispatch<S: StorageIO>(state: &HandlerState<S>, request: MetastoreRequest) {
fn dispatch<S: StorageIO + 'static>(state: &HandlerState<S>, request: MetastoreRequest) {
match request {
MetastoreRequest::Repo(r) => dispatch_repo(state, r),
MetastoreRequest::Record(r) => dispatch_record(state, r),
@@ -5023,7 +4997,7 @@ fn dispatch<S: StorageIO>(state: &HandlerState<S>, request: MetastoreRequest) {
}
}
fn dispatch_user<S: StorageIO>(state: &HandlerState<S>, req: UserRequest) {
fn dispatch_user<S: StorageIO + 'static>(state: &HandlerState<S>, req: UserRequest) {
let user = state.metastore.user_ops();
match req {
UserRequest::GetByDid { did, tx } => {
@@ -5768,7 +5742,17 @@ fn dispatch_user<S: StorageIO>(state: &HandlerState<S>, req: UserRequest) {
let _ = tx.send(result.map(|seq| seq.as_i64()));
}
UserRequest::CreatePasswordAccount { input, tx } => {
let _ = tx.send(user.create_password_account(&input));
let result = user.create_password_account(&input).and_then(|result| {
if let Some(key_id) = input.reserved_key_id {
state
.metastore
.infra_ops()
.mark_signing_key_used(key_id)
.map_err(|e| CreateAccountError::Database(e.to_string()))?;
}
Ok(result)
});
let _ = tx.send(result);
}
UserRequest::CreateDelegatedAccount { input, tx } => {
let result = user.create_delegated_account(&input).and_then(|account| {
@@ -5797,10 +5781,39 @@ fn dispatch_user<S: StorageIO>(state: &HandlerState<S>, req: UserRequest) {
let _ = tx.send(result);
}
UserRequest::CreatePasskeyAccount { input, tx } => {
let _ = tx.send(user.create_passkey_account(&input));
let result = user.create_passkey_account(&input).and_then(|result| {
if let Some(key_id) = input.reserved_key_id {
state
.metastore
.infra_ops()
.mark_signing_key_used(key_id)
.map_err(|e| CreateAccountError::Database(e.to_string()))?;
}
Ok(result)
});
let _ = tx.send(result);
}
UserRequest::CreateSsoAccount { input, tx } => {
let _ = tx.send(user.create_sso_account(&input));
let sso_ops = state.metastore.sso_ops();
let result = sso_ops
.consume_pending_registration(&input.pending_registration_token)
.map_err(|e| CreateAccountError::Database(e.to_string()))
.and_then(|consumed| match consumed {
Some(_) => user.create_sso_account(&input).and_then(|result| {
sso_ops
.create_external_identity(
&input.did,
input.sso_provider,
&input.sso_provider_user_id,
input.sso_provider_username.as_deref(),
input.sso_provider_email.as_deref(),
)
.map_err(|e| CreateAccountError::Database(e.to_string()))?;
Ok(result)
}),
None => Err(CreateAccountError::InvalidToken),
});
let _ = tx.send(result);
}
UserRequest::ReactivateMigrationAccount { input, tx } => {
let _ = tx.send(user.reactivate_migration_account(&input));
@@ -5884,7 +5897,7 @@ fn dispatch_user<S: StorageIO>(state: &HandlerState<S>, req: UserRequest) {
}
}
fn handler_loop<S: StorageIO>(
fn handler_loop<S: StorageIO + 'static>(
metastore: Metastore,
bridge: Arc<EventLogBridge<S>>,
blockstore: Option<TranquilBlockStore>,
crates/tranquil-store/src/metastore/infra_ops.rs
+55 -31
View File
@@ -2,6 +2,7 @@ use std::sync::Arc;
use chrono::{DateTime, Utc};
use fjall::{Database, Keyspace};
use smallvec::SmallVec;
use uuid::Uuid;
use super::MetastoreError;
@@ -35,6 +36,7 @@ pub struct InfraOps {
repo_data: Keyspace,
users: Keyspace,
user_hashes: Arc<UserHashMap>,
comms_seq: Arc<std::sync::atomic::AtomicU32>,
}
impl InfraOps {
@@ -44,6 +46,7 @@ impl InfraOps {
repo_data: Keyspace,
users: Keyspace,
user_hashes: Arc<UserHashMap>,
comms_seq: Arc<std::sync::atomic::AtomicU32>,
) -> Self {
Self {
db,
@@ -51,6 +54,7 @@ impl InfraOps {
repo_data,
users,
user_hashes,
comms_seq,
}
}
@@ -189,7 +193,10 @@ impl InfraOps {
status: status_to_u8(CommsStatus::Pending),
created_at_ms: now_ms,
};
let history_key = comms_history_key(uid, now_ms, id);
let seq = self
.comms_seq
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let history_key = comms_history_key(uid, now_ms, seq, id);
batch.insert(
&self.infra,
history_key.as_slice(),
@@ -245,19 +252,11 @@ impl InfraOps {
let mut batch = self.db.batch();
batch.insert(&self.infra, key.as_slice(), val.serialize());
let history_key = comms_history_key(
val.user_id.unwrap_or(Uuid::nil()),
val.created_at_ms,
val.id,
);
if let Some(mut history_val) = point_lookup(
&self.infra,
history_key.as_slice(),
NotificationHistoryValue::deserialize,
"corrupt notification history",
)? {
history_val.status = status_to_u8(CommsStatus::Sent);
batch.insert(&self.infra, history_key.as_slice(), history_val.serialize());
if let Some((hk, mut hv)) =
self.find_history_entry(val.user_id.unwrap_or(Uuid::nil()), val.id)?
{
hv.status = status_to_u8(CommsStatus::Sent);
batch.insert(&self.infra, hk.as_slice(), hv.serialize());
}
batch.commit().map_err(MetastoreError::Fjall)
@@ -280,24 +279,46 @@ impl InfraOps {
let mut batch = self.db.batch();
batch.insert(&self.infra, key.as_slice(), val.serialize());
let history_key = comms_history_key(
val.user_id.unwrap_or(Uuid::nil()),
val.created_at_ms,
val.id,
);
if let Some(mut history_val) = point_lookup(
&self.infra,
history_key.as_slice(),
NotificationHistoryValue::deserialize,
"corrupt notification history",
)? {
history_val.status = status_to_u8(CommsStatus::Failed);
batch.insert(&self.infra, history_key.as_slice(), history_val.serialize());
if let Some((hk, mut hv)) =
self.find_history_entry(val.user_id.unwrap_or(Uuid::nil()), val.id)?
{
hv.status = status_to_u8(CommsStatus::Failed);
batch.insert(&self.infra, hk.as_slice(), hv.serialize());
}
batch.commit().map_err(MetastoreError::Fjall)
}
#[allow(clippy::type_complexity)]
fn find_history_entry(
&self,
user_id: Uuid,
comms_id: Uuid,
) -> Result<Option<(SmallVec<[u8; 128]>, NotificationHistoryValue)>, MetastoreError> {
let prefix = comms_history_prefix(user_id);
self.infra
.prefix(prefix.as_slice())
.find_map(|guard| {
let (key_bytes, val_bytes) = match guard.into_inner() {
Ok(kv) => kv,
Err(e) => return Some(Err(MetastoreError::Fjall(e))),
};
let val = match NotificationHistoryValue::deserialize(&val_bytes) {
Some(v) => v,
None => {
return Some(Err(MetastoreError::CorruptData(
"corrupt notification history",
)));
}
};
match val.id == comms_id {
true => Some(Ok((SmallVec::from_slice(&key_bytes), val))),
false => None,
}
})
.transpose()
}
pub fn create_invite_code(
&self,
code: &str,
@@ -743,10 +764,13 @@ impl InfraOps {
SigningKeyValue::deserialize,
"corrupt signing key",
)?;
Ok(val.map(|v| ReservedSigningKey {
id: v.id,
private_key_bytes: v.private_key_bytes,
}))
let now_ms = Utc::now().timestamp_millis();
Ok(val
.filter(|v| !v.used && v.expires_at_ms > now_ms)
.map(|v| ReservedSigningKey {
id: v.id,
private_key_bytes: v.private_key_bytes,
}))
}
pub fn mark_signing_key_used(&self, key_id: Uuid) -> Result<(), MetastoreError> {
crates/tranquil-store/src/metastore/infra_schema.rs
+5 -2
View File
@@ -452,13 +452,16 @@ pub fn plc_token_prefix(user_id: uuid::Uuid) -> SmallVec<[u8; 128]> {
pub fn comms_history_key(
user_id: uuid::Uuid,
created_at_ms: i64,
seq: u32,
id: uuid::Uuid,
) -> SmallVec<[u8; 128]> {
let reversed_ts = i64::MAX.saturating_sub(created_at_ms);
let reversed_seq = u32::MAX.saturating_sub(seq);
KeyBuilder::new()
.tag(KeyTag::INFRA_COMMS_HISTORY)
.bytes(user_id.as_bytes())
.i64(reversed_ts)
.bytes(&reversed_seq.to_be_bytes())
.bytes(id.as_bytes())
.build()
}
@@ -655,8 +658,8 @@ mod tests {
let user_id = uuid::Uuid::new_v4();
let id_a = uuid::Uuid::new_v4();
let id_b = uuid::Uuid::new_v4();
let key_old = comms_history_key(user_id, 1000, id_a);
let key_new = comms_history_key(user_id, 2000, id_b);
let key_old = comms_history_key(user_id, 1000, 0, id_a);
let key_new = comms_history_key(user_id, 2000, 0, id_b);
assert!(key_new.as_slice() < key_old.as_slice());
}
crates/tranquil-store/src/metastore/keys.rs
-2
View File
@@ -47,7 +47,6 @@ impl KeyTag {
pub const USER_MAP_REVERSE: Self = Self(0x11);
pub const REV_TO_SEQ: Self = Self(0x20);
pub const SEQ_META: Self = Self(0x21);
pub const SEQ_TOMBSTONE: Self = Self(0x22);
pub const METASTORE_CURSOR: Self = Self(0x23);
pub const DID_EVENTS: Self = Self(0x24);
@@ -181,7 +180,6 @@ mod tests {
KeyTag::USER_MAP,
KeyTag::USER_MAP_REVERSE,
KeyTag::REV_TO_SEQ,
KeyTag::SEQ_META,
KeyTag::SEQ_TOMBSTONE,
KeyTag::METASTORE_CURSOR,
KeyTag::DID_EVENTS,
crates/tranquil-store/src/metastore/mod.rs
+18 -4
View File
@@ -30,7 +30,7 @@ pub mod user_hash;
pub mod user_ops;
pub mod users;
use std::path::Path;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use fjall::{Database, Keyspace};
@@ -50,9 +50,10 @@ impl Default for MetastoreConfig {
fn default() -> Self {
let total_ram = total_system_ram_bytes();
let twenty_percent = total_ram / 5;
let max_cache: u64 = 4 * 1024 * 1024 * 1024;
Self {
cache_size_bytes: twenty_percent,
cache_size_bytes: twenty_percent.min(max_cache),
}
}
}
@@ -97,6 +98,7 @@ pub enum MetastoreError {
existing_uuid: uuid::Uuid,
new_uuid: uuid::Uuid,
},
UniqueViolation(&'static str),
}
impl std::fmt::Display for MetastoreError {
@@ -120,6 +122,9 @@ impl std::fmt::Display for MetastoreError {
f,
"user hash collision: hash {hash} maps to both {existing_uuid} and {new_uuid}"
),
Self::UniqueViolation(constraint) => {
write!(f, "unique constraint violated: {constraint}")
}
}
}
}
@@ -158,6 +163,8 @@ pub struct Metastore {
partitions: [Keyspace; Partition::ALL.len()],
user_hashes: Arc<UserHashMap>,
counter_lock: Arc<parking_lot::Mutex<()>>,
comms_seq: Arc<std::sync::atomic::AtomicU32>,
path: PathBuf,
}
impl Metastore {
@@ -199,6 +206,8 @@ impl Metastore {
partitions,
user_hashes,
counter_lock: Arc::new(parking_lot::Mutex::new(())),
comms_seq: Arc::new(std::sync::atomic::AtomicU32::new(0)),
path: path.to_path_buf(),
})
}
@@ -229,6 +238,10 @@ impl Metastore {
}
}
pub fn path(&self) -> &Path {
&self.path
}
pub fn partition(&self, p: Partition) -> &Keyspace {
&self.partitions[p.index()]
}
@@ -266,7 +279,7 @@ impl Metastore {
)
}
pub fn event_ops<S: crate::io::StorageIO>(
pub fn event_ops<S: crate::io::StorageIO + 'static>(
&self,
bridge: Arc<crate::eventlog::EventLogBridge<S>>,
) -> event_ops::EventOps<S> {
@@ -325,6 +338,7 @@ impl Metastore {
self.partitions[Partition::RepoData.index()].clone(),
self.partitions[Partition::Users.index()].clone(),
Arc::clone(&self.user_hashes),
Arc::clone(&self.comms_seq),
)
}
@@ -347,7 +361,7 @@ impl Metastore {
)
}
pub fn commit_ops<S: crate::io::StorageIO>(
pub fn commit_ops<S: crate::io::StorageIO + 'static>(
&self,
bridge: Arc<crate::eventlog::EventLogBridge<S>>,
) -> commit_ops::CommitOps<S> {

Some files were not shown because too many files have changed in this diff Show More