From a5c68a350621f30439e5ae35217baef222cfa77e Mon Sep 17 00:00:00 2001 From: Lewis Date: Fri, 27 Mar 2026 11:14:46 -0700 Subject: [PATCH] feat(tranquil-store): metastore Lewis: May this revision serve well! --- ...a40e984082851741b22f63a170728e734763b.json | 22 + ...28e4905d69f9fe527b4d4b49c69deff52cea8.json | 22 + ...93417769a4a37affe0e3972e324ec27ec532f.json | 17 + ...b67ec7c534b3c821d55481b110e3e462ee366.json | 14 + ...c5655aae248a0a69dc260d1da9cf1d9ed2c49.json | 14 + Cargo.lock | 13 + Cargo.toml | 4 + crates/tranquil-api/src/admin/status.rs | 26 +- crates/tranquil-api/src/delegation.rs | 17 + crates/tranquil-api/src/identity/account.rs | 20 +- crates/tranquil-api/src/repo/import.rs | 10 + crates/tranquil-api/src/repo/record/batch.rs | 24 +- crates/tranquil-api/src/repo/record/delete.rs | 16 +- crates/tranquil-api/src/repo/record/read.rs | 2 +- crates/tranquil-api/src/repo/record/write.rs | 63 +- .../tranquil-api/src/server/account_status.rs | 16 + .../src/server/passkey_account.rs | 17 + crates/tranquil-api/src/server/password.rs | 4 +- crates/tranquil-api/src/server/session.rs | 6 +- crates/tranquil-config/src/lib.rs | 83 + crates/tranquil-db-traits/src/repo.rs | 18 + crates/tranquil-db-traits/src/sequence.rs | 4 + crates/tranquil-db/src/postgres/repo.rs | 83 + crates/tranquil-pds/Cargo.toml | 1 + crates/tranquil-pds/src/delegation/mod.rs | 9 +- crates/tranquil-pds/src/did.rs | 10 +- crates/tranquil-pds/src/repo_ops.rs | 15 + crates/tranquil-pds/src/scheduled.rs | 64 +- crates/tranquil-pds/src/state.rs | 76 +- crates/tranquil-pds/tests/helpers/mod.rs | 64 +- crates/tranquil-repo/src/lib.rs | 26 + crates/tranquil-server/src/main.rs | 2 + crates/tranquil-store/Cargo.toml | 5 + crates/tranquil-store/benches/metastore.rs | 812 ++++++++ .../tranquil-store/benches/metastore_scale.rs | 1083 ++++++++++ .../tranquil-store/benches/profile_reads.rs | 358 ++++ crates/tranquil-store/src/lib.rs | 1 + .../src/metastore/backlink_ops.rs | 638 ++++++ .../tranquil-store/src/metastore/backlinks.rs | 257 +++ .../tranquil-store/src/metastore/blob_ops.rs | 785 ++++++++ crates/tranquil-store/src/metastore/blobs.rs | 144 ++ crates/tranquil-store/src/metastore/client.rs | 1023 ++++++++++ .../src/metastore/commit_ops.rs | 1504 ++++++++++++++ .../tranquil-store/src/metastore/encoding.rs | 599 ++++++ .../src/metastore/event_keys.rs | 230 +++ .../tranquil-store/src/metastore/event_ops.rs | 1734 +++++++++++++++++ .../tranquil-store/src/metastore/handler.rs | 1729 ++++++++++++++++ crates/tranquil-store/src/metastore/keys.rs | 149 ++ crates/tranquil-store/src/metastore/mod.rs | 417 ++++ .../src/metastore/partitions.rs | 129 ++ .../src/metastore/record_ops.rs | 1506 ++++++++++++++ .../tranquil-store/src/metastore/records.rs | 162 ++ .../tranquil-store/src/metastore/recovery.rs | 307 +++ .../tranquil-store/src/metastore/repo_meta.rs | 226 +++ .../tranquil-store/src/metastore/repo_ops.rs | 1146 +++++++++++ crates/tranquil-store/src/metastore/scan.rs | 36 + .../src/metastore/user_block_ops.rs | 560 ++++++ .../src/metastore/user_blocks.rs | 112 ++ .../tranquil-store/src/metastore/user_hash.rs | 437 +++++ example.toml | 34 + 60 files changed, 16813 insertions(+), 92 deletions(-) create mode 100644 .sqlx/query-12f5864ebff622fc52643de7151a40e984082851741b22f63a170728e734763b.json create mode 100644 .sqlx/query-18fa821e4bd00ccf5d1d8395ba728e4905d69f9fe527b4d4b49c69deff52cea8.json create mode 100644 .sqlx/query-47149c0577ad9e9b9b089820b0c93417769a4a37affe0e3972e324ec27ec532f.json create mode 100644 .sqlx/query-8eecf8fef308716be88815eb59bb67ec7c534b3c821d55481b110e3e462ee366.json create mode 100644 .sqlx/query-cffe4c37fe949fbdc3d5cd83ccec5655aae248a0a69dc260d1da9cf1d9ed2c49.json create mode 100644 crates/tranquil-store/benches/metastore.rs create mode 100644 crates/tranquil-store/benches/metastore_scale.rs create mode 100644 crates/tranquil-store/benches/profile_reads.rs create mode 100644 crates/tranquil-store/src/metastore/backlink_ops.rs create mode 100644 crates/tranquil-store/src/metastore/backlinks.rs create mode 100644 crates/tranquil-store/src/metastore/blob_ops.rs create mode 100644 crates/tranquil-store/src/metastore/blobs.rs create mode 100644 crates/tranquil-store/src/metastore/client.rs create mode 100644 crates/tranquil-store/src/metastore/commit_ops.rs create mode 100644 crates/tranquil-store/src/metastore/encoding.rs create mode 100644 crates/tranquil-store/src/metastore/event_keys.rs create mode 100644 crates/tranquil-store/src/metastore/event_ops.rs create mode 100644 crates/tranquil-store/src/metastore/handler.rs create mode 100644 crates/tranquil-store/src/metastore/keys.rs create mode 100644 crates/tranquil-store/src/metastore/mod.rs create mode 100644 crates/tranquil-store/src/metastore/partitions.rs create mode 100644 crates/tranquil-store/src/metastore/record_ops.rs create mode 100644 crates/tranquil-store/src/metastore/records.rs create mode 100644 crates/tranquil-store/src/metastore/recovery.rs create mode 100644 crates/tranquil-store/src/metastore/repo_meta.rs create mode 100644 crates/tranquil-store/src/metastore/repo_ops.rs create mode 100644 crates/tranquil-store/src/metastore/scan.rs create mode 100644 crates/tranquil-store/src/metastore/user_block_ops.rs create mode 100644 crates/tranquil-store/src/metastore/user_blocks.rs create mode 100644 crates/tranquil-store/src/metastore/user_hash.rs diff --git a/.sqlx/query-12f5864ebff622fc52643de7151a40e984082851741b22f63a170728e734763b.json b/.sqlx/query-12f5864ebff622fc52643de7151a40e984082851741b22f63a170728e734763b.json new file mode 100644 index 0000000..4b705cb --- /dev/null +++ b/.sqlx/query-12f5864ebff622fc52643de7151a40e984082851741b22f63a170728e734763b.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT cid FROM blocks ORDER BY created_at ASC LIMIT $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "cid", + "type_info": "Bytea" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "12f5864ebff622fc52643de7151a40e984082851741b22f63a170728e734763b" +} diff --git a/.sqlx/query-18fa821e4bd00ccf5d1d8395ba728e4905d69f9fe527b4d4b49c69deff52cea8.json b/.sqlx/query-18fa821e4bd00ccf5d1d8395ba728e4905d69f9fe527b4d4b49c69deff52cea8.json new file mode 100644 index 0000000..a4d22d1 --- /dev/null +++ b/.sqlx/query-18fa821e4bd00ccf5d1d8395ba728e4905d69f9fe527b4d4b49c69deff52cea8.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT t.cid FROM UNNEST($1::bytea[]) AS t(cid)\n WHERE NOT EXISTS (\n SELECT 1 FROM user_blocks WHERE block_cid = t.cid\n )\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "cid", + "type_info": "Bytea" + } + ], + "parameters": { + "Left": [ + "ByteaArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "18fa821e4bd00ccf5d1d8395ba728e4905d69f9fe527b4d4b49c69deff52cea8" +} diff --git a/.sqlx/query-47149c0577ad9e9b9b089820b0c93417769a4a37affe0e3972e324ec27ec532f.json b/.sqlx/query-47149c0577ad9e9b9b089820b0c93417769a4a37affe0e3972e324ec27ec532f.json new file mode 100644 index 0000000..deea9b1 --- /dev/null +++ b/.sqlx/query-47149c0577ad9e9b9b089820b0c93417769a4a37affe0e3972e324ec27ec532f.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO backlinks (uri, path, link_to, repo_id)\n SELECT unnest($1::text[]), unnest($2::text[]), unnest($3::text[]), $4\n ON CONFLICT (uri, path) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TextArray", + "TextArray", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "47149c0577ad9e9b9b089820b0c93417769a4a37affe0e3972e324ec27ec532f" +} diff --git a/.sqlx/query-8eecf8fef308716be88815eb59bb67ec7c534b3c821d55481b110e3e462ee366.json b/.sqlx/query-8eecf8fef308716be88815eb59bb67ec7c534b3c821d55481b110e3e462ee366.json new file mode 100644 index 0000000..afdba00 --- /dev/null +++ b/.sqlx/query-8eecf8fef308716be88815eb59bb67ec7c534b3c821d55481b110e3e462ee366.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM blocks WHERE cid = ANY($1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "ByteaArray" + ] + }, + "nullable": [] + }, + "hash": "8eecf8fef308716be88815eb59bb67ec7c534b3c821d55481b110e3e462ee366" +} diff --git a/.sqlx/query-cffe4c37fe949fbdc3d5cd83ccec5655aae248a0a69dc260d1da9cf1d9ed2c49.json b/.sqlx/query-cffe4c37fe949fbdc3d5cd83ccec5655aae248a0a69dc260d1da9cf1d9ed2c49.json new file mode 100644 index 0000000..659eb09 --- /dev/null +++ b/.sqlx/query-cffe4c37fe949fbdc3d5cd83ccec5655aae248a0a69dc260d1da9cf1d9ed2c49.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM backlinks WHERE uri = ANY($1::text[])", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "cffe4c37fe949fbdc3d5cd83ccec5655aae248a0a69dc260d1da9cf1d9ed2c49" +} diff --git a/Cargo.lock b/Cargo.lock index 6d72047..5c2b887 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6389,6 +6389,12 @@ dependencies = [ "time", ] +[[package]] +name = "siphasher" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" + [[package]] name = "sketches-ddsketch" version = "0.3.1" @@ -7705,6 +7711,7 @@ dependencies = [ "tranquil-scopes", "tranquil-signal", "tranquil-storage", + "tranquil-store", "tranquil-sync", "tranquil-types", "urlencoding", @@ -7833,12 +7840,14 @@ dependencies = [ "bytes", "chrono", "cid", + "dashmap", "fjall", "flume 0.11.1", "futures", "jacquard-common", "jacquard-repo", "k256", + "lsm-tree", "memmap2", "multihash", "parking_lot", @@ -7849,14 +7858,18 @@ dependencies = [ "serde_ipld_dagcbor", "serde_json", "sha2", + "siphasher", + "smallvec", "sqlx", "tempfile", "thiserror 2.0.18", "tokio", "tracing", + "tranquil-db", "tranquil-db-traits", "tranquil-repo", "tranquil-types", + "uuid", "xxhash-rust", ] diff --git a/Cargo.toml b/Cargo.toml index 1ab2d29..ff64949 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -144,3 +144,7 @@ lto = "fat" strip = true codegen-units = 1 panic = "abort" + +[profile.bench] +debug = 1 +strip = false diff --git a/crates/tranquil-api/src/admin/status.rs b/crates/tranquil-api/src/admin/status.rs index 956f1d7..3c7414b 100644 --- a/crates/tranquil-api/src/admin/status.rs +++ b/crates/tranquil-api/src/admin/status.rs @@ -195,11 +195,22 @@ pub async fn update_subject_status( ApiError::InternalError(Some("Failed to update deactivation status".into())) })?; } + let takedown_update = input.takedown.as_ref().map(|t| t.applied); + let takedown_ref = input.takedown.as_ref().and_then(|t| t.r#ref.as_deref()); + let deactivated_update = input.deactivated.as_ref().map(|d| d.applied); + if (takedown_update.is_some() || deactivated_update.is_some()) + && let Err(e) = state + .repos + .repo + .update_repo_status(&did, takedown_update, takedown_ref, deactivated_update) + .await + { + warn!("failed to sync status to repo backend: {e:?}"); + } if let Some(takedown) = &input.takedown { - let status = if takedown.applied { - tranquil_db_traits::AccountStatus::Takendown - } else { - tranquil_db_traits::AccountStatus::Active + let status = match takedown.applied { + true => tranquil_db_traits::AccountStatus::Takendown, + false => tranquil_db_traits::AccountStatus::Active, }; if let Err(e) = tranquil_pds::repo_ops::sequence_account_event(&state, &did, status).await @@ -208,10 +219,9 @@ pub async fn update_subject_status( } } if let Some(deactivated) = &input.deactivated { - let status = if deactivated.applied { - tranquil_db_traits::AccountStatus::Deactivated - } else { - tranquil_db_traits::AccountStatus::Active + let status = match deactivated.applied { + true => tranquil_db_traits::AccountStatus::Deactivated, + false => tranquil_db_traits::AccountStatus::Active, }; if let Err(e) = tranquil_pds::repo_ops::sequence_account_event(&state, &did, status).await diff --git a/crates/tranquil-api/src/delegation.rs b/crates/tranquil-api/src/delegation.rs index b2a30dc..39bd140 100644 --- a/crates/tranquil-api/src/delegation.rs +++ b/crates/tranquil-api/src/delegation.rs @@ -18,6 +18,7 @@ use tranquil_pds::delegation::{ use tranquil_pds::rate_limit::{AccountCreationLimit, RateLimited}; use tranquil_pds::state::AppState; use tranquil_pds::types::{Did, Handle}; +use tranquil_types::CidLink; pub async fn list_controllers( State(state): State, @@ -419,6 +420,22 @@ pub async fn create_delegated_account( } }; + state + .repos + .repo + .create_repo( + user_id, + &did, + &handle, + &CidLink::from(&repo.commit_cid), + &repo.repo_rev, + ) + .await + .map_err(|e| { + error!("failed to register repo in backend: {e:?}"); + ApiError::InternalError(None) + })?; + if let Some(validated) = validated_invite_code && let Err(e) = state .repos diff --git a/crates/tranquil-api/src/identity/account.rs b/crates/tranquil-api/src/identity/account.rs index d67fcf2..289a9e6 100644 --- a/crates/tranquil-api/src/identity/account.rs +++ b/crates/tranquil-api/src/identity/account.rs @@ -15,6 +15,7 @@ use tranquil_pds::rate_limit::{AccountCreationLimit, RateLimited}; use tranquil_pds::state::AppState; use tranquil_pds::types::{Did, Handle, PlainPassword}; use tranquil_pds::validation::validate_password; +use tranquil_types::CidLink; #[derive(Deserialize)] #[serde(rename_all = "camelCase")] @@ -160,7 +161,7 @@ async fn try_reactivate_migration( .fetch_did_document(did) .await .ok() - .and_then(|f| Some((*f).clone())), + .map(|f| (*f).clone()), access_jwt: access_meta.token, refresh_jwt: refresh_meta.token, verification_required, @@ -547,6 +548,21 @@ pub async fn create_account( } }; let user_id = create_result.user_id; + if let Err(e) = state + .repos + .repo + .create_repo( + user_id, + &did_for_commit, + &handle_typed, + &CidLink::from(&repo.commit_cid), + &repo.repo_rev, + ) + .await + { + error!("failed to register repo in backend: {e:?}"); + return ApiError::InternalError(None).into_response(); + } if !is_migration && !is_did_web_byod { super::provision::sequence_new_account( &state, @@ -607,7 +623,7 @@ pub async fn create_account( Json(CreateAccountOutput { handle: handle.clone().into(), did: did_for_commit, - did_doc: did_doc.and_then(|f| Some((*f).clone())), + did_doc: did_doc.map(|f| (*f).clone()), access_jwt: session.access_jwt, refresh_jwt: session.refresh_jwt, verification_required: !is_migration, diff --git a/crates/tranquil-api/src/repo/import.rs b/crates/tranquil-api/src/repo/import.rs index 494cfff..b4d3945 100644 --- a/crates/tranquil-api/src/repo/import.rs +++ b/crates/tranquil-api/src/repo/import.rs @@ -195,6 +195,16 @@ pub async fn import_repo( } let max_blocks = tranquil_config::get().import.max_blocks as usize; let _write_lock = state.repo_write_locks.lock(user_id).await; + + state + .block_store + .put_many(blocks.clone()) + .await + .map_err(|e| { + error!("Failed to store import blocks: {:?}", e); + ApiError::InternalError(None) + })?; + match apply_import( &state.repos.repo, user_id, diff --git a/crates/tranquil-api/src/repo/record/batch.rs b/crates/tranquil-api/src/repo/record/batch.rs index cdd92c6..5e6e630 100644 --- a/crates/tranquil-api/src/repo/record/batch.rs +++ b/crates/tranquil-api/src/repo/record/batch.rs @@ -6,6 +6,7 @@ use jacquard_repo::{mst::Mst, storage::BlockStore}; use serde::{Deserialize, Serialize}; use serde_json::json; use tracing::info; +use tranquil_db_traits::Backlink; use tranquil_pds::api::error::{ApiError, DbResultExt}; use tranquil_pds::auth::{ Active, Auth, WriteOpKind, require_not_migrated, require_verified_or_delegated, @@ -13,7 +14,8 @@ use tranquil_pds::auth::{ }; use tranquil_pds::repo::TrackingBlockStore; use tranquil_pds::repo_ops::{ - FinalizeParams, RecordOp, begin_repo_write, extract_blob_cids, finalize_repo_write, + FinalizeParams, RecordOp, begin_repo_write, extract_backlinks, extract_blob_cids, + finalize_repo_write, }; use tranquil_pds::state::AppState; use tranquil_pds::types::{AtIdentifier, AtUri, Did, Nsid, Rkey}; @@ -27,6 +29,8 @@ struct WriteAccumulator { ops: Vec, modified_keys: Vec, all_blob_cids: Vec, + backlinks_to_add: Vec, + backlinks_to_remove: Vec, } async fn process_single_write( @@ -42,6 +46,8 @@ async fn process_single_write( mut ops, mut modified_keys, mut all_blob_cids, + mut backlinks_to_add, + mut backlinks_to_remove, } = acc; match write { @@ -79,6 +85,7 @@ async fn process_single_write( .await .map_err(|_| ApiError::InternalError(Some("Failed to add to MST".into())))?; let uri = AtUri::from_parts(did, collection, &rkey); + backlinks_to_add.extend(extract_backlinks(&uri, value)); results.push(WriteResult::CreateResult { uri, cid: record_cid.to_string(), @@ -95,6 +102,8 @@ async fn process_single_write( ops, modified_keys, all_blob_cids, + backlinks_to_add, + backlinks_to_remove, }) } WriteOp::Update { @@ -131,6 +140,8 @@ async fn process_single_write( .await .map_err(|_| ApiError::InternalError(Some("Failed to update MST".into())))?; let uri = AtUri::from_parts(did, collection, rkey); + backlinks_to_remove.push(uri.clone()); + backlinks_to_add.extend(extract_backlinks(&uri, value)); results.push(WriteResult::UpdateResult { uri, cid: record_cid.to_string(), @@ -148,6 +159,8 @@ async fn process_single_write( ops, modified_keys, all_blob_cids, + backlinks_to_add, + backlinks_to_remove, }) } WriteOp::Delete { collection, rkey } => { @@ -158,6 +171,7 @@ async fn process_single_write( .delete(&key) .await .map_err(|_| ApiError::InternalError(Some("Failed to delete from MST".into())))?; + backlinks_to_remove.push(AtUri::from_parts(did, collection, rkey)); results.push(WriteResult::DeleteResult {}); ops.push(RecordOp::Delete { collection: collection.clone(), @@ -170,6 +184,8 @@ async fn process_single_write( ops, modified_keys, all_blob_cids, + backlinks_to_add, + backlinks_to_remove, }) } } @@ -189,6 +205,8 @@ async fn process_writes( ops: Vec::new(), modified_keys: Vec::new(), all_blob_cids: Vec::new(), + backlinks_to_add: Vec::new(), + backlinks_to_remove: Vec::new(), }; stream::iter(writes.iter().map(Ok::<_, ApiError>)) .try_fold(initial_acc, |acc, write| async move { @@ -319,6 +337,8 @@ pub async fn apply_writes( ops, modified_keys, all_blob_cids, + backlinks_to_add, + backlinks_to_remove, } = process_writes( &input.writes, mst, @@ -373,6 +393,8 @@ pub async fn apply_writes( ops, modified_keys: &modified_keys, blob_cids: &all_blob_cids, + backlinks_to_add, + backlinks_to_remove, }, ) .await?; diff --git a/crates/tranquil-api/src/repo/record/delete.rs b/crates/tranquil-api/src/repo/record/delete.rs index cf2f6e6..a1b804b 100644 --- a/crates/tranquil-api/src/repo/record/delete.rs +++ b/crates/tranquil-api/src/repo/record/delete.rs @@ -76,6 +76,7 @@ pub async fn delete_record( }; let modified_keys = [key]; + let deleted_uri = AtUri::from_parts(&did, &input.collection, &input.rkey); let commit_result = finalize_repo_write( &state, @@ -95,20 +96,12 @@ pub async fn delete_record( ops: vec![op], modified_keys: &modified_keys, blob_cids: &[], + backlinks_to_add: vec![], + backlinks_to_remove: vec![deleted_uri], }, ) .await?; - let deleted_uri = AtUri::from_parts(&did, &input.collection, &input.rkey); - if let Err(e) = state - .repos - .backlink - .remove_backlinks_by_uri(&deleted_uri) - .await - { - error!("Failed to remove backlinks for {}: {}", deleted_uri, e); - } - Ok(Json(DeleteRecordOutput { commit: Some(CommitInfo { cid: commit_result.commit_cid.to_string(), @@ -216,6 +209,7 @@ pub async fn delete_record_internal( let written_cids_str: Vec = written_cids.iter().map(ToString::to_string).collect(); + let deleted_uri = AtUri::from_parts(did.as_str(), collection.as_str(), rkey.as_str()); commit_and_log( state, CommitParams { @@ -228,6 +222,8 @@ pub async fn delete_record_internal( blocks_cids: &written_cids_str, blobs: &[], obsolete_cids, + backlinks_to_add: vec![], + backlinks_to_remove: vec![deleted_uri], }, ) .await?; diff --git a/crates/tranquil-api/src/repo/record/read.rs b/crates/tranquil-api/src/repo/record/read.rs index 547da78..036b13a 100644 --- a/crates/tranquil-api/src/repo/record/read.rs +++ b/crates/tranquil-api/src/repo/record/read.rs @@ -178,7 +178,7 @@ pub async fn list_records( }; let records: Vec = parsed_rows .iter() - .zip(blocks.into_iter()) + .zip(blocks) .filter_map(|((_, rkey, cid_str), block_opt)| { block_opt.and_then(|block| { serde_ipld_dagcbor::from_slice::(&block) diff --git a/crates/tranquil-api/src/repo/record/write.rs b/crates/tranquil-api/src/repo/record/write.rs index 7325450..b039a76 100644 --- a/crates/tranquil-api/src/repo/record/write.rs +++ b/crates/tranquil-api/src/repo/record/write.rs @@ -147,20 +147,28 @@ pub async fn create_record( let prev_cid = match mst.get(&conflict_key).await { Ok(Some(cid)) => cid, - _ => continue, - }; - - mst = match mst.delete(&conflict_key).await { - Ok(m) => m, + Ok(None) => continue, Err(e) => { error!( - "Failed to delete conflict from MST {}: {:?}", + "Failed to read conflict record from MST {}: {:?}", conflict_uri, e ); - continue; + return Err(ApiError::InternalError(Some( + "Failed to read conflicting record from MST".into(), + ))); } }; + mst = mst.delete(&conflict_key).await.map_err(|e| { + error!( + "Failed to delete conflict from MST {}: {:?}", + conflict_uri, e + ); + ApiError::InternalError(Some( + "Failed to delete conflicting record from MST".into(), + )) + })?; + ops.push(RecordOp::Delete { collection: conflict_collection, rkey: conflict_rkey, @@ -208,6 +216,9 @@ pub async fn create_record( .collect(); let blob_cids = extract_blob_cids(&input.record); + let created_uri = AtUri::from_parts(&did, &input.collection, &rkey); + let backlinks_to_add = extract_backlinks(&created_uri, &input.record); + let commit_result = finalize_repo_write( &state, ctx, @@ -226,35 +237,12 @@ pub async fn create_record( ops, modified_keys: &modified_keys, blob_cids: &blob_cids, + backlinks_to_add, + backlinks_to_remove: conflict_uris_to_cleanup, }, ) .await?; - { - let backlink_repo = state.repos.backlink.clone(); - futures::future::join_all(conflict_uris_to_cleanup.iter().map(|uri| { - let backlink_repo = backlink_repo.clone(); - async move { - if let Err(e) = backlink_repo.remove_backlinks_by_uri(uri).await { - error!("Failed to remove backlinks for {}: {}", uri, e); - } - } - })) - .await; - } - - let created_uri = AtUri::from_parts(&did, &input.collection, &rkey); - let backlinks = extract_backlinks(&created_uri, &input.record); - if !backlinks.is_empty() - && let Err(e) = state - .repos - .backlink - .add_backlinks(user_id, &backlinks) - .await - { - error!("Failed to add backlinks for {}: {}", created_uri, e); - } - Ok(Json(CreateRecordOutput { uri: created_uri, cid: record_cid.to_string(), @@ -379,6 +367,13 @@ pub async fn put_record( let modified_keys = [key]; let blob_cids = extract_blob_cids(&input.record); + let record_uri = AtUri::from_parts(&did, &input.collection, &input.rkey); + let backlinks_to_add = extract_backlinks(&record_uri, &input.record); + let backlinks_to_remove = match is_update { + true => vec![record_uri.clone()], + false => vec![], + }; + let commit_result = finalize_repo_write( &state, ctx, @@ -397,12 +392,14 @@ pub async fn put_record( ops: vec![op], modified_keys: &modified_keys, blob_cids: &blob_cids, + backlinks_to_add, + backlinks_to_remove, }, ) .await?; Ok(Json(PutRecordOutput { - uri: AtUri::from_parts(&did, &input.collection, &input.rkey), + uri: record_uri, cid: record_cid.to_string(), commit: Some(CommitInfo { cid: commit_result.commit_cid.to_string(), diff --git a/crates/tranquil-api/src/server/account_status.rs b/crates/tranquil-api/src/server/account_status.rs index 730dec6..74e73a1 100644 --- a/crates/tranquil-api/src/server/account_status.rs +++ b/crates/tranquil-api/src/server/account_status.rs @@ -382,6 +382,14 @@ pub async fn activate_account( did ); } + if let Err(e) = state + .repos + .repo + .update_repo_status(&did, None, None, Some(false)) + .await + { + warn!("failed to sync activation to repo backend: {e:?}"); + } info!( "[MIGRATION] activateAccount: Sequencing account event (active=true) for did={}", did @@ -514,6 +522,14 @@ pub async fn deactivate_account( .delete(&tranquil_pds::cache_keys::handle_key(h)) .await; } + if let Err(e) = state + .repos + .repo + .update_repo_status(&did, None, None, Some(true)) + .await + { + warn!("failed to sync deactivation to repo backend: {e:?}"); + } if let Err(e) = tranquil_pds::repo_ops::sequence_account_event( &state, &did, diff --git a/crates/tranquil-api/src/server/passkey_account.rs b/crates/tranquil-api/src/server/passkey_account.rs index 9b3ff3e..bbb96fc 100644 --- a/crates/tranquil-api/src/server/passkey_account.rs +++ b/crates/tranquil-api/src/server/passkey_account.rs @@ -15,6 +15,7 @@ use tranquil_pds::rate_limit::{AccountCreationLimit, PasswordResetLimit, RateLim use tranquil_pds::state::AppState; use tranquil_pds::types::{Did, Handle, PlainPassword}; use tranquil_pds::validation::validate_password; +use tranquil_types::CidLink; fn generate_setup_token() -> String { let mut rng = rand::thread_rng(); @@ -366,6 +367,22 @@ pub async fn create_passkey_account( }; let user_id = create_result.user_id; + state + .repos + .repo + .create_repo( + user_id, + &did_typed, + &handle_typed, + &CidLink::from(&repo.commit_cid), + &repo.repo_rev, + ) + .await + .map_err(|e| { + error!("failed to register repo in backend: {e:?}"); + ApiError::InternalError(None) + })?; + if !is_byod_did_web { crate::identity::provision::sequence_new_account( &state, diff --git a/crates/tranquil-api/src/server/password.rs b/crates/tranquil-api/src/server/password.rs index c88b3d2..7d81064 100644 --- a/crates/tranquil-api/src/server/password.rs +++ b/crates/tranquil-api/src/server/password.rs @@ -153,7 +153,7 @@ pub async fn reset_password( } return Err(ApiError::ExpiredToken(None)); } - let password_hash = crate::common::hash_password_async(&password).await?; + let password_hash = crate::common::hash_password_async(password).await?; let result = match state .repos .user @@ -345,7 +345,7 @@ pub async fn set_password( )); } - let new_hash = crate::common::hash_password_async(&new_password).await?; + let new_hash = crate::common::hash_password_async(new_password).await?; state .repos diff --git a/crates/tranquil-api/src/server/session.rs b/crates/tranquil-api/src/server/session.rs index 3735793..3fb1075 100644 --- a/crates/tranquil-api/src/server/session.rs +++ b/crates/tranquil-api/src/server/session.rs @@ -313,7 +313,7 @@ pub async fn create_session( refresh_jwt: refresh_meta.token, handle, did: row.did, - did_doc: did_doc.ok().and_then(|f| Some((*f).clone())), + did_doc: did_doc.ok().map(|f| (*f).clone()), email: row.email, email_confirmed: Some(row.channel_verification.email), email_auth_factor: email_auth_factor_out, @@ -406,7 +406,7 @@ pub async fn get_session( status: account_state.status_for_session().map(String::from), migrated_to_pds, migrated_at, - did_doc: did_doc.ok().and_then(|f| Some((*f).clone())), + did_doc: did_doc.ok().map(|f| (*f).clone()), })) } Ok(None) => Err(ApiError::AuthenticationFailed(None)), @@ -604,7 +604,7 @@ pub async fn refresh_session( preferred_locale: u.preferred_locale, is_admin: u.is_admin, active: account_state.is_active(), - did_doc: did_doc.ok().and_then(|f| Some((*f).clone())), + did_doc: did_doc.ok().map(|f| (*f).clone()), status: account_state.status_for_session().map(String::from), })) } diff --git a/crates/tranquil-config/src/lib.rs b/crates/tranquil-config/src/lib.rs index b4c1656..4ae86d1 100644 --- a/crates/tranquil-config/src/lib.rs +++ b/crates/tranquil-config/src/lib.rs @@ -143,6 +143,9 @@ pub struct TranquilConfig { #[config(nested)] pub scheduled: ScheduledConfig, + + #[config(nested)] + pub tranquil_store: TranquilStoreConfig, } impl TranquilConfig { @@ -250,6 +253,23 @@ impl TranquilConfig { ); } + // -- repo backend ----------------------------------------------------- + if let Err(e) = self.storage.repo_backend.parse::() { + errors.push(e); + } + + // -- tranquil-store --------------------------------------------------- + if let Some(mb) = self.tranquil_store.memory_budget_mb + && mb == 0 + { + errors.push("tranquil_store.memory_budget_mb must be at least 1".to_string()); + } + if let Some(threads) = self.tranquil_store.handler_threads + && threads == 0 + { + errors.push("tranquil_store.handler_threads must be at least 1".to_string()); + } + // -- cache ------------------------------------------------------------ match self.cache.backend.as_str() { "valkey" => { @@ -561,6 +581,35 @@ impl SecretsConfig { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RepoBackend { + Postgres, + TranquilStore, +} + +impl std::str::FromStr for RepoBackend { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "postgres" => Ok(Self::Postgres), + "tranquil-store" => Ok(Self::TranquilStore), + other => Err(format!( + "unknown repo backend \"{other}\", expected \"postgres\" or \"tranquil-store\"" + )), + } + } +} + +impl fmt::Display for RepoBackend { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Postgres => f.write_str("postgres"), + Self::TranquilStore => f.write_str("tranquil-store"), + } + } +} + #[derive(Debug, Config)] pub struct StorageConfig { /// Storage backend: `filesystem` or `s3`. @@ -578,6 +627,17 @@ pub struct StorageConfig { /// Custom S3 endpoint URL (for MinIO, R2, etc.). #[config(env = "S3_ENDPOINT")] pub s3_endpoint: Option, + + #[config(env = "REPO_BACKEND", default = "postgres")] + pub repo_backend: String, +} + +impl StorageConfig { + pub fn repo_backend(&self) -> RepoBackend { + self.repo_backend + .parse() + .expect("repo_backend must be validated before use") + } } #[derive(Debug, Config)] @@ -996,6 +1056,29 @@ pub struct ScheduledConfig { /// Interval in seconds between scheduled delete checks. #[config(env = "SCHEDULED_DELETE_CHECK_INTERVAL_SECS", default = 3600)] pub delete_check_interval_secs: u64, + + /// Interval in seconds between block garbage collection cycles. + #[config(env = "BLOCK_GC_INTERVAL_SECS", default = 21600)] + pub block_gc_interval_secs: u64, +} + +#[derive(Debug, Config)] +pub struct TranquilStoreConfig { + /// Directory for tranquil-store data (metastore, eventlog). + #[config( + env = "TRANQUIL_STORE_DATA_DIR", + default = "/var/lib/tranquil-pds/store" + )] + pub data_dir: String, + + /// Fjall block cache size in megabytes. Defaults to 20% of system RAM + /// when unset. + #[config(env = "TRANQUIL_STORE_MEMORY_BUDGET_MB")] + pub memory_budget_mb: Option, + + /// Number of handler threads. Defaults to available_parallelism / 2. + #[config(env = "TRANQUIL_STORE_HANDLER_THREADS")] + pub handler_threads: Option, } /// Generate a TOML configuration template with all available options, diff --git a/crates/tranquil-db-traits/src/repo.rs b/crates/tranquil-db-traits/src/repo.rs index d5f6f29..706fdb7 100644 --- a/crates/tranquil-db-traits/src/repo.rs +++ b/crates/tranquil-db-traits/src/repo.rs @@ -5,6 +5,7 @@ use tranquil_types::{AtUri, CidLink, Did, Handle, Nsid, Rkey}; use uuid::Uuid; use crate::DbError; +use crate::backlink::Backlink; use crate::sequence::SequenceNumber; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, sqlx::Type)] @@ -279,6 +280,8 @@ pub struct ApplyCommitInput { pub obsolete_block_cids: Vec>, pub record_upserts: Vec, pub record_deletes: Vec, + pub backlinks_to_add: Vec, + pub backlinks_to_remove: Vec, pub commit_event: CommitEventData, } @@ -300,6 +303,8 @@ pub trait RepoRepository: Send + Sync { async fn create_repo( &self, user_id: Uuid, + did: &Did, + handle: &Handle, repo_root_cid: &CidLink, repo_rev: &str, ) -> Result<(), DbError>; @@ -313,6 +318,14 @@ pub trait RepoRepository: Send + Sync { async fn update_repo_rev(&self, user_id: Uuid, repo_rev: &str) -> Result<(), DbError>; + async fn update_repo_status( + &self, + did: &Did, + takedown: Option, + takedown_ref: Option<&str>, + deactivated: Option, + ) -> Result<(), DbError>; + async fn delete_repo(&self, user_id: Uuid) -> Result<(), DbError>; async fn get_repo_root_for_update(&self, user_id: Uuid) -> Result, DbError>; @@ -400,6 +413,11 @@ pub trait RepoRepository: Send + Sync { async fn count_user_blocks(&self, user_id: Uuid) -> Result; + async fn find_unreferenced_blocks( + &self, + candidate_cids: &[Vec], + ) -> Result>, DbError>; + async fn insert_commit_event(&self, data: &CommitEventData) -> Result; async fn insert_identity_event( diff --git a/crates/tranquil-db-traits/src/sequence.rs b/crates/tranquil-db-traits/src/sequence.rs index dcce24c..1f6f85a 100644 --- a/crates/tranquil-db-traits/src/sequence.rs +++ b/crates/tranquil-db-traits/src/sequence.rs @@ -23,6 +23,10 @@ impl SequenceNumber { pub fn is_valid(&self) -> bool { self.0 >= 0 } + + pub fn as_u64(&self) -> Option { + u64::try_from(self.0).ok() + } } impl fmt::Display for SequenceNumber { diff --git a/crates/tranquil-db/src/postgres/repo.rs b/crates/tranquil-db/src/postgres/repo.rs index f695da9..37ccec8 100644 --- a/crates/tranquil-db/src/postgres/repo.rs +++ b/crates/tranquil-db/src/postgres/repo.rs @@ -46,9 +46,21 @@ impl PostgresRepoRepository { #[async_trait] impl RepoRepository for PostgresRepoRepository { + async fn update_repo_status( + &self, + _did: &Did, + _takedown: Option, + _takedown_ref: Option<&str>, + _deactivated: Option, + ) -> Result<(), DbError> { + Ok(()) + } + async fn create_repo( &self, user_id: Uuid, + _did: &Did, + _handle: &Handle, repo_root_cid: &CidLink, repo_rev: &str, ) -> Result<(), DbError> { @@ -606,6 +618,30 @@ impl RepoRepository for PostgresRepoRepository { Ok(count) } + async fn find_unreferenced_blocks( + &self, + candidate_cids: &[Vec], + ) -> Result>, DbError> { + match candidate_cids.is_empty() { + true => Ok(Vec::new()), + false => { + let rows = sqlx::query!( + r#" + SELECT t.cid FROM UNNEST($1::bytea[]) AS t(cid) + WHERE NOT EXISTS ( + SELECT 1 FROM user_blocks WHERE block_cid = t.cid + ) + "#, + candidate_cids, + ) + .fetch_all(&self.pool) + .await + .map_err(map_sqlx_error)?; + Ok(rows.into_iter().filter_map(|r| r.cid).collect()) + } + } + } + async fn get_user_block_cids_since_rev( &self, user_id: Uuid, @@ -1367,6 +1403,53 @@ impl RepoRepository for PostgresRepoRepository { .map_err(|e| ApplyCommitError::Database(e.to_string()))?; } + if !input.backlinks_to_remove.is_empty() { + let remove_uris: Vec<&str> = input + .backlinks_to_remove + .iter() + .map(|u| u.as_str()) + .collect(); + sqlx::query!( + "DELETE FROM backlinks WHERE uri = ANY($1::text[])", + &remove_uris as &[&str], + ) + .execute(&mut *tx) + .await + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; + } + + if !input.backlinks_to_add.is_empty() { + let uris: Vec<&str> = input + .backlinks_to_add + .iter() + .map(|b| b.uri.as_str()) + .collect(); + let paths: Vec<&str> = input + .backlinks_to_add + .iter() + .map(|b| b.path.as_str()) + .collect(); + let link_tos: Vec<&str> = input + .backlinks_to_add + .iter() + .map(|b| b.link_to.as_str()) + .collect(); + sqlx::query!( + r#" + INSERT INTO backlinks (uri, path, link_to, repo_id) + SELECT unnest($1::text[]), unnest($2::text[]), unnest($3::text[]), $4 + ON CONFLICT (uri, path) DO NOTHING + "#, + &uris as &[&str], + &paths as &[&str], + &link_tos as &[&str], + input.user_id, + ) + .execute(&mut *tx) + .await + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; + } + let event = &input.commit_event; let seq: i64 = sqlx::query_scalar( r#" diff --git a/crates/tranquil-pds/Cargo.toml b/crates/tranquil-pds/Cargo.toml index 0be34a8..d4ec5d6 100644 --- a/crates/tranquil-pds/Cargo.toml +++ b/crates/tranquil-pds/Cargo.toml @@ -18,6 +18,7 @@ tranquil-comms = { workspace = true } tranquil-signal = { workspace = true } tranquil-db = { workspace = true } tranquil-db-traits = { workspace = true } +tranquil-store = { workspace = true } tranquil-lexicon = { workspace = true, features = ["resolve"] } aes-gcm = { workspace = true } diff --git a/crates/tranquil-pds/src/delegation/mod.rs b/crates/tranquil-pds/src/delegation/mod.rs index 2f0f662..2a29b5b 100644 --- a/crates/tranquil-pds/src/delegation/mod.rs +++ b/crates/tranquil-pds/src/delegation/mod.rs @@ -49,11 +49,10 @@ pub async fn resolve_identity( None } }); - let handle = did_doc.also_known_as.iter().find_map(|alias| { - alias - .strip_prefix("at://") - .and_then(|s| Some(s.to_string())) - }); + let handle = did_doc + .also_known_as + .iter() + .find_map(|alias| alias.strip_prefix("at://").map(|s| s.to_string())); Ok(ResolvedIdentity { did: did.clone(), diff --git a/crates/tranquil-pds/src/did.rs b/crates/tranquil-pds/src/did.rs index dd883f9..0fe5107 100644 --- a/crates/tranquil-pds/src/did.rs +++ b/crates/tranquil-pds/src/did.rs @@ -4,7 +4,7 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::sync::RwLock; -use tracing::{debug, error, info, warn}; +use tracing::{debug, info, warn}; #[derive(Debug, thiserror::Error)] pub enum DidResolutionError { @@ -55,10 +55,12 @@ pub struct ResolvedService { pub service_id: String, } +type TimedCache = RwLock, (Instant, Arc)>>; + pub struct DidResolver { - did_doc_cache: RwLock, (Instant, Arc)>>, - parsed_did_doc_cache: RwLock, (Instant, Arc)>>, - service_cache: RwLock, (Instant, Arc)>>, + did_doc_cache: TimedCache, + parsed_did_doc_cache: TimedCache, + service_cache: TimedCache, client: Client, cache_ttl: Duration, plc_directory_url: String, diff --git a/crates/tranquil-pds/src/repo_ops.rs b/crates/tranquil-pds/src/repo_ops.rs index c4b609c..e17d623 100644 --- a/crates/tranquil-pds/src/repo_ops.rs +++ b/crates/tranquil-pds/src/repo_ops.rs @@ -159,6 +159,8 @@ pub struct FinalizeParams<'a> { pub ops: Vec, pub modified_keys: &'a [String], pub blob_cids: &'a [String], + pub backlinks_to_add: Vec, + pub backlinks_to_remove: Vec, } pub async fn begin_repo_write( @@ -249,6 +251,8 @@ pub async fn finalize_repo_write( blocks_cids: &written_cids_str, blobs: params.blob_cids, obsolete_cids: vec![ctx.current_root_cid], + backlinks_to_add: params.backlinks_to_add, + backlinks_to_remove: params.backlinks_to_remove, }, ) .await?; @@ -331,6 +335,8 @@ pub struct CommitParams<'a> { pub blocks_cids: &'a [String], pub blobs: &'a [String], pub obsolete_cids: Vec, + pub backlinks_to_add: Vec, + pub backlinks_to_remove: Vec, } pub async fn commit_and_log( @@ -342,6 +348,8 @@ pub async fn commit_and_log( RepoEventType, }; + let backlinks_to_add = params.backlinks_to_add; + let backlinks_to_remove = params.backlinks_to_remove; let CommitParams { did, user_id, @@ -352,6 +360,7 @@ pub async fn commit_and_log( blocks_cids, blobs, obsolete_cids, + .. } = params; let key_row = state .repos @@ -485,6 +494,8 @@ pub async fn commit_and_log( obsolete_block_cids: obsolete_bytes, record_upserts, record_deletes, + backlinks_to_add, + backlinks_to_remove, commit_event, }; @@ -592,6 +603,8 @@ pub async fn create_record_internal( .collect(); let written_cids_str: Vec = written_cids.iter().map(|c| c.to_string()).collect(); let blob_cids = extract_blob_cids(record); + let record_uri = AtUri::from_parts(did.as_str(), collection.as_str(), rkey.as_str()); + let backlinks = extract_backlinks(&record_uri, record); let result = commit_and_log( state, CommitParams { @@ -604,6 +617,8 @@ pub async fn create_record_internal( blocks_cids: &written_cids_str, blobs: &blob_cids, obsolete_cids, + backlinks_to_add: backlinks, + backlinks_to_remove: vec![], }, ) .await?; diff --git a/crates/tranquil-pds/src/scheduled.rs b/crates/tranquil-pds/src/scheduled.rs index 1e92a1f..0626ea7 100644 --- a/crates/tranquil-pds/src/scheduled.rs +++ b/crates/tranquil-pds/src/scheduled.rs @@ -436,19 +436,26 @@ pub async fn start_scheduled_tasks( blob_repo: Arc, blob_store: Arc, sso_repo: Arc, + repo_repo: Arc, + block_store: PostgresBlockStore, shutdown: CancellationToken, ) { - let check_interval = - Duration::from_secs(tranquil_config::get().scheduled.delete_check_interval_secs); + let cfg = tranquil_config::get(); + let check_interval = Duration::from_secs(cfg.scheduled.delete_check_interval_secs); + let gc_interval = Duration::from_secs(cfg.scheduled.block_gc_interval_secs); info!( check_interval_secs = check_interval.as_secs(), + gc_interval_secs = gc_interval.as_secs(), "Starting scheduled tasks service" ); let mut ticker = interval(check_interval); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut gc_ticker = interval(gc_interval); + gc_ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { tokio::select! { _ = shutdown.cancelled() => { @@ -494,10 +501,63 @@ pub async fn start_scheduled_tasks( } } } + _ = gc_ticker.tick() => { + if let Err(e) = run_block_gc(repo_repo.as_ref(), &block_store).await { + error!("Block GC error: {e}"); + } + } } } } +const BLOCK_GC_BATCH_SIZE: i64 = 1000; + +async fn run_block_gc( + repo_repo: &dyn RepoRepository, + block_store: &PostgresBlockStore, +) -> anyhow::Result<()> { + let mut total_deleted: u64 = 0; + + loop { + let candidates = block_store + .get_oldest_block_cids(BLOCK_GC_BATCH_SIZE) + .await + .context("failed to fetch candidate blocks")?; + + match candidates.is_empty() { + true => break, + false => { + let batch_len = candidates.len(); + let unreferenced = repo_repo + .find_unreferenced_blocks(&candidates) + .await + .context("failed to check block references")?; + + let deleted = match unreferenced.is_empty() { + true => 0, + false => block_store + .delete_blocks(&unreferenced) + .await + .context("failed to delete unreferenced blocks")?, + }; + + total_deleted = total_deleted.saturating_add(deleted); + + match unreferenced.len() == batch_len { + true => continue, + false => break, + } + } + } + } + + match total_deleted > 0 { + true => info!(total_deleted, "Block GC cycle complete"), + false => debug!("Block GC cycle: no orphaned blocks found"), + } + Ok(()) +} + async fn process_scheduled_deletions( user_repo: &dyn UserRepository, blob_repo: &dyn BlobRepository, diff --git a/crates/tranquil-pds/src/state.rs b/crates/tranquil-pds/src/state.rs index 813f24b..fe05333 100644 --- a/crates/tranquil-pds/src/state.rs +++ b/crates/tranquil-pds/src/state.rs @@ -12,6 +12,7 @@ use crate::sso::{SsoConfig, SsoManager}; use crate::storage::{BlobStorage, create_blob_storage}; use sqlx::PgPool; use std::error::Error; +use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use tokio::sync::broadcast; @@ -262,7 +263,14 @@ impl AppState { AuthConfig::init(); init_rate_limit_override(); - let repos = Arc::new(PostgresRepositories::new(db.clone())); + let mut repos = PostgresRepositories::new(db.clone()); + + let cfg = tranquil_config::get(); + if cfg.storage.repo_backend() == tranquil_config::RepoBackend::TranquilStore { + wire_tranquil_store(&mut repos, &cfg.tranquil_store); + } + + let repos = Arc::new(repos); let block_store = PostgresBlockStore::new(db); let blob_store = create_blob_storage().await; @@ -377,3 +385,69 @@ impl AppState { true } } + +fn wire_tranquil_store( + repos: &mut PostgresRepositories, + store_cfg: &tranquil_config::TranquilStoreConfig, +) { + use tranquil_store::RealIO; + use tranquil_store::eventlog::{EventLog, EventLogBridge, EventLogConfig}; + use tranquil_store::metastore::client::MetastoreClient; + use tranquil_store::metastore::handler::HandlerPool; + use tranquil_store::metastore::partitions::Partition; + use tranquil_store::metastore::{Metastore, MetastoreConfig}; + + let base_dir = PathBuf::from(&store_cfg.data_dir); + let data_dir = match std::env::var("TRANQUIL_PDS_TEST_INFRA_READY").as_deref() { + Ok("1") => base_dir.join(format!("pid-{}", std::process::id())), + _ => base_dir, + }; + let metastore_dir = data_dir.join("metastore"); + let segments_dir = data_dir.join("eventlog").join("segments"); + + std::fs::create_dir_all(&metastore_dir).expect("failed to create metastore directory"); + std::fs::create_dir_all(&segments_dir).expect("failed to create eventlog segments directory"); + + let metastore_config = store_cfg + .memory_budget_mb + .map(|mb| MetastoreConfig { + cache_size_bytes: mb.saturating_mul(1024 * 1024), + }) + .unwrap_or_default(); + + let metastore = + Metastore::open(&metastore_dir, metastore_config).expect("failed to open metastore"); + + let event_log = EventLog::open( + EventLogConfig { + segments_dir, + ..EventLogConfig::default() + }, + RealIO::new(), + ) + .expect("failed to open eventlog"); + let event_log = Arc::new(event_log); + + let bridge = Arc::new(EventLogBridge::new(Arc::clone(&event_log))); + + let indexes = metastore.partition(Partition::Indexes).clone(); + let event_ops = metastore.event_ops(Arc::clone(&bridge)); + let recovered = event_ops + .recover_metastore_mutations(&indexes) + .expect("metastore crash recovery failed"); + if recovered > 0 { + tracing::info!(recovered, "replayed metastore mutations from eventlog"); + } + + let notifier = bridge.notifier(); + + let pool = HandlerPool::spawn::(metastore, bridge, None, store_cfg.handler_threads); + + let client = MetastoreClient::::new(Arc::new(pool)); + + tracing::info!(data_dir = %store_cfg.data_dir, "tranquil-store data directory"); + + repos.repo = Arc::new(client.clone()); + repos.backlink = Arc::new(client); + repos.event_notifier = Arc::new(notifier); +} diff --git a/crates/tranquil-pds/tests/helpers/mod.rs b/crates/tranquil-pds/tests/helpers/mod.rs index 522663a..ae13c75 100644 --- a/crates/tranquil-pds/tests/helpers/mod.rs +++ b/crates/tranquil-pds/tests/helpers/mod.rs @@ -302,30 +302,54 @@ pub async fn create_repost( #[allow(dead_code)] pub async fn set_account_takedown(did: &str, takedown_ref: Option<&str>) { - let pool = get_test_db_pool().await; - sqlx::query!( - "UPDATE users SET takedown_ref = $1 WHERE did = $2", - takedown_ref, - did - ) - .execute(pool) - .await - .expect("Failed to update takedown_ref"); + let client = client(); + let (admin_jwt, _) = create_admin_account_and_login(&client).await; + let applied = takedown_ref.is_some(); + let res = client + .post(format!( + "{}/xrpc/com.atproto.admin.updateSubjectStatus", + base_url().await, + )) + .bearer_auth(&admin_jwt) + .json(&json!({ + "subject": { + "$type": "com.atproto.admin.defs#repoRef", + "did": did + }, + "takedown": { + "applied": applied, + "ref": takedown_ref + } + })) + .send() + .await + .expect("Failed to send takedown request"); + assert_eq!(res.status(), StatusCode::OK, "Failed to set takedown"); } #[allow(dead_code)] pub async fn set_account_deactivated(did: &str, deactivated: bool) { - let pool = get_test_db_pool().await; - let deactivated_at: Option> = - if deactivated { Some(Utc::now()) } else { None }; - sqlx::query!( - "UPDATE users SET deactivated_at = $1 WHERE did = $2", - deactivated_at, - did - ) - .execute(pool) - .await - .expect("Failed to update deactivated_at"); + let client = client(); + let (admin_jwt, _) = create_admin_account_and_login(&client).await; + let res = client + .post(format!( + "{}/xrpc/com.atproto.admin.updateSubjectStatus", + base_url().await, + )) + .bearer_auth(&admin_jwt) + .json(&json!({ + "subject": { + "$type": "com.atproto.admin.defs#repoRef", + "did": did + }, + "deactivated": { + "applied": deactivated + } + })) + .send() + .await + .expect("Failed to send deactivation request"); + assert_eq!(res.status(), StatusCode::OK, "Failed to set deactivation"); } #[allow(dead_code)] diff --git a/crates/tranquil-repo/src/lib.rs b/crates/tranquil-repo/src/lib.rs index 6e44e10..98c5037 100644 --- a/crates/tranquil-repo/src/lib.rs +++ b/crates/tranquil-repo/src/lib.rs @@ -24,6 +24,32 @@ impl PostgresBlockStore { } } +impl PostgresBlockStore { + pub async fn get_oldest_block_cids(&self, limit: i64) -> Result>, RepoError> { + let rows = sqlx::query!( + "SELECT cid FROM blocks ORDER BY created_at ASC LIMIT $1", + limit, + ) + .fetch_all(&self.pool) + .await + .map_err(RepoError::storage)?; + Ok(rows.into_iter().map(|r| r.cid).collect()) + } + + pub async fn delete_blocks(&self, cids: &[Vec]) -> Result { + match cids.is_empty() { + true => Ok(0), + false => { + let result = sqlx::query!("DELETE FROM blocks WHERE cid = ANY($1)", cids,) + .execute(&self.pool) + .await + .map_err(RepoError::storage)?; + Ok(result.rows_affected()) + } + } + } +} + impl BlockStore for PostgresBlockStore { async fn get(&self, cid: &Cid) -> Result, RepoError> { let cid_bytes = cid.to_bytes(); diff --git a/crates/tranquil-server/src/main.rs b/crates/tranquil-server/src/main.rs index c7f2ce4..e69e5a5 100644 --- a/crates/tranquil-server/src/main.rs +++ b/crates/tranquil-server/src/main.rs @@ -253,6 +253,8 @@ async fn run() -> Result<(), Box> { state.repos.blob.clone(), state.blob_store.clone(), state.repos.sso.clone(), + state.repos.repo.clone(), + state.block_store.clone(), shutdown.clone(), )); diff --git a/crates/tranquil-store/Cargo.toml b/crates/tranquil-store/Cargo.toml index f8d18a1..8b50fd3 100644 --- a/crates/tranquil-store/Cargo.toml +++ b/crates/tranquil-store/Cargo.toml @@ -12,6 +12,7 @@ serde = { workspace = true } postcard = { version = "1", features = ["alloc"] } parking_lot = { workspace = true } fjall = "3" +lsm-tree = "3" flume = "0.11" tokio = { workspace = true, features = ["sync", "rt"] } bytes = "1" @@ -26,6 +27,10 @@ jacquard-repo = { workspace = true } cid = { workspace = true } multihash = { workspace = true } sha2 = { workspace = true } +siphasher = "1" +dashmap = "6" +smallvec = "1" +uuid = { workspace = true } [features] test-harness = [] diff --git a/crates/tranquil-store/benches/metastore.rs b/crates/tranquil-store/benches/metastore.rs new file mode 100644 index 0000000..e3c5090 --- /dev/null +++ b/crates/tranquil-store/benches/metastore.rs @@ -0,0 +1,812 @@ +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use futures::StreamExt; +use tokio::sync::oneshot; +use tranquil_db_traits::{ + ApplyCommitInput, CommitEventData, RecordUpsert, RepoEventType, RepoRepository, +}; +use tranquil_types::{CidLink, Did, Handle, Nsid, Rkey}; +use uuid::Uuid; + +use tranquil_store::RealIO; +use tranquil_store::eventlog::{EventLog, EventLogConfig}; +use tranquil_store::metastore::handler::{ + CommitRequest, HandlerPool, MetastoreRequest, RecordRequest, RepoRequest, +}; +use tranquil_store::metastore::{Metastore, MetastoreConfig}; + +struct LatencyStats { + p50: Duration, + p95: Duration, + p99: Duration, + max: Duration, + mean: Duration, +} + +fn compute_stats(durations: &mut [Duration]) -> Option { + match durations.is_empty() { + true => None, + false => { + durations.sort(); + let len = durations.len(); + let sum: Duration = durations.iter().sum(); + let divisor = u32::try_from(len).unwrap_or(u32::MAX); + let last = len - 1; + Some(LatencyStats { + p50: durations[last * 50 / 100], + p95: durations[last * 95 / 100], + p99: durations[last * 99 / 100], + max: durations[last], + mean: sum / divisor, + }) + } + } +} + +fn print_result(ops: usize, elapsed: Duration, stats: Option<&LatencyStats>) { + let throughput = ops as f64 / elapsed.as_secs_f64(); + match stats { + Some(s) => println!( + "{throughput:.0} ops/sec, {:.1}ms | p50={:?} p95={:?} p99={:?} max={:?} mean={:?}", + elapsed.as_secs_f64() * 1000.0, + s.p50, + s.p95, + s.p99, + s.max, + s.mean + ), + None => println!( + "{throughput:.0} ops/sec, {:.1}ms", + elapsed.as_secs_f64() * 1000.0, + ), + } +} + +async fn collect_latencies(handles: Vec>>) -> Vec { + futures::stream::iter(handles) + .fold(Vec::new(), |mut acc, h| async move { + acc.extend(h.await.unwrap()); + acc + }) + .await +} + +fn test_cid(seed: u8) -> CidLink { + let digest: [u8; 32] = std::array::from_fn(|i| seed.wrapping_add(i as u8)); + let mh = multihash::Multihash::<64>::wrap(0x12, &digest).unwrap(); + let c = cid::Cid::new_v1(0x71, mh); + CidLink::from_cid(&c) +} + +fn test_cid_bytes(seed: u8) -> Vec { + let digest: [u8; 32] = std::array::from_fn(|i| seed.wrapping_add(i as u8)); + let mh = multihash::Multihash::<64>::wrap(0x12, &digest).unwrap(); + cid::Cid::new_v1(0x71, mh).to_bytes() +} + +fn make_rev(n: u64) -> String { + format!("rev{n:010}") +} + +struct BenchHarness { + pool: Arc, + _metastore_dir: tempfile::TempDir, + _eventlog_dir: tempfile::TempDir, +} + +fn setup(thread_count: usize) -> BenchHarness { + let metastore_dir = tempfile::TempDir::new().unwrap(); + let eventlog_dir = tempfile::TempDir::new().unwrap(); + let segments_dir = eventlog_dir.path().join("segments"); + std::fs::create_dir_all(&segments_dir).unwrap(); + + let metastore = Metastore::open( + metastore_dir.path(), + MetastoreConfig { + cache_size_bytes: 256 * 1024 * 1024, + }, + ) + .unwrap(); + + let event_log = EventLog::open( + EventLogConfig { + segments_dir, + ..EventLogConfig::default() + }, + RealIO::new(), + ) + .unwrap(); + + let bridge = Arc::new(tranquil_store::eventlog::EventLogBridge::new(Arc::new( + event_log, + ))); + + let pool = Arc::new(HandlerPool::spawn::( + metastore, + bridge, + None, + Some(thread_count), + )); + + BenchHarness { + pool, + _metastore_dir: metastore_dir, + _eventlog_dir: eventlog_dir, + } +} + +async fn create_user(pool: &HandlerPool, user_id: Uuid, did: &Did, cid: &CidLink) { + let (tx, rx) = oneshot::channel(); + pool.send(MetastoreRequest::Repo(RepoRequest::CreateRepoFull { + user_id, + did: did.clone(), + handle: Handle::from(format!("bench.{}.invalid", user_id.as_simple())), + repo_root_cid: cid.clone(), + repo_rev: "rev0000000000".to_string(), + tx, + })) + .unwrap(); + rx.await.unwrap().unwrap(); +} + +fn make_commit_input( + user_id: Uuid, + did: &Did, + collection: &Nsid, + rev_n: u64, + cid_seed: u8, +) -> ApplyCommitInput { + ApplyCommitInput { + user_id, + did: did.clone(), + expected_root_cid: None, + new_root_cid: test_cid(cid_seed), + new_rev: make_rev(rev_n), + new_block_cids: vec![test_cid_bytes(cid_seed)], + obsolete_block_cids: vec![], + record_upserts: vec![RecordUpsert { + collection: collection.clone(), + rkey: Rkey::from(format!("r{rev_n:010}")), + cid: test_cid(cid_seed), + }], + record_deletes: vec![], + backlinks_to_add: vec![], + backlinks_to_remove: vec![], + commit_event: CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(test_cid(cid_seed)), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some(make_rev(rev_n)), + }, + } +} + +async fn seed_records( + pool: &HandlerPool, + user_id: Uuid, + did: &Did, + collection: &Nsid, + count: usize, +) { + let batches: Vec<(usize, usize, u64, u8)> = (0..) + .map(|i| { + let start = i * 50; + let end = (start + 50).min(count); + let rev_n = (i as u64) + 1; + let cid_seed = ((i + 10) & 0xFF) as u8; + (start, end, rev_n, cid_seed) + }) + .take_while(|(start, _, _, _)| *start < count) + .collect(); + + futures::stream::iter(batches) + .fold((), |(), (batch_start, batch_end, rev_n, cid_seed)| { + let did = did.clone(); + let collection = collection.clone(); + async move { + let record_upserts: Vec = (batch_start..batch_end) + .map(|i| RecordUpsert { + collection: collection.clone(), + rkey: Rkey::from(format!("rec{i:08}")), + cid: test_cid(((i * 7 + 3) & 0xFF) as u8), + }) + .collect(); + + let new_block_cids: Vec> = (batch_start..batch_end) + .map(|i| test_cid_bytes(((i * 11 + 5) & 0xFF) as u8)) + .collect(); + + let input = ApplyCommitInput { + user_id, + did: did.clone(), + expected_root_cid: None, + new_root_cid: test_cid(cid_seed), + new_rev: make_rev(rev_n), + new_block_cids, + obsolete_block_cids: vec![], + record_upserts, + record_deletes: vec![], + backlinks_to_add: vec![], + backlinks_to_remove: vec![], + commit_event: CommitEventData { + did, + event_type: RepoEventType::Commit, + commit_cid: Some(test_cid(cid_seed)), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some(make_rev(rev_n)), + }, + }; + + let (tx, rx) = oneshot::channel(); + pool.send(MetastoreRequest::Commit(Box::new( + CommitRequest::ApplyCommit { + input: Box::new(input), + tx, + }, + ))) + .unwrap(); + rx.await.unwrap().unwrap(); + } + }) + .await; +} + +async fn bench_apply_commit(pool: &Arc, concurrency: usize, ops_per_task: usize) { + let user_ids: Vec = (0..concurrency).map(|_| Uuid::new_v4()).collect(); + let dids: Vec = user_ids + .iter() + .map(|u| Did::from(format!("did:plc:bench{}", u.as_simple()))) + .collect(); + + futures::stream::iter(user_ids.iter().zip(dids.iter())) + .fold((), |(), (uid, did)| async { + create_user(pool, *uid, did, &test_cid(1)).await; + }) + .await; + + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let start = Instant::now(); + let handles: Vec<_> = (0..concurrency) + .map(|task_id| { + let pool = Arc::clone(pool); + let user_id = user_ids[task_id]; + let did = dids[task_id].clone(); + let collection = collection.clone(); + tokio::spawn(async move { + futures::stream::iter(0..ops_per_task) + .fold(Vec::with_capacity(ops_per_task), |mut latencies, i| { + let pool = &pool; + let did = &did; + let collection = &collection; + async move { + let rev_n = (task_id * ops_per_task + i + 1) as u64; + let cid_seed = ((task_id * 31 + i * 7) & 0xFF) as u8; + let input = + make_commit_input(user_id, did, collection, rev_n, cid_seed); + let t = Instant::now(); + let (tx, rx) = oneshot::channel(); + pool.send(MetastoreRequest::Commit(Box::new( + CommitRequest::ApplyCommit { + input: Box::new(input), + tx, + }, + ))) + .unwrap(); + rx.await.unwrap().unwrap(); + latencies.push(t.elapsed()); + latencies + } + }) + .await + }) + }) + .collect(); + + let mut all_latencies = collect_latencies(handles).await; + let elapsed = start.elapsed(); + let total_ops = concurrency * ops_per_task; + let stats = compute_stats(&mut all_latencies); + print_result(total_ops, elapsed, stats.as_ref()); +} + +async fn bench_get_record_cid(pool: &Arc, concurrency: usize, ops_per_task: usize) { + let user_id = Uuid::new_v4(); + let did = Did::from(format!("did:plc:getrecord{}", user_id.as_simple())); + let collection = Nsid::from("app.bsky.feed.post".to_string()); + create_user(pool, user_id, &did, &test_cid(1)).await; + seed_records(pool, user_id, &did, &collection, 1000).await; + + let total_records = 1000usize; + let start = Instant::now(); + let handles: Vec<_> = (0..concurrency) + .map(|task_id| { + let pool = Arc::clone(pool); + let collection = collection.clone(); + tokio::spawn(async move { + futures::stream::iter(0..ops_per_task) + .fold(Vec::with_capacity(ops_per_task), |mut latencies, i| { + let pool = &pool; + let collection = &collection; + async move { + let rec_idx = (task_id * 7 + i * 13) % total_records; + let rkey = Rkey::from(format!("rec{rec_idx:08}")); + let t = Instant::now(); + let (tx, rx) = oneshot::channel(); + pool.send(MetastoreRequest::Record(RecordRequest::GetRecordCid { + repo_id: user_id, + collection: collection.clone(), + rkey, + tx, + })) + .unwrap(); + let result = rx.await.unwrap().unwrap(); + assert!(result.is_some()); + latencies.push(t.elapsed()); + latencies + } + }) + .await + }) + }) + .collect(); + + let mut all_latencies = collect_latencies(handles).await; + let elapsed = start.elapsed(); + let total_ops = concurrency * ops_per_task; + let stats = compute_stats(&mut all_latencies); + print_result(total_ops, elapsed, stats.as_ref()); +} + +async fn bench_list_records(pool: &Arc, concurrency: usize, ops_per_task: usize) { + let user_id = Uuid::new_v4(); + let did = Did::from(format!("did:plc:listrecords{}", user_id.as_simple())); + let collection = Nsid::from("app.bsky.feed.post".to_string()); + create_user(pool, user_id, &did, &test_cid(1)).await; + seed_records(pool, user_id, &did, &collection, 1000).await; + + let start = Instant::now(); + let handles: Vec<_> = (0..concurrency) + .map(|_| { + let pool = Arc::clone(pool); + let collection = collection.clone(); + tokio::spawn(async move { + futures::stream::iter(0..ops_per_task) + .fold(Vec::with_capacity(ops_per_task), |mut latencies, _| { + let pool = &pool; + let collection = &collection; + async move { + let t = Instant::now(); + let (tx, rx) = oneshot::channel(); + pool.send(MetastoreRequest::Record(RecordRequest::ListRecords { + repo_id: user_id, + collection: collection.clone(), + cursor: None, + limit: 50, + reverse: false, + rkey_start: None, + rkey_end: None, + tx, + })) + .unwrap(); + let result = rx.await.unwrap().unwrap(); + assert!(!result.is_empty()); + latencies.push(t.elapsed()); + latencies + } + }) + .await + }) + }) + .collect(); + + let mut all_latencies = collect_latencies(handles).await; + let elapsed = start.elapsed(); + let total_ops = concurrency * ops_per_task; + let stats = compute_stats(&mut all_latencies); + print_result(total_ops, elapsed, stats.as_ref()); +} + +async fn setup_pg_bench_schema(pool: &sqlx::PgPool) { + sqlx::query( + "CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + handle TEXT NOT NULL UNIQUE, + email TEXT, + did TEXT NOT NULL UNIQUE, + password_hash TEXT NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + deactivated_at TIMESTAMPTZ, + invites_disabled BOOLEAN DEFAULT FALSE, + takedown_ref TEXT, + preferred_comms_channel TEXT NOT NULL DEFAULT 'email', + password_reset_code TEXT, + password_reset_code_expires_at TIMESTAMPTZ, + email_verified BOOLEAN NOT NULL DEFAULT FALSE, + two_factor_enabled BOOLEAN NOT NULL DEFAULT FALSE, + discord_id TEXT, + discord_verified BOOLEAN NOT NULL DEFAULT FALSE, + telegram_username TEXT, + telegram_verified BOOLEAN NOT NULL DEFAULT FALSE, + signal_number TEXT, + signal_verified BOOLEAN NOT NULL DEFAULT FALSE, + is_admin BOOLEAN NOT NULL DEFAULT FALSE, + migrated_to_pds TEXT, + migrated_at TIMESTAMPTZ, + preferred_locale TEXT, + signal_uuid TEXT + )", + ) + .execute(pool) + .await + .unwrap(); + + sqlx::query( + "CREATE TABLE IF NOT EXISTS repos ( + user_id UUID PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, + repo_root_cid TEXT NOT NULL, + repo_rev TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + )", + ) + .execute(pool) + .await + .unwrap(); + + sqlx::query( + "CREATE TABLE IF NOT EXISTS records ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + repo_id UUID NOT NULL REFERENCES repos(user_id) ON DELETE CASCADE, + collection TEXT NOT NULL, + rkey TEXT NOT NULL, + record_cid TEXT NOT NULL, + takedown_ref TEXT, + repo_rev TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE(repo_id, collection, rkey) + )", + ) + .execute(pool) + .await + .unwrap(); +} + +async fn pg_create_user(pg: &sqlx::PgPool, user_id: Uuid, did: &str) { + sqlx::query("INSERT INTO users (id, handle, did) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING") + .bind(user_id) + .bind(format!("bench.{}.invalid", Uuid::new_v4().as_simple())) + .bind(did) + .execute(pg) + .await + .unwrap(); +} + +async fn bench_pg_upsert_records( + repo: &dyn RepoRepository, + pg: &sqlx::PgPool, + concurrency: usize, + ops_per_task: usize, +) { + let user_ids: Vec = (0..concurrency).map(|_| Uuid::new_v4()).collect(); + let dids: Vec = user_ids + .iter() + .map(|u| format!("did:plc:pgbench{}", u.as_simple())) + .collect(); + + futures::stream::iter(user_ids.iter().zip(dids.iter())) + .fold((), |(), (uid, did)| async { + pg_create_user(pg, *uid, did).await; + let did_typed = Did::from(did.clone()); + let handle = Handle::from(format!("bench.{}.invalid", uid.as_simple())); + repo.create_repo(*uid, &did_typed, &handle, &test_cid(1), "rev0000000000") + .await + .unwrap(); + }) + .await; + + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let start = Instant::now(); + let handles: Vec<_> = (0..concurrency) + .map(|task_id| { + let user_id = user_ids[task_id]; + let collection = collection.clone(); + let pg = pg.clone(); + tokio::spawn(async move { + let repo = tranquil_db::postgres::PostgresRepoRepository::new(pg); + futures::stream::iter(0..ops_per_task) + .fold(Vec::with_capacity(ops_per_task), |mut latencies, i| { + let repo = &repo; + let collection = &collection; + async move { + let rev_n = (task_id * ops_per_task + i + 1) as u64; + let cid_seed = ((task_id * 31 + i * 7) & 0xFF) as u8; + let rkey = Rkey::from(format!("r{rev_n:010}")); + let t = Instant::now(); + repo.upsert_records( + user_id, + std::slice::from_ref(collection), + &[rkey], + &[test_cid(cid_seed)], + &make_rev(rev_n), + ) + .await + .unwrap(); + latencies.push(t.elapsed()); + latencies + } + }) + .await + }) + }) + .collect(); + + let mut all_latencies = collect_latencies(handles).await; + let elapsed = start.elapsed(); + let total_ops = concurrency * ops_per_task; + let stats = compute_stats(&mut all_latencies); + print_result(total_ops, elapsed, stats.as_ref()); +} + +async fn pg_seed_records( + repo: &dyn RepoRepository, + user_id: Uuid, + collection: &Nsid, + count: usize, +) { + let batches: Vec<(usize, usize)> = (0..) + .map(|i| { + let start = i * 50; + let end = (start + 50).min(count); + (start, end) + }) + .take_while(|(start, _)| *start < count) + .collect(); + + futures::stream::iter(batches) + .fold((), |(), (batch_start, batch_end)| { + let collection = collection.clone(); + async move { + let collections: Vec = (batch_start..batch_end) + .map(|_| collection.clone()) + .collect(); + let rkeys: Vec = (batch_start..batch_end) + .map(|i| Rkey::from(format!("rec{i:08}"))) + .collect(); + let cids: Vec = (batch_start..batch_end) + .map(|i| test_cid(((i * 7 + 3) & 0xFF) as u8)) + .collect(); + repo.upsert_records( + user_id, + &collections, + &rkeys, + &cids, + &make_rev(batch_start as u64), + ) + .await + .unwrap(); + } + }) + .await; +} + +async fn bench_pg_get_record_cid(pg: &sqlx::PgPool, concurrency: usize, ops_per_task: usize) { + let user_id = Uuid::new_v4(); + let did = format!("did:plc:pgget{}", user_id.as_simple()); + let collection = Nsid::from("app.bsky.feed.post".to_string()); + pg_create_user(pg, user_id, &did).await; + let repo = tranquil_db::postgres::PostgresRepoRepository::new(pg.clone()); + let did_typed = Did::from(did.clone()); + let handle = Handle::from(format!("bench.{}.invalid", user_id.as_simple())); + repo.create_repo(user_id, &did_typed, &handle, &test_cid(1), "rev0000000000") + .await + .unwrap(); + pg_seed_records(&repo, user_id, &collection, 1000).await; + + let total_records = 1000usize; + let start = Instant::now(); + let handles: Vec<_> = (0..concurrency) + .map(|task_id| { + let pg = pg.clone(); + let collection = collection.clone(); + tokio::spawn(async move { + let repo = tranquil_db::postgres::PostgresRepoRepository::new(pg); + futures::stream::iter(0..ops_per_task) + .fold(Vec::with_capacity(ops_per_task), |mut latencies, i| { + let repo = &repo; + let collection = &collection; + async move { + let rec_idx = (task_id * 7 + i * 13) % total_records; + let rkey = Rkey::from(format!("rec{rec_idx:08}")); + let t = Instant::now(); + let result = repo + .get_record_cid(user_id, collection, &rkey) + .await + .unwrap(); + assert!(result.is_some()); + latencies.push(t.elapsed()); + latencies + } + }) + .await + }) + }) + .collect(); + + let mut all_latencies = collect_latencies(handles).await; + let elapsed = start.elapsed(); + let total_ops = concurrency * ops_per_task; + let stats = compute_stats(&mut all_latencies); + print_result(total_ops, elapsed, stats.as_ref()); +} + +async fn bench_pg_list_records(pg: &sqlx::PgPool, concurrency: usize, ops_per_task: usize) { + let user_id = Uuid::new_v4(); + let did = format!("did:plc:pglist{}", user_id.as_simple()); + let collection = Nsid::from("app.bsky.feed.post".to_string()); + pg_create_user(pg, user_id, &did).await; + let repo = tranquil_db::postgres::PostgresRepoRepository::new(pg.clone()); + let did_typed = Did::from(did.clone()); + let handle = Handle::from(format!("bench.{}.invalid", user_id.as_simple())); + repo.create_repo(user_id, &did_typed, &handle, &test_cid(1), "rev0000000000") + .await + .unwrap(); + pg_seed_records(&repo, user_id, &collection, 1000).await; + + let start = Instant::now(); + let handles: Vec<_> = (0..concurrency) + .map(|_| { + let pg = pg.clone(); + let collection = collection.clone(); + tokio::spawn(async move { + let repo = tranquil_db::postgres::PostgresRepoRepository::new(pg); + futures::stream::iter(0..ops_per_task) + .fold(Vec::with_capacity(ops_per_task), |mut latencies, _| { + let repo = &repo; + let collection = &collection; + async move { + let t = Instant::now(); + let result = repo + .list_records(user_id, collection, None, 50, false, None, None) + .await + .unwrap(); + assert!(!result.is_empty()); + latencies.push(t.elapsed()); + latencies + } + }) + .await + }) + }) + .collect(); + + let mut all_latencies = collect_latencies(handles).await; + let elapsed = start.elapsed(); + let total_ops = concurrency * ops_per_task; + let stats = compute_stats(&mut all_latencies); + print_result(total_ops, elapsed, stats.as_ref()); +} + +#[tokio::main] +async fn main() { + let handler_threads = std::thread::available_parallelism() + .map(|n| n.get().max(2) / 2) + .unwrap_or(2); + println!("handler threads: {handler_threads}"); + + let concurrency_levels = [1, 10, 100, 1000]; + let ops_per_concurrency = |c: usize| match c { + 1 => 5000, + 10 => 1000, + 100 => 200, + 1000 => 50, + _ => 100, + }; + + futures::stream::iter(concurrency_levels.iter()) + .fold((), |(), &c| async move { + let ops = ops_per_concurrency(c); + + println!("-- apply_commit: {} ops, {} callers --", ops * c, c); + let h = setup(handler_threads); + bench_apply_commit(&h.pool, c, ops).await; + + println!("-- get_record_cid: {} ops, {} callers --", ops * c, c); + let h = setup(handler_threads); + bench_get_record_cid(&h.pool, c, ops).await; + + println!("-- list_records: {} ops, {} callers --", ops * c, c); + let h = setup(handler_threads); + bench_list_records(&h.pool, c, ops).await; + }) + .await; + + let pg_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => { + println!("set DATABASE_URL for postgres comparison"); + return; + } + }; + + let pg_concurrency_levels: &[usize] = &[1, 10, 50]; + + let setup_pg = |max_conns: u32| { + let url = pg_url.clone(); + async move { + sqlx::postgres::PgPoolOptions::new() + .max_connections(max_conns) + .connect(&url) + .await + .unwrap() + } + }; + + let pg = setup_pg(60).await; + setup_pg_bench_schema(&pg).await; + pg.close().await; + + futures::stream::iter(pg_concurrency_levels.iter()) + .fold((), |(), &c| { + let setup_pg = &setup_pg; + async move { + let ops = ops_per_concurrency(c); + let max_conns = u32::try_from(c).unwrap_or(50) + 10; + let pg = setup_pg(max_conns).await; + let repo = tranquil_db::postgres::PostgresRepoRepository::new(pg.clone()); + + println!( + "-- postgres upsert_records: {} ops, {} callers --", + ops * c, + c + ); + bench_pg_upsert_records(&repo, &pg, c, ops).await; + + println!( + "-- postgres get_record_cid: {} ops, {} callers --", + ops * c, + c + ); + bench_pg_get_record_cid(&pg, c, ops).await; + + println!( + "-- postgres list_records: {} ops, {} callers --", + ops * c, + c + ); + bench_pg_list_records(&pg, c, ops).await; + + sqlx::query("TRUNCATE records, repos, users CASCADE") + .execute(&pg) + .await + .unwrap(); + pg.close().await; + } + }) + .await; + + let pg = setup_pg(5).await; + sqlx::query("DROP TABLE IF EXISTS records CASCADE") + .execute(&pg) + .await + .unwrap(); + sqlx::query("DROP TABLE IF EXISTS repos CASCADE") + .execute(&pg) + .await + .unwrap(); + sqlx::query("DROP TABLE IF EXISTS users CASCADE") + .execute(&pg) + .await + .unwrap(); + pg.close().await; +} diff --git a/crates/tranquil-store/benches/metastore_scale.rs b/crates/tranquil-store/benches/metastore_scale.rs new file mode 100644 index 0000000..98810aa --- /dev/null +++ b/crates/tranquil-store/benches/metastore_scale.rs @@ -0,0 +1,1083 @@ +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use futures::StreamExt; +use tokio::sync::oneshot; +use tranquil_db_traits::{ + ApplyCommitInput, CommitEventData, RecordUpsert, RepoEventType, RepoRepository, +}; +use tranquil_types::{CidLink, Did, Handle, Nsid, Rkey}; +use uuid::Uuid; + +use tranquil_store::RealIO; +use tranquil_store::eventlog::{EventLog, EventLogConfig}; +use tranquil_store::metastore::handler::{ + CommitRequest, HandlerPool, MetastoreRequest, RecordRequest, RepoRequest, +}; +use tranquil_store::metastore::{Metastore, MetastoreConfig}; + +struct LatencyStats { + p50: Duration, + p95: Duration, + p99: Duration, + max: Duration, + mean: Duration, +} + +fn compute_stats(durations: &mut [Duration]) -> Option { + match durations.is_empty() { + true => None, + false => { + durations.sort(); + let len = durations.len(); + let sum: Duration = durations.iter().sum(); + let divisor = u32::try_from(len).unwrap_or(u32::MAX); + let last = len - 1; + Some(LatencyStats { + p50: durations[last * 50 / 100], + p95: durations[last * 95 / 100], + p99: durations[last * 99 / 100], + max: durations[last], + mean: sum / divisor, + }) + } + } +} + +fn print_result(label: &str, ops: usize, elapsed: Duration, stats: Option<&LatencyStats>) { + let throughput = ops as f64 / elapsed.as_secs_f64(); + match stats { + Some(s) => println!( + "{label}: {throughput:.0} ops/sec, {:.1}ms | p50={:?} p95={:?} p99={:?} max={:?} mean={:?}", + elapsed.as_secs_f64() * 1000.0, + s.p50, + s.p95, + s.p99, + s.max, + s.mean + ), + None => println!( + "{label}: {throughput:.0} ops/sec, {:.1}ms", + elapsed.as_secs_f64() * 1000.0, + ), + } +} + +async fn collect_latencies(handles: Vec>>) -> Vec { + futures::stream::iter(handles) + .fold(Vec::new(), |mut acc, h| async move { + acc.extend(h.await.unwrap()); + acc + }) + .await +} + +fn test_cid(seed: u8) -> CidLink { + let digest: [u8; 32] = std::array::from_fn(|i| seed.wrapping_add(i as u8)); + let mh = multihash::Multihash::<64>::wrap(0x12, &digest).unwrap(); + let c = cid::Cid::new_v1(0x71, mh); + CidLink::from_cid(&c) +} + +fn test_cid_bytes(seed: u8) -> Vec { + let digest: [u8; 32] = std::array::from_fn(|i| seed.wrapping_add(i as u8)); + let mh = multihash::Multihash::<64>::wrap(0x12, &digest).unwrap(); + cid::Cid::new_v1(0x71, mh).to_bytes() +} + +fn make_rev(n: u64) -> String { + format!("rev{n:010}") +} + +struct BenchHarness { + pool: Arc, + metastore: Metastore, + _metastore_dir: tempfile::TempDir, + _eventlog_dir: tempfile::TempDir, +} + +fn cache_size_for_users(user_count: usize) -> u64 { + let estimated_dataset_bytes = user_count as u64 * 10 * 300; + let cache_bytes = estimated_dataset_bytes + .saturating_mul(2) + .max(512 * 1024 * 1024); + let cap = 8u64 * 1024 * 1024 * 1024; + cache_bytes.min(cap) +} + +fn setup(thread_count: usize, user_count: usize) -> BenchHarness { + let cache_bytes = cache_size_for_users(user_count); + println!( + "cache size: {} MB (for {user_count} users)", + cache_bytes / (1024 * 1024) + ); + + let metastore_dir = tempfile::TempDir::new().unwrap(); + let eventlog_dir = tempfile::TempDir::new().unwrap(); + let segments_dir = eventlog_dir.path().join("segments"); + std::fs::create_dir_all(&segments_dir).unwrap(); + + let metastore = Metastore::open( + metastore_dir.path(), + MetastoreConfig { + cache_size_bytes: cache_bytes, + }, + ) + .unwrap(); + + let event_log = EventLog::open( + EventLogConfig { + segments_dir, + ..EventLogConfig::default() + }, + RealIO::new(), + ) + .unwrap(); + + let bridge = Arc::new(tranquil_store::eventlog::EventLogBridge::new(Arc::new( + event_log, + ))); + + let pool = Arc::new(HandlerPool::spawn::( + metastore.clone(), + bridge, + None, + Some(thread_count), + )); + + BenchHarness { + pool, + metastore, + _metastore_dir: metastore_dir, + _eventlog_dir: eventlog_dir, + } +} + +fn compact_and_report(metastore: &Metastore) { + println!("running major compaction..."); + let start = Instant::now(); + metastore.major_compact().unwrap(); + println!( + "major compaction complete in {:.1}s", + start.elapsed().as_secs_f64() + ); +} + +struct UserInfo { + user_id: Uuid, + did: Did, +} + +async fn seed_users(pool: &HandlerPool, count: usize) -> Vec { + let users: Vec = (0..count) + .map(|i| { + let user_id = Uuid::new_v4(); + UserInfo { + did: Did::from(format!("did:plc:scale{i:06x}{}", user_id.as_simple())), + user_id, + } + }) + .collect(); + + let batch_size = 500; + let batches: Vec<&[UserInfo]> = users.chunks(batch_size).collect(); + let total_batches = batches.len(); + + let start = Instant::now(); + futures::stream::iter(batches.into_iter().enumerate()) + .fold((), |(), (batch_idx, batch)| async move { + futures::stream::iter(batch.iter()) + .fold((), |(), user| async { + let (tx, rx) = oneshot::channel(); + pool.send(MetastoreRequest::Repo(RepoRequest::CreateRepoFull { + user_id: user.user_id, + did: user.did.clone(), + handle: Handle::from(format!( + "u{}.scale.invalid", + user.user_id.as_simple() + )), + repo_root_cid: test_cid(1), + repo_rev: "rev0000000000".to_string(), + tx, + })) + .unwrap(); + rx.await.unwrap().unwrap(); + }) + .await; + if (batch_idx + 1) % 20 == 0 || batch_idx + 1 == total_batches { + println!( + "seeded {}/{} users, {:.1}s", + (batch_idx + 1) * batch_size, + count, + start.elapsed().as_secs_f64() + ); + } + }) + .await; + println!( + "seeded {} users in {:.1}s, {:.0} users/sec", + count, + start.elapsed().as_secs_f64(), + count as f64 / start.elapsed().as_secs_f64() + ); + + users +} + +async fn seed_records_for_user(pool: &HandlerPool, user: &UserInfo, record_count: usize) { + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let record_upserts: Vec = (0..record_count) + .map(|i| RecordUpsert { + collection: collection.clone(), + rkey: Rkey::from(format!("rec{i:08}")), + cid: test_cid(((i * 7 + 3) & 0xFF) as u8), + }) + .collect(); + + let new_block_cids: Vec> = (0..record_count) + .map(|i| test_cid_bytes(((i * 11 + 5) & 0xFF) as u8)) + .collect(); + + let input = ApplyCommitInput { + user_id: user.user_id, + did: user.did.clone(), + expected_root_cid: None, + new_root_cid: test_cid(2), + new_rev: make_rev(1), + new_block_cids, + obsolete_block_cids: vec![], + record_upserts, + record_deletes: vec![], + backlinks_to_add: vec![], + backlinks_to_remove: vec![], + commit_event: CommitEventData { + did: user.did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(test_cid(2)), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some(make_rev(1)), + }, + }; + + let (tx, rx) = oneshot::channel(); + pool.send(MetastoreRequest::Commit(Box::new( + CommitRequest::ApplyCommit { + input: Box::new(input), + tx, + }, + ))) + .unwrap(); + rx.await.unwrap().unwrap(); +} + +async fn seed_all_records(pool: &Arc, users: &[UserInfo], records_per_user: usize) { + let start = Instant::now(); + let total = users.len(); + let chunk_size = 500; + let chunks: Vec<&[UserInfo]> = users.chunks(chunk_size).collect(); + let total_chunks = chunks.len(); + + futures::stream::iter(chunks.into_iter().enumerate()) + .fold((), |(), (chunk_idx, chunk)| { + let pool = Arc::clone(pool); + async move { + futures::stream::iter(chunk.iter()) + .fold((), |(), user| { + let pool = &pool; + async move { + seed_records_for_user(pool, user, records_per_user).await; + } + }) + .await; + if (chunk_idx + 1) % 20 == 0 || chunk_idx + 1 == total_chunks { + println!( + "seeded records for {}/{} users, {:.1}s", + (chunk_idx + 1) * chunk_size, + total, + start.elapsed().as_secs_f64() + ); + } + } + }) + .await; + println!( + "seeded {} records across {} users in {:.1}s", + total * records_per_user, + total, + start.elapsed().as_secs_f64() + ); +} + +async fn bench_single_user_commit(pool: &Arc, user: &UserInfo, ops: usize) { + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let start = Instant::now(); + let mut latencies: Vec = Vec::with_capacity(ops); + + futures::stream::iter(0..ops) + .fold(&mut latencies, |latencies, i| { + let pool = &pool; + let user = &user; + let collection = &collection; + async move { + let rev_n = (i + 100) as u64; + let cid_seed = ((i * 7 + 42) & 0xFF) as u8; + let input = ApplyCommitInput { + user_id: user.user_id, + did: user.did.clone(), + expected_root_cid: None, + new_root_cid: test_cid(cid_seed), + new_rev: make_rev(rev_n), + new_block_cids: vec![test_cid_bytes(cid_seed)], + obsolete_block_cids: vec![], + record_upserts: vec![RecordUpsert { + collection: collection.clone(), + rkey: Rkey::from(format!("new{rev_n:010}")), + cid: test_cid(cid_seed), + }], + record_deletes: vec![], + backlinks_to_add: vec![], + backlinks_to_remove: vec![], + commit_event: CommitEventData { + did: user.did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(test_cid(cid_seed)), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some(make_rev(rev_n)), + }, + }; + let t = Instant::now(); + let (tx, rx) = oneshot::channel(); + pool.send(MetastoreRequest::Commit(Box::new( + CommitRequest::ApplyCommit { + input: Box::new(input), + tx, + }, + ))) + .unwrap(); + rx.await.unwrap().unwrap(); + latencies.push(t.elapsed()); + latencies + } + }) + .await; + + let elapsed = start.elapsed(); + let stats = compute_stats(&mut latencies); + print_result("single-user commit", ops, elapsed, stats.as_ref()); +} + +async fn bench_multi_user_commit( + pool: &Arc, + users: &[UserInfo], + concurrency: usize, + ops_per_task: usize, +) { + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let active_users: Vec<&UserInfo> = users.iter().take(concurrency).collect(); + + let start = Instant::now(); + let handles: Vec<_> = active_users + .iter() + .enumerate() + .map(|(task_id, user)| { + let pool = Arc::clone(pool); + let user_id = user.user_id; + let did = user.did.clone(); + let collection = collection.clone(); + tokio::spawn(async move { + futures::stream::iter(0..ops_per_task) + .fold(Vec::with_capacity(ops_per_task), |mut latencies, i| { + let pool = &pool; + let did = &did; + let collection = &collection; + async move { + let rev_n = (task_id * ops_per_task + i + 200) as u64; + let cid_seed = ((task_id * 31 + i * 7) & 0xFF) as u8; + let input = ApplyCommitInput { + user_id, + did: did.clone(), + expected_root_cid: None, + new_root_cid: test_cid(cid_seed), + new_rev: make_rev(rev_n), + new_block_cids: vec![test_cid_bytes(cid_seed)], + obsolete_block_cids: vec![], + record_upserts: vec![RecordUpsert { + collection: collection.clone(), + rkey: Rkey::from(format!("mu{rev_n:010}")), + cid: test_cid(cid_seed), + }], + record_deletes: vec![], + backlinks_to_add: vec![], + backlinks_to_remove: vec![], + commit_event: CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(test_cid(cid_seed)), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some(make_rev(rev_n)), + }, + }; + let t = Instant::now(); + let (tx, rx) = oneshot::channel(); + pool.send(MetastoreRequest::Commit(Box::new( + CommitRequest::ApplyCommit { + input: Box::new(input), + tx, + }, + ))) + .unwrap(); + rx.await.unwrap().unwrap(); + latencies.push(t.elapsed()); + latencies + } + }) + .await + }) + }) + .collect(); + + let mut all_latencies = collect_latencies(handles).await; + let elapsed = start.elapsed(); + let total_ops = concurrency * ops_per_task; + let stats = compute_stats(&mut all_latencies); + print_result( + &format!("multi-user commit ({concurrency} writers)"), + total_ops, + elapsed, + stats.as_ref(), + ); +} + +async fn bench_list_records_at_scale( + pool: &Arc, + users: &[UserInfo], + concurrency: usize, + ops_per_task: usize, +) { + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let user_count = users.len(); + + let start = Instant::now(); + let handles: Vec<_> = (0..concurrency) + .map(|task_id| { + let pool = Arc::clone(pool); + let collection = collection.clone(); + let users: Vec<(Uuid, Did)> = + users.iter().map(|u| (u.user_id, u.did.clone())).collect(); + tokio::spawn(async move { + futures::stream::iter(0..ops_per_task) + .fold(Vec::with_capacity(ops_per_task), |mut latencies, i| { + let pool = &pool; + let collection = &collection; + let users = &users; + async move { + let idx = (task_id * 997 + i * 31) % user_count; + let (user_id, _) = &users[idx]; + let t = Instant::now(); + let (tx, rx) = oneshot::channel(); + pool.send(MetastoreRequest::Record(RecordRequest::ListRecords { + repo_id: *user_id, + collection: collection.clone(), + cursor: None, + limit: 50, + reverse: false, + rkey_start: None, + rkey_end: None, + tx, + })) + .unwrap(); + let _result = rx.await.unwrap().unwrap(); + latencies.push(t.elapsed()); + latencies + } + }) + .await + }) + }) + .collect(); + + let mut all_latencies = collect_latencies(handles).await; + let elapsed = start.elapsed(); + let total_ops = concurrency * ops_per_task; + let stats = compute_stats(&mut all_latencies); + print_result( + &format!("listRecords ({concurrency} readers)"), + total_ops, + elapsed, + stats.as_ref(), + ); +} + +async fn bench_get_record_at_scale( + pool: &Arc, + users: &[UserInfo], + concurrency: usize, + ops_per_task: usize, +) { + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let user_count = users.len(); + let records_per_user = 10usize; + + let start = Instant::now(); + let handles: Vec<_> = (0..concurrency) + .map(|task_id| { + let pool = Arc::clone(pool); + let collection = collection.clone(); + let users: Vec = users.iter().map(|u| u.user_id).collect(); + tokio::spawn(async move { + futures::stream::iter(0..ops_per_task) + .fold(Vec::with_capacity(ops_per_task), |mut latencies, i| { + let pool = &pool; + let collection = &collection; + let users = &users; + async move { + let user_idx = (task_id * 997 + i * 31) % user_count; + let rec_idx = (task_id * 13 + i * 7) % records_per_user; + let rkey = Rkey::from(format!("rec{rec_idx:08}")); + let t = Instant::now(); + let (tx, rx) = oneshot::channel(); + pool.send(MetastoreRequest::Record(RecordRequest::GetRecordCid { + repo_id: users[user_idx], + collection: collection.clone(), + rkey, + tx, + })) + .unwrap(); + let _result = rx.await.unwrap().unwrap(); + latencies.push(t.elapsed()); + latencies + } + }) + .await + }) + }) + .collect(); + + let mut all_latencies = collect_latencies(handles).await; + let elapsed = start.elapsed(); + let total_ops = concurrency * ops_per_task; + let stats = compute_stats(&mut all_latencies); + print_result( + &format!("getRecordCid ({concurrency} readers)"), + total_ops, + elapsed, + stats.as_ref(), + ); +} + +async fn pg_seed_users(pg: &sqlx::PgPool, count: usize) -> Vec { + let users: Vec = (0..count) + .map(|i| { + let user_id = Uuid::new_v4(); + UserInfo { + did: Did::from(format!("did:plc:pgscale{i:06x}{}", user_id.as_simple())), + user_id, + } + }) + .collect(); + + let start = Instant::now(); + let batch_size = 500; + let batches: Vec<&[UserInfo]> = users.chunks(batch_size).collect(); + let total_batches = batches.len(); + + futures::stream::iter(batches.into_iter().enumerate()) + .fold((), |(), (batch_idx, batch)| async move { + futures::stream::iter(batch.iter()) + .fold((), |(), user| async { + sqlx::query( + "INSERT INTO users (id, handle, did) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", + ) + .bind(user.user_id) + .bind(format!("u{}.pgscale.invalid", user.user_id.as_simple())) + .bind(user.did.as_str()) + .execute(pg) + .await + .unwrap(); + + let repo = tranquil_db::postgres::PostgresRepoRepository::new(pg.clone()); + let handle = Handle::from(format!( + "u{}.pgscale.invalid", + user.user_id.as_simple() + )); + repo.create_repo(user.user_id, &user.did, &handle, &test_cid(1), "rev0000000000") + .await + .unwrap(); + }) + .await; + if (batch_idx + 1) % 20 == 0 || batch_idx + 1 == total_batches { + println!( + "seeded {}/{} postgres users, {:.1}s", + (batch_idx + 1) * batch_size, + count, + start.elapsed().as_secs_f64() + ); + } + }) + .await; + println!( + "seeded {} postgres users in {:.1}s, {:.0} users/sec", + count, + start.elapsed().as_secs_f64(), + count as f64 / start.elapsed().as_secs_f64() + ); + + users +} + +async fn pg_seed_all_records(pg: &sqlx::PgPool, users: &[UserInfo], records_per_user: usize) { + let start = Instant::now(); + let total = users.len(); + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let chunk_size = 500; + let chunks: Vec<&[UserInfo]> = users.chunks(chunk_size).collect(); + let total_chunks = chunks.len(); + + futures::stream::iter(chunks.into_iter().enumerate()) + .fold((), |(), (chunk_idx, chunk)| { + let collection = collection.clone(); + let pg = pg.clone(); + async move { + let repo = tranquil_db::postgres::PostgresRepoRepository::new(pg); + futures::stream::iter(chunk.iter()) + .fold((), |(), user| { + let repo = &repo; + let collection = &collection; + async move { + let collections: Vec = + (0..records_per_user).map(|_| collection.clone()).collect(); + let rkeys: Vec = (0..records_per_user) + .map(|i| Rkey::from(format!("rec{i:08}"))) + .collect(); + let cids: Vec = (0..records_per_user) + .map(|i| test_cid(((i * 7 + 3) & 0xFF) as u8)) + .collect(); + repo.upsert_records( + user.user_id, + &collections, + &rkeys, + &cids, + "rev0000000001", + ) + .await + .unwrap(); + } + }) + .await; + if (chunk_idx + 1) % 20 == 0 || chunk_idx + 1 == total_chunks { + println!( + "seeded records for {}/{} postgres users, {:.1}s", + (chunk_idx + 1) * chunk_size, + total, + start.elapsed().as_secs_f64() + ); + } + } + }) + .await; + println!( + "seeded {} postgres records in {:.1}s", + total * records_per_user, + start.elapsed().as_secs_f64() + ); +} + +async fn bench_pg_single_user_commit(pg: &sqlx::PgPool, user: &UserInfo, ops: usize) { + let repo = tranquil_db::postgres::PostgresRepoRepository::new(pg.clone()); + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let start = Instant::now(); + let mut latencies: Vec = Vec::with_capacity(ops); + + futures::stream::iter(0..ops) + .fold(&mut latencies, |latencies, i| { + let repo = &repo; + let user = &user; + let collection = &collection; + async move { + let rev_n = (i + 100) as u64; + let cid_seed = ((i * 7 + 42) & 0xFF) as u8; + let rkey = Rkey::from(format!("new{rev_n:010}")); + let t = Instant::now(); + repo.upsert_records( + user.user_id, + std::slice::from_ref(collection), + &[rkey], + &[test_cid(cid_seed)], + &make_rev(rev_n), + ) + .await + .unwrap(); + latencies.push(t.elapsed()); + latencies + } + }) + .await; + + let elapsed = start.elapsed(); + let stats = compute_stats(&mut latencies); + print_result("single-user commit", ops, elapsed, stats.as_ref()); +} + +async fn bench_pg_multi_user_commit( + pg: &sqlx::PgPool, + users: &[UserInfo], + concurrency: usize, + ops_per_task: usize, +) { + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let active_users: Vec<&UserInfo> = users.iter().take(concurrency).collect(); + + let start = Instant::now(); + let handles: Vec<_> = active_users + .iter() + .enumerate() + .map(|(task_id, user)| { + let pg = pg.clone(); + let user_id = user.user_id; + let collection = collection.clone(); + tokio::spawn(async move { + let repo = tranquil_db::postgres::PostgresRepoRepository::new(pg); + futures::stream::iter(0..ops_per_task) + .fold(Vec::with_capacity(ops_per_task), |mut latencies, i| { + let repo = &repo; + let collection = &collection; + async move { + let rev_n = (task_id * ops_per_task + i + 200) as u64; + let cid_seed = ((task_id * 31 + i * 7) & 0xFF) as u8; + let rkey = Rkey::from(format!("mu{rev_n:010}")); + let t = Instant::now(); + repo.upsert_records( + user_id, + std::slice::from_ref(collection), + &[rkey], + &[test_cid(cid_seed)], + &make_rev(rev_n), + ) + .await + .unwrap(); + latencies.push(t.elapsed()); + latencies + } + }) + .await + }) + }) + .collect(); + + let mut all_latencies = collect_latencies(handles).await; + let elapsed = start.elapsed(); + let total_ops = concurrency * ops_per_task; + let stats = compute_stats(&mut all_latencies); + print_result( + &format!("multi-user commit ({concurrency} writers)"), + total_ops, + elapsed, + stats.as_ref(), + ); +} + +async fn bench_pg_list_records( + pg: &sqlx::PgPool, + users: &[UserInfo], + concurrency: usize, + ops_per_task: usize, +) { + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let user_count = users.len(); + + let start = Instant::now(); + let handles: Vec<_> = (0..concurrency) + .map(|task_id| { + let pg = pg.clone(); + let collection = collection.clone(); + let user_ids: Vec = users.iter().map(|u| u.user_id).collect(); + tokio::spawn(async move { + let repo = tranquil_db::postgres::PostgresRepoRepository::new(pg); + futures::stream::iter(0..ops_per_task) + .fold(Vec::with_capacity(ops_per_task), |mut latencies, i| { + let repo = &repo; + let collection = &collection; + let user_ids = &user_ids; + async move { + let idx = (task_id * 997 + i * 31) % user_count; + let t = Instant::now(); + let _result = repo + .list_records( + user_ids[idx], + collection, + None, + 50, + false, + None, + None, + ) + .await + .unwrap(); + latencies.push(t.elapsed()); + latencies + } + }) + .await + }) + }) + .collect(); + + let mut all_latencies = collect_latencies(handles).await; + let elapsed = start.elapsed(); + let total_ops = concurrency * ops_per_task; + let stats = compute_stats(&mut all_latencies); + print_result( + &format!("listRecords ({concurrency} readers)"), + total_ops, + elapsed, + stats.as_ref(), + ); +} + +async fn bench_pg_get_record( + pg: &sqlx::PgPool, + users: &[UserInfo], + concurrency: usize, + ops_per_task: usize, +) { + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let user_count = users.len(); + let records_per_user = 10usize; + + let start = Instant::now(); + let handles: Vec<_> = (0..concurrency) + .map(|task_id| { + let pg = pg.clone(); + let collection = collection.clone(); + let user_ids: Vec = users.iter().map(|u| u.user_id).collect(); + tokio::spawn(async move { + let repo = tranquil_db::postgres::PostgresRepoRepository::new(pg); + futures::stream::iter(0..ops_per_task) + .fold(Vec::with_capacity(ops_per_task), |mut latencies, i| { + let repo = &repo; + let collection = &collection; + let user_ids = &user_ids; + async move { + let user_idx = (task_id * 997 + i * 31) % user_count; + let rec_idx = (task_id * 13 + i * 7) % records_per_user; + let rkey = Rkey::from(format!("rec{rec_idx:08}")); + let t = Instant::now(); + let _result = repo + .get_record_cid(user_ids[user_idx], collection, &rkey) + .await + .unwrap(); + latencies.push(t.elapsed()); + latencies + } + }) + .await + }) + }) + .collect(); + + let mut all_latencies = collect_latencies(handles).await; + let elapsed = start.elapsed(); + let total_ops = concurrency * ops_per_task; + let stats = compute_stats(&mut all_latencies); + print_result( + &format!("getRecordCid ({concurrency} readers)"), + total_ops, + elapsed, + stats.as_ref(), + ); +} + +async fn setup_pg_bench_schema(pool: &sqlx::PgPool) { + sqlx::query( + "CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + handle TEXT NOT NULL UNIQUE, + email TEXT, + did TEXT NOT NULL UNIQUE, + password_hash TEXT NOT NULL DEFAULT '', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + deactivated_at TIMESTAMPTZ, + invites_disabled BOOLEAN DEFAULT FALSE, + takedown_ref TEXT, + preferred_comms_channel TEXT NOT NULL DEFAULT 'email', + password_reset_code TEXT, + password_reset_code_expires_at TIMESTAMPTZ, + email_verified BOOLEAN NOT NULL DEFAULT FALSE, + two_factor_enabled BOOLEAN NOT NULL DEFAULT FALSE, + discord_id TEXT, + discord_verified BOOLEAN NOT NULL DEFAULT FALSE, + telegram_username TEXT, + telegram_verified BOOLEAN NOT NULL DEFAULT FALSE, + signal_number TEXT, + signal_verified BOOLEAN NOT NULL DEFAULT FALSE, + is_admin BOOLEAN NOT NULL DEFAULT FALSE, + migrated_to_pds TEXT, + migrated_at TIMESTAMPTZ, + preferred_locale TEXT, + signal_uuid TEXT + )", + ) + .execute(pool) + .await + .unwrap(); + + sqlx::query( + "CREATE TABLE IF NOT EXISTS repos ( + user_id UUID PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, + repo_root_cid TEXT NOT NULL, + repo_rev TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + )", + ) + .execute(pool) + .await + .unwrap(); + + sqlx::query( + "CREATE TABLE IF NOT EXISTS records ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + repo_id UUID NOT NULL REFERENCES repos(user_id) ON DELETE CASCADE, + collection TEXT NOT NULL, + rkey TEXT NOT NULL, + record_cid TEXT NOT NULL, + takedown_ref TEXT, + repo_rev TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE(repo_id, collection, rkey) + )", + ) + .execute(pool) + .await + .unwrap(); + + sqlx::query( + "CREATE TABLE IF NOT EXISTS user_blocks ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + block_cid BYTEA NOT NULL, + repo_rev TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE(user_id, block_cid) + )", + ) + .execute(pool) + .await + .unwrap(); +} + +#[tokio::main] +async fn main() { + let handler_threads = std::thread::available_parallelism() + .map(|n| n.get().max(2) / 2) + .unwrap_or(2); + println!("-- metastore scale --"); + println!("handler threads: {handler_threads}"); + + let scale_levels: &[usize] = &[1_000, 10_000, 100_000, 300_000]; + let records_per_user = 10; + let bench_ops = 2000; + let concurrency = 100; + let ops_per_writer = 20; + + futures::stream::iter(scale_levels.iter()) + .fold((), |(), &user_count| async move { + println!("-- tranquil-store, {user_count} users, {records_per_user} records each --"); + + let h = setup(handler_threads, user_count); + let users = seed_users(&h.pool, user_count).await; + seed_all_records(&h.pool, &users, records_per_user).await; + + compact_and_report(&h.metastore); + + bench_single_user_commit(&h.pool, &users[0], bench_ops).await; + + let writers = concurrency.min(user_count); + bench_multi_user_commit(&h.pool, &users, writers, ops_per_writer).await; + + let warmup_ops = 50; + println!("warming read cache, {} ops per task...", warmup_ops); + bench_list_records_at_scale(&h.pool, &users, concurrency, warmup_ops).await; + bench_get_record_at_scale(&h.pool, &users, concurrency, warmup_ops).await; + + let read_ops = 500; + println!("measuring reads, {} ops per task...", read_ops); + bench_list_records_at_scale(&h.pool, &users, concurrency, read_ops).await; + bench_get_record_at_scale(&h.pool, &users, concurrency, read_ops).await; + }) + .await; + + let pg_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => { + println!("set DATABASE_URL for postgres comparison"); + return; + } + }; + + let pg_scale_levels: &[usize] = &[1_000, 10_000, 100_000]; + + let setup_pg = |max_conns: u32| { + let url = pg_url.clone(); + async move { + sqlx::postgres::PgPoolOptions::new() + .max_connections(max_conns) + .acquire_timeout(Duration::from_secs(30)) + .connect(&url) + .await + .unwrap() + } + }; + + let pg = setup_pg(60).await; + setup_pg_bench_schema(&pg).await; + pg.close().await; + + futures::stream::iter(pg_scale_levels.iter()) + .fold((), |(), &user_count| { + let setup_pg = &setup_pg; + async move { + println!("-- postgres, {user_count} users, {records_per_user} records each --"); + + let pg = setup_pg(120).await; + + let users = pg_seed_users(&pg, user_count).await; + pg_seed_all_records(&pg, &users, records_per_user).await; + + bench_pg_single_user_commit(&pg, &users[0], bench_ops).await; + + let writers = concurrency.min(user_count); + bench_pg_multi_user_commit(&pg, &users, writers, ops_per_writer).await; + + bench_pg_list_records(&pg, &users, concurrency, ops_per_writer).await; + bench_pg_get_record(&pg, &users, concurrency, ops_per_writer).await; + + sqlx::query("TRUNCATE user_blocks, records, repos, users CASCADE") + .execute(&pg) + .await + .unwrap(); + pg.close().await; + } + }) + .await; + + let pg = setup_pg(5).await; + sqlx::query("DROP TABLE IF EXISTS user_blocks, records, repos, users CASCADE") + .execute(&pg) + .await + .unwrap(); + pg.close().await; +} diff --git a/crates/tranquil-store/benches/profile_reads.rs b/crates/tranquil-store/benches/profile_reads.rs new file mode 100644 index 0000000..349d7d6 --- /dev/null +++ b/crates/tranquil-store/benches/profile_reads.rs @@ -0,0 +1,358 @@ +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use futures::StreamExt; +use tokio::sync::oneshot; +use tranquil_db_traits::{ApplyCommitInput, CommitEventData, RecordUpsert, RepoEventType}; +use tranquil_types::{CidLink, Did, Handle, Nsid, Rkey}; +use uuid::Uuid; + +use tranquil_store::RealIO; +use tranquil_store::eventlog::{EventLog, EventLogConfig}; +use tranquil_store::metastore::handler::{ + CommitRequest, HandlerPool, MetastoreRequest, RecordRequest, RepoRequest, +}; +use tranquil_store::metastore::{Metastore, MetastoreConfig}; + +fn test_cid(seed: u8) -> CidLink { + let digest: [u8; 32] = std::array::from_fn(|i| seed.wrapping_add(i as u8)); + let mh = multihash::Multihash::<64>::wrap(0x12, &digest).unwrap(); + let c = cid::Cid::new_v1(0x71, mh); + CidLink::from_cid(&c) +} + +fn test_cid_bytes(seed: u8) -> Vec { + let digest: [u8; 32] = std::array::from_fn(|i| seed.wrapping_add(i as u8)); + let mh = multihash::Multihash::<64>::wrap(0x12, &digest).unwrap(); + cid::Cid::new_v1(0x71, mh).to_bytes() +} + +struct UserInfo { + user_id: Uuid, + did: Did, +} + +async fn seed_users(pool: &HandlerPool, count: usize) -> Vec { + let users: Vec = (0..count) + .map(|i| { + let user_id = Uuid::new_v4(); + UserInfo { + did: Did::from(format!("did:plc:prof{i:06x}{}", user_id.as_simple())), + user_id, + } + }) + .collect(); + + let batch_size = 500; + let start = Instant::now(); + let total_batches = count.div_ceil(batch_size); + futures::stream::iter(users.chunks(batch_size).enumerate()) + .fold((), |(), (batch_idx, batch)| async move { + futures::stream::iter(batch.iter()) + .fold((), |(), user| async { + let (tx, rx) = oneshot::channel(); + pool.send(MetastoreRequest::Repo(RepoRequest::CreateRepoFull { + user_id: user.user_id, + did: user.did.clone(), + handle: Handle::from(format!("u{}.prof.invalid", user.user_id.as_simple())), + repo_root_cid: test_cid(1), + repo_rev: "rev0000000000".to_string(), + tx, + })) + .unwrap(); + rx.await.unwrap().unwrap(); + }) + .await; + if (batch_idx + 1) % 100 == 0 || batch_idx + 1 == total_batches { + println!( + "seeded {}/{} users, {:.1}s", + ((batch_idx + 1) * batch_size).min(count), + count, + start.elapsed().as_secs_f64() + ); + } + }) + .await; + println!( + "seeded {} users in {:.1}s, {:.0} users/sec", + count, + start.elapsed().as_secs_f64(), + count as f64 / start.elapsed().as_secs_f64() + ); + users +} + +async fn seed_records(pool: &Arc, users: &[UserInfo], records_per_user: usize) { + let start = Instant::now(); + let total = users.len(); + let batch_size = 500; + let total_batches = total.div_ceil(batch_size); + let collection = Nsid::from("app.bsky.feed.post".to_string()); + + futures::stream::iter(users.chunks(batch_size).enumerate()) + .fold((), |(), (chunk_idx, chunk)| { + let pool = Arc::clone(pool); + let collection = collection.clone(); + async move { + futures::stream::iter(chunk.iter()) + .fold((), |(), user| { + let pool = &pool; + let collection = &collection; + async move { + let record_upserts: Vec = (0..records_per_user) + .map(|i| RecordUpsert { + collection: collection.clone(), + rkey: Rkey::from(format!("rec{i:08}")), + cid: test_cid(((i * 7 + 3) & 0xFF) as u8), + }) + .collect(); + let new_block_cids: Vec> = (0..records_per_user) + .map(|i| test_cid_bytes(((i * 11 + 5) & 0xFF) as u8)) + .collect(); + let input = ApplyCommitInput { + user_id: user.user_id, + did: user.did.clone(), + expected_root_cid: None, + new_root_cid: test_cid(2), + new_rev: "rev0000000001".to_string(), + new_block_cids, + obsolete_block_cids: vec![], + record_upserts, + record_deletes: vec![], + backlinks_to_add: vec![], + backlinks_to_remove: vec![], + commit_event: CommitEventData { + did: user.did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(test_cid(2)), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev0000000001".to_string()), + }, + }; + let (tx, rx) = oneshot::channel(); + pool.send(MetastoreRequest::Commit(Box::new( + CommitRequest::ApplyCommit { + input: Box::new(input), + tx, + }, + ))) + .unwrap(); + rx.await.unwrap().unwrap(); + } + }) + .await; + if (chunk_idx + 1) % 100 == 0 || chunk_idx + 1 == total_batches { + println!( + "seeded records for {}/{} users, {:.1}s", + ((chunk_idx + 1) * batch_size).min(total), + total, + start.elapsed().as_secs_f64() + ); + } + } + }) + .await; + println!( + "seeded {} records across {} users in {:.1}s", + total * records_per_user, + total, + start.elapsed().as_secs_f64() + ); +} + +async fn profile_list_records( + pool: &Arc, + user_ids: &Arc>, + concurrency: usize, + seconds: u64, +) -> u64 { + let deadline = Instant::now() + Duration::from_secs(seconds); + let user_count = user_ids.len(); + + let handles: Vec<_> = (0..concurrency) + .map(|task_id| { + let pool = Arc::clone(pool); + let user_ids = Arc::clone(user_ids); + tokio::spawn(async move { + futures::stream::unfold(0usize, |i| { + let cont = Instant::now() < deadline; + async move { cont.then_some((i, i + 1)) } + }) + .fold(0u64, |ops, i| { + let pool = &pool; + let user_ids = &user_ids; + async move { + let idx = (task_id * 997 + i * 31) % user_count; + let (tx, rx) = oneshot::channel(); + pool.send(MetastoreRequest::Record(RecordRequest::ListRecords { + repo_id: user_ids[idx], + collection: Nsid::from("app.bsky.feed.post".to_string()), + cursor: None, + limit: 50, + reverse: false, + rkey_start: None, + rkey_end: None, + tx, + })) + .unwrap(); + let _ = rx.await.unwrap().unwrap(); + ops + 1 + } + }) + .await + }) + }) + .collect(); + + futures::stream::iter(handles) + .fold(0u64, |acc, h| async move { acc + h.await.unwrap() }) + .await +} + +async fn profile_get_record_cid( + pool: &Arc, + user_ids: &Arc>, + concurrency: usize, + seconds: u64, + records_per_user: usize, +) -> u64 { + let deadline = Instant::now() + Duration::from_secs(seconds); + let user_count = user_ids.len(); + + let handles: Vec<_> = (0..concurrency) + .map(|task_id| { + let pool = Arc::clone(pool); + let user_ids = Arc::clone(user_ids); + tokio::spawn(async move { + futures::stream::unfold(0usize, |i| { + let cont = Instant::now() < deadline; + async move { cont.then_some((i, i + 1)) } + }) + .fold(0u64, |ops, i| { + let pool = &pool; + let user_ids = &user_ids; + async move { + let user_idx = (task_id * 997 + i * 31) % user_count; + let rec_idx = (task_id * 13 + i * 7) % records_per_user; + let rkey = Rkey::from(format!("rec{rec_idx:08}")); + let (tx, rx) = oneshot::channel(); + pool.send(MetastoreRequest::Record(RecordRequest::GetRecordCid { + repo_id: user_ids[user_idx], + collection: Nsid::from("app.bsky.feed.post".to_string()), + rkey, + tx, + })) + .unwrap(); + let _ = rx.await.unwrap().unwrap(); + ops + 1 + } + }) + .await + }) + }) + .collect(); + + futures::stream::iter(handles) + .fold(0u64, |acc, h| async move { acc + h.await.unwrap() }) + .await +} + +#[tokio::main] +async fn main() { + let user_count = std::env::var("PROFILE_USERS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(300_000usize); + let records_per_user = 10; + let profile_seconds = std::env::var("PROFILE_SECONDS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(30u64); + let concurrency = 100usize; + + let handler_threads = std::thread::available_parallelism() + .map(|n| n.get().max(2) / 2) + .unwrap_or(2); + + println!("-- profile reads --"); + println!("handler threads: {handler_threads}"); + println!("{user_count} users, {records_per_user} records each, {profile_seconds}s per phase"); + + let metastore_dir = tempfile::TempDir::new().unwrap(); + let eventlog_dir = tempfile::TempDir::new().unwrap(); + let segments_dir = eventlog_dir.path().join("segments"); + std::fs::create_dir_all(&segments_dir).unwrap(); + + let cache_bytes = (user_count as u64 * 10 * 300) + .saturating_mul(2) + .clamp(512 * 1024 * 1024, 8 * 1024 * 1024 * 1024); + println!("cache size: {} MB", cache_bytes / (1024 * 1024)); + + let metastore = Metastore::open( + metastore_dir.path(), + MetastoreConfig { + cache_size_bytes: cache_bytes, + }, + ) + .unwrap(); + + let event_log = EventLog::open( + EventLogConfig { + segments_dir, + ..EventLogConfig::default() + }, + RealIO::new(), + ) + .unwrap(); + + let bridge = Arc::new(tranquil_store::eventlog::EventLogBridge::new(Arc::new( + event_log, + ))); + + let pool = Arc::new(HandlerPool::spawn::( + metastore.clone(), + bridge, + None, + Some(handler_threads), + )); + + let users = seed_users(&pool, user_count).await; + seed_records(&pool, &users, records_per_user).await; + + println!("running major compaction..."); + let t = Instant::now(); + metastore.major_compact().unwrap(); + println!( + "major compaction complete in {:.1}s", + t.elapsed().as_secs_f64() + ); + + let user_ids: Arc> = Arc::new(users.iter().map(|u| u.user_id).collect()); + + println!("-- listRecords, {concurrency} readers, {profile_seconds}s --"); + let list_ops = profile_list_records(&pool, &user_ids, concurrency, profile_seconds).await; + println!( + "listRecords: {list_ops} ops, {:.0} ops/sec", + list_ops as f64 / profile_seconds as f64 + ); + + println!("-- getRecordCid, {concurrency} readers, {profile_seconds}s --"); + let get_ops = profile_get_record_cid( + &pool, + &user_ids, + concurrency, + profile_seconds, + records_per_user, + ) + .await; + println!( + "getRecordCid: {get_ops} ops, {:.0} ops/sec", + get_ops as f64 / profile_seconds as f64 + ); + + println!("-- profile reads complete :D --"); +} diff --git a/crates/tranquil-store/src/lib.rs b/crates/tranquil-store/src/lib.rs index 349daac..fd9a379 100644 --- a/crates/tranquil-store/src/lib.rs +++ b/crates/tranquil-store/src/lib.rs @@ -3,6 +3,7 @@ pub mod eventlog; pub mod fsync_order; mod harness; mod io; +pub mod metastore; mod record; #[cfg(any(test, feature = "test-harness"))] mod sim; diff --git a/crates/tranquil-store/src/metastore/backlink_ops.rs b/crates/tranquil-store/src/metastore/backlink_ops.rs new file mode 100644 index 0000000..281a93b --- /dev/null +++ b/crates/tranquil-store/src/metastore/backlink_ops.rs @@ -0,0 +1,638 @@ +use std::sync::Arc; + +use fjall::{Keyspace, OwnedWriteBatch}; +use uuid::Uuid; + +use super::MetastoreError; +use super::backlinks::{ + BacklinkValue, backlink_by_user_key, backlink_by_user_prefix, backlink_by_user_record_prefix, + backlink_key, backlink_target_user_prefix, discriminant_to_path, path_to_discriminant, +}; +use super::encoding::KeyReader; +use super::keys::{KeyTag, UserHash}; +use super::user_hash::UserHashMap; + +use tranquil_db_traits::Backlink; +use tranquil_types::{AtUri, Nsid}; + +pub(super) fn parse_backlink_by_user_fields(key_bytes: &[u8]) -> Option<(String, String, String)> { + let mut reader = KeyReader::new(key_bytes); + let tag = reader.tag()?; + match tag == KeyTag::BACKLINK_BY_USER.raw() { + true => { + let _user_hash = reader.u64()?; + let collection = reader.string()?; + let rkey = reader.string()?; + let link_target = reader.string()?; + Some((collection, rkey, link_target)) + } + false => None, + } +} + +pub(super) fn remove_backlinks_for_record( + batch: &mut OwnedWriteBatch, + indexes: &Keyspace, + user_hash: UserHash, + collection: &str, + rkey: &str, +) -> Result<(), MetastoreError> { + let record_prefix = backlink_by_user_record_prefix(user_hash, collection, rkey); + indexes + .prefix(record_prefix.as_slice()) + .try_for_each(|guard| { + let (key_bytes, _) = guard.into_inner().map_err(MetastoreError::Fjall)?; + + let (_, _, link_target) = parse_backlink_by_user_fields(&key_bytes).ok_or( + MetastoreError::CorruptData("unparseable BACKLINK_BY_USER key"), + )?; + + let primary = backlink_key(&link_target, user_hash, collection, rkey); + batch.remove(indexes, primary.as_slice()); + batch.remove(indexes, key_bytes.as_ref()); + + Ok::<_, MetastoreError>(()) + }) +} + +pub struct BacklinkOps { + indexes: Keyspace, + user_hashes: Arc, +} + +impl BacklinkOps { + pub fn new(indexes: Keyspace, user_hashes: Arc) -> Self { + Self { + indexes, + user_hashes, + } + } + + pub fn add_backlinks( + &self, + batch: &mut OwnedWriteBatch, + user_hash: UserHash, + backlinks: &[Backlink], + ) -> Result<(), MetastoreError> { + backlinks + .iter() + .try_for_each(|bl| self.add_single_backlink(batch, user_hash, bl)) + } + + fn add_single_backlink( + &self, + batch: &mut OwnedWriteBatch, + user_hash: UserHash, + bl: &Backlink, + ) -> Result<(), MetastoreError> { + let collection = bl.uri.collection().ok_or(MetastoreError::InvalidInput( + "backlink uri missing collection", + ))?; + let rkey = bl + .uri + .rkey() + .ok_or(MetastoreError::InvalidInput("backlink uri missing rkey"))?; + + let primary = backlink_key(&bl.link_to, user_hash, collection, rkey); + let value = BacklinkValue { + source_uri: bl.uri.as_str().to_owned(), + path: path_to_discriminant(bl.path), + }; + batch.insert(&self.indexes, primary.as_slice(), value.serialize()); + + let reverse = backlink_by_user_key(user_hash, collection, rkey, &bl.link_to); + batch.insert(&self.indexes, reverse.as_slice(), []); + Ok(()) + } + + pub fn remove_backlinks_by_uri( + &self, + batch: &mut OwnedWriteBatch, + user_hash: UserHash, + uri: &AtUri, + ) -> Result<(), MetastoreError> { + let collection = uri.collection().ok_or(MetastoreError::InvalidInput( + "backlink uri missing collection", + ))?; + let rkey = uri + .rkey() + .ok_or(MetastoreError::InvalidInput("backlink uri missing rkey"))?; + + remove_backlinks_for_record(batch, &self.indexes, user_hash, collection, rkey) + } + + pub fn remove_backlinks_by_repo( + &self, + batch: &mut OwnedWriteBatch, + user_hash: UserHash, + ) -> Result<(), MetastoreError> { + let user_prefix = backlink_by_user_prefix(user_hash); + self.indexes + .prefix(user_prefix.as_slice()) + .try_for_each(|guard| { + let (key_bytes, _) = guard.into_inner().map_err(MetastoreError::Fjall)?; + + let (collection, rkey, link_target) = parse_backlink_by_user_fields(&key_bytes) + .ok_or(MetastoreError::CorruptData( + "unparseable BACKLINK_BY_USER key", + ))?; + + let primary = backlink_key(&link_target, user_hash, &collection, &rkey); + batch.remove(&self.indexes, primary.as_slice()); + batch.remove(&self.indexes, key_bytes.as_ref()); + + Ok::<_, MetastoreError>(()) + }) + } + + pub fn get_backlink_conflicts( + &self, + repo_id: Uuid, + collection: &Nsid, + backlinks: &[Backlink], + ) -> Result, MetastoreError> { + if backlinks.is_empty() { + return Ok(Vec::new()); + } + + let user_hash = self + .user_hashes + .get(&repo_id) + .ok_or(MetastoreError::InvalidInput("unknown repo_id"))?; + + let collection_str = collection.as_str(); + + let mut seen = std::collections::HashSet::new(); + backlinks.iter().try_fold(Vec::new(), |mut conflicts, bl| { + let prefix = backlink_target_user_prefix(&bl.link_to, user_hash); + self.indexes + .prefix(prefix.as_slice()) + .try_for_each(|guard| { + let (_, val_bytes) = guard.into_inner().map_err(MetastoreError::Fjall)?; + + let val = BacklinkValue::deserialize(&val_bytes).ok_or( + MetastoreError::CorruptData("corrupt backlink value in indexes partition"), + )?; + + let uri: AtUri = val.source_uri.into(); + let is_self = uri.as_str() == bl.uri.as_str(); + let matches_collection = uri.collection().is_some_and(|c| c == collection_str); + let matches_path = match discriminant_to_path(val.path) { + Some(p) => p == bl.path, + None => { + tracing::warn!( + discriminant = val.path, + uri = %uri, + "unknown backlink path discriminant in indexes partition" + ); + false + } + }; + if !is_self + && matches_collection + && matches_path + && !seen.contains(uri.as_str()) + { + seen.insert(uri.as_str().to_owned()); + conflicts.push(uri); + } + + Ok::<_, MetastoreError>(()) + })?; + Ok(conflicts) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::metastore::backlinks::{backlink_by_user_prefix, backlink_target_prefix}; + use crate::metastore::partitions::Partition; + use crate::metastore::{Metastore, MetastoreConfig}; + use tranquil_db_traits::{Backlink, BacklinkPath}; + use tranquil_types::{Did, Handle, Nsid}; + + struct TestHarness { + _dir: tempfile::TempDir, + metastore: Metastore, + } + + fn setup() -> TestHarness { + let dir = tempfile::TempDir::new().unwrap(); + let metastore = Metastore::open( + dir.path(), + MetastoreConfig { + cache_size_bytes: 64 * 1024 * 1024, + }, + ) + .unwrap(); + TestHarness { + _dir: dir, + metastore, + } + } + + fn test_cid_link(seed: u8) -> tranquil_types::CidLink { + let digest: [u8; 32] = std::array::from_fn(|i| seed.wrapping_add(i as u8)); + let mh = multihash::Multihash::<64>::wrap(0x12, &digest).unwrap(); + let c = cid::Cid::new_v1(0x71, mh); + tranquil_types::CidLink::from_cid(&c) + } + + fn create_repo(h: &TestHarness, name: &str, seed: u8) -> (Uuid, UserHash) { + let user_id = Uuid::new_v4(); + let did = Did::from(format!("did:plc:{name}")); + let handle = Handle::from(format!("{name}.test.invalid")); + let cid = test_cid_link(seed); + h.metastore + .repo_ops() + .create_repo(h.metastore.database(), user_id, &did, &handle, &cid, "rev0") + .unwrap(); + let user_hash = h.metastore.user_hashes().get(&user_id).unwrap(); + (user_id, user_hash) + } + + fn count_prefix(ks: &fjall::Keyspace, prefix: &[u8]) -> usize { + ks.prefix(prefix) + .map(|g| g.into_inner().expect("prefix scan must not fail in test")) + .fold(0, |acc, _| acc + 1) + } + + #[test] + fn add_and_query_by_target() { + let h = setup(); + let ops = h.metastore.backlink_ops(); + let (_user_id, user_hash) = create_repo(&h, "alice", 1); + + let uri = AtUri::from_parts("did:plc:alice", "app.bsky.feed.like", "3k2abc"); + let backlinks = vec![Backlink { + uri: uri.clone(), + path: BacklinkPath::SubjectUri, + link_to: "at://did:plc:target/app.bsky.feed.post/3k2post".to_string(), + }]; + + let mut batch = h.metastore.database().batch(); + ops.add_backlinks(&mut batch, user_hash, &backlinks) + .unwrap(); + batch.commit().unwrap(); + + let indexes = h.metastore.partition(Partition::Indexes); + let target_prefix = + backlink_target_prefix("at://did:plc:target/app.bsky.feed.post/3k2post"); + assert_eq!(count_prefix(indexes, target_prefix.as_slice()), 1); + + let user_prefix = backlink_by_user_prefix(user_hash); + assert_eq!(count_prefix(indexes, user_prefix.as_slice()), 1); + } + + #[test] + fn remove_by_uri_deletes_both_indexes() { + let h = setup(); + let ops = h.metastore.backlink_ops(); + let (_user_id, user_hash) = create_repo(&h, "bob", 2); + + let uri = AtUri::from_parts("did:plc:bob", "app.bsky.graph.follow", "3k2fol"); + let backlinks = vec![Backlink { + uri: uri.clone(), + path: BacklinkPath::Subject, + link_to: "did:plc:target_user".to_string(), + }]; + + let mut batch = h.metastore.database().batch(); + ops.add_backlinks(&mut batch, user_hash, &backlinks) + .unwrap(); + batch.commit().unwrap(); + + let indexes = h.metastore.partition(Partition::Indexes); + assert_eq!( + count_prefix( + indexes, + backlink_target_prefix("did:plc:target_user").as_slice() + ), + 1 + ); + + let mut batch = h.metastore.database().batch(); + ops.remove_backlinks_by_uri(&mut batch, user_hash, &uri) + .unwrap(); + batch.commit().unwrap(); + + assert_eq!( + count_prefix( + indexes, + backlink_target_prefix("did:plc:target_user").as_slice() + ), + 0 + ); + assert_eq!( + count_prefix(indexes, backlink_by_user_prefix(user_hash).as_slice()), + 0 + ); + } + + #[test] + fn remove_by_repo_deletes_all_user_backlinks() { + let h = setup(); + let ops = h.metastore.backlink_ops(); + let (_user_id, user_hash) = create_repo(&h, "carol", 3); + + let backlinks: Vec = (0..5) + .map(|i| Backlink { + uri: AtUri::from_parts("did:plc:carol", "app.bsky.feed.like", &format!("3k2r{i}")), + path: BacklinkPath::SubjectUri, + link_to: format!("at://did:plc:target{i}/app.bsky.feed.post/3k2p{i}"), + }) + .collect(); + + let mut batch = h.metastore.database().batch(); + ops.add_backlinks(&mut batch, user_hash, &backlinks) + .unwrap(); + batch.commit().unwrap(); + + let indexes = h.metastore.partition(Partition::Indexes); + assert_eq!( + count_prefix(indexes, backlink_by_user_prefix(user_hash).as_slice()), + 5 + ); + + let mut batch = h.metastore.database().batch(); + ops.remove_backlinks_by_repo(&mut batch, user_hash).unwrap(); + batch.commit().unwrap(); + + assert_eq!( + count_prefix(indexes, backlink_by_user_prefix(user_hash).as_slice()), + 0 + ); + + (0..5).for_each(|i| { + let target = format!("at://did:plc:target{i}/app.bsky.feed.post/3k2p{i}"); + let prefix = backlink_target_prefix(&target); + assert_eq!(count_prefix(indexes, prefix.as_slice()), 0); + }); + } + + #[test] + fn get_backlink_conflicts_finds_matching() { + let h = setup(); + let ops = h.metastore.backlink_ops(); + let (user_id, user_hash) = create_repo(&h, "dave", 4); + + let existing = Backlink { + uri: AtUri::from_parts("did:plc:dave", "app.bsky.feed.like", "3k2old"), + path: BacklinkPath::SubjectUri, + link_to: "at://did:plc:someone/app.bsky.feed.post/3k2p1".to_string(), + }; + + let mut batch = h.metastore.database().batch(); + ops.add_backlinks(&mut batch, user_hash, &[existing]) + .unwrap(); + batch.commit().unwrap(); + + let proposed = vec![Backlink { + uri: AtUri::from_parts("did:plc:dave", "app.bsky.feed.like", "3k2new"), + path: BacklinkPath::SubjectUri, + link_to: "at://did:plc:someone/app.bsky.feed.post/3k2p1".to_string(), + }]; + + let collection = Nsid::from("app.bsky.feed.like".to_string()); + let conflicts = ops + .get_backlink_conflicts(user_id, &collection, &proposed) + .unwrap(); + + assert_eq!(conflicts.len(), 1); + assert_eq!( + conflicts[0].as_str(), + "at://did:plc:dave/app.bsky.feed.like/3k2old" + ); + } + + #[test] + fn get_backlink_conflicts_ignores_different_collection() { + let h = setup(); + let ops = h.metastore.backlink_ops(); + let (user_id, user_hash) = create_repo(&h, "eve", 5); + + let existing = Backlink { + uri: AtUri::from_parts("did:plc:eve", "app.bsky.feed.like", "3k2old"), + path: BacklinkPath::SubjectUri, + link_to: "at://did:plc:someone/app.bsky.feed.post/3k2p1".to_string(), + }; + + let mut batch = h.metastore.database().batch(); + ops.add_backlinks(&mut batch, user_hash, &[existing]) + .unwrap(); + batch.commit().unwrap(); + + let proposed = vec![Backlink { + uri: AtUri::from_parts("did:plc:eve", "app.bsky.feed.repost", "3k2new"), + path: BacklinkPath::SubjectUri, + link_to: "at://did:plc:someone/app.bsky.feed.post/3k2p1".to_string(), + }]; + + let collection = Nsid::from("app.bsky.feed.repost".to_string()); + let conflicts = ops + .get_backlink_conflicts(user_id, &collection, &proposed) + .unwrap(); + + assert!(conflicts.is_empty()); + } + + #[test] + fn get_backlink_conflicts_ignores_different_path() { + let h = setup(); + let ops = h.metastore.backlink_ops(); + let (user_id, user_hash) = create_repo(&h, "frank", 6); + + let existing = Backlink { + uri: AtUri::from_parts("did:plc:frank", "app.bsky.graph.follow", "3k2old"), + path: BacklinkPath::Subject, + link_to: "did:plc:target".to_string(), + }; + + let mut batch = h.metastore.database().batch(); + ops.add_backlinks(&mut batch, user_hash, &[existing]) + .unwrap(); + batch.commit().unwrap(); + + let proposed = vec![Backlink { + uri: AtUri::from_parts("did:plc:frank", "app.bsky.graph.follow", "3k2new"), + path: BacklinkPath::SubjectUri, + link_to: "did:plc:target".to_string(), + }]; + + let collection = Nsid::from("app.bsky.graph.follow".to_string()); + let conflicts = ops + .get_backlink_conflicts(user_id, &collection, &proposed) + .unwrap(); + + assert!(conflicts.is_empty()); + } + + #[test] + fn get_backlink_conflicts_ignores_other_users() { + let h = setup(); + let ops = h.metastore.backlink_ops(); + let (_user_id_a, user_hash_a) = create_repo(&h, "grace", 7); + let (user_id_b, _user_hash_b) = create_repo(&h, "henry", 8); + + let existing = Backlink { + uri: AtUri::from_parts("did:plc:grace", "app.bsky.feed.like", "3k2old"), + path: BacklinkPath::SubjectUri, + link_to: "at://did:plc:target/app.bsky.feed.post/3k2p1".to_string(), + }; + + let mut batch = h.metastore.database().batch(); + ops.add_backlinks(&mut batch, user_hash_a, &[existing]) + .unwrap(); + batch.commit().unwrap(); + + let proposed = vec![Backlink { + uri: AtUri::from_parts("did:plc:henry", "app.bsky.feed.like", "3k2new"), + path: BacklinkPath::SubjectUri, + link_to: "at://did:plc:target/app.bsky.feed.post/3k2p1".to_string(), + }]; + + let collection = Nsid::from("app.bsky.feed.like".to_string()); + let conflicts = ops + .get_backlink_conflicts(user_id_b, &collection, &proposed) + .unwrap(); + + assert!(conflicts.is_empty()); + } + + #[test] + fn get_backlink_conflicts_excludes_self_match() { + let h = setup(); + let ops = h.metastore.backlink_ops(); + let (user_id, user_hash) = create_repo(&h, "luna", 12); + + let existing = Backlink { + uri: AtUri::from_parts("did:plc:luna", "app.bsky.feed.like", "3k2same"), + path: BacklinkPath::SubjectUri, + link_to: "at://did:plc:someone/app.bsky.feed.post/3k2p1".to_string(), + }; + + let mut batch = h.metastore.database().batch(); + ops.add_backlinks(&mut batch, user_hash, &[existing]) + .unwrap(); + batch.commit().unwrap(); + + let proposed = vec![Backlink { + uri: AtUri::from_parts("did:plc:luna", "app.bsky.feed.like", "3k2same"), + path: BacklinkPath::SubjectUri, + link_to: "at://did:plc:someone/app.bsky.feed.post/3k2p1".to_string(), + }]; + + let collection = Nsid::from("app.bsky.feed.like".to_string()); + let conflicts = ops + .get_backlink_conflicts(user_id, &collection, &proposed) + .unwrap(); + + assert!(conflicts.is_empty()); + } + + #[test] + fn empty_backlinks_returns_empty_conflicts() { + let h = setup(); + let ops = h.metastore.backlink_ops(); + let (user_id, _user_hash) = create_repo(&h, "ivan", 9); + + let collection = Nsid::from("app.bsky.feed.like".to_string()); + let conflicts = ops + .get_backlink_conflicts(user_id, &collection, &[]) + .unwrap(); + assert!(conflicts.is_empty()); + } + + #[test] + fn remove_by_uri_only_removes_matching_rkey() { + let h = setup(); + let ops = h.metastore.backlink_ops(); + let (_user_id, user_hash) = create_repo(&h, "julia", 10); + + let bl1 = Backlink { + uri: AtUri::from_parts("did:plc:julia", "app.bsky.feed.like", "3k2aaa"), + path: BacklinkPath::SubjectUri, + link_to: "at://did:plc:t1/app.bsky.feed.post/p1".to_string(), + }; + let bl2 = Backlink { + uri: AtUri::from_parts("did:plc:julia", "app.bsky.feed.like", "3k2bbb"), + path: BacklinkPath::SubjectUri, + link_to: "at://did:plc:t2/app.bsky.feed.post/p2".to_string(), + }; + + let mut batch = h.metastore.database().batch(); + ops.add_backlinks(&mut batch, user_hash, &[bl1.clone(), bl2]) + .unwrap(); + batch.commit().unwrap(); + + let indexes = h.metastore.partition(Partition::Indexes); + assert_eq!( + count_prefix(indexes, backlink_by_user_prefix(user_hash).as_slice()), + 2 + ); + + let mut batch = h.metastore.database().batch(); + ops.remove_backlinks_by_uri(&mut batch, user_hash, &bl1.uri) + .unwrap(); + batch.commit().unwrap(); + + assert_eq!( + count_prefix(indexes, backlink_by_user_prefix(user_hash).as_slice()), + 1 + ); + assert_eq!( + count_prefix( + indexes, + backlink_target_prefix("at://did:plc:t1/app.bsky.feed.post/p1").as_slice() + ), + 0 + ); + assert_eq!( + count_prefix( + indexes, + backlink_target_prefix("at://did:plc:t2/app.bsky.feed.post/p2").as_slice() + ), + 1 + ); + } + + #[test] + fn conflicts_deduplicates_results() { + let h = setup(); + let ops = h.metastore.backlink_ops(); + let (user_id, user_hash) = create_repo(&h, "kate", 11); + + let existing = Backlink { + uri: AtUri::from_parts("did:plc:kate", "app.bsky.feed.like", "3k2old"), + path: BacklinkPath::SubjectUri, + link_to: "at://did:plc:someone/app.bsky.feed.post/3k2p1".to_string(), + }; + + let mut batch = h.metastore.database().batch(); + ops.add_backlinks(&mut batch, user_hash, &[existing]) + .unwrap(); + batch.commit().unwrap(); + + let proposed = vec![ + Backlink { + uri: AtUri::from_parts("did:plc:kate", "app.bsky.feed.like", "3k2new1"), + path: BacklinkPath::SubjectUri, + link_to: "at://did:plc:someone/app.bsky.feed.post/3k2p1".to_string(), + }, + Backlink { + uri: AtUri::from_parts("did:plc:kate", "app.bsky.feed.like", "3k2new2"), + path: BacklinkPath::SubjectUri, + link_to: "at://did:plc:someone/app.bsky.feed.post/3k2p1".to_string(), + }, + ]; + + let collection = Nsid::from("app.bsky.feed.like".to_string()); + let conflicts = ops + .get_backlink_conflicts(user_id, &collection, &proposed) + .unwrap(); + + assert_eq!(conflicts.len(), 1); + } +} diff --git a/crates/tranquil-store/src/metastore/backlinks.rs b/crates/tranquil-store/src/metastore/backlinks.rs new file mode 100644 index 0000000..131f2a3 --- /dev/null +++ b/crates/tranquil-store/src/metastore/backlinks.rs @@ -0,0 +1,257 @@ +use serde::{Deserialize, Serialize}; +use smallvec::SmallVec; + +use super::encoding::KeyBuilder; +use super::keys::{KeyTag, UserHash}; + +use tranquil_db_traits::BacklinkPath; + +const SCHEMA_VERSION: u8 = 1; + +pub fn path_to_discriminant(path: BacklinkPath) -> u8 { + match path { + BacklinkPath::Subject => 0, + BacklinkPath::SubjectUri => 1, + } +} + +pub fn discriminant_to_path(d: u8) -> Option { + match d { + 0 => Some(BacklinkPath::Subject), + 1 => Some(BacklinkPath::SubjectUri), + _ => None, + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BacklinkValue { + pub source_uri: String, + pub path: u8, +} + +impl BacklinkValue { + pub fn serialize(&self) -> Vec { + let payload = postcard::to_allocvec(self).expect("BacklinkValue serialization cannot fail"); + let mut buf = Vec::with_capacity(1 + payload.len()); + buf.push(SCHEMA_VERSION); + buf.extend_from_slice(&payload); + buf + } + + pub fn deserialize(bytes: &[u8]) -> Option { + let (&version, payload) = bytes.split_first()?; + match version { + SCHEMA_VERSION => postcard::from_bytes(payload).ok(), + _ => None, + } + } +} + +pub fn backlink_key( + link_target: &str, + user_hash: UserHash, + collection: &str, + rkey: &str, +) -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::BACKLINKS) + .string(link_target) + .u64(user_hash.raw()) + .string(collection) + .string(rkey) + .build() +} + +pub fn backlink_target_prefix(link_target: &str) -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::BACKLINKS) + .string(link_target) + .build() +} + +pub fn backlink_target_user_prefix(link_target: &str, user_hash: UserHash) -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::BACKLINKS) + .string(link_target) + .u64(user_hash.raw()) + .build() +} + +pub fn backlink_by_user_key( + user_hash: UserHash, + collection: &str, + rkey: &str, + link_target: &str, +) -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::BACKLINK_BY_USER) + .u64(user_hash.raw()) + .string(collection) + .string(rkey) + .string(link_target) + .build() +} + +pub fn backlink_by_user_prefix(user_hash: UserHash) -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::BACKLINK_BY_USER) + .u64(user_hash.raw()) + .build() +} + +pub fn backlink_by_user_record_prefix( + user_hash: UserHash, + collection: &str, + rkey: &str, +) -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::BACKLINK_BY_USER) + .u64(user_hash.raw()) + .string(collection) + .string(rkey) + .build() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::metastore::encoding::KeyReader; + + #[test] + fn discriminant_roundtrip() { + assert_eq!( + discriminant_to_path(path_to_discriminant(BacklinkPath::Subject)), + Some(BacklinkPath::Subject) + ); + assert_eq!( + discriminant_to_path(path_to_discriminant(BacklinkPath::SubjectUri)), + Some(BacklinkPath::SubjectUri) + ); + assert_eq!(discriminant_to_path(255), None); + assert_eq!(discriminant_to_path(2), None); + } + + #[test] + fn backlink_value_roundtrip() { + let value = BacklinkValue { + source_uri: "at://did:plc:abc/app.bsky.feed.like/3k2xyz".to_string(), + path: path_to_discriminant(BacklinkPath::SubjectUri), + }; + let bytes = value.serialize(); + let decoded = BacklinkValue::deserialize(&bytes).unwrap(); + assert_eq!(decoded, value); + } + + #[test] + fn schema_version_is_first_byte() { + let value = BacklinkValue { + source_uri: "at://x".to_string(), + path: path_to_discriminant(BacklinkPath::Subject), + }; + let bytes = value.serialize(); + assert_eq!(bytes[0], SCHEMA_VERSION); + } + + #[test] + fn deserialize_rejects_unknown_version() { + let value = BacklinkValue { + source_uri: "at://x".to_string(), + path: path_to_discriminant(BacklinkPath::Subject), + }; + let mut bytes = value.serialize(); + bytes[0] = 99; + assert!(BacklinkValue::deserialize(&bytes).is_none()); + } + + #[test] + fn deserialize_rejects_empty() { + assert!(BacklinkValue::deserialize(&[]).is_none()); + } + + #[test] + fn backlink_key_roundtrip() { + let hash = UserHash::from_raw(0xCAFE_BABE_DEAD_BEEF); + let key = backlink_key( + "at://did:plc:target/app.bsky.feed.post/3k2abc", + hash, + "app.bsky.feed.like", + "3k2xyz", + ); + let mut reader = KeyReader::new(&key); + assert_eq!(reader.tag(), Some(KeyTag::BACKLINKS.raw())); + assert_eq!( + reader.string(), + Some("at://did:plc:target/app.bsky.feed.post/3k2abc".to_string()) + ); + assert_eq!(reader.u64(), Some(0xCAFE_BABE_DEAD_BEEF)); + assert_eq!(reader.string(), Some("app.bsky.feed.like".to_string())); + assert_eq!(reader.string(), Some("3k2xyz".to_string())); + assert!(reader.is_empty()); + } + + #[test] + fn backlink_keys_sort_by_target_then_user_then_collection_then_rkey() { + let h1 = UserHash::from_raw(1); + let h2 = UserHash::from_raw(2); + + let k1 = backlink_key("aaa", h1, "col_a", "r1"); + let k2 = backlink_key("aaa", h1, "col_a", "r2"); + let k3 = backlink_key("aaa", h1, "col_b", "r1"); + let k4 = backlink_key("aaa", h2, "col_a", "r1"); + let k5 = backlink_key("bbb", h1, "col_a", "r1"); + + assert!(k1.as_slice() < k2.as_slice()); + assert!(k2.as_slice() < k3.as_slice()); + assert!(k3.as_slice() < k4.as_slice()); + assert!(k4.as_slice() < k5.as_slice()); + } + + #[test] + fn target_prefix_is_prefix_of_full_key() { + let hash = UserHash::from_raw(42); + let prefix = backlink_target_prefix("did:plc:target"); + let full = backlink_key("did:plc:target", hash, "col", "rk"); + assert!(full.as_slice().starts_with(prefix.as_slice())); + } + + #[test] + fn by_user_key_roundtrip() { + let hash = UserHash::from_raw(0xDEAD_BEEF_1234_5678); + let key = backlink_by_user_key(hash, "app.bsky.feed.like", "3k2abc", "did:plc:target"); + let mut reader = KeyReader::new(&key); + assert_eq!(reader.tag(), Some(KeyTag::BACKLINK_BY_USER.raw())); + assert_eq!(reader.u64(), Some(0xDEAD_BEEF_1234_5678)); + assert_eq!(reader.string(), Some("app.bsky.feed.like".to_string())); + assert_eq!(reader.string(), Some("3k2abc".to_string())); + assert_eq!(reader.string(), Some("did:plc:target".to_string())); + assert!(reader.is_empty()); + } + + #[test] + fn by_user_prefix_is_prefix_of_full_key() { + let hash = UserHash::from_raw(42); + let prefix = backlink_by_user_prefix(hash); + let full = backlink_by_user_key(hash, "col", "rk", "target"); + assert!(full.as_slice().starts_with(prefix.as_slice())); + } + + #[test] + fn by_user_record_prefix_is_prefix_of_full_key() { + let hash = UserHash::from_raw(42); + let prefix = backlink_by_user_record_prefix(hash, "col", "rk"); + let full = backlink_by_user_key(hash, "col", "rk", "target"); + assert!(full.as_slice().starts_with(prefix.as_slice())); + } + + #[test] + fn same_rkey_different_collection_produces_distinct_keys() { + let hash = UserHash::from_raw(42); + let k1 = backlink_key("target", hash, "app.bsky.feed.like", "self"); + let k2 = backlink_key("target", hash, "app.bsky.graph.follow", "self"); + assert_ne!(k1.as_slice(), k2.as_slice()); + + let r1 = backlink_by_user_key(hash, "app.bsky.feed.like", "self", "target"); + let r2 = backlink_by_user_key(hash, "app.bsky.graph.follow", "self", "target"); + assert_ne!(r1.as_slice(), r2.as_slice()); + } +} diff --git a/crates/tranquil-store/src/metastore/blob_ops.rs b/crates/tranquil-store/src/metastore/blob_ops.rs new file mode 100644 index 0000000..0703423 --- /dev/null +++ b/crates/tranquil-store/src/metastore/blob_ops.rs @@ -0,0 +1,785 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::ops::Bound; +use std::sync::Arc; + +use fjall::{Database, Keyspace}; +use smallvec::SmallVec; +use uuid::Uuid; + +use super::MetastoreError; +use super::blobs::{BlobMetaValue, blob_by_cid_key, blob_meta_key, blob_user_prefix, blobs_prefix}; +use super::commit_ops::{RecordBlobsValue, record_blobs_user_prefix}; +use super::encoding::{KeyReader, exclusive_upper_bound}; +use super::keys::{KeyTag, UserHash}; +use super::repo_ops::bytes_to_cid_link; +use super::scan::{count_prefix, point_lookup}; +use super::user_hash::UserHashMap; +use tranquil_types::CidLink; + +const DELETE_BATCH_SIZE: usize = 1024; + +pub struct BlobOps { + db: Database, + repo_data: Keyspace, + user_hashes: Arc, +} + +impl BlobOps { + pub fn new(db: Database, repo_data: Keyspace, user_hashes: Arc) -> Self { + Self { + db, + repo_data, + user_hashes, + } + } + + fn resolve_user_hash(&self, user_id: Uuid) -> Result { + self.user_hashes + .get(&user_id) + .ok_or(MetastoreError::InvalidInput("unknown user_id")) + } + + pub fn insert_blob( + &self, + cid: &CidLink, + mime_type: &str, + size_bytes: i64, + created_by_user: Uuid, + storage_key: &str, + ) -> Result, MetastoreError> { + if size_bytes < 0 { + return Err(MetastoreError::InvalidInput( + "size_bytes must be non-negative", + )); + } + + let user_hash = self.resolve_user_hash(created_by_user)?; + let cid_str = cid.as_str(); + + let cid_index_key = blob_by_cid_key(cid_str); + let existing = self + .repo_data + .get(cid_index_key.as_slice()) + .map_err(MetastoreError::Fjall)?; + if existing.is_some() { + return Ok(None); + } + + let value = BlobMetaValue { + size_bytes, + mime_type: mime_type.to_owned(), + storage_key: storage_key.to_owned(), + takedown_ref: None, + created_at_ms: chrono::Utc::now().timestamp_millis(), + }; + + let primary_key = blob_meta_key(user_hash, cid_str); + + let mut batch = self.db.batch(); + batch.insert(&self.repo_data, primary_key.as_slice(), value.serialize()); + batch.insert( + &self.repo_data, + cid_index_key.as_slice(), + user_hash.raw().to_be_bytes(), + ); + batch.commit().map_err(MetastoreError::Fjall)?; + + Ok(Some(cid.clone())) + } + + fn lookup_user_hash_by_cid(&self, cid_str: &str) -> Result, MetastoreError> { + let key = blob_by_cid_key(cid_str); + match self + .repo_data + .get(key.as_slice()) + .map_err(MetastoreError::Fjall)? + { + Some(raw) => { + let arr: [u8; 8] = raw + .as_ref() + .try_into() + .map_err(|_| MetastoreError::CorruptData("blob_by_cid value not 8 bytes"))?; + Ok(Some(UserHash::from_raw(u64::from_be_bytes(arr)))) + } + None => Ok(None), + } + } + + fn get_blob_value(&self, cid: &CidLink) -> Result, MetastoreError> { + let cid_str = cid.as_str(); + let user_hash = match self.lookup_user_hash_by_cid(cid_str)? { + Some(h) => h, + None => return Ok(None), + }; + let key = blob_meta_key(user_hash, cid_str); + point_lookup( + &self.repo_data, + key.as_slice(), + BlobMetaValue::deserialize, + "corrupt blob_meta value", + ) + } + + pub fn get_blob_metadata( + &self, + cid: &CidLink, + ) -> Result, MetastoreError> { + Ok(self + .get_blob_value(cid)? + .map(|v| tranquil_db_traits::BlobMetadata { + storage_key: v.storage_key, + mime_type: v.mime_type, + size_bytes: v.size_bytes, + })) + } + + pub fn get_blob_with_takedown( + &self, + cid: &CidLink, + ) -> Result, MetastoreError> { + Ok(self + .get_blob_value(cid)? + .map(|v| tranquil_db_traits::BlobWithTakedown { + cid: cid.clone(), + takedown_ref: v.takedown_ref, + })) + } + + pub fn get_blob_storage_key(&self, cid: &CidLink) -> Result, MetastoreError> { + Ok(self.get_blob_value(cid)?.map(|v| v.storage_key)) + } + + pub fn list_blobs_by_user( + &self, + user_id: Uuid, + cursor: Option<&str>, + limit: usize, + ) -> Result, MetastoreError> { + let user_hash = self.resolve_user_hash(user_id)?; + let prefix = blob_user_prefix(user_hash); + let upper = exclusive_upper_bound(prefix.as_slice()) + .expect("blob user prefix always contains non-0xFF bytes"); + + let range_start: SmallVec<[u8; 128]> = match cursor { + Some(c) => { + let mut cursor_key = blob_meta_key(user_hash, c); + cursor_key.push(0x00); + cursor_key + } + None => prefix, + }; + + self.repo_data + .range(range_start.as_slice()..upper.as_slice()) + .map(|guard| { + let (key_bytes, _) = guard.into_inner().map_err(MetastoreError::Fjall)?; + parse_blob_cid_from_key(key_bytes.as_ref()) + }) + .take(limit) + .collect() + } + + pub fn count_blobs_by_user(&self, user_id: Uuid) -> Result { + let user_hash = self.resolve_user_hash(user_id)?; + let prefix = blob_user_prefix(user_hash); + count_prefix(&self.repo_data, prefix.as_slice()) + } + + pub fn sum_blob_storage(&self) -> Result { + let prefix = blobs_prefix(); + self.repo_data + .prefix(prefix.as_slice()) + .try_fold(0i64, |acc, guard| { + let (_, val_bytes) = guard.into_inner().map_err(MetastoreError::Fjall)?; + let value = BlobMetaValue::deserialize(&val_bytes) + .ok_or(MetastoreError::CorruptData("corrupt blob_meta in sum"))?; + Ok::<_, MetastoreError>(acc.saturating_add(value.size_bytes)) + }) + } + + pub fn update_blob_takedown( + &self, + cid: &CidLink, + takedown_ref: Option<&str>, + ) -> Result { + let cid_str = cid.as_str(); + let user_hash = match self.lookup_user_hash_by_cid(cid_str)? { + Some(h) => h, + None => return Ok(false), + }; + let key = blob_meta_key(user_hash, cid_str); + let mut value = match point_lookup( + &self.repo_data, + key.as_slice(), + BlobMetaValue::deserialize, + "corrupt blob_meta value", + )? { + Some(v) => v, + None => return Ok(false), + }; + + value.takedown_ref = takedown_ref.map(str::to_owned); + let mut batch = self.db.batch(); + batch.insert(&self.repo_data, key.as_slice(), value.serialize()); + batch.commit().map_err(MetastoreError::Fjall)?; + Ok(true) + } + + pub fn delete_blob_by_cid(&self, cid: &CidLink) -> Result { + let cid_str = cid.as_str(); + let user_hash = match self.lookup_user_hash_by_cid(cid_str)? { + Some(h) => h, + None => return Ok(false), + }; + + let primary_key = blob_meta_key(user_hash, cid_str); + let exists = self + .repo_data + .get(primary_key.as_slice()) + .map_err(MetastoreError::Fjall)? + .is_some(); + if !exists { + return Ok(false); + } + + let cid_index_key = blob_by_cid_key(cid_str); + + let mut batch = self.db.batch(); + batch.remove(&self.repo_data, primary_key.as_slice()); + batch.remove(&self.repo_data, cid_index_key.as_slice()); + batch.commit().map_err(MetastoreError::Fjall)?; + + Ok(true) + } + + pub fn delete_blobs_by_user(&self, user_id: Uuid) -> Result { + let user_hash = self.resolve_user_hash(user_id)?; + let prefix = blob_user_prefix(user_hash); + let user_hash_bytes = user_hash.raw().to_be_bytes(); + + let (final_batch, remaining, total) = self + .repo_data + .prefix(prefix.as_slice()) + .map(|guard| { + let (key_bytes, _) = guard.into_inner().map_err(MetastoreError::Fjall)?; + parse_blob_cid_from_key(key_bytes.as_ref()).map(|c| c.as_str().to_owned()) + }) + .try_fold( + (self.db.batch(), 0usize, 0u64), + |(mut batch, count, total), entry: Result<_, MetastoreError>| { + let cid_str = entry?; + batch.remove( + &self.repo_data, + blob_meta_key(user_hash, &cid_str).as_slice(), + ); + let cid_index_key = blob_by_cid_key(&cid_str); + let owns_cid = self + .repo_data + .get(cid_index_key.as_slice()) + .map_err(MetastoreError::Fjall)? + .is_some_and(|raw| raw.as_ref() == user_hash_bytes); + if owns_cid { + batch.remove(&self.repo_data, cid_index_key.as_slice()); + } + let new_count = count + 1; + if new_count >= DELETE_BATCH_SIZE { + batch.commit().map_err(MetastoreError::Fjall)?; + let flushed = u64::try_from(new_count).unwrap_or(u64::MAX); + Ok::<_, MetastoreError>((self.db.batch(), 0, total.saturating_add(flushed))) + } else { + Ok((batch, new_count, total)) + } + }, + )?; + + if remaining > 0 { + final_batch.commit().map_err(MetastoreError::Fjall)?; + let flushed = u64::try_from(remaining).unwrap_or(u64::MAX); + Ok(total.saturating_add(flushed)) + } else { + Ok(total) + } + } + + pub fn get_blob_storage_keys_by_user( + &self, + user_id: Uuid, + ) -> Result, MetastoreError> { + let user_hash = self.resolve_user_hash(user_id)?; + let prefix = blob_user_prefix(user_hash); + + self.repo_data + .prefix(prefix.as_slice()) + .map(|guard| { + let (_, val_bytes) = guard.into_inner().map_err(MetastoreError::Fjall)?; + let value = BlobMetaValue::deserialize(&val_bytes) + .ok_or(MetastoreError::CorruptData("corrupt blob_meta value"))?; + Ok(value.storage_key) + }) + .collect() + } + + pub fn list_missing_blobs( + &self, + repo_id: Uuid, + cursor: Option<&str>, + limit: usize, + ) -> Result, MetastoreError> { + let user_hash = self.resolve_user_hash(repo_id)?; + let rb_prefix = record_blobs_user_prefix(user_hash); + + let missing: BTreeMap = self + .repo_data + .prefix(rb_prefix.as_slice()) + .try_fold(BTreeMap::new(), |mut acc, guard| { + let (key_bytes, val_bytes) = guard.into_inner().map_err(MetastoreError::Fjall)?; + let record_uri = parse_record_blobs_uri(&key_bytes) + .ok_or(MetastoreError::CorruptData("corrupt record_blobs key"))?; + let blob_cid_bytes = RecordBlobsValue::deserialize(&val_bytes) + .map(|v| v.blob_cid_bytes) + .ok_or(MetastoreError::CorruptData("corrupt record_blobs value"))?; + + blob_cid_bytes.into_iter().try_for_each( + |cid_bytes| -> Result<(), MetastoreError> { + let cid_link = bytes_to_cid_link(&cid_bytes)?; + let cid_str = cid_link.as_str().to_owned(); + if acc.contains_key(&cid_str) { + return Ok(()); + } + let key = blob_meta_key(user_hash, &cid_str); + let exists = self + .repo_data + .get(key.as_slice()) + .map_err(MetastoreError::Fjall)? + .is_some(); + if !exists { + acc.insert(cid_str, record_uri.clone()); + } + Ok(()) + }, + )?; + + Ok::<_, MetastoreError>(acc) + })?; + + let start = cursor.map_or(Bound::Unbounded, Bound::Excluded); + Ok(missing + .range::((start, Bound::Unbounded)) + .take(limit) + .map(|(cid_str, uri)| tranquil_db_traits::MissingBlobInfo { + blob_cid: CidLink::from(cid_str.clone()), + record_uri: tranquil_types::AtUri::from(uri.clone()), + }) + .collect()) + } + + fn collect_referenced_cid_bytes( + &self, + user_hash: UserHash, + ) -> Result>, MetastoreError> { + let rb_prefix = record_blobs_user_prefix(user_hash); + self.repo_data + .prefix(rb_prefix.as_slice()) + .try_fold(BTreeSet::new(), |mut acc, guard| { + let (_, val_bytes) = guard.into_inner().map_err(MetastoreError::Fjall)?; + let blob_cids = RecordBlobsValue::deserialize(&val_bytes) + .map(|v| v.blob_cid_bytes) + .ok_or(MetastoreError::CorruptData("corrupt record_blobs value"))?; + acc.extend(blob_cids); + Ok::<_, MetastoreError>(acc) + }) + } + + pub fn count_distinct_record_blobs(&self, repo_id: Uuid) -> Result { + let user_hash = self.resolve_user_hash(repo_id)?; + let distinct = self.collect_referenced_cid_bytes(user_hash)?; + Ok(i64::try_from(distinct.len()).unwrap_or(i64::MAX)) + } + + pub fn get_blobs_for_export( + &self, + repo_id: Uuid, + ) -> Result, MetastoreError> { + let user_hash = self.resolve_user_hash(repo_id)?; + let referenced_cids = self.collect_referenced_cid_bytes(user_hash)?; + + referenced_cids + .into_iter() + .filter_map(|cid_bytes| { + let cid_link = match bytes_to_cid_link(&cid_bytes) { + Ok(c) => c, + Err(e) => return Some(Err(e)), + }; + let key = blob_meta_key(user_hash, cid_link.as_str()); + match point_lookup( + &self.repo_data, + key.as_slice(), + BlobMetaValue::deserialize, + "corrupt blob_meta value", + ) { + Ok(Some(v)) => Some(Ok(tranquil_db_traits::BlobForExport { + cid: cid_link, + storage_key: v.storage_key, + mime_type: v.mime_type, + })), + Ok(None) => None, + Err(e) => Some(Err(e)), + } + }) + .collect() + } +} + +fn parse_blob_cid_from_key(key: &[u8]) -> Result { + let mut reader = KeyReader::new(key); + let tag = reader + .tag() + .ok_or(MetastoreError::CorruptData("corrupt blob key: missing tag"))?; + if tag != KeyTag::BLOBS.raw() { + return Err(MetastoreError::CorruptData( + "corrupt blob key: unexpected tag", + )); + } + reader.u64().ok_or(MetastoreError::CorruptData( + "corrupt blob key: missing user_hash", + ))?; + reader + .string() + .map(CidLink::from) + .ok_or(MetastoreError::CorruptData("corrupt blob key: missing cid")) +} + +fn parse_record_blobs_uri(key: &[u8]) -> Option { + let mut reader = KeyReader::new(key); + let tag = reader.tag()?; + if tag != KeyTag::RECORD_BLOBS.raw() { + return None; + } + reader.u64()?; + reader.string() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::metastore::{Metastore, MetastoreConfig}; + + fn open_fresh() -> (tempfile::TempDir, Metastore) { + let dir = tempfile::TempDir::new().unwrap(); + let ms = Metastore::open( + dir.path(), + MetastoreConfig { + cache_size_bytes: 64 * 1024 * 1024, + }, + ) + .unwrap(); + (dir, ms) + } + + fn setup_user(ms: &Metastore) -> (Uuid, UserHash) { + let user_id = Uuid::new_v4(); + let did = format!("did:plc:blob_test_{}", user_id); + let user_hash = UserHash::from_did(&did); + let mut batch = ms.database().batch(); + ms.user_hashes() + .stage_insert(&mut batch, user_id, user_hash) + .unwrap(); + batch.commit().unwrap(); + (user_id, user_hash) + } + + fn test_cid_link(seed: u8) -> CidLink { + let digest: [u8; 32] = std::array::from_fn(|i| seed.wrapping_add(i as u8)); + let mh = multihash::Multihash::<64>::wrap(0x12, &digest).unwrap(); + let c = cid::Cid::new_v1(0x71, mh); + CidLink::from_cid(&c) + } + + #[test] + fn insert_and_get_metadata_roundtrip() { + let (_dir, ms) = open_fresh(); + let (user_id, _) = setup_user(&ms); + let ops = ms.blob_ops(); + + let cid = test_cid_link(1); + let result = ops + .insert_blob(&cid, "image/png", 1024, user_id, "blobs/a/b") + .unwrap(); + assert_eq!(result, Some(cid.clone())); + + let meta = ops.get_blob_metadata(&cid).unwrap().unwrap(); + assert_eq!(meta.storage_key, "blobs/a/b"); + assert_eq!(meta.mime_type, "image/png"); + assert_eq!(meta.size_bytes, 1024); + } + + #[test] + fn insert_duplicate_returns_none() { + let (_dir, ms) = open_fresh(); + let (user_id, _) = setup_user(&ms); + let ops = ms.blob_ops(); + + let cid = test_cid_link(2); + assert!( + ops.insert_blob(&cid, "image/png", 100, user_id, "k1") + .unwrap() + .is_some() + ); + assert!( + ops.insert_blob(&cid, "image/png", 100, user_id, "k1") + .unwrap() + .is_none() + ); + } + + #[test] + fn insert_same_cid_different_user_returns_none() { + let (_dir, ms) = open_fresh(); + let (user_a, _) = setup_user(&ms); + let (user_b, _) = setup_user(&ms); + let ops = ms.blob_ops(); + + let cid = test_cid_link(80); + assert!( + ops.insert_blob(&cid, "image/png", 100, user_a, "ka") + .unwrap() + .is_some() + ); + assert!( + ops.insert_blob(&cid, "image/png", 100, user_b, "kb") + .unwrap() + .is_none() + ); + } + + #[test] + fn get_blob_with_takedown_no_takedown() { + let (_dir, ms) = open_fresh(); + let (user_id, _) = setup_user(&ms); + let ops = ms.blob_ops(); + + let cid = test_cid_link(3); + ops.insert_blob(&cid, "text/plain", 10, user_id, "k") + .unwrap(); + + let result = ops.get_blob_with_takedown(&cid).unwrap().unwrap(); + assert_eq!(result.cid, cid); + assert!(result.takedown_ref.is_none()); + } + + #[test] + fn update_takedown_and_read_back() { + let (_dir, ms) = open_fresh(); + let (user_id, _) = setup_user(&ms); + let ops = ms.blob_ops(); + + let cid = test_cid_link(4); + ops.insert_blob(&cid, "text/plain", 10, user_id, "k") + .unwrap(); + + assert!(ops.update_blob_takedown(&cid, Some("mod-42")).unwrap()); + let result = ops.get_blob_with_takedown(&cid).unwrap().unwrap(); + assert_eq!(result.takedown_ref.as_deref(), Some("mod-42")); + + assert!(ops.update_blob_takedown(&cid, None).unwrap()); + let result = ops.get_blob_with_takedown(&cid).unwrap().unwrap(); + assert!(result.takedown_ref.is_none()); + } + + #[test] + fn update_takedown_nonexistent_returns_false() { + let (_dir, ms) = open_fresh(); + let ops = ms.blob_ops(); + let cid = test_cid_link(99); + assert!(!ops.update_blob_takedown(&cid, Some("x")).unwrap()); + } + + #[test] + fn get_blob_storage_key() { + let (_dir, ms) = open_fresh(); + let (user_id, _) = setup_user(&ms); + let ops = ms.blob_ops(); + + let cid = test_cid_link(5); + ops.insert_blob(&cid, "image/jpeg", 500, user_id, "blobs/x/y") + .unwrap(); + + assert_eq!( + ops.get_blob_storage_key(&cid).unwrap().as_deref(), + Some("blobs/x/y") + ); + } + + #[test] + fn list_blobs_by_user_with_pagination() { + let (_dir, ms) = open_fresh(); + let (user_id, _) = setup_user(&ms); + let ops = ms.blob_ops(); + + let cids: Vec = (0..5).map(|i| test_cid_link(10 + i)).collect(); + cids.iter().enumerate().for_each(|(i, cid)| { + ops.insert_blob(cid, "image/png", i as i64, user_id, &format!("k{i}")) + .unwrap(); + }); + + let page1 = ops.list_blobs_by_user(user_id, None, 3).unwrap(); + assert_eq!(page1.len(), 3); + + let cursor = page1.last().unwrap().as_str(); + let page2 = ops.list_blobs_by_user(user_id, Some(cursor), 3).unwrap(); + assert_eq!(page2.len(), 2); + + let all = ops.list_blobs_by_user(user_id, None, 100).unwrap(); + assert_eq!(all.len(), 5); + } + + #[test] + fn count_blobs_by_user() { + let (_dir, ms) = open_fresh(); + let (user_id, _) = setup_user(&ms); + let ops = ms.blob_ops(); + + assert_eq!(ops.count_blobs_by_user(user_id).unwrap(), 0); + + (0..3).for_each(|i| { + ops.insert_blob( + &test_cid_link(20 + i), + "image/png", + 100, + user_id, + &format!("k{i}"), + ) + .unwrap(); + }); + + assert_eq!(ops.count_blobs_by_user(user_id).unwrap(), 3); + } + + #[test] + fn sum_blob_storage() { + let (_dir, ms) = open_fresh(); + let (user_id, _) = setup_user(&ms); + let ops = ms.blob_ops(); + + assert_eq!(ops.sum_blob_storage().unwrap(), 0); + + ops.insert_blob(&test_cid_link(30), "a/b", 100, user_id, "k0") + .unwrap(); + ops.insert_blob(&test_cid_link(31), "a/b", 250, user_id, "k1") + .unwrap(); + + assert_eq!(ops.sum_blob_storage().unwrap(), 350); + } + + #[test] + fn delete_blob_by_cid() { + let (_dir, ms) = open_fresh(); + let (user_id, _) = setup_user(&ms); + let ops = ms.blob_ops(); + + let cid = test_cid_link(40); + ops.insert_blob(&cid, "image/png", 100, user_id, "k") + .unwrap(); + + assert!(ops.delete_blob_by_cid(&cid).unwrap()); + assert!(ops.get_blob_metadata(&cid).unwrap().is_none()); + assert!(!ops.delete_blob_by_cid(&cid).unwrap()); + } + + #[test] + fn delete_blob_cleans_up_indexes() { + let (_dir, ms) = open_fresh(); + let (user_id, _) = setup_user(&ms); + let ops = ms.blob_ops(); + + let cid = test_cid_link(41); + ops.insert_blob(&cid, "image/png", 100, user_id, "storage/abc") + .unwrap(); + assert!(ops.get_blob_storage_key(&cid).unwrap().is_some()); + + ops.delete_blob_by_cid(&cid).unwrap(); + + assert!(ops.lookup_user_hash_by_cid(cid.as_str()).unwrap().is_none()); + } + + #[test] + fn delete_blobs_by_user() { + let (_dir, ms) = open_fresh(); + let (user_id, _) = setup_user(&ms); + let ops = ms.blob_ops(); + + (0..4).for_each(|i| { + ops.insert_blob(&test_cid_link(50 + i), "a/b", 10, user_id, &format!("k{i}")) + .unwrap(); + }); + + let deleted = ops.delete_blobs_by_user(user_id).unwrap(); + assert_eq!(deleted, 4); + assert_eq!(ops.count_blobs_by_user(user_id).unwrap(), 0); + } + + #[test] + fn delete_blobs_by_user_cleans_all_indexes() { + let (_dir, ms) = open_fresh(); + let (user_id, _) = setup_user(&ms); + let ops = ms.blob_ops(); + + let cid = test_cid_link(81); + ops.insert_blob(&cid, "a/b", 10, user_id, "storage/del_test") + .unwrap(); + + ops.delete_blobs_by_user(user_id).unwrap(); + + assert!(ops.lookup_user_hash_by_cid(cid.as_str()).unwrap().is_none()); + } + + #[test] + fn insert_blob_rejects_negative_size() { + let (_dir, ms) = open_fresh(); + let (user_id, _) = setup_user(&ms); + let ops = ms.blob_ops(); + + let result = ops.insert_blob(&test_cid_link(90), "a/b", -1, user_id, "k"); + assert!(result.is_err()); + } + + #[test] + fn get_blob_storage_keys_by_user() { + let (_dir, ms) = open_fresh(); + let (user_id, _) = setup_user(&ms); + let ops = ms.blob_ops(); + + ops.insert_blob(&test_cid_link(60), "a/b", 10, user_id, "alpha") + .unwrap(); + ops.insert_blob(&test_cid_link(61), "a/b", 10, user_id, "beta") + .unwrap(); + + let mut keys = ops.get_blob_storage_keys_by_user(user_id).unwrap(); + keys.sort(); + assert_eq!(keys, vec!["alpha", "beta"]); + } + + #[test] + fn get_metadata_for_nonexistent_returns_none() { + let (_dir, ms) = open_fresh(); + let ops = ms.blob_ops(); + assert!(ops.get_blob_metadata(&test_cid_link(99)).unwrap().is_none()); + } + + #[test] + fn blobs_isolated_between_users() { + let (_dir, ms) = open_fresh(); + let (user_a, _) = setup_user(&ms); + let (user_b, _) = setup_user(&ms); + let ops = ms.blob_ops(); + + ops.insert_blob(&test_cid_link(70), "a/b", 10, user_a, "ka") + .unwrap(); + ops.insert_blob(&test_cid_link(71), "a/b", 20, user_b, "kb") + .unwrap(); + + assert_eq!(ops.count_blobs_by_user(user_a).unwrap(), 1); + assert_eq!(ops.count_blobs_by_user(user_b).unwrap(), 1); + assert_eq!(ops.sum_blob_storage().unwrap(), 30); + } +} diff --git a/crates/tranquil-store/src/metastore/blobs.rs b/crates/tranquil-store/src/metastore/blobs.rs new file mode 100644 index 0000000..94733ee --- /dev/null +++ b/crates/tranquil-store/src/metastore/blobs.rs @@ -0,0 +1,144 @@ +use serde::{Deserialize, Serialize}; +use smallvec::SmallVec; + +use super::encoding::KeyBuilder; +use super::keys::{KeyTag, UserHash}; + +const BLOB_META_SCHEMA_VERSION: u8 = 1; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BlobMetaValue { + pub size_bytes: i64, + pub mime_type: String, + pub storage_key: String, + pub takedown_ref: Option, + pub created_at_ms: i64, +} + +impl BlobMetaValue { + pub fn serialize(&self) -> Vec { + let payload = postcard::to_allocvec(self).expect("BlobMetaValue serialization cannot fail"); + let mut buf = Vec::with_capacity(1 + payload.len()); + buf.push(BLOB_META_SCHEMA_VERSION); + buf.extend_from_slice(&payload); + buf + } + + pub fn deserialize(bytes: &[u8]) -> Option { + let (&version, payload) = bytes.split_first()?; + match version { + BLOB_META_SCHEMA_VERSION => postcard::from_bytes(payload).ok(), + _ => None, + } + } +} + +pub fn blob_meta_key(user_hash: UserHash, cid_str: &str) -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::BLOBS) + .u64(user_hash.raw()) + .string(cid_str) + .build() +} + +pub fn blob_user_prefix(user_hash: UserHash) -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::BLOBS) + .u64(user_hash.raw()) + .build() +} + +pub fn blobs_prefix() -> SmallVec<[u8; 128]> { + KeyBuilder::new().tag(KeyTag::BLOBS).build() +} + +pub fn blob_by_cid_key(cid_str: &str) -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::BLOB_BY_CID) + .string(cid_str) + .build() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::metastore::encoding::KeyReader; + + #[test] + fn blob_meta_value_roundtrip() { + let val = BlobMetaValue { + size_bytes: 1024, + mime_type: "image/png".to_owned(), + storage_key: "blobs/abc/def".to_owned(), + takedown_ref: None, + created_at_ms: 1700000000000, + }; + let bytes = val.serialize(); + let decoded = BlobMetaValue::deserialize(&bytes).unwrap(); + assert_eq!(val, decoded); + } + + #[test] + fn blob_meta_value_with_takedown_roundtrip() { + let val = BlobMetaValue { + size_bytes: 42, + mime_type: "text/plain".to_owned(), + storage_key: "k".to_owned(), + takedown_ref: Some("mod-123".to_owned()), + created_at_ms: 0, + }; + let bytes = val.serialize(); + let decoded = BlobMetaValue::deserialize(&bytes).unwrap(); + assert_eq!(val, decoded); + } + + #[test] + fn blob_meta_key_roundtrip() { + let uh = UserHash::from_did("did:plc:test"); + let key = blob_meta_key(uh, "bafyreiabc"); + let mut reader = KeyReader::new(&key); + assert_eq!(reader.tag(), Some(KeyTag::BLOBS.raw())); + assert_eq!(reader.u64(), Some(uh.raw())); + assert_eq!(reader.string(), Some("bafyreiabc".to_owned())); + assert!(reader.is_empty()); + } + + #[test] + fn blob_user_prefix_is_prefix_of_key() { + let uh = UserHash::from_did("did:plc:test"); + let prefix = blob_user_prefix(uh); + let key = blob_meta_key(uh, "bafyreiabc"); + assert!(key.starts_with(prefix.as_slice())); + } + + #[test] + fn blob_by_cid_key_roundtrip() { + let key = blob_by_cid_key("bafyreiabc"); + let mut reader = KeyReader::new(&key); + assert_eq!(reader.tag(), Some(KeyTag::BLOB_BY_CID.raw())); + assert_eq!(reader.string(), Some("bafyreiabc".to_owned())); + assert!(reader.is_empty()); + } + + #[test] + fn blob_meta_key_ordering_by_cid() { + let uh = UserHash::from_did("did:plc:test"); + let key_a = blob_meta_key(uh, "aaa"); + let key_b = blob_meta_key(uh, "bbb"); + assert!(key_a.as_slice() < key_b.as_slice()); + } + + #[test] + fn deserialize_unknown_version_returns_none() { + let val = BlobMetaValue { + size_bytes: 0, + mime_type: String::new(), + storage_key: String::new(), + takedown_ref: None, + created_at_ms: 0, + }; + let mut bytes = val.serialize(); + bytes[0] = 99; + assert!(BlobMetaValue::deserialize(&bytes).is_none()); + } +} diff --git a/crates/tranquil-store/src/metastore/client.rs b/crates/tranquil-store/src/metastore/client.rs new file mode 100644 index 0000000..c398a54 --- /dev/null +++ b/crates/tranquil-store/src/metastore/client.rs @@ -0,0 +1,1023 @@ +use std::marker::PhantomData; +use std::sync::Arc; + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use tokio::sync::oneshot; +use tranquil_db_traits::{ + AccountStatus, ApplyCommitError, ApplyCommitInput, ApplyCommitResult, Backlink, + BrokenGenesisCommit, CommitEventData, DbError, EventBlocksCids, ImportBlock, ImportRecord, + ImportRepoError, RepoAccountInfo, RepoInfo, RepoListItem, RepoWithoutRev, SequenceNumber, + SequencedEvent, UserNeedingRecordBlobsBackfill, UserWithoutBlocks, +}; +use tranquil_types::{AtUri, CidLink, Did, Handle, Nsid, Rkey}; +use uuid::Uuid; + +use super::handler::{ + BacklinkRequest, BlobRequest, CommitRequest, EventRequest, HandlerPool, MetastoreRequest, + RecordRequest, RepoRequest, UserBlockRequest, +}; +use super::keys::UserHash; +use crate::io::StorageIO; + +async fn recv(rx: oneshot::Receiver>) -> Result { + rx.await + .map_err(|_| DbError::Connection("metastore handler thread closed".to_string()))? +} + +async fn recv_commit( + rx: oneshot::Receiver>, +) -> Result { + rx.await + .map_err(|_| ApplyCommitError::Database("metastore handler thread closed".to_string()))? +} + +async fn recv_import( + rx: oneshot::Receiver>, +) -> Result<(), ImportRepoError> { + rx.await + .map_err(|_| ImportRepoError::Database("metastore handler thread closed".to_string()))? +} + +pub struct MetastoreClient { + pool: Arc, + _phantom: PhantomData, +} + +impl Clone for MetastoreClient { + fn clone(&self) -> Self { + Self { + pool: Arc::clone(&self.pool), + _phantom: PhantomData, + } + } +} + +impl MetastoreClient { + pub fn new(pool: Arc) -> Self { + Self { + pool, + _phantom: PhantomData, + } + } + + pub fn pool(&self) -> &Arc { + &self.pool + } + + pub async fn create_repo_full( + &self, + user_id: Uuid, + did: &Did, + handle: &Handle, + repo_root_cid: &CidLink, + repo_rev: &str, + ) -> Result<(), DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Repo(RepoRequest::CreateRepoFull { + user_id, + did: did.clone(), + handle: handle.clone(), + repo_root_cid: repo_root_cid.clone(), + repo_rev: repo_rev.to_string(), + tx, + }))?; + recv(rx).await + } +} + +#[async_trait] +impl tranquil_db_traits::RepoRepository for MetastoreClient { + async fn create_repo( + &self, + user_id: Uuid, + did: &Did, + handle: &Handle, + repo_root_cid: &CidLink, + repo_rev: &str, + ) -> Result<(), DbError> { + self.create_repo_full(user_id, did, handle, repo_root_cid, repo_rev) + .await + } + + async fn update_repo_status( + &self, + did: &Did, + takedown: Option, + takedown_ref: Option<&str>, + deactivated: Option, + ) -> Result<(), DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Repo(RepoRequest::UpdateRepoStatus { + did: did.clone(), + takedown, + takedown_ref: takedown_ref.map(str::to_owned), + deactivated, + tx, + }))?; + recv(rx).await + } + + async fn update_repo_root( + &self, + user_id: Uuid, + repo_root_cid: &CidLink, + repo_rev: &str, + ) -> Result<(), DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Repo(RepoRequest::UpdateRepoRoot { + user_id, + repo_root_cid: repo_root_cid.clone(), + repo_rev: repo_rev.to_string(), + tx, + }))?; + recv(rx).await + } + + async fn update_repo_rev(&self, user_id: Uuid, repo_rev: &str) -> Result<(), DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Repo(RepoRequest::UpdateRepoRev { + user_id, + repo_rev: repo_rev.to_string(), + tx, + }))?; + recv(rx).await + } + + async fn delete_repo(&self, user_id: Uuid) -> Result<(), DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Repo(RepoRequest::DeleteRepo { + user_id, + tx, + }))?; + recv(rx).await + } + + async fn get_repo_root_for_update(&self, user_id: Uuid) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Repo(RepoRequest::GetRepoRootForUpdate { + user_id, + tx, + }))?; + recv(rx).await + } + + async fn get_repo(&self, user_id: Uuid) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Repo(RepoRequest::GetRepo { user_id, tx }))?; + recv(rx).await + } + + async fn get_repo_root_by_did(&self, did: &Did) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Repo(RepoRequest::GetRepoRootByDid { + did: did.clone(), + tx, + }))?; + recv(rx).await + } + + async fn count_repos(&self) -> Result { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Repo(RepoRequest::CountRepos { tx }))?; + recv(rx).await + } + + async fn get_repos_without_rev(&self) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Repo(RepoRequest::GetReposWithoutRev { + tx, + }))?; + recv(rx).await + } + + async fn upsert_records( + &self, + repo_id: Uuid, + collections: &[Nsid], + rkeys: &[Rkey], + record_cids: &[CidLink], + repo_rev: &str, + ) -> Result<(), DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Record(RecordRequest::UpsertRecords { + repo_id, + collections: collections.to_vec(), + rkeys: rkeys.to_vec(), + record_cids: record_cids.to_vec(), + repo_rev: repo_rev.to_string(), + tx, + }))?; + recv(rx).await + } + + async fn delete_records( + &self, + repo_id: Uuid, + collections: &[Nsid], + rkeys: &[Rkey], + ) -> Result<(), DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Record(RecordRequest::DeleteRecords { + repo_id, + collections: collections.to_vec(), + rkeys: rkeys.to_vec(), + tx, + }))?; + recv(rx).await + } + + async fn delete_all_records(&self, repo_id: Uuid) -> Result<(), DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Record(RecordRequest::DeleteAllRecords { + repo_id, + tx, + }))?; + recv(rx).await + } + + async fn get_record_cid( + &self, + repo_id: Uuid, + collection: &Nsid, + rkey: &Rkey, + ) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Record(RecordRequest::GetRecordCid { + repo_id, + collection: collection.clone(), + rkey: rkey.clone(), + tx, + }))?; + recv(rx).await + } + + async fn list_records( + &self, + repo_id: Uuid, + collection: &Nsid, + cursor: Option<&Rkey>, + limit: i64, + reverse: bool, + rkey_start: Option<&Rkey>, + rkey_end: Option<&Rkey>, + ) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Record(RecordRequest::ListRecords { + repo_id, + collection: collection.clone(), + cursor: cursor.cloned(), + limit, + reverse, + rkey_start: rkey_start.cloned(), + rkey_end: rkey_end.cloned(), + tx, + }))?; + recv(rx).await + } + + async fn get_all_records( + &self, + repo_id: Uuid, + ) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Record(RecordRequest::GetAllRecords { + repo_id, + tx, + }))?; + recv(rx).await + } + + async fn list_collections(&self, repo_id: Uuid) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Record(RecordRequest::ListCollections { + repo_id, + tx, + }))?; + recv(rx).await + } + + async fn count_records(&self, repo_id: Uuid) -> Result { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Record(RecordRequest::CountRecords { + repo_id, + tx, + }))?; + recv(rx).await + } + + async fn count_all_records(&self) -> Result { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Record(RecordRequest::CountAllRecords { + tx, + }))?; + recv(rx).await + } + + async fn get_record_by_cid( + &self, + cid: &CidLink, + ) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Record(RecordRequest::GetRecordByCid { + cid: cid.clone(), + tx, + }))?; + recv(rx).await + } + + async fn set_record_takedown( + &self, + cid: &CidLink, + takedown_ref: Option<&str>, + ) -> Result<(), DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Record(RecordRequest::SetRecordTakedown { + cid: cid.clone(), + takedown_ref: takedown_ref.map(str::to_owned), + scope_user: None, + tx, + }))?; + recv(rx).await + } + + async fn insert_user_blocks( + &self, + user_id: Uuid, + block_cids: &[Vec], + repo_rev: &str, + ) -> Result<(), DbError> { + let (tx, rx) = oneshot::channel(); + self.pool.send(MetastoreRequest::UserBlock( + UserBlockRequest::InsertUserBlocks { + user_id, + block_cids: block_cids.to_vec(), + repo_rev: repo_rev.to_string(), + tx, + }, + ))?; + recv(rx).await + } + + async fn delete_user_blocks( + &self, + user_id: Uuid, + block_cids: &[Vec], + ) -> Result<(), DbError> { + let (tx, rx) = oneshot::channel(); + self.pool.send(MetastoreRequest::UserBlock( + UserBlockRequest::DeleteUserBlocks { + user_id, + block_cids: block_cids.to_vec(), + tx, + }, + ))?; + recv(rx).await + } + + async fn get_user_block_cids_since_rev( + &self, + user_id: Uuid, + since_rev: &str, + ) -> Result>, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool.send(MetastoreRequest::UserBlock( + UserBlockRequest::GetUserBlockCidsSinceRev { + user_id, + since_rev: since_rev.to_string(), + tx, + }, + ))?; + recv(rx).await + } + + async fn count_user_blocks(&self, user_id: Uuid) -> Result { + let (tx, rx) = oneshot::channel(); + self.pool.send(MetastoreRequest::UserBlock( + UserBlockRequest::CountUserBlocks { user_id, tx }, + ))?; + recv(rx).await + } + + async fn find_unreferenced_blocks( + &self, + candidate_cids: &[Vec], + ) -> Result>, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool.send(MetastoreRequest::UserBlock( + UserBlockRequest::FindUnreferencedBlocks { + candidate_cids: candidate_cids.to_vec(), + tx, + }, + ))?; + recv(rx).await + } + + async fn insert_commit_event(&self, data: &CommitEventData) -> Result { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Event(EventRequest::InsertCommitEvent { + data: data.clone(), + tx, + }))?; + recv(rx).await + } + + async fn insert_identity_event( + &self, + did: &Did, + handle: Option<&Handle>, + ) -> Result { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Event(EventRequest::InsertIdentityEvent { + did: did.clone(), + handle: handle.cloned(), + tx, + }))?; + recv(rx).await + } + + async fn insert_account_event( + &self, + did: &Did, + status: AccountStatus, + ) -> Result { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Event(EventRequest::InsertAccountEvent { + did: did.clone(), + status, + tx, + }))?; + recv(rx).await + } + + async fn insert_sync_event( + &self, + did: &Did, + commit_cid: &CidLink, + rev: Option<&str>, + ) -> Result { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Event(EventRequest::InsertSyncEvent { + did: did.clone(), + commit_cid: commit_cid.clone(), + rev: rev.map(str::to_owned), + tx, + }))?; + recv(rx).await + } + + async fn insert_genesis_commit_event( + &self, + did: &Did, + commit_cid: &CidLink, + mst_root_cid: &CidLink, + rev: &str, + ) -> Result { + let (tx, rx) = oneshot::channel(); + self.pool.send(MetastoreRequest::Event( + EventRequest::InsertGenesisCommitEvent { + did: did.clone(), + commit_cid: commit_cid.clone(), + mst_root_cid: mst_root_cid.clone(), + rev: rev.to_string(), + tx, + }, + ))?; + recv(rx).await + } + + async fn update_seq_blocks_cids( + &self, + seq: SequenceNumber, + blocks_cids: &[String], + ) -> Result<(), DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Event(EventRequest::UpdateSeqBlocksCids { + seq, + blocks_cids: blocks_cids.to_vec(), + tx, + }))?; + recv(rx).await + } + + async fn delete_sequences_except( + &self, + did: &Did, + keep_seq: SequenceNumber, + ) -> Result<(), DbError> { + let (tx, rx) = oneshot::channel(); + self.pool.send(MetastoreRequest::Event( + EventRequest::DeleteSequencesExcept { + did: did.clone(), + keep_seq, + tx, + }, + ))?; + recv(rx).await + } + + async fn get_max_seq(&self) -> Result { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Event(EventRequest::GetMaxSeq { tx }))?; + recv(rx).await + } + + async fn get_min_seq_since( + &self, + since: DateTime, + ) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Event(EventRequest::GetMinSeqSince { + since, + tx, + }))?; + recv(rx).await + } + + async fn get_account_with_repo(&self, did: &Did) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Repo(RepoRequest::GetAccountWithRepo { + did: did.clone(), + tx, + }))?; + recv(rx).await + } + + async fn get_events_since_seq( + &self, + since_seq: SequenceNumber, + limit: Option, + ) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Event(EventRequest::GetEventsSinceSeq { + since_seq, + limit, + tx, + }))?; + recv(rx).await + } + + async fn get_events_in_seq_range( + &self, + start_seq: SequenceNumber, + end_seq: SequenceNumber, + ) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Event(EventRequest::GetEventsInSeqRange { + start_seq, + end_seq, + tx, + }))?; + recv(rx).await + } + + async fn get_event_by_seq( + &self, + seq: SequenceNumber, + ) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Event(EventRequest::GetEventBySeq { + seq, + tx, + }))?; + recv(rx).await + } + + async fn get_events_since_cursor( + &self, + cursor: SequenceNumber, + limit: i64, + ) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool.send(MetastoreRequest::Event( + EventRequest::GetEventsSinceCursor { cursor, limit, tx }, + ))?; + recv(rx).await + } + + async fn get_events_since_rev( + &self, + did: &Did, + since_rev: &str, + ) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Event(EventRequest::GetEventsSinceRev { + did: did.clone(), + since_rev: since_rev.to_string(), + tx, + }))?; + recv(rx).await + } + + async fn list_repos_paginated( + &self, + cursor_did: Option<&Did>, + limit: i64, + ) -> Result, DbError> { + let cursor_hash = cursor_did.map(|d| UserHash::from_did(d.as_str()).raw()); + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Repo(RepoRequest::ListReposPaginated { + cursor_user_hash: cursor_hash, + limit: usize::try_from(limit).unwrap_or(usize::MAX), + tx, + }))?; + recv(rx).await + } + + async fn get_repo_root_cid_by_user_id( + &self, + user_id: Uuid, + ) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool.send(MetastoreRequest::Repo( + RepoRequest::GetRepoRootCidByUserId { user_id, tx }, + ))?; + recv(rx).await + } + + async fn notify_update(&self, seq: SequenceNumber) -> Result<(), DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Event(EventRequest::NotifyUpdate { + seq, + tx, + }))?; + recv(rx).await + } + + async fn import_repo_data( + &self, + user_id: Uuid, + blocks: &[ImportBlock], + records: &[ImportRecord], + expected_root_cid: Option<&CidLink>, + ) -> Result<(), ImportRepoError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Commit(Box::new( + CommitRequest::ImportRepoData { + user_id, + blocks: blocks.to_vec(), + records: records.to_vec(), + expected_root_cid: expected_root_cid.cloned(), + tx, + }, + ))) + .map_err(|e| ImportRepoError::Database(e.to_string()))?; + recv_import(rx).await + } + + async fn apply_commit( + &self, + input: ApplyCommitInput, + ) -> Result { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Commit(Box::new( + CommitRequest::ApplyCommit { + input: Box::new(input), + tx, + }, + ))) + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; + recv_commit(rx).await + } + + async fn get_broken_genesis_commits(&self) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool.send(MetastoreRequest::Commit(Box::new( + CommitRequest::GetBrokenGenesisCommits { tx }, + )))?; + recv(rx).await + } + + async fn get_users_without_blocks(&self) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool.send(MetastoreRequest::Commit(Box::new( + CommitRequest::GetUsersWithoutBlocks { tx }, + )))?; + recv(rx).await + } + + async fn get_users_needing_record_blobs_backfill( + &self, + limit: i64, + ) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool.send(MetastoreRequest::Commit(Box::new( + CommitRequest::GetUsersNeedingRecordBlobsBackfill { limit, tx }, + )))?; + recv(rx).await + } + + async fn insert_record_blobs( + &self, + repo_id: Uuid, + record_uris: &[AtUri], + blob_cids: &[CidLink], + ) -> Result<(), DbError> { + let (tx, rx) = oneshot::channel(); + self.pool.send(MetastoreRequest::Commit(Box::new( + CommitRequest::InsertRecordBlobs { + repo_id, + record_uris: record_uris.to_vec(), + blob_cids: blob_cids.to_vec(), + tx, + }, + )))?; + recv(rx).await + } +} + +#[async_trait] +impl tranquil_db_traits::BacklinkRepository for MetastoreClient { + async fn get_backlink_conflicts( + &self, + repo_id: Uuid, + collection: &Nsid, + backlinks: &[Backlink], + ) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool.send(MetastoreRequest::Backlink( + BacklinkRequest::GetBacklinkConflicts { + repo_id, + collection: collection.clone(), + backlinks: backlinks.to_vec(), + tx, + }, + ))?; + recv(rx).await + } + + async fn add_backlinks(&self, repo_id: Uuid, backlinks: &[Backlink]) -> Result<(), DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Backlink(BacklinkRequest::AddBacklinks { + repo_id, + backlinks: backlinks.to_vec(), + tx, + }))?; + recv(rx).await + } + + async fn remove_backlinks_by_uri(&self, uri: &AtUri) -> Result<(), DbError> { + let (tx, rx) = oneshot::channel(); + self.pool.send(MetastoreRequest::Backlink( + BacklinkRequest::RemoveBacklinksByUri { + uri: uri.clone(), + tx, + }, + ))?; + recv(rx).await + } + + async fn remove_backlinks_by_repo(&self, repo_id: Uuid) -> Result<(), DbError> { + let (tx, rx) = oneshot::channel(); + self.pool.send(MetastoreRequest::Backlink( + BacklinkRequest::RemoveBacklinksByRepo { repo_id, tx }, + ))?; + recv(rx).await + } +} + +#[async_trait] +impl tranquil_db_traits::BlobRepository for MetastoreClient { + async fn insert_blob( + &self, + cid: &CidLink, + mime_type: &str, + size_bytes: i64, + created_by_user: Uuid, + storage_key: &str, + ) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Blob(BlobRequest::InsertBlob { + cid: cid.clone(), + mime_type: mime_type.to_owned(), + size_bytes, + created_by_user, + storage_key: storage_key.to_owned(), + tx, + }))?; + recv(rx).await + } + + async fn get_blob_metadata( + &self, + cid: &CidLink, + ) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Blob(BlobRequest::GetBlobMetadata { + cid: cid.clone(), + tx, + }))?; + recv(rx).await + } + + async fn get_blob_with_takedown( + &self, + cid: &CidLink, + ) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Blob(BlobRequest::GetBlobWithTakedown { + cid: cid.clone(), + tx, + }))?; + recv(rx).await + } + + async fn get_blob_storage_key(&self, cid: &CidLink) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Blob(BlobRequest::GetBlobStorageKey { + cid: cid.clone(), + tx, + }))?; + recv(rx).await + } + + async fn list_blobs_by_user( + &self, + user_id: Uuid, + cursor: Option<&str>, + limit: i64, + ) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Blob(BlobRequest::ListBlobsByUser { + user_id, + cursor: cursor.map(str::to_owned), + limit, + tx, + }))?; + recv(rx).await + } + + async fn list_blobs_since_rev( + &self, + did: &tranquil_types::Did, + since: &str, + ) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Blob(BlobRequest::ListBlobsSinceRev { + did: did.clone(), + since: since.to_owned(), + tx, + }))?; + recv(rx).await + } + + async fn count_blobs_by_user(&self, user_id: Uuid) -> Result { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Blob(BlobRequest::CountBlobsByUser { + user_id, + tx, + }))?; + recv(rx).await + } + + async fn sum_blob_storage(&self) -> Result { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Blob(BlobRequest::SumBlobStorage { tx }))?; + recv(rx).await + } + + async fn update_blob_takedown( + &self, + cid: &CidLink, + takedown_ref: Option<&str>, + ) -> Result { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Blob(BlobRequest::UpdateBlobTakedown { + cid: cid.clone(), + takedown_ref: takedown_ref.map(str::to_owned), + tx, + }))?; + recv(rx).await + } + + async fn delete_blob_by_cid(&self, cid: &CidLink) -> Result { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Blob(BlobRequest::DeleteBlobByCid { + cid: cid.clone(), + tx, + }))?; + recv(rx).await + } + + async fn delete_blobs_by_user(&self, user_id: Uuid) -> Result { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Blob(BlobRequest::DeleteBlobsByUser { + user_id, + tx, + }))?; + recv(rx).await + } + + async fn get_blob_storage_keys_by_user(&self, user_id: Uuid) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool.send(MetastoreRequest::Blob( + BlobRequest::GetBlobStorageKeysByUser { user_id, tx }, + ))?; + recv(rx).await + } + + async fn insert_record_blobs( + &self, + repo_id: Uuid, + record_uris: &[AtUri], + blob_cids: &[CidLink], + ) -> Result<(), DbError> { + let (tx, rx) = oneshot::channel(); + self.pool.send(MetastoreRequest::Commit(Box::new( + CommitRequest::InsertRecordBlobs { + repo_id, + record_uris: record_uris.to_vec(), + blob_cids: blob_cids.to_vec(), + tx, + }, + )))?; + recv(rx).await + } + + async fn list_missing_blobs( + &self, + repo_id: Uuid, + cursor: Option<&str>, + limit: i64, + ) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Blob(BlobRequest::ListMissingBlobs { + repo_id, + cursor: cursor.map(str::to_owned), + limit, + tx, + }))?; + recv(rx).await + } + + async fn count_distinct_record_blobs(&self, repo_id: Uuid) -> Result { + let (tx, rx) = oneshot::channel(); + self.pool.send(MetastoreRequest::Blob( + BlobRequest::CountDistinctRecordBlobs { repo_id, tx }, + ))?; + recv(rx).await + } + + async fn get_blobs_for_export( + &self, + repo_id: Uuid, + ) -> Result, DbError> { + let (tx, rx) = oneshot::channel(); + self.pool + .send(MetastoreRequest::Blob(BlobRequest::GetBlobsForExport { + repo_id, + tx, + }))?; + recv(rx).await + } +} diff --git a/crates/tranquil-store/src/metastore/commit_ops.rs b/crates/tranquil-store/src/metastore/commit_ops.rs new file mode 100644 index 0000000..5bcd64b --- /dev/null +++ b/crates/tranquil-store/src/metastore/commit_ops.rs @@ -0,0 +1,1504 @@ +use std::sync::Arc; + +use fjall::{Database, Keyspace}; +use smallvec::SmallVec; +use uuid::Uuid; + +use super::MetastoreError; +use super::backlink_ops::BacklinkOps; +use super::backlinks::path_to_discriminant; +use super::encoding::KeyBuilder; +use super::event_ops::EventOps; +use super::keys::{KeyTag, UserHash}; +use super::record_ops::{RecordDelete, RecordOps, RecordWrite}; +use super::recovery::{ + BacklinkMutation, CommitMutationSet, RecordMutationDelete, RecordMutationUpsert, +}; +use super::repo_meta::{RepoMetaValue, repo_meta_key, repo_meta_prefix}; +use super::repo_ops::{RepoOps, bytes_to_cid_link, cid_link_to_bytes}; +use super::user_block_ops::UserBlockOps; +use super::user_blocks::user_block_user_prefix; +use super::user_hash::UserHashMap; +use crate::blockstore::TranquilBlockStore; +use crate::eventlog::EventLogBridge; +use crate::io::StorageIO; + +use tranquil_db_traits::{ + ApplyCommitError, ApplyCommitInput, ApplyCommitResult, BrokenGenesisCommit, ImportBlock, + ImportRecord, ImportRepoError, RepoEventType, SequenceNumber, UserNeedingRecordBlobsBackfill, + UserWithoutBlocks, +}; +use tranquil_types::{AtUri, CidLink, Did}; + +use serde::{Deserialize, Serialize}; + +pub(crate) const RECORD_BLOBS_SCHEMA_VERSION: u8 = 1; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct RecordBlobsValue { + pub(crate) blob_cid_bytes: Vec>, +} + +impl RecordBlobsValue { + pub(crate) fn serialize(&self) -> Vec { + let payload = + postcard::to_allocvec(self).expect("RecordBlobsValue serialization cannot fail"); + let mut buf = Vec::with_capacity(1 + payload.len()); + buf.push(RECORD_BLOBS_SCHEMA_VERSION); + buf.extend_from_slice(&payload); + buf + } + + pub(crate) fn deserialize(bytes: &[u8]) -> Option { + let (&version, payload) = bytes.split_first()?; + match version { + RECORD_BLOBS_SCHEMA_VERSION => postcard::from_bytes(payload).ok(), + _ => None, + } + } +} + +fn record_blobs_key(user_hash: UserHash, uri: &AtUri) -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::RECORD_BLOBS) + .u64(user_hash.raw()) + .string(uri.as_str()) + .build() +} + +pub(crate) fn record_blobs_user_prefix(user_hash: UserHash) -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::RECORD_BLOBS) + .u64(user_hash.raw()) + .build() +} + +pub struct CommitOps { + db: Database, + repo_data: Keyspace, + user_hashes: Arc, + repo_ops: RepoOps, + record_ops: RecordOps, + user_block_ops: UserBlockOps, + backlink_ops: BacklinkOps, + event_ops: EventOps, + blockstore: Option, +} + +impl CommitOps { + pub fn new( + db: Database, + repo_data: Keyspace, + indexes: Keyspace, + user_hashes: Arc, + bridge: Arc>, + ) -> Self { + let repo_ops = RepoOps::new(repo_data.clone(), Arc::clone(&user_hashes)); + let record_ops = RecordOps::new(repo_data.clone(), Arc::clone(&user_hashes)); + let user_block_ops = UserBlockOps::new(repo_data.clone(), Arc::clone(&user_hashes)); + let backlink_ops = BacklinkOps::new(indexes, Arc::clone(&user_hashes)); + let event_ops = EventOps::new(db.clone(), repo_data.clone(), bridge); + Self { + db, + repo_data, + user_hashes, + repo_ops, + record_ops, + user_block_ops, + backlink_ops, + event_ops, + blockstore: None, + } + } + + pub fn with_blockstore(mut self, blockstore: TranquilBlockStore) -> Self { + self.blockstore = Some(blockstore); + self + } + + pub fn apply_commit( + &self, + input: ApplyCommitInput, + ) -> Result { + let user_hash = self + .user_hashes + .get(&input.user_id) + .ok_or(ApplyCommitError::RepoNotFound)?; + + let key = repo_meta_key(user_hash); + let meta = self + .repo_data + .get(key.as_slice()) + .map_err(|e| ApplyCommitError::Database(e.to_string()))? + .and_then(|raw| RepoMetaValue::deserialize(&raw)) + .ok_or(ApplyCommitError::RepoNotFound)?; + + if let Some(expected) = &input.expected_root_cid { + let current = bytes_to_cid_link(&meta.repo_root_cid) + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; + if current != *expected { + return Err(ApplyCommitError::ConcurrentModification); + } + } + + let new_cid_bytes = cid_link_to_bytes(&input.new_root_cid) + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; + + let is_active = meta.status.is_active(); + + let updated_meta = RepoMetaValue { + repo_root_cid: new_cid_bytes.clone(), + repo_rev: input.new_rev.clone(), + ..meta + }; + + let mut batch = self.db.batch(); + + self.repo_ops + .write_repo_meta(&mut batch, user_hash, &updated_meta); + + let upserts: Vec> = input + .record_upserts + .iter() + .map(|u| RecordWrite { + collection: &u.collection, + rkey: &u.rkey, + cid: &u.cid, + }) + .collect(); + + self.record_ops + .upsert_records(&mut batch, user_hash, &upserts) + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; + + let deletes: Vec> = input + .record_deletes + .iter() + .map(|d| RecordDelete { + collection: &d.collection, + rkey: &d.rkey, + }) + .collect(); + + self.record_ops + .delete_records(&mut batch, user_hash, &deletes); + + self.user_block_ops + .insert_user_blocks(&mut batch, user_hash, &input.new_block_cids, &input.new_rev) + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; + + self.user_block_ops + .delete_user_blocks_by_cid(&mut batch, user_hash, &input.obsolete_block_cids) + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; + + input.backlinks_to_remove.iter().try_for_each(|uri| { + self.backlink_ops + .remove_backlinks_by_uri(&mut batch, user_hash, uri) + .map_err(|e| ApplyCommitError::Database(e.to_string())) + })?; + + self.backlink_ops + .add_backlinks(&mut batch, user_hash, &input.backlinks_to_add) + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; + + let mutation_set = CommitMutationSet { + new_root_cid: new_cid_bytes.clone(), + new_rev: input.new_rev.clone(), + record_upserts: input + .record_upserts + .iter() + .map(|u| { + let cid_bytes = cid_link_to_bytes(&u.cid) + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; + Ok(RecordMutationUpsert { + collection: u.collection.as_str().to_owned(), + rkey: u.rkey.as_str().to_owned(), + cid_bytes, + }) + }) + .collect::, ApplyCommitError>>()?, + record_deletes: input + .record_deletes + .iter() + .map(|d| RecordMutationDelete { + collection: d.collection.as_str().to_owned(), + rkey: d.rkey.as_str().to_owned(), + }) + .collect(), + block_inserts: input.new_block_cids.clone(), + block_deletes: input.obsolete_block_cids.clone(), + backlink_adds: input + .backlinks_to_add + .iter() + .map(|bl| BacklinkMutation { + uri: bl.uri.as_str().to_owned(), + path: path_to_discriminant(bl.path), + link_to: bl.link_to.clone(), + }) + .collect(), + backlink_remove_uris: input + .backlinks_to_remove + .iter() + .map(|uri| uri.as_str().to_owned()) + .collect(), + }; + let mutation_set_bytes = mutation_set + .serialize() + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; + + let (seq, deferred) = self + .event_ops + .append_commit_event_into_batch( + &mut batch, + &input.commit_event, + Some(&mutation_set_bytes), + ) + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; + + batch + .commit() + .map_err(|e| ApplyCommitError::Database(e.to_string()))?; + + self.event_ops.complete_broadcast(deferred); + + Ok(ApplyCommitResult { + seq: seq.as_i64(), + is_account_active: is_active, + }) + } + + pub fn import_repo_data( + &self, + user_id: Uuid, + blocks: &[ImportBlock], + records: &[ImportRecord], + expected_root_cid: Option<&CidLink>, + ) -> Result<(), ImportRepoError> { + let user_hash = self + .user_hashes + .get(&user_id) + .ok_or(ImportRepoError::RepoNotFound)?; + + let key = repo_meta_key(user_hash); + let meta = self + .repo_data + .get(key.as_slice()) + .map_err(|e| ImportRepoError::Database(e.to_string()))? + .and_then(|raw| RepoMetaValue::deserialize(&raw)) + .ok_or(ImportRepoError::RepoNotFound)?; + + if let Some(expected) = expected_root_cid { + let current = bytes_to_cid_link(&meta.repo_root_cid) + .map_err(|e| ImportRepoError::Database(e.to_string()))?; + if current != *expected { + return Err(ImportRepoError::ConcurrentModification); + } + } + + if let Some(bs) = &self.blockstore + && !blocks.is_empty() + { + let block_pairs: Vec<([u8; 36], Vec)> = blocks + .iter() + .map(|b| { + let cid: [u8; 36] = b.cid_bytes.as_slice().try_into().map_err(|_| { + ImportRepoError::Database(format!( + "block CID has invalid length: {} (expected 36)", + b.cid_bytes.len() + )) + })?; + Ok((cid, b.data.clone())) + }) + .collect::, ImportRepoError>>()?; + bs.put_blocks_blocking(block_pairs) + .map_err(|e| ImportRepoError::Database(e.to_string()))?; + } + + let mut batch = self.db.batch(); + + let upserts: Vec> = records + .iter() + .map(|r| RecordWrite { + collection: &r.collection, + rkey: &r.rkey, + cid: &r.record_cid, + }) + .collect(); + + self.record_ops + .upsert_records(&mut batch, user_hash, &upserts) + .map_err(|e| ImportRepoError::Database(e.to_string()))?; + + batch + .commit() + .map_err(|e| ImportRepoError::Database(e.to_string())) + } + + pub fn insert_record_blobs( + &self, + repo_id: Uuid, + record_uris: &[AtUri], + blob_cids: &[CidLink], + ) -> Result<(), MetastoreError> { + let user_hash = self + .user_hashes + .get(&repo_id) + .ok_or(MetastoreError::InvalidInput("unknown user_id"))?; + + let blob_bytes: Vec> = blob_cids + .iter() + .map(cid_link_to_bytes) + .collect::>()?; + + let serialized = RecordBlobsValue { + blob_cid_bytes: blob_bytes, + } + .serialize(); + + let mut batch = self.db.batch(); + record_uris.iter().for_each(|uri| { + let key = record_blobs_key(user_hash, uri); + batch.insert(&self.repo_data, key.as_slice(), serialized.as_slice()); + }); + batch.commit().map_err(MetastoreError::Fjall) + } + + pub fn get_users_needing_record_blobs_backfill( + &self, + limit: i64, + ) -> Result, MetastoreError> { + let limit_usize = usize::try_from(limit).unwrap_or(usize::MAX); + + self.scan_users_missing_prefix( + record_blobs_user_prefix, + |meta, user_id| { + let did = meta + .did + .map(Did::from) + .ok_or(MetastoreError::CorruptData("repo_meta missing did field"))?; + Ok(UserNeedingRecordBlobsBackfill { user_id, did }) + }, + limit_usize, + ) + } + + pub fn get_broken_genesis_commits(&self) -> Result, MetastoreError> { + const PAGE_SIZE: usize = 4096; + self.collect_broken_genesis_page(SequenceNumber::ZERO, Vec::new(), PAGE_SIZE) + } + + fn collect_broken_genesis_page( + &self, + cursor: SequenceNumber, + acc: Vec, + page_size: usize, + ) -> Result, MetastoreError> { + let limit = i64::try_from(page_size).unwrap_or(i64::MAX); + let events = self + .event_ops + .get_events_since_seq(cursor, Some(limit)) + .map_err(|_| MetastoreError::CorruptData("failed to read events"))?; + + let page_len = events.len(); + let page_high_seq = events.last().map(|e| e.seq).unwrap_or(cursor); + + let results = events.into_iter().fold(acc, |mut results, e| { + if e.event_type == RepoEventType::Commit + && e.prev_cid.is_none() + && e.commit_cid.is_none() + { + results.push(BrokenGenesisCommit { + seq: e.seq, + did: e.did, + commit_cid: e.commit_cid, + }); + } + results + }); + + match page_len < page_size { + true => Ok(results), + false => self.collect_broken_genesis_page(page_high_seq, results, page_size), + } + } + + pub fn get_users_without_blocks(&self) -> Result, MetastoreError> { + const MAX_RESULTS: usize = 10_000; + + self.scan_users_missing_prefix( + user_block_user_prefix, + |meta, user_id| { + let root_cid = bytes_to_cid_link(&meta.repo_root_cid)?; + Ok(UserWithoutBlocks { + user_id, + repo_root_cid: root_cid, + repo_rev: match meta.repo_rev.is_empty() { + true => None, + false => Some(meta.repo_rev), + }, + }) + }, + MAX_RESULTS, + ) + } + + fn scan_users_missing_prefix( + &self, + make_prefix: P, + build_result: F, + limit: usize, + ) -> Result, MetastoreError> + where + F: Fn(RepoMetaValue, Uuid) -> Result, + P: Fn(UserHash) -> SmallVec<[u8; 128]>, + { + let prefix = repo_meta_prefix(); + + self.repo_data + .prefix(prefix.as_slice()) + .filter_map(|guard| { + let (key_bytes, val_bytes) = match guard.into_inner() { + Ok(pair) => pair, + Err(e) => return Some(Err(MetastoreError::Fjall(e))), + }; + + let user_hash = match parse_user_hash_from_key(&key_bytes) { + Some(h) => h, + None => return Some(Err(MetastoreError::CorruptData("invalid repo_meta key"))), + }; + + let check_prefix = make_prefix(user_hash); + let has_entries = match self.repo_data.prefix(check_prefix.as_slice()).next() { + Some(guard) => match guard.into_inner() { + Ok(_) => true, + Err(e) => return Some(Err(MetastoreError::Fjall(e))), + }, + None => false, + }; + + match has_entries { + true => None, + false => { + let meta = match RepoMetaValue::deserialize(&val_bytes) { + Some(v) => v, + None => { + return Some(Err(MetastoreError::CorruptData( + "invalid repo_meta value", + ))); + } + }; + let user_id = match self.user_hashes.get_uuid(&user_hash) { + Some(id) => id, + None => { + return Some(Err(MetastoreError::CorruptData( + "user_hash has no reverse mapping", + ))); + } + }; + Some(build_result(meta, user_id)) + } + } + }) + .take(limit) + .collect() + } +} + +fn parse_user_hash_from_key(key_bytes: &[u8]) -> Option { + use super::encoding::KeyReader; + let mut reader = KeyReader::new(key_bytes); + let _tag = reader.tag()?; + let hash = reader.u64()?; + Some(UserHash::from_raw(hash)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::eventlog::{EventLog, EventLogConfig}; + use crate::io::RealIO; + use crate::metastore::{Metastore, MetastoreConfig}; + use tranquil_db_traits::CommitEventData; + use tranquil_types::{Handle, Nsid, Rkey}; + + struct TestHarness { + _metastore_dir: tempfile::TempDir, + _eventlog_dir: tempfile::TempDir, + metastore: Metastore, + bridge: Arc>, + } + + fn setup() -> TestHarness { + let metastore_dir = tempfile::TempDir::new().unwrap(); + let eventlog_dir = tempfile::TempDir::new().unwrap(); + let segments_dir = eventlog_dir.path().join("segments"); + std::fs::create_dir_all(&segments_dir).unwrap(); + + let metastore = Metastore::open( + metastore_dir.path(), + MetastoreConfig { + cache_size_bytes: 64 * 1024 * 1024, + }, + ) + .unwrap(); + + let event_log = EventLog::open( + EventLogConfig { + segments_dir, + ..EventLogConfig::default() + }, + RealIO::new(), + ) + .unwrap(); + + let bridge = Arc::new(EventLogBridge::new(Arc::new(event_log))); + + TestHarness { + _metastore_dir: metastore_dir, + _eventlog_dir: eventlog_dir, + metastore, + bridge, + } + } + + fn test_cid_link(seed: u8) -> CidLink { + let digest: [u8; 32] = std::array::from_fn(|i| seed.wrapping_add(i as u8)); + let mh = multihash::Multihash::<64>::wrap(0x12, &digest).unwrap(); + let c = cid::Cid::new_v1(0x71, mh); + CidLink::from_cid(&c) + } + + fn test_did(name: &str) -> Did { + Did::from(format!("did:plc:{name}")) + } + + fn test_handle(name: &str) -> Handle { + Handle::from(format!("{name}.test.invalid")) + } + + fn make_commit_ops(h: &TestHarness) -> CommitOps { + use crate::metastore::partitions::Partition; + CommitOps::new( + h.metastore.database().clone(), + h.metastore.partition(Partition::RepoData).clone(), + h.metastore.partition(Partition::Indexes).clone(), + Arc::clone(h.metastore.user_hashes()), + Arc::clone(&h.bridge), + ) + } + + fn create_test_repo(h: &TestHarness, name: &str, seed: u8) -> (Uuid, Did, CidLink) { + let user_id = Uuid::new_v4(); + let did = test_did(name); + let handle = test_handle(name); + let cid = test_cid_link(seed); + h.metastore + .repo_ops() + .create_repo(h.metastore.database(), user_id, &did, &handle, &cid, "rev0") + .unwrap(); + (user_id, did, cid) + } + + #[test] + fn apply_commit_updates_records_and_meta() { + let h = setup(); + let ops = make_commit_ops(&h); + let (user_id, did, root_cid) = create_test_repo(&h, "alice", 1); + + let new_root = test_cid_link(2); + let record_cid = test_cid_link(3); + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkey = Rkey::from("3k2abc".to_string()); + + let input = ApplyCommitInput { + user_id, + did: did.clone(), + expected_root_cid: Some(root_cid.clone()), + new_root_cid: new_root.clone(), + new_rev: "rev1".to_string(), + new_block_cids: vec![vec![0x01, 0x02]], + obsolete_block_cids: vec![], + record_upserts: vec![tranquil_db_traits::RecordUpsert { + collection: collection.clone(), + rkey: rkey.clone(), + cid: record_cid.clone(), + }], + record_deletes: vec![], + backlinks_to_add: vec![], + backlinks_to_remove: vec![], + commit_event: CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(new_root.clone()), + prev_cid: Some(root_cid.clone()), + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev1".to_string()), + }, + }; + + let result = ops.apply_commit(input).unwrap(); + assert!(result.seq > 0); + assert!(result.is_account_active); + + let repo = h.metastore.repo_ops().get_repo(user_id).unwrap().unwrap(); + assert_eq!(repo.repo_root_cid, new_root); + assert_eq!(repo.repo_rev.as_deref(), Some("rev1")); + + let found_cid = h + .metastore + .record_ops() + .get_record_cid(user_id, &collection, &rkey) + .unwrap() + .unwrap(); + assert_eq!(found_cid, record_cid); + } + + #[test] + fn apply_commit_cas_rejects_stale_root() { + let h = setup(); + let ops = make_commit_ops(&h); + let (user_id, did, _root_cid) = create_test_repo(&h, "bob", 10); + + let stale_root = test_cid_link(99); + let new_root = test_cid_link(11); + + let input = ApplyCommitInput { + user_id, + did, + expected_root_cid: Some(stale_root), + new_root_cid: new_root, + new_rev: "rev1".to_string(), + new_block_cids: vec![], + obsolete_block_cids: vec![], + record_upserts: vec![], + record_deletes: vec![], + backlinks_to_add: vec![], + backlinks_to_remove: vec![], + commit_event: CommitEventData { + did: test_did("bob"), + event_type: RepoEventType::Commit, + commit_cid: None, + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev1".to_string()), + }, + }; + + let result = ops.apply_commit(input); + assert_eq!( + result.unwrap_err(), + ApplyCommitError::ConcurrentModification + ); + } + + #[test] + fn apply_commit_returns_repo_not_found_for_unknown_user() { + let h = setup(); + let ops = make_commit_ops(&h); + + let input = ApplyCommitInput { + user_id: Uuid::new_v4(), + did: test_did("nobody"), + expected_root_cid: None, + new_root_cid: test_cid_link(1), + new_rev: "rev1".to_string(), + new_block_cids: vec![], + obsolete_block_cids: vec![], + record_upserts: vec![], + record_deletes: vec![], + backlinks_to_add: vec![], + backlinks_to_remove: vec![], + commit_event: CommitEventData { + did: test_did("nobody"), + event_type: RepoEventType::Commit, + commit_cid: None, + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: None, + }, + }; + + assert_eq!( + ops.apply_commit(input).unwrap_err(), + ApplyCommitError::RepoNotFound + ); + } + + #[test] + fn apply_commit_record_deletes() { + let h = setup(); + let ops = make_commit_ops(&h); + let (user_id, did, root_cid) = create_test_repo(&h, "carol", 20); + + let mid_root = test_cid_link(21); + let record_cid = test_cid_link(22); + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkey = Rkey::from("3k2del".to_string()); + + let insert_input = ApplyCommitInput { + user_id, + did: did.clone(), + expected_root_cid: Some(root_cid.clone()), + new_root_cid: mid_root.clone(), + new_rev: "rev1".to_string(), + new_block_cids: vec![], + obsolete_block_cids: vec![], + record_upserts: vec![tranquil_db_traits::RecordUpsert { + collection: collection.clone(), + rkey: rkey.clone(), + cid: record_cid, + }], + record_deletes: vec![], + backlinks_to_add: vec![], + backlinks_to_remove: vec![], + commit_event: CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(mid_root.clone()), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev1".to_string()), + }, + }; + ops.apply_commit(insert_input).unwrap(); + + assert!( + h.metastore + .record_ops() + .get_record_cid(user_id, &collection, &rkey) + .unwrap() + .is_some() + ); + + let final_root = test_cid_link(23); + let delete_input = ApplyCommitInput { + user_id, + did: did.clone(), + expected_root_cid: Some(mid_root.clone()), + new_root_cid: final_root.clone(), + new_rev: "rev2".to_string(), + new_block_cids: vec![], + obsolete_block_cids: vec![], + record_upserts: vec![], + record_deletes: vec![tranquil_db_traits::RecordDelete { + collection: collection.clone(), + rkey: rkey.clone(), + }], + backlinks_to_add: vec![], + backlinks_to_remove: vec![], + commit_event: CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(final_root.clone()), + prev_cid: Some(mid_root.clone()), + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev2".to_string()), + }, + }; + ops.apply_commit(delete_input).unwrap(); + + assert!( + h.metastore + .record_ops() + .get_record_cid(user_id, &collection, &rkey) + .unwrap() + .is_none() + ); + } + + #[test] + fn apply_commit_event_visible_after_commit() { + let h = setup(); + let ops = make_commit_ops(&h); + let (user_id, did, root_cid) = create_test_repo(&h, "dave", 30); + + let new_root = test_cid_link(31); + let input = ApplyCommitInput { + user_id, + did: did.clone(), + expected_root_cid: Some(root_cid.clone()), + new_root_cid: new_root.clone(), + new_rev: "rev1".to_string(), + new_block_cids: vec![], + obsolete_block_cids: vec![], + record_upserts: vec![], + record_deletes: vec![], + backlinks_to_add: vec![], + backlinks_to_remove: vec![], + commit_event: CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(new_root.clone()), + prev_cid: Some(root_cid.clone()), + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev1".to_string()), + }, + }; + + let result = ops.apply_commit(input).unwrap(); + let seq = SequenceNumber::from_raw(result.seq); + + let event = ops.event_ops.get_event_by_seq(seq).unwrap().unwrap(); + assert_eq!(event.did, did); + assert_eq!(event.event_type, RepoEventType::Commit); + assert_eq!(event.rev.as_deref(), Some("rev1")); + } + + #[test] + fn import_repo_data_inserts_records() { + let h = setup(); + let ops = make_commit_ops(&h); + let (user_id, _did, root_cid) = create_test_repo(&h, "eve", 40); + + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkey = Rkey::from("3k2import".to_string()); + let record_cid = test_cid_link(41); + + ops.import_repo_data( + user_id, + &[], + &[ImportRecord { + collection: collection.clone(), + rkey: rkey.clone(), + record_cid: record_cid.clone(), + }], + Some(&root_cid), + ) + .unwrap(); + + let found = h + .metastore + .record_ops() + .get_record_cid(user_id, &collection, &rkey) + .unwrap() + .unwrap(); + assert_eq!(found, record_cid); + } + + #[test] + fn import_repo_data_cas_rejects_stale_root() { + let h = setup(); + let ops = make_commit_ops(&h); + let (user_id, _did, _root_cid) = create_test_repo(&h, "frank", 50); + + let stale = test_cid_link(99); + let result = ops.import_repo_data(user_id, &[], &[], Some(&stale)); + assert_eq!(result.unwrap_err(), ImportRepoError::ConcurrentModification); + } + + #[test] + fn insert_record_blobs_and_backfill_query() { + let h = setup(); + let ops = make_commit_ops(&h); + let (user_id_a, did_a, _) = create_test_repo(&h, "grace", 60); + let (user_id_b, _did_b, _) = create_test_repo(&h, "henry", 61); + + let needing = ops.get_users_needing_record_blobs_backfill(100).unwrap(); + assert_eq!(needing.len(), 2); + + let uri = AtUri::from_parts(did_a.as_str(), "app.bsky.feed.post", "3k2abc"); + let blob_cid = test_cid_link(62); + ops.insert_record_blobs(user_id_a, &[uri], &[blob_cid]) + .unwrap(); + + let needing_after = ops.get_users_needing_record_blobs_backfill(100).unwrap(); + assert_eq!(needing_after.len(), 1); + assert_eq!(needing_after[0].user_id, user_id_b); + } + + #[test] + fn get_users_without_blocks_returns_users_with_no_blocks() { + let h = setup(); + let ops = make_commit_ops(&h); + let (user_id_a, did_a, root_a) = create_test_repo(&h, "ivan", 70); + let (user_id_b, _did_b, _root_b) = create_test_repo(&h, "julia", 71); + + let new_root = test_cid_link(72); + let input = ApplyCommitInput { + user_id: user_id_a, + did: did_a.clone(), + expected_root_cid: Some(root_a), + new_root_cid: new_root.clone(), + new_rev: "rev1".to_string(), + new_block_cids: vec![vec![0x01, 0x02, 0x03]], + obsolete_block_cids: vec![], + record_upserts: vec![], + record_deletes: vec![], + backlinks_to_add: vec![], + backlinks_to_remove: vec![], + commit_event: CommitEventData { + did: did_a.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(new_root), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev1".to_string()), + }, + }; + ops.apply_commit(input).unwrap(); + + let without = ops.get_users_without_blocks().unwrap(); + assert_eq!(without.len(), 1); + assert_eq!(without[0].user_id, user_id_b); + } + + #[test] + fn apply_commit_without_expected_root_skips_cas() { + let h = setup(); + let ops = make_commit_ops(&h); + let (user_id, did, _root_cid) = create_test_repo(&h, "kate", 80); + + let new_root = test_cid_link(81); + let input = ApplyCommitInput { + user_id, + did: did.clone(), + expected_root_cid: None, + new_root_cid: new_root.clone(), + new_rev: "rev_force".to_string(), + new_block_cids: vec![], + obsolete_block_cids: vec![], + record_upserts: vec![], + record_deletes: vec![], + backlinks_to_add: vec![], + backlinks_to_remove: vec![], + commit_event: CommitEventData { + did, + event_type: RepoEventType::Commit, + commit_cid: None, + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev_force".to_string()), + }, + }; + + let result = ops.apply_commit(input).unwrap(); + assert!(result.seq > 0); + } + + #[test] + fn apply_commit_update_preserves_new_backlinks() { + use crate::metastore::backlinks::backlink_target_prefix; + use crate::metastore::partitions::Partition; + + let h = setup(); + let ops = make_commit_ops(&h); + let (user_id, did, root_cid) = create_test_repo(&h, "backlink_upd", 90); + + let collection = Nsid::from("app.bsky.feed.like".to_string()); + let rkey = Rkey::from("3k2like1".to_string()); + let record_cid = test_cid_link(91); + let record_uri = AtUri::from_parts(did.as_str(), collection.as_str(), rkey.as_str()); + + let mid_root = test_cid_link(92); + let create_input = ApplyCommitInput { + user_id, + did: did.clone(), + expected_root_cid: Some(root_cid.clone()), + new_root_cid: mid_root.clone(), + new_rev: "rev1".to_string(), + new_block_cids: vec![], + obsolete_block_cids: vec![], + record_upserts: vec![tranquil_db_traits::RecordUpsert { + collection: collection.clone(), + rkey: rkey.clone(), + cid: record_cid.clone(), + }], + record_deletes: vec![], + backlinks_to_add: vec![tranquil_db_traits::Backlink { + uri: record_uri.clone(), + path: tranquil_db_traits::BacklinkPath::SubjectUri, + link_to: "at://did:plc:target_a/app.bsky.feed.post/p1".to_string(), + }], + backlinks_to_remove: vec![], + commit_event: CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(mid_root.clone()), + prev_cid: Some(root_cid.clone()), + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev1".to_string()), + }, + }; + ops.apply_commit(create_input).unwrap(); + + let indexes = h.metastore.partition(Partition::Indexes); + let target_a_prefix = backlink_target_prefix("at://did:plc:target_a/app.bsky.feed.post/p1"); + let count_a_before = indexes + .prefix(target_a_prefix.as_slice()) + .map(|g| g.into_inner().expect("scan must not fail")) + .fold(0, |acc, _| acc + 1); + assert_eq!(count_a_before, 1); + + let final_root = test_cid_link(93); + let new_record_cid = test_cid_link(94); + let update_input = ApplyCommitInput { + user_id, + did: did.clone(), + expected_root_cid: Some(mid_root.clone()), + new_root_cid: final_root.clone(), + new_rev: "rev2".to_string(), + new_block_cids: vec![], + obsolete_block_cids: vec![], + record_upserts: vec![tranquil_db_traits::RecordUpsert { + collection: collection.clone(), + rkey: rkey.clone(), + cid: new_record_cid.clone(), + }], + record_deletes: vec![], + backlinks_to_add: vec![tranquil_db_traits::Backlink { + uri: record_uri.clone(), + path: tranquil_db_traits::BacklinkPath::SubjectUri, + link_to: "at://did:plc:target_b/app.bsky.feed.post/p2".to_string(), + }], + backlinks_to_remove: vec![record_uri], + commit_event: CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(final_root.clone()), + prev_cid: Some(mid_root.clone()), + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev2".to_string()), + }, + }; + ops.apply_commit(update_input).unwrap(); + + let count_a_after = indexes + .prefix(target_a_prefix.as_slice()) + .map(|g| g.into_inner().expect("scan must not fail")) + .fold(0, |acc, _| acc + 1); + assert_eq!(count_a_after, 0); + + let target_b_prefix = backlink_target_prefix("at://did:plc:target_b/app.bsky.feed.post/p2"); + let count_b = indexes + .prefix(target_b_prefix.as_slice()) + .map(|g| g.into_inner().expect("scan must not fail")) + .fold(0, |acc, _| acc + 1); + assert_eq!(count_b, 1); + } + + #[test] + fn crash_recovery_replays_mutation_set() { + let metastore_dir = tempfile::TempDir::new().unwrap(); + let eventlog_dir = tempfile::TempDir::new().unwrap(); + let segments_dir = eventlog_dir.path().join("segments"); + std::fs::create_dir_all(&segments_dir).unwrap(); + + let user_id = Uuid::new_v4(); + let did = test_did("crash_alice"); + let handle = test_handle("crash_alice"); + let initial_root = test_cid_link(200); + let new_root = test_cid_link(201); + let record_cid = test_cid_link(202); + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkey = Rkey::from("3k2crash".to_string()); + + let event_log = EventLog::open( + EventLogConfig { + segments_dir: segments_dir.clone(), + ..EventLogConfig::default() + }, + RealIO::new(), + ) + .unwrap(); + let event_log = Arc::new(event_log); + let bridge = Arc::new(EventLogBridge::new(Arc::clone(&event_log))); + + { + let metastore = Metastore::open( + metastore_dir.path(), + MetastoreConfig { + cache_size_bytes: 64 * 1024 * 1024, + }, + ) + .unwrap(); + + metastore + .repo_ops() + .create_repo( + metastore.database(), + user_id, + &did, + &handle, + &initial_root, + "rev0", + ) + .unwrap(); + metastore.persist().unwrap(); + + let ops = make_commit_ops_from(&metastore, &bridge); + let input = ApplyCommitInput { + user_id, + did: did.clone(), + expected_root_cid: Some(initial_root.clone()), + new_root_cid: new_root.clone(), + new_rev: "rev1".to_string(), + new_block_cids: vec![vec![0xAA, 0xBB]], + obsolete_block_cids: vec![], + record_upserts: vec![tranquil_db_traits::RecordUpsert { + collection: collection.clone(), + rkey: rkey.clone(), + cid: record_cid.clone(), + }], + record_deletes: vec![], + backlinks_to_add: vec![], + backlinks_to_remove: vec![], + commit_event: CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(new_root.clone()), + prev_cid: Some(initial_root.clone()), + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev1".to_string()), + }, + }; + + let result = ops.apply_commit(input).unwrap(); + assert!(result.seq > 0); + metastore.persist().unwrap(); + } + + { + let metastore = Metastore::open( + metastore_dir.path(), + MetastoreConfig { + cache_size_bytes: 64 * 1024 * 1024, + }, + ) + .unwrap(); + + let event_ops = metastore.event_ops(Arc::clone(&bridge)); + + event_ops.write_last_applied_cursor_direct(0).unwrap(); + metastore.persist().unwrap(); + } + + { + let metastore = Metastore::open( + metastore_dir.path(), + MetastoreConfig { + cache_size_bytes: 64 * 1024 * 1024, + }, + ) + .unwrap(); + + let repo_before = metastore.repo_ops().get_repo(user_id).unwrap().unwrap(); + assert_eq!(repo_before.repo_root_cid, new_root); + + let event_ops = metastore.event_ops(Arc::clone(&bridge)); + let cursor_before = event_ops.read_last_applied_cursor().unwrap(); + assert_eq!(cursor_before, Some(0)); + + let indexes = metastore + .partition(crate::metastore::partitions::Partition::Indexes) + .clone(); + let recovered = event_ops.recover_metastore_mutations(&indexes).unwrap(); + assert!(recovered > 0, "should replay at least one event"); + + let cursor_after = event_ops.read_last_applied_cursor().unwrap(); + assert!(cursor_after.unwrap_or(0) > 0); + } + } + + #[test] + fn crash_recovery_with_uncommitted_batch() { + let metastore_dir = tempfile::TempDir::new().unwrap(); + let eventlog_dir = tempfile::TempDir::new().unwrap(); + let segments_dir = eventlog_dir.path().join("segments"); + std::fs::create_dir_all(&segments_dir).unwrap(); + + let user_id = Uuid::new_v4(); + let did = test_did("crash_bob"); + let handle = test_handle("crash_bob"); + let initial_root = test_cid_link(210); + let new_root = test_cid_link(211); + let record_cid = test_cid_link(212); + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkey = Rkey::from("3k2bob".to_string()); + + let event_log = EventLog::open( + EventLogConfig { + segments_dir: segments_dir.clone(), + ..EventLogConfig::default() + }, + RealIO::new(), + ) + .unwrap(); + let event_log = Arc::new(event_log); + let bridge = Arc::new(EventLogBridge::new(Arc::clone(&event_log))); + + { + let metastore = Metastore::open( + metastore_dir.path(), + MetastoreConfig { + cache_size_bytes: 64 * 1024 * 1024, + }, + ) + .unwrap(); + + metastore + .repo_ops() + .create_repo( + metastore.database(), + user_id, + &did, + &handle, + &initial_root, + "rev0", + ) + .unwrap(); + metastore.persist().unwrap(); + + let event_ops = metastore.event_ops(Arc::clone(&bridge)); + let mutation_set = super::CommitMutationSet { + new_root_cid: super::cid_link_to_bytes(&new_root).unwrap(), + new_rev: "rev1".to_string(), + record_upserts: vec![super::RecordMutationUpsert { + collection: collection.as_str().to_owned(), + rkey: rkey.as_str().to_owned(), + cid_bytes: super::cid_link_to_bytes(&record_cid).unwrap(), + }], + record_deletes: vec![], + block_inserts: vec![vec![0xCC, 0xDD]], + block_deletes: vec![], + backlink_adds: vec![], + backlink_remove_uris: vec![], + }; + let ms_bytes = mutation_set.serialize().unwrap(); + + let commit_data = CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(new_root.clone()), + prev_cid: Some(initial_root.clone()), + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev1".to_string()), + }; + + let mut batch = metastore.database().batch(); + let (_seq, deferred) = event_ops + .append_commit_event_into_batch(&mut batch, &commit_data, Some(&ms_bytes)) + .unwrap(); + + event_ops.complete_broadcast(deferred); + + drop(batch); + + metastore.persist().unwrap(); + } + + { + let metastore = Metastore::open( + metastore_dir.path(), + MetastoreConfig { + cache_size_bytes: 64 * 1024 * 1024, + }, + ) + .unwrap(); + + let repo = metastore.repo_ops().get_repo(user_id).unwrap().unwrap(); + assert_eq!(repo.repo_root_cid, initial_root); + + let record = metastore + .record_ops() + .get_record_cid(user_id, &collection, &rkey) + .unwrap(); + assert!(record.is_none()); + + let event_ops = metastore.event_ops(Arc::clone(&bridge)); + let indexes = metastore + .partition(crate::metastore::partitions::Partition::Indexes) + .clone(); + let recovered = event_ops.recover_metastore_mutations(&indexes).unwrap(); + assert_eq!(recovered, 1); + + let repo_after = metastore.repo_ops().get_repo(user_id).unwrap().unwrap(); + assert_eq!(repo_after.repo_root_cid, new_root); + assert_eq!(repo_after.repo_rev.as_deref(), Some("rev1")); + + let record_after = metastore + .record_ops() + .get_record_cid(user_id, &collection, &rkey) + .unwrap(); + assert_eq!(record_after, Some(record_cid)); + } + } + + fn make_commit_ops_from( + metastore: &Metastore, + bridge: &Arc>, + ) -> CommitOps { + use crate::metastore::partitions::Partition; + CommitOps::new( + metastore.database().clone(), + metastore.partition(Partition::RepoData).clone(), + metastore.partition(Partition::Indexes).clone(), + Arc::clone(metastore.user_hashes()), + Arc::clone(bridge), + ) + } + + #[test] + fn apply_commit_backlinks_isolated_by_collection() { + use crate::metastore::backlinks::{backlink_by_user_prefix, backlink_target_prefix}; + use crate::metastore::partitions::Partition; + + let h = setup(); + let ops = make_commit_ops(&h); + let (user_id, did, root_cid) = create_test_repo(&h, "col_iso", 95); + + let col_like = Nsid::from("app.bsky.feed.like".to_string()); + let col_repost = Nsid::from("app.bsky.feed.repost".to_string()); + let rkey = Rkey::from("same_rkey".to_string()); + let target = "at://did:plc:someone/app.bsky.feed.post/p1"; + + let mid_root = test_cid_link(96); + let uri_like = AtUri::from_parts(did.as_str(), col_like.as_str(), rkey.as_str()); + let uri_repost = AtUri::from_parts(did.as_str(), col_repost.as_str(), rkey.as_str()); + + let input = ApplyCommitInput { + user_id, + did: did.clone(), + expected_root_cid: Some(root_cid.clone()), + new_root_cid: mid_root.clone(), + new_rev: "rev1".to_string(), + new_block_cids: vec![], + obsolete_block_cids: vec![], + record_upserts: vec![ + tranquil_db_traits::RecordUpsert { + collection: col_like.clone(), + rkey: rkey.clone(), + cid: test_cid_link(97), + }, + tranquil_db_traits::RecordUpsert { + collection: col_repost.clone(), + rkey: rkey.clone(), + cid: test_cid_link(98), + }, + ], + record_deletes: vec![], + backlinks_to_add: vec![ + tranquil_db_traits::Backlink { + uri: uri_like.clone(), + path: tranquil_db_traits::BacklinkPath::SubjectUri, + link_to: target.to_string(), + }, + tranquil_db_traits::Backlink { + uri: uri_repost.clone(), + path: tranquil_db_traits::BacklinkPath::SubjectUri, + link_to: target.to_string(), + }, + ], + backlinks_to_remove: vec![], + commit_event: CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(mid_root.clone()), + prev_cid: Some(root_cid.clone()), + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev1".to_string()), + }, + }; + ops.apply_commit(input).unwrap(); + + let indexes = h.metastore.partition(Partition::Indexes); + let user_hash = h.metastore.user_hashes().get(&user_id).unwrap(); + let target_prefix = backlink_target_prefix(target); + let user_prefix = backlink_by_user_prefix(user_hash); + + assert_eq!( + indexes + .prefix(target_prefix.as_slice()) + .map(|g| g.into_inner().expect("scan must not fail")) + .fold(0, |acc, _| acc + 1), + 2 + ); + assert_eq!( + indexes + .prefix(user_prefix.as_slice()) + .map(|g| g.into_inner().expect("scan must not fail")) + .fold(0, |acc, _| acc + 1), + 2 + ); + + let final_root = test_cid_link(99); + let remove_like = ApplyCommitInput { + user_id, + did: did.clone(), + expected_root_cid: Some(mid_root.clone()), + new_root_cid: final_root.clone(), + new_rev: "rev2".to_string(), + new_block_cids: vec![], + obsolete_block_cids: vec![], + record_upserts: vec![], + record_deletes: vec![], + backlinks_to_add: vec![], + backlinks_to_remove: vec![uri_like], + commit_event: CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(final_root.clone()), + prev_cid: Some(mid_root.clone()), + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev2".to_string()), + }, + }; + ops.apply_commit(remove_like).unwrap(); + + assert_eq!( + indexes + .prefix(target_prefix.as_slice()) + .map(|g| g.into_inner().expect("scan must not fail")) + .fold(0, |acc, _| acc + 1), + 1 + ); + assert_eq!( + indexes + .prefix(user_prefix.as_slice()) + .map(|g| g.into_inner().expect("scan must not fail")) + .fold(0, |acc, _| acc + 1), + 1 + ); + } +} diff --git a/crates/tranquil-store/src/metastore/encoding.rs b/crates/tranquil-store/src/metastore/encoding.rs new file mode 100644 index 0000000..82f1fa1 --- /dev/null +++ b/crates/tranquil-store/src/metastore/encoding.rs @@ -0,0 +1,599 @@ +use smallvec::SmallVec; + +const NULL_ESCAPE: u8 = 0x01; +const NULL_TERMINATOR: [u8; 2] = [0x00, 0x00]; + +pub fn encode_u64(buf: &mut SmallVec<[u8; 128]>, value: u64) { + buf.extend_from_slice(&value.to_be_bytes()); +} + +pub fn decode_u64(src: &[u8]) -> Option<(u64, &[u8])> { + let (bytes, rest) = src.split_first_chunk::<8>()?; + Some((u64::from_be_bytes(*bytes), rest)) +} + +pub fn encode_i64(buf: &mut SmallVec<[u8; 128]>, value: i64) { + let encoded = (value as u64) ^ (1u64 << 63); + buf.extend_from_slice(&encoded.to_be_bytes()); +} + +pub fn decode_i64(src: &[u8]) -> Option<(i64, &[u8])> { + let (bytes, rest) = src.split_first_chunk::<8>()?; + let raw = u64::from_be_bytes(*bytes) ^ (1u64 << 63); + Some((raw as i64, rest)) +} + +pub fn encode_u32(buf: &mut SmallVec<[u8; 128]>, value: u32) { + buf.extend_from_slice(&value.to_be_bytes()); +} + +pub fn decode_u32(src: &[u8]) -> Option<(u32, &[u8])> { + let (bytes, rest) = src.split_first_chunk::<4>()?; + Some((u32::from_be_bytes(*bytes), rest)) +} + +pub fn encode_u16(buf: &mut SmallVec<[u8; 128]>, value: u16) { + buf.extend_from_slice(&value.to_be_bytes()); +} + +pub fn decode_u16(src: &[u8]) -> Option<(u16, &[u8])> { + let (bytes, rest) = src.split_first_chunk::<2>()?; + Some((u16::from_be_bytes(*bytes), rest)) +} + +pub fn encode_bool(buf: &mut SmallVec<[u8; 128]>, value: bool) { + buf.push(u8::from(value)); +} + +pub fn decode_bool(src: &[u8]) -> Option<(bool, &[u8])> { + let (&byte, rest) = src.split_first()?; + match byte { + 0 => Some((false, rest)), + 1 => Some((true, rest)), + _ => None, + } +} + +pub fn encode_bytes(buf: &mut SmallVec<[u8; 128]>, value: &[u8]) { + value.iter().for_each(|&b| match b { + 0x00 => { + buf.push(0x00); + buf.push(NULL_ESCAPE); + } + other => buf.push(other), + }); + buf.extend_from_slice(&NULL_TERMINATOR); +} + +pub fn decode_bytes(src: &[u8]) -> Option<(Vec, &[u8])> { + let mut result = Vec::new(); + let mut i = 0; + loop { + match src.get(i)? { + 0x00 => match src.get(i + 1)? { + 0x00 => return Some((result, &src[i + 2..])), + &NULL_ESCAPE => { + result.push(0x00); + i += 2; + } + _ => return None, + }, + &b => { + result.push(b); + i += 1; + } + } + } +} + +pub fn encode_string(buf: &mut SmallVec<[u8; 128]>, value: &str) { + encode_bytes(buf, value.as_bytes()); +} + +pub fn decode_string(src: &[u8]) -> Option<(String, &[u8])> { + let (bytes, rest) = decode_bytes(src)?; + String::from_utf8(bytes).ok().map(|s| (s, rest)) +} + +pub struct KeyBuilder(SmallVec<[u8; 128]>); + +impl KeyBuilder { + pub fn new() -> Self { + Self(SmallVec::new()) + } + + pub fn with_capacity(cap: usize) -> Self { + Self(SmallVec::with_capacity(cap)) + } + + pub fn u64(mut self, value: u64) -> Self { + encode_u64(&mut self.0, value); + self + } + + pub fn i64(mut self, value: i64) -> Self { + encode_i64(&mut self.0, value); + self + } + + pub fn u32(mut self, value: u32) -> Self { + encode_u32(&mut self.0, value); + self + } + + pub fn u16(mut self, value: u16) -> Self { + encode_u16(&mut self.0, value); + self + } + + pub fn bool(mut self, value: bool) -> Self { + encode_bool(&mut self.0, value); + self + } + + pub fn bytes(mut self, value: &[u8]) -> Self { + encode_bytes(&mut self.0, value); + self + } + + pub fn string(mut self, value: &str) -> Self { + encode_string(&mut self.0, value); + self + } + + pub fn tag(mut self, tag: super::keys::KeyTag) -> Self { + self.0.push(tag.raw()); + self + } + + pub fn fixed(mut self, bytes: &[u8; N]) -> Self { + self.0.extend_from_slice(bytes); + self + } + + pub fn raw(mut self, bytes: &[u8]) -> Self { + self.0.extend_from_slice(bytes); + self + } + + pub fn build(self) -> SmallVec<[u8; 128]> { + self.0 + } + + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } +} + +impl Default for KeyBuilder { + fn default() -> Self { + Self::new() + } +} + +pub struct KeyReader<'a>(&'a [u8]); + +impl<'a> KeyReader<'a> { + pub fn new(src: &'a [u8]) -> Self { + Self(src) + } + + pub fn u64(&mut self) -> Option { + let (val, rest) = decode_u64(self.0)?; + self.0 = rest; + Some(val) + } + + pub fn i64(&mut self) -> Option { + let (val, rest) = decode_i64(self.0)?; + self.0 = rest; + Some(val) + } + + pub fn u32(&mut self) -> Option { + let (val, rest) = decode_u32(self.0)?; + self.0 = rest; + Some(val) + } + + pub fn u16(&mut self) -> Option { + let (val, rest) = decode_u16(self.0)?; + self.0 = rest; + Some(val) + } + + pub fn bool(&mut self) -> Option { + let (val, rest) = decode_bool(self.0)?; + self.0 = rest; + Some(val) + } + + pub fn bytes(&mut self) -> Option> { + let (val, rest) = decode_bytes(self.0)?; + self.0 = rest; + Some(val) + } + + pub fn string(&mut self) -> Option { + let (val, rest) = decode_string(self.0)?; + self.0 = rest; + Some(val) + } + + pub fn tag(&mut self) -> Option { + let (&tag, rest) = self.0.split_first()?; + self.0 = rest; + Some(tag) + } + + pub fn remaining(&self) -> &'a [u8] { + self.0 + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +pub fn exclusive_upper_bound(prefix: &[u8]) -> Option> { + prefix.iter().rposition(|&b| b != 0xFF).map(|pos| { + let mut result = SmallVec::from_slice(&prefix[..=pos]); + result[pos] = prefix[pos].wrapping_add(1); + result + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use proptest::prelude::*; + + #[test] + fn u64_roundtrip_boundaries() { + [0u64, 1, u64::MAX / 2, u64::MAX - 1, u64::MAX] + .iter() + .for_each(|&v| { + let mut buf = SmallVec::new(); + encode_u64(&mut buf, v); + let (decoded, rest) = decode_u64(&buf).unwrap(); + assert_eq!(decoded, v); + assert!(rest.is_empty()); + }); + } + + #[test] + fn i64_roundtrip_boundaries() { + [i64::MIN, -1, 0, 1, i64::MAX].iter().for_each(|&v| { + let mut buf = SmallVec::new(); + encode_i64(&mut buf, v); + let (decoded, rest) = decode_i64(&buf).unwrap(); + assert_eq!(decoded, v); + assert!(rest.is_empty()); + }); + } + + #[test] + fn bool_roundtrip() { + [false, true].iter().for_each(|&v| { + let mut buf = SmallVec::new(); + encode_bool(&mut buf, v); + let (decoded, rest) = decode_bool(&buf).unwrap(); + assert_eq!(decoded, v); + assert!(rest.is_empty()); + }); + } + + #[test] + fn bytes_with_nulls() { + let input = &[0x00, 0x01, 0x00, 0xFF, 0x00]; + let mut buf = SmallVec::new(); + encode_bytes(&mut buf, input); + let (decoded, rest) = decode_bytes(&buf).unwrap(); + assert_eq!(decoded, input); + assert!(rest.is_empty()); + } + + #[test] + fn empty_bytes_roundtrip() { + let mut buf = SmallVec::new(); + encode_bytes(&mut buf, &[]); + let (decoded, rest) = decode_bytes(&buf).unwrap(); + assert!(decoded.is_empty()); + assert!(rest.is_empty()); + } + + #[test] + fn empty_string_roundtrip() { + let mut buf = SmallVec::new(); + encode_string(&mut buf, ""); + let (decoded, rest) = decode_string(&buf).unwrap(); + assert_eq!(decoded, ""); + assert!(rest.is_empty()); + } + + #[test] + fn string_with_null_bytes() { + let input = "hello\x00world"; + let mut buf = SmallVec::new(); + encode_string(&mut buf, input); + let (decoded, rest) = decode_string(&buf).unwrap(); + assert_eq!(decoded, input); + assert!(rest.is_empty()); + } + + #[test] + fn key_builder_composite_roundtrip() { + let key = KeyBuilder::new() + .tag(super::super::keys::KeyTag::RECORDS) + .u64(42) + .string("app.bsky.feed.post") + .string("3k2a") + .build(); + + let mut reader = KeyReader::new(&key); + assert_eq!( + reader.tag(), + Some(super::super::keys::KeyTag::RECORDS.raw()) + ); + assert_eq!(reader.u64(), Some(42)); + assert_eq!(reader.string(), Some("app.bsky.feed.post".to_string())); + assert_eq!(reader.string(), Some("3k2a".to_string())); + assert!(reader.is_empty()); + } + + #[test] + fn key_builder_ordering_preserves_field_order() { + let key_a = KeyBuilder::new().u64(1).string("aaa").build(); + let key_b = KeyBuilder::new().u64(1).string("bbb").build(); + let key_c = KeyBuilder::new().u64(2).string("aaa").build(); + + assert!(key_a.as_slice() < key_b.as_slice()); + assert!(key_b.as_slice() < key_c.as_slice()); + } + + #[test] + fn decode_bytes_rejects_invalid_escape() { + assert!(decode_bytes(&[0x00, 0x02]).is_none()); + assert!(decode_bytes(&[0x00, 0xFF]).is_none()); + assert!(decode_bytes(&[0x41, 0x00, 0x03]).is_none()); + } + + #[test] + fn decode_bytes_rejects_truncated_input() { + assert!(decode_bytes(&[]).is_none()); + assert!(decode_bytes(&[0x00]).is_none()); + assert!(decode_bytes(&[0x41]).is_none()); + assert!(decode_bytes(&[0x41, 0x00]).is_none()); + assert!(decode_bytes(&[0x00, 0x01]).is_none()); + } + + #[test] + fn decode_bool_rejects_invalid_byte() { + assert!(decode_bool(&[0x02]).is_none()); + assert!(decode_bool(&[0xFF]).is_none()); + assert!(decode_bool(&[]).is_none()); + } + + #[test] + fn decode_string_rejects_invalid_utf8() { + let mut buf = SmallVec::new(); + encode_bytes(&mut buf, &[0xFF, 0xFE]); + assert!(decode_string(&buf).is_none()); + } + + #[test] + fn decode_u64_rejects_short_input() { + assert!(decode_u64(&[]).is_none()); + assert!(decode_u64(&[0x00; 7]).is_none()); + } + + #[test] + fn decode_u32_rejects_short_input() { + assert!(decode_u32(&[]).is_none()); + assert!(decode_u32(&[0x00; 3]).is_none()); + } + + #[test] + fn decode_u16_rejects_short_input() { + assert!(decode_u16(&[]).is_none()); + assert!(decode_u16(&[0x00]).is_none()); + } + + #[test] + fn decode_i64_rejects_short_input() { + assert!(decode_i64(&[]).is_none()); + assert!(decode_i64(&[0x00; 7]).is_none()); + } + + #[test] + fn fixed_key_roundtrip() { + let data: [u8; 4] = [0xDE, 0xAD, 0xBE, 0xEF]; + let key = KeyBuilder::new() + .tag(super::super::keys::KeyTag::RECORDS) + .fixed(&data) + .build(); + + let mut reader = KeyReader::new(&key); + assert_eq!( + reader.tag(), + Some(super::super::keys::KeyTag::RECORDS.raw()) + ); + assert_eq!(reader.remaining(), &data); + } + + proptest! { + #[test] + fn prop_u64_roundtrip(v: u64) { + let mut buf = SmallVec::new(); + encode_u64(&mut buf, v); + let (decoded, rest) = decode_u64(&buf).unwrap(); + prop_assert_eq!(decoded, v); + prop_assert!(rest.is_empty()); + } + + #[test] + fn prop_u64_ordering(a: u64, b: u64) { + let mut buf_a = SmallVec::new(); + let mut buf_b = SmallVec::new(); + encode_u64(&mut buf_a, a); + encode_u64(&mut buf_b, b); + prop_assert_eq!(buf_a.as_slice().cmp(buf_b.as_slice()), a.cmp(&b)); + } + + #[test] + fn prop_i64_roundtrip(v: i64) { + let mut buf = SmallVec::new(); + encode_i64(&mut buf, v); + let (decoded, rest) = decode_i64(&buf).unwrap(); + prop_assert_eq!(decoded, v); + prop_assert!(rest.is_empty()); + } + + #[test] + fn prop_i64_ordering(a: i64, b: i64) { + let mut buf_a = SmallVec::new(); + let mut buf_b = SmallVec::new(); + encode_i64(&mut buf_a, a); + encode_i64(&mut buf_b, b); + prop_assert_eq!(buf_a.as_slice().cmp(buf_b.as_slice()), a.cmp(&b)); + } + + #[test] + fn prop_u32_roundtrip(v: u32) { + let mut buf = SmallVec::new(); + encode_u32(&mut buf, v); + let (decoded, rest) = decode_u32(&buf).unwrap(); + prop_assert_eq!(decoded, v); + prop_assert!(rest.is_empty()); + } + + #[test] + fn prop_u32_ordering(a: u32, b: u32) { + let mut buf_a = SmallVec::new(); + let mut buf_b = SmallVec::new(); + encode_u32(&mut buf_a, a); + encode_u32(&mut buf_b, b); + prop_assert_eq!(buf_a.as_slice().cmp(buf_b.as_slice()), a.cmp(&b)); + } + + #[test] + fn prop_u16_roundtrip(v: u16) { + let mut buf = SmallVec::new(); + encode_u16(&mut buf, v); + let (decoded, rest) = decode_u16(&buf).unwrap(); + prop_assert_eq!(decoded, v); + prop_assert!(rest.is_empty()); + } + + #[test] + fn prop_u16_ordering(a: u16, b: u16) { + let mut buf_a = SmallVec::new(); + let mut buf_b = SmallVec::new(); + encode_u16(&mut buf_a, a); + encode_u16(&mut buf_b, b); + prop_assert_eq!(buf_a.as_slice().cmp(buf_b.as_slice()), a.cmp(&b)); + } + + #[test] + fn prop_bool_roundtrip(v: bool) { + let mut buf = SmallVec::new(); + encode_bool(&mut buf, v); + let (decoded, rest) = decode_bool(&buf).unwrap(); + prop_assert_eq!(decoded, v); + prop_assert!(rest.is_empty()); + } + + #[test] + fn prop_bool_ordering(a: bool, b: bool) { + let mut buf_a = SmallVec::new(); + let mut buf_b = SmallVec::new(); + encode_bool(&mut buf_a, a); + encode_bool(&mut buf_b, b); + prop_assert_eq!(buf_a.as_slice().cmp(buf_b.as_slice()), a.cmp(&b)); + } + + #[test] + fn prop_bytes_roundtrip(v in proptest::collection::vec(any::(), 0..256)) { + let mut buf = SmallVec::new(); + encode_bytes(&mut buf, &v); + let (decoded, rest) = decode_bytes(&buf).unwrap(); + prop_assert_eq!(decoded, v); + prop_assert!(rest.is_empty()); + } + + #[test] + fn prop_bytes_ordering( + a in proptest::collection::vec(any::(), 0..64), + b in proptest::collection::vec(any::(), 0..64), + ) { + let mut buf_a = SmallVec::new(); + let mut buf_b = SmallVec::new(); + encode_bytes(&mut buf_a, &a); + encode_bytes(&mut buf_b, &b); + prop_assert_eq!(buf_a.as_slice().cmp(buf_b.as_slice()), a.cmp(&b)); + } + + #[test] + fn prop_string_roundtrip(v in "\\PC{0,128}") { + let mut buf = SmallVec::new(); + encode_string(&mut buf, &v); + let (decoded, rest) = decode_string(&buf).unwrap(); + prop_assert_eq!(decoded, v); + prop_assert!(rest.is_empty()); + } + + #[test] + fn prop_string_ordering( + a in "[\\x00-\\xff]{0,32}", + b in "[\\x00-\\xff]{0,32}", + ) { + let mut buf_a = SmallVec::new(); + let mut buf_b = SmallVec::new(); + encode_string(&mut buf_a, &a); + encode_string(&mut buf_b, &b); + prop_assert_eq!( + buf_a.as_slice().cmp(buf_b.as_slice()), + a.as_bytes().cmp(b.as_bytes()) + ); + } + + #[test] + fn prop_composite_roundtrip( + tag_raw in 0u8..=255, + num in any::(), + s1 in "\\PC{0,32}", + s2 in "\\PC{0,32}", + ) { + let tag = super::super::keys::KeyTag::from_raw_unchecked(tag_raw); + let key = KeyBuilder::new() + .tag(tag) + .u64(num) + .string(&s1) + .string(&s2) + .build(); + + let mut reader = KeyReader::new(&key); + prop_assert_eq!(reader.tag(), Some(tag_raw)); + prop_assert_eq!(reader.u64(), Some(num)); + prop_assert_eq!(reader.string(), Some(s1)); + prop_assert_eq!(reader.string(), Some(s2)); + prop_assert!(reader.is_empty()); + } + + #[test] + fn prop_composite_ordering( + tag_raw in 0u8..=10, + a_num in any::(), + b_num in any::(), + a_str in "[a-z]{0,8}", + b_str in "[a-z]{0,8}", + ) { + let tag = super::super::keys::KeyTag::from_raw_unchecked(tag_raw); + let key_a = KeyBuilder::new().tag(tag).u64(a_num).string(&a_str).build(); + let key_b = KeyBuilder::new().tag(tag).u64(b_num).string(&b_str).build(); + + let expected = a_num.cmp(&b_num).then_with(|| a_str.as_bytes().cmp(b_str.as_bytes())); + prop_assert_eq!(key_a.as_slice().cmp(key_b.as_slice()), expected); + } + } +} diff --git a/crates/tranquil-store/src/metastore/event_keys.rs b/crates/tranquil-store/src/metastore/event_keys.rs new file mode 100644 index 0000000..112e612 --- /dev/null +++ b/crates/tranquil-store/src/metastore/event_keys.rs @@ -0,0 +1,230 @@ +use serde::{Deserialize, Serialize}; +use smallvec::SmallVec; + +use super::encoding::KeyBuilder; +use super::keys::{KeyTag, UserHash}; + +const SEQ_META_SCHEMA_VERSION: u8 = 1; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SeqMetaValue { + pub blocks_cids: Vec, +} + +impl SeqMetaValue { + pub fn serialize(&self) -> Vec { + let payload = postcard::to_allocvec(self).expect("SeqMetaValue serialization cannot fail"); + let mut buf = Vec::with_capacity(1 + payload.len()); + buf.push(SEQ_META_SCHEMA_VERSION); + buf.extend_from_slice(&payload); + buf + } + + pub fn deserialize(bytes: &[u8]) -> Option { + let (&version, payload) = bytes.split_first()?; + match version { + SEQ_META_SCHEMA_VERSION => postcard::from_bytes(payload).ok(), + _ => None, + } + } +} + +pub fn rev_to_seq_key(user_hash: UserHash, rev: &str) -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::REV_TO_SEQ) + .u64(user_hash.raw()) + .string(rev) + .build() +} + +pub fn rev_to_seq_user_prefix(user_hash: UserHash) -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::REV_TO_SEQ) + .u64(user_hash.raw()) + .build() +} + +pub fn seq_meta_key(seq: u64) -> SmallVec<[u8; 128]> { + KeyBuilder::new().tag(KeyTag::SEQ_META).u64(seq).build() +} + +pub fn seq_tombstone_key(seq: u64) -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::SEQ_TOMBSTONE) + .u64(seq) + .build() +} + +pub fn did_events_key(user_hash: UserHash, seq: u64) -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::DID_EVENTS) + .u64(user_hash.raw()) + .u64(seq) + .build() +} + +pub fn did_events_prefix(user_hash: UserHash) -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::DID_EVENTS) + .u64(user_hash.raw()) + .build() +} + +pub fn metastore_cursor_key() -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::METASTORE_CURSOR) + .raw(&[0x00]) + .build() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::metastore::encoding::KeyReader; + + #[test] + fn seq_meta_value_roundtrip() { + let value = SeqMetaValue { + blocks_cids: vec!["bafyreiblock1".to_owned(), "bafyreiblock2".to_owned()], + }; + let bytes = value.serialize(); + let decoded = SeqMetaValue::deserialize(&bytes).unwrap(); + assert_eq!(decoded, value); + } + + #[test] + fn seq_meta_value_empty_blocks() { + let value = SeqMetaValue { + blocks_cids: vec![], + }; + let bytes = value.serialize(); + let decoded = SeqMetaValue::deserialize(&bytes).unwrap(); + assert_eq!(decoded, value); + } + + #[test] + fn seq_meta_schema_version_first_byte() { + let value = SeqMetaValue { + blocks_cids: vec![], + }; + let bytes = value.serialize(); + assert_eq!(bytes[0], SEQ_META_SCHEMA_VERSION); + } + + #[test] + fn seq_meta_rejects_unknown_version() { + let value = SeqMetaValue { + blocks_cids: vec![], + }; + let mut bytes = value.serialize(); + bytes[0] = 99; + assert!(SeqMetaValue::deserialize(&bytes).is_none()); + } + + #[test] + fn seq_meta_rejects_empty_input() { + assert!(SeqMetaValue::deserialize(&[]).is_none()); + } + + #[test] + fn rev_to_seq_key_roundtrip() { + let hash = UserHash::from_raw(0xDEAD_BEEF_CAFE_BABE); + let key = rev_to_seq_key(hash, "3k2abcde"); + let mut reader = KeyReader::new(&key); + assert_eq!(reader.tag(), Some(KeyTag::REV_TO_SEQ.raw())); + assert_eq!(reader.u64(), Some(0xDEAD_BEEF_CAFE_BABE)); + assert_eq!(reader.string(), Some("3k2abcde".to_owned())); + assert!(reader.is_empty()); + } + + #[test] + fn rev_to_seq_keys_sort_by_user_then_rev() { + let h1 = UserHash::from_raw(1); + let h2 = UserHash::from_raw(2); + let k1 = rev_to_seq_key(h1, "abc"); + let k2 = rev_to_seq_key(h1, "def"); + let k3 = rev_to_seq_key(h2, "abc"); + assert!(k1.as_slice() < k2.as_slice()); + assert!(k2.as_slice() < k3.as_slice()); + } + + #[test] + fn rev_to_seq_user_prefix_is_prefix_of_full_key() { + let hash = UserHash::from_raw(42); + let prefix = rev_to_seq_user_prefix(hash); + let full = rev_to_seq_key(hash, "some_rev"); + assert!(full.as_slice().starts_with(prefix.as_slice())); + } + + #[test] + fn seq_meta_key_roundtrip() { + let key = seq_meta_key(12345); + let mut reader = KeyReader::new(&key); + assert_eq!(reader.tag(), Some(KeyTag::SEQ_META.raw())); + assert_eq!(reader.u64(), Some(12345)); + assert!(reader.is_empty()); + } + + #[test] + fn seq_meta_keys_sort_by_seq() { + let k1 = seq_meta_key(1); + let k2 = seq_meta_key(2); + let k3 = seq_meta_key(100); + assert!(k1.as_slice() < k2.as_slice()); + assert!(k2.as_slice() < k3.as_slice()); + } + + #[test] + fn seq_tombstone_key_roundtrip() { + let key = seq_tombstone_key(999); + let mut reader = KeyReader::new(&key); + assert_eq!(reader.tag(), Some(KeyTag::SEQ_TOMBSTONE.raw())); + assert_eq!(reader.u64(), Some(999)); + assert!(reader.is_empty()); + } + + #[test] + fn did_events_key_roundtrip() { + let hash = UserHash::from_raw(0xCAFE_BABE_DEAD_BEEF); + let key = did_events_key(hash, 42); + let mut reader = KeyReader::new(&key); + assert_eq!(reader.tag(), Some(KeyTag::DID_EVENTS.raw())); + assert_eq!(reader.u64(), Some(0xCAFE_BABE_DEAD_BEEF)); + assert_eq!(reader.u64(), Some(42)); + assert!(reader.is_empty()); + } + + #[test] + fn did_events_keys_sort_by_user_then_seq() { + let h1 = UserHash::from_raw(1); + let h2 = UserHash::from_raw(2); + let k1 = did_events_key(h1, 10); + let k2 = did_events_key(h1, 20); + let k3 = did_events_key(h2, 5); + assert!(k1.as_slice() < k2.as_slice()); + assert!(k2.as_slice() < k3.as_slice()); + } + + #[test] + fn did_events_prefix_is_prefix_of_full_key() { + let hash = UserHash::from_raw(99); + let prefix = did_events_prefix(hash); + let full = did_events_key(hash, 1); + assert!(full.as_slice().starts_with(prefix.as_slice())); + } + + #[test] + fn metastore_cursor_key_roundtrip() { + let key = metastore_cursor_key(); + let mut reader = KeyReader::new(&key); + assert_eq!(reader.tag(), Some(KeyTag::METASTORE_CURSOR.raw())); + assert_eq!(reader.remaining(), &[0x00]); + } + + #[test] + fn metastore_cursor_key_is_stable() { + let k1 = metastore_cursor_key(); + let k2 = metastore_cursor_key(); + assert_eq!(k1.as_slice(), k2.as_slice()); + } +} diff --git a/crates/tranquil-store/src/metastore/event_ops.rs b/crates/tranquil-store/src/metastore/event_ops.rs new file mode 100644 index 0000000..38aa485 --- /dev/null +++ b/crates/tranquil-store/src/metastore/event_ops.rs @@ -0,0 +1,1734 @@ +use std::collections::HashSet; +use std::sync::Arc; + +use chrono::{DateTime, Utc}; +use fjall::{Database, Keyspace}; +use tracing::warn; +use tranquil_db_traits::{ + AccountStatus, CommitEventData, DbError, EventBlocksCids, RepoEventType, SequenceNumber, + SequencedEvent, +}; +use tranquil_types::{CidLink, Did, Handle}; + +use super::encoding::{KeyReader, exclusive_upper_bound}; +use super::event_keys::{ + SeqMetaValue, did_events_key, did_events_prefix, metastore_cursor_key, rev_to_seq_key, + rev_to_seq_user_prefix, seq_meta_key, seq_tombstone_key, +}; +use super::keys::UserHash; +use super::recovery::CommitMutationSet; +use super::repo_meta::RepoMetaValue; +use crate::eventlog::{DeferredBroadcast, EventLogBridge, EventLogNotifier}; +use crate::io::StorageIO; + +const RECOVERY_BATCH_SIZE: usize = 4096; + +pub struct EventOps { + db: Database, + repo_data: Keyspace, + bridge: Arc>, +} + +impl EventOps { + pub fn new(db: Database, repo_data: Keyspace, bridge: Arc>) -> Self { + Self { + db, + repo_data, + bridge, + } + } + + pub fn notifier(&self) -> EventLogNotifier { + self.bridge.notifier() + } + + pub fn notify_update(&self, _seq: SequenceNumber) -> Result<(), DbError> { + Ok(()) + } + + pub fn insert_commit_event(&self, data: &CommitEventData) -> Result { + let event = Self::build_commit_event(data); + self.append_and_index(&event, &data.did, data.rev.as_deref()) + } + + pub fn append_commit_event_into_batch( + &self, + batch: &mut fjall::OwnedWriteBatch, + data: &CommitEventData, + mutation_set_bytes: Option<&[u8]>, + ) -> Result<(SequenceNumber, DeferredBroadcast), DbError> { + let event = Self::build_commit_event(data); + let payload = crate::eventlog::encode_payload_with_mutations(&event, mutation_set_bytes); + let (seq, deferred) = self + .bridge + .insert_event_deferred_raw(&data.did, data.event_type, payload) + .map_err(|e| DbError::Query(e.to_string()))?; + + let seq_u64 = seq_to_u64(seq)?; + let user_hash = UserHash::from_did(data.did.as_str()); + self.stage_did_event(batch, user_hash, seq_u64); + if let Some(rev) = &data.rev { + self.stage_rev_to_seq(batch, user_hash, rev, seq_u64); + } + self.write_last_applied_cursor(batch, seq_u64); + + Ok((seq, deferred)) + } + + pub fn complete_broadcast(&self, deferred: DeferredBroadcast) { + self.bridge.complete_broadcast(deferred); + } + + fn build_commit_event(data: &CommitEventData) -> SequencedEvent { + SequencedEvent { + seq: SequenceNumber::ZERO, + did: data.did.clone(), + created_at: Utc::now(), + event_type: data.event_type, + commit_cid: data.commit_cid.clone(), + prev_cid: data.prev_cid.clone(), + prev_data_cid: data.prev_data_cid.clone(), + ops: data.ops.clone(), + blobs: data.blobs.clone(), + blocks_cids: data.blocks_cids.clone(), + handle: None, + active: None, + status: None, + rev: data.rev.clone(), + } + } + + pub fn insert_identity_event( + &self, + did: &Did, + handle: Option<&Handle>, + ) -> Result { + let event = SequencedEvent { + seq: SequenceNumber::ZERO, + did: did.clone(), + created_at: Utc::now(), + event_type: RepoEventType::Identity, + commit_cid: None, + prev_cid: None, + prev_data_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + handle: handle.cloned(), + active: None, + status: None, + rev: None, + }; + + self.append_and_index(&event, did, None) + } + + pub fn insert_account_event( + &self, + did: &Did, + status: AccountStatus, + ) -> Result { + let active = Some(status.is_active()); + let event = SequencedEvent { + seq: SequenceNumber::ZERO, + did: did.clone(), + created_at: Utc::now(), + event_type: RepoEventType::Account, + commit_cid: None, + prev_cid: None, + prev_data_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + handle: None, + active, + status: Some(status), + rev: None, + }; + + self.append_and_index(&event, did, None) + } + + pub fn insert_sync_event( + &self, + did: &Did, + commit_cid: &CidLink, + rev: Option<&str>, + ) -> Result { + let event = SequencedEvent { + seq: SequenceNumber::ZERO, + did: did.clone(), + created_at: Utc::now(), + event_type: RepoEventType::Sync, + commit_cid: Some(commit_cid.clone()), + prev_cid: None, + prev_data_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + handle: None, + active: None, + status: None, + rev: rev.map(str::to_owned), + }; + + self.append_and_index(&event, did, rev) + } + + pub fn insert_genesis_commit_event( + &self, + did: &Did, + commit_cid: &CidLink, + mst_root_cid: &CidLink, + rev: &str, + ) -> Result { + let event = SequencedEvent { + seq: SequenceNumber::ZERO, + did: did.clone(), + created_at: Utc::now(), + event_type: RepoEventType::Commit, + commit_cid: Some(commit_cid.clone()), + prev_cid: None, + prev_data_cid: Some(mst_root_cid.clone()), + ops: None, + blobs: None, + blocks_cids: None, + handle: None, + active: None, + status: None, + rev: Some(rev.to_owned()), + }; + + self.append_and_index(&event, did, Some(rev)) + } + + pub fn get_events_since_seq( + &self, + since: SequenceNumber, + limit: Option, + ) -> Result, DbError> { + let events = self.bridge.get_events_since_seq(since, limit)?; + self.apply_sidecars_and_filter(events) + } + + pub fn get_events_in_seq_range( + &self, + start: SequenceNumber, + end: SequenceNumber, + ) -> Result, DbError> { + let events = self.bridge.get_events_in_seq_range(start, end)?; + self.apply_sidecars_and_filter(events) + } + + pub fn get_event_by_seq(&self, seq: SequenceNumber) -> Result, DbError> { + let seq_u64 = match seq.as_u64() { + Some(v) => v, + None => return Ok(None), + }; + + if self.is_tombstoned(seq_u64)? { + return Ok(None); + } + + self.bridge + .get_event_by_seq(seq) + .map(|opt| opt.map(|e| self.merge_sidecar(e))) + } + + pub fn get_events_since_cursor( + &self, + cursor: SequenceNumber, + limit: i64, + ) -> Result, DbError> { + let events = self.bridge.get_events_since_cursor(cursor, limit)?; + self.apply_sidecars_and_filter(events) + } + + pub fn get_max_seq(&self) -> SequenceNumber { + self.bridge.get_max_seq() + } + + pub fn get_min_seq_since( + &self, + since: DateTime, + ) -> Result, DbError> { + self.bridge.get_min_seq_since(since) + } + + pub fn get_events_since_rev( + &self, + did: &Did, + since_rev: &str, + ) -> Result, DbError> { + let user_hash = UserHash::from_did(did.as_str()); + + let key = rev_to_seq_key(user_hash, since_rev); + let since_seq_u64 = match self.repo_data.get(key).map_err(fjall_to_db)? { + Some(bytes) => { + let arr: [u8; 8] = bytes + .as_ref() + .try_into() + .map_err(|_| DbError::Query("corrupt rev_to_seq value".to_owned()))?; + u64::from_be_bytes(arr) + } + None => return Ok(Vec::new()), + }; + + let start_seq = match since_seq_u64.checked_add(1) { + Some(s) => s, + None => return Ok(Vec::new()), + }; + + let user_seqs = self.scan_did_events(user_hash, start_seq)?; + + user_seqs + .into_iter() + .try_fold(Vec::new(), |mut acc, seq_u64| { + if self.is_tombstoned(seq_u64)? { + return Ok(acc); + } + let seq_sn = SequenceNumber::from_raw( + i64::try_from(seq_u64) + .map_err(|_| DbError::Query("seq exceeds i64::MAX".to_owned()))?, + ); + match self.bridge.get_event_by_seq(seq_sn)? { + Some(event) if event.rev.is_some() => { + let merged = self.merge_sidecar(event); + acc.push(EventBlocksCids { + blocks_cids: merged.blocks_cids, + commit_cid: merged.commit_cid, + }); + Ok(acc) + } + _ => Ok(acc), + } + }) + } + + pub fn get_blob_cids_since_rev( + &self, + did: &Did, + since_rev: &str, + ) -> Result, DbError> { + let user_hash = UserHash::from_did(did.as_str()); + + let key = rev_to_seq_key(user_hash, since_rev); + let since_seq_u64 = match self.repo_data.get(key).map_err(fjall_to_db)? { + Some(bytes) => { + let arr: [u8; 8] = bytes + .as_ref() + .try_into() + .map_err(|_| DbError::Query("corrupt rev_to_seq value".to_owned()))?; + u64::from_be_bytes(arr) + } + None => return Ok(Vec::new()), + }; + + let start_seq = match since_seq_u64.checked_add(1) { + Some(s) => s, + None => return Ok(Vec::new()), + }; + + let user_seqs = self.scan_did_events(user_hash, start_seq)?; + + let mut seen = std::collections::BTreeSet::new(); + user_seqs + .into_iter() + .try_fold(Vec::new(), |mut acc, seq_u64| { + if self.is_tombstoned(seq_u64)? { + return Ok(acc); + } + let seq_sn = SequenceNumber::from_raw( + i64::try_from(seq_u64) + .map_err(|_| DbError::Query("seq exceeds i64::MAX".to_owned()))?, + ); + match self.bridge.get_event_by_seq(seq_sn)? { + Some(event) if event.rev.is_some() => { + if let Some(blobs) = event.blobs { + acc.extend( + blobs + .into_iter() + .filter(|b| seen.insert(b.clone())) + .map(CidLink::from), + ); + } + Ok(acc) + } + _ => Ok(acc), + } + }) + } + + pub fn update_seq_blocks_cids( + &self, + seq: SequenceNumber, + blocks_cids: &[String], + ) -> Result<(), DbError> { + let seq_u64 = seq + .as_u64() + .ok_or_else(|| DbError::Query("invalid sequence number".to_owned()))?; + let key = seq_meta_key(seq_u64); + let value = SeqMetaValue { + blocks_cids: blocks_cids.to_vec(), + }; + self.repo_data + .insert(key.as_slice(), value.serialize()) + .map_err(fjall_to_db) + } + + pub fn delete_sequences_except( + &self, + did: &Did, + keep_seq: SequenceNumber, + ) -> Result<(), DbError> { + let keep_raw = keep_seq.as_u64().ok_or_else(|| { + DbError::Query("invalid keep_seq: negative sequence number".to_owned()) + })?; + + let user_hash = UserHash::from_did(did.as_str()); + + let prefix = did_events_prefix(user_hash); + let upper = exclusive_upper_bound(prefix.as_slice()) + .expect("did_events prefix can never be all-0xFF"); + + let seqs_to_tombstone: Result, DbError> = self + .repo_data + .range(prefix.as_slice()..upper.as_slice()) + .map(|guard| { + let (key, _) = guard.into_inner().map_err(fjall_to_db)?; + decode_did_events_seq(key.as_ref()) + }) + .filter(|result| match result { + Ok(seq) => *seq != keep_raw, + Err(_) => true, + }) + .collect(); + + let seqs = seqs_to_tombstone?; + let tombstone_set: HashSet = seqs.iter().copied().collect(); + + let stale_rev_keys = self.collect_stale_rev_keys(user_hash, &tombstone_set)?; + + let mut batch = self.db.batch(); + seqs.iter().for_each(|&seq| { + batch.insert(&self.repo_data, seq_tombstone_key(seq).as_slice(), []); + batch.remove(&self.repo_data, did_events_key(user_hash, seq).as_slice()); + batch.remove(&self.repo_data, seq_meta_key(seq).as_slice()); + }); + stale_rev_keys.iter().for_each(|key| { + batch.remove(&self.repo_data, key.as_slice()); + }); + batch.commit().map_err(fjall_to_db)?; + + Ok(()) + } + + pub fn read_last_applied_cursor(&self) -> Result, DbError> { + let key = metastore_cursor_key(); + match self.repo_data.get(key.as_slice()).map_err(fjall_to_db)? { + Some(bytes) => { + let arr: [u8; 8] = bytes + .as_ref() + .try_into() + .map_err(|_| DbError::Query("corrupt metastore cursor".to_owned()))?; + Ok(Some(u64::from_be_bytes(arr))) + } + None => Ok(None), + } + } + + pub fn write_last_applied_cursor(&self, batch: &mut fjall::OwnedWriteBatch, seq: u64) { + let key = metastore_cursor_key(); + batch.insert(&self.repo_data, key.as_slice(), seq.to_be_bytes()); + } + + pub fn write_last_applied_cursor_direct(&self, seq: u64) -> Result<(), DbError> { + let key = metastore_cursor_key(); + self.repo_data + .insert(key.as_slice(), seq.to_be_bytes()) + .map_err(fjall_to_db) + } + + pub fn recover_sidecar_indexes(&self) -> Result { + let cursor_seq = self.read_last_applied_cursor()?.unwrap_or(0); + let max_raw = self.bridge.get_max_seq().as_u64().unwrap_or(0); + + match max_raw <= cursor_seq { + true => Ok(0), + false => self.recover_page(cursor_seq, 0), + } + } + + pub fn recover_metastore_mutations(&self, indexes: &fjall::Keyspace) -> Result { + let cursor_seq = self.read_last_applied_cursor()?.unwrap_or(0); + let max_raw = self.bridge.get_max_seq().as_u64().unwrap_or(0); + + match max_raw <= cursor_seq { + true => Ok(0), + false => { + tracing::info!( + cursor = cursor_seq, + eventlog_max = max_raw, + gap = max_raw.saturating_sub(cursor_seq), + "replaying metastore mutations from eventlog" + ); + self.recover_mutations_page(indexes, cursor_seq, 0) + } + } + } + + fn recover_mutations_page( + &self, + indexes: &fjall::Keyspace, + cursor: u64, + total: u64, + ) -> Result { + let cursor_sn = SequenceNumber::from_raw( + i64::try_from(cursor) + .map_err(|_| DbError::Query("recovery cursor exceeds i64::MAX".to_owned()))?, + ); + let events_with_mutations = self + .bridge + .get_events_with_mutations_since(cursor_sn, RECOVERY_BATCH_SIZE)?; + + match events_with_mutations.is_empty() { + true => Ok(total), + false => { + let page_len = events_with_mutations.len(); + let mut page_high = cursor; + let mut count = 0u64; + + events_with_mutations.iter().try_for_each(|ewm| { + let seq_u64 = match ewm.event.seq.as_u64() { + Some(v) => v, + None => return Ok(()), + }; + let user_hash = UserHash::from_did(ewm.event.did.as_str()); + + let mut batch = self.db.batch(); + + self.stage_did_event(&mut batch, user_hash, seq_u64); + if let Some(rev) = &ewm.event.rev { + self.stage_rev_to_seq(&mut batch, user_hash, rev, seq_u64); + } + + if let Some(ms_bytes) = &ewm.mutation_set { + let ms = CommitMutationSet::deserialize(ms_bytes).ok_or_else(|| { + DbError::Query(format!("corrupt CommitMutationSet at seq {seq_u64}")) + })?; + + let meta_key = super::repo_meta::repo_meta_key(user_hash); + let current_meta = self + .repo_data + .get(meta_key.as_slice()) + .map_err(fjall_to_db)? + .and_then(|raw| RepoMetaValue::deserialize(&raw)) + .unwrap_or_else(|| RepoMetaValue { + repo_root_cid: vec![], + repo_rev: String::new(), + handle: String::new(), + status: super::repo_meta::RepoStatus::Active, + deactivated_at_ms: None, + takedown_ref: None, + did: Some(ewm.event.did.as_str().to_owned()), + }); + + super::recovery::replay_mutation_set( + &mut batch, + &self.repo_data, + indexes, + user_hash, + ¤t_meta, + &ms, + ) + .map_err(|e| DbError::Query(e.to_string()))?; + } + + self.write_last_applied_cursor(&mut batch, seq_u64); + batch.commit().map_err(fjall_to_db)?; + + page_high = seq_u64.max(page_high); + count = count.saturating_add(1); + Ok::<_, DbError>(()) + })?; + + let new_total = total.saturating_add(count); + match page_len < RECOVERY_BATCH_SIZE { + true => Ok(new_total), + false => self.recover_mutations_page(indexes, page_high, new_total), + } + } + } + } + + fn recover_page(&self, cursor: u64, total: u64) -> Result { + let cursor_sn = SequenceNumber::from_raw( + i64::try_from(cursor) + .map_err(|_| DbError::Query("recovery cursor exceeds i64::MAX".to_owned()))?, + ); + let events = self + .bridge + .get_events_since_seq(cursor_sn, Some(RECOVERY_BATCH_SIZE as i64))?; + + match events.is_empty() { + true => Ok(total), + false => { + let page_len = events.len(); + let mut batch = self.db.batch(); + + let (batch_high, count) = + events.iter().fold((cursor, 0u64), |(high, count), event| { + let seq_u64 = match event.seq.as_u64() { + Some(v) => v, + None => return (high, count), + }; + let user_hash = UserHash::from_did(event.did.as_str()); + + self.stage_did_event(&mut batch, user_hash, seq_u64); + if let Some(rev) = &event.rev { + self.stage_rev_to_seq(&mut batch, user_hash, rev, seq_u64); + } + + (seq_u64.max(high), count.saturating_add(1)) + }); + + self.write_last_applied_cursor(&mut batch, batch_high); + batch.commit().map_err(fjall_to_db)?; + + let new_total = total.saturating_add(count); + match page_len < RECOVERY_BATCH_SIZE { + true => Ok(new_total), + false => self.recover_page(batch_high, new_total), + } + } + } + } + + pub fn append_and_stage_indexes( + &self, + batch: &mut fjall::OwnedWriteBatch, + event: &SequencedEvent, + did: &Did, + rev: Option<&str>, + ) -> Result { + let seq = self + .bridge + .insert_event(event) + .map_err(|e| DbError::Query(e.to_string()))?; + + let seq_u64 = seq_to_u64(seq)?; + let user_hash = UserHash::from_did(did.as_str()); + self.stage_did_event(batch, user_hash, seq_u64); + if let Some(rev) = rev { + self.stage_rev_to_seq(batch, user_hash, rev, seq_u64); + } + self.write_last_applied_cursor(batch, seq_u64); + + Ok(seq) + } + + fn append_and_index( + &self, + event: &SequencedEvent, + did: &Did, + rev: Option<&str>, + ) -> Result { + let mut batch = self.db.batch(); + let seq = self.append_and_stage_indexes(&mut batch, event, did, rev)?; + batch.commit().map_err(fjall_to_db)?; + Ok(seq) + } + + fn stage_did_event(&self, batch: &mut fjall::OwnedWriteBatch, user_hash: UserHash, seq: u64) { + let key = did_events_key(user_hash, seq); + batch.insert(&self.repo_data, key.as_slice(), []); + } + + fn stage_rev_to_seq( + &self, + batch: &mut fjall::OwnedWriteBatch, + user_hash: UserHash, + rev: &str, + seq: u64, + ) { + let key = rev_to_seq_key(user_hash, rev); + batch.insert(&self.repo_data, key.as_slice(), seq.to_be_bytes()); + } + + fn scan_did_events(&self, user_hash: UserHash, start_seq: u64) -> Result, DbError> { + let range_start = did_events_key(user_hash, start_seq); + let range_end = exclusive_upper_bound(did_events_prefix(user_hash).as_slice()) + .expect("did_events prefix can never be all-0xFF"); + + self.repo_data + .range(range_start.as_slice()..range_end.as_slice()) + .try_fold(Vec::new(), |mut acc, guard| { + let (key, _) = guard.into_inner().map_err(fjall_to_db)?; + let seq = decode_did_events_seq(key.as_ref())?; + acc.push(seq); + Ok(acc) + }) + } + + fn collect_stale_rev_keys( + &self, + user_hash: UserHash, + tombstone_set: &HashSet, + ) -> Result>, DbError> { + let rev_prefix = rev_to_seq_user_prefix(user_hash); + let rev_upper = exclusive_upper_bound(rev_prefix.as_slice()) + .expect("rev_to_seq prefix can never be all-0xFF"); + + self.repo_data + .range(rev_prefix.as_slice()..rev_upper.as_slice()) + .try_fold(Vec::new(), |mut acc, guard| { + let (key, val) = guard.into_inner().map_err(fjall_to_db)?; + let val_arr: [u8; 8] = val + .as_ref() + .try_into() + .map_err(|_| DbError::Query("corrupt rev_to_seq value".to_owned()))?; + let stored_seq = u64::from_be_bytes(val_arr); + if tombstone_set.contains(&stored_seq) { + acc.push(key.as_ref().to_vec()); + } + Ok(acc) + }) + } + + fn is_tombstoned(&self, seq: u64) -> Result { + let key = seq_tombstone_key(seq); + match self.repo_data.get(key.as_slice()) { + Ok(Some(_)) => Ok(true), + Ok(None) => Ok(false), + Err(e) => { + warn!(seq, error = %e, "tombstone check failed, propagating error"); + Err(fjall_to_db(e)) + } + } + } + + fn merge_sidecar(&self, mut event: SequencedEvent) -> SequencedEvent { + let seq_u64 = match event.seq.as_u64() { + Some(v) => v, + None => return event, + }; + + let key = seq_meta_key(seq_u64); + match self.repo_data.get(key.as_slice()) { + Ok(Some(sidecar_bytes)) => { + if let Some(sidecar) = SeqMetaValue::deserialize(sidecar_bytes.as_ref()) { + event.blocks_cids = Some(sidecar.blocks_cids); + } + } + Ok(None) => {} + Err(e) => { + warn!(seq = seq_u64, error = %e, "failed to read seq sidecar, returning event without sidecar merge"); + } + } + + event + } + + fn apply_sidecars_and_filter( + &self, + events: Vec, + ) -> Result, DbError> { + events.into_iter().try_fold(Vec::new(), |mut acc, e| { + let tombstoned = match e.seq.as_u64() { + Some(seq_u64) => self.is_tombstoned(seq_u64)?, + None => false, + }; + if !tombstoned { + acc.push(self.merge_sidecar(e)); + } + Ok(acc) + }) + } +} + +fn fjall_to_db(e: fjall::Error) -> DbError { + DbError::Query(e.to_string()) +} + +fn seq_to_u64(seq: SequenceNumber) -> Result { + seq.as_u64() + .ok_or_else(|| DbError::Query("sequence number is negative".to_owned())) +} + +fn decode_did_events_seq(key_bytes: &[u8]) -> Result { + let mut reader = KeyReader::new(key_bytes); + let _tag = reader.tag(); + let _user_hash = reader.u64(); + reader + .u64() + .ok_or_else(|| DbError::Query("corrupt did_events key: missing seq field".to_owned())) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::eventlog::{EventLog, EventLogConfig}; + use crate::io::RealIO; + use sha2::Digest; + use tranquil_db_traits::RepoEventType; + + struct TestHarness { + _metastore_dir: tempfile::TempDir, + _eventlog_dir: tempfile::TempDir, + event_ops: EventOps, + } + + fn setup() -> TestHarness { + let metastore_dir = tempfile::TempDir::new().unwrap(); + let eventlog_dir = tempfile::TempDir::new().unwrap(); + let segments_dir = eventlog_dir.path().join("segments"); + std::fs::create_dir_all(&segments_dir).unwrap(); + + let db = fjall::Database::builder(metastore_dir.path()) + .open() + .unwrap(); + let repo_data = db + .keyspace("repo_data", fjall::KeyspaceCreateOptions::default) + .unwrap(); + + let event_log = EventLog::open( + EventLogConfig { + segments_dir, + ..EventLogConfig::default() + }, + RealIO::new(), + ) + .unwrap(); + + let bridge = Arc::new(EventLogBridge::new(Arc::new(event_log))); + let event_ops = EventOps::new(db, repo_data, bridge); + + TestHarness { + _metastore_dir: metastore_dir, + _eventlog_dir: eventlog_dir, + event_ops, + } + } + + fn test_did() -> Did { + Did::new("did:plc:testuser1234567890abcdef").unwrap() + } + + fn test_cid_link() -> CidLink { + let hash = sha2::Digest::finalize(sha2::Sha256::new()); + let mh = multihash::Multihash::<64>::wrap(0x12, &hash).unwrap(); + let c = cid::Cid::new_v1(0x71, mh); + CidLink::from_cid(&c) + } + + #[test] + fn insert_and_query_commit_event() { + let h = setup(); + let cid = test_cid_link(); + let data = CommitEventData { + did: test_did(), + event_type: RepoEventType::Commit, + commit_cid: Some(cid.clone()), + prev_cid: None, + ops: Some(serde_json::json!([{"action": "create", "path": "app.bsky.feed.post/abc"}])), + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("3k2abcde".to_owned()), + }; + + let seq = h.event_ops.insert_commit_event(&data).unwrap(); + assert!(seq.as_i64() > 0); + + let event = h.event_ops.get_event_by_seq(seq).unwrap().unwrap(); + assert_eq!(event.did.as_str(), test_did().as_str()); + assert_eq!(event.event_type, RepoEventType::Commit); + assert_eq!(event.commit_cid, Some(cid)); + assert_eq!(event.rev, Some("3k2abcde".to_owned())); + } + + #[test] + fn insert_and_query_identity_event() { + let h = setup(); + let handle = Handle::new("alice.test").unwrap(); + + let seq = h + .event_ops + .insert_identity_event(&test_did(), Some(&handle)) + .unwrap(); + assert!(seq.as_i64() > 0); + + let event = h.event_ops.get_event_by_seq(seq).unwrap().unwrap(); + assert_eq!(event.event_type, RepoEventType::Identity); + assert_eq!( + event.handle.as_ref().map(|h| h.as_str()), + Some("alice.test") + ); + } + + #[test] + fn insert_and_query_account_event() { + let h = setup(); + + let seq = h + .event_ops + .insert_account_event(&test_did(), AccountStatus::Deactivated) + .unwrap(); + assert!(seq.as_i64() > 0); + + let event = h.event_ops.get_event_by_seq(seq).unwrap().unwrap(); + assert_eq!(event.event_type, RepoEventType::Account); + assert_eq!(event.status, Some(AccountStatus::Deactivated)); + } + + #[test] + fn insert_and_query_sync_event() { + let h = setup(); + let cid = test_cid_link(); + + let seq = h + .event_ops + .insert_sync_event(&test_did(), &cid, Some("rev1")) + .unwrap(); + assert!(seq.as_i64() > 0); + + let event = h.event_ops.get_event_by_seq(seq).unwrap().unwrap(); + assert_eq!(event.event_type, RepoEventType::Sync); + assert_eq!(event.commit_cid, Some(cid)); + assert_eq!(event.rev, Some("rev1".to_owned())); + } + + #[test] + fn insert_genesis_commit_event() { + let h = setup(); + let commit_cid = test_cid_link(); + let mst_cid = test_cid_link(); + + let seq = h + .event_ops + .insert_genesis_commit_event(&test_did(), &commit_cid, &mst_cid, "genesis_rev") + .unwrap(); + assert!(seq.as_i64() > 0); + + let event = h.event_ops.get_event_by_seq(seq).unwrap().unwrap(); + assert_eq!(event.event_type, RepoEventType::Commit); + assert_eq!(event.commit_cid, Some(commit_cid)); + assert_eq!(event.prev_data_cid, Some(mst_cid)); + assert_eq!(event.rev, Some("genesis_rev".to_owned())); + } + + #[test] + fn get_events_since_seq_returns_ordered() { + let h = setup(); + let did = test_did(); + + let seq1 = h + .event_ops + .insert_account_event(&did, AccountStatus::Active) + .unwrap(); + let seq2 = h.event_ops.insert_identity_event(&did, None).unwrap(); + + let events = h + .event_ops + .get_events_since_seq(SequenceNumber::ZERO, None) + .unwrap(); + assert_eq!(events.len(), 2); + assert_eq!(events[0].seq, seq1); + assert_eq!(events[1].seq, seq2); + } + + #[test] + fn get_events_since_seq_with_limit() { + let h = setup(); + let did = test_did(); + + h.event_ops + .insert_account_event(&did, AccountStatus::Active) + .unwrap(); + h.event_ops.insert_identity_event(&did, None).unwrap(); + + let events = h + .event_ops + .get_events_since_seq(SequenceNumber::ZERO, Some(1)) + .unwrap(); + assert_eq!(events.len(), 1); + } + + #[test] + fn get_events_in_seq_range() { + let h = setup(); + let did = test_did(); + + let seq1 = h + .event_ops + .insert_account_event(&did, AccountStatus::Active) + .unwrap(); + let seq2 = h.event_ops.insert_identity_event(&did, None).unwrap(); + let _seq3 = h.event_ops.insert_identity_event(&did, None).unwrap(); + + let events = h + .event_ops + .get_events_in_seq_range(SequenceNumber::ZERO, seq2) + .unwrap(); + assert_eq!(events.len(), 1); + assert_eq!(events[0].seq, seq1); + } + + #[test] + fn cursor_pagination() { + let h = setup(); + let did = test_did(); + + let seqs: Vec = (0..5) + .map(|_| h.event_ops.insert_identity_event(&did, None).unwrap()) + .collect(); + + let page1 = h + .event_ops + .get_events_since_cursor(SequenceNumber::ZERO, 2) + .unwrap(); + assert_eq!(page1.len(), 2); + assert_eq!(page1[0].seq, seqs[0]); + assert_eq!(page1[1].seq, seqs[1]); + + let page2 = h + .event_ops + .get_events_since_cursor(page1[1].seq, 2) + .unwrap(); + assert_eq!(page2.len(), 2); + assert_eq!(page2[0].seq, seqs[2]); + assert_eq!(page2[1].seq, seqs[3]); + + let page3 = h + .event_ops + .get_events_since_cursor(page2[1].seq, 2) + .unwrap(); + assert_eq!(page3.len(), 1); + assert_eq!(page3[0].seq, seqs[4]); + } + + #[test] + fn get_max_seq() { + let h = setup(); + assert_eq!(h.event_ops.get_max_seq(), SequenceNumber::ZERO); + + let seq = h + .event_ops + .insert_identity_event(&test_did(), None) + .unwrap(); + assert_eq!(h.event_ops.get_max_seq(), seq); + } + + #[test] + fn update_seq_blocks_cids_merges_on_query() { + let h = setup(); + let data = CommitEventData { + did: test_did(), + event_type: RepoEventType::Commit, + commit_cid: Some(test_cid_link()), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev1".to_owned()), + }; + + let seq = h.event_ops.insert_commit_event(&data).unwrap(); + + let blocks = vec!["bafyblock1".to_owned(), "bafyblock2".to_owned()]; + h.event_ops.update_seq_blocks_cids(seq, &blocks).unwrap(); + + let event = h.event_ops.get_event_by_seq(seq).unwrap().unwrap(); + assert_eq!(event.blocks_cids, Some(blocks)); + } + + #[test] + fn delete_sequences_except_tombstones_others() { + let h = setup(); + let did = test_did(); + let cid = test_cid_link(); + + let seq1 = h + .event_ops + .insert_commit_event(&CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(cid.clone()), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev_a".to_owned()), + }) + .unwrap(); + + let seq2 = h + .event_ops + .insert_commit_event(&CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(cid.clone()), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev_b".to_owned()), + }) + .unwrap(); + + h.event_ops.delete_sequences_except(&did, seq2).unwrap(); + + assert!(h.event_ops.get_event_by_seq(seq1).unwrap().is_none()); + assert!(h.event_ops.get_event_by_seq(seq2).unwrap().is_some()); + } + + #[test] + fn tombstoned_events_filtered_from_range_queries() { + let h = setup(); + let did = test_did(); + let cid = test_cid_link(); + + let _seq1 = h + .event_ops + .insert_commit_event(&CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(cid.clone()), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev_x".to_owned()), + }) + .unwrap(); + + let seq2 = h + .event_ops + .insert_commit_event(&CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(cid.clone()), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev_y".to_owned()), + }) + .unwrap(); + + h.event_ops.delete_sequences_except(&did, seq2).unwrap(); + + let events = h + .event_ops + .get_events_since_seq(SequenceNumber::ZERO, None) + .unwrap(); + assert_eq!(events.len(), 1); + assert_eq!(events[0].seq, seq2); + } + + #[test] + fn get_events_since_rev() { + let h = setup(); + let did = test_did(); + let cid = test_cid_link(); + + h.event_ops + .insert_commit_event(&CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(cid.clone()), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: Some(vec!["block_a".to_owned()]), + prev_data_cid: None, + rev: Some("rev_1".to_owned()), + }) + .unwrap(); + + h.event_ops + .insert_commit_event(&CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(cid.clone()), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: Some(vec!["block_b".to_owned()]), + prev_data_cid: None, + rev: Some("rev_2".to_owned()), + }) + .unwrap(); + + let events = h.event_ops.get_events_since_rev(&did, "rev_1").unwrap(); + + assert_eq!(events.len(), 1); + assert_eq!(events[0].blocks_cids, Some(vec!["block_b".to_owned()])); + } + + #[test] + fn get_events_since_rev_unknown_rev_returns_empty() { + let h = setup(); + let events = h + .event_ops + .get_events_since_rev(&test_did(), "nonexistent_rev") + .unwrap(); + assert!(events.is_empty()); + } + + #[test] + fn metastore_cursor_read_write() { + let h = setup(); + assert_eq!(h.event_ops.read_last_applied_cursor().unwrap(), None); + + h.event_ops.write_last_applied_cursor_direct(42).unwrap(); + assert_eq!(h.event_ops.read_last_applied_cursor().unwrap(), Some(42)); + + h.event_ops.write_last_applied_cursor_direct(100).unwrap(); + assert_eq!(h.event_ops.read_last_applied_cursor().unwrap(), Some(100)); + } + + #[test] + fn inserts_advance_cursor() { + let h = setup(); + let did = test_did(); + assert_eq!(h.event_ops.read_last_applied_cursor().unwrap(), None); + + let seq1 = h.event_ops.insert_identity_event(&did, None).unwrap(); + assert_eq!( + h.event_ops.read_last_applied_cursor().unwrap(), + seq1.as_u64() + ); + + let seq2 = h + .event_ops + .insert_account_event(&did, AccountStatus::Active) + .unwrap(); + assert_eq!( + h.event_ops.read_last_applied_cursor().unwrap(), + seq2.as_u64() + ); + assert!(seq2 > seq1); + } + + #[test] + fn get_event_by_seq_none_for_missing() { + let h = setup(); + let result = h + .event_ops + .get_event_by_seq(SequenceNumber::from_raw(9999)) + .unwrap(); + assert!(result.is_none()); + } + + #[test] + fn get_event_by_seq_none_for_negative() { + let h = setup(); + let result = h + .event_ops + .get_event_by_seq(SequenceNumber::from_raw(-1)) + .unwrap(); + assert!(result.is_none()); + } + + #[test] + fn multiple_event_types_interleaved() { + let h = setup(); + let did = test_did(); + let cid = test_cid_link(); + + let s1 = h + .event_ops + .insert_commit_event(&CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(cid.clone()), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("r1".to_owned()), + }) + .unwrap(); + let s2 = h.event_ops.insert_identity_event(&did, None).unwrap(); + let s3 = h + .event_ops + .insert_account_event(&did, AccountStatus::Active) + .unwrap(); + let s4 = h + .event_ops + .insert_sync_event(&did, &cid, Some("r2")) + .unwrap(); + + let events = h + .event_ops + .get_events_since_seq(SequenceNumber::ZERO, None) + .unwrap(); + assert_eq!(events.len(), 4); + assert_eq!(events[0].event_type, RepoEventType::Commit); + assert_eq!(events[1].event_type, RepoEventType::Identity); + assert_eq!(events[2].event_type, RepoEventType::Account); + assert_eq!(events[3].event_type, RepoEventType::Sync); + + assert!(s1 < s2); + assert!(s2 < s3); + assert!(s3 < s4); + } + + #[test] + fn delete_sequences_except_tombstones_all_event_types() { + let h = setup(); + let did = test_did(); + let cid = test_cid_link(); + + let _commit_seq = h + .event_ops + .insert_commit_event(&CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(cid.clone()), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev_keep".to_owned()), + }) + .unwrap(); + + let identity_seq = h.event_ops.insert_identity_event(&did, None).unwrap(); + + let account_seq = h + .event_ops + .insert_account_event(&did, AccountStatus::Active) + .unwrap(); + + let keep_seq = h + .event_ops + .insert_sync_event(&did, &cid, Some("rev_sync")) + .unwrap(); + + h.event_ops.delete_sequences_except(&did, keep_seq).unwrap(); + + assert!( + h.event_ops + .get_event_by_seq(identity_seq) + .unwrap() + .is_none() + ); + assert!(h.event_ops.get_event_by_seq(account_seq).unwrap().is_none()); + assert!(h.event_ops.get_event_by_seq(keep_seq).unwrap().is_some()); + } + + #[test] + fn delete_sequences_except_rejects_negative_keep_seq() { + let h = setup(); + let result = h + .event_ops + .delete_sequences_except(&test_did(), SequenceNumber::from_raw(-1)); + assert!(result.is_err()); + } + + #[test] + fn delete_sequences_except_cleans_rev_to_seq_entries() { + let h = setup(); + let did = test_did(); + let cid = test_cid_link(); + let user_hash = super::UserHash::from_did(did.as_str()); + + let _seq1 = h + .event_ops + .insert_commit_event(&CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(cid.clone()), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev_old".to_owned()), + }) + .unwrap(); + + let seq2 = h + .event_ops + .insert_commit_event(&CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(cid.clone()), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev_keep".to_owned()), + }) + .unwrap(); + + let old_key = super::super::event_keys::rev_to_seq_key(user_hash, "rev_old"); + assert!( + h.event_ops + .repo_data + .get(old_key.as_slice()) + .unwrap() + .is_some() + ); + + h.event_ops.delete_sequences_except(&did, seq2).unwrap(); + + assert!( + h.event_ops + .repo_data + .get(old_key.as_slice()) + .unwrap() + .is_none() + ); + + let keep_key = super::super::event_keys::rev_to_seq_key(user_hash, "rev_keep"); + assert!( + h.event_ops + .repo_data + .get(keep_key.as_slice()) + .unwrap() + .is_some() + ); + } + + #[test] + fn delete_sequences_except_cleans_did_events_entries() { + let h = setup(); + let did = test_did(); + let cid = test_cid_link(); + let user_hash = super::UserHash::from_did(did.as_str()); + + let seq1 = h + .event_ops + .insert_commit_event(&CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(cid.clone()), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev_a".to_owned()), + }) + .unwrap(); + + let seq2 = h + .event_ops + .insert_commit_event(&CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(cid.clone()), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev_b".to_owned()), + }) + .unwrap(); + + h.event_ops.delete_sequences_except(&did, seq2).unwrap(); + + let removed_key = + super::super::event_keys::did_events_key(user_hash, seq1.as_u64().unwrap()); + assert!( + h.event_ops + .repo_data + .get(removed_key.as_slice()) + .unwrap() + .is_none() + ); + + let kept_key = super::super::event_keys::did_events_key(user_hash, seq2.as_u64().unwrap()); + assert!( + h.event_ops + .repo_data + .get(kept_key.as_slice()) + .unwrap() + .is_some() + ); + } + + #[test] + fn delete_sequences_except_cleans_seq_meta_entries() { + let h = setup(); + let did = test_did(); + let cid = test_cid_link(); + + let seq1 = h + .event_ops + .insert_commit_event(&CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(cid.clone()), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev_a".to_owned()), + }) + .unwrap(); + + let seq2 = h + .event_ops + .insert_commit_event(&CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(cid.clone()), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev_b".to_owned()), + }) + .unwrap(); + + h.event_ops + .update_seq_blocks_cids(seq1, &["block1".to_owned()]) + .unwrap(); + h.event_ops + .update_seq_blocks_cids(seq2, &["block2".to_owned()]) + .unwrap(); + + let stale_key = super::super::event_keys::seq_meta_key(seq1.as_u64().unwrap()); + assert!( + h.event_ops + .repo_data + .get(stale_key.as_slice()) + .unwrap() + .is_some() + ); + + h.event_ops.delete_sequences_except(&did, seq2).unwrap(); + + assert!( + h.event_ops + .repo_data + .get(stale_key.as_slice()) + .unwrap() + .is_none() + ); + + let kept_key = super::super::event_keys::seq_meta_key(seq2.as_u64().unwrap()); + assert!( + h.event_ops + .repo_data + .get(kept_key.as_slice()) + .unwrap() + .is_some() + ); + } + + #[test] + fn get_events_since_rev_excludes_events_without_rev() { + let h = setup(); + let did = test_did(); + let cid = test_cid_link(); + + h.event_ops + .insert_commit_event(&CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(cid.clone()), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: Some(vec!["block_1".to_owned()]), + prev_data_cid: None, + rev: Some("rev_1".to_owned()), + }) + .unwrap(); + + h.event_ops.insert_identity_event(&did, None).unwrap(); + + h.event_ops + .insert_account_event(&did, AccountStatus::Active) + .unwrap(); + + h.event_ops + .insert_commit_event(&CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(cid.clone()), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: Some(vec!["block_2".to_owned()]), + prev_data_cid: None, + rev: Some("rev_2".to_owned()), + }) + .unwrap(); + + let events = h.event_ops.get_events_since_rev(&did, "rev_1").unwrap(); + + assert_eq!(events.len(), 1); + assert_eq!(events[0].blocks_cids, Some(vec!["block_2".to_owned()])); + } + + #[test] + fn sync_event_with_rev_appears_in_get_events_since_rev() { + let h = setup(); + let did = test_did(); + let cid = test_cid_link(); + + h.event_ops + .insert_commit_event(&CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(cid.clone()), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev_a".to_owned()), + }) + .unwrap(); + + h.event_ops + .insert_sync_event(&did, &cid, Some("rev_b")) + .unwrap(); + + let events = h.event_ops.get_events_since_rev(&did, "rev_a").unwrap(); + + assert_eq!(events.len(), 1); + assert_eq!(events[0].commit_cid, Some(cid)); + } + + #[test] + fn recover_sidecar_indexes_no_gap() { + let h = setup(); + let did = test_did(); + + let seq = h + .event_ops + .insert_commit_event(&CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(test_cid_link()), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev_1".to_owned()), + }) + .unwrap(); + + h.event_ops + .write_last_applied_cursor_direct(seq.as_u64().unwrap()) + .unwrap(); + + let recovered = h.event_ops.recover_sidecar_indexes().unwrap(); + assert_eq!(recovered, 0); + } + + #[test] + fn recover_sidecar_indexes_rebuilds_after_gap() { + let h = setup(); + let did = test_did(); + let cid = test_cid_link(); + + let seq1 = h + .event_ops + .insert_commit_event(&CommitEventData { + did: did.clone(), + event_type: RepoEventType::Commit, + commit_cid: Some(cid.clone()), + prev_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + prev_data_cid: None, + rev: Some("rev_1".to_owned()), + }) + .unwrap(); + + h.event_ops + .write_last_applied_cursor_direct(seq1.as_u64().unwrap()) + .unwrap(); + + let crash_event = SequencedEvent { + seq: SequenceNumber::ZERO, + did: did.clone(), + created_at: Utc::now(), + event_type: RepoEventType::Commit, + commit_cid: Some(cid.clone()), + prev_cid: None, + prev_data_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + handle: None, + active: None, + status: None, + rev: Some("rev_2".to_owned()), + }; + h.event_ops.bridge.insert_event(&crash_event).unwrap(); + + let identity_event = SequencedEvent { + seq: SequenceNumber::ZERO, + did: did.clone(), + created_at: Utc::now(), + event_type: RepoEventType::Identity, + commit_cid: None, + prev_cid: None, + prev_data_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + handle: None, + active: None, + status: None, + rev: None, + }; + h.event_ops.bridge.insert_event(&identity_event).unwrap(); + + let crash_event_3 = SequencedEvent { + seq: SequenceNumber::ZERO, + did: did.clone(), + created_at: Utc::now(), + event_type: RepoEventType::Commit, + commit_cid: Some(cid.clone()), + prev_cid: None, + prev_data_cid: None, + ops: None, + blobs: None, + blocks_cids: None, + handle: None, + active: None, + status: None, + rev: Some("rev_3".to_owned()), + }; + h.event_ops.bridge.insert_event(&crash_event_3).unwrap(); + + assert!( + h.event_ops + .get_events_since_rev(&did, "rev_2") + .unwrap() + .is_empty() + ); + + let recovered = h.event_ops.recover_sidecar_indexes().unwrap(); + assert_eq!(recovered, 3); + + let events = h.event_ops.get_events_since_rev(&did, "rev_2").unwrap(); + assert_eq!(events.len(), 1); + + let cursor = h.event_ops.read_last_applied_cursor().unwrap(); + assert!(cursor.is_some()); + } +} diff --git a/crates/tranquil-store/src/metastore/handler.rs b/crates/tranquil-store/src/metastore/handler.rs new file mode 100644 index 0000000..db15f0d --- /dev/null +++ b/crates/tranquil-store/src/metastore/handler.rs @@ -0,0 +1,1729 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::thread::JoinHandle; + +use chrono::{DateTime, Utc}; +use tokio::sync::oneshot; +use tranquil_db_traits::{ + AccountStatus, ApplyCommitError, ApplyCommitInput, ApplyCommitResult, Backlink, + BrokenGenesisCommit, CommitEventData, DbError, EventBlocksCids, ImportBlock, ImportRecord, + ImportRepoError, SequenceNumber, SequencedEvent, UserNeedingRecordBlobsBackfill, + UserWithoutBlocks, +}; +use tranquil_types::{AtUri, CidLink, Did, Handle, Nsid, Rkey}; +use uuid::Uuid; + +use super::MetastoreError; +use super::commit_ops::CommitOps; +use super::event_ops::EventOps; +use super::keys::UserHash; +use super::record_ops::ListRecordsQuery; +use super::user_hash::UserHashMap; +use crate::blockstore::TranquilBlockStore; +use crate::eventlog::EventLogBridge; +use crate::io::StorageIO; +use crate::metastore::Metastore; + +type Tx = oneshot::Sender>; + +fn metastore_to_db(e: MetastoreError) -> DbError { + match e { + MetastoreError::Fjall(e) => DbError::Query(e.to_string()), + MetastoreError::Lsm(e) => DbError::Query(e.to_string()), + MetastoreError::VersionMismatch { expected, found } => DbError::Query(format!( + "format version mismatch: expected {expected}, found {found}" + )), + MetastoreError::CorruptData(msg) => DbError::CorruptData(msg), + MetastoreError::InvalidInput(msg) => DbError::Query(msg.to_string()), + MetastoreError::UserHashCollision { + hash, + existing_uuid, + new_uuid, + } => DbError::Constraint(format!( + "user hash collision: {hash} maps to both {existing_uuid} and {new_uuid}" + )), + } +} + +enum Routing { + Sharded(u64), + Global, +} + +fn uuid_to_routing(user_hashes: &UserHashMap, user_id: &Uuid) -> Routing { + match user_hashes.get(user_id) { + Some(h) => Routing::Sharded(h.raw()), + None => Routing::Sharded(user_id.as_u128() as u64), + } +} + +fn did_to_routing(did: &str) -> Routing { + Routing::Sharded(UserHash::from_did(did).raw()) +} + +fn cid_to_routing(cid: &CidLink) -> Routing { + use siphasher::sip::SipHasher24; + use std::hash::{Hash, Hasher}; + let mut hasher = SipHasher24::new(); + cid.as_str().hash(&mut hasher); + Routing::Sharded(hasher.finish()) +} + +pub enum MetastoreRequest { + Repo(RepoRequest), + Record(RecordRequest), + UserBlock(UserBlockRequest), + Event(EventRequest), + Commit(Box), + Backlink(BacklinkRequest), + Blob(BlobRequest), +} + +impl MetastoreRequest { + fn routing(&self, user_hashes: &UserHashMap) -> Routing { + match self { + Self::Repo(r) => r.routing(user_hashes), + Self::Record(r) => r.routing(user_hashes), + Self::UserBlock(r) => r.routing(user_hashes), + Self::Event(r) => r.routing(), + Self::Commit(r) => r.routing(user_hashes), + Self::Backlink(r) => r.routing(user_hashes), + Self::Blob(r) => r.routing(user_hashes), + } + } +} + +pub enum RepoRequest { + CreateRepoFull { + user_id: Uuid, + did: Did, + handle: Handle, + repo_root_cid: CidLink, + repo_rev: String, + tx: Tx<()>, + }, + UpdateRepoRoot { + user_id: Uuid, + repo_root_cid: CidLink, + repo_rev: String, + tx: Tx<()>, + }, + UpdateRepoRev { + user_id: Uuid, + repo_rev: String, + tx: Tx<()>, + }, + DeleteRepo { + user_id: Uuid, + tx: Tx<()>, + }, + GetRepoRootForUpdate { + user_id: Uuid, + tx: Tx>, + }, + GetRepo { + user_id: Uuid, + tx: Tx>, + }, + GetRepoRootByDid { + did: Did, + tx: Tx>, + }, + CountRepos { + tx: Tx, + }, + GetReposWithoutRev { + tx: Tx>, + }, + GetRepoRootCidByUserId { + user_id: Uuid, + tx: Tx>, + }, + GetAccountWithRepo { + did: Did, + tx: Tx>, + }, + ListReposPaginated { + cursor_user_hash: Option, + limit: usize, + tx: Tx>, + }, + UpdateRepoStatus { + did: Did, + takedown: Option, + takedown_ref: Option, + deactivated: Option, + tx: Tx<()>, + }, +} + +impl RepoRequest { + fn routing(&self, user_hashes: &UserHashMap) -> Routing { + match self { + Self::CreateRepoFull { did, .. } => did_to_routing(did.as_str()), + Self::UpdateRepoRoot { user_id, .. } + | Self::UpdateRepoRev { user_id, .. } + | Self::DeleteRepo { user_id, .. } + | Self::GetRepoRootForUpdate { user_id, .. } + | Self::GetRepo { user_id, .. } + | Self::GetRepoRootCidByUserId { user_id, .. } => uuid_to_routing(user_hashes, user_id), + Self::GetRepoRootByDid { did, .. } + | Self::GetAccountWithRepo { did, .. } + | Self::UpdateRepoStatus { did, .. } => did_to_routing(did.as_str()), + Self::CountRepos { .. } + | Self::GetReposWithoutRev { .. } + | Self::ListReposPaginated { .. } => Routing::Global, + } + } +} + +pub enum RecordRequest { + UpsertRecords { + repo_id: Uuid, + collections: Vec, + rkeys: Vec, + record_cids: Vec, + repo_rev: String, + tx: Tx<()>, + }, + DeleteRecords { + repo_id: Uuid, + collections: Vec, + rkeys: Vec, + tx: Tx<()>, + }, + DeleteAllRecords { + repo_id: Uuid, + tx: Tx<()>, + }, + GetRecordCid { + repo_id: Uuid, + collection: Nsid, + rkey: Rkey, + tx: Tx>, + }, + ListRecords { + repo_id: Uuid, + collection: Nsid, + cursor: Option, + limit: i64, + reverse: bool, + rkey_start: Option, + rkey_end: Option, + tx: Tx>, + }, + GetAllRecords { + repo_id: Uuid, + tx: Tx>, + }, + ListCollections { + repo_id: Uuid, + tx: Tx>, + }, + CountRecords { + repo_id: Uuid, + tx: Tx, + }, + CountAllRecords { + tx: Tx, + }, + GetRecordByCid { + cid: CidLink, + tx: Tx>, + }, + SetRecordTakedown { + cid: CidLink, + takedown_ref: Option, + scope_user: Option, + tx: Tx<()>, + }, +} + +impl RecordRequest { + fn routing(&self, user_hashes: &UserHashMap) -> Routing { + match self { + Self::UpsertRecords { repo_id, .. } + | Self::DeleteRecords { repo_id, .. } + | Self::DeleteAllRecords { repo_id, .. } + | Self::GetRecordCid { repo_id, .. } + | Self::ListRecords { repo_id, .. } + | Self::GetAllRecords { repo_id, .. } + | Self::ListCollections { repo_id, .. } + | Self::CountRecords { repo_id, .. } => uuid_to_routing(user_hashes, repo_id), + Self::CountAllRecords { .. } | Self::GetRecordByCid { .. } => Routing::Global, + Self::SetRecordTakedown { + scope_user: Some(user_id), + .. + } => uuid_to_routing(user_hashes, user_id), + Self::SetRecordTakedown { .. } => Routing::Global, + } + } +} + +pub enum UserBlockRequest { + InsertUserBlocks { + user_id: Uuid, + block_cids: Vec>, + repo_rev: String, + tx: Tx<()>, + }, + DeleteUserBlocks { + user_id: Uuid, + block_cids: Vec>, + tx: Tx<()>, + }, + GetUserBlockCidsSinceRev { + user_id: Uuid, + since_rev: String, + tx: Tx>>, + }, + CountUserBlocks { + user_id: Uuid, + tx: Tx, + }, + FindUnreferencedBlocks { + candidate_cids: Vec>, + tx: Tx>>, + }, +} + +impl UserBlockRequest { + fn routing(&self, user_hashes: &UserHashMap) -> Routing { + match self { + Self::InsertUserBlocks { user_id, .. } + | Self::DeleteUserBlocks { user_id, .. } + | Self::GetUserBlockCidsSinceRev { user_id, .. } + | Self::CountUserBlocks { user_id, .. } => uuid_to_routing(user_hashes, user_id), + Self::FindUnreferencedBlocks { .. } => Routing::Global, + } + } +} + +pub enum EventRequest { + InsertCommitEvent { + data: CommitEventData, + tx: Tx, + }, + InsertIdentityEvent { + did: Did, + handle: Option, + tx: Tx, + }, + InsertAccountEvent { + did: Did, + status: AccountStatus, + tx: Tx, + }, + InsertSyncEvent { + did: Did, + commit_cid: CidLink, + rev: Option, + tx: Tx, + }, + InsertGenesisCommitEvent { + did: Did, + commit_cid: CidLink, + mst_root_cid: CidLink, + rev: String, + tx: Tx, + }, + UpdateSeqBlocksCids { + seq: SequenceNumber, + blocks_cids: Vec, + tx: Tx<()>, + }, + DeleteSequencesExcept { + did: Did, + keep_seq: SequenceNumber, + tx: Tx<()>, + }, + GetMaxSeq { + tx: Tx, + }, + GetMinSeqSince { + since: DateTime, + tx: Tx>, + }, + GetEventsSinceSeq { + since_seq: SequenceNumber, + limit: Option, + tx: Tx>, + }, + GetEventsInSeqRange { + start_seq: SequenceNumber, + end_seq: SequenceNumber, + tx: Tx>, + }, + GetEventBySeq { + seq: SequenceNumber, + tx: Tx>, + }, + GetEventsSinceCursor { + cursor: SequenceNumber, + limit: i64, + tx: Tx>, + }, + GetEventsSinceRev { + did: Did, + since_rev: String, + tx: Tx>, + }, + NotifyUpdate { + seq: SequenceNumber, + tx: Tx<()>, + }, +} + +impl EventRequest { + fn routing(&self) -> Routing { + match self { + Self::InsertCommitEvent { data, .. } => { + Routing::Sharded(UserHash::from_did(data.did.as_str()).raw()) + } + Self::InsertIdentityEvent { did, .. } + | Self::InsertAccountEvent { did, .. } + | Self::InsertSyncEvent { did, .. } + | Self::InsertGenesisCommitEvent { did, .. } + | Self::DeleteSequencesExcept { did, .. } + | Self::GetEventsSinceRev { did, .. } => { + Routing::Sharded(UserHash::from_did(did.as_str()).raw()) + } + Self::UpdateSeqBlocksCids { .. } + | Self::GetMaxSeq { .. } + | Self::GetMinSeqSince { .. } + | Self::GetEventsSinceSeq { .. } + | Self::GetEventsInSeqRange { .. } + | Self::GetEventBySeq { .. } + | Self::GetEventsSinceCursor { .. } + | Self::NotifyUpdate { .. } => Routing::Global, + } + } +} + +pub enum CommitRequest { + ApplyCommit { + input: Box, + tx: oneshot::Sender>, + }, + ImportRepoData { + user_id: Uuid, + blocks: Vec, + records: Vec, + expected_root_cid: Option, + tx: oneshot::Sender>, + }, + GetBrokenGenesisCommits { + tx: Tx>, + }, + GetUsersWithoutBlocks { + tx: Tx>, + }, + GetUsersNeedingRecordBlobsBackfill { + limit: i64, + tx: Tx>, + }, + InsertRecordBlobs { + repo_id: Uuid, + record_uris: Vec, + blob_cids: Vec, + tx: Tx<()>, + }, +} + +impl CommitRequest { + fn routing(&self, user_hashes: &UserHashMap) -> Routing { + match self { + Self::ApplyCommit { input, .. } => did_to_routing(input.did.as_str()), + Self::ImportRepoData { user_id, .. } + | Self::InsertRecordBlobs { + repo_id: user_id, .. + } => uuid_to_routing(user_hashes, user_id), + Self::GetBrokenGenesisCommits { .. } + | Self::GetUsersWithoutBlocks { .. } + | Self::GetUsersNeedingRecordBlobsBackfill { .. } => Routing::Global, + } + } +} + +pub enum BacklinkRequest { + GetBacklinkConflicts { + repo_id: Uuid, + collection: Nsid, + backlinks: Vec, + tx: Tx>, + }, + AddBacklinks { + repo_id: Uuid, + backlinks: Vec, + tx: Tx<()>, + }, + RemoveBacklinksByUri { + uri: AtUri, + tx: Tx<()>, + }, + RemoveBacklinksByRepo { + repo_id: Uuid, + tx: Tx<()>, + }, +} + +impl BacklinkRequest { + fn routing(&self, user_hashes: &UserHashMap) -> Routing { + match self { + Self::GetBacklinkConflicts { repo_id, .. } + | Self::AddBacklinks { repo_id, .. } + | Self::RemoveBacklinksByRepo { repo_id, .. } => uuid_to_routing(user_hashes, repo_id), + Self::RemoveBacklinksByUri { uri, .. } => match uri.did() { + Some(did) => did_to_routing(did), + None => Routing::Global, + }, + } + } +} + +pub enum BlobRequest { + InsertBlob { + cid: CidLink, + mime_type: String, + size_bytes: i64, + created_by_user: Uuid, + storage_key: String, + tx: Tx>, + }, + GetBlobMetadata { + cid: CidLink, + tx: Tx>, + }, + GetBlobWithTakedown { + cid: CidLink, + tx: Tx>, + }, + GetBlobStorageKey { + cid: CidLink, + tx: Tx>, + }, + ListBlobsByUser { + user_id: Uuid, + cursor: Option, + limit: i64, + tx: Tx>, + }, + ListBlobsSinceRev { + did: Did, + since: String, + tx: Tx>, + }, + CountBlobsByUser { + user_id: Uuid, + tx: Tx, + }, + SumBlobStorage { + tx: Tx, + }, + UpdateBlobTakedown { + cid: CidLink, + takedown_ref: Option, + tx: Tx, + }, + DeleteBlobByCid { + cid: CidLink, + tx: Tx, + }, + DeleteBlobsByUser { + user_id: Uuid, + tx: Tx, + }, + GetBlobStorageKeysByUser { + user_id: Uuid, + tx: Tx>, + }, + ListMissingBlobs { + repo_id: Uuid, + cursor: Option, + limit: i64, + tx: Tx>, + }, + CountDistinctRecordBlobs { + repo_id: Uuid, + tx: Tx, + }, + GetBlobsForExport { + repo_id: Uuid, + tx: Tx>, + }, +} + +impl BlobRequest { + fn routing(&self, user_hashes: &UserHashMap) -> Routing { + match self { + Self::InsertBlob { cid, .. } + | Self::UpdateBlobTakedown { cid, .. } + | Self::DeleteBlobByCid { cid, .. } => cid_to_routing(cid), + + Self::DeleteBlobsByUser { user_id, .. } => uuid_to_routing(user_hashes, user_id), + + Self::GetBlobMetadata { .. } + | Self::GetBlobWithTakedown { .. } + | Self::GetBlobStorageKey { .. } + | Self::SumBlobStorage { .. } => Routing::Global, + + Self::ListBlobsByUser { user_id, .. } + | Self::CountBlobsByUser { user_id, .. } + | Self::GetBlobStorageKeysByUser { user_id, .. } => { + uuid_to_routing(user_hashes, user_id) + } + Self::ListMissingBlobs { repo_id, .. } + | Self::CountDistinctRecordBlobs { repo_id, .. } + | Self::GetBlobsForExport { repo_id, .. } => uuid_to_routing(user_hashes, repo_id), + Self::ListBlobsSinceRev { did, .. } => did_to_routing(did.as_str()), + } + } +} + +fn convert_repo_info(r: super::repo_ops::RepoInfo) -> tranquil_db_traits::RepoInfo { + tranquil_db_traits::RepoInfo { + user_id: r.user_id, + repo_root_cid: r.repo_root_cid, + repo_rev: r.repo_rev, + } +} + +fn convert_repo_account( + r: super::repo_ops::RepoAccountEntry, +) -> tranquil_db_traits::RepoAccountInfo { + tranquil_db_traits::RepoAccountInfo { + user_id: r.user_id, + did: r.did, + deactivated_at: r.deactivated_at, + takedown_ref: r.takedown_ref, + repo_root_cid: r.repo_root_cid, + } +} + +fn convert_repo_list_entry( + r: super::repo_ops::RepoListEntry, +) -> Result { + let did = r + .did + .ok_or(DbError::CorruptData("repo_meta missing DID field"))?; + Ok(tranquil_db_traits::RepoListItem { + did: Did::from(did), + deactivated_at: r.deactivated_at, + takedown_ref: r.takedown_ref, + repo_root_cid: r.repo_root_cid, + repo_rev: r.repo_rev, + }) +} + +fn convert_record_info(r: super::record_ops::RecordInfo) -> tranquil_db_traits::RecordInfo { + tranquil_db_traits::RecordInfo { + rkey: r.rkey, + record_cid: r.record_cid, + } +} + +fn convert_full_record_info( + r: super::record_ops::FullRecordInfo, +) -> tranquil_db_traits::FullRecordInfo { + tranquil_db_traits::FullRecordInfo { + collection: r.collection, + rkey: r.rkey, + record_cid: r.record_cid, + } +} + +fn convert_record_with_takedown( + r: super::record_ops::RecordWithTakedown, +) -> tranquil_db_traits::RecordWithTakedown { + tranquil_db_traits::RecordWithTakedown { + id: r.id, + takedown_ref: r.takedown_ref, + } +} + +fn convert_without_rev( + r: super::repo_ops::RepoWithoutRevEntry, +) -> tranquil_db_traits::RepoWithoutRev { + tranquil_db_traits::RepoWithoutRev { + user_id: r.user_id, + repo_root_cid: r.repo_root_cid, + } +} + +struct HandlerState { + metastore: Metastore, + event_ops: EventOps, + commit_ops: CommitOps, +} + +fn dispatch_repo(state: &HandlerState, req: RepoRequest) { + match req { + RepoRequest::CreateRepoFull { + user_id, + did, + handle, + repo_root_cid, + repo_rev, + tx, + } => { + let result = state + .metastore + .repo_ops() + .create_repo( + state.metastore.database(), + user_id, + &did, + &handle, + &repo_root_cid, + &repo_rev, + ) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + RepoRequest::UpdateRepoRoot { + user_id, + repo_root_cid, + repo_rev, + tx, + } => { + let result = state + .metastore + .repo_ops() + .update_repo_root( + state.metastore.database(), + user_id, + &repo_root_cid, + &repo_rev, + ) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + RepoRequest::UpdateRepoRev { + user_id, + repo_rev, + tx, + } => { + let result = state + .metastore + .repo_ops() + .update_repo_rev(state.metastore.database(), user_id, &repo_rev) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + RepoRequest::DeleteRepo { user_id, tx } => { + let result = state + .metastore + .repo_ops() + .delete_repo(state.metastore.database(), user_id) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + RepoRequest::GetRepoRootForUpdate { user_id, tx } => { + let result = state + .metastore + .repo_ops() + .get_repo_root_for_update(user_id) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + RepoRequest::GetRepo { user_id, tx } => { + let result = state + .metastore + .repo_ops() + .get_repo(user_id) + .map(|opt| opt.map(convert_repo_info)) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + RepoRequest::GetRepoRootByDid { did, tx } => { + let result = state + .metastore + .repo_ops() + .get_repo_root_by_did(&did) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + RepoRequest::CountRepos { tx } => { + let result = state + .metastore + .repo_ops() + .count_repos() + .map_err(metastore_to_db); + let _ = tx.send(result); + } + RepoRequest::GetReposWithoutRev { tx } => { + let result = state + .metastore + .repo_ops() + .get_repos_without_rev(MAX_REPOS_WITHOUT_REV) + .map(|v| v.into_iter().map(convert_without_rev).collect()) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + RepoRequest::GetRepoRootCidByUserId { user_id, tx } => { + let result = state + .metastore + .repo_ops() + .get_repo_root_cid_by_user_id(user_id) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + RepoRequest::GetAccountWithRepo { did, tx } => { + let result = state + .metastore + .repo_ops() + .get_account_with_repo(&did) + .map(|opt| opt.map(convert_repo_account)) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + RepoRequest::ListReposPaginated { + cursor_user_hash, + limit, + tx, + } => { + let result = state + .metastore + .repo_ops() + .list_repos_paginated(cursor_user_hash, limit) + .map_err(metastore_to_db) + .and_then(|entries| entries.into_iter().map(convert_repo_list_entry).collect()); + let _ = tx.send(result); + } + RepoRequest::UpdateRepoStatus { + did, + takedown, + takedown_ref, + deactivated, + tx, + } => { + let result = state + .metastore + .repo_ops() + .update_repo_status( + state.metastore.database(), + &did, + takedown, + takedown_ref.as_deref(), + deactivated, + ) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + } +} + +fn dispatch_record(state: &HandlerState, req: RecordRequest) { + match req { + RecordRequest::UpsertRecords { + repo_id, + collections, + rkeys, + record_cids, + repo_rev, + tx, + } => { + let result = (|| { + let (user_hash, mut meta) = state + .metastore + .repo_ops() + .get_repo_meta(repo_id) + .map_err(metastore_to_db)? + .ok_or(DbError::Query("unknown user_id".to_string()))?; + let writes: Vec> = collections + .iter() + .zip(rkeys.iter()) + .zip(record_cids.iter()) + .map(|((c, r), cid)| super::record_ops::RecordWrite { + collection: c, + rkey: r, + cid, + }) + .collect(); + let mut batch = state.metastore.database().batch(); + state + .metastore + .record_ops() + .upsert_records(&mut batch, user_hash, &writes) + .map_err(metastore_to_db)?; + meta.repo_rev = repo_rev; + state + .metastore + .repo_ops() + .write_repo_meta(&mut batch, user_hash, &meta); + batch.commit().map_err(|e| DbError::Query(e.to_string())) + })(); + let _ = tx.send(result); + } + RecordRequest::DeleteRecords { + repo_id, + collections, + rkeys, + tx, + } => { + let result = (|| { + let user_hash = state + .metastore + .user_hashes() + .get(&repo_id) + .ok_or(DbError::Query("unknown user_id".to_string()))?; + let deletes: Vec> = collections + .iter() + .zip(rkeys.iter()) + .map(|(c, r)| super::record_ops::RecordDelete { + collection: c, + rkey: r, + }) + .collect(); + let mut batch = state.metastore.database().batch(); + state + .metastore + .record_ops() + .delete_records(&mut batch, user_hash, &deletes); + batch.commit().map_err(|e| DbError::Query(e.to_string())) + })(); + let _ = tx.send(result); + } + RecordRequest::DeleteAllRecords { repo_id, tx } => { + let result = (|| { + let user_hash = state + .metastore + .user_hashes() + .get(&repo_id) + .ok_or(DbError::Query("unknown user_id".to_string()))?; + let mut batch = state.metastore.database().batch(); + state + .metastore + .record_ops() + .delete_all_records(&mut batch, user_hash) + .map_err(metastore_to_db)?; + batch.commit().map_err(|e| DbError::Query(e.to_string())) + })(); + let _ = tx.send(result); + } + RecordRequest::GetRecordCid { + repo_id, + collection, + rkey, + tx, + } => { + let result = state + .metastore + .record_ops() + .get_record_cid(repo_id, &collection, &rkey) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + RecordRequest::ListRecords { + repo_id, + collection, + cursor, + limit, + reverse, + rkey_start, + rkey_end, + tx, + } => { + let query = ListRecordsQuery { + user_id: repo_id, + collection: &collection, + cursor: cursor.as_ref(), + limit: usize::try_from(limit).unwrap_or(usize::MAX), + reverse, + rkey_start: rkey_start.as_ref(), + rkey_end: rkey_end.as_ref(), + }; + let result = state + .metastore + .record_ops() + .list_records(&query) + .map(|v| v.into_iter().map(convert_record_info).collect()) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + RecordRequest::GetAllRecords { repo_id, tx } => { + let result = state + .metastore + .record_ops() + .get_all_records(repo_id) + .map(|v| v.into_iter().map(convert_full_record_info).collect()) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + RecordRequest::ListCollections { repo_id, tx } => { + let result = state + .metastore + .record_ops() + .list_collections(repo_id) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + RecordRequest::CountRecords { repo_id, tx } => { + let result = state + .metastore + .record_ops() + .count_records(repo_id) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + RecordRequest::CountAllRecords { tx } => { + let result = state + .metastore + .record_ops() + .count_all_records() + .map_err(metastore_to_db); + let _ = tx.send(result); + } + RecordRequest::GetRecordByCid { cid, tx } => { + let result = state + .metastore + .record_ops() + .get_record_by_cid(&cid, None) + .map(|opt| opt.map(convert_record_with_takedown)) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + RecordRequest::SetRecordTakedown { + cid, + takedown_ref, + scope_user, + tx, + } => { + let result = state + .metastore + .record_ops() + .set_record_takedown( + state.metastore.database(), + &cid, + takedown_ref.as_deref(), + scope_user, + ) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + } +} + +fn dispatch_user_block(state: &HandlerState, req: UserBlockRequest) { + match req { + UserBlockRequest::InsertUserBlocks { + user_id, + block_cids, + repo_rev, + tx, + } => { + let result = (|| { + let user_hash = state + .metastore + .user_hashes() + .get(&user_id) + .ok_or(DbError::Query("unknown user_id".to_string()))?; + let mut batch = state.metastore.database().batch(); + state + .metastore + .user_block_ops() + .insert_user_blocks(&mut batch, user_hash, &block_cids, &repo_rev) + .map_err(metastore_to_db)?; + batch.commit().map_err(|e| DbError::Query(e.to_string())) + })(); + let _ = tx.send(result); + } + UserBlockRequest::DeleteUserBlocks { + user_id, + block_cids, + tx, + } => { + let result = (|| { + let user_hash = state + .metastore + .user_hashes() + .get(&user_id) + .ok_or(DbError::Query("unknown user_id".to_string()))?; + let mut batch = state.metastore.database().batch(); + state + .metastore + .user_block_ops() + .delete_user_blocks_by_cid(&mut batch, user_hash, &block_cids) + .map_err(metastore_to_db)?; + batch.commit().map_err(|e| DbError::Query(e.to_string())) + })(); + let _ = tx.send(result); + } + UserBlockRequest::GetUserBlockCidsSinceRev { + user_id, + since_rev, + tx, + } => { + let result = state + .metastore + .user_block_ops() + .get_user_block_cids_since_rev(user_id, &since_rev) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + UserBlockRequest::CountUserBlocks { user_id, tx } => { + let result = state + .metastore + .user_block_ops() + .count_user_blocks(user_id) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + UserBlockRequest::FindUnreferencedBlocks { candidate_cids, tx } => { + let result = state + .metastore + .user_block_ops() + .find_unreferenced(&candidate_cids); + let _ = tx.send(Ok(result)); + } + } +} + +fn dispatch_event(state: &HandlerState, req: EventRequest) { + match req { + EventRequest::InsertCommitEvent { data, tx } => { + let result = state.event_ops.insert_commit_event(&data); + let _ = tx.send(result); + } + EventRequest::InsertIdentityEvent { did, handle, tx } => { + let result = state.event_ops.insert_identity_event(&did, handle.as_ref()); + let _ = tx.send(result); + } + EventRequest::InsertAccountEvent { did, status, tx } => { + let result = state.event_ops.insert_account_event(&did, status); + let _ = tx.send(result); + } + EventRequest::InsertSyncEvent { + did, + commit_cid, + rev, + tx, + } => { + let result = state + .event_ops + .insert_sync_event(&did, &commit_cid, rev.as_deref()); + let _ = tx.send(result); + } + EventRequest::InsertGenesisCommitEvent { + did, + commit_cid, + mst_root_cid, + rev, + tx, + } => { + let result = + state + .event_ops + .insert_genesis_commit_event(&did, &commit_cid, &mst_root_cid, &rev); + let _ = tx.send(result); + } + EventRequest::UpdateSeqBlocksCids { + seq, + blocks_cids, + tx, + } => { + let result = state.event_ops.update_seq_blocks_cids(seq, &blocks_cids); + let _ = tx.send(result); + } + EventRequest::DeleteSequencesExcept { did, keep_seq, tx } => { + let result = state.event_ops.delete_sequences_except(&did, keep_seq); + let _ = tx.send(result); + } + EventRequest::GetMaxSeq { tx } => { + let _ = tx.send(Ok(state.event_ops.get_max_seq())); + } + EventRequest::GetMinSeqSince { since, tx } => { + let _ = tx.send(state.event_ops.get_min_seq_since(since)); + } + EventRequest::GetEventsSinceSeq { + since_seq, + limit, + tx, + } => { + let _ = tx.send(state.event_ops.get_events_since_seq(since_seq, limit)); + } + EventRequest::GetEventsInSeqRange { + start_seq, + end_seq, + tx, + } => { + let _ = tx.send(state.event_ops.get_events_in_seq_range(start_seq, end_seq)); + } + EventRequest::GetEventBySeq { seq, tx } => { + let _ = tx.send(state.event_ops.get_event_by_seq(seq)); + } + EventRequest::GetEventsSinceCursor { cursor, limit, tx } => { + let _ = tx.send(state.event_ops.get_events_since_cursor(cursor, limit)); + } + EventRequest::GetEventsSinceRev { did, since_rev, tx } => { + let _ = tx.send(state.event_ops.get_events_since_rev(&did, &since_rev)); + } + EventRequest::NotifyUpdate { seq, tx } => { + let _ = tx.send(state.event_ops.notify_update(seq)); + } + } +} + +fn dispatch_commit(state: &HandlerState, req: CommitRequest) { + match req { + CommitRequest::ApplyCommit { input, tx } => { + let _ = tx.send(state.commit_ops.apply_commit(*input)); + } + CommitRequest::ImportRepoData { + user_id, + blocks, + records, + expected_root_cid, + tx, + } => { + let _ = tx.send(state.commit_ops.import_repo_data( + user_id, + &blocks, + &records, + expected_root_cid.as_ref(), + )); + } + CommitRequest::GetBrokenGenesisCommits { tx } => { + let _ = tx.send( + state + .commit_ops + .get_broken_genesis_commits() + .map_err(metastore_to_db), + ); + } + CommitRequest::GetUsersWithoutBlocks { tx } => { + let _ = tx.send( + state + .commit_ops + .get_users_without_blocks() + .map_err(metastore_to_db), + ); + } + CommitRequest::GetUsersNeedingRecordBlobsBackfill { limit, tx } => { + let _ = tx.send( + state + .commit_ops + .get_users_needing_record_blobs_backfill(limit) + .map_err(metastore_to_db), + ); + } + CommitRequest::InsertRecordBlobs { + repo_id, + record_uris, + blob_cids, + tx, + } => { + let _ = tx.send( + state + .commit_ops + .insert_record_blobs(repo_id, &record_uris, &blob_cids) + .map_err(metastore_to_db), + ); + } + } +} + +fn dispatch_backlink(state: &HandlerState, req: BacklinkRequest) { + match req { + BacklinkRequest::GetBacklinkConflicts { + repo_id, + collection, + backlinks, + tx, + } => { + let result = state + .metastore + .backlink_ops() + .get_backlink_conflicts(repo_id, &collection, &backlinks) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + BacklinkRequest::AddBacklinks { + repo_id, + backlinks, + tx, + } => { + let result = (|| { + let user_hash = state + .metastore + .user_hashes() + .get(&repo_id) + .ok_or(DbError::Query("unknown user_id".to_string()))?; + let mut batch = state.metastore.database().batch(); + state + .metastore + .backlink_ops() + .add_backlinks(&mut batch, user_hash, &backlinks) + .map_err(metastore_to_db)?; + batch.commit().map_err(|e| DbError::Query(e.to_string())) + })(); + let _ = tx.send(result); + } + BacklinkRequest::RemoveBacklinksByUri { uri, tx } => { + let result = (|| { + let did_str = uri + .did() + .ok_or(DbError::Query("backlink uri missing did".to_string()))?; + let user_hash = UserHash::from_did(did_str); + let mut batch = state.metastore.database().batch(); + state + .metastore + .backlink_ops() + .remove_backlinks_by_uri(&mut batch, user_hash, &uri) + .map_err(metastore_to_db)?; + batch.commit().map_err(|e| DbError::Query(e.to_string())) + })(); + let _ = tx.send(result); + } + BacklinkRequest::RemoveBacklinksByRepo { repo_id, tx } => { + let result = (|| { + let user_hash = state + .metastore + .user_hashes() + .get(&repo_id) + .ok_or(DbError::Query("unknown user_id".to_string()))?; + let mut batch = state.metastore.database().batch(); + state + .metastore + .backlink_ops() + .remove_backlinks_by_repo(&mut batch, user_hash) + .map_err(metastore_to_db)?; + batch.commit().map_err(|e| DbError::Query(e.to_string())) + })(); + let _ = tx.send(result); + } + } +} + +fn dispatch_blob(state: &HandlerState, req: BlobRequest) { + match req { + BlobRequest::InsertBlob { + cid, + mime_type, + size_bytes, + created_by_user, + storage_key, + tx, + } => { + let result = state + .metastore + .blob_ops() + .insert_blob(&cid, &mime_type, size_bytes, created_by_user, &storage_key) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + BlobRequest::GetBlobMetadata { cid, tx } => { + let result = state + .metastore + .blob_ops() + .get_blob_metadata(&cid) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + BlobRequest::GetBlobWithTakedown { cid, tx } => { + let result = state + .metastore + .blob_ops() + .get_blob_with_takedown(&cid) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + BlobRequest::GetBlobStorageKey { cid, tx } => { + let result = state + .metastore + .blob_ops() + .get_blob_storage_key(&cid) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + BlobRequest::ListBlobsByUser { + user_id, + cursor, + limit, + tx, + } => { + let result = state + .metastore + .blob_ops() + .list_blobs_by_user( + user_id, + cursor.as_deref(), + usize::try_from(limit).unwrap_or(usize::MAX), + ) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + BlobRequest::ListBlobsSinceRev { did, since, tx } => { + let result = state.event_ops.get_blob_cids_since_rev(&did, &since); + let _ = tx.send(result); + } + BlobRequest::CountBlobsByUser { user_id, tx } => { + let result = state + .metastore + .blob_ops() + .count_blobs_by_user(user_id) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + BlobRequest::SumBlobStorage { tx } => { + let result = state + .metastore + .blob_ops() + .sum_blob_storage() + .map_err(metastore_to_db); + let _ = tx.send(result); + } + BlobRequest::UpdateBlobTakedown { + cid, + takedown_ref, + tx, + } => { + let result = state + .metastore + .blob_ops() + .update_blob_takedown(&cid, takedown_ref.as_deref()) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + BlobRequest::DeleteBlobByCid { cid, tx } => { + let result = state + .metastore + .blob_ops() + .delete_blob_by_cid(&cid) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + BlobRequest::DeleteBlobsByUser { user_id, tx } => { + let result = state + .metastore + .blob_ops() + .delete_blobs_by_user(user_id) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + BlobRequest::GetBlobStorageKeysByUser { user_id, tx } => { + let result = state + .metastore + .blob_ops() + .get_blob_storage_keys_by_user(user_id) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + BlobRequest::ListMissingBlobs { + repo_id, + cursor, + limit, + tx, + } => { + let result = state + .metastore + .blob_ops() + .list_missing_blobs( + repo_id, + cursor.as_deref(), + usize::try_from(limit).unwrap_or(usize::MAX), + ) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + BlobRequest::CountDistinctRecordBlobs { repo_id, tx } => { + let result = state + .metastore + .blob_ops() + .count_distinct_record_blobs(repo_id) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + BlobRequest::GetBlobsForExport { repo_id, tx } => { + let result = state + .metastore + .blob_ops() + .get_blobs_for_export(repo_id) + .map_err(metastore_to_db); + let _ = tx.send(result); + } + } +} + +fn dispatch(state: &HandlerState, request: MetastoreRequest) { + match request { + MetastoreRequest::Repo(r) => dispatch_repo(state, r), + MetastoreRequest::Record(r) => dispatch_record(state, r), + MetastoreRequest::UserBlock(r) => dispatch_user_block(state, r), + MetastoreRequest::Event(r) => dispatch_event(state, r), + MetastoreRequest::Commit(r) => dispatch_commit(state, *r), + MetastoreRequest::Backlink(r) => dispatch_backlink(state, r), + MetastoreRequest::Blob(r) => dispatch_blob(state, r), + } +} + +fn handler_loop( + metastore: Metastore, + bridge: Arc>, + blockstore: Option, + rx: flume::Receiver, + thread_index: usize, +) { + let event_ops = metastore.event_ops(Arc::clone(&bridge)); + let mut commit_ops = metastore.commit_ops(bridge); + if let Some(bs) = blockstore { + commit_ops = commit_ops.with_blockstore(bs); + } + let state = HandlerState { + metastore, + event_ops, + commit_ops, + }; + tracing::info!(thread_index, "metastore handler thread started"); + rx.iter().for_each(|req| { + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| dispatch(&state, req))) { + Ok(()) => {} + Err(e) => { + let msg = match e.downcast_ref::<&str>() { + Some(s) => (*s).to_owned(), + None => match e.downcast_ref::() { + Some(s) => s.clone(), + None => "unknown panic payload".to_owned(), + }, + }; + tracing::error!(thread_index, msg, "metastore handler panic (recovered)"); + } + } + }); + tracing::info!(thread_index, "metastore handler thread exiting"); +} + +const DEFAULT_CHANNEL_BOUND: usize = 256; +const MAX_REPOS_WITHOUT_REV: usize = 10_000; + +pub struct HandlerPool { + senders: Vec>, + handles: Option>>, + user_hashes: Arc, + round_robin: AtomicUsize, +} + +impl HandlerPool { + pub fn spawn( + metastore: Metastore, + bridge: Arc>, + blockstore: Option, + thread_count: Option, + ) -> Self { + let count = thread_count + .unwrap_or_else(|| { + std::thread::available_parallelism() + .map(|n| n.get().max(2) / 2) + .unwrap_or(1) + }) + .max(1); + + let user_hashes = Arc::clone(metastore.user_hashes()); + + let (senders, handles): (Vec<_>, Vec<_>) = (0..count) + .map(|i| { + let (tx, rx) = flume::bounded(DEFAULT_CHANNEL_BOUND); + let ms = metastore.clone(); + let br = Arc::clone(&bridge); + let bs = blockstore.clone(); + let handle = std::thread::Builder::new() + .name(format!("metastore-{i}")) + .spawn(move || handler_loop(ms, br, bs, rx, i)) + .expect("failed to spawn metastore handler thread"); + (tx, handle) + }) + .unzip(); + + Self { + senders, + handles: Some(handles), + user_hashes, + round_robin: AtomicUsize::new(0), + } + } + + pub fn send(&self, request: MetastoreRequest) -> Result<(), DbError> { + let index = match request.routing(&self.user_hashes) { + Routing::Sharded(bits) => (bits as usize) % self.senders.len(), + Routing::Global => { + self.round_robin.fetch_add(1, Ordering::Relaxed) % self.senders.len() + } + }; + self.senders[index].try_send(request).map_err(|e| match e { + flume::TrySendError::Full(_) => { + DbError::Query("metastore handler backpressure".to_string()) + } + flume::TrySendError::Disconnected(_) => { + DbError::Connection("metastore handler pool shut down".to_string()) + } + }) + } + + pub fn thread_count(&self) -> usize { + self.senders.len() + } + + pub async fn shutdown(&mut self) { + self.senders.clear(); + if let Some(handles) = self.handles.take() { + let join_fut = tokio::task::spawn_blocking(move || { + handles.into_iter().for_each(|h| { + if let Err(e) = h.join() { + tracing::error!("metastore handler thread panicked: {e:?}"); + } + }); + }); + match tokio::time::timeout(std::time::Duration::from_secs(30), join_fut).await { + Ok(_) => tracing::info!("metastore handler threads shut down cleanly"), + Err(_) => tracing::error!("metastore handler thread shutdown timed out after 30s"), + } + } + } +} + +impl Drop for HandlerPool { + fn drop(&mut self) { + self.senders.clear(); + if let Some(handles) = self.handles.take() { + tracing::warn!( + "HandlerPool dropped without calling shutdown(); blocking on thread join" + ); + handles.into_iter().for_each(|h| { + if let Err(e) = h.join() { + tracing::error!("metastore handler thread panicked: {e:?}"); + } + }); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::eventlog::{EventLog, EventLogConfig}; + use crate::io::RealIO; + use crate::metastore::MetastoreConfig; + use tranquil_types::{Did, Handle}; + + struct TestHarness { + _metastore_dir: tempfile::TempDir, + _eventlog_dir: tempfile::TempDir, + pool: HandlerPool, + } + + fn setup() -> TestHarness { + let metastore_dir = tempfile::TempDir::new().unwrap(); + let eventlog_dir = tempfile::TempDir::new().unwrap(); + let segments_dir = eventlog_dir.path().join("segments"); + std::fs::create_dir_all(&segments_dir).unwrap(); + + let metastore = Metastore::open( + metastore_dir.path(), + MetastoreConfig { + cache_size_bytes: 64 * 1024 * 1024, + }, + ) + .unwrap(); + + let event_log = EventLog::open( + EventLogConfig { + segments_dir, + ..EventLogConfig::default() + }, + RealIO::new(), + ) + .unwrap(); + + let bridge = Arc::new(EventLogBridge::new(Arc::new(event_log))); + + let pool = HandlerPool::spawn::(metastore, bridge, None, Some(2)); + + TestHarness { + _metastore_dir: metastore_dir, + _eventlog_dir: eventlog_dir, + pool, + } + } + + fn test_cid_link(seed: u8) -> CidLink { + let digest: [u8; 32] = std::array::from_fn(|i| seed.wrapping_add(i as u8)); + let mh = multihash::Multihash::<64>::wrap(0x12, &digest).unwrap(); + let c = cid::Cid::new_v1(0x71, mh); + CidLink::from_cid(&c) + } + + #[tokio::test] + async fn create_and_get_roundtrip() { + let h = setup(); + let user_id = Uuid::new_v4(); + let did = Did::from("did:plc:handler_test".to_string()); + let handle = Handle::from("handler.test.invalid".to_string()); + let cid = test_cid_link(1); + + let (tx, rx) = oneshot::channel(); + h.pool + .send(MetastoreRequest::Repo(RepoRequest::CreateRepoFull { + user_id, + did, + handle, + repo_root_cid: cid.clone(), + repo_rev: "rev1".to_string(), + tx, + })) + .unwrap(); + rx.await.unwrap().unwrap(); + + let (tx, rx) = oneshot::channel(); + h.pool + .send(MetastoreRequest::Repo(RepoRequest::GetRepo { user_id, tx })) + .unwrap(); + let repo = rx.await.unwrap().unwrap().unwrap(); + assert_eq!(repo.repo_root_cid, cid); + assert_eq!(repo.repo_rev.as_deref(), Some("rev1")); + } + + #[test] + fn routing_determinism() { + let user_id = Uuid::from_u128(0x12345678); + let bits = user_id.as_u128() as u64; + let thread_count = 4usize; + let expected = (bits as usize) % thread_count; + (0..100).for_each(|_| { + assert_eq!((bits as usize) % thread_count, expected); + }); + } + + #[test] + fn global_round_robin_distributes() { + let counter = AtomicUsize::new(0); + let thread_count = 4usize; + let indices: Vec = (0..8) + .map(|_| counter.fetch_add(1, Ordering::Relaxed) % thread_count) + .collect(); + assert_eq!(indices, vec![0, 1, 2, 3, 0, 1, 2, 3]); + } + + #[tokio::test] + async fn shutdown_completes_inflight() { + let mut h = setup(); + let user_id = Uuid::new_v4(); + let did = Did::from("did:plc:shutdown_test".to_string()); + let handle = Handle::from("shutdown.test.invalid".to_string()); + let cid = test_cid_link(2); + + let (tx, rx) = oneshot::channel(); + h.pool + .send(MetastoreRequest::Repo(RepoRequest::CreateRepoFull { + user_id, + did, + handle, + repo_root_cid: cid, + repo_rev: "rev1".to_string(), + tx, + })) + .unwrap(); + rx.await.unwrap().unwrap(); + + h.pool.shutdown().await; + } +} diff --git a/crates/tranquil-store/src/metastore/keys.rs b/crates/tranquil-store/src/metastore/keys.rs new file mode 100644 index 0000000..c43bd31 --- /dev/null +++ b/crates/tranquil-store/src/metastore/keys.rs @@ -0,0 +1,149 @@ +use serde::{Deserialize, Serialize}; +use siphasher::sip::SipHasher24; +use std::hash::Hasher; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct UserHash(u64); + +const SIPHASH_KEY0: u64 = 0x7472_616e_7175_696c; +const SIPHASH_KEY1: u64 = 0x7064_735f_7573_6572; + +impl UserHash { + pub fn from_did(did: &str) -> Self { + let mut hasher = SipHasher24::new_with_keys(SIPHASH_KEY0, SIPHASH_KEY1); + hasher.write(did.as_bytes()); + Self(hasher.finish()) + } + + pub fn from_raw(raw: u64) -> Self { + Self(raw) + } + + pub fn raw(self) -> u64 { + self.0 + } +} + +impl std::fmt::Display for UserHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:016x}", self.0) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct KeyTag(u8); + +impl KeyTag { + pub const REPO_META: Self = Self(0x01); + pub const RECORDS: Self = Self(0x02); + pub const USER_BLOCKS: Self = Self(0x03); + pub const HANDLES: Self = Self(0x04); + pub const BLOBS: Self = Self(0x05); + pub const BACKLINKS: Self = Self(0x06); + pub const BLOB_BY_CID: Self = Self(0x07); + + pub const USER_MAP: Self = Self(0x10); + pub const USER_MAP_REVERSE: Self = Self(0x11); + + pub const REV_TO_SEQ: Self = Self(0x20); + pub const SEQ_META: Self = Self(0x21); + pub const SEQ_TOMBSTONE: Self = Self(0x22); + pub const METASTORE_CURSOR: Self = Self(0x23); + pub const DID_EVENTS: Self = Self(0x24); + + pub const RECORD_BLOBS: Self = Self(0x30); + pub const BACKLINK_BY_USER: Self = Self(0x31); + + pub const FORMAT_VERSION: Self = Self(0xFF); + + pub const fn raw(self) -> u8 { + self.0 + } + + pub fn exclusive_prefix_bound(self) -> [u8; 1] { + match self.0.checked_add(1) { + Some(next) => [next], + None => panic!("cannot compute exclusive upper bound for tag 0xFF"), + } + } + + #[cfg(test)] + pub fn from_raw_unchecked(raw: u8) -> Self { + Self(raw) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn user_hash_deterministic() { + let a = UserHash::from_did("did:plc:abc123"); + let b = UserHash::from_did("did:plc:abc123"); + assert_eq!(a, b); + } + + #[test] + fn user_hash_different_dids_differ() { + let a = UserHash::from_did("did:plc:abc123"); + let b = UserHash::from_did("did:plc:xyz789"); + assert_ne!(a, b); + } + + #[test] + fn user_hash_display_is_hex() { + let h = UserHash::from_raw(0xDEAD_BEEF_CAFE_BABE); + assert_eq!(h.to_string(), "deadbeefcafebabe"); + } + + #[test] + fn key_tags_are_distinct() { + let tags = [ + KeyTag::REPO_META, + KeyTag::RECORDS, + KeyTag::USER_BLOCKS, + KeyTag::HANDLES, + KeyTag::BLOBS, + KeyTag::BACKLINKS, + KeyTag::BLOB_BY_CID, + KeyTag::USER_MAP, + KeyTag::USER_MAP_REVERSE, + KeyTag::REV_TO_SEQ, + KeyTag::SEQ_META, + KeyTag::SEQ_TOMBSTONE, + KeyTag::METASTORE_CURSOR, + KeyTag::DID_EVENTS, + KeyTag::RECORD_BLOBS, + KeyTag::BACKLINK_BY_USER, + KeyTag::FORMAT_VERSION, + ]; + let mut raw: Vec = tags.iter().map(|t| t.raw()).collect(); + let original_len = raw.len(); + raw.sort(); + raw.dedup(); + assert_eq!(raw.len(), original_len); + } + + #[test] + fn key_tag_ordering() { + assert!(KeyTag::REPO_META < KeyTag::RECORDS); + assert!(KeyTag::RECORDS < KeyTag::USER_BLOCKS); + } + + #[test] + fn exclusive_prefix_bound_is_tag_plus_one() { + assert_eq!( + KeyTag::REPO_META.exclusive_prefix_bound(), + [KeyTag::REPO_META.raw() + 1] + ); + assert_eq!(KeyTag::HANDLES.exclusive_prefix_bound(), [0x05]); + } + + #[test] + #[should_panic(expected = "cannot compute exclusive upper bound for tag 0xFF")] + fn exclusive_prefix_bound_panics_for_0xff() { + KeyTag::FORMAT_VERSION.exclusive_prefix_bound(); + } +} diff --git a/crates/tranquil-store/src/metastore/mod.rs b/crates/tranquil-store/src/metastore/mod.rs new file mode 100644 index 0000000..ad4e6fb --- /dev/null +++ b/crates/tranquil-store/src/metastore/mod.rs @@ -0,0 +1,417 @@ +pub mod backlink_ops; +pub mod backlinks; +pub mod blob_ops; +pub mod blobs; +pub mod commit_ops; +pub mod encoding; +pub mod event_keys; +pub mod event_ops; +pub mod keys; +pub mod partitions; +pub mod record_ops; +pub mod records; +pub mod recovery; +pub mod repo_meta; +pub mod repo_ops; +pub mod scan; +pub mod user_block_ops; +pub mod user_blocks; +pub mod user_hash; + +use std::path::Path; +use std::sync::Arc; + +use fjall::{Database, Keyspace}; + +use self::keys::KeyTag; +use self::partitions::Partition; +use self::user_hash::UserHashMap; + +const CURRENT_FORMAT_VERSION: u64 = 1; + +#[derive(Debug, Clone)] +pub struct MetastoreConfig { + pub cache_size_bytes: u64, +} + +impl Default for MetastoreConfig { + fn default() -> Self { + let total_ram = total_system_ram_bytes(); + let twenty_percent = total_ram / 5; + + Self { + cache_size_bytes: twenty_percent, + } + } +} + +fn total_system_ram_bytes() -> u64 { + #[cfg(target_os = "linux")] + { + std::fs::read_to_string("/proc/meminfo") + .ok() + .and_then(|contents| { + contents + .lines() + .find(|line| line.starts_with("MemTotal:")) + .and_then(|line| { + line.split_whitespace() + .nth(1) + .and_then(|kb| kb.parse::().ok()) + .map(|kb| kb.saturating_mul(1024)) + }) + }) + .unwrap_or(4 * 1024 * 1024 * 1024) + } + #[cfg(not(target_os = "linux"))] + { + tracing::warn!("cannot detect system RAM on this platform, defaulting to 4GB"); + 4 * 1024 * 1024 * 1024 + } +} + +#[derive(Debug)] +pub enum MetastoreError { + Fjall(fjall::Error), + Lsm(lsm_tree::Error), + VersionMismatch { + expected: u64, + found: u64, + }, + CorruptData(&'static str), + InvalidInput(&'static str), + UserHashCollision { + hash: keys::UserHash, + existing_uuid: uuid::Uuid, + new_uuid: uuid::Uuid, + }, +} + +impl std::fmt::Display for MetastoreError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Fjall(e) => write!(f, "fjall: {e}"), + Self::Lsm(e) => write!(f, "lsm: {e}"), + Self::VersionMismatch { expected, found } => { + write!( + f, + "format version mismatch: expected {expected}, found {found}" + ) + } + Self::CorruptData(msg) => write!(f, "corrupt data: {msg}"), + Self::InvalidInput(msg) => write!(f, "invalid input: {msg}"), + Self::UserHashCollision { + hash, + existing_uuid, + new_uuid, + } => write!( + f, + "user hash collision: hash {hash} maps to both {existing_uuid} and {new_uuid}" + ), + } + } +} + +impl std::error::Error for MetastoreError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Fjall(e) => Some(e), + Self::Lsm(e) => Some(e), + _ => None, + } + } +} + +impl From for MetastoreError { + fn from(e: fjall::Error) -> Self { + Self::Fjall(e) + } +} + +impl From for MetastoreError { + fn from(e: lsm_tree::Error) -> Self { + Self::Lsm(e) + } +} + +type CompactionFilterFn = + Arc Option> + Send + Sync>; + +pub mod client; +pub mod handler; + +#[derive(Clone)] +pub struct Metastore { + db: Database, + partitions: [Keyspace; Partition::ALL.len()], + user_hashes: Arc, +} + +impl Metastore { + pub fn open(path: &Path, config: MetastoreConfig) -> Result { + let auth_name = Partition::Auth.name(); + let filter_factory: CompactionFilterFn = + Arc::new(move |name: &str| match name == auth_name { + true => Some(Arc::new(partitions::TtlFilterFactory)), + false => None, + }); + + let db = Database::builder(path) + .cache_size(config.cache_size_bytes) + .with_compaction_filter_factories(filter_factory) + .open()?; + + let opened: Vec = Partition::ALL + .iter() + .map(|&p| { + let opts = p.create_options(); + db.keyspace(p.name(), || opts) + }) + .collect::>()?; + + let partitions: [Keyspace; Partition::ALL.len()] = opened + .try_into() + .ok() + .expect("opened exactly Partition::ALL.len() keyspaces"); + + let repo_data = partitions[Partition::RepoData.index()].clone(); + Self::check_or_write_version(&db, &repo_data)?; + + let user_hashes = Arc::new(UserHashMap::new(repo_data)); + let loaded = user_hashes.load_all()?; + tracing::info!(count = loaded, "loaded user hash mappings"); + + Ok(Self { + db, + partitions, + user_hashes, + }) + } + + fn check_or_write_version(db: &Database, repo_data: &Keyspace) -> Result<(), MetastoreError> { + let version_key = [KeyTag::FORMAT_VERSION.raw()]; + let version_bytes = CURRENT_FORMAT_VERSION.to_be_bytes(); + + match repo_data.get(version_key)? { + Some(existing) => { + let found_bytes: [u8; 8] = existing + .as_ref() + .try_into() + .map_err(|_| MetastoreError::CorruptData("format version not 8 bytes"))?; + let found = u64::from_be_bytes(found_bytes); + match found == CURRENT_FORMAT_VERSION { + true => Ok(()), + false => Err(MetastoreError::VersionMismatch { + expected: CURRENT_FORMAT_VERSION, + found, + }), + } + } + None => { + repo_data.insert(version_key, version_bytes)?; + db.persist(fjall::PersistMode::SyncData)?; + Ok(()) + } + } + } + + pub fn partition(&self, p: Partition) -> &Keyspace { + &self.partitions[p.index()] + } + + pub fn user_hashes(&self) -> &Arc { + &self.user_hashes + } + + pub fn database(&self) -> &Database { + &self.db + } + + pub fn repo_ops(&self) -> repo_ops::RepoOps { + repo_ops::RepoOps::new( + self.partitions[Partition::RepoData.index()].clone(), + Arc::clone(&self.user_hashes), + ) + } + + pub fn record_ops(&self) -> record_ops::RecordOps { + record_ops::RecordOps::new( + self.partitions[Partition::RepoData.index()].clone(), + Arc::clone(&self.user_hashes), + ) + } + + pub fn user_block_ops(&self) -> user_block_ops::UserBlockOps { + user_block_ops::UserBlockOps::new( + self.partitions[Partition::RepoData.index()].clone(), + Arc::clone(&self.user_hashes), + ) + } + + pub fn event_ops( + &self, + bridge: Arc>, + ) -> event_ops::EventOps { + event_ops::EventOps::new( + self.db.clone(), + self.partitions[Partition::RepoData.index()].clone(), + bridge, + ) + } + + pub fn blob_ops(&self) -> blob_ops::BlobOps { + blob_ops::BlobOps::new( + self.db.clone(), + self.partitions[Partition::RepoData.index()].clone(), + Arc::clone(&self.user_hashes), + ) + } + + pub fn backlink_ops(&self) -> backlink_ops::BacklinkOps { + backlink_ops::BacklinkOps::new( + self.partitions[Partition::Indexes.index()].clone(), + Arc::clone(&self.user_hashes), + ) + } + + pub fn commit_ops( + &self, + bridge: Arc>, + ) -> commit_ops::CommitOps { + commit_ops::CommitOps::new( + self.db.clone(), + self.partitions[Partition::RepoData.index()].clone(), + self.partitions[Partition::Indexes.index()].clone(), + Arc::clone(&self.user_hashes), + bridge, + ) + } + + pub fn persist(&self) -> Result<(), MetastoreError> { + self.db + .persist(fjall::PersistMode::SyncData) + .map_err(MetastoreError::Fjall) + } + + pub fn major_compact(&self) -> Result<(), MetastoreError> { + Partition::ALL.iter().try_for_each(|&p| { + tracing::info!(partition = p.name(), "starting major compaction"); + self.partitions[p.index()] + .major_compact() + .map_err(MetastoreError::Fjall)?; + tracing::info!(partition = p.name(), "major compaction complete"); + Ok::<(), MetastoreError>(()) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn open_fresh() -> (tempfile::TempDir, Metastore) { + let dir = tempfile::TempDir::new().unwrap(); + let ms = Metastore::open( + dir.path(), + MetastoreConfig { + cache_size_bytes: 64 * 1024 * 1024, + }, + ) + .unwrap(); + (dir, ms) + } + + fn test_config() -> MetastoreConfig { + MetastoreConfig { + cache_size_bytes: 64 * 1024 * 1024, + } + } + + #[test] + fn open_fresh_directory_succeeds() { + let (_dir, ms) = open_fresh(); + assert_eq!(ms.user_hashes().len(), 0); + } + + #[test] + fn all_partitions_accessible() { + let (_dir, ms) = open_fresh(); + Partition::ALL.iter().for_each(|&p| { + let _ = ms.partition(p); + }); + } + + #[test] + fn reopen_preserves_partitions() { + let dir = tempfile::TempDir::new().unwrap(); + + { + let ms = Metastore::open(dir.path(), test_config()).unwrap(); + let repo_data = ms.partition(Partition::RepoData); + repo_data.insert(b"test_key", b"test_value").unwrap(); + ms.persist().unwrap(); + } + + { + let ms = Metastore::open(dir.path(), test_config()).unwrap(); + let repo_data = ms.partition(Partition::RepoData); + let val = repo_data.get(b"test_key").unwrap().unwrap(); + assert_eq!(val.as_ref(), b"test_value"); + } + } + + #[test] + fn version_mismatch_returns_error() { + let dir = tempfile::TempDir::new().unwrap(); + + { + let ms = Metastore::open(dir.path(), test_config()).unwrap(); + let repo_data = ms.partition(Partition::RepoData); + let version_key = [KeyTag::FORMAT_VERSION.raw()]; + repo_data.insert(version_key, 999u64.to_be_bytes()).unwrap(); + ms.persist().unwrap(); + } + + { + let result = Metastore::open(dir.path(), test_config()); + assert!(matches!( + result, + Err(MetastoreError::VersionMismatch { + expected: 1, + found: 999 + }) + )); + } + } + + #[test] + fn user_hash_mappings_survive_reopen() { + let dir = tempfile::TempDir::new().unwrap(); + let uuid = uuid::Uuid::new_v4(); + let hash = keys::UserHash::from_did("did:plc:survivor"); + + { + let ms = Metastore::open(dir.path(), test_config()).unwrap(); + let mut batch = ms.database().batch(); + ms.user_hashes() + .stage_insert(&mut batch, uuid, hash) + .unwrap(); + batch.commit().unwrap(); + ms.persist().unwrap(); + } + + { + let ms = Metastore::open(dir.path(), test_config()).unwrap(); + assert_eq!(ms.user_hashes().len(), 1); + assert_eq!(ms.user_hashes().get(&uuid), Some(hash)); + assert_eq!(ms.user_hashes().get_uuid(&hash), Some(uuid)); + } + } + + #[test] + fn default_config_has_reasonable_cache_size() { + let config = MetastoreConfig::default(); + assert!(config.cache_size_bytes > 0); + assert!(config.cache_size_bytes <= 4 * 1024 * 1024 * 1024); + } +} diff --git a/crates/tranquil-store/src/metastore/partitions.rs b/crates/tranquil-store/src/metastore/partitions.rs new file mode 100644 index 0000000..4727064 --- /dev/null +++ b/crates/tranquil-store/src/metastore/partitions.rs @@ -0,0 +1,129 @@ +use std::time::{SystemTime, UNIX_EPOCH}; + +use fjall::KeyspaceCreateOptions; +use fjall::compaction::filter::{CompactionFilter, Context, Factory, ItemAccessor, Verdict}; +use fjall::config::{BloomConstructionPolicy, FilterPolicy, FilterPolicyEntry}; + +pub const EXPIRES_AT_MS_SIZE: usize = 8; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Partition { + RepoData, + Auth, + Users, + Infra, + Indexes, +} + +impl Partition { + pub const ALL: [Partition; 5] = [ + Partition::RepoData, + Partition::Auth, + Partition::Users, + Partition::Infra, + Partition::Indexes, + ]; + + pub const fn index(self) -> usize { + match self { + Self::RepoData => 0, + Self::Auth => 1, + Self::Users => 2, + Self::Infra => 3, + Self::Indexes => 4, + } + } + + pub fn name(self) -> &'static str { + match self { + Self::RepoData => "repo_data", + Self::Auth => "auth", + Self::Users => "users", + Self::Infra => "infra", + Self::Indexes => "indexes", + } + } + + pub fn create_options(self) -> KeyspaceCreateOptions { + match self { + Self::RepoData | Self::Indexes => { + KeyspaceCreateOptions::default().filter_policy(FilterPolicy::new([ + FilterPolicyEntry::Bloom(BloomConstructionPolicy::FalsePositiveRate(0.01)), + FilterPolicyEntry::Bloom(BloomConstructionPolicy::BitsPerKey(10.0)), + ])) + } + Self::Auth | Self::Users | Self::Infra => KeyspaceCreateOptions::default(), + } + } +} + +pub(crate) struct TtlFilterFactory; + +impl Factory for TtlFilterFactory { + fn name(&self) -> &str { + "ttl_expiry" + } + + fn make_filter(&self, _ctx: &Context) -> Box { + let now_ms = u64::try_from( + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock before unix epoch") + .as_millis(), + ) + .unwrap_or(u64::MAX); + Box::new(TtlFilter { now_ms }) + } +} + +struct TtlFilter { + now_ms: u64, +} + +impl CompactionFilter for TtlFilter { + fn filter_item(&mut self, item: ItemAccessor<'_>, _ctx: &Context) -> lsm_tree::Result { + let value = item.value()?; + match value.get(..EXPIRES_AT_MS_SIZE) { + Some(bytes) => { + let expires_at_ms = + u64::from_be_bytes(bytes.try_into().expect("slice is exactly 8 bytes")); + match expires_at_ms > 0 && expires_at_ms < self.now_ms { + true => Ok(Verdict::Remove), + false => Ok(Verdict::Keep), + } + } + None => Ok(Verdict::Keep), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn partition_names_are_distinct() { + let names: Vec<_> = Partition::ALL.iter().map(|p| p.name()).collect(); + let mut deduped = names.clone(); + deduped.sort(); + deduped.dedup(); + assert_eq!(names.len(), deduped.len()); + } + + #[test] + fn all_partitions_covered() { + assert_eq!(Partition::ALL.len(), 5); + } + + #[test] + fn auth_partition_has_filter() { + assert_eq!(Partition::Auth.name(), "auth"); + } + + #[test] + fn index_matches_all_array_position() { + Partition::ALL.iter().enumerate().for_each(|(i, &p)| { + assert_eq!(p.index(), i, "Partition::{:?} index mismatch", p); + }); + } +} diff --git a/crates/tranquil-store/src/metastore/record_ops.rs b/crates/tranquil-store/src/metastore/record_ops.rs new file mode 100644 index 0000000..31dddf9 --- /dev/null +++ b/crates/tranquil-store/src/metastore/record_ops.rs @@ -0,0 +1,1506 @@ +use std::sync::Arc; + +use fjall::Keyspace; +use smallvec::SmallVec; +use uuid::Uuid; + +use super::MetastoreError; +use super::encoding::{KeyReader, exclusive_upper_bound}; +use super::keys::UserHash; +use super::records::{ + RecordValue, record_collection_prefix, record_key, record_user_prefix, records_prefix, +}; +use super::repo_ops::{bytes_to_cid_link, cid_link_to_bytes}; +use super::scan::{count_prefix, delete_all_by_prefix, point_lookup}; +use super::user_hash::UserHashMap; + +use tranquil_types::{CidLink, Nsid, Rkey}; + +pub struct RecordWrite<'a> { + pub collection: &'a Nsid, + pub rkey: &'a Rkey, + pub cid: &'a CidLink, +} + +pub struct RecordDelete<'a> { + pub collection: &'a Nsid, + pub rkey: &'a Rkey, +} + +pub struct ListRecordsQuery<'a> { + pub user_id: Uuid, + pub collection: &'a Nsid, + pub cursor: Option<&'a Rkey>, + pub limit: usize, + pub reverse: bool, + pub rkey_start: Option<&'a Rkey>, + pub rkey_end: Option<&'a Rkey>, +} + +#[derive(Debug, Clone)] +pub struct RecordInfo { + pub rkey: Rkey, + pub record_cid: CidLink, +} + +#[derive(Debug, Clone)] +pub struct FullRecordInfo { + pub collection: Nsid, + pub rkey: Rkey, + pub record_cid: CidLink, +} + +#[derive(Debug, Clone)] +pub struct RecordWithTakedown { + pub id: Uuid, + pub collection: Nsid, + pub rkey: Rkey, + pub takedown_ref: Option, +} + +pub struct RecordOps { + repo_data: Keyspace, + user_hashes: Arc, +} + +impl RecordOps { + pub fn new(repo_data: Keyspace, user_hashes: Arc) -> Self { + Self { + repo_data, + user_hashes, + } + } + + pub fn upsert_records( + &self, + batch: &mut fjall::OwnedWriteBatch, + user_hash: UserHash, + records: &[RecordWrite<'_>], + ) -> Result<(), MetastoreError> { + records.iter().try_for_each(|rec| { + let key = record_key(user_hash, rec.collection.as_str(), rec.rkey.as_str()); + let cid_bytes = cid_link_to_bytes(rec.cid)?; + let existing_takedown = self + .repo_data + .get(key.as_slice()) + .map_err(MetastoreError::Fjall)? + .and_then(|raw| RecordValue::deserialize(&raw)) + .and_then(|v| v.takedown_ref); + let value = RecordValue { + record_cid: cid_bytes, + takedown_ref: existing_takedown, + }; + batch.insert(&self.repo_data, key.as_slice(), value.serialize()); + Ok::<(), MetastoreError>(()) + }) + } + + pub fn delete_records( + &self, + batch: &mut fjall::OwnedWriteBatch, + user_hash: UserHash, + records: &[RecordDelete<'_>], + ) { + records.iter().for_each(|rec| { + let key = record_key(user_hash, rec.collection.as_str(), rec.rkey.as_str()); + batch.remove(&self.repo_data, key.as_slice()); + }); + } + + pub fn delete_all_records( + &self, + batch: &mut fjall::OwnedWriteBatch, + user_hash: UserHash, + ) -> Result<(), MetastoreError> { + let prefix = record_user_prefix(user_hash); + delete_all_by_prefix(&self.repo_data, batch, prefix.as_slice()) + } + + pub fn get_record_cid( + &self, + user_id: Uuid, + collection: &Nsid, + rkey: &Rkey, + ) -> Result, MetastoreError> { + let user_hash = match self.user_hashes.get(&user_id) { + Some(h) => h, + None => return Ok(None), + }; + let key = record_key(user_hash, collection.as_str(), rkey.as_str()); + + point_lookup( + &self.repo_data, + key.as_slice(), + RecordValue::deserialize, + "invalid record value", + )? + .map(|v| bytes_to_cid_link(&v.record_cid)) + .transpose() + } + + pub fn list_records( + &self, + query: &ListRecordsQuery<'_>, + ) -> Result, MetastoreError> { + let user_hash = match self.user_hashes.get(&query.user_id) { + Some(h) => h, + None => return Ok(Vec::new()), + }; + + let coll_str = query.collection.as_str(); + let coll_prefix = record_collection_prefix(user_hash, coll_str); + let coll_upper = exclusive_upper_bound(coll_prefix.as_slice()) + .expect("collection prefix always contains non-0xFF bytes"); + + let start_key = query + .rkey_start + .map(|rs| record_key(user_hash, coll_str, rs.as_str())); + let end_key_upper = query + .rkey_end + .map(|re| record_key(user_hash, coll_str, re.as_str())) + .map(|ek| { + exclusive_upper_bound(ek.as_slice()) + .expect("record key always contains non-0xFF bytes") + }); + let cursor_key = query + .cursor + .map(|c| record_key(user_hash, coll_str, c.as_str())); + + let mut range_lo: &[u8] = coll_prefix.as_slice(); + let mut range_hi: &[u8] = coll_upper.as_slice(); + + if let Some(sk) = start_key.as_ref().filter(|sk| sk.as_slice() > range_lo) { + range_lo = sk.as_slice(); + } + + if let Some(eu) = end_key_upper.as_ref().filter(|eu| eu.as_slice() < range_hi) { + range_hi = eu.as_slice(); + } + + let effective_cursor = match query.reverse { + false => { + let narrowed = cursor_key.as_ref().filter(|ck| ck.as_slice() > range_lo); + match narrowed { + Some(ck) => { + range_lo = ck.as_slice(); + Some(ck.as_slice()) + } + None => None, + } + } + true => { + if let Some(ck) = cursor_key.as_ref().filter(|ck| ck.as_slice() < range_hi) { + range_hi = ck.as_slice(); + } + None + } + }; + + match range_lo >= range_hi { + true => Ok(Vec::new()), + false => match query.reverse { + false => { + self.list_records_forward(range_lo, range_hi, effective_cursor, query.limit) + } + true => self.list_records_reverse(range_lo, range_hi, query.limit), + }, + } + } + + fn list_records_forward( + &self, + range_start: &[u8], + range_end: &[u8], + cursor_key: Option<&[u8]>, + limit: usize, + ) -> Result, MetastoreError> { + self.repo_data + .range(range_start..range_end) + .filter_map(|guard| { + let (key_bytes, val_bytes) = match guard.into_inner() { + Ok(pair) => pair, + Err(e) => return Some(Err(MetastoreError::Fjall(e))), + }; + + match cursor_key { + Some(ck) if key_bytes.as_ref() <= ck => None, + _ => Some(decode_record_info(&key_bytes, &val_bytes)), + } + }) + .take(limit) + .collect() + } + + fn list_records_reverse( + &self, + range_start: &[u8], + range_end: &[u8], + limit: usize, + ) -> Result, MetastoreError> { + self.repo_data + .range(range_start..range_end) + .rev() + .map(|guard| { + let (key_bytes, val_bytes) = guard.into_inner().map_err(MetastoreError::Fjall)?; + decode_record_info(&key_bytes, &val_bytes) + }) + .take(limit) + .collect() + } + + pub fn get_all_records(&self, user_id: Uuid) -> Result, MetastoreError> { + let user_hash = match self.user_hashes.get(&user_id) { + Some(h) => h, + None => return Ok(Vec::new()), + }; + let prefix = record_user_prefix(user_hash); + + self.repo_data + .prefix(prefix.as_slice()) + .map(|guard| { + let (key_bytes, val_bytes) = guard.into_inner().map_err(MetastoreError::Fjall)?; + decode_full_record_info(&key_bytes, &val_bytes) + }) + .collect() + } + + pub fn list_collections(&self, user_id: Uuid) -> Result, MetastoreError> { + let user_hash = match self.user_hashes.get(&user_id) { + Some(h) => h, + None => return Ok(Vec::new()), + }; + let user_pfx = record_user_prefix(user_hash); + let user_upper = exclusive_upper_bound(user_pfx.as_slice()) + .expect("user prefix always contains non-0xFF bytes"); + + let mut collections: Vec = Vec::new(); + let mut seek_from: SmallVec<[u8; 128]> = user_pfx.clone(); + + loop { + let entry = self + .repo_data + .range(seek_from.as_slice()..user_upper.as_slice()) + .next(); + let guard = match entry { + Some(g) => g, + None => break, + }; + let (key_bytes, _) = guard.into_inner().map_err(MetastoreError::Fjall)?; + let collection = parse_record_key_collection(&key_bytes) + .ok_or(MetastoreError::CorruptData("invalid record key"))?; + let coll_prefix = record_collection_prefix(user_hash, &collection); + seek_from = exclusive_upper_bound(coll_prefix.as_slice()) + .expect("collection prefix always contains non-0xFF bytes"); + collections.push(Nsid::from(collection)); + } + + Ok(collections) + } + + pub fn count_records(&self, user_id: Uuid) -> Result { + let user_hash = match self.user_hashes.get(&user_id) { + Some(h) => h, + None => return Ok(0), + }; + let prefix = record_user_prefix(user_hash); + count_prefix(&self.repo_data, prefix.as_slice()) + } + + pub fn count_all_records(&self) -> Result { + let prefix = records_prefix(); + count_prefix(&self.repo_data, prefix.as_slice()) + } + + pub fn get_record_by_cid( + &self, + cid: &CidLink, + scope_user: Option, + ) -> Result, MetastoreError> { + let target_bytes = cid_link_to_bytes(cid)?; + let prefix = match scope_user { + Some(uid) => match self.user_hashes.get(&uid) { + Some(hash) => record_user_prefix(hash), + None => return Ok(None), + }, + None => records_prefix(), + }; + + self.repo_data + .prefix(prefix.as_slice()) + .find_map(|guard| { + let (key_bytes, val_bytes) = match guard.into_inner() { + Ok(pair) => pair, + Err(e) => return Some(Err(MetastoreError::Fjall(e))), + }; + let value = match RecordValue::deserialize(&val_bytes) { + Some(v) => v, + None => return Some(Err(MetastoreError::CorruptData("invalid record value"))), + }; + + match value.record_cid == target_bytes { + true => { + let (coll_str, rkey_str) = match parse_record_key_fields(&key_bytes) { + Some(pair) => pair, + None => { + return Some(Err(MetastoreError::CorruptData( + "invalid record key", + ))); + } + }; + let user_hash = match parse_record_key_user_hash(&key_bytes) { + Some(h) => h, + None => { + return Some(Err(MetastoreError::CorruptData( + "invalid record key", + ))); + } + }; + let user_id = match self.user_hashes.get_uuid(&user_hash) { + Some(id) => id, + None => { + return Some(Err(MetastoreError::CorruptData( + "record user_hash has no reverse mapping", + ))); + } + }; + Some(Ok(RecordWithTakedown { + id: user_id, + collection: Nsid::from(coll_str), + rkey: Rkey::from(rkey_str), + takedown_ref: value.takedown_ref, + })) + } + false => None, + } + }) + .transpose() + } + + pub fn set_record_takedown( + &self, + db: &fjall::Database, + cid: &CidLink, + takedown_ref: Option<&str>, + scope_user: Option, + ) -> Result<(), MetastoreError> { + let target_bytes = cid_link_to_bytes(cid)?; + let prefix = match scope_user { + Some(uid) => match self.user_hashes.get(&uid) { + Some(hash) => record_user_prefix(hash), + None => return Ok(()), + }, + None => records_prefix(), + }; + + let found = self + .repo_data + .prefix(prefix.as_slice()) + .find_map(|guard| { + let (key_bytes, val_bytes) = match guard.into_inner() { + Ok(pair) => pair, + Err(e) => return Some(Err(MetastoreError::Fjall(e))), + }; + let value = match RecordValue::deserialize(&val_bytes) { + Some(v) => v, + None => return Some(Err(MetastoreError::CorruptData("invalid record value"))), + }; + match value.record_cid == target_bytes { + true => Some(Ok((key_bytes.to_vec(), value))), + false => None, + } + }) + .transpose()?; + + match found { + Some((key, mut value)) => { + value.takedown_ref = takedown_ref.map(str::to_string); + let mut batch = db.batch(); + batch.insert(&self.repo_data, &key, value.serialize()); + batch.commit().map_err(MetastoreError::Fjall) + } + None => Ok(()), + } + } +} + +fn decode_record_info(key_bytes: &[u8], val_bytes: &[u8]) -> Result { + let value = RecordValue::deserialize(val_bytes) + .ok_or(MetastoreError::CorruptData("invalid record value"))?; + let (_collection, rkey) = parse_record_key_fields(key_bytes) + .ok_or(MetastoreError::CorruptData("invalid record key"))?; + let cid = bytes_to_cid_link(&value.record_cid)?; + Ok(RecordInfo { + rkey: Rkey::from(rkey), + record_cid: cid, + }) +} + +fn decode_full_record_info( + key_bytes: &[u8], + val_bytes: &[u8], +) -> Result { + let value = RecordValue::deserialize(val_bytes) + .ok_or(MetastoreError::CorruptData("invalid record value"))?; + let (collection, rkey) = parse_record_key_fields(key_bytes) + .ok_or(MetastoreError::CorruptData("invalid record key"))?; + let cid = bytes_to_cid_link(&value.record_cid)?; + Ok(FullRecordInfo { + collection: Nsid::from(collection), + rkey: Rkey::from(rkey), + record_cid: cid, + }) +} + +fn parse_record_key_fields(key_bytes: &[u8]) -> Option<(String, String)> { + let mut reader = KeyReader::new(key_bytes); + let _tag = reader.tag()?; + let _user_hash = reader.u64()?; + let collection = reader.string()?; + let rkey = reader.string()?; + Some((collection, rkey)) +} + +fn parse_record_key_collection(key_bytes: &[u8]) -> Option { + let mut reader = KeyReader::new(key_bytes); + let _tag = reader.tag()?; + let _user_hash = reader.u64()?; + reader.string() +} + +fn parse_record_key_user_hash(key_bytes: &[u8]) -> Option { + let mut reader = KeyReader::new(key_bytes); + let _tag = reader.tag()?; + let hash = reader.u64()?; + Some(UserHash::from_raw(hash)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::metastore::{Metastore, MetastoreConfig}; + + fn test_config() -> MetastoreConfig { + MetastoreConfig { + cache_size_bytes: 64 * 1024 * 1024, + } + } + + fn test_cid_link(seed: u8) -> CidLink { + let digest: [u8; 32] = std::array::from_fn(|i| seed.wrapping_add(i as u8)); + let mh = multihash::Multihash::<64>::wrap(0x12, &digest).unwrap(); + let c = cid::Cid::new_v1(0x71, mh); + CidLink::from_cid(&c) + } + + fn open_fresh() -> (tempfile::TempDir, Metastore) { + let dir = tempfile::TempDir::new().unwrap(); + let ms = Metastore::open(dir.path(), test_config()).unwrap(); + (dir, ms) + } + + fn test_did(name: &str) -> tranquil_types::Did { + tranquil_types::Did::from(format!("did:plc:{name}")) + } + + fn test_handle(name: &str) -> tranquil_types::Handle { + tranquil_types::Handle::from(format!("{name}.test.invalid")) + } + + fn setup_user(ms: &Metastore) -> (Uuid, super::super::keys::UserHash) { + let user_id = Uuid::new_v4(); + let did = test_did("testuser"); + let handle = test_handle("testuser"); + let cid = test_cid_link(0); + ms.repo_ops() + .create_repo(ms.database(), user_id, &did, &handle, &cid, "rev0") + .unwrap(); + let user_hash = ms.user_hashes().get(&user_id).unwrap(); + (user_id, user_hash) + } + + fn rw<'a>(collection: &'a Nsid, rkey: &'a Rkey, cid: &'a CidLink) -> RecordWrite<'a> { + RecordWrite { + collection, + rkey, + cid, + } + } + + fn rd<'a>(collection: &'a Nsid, rkey: &'a Rkey) -> RecordDelete<'a> { + RecordDelete { collection, rkey } + } + + fn lrq<'a>( + user_id: Uuid, + collection: &'a Nsid, + cursor: Option<&'a Rkey>, + limit: usize, + reverse: bool, + rkey_start: Option<&'a Rkey>, + rkey_end: Option<&'a Rkey>, + ) -> ListRecordsQuery<'a> { + ListRecordsQuery { + user_id, + collection, + cursor, + limit, + reverse, + rkey_start, + rkey_end, + } + } + + #[test] + fn upsert_and_get_record() { + let (_dir, ms) = open_fresh(); + let (user_id, user_hash) = setup_user(&ms); + let rec_ops = ms.record_ops(); + + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkey = Rkey::from("3k2abcd".to_string()); + let cid = test_cid_link(1); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records(&mut batch, user_hash, &[rw(&collection, &rkey, &cid)]) + .unwrap(); + batch.commit().unwrap(); + + let found = rec_ops.get_record_cid(user_id, &collection, &rkey).unwrap(); + assert_eq!(found, Some(cid)); + } + + #[test] + fn get_record_returns_none_for_missing() { + let (_dir, ms) = open_fresh(); + let (user_id, _) = setup_user(&ms); + let rec_ops = ms.record_ops(); + + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkey = Rkey::from("nonexistent".to_string()); + assert!( + rec_ops + .get_record_cid(user_id, &collection, &rkey) + .unwrap() + .is_none() + ); + } + + #[test] + fn upsert_overwrites_existing() { + let (_dir, ms) = open_fresh(); + let (user_id, user_hash) = setup_user(&ms); + let rec_ops = ms.record_ops(); + + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkey = Rkey::from("3k2abcd".to_string()); + let cid1 = test_cid_link(1); + let cid2 = test_cid_link(2); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records(&mut batch, user_hash, &[rw(&collection, &rkey, &cid1)]) + .unwrap(); + batch.commit().unwrap(); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records(&mut batch, user_hash, &[rw(&collection, &rkey, &cid2)]) + .unwrap(); + batch.commit().unwrap(); + + let found = rec_ops.get_record_cid(user_id, &collection, &rkey).unwrap(); + assert_eq!(found, Some(cid2)); + } + + #[test] + fn delete_records_removes_entries() { + let (_dir, ms) = open_fresh(); + let (user_id, user_hash) = setup_user(&ms); + let rec_ops = ms.record_ops(); + + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkey = Rkey::from("3k2abcd".to_string()); + let cid = test_cid_link(1); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records(&mut batch, user_hash, &[rw(&collection, &rkey, &cid)]) + .unwrap(); + batch.commit().unwrap(); + + let mut batch = ms.database().batch(); + rec_ops.delete_records(&mut batch, user_hash, &[rd(&collection, &rkey)]); + batch.commit().unwrap(); + + assert!( + rec_ops + .get_record_cid(user_id, &collection, &rkey) + .unwrap() + .is_none() + ); + } + + #[test] + fn delete_all_records_clears_user() { + let (_dir, ms) = open_fresh(); + let (user_id, user_hash) = setup_user(&ms); + let rec_ops = ms.record_ops(); + + let coll1 = Nsid::from("app.bsky.feed.post".to_string()); + let coll2 = Nsid::from("app.bsky.feed.like".to_string()); + let rkey_a = Rkey::from("a".to_string()); + let rkey_b = Rkey::from("b".to_string()); + let cid1 = test_cid_link(1); + let cid2 = test_cid_link(2); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records( + &mut batch, + user_hash, + &[rw(&coll1, &rkey_a, &cid1), rw(&coll2, &rkey_b, &cid2)], + ) + .unwrap(); + batch.commit().unwrap(); + + assert_eq!(rec_ops.count_records(user_id).unwrap(), 2); + + let mut batch = ms.database().batch(); + rec_ops.delete_all_records(&mut batch, user_hash).unwrap(); + batch.commit().unwrap(); + + assert_eq!(rec_ops.count_records(user_id).unwrap(), 0); + } + + #[test] + fn list_records_forward_with_limit() { + let (_dir, ms) = open_fresh(); + let (user_id, user_hash) = setup_user(&ms); + let rec_ops = ms.record_ops(); + + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkeys: Vec = (0..5).map(|i| Rkey::from(format!("rkey{i:03}"))).collect(); + let cids: Vec = (0..5).map(|i| test_cid_link(i + 1)).collect(); + + let writes: Vec> = rkeys + .iter() + .zip(cids.iter()) + .map(|(rk, c)| rw(&collection, rk, c)) + .collect(); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records(&mut batch, user_hash, &writes) + .unwrap(); + batch.commit().unwrap(); + + let results = rec_ops + .list_records(&lrq(user_id, &collection, None, 3, false, None, None)) + .unwrap(); + assert_eq!(results.len(), 3); + assert_eq!(results[0].rkey.as_str(), "rkey000"); + assert_eq!(results[1].rkey.as_str(), "rkey001"); + assert_eq!(results[2].rkey.as_str(), "rkey002"); + } + + #[test] + fn list_records_with_cursor() { + let (_dir, ms) = open_fresh(); + let (user_id, user_hash) = setup_user(&ms); + let rec_ops = ms.record_ops(); + + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkeys: Vec = (0..5).map(|i| Rkey::from(format!("rkey{i:03}"))).collect(); + let cids: Vec = (0..5).map(|i| test_cid_link(i + 1)).collect(); + + let writes: Vec> = rkeys + .iter() + .zip(cids.iter()) + .map(|(rk, c)| rw(&collection, rk, c)) + .collect(); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records(&mut batch, user_hash, &writes) + .unwrap(); + batch.commit().unwrap(); + + let cursor = Rkey::from("rkey001".to_string()); + let results = rec_ops + .list_records(&lrq( + user_id, + &collection, + Some(&cursor), + 10, + false, + None, + None, + )) + .unwrap(); + assert_eq!(results.len(), 3); + assert_eq!(results[0].rkey.as_str(), "rkey002"); + assert_eq!(results[1].rkey.as_str(), "rkey003"); + assert_eq!(results[2].rkey.as_str(), "rkey004"); + } + + #[test] + fn list_records_reverse() { + let (_dir, ms) = open_fresh(); + let (user_id, user_hash) = setup_user(&ms); + let rec_ops = ms.record_ops(); + + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkeys: Vec = (0..5).map(|i| Rkey::from(format!("rkey{i:03}"))).collect(); + let cids: Vec = (0..5).map(|i| test_cid_link(i + 1)).collect(); + + let writes: Vec> = rkeys + .iter() + .zip(cids.iter()) + .map(|(rk, c)| rw(&collection, rk, c)) + .collect(); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records(&mut batch, user_hash, &writes) + .unwrap(); + batch.commit().unwrap(); + + let results = rec_ops + .list_records(&lrq(user_id, &collection, None, 3, true, None, None)) + .unwrap(); + assert_eq!(results.len(), 3); + assert_eq!(results[0].rkey.as_str(), "rkey004"); + assert_eq!(results[1].rkey.as_str(), "rkey003"); + assert_eq!(results[2].rkey.as_str(), "rkey002"); + } + + #[test] + fn list_records_reverse_with_cursor() { + let (_dir, ms) = open_fresh(); + let (user_id, user_hash) = setup_user(&ms); + let rec_ops = ms.record_ops(); + + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkeys: Vec = (0..5).map(|i| Rkey::from(format!("rkey{i:03}"))).collect(); + let cids: Vec = (0..5).map(|i| test_cid_link(i + 1)).collect(); + + let writes: Vec> = rkeys + .iter() + .zip(cids.iter()) + .map(|(rk, c)| rw(&collection, rk, c)) + .collect(); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records(&mut batch, user_hash, &writes) + .unwrap(); + batch.commit().unwrap(); + + let cursor = Rkey::from("rkey003".to_string()); + let results = rec_ops + .list_records(&lrq( + user_id, + &collection, + Some(&cursor), + 10, + true, + None, + None, + )) + .unwrap(); + assert_eq!(results.len(), 3); + assert_eq!(results[0].rkey.as_str(), "rkey002"); + assert_eq!(results[1].rkey.as_str(), "rkey001"); + assert_eq!(results[2].rkey.as_str(), "rkey000"); + } + + #[test] + fn list_records_rkey_range_bounds() { + let (_dir, ms) = open_fresh(); + let (user_id, user_hash) = setup_user(&ms); + let rec_ops = ms.record_ops(); + + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkeys: Vec = (0..10).map(|i| Rkey::from(format!("rkey{i:03}"))).collect(); + let cids: Vec = (0..10).map(|i| test_cid_link(i + 1)).collect(); + + let writes: Vec> = rkeys + .iter() + .zip(cids.iter()) + .map(|(rk, c)| rw(&collection, rk, c)) + .collect(); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records(&mut batch, user_hash, &writes) + .unwrap(); + batch.commit().unwrap(); + + let rkey_start = Rkey::from("rkey003".to_string()); + let rkey_end = Rkey::from("rkey006".to_string()); + let results = rec_ops + .list_records(&lrq( + user_id, + &collection, + None, + 100, + false, + Some(&rkey_start), + Some(&rkey_end), + )) + .unwrap(); + assert_eq!(results.len(), 4); + assert_eq!(results[0].rkey.as_str(), "rkey003"); + assert_eq!(results[3].rkey.as_str(), "rkey006"); + } + + #[test] + fn get_all_records_across_collections() { + let (_dir, ms) = open_fresh(); + let (user_id, user_hash) = setup_user(&ms); + let rec_ops = ms.record_ops(); + + let coll1 = Nsid::from("app.bsky.feed.like".to_string()); + let coll2 = Nsid::from("app.bsky.feed.post".to_string()); + let rkey_a = Rkey::from("a".to_string()); + let rkey_b = Rkey::from("b".to_string()); + let rkey_c = Rkey::from("c".to_string()); + let cid1 = test_cid_link(1); + let cid2 = test_cid_link(2); + let cid3 = test_cid_link(3); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records( + &mut batch, + user_hash, + &[ + rw(&coll1, &rkey_a, &cid1), + rw(&coll2, &rkey_b, &cid2), + rw(&coll1, &rkey_c, &cid3), + ], + ) + .unwrap(); + batch.commit().unwrap(); + + let all = rec_ops.get_all_records(user_id).unwrap(); + assert_eq!(all.len(), 3); + assert_eq!(all[0].collection.as_str(), "app.bsky.feed.like"); + assert_eq!(all[1].collection.as_str(), "app.bsky.feed.like"); + assert_eq!(all[2].collection.as_str(), "app.bsky.feed.post"); + } + + #[test] + fn list_collections_returns_distinct_sorted() { + let (_dir, ms) = open_fresh(); + let (user_id, user_hash) = setup_user(&ms); + let rec_ops = ms.record_ops(); + + let coll1 = Nsid::from("app.bsky.feed.like".to_string()); + let coll2 = Nsid::from("app.bsky.feed.post".to_string()); + let coll3 = Nsid::from("app.bsky.graph.follow".to_string()); + let rkeys: Vec = ["a", "b", "c", "d", "e"] + .iter() + .map(|s| Rkey::from(s.to_string())) + .collect(); + let cids: Vec = (1..=5).map(test_cid_link).collect(); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records( + &mut batch, + user_hash, + &[ + rw(&coll1, &rkeys[0], &cids[0]), + rw(&coll2, &rkeys[1], &cids[1]), + rw(&coll1, &rkeys[2], &cids[2]), + rw(&coll3, &rkeys[3], &cids[3]), + rw(&coll2, &rkeys[4], &cids[4]), + ], + ) + .unwrap(); + batch.commit().unwrap(); + + let collections = rec_ops.list_collections(user_id).unwrap(); + assert_eq!(collections.len(), 3); + assert_eq!(collections[0].as_str(), "app.bsky.feed.like"); + assert_eq!(collections[1].as_str(), "app.bsky.feed.post"); + assert_eq!(collections[2].as_str(), "app.bsky.graph.follow"); + } + + #[test] + fn count_records_and_count_all() { + let (_dir, ms) = open_fresh(); + let (user_id, user_hash) = setup_user(&ms); + let rec_ops = ms.record_ops(); + + assert_eq!(rec_ops.count_records(user_id).unwrap(), 0); + assert_eq!(rec_ops.count_all_records().unwrap(), 0); + + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkey_a = Rkey::from("a".to_string()); + let rkey_b = Rkey::from("b".to_string()); + let cid1 = test_cid_link(1); + let cid2 = test_cid_link(2); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records( + &mut batch, + user_hash, + &[ + rw(&collection, &rkey_a, &cid1), + rw(&collection, &rkey_b, &cid2), + ], + ) + .unwrap(); + batch.commit().unwrap(); + + assert_eq!(rec_ops.count_records(user_id).unwrap(), 2); + assert_eq!(rec_ops.count_all_records().unwrap(), 2); + } + + #[test] + fn records_isolated_between_users() { + let (_dir, ms) = open_fresh(); + let rec_ops = ms.record_ops(); + + let user1 = Uuid::new_v4(); + let did1 = test_did("user1"); + let handle1 = test_handle("user1"); + ms.repo_ops() + .create_repo( + ms.database(), + user1, + &did1, + &handle1, + &test_cid_link(0), + "r", + ) + .unwrap(); + let hash1 = ms.user_hashes().get(&user1).unwrap(); + + let user2 = Uuid::new_v4(); + let did2 = test_did("user2"); + let handle2 = test_handle("user2"); + ms.repo_ops() + .create_repo( + ms.database(), + user2, + &did2, + &handle2, + &test_cid_link(0), + "r", + ) + .unwrap(); + let hash2 = ms.user_hashes().get(&user2).unwrap(); + + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkey_a = Rkey::from("a".to_string()); + let rkey_b = Rkey::from("b".to_string()); + let cid1 = test_cid_link(1); + let cid2 = test_cid_link(2); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records(&mut batch, hash1, &[rw(&collection, &rkey_a, &cid1)]) + .unwrap(); + rec_ops + .upsert_records(&mut batch, hash2, &[rw(&collection, &rkey_b, &cid2)]) + .unwrap(); + batch.commit().unwrap(); + + assert_eq!(rec_ops.count_records(user1).unwrap(), 1); + assert_eq!(rec_ops.count_records(user2).unwrap(), 1); + assert_eq!(rec_ops.count_all_records().unwrap(), 2); + } + + #[test] + fn get_record_by_cid_finds_match() { + let (_dir, ms) = open_fresh(); + let (user_id, user_hash) = setup_user(&ms); + let rec_ops = ms.record_ops(); + + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkey = Rkey::from("r1".to_string()); + let cid = test_cid_link(42); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records(&mut batch, user_hash, &[rw(&collection, &rkey, &cid)]) + .unwrap(); + batch.commit().unwrap(); + + let found = rec_ops.get_record_by_cid(&cid, None).unwrap().unwrap(); + assert_eq!(found.id, user_id); + assert_eq!(found.collection.as_str(), "app.bsky.feed.post"); + assert_eq!(found.rkey.as_str(), "r1"); + assert!(found.takedown_ref.is_none()); + } + + #[test] + fn get_record_by_cid_returns_none_for_missing() { + let (_dir, ms) = open_fresh(); + let rec_ops = ms.record_ops(); + assert!( + rec_ops + .get_record_by_cid(&test_cid_link(99), None) + .unwrap() + .is_none() + ); + } + + #[test] + fn get_record_by_cid_scoped_to_user() { + let (_dir, ms) = open_fresh(); + let (user_id, user_hash) = setup_user(&ms); + let rec_ops = ms.record_ops(); + + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkey = Rkey::from("r1".to_string()); + let cid = test_cid_link(42); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records(&mut batch, user_hash, &[rw(&collection, &rkey, &cid)]) + .unwrap(); + batch.commit().unwrap(); + + let found = rec_ops + .get_record_by_cid(&cid, Some(user_id)) + .unwrap() + .unwrap(); + assert_eq!(found.id, user_id); + + let other_user = Uuid::new_v4(); + assert!( + rec_ops + .get_record_by_cid(&cid, Some(other_user)) + .unwrap() + .is_none() + ); + } + + #[test] + fn set_and_get_takedown() { + let (_dir, ms) = open_fresh(); + let (user_id, user_hash) = setup_user(&ms); + let rec_ops = ms.record_ops(); + + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkey = Rkey::from("r1".to_string()); + let cid = test_cid_link(42); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records(&mut batch, user_hash, &[rw(&collection, &rkey, &cid)]) + .unwrap(); + batch.commit().unwrap(); + + rec_ops + .set_record_takedown(ms.database(), &cid, Some("DMCA-789"), None) + .unwrap(); + + let found = rec_ops.get_record_by_cid(&cid, None).unwrap().unwrap(); + assert_eq!(found.id, user_id); + assert_eq!(found.takedown_ref.as_deref(), Some("DMCA-789")); + + rec_ops + .set_record_takedown(ms.database(), &cid, None, None) + .unwrap(); + + let found = rec_ops.get_record_by_cid(&cid, None).unwrap().unwrap(); + assert!(found.takedown_ref.is_none()); + } + + #[test] + fn upsert_preserves_existing_takedown() { + let (_dir, ms) = open_fresh(); + let (_user_id, user_hash) = setup_user(&ms); + let rec_ops = ms.record_ops(); + + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkey = Rkey::from("r1".to_string()); + let cid1 = test_cid_link(1); + let cid2 = test_cid_link(2); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records(&mut batch, user_hash, &[rw(&collection, &rkey, &cid1)]) + .unwrap(); + batch.commit().unwrap(); + + rec_ops + .set_record_takedown(ms.database(), &cid1, Some("DMCA-999"), None) + .unwrap(); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records(&mut batch, user_hash, &[rw(&collection, &rkey, &cid2)]) + .unwrap(); + batch.commit().unwrap(); + + let found = rec_ops.get_record_by_cid(&cid2, None).unwrap().unwrap(); + assert_eq!(found.takedown_ref.as_deref(), Some("DMCA-999")); + } + + #[test] + fn records_survive_reopen() { + let dir = tempfile::TempDir::new().unwrap(); + let user_id; + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkey = Rkey::from("durable".to_string()); + let cid = test_cid_link(77); + + { + let ms = Metastore::open(dir.path(), test_config()).unwrap(); + user_id = Uuid::new_v4(); + let did = test_did("persist"); + let handle = test_handle("persist"); + ms.repo_ops() + .create_repo( + ms.database(), + user_id, + &did, + &handle, + &test_cid_link(0), + "r", + ) + .unwrap(); + let user_hash = ms.user_hashes().get(&user_id).unwrap(); + + let rec_ops = ms.record_ops(); + let mut batch = ms.database().batch(); + rec_ops + .upsert_records(&mut batch, user_hash, &[rw(&collection, &rkey, &cid)]) + .unwrap(); + batch.commit().unwrap(); + ms.persist().unwrap(); + } + + { + let ms = Metastore::open(dir.path(), test_config()).unwrap(); + let rec_ops = ms.record_ops(); + let found = rec_ops.get_record_cid(user_id, &collection, &rkey).unwrap(); + assert_eq!(found, Some(cid)); + } + } + + #[test] + fn rkey_ordering_matches_lexicographic() { + let (_dir, ms) = open_fresh(); + let (user_id, user_hash) = setup_user(&ms); + let rec_ops = ms.record_ops(); + + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkeys_unordered = ["zebra", "apple", "mango", "banana"]; + + let rkeys: Vec = rkeys_unordered + .iter() + .map(|rk| Rkey::from(rk.to_string())) + .collect(); + let cids: Vec = (0..rkeys.len()) + .map(|i| test_cid_link(i as u8 + 1)) + .collect(); + + let writes: Vec> = rkeys + .iter() + .zip(cids.iter()) + .map(|(rk, c)| rw(&collection, rk, c)) + .collect(); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records(&mut batch, user_hash, &writes) + .unwrap(); + batch.commit().unwrap(); + + let results = rec_ops + .list_records(&lrq(user_id, &collection, None, 100, false, None, None)) + .unwrap(); + let result_rkeys: Vec<&str> = results.iter().map(|r| r.rkey.as_str()).collect(); + assert_eq!(result_rkeys, ["apple", "banana", "mango", "zebra"]); + } + + #[test] + fn record_with_empty_rkey() { + let (_dir, ms) = open_fresh(); + let (user_id, user_hash) = setup_user(&ms); + let rec_ops = ms.record_ops(); + + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkey = Rkey::from(String::new()); + let cid = test_cid_link(1); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records(&mut batch, user_hash, &[rw(&collection, &rkey, &cid)]) + .unwrap(); + batch.commit().unwrap(); + + let found = rec_ops.get_record_cid(user_id, &collection, &rkey).unwrap(); + assert_eq!(found, Some(cid)); + + let results = rec_ops + .list_records(&lrq(user_id, &collection, None, 100, false, None, None)) + .unwrap(); + assert_eq!(results.len(), 1); + } + + #[test] + fn record_with_null_bytes_in_rkey() { + let (_dir, ms) = open_fresh(); + let (user_id, user_hash) = setup_user(&ms); + let rec_ops = ms.record_ops(); + + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkey_with_null = Rkey::from("abc\x00def".to_string()); + let rkey_plain = Rkey::from("abc".to_string()); + let cid1 = test_cid_link(1); + let cid2 = test_cid_link(2); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records( + &mut batch, + user_hash, + &[ + rw(&collection, &rkey_with_null, &cid1), + rw(&collection, &rkey_plain, &cid2), + ], + ) + .unwrap(); + batch.commit().unwrap(); + + let found1 = rec_ops + .get_record_cid(user_id, &collection, &rkey_with_null) + .unwrap(); + assert_eq!(found1, Some(cid1)); + + let found2 = rec_ops + .get_record_cid(user_id, &collection, &rkey_plain) + .unwrap(); + assert_eq!(found2, Some(cid2)); + + let results = rec_ops + .list_records(&lrq(user_id, &collection, None, 100, false, None, None)) + .unwrap(); + assert_eq!(results.len(), 2); + assert_eq!(results[0].rkey.as_str(), "abc"); + assert_eq!(results[1].rkey.as_str(), "abc\x00def"); + } + + #[test] + fn record_with_null_bytes_in_collection() { + let (_dir, ms) = open_fresh(); + let (user_id, user_hash) = setup_user(&ms); + let rec_ops = ms.record_ops(); + + let coll_normal = Nsid::from("app.bsky.feed.post".to_string()); + let coll_with_null = Nsid::from("app.bsky.feed.post\x00extra".to_string()); + let rkey = Rkey::from("r1".to_string()); + let cid1 = test_cid_link(1); + let cid2 = test_cid_link(2); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records( + &mut batch, + user_hash, + &[ + rw(&coll_normal, &rkey, &cid1), + rw(&coll_with_null, &rkey, &cid2), + ], + ) + .unwrap(); + batch.commit().unwrap(); + + let found1 = rec_ops + .get_record_cid(user_id, &coll_normal, &rkey) + .unwrap(); + assert_eq!(found1, Some(cid1)); + + let found2 = rec_ops + .get_record_cid(user_id, &coll_with_null, &rkey) + .unwrap(); + assert_eq!(found2, Some(cid2)); + + let results_normal = rec_ops + .list_records(&lrq(user_id, &coll_normal, None, 100, false, None, None)) + .unwrap(); + assert_eq!(results_normal.len(), 1); + + let results_null = rec_ops + .list_records(&lrq(user_id, &coll_with_null, None, 100, false, None, None)) + .unwrap(); + assert_eq!(results_null.len(), 1); + + let collections = rec_ops.list_collections(user_id).unwrap(); + assert_eq!(collections.len(), 2); + } + + #[test] + fn list_records_cursor_past_rkey_end_returns_empty() { + let (_dir, ms) = open_fresh(); + let (user_id, user_hash) = setup_user(&ms); + let rec_ops = ms.record_ops(); + + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkeys: Vec = (0..5).map(|i| Rkey::from(format!("rkey{i:03}"))).collect(); + let cids: Vec = (0..5).map(|i| test_cid_link(i + 1)).collect(); + + let writes: Vec> = rkeys + .iter() + .zip(cids.iter()) + .map(|(rk, c)| rw(&collection, rk, c)) + .collect(); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records(&mut batch, user_hash, &writes) + .unwrap(); + batch.commit().unwrap(); + + let cursor = Rkey::from("rkey010".to_string()); + let rkey_end = Rkey::from("rkey003".to_string()); + let results = rec_ops + .list_records(&lrq( + user_id, + &collection, + Some(&cursor), + 100, + false, + None, + Some(&rkey_end), + )) + .unwrap(); + assert!(results.is_empty()); + } + + #[test] + fn list_records_reverse_with_rkey_bounds() { + let (_dir, ms) = open_fresh(); + let (user_id, user_hash) = setup_user(&ms); + let rec_ops = ms.record_ops(); + + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkeys: Vec = (0..10).map(|i| Rkey::from(format!("rkey{i:03}"))).collect(); + let cids: Vec = (0..10).map(|i| test_cid_link(i + 1)).collect(); + + let writes: Vec> = rkeys + .iter() + .zip(cids.iter()) + .map(|(rk, c)| rw(&collection, rk, c)) + .collect(); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records(&mut batch, user_hash, &writes) + .unwrap(); + batch.commit().unwrap(); + + let rkey_start = Rkey::from("rkey002".to_string()); + let rkey_end = Rkey::from("rkey007".to_string()); + let results = rec_ops + .list_records(&lrq( + user_id, + &collection, + None, + 100, + true, + Some(&rkey_start), + Some(&rkey_end), + )) + .unwrap(); + assert_eq!(results.len(), 6); + assert_eq!(results[0].rkey.as_str(), "rkey007"); + assert_eq!(results[5].rkey.as_str(), "rkey002"); + } + + #[test] + fn list_records_reverse_cursor_narrows_range() { + let (_dir, ms) = open_fresh(); + let (user_id, user_hash) = setup_user(&ms); + let rec_ops = ms.record_ops(); + + let collection = Nsid::from("app.bsky.feed.post".to_string()); + let rkeys: Vec = (0..10).map(|i| Rkey::from(format!("rkey{i:03}"))).collect(); + let cids: Vec = (0..10).map(|i| test_cid_link(i + 1)).collect(); + + let writes: Vec> = rkeys + .iter() + .zip(cids.iter()) + .map(|(rk, c)| rw(&collection, rk, c)) + .collect(); + + let mut batch = ms.database().batch(); + rec_ops + .upsert_records(&mut batch, user_hash, &writes) + .unwrap(); + batch.commit().unwrap(); + + let cursor = Rkey::from("rkey005".to_string()); + let results = rec_ops + .list_records(&lrq( + user_id, + &collection, + Some(&cursor), + 3, + true, + None, + None, + )) + .unwrap(); + assert_eq!(results.len(), 3); + assert_eq!(results[0].rkey.as_str(), "rkey004"); + assert_eq!(results[1].rkey.as_str(), "rkey003"); + assert_eq!(results[2].rkey.as_str(), "rkey002"); + } + + #[test] + fn exclusive_upper_bound_basic() { + let prefix = &[0x01, 0x02, 0x03]; + let upper = exclusive_upper_bound(prefix).unwrap(); + assert_eq!(upper.as_slice(), &[0x01, 0x02, 0x04]); + } + + #[test] + fn exclusive_upper_bound_with_trailing_ff() { + let prefix = &[0x01, 0xFF, 0xFF]; + let upper = exclusive_upper_bound(prefix).unwrap(); + assert_eq!(upper.as_slice(), &[0x02]); + } + + #[test] + fn exclusive_upper_bound_all_ff_returns_none() { + assert!(exclusive_upper_bound(&[0xFF, 0xFF, 0xFF]).is_none()); + } + + #[test] + fn exclusive_upper_bound_empty_returns_none() { + assert!(exclusive_upper_bound(&[]).is_none()); + } + + #[test] + fn exclusive_upper_bound_preserves_prefix_ordering() { + let prefix = &[ + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2A, 0x61, 0x00, 0x00, + ]; + let upper = exclusive_upper_bound(prefix).unwrap(); + + let key_inside = { + let mut k = prefix.to_vec(); + k.extend_from_slice(&[0x62, 0x00, 0x00]); + k + }; + assert!(key_inside.as_slice() < upper.as_slice()); + + let key_outside = { + let mut k = Vec::from(&prefix[..prefix.len() - 2]); + k.extend_from_slice(&[0x00, 0x01, 0x00, 0x00]); + k + }; + assert!(key_outside.as_slice() >= upper.as_slice()); + } +} diff --git a/crates/tranquil-store/src/metastore/records.rs b/crates/tranquil-store/src/metastore/records.rs new file mode 100644 index 0000000..4973f88 --- /dev/null +++ b/crates/tranquil-store/src/metastore/records.rs @@ -0,0 +1,162 @@ +use serde::{Deserialize, Serialize}; +use smallvec::SmallVec; + +use super::encoding::KeyBuilder; +use super::keys::{KeyTag, UserHash}; + +const SCHEMA_VERSION: u8 = 1; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RecordValue { + pub record_cid: Vec, + pub takedown_ref: Option, +} + +impl RecordValue { + pub fn serialize(&self) -> Vec { + let payload = postcard::to_allocvec(self).expect("RecordValue serialization cannot fail"); + let mut buf = Vec::with_capacity(1 + payload.len()); + buf.push(SCHEMA_VERSION); + buf.extend_from_slice(&payload); + buf + } + + pub fn deserialize(bytes: &[u8]) -> Option { + let (&version, payload) = bytes.split_first()?; + match version { + SCHEMA_VERSION => postcard::from_bytes(payload).ok(), + _ => None, + } + } +} + +pub fn record_key(user_hash: UserHash, collection: &str, rkey: &str) -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::RECORDS) + .u64(user_hash.raw()) + .string(collection) + .string(rkey) + .build() +} + +pub fn record_collection_prefix(user_hash: UserHash, collection: &str) -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::RECORDS) + .u64(user_hash.raw()) + .string(collection) + .build() +} + +pub fn record_user_prefix(user_hash: UserHash) -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::RECORDS) + .u64(user_hash.raw()) + .build() +} + +pub fn records_prefix() -> SmallVec<[u8; 128]> { + KeyBuilder::new().tag(KeyTag::RECORDS).build() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::metastore::encoding::KeyReader; + + #[test] + fn record_value_roundtrip() { + let value = RecordValue { + record_cid: vec![0x01, 0x71, 0x12, 0x20, 0xAB], + takedown_ref: None, + }; + let bytes = value.serialize(); + let decoded = RecordValue::deserialize(&bytes).unwrap(); + assert_eq!(decoded, value); + } + + #[test] + fn record_value_with_takedown() { + let value = RecordValue { + record_cid: vec![0x01], + takedown_ref: Some("DMCA-456".to_string()), + }; + let bytes = value.serialize(); + let decoded = RecordValue::deserialize(&bytes).unwrap(); + assert_eq!(decoded, value); + } + + #[test] + fn schema_version_is_first_byte() { + let value = RecordValue { + record_cid: vec![0x01], + takedown_ref: None, + }; + let bytes = value.serialize(); + assert_eq!(bytes[0], SCHEMA_VERSION); + } + + #[test] + fn deserialize_rejects_unknown_schema_version() { + let value = RecordValue { + record_cid: vec![0x01], + takedown_ref: None, + }; + let mut bytes = value.serialize(); + bytes[0] = 99; + assert!(RecordValue::deserialize(&bytes).is_none()); + } + + #[test] + fn deserialize_rejects_empty_input() { + assert!(RecordValue::deserialize(&[]).is_none()); + } + + #[test] + fn record_key_roundtrip() { + let hash = UserHash::from_raw(0xDEAD_BEEF_CAFE_BABE); + let key = record_key(hash, "app.bsky.feed.post", "3k2abcd"); + let mut reader = KeyReader::new(&key); + assert_eq!(reader.tag(), Some(KeyTag::RECORDS.raw())); + assert_eq!(reader.u64(), Some(0xDEAD_BEEF_CAFE_BABE)); + assert_eq!(reader.string(), Some("app.bsky.feed.post".to_string())); + assert_eq!(reader.string(), Some("3k2abcd".to_string())); + assert!(reader.is_empty()); + } + + #[test] + fn record_keys_sort_by_user_then_collection_then_rkey() { + let h1 = UserHash::from_raw(1); + let h2 = UserHash::from_raw(2); + + let k1 = record_key(h1, "app.bsky.feed.like", "aaa"); + let k2 = record_key(h1, "app.bsky.feed.post", "aaa"); + let k3 = record_key(h1, "app.bsky.feed.post", "bbb"); + let k4 = record_key(h2, "app.bsky.feed.like", "aaa"); + + assert!(k1.as_slice() < k2.as_slice()); + assert!(k2.as_slice() < k3.as_slice()); + assert!(k3.as_slice() < k4.as_slice()); + } + + #[test] + fn collection_prefix_is_prefix_of_full_key() { + let hash = UserHash::from_raw(42); + let prefix = record_collection_prefix(hash, "app.bsky.feed.post"); + let full = record_key(hash, "app.bsky.feed.post", "some_rkey"); + assert!(full.as_slice().starts_with(prefix.as_slice())); + } + + #[test] + fn user_prefix_is_prefix_of_collection_prefix() { + let hash = UserHash::from_raw(42); + let user_pfx = record_user_prefix(hash); + let coll_pfx = record_collection_prefix(hash, "app.bsky.feed.post"); + assert!(coll_pfx.as_slice().starts_with(user_pfx.as_slice())); + } + + #[test] + fn records_prefix_is_just_tag() { + let pfx = records_prefix(); + assert_eq!(pfx.as_slice(), &[KeyTag::RECORDS.raw()]); + } +} diff --git a/crates/tranquil-store/src/metastore/recovery.rs b/crates/tranquil-store/src/metastore/recovery.rs new file mode 100644 index 0000000..535d64b --- /dev/null +++ b/crates/tranquil-store/src/metastore/recovery.rs @@ -0,0 +1,307 @@ +use std::collections::HashSet; + +use serde::{Deserialize, Serialize}; + +use super::backlink_ops::remove_backlinks_for_record; +use super::backlinks::{BacklinkValue, backlink_by_user_key, backlink_key, discriminant_to_path}; +use super::encoding::KeyReader; +use super::keys::{KeyTag, UserHash}; +use super::records::{RecordValue, record_key}; +use super::repo_meta::{RepoMetaValue, repo_meta_key}; +use super::user_blocks::{user_block_key, user_block_user_prefix}; +use crate::metastore::MetastoreError; + +const MUTATION_SET_VERSION: u8 = 1; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CommitMutationSet { + pub new_root_cid: Vec, + pub new_rev: String, + pub record_upserts: Vec, + pub record_deletes: Vec, + pub block_inserts: Vec>, + pub block_deletes: Vec>, + pub backlink_adds: Vec, + pub backlink_remove_uris: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RecordMutationUpsert { + pub collection: String, + pub rkey: String, + pub cid_bytes: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RecordMutationDelete { + pub collection: String, + pub rkey: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BacklinkMutation { + pub uri: String, + pub path: u8, + pub link_to: String, +} + +const MAX_MUTATION_SET_ENTRIES: usize = 50_000; + +impl CommitMutationSet { + pub fn serialize(&self) -> Result, MetastoreError> { + self.validate_size()?; + let payload = postcard::to_allocvec(self) + .map_err(|_| MetastoreError::CorruptData("CommitMutationSet serialization failed"))?; + let mut buf = Vec::with_capacity(1 + payload.len()); + buf.push(MUTATION_SET_VERSION); + buf.extend_from_slice(&payload); + Ok(buf) + } + + fn validate_size(&self) -> Result<(), MetastoreError> { + let total = self.record_upserts.len() + + self.record_deletes.len() + + self.block_inserts.len() + + self.block_deletes.len() + + self.backlink_adds.len() + + self.backlink_remove_uris.len(); + match total <= MAX_MUTATION_SET_ENTRIES { + true => Ok(()), + false => { + tracing::warn!( + total_entries = total, + max = MAX_MUTATION_SET_ENTRIES, + "CommitMutationSet exceeds entry limit" + ); + Err(MetastoreError::InvalidInput( + "CommitMutationSet exceeds maximum entry count", + )) + } + } + } + + pub fn deserialize(bytes: &[u8]) -> Option { + let (&version, payload) = bytes.split_first()?; + match version { + MUTATION_SET_VERSION => match postcard::from_bytes(payload) { + Ok(v) => Some(v), + Err(e) => { + tracing::warn!(%e, "failed to deserialize CommitMutationSet payload"); + None + } + }, + _ => { + tracing::warn!(version, "unknown CommitMutationSet version"); + None + } + } + } +} + +pub fn replay_mutation_set( + batch: &mut fjall::OwnedWriteBatch, + repo_data: &fjall::Keyspace, + indexes: &fjall::Keyspace, + user_hash: UserHash, + current_meta: &RepoMetaValue, + mutation_set: &CommitMutationSet, +) -> Result<(), MetastoreError> { + mutation_set.validate_size()?; + + let updated_meta = RepoMetaValue { + repo_root_cid: mutation_set.new_root_cid.clone(), + repo_rev: mutation_set.new_rev.clone(), + ..current_meta.clone() + }; + let meta_key = repo_meta_key(user_hash); + batch.insert(repo_data, meta_key.as_slice(), updated_meta.serialize()); + + mutation_set.record_upserts.iter().for_each(|u| { + let key = record_key(user_hash, &u.collection, &u.rkey); + let value = RecordValue { + record_cid: u.cid_bytes.clone(), + takedown_ref: None, + }; + batch.insert(repo_data, key.as_slice(), value.serialize()); + }); + + mutation_set.record_deletes.iter().for_each(|d| { + let key = record_key(user_hash, &d.collection, &d.rkey); + batch.remove(repo_data, key.as_slice()); + }); + + mutation_set.block_inserts.iter().for_each(|cid_bytes| { + let key = user_block_key(user_hash, &mutation_set.new_rev, cid_bytes); + batch.insert(repo_data, key.as_slice(), []); + }); + + delete_user_blocks_by_cid_scan(batch, repo_data, user_hash, &mutation_set.block_deletes)?; + + mutation_set + .backlink_remove_uris + .iter() + .try_for_each(|uri_str| { + let uri = tranquil_types::AtUri::from(uri_str.clone()); + let collection = uri.collection().ok_or(MetastoreError::CorruptData( + "backlink URI missing collection", + ))?; + let rkey = uri + .rkey() + .ok_or(MetastoreError::CorruptData("backlink URI missing rkey"))?; + + remove_backlinks_for_record(batch, indexes, user_hash, collection, rkey) + })?; + + mutation_set.backlink_adds.iter().try_for_each(|bl| { + let uri = tranquil_types::AtUri::from(bl.uri.clone()); + let collection = uri.collection().ok_or(MetastoreError::CorruptData( + "backlink URI missing collection", + ))?; + let rkey = uri + .rkey() + .ok_or(MetastoreError::CorruptData("backlink URI missing rkey"))?; + + match discriminant_to_path(bl.path) { + None => { + tracing::warn!( + path = bl.path, + uri = %bl.uri, + "skipping backlink with unknown path discriminant during recovery" + ); + } + Some(_) => { + let primary = backlink_key(&bl.link_to, user_hash, collection, rkey); + let value = BacklinkValue { + source_uri: bl.uri.clone(), + path: bl.path, + }; + batch.insert(indexes, primary.as_slice(), value.serialize()); + + let reverse = backlink_by_user_key(user_hash, collection, rkey, &bl.link_to); + batch.insert(indexes, reverse.as_slice(), []); + } + } + Ok::<_, MetastoreError>(()) + }) +} + +fn delete_user_blocks_by_cid_scan( + batch: &mut fjall::OwnedWriteBatch, + repo_data: &fjall::Keyspace, + user_hash: UserHash, + block_cids: &[Vec], +) -> Result<(), MetastoreError> { + match block_cids.is_empty() { + true => Ok(()), + false => { + let cid_set: HashSet<&[u8]> = block_cids.iter().map(|c| c.as_slice()).collect(); + let prefix = user_block_user_prefix(user_hash); + repo_data.prefix(prefix.as_slice()).try_for_each(|guard| { + let (key_bytes, _) = guard.into_inner().map_err(MetastoreError::Fjall)?; + match extract_cid_from_user_block_key(&key_bytes) { + Some(cid) if cid_set.contains(cid) => { + batch.remove(repo_data, key_bytes.as_ref()); + Ok(()) + } + _ => Ok(()), + } + }) + } + } +} + +fn extract_cid_from_user_block_key(key_bytes: &[u8]) -> Option<&[u8]> { + let mut reader = KeyReader::new(key_bytes); + let tag = reader.tag()?; + + if tag != KeyTag::USER_BLOCKS.raw() { + tracing::warn!( + tag, + "unexpected key tag in user_block prefix scan during recovery" + ); + return None; + } + + if reader.u64().and_then(|_| reader.string()).is_none() { + tracing::warn!("user_block key has corrupt user_hash or rev during recovery"); + return None; + } + + let remaining = reader.remaining(); + match remaining.is_empty() { + true => { + tracing::warn!("user_block key has no CID suffix during recovery"); + None + } + false => Some(remaining), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn mutation_set_roundtrip() { + let ms = CommitMutationSet { + new_root_cid: vec![0x01, 0x71, 0x12, 0x20], + new_rev: "rev1".to_owned(), + record_upserts: vec![RecordMutationUpsert { + collection: "app.bsky.feed.post".to_owned(), + rkey: "3k2abc".to_owned(), + cid_bytes: vec![0xDE, 0xAD], + }], + record_deletes: vec![RecordMutationDelete { + collection: "app.bsky.feed.like".to_owned(), + rkey: "3k2del".to_owned(), + }], + block_inserts: vec![vec![0x01, 0x02]], + block_deletes: vec![vec![0x03, 0x04]], + backlink_adds: vec![BacklinkMutation { + uri: "at://did:plc:alice/app.bsky.feed.like/3k2abc".to_owned(), + path: 1, + link_to: "at://did:plc:bob/app.bsky.feed.post/3k2xyz".to_owned(), + }], + backlink_remove_uris: vec!["at://did:plc:alice/app.bsky.feed.like/3k2old".to_owned()], + }; + + let bytes = ms.serialize().unwrap(); + assert_eq!(bytes[0], MUTATION_SET_VERSION); + let recovered = CommitMutationSet::deserialize(&bytes).unwrap(); + assert_eq!(recovered, ms); + } + + #[test] + fn mutation_set_empty_roundtrip() { + let ms = CommitMutationSet { + new_root_cid: vec![], + new_rev: String::new(), + record_upserts: vec![], + record_deletes: vec![], + block_inserts: vec![], + block_deletes: vec![], + backlink_adds: vec![], + backlink_remove_uris: vec![], + }; + + let recovered = CommitMutationSet::deserialize(&ms.serialize().unwrap()).unwrap(); + assert_eq!(recovered, ms); + } + + #[test] + fn unknown_version_returns_none() { + let ms = CommitMutationSet { + new_root_cid: vec![], + new_rev: String::new(), + record_upserts: vec![], + record_deletes: vec![], + block_inserts: vec![], + block_deletes: vec![], + backlink_adds: vec![], + backlink_remove_uris: vec![], + }; + let mut bytes = ms.serialize().unwrap(); + bytes[0] = 99; + assert!(CommitMutationSet::deserialize(&bytes).is_none()); + } +} diff --git a/crates/tranquil-store/src/metastore/repo_meta.rs b/crates/tranquil-store/src/metastore/repo_meta.rs new file mode 100644 index 0000000..bf80e52 --- /dev/null +++ b/crates/tranquil-store/src/metastore/repo_meta.rs @@ -0,0 +1,226 @@ +use serde::{Deserialize, Serialize}; +use smallvec::SmallVec; + +use super::encoding::KeyBuilder; +use super::keys::{KeyTag, UserHash}; + +const SCHEMA_VERSION: u8 = 1; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[repr(u8)] +pub enum RepoStatus { + Active = 0, + Takendown = 1, + Suspended = 2, + Deactivated = 3, + Deleted = 4, +} + +impl RepoStatus { + pub fn is_active(self) -> bool { + matches!(self, Self::Active) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RepoMetaValue { + pub repo_root_cid: Vec, + pub repo_rev: String, + pub handle: String, + pub status: RepoStatus, + pub deactivated_at_ms: Option, + pub takedown_ref: Option, + #[serde(default)] + pub did: Option, +} + +impl RepoMetaValue { + pub fn serialize(&self) -> Vec { + let payload = postcard::to_allocvec(self).expect("RepoMetaValue serialization cannot fail"); + let mut buf = Vec::with_capacity(1 + payload.len()); + buf.push(SCHEMA_VERSION); + buf.extend_from_slice(&payload); + buf + } + + pub fn deserialize(bytes: &[u8]) -> Option { + let (&version, payload) = bytes.split_first()?; + match version { + SCHEMA_VERSION => postcard::from_bytes(payload).ok(), + _ => None, + } + } +} + +pub fn repo_meta_key(user_hash: UserHash) -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::REPO_META) + .u64(user_hash.raw()) + .build() +} + +pub fn repo_meta_prefix() -> SmallVec<[u8; 128]> { + KeyBuilder::new().tag(KeyTag::REPO_META).build() +} + +pub fn handle_key(handle_lower: &str) -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::HANDLES) + .string(handle_lower) + .build() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::metastore::encoding::KeyReader; + + #[test] + fn repo_meta_value_roundtrip() { + let value = RepoMetaValue { + repo_root_cid: vec![0x01, 0x71, 0x12, 0x20, 0xAB], + repo_rev: "3k2a7bcd".to_string(), + handle: "alice.bsky.social".to_string(), + status: RepoStatus::Active, + deactivated_at_ms: None, + takedown_ref: None, + did: None, + }; + let bytes = value.serialize(); + let decoded = RepoMetaValue::deserialize(&bytes).unwrap(); + assert_eq!(decoded, value); + } + + #[test] + fn repo_meta_value_with_optional_fields() { + let value = RepoMetaValue { + repo_root_cid: vec![0x01], + repo_rev: "rev1".to_string(), + handle: "bob.example.com".to_string(), + status: RepoStatus::Deactivated, + deactivated_at_ms: Some(1700000000000), + takedown_ref: Some("DMCA-123".to_string()), + did: Some("did:plc:bob".to_string()), + }; + let bytes = value.serialize(); + let decoded = RepoMetaValue::deserialize(&bytes).unwrap(); + assert_eq!(decoded, value); + } + + #[test] + fn repo_meta_key_roundtrip() { + let hash = UserHash::from_raw(0xDEAD_BEEF_CAFE_BABE); + let key = repo_meta_key(hash); + let mut reader = KeyReader::new(&key); + assert_eq!(reader.tag(), Some(KeyTag::REPO_META.raw())); + assert_eq!(reader.u64(), Some(0xDEAD_BEEF_CAFE_BABE)); + assert!(reader.is_empty()); + } + + #[test] + fn repo_meta_keys_sort_by_user_hash() { + let k1 = repo_meta_key(UserHash::from_raw(1)); + let k2 = repo_meta_key(UserHash::from_raw(2)); + let k3 = repo_meta_key(UserHash::from_raw(u64::MAX)); + assert!(k1.as_slice() < k2.as_slice()); + assert!(k2.as_slice() < k3.as_slice()); + } + + #[test] + fn handle_key_roundtrip() { + let key = handle_key("alice.bsky.social"); + let mut reader = KeyReader::new(&key); + assert_eq!(reader.tag(), Some(KeyTag::HANDLES.raw())); + assert_eq!(reader.string(), Some("alice.bsky.social".to_string())); + assert!(reader.is_empty()); + } + + #[test] + fn handle_keys_sort_lexicographically() { + let k1 = handle_key("alice.example.com"); + let k2 = handle_key("bob.example.com"); + assert!(k1.as_slice() < k2.as_slice()); + } + + #[test] + fn all_repo_statuses_roundtrip() { + [ + RepoStatus::Active, + RepoStatus::Takendown, + RepoStatus::Suspended, + RepoStatus::Deactivated, + RepoStatus::Deleted, + ] + .iter() + .for_each(|&status| { + let value = RepoMetaValue { + repo_root_cid: vec![0x01], + repo_rev: "r".to_string(), + handle: "h.test".to_string(), + status, + deactivated_at_ms: None, + takedown_ref: None, + did: None, + }; + let decoded = RepoMetaValue::deserialize(&value.serialize()).unwrap(); + assert_eq!(decoded.status, status); + }); + } + + #[test] + fn repo_status_serialization_stability() { + [ + (RepoStatus::Active, 0u8), + (RepoStatus::Takendown, 1), + (RepoStatus::Suspended, 2), + (RepoStatus::Deactivated, 3), + (RepoStatus::Deleted, 4), + ] + .iter() + .for_each(|&(status, expected_byte)| { + let bytes = postcard::to_allocvec(&status).unwrap(); + assert_eq!( + bytes, + [expected_byte], + "{status:?} serialized to {bytes:?}, expected [{expected_byte}]" + ); + }); + } + + #[test] + fn schema_version_is_first_byte() { + let value = RepoMetaValue { + repo_root_cid: vec![0x01], + repo_rev: "r".to_string(), + handle: "h.test".to_string(), + status: RepoStatus::Active, + deactivated_at_ms: None, + takedown_ref: None, + did: None, + }; + let bytes = value.serialize(); + assert_eq!(bytes[0], SCHEMA_VERSION); + assert_eq!(bytes[0], 1, "schema version must remain 1 for this format"); + } + + #[test] + fn deserialize_rejects_unknown_schema_version() { + let value = RepoMetaValue { + repo_root_cid: vec![0x01], + repo_rev: "r".to_string(), + handle: "h.test".to_string(), + status: RepoStatus::Active, + deactivated_at_ms: None, + takedown_ref: None, + did: None, + }; + let mut bytes = value.serialize(); + bytes[0] = 99; + assert!(RepoMetaValue::deserialize(&bytes).is_none()); + } + + #[test] + fn deserialize_rejects_empty_input() { + assert!(RepoMetaValue::deserialize(&[]).is_none()); + } +} diff --git a/crates/tranquil-store/src/metastore/repo_ops.rs b/crates/tranquil-store/src/metastore/repo_ops.rs new file mode 100644 index 0000000..fd28057 --- /dev/null +++ b/crates/tranquil-store/src/metastore/repo_ops.rs @@ -0,0 +1,1146 @@ +use chrono::{DateTime, TimeZone, Utc}; +use fjall::Keyspace; +use std::sync::Arc; +use uuid::Uuid; + +use super::MetastoreError; +use super::encoding::KeyReader; +use super::keys::{KeyTag, UserHash}; +use super::repo_meta::{RepoMetaValue, RepoStatus, handle_key, repo_meta_key, repo_meta_prefix}; +use super::scan::{count_prefix, point_lookup}; +use super::user_hash::UserHashMap; + +use tranquil_types::{CidLink, Did, Handle}; + +pub struct RepoOps { + repo_data: Keyspace, + user_hashes: Arc, +} + +impl RepoOps { + pub fn new(repo_data: Keyspace, user_hashes: Arc) -> Self { + Self { + repo_data, + user_hashes, + } + } + + pub fn create_repo( + &self, + db: &fjall::Database, + user_id: Uuid, + did: &Did, + handle: &Handle, + repo_root_cid: &CidLink, + repo_rev: &str, + ) -> Result<(), MetastoreError> { + let user_hash = UserHash::from_did(did.as_str()); + let mut batch = db.batch(); + + self.user_hashes + .stage_insert(&mut batch, user_id, user_hash)?; + + let cid_bytes = cid_link_to_bytes(repo_root_cid)?; + let handle_lower = handle.as_str().to_ascii_lowercase(); + + let value = RepoMetaValue { + repo_root_cid: cid_bytes, + repo_rev: repo_rev.to_string(), + handle: handle_lower.clone(), + status: RepoStatus::Active, + deactivated_at_ms: None, + takedown_ref: None, + did: Some(did.as_str().to_string()), + }; + + batch.insert( + &self.repo_data, + repo_meta_key(user_hash).as_slice(), + value.serialize(), + ); + + batch.insert( + &self.repo_data, + handle_key(&handle_lower).as_slice(), + user_hash.raw().to_be_bytes(), + ); + + match batch.commit() { + Ok(()) => Ok(()), + Err(e) => { + self.user_hashes.rollback_insert(&user_id, &user_hash); + Err(MetastoreError::Fjall(e)) + } + } + } + + pub fn get_repo_meta( + &self, + user_id: Uuid, + ) -> Result, MetastoreError> { + let user_hash = match self.user_hashes.get(&user_id) { + Some(h) => h, + None => return Ok(None), + }; + let key = repo_meta_key(user_hash); + Ok(point_lookup( + &self.repo_data, + key.as_slice(), + RepoMetaValue::deserialize, + "invalid repo_meta value", + )? + .map(|v| (user_hash, v))) + } + + pub fn write_repo_meta( + &self, + batch: &mut fjall::OwnedWriteBatch, + user_hash: UserHash, + value: &RepoMetaValue, + ) { + batch.insert( + &self.repo_data, + repo_meta_key(user_hash).as_slice(), + value.serialize(), + ); + } + + pub(crate) fn update_repo_root( + &self, + db: &fjall::Database, + user_id: Uuid, + repo_root_cid: &CidLink, + repo_rev: &str, + ) -> Result<(), MetastoreError> { + let user_hash = self.resolve_user_hash(user_id)?; + let key = repo_meta_key(user_hash); + + let mut value = self.get_meta_value(key.as_slice())?; + let cid_bytes = cid_link_to_bytes(repo_root_cid)?; + value.repo_root_cid = cid_bytes; + value.repo_rev = repo_rev.to_string(); + + let mut batch = db.batch(); + batch.insert(&self.repo_data, key.as_slice(), value.serialize()); + batch.commit().map_err(MetastoreError::Fjall) + } + + pub(crate) fn update_repo_rev( + &self, + db: &fjall::Database, + user_id: Uuid, + repo_rev: &str, + ) -> Result<(), MetastoreError> { + let user_hash = self.resolve_user_hash(user_id)?; + let key = repo_meta_key(user_hash); + + let mut value = self.get_meta_value(key.as_slice())?; + value.repo_rev = repo_rev.to_string(); + + let mut batch = db.batch(); + batch.insert(&self.repo_data, key.as_slice(), value.serialize()); + batch.commit().map_err(MetastoreError::Fjall) + } + + pub fn update_repo_status( + &self, + db: &fjall::Database, + did: &Did, + takedown: Option, + takedown_ref: Option<&str>, + deactivated: Option, + ) -> Result<(), MetastoreError> { + let user_hash = UserHash::from_did(did.as_str()); + let key = repo_meta_key(user_hash); + let existing = point_lookup( + &self.repo_data, + key.as_slice(), + RepoMetaValue::deserialize, + "invalid repo_meta value", + )?; + let mut value = match existing { + Some(v) => v, + None => { + tracing::warn!( + did = did.as_str(), + "update_repo_status: repo not found in metastore" + ); + return Ok(()); + } + }; + + match value.status { + RepoStatus::Suspended | RepoStatus::Deleted => return Ok(()), + _ => {} + } + + if let Some(taken_down) = takedown { + value.takedown_ref = match taken_down { + true => Some(takedown_ref.unwrap_or("").to_owned()), + false => None, + }; + } + if let Some(now_deactivated) = deactivated { + value.deactivated_at_ms = match now_deactivated { + true => value.deactivated_at_ms.or_else(|| { + Some(u64::try_from(chrono::Utc::now().timestamp_millis()).unwrap_or(0)) + }), + false => None, + }; + } + + let is_taken_down = match takedown { + Some(v) => v, + None => value.takedown_ref.is_some(), + }; + value.status = match (is_taken_down, value.deactivated_at_ms.is_some()) { + (true, _) => RepoStatus::Takendown, + (false, true) => RepoStatus::Deactivated, + (false, false) => RepoStatus::Active, + }; + + let mut batch = db.batch(); + batch.insert(&self.repo_data, key.as_slice(), value.serialize()); + batch.commit().map_err(MetastoreError::Fjall) + } + + pub fn update_handle( + &self, + db: &fjall::Database, + user_id: Uuid, + new_handle: &Handle, + ) -> Result<(), MetastoreError> { + let user_hash = self.resolve_user_hash(user_id)?; + let key = repo_meta_key(user_hash); + let mut value = self.get_meta_value(key.as_slice())?; + let new_lower = new_handle.as_str().to_ascii_lowercase(); + + let mut batch = db.batch(); + + match value.handle.is_empty() { + true => {} + false => batch.remove(&self.repo_data, handle_key(&value.handle).as_slice()), + } + + batch.insert( + &self.repo_data, + handle_key(&new_lower).as_slice(), + user_hash.raw().to_be_bytes(), + ); + + value.handle = new_lower; + batch.insert(&self.repo_data, key.as_slice(), value.serialize()); + + batch.commit().map_err(MetastoreError::Fjall) + } + + pub fn delete_repo(&self, db: &fjall::Database, user_id: Uuid) -> Result<(), MetastoreError> { + let user_hash = self.resolve_user_hash(user_id)?; + let key = repo_meta_key(user_hash); + + let meta = self.get_meta_value(key.as_slice())?; + + let mut batch = db.batch(); + batch.remove(&self.repo_data, key.as_slice()); + + match meta.handle.is_empty() { + true => {} + false => batch.remove(&self.repo_data, handle_key(&meta.handle).as_slice()), + } + + self.user_hashes.stage_remove(&mut batch, &user_id); + + match batch.commit() { + Ok(()) => Ok(()), + Err(e) => { + self.user_hashes.rollback_remove(user_id, user_hash); + Err(MetastoreError::Fjall(e)) + } + } + } + + pub fn get_repo(&self, user_id: Uuid) -> Result, MetastoreError> { + let user_hash = match self.user_hashes.get(&user_id) { + Some(h) => h, + None => return Ok(None), + }; + let key = repo_meta_key(user_hash); + point_lookup( + &self.repo_data, + key.as_slice(), + RepoMetaValue::deserialize, + "invalid repo_meta value", + )? + .map(|value| { + let cid = bytes_to_cid_link(&value.repo_root_cid)?; + Ok(RepoInfo { + user_id, + repo_root_cid: cid, + repo_rev: Some(value.repo_rev), + }) + }) + .transpose() + } + + pub fn get_repo_root_for_update( + &self, + user_id: Uuid, + ) -> Result, MetastoreError> { + let user_hash = match self.user_hashes.get(&user_id) { + Some(h) => h, + None => return Ok(None), + }; + let key = repo_meta_key(user_hash); + point_lookup( + &self.repo_data, + key.as_slice(), + RepoMetaValue::deserialize, + "invalid repo_meta value", + )? + .map(|v| bytes_to_cid_link(&v.repo_root_cid)) + .transpose() + } + + pub fn get_repo_root_by_did(&self, did: &Did) -> Result, MetastoreError> { + let user_hash = UserHash::from_did(did.as_str()); + let key = repo_meta_key(user_hash); + point_lookup( + &self.repo_data, + key.as_slice(), + RepoMetaValue::deserialize, + "invalid repo_meta value", + )? + .map(|v| bytes_to_cid_link(&v.repo_root_cid)) + .transpose() + } + + pub fn get_repo_root_cid_by_user_id( + &self, + user_id: Uuid, + ) -> Result, MetastoreError> { + self.get_repo_root_for_update(user_id) + } + + pub fn count_repos(&self) -> Result { + let prefix = repo_meta_prefix(); + count_prefix(&self.repo_data, prefix.as_slice()) + } + + pub fn get_repos_without_rev( + &self, + limit: usize, + ) -> Result, MetastoreError> { + let prefix = repo_meta_prefix(); + self.repo_data + .prefix(prefix.as_slice()) + .filter_map(|guard| { + let (key_bytes, val_bytes) = match guard.into_inner() { + Ok(pair) => pair, + Err(e) => return Some(Err(MetastoreError::Fjall(e))), + }; + let value = match RepoMetaValue::deserialize(&val_bytes) { + Some(v) => v, + None => { + return Some(Err(MetastoreError::CorruptData("invalid repo_meta value"))); + } + }; + match value.repo_rev.is_empty() { + true => Some(decode_without_rev_entry( + &key_bytes, + &value, + &self.user_hashes, + )), + false => None, + } + }) + .take(limit) + .collect() + } + + pub fn get_account_with_repo( + &self, + did: &Did, + ) -> Result, MetastoreError> { + let user_hash = UserHash::from_did(did.as_str()); + let key = repo_meta_key(user_hash); + point_lookup( + &self.repo_data, + key.as_slice(), + RepoMetaValue::deserialize, + "invalid repo_meta value", + )? + .map(|value| { + let user_id = + self.user_hashes + .get_uuid(&user_hash) + .ok_or(MetastoreError::CorruptData( + "user_hash has no reverse mapping", + ))?; + let cid = Some(bytes_to_cid_link(&value.repo_root_cid)?); + let deactivated_at = value + .deactivated_at_ms + .and_then(|ms| i64::try_from(ms).ok()) + .and_then(|ms| Utc.timestamp_millis_opt(ms).single()); + Ok(RepoAccountEntry { + user_id, + did: did.clone(), + deactivated_at, + takedown_ref: value.takedown_ref, + repo_root_cid: cid, + }) + }) + .transpose() + } + + pub fn list_repos_paginated( + &self, + cursor_user_hash: Option, + limit: usize, + ) -> Result, MetastoreError> { + const _: () = assert!(KeyTag::REPO_META.raw() < 0xFF); + let upper = KeyTag::REPO_META.exclusive_prefix_bound(); + + let start = match cursor_user_hash { + Some(cursor) => match cursor.checked_add(1) { + Some(next) => repo_meta_key(UserHash::from_raw(next)), + None => return Ok(Vec::new()), + }, + None => repo_meta_prefix(), + }; + + self.repo_data + .range(start.as_slice()..upper.as_slice()) + .take(limit) + .map(|guard| { + let (k, v) = guard.into_inner().map_err(MetastoreError::Fjall)?; + decode_list_entry_from_kv(&k, &v, &self.user_hashes) + }) + .collect() + } + + fn resolve_user_hash(&self, user_id: Uuid) -> Result { + self.user_hashes + .get(&user_id) + .ok_or(MetastoreError::InvalidInput("unknown user_id")) + } + + fn get_meta_value(&self, key: &[u8]) -> Result { + point_lookup( + &self.repo_data, + key, + RepoMetaValue::deserialize, + "invalid repo_meta value", + )? + .ok_or(MetastoreError::CorruptData("repo_meta not found")) + } + + pub fn lookup_handle(&self, handle: &Handle) -> Result, MetastoreError> { + let handle_lower = handle.as_str().to_ascii_lowercase(); + let key = handle_key(&handle_lower); + + match self + .repo_data + .get(key.as_slice()) + .map_err(MetastoreError::Fjall)? + { + Some(raw) => { + let user_hash = parse_handle_value(&raw)?; + self.user_hashes + .get_uuid(&user_hash) + .ok_or(MetastoreError::CorruptData( + "handle maps to unknown user_hash", + )) + .map(Some) + } + None => Ok(None), + } + } +} + +#[derive(Debug, Clone)] +pub struct RepoInfo { + pub user_id: Uuid, + pub repo_root_cid: CidLink, + pub repo_rev: Option, +} + +#[derive(Debug, Clone)] +pub struct RepoWithoutRevEntry { + pub user_id: Uuid, + pub repo_root_cid: CidLink, +} + +#[derive(Debug, Clone)] +pub struct RepoAccountEntry { + pub user_id: Uuid, + pub did: Did, + pub deactivated_at: Option>, + pub takedown_ref: Option, + pub repo_root_cid: Option, +} + +#[derive(Debug, Clone)] +pub struct RepoListEntry { + pub user_id: Uuid, + pub user_hash: UserHash, + pub did: Option, + pub deactivated_at: Option>, + pub takedown_ref: Option, + pub repo_root_cid: CidLink, + pub repo_rev: Option, +} + +fn decode_list_entry_from_kv( + key_bytes: &[u8], + val_bytes: &[u8], + user_hashes: &UserHashMap, +) -> Result { + let value = RepoMetaValue::deserialize(val_bytes) + .ok_or(MetastoreError::CorruptData("invalid repo_meta value"))?; + let user_hash = parse_repo_meta_key_hash(key_bytes) + .ok_or(MetastoreError::CorruptData("invalid repo_meta key"))?; + let user_id = user_hashes + .get_uuid(&user_hash) + .ok_or(MetastoreError::CorruptData( + "user_hash has no reverse mapping", + ))?; + let cid = bytes_to_cid_link(&value.repo_root_cid)?; + let deactivated_at = value + .deactivated_at_ms + .and_then(|ms| i64::try_from(ms).ok()) + .and_then(|ms| Utc.timestamp_millis_opt(ms).single()); + Ok(RepoListEntry { + user_id, + user_hash, + did: value.did, + deactivated_at, + takedown_ref: value.takedown_ref, + repo_root_cid: cid, + repo_rev: match value.repo_rev.is_empty() { + true => None, + false => Some(value.repo_rev), + }, + }) +} + +fn decode_without_rev_entry( + key_bytes: &[u8], + value: &RepoMetaValue, + user_hashes: &UserHashMap, +) -> Result { + let user_hash = parse_repo_meta_key_hash(key_bytes) + .ok_or(MetastoreError::CorruptData("invalid repo_meta key"))?; + let user_id = user_hashes + .get_uuid(&user_hash) + .ok_or(MetastoreError::CorruptData( + "user_hash has no reverse mapping", + ))?; + let cid = bytes_to_cid_link(&value.repo_root_cid)?; + Ok(RepoWithoutRevEntry { + user_id, + repo_root_cid: cid, + }) +} + +pub(crate) fn cid_link_to_bytes(cid_link: &CidLink) -> Result, MetastoreError> { + let cid = cid_link.to_cid().ok_or(MetastoreError::InvalidInput( + "CidLink does not contain a valid CID", + ))?; + Ok(cid.to_bytes()) +} + +pub(crate) fn bytes_to_cid_link(bytes: &[u8]) -> Result { + let cid = cid::Cid::read_bytes(std::io::Cursor::new(bytes)) + .map_err(|_| MetastoreError::CorruptData("invalid CID bytes in repo_meta"))?; + Ok(CidLink::from_cid(&cid)) +} + +fn parse_handle_value(raw: &[u8]) -> Result { + let bytes: [u8; 8] = raw + .try_into() + .map_err(|_| MetastoreError::CorruptData("handle value not 8 bytes"))?; + Ok(UserHash::from_raw(u64::from_be_bytes(bytes))) +} + +fn parse_repo_meta_key_hash(key_bytes: &[u8]) -> Option { + let mut reader = KeyReader::new(key_bytes); + let _tag = reader.tag()?; + let hash = reader.u64()?; + Some(UserHash::from_raw(hash)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::metastore::{Metastore, MetastoreConfig}; + + fn test_config() -> MetastoreConfig { + MetastoreConfig { + cache_size_bytes: 64 * 1024 * 1024, + } + } + + fn test_cid_link(seed: u8) -> CidLink { + let digest: [u8; 32] = std::array::from_fn(|i| seed.wrapping_add(i as u8)); + let mh = multihash::Multihash::<64>::wrap(0x12, &digest).unwrap(); + let c = cid::Cid::new_v1(0x71, mh); + CidLink::from_cid(&c) + } + + fn open_fresh() -> (tempfile::TempDir, Metastore) { + let dir = tempfile::TempDir::new().unwrap(); + let ms = Metastore::open(dir.path(), test_config()).unwrap(); + (dir, ms) + } + + fn test_did(name: &str) -> Did { + Did::from(format!("did:plc:{name}")) + } + + fn test_handle(name: &str) -> Handle { + Handle::from(format!("{name}.test.invalid")) + } + + #[test] + fn create_and_get_repo() { + let (_dir, ms) = open_fresh(); + let ops = ms.repo_ops(); + let user_id = uuid::Uuid::new_v4(); + let did = test_did("alice"); + let handle = test_handle("alice"); + let cid = test_cid_link(1); + + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "rev1") + .unwrap(); + + let repo = ops.get_repo(user_id).unwrap().unwrap(); + assert_eq!(repo.user_id, user_id); + assert_eq!(repo.repo_root_cid, cid); + assert_eq!(repo.repo_rev.as_deref(), Some("rev1")); + } + + #[test] + fn get_repo_returns_none_for_unknown() { + let (_dir, ms) = open_fresh(); + let ops = ms.repo_ops(); + assert!(ops.get_repo(uuid::Uuid::new_v4()).unwrap().is_none()); + } + + #[test] + fn update_repo_root() { + let (_dir, ms) = open_fresh(); + let ops = ms.repo_ops(); + let user_id = uuid::Uuid::new_v4(); + let did = test_did("bob"); + let handle = test_handle("bob"); + let cid1 = test_cid_link(1); + let cid2 = test_cid_link(2); + + ops.create_repo(ms.database(), user_id, &did, &handle, &cid1, "rev1") + .unwrap(); + ops.update_repo_root(ms.database(), user_id, &cid2, "rev2") + .unwrap(); + + let repo = ops.get_repo(user_id).unwrap().unwrap(); + assert_eq!(repo.repo_root_cid, cid2); + assert_eq!(repo.repo_rev.as_deref(), Some("rev2")); + } + + #[test] + fn update_repo_rev() { + let (_dir, ms) = open_fresh(); + let ops = ms.repo_ops(); + let user_id = uuid::Uuid::new_v4(); + let did = test_did("carol"); + let handle = test_handle("carol"); + let cid = test_cid_link(3); + + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "rev1") + .unwrap(); + ops.update_repo_rev(ms.database(), user_id, "rev_updated") + .unwrap(); + + let repo = ops.get_repo(user_id).unwrap().unwrap(); + assert_eq!(repo.repo_root_cid, cid); + assert_eq!(repo.repo_rev.as_deref(), Some("rev_updated")); + } + + #[test] + fn delete_repo_removes_meta_and_handle() { + let (_dir, ms) = open_fresh(); + let ops = ms.repo_ops(); + let user_id = uuid::Uuid::new_v4(); + let did = test_did("dave"); + let handle = test_handle("dave"); + let cid = test_cid_link(4); + + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "rev1") + .unwrap(); + assert!(ops.get_repo(user_id).unwrap().is_some()); + assert!(ops.lookup_handle(&handle).unwrap().is_some()); + + ops.delete_repo(ms.database(), user_id).unwrap(); + assert!(ops.get_repo(user_id).unwrap().is_none()); + assert!(ops.lookup_handle(&handle).unwrap().is_none()); + } + + #[test] + fn delete_repo_clears_user_hash_mapping() { + let (_dir, ms) = open_fresh(); + let ops = ms.repo_ops(); + let user_id = uuid::Uuid::new_v4(); + let did = test_did("mapped"); + let handle = test_handle("mapped"); + let cid = test_cid_link(4); + + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "rev1") + .unwrap(); + assert!(ms.user_hashes().get(&user_id).is_some()); + + ops.delete_repo(ms.database(), user_id).unwrap(); + assert!(ms.user_hashes().get(&user_id).is_none()); + } + + #[test] + fn delete_repo_allows_recreate_with_same_did() { + let (_dir, ms) = open_fresh(); + let ops = ms.repo_ops(); + let did = test_did("recreate"); + let handle_a = test_handle("recreate_a"); + let handle_b = test_handle("recreate_b"); + let uid_a = uuid::Uuid::new_v4(); + let uid_b = uuid::Uuid::new_v4(); + let cid = test_cid_link(70); + + ops.create_repo(ms.database(), uid_a, &did, &handle_a, &cid, "r1") + .unwrap(); + ops.delete_repo(ms.database(), uid_a).unwrap(); + + ops.create_repo(ms.database(), uid_b, &did, &handle_b, &cid, "r2") + .unwrap(); + + let repo = ops.get_repo(uid_b).unwrap().unwrap(); + assert_eq!(repo.user_id, uid_b); + assert_eq!(repo.repo_rev.as_deref(), Some("r2")); + assert!(ops.get_repo(uid_a).unwrap().is_none()); + } + + #[test] + fn delete_nonexistent_user_returns_error() { + let (_dir, ms) = open_fresh(); + let ops = ms.repo_ops(); + let result = ops.delete_repo(ms.database(), uuid::Uuid::new_v4()); + assert!(matches!(result, Err(MetastoreError::InvalidInput(_)))); + } + + #[test] + fn handle_lookup_case_insensitive() { + let (_dir, ms) = open_fresh(); + let ops = ms.repo_ops(); + let user_id = uuid::Uuid::new_v4(); + let did = test_did("eve"); + let handle = test_handle("eve"); + let cid = test_cid_link(5); + + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "rev1") + .unwrap(); + + let upper_handle = Handle::from("EVE.TEST.INVALID".to_string()); + let found = ops.lookup_handle(&upper_handle).unwrap(); + assert_eq!(found, Some(user_id)); + } + + #[test] + fn get_repo_root_by_did() { + let (_dir, ms) = open_fresh(); + let ops = ms.repo_ops(); + let user_id = uuid::Uuid::new_v4(); + let did = test_did("frank"); + let handle = test_handle("frank"); + let cid = test_cid_link(6); + + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "rev1") + .unwrap(); + + let root = ops.get_repo_root_by_did(&did).unwrap().unwrap(); + assert_eq!(root, cid); + + let unknown = test_did("nobody"); + assert!(ops.get_repo_root_by_did(&unknown).unwrap().is_none()); + } + + #[test] + fn get_repo_root_for_update() { + let (_dir, ms) = open_fresh(); + let ops = ms.repo_ops(); + let user_id = uuid::Uuid::new_v4(); + let did = test_did("grace"); + let handle = test_handle("grace"); + let cid = test_cid_link(7); + + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "rev1") + .unwrap(); + + let root = ops.get_repo_root_for_update(user_id).unwrap().unwrap(); + assert_eq!(root, cid); + } + + #[test] + fn count_repos() { + let (_dir, ms) = open_fresh(); + let ops = ms.repo_ops(); + + assert_eq!(ops.count_repos().unwrap(), 0); + + (0..5u8).for_each(|i| { + let user_id = uuid::Uuid::new_v4(); + let did = test_did(&format!("user{i}")); + let handle = test_handle(&format!("user{i}")); + let cid = test_cid_link(i); + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "rev1") + .unwrap(); + }); + + assert_eq!(ops.count_repos().unwrap(), 5); + } + + #[test] + fn get_account_with_repo() { + let (_dir, ms) = open_fresh(); + let ops = ms.repo_ops(); + let user_id = uuid::Uuid::new_v4(); + let did = test_did("henry"); + let handle = test_handle("henry"); + let cid = test_cid_link(8); + + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "rev1") + .unwrap(); + + let account = ops.get_account_with_repo(&did).unwrap().unwrap(); + assert_eq!(account.user_id, user_id); + assert_eq!(account.did, did); + assert_eq!(account.repo_root_cid, Some(cid)); + assert!(account.deactivated_at.is_none()); + assert!(account.takedown_ref.is_none()); + } + + #[test] + fn list_repos_paginated_all() { + let (_dir, ms) = open_fresh(); + let ops = ms.repo_ops(); + + (0..5u8).for_each(|i| { + let user_id = uuid::Uuid::new_v4(); + let did = test_did(&format!("page{i}")); + let handle = test_handle(&format!("page{i}")); + let cid = test_cid_link(10 + i); + ops.create_repo( + ms.database(), + user_id, + &did, + &handle, + &cid, + &format!("rev{i}"), + ) + .unwrap(); + }); + + let all = ops.list_repos_paginated(None, 100).unwrap(); + assert_eq!(all.len(), 5); + + all.iter() + .zip(all.iter().skip(1)) + .for_each(|(a, b)| assert!(a.user_hash.raw() < b.user_hash.raw())); + } + + #[test] + fn list_repos_paginated_with_cursor() { + let (_dir, ms) = open_fresh(); + let ops = ms.repo_ops(); + + (0..10u8).for_each(|i| { + let user_id = uuid::Uuid::new_v4(); + let did = test_did(&format!("cursor{i}")); + let handle = test_handle(&format!("cursor{i}")); + let cid = test_cid_link(20 + i); + ops.create_repo( + ms.database(), + user_id, + &did, + &handle, + &cid, + &format!("rev{i}"), + ) + .unwrap(); + }); + + let page1 = ops.list_repos_paginated(None, 3).unwrap(); + assert_eq!(page1.len(), 3); + + let cursor = page1.last().unwrap().user_hash.raw(); + let page2 = ops.list_repos_paginated(Some(cursor), 3).unwrap(); + assert_eq!(page2.len(), 3); + + assert!(page2.first().unwrap().user_hash.raw() > cursor); + + let page2_cursor = page2.last().unwrap().user_hash.raw(); + let page3 = ops.list_repos_paginated(Some(page2_cursor), 100).unwrap(); + assert_eq!(page3.len(), 4); + + let total = page1.len() + page2.len() + page3.len(); + assert_eq!(total, 10); + } + + #[test] + fn data_survives_reopen() { + let dir = tempfile::TempDir::new().unwrap(); + let user_id = uuid::Uuid::new_v4(); + let did = test_did("persist"); + let handle = test_handle("persist"); + let cid = test_cid_link(99); + + { + let ms = Metastore::open(dir.path(), test_config()).unwrap(); + let ops = ms.repo_ops(); + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "rev_persist") + .unwrap(); + ms.persist().unwrap(); + } + + { + let ms = Metastore::open(dir.path(), test_config()).unwrap(); + let ops = ms.repo_ops(); + let repo = ops.get_repo(user_id).unwrap().unwrap(); + assert_eq!(repo.repo_root_cid, cid); + assert_eq!(repo.repo_rev.as_deref(), Some("rev_persist")); + + let found = ops.lookup_handle(&handle).unwrap(); + assert_eq!(found, Some(user_id)); + } + } + + #[test] + fn get_repos_without_rev() { + let (_dir, ms) = open_fresh(); + let ops = ms.repo_ops(); + + let uid_with = uuid::Uuid::new_v4(); + let did_with = test_did("with_rev"); + let handle_with = test_handle("with_rev"); + ops.create_repo( + ms.database(), + uid_with, + &did_with, + &handle_with, + &test_cid_link(40), + "some_rev", + ) + .unwrap(); + + let uid_without = uuid::Uuid::new_v4(); + let did_without = test_did("without_rev"); + let handle_without = test_handle("without_rev"); + ops.create_repo( + ms.database(), + uid_without, + &did_without, + &handle_without, + &test_cid_link(41), + "", + ) + .unwrap(); + + let result = ops.get_repos_without_rev(100).unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].user_id, uid_without); + } + + #[test] + fn get_repo_root_cid_by_user_id() { + let (_dir, ms) = open_fresh(); + let ops = ms.repo_ops(); + let user_id = uuid::Uuid::new_v4(); + let did = test_did("root_cid"); + let handle = test_handle("root_cid"); + let cid = test_cid_link(50); + + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "rev1") + .unwrap(); + + let root = ops.get_repo_root_cid_by_user_id(user_id).unwrap().unwrap(); + assert_eq!(root, cid); + + assert!( + ops.get_repo_root_cid_by_user_id(uuid::Uuid::new_v4()) + .unwrap() + .is_none() + ); + } + + #[test] + fn delete_only_removes_target_handle() { + let (_dir, ms) = open_fresh(); + let ops = ms.repo_ops(); + + let uid_a = uuid::Uuid::new_v4(); + let did_a = test_did("keep_a"); + let handle_a = test_handle("keep_a"); + ops.create_repo( + ms.database(), + uid_a, + &did_a, + &handle_a, + &test_cid_link(60), + "r", + ) + .unwrap(); + + let uid_b = uuid::Uuid::new_v4(); + let did_b = test_did("delete_b"); + let handle_b = test_handle("delete_b"); + ops.create_repo( + ms.database(), + uid_b, + &did_b, + &handle_b, + &test_cid_link(61), + "r", + ) + .unwrap(); + + ops.delete_repo(ms.database(), uid_b).unwrap(); + + assert!(ops.lookup_handle(&handle_a).unwrap().is_some()); + assert!(ops.lookup_handle(&handle_b).unwrap().is_none()); + assert!(ops.get_repo(uid_a).unwrap().is_some()); + } + + #[test] + fn create_repo_rejects_hash_collision() { + let (_dir, ms) = open_fresh(); + let ops = ms.repo_ops(); + + let uid_a = uuid::Uuid::new_v4(); + let did_a = test_did("collision_a"); + let handle_a = test_handle("collision_a"); + let cid = test_cid_link(80); + + ops.create_repo(ms.database(), uid_a, &did_a, &handle_a, &cid, "r1") + .unwrap(); + + let uid_b = uuid::Uuid::new_v4(); + let handle_b = test_handle("collision_b"); + + let result = ops.create_repo(ms.database(), uid_b, &did_a, &handle_b, &cid, "r2"); + + match result { + Ok(()) => { + let repo = ops.get_repo(uid_a).unwrap().unwrap(); + assert_eq!(repo.repo_root_cid, cid); + } + Err(MetastoreError::UserHashCollision { .. }) => { + let repo = ops.get_repo(uid_a).unwrap().unwrap(); + assert_eq!(repo.repo_root_cid, cid); + assert_eq!(repo.repo_rev.as_deref(), Some("r1")); + } + Err(e) => panic!("unexpected error: {e}"), + } + } + + #[test] + fn get_repo_meta_returns_raw_value() { + let (_dir, ms) = open_fresh(); + let ops = ms.repo_ops(); + let user_id = uuid::Uuid::new_v4(); + let did = test_did("meta_raw"); + let handle = test_handle("meta_raw"); + let cid = test_cid_link(81); + + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "rev_meta") + .unwrap(); + + let (user_hash, value) = ops.get_repo_meta(user_id).unwrap().unwrap(); + assert_eq!(user_hash, UserHash::from_did(did.as_str())); + assert_eq!(value.repo_rev, "rev_meta"); + assert_eq!(value.handle, "meta_raw.test.invalid"); + assert_eq!(value.status, RepoStatus::Active); + } + + #[test] + fn get_repo_meta_returns_none_for_unknown() { + let (_dir, ms) = open_fresh(); + let ops = ms.repo_ops(); + assert!(ops.get_repo_meta(uuid::Uuid::new_v4()).unwrap().is_none()); + } + + #[test] + fn write_repo_meta_via_batch() { + let (_dir, ms) = open_fresh(); + let ops = ms.repo_ops(); + let user_id = uuid::Uuid::new_v4(); + let did = test_did("batch_write"); + let handle = test_handle("batch_write"); + let cid1 = test_cid_link(82); + let cid2 = test_cid_link(83); + + ops.create_repo(ms.database(), user_id, &did, &handle, &cid1, "rev1") + .unwrap(); + + let (user_hash, mut value) = ops.get_repo_meta(user_id).unwrap().unwrap(); + value.repo_root_cid = cid_link_to_bytes(&cid2).unwrap(); + value.repo_rev = "rev2".to_string(); + + let mut batch = ms.database().batch(); + ops.write_repo_meta(&mut batch, user_hash, &value); + batch.commit().unwrap(); + + let repo = ops.get_repo(user_id).unwrap().unwrap(); + assert_eq!(repo.repo_root_cid, cid2); + assert_eq!(repo.repo_rev.as_deref(), Some("rev2")); + } + + #[test] + fn update_handle_swaps_lookup() { + let (_dir, ms) = open_fresh(); + let ops = ms.repo_ops(); + let user_id = uuid::Uuid::new_v4(); + let did = test_did("handle_swap"); + let old_handle = test_handle("old_name"); + let new_handle = test_handle("new_name"); + let cid = test_cid_link(84); + + ops.create_repo(ms.database(), user_id, &did, &old_handle, &cid, "r1") + .unwrap(); + assert!(ops.lookup_handle(&old_handle).unwrap().is_some()); + + ops.update_handle(ms.database(), user_id, &new_handle) + .unwrap(); + + assert!(ops.lookup_handle(&old_handle).unwrap().is_none()); + assert_eq!(ops.lookup_handle(&new_handle).unwrap(), Some(user_id)); + + let (_, meta) = ops.get_repo_meta(user_id).unwrap().unwrap(); + assert_eq!(meta.handle, "new_name.test.invalid"); + } + + #[test] + fn update_handle_case_insensitive() { + let (_dir, ms) = open_fresh(); + let ops = ms.repo_ops(); + let user_id = uuid::Uuid::new_v4(); + let did = test_did("handle_case"); + let handle = test_handle("original"); + let cid = test_cid_link(85); + + ops.create_repo(ms.database(), user_id, &did, &handle, &cid, "r1") + .unwrap(); + + let mixed_case = Handle::from("UPPER.TEST.INVALID".to_string()); + ops.update_handle(ms.database(), user_id, &mixed_case) + .unwrap(); + + let lower_lookup = Handle::from("upper.test.invalid".to_string()); + assert_eq!(ops.lookup_handle(&lower_lookup).unwrap(), Some(user_id)); + assert!(ops.lookup_handle(&handle).unwrap().is_none()); + } +} diff --git a/crates/tranquil-store/src/metastore/scan.rs b/crates/tranquil-store/src/metastore/scan.rs new file mode 100644 index 0000000..8f4a217 --- /dev/null +++ b/crates/tranquil-store/src/metastore/scan.rs @@ -0,0 +1,36 @@ +use fjall::Keyspace; + +use super::MetastoreError; + +pub fn count_prefix(keyspace: &Keyspace, prefix: &[u8]) -> Result { + keyspace.prefix(prefix).try_fold(0i64, |acc, guard| { + guard.into_inner().map_err(MetastoreError::Fjall)?; + Ok::<_, MetastoreError>(acc.saturating_add(1)) + }) +} + +pub fn delete_all_by_prefix( + keyspace: &Keyspace, + batch: &mut fjall::OwnedWriteBatch, + prefix: &[u8], +) -> Result<(), MetastoreError> { + keyspace.prefix(prefix).try_for_each(|guard| { + let (key_bytes, _) = guard.into_inner().map_err(MetastoreError::Fjall)?; + batch.remove(keyspace, key_bytes.as_ref()); + Ok::<(), MetastoreError>(()) + }) +} + +pub fn point_lookup( + keyspace: &Keyspace, + key: &[u8], + deserialize: impl FnOnce(&[u8]) -> Option, + corrupt_msg: &'static str, +) -> Result, MetastoreError> { + match keyspace.get(key).map_err(MetastoreError::Fjall)? { + Some(raw) => deserialize(&raw) + .ok_or(MetastoreError::CorruptData(corrupt_msg)) + .map(Some), + None => Ok(None), + } +} diff --git a/crates/tranquil-store/src/metastore/user_block_ops.rs b/crates/tranquil-store/src/metastore/user_block_ops.rs new file mode 100644 index 0000000..f148c5f --- /dev/null +++ b/crates/tranquil-store/src/metastore/user_block_ops.rs @@ -0,0 +1,560 @@ +use std::collections::HashSet; +use std::sync::Arc; + +use fjall::Keyspace; +use uuid::Uuid; + +use super::MetastoreError; +use super::encoding::{KeyReader, exclusive_upper_bound}; +use super::keys::UserHash; +use super::scan::{count_prefix, delete_all_by_prefix}; +use super::user_blocks::{user_block_key, user_block_rev_prefix, user_block_user_prefix}; +use super::user_hash::UserHashMap; + +pub struct UserBlockOps { + repo_data: Keyspace, + user_hashes: Arc, +} + +impl UserBlockOps { + pub fn new(repo_data: Keyspace, user_hashes: Arc) -> Self { + Self { + repo_data, + user_hashes, + } + } + + pub fn insert_user_blocks>( + &self, + batch: &mut fjall::OwnedWriteBatch, + user_hash: UserHash, + block_cids: &[C], + repo_rev: &str, + ) -> Result<(), MetastoreError> { + let existing: HashSet> = match block_cids.is_empty() { + true => HashSet::new(), + false => { + let prefix = user_block_user_prefix(user_hash); + self.repo_data + .prefix(prefix.as_slice()) + .filter_map(|guard| { + let (key_bytes, _) = guard.into_inner().ok()?; + extract_cid_from_key(&key_bytes).map(|c| c.to_vec()) + }) + .collect() + } + }; + + block_cids.iter().try_for_each(|cid| { + let cid = cid.as_ref(); + match cid.is_empty() { + true => Err(MetastoreError::InvalidInput("block CID must not be empty")), + false => { + if !existing.contains(cid) { + let key = user_block_key(user_hash, repo_rev, cid); + batch.insert(&self.repo_data, key.as_slice(), []); + } + Ok(()) + } + } + }) + } + + pub fn delete_user_blocks>( + &self, + batch: &mut fjall::OwnedWriteBatch, + user_hash: UserHash, + block_cids: &[C], + rev: &str, + ) -> Result<(), MetastoreError> { + block_cids.iter().try_for_each(|cid| { + let cid = cid.as_ref(); + match cid.is_empty() { + true => Err(MetastoreError::InvalidInput("block CID must not be empty")), + false => { + let key = user_block_key(user_hash, rev, cid); + batch.remove(&self.repo_data, key.as_slice()); + Ok(()) + } + } + }) + } + + pub fn delete_user_blocks_by_cid>( + &self, + batch: &mut fjall::OwnedWriteBatch, + user_hash: UserHash, + block_cids: &[C], + ) -> Result<(), MetastoreError> { + match block_cids.is_empty() { + true => Ok(()), + false => { + let cid_set: HashSet<&[u8]> = block_cids.iter().map(|c| c.as_ref()).collect(); + let prefix = user_block_user_prefix(user_hash); + self.repo_data + .prefix(prefix.as_slice()) + .try_for_each(|guard| { + let (key_bytes, _) = guard.into_inner().map_err(MetastoreError::Fjall)?; + match extract_cid_from_key(&key_bytes) { + Some(cid) if cid_set.contains(cid.as_slice()) => { + batch.remove(&self.repo_data, key_bytes.as_ref()); + Ok(()) + } + _ => Ok(()), + } + }) + } + } + } + + pub fn delete_all_user_blocks( + &self, + batch: &mut fjall::OwnedWriteBatch, + user_hash: UserHash, + ) -> Result<(), MetastoreError> { + let prefix = user_block_user_prefix(user_hash); + delete_all_by_prefix(&self.repo_data, batch, prefix.as_slice()) + } + + pub fn get_user_block_cids_since_rev( + &self, + user_id: Uuid, + since_rev: &str, + ) -> Result>, MetastoreError> { + let user_hash = match self.user_hashes.get(&user_id) { + Some(h) => h, + None => return Ok(Vec::new()), + }; + + let since_prefix = user_block_rev_prefix(user_hash, since_rev); + let since_upper = exclusive_upper_bound(since_prefix.as_slice()) + .expect("user block rev prefix always contains non-0xFF bytes"); + + let user_prefix = user_block_user_prefix(user_hash); + let user_upper = exclusive_upper_bound(user_prefix.as_slice()) + .expect("user block user prefix always contains non-0xFF bytes"); + + self.repo_data + .range(since_upper.as_slice()..user_upper.as_slice()) + .map(|guard| { + let (key_bytes, _) = guard.into_inner().map_err(MetastoreError::Fjall)?; + extract_cid_from_key(&key_bytes) + .ok_or(MetastoreError::CorruptData("invalid user_blocks key")) + }) + .collect() + } + + pub fn find_unreferenced(&self, candidate_cids: &[Vec]) -> Vec> { + match candidate_cids.is_empty() { + true => Vec::new(), + false => { + let mut remaining: HashSet> = candidate_cids.iter().cloned().collect(); + let tag_prefix = super::keys::KeyTag::USER_BLOCKS.raw(); + let mut iter = self.repo_data.prefix([tag_prefix]); + loop { + match remaining.is_empty() { + true => break, + false => match iter.next() { + None => break, + Some(guard) => { + if let Ok((key_bytes, _)) = guard.into_inner() + && let Some(cid) = extract_cid_from_key(&key_bytes) + { + remaining.remove(&cid); + } + } + }, + } + } + remaining.into_iter().collect() + } + } + } + + pub fn count_user_blocks(&self, user_id: Uuid) -> Result { + let user_hash = match self.user_hashes.get(&user_id) { + Some(h) => h, + None => return Ok(0), + }; + let prefix = user_block_user_prefix(user_hash); + count_prefix(&self.repo_data, prefix.as_slice()) + } +} + +fn extract_cid_from_key(key_bytes: &[u8]) -> Option> { + let mut reader = KeyReader::new(key_bytes); + reader.tag()?; + reader.u64()?; + reader.string()?; + match reader.remaining().is_empty() { + true => None, + false => Some(reader.remaining().to_vec()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::metastore::{Metastore, MetastoreConfig}; + + fn open_fresh() -> (tempfile::TempDir, Metastore) { + let dir = tempfile::TempDir::new().unwrap(); + let ms = Metastore::open( + dir.path(), + MetastoreConfig { + cache_size_bytes: 64 * 1024 * 1024, + }, + ) + .unwrap(); + (dir, ms) + } + + fn setup_user(ms: &Metastore) -> (Uuid, UserHash) { + let uuid = Uuid::new_v4(); + let hash = UserHash::from_did("did:plc:testuser1"); + let mut batch = ms.database().batch(); + ms.user_hashes() + .stage_insert(&mut batch, uuid, hash) + .unwrap(); + batch.commit().unwrap(); + (uuid, hash) + } + + #[test] + fn insert_and_count() { + let (_dir, ms) = open_fresh(); + let (uuid, hash) = setup_user(&ms); + let ops = ms.user_block_ops(); + + let cids = vec![vec![0x01, 0x71], vec![0x02, 0x72], vec![0x03, 0x73]]; + + let mut batch = ms.database().batch(); + ops.insert_user_blocks(&mut batch, hash, &cids, "rev1") + .unwrap(); + batch.commit().unwrap(); + + assert_eq!(ops.count_user_blocks(uuid).unwrap(), 3); + } + + #[test] + fn get_since_rev_returns_later_revisions() { + let (_dir, ms) = open_fresh(); + let (uuid, hash) = setup_user(&ms); + let ops = ms.user_block_ops(); + + let cids_abc = vec![vec![0x01], vec![0x02]]; + let cids_def = vec![vec![0x03]]; + + let mut batch = ms.database().batch(); + ops.insert_user_blocks(&mut batch, hash, &cids_abc, "abc") + .unwrap(); + ops.insert_user_blocks(&mut batch, hash, &cids_def, "def") + .unwrap(); + batch.commit().unwrap(); + + let since_abc = ops.get_user_block_cids_since_rev(uuid, "abc").unwrap(); + assert_eq!(since_abc.len(), 1); + assert_eq!(since_abc[0], vec![0x03]); + + let since_def = ops.get_user_block_cids_since_rev(uuid, "def").unwrap(); + assert!(since_def.is_empty()); + } + + #[test] + fn get_since_rev_with_both_revisions() { + let (_dir, ms) = open_fresh(); + let (uuid, hash) = setup_user(&ms); + let ops = ms.user_block_ops(); + + let cids_r1 = vec![vec![0x01]]; + let cids_r2 = vec![vec![0x02]]; + + let mut batch = ms.database().batch(); + ops.insert_user_blocks(&mut batch, hash, &cids_r1, "aaa") + .unwrap(); + ops.insert_user_blocks(&mut batch, hash, &cids_r2, "bbb") + .unwrap(); + batch.commit().unwrap(); + + let since_before = ops.get_user_block_cids_since_rev(uuid, "aaa").unwrap(); + assert_eq!(since_before.len(), 1); + assert_eq!(since_before[0], vec![0x02]); + + let all = ops.get_user_block_cids_since_rev(uuid, "").unwrap(); + assert_eq!(all.len(), 2); + } + + #[test] + fn delete_blocks_at_rev() { + let (_dir, ms) = open_fresh(); + let (uuid, hash) = setup_user(&ms); + let ops = ms.user_block_ops(); + + let cids = vec![vec![0x01], vec![0x02], vec![0x03]]; + + let mut batch = ms.database().batch(); + ops.insert_user_blocks(&mut batch, hash, &cids, "rev1") + .unwrap(); + batch.commit().unwrap(); + assert_eq!(ops.count_user_blocks(uuid).unwrap(), 3); + + let to_delete = vec![vec![0x01], vec![0x03]]; + let mut batch = ms.database().batch(); + ops.delete_user_blocks(&mut batch, hash, &to_delete, "rev1") + .unwrap(); + batch.commit().unwrap(); + + assert_eq!(ops.count_user_blocks(uuid).unwrap(), 1); + } + + #[test] + fn delete_wrong_rev_is_noop() { + let (_dir, ms) = open_fresh(); + let (uuid, hash) = setup_user(&ms); + let ops = ms.user_block_ops(); + + let cids = vec![vec![0x01], vec![0x02]]; + + let mut batch = ms.database().batch(); + ops.insert_user_blocks(&mut batch, hash, &cids, "rev1") + .unwrap(); + batch.commit().unwrap(); + + let mut batch = ms.database().batch(); + ops.delete_user_blocks(&mut batch, hash, &cids, "wrong_rev") + .unwrap(); + batch.commit().unwrap(); + + assert_eq!(ops.count_user_blocks(uuid).unwrap(), 2); + } + + #[test] + fn delete_all_user_blocks_clears_all_revisions() { + let (_dir, ms) = open_fresh(); + let (uuid, hash) = setup_user(&ms); + let ops = ms.user_block_ops(); + + let mut batch = ms.database().batch(); + ops.insert_user_blocks(&mut batch, hash, &[vec![0x01], vec![0x02]], "rev1") + .unwrap(); + ops.insert_user_blocks(&mut batch, hash, &[vec![0x03]], "rev2") + .unwrap(); + batch.commit().unwrap(); + assert_eq!(ops.count_user_blocks(uuid).unwrap(), 3); + + let mut batch = ms.database().batch(); + ops.delete_all_user_blocks(&mut batch, hash).unwrap(); + batch.commit().unwrap(); + + assert_eq!(ops.count_user_blocks(uuid).unwrap(), 0); + } + + #[test] + fn empty_rev_scan_returns_empty() { + let (_dir, ms) = open_fresh(); + let (uuid, _hash) = setup_user(&ms); + let ops = ms.user_block_ops(); + + let result = ops.get_user_block_cids_since_rev(uuid, "anything").unwrap(); + assert!(result.is_empty()); + } + + #[test] + fn unknown_user_returns_zero_count() { + let (_dir, ms) = open_fresh(); + let ops = ms.user_block_ops(); + let unknown = Uuid::new_v4(); + assert_eq!(ops.count_user_blocks(unknown).unwrap(), 0); + } + + #[test] + fn blocks_survive_reopen() { + let dir = tempfile::TempDir::new().unwrap(); + let uuid = Uuid::new_v4(); + let hash = UserHash::from_did("did:plc:persist"); + + { + let ms = Metastore::open( + dir.path(), + MetastoreConfig { + cache_size_bytes: 64 * 1024 * 1024, + }, + ) + .unwrap(); + let mut batch = ms.database().batch(); + ms.user_hashes() + .stage_insert(&mut batch, uuid, hash) + .unwrap(); + batch.commit().unwrap(); + + let ops = ms.user_block_ops(); + let cids = vec![vec![0xAA, 0xBB], vec![0xCC, 0xDD]]; + let mut batch = ms.database().batch(); + ops.insert_user_blocks(&mut batch, hash, &cids, "rev1") + .unwrap(); + batch.commit().unwrap(); + ms.persist().unwrap(); + } + + { + let ms = Metastore::open( + dir.path(), + MetastoreConfig { + cache_size_bytes: 64 * 1024 * 1024, + }, + ) + .unwrap(); + let ops = ms.user_block_ops(); + assert_eq!(ops.count_user_blocks(uuid).unwrap(), 2); + } + } + + #[test] + fn multiple_users_isolated() { + let (_dir, ms) = open_fresh(); + + let uuid1 = Uuid::new_v4(); + let hash1 = UserHash::from_did("did:plc:user1"); + let uuid2 = Uuid::new_v4(); + let hash2 = UserHash::from_did("did:plc:user2"); + + let mut batch = ms.database().batch(); + ms.user_hashes() + .stage_insert(&mut batch, uuid1, hash1) + .unwrap(); + ms.user_hashes() + .stage_insert(&mut batch, uuid2, hash2) + .unwrap(); + batch.commit().unwrap(); + + let ops = ms.user_block_ops(); + + let mut batch = ms.database().batch(); + ops.insert_user_blocks(&mut batch, hash1, &[vec![0x01], vec![0x02]], "rev1") + .unwrap(); + ops.insert_user_blocks(&mut batch, hash2, &[vec![0x03]], "rev1") + .unwrap(); + batch.commit().unwrap(); + + assert_eq!(ops.count_user_blocks(uuid1).unwrap(), 2); + assert_eq!(ops.count_user_blocks(uuid2).unwrap(), 1); + + let mut batch = ms.database().batch(); + ops.delete_user_blocks(&mut batch, hash1, &[vec![0x01]], "rev1") + .unwrap(); + batch.commit().unwrap(); + + assert_eq!(ops.count_user_blocks(uuid1).unwrap(), 1); + assert_eq!(ops.count_user_blocks(uuid2).unwrap(), 1); + } + + #[test] + fn delete_all_does_not_affect_other_users() { + let (_dir, ms) = open_fresh(); + + let uuid1 = Uuid::new_v4(); + let hash1 = UserHash::from_did("did:plc:user1"); + let uuid2 = Uuid::new_v4(); + let hash2 = UserHash::from_did("did:plc:user2"); + + let mut batch = ms.database().batch(); + ms.user_hashes() + .stage_insert(&mut batch, uuid1, hash1) + .unwrap(); + ms.user_hashes() + .stage_insert(&mut batch, uuid2, hash2) + .unwrap(); + batch.commit().unwrap(); + + let ops = ms.user_block_ops(); + + let mut batch = ms.database().batch(); + ops.insert_user_blocks(&mut batch, hash1, &[vec![0x01]], "rev1") + .unwrap(); + ops.insert_user_blocks(&mut batch, hash2, &[vec![0x02]], "rev1") + .unwrap(); + batch.commit().unwrap(); + + let mut batch = ms.database().batch(); + ops.delete_all_user_blocks(&mut batch, hash1).unwrap(); + batch.commit().unwrap(); + + assert_eq!(ops.count_user_blocks(uuid1).unwrap(), 0); + assert_eq!(ops.count_user_blocks(uuid2).unwrap(), 1); + } + + #[test] + fn cids_with_null_bytes_roundtrip_through_storage() { + let (_dir, ms) = open_fresh(); + let (uuid, hash) = setup_user(&ms); + let ops = ms.user_block_ops(); + + let cids = vec![ + vec![0x00, 0x00, 0x01], + vec![0x00, 0x01, 0x00, 0x00], + vec![0x00, 0x00], + ]; + + let mut batch = ms.database().batch(); + ops.insert_user_blocks(&mut batch, hash, &cids, "rev1") + .unwrap(); + batch.commit().unwrap(); + + assert_eq!(ops.count_user_blocks(uuid).unwrap(), 3); + + let retrieved = ops.get_user_block_cids_since_rev(uuid, "").unwrap(); + assert_eq!(retrieved.len(), 3); + let mut expected = cids.clone(); + expected.sort(); + assert_eq!(retrieved, expected); + } + + #[test] + fn insert_empty_cid_is_rejected() { + let (_dir, ms) = open_fresh(); + let (_uuid, hash) = setup_user(&ms); + let ops = ms.user_block_ops(); + + let cids = vec![vec![]]; + let mut batch = ms.database().batch(); + let result = ops.insert_user_blocks(&mut batch, hash, &cids, "rev1"); + assert!(matches!(result, Err(MetastoreError::InvalidInput(_)))); + } + + #[test] + fn delete_empty_cid_is_rejected() { + let (_dir, ms) = open_fresh(); + let (_uuid, hash) = setup_user(&ms); + let ops = ms.user_block_ops(); + + let cids = vec![vec![]]; + let mut batch = ms.database().batch(); + let result = ops.delete_user_blocks(&mut batch, hash, &cids, "rev1"); + assert!(matches!(result, Err(MetastoreError::InvalidInput(_)))); + } + + #[test] + fn since_rev_nonexistent_returns_later_revisions() { + let (_dir, ms) = open_fresh(); + let (uuid, hash) = setup_user(&ms); + let ops = ms.user_block_ops(); + + let mut batch = ms.database().batch(); + ops.insert_user_blocks(&mut batch, hash, &[vec![0x01]], "aaa") + .unwrap(); + ops.insert_user_blocks(&mut batch, hash, &[vec![0x02]], "bbb") + .unwrap(); + ops.insert_user_blocks(&mut batch, hash, &[vec![0x03]], "ddd") + .unwrap(); + batch.commit().unwrap(); + + let result = ops.get_user_block_cids_since_rev(uuid, "aab").unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0], vec![0x02]); + assert_eq!(result[1], vec![0x03]); + + let result = ops.get_user_block_cids_since_rev(uuid, "ccc").unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0], vec![0x03]); + } +} diff --git a/crates/tranquil-store/src/metastore/user_blocks.rs b/crates/tranquil-store/src/metastore/user_blocks.rs new file mode 100644 index 0000000..528a59b --- /dev/null +++ b/crates/tranquil-store/src/metastore/user_blocks.rs @@ -0,0 +1,112 @@ +use smallvec::SmallVec; + +use super::encoding::KeyBuilder; +use super::keys::{KeyTag, UserHash}; + +pub fn user_block_key(user_hash: UserHash, rev: &str, cid_bytes: &[u8]) -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::USER_BLOCKS) + .u64(user_hash.raw()) + .string(rev) + .raw(cid_bytes) + .build() +} + +pub fn user_block_user_prefix(user_hash: UserHash) -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::USER_BLOCKS) + .u64(user_hash.raw()) + .build() +} + +pub fn user_block_rev_prefix(user_hash: UserHash, rev: &str) -> SmallVec<[u8; 128]> { + KeyBuilder::new() + .tag(KeyTag::USER_BLOCKS) + .u64(user_hash.raw()) + .string(rev) + .build() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::metastore::encoding::KeyReader; + + #[test] + fn user_block_key_roundtrip() { + let hash = UserHash::from_raw(0xDEAD_BEEF_CAFE_BABE); + let cid = [0x01, 0x71, 0x12, 0x20, 0xAB]; + let key = user_block_key(hash, "3k2abcde", &cid); + + let mut reader = KeyReader::new(&key); + assert_eq!(reader.tag(), Some(KeyTag::USER_BLOCKS.raw())); + assert_eq!(reader.u64(), Some(0xDEAD_BEEF_CAFE_BABE)); + assert_eq!(reader.string(), Some("3k2abcde".to_string())); + assert_eq!(reader.remaining(), &cid); + } + + #[test] + fn keys_sort_by_user_then_rev_then_cid() { + let h1 = UserHash::from_raw(1); + let h2 = UserHash::from_raw(2); + + let k1 = user_block_key(h1, "abc", &[0x01]); + let k2 = user_block_key(h1, "abc", &[0x02]); + let k3 = user_block_key(h1, "def", &[0x01]); + let k4 = user_block_key(h2, "abc", &[0x01]); + + assert!(k1.as_slice() < k2.as_slice()); + assert!(k2.as_slice() < k3.as_slice()); + assert!(k3.as_slice() < k4.as_slice()); + } + + #[test] + fn user_prefix_is_prefix_of_rev_prefix() { + let hash = UserHash::from_raw(42); + let user_pfx = user_block_user_prefix(hash); + let rev_pfx = user_block_rev_prefix(hash, "some_rev"); + assert!(rev_pfx.as_slice().starts_with(user_pfx.as_slice())); + } + + #[test] + fn rev_prefix_is_prefix_of_full_key() { + let hash = UserHash::from_raw(42); + let rev_pfx = user_block_rev_prefix(hash, "some_rev"); + let full = user_block_key(hash, "some_rev", &[0x01, 0x02]); + assert!(full.as_slice().starts_with(rev_pfx.as_slice())); + } + + #[test] + fn cid_with_null_bytes_roundtrips() { + let hash = UserHash::from_raw(99); + let cid = [0x00, 0x01, 0x00, 0xFF]; + let key = user_block_key(hash, "rev1", &cid); + + let mut reader = KeyReader::new(&key); + assert_eq!(reader.tag(), Some(KeyTag::USER_BLOCKS.raw())); + assert_eq!(reader.u64(), Some(99)); + assert_eq!(reader.string(), Some("rev1".to_string())); + assert_eq!(reader.remaining(), &cid); + } + + #[test] + fn cid_with_double_null_bytes_roundtrips() { + let hash = UserHash::from_raw(99); + let cid = [0x00, 0x00, 0x01, 0x00, 0x00]; + let key = user_block_key(hash, "rev1", &cid); + + let mut reader = KeyReader::new(&key); + assert_eq!(reader.tag(), Some(KeyTag::USER_BLOCKS.raw())); + assert_eq!(reader.u64(), Some(99)); + assert_eq!(reader.string(), Some("rev1".to_string())); + assert_eq!(reader.remaining(), &cid); + } + + #[test] + fn empty_cid_produces_key_equal_to_rev_prefix() { + let hash = UserHash::from_raw(42); + let rev_pfx = user_block_rev_prefix(hash, "rev1"); + let full = user_block_key(hash, "rev1", &[]); + assert_eq!(full.as_slice(), rev_pfx.as_slice()); + } +} diff --git a/crates/tranquil-store/src/metastore/user_hash.rs b/crates/tranquil-store/src/metastore/user_hash.rs new file mode 100644 index 0000000..46a9faf --- /dev/null +++ b/crates/tranquil-store/src/metastore/user_hash.rs @@ -0,0 +1,437 @@ +use dashmap::DashMap; +use fjall::Keyspace; +use parking_lot::Mutex; +use uuid::Uuid; + +use super::MetastoreError; +use super::encoding::KeyBuilder; +use super::keys::{KeyTag, UserHash}; + +pub struct UserHashMap { + cache: DashMap, + reverse: DashMap, + repo_data: Keyspace, + write_guard: Mutex<()>, +} + +impl UserHashMap { + pub fn new(repo_data: Keyspace) -> Self { + Self { + cache: DashMap::new(), + reverse: DashMap::new(), + repo_data, + write_guard: Mutex::new(()), + } + } + + pub fn load_all(&self) -> Result { + let prefix = [KeyTag::USER_MAP.raw()]; + let mut count = 0usize; + + self.repo_data.prefix(prefix).try_for_each(|guard| { + let (key_bytes, value_bytes) = guard.into_inner().map_err(MetastoreError::Fjall)?; + + let uuid_bytes: [u8; 16] = key_bytes + .get(1..17) + .and_then(|s| <[u8; 16]>::try_from(s).ok()) + .ok_or(MetastoreError::CorruptData( + "user_map key too short for UUID", + ))?; + + let hash_bytes: [u8; 8] = <[u8; 8]>::try_from(value_bytes.as_ref()) + .map_err(|_| MetastoreError::CorruptData("user_map value not 8 bytes"))?; + + let uuid = Uuid::from_bytes(uuid_bytes); + let user_hash = UserHash::from_raw(u64::from_be_bytes(hash_bytes)); + + let existing = self.reverse.get(&user_hash).map(|r| *r); + if let Some(existing_uuid) = existing + && existing_uuid != uuid + { + tracing::error!( + existing_uuid = %existing_uuid, + new_uuid = %uuid, + user_hash = %user_hash, + "user hash collision in persisted data" + ); + return Err(MetastoreError::UserHashCollision { + hash: user_hash, + existing_uuid, + new_uuid: uuid, + }); + } + + self.cache.insert(uuid, user_hash); + self.reverse.insert(user_hash, uuid); + count = count.saturating_add(1); + + if count.is_multiple_of(100_000) { + tracing::info!(count, "loading user hash mappings"); + } + + Ok::<_, MetastoreError>(()) + })?; + + Ok(count) + } + + pub fn stage_insert( + &self, + batch: &mut fjall::OwnedWriteBatch, + uuid: Uuid, + user_hash: UserHash, + ) -> Result<(), MetastoreError> { + let _guard = self.write_guard.lock(); + + let existing = self.reverse.get(&user_hash).map(|r| *r); + if let Some(existing_uuid) = existing + && existing_uuid != uuid + { + tracing::error!( + existing_uuid = %existing_uuid, + new_uuid = %uuid, + user_hash = %user_hash, + "user hash collision detected" + ); + return Err(MetastoreError::UserHashCollision { + hash: user_hash, + existing_uuid, + new_uuid: uuid, + }); + } + + let forward_key = KeyBuilder::new() + .tag(KeyTag::USER_MAP) + .fixed(uuid.as_bytes()) + .build(); + + let reverse_key = KeyBuilder::new() + .tag(KeyTag::USER_MAP_REVERSE) + .u64(user_hash.raw()) + .build(); + + batch.insert( + &self.repo_data, + forward_key.as_slice(), + user_hash.raw().to_be_bytes(), + ); + batch.insert( + &self.repo_data, + reverse_key.as_slice(), + uuid.as_bytes().as_slice(), + ); + + self.cache.insert(uuid, user_hash); + self.reverse.insert(user_hash, uuid); + + Ok(()) + } + + pub fn rollback_insert(&self, uuid: &Uuid, user_hash: &UserHash) { + self.cache.remove(uuid); + self.reverse.remove(user_hash); + } + + pub fn stage_remove( + &self, + batch: &mut fjall::OwnedWriteBatch, + uuid: &Uuid, + ) -> Option { + let _guard = self.write_guard.lock(); + + let (_, user_hash) = self.cache.remove(uuid)?; + self.reverse.remove(&user_hash); + + let forward_key = KeyBuilder::new() + .tag(KeyTag::USER_MAP) + .fixed(uuid.as_bytes()) + .build(); + let reverse_key = KeyBuilder::new() + .tag(KeyTag::USER_MAP_REVERSE) + .u64(user_hash.raw()) + .build(); + + batch.remove(&self.repo_data, forward_key.as_slice()); + batch.remove(&self.repo_data, reverse_key.as_slice()); + + Some(user_hash) + } + + pub fn rollback_remove(&self, uuid: Uuid, user_hash: UserHash) { + self.cache.insert(uuid, user_hash); + self.reverse.insert(user_hash, uuid); + } + + pub fn get(&self, uuid: &Uuid) -> Option { + self.cache.get(uuid).map(|r| *r) + } + + pub fn get_uuid(&self, user_hash: &UserHash) -> Option { + self.reverse.get(user_hash).map(|r| *r) + } + + pub fn len(&self) -> usize { + self.cache.len() + } + + pub fn is_empty(&self) -> bool { + self.cache.is_empty() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn open_temp() -> (tempfile::TempDir, fjall::Database, Keyspace) { + let dir = tempfile::TempDir::new().unwrap(); + let db = fjall::Database::builder(dir.path()).open().unwrap(); + let ks = db + .keyspace("repo_data", fjall::KeyspaceCreateOptions::default) + .unwrap(); + (dir, db, ks) + } + + #[test] + fn insert_and_lookup() { + let (_dir, db, ks) = open_temp(); + let map = UserHashMap::new(ks); + + let uuid = Uuid::new_v4(); + let hash = UserHash::from_did("did:plc:test123"); + + let mut batch = db.batch(); + map.stage_insert(&mut batch, uuid, hash).unwrap(); + batch.commit().unwrap(); + + assert_eq!(map.get(&uuid), Some(hash)); + assert_eq!(map.get_uuid(&hash), Some(uuid)); + assert_eq!(map.len(), 1); + } + + #[test] + fn cache_populated_at_stage_time() { + let (_dir, db, ks) = open_temp(); + let map = UserHashMap::new(ks); + + let uuid = Uuid::new_v4(); + let hash = UserHash::from_did("did:plc:staged_only"); + + let mut batch = db.batch(); + map.stage_insert(&mut batch, uuid, hash).unwrap(); + + assert_eq!(map.get(&uuid), Some(hash)); + assert_eq!(map.get_uuid(&hash), Some(uuid)); + assert_eq!(map.len(), 1); + } + + #[test] + fn rollback_removes_from_cache() { + let (_dir, db, ks) = open_temp(); + let map = UserHashMap::new(ks); + + let uuid = Uuid::new_v4(); + let hash = UserHash::from_did("did:plc:rollback"); + + let mut batch = db.batch(); + map.stage_insert(&mut batch, uuid, hash).unwrap(); + assert_eq!(map.len(), 1); + + map.rollback_insert(&uuid, &hash); + assert!(map.is_empty()); + assert_eq!(map.get(&uuid), None); + assert_eq!(map.get_uuid(&hash), None); + + drop(batch); + } + + #[test] + fn load_all_after_reopen() { + let dir = tempfile::TempDir::new().unwrap(); + let uuid = Uuid::new_v4(); + let hash = UserHash::from_did("did:plc:persist"); + + { + let db = fjall::Database::builder(dir.path()).open().unwrap(); + let ks = db + .keyspace("repo_data", fjall::KeyspaceCreateOptions::default) + .unwrap(); + let map = UserHashMap::new(ks); + let mut batch = db.batch(); + map.stage_insert(&mut batch, uuid, hash).unwrap(); + batch.commit().unwrap(); + db.persist(fjall::PersistMode::SyncData).unwrap(); + } + + { + let db = fjall::Database::builder(dir.path()).open().unwrap(); + let ks = db + .keyspace("repo_data", fjall::KeyspaceCreateOptions::default) + .unwrap(); + let map = UserHashMap::new(ks); + let count = map.load_all().unwrap(); + assert_eq!(count, 1); + assert_eq!(map.get(&uuid), Some(hash)); + assert_eq!(map.get_uuid(&hash), Some(uuid)); + } + } + + #[test] + fn multiple_users() { + let (_dir, db, ks) = open_temp(); + let map = UserHashMap::new(ks); + + let pairs: Vec<_> = (0..10) + .map(|i| { + let uuid = Uuid::new_v4(); + let hash = UserHash::from_did(&format!("did:plc:user{i}")); + (uuid, hash) + }) + .collect(); + + let mut batch = db.batch(); + pairs.iter().for_each(|(uuid, hash)| { + map.stage_insert(&mut batch, *uuid, *hash).unwrap(); + }); + batch.commit().unwrap(); + + assert_eq!(map.len(), 10); + pairs.iter().for_each(|(uuid, hash)| { + assert_eq!(map.get(uuid), Some(*hash)); + assert_eq!(map.get_uuid(hash), Some(*uuid)); + }); + } + + #[test] + fn stage_insert_idempotent_for_same_uuid() { + let (_dir, db, ks) = open_temp(); + let map = UserHashMap::new(ks); + + let uuid = Uuid::new_v4(); + let hash = UserHash::from_did("did:plc:same"); + + let mut batch = db.batch(); + map.stage_insert(&mut batch, uuid, hash).unwrap(); + batch.commit().unwrap(); + + let mut batch2 = db.batch(); + map.stage_insert(&mut batch2, uuid, hash).unwrap(); + } + + #[test] + fn stage_insert_rejects_collision() { + let (_dir, db, ks) = open_temp(); + let map = UserHashMap::new(ks); + + let uuid_a = Uuid::new_v4(); + let uuid_b = Uuid::new_v4(); + let same_hash = UserHash::from_raw(0xDEAD_BEEF); + + let mut batch = db.batch(); + map.stage_insert(&mut batch, uuid_a, same_hash).unwrap(); + batch.commit().unwrap(); + + let mut batch2 = db.batch(); + let result = map.stage_insert(&mut batch2, uuid_b, same_hash); + assert!(matches!( + result, + Err(MetastoreError::UserHashCollision { .. }) + )); + } + + #[test] + fn stage_remove_clears_cache_and_persists() { + let dir = tempfile::TempDir::new().unwrap(); + let uuid = Uuid::new_v4(); + let hash = UserHash::from_did("did:plc:removable"); + + let db = fjall::Database::builder(dir.path()).open().unwrap(); + let ks = db + .keyspace("repo_data", fjall::KeyspaceCreateOptions::default) + .unwrap(); + let map = UserHashMap::new(ks); + + let mut batch = db.batch(); + map.stage_insert(&mut batch, uuid, hash).unwrap(); + batch.commit().unwrap(); + assert_eq!(map.len(), 1); + + let mut remove_batch = db.batch(); + let removed = map.stage_remove(&mut remove_batch, &uuid); + assert_eq!(removed, Some(hash)); + remove_batch.commit().unwrap(); + + assert!(map.is_empty()); + assert_eq!(map.get(&uuid), None); + assert_eq!(map.get_uuid(&hash), None); + + db.persist(fjall::PersistMode::SyncData).unwrap(); + drop(map); + drop(db); + + let db2 = fjall::Database::builder(dir.path()).open().unwrap(); + let ks2 = db2 + .keyspace("repo_data", fjall::KeyspaceCreateOptions::default) + .unwrap(); + let map2 = UserHashMap::new(ks2); + assert_eq!(map2.load_all().unwrap(), 0); + } + + #[test] + fn stage_remove_returns_none_for_unknown() { + let (_dir, db, ks) = open_temp(); + let map = UserHashMap::new(ks); + let mut batch = db.batch(); + assert_eq!(map.stage_remove(&mut batch, &Uuid::new_v4()), None); + } + + #[test] + fn rollback_remove_restores_cache() { + let (_dir, db, ks) = open_temp(); + let map = UserHashMap::new(ks); + + let uuid = Uuid::new_v4(); + let hash = UserHash::from_did("did:plc:rollback_remove"); + + let mut batch = db.batch(); + map.stage_insert(&mut batch, uuid, hash).unwrap(); + batch.commit().unwrap(); + + let mut remove_batch = db.batch(); + map.stage_remove(&mut remove_batch, &uuid); + assert!(map.is_empty()); + + map.rollback_remove(uuid, hash); + assert_eq!(map.get(&uuid), Some(hash)); + assert_eq!(map.get_uuid(&hash), Some(uuid)); + assert_eq!(map.len(), 1); + + drop(remove_batch); + } + + #[test] + fn stage_remove_then_reinsert_same_did() { + let (_dir, db, ks) = open_temp(); + let map = UserHashMap::new(ks); + + let uuid_a = Uuid::new_v4(); + let hash = UserHash::from_did("did:plc:reinsert"); + + let mut batch = db.batch(); + map.stage_insert(&mut batch, uuid_a, hash).unwrap(); + batch.commit().unwrap(); + + let mut remove_batch = db.batch(); + map.stage_remove(&mut remove_batch, &uuid_a); + remove_batch.commit().unwrap(); + + let uuid_b = Uuid::new_v4(); + let mut batch2 = db.batch(); + map.stage_insert(&mut batch2, uuid_b, hash).unwrap(); + batch2.commit().unwrap(); + + assert_eq!(map.get(&uuid_b), Some(hash)); + assert_eq!(map.get_uuid(&hash), Some(uuid_b)); + assert_eq!(map.get(&uuid_a), None); + } +} diff --git a/example.toml b/example.toml index 776ae6e..a7422e0 100644 --- a/example.toml +++ b/example.toml @@ -180,6 +180,32 @@ # Can also be specified via environment variable `S3_ENDPOINT`. #s3_endpoint = +# Repository backend: `postgres` by default, or `tranquil-store`, our embedded db. +# tranquil-store is EXPERIMENTAL!!!! RISK OF TOTAL DATA LOSS. +# +# Can also be specified via environment variable `REPO_BACKEND`. +# +# Default value: "postgres" +#repo_backend = "postgres" + +[tranquil_store] +# Directory for tranquil-store data: the metastore, eventlog, and blockstore. +# +# Can also be specified via environment variable `TRANQUIL_STORE_DATA_DIR`. +# +# Default value: "/var/lib/tranquil-pds/store" +#data_dir = "/var/lib/tranquil-pds/store" + +# Fjall block cache size in megabytes. Defaults to 20% of system RAM when unset. +# +# Can also be specified via environment variable `TRANQUIL_STORE_MEMORY_BUDGET_MB`. +#memory_budget_mb = + +# Number of handler threads. Defaults to available_parallelism / 2. +# +# Can also be specified via environment variable `TRANQUIL_STORE_HANDLER_THREADS`. +#handler_threads = + [cache] # Cache backend: `ripple` (default, built-in gossip) or `valkey`. # @@ -482,3 +508,11 @@ # # Default value: 3600 #delete_check_interval_secs = 3600 + +# Interval in seconds between block garbage collection cycles. +# Reclaims orphaned ipld blocks that were stored but never committed. +# +# Can also be specified via environment variable `BLOCK_GC_INTERVAL_SECS`. +# +# Default value: 21600 +#block_gc_interval_secs = 21600